Hadoop default configuration parameters and values

hi all,

This is the default configuration for hadoop-1.2.1. Feel free to take this and have a reference.

dfs.access.time.precision=3600000
dfs.balance.bandwidthPerSec=1048576
dfs.block.access.key.update.interval=600
dfs.block.access.token.enable=false
dfs.block.access.token.lifetime=600
dfs.blockreport.initialDelay=0
dfs.blockreport.intervalMsec=3600000
dfs.block.size=67108864
dfs.client.block.write.retries=3
dfs.client.use.datanode.hostname=false
dfs.data.dir=${hadoop.tmp.dir}/dfs/data
dfs.datanode.address=0.0.0.0:50010
dfs.datanode.data.dir.perm=755
dfs.datanode.dns.interface=default
dfs.datanode.dns.nameserver=default
dfs.datanode.drop.cache.behind.reads=false
dfs.datanode.drop.cache.behind.writes=false
dfs.datanode.du.reserved=0
dfs.datanode.failed.volumes.tolerated=0
dfs.datanode.handler.count=3
dfs.datanode.http.address=0.0.0.0:50075
dfs.datanode.https.address=0.0.0.0:50475
dfs.datanode.ipc.address=0.0.0.0:50020
dfs.datanode.max.xcievers=4096
dfs.datanode.readahead.bytes=4193404
dfs.datanode.sync.behind.writes=false
dfs.datanode.use.datanode.hostname=false
dfs.default.chunk.view.size=32768
dfs.df.interval=60000
dfs.heartbeat.interval=3
dfs.http.address=0.0.0.0:50070
dfs.https.address=0.0.0.0:50470
dfs.https.client.keystore.resource=ssl-client.xml
dfs.https.enable=false
dfs.https.need.client.auth=false
dfs.https.server.keystore.resource=ssl-server.xml
dfs.image.transfer.bandwidthPerSec=0
dfs.max.objects=0
dfs.name.dir=${hadoop.tmp.dir}/dfs/name
dfs.name.edits.dir=${dfs.name.dir}
dfs.namenode.avoid.read.stale.datanode=false
dfs.namenode.avoid.write.stale.datanode=false
dfs.namenode.decommission.interval=30
dfs.namenode.decommission.nodes.per.interval=5
dfs.namenode.delegation.key.update-interval=86400000
dfs.namenode.delegation.token.max-lifetime=604800000
dfs.namenode.delegation.token.renew-interval=86400000
dfs.namenode.edits.toleration.length=0
dfs.namenode.handler.count=10
dfs.namenode.invalidate.work.pct.per.iteration=0.32f
dfs.namenode.kerberos.internal.spnego.principal=${dfs.web.authentication.kerberos.principal}
dfs.namenode.logging.level=info
dfs.namenode.replication.work.multiplier.per.iteration=2
dfs.namenode.safemode.min.datanodes=0
dfs.namenode.stale.datanode.interval=30000
dfs.namenode.write.stale.datanode.ratio=0.5f
dfs.permissions.supergroup=supergroup
dfs.permissions=true
dfs.replication=3
dfs.replication.considerLoad=true
dfs.replication.interval=3
dfs.replication.max=512
dfs.replication.min=1
dfs.safemode.extension=30000
dfs.safemode.threshold.pct=0.999f
dfs.secondary.http.address=0.0.0.0:50090
dfs.secondary.namenode.kerberos.internal.spnego.principal=${dfs.web.authentication.kerberos.principal}
dfs.webhdfs.enabled=false
dfs.web.ugi=webuser,webgroup
fs.checkpoint.dir=${hadoop.tmp.dir}/dfs/namesecondary
fs.checkpoint.edits.dir=${fs.checkpoint.dir}
fs.checkpoint.period=3600
fs.checkpoint.size=67108864
fs.default.name=file:///
fs.file.impl=org.apache.hadoop.fs.LocalFileSystem
fs.ftp.impl=org.apache.hadoop.fs.ftp.FTPFileSystem
fs.har.impl.disable.cache=true
fs.har.impl=org.apache.hadoop.fs.HarFileSystem
fs.hdfs.impl=org.apache.hadoop.hdfs.DistributedFileSystem
fs.hftp.impl=org.apache.hadoop.hdfs.HftpFileSystem
fs.hsftp.impl=org.apache.hadoop.hdfs.HsftpFileSystem
fs.kfs.impl=org.apache.hadoop.fs.kfs.KosmosFileSystem
fs.ramfs.impl=org.apache.hadoop.fs.InMemoryFileSystem
fs.s3.block.size=67108864
fs.s3.buffer.dir=${hadoop.tmp.dir}/s3
fs.s3.impl=org.apache.hadoop.fs.s3.S3FileSystem
fs.s3.maxRetries=4
fs.s3n.impl=org.apache.hadoop.fs.s3native.NativeS3FileSystem
fs.s3.sleepTimeSeconds=10
fs.trash.interval=0
fs.webhdfs.impl=org.apache.hadoop.hdfs.web.WebHdfsFileSystem
hadoop.http.authentication.kerberos.keytab=${user.home}/hadoop.keytab
hadoop.http.authentication.kerberos.principal=HTTP/localhost@LOCALHOST
hadoop.http.authentication.signature.secret.file=${user.home}/hadoop-http-auth-signature-secret
hadoop.http.authentication.simple.anonymous.allowed=true
hadoop.http.authentication.token.validity=36000
hadoop.http.authentication.type=simple
hadoop.jetty.logs.serve.aliases=true
hadoop.logfile.count=10
hadoop.logfile.size=10000000
hadoop.native.lib=true
hadoop.relaxed.worker.version.check=false
hadoop.rpc.socket.factory.class.default=org.apache.hadoop.net.StandardSocketFactory
hadoop.security.authentication=simple
hadoop.security.authorization=false
hadoop.security.group.mapping=org.apache.hadoop.security.ShellBasedUnixGroupsMapping
hadoop.security.instrumentation.requires.admin=false
hadoop.security.token.service.use_ip=true
hadoop.security.uid.cache.secs=14400
hadoop.security.use-weak-http-crypto=false
hadoop.skip.worker.version.check=false
hadoop.tmp.dir=/tmp/hadoop-${user.name}
hadoop.util.hash.type=murmur
io.bytes.per.checksum=512
io.compression.codecs=org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.SnappyCodec
io.file.buffer.size=4096
io.mapfile.bloom.error.rate=0.005
io.mapfile.bloom.size=1048576
io.map.index.skip=0
io.seqfile.compress.blocksize=1000000
io.seqfile.lazydecompress=true
io.seqfile.sorter.recordlimit=1000000
io.serializations=org.apache.hadoop.io.serializer.WritableSerialization
io.skip.checksum.errors=false
io.sort.factor=10
io.sort.mb=100
io.sort.record.percent=0.05
io.sort.spill.percent=0.80
ipc.client.connection.maxidletime=10000
ipc.client.connect.max.retries=10
ipc.client.fallback-to-simple-auth-allowed=false
ipc.client.idlethreshold=4000
ipc.client.kill.max=10
ipc.client.tcpnodelay=false
ipc.server.listen.queue.size=128
ipc.server.tcpnodelay=false
jobclient.output.filter=FAILED
job.end.retry.attempts=0
job.end.retry.interval=30000
keep.failed.task.files=false
local.cache.size=10737418240
mapred.acls.enabled=false
mapred.child.java.opts=-Xmx200m
mapred.child.tmp=./tmp
mapred.cluster.map.memory.mb=-1
mapred.cluster.max.map.memory.mb=-1
mapred.cluster.max.reduce.memory.mb=-1
mapred.cluster.reduce.memory.mb=-1
mapred.combine.recordsBeforeProgress=10000
mapred.compress.map.output=false
mapred.disk.healthChecker.interval=60000
mapred.healthChecker.interval=60000
mapred.healthChecker.script.timeout=600000
mapred.heartbeats.in.second=100
mapred.inmem.merge.threshold=1000
mapred.job.map.memory.mb=-1
mapred.job.queue.name=default
mapred.job.reduce.input.buffer.percent=0.0
mapred.job.reduce.memory.mb=-1
mapred.job.reuse.jvm.num.tasks=1
mapred.job.shuffle.input.buffer.percent=0.70
mapred.job.shuffle.merge.percent=0.66
mapred.jobtracker.blacklist.fault-bucket-width=15
mapred.jobtracker.blacklist.fault-timeout-window=180
mapred.jobtracker.completeuserjobs.maximum=100
mapred.job.tracker.handler.count=10
mapred.job.tracker.http.address=0.0.0.0:50030
mapred.jobtracker.job.history.block.size=3145728
mapred.job.tracker.jobhistory.lru.cache.size=5
mapred.jobtracker.jobSchedulable=org.apache.hadoop.mapred.JobSchedulable
mapred.job.tracker=local
mapred.jobtracker.maxtasks.per.job=-1
mapred.jobtracker.nodegroup.aware=false
mapred.job.tracker.persist.jobstatus.active=false
mapred.job.tracker.persist.jobstatus.dir=/jobtracker/jobsInfo
mapred.job.tracker.persist.jobstatus.hours=0
mapred.jobtracker.restart.recover=false
mapred.job.tracker.retiredjobs.cache.size=1000
mapred.jobtracker.taskScheduler=org.apache.hadoop.mapred.JobQueueTaskScheduler
mapred.line.input.format.linespermap=1
mapred.local.dir=${hadoop.tmp.dir}/mapred/local
mapred.local.dir.minspacekill=0
mapred.local.dir.minspacestart=0
mapred.map.max.attempts=4
mapred.map.output.compression.codec=org.apache.hadoop.io.compress.DefaultCodec
mapred.map.tasks=2
mapred.map.tasks.speculative.execution=true
mapred.max.tracker.blacklists=4
mapred.max.tracker.failures=4
mapred.merge.recordsBeforeProgress=10000
mapred.min.split.size=0
mapred.output.compress=false
mapred.output.compression.codec=org.apache.hadoop.io.compress.DefaultCodec
mapred.output.compression.type=RECORD
mapred.queue.default.state=RUNNING
mapred.queue.names=default
mapred.reduce.max.attempts=4
mapred.reduce.parallel.copies=5
mapred.reduce.slowstart.completed.maps=0.05
mapred.reduce.tasks=1
mapred.reduce.tasks.speculative.execution=true
mapred.skip.attempts.to.start.skipping=2
mapred.skip.map.auto.incr.proc.count=true
mapred.skip.map.max.skip.records=0
mapred.skip.reduce.auto.incr.proc.count=true
mapred.skip.reduce.max.skip.groups=0
mapred.submit.replication=10
mapred.system.dir=${hadoop.tmp.dir}/mapred/system
mapred.task.cache.levels=2
mapred.task.profile=false
mapred.task.profile.maps=0-2
mapred.task.profile.reduces=0-2
mapred.task.timeout=600000
mapred.tasktracker.dns.interface=default
mapred.tasktracker.dns.nameserver=default
mapred.tasktracker.expiry.interval=600000
mapred.task.tracker.http.address=0.0.0.0:50060
mapred.tasktracker.indexcache.mb=10
mapred.tasktracker.map.tasks.maximum=2
mapred.tasktracker.reduce.tasks.maximum=2
mapred.task.tracker.report.address=127.0.0.1:0
mapred.task.tracker.task-controller=org.apache.hadoop.mapred.DefaultTaskController
mapred.tasktracker.taskmemorymanager.monitoring-interval=5000
mapred.tasktracker.tasks.sleeptime-before-sigkill=5000
mapred.temp.dir=${hadoop.tmp.dir}/mapred/temp
mapreduce.ifile.readahead.bytes=4194304
mapreduce.ifile.readahead=true
mapreduce.job.acl-modify-job=
mapreduce.job.acl-view-job=
mapreduce.job.complete.cancel.delegation.tokens=true
mapreduce.job.counters.counter.name.max=64
mapreduce.job.counters.group.name.max=128
mapreduce.job.counters.groups.max=50
mapreduce.job.counters.max=120
mapreduce.jobhistory.cleaner.interval-ms=86400000
mapreduce.jobhistory.max-age-ms=2592000000
mapreduce.job.restart.recover=true
mapreduce.job.split.metainfo.maxsize=10000000
mapreduce.jobtracker.staging.root.dir=${hadoop.tmp.dir}/mapred/staging
mapreduce.reduce.input.limit=-1
mapreduce.reduce.shuffle.connect.timeout=180000
mapreduce.reduce.shuffle.maxfetchfailures=10
mapreduce.reduce.shuffle.read.timeout=180000
mapreduce.tasktracker.outofband.heartbeat.damper=1000000
mapreduce.tasktracker.outofband.heartbeat=false
mapred.used.genericoptionsparser=true
mapred.user.jobconf.limit=5242880
mapred.userlog.limit.kb=0
mapred.userlog.retain.hours=24
map.sort.class=org.apache.hadoop.util.QuickSort
net.topology.impl=org.apache.hadoop.net.NetworkTopology
tasktracker.http.threads=40
topology.node.switch.mapping.impl=org.apache.hadoop.net.ScriptBasedMapping
topology.script.number.args=100
webinterface.private.actions=false

Managing bad disks in Hadoop cluster

hi all,

I was tasked to do some log files massaging and pipe them onto a specific custom log. The whole notion is to detect any of the disk failures that happened on the hadoop cluster. For example, if we are running a 30+ nodes in a hadoop cluster, with each node having 10 local disks attached onto the it. The possibility of having worn/bad disks is really high when the cluster is serving for a business for a period of times. So, instead of proactively scanning the disks on 30+ nodes everyday, we can make use of the existing tool to help us to achieve our goal.

These are the existing tool.

1. rsyslog
2. any sort of monitoring tool, like nagios, or OVO agent/BMC patrol agent.


I want to document on the steps how I manage to harvest the disk failure messages from the standard system log, /var/log/message*.

1. put a conf file like this.

[root@centos65-1 ~]# cat /etc/rsyslog.d/hadoop.conf
:msg, contains, "offline"    /var/log/hadoop_disk.log

2. touch the /var/log/hadoop_disk.log

3. restart the rsyslog daemon.


Voila you are done!


Now, we want to test it. Here are the steps.

1. Create a small disk from the VM, carve it, format, and mount it.

 [root@centos65-1 ~]# df /mnt/test
Filesystem     1K-blocks  Used Available Use% Mounted on
/dev/sda1        2063504 35840   1922844   2% /mnt/test

2. Offline the disk state.

[root@centos65-1 ~]# echo "offline" > /sys/block/sda/device/state
[root@centos65-1 ~]#

3. Jump/cd onto the mount point.

[root@centos65-1 ~]# cd /mnt/test
[root@centos65-1 test]# ls
ls: reading directory .: Input/output error

Now, you should be getting the system log files redirected to /var/log/hadoop_disk.log.

[root@centos65-1 ~]# tail -f /var/log/hadoop_disk.log
Jun 25 10:15:06 centos65-1 kernel: sd 0:0:0:0: rejecting I/O to offline device
Jun 25 10:15:12 centos65-1 kernel: sd 0:0:0:0: rejecting I/O to offline device
Jun 25 10:15:12 centos65-1 kernel: sd 0:0:0:0: rejecting I/O to offline device
Jun 25 10:15:12 centos65-1 kernel: sd 0:0:0:0: rejecting I/O to offline device

Good stuffs! This is what we can do achieve. Next step, we can configure our monitoring tool to watch this log file. And eventually generate ticket to alerting the business.

Creating sparse file to show how thin-provisioning is possible on Linux

hi,

Recently I have a good lunch meet up with Wing Loon to investigate on some of the questions that I kept for weeks. We were discussing on how Docker can store the container with extra disk space (by default it is allocating 10Gb for each container), yet my VM has not been allocating such a huge disk space. We were pondering and have no clue, until we bumped into a computer science term called, Sparse file. In layman term, it is something like thin-provisioning allowing a system to allocate a temporarily space, but will not really claiming data block size until you write it onto the disk. This type of file will eventually grow as it goes.

Now, I would like to hand-held you on a few steps to demonstrate/create this type of file.


[root@localhost ~]# dd of=sparse-file bs=1M seek=102400 count=0
0+0 records in
0+0 records out
0 bytes (0 B) copied, 5.3053e-05 s, 0.0 kB/s
[root@localhost ~]# ls -al
total 64
dr-xr-x---.  5 root root         4096 Jun 13 11:10 .
drwxr-xr-x. 18 root root         4096 Jun 13 10:28 ..
-rw-------.  1 root root         1082 Jun  6 20:26 anaconda-ks.cfg
-rw-------.  1 root root        12916 Jun 13 10:06 .bash_history
-rw-r--r--.  1 root root           18 Dec  4  2013 .bash_logout
-rw-r--r--.  1 root root          262 Jun  9 11:12 .bash_profile
-rw-r--r--.  1 root root          176 Dec  4  2013 .bashrc
-rw-r--r--.  1 root root          100 Dec  4  2013 .cshrc
-rw-------.  1 root root           98 Jun 13 10:00 .lesshst
drwxr-xr-x.  3 root root         4096 Jun  6 21:01 .local
drwxr-xr-x.  4 root root         4096 Dec 12  2013 .mozilla
drwxr-----.  3 root root         4096 Jun  6 20:30 .pki
-rw-r--r--.  1 root root 107374182400 Jun 13 11:10 sparse-file
-rw-r--r--.  1 root root          129 Dec  4  2013 .tcshrc
[root@localhost ~]#
[root@localhost ~]# du -s sparse-file
0    sparse-file

If you understand these commands above, you will curious why ls -al sparse-file is showing such a huge numbers, and yet, du -s sparse-file is just a zeroed-file.That's what we called it sparse file.


So, next we will making use of this file, I can create a device mapper file, and put do a prank tricks to my friends. This is how it goes.


[root@localhost ~]# losetup -f
/dev/loop0
[root@localhost ~]# losetup /dev/loop0 /root/sparse-file
[root@localhost ~]# losetup
NAME       SIZELIMIT OFFSET AUTOCLEAR RO BACK-FILE
/dev/loop0         0      0         0  0 /root/sparse-file
[root@localhost ~]#
[root@localhost ~]#
[root@localhost ~]# blockdev --getsz /root/sparse-file
209715200
[root@localhost ~]#
[root@localhost ~]#
[root@localhost ~]# dmsetup create hiu_prank --table "0 209715200 linear /dev/loop0 0"
[root@localhost ~]#
[root@localhost ~]#
[root@localhost ~]# dmsetup ls
fedora-swap    (253:1)
fedora-root    (253:0)
hiu_prank    (253:2)
[root@localhost ~]#
[root@localhost ~]#
[root@localhost ~]# fdisk -l /dev/mapper/hiu_prank

Disk /dev/mapper/hiu_prank: 100 GiB, 107374182400 bytes, 209715200 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes


Now, i have a fake disk that claimed itself having 100Gb. I guess you know how to format it and mount it and write data onto it.

By the way, /var/log/lastlog is also another type of sparse file, which we use it daily.

Enjoy!
 (P.S. thank you wingloon.com)

Searching big files over in linux with python script.

hi all,

Welcome to download and put this in your /usr/bin directory for handy big files searching..

import os
import sys

# searching for bigfile
def searchbigfile( search_dir ):
        filesize = 0
        filename = ''
        print "Searching directory: %s " %(search_dir)
        for root, dirs, files in os.walk(search_dir):
                for f in files:
                        myfile = os.path.join (root, f)
                        if os.path.isfile(myfile) and os.path.getsize(myfile) > filesize:
                                filesize = int(os.path.getsize(myfile))
                                filename = os.path.realpath(myfile)
                                print "file: %s (%s) " % (filename, sizeof_fmt(filesize))


# get the size in human readable format
def sizeof_fmt(num):
        for x in ['bytes', 'KB', 'MB', 'GB', 'TB' ]:
                if num < 1024.0:
                        return "%3.1f %s" % (num, x)
                num /= 1024.0


if __name__ == '__main__':
        if len(sys.argv) == 2:
                if str(sys.argv)[1]:
                        search_dir = str((sys.argv)[1])
        else:
                search_dir = "/"
        try:
                searchbigfile( search_dir )
        except KeyboardInterrupt, e:
                print "Caught CRTL-C, script is terminated.", e

The result will be something like this.

[root@nasilemak~]# ./bigfile.py
Searching directory: /
file: /javacerts.tgz ==> size: 242.6 KB
file: /lib/modules/2.6.32-358.18.1.el6.x86_64/kernel/fs/cifs/cifs.ko ==> size: 497.5 KB
file: /lib/modules/2.6.32-358.18.1.el6.x86_64/kernel/fs/btrfs/btrfs.ko ==> size: 1.2 MB
file: /lib/modules/2.6.32-358.18.1.el6.x86_64/kernel/fs/xfs/xfs.ko ==> size: 1.7 MB
file: /boot/System.map-2.6.32-279.el6.x86_64 ==> size: 2.2 MB
file: /boot/vmlinuz-2.6.32-358.18.1.el6.x86_64 ==> size: 3.9 MB
file: /usr/share/dict/linux.words ==> size: 4.7 MB
file: /usr/share/cracklib/pw_dict.pwd ==> size: 8.2 MB
file: /usr/lib/locale/locale-archive ==> size: 94.6 MB