├── EMR Scripts ├── teragenEMR.sh ├── teragen_results_2016-12-15:03:21:30 ├── terasortEMR.sh ├── terasort_results_2016-12-15:15:04:51 ├── teravalidateEMR.sh └── teravalidate_results_2016-12-15:20:13:11 ├── README.md ├── ambari-blueprint ├── capacity-scheduler.xml ├── core-site.xml ├── dfsio.sh ├── mapred-site.xml ├── performance Test Results ├── dfsio_read_output3n.txt ├── dfsio_read_results5n.txt ├── dfsio_write_output3n.txt ├── dfsio_write_results5n.txt ├── teragen 3 node.jpg ├── teragen 5 node.jpg ├── teragen_results_2016-08-22:16:43:22 ├── terasort 3 node.jpg ├── terasort 5 node.jpg ├── terasort.jpg ├── terasort_results_2016-08-22:16:52:42 ├── teravalidate 3 node.jpg ├── teravalidate 5 node.jpg ├── teravalidate.jpg └── teravalidate_results_2016-08-22:22:14:17 ├── teragen.sh ├── terasort.sh ├── validate.sh └── yarn-site.xml /EMR Scripts/teragenEMR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | trap "" HUP 4 | 5 | #if [ $EUID -eq 0 ]; then 6 | # echo "this script must not be run as root. su to hdfs user to run" 7 | # exit 1 8 | #fi 9 | 10 | #MR_EXAMPLES_JAR=/usr/hdp/2.2.0.0-2041/hadoop-mapreduce/hadoop-mapreduce-examples.jar 11 | MR_EXAMPLES_JAR=/usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples-2.7.3-amzn-0.jar 12 | 13 | #SIZE=500G 14 | #ROWS=5000000000 15 | 16 | #SIZE=100G 17 | #ROWS=1000000000 18 | 19 | SIZE=1T 20 | ROWS=10000000000 21 | 22 | # SIZE=10G 23 | # ROWS=100000000 24 | 25 | # SIZE=1G 26 | # ROWS=10000000 27 | 28 | 29 | LOGDIR=logs 30 | 31 | if [ ! -d "$LOGDIR" ] 32 | then 33 | mkdir ./$LOGDIR 34 | fi 35 | 36 | DATE=`date +%Y-%m-%d:%H:%M:%S` 37 | 38 | RESULTSFILE="./$LOGDIR/teragen_results_$DATE" 39 | 40 | 41 | OUTPUT=s3://sunileman1/data/sandbox/poc/teragen/${SIZE}-terasort-input 42 | 43 | # teragen.sh 44 | # Kill any running MapReduce jobs 45 | mapred job -list | grep job_ | awk ' { system("mapred job -kill " $1) } ' 46 | # Delete the output directory 47 | hadoop fs -rm -r -f -skipTrash ${OUTPUT} 48 | 49 | # Run teragen 50 | time hadoop jar $MR_EXAMPLES_JAR teragen \ 51 | -Dmapreduce.map.log.level=INFO \ 52 | -Dmapreduce.reduce.log.level=INFO \ 53 | -Dyarn.app.mapreduce.am.log.level=INFO \ 54 | -Dio.file.buffer.size=131072 \ 55 | -Dmapreduce.map.cpu.vcores=1 \ 56 | -Dmapreduce.map.java.opts=-Xmx1536m \ 57 | -Dmapreduce.map.maxattempts=1 \ 58 | -Dmapreduce.map.memory.mb=2048 \ 59 | -Dmapreduce.map.output.compress=true \ 60 | -Dmapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.Lz4Codec \ 61 | -Dmapreduce.reduce.cpu.vcores=1 \ 62 | -Dmapreduce.reduce.java.opts=-Xmx1536m \ 63 | -Dmapreduce.reduce.maxattempts=1 \ 64 | -Dmapreduce.reduce.memory.mb=2048 \ 65 | -Dmapreduce.task.io.sort.factor=100 \ 66 | -Dmapreduce.task.io.sort.mb=384 \ 67 | -Dyarn.app.mapreduce.am.command.opts=-Xmx1900m \ 68 | -Dyarn.app.mapreduce.am.resource.mb=2024 \ 69 | -Dmapred.task.timeout=12000000 \ 70 | -Dmapred.map.tasks=92 \ 71 | ${ROWS} ${OUTPUT} >> $RESULTSFILE 2>&1 72 | 73 | #-Dmapreduce.map.log.level=TRACE \ 74 | #-Dmapreduce.reduce.log.level=TRACE \ 75 | #-Dyarn.app.mapreqduce.am.log.level=TRACE \ 76 | -------------------------------------------------------------------------------- /EMR Scripts/teragen_results_2016-12-15:03:21:30: -------------------------------------------------------------------------------- 1 | 16/12/15 03:21:38 INFO client.RMProxy: Connecting to ResourceManager at ip-10-0-0-196.us-west-2.compute.internal/10.0.0.196:8032 2 | 16/12/15 03:21:39 INFO terasort.TeraSort: Generating 10000000000 using 92 3 | 16/12/15 03:21:39 INFO mapreduce.JobSubmitter: number of splits:92 4 | 16/12/15 03:21:39 INFO Configuration.deprecation: mapred.task.timeout is deprecated. Instead, use mapreduce.task.timeout 5 | 16/12/15 03:21:39 INFO Configuration.deprecation: mapred.map.tasks is deprecated. Instead, use mapreduce.job.maps 6 | 16/12/15 03:21:39 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1481769222473_0002 7 | 16/12/15 03:21:40 INFO impl.YarnClientImpl: Submitted application application_1481769222473_0002 8 | 16/12/15 03:21:40 INFO mapreduce.Job: The url to track the job: http://ip-10-0-0-196.us-west-2.compute.internal:20888/proxy/application_1481769222473_0002/ 9 | 16/12/15 03:21:40 INFO mapreduce.Job: Running job: job_1481769222473_0002 10 | 16/12/15 03:21:48 INFO mapreduce.Job: Job job_1481769222473_0002 running in uber mode : false 11 | 16/12/15 03:21:48 INFO mapreduce.Job: map 0% reduce 0% 12 | 16/12/15 03:22:07 INFO mapreduce.Job: map 1% reduce 0% 13 | 16/12/15 03:22:32 INFO mapreduce.Job: map 2% reduce 0% 14 | 16/12/15 03:22:47 INFO mapreduce.Job: map 3% reduce 0% 15 | 16/12/15 03:22:57 INFO mapreduce.Job: map 4% reduce 0% 16 | 16/12/15 03:23:06 INFO mapreduce.Job: map 5% reduce 0% 17 | 16/12/15 03:23:14 INFO mapreduce.Job: map 6% reduce 0% 18 | 16/12/15 03:23:22 INFO mapreduce.Job: map 7% reduce 0% 19 | 16/12/15 03:23:29 INFO mapreduce.Job: map 8% reduce 0% 20 | 16/12/15 03:23:37 INFO mapreduce.Job: map 9% reduce 0% 21 | 16/12/15 03:23:43 INFO mapreduce.Job: map 10% reduce 0% 22 | 16/12/15 03:23:51 INFO mapreduce.Job: map 11% reduce 0% 23 | 16/12/15 03:23:58 INFO mapreduce.Job: map 12% reduce 0% 24 | 16/12/15 03:24:05 INFO mapreduce.Job: map 13% reduce 0% 25 | 16/12/15 03:24:13 INFO mapreduce.Job: map 14% reduce 0% 26 | 16/12/15 03:24:20 INFO mapreduce.Job: map 15% reduce 0% 27 | 16/12/15 03:24:28 INFO mapreduce.Job: map 16% reduce 0% 28 | 16/12/15 03:24:35 INFO mapreduce.Job: map 17% reduce 0% 29 | 16/12/15 03:24:42 INFO mapreduce.Job: map 18% reduce 0% 30 | 16/12/15 03:24:49 INFO mapreduce.Job: map 19% reduce 0% 31 | 16/12/15 03:24:57 INFO mapreduce.Job: map 20% reduce 0% 32 | 16/12/15 03:25:04 INFO mapreduce.Job: map 21% reduce 0% 33 | 16/12/15 03:25:12 INFO mapreduce.Job: map 22% reduce 0% 34 | 16/12/15 03:25:20 INFO mapreduce.Job: map 23% reduce 0% 35 | 16/12/15 03:25:27 INFO mapreduce.Job: map 24% reduce 0% 36 | 16/12/15 03:25:35 INFO mapreduce.Job: map 25% reduce 0% 37 | 16/12/15 03:25:43 INFO mapreduce.Job: map 26% reduce 0% 38 | 16/12/15 03:25:50 INFO mapreduce.Job: map 27% reduce 0% 39 | 16/12/15 03:25:58 INFO mapreduce.Job: map 28% reduce 0% 40 | 16/12/15 03:26:05 INFO mapreduce.Job: map 29% reduce 0% 41 | 16/12/15 03:26:13 INFO mapreduce.Job: map 30% reduce 0% 42 | 16/12/15 03:26:20 INFO mapreduce.Job: map 31% reduce 0% 43 | 16/12/15 03:26:27 INFO mapreduce.Job: map 32% reduce 0% 44 | 16/12/15 03:26:35 INFO mapreduce.Job: map 33% reduce 0% 45 | 16/12/15 03:26:42 INFO mapreduce.Job: map 34% reduce 0% 46 | 16/12/15 03:26:51 INFO mapreduce.Job: map 35% reduce 0% 47 | 16/12/15 03:26:58 INFO mapreduce.Job: map 36% reduce 0% 48 | 16/12/15 03:27:04 INFO mapreduce.Job: map 37% reduce 0% 49 | 16/12/15 03:27:12 INFO mapreduce.Job: map 38% reduce 0% 50 | 16/12/15 03:27:19 INFO mapreduce.Job: map 39% reduce 0% 51 | 16/12/15 03:27:27 INFO mapreduce.Job: map 40% reduce 0% 52 | 16/12/15 03:27:35 INFO mapreduce.Job: map 41% reduce 0% 53 | 16/12/15 03:27:42 INFO mapreduce.Job: map 42% reduce 0% 54 | 16/12/15 03:27:49 INFO mapreduce.Job: map 43% reduce 0% 55 | 16/12/15 03:27:56 INFO mapreduce.Job: map 44% reduce 0% 56 | 16/12/15 03:28:04 INFO mapreduce.Job: map 45% reduce 0% 57 | 16/12/15 03:28:10 INFO mapreduce.Job: map 46% reduce 0% 58 | 16/12/15 03:28:18 INFO mapreduce.Job: map 47% reduce 0% 59 | 16/12/15 03:28:26 INFO mapreduce.Job: map 48% reduce 0% 60 | 16/12/15 03:28:34 INFO mapreduce.Job: map 49% reduce 0% 61 | 16/12/15 03:28:40 INFO mapreduce.Job: map 50% reduce 0% 62 | 16/12/15 03:28:48 INFO mapreduce.Job: map 51% reduce 0% 63 | 16/12/15 03:28:56 INFO mapreduce.Job: map 52% reduce 0% 64 | 16/12/15 03:29:03 INFO mapreduce.Job: map 53% reduce 0% 65 | 16/12/15 03:29:11 INFO mapreduce.Job: map 54% reduce 0% 66 | 16/12/15 03:29:18 INFO mapreduce.Job: map 55% reduce 0% 67 | 16/12/15 03:29:27 INFO mapreduce.Job: map 56% reduce 0% 68 | 16/12/15 03:29:33 INFO mapreduce.Job: map 57% reduce 0% 69 | 16/12/15 03:29:41 INFO mapreduce.Job: map 58% reduce 0% 70 | 16/12/15 03:29:48 INFO mapreduce.Job: map 59% reduce 0% 71 | 16/12/15 03:29:55 INFO mapreduce.Job: map 60% reduce 0% 72 | 16/12/15 03:30:03 INFO mapreduce.Job: map 61% reduce 0% 73 | 16/12/15 03:30:11 INFO mapreduce.Job: map 62% reduce 0% 74 | 16/12/15 03:30:17 INFO mapreduce.Job: map 63% reduce 0% 75 | 16/12/15 03:30:26 INFO mapreduce.Job: map 64% reduce 0% 76 | 16/12/15 03:30:34 INFO mapreduce.Job: map 65% reduce 0% 77 | 16/12/15 03:30:42 INFO mapreduce.Job: map 66% reduce 0% 78 | 16/12/15 03:30:49 INFO mapreduce.Job: map 67% reduce 0% 79 | 16/12/15 03:30:55 INFO mapreduce.Job: map 68% reduce 0% 80 | 16/12/15 03:31:04 INFO mapreduce.Job: map 69% reduce 0% 81 | 16/12/15 03:31:11 INFO mapreduce.Job: map 70% reduce 0% 82 | 16/12/15 03:31:19 INFO mapreduce.Job: map 71% reduce 0% 83 | 16/12/15 03:31:28 INFO mapreduce.Job: map 72% reduce 0% 84 | 16/12/15 03:31:34 INFO mapreduce.Job: map 73% reduce 0% 85 | 16/12/15 03:31:42 INFO mapreduce.Job: map 74% reduce 0% 86 | 16/12/15 03:31:50 INFO mapreduce.Job: map 75% reduce 0% 87 | 16/12/15 03:31:57 INFO mapreduce.Job: map 76% reduce 0% 88 | 16/12/15 03:32:05 INFO mapreduce.Job: map 77% reduce 0% 89 | 16/12/15 03:32:13 INFO mapreduce.Job: map 78% reduce 0% 90 | 16/12/15 03:32:20 INFO mapreduce.Job: map 79% reduce 0% 91 | 16/12/15 03:32:28 INFO mapreduce.Job: map 80% reduce 0% 92 | 16/12/15 03:32:36 INFO mapreduce.Job: map 81% reduce 0% 93 | 16/12/15 03:32:43 INFO mapreduce.Job: map 82% reduce 0% 94 | 16/12/15 03:32:50 INFO mapreduce.Job: map 83% reduce 0% 95 | 16/12/15 03:32:58 INFO mapreduce.Job: map 84% reduce 0% 96 | 16/12/15 03:33:05 INFO mapreduce.Job: map 85% reduce 0% 97 | 16/12/15 03:33:13 INFO mapreduce.Job: map 86% reduce 0% 98 | 16/12/15 03:33:20 INFO mapreduce.Job: map 87% reduce 0% 99 | 16/12/15 03:33:28 INFO mapreduce.Job: map 88% reduce 0% 100 | 16/12/15 03:33:36 INFO mapreduce.Job: map 89% reduce 0% 101 | 16/12/15 03:33:43 INFO mapreduce.Job: map 90% reduce 0% 102 | 16/12/15 03:33:51 INFO mapreduce.Job: map 91% reduce 0% 103 | 16/12/15 03:33:58 INFO mapreduce.Job: map 92% reduce 0% 104 | 16/12/15 03:34:06 INFO mapreduce.Job: map 93% reduce 0% 105 | 16/12/15 03:34:13 INFO mapreduce.Job: map 94% reduce 0% 106 | 16/12/15 03:34:21 INFO mapreduce.Job: map 95% reduce 0% 107 | 16/12/15 03:34:28 INFO mapreduce.Job: map 96% reduce 0% 108 | 16/12/15 03:34:35 INFO mapreduce.Job: map 97% reduce 0% 109 | 16/12/15 03:34:43 INFO mapreduce.Job: map 98% reduce 0% 110 | 16/12/15 03:34:55 INFO mapreduce.Job: map 99% reduce 0% 111 | 16/12/15 03:35:12 INFO mapreduce.Job: map 100% reduce 0% 112 | 16/12/15 03:48:26 INFO mapreduce.Job: Job job_1481769222473_0002 completed successfully 113 | 16/12/15 03:48:26 INFO mapreduce.Job: Counters: 36 114 | File System Counters 115 | FILE: Number of bytes read=0 116 | FILE: Number of bytes written=11715178 117 | FILE: Number of read operations=0 118 | FILE: Number of large read operations=0 119 | FILE: Number of write operations=0 120 | HDFS: Number of bytes read=8052 121 | HDFS: Number of bytes written=0 122 | HDFS: Number of read operations=92 123 | HDFS: Number of large read operations=0 124 | HDFS: Number of write operations=0 125 | S3: Number of bytes read=0 126 | S3: Number of bytes written=1000000000000 127 | S3: Number of read operations=0 128 | S3: Number of large read operations=0 129 | S3: Number of write operations=0 130 | Job Counters 131 | Launched map tasks=92 132 | Other local map tasks=92 133 | Total time spent by all maps in occupied slots (ms)=9138155264 134 | Total time spent by all reduces in occupied slots (ms)=0 135 | Total time spent by all map tasks (ms)=142783676 136 | Total vcore-milliseconds taken by all map tasks=142783676 137 | Total megabyte-milliseconds taken by all map tasks=292420968448 138 | Map-Reduce Framework 139 | Map input records=10000000000 140 | Map output records=10000000000 141 | Input split bytes=8052 142 | Spilled Records=0 143 | Failed Shuffles=0 144 | Merged Map outputs=0 145 | GC time elapsed (ms)=226153 146 | CPU time spent (ms)=44931650 147 | Physical memory (bytes) snapshot=60609339392 148 | Virtual memory (bytes) snapshot=340085231616 149 | Total committed heap usage (bytes)=111961702400 150 | org.apache.hadoop.examples.terasort.TeraGen$Counters 151 | CHECKSUM=3028416809717741100 152 | File Input Format Counters 153 | Bytes Read=0 154 | File Output Format Counters 155 | Bytes Written=1000000000000 156 | -------------------------------------------------------------------------------- /EMR Scripts/terasortEMR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | trap "" HUP 4 | 5 | #if [ $EUID -eq 0 ]; then 6 | # echo "this script must not be run as root. su to hdfs user to run" 7 | # exit 1 8 | #fi 9 | 10 | MR_EXAMPLES_JAR=/usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples-2.7.3-amzn-0.jar 11 | 12 | 13 | #SIZE=500G 14 | #SIZE=100G 15 | SIZE=1T 16 | #SIZE=1G 17 | #SIZE=10G 18 | #INPUT=/${SIZE}-terasort-input 19 | #OUTPUT=/${SIZE}-terasort-output 20 | 21 | 22 | LOGDIR=logs 23 | 24 | if [ ! -d "$LOGDIR" ] 25 | then 26 | mkdir ./$LOGDIR 27 | fi 28 | 29 | DATE=`date +%Y-%m-%d:%H:%M:%S` 30 | 31 | RESULTSFILE="./$LOGDIR/terasort_results_$DATE" 32 | 33 | 34 | INPUT=s3://sunileman1/data/sandbox/poc/teragen/${SIZE}-terasort-input 35 | OUTPUT=s3://sunileman1/data/sandbox/poc/teragen/${SIZE}-terasort-output 36 | 37 | # terasort.sh 38 | # Kill any running MapReduce jobs 39 | mapred job -list | grep job_ | awk ' { system("mapred job -kill " $1) } ' 40 | # Delete the output directory 41 | hadoop fs -rm -r -f -skipTrash ${OUTPUT} 42 | 43 | # Run terasort 44 | time hadoop jar $MR_EXAMPLES_JAR terasort \ 45 | -Dmapreduce.map.log.level=INFO \ 46 | -Dmapreduce.reduce.log.level=INFO \ 47 | -Dyarn.app.mapreduce.am.log.level=INFO \ 48 | -Dio.file.buffer.size=131072 \ 49 | -Dmapreduce.map.cpu.vcores=1 \ 50 | -Dmapreduce.map.java.opts=-Xmx1536m \ 51 | -Dmapreduce.map.maxattempts=1 \ 52 | -Dmapreduce.map.memory.mb=2048 \ 53 | -Dmapreduce.map.output.compress=true \ 54 | -Dmapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.Lz4Codec \ 55 | -Dmapreduce.reduce.cpu.vcores=1 \ 56 | -Dmapreduce.reduce.java.opts=-Xmx1536m \ 57 | -Dmapreduce.reduce.maxattempts=1 \ 58 | -Dmapreduce.reduce.memory.mb=2048 \ 59 | -Dmapreduce.task.io.sort.factor=300 \ 60 | -Dmapreduce.task.io.sort.mb=384 \ 61 | -Dyarn.app.mapreduce.am.command.opts=-Xmx1900m \ 62 | -Dyarn.app.mapreduce.am.resource.mb=2024 \ 63 | -Dmapred.task.timeout=12000000 \ 64 | -Dmapred.reduce.tasks=92 \ 65 | -Dmapreduce.terasort.output.replication=1 \ 66 | ${INPUT} ${OUTPUT} >> $RESULTSFILE 2>&1 67 | -------------------------------------------------------------------------------- /EMR Scripts/terasort_results_2016-12-15:15:04:51: -------------------------------------------------------------------------------- 1 | 16/12/15 15:04:57 INFO terasort.TeraSort: starting 2 | 16/12/15 15:04:59 INFO input.FileInputFormat: Total input paths to process : 92 3 | Spent 346ms computing base-splits. 4 | Spent 34ms computing TeraScheduler splits. 5 | Computing input splits took 381ms 6 | Sampling 10 splits of 14904 7 | Making 92 from 100000 sampled records 8 | Computing parititions took 1249ms 9 | Spent 1632ms computing partitions. 10 | 16/12/15 15:05:01 INFO client.RMProxy: Connecting to ResourceManager at ip-10-0-0-230.us-west-2.compute.internal/10.0.0.230:8032 11 | 16/12/15 15:05:02 INFO mapreduce.JobSubmitter: number of splits:14904 12 | 16/12/15 15:05:02 INFO Configuration.deprecation: mapred.reduce.tasks is deprecated. Instead, use mapreduce.job.reduces 13 | 16/12/15 15:05:02 INFO Configuration.deprecation: mapred.task.timeout is deprecated. Instead, use mapreduce.task.timeout 14 | 16/12/15 15:05:02 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1481813039325_0002 15 | 16/12/15 15:05:02 INFO impl.YarnClientImpl: Submitted application application_1481813039325_0002 16 | 16/12/15 15:05:02 INFO mapreduce.Job: The url to track the job: http://ip-10-0-0-230.us-west-2.compute.internal:20888/proxy/application_1481813039325_0002/ 17 | 16/12/15 15:05:02 INFO mapreduce.Job: Running job: job_1481813039325_0002 18 | 16/12/15 15:05:10 INFO mapreduce.Job: Job job_1481813039325_0002 running in uber mode : false 19 | 16/12/15 15:05:10 INFO mapreduce.Job: map 0% reduce 0% 20 | 16/12/15 15:06:23 INFO mapreduce.Job: map 1% reduce 0% 21 | 16/12/15 15:07:41 INFO mapreduce.Job: map 2% reduce 0% 22 | 16/12/15 15:09:00 INFO mapreduce.Job: map 3% reduce 0% 23 | 16/12/15 15:10:26 INFO mapreduce.Job: map 4% reduce 0% 24 | 16/12/15 15:11:48 INFO mapreduce.Job: map 5% reduce 0% 25 | 16/12/15 15:13:13 INFO mapreduce.Job: map 6% reduce 0% 26 | 16/12/15 15:14:39 INFO mapreduce.Job: map 7% reduce 0% 27 | 16/12/15 15:16:02 INFO mapreduce.Job: map 8% reduce 0% 28 | 16/12/15 15:17:23 INFO mapreduce.Job: map 9% reduce 0% 29 | 16/12/15 15:18:43 INFO mapreduce.Job: map 10% reduce 0% 30 | 16/12/15 15:19:07 INFO mapreduce.Job: map 10% reduce 1% 31 | 16/12/15 15:20:03 INFO mapreduce.Job: map 11% reduce 1% 32 | 16/12/15 15:21:24 INFO mapreduce.Job: map 12% reduce 1% 33 | 16/12/15 15:22:48 INFO mapreduce.Job: map 13% reduce 1% 34 | 16/12/15 15:24:15 INFO mapreduce.Job: map 14% reduce 1% 35 | 16/12/15 15:25:42 INFO mapreduce.Job: map 15% reduce 1% 36 | 16/12/15 15:27:10 INFO mapreduce.Job: map 16% reduce 1% 37 | 16/12/15 15:28:35 INFO mapreduce.Job: map 17% reduce 1% 38 | 16/12/15 15:29:01 INFO mapreduce.Job: map 17% reduce 2% 39 | 16/12/15 15:30:01 INFO mapreduce.Job: map 18% reduce 2% 40 | 16/12/15 15:31:24 INFO mapreduce.Job: map 19% reduce 2% 41 | 16/12/15 15:32:51 INFO mapreduce.Job: map 20% reduce 2% 42 | 16/12/15 15:34:22 INFO mapreduce.Job: map 21% reduce 2% 43 | 16/12/15 15:35:51 INFO mapreduce.Job: map 22% reduce 2% 44 | 16/12/15 15:36:01 INFO mapreduce.Job: map 22% reduce 3% 45 | 16/12/15 15:37:18 INFO mapreduce.Job: map 23% reduce 3% 46 | 16/12/15 15:38:47 INFO mapreduce.Job: map 24% reduce 3% 47 | 16/12/15 15:40:15 INFO mapreduce.Job: map 25% reduce 3% 48 | 16/12/15 15:41:44 INFO mapreduce.Job: map 26% reduce 3% 49 | 16/12/15 15:41:45 INFO mapreduce.Job: map 26% reduce 4% 50 | 16/12/15 15:43:14 INFO mapreduce.Job: map 27% reduce 4% 51 | 16/12/15 15:44:46 INFO mapreduce.Job: map 28% reduce 4% 52 | 16/12/15 15:46:12 INFO mapreduce.Job: map 29% reduce 4% 53 | 16/12/15 15:46:28 INFO mapreduce.Job: map 29% reduce 5% 54 | 16/12/15 15:47:40 INFO mapreduce.Job: map 30% reduce 5% 55 | 16/12/15 15:49:07 INFO mapreduce.Job: map 31% reduce 5% 56 | 16/12/15 15:50:47 INFO mapreduce.Job: map 32% reduce 5% 57 | 16/12/15 15:51:10 INFO mapreduce.Job: map 32% reduce 6% 58 | 16/12/15 15:52:18 INFO mapreduce.Job: map 33% reduce 6% 59 | 16/12/15 15:53:47 INFO mapreduce.Job: map 34% reduce 6% 60 | 16/12/15 15:55:13 INFO mapreduce.Job: map 34% reduce 7% 61 | 16/12/15 15:55:27 INFO mapreduce.Job: map 35% reduce 7% 62 | 16/12/15 15:56:55 INFO mapreduce.Job: map 36% reduce 7% 63 | 16/12/15 15:58:30 INFO mapreduce.Job: map 37% reduce 7% 64 | 16/12/15 15:59:03 INFO mapreduce.Job: map 37% reduce 8% 65 | 16/12/15 16:00:01 INFO mapreduce.Job: map 38% reduce 8% 66 | 16/12/15 16:01:34 INFO mapreduce.Job: map 39% reduce 8% 67 | 16/12/15 16:02:56 INFO mapreduce.Job: map 39% reduce 9% 68 | 16/12/15 16:03:18 INFO mapreduce.Job: map 40% reduce 9% 69 | 16/12/15 16:04:54 INFO mapreduce.Job: map 41% reduce 9% 70 | 16/12/15 16:06:19 INFO mapreduce.Job: map 41% reduce 10% 71 | 16/12/15 16:06:22 INFO mapreduce.Job: map 42% reduce 10% 72 | 16/12/15 16:07:59 INFO mapreduce.Job: map 43% reduce 10% 73 | 16/12/15 16:09:29 INFO mapreduce.Job: map 44% reduce 10% 74 | 16/12/15 16:09:31 INFO mapreduce.Job: map 44% reduce 11% 75 | 16/12/15 16:11:15 INFO mapreduce.Job: map 45% reduce 11% 76 | 16/12/15 16:12:47 INFO mapreduce.Job: map 45% reduce 12% 77 | 16/12/15 16:12:59 INFO mapreduce.Job: map 46% reduce 12% 78 | 16/12/15 16:14:29 INFO mapreduce.Job: map 47% reduce 12% 79 | 16/12/15 16:15:40 INFO mapreduce.Job: map 47% reduce 13% 80 | 16/12/15 16:16:07 INFO mapreduce.Job: map 48% reduce 13% 81 | 16/12/15 16:17:43 INFO mapreduce.Job: map 49% reduce 13% 82 | 16/12/15 16:18:29 INFO mapreduce.Job: map 49% reduce 14% 83 | 16/12/15 16:19:21 INFO mapreduce.Job: map 50% reduce 14% 84 | 16/12/15 16:20:55 INFO mapreduce.Job: map 51% reduce 14% 85 | 16/12/15 16:21:38 INFO mapreduce.Job: map 51% reduce 15% 86 | 16/12/15 16:22:39 INFO mapreduce.Job: map 52% reduce 15% 87 | 16/12/15 16:24:21 INFO mapreduce.Job: map 53% reduce 15% 88 | 16/12/15 16:24:45 INFO mapreduce.Job: map 53% reduce 16% 89 | 16/12/15 16:25:58 INFO mapreduce.Job: map 54% reduce 16% 90 | 16/12/15 16:27:32 INFO mapreduce.Job: map 55% reduce 16% 91 | 16/12/15 16:27:57 INFO mapreduce.Job: map 55% reduce 17% 92 | 16/12/15 16:29:02 INFO mapreduce.Job: map 56% reduce 17% 93 | 16/12/15 16:30:39 INFO mapreduce.Job: map 57% reduce 17% 94 | 16/12/15 16:31:51 INFO mapreduce.Job: map 57% reduce 18% 95 | 16/12/15 16:32:17 INFO mapreduce.Job: map 58% reduce 18% 96 | 16/12/15 16:33:42 INFO mapreduce.Job: map 59% reduce 18% 97 | 16/12/15 16:35:17 INFO mapreduce.Job: map 60% reduce 18% 98 | 16/12/15 16:36:25 INFO mapreduce.Job: map 60% reduce 19% 99 | 16/12/15 16:36:49 INFO mapreduce.Job: map 61% reduce 19% 100 | 16/12/15 16:38:15 INFO mapreduce.Job: map 62% reduce 19% 101 | 16/12/15 16:39:48 INFO mapreduce.Job: map 63% reduce 19% 102 | 16/12/15 16:41:23 INFO mapreduce.Job: map 63% reduce 20% 103 | 16/12/15 16:41:25 INFO mapreduce.Job: map 64% reduce 20% 104 | 16/12/15 16:42:57 INFO mapreduce.Job: map 65% reduce 20% 105 | 16/12/15 16:44:18 INFO mapreduce.Job: map 66% reduce 20% 106 | 16/12/15 16:45:50 INFO mapreduce.Job: map 67% reduce 20% 107 | 16/12/15 16:46:06 INFO mapreduce.Job: map 67% reduce 21% 108 | 16/12/15 16:47:20 INFO mapreduce.Job: map 68% reduce 21% 109 | 16/12/15 16:48:50 INFO mapreduce.Job: map 69% reduce 21% 110 | 16/12/15 16:50:25 INFO mapreduce.Job: map 70% reduce 21% 111 | 16/12/15 16:51:10 INFO mapreduce.Job: map 70% reduce 22% 112 | 16/12/15 16:51:59 INFO mapreduce.Job: map 71% reduce 22% 113 | 16/12/15 16:53:26 INFO mapreduce.Job: map 72% reduce 22% 114 | 16/12/15 16:54:56 INFO mapreduce.Job: map 73% reduce 22% 115 | 16/12/15 16:55:57 INFO mapreduce.Job: map 73% reduce 23% 116 | 16/12/15 16:56:23 INFO mapreduce.Job: map 74% reduce 23% 117 | 16/12/15 16:57:52 INFO mapreduce.Job: map 75% reduce 23% 118 | 16/12/15 16:59:27 INFO mapreduce.Job: map 76% reduce 23% 119 | 16/12/15 17:00:44 INFO mapreduce.Job: map 76% reduce 24% 120 | 16/12/15 17:01:01 INFO mapreduce.Job: map 77% reduce 24% 121 | 16/12/15 17:02:17 INFO mapreduce.Job: map 78% reduce 24% 122 | 16/12/15 17:03:52 INFO mapreduce.Job: map 79% reduce 24% 123 | 16/12/15 17:05:26 INFO mapreduce.Job: map 80% reduce 24% 124 | 16/12/15 17:05:41 INFO mapreduce.Job: map 80% reduce 25% 125 | 16/12/15 17:06:55 INFO mapreduce.Job: map 81% reduce 25% 126 | 16/12/15 17:08:24 INFO mapreduce.Job: map 82% reduce 25% 127 | 16/12/15 17:10:00 INFO mapreduce.Job: map 83% reduce 25% 128 | 16/12/15 17:10:34 INFO mapreduce.Job: map 83% reduce 26% 129 | 16/12/15 17:11:33 INFO mapreduce.Job: map 84% reduce 26% 130 | 16/12/15 17:12:50 INFO mapreduce.Job: map 85% reduce 26% 131 | 16/12/15 17:14:24 INFO mapreduce.Job: map 86% reduce 26% 132 | 16/12/15 17:15:22 INFO mapreduce.Job: map 86% reduce 27% 133 | 16/12/15 17:15:56 INFO mapreduce.Job: map 87% reduce 27% 134 | 16/12/15 17:17:30 INFO mapreduce.Job: map 88% reduce 27% 135 | 16/12/15 17:18:56 INFO mapreduce.Job: map 89% reduce 27% 136 | 16/12/15 17:20:21 INFO mapreduce.Job: map 89% reduce 28% 137 | 16/12/15 17:20:27 INFO mapreduce.Job: map 90% reduce 28% 138 | 16/12/15 17:21:56 INFO mapreduce.Job: map 91% reduce 28% 139 | 16/12/15 17:23:28 INFO mapreduce.Job: map 92% reduce 28% 140 | 16/12/15 17:24:59 INFO mapreduce.Job: map 93% reduce 28% 141 | 16/12/15 17:25:14 INFO mapreduce.Job: map 93% reduce 29% 142 | 16/12/15 17:26:31 INFO mapreduce.Job: map 94% reduce 29% 143 | 16/12/15 17:28:00 INFO mapreduce.Job: map 95% reduce 29% 144 | 16/12/15 17:29:29 INFO mapreduce.Job: map 96% reduce 29% 145 | 16/12/15 17:30:03 INFO mapreduce.Job: map 96% reduce 30% 146 | 16/12/15 17:30:51 INFO mapreduce.Job: map 97% reduce 30% 147 | 16/12/15 17:32:24 INFO mapreduce.Job: map 98% reduce 30% 148 | 16/12/15 17:33:58 INFO mapreduce.Job: map 99% reduce 30% 149 | 16/12/15 17:34:56 INFO mapreduce.Job: map 99% reduce 31% 150 | 16/12/15 17:35:26 INFO mapreduce.Job: map 100% reduce 31% 151 | 16/12/15 17:36:05 INFO mapreduce.Job: map 100% reduce 32% 152 | 16/12/15 17:36:06 INFO mapreduce.Job: map 100% reduce 34% 153 | 16/12/15 17:36:07 INFO mapreduce.Job: map 100% reduce 36% 154 | 16/12/15 17:36:08 INFO mapreduce.Job: map 100% reduce 38% 155 | 16/12/15 17:36:09 INFO mapreduce.Job: map 100% reduce 40% 156 | 16/12/15 17:36:10 INFO mapreduce.Job: map 100% reduce 41% 157 | 16/12/15 17:36:11 INFO mapreduce.Job: map 100% reduce 43% 158 | 16/12/15 17:36:12 INFO mapreduce.Job: map 100% reduce 44% 159 | 16/12/15 17:36:13 INFO mapreduce.Job: map 100% reduce 45% 160 | 16/12/15 17:36:14 INFO mapreduce.Job: map 100% reduce 46% 161 | 16/12/15 17:36:15 INFO mapreduce.Job: map 100% reduce 47% 162 | 16/12/15 17:36:16 INFO mapreduce.Job: map 100% reduce 48% 163 | 16/12/15 17:36:18 INFO mapreduce.Job: map 100% reduce 49% 164 | 16/12/15 17:36:19 INFO mapreduce.Job: map 100% reduce 50% 165 | 16/12/15 17:36:21 INFO mapreduce.Job: map 100% reduce 51% 166 | 16/12/15 17:36:22 INFO mapreduce.Job: map 100% reduce 52% 167 | 16/12/15 17:36:24 INFO mapreduce.Job: map 100% reduce 53% 168 | 16/12/15 17:36:27 INFO mapreduce.Job: map 100% reduce 54% 169 | 16/12/15 17:36:29 INFO mapreduce.Job: map 100% reduce 55% 170 | 16/12/15 17:36:32 INFO mapreduce.Job: map 100% reduce 56% 171 | 16/12/15 17:36:35 INFO mapreduce.Job: map 100% reduce 57% 172 | 16/12/15 17:36:38 INFO mapreduce.Job: map 100% reduce 58% 173 | 16/12/15 17:36:43 INFO mapreduce.Job: map 100% reduce 59% 174 | 16/12/15 17:36:48 INFO mapreduce.Job: map 100% reduce 60% 175 | 16/12/15 17:36:55 INFO mapreduce.Job: map 100% reduce 61% 176 | 16/12/15 17:37:05 INFO mapreduce.Job: map 100% reduce 62% 177 | 16/12/15 17:37:12 INFO mapreduce.Job: map 100% reduce 63% 178 | 16/12/15 17:37:27 INFO mapreduce.Job: map 100% reduce 64% 179 | 16/12/15 17:37:56 INFO mapreduce.Job: map 100% reduce 65% 180 | 16/12/15 17:38:21 INFO mapreduce.Job: map 100% reduce 66% 181 | 16/12/15 17:38:50 INFO mapreduce.Job: map 100% reduce 67% 182 | 16/12/15 17:39:17 INFO mapreduce.Job: map 100% reduce 68% 183 | 16/12/15 17:39:44 INFO mapreduce.Job: map 100% reduce 69% 184 | 16/12/15 17:40:13 INFO mapreduce.Job: map 100% reduce 70% 185 | 16/12/15 17:40:41 INFO mapreduce.Job: map 100% reduce 71% 186 | 16/12/15 17:41:09 INFO mapreduce.Job: map 100% reduce 72% 187 | 16/12/15 17:41:37 INFO mapreduce.Job: map 100% reduce 73% 188 | 16/12/15 17:42:05 INFO mapreduce.Job: map 100% reduce 74% 189 | 16/12/15 17:42:34 INFO mapreduce.Job: map 100% reduce 75% 190 | 16/12/15 17:43:04 INFO mapreduce.Job: map 100% reduce 76% 191 | 16/12/15 17:43:34 INFO mapreduce.Job: map 100% reduce 77% 192 | 16/12/15 17:44:02 INFO mapreduce.Job: map 100% reduce 78% 193 | 16/12/15 17:44:31 INFO mapreduce.Job: map 100% reduce 79% 194 | 16/12/15 17:44:59 INFO mapreduce.Job: map 100% reduce 80% 195 | 16/12/15 17:45:29 INFO mapreduce.Job: map 100% reduce 81% 196 | 16/12/15 17:45:56 INFO mapreduce.Job: map 100% reduce 82% 197 | 16/12/15 17:46:25 INFO mapreduce.Job: map 100% reduce 83% 198 | 16/12/15 17:46:53 INFO mapreduce.Job: map 100% reduce 84% 199 | 16/12/15 17:47:22 INFO mapreduce.Job: map 100% reduce 85% 200 | 16/12/15 17:47:50 INFO mapreduce.Job: map 100% reduce 86% 201 | 16/12/15 17:48:20 INFO mapreduce.Job: map 100% reduce 87% 202 | 16/12/15 17:48:48 INFO mapreduce.Job: map 100% reduce 88% 203 | 16/12/15 17:49:16 INFO mapreduce.Job: map 100% reduce 89% 204 | 16/12/15 17:49:45 INFO mapreduce.Job: map 100% reduce 90% 205 | 16/12/15 17:50:14 INFO mapreduce.Job: map 100% reduce 91% 206 | 16/12/15 17:50:42 INFO mapreduce.Job: map 100% reduce 92% 207 | 16/12/15 17:51:16 INFO mapreduce.Job: map 100% reduce 93% 208 | 16/12/15 17:52:00 INFO mapreduce.Job: map 100% reduce 94% 209 | 16/12/15 17:52:28 INFO mapreduce.Job: map 100% reduce 95% 210 | 16/12/15 17:53:01 INFO mapreduce.Job: map 100% reduce 96% 211 | 16/12/15 17:54:20 INFO mapreduce.Job: map 100% reduce 97% 212 | 16/12/15 17:54:46 INFO mapreduce.Job: map 100% reduce 98% 213 | 16/12/15 17:55:17 INFO mapreduce.Job: map 100% reduce 99% 214 | 16/12/15 17:57:10 INFO mapreduce.Job: map 100% reduce 100% 215 | 16/12/15 18:02:53 INFO mapreduce.Job: Job job_1481813039325_0002 completed successfully 216 | 16/12/15 18:02:53 INFO mapreduce.Job: Counters: 55 217 | File System Counters 218 | FILE: Number of bytes read=562586977882 219 | FILE: Number of bytes written=1112968940412 220 | FILE: Number of read operations=0 221 | FILE: Number of large read operations=0 222 | FILE: Number of write operations=0 223 | HDFS: Number of bytes read=2026944 224 | HDFS: Number of bytes written=0 225 | HDFS: Number of read operations=14904 226 | HDFS: Number of large read operations=0 227 | HDFS: Number of write operations=0 228 | S3: Number of bytes read=0 229 | S3: Number of bytes written=1000000000000 230 | S3: Number of read operations=0 231 | S3: Number of large read operations=0 232 | S3: Number of write operations=0 233 | Job Counters 234 | Killed map tasks=3 235 | Launched map tasks=14907 236 | Launched reduce tasks=92 237 | Data-local map tasks=14907 238 | Total time spent by all maps in occupied slots (ms)=60157948864 239 | Total time spent by all reduces in occupied slots (ms)=45815133824 240 | Total time spent by all map tasks (ms)=939967951 241 | Total time spent by all reduce tasks (ms)=715861466 242 | Total vcore-milliseconds taken by all map tasks=939967951 243 | Total vcore-milliseconds taken by all reduce tasks=715861466 244 | Total megabyte-milliseconds taken by all map tasks=1925054363648 245 | Total megabyte-milliseconds taken by all reduce tasks=1466084282368 246 | Map-Reduce Framework 247 | Map input records=10000000000 248 | Map output records=10000000000 249 | Map output bytes=1020000000000 250 | Map output materialized bytes=548430807028 251 | Input split bytes=2026944 252 | Combine input records=0 253 | Combine output records=0 254 | Reduce input groups=10000000000 255 | Reduce shuffle bytes=548430807028 256 | Reduce input records=10000000000 257 | Reduce output records=10000000000 258 | Spilled Records=20000000000 259 | Shuffled Maps =1371168 260 | Failed Shuffles=0 261 | Merged Map outputs=1371168 262 | GC time elapsed (ms)=14424091 263 | CPU time spent (ms)=231584060 264 | Physical memory (bytes) snapshot=15973874188288 265 | Virtual memory (bytes) snapshot=55079509270528 266 | Total committed heap usage (bytes)=22899044384768 267 | Shuffle Errors 268 | BAD_ID=0 269 | CONNECTION=0 270 | IO_ERROR=0 271 | WRONG_LENGTH=0 272 | WRONG_MAP=0 273 | WRONG_REDUCE=0 274 | File Input Format Counters 275 | Bytes Read=0 276 | File Output Format Counters 277 | Bytes Written=1000000000000 278 | 16/12/15 18:02:53 INFO terasort.TeraSort: done 279 | -------------------------------------------------------------------------------- /EMR Scripts/teravalidateEMR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | trap "" HUP 4 | 5 | #if [ $EUID -eq 0 ]; then 6 | # echo "this script must not be run as root. su to hdfs user to run" 7 | # exit 1 8 | #fi 9 | 10 | MR_EXAMPLES_JAR=/usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples-2.7.3-amzn-0.jar 11 | 12 | #SIZE=500G 13 | #SIZE=100G 14 | SIZE=1T 15 | #SIZE=1G 16 | #SIZE=10G 17 | 18 | 19 | LOGDIR=logs 20 | 21 | if [ ! -d "$LOGDIR" ] 22 | then 23 | mkdir ./$LOGDIR 24 | fi 25 | 26 | DATE=`date +%Y-%m-%d:%H:%M:%S` 27 | 28 | RESULTSFILE="./$LOGDIR/teravalidate_results_$DATE" 29 | 30 | 31 | OUTPUT=s3://sunileman1/data/sandbox/poc/teragen/${SIZE}-terasort-output 32 | REPORT=s3://sunileman1/data/sandbox/poc/teragen/${SIZE}-terasort-report 33 | 34 | 35 | # teravalidate.sh 36 | # Kill any running MapReduce jobs 37 | mapred job -list | grep job_ | awk ' { system("mapred job -kill " $1) } ' 38 | # Delete the output directory 39 | hadoop fs -rm -r -f -skipTrash ${REPORT} 40 | 41 | # Run teravalidate 42 | time hadoop jar $MR_EXAMPLES_JAR teravalidate \ 43 | -Ddfs.blocksize=256M \ 44 | -Dio.file.buffer.size=131072 \ 45 | -Dmapreduce.map.memory.mb=2048 \ 46 | -Dmapreduce.map.java.opts=-Xmx1536m \ 47 | -Dmapreduce.reduce.memory.mb=2048 \ 48 | -Dmapreduce.reduce.java.opts=-Xmx1536m \ 49 | -Dyarn.app.mapreduce.am.command.opts=-Xmx1900m \ 50 | -Dyarn.app.mapreduce.am.resource.mb=2024 \ 51 | -Dmapred.task.timeout=12000000 \ 52 | -Dmapreduce.task.io.sort.mb=1 \ 53 | -Dmapred.map.tasks=185 \ 54 | -Dmapred.reduce.tasks=185 \ 55 | ${OUTPUT} ${REPORT} >> $RESULTSFILE 2>&1 56 | -------------------------------------------------------------------------------- /EMR Scripts/teravalidate_results_2016-12-15:20:13:11: -------------------------------------------------------------------------------- 1 | 16/12/15 20:13:19 INFO client.RMProxy: Connecting to ResourceManager at ip-10-0-0-230.us-west-2.compute.internal/10.0.0.230:8032 2 | 16/12/15 20:13:20 INFO input.FileInputFormat: Total input paths to process : 92 3 | Spent 263ms computing base-splits. 4 | Spent 2ms computing TeraScheduler splits. 5 | 16/12/15 20:13:21 INFO mapreduce.JobSubmitter: number of splits:92 6 | 16/12/15 20:13:21 INFO Configuration.deprecation: mapred.reduce.tasks is deprecated. Instead, use mapreduce.job.reduces 7 | 16/12/15 20:13:21 INFO Configuration.deprecation: mapred.task.timeout is deprecated. Instead, use mapreduce.task.timeout 8 | 16/12/15 20:13:21 INFO Configuration.deprecation: mapred.map.tasks is deprecated. Instead, use mapreduce.job.maps 9 | 16/12/15 20:13:21 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1481813039325_0003 10 | 16/12/15 20:13:21 INFO impl.YarnClientImpl: Submitted application application_1481813039325_0003 11 | 16/12/15 20:13:21 INFO mapreduce.Job: The url to track the job: http://ip-10-0-0-230.us-west-2.compute.internal:20888/proxy/application_1481813039325_0003/ 12 | 16/12/15 20:13:21 INFO mapreduce.Job: Running job: job_1481813039325_0003 13 | 16/12/15 20:13:33 INFO mapreduce.Job: Job job_1481813039325_0003 running in uber mode : false 14 | 16/12/15 20:13:33 INFO mapreduce.Job: map 0% reduce 0% 15 | 16/12/15 20:13:57 INFO mapreduce.Job: map 1% reduce 0% 16 | 16/12/15 20:14:18 INFO mapreduce.Job: map 2% reduce 0% 17 | 16/12/15 20:14:38 INFO mapreduce.Job: map 3% reduce 0% 18 | 16/12/15 20:15:00 INFO mapreduce.Job: map 4% reduce 0% 19 | 16/12/15 20:15:20 INFO mapreduce.Job: map 5% reduce 0% 20 | 16/12/15 20:15:41 INFO mapreduce.Job: map 6% reduce 0% 21 | 16/12/15 20:16:02 INFO mapreduce.Job: map 7% reduce 0% 22 | 16/12/15 20:16:23 INFO mapreduce.Job: map 8% reduce 0% 23 | 16/12/15 20:16:44 INFO mapreduce.Job: map 9% reduce 0% 24 | 16/12/15 20:17:05 INFO mapreduce.Job: map 10% reduce 0% 25 | 16/12/15 20:17:26 INFO mapreduce.Job: map 11% reduce 0% 26 | 16/12/15 20:17:47 INFO mapreduce.Job: map 12% reduce 0% 27 | 16/12/15 20:18:07 INFO mapreduce.Job: map 13% reduce 0% 28 | 16/12/15 20:18:29 INFO mapreduce.Job: map 14% reduce 0% 29 | 16/12/15 20:18:49 INFO mapreduce.Job: map 15% reduce 0% 30 | 16/12/15 20:19:10 INFO mapreduce.Job: map 16% reduce 0% 31 | 16/12/15 20:19:31 INFO mapreduce.Job: map 17% reduce 0% 32 | 16/12/15 20:19:52 INFO mapreduce.Job: map 18% reduce 0% 33 | 16/12/15 20:20:13 INFO mapreduce.Job: map 19% reduce 0% 34 | 16/12/15 20:20:33 INFO mapreduce.Job: map 20% reduce 0% 35 | 16/12/15 20:20:55 INFO mapreduce.Job: map 21% reduce 0% 36 | 16/12/15 20:21:15 INFO mapreduce.Job: map 22% reduce 0% 37 | 16/12/15 20:21:36 INFO mapreduce.Job: map 23% reduce 0% 38 | 16/12/15 20:21:58 INFO mapreduce.Job: map 24% reduce 0% 39 | 16/12/15 20:22:18 INFO mapreduce.Job: map 25% reduce 0% 40 | 16/12/15 20:22:39 INFO mapreduce.Job: map 26% reduce 0% 41 | 16/12/15 20:23:00 INFO mapreduce.Job: map 27% reduce 0% 42 | 16/12/15 20:23:20 INFO mapreduce.Job: map 28% reduce 0% 43 | 16/12/15 20:23:42 INFO mapreduce.Job: map 29% reduce 0% 44 | 16/12/15 20:24:03 INFO mapreduce.Job: map 30% reduce 0% 45 | 16/12/15 20:24:23 INFO mapreduce.Job: map 31% reduce 0% 46 | 16/12/15 20:24:44 INFO mapreduce.Job: map 32% reduce 0% 47 | 16/12/15 20:25:05 INFO mapreduce.Job: map 33% reduce 0% 48 | 16/12/15 20:25:26 INFO mapreduce.Job: map 34% reduce 0% 49 | 16/12/15 20:25:47 INFO mapreduce.Job: map 35% reduce 0% 50 | 16/12/15 20:26:08 INFO mapreduce.Job: map 36% reduce 0% 51 | 16/12/15 20:26:29 INFO mapreduce.Job: map 37% reduce 0% 52 | 16/12/15 20:26:49 INFO mapreduce.Job: map 38% reduce 0% 53 | 16/12/15 20:27:11 INFO mapreduce.Job: map 39% reduce 0% 54 | 16/12/15 20:27:31 INFO mapreduce.Job: map 40% reduce 0% 55 | 16/12/15 20:27:53 INFO mapreduce.Job: map 41% reduce 0% 56 | 16/12/15 20:28:13 INFO mapreduce.Job: map 42% reduce 0% 57 | 16/12/15 20:28:34 INFO mapreduce.Job: map 43% reduce 0% 58 | 16/12/15 20:28:55 INFO mapreduce.Job: map 44% reduce 0% 59 | 16/12/15 20:29:16 INFO mapreduce.Job: map 45% reduce 0% 60 | 16/12/15 20:29:37 INFO mapreduce.Job: map 46% reduce 0% 61 | 16/12/15 20:29:58 INFO mapreduce.Job: map 47% reduce 0% 62 | 16/12/15 20:30:18 INFO mapreduce.Job: map 48% reduce 0% 63 | 16/12/15 20:30:39 INFO mapreduce.Job: map 49% reduce 0% 64 | 16/12/15 20:31:00 INFO mapreduce.Job: map 50% reduce 0% 65 | 16/12/15 20:31:21 INFO mapreduce.Job: map 51% reduce 0% 66 | 16/12/15 20:31:42 INFO mapreduce.Job: map 52% reduce 0% 67 | 16/12/15 20:32:03 INFO mapreduce.Job: map 53% reduce 0% 68 | 16/12/15 20:32:24 INFO mapreduce.Job: map 54% reduce 0% 69 | 16/12/15 20:32:44 INFO mapreduce.Job: map 55% reduce 0% 70 | 16/12/15 20:33:06 INFO mapreduce.Job: map 56% reduce 0% 71 | 16/12/15 20:33:26 INFO mapreduce.Job: map 57% reduce 0% 72 | 16/12/15 20:33:47 INFO mapreduce.Job: map 58% reduce 0% 73 | 16/12/15 20:34:08 INFO mapreduce.Job: map 59% reduce 0% 74 | 16/12/15 20:34:29 INFO mapreduce.Job: map 60% reduce 0% 75 | 16/12/15 20:34:50 INFO mapreduce.Job: map 61% reduce 0% 76 | 16/12/15 20:35:03 INFO mapreduce.Job: map 62% reduce 0% 77 | 16/12/15 20:35:24 INFO mapreduce.Job: map 63% reduce 0% 78 | 16/12/15 20:35:38 INFO mapreduce.Job: map 64% reduce 0% 79 | 16/12/15 20:35:44 INFO mapreduce.Job: map 65% reduce 0% 80 | 16/12/15 20:35:47 INFO mapreduce.Job: map 66% reduce 0% 81 | 16/12/15 20:35:55 INFO mapreduce.Job: map 67% reduce 0% 82 | 16/12/15 20:35:57 INFO mapreduce.Job: map 67% reduce 3% 83 | 16/12/15 20:36:01 INFO mapreduce.Job: map 68% reduce 3% 84 | 16/12/15 20:36:03 INFO mapreduce.Job: map 68% reduce 4% 85 | 16/12/15 20:36:08 INFO mapreduce.Job: map 69% reduce 4% 86 | 16/12/15 20:36:13 INFO mapreduce.Job: map 70% reduce 4% 87 | 16/12/15 20:36:15 INFO mapreduce.Job: map 70% reduce 6% 88 | 16/12/15 20:36:16 INFO mapreduce.Job: map 71% reduce 6% 89 | 16/12/15 20:36:17 INFO mapreduce.Job: map 72% reduce 6% 90 | 16/12/15 20:36:18 INFO mapreduce.Job: map 72% reduce 7% 91 | 16/12/15 20:36:19 INFO mapreduce.Job: map 73% reduce 7% 92 | 16/12/15 20:36:21 INFO mapreduce.Job: map 74% reduce 8% 93 | 16/12/15 20:36:22 INFO mapreduce.Job: map 75% reduce 8% 94 | 16/12/15 20:36:23 INFO mapreduce.Job: map 76% reduce 8% 95 | 16/12/15 20:36:24 INFO mapreduce.Job: map 77% reduce 11% 96 | 16/12/15 20:36:25 INFO mapreduce.Job: map 78% reduce 11% 97 | 16/12/15 20:36:26 INFO mapreduce.Job: map 79% reduce 11% 98 | 16/12/15 20:36:27 INFO mapreduce.Job: map 80% reduce 14% 99 | 16/12/15 20:36:30 INFO mapreduce.Job: map 81% reduce 16% 100 | 16/12/15 20:36:31 INFO mapreduce.Job: map 82% reduce 16% 101 | 16/12/15 20:36:33 INFO mapreduce.Job: map 83% reduce 17% 102 | 16/12/15 20:36:34 INFO mapreduce.Job: map 84% reduce 17% 103 | 16/12/15 20:36:35 INFO mapreduce.Job: map 85% reduce 17% 104 | 16/12/15 20:36:36 INFO mapreduce.Job: map 85% reduce 19% 105 | 16/12/15 20:36:39 INFO mapreduce.Job: map 85% reduce 20% 106 | 16/12/15 20:36:40 INFO mapreduce.Job: map 86% reduce 20% 107 | 16/12/15 20:36:42 INFO mapreduce.Job: map 86% reduce 21% 108 | 16/12/15 20:36:44 INFO mapreduce.Job: map 87% reduce 21% 109 | 16/12/15 20:36:46 INFO mapreduce.Job: map 88% reduce 21% 110 | 16/12/15 20:36:48 INFO mapreduce.Job: map 88% reduce 22% 111 | 16/12/15 20:36:53 INFO mapreduce.Job: map 89% reduce 22% 112 | 16/12/15 20:36:54 INFO mapreduce.Job: map 89% reduce 23% 113 | 16/12/15 20:36:56 INFO mapreduce.Job: map 90% reduce 23% 114 | 16/12/15 20:36:57 INFO mapreduce.Job: map 90% reduce 24% 115 | 16/12/15 20:36:58 INFO mapreduce.Job: map 92% reduce 24% 116 | 16/12/15 20:37:00 INFO mapreduce.Job: map 92% reduce 26% 117 | 16/12/15 20:37:01 INFO mapreduce.Job: map 93% reduce 26% 118 | 16/12/15 20:37:02 INFO mapreduce.Job: map 94% reduce 26% 119 | 16/12/15 20:37:03 INFO mapreduce.Job: map 94% reduce 28% 120 | 16/12/15 20:37:05 INFO mapreduce.Job: map 95% reduce 28% 121 | 16/12/15 20:37:06 INFO mapreduce.Job: map 95% reduce 29% 122 | 16/12/15 20:37:07 INFO mapreduce.Job: map 97% reduce 29% 123 | 16/12/15 20:37:09 INFO mapreduce.Job: map 97% reduce 31% 124 | 16/12/15 20:37:10 INFO mapreduce.Job: map 98% reduce 31% 125 | 16/12/15 20:37:11 INFO mapreduce.Job: map 99% reduce 31% 126 | 16/12/15 20:37:12 INFO mapreduce.Job: map 99% reduce 33% 127 | 16/12/15 20:37:15 INFO mapreduce.Job: map 100% reduce 33% 128 | 16/12/15 20:37:17 INFO mapreduce.Job: map 100% reduce 100% 129 | 16/12/15 20:37:18 INFO mapreduce.Job: Job job_1481813039325_0003 completed successfully 130 | 16/12/15 20:37:19 INFO mapreduce.Job: Counters: 54 131 | File System Counters 132 | FILE: Number of bytes read=4539 133 | FILE: Number of bytes written=11880200 134 | FILE: Number of read operations=0 135 | FILE: Number of large read operations=0 136 | FILE: Number of write operations=0 137 | HDFS: Number of bytes read=12604 138 | HDFS: Number of bytes written=0 139 | HDFS: Number of read operations=92 140 | HDFS: Number of large read operations=0 141 | HDFS: Number of write operations=0 142 | S3: Number of bytes read=0 143 | S3: Number of bytes written=27 144 | S3: Number of read operations=0 145 | S3: Number of large read operations=0 146 | S3: Number of write operations=0 147 | Job Counters 148 | Launched map tasks=92 149 | Launched reduce tasks=1 150 | Data-local map tasks=92 151 | Total time spent by all maps in occupied slots (ms)=8036067200 152 | Total time spent by all reduces in occupied slots (ms)=5665536 153 | Total time spent by all map tasks (ms)=125563550 154 | Total time spent by all reduce tasks (ms)=88524 155 | Total vcore-milliseconds taken by all map tasks=125563550 156 | Total vcore-milliseconds taken by all reduce tasks=88524 157 | Total megabyte-milliseconds taken by all map tasks=257154150400 158 | Total megabyte-milliseconds taken by all reduce tasks=181297152 159 | Map-Reduce Framework 160 | Map input records=10000000000 161 | Map output records=276 162 | Map output bytes=7636 163 | Map output materialized bytes=9019 164 | Input split bytes=12604 165 | Combine input records=0 166 | Combine output records=0 167 | Reduce input groups=185 168 | Reduce shuffle bytes=9019 169 | Reduce input records=276 170 | Reduce output records=1 171 | Spilled Records=552 172 | Shuffled Maps =92 173 | Failed Shuffles=0 174 | Merged Map outputs=92 175 | GC time elapsed (ms)=85055 176 | CPU time spent (ms)=18501730 177 | Physical memory (bytes) snapshot=56685854720 178 | Virtual memory (bytes) snapshot=341189570560 179 | Total committed heap usage (bytes)=111058354176 180 | Shuffle Errors 181 | BAD_ID=0 182 | CONNECTION=0 183 | IO_ERROR=0 184 | WRONG_LENGTH=0 185 | WRONG_MAP=0 186 | WRONG_REDUCE=0 187 | File Input Format Counters 188 | Bytes Read=0 189 | File Output Format Counters 190 | Bytes Written=27 191 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MapReduce-Performance_Testing 2 | MapReduce performance testing using TeraGen, TeraSort, and TeraValidate 3 | 4 | Full "How To" on running teragen, terasort, and teravalidate using my script here 5 | https://community.hortonworks.com/articles/71173/benchmarking-hadoop-with-teragen-terasort-and-tera.html 6 | -------------------------------------------------------------------------------- /ambari-blueprint: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "hive-site": { 5 | "javax.jdo.option.ConnectionUserName": "hive", 6 | "javax.jdo.option.ConnectionPassword": "hive" 7 | } 8 | } 9 | ], 10 | "host_groups": [ 11 | { 12 | "name": "host_group_slave_1", 13 | "configurations": [], 14 | "components": [ 15 | { 16 | "name": "NODEMANAGER" 17 | }, 18 | { 19 | "name": "METRICS_MONITOR" 20 | }, 21 | { 22 | "name": "DATANODE" 23 | }, 24 | { 25 | "name": "ZOOKEEPER_CLIENT" 26 | }, 27 | { 28 | "name": "HDFS_CLIENT" 29 | }, 30 | { 31 | "name": "YARN_CLIENT" 32 | }, 33 | { 34 | "name": "MAPREDUCE2_CLIENT" 35 | } 36 | ], 37 | "cardinality": "1" 38 | }, 39 | { 40 | "name": "host_group_master_1", 41 | "configurations": [], 42 | "components": [ 43 | { 44 | "name": "ZOOKEEPER_SERVER" 45 | }, 46 | { 47 | "name": "HISTORYSERVER" 48 | }, 49 | { 50 | "name": "NAMENODE" 51 | }, 52 | { 53 | "name": "HDFS_CLIENT" 54 | }, 55 | { 56 | "name": "YARN_CLIENT" 57 | }, 58 | { 59 | "name": "METRICS_MONITOR" 60 | }, 61 | { 62 | "name": "MAPREDUCE2_CLIENT" 63 | }, 64 | { 65 | "name": "APP_TIMELINE_SERVER" 66 | }, 67 | { 68 | "name": "SECONDARY_NAMENODE" 69 | }, 70 | { 71 | "name": "ZOOKEEPER_CLIENT" 72 | }, 73 | { 74 | "name": "RESOURCEMANAGER" 75 | }, 76 | { 77 | "name": "METRICS_COLLECTOR" 78 | } 79 | ], 80 | "cardinality": "1" 81 | } 82 | ], 83 | "Blueprints": { 84 | "blueprint_name": "hdp-small-default", 85 | "stack_name": "HDP", 86 | "stack_version": "2.4" 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /capacity-scheduler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | yarn.scheduler.capacity.default.minimum-user-limit-percent 5 | 100 6 | 7 | 8 | 9 | yarn.scheduler.capacity.maximum-am-resource-percent 10 | 0.2 11 | 12 | 13 | 14 | yarn.scheduler.capacity.maximum-applications 15 | 10000 16 | 17 | 18 | 19 | yarn.scheduler.capacity.node-locality-delay 20 | 40 21 | 22 | 23 | 24 | yarn.scheduler.capacity.resource-calculator 25 | org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator 26 | 27 | 28 | 29 | yarn.scheduler.capacity.root.accessible-node-labels 30 | * 31 | 32 | 33 | 34 | yarn.scheduler.capacity.root.acl_administer_queue 35 | * 36 | 37 | 38 | 39 | yarn.scheduler.capacity.root.capacity 40 | 100 41 | 42 | 43 | 44 | yarn.scheduler.capacity.root.default.acl_administer_jobs 45 | * 46 | 47 | 48 | 49 | yarn.scheduler.capacity.root.default.acl_submit_applications 50 | * 51 | 52 | 53 | 54 | yarn.scheduler.capacity.root.default.capacity 55 | 100 56 | 57 | 58 | 59 | yarn.scheduler.capacity.root.default.maximum-capacity 60 | 100 61 | 62 | 63 | 64 | yarn.scheduler.capacity.root.default.state 65 | RUNNING 66 | 67 | 68 | 69 | yarn.scheduler.capacity.root.default.user-limit-factor 70 | 1 71 | 72 | 73 | 74 | yarn.scheduler.capacity.root.queues 75 | default 76 | 77 | -------------------------------------------------------------------------------- /core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | fs.defaultFS 5 | hdfs://ip-10-0-73-146.ec2.internal:8020 6 | true 7 | 8 | 9 | 10 | fs.trash.interval 11 | 360 12 | 13 | 14 | 15 | ha.failover-controller.active-standby-elector.zk.op.retries 16 | 120 17 | 18 | 19 | 20 | hadoop.http.authentication.simple.anonymous.allowed 21 | true 22 | 23 | 24 | 25 | hadoop.proxyuser.falcon.groups 26 | * 27 | 28 | 29 | 30 | hadoop.proxyuser.hbase.groups 31 | * 32 | 33 | 34 | 35 | hadoop.proxyuser.hcat.groups 36 | * 37 | 38 | 39 | 40 | hadoop.proxyuser.hive.groups 41 | * 42 | 43 | 44 | 45 | hadoop.proxyuser.oozie.groups 46 | * 47 | 48 | 49 | 50 | hadoop.proxyuser.root.groups 51 | * 52 | 53 | 54 | 55 | hadoop.proxyuser.root.hosts 56 | * 57 | 58 | 59 | 60 | hadoop.security.auth_to_local 61 | DEFAULT 62 | 63 | 64 | 65 | hadoop.security.authentication 66 | simple 67 | 68 | 69 | 70 | hadoop.security.authorization 71 | false 72 | 73 | 74 | 75 | hadoop.security.key.provider.path 76 | 77 | 78 | 79 | 80 | io.compression.codecs 81 | org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec 82 | 83 | 84 | 85 | io.file.buffer.size 86 | 131072 87 | 88 | 89 | 90 | io.serializations 91 | org.apache.hadoop.io.serializer.WritableSerialization 92 | 93 | 94 | 95 | ipc.client.connect.max.retries 96 | 50 97 | 98 | 99 | 100 | ipc.client.connection.maxidletime 101 | 30000 102 | 103 | 104 | 105 | ipc.client.idlethreshold 106 | 8000 107 | 108 | 109 | 110 | ipc.server.tcpnodelay 111 | true 112 | 113 | 114 | 115 | mapreduce.jobtracker.webinterface.trusted 116 | false 117 | 118 | 119 | 120 | net.topology.script.file.name 121 | /etc/hadoop/conf/topology_script.py 122 | 123 | 124 | 125 | proxyuser_group 126 | hadoop 127 | -------------------------------------------------------------------------------- /dfsio.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | trap "" HUP 4 | 5 | #if [ $EUID -eq 0 ]; then 6 | # echo "this script must not be run as root. su to hdfs user to run" 7 | # exit 1 8 | #fi 9 | 10 | 11 | DFSIO_JAR=/usr/hdp/current/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient-tests.jar 12 | FILES=10 13 | FILESIZE=100000 14 | 15 | LOGDIR=logs 16 | 17 | if [ ! -d "$LOGDIR" ] 18 | then 19 | mkdir ./$LOGDIR 20 | fi 21 | 22 | DATE=`date +%Y-%m-%d:%H:%M:%S` 23 | 24 | 25 | DFSIO_WRITE_OUTPUT_FILE="./$LOGDIR/dfsio_write_results.txt_$DATE" 26 | 27 | DFSIO_READ_OUTPUT_FILE="./$LOGDIR/dfsio_read_results.txt_$DATE" 28 | 29 | 30 | 31 | echo Running DFSIO CLEAN job 32 | echo =============================================================== 33 | yarn jar $DFSIO_JAR TestDFSIO -clean 34 | 35 | 36 | 37 | echo Running DFSIO WRITE job 38 | echo =============================================================== 39 | yarn jar $DFSIO_JAR TestDFSIO \ 40 | -write -nrFiles $FILES \ 41 | -fileSize $FILESIZE \ 42 | -resFile $DFSIO_WRITE_OUTPUT_FILE 43 | 44 | 45 | echo Running DFSIO READ job 46 | echo =============================================================== 47 | yarn jar $DFSIO_JAR TestDFSIO \ 48 | -read -nrFiles $FILES \ 49 | -fileSize $FILESIZE \ 50 | -resFile $DFSIO_READ_OUTPUT_FILE 51 | 52 | -------------------------------------------------------------------------------- /mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | mapreduce.admin.map.child.java.opts 5 | -server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version} 6 | 7 | 8 | 9 | mapreduce.admin.reduce.child.java.opts 10 | -server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version} 11 | 12 | 13 | 14 | mapreduce.admin.user.env 15 | LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64 16 | 17 | 18 | 19 | mapreduce.am.max-attempts 20 | 2 21 | 22 | 23 | 24 | mapreduce.application.classpath 25 | $PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure 26 | 27 | 28 | 29 | mapreduce.application.framework.path 30 | /hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework 31 | 32 | 33 | 34 | mapreduce.cluster.administrators 35 | hadoop 36 | 37 | 38 | 39 | mapreduce.framework.name 40 | yarn 41 | 42 | 43 | 44 | mapreduce.job.counters.max 45 | 130 46 | 47 | 48 | 49 | mapreduce.job.emit-timeline-data 50 | false 51 | 52 | 53 | 54 | mapreduce.job.reduce.slowstart.completedmaps 55 | 0.05 56 | 57 | 58 | 59 | mapreduce.jobhistory.address 60 | ip-10-0-73-146.ec2.internal:10020 61 | 62 | 63 | 64 | mapreduce.jobhistory.bind-host 65 | 0.0.0.0 66 | 67 | 68 | 69 | mapreduce.jobhistory.done-dir 70 | /mr-history/done 71 | 72 | 73 | 74 | mapreduce.jobhistory.intermediate-done-dir 75 | /mr-history/tmp 76 | 77 | 78 | 79 | mapreduce.jobhistory.recovery.enable 80 | true 81 | 82 | 83 | 84 | mapreduce.jobhistory.recovery.store.class 85 | org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService 86 | 87 | 88 | 89 | mapreduce.jobhistory.recovery.store.leveldb.path 90 | /hadoop/mapreduce/jhs 91 | 92 | 93 | 94 | mapreduce.jobhistory.webapp.address 95 | ip-10-0-73-146.ec2.internal:19888 96 | 97 | 98 | 99 | mapreduce.map.java.opts 100 | -Xmx614m 101 | 102 | 103 | 104 | mapreduce.map.log.level 105 | INFO 106 | 107 | 108 | 109 | mapreduce.map.memory.mb 110 | 768 111 | 112 | 113 | 114 | mapreduce.map.output.compress 115 | false 116 | 117 | 118 | 119 | mapreduce.map.sort.spill.percent 120 | 0.7 121 | 122 | 123 | 124 | mapreduce.map.speculative 125 | false 126 | 127 | 128 | 129 | mapreduce.output.fileoutputformat.compress 130 | false 131 | 132 | 133 | 134 | mapreduce.output.fileoutputformat.compress.type 135 | BLOCK 136 | 137 | 138 | 139 | mapreduce.reduce.input.buffer.percent 140 | 0.0 141 | 142 | 143 | 144 | mapreduce.reduce.java.opts 145 | -Xmx1228m 146 | 147 | 148 | 149 | mapreduce.reduce.log.level 150 | INFO 151 | 152 | 153 | 154 | mapreduce.reduce.memory.mb 155 | 1536 156 | 157 | 158 | 159 | mapreduce.reduce.shuffle.fetch.retry.enabled 160 | 1 161 | 162 | 163 | 164 | mapreduce.reduce.shuffle.fetch.retry.interval-ms 165 | 1000 166 | 167 | 168 | 169 | mapreduce.reduce.shuffle.fetch.retry.timeout-ms 170 | 30000 171 | 172 | 173 | 174 | mapreduce.reduce.shuffle.input.buffer.percent 175 | 0.7 176 | 177 | 178 | 179 | mapreduce.reduce.shuffle.merge.percent 180 | 0.66 181 | 182 | 183 | 184 | mapreduce.reduce.shuffle.parallelcopies 185 | 30 186 | 187 | 188 | 189 | mapreduce.reduce.speculative 190 | false 191 | 192 | 193 | 194 | mapreduce.shuffle.port 195 | 13562 196 | 197 | 198 | 199 | mapreduce.task.io.sort.factor 200 | 100 201 | 202 | 203 | 204 | mapreduce.task.io.sort.mb 205 | 429 206 | 207 | 208 | 209 | mapreduce.task.timeout 210 | 300000 211 | 212 | 213 | 214 | yarn.app.mapreduce.am.admin-command-opts 215 | -Dhdp.version=${hdp.version} 216 | 217 | 218 | 219 | yarn.app.mapreduce.am.command-opts 220 | -Xmx614m -Dhdp.version=${hdp.version} 221 | 222 | 223 | 224 | yarn.app.mapreduce.am.log.level 225 | INFO 226 | 227 | 228 | 229 | yarn.app.mapreduce.am.resource.mb 230 | 768 231 | 232 | 233 | 234 | yarn.app.mapreduce.am.staging-dir 235 | /user 236 | 237 | -------------------------------------------------------------------------------- /performance Test Results/dfsio_read_output3n.txt: -------------------------------------------------------------------------------- 1 | ----- TestDFSIO ----- : read 2 | Date & time: Fri Aug 19 23:24:56 EDT 2016 3 | Number of files: 10 4 | Total MBytes processed: 1000000.0 5 | Throughput mb/sec: 90.27605062139263 6 | Average IO rate mb/sec: 90.29936218261719 7 | IO rate std deviation: 1.4534757501502815 8 | Test exec time sec: 1153.225 9 | 10 | -------------------------------------------------------------------------------- /performance Test Results/dfsio_read_results5n.txt: -------------------------------------------------------------------------------- 1 | ----- TestDFSIO ----- : read 2 | Date & time: Sat Aug 20 00:02:57 EDT 2016 3 | Number of files: 10 4 | Total MBytes processed: 1000000.0 5 | Throughput mb/sec: 138.81604902316533 6 | Average IO rate mb/sec: 139.01646423339844 7 | IO rate std deviation: 5.262152617442122 8 | Test exec time sec: 776.774 9 | 10 | -------------------------------------------------------------------------------- /performance Test Results/dfsio_write_output3n.txt: -------------------------------------------------------------------------------- 1 | nohup: ignoring input 2 | Not a valid JAR: /usr/hdp/2.4.0.0-169/hadoop-mapreduce/hadoop-mapreduce-client-jobclient-2.7.1.2.3.2.0-2950-tests.jar 3 | nohup: ignoring input 4 | 16/08/19 22:00:11 INFO fs.TestDFSIO: TestDFSIO.1.8 5 | 16/08/19 22:00:11 INFO fs.TestDFSIO: nrFiles = 10 6 | 16/08/19 22:00:11 INFO fs.TestDFSIO: nrBytes (MB) = 100000.0 7 | 16/08/19 22:00:11 INFO fs.TestDFSIO: bufferSize = 1000000 8 | 16/08/19 22:00:11 INFO fs.TestDFSIO: baseDir = /benchmarks/TestDFSIO 9 | 16/08/19 22:00:12 INFO fs.TestDFSIO: creating control file: 104857600000 bytes, 10 files 10 | 16/08/19 22:00:12 INFO fs.TestDFSIO: created control files for: 10 files 11 | 16/08/19 22:00:13 INFO impl.TimelineClientImpl: Timeline service address: http://instance-21736.bigstep.io:8188/ws/v1/timeline/ 12 | 16/08/19 22:00:13 INFO client.RMProxy: Connecting to ResourceManager at instance-21735.bigstep.io/176.223.251.138:8050 13 | 16/08/19 22:00:13 INFO impl.TimelineClientImpl: Timeline service address: http://instance-21736.bigstep.io:8188/ws/v1/timeline/ 14 | 16/08/19 22:00:13 INFO client.RMProxy: Connecting to ResourceManager at instance-21735.bigstep.io/176.223.251.138:8050 15 | 16/08/19 22:00:13 INFO mapred.FileInputFormat: Total input paths to process : 10 16 | 16/08/19 22:00:13 INFO mapreduce.JobSubmitter: number of splits:10 17 | 16/08/19 22:00:13 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1471444967480_0013 18 | 16/08/19 22:00:14 INFO impl.YarnClientImpl: Submitted application application_1471444967480_0013 19 | 16/08/19 22:00:14 INFO mapreduce.Job: The url to track the job: http://instance-21735.bigstep.io:8088/proxy/application_1471444967480_0013/ 20 | 16/08/19 22:00:14 INFO mapreduce.Job: Running job: job_1471444967480_0013 21 | 16/08/19 22:00:21 INFO mapreduce.Job: Job job_1471444967480_0013 running in uber mode : false 22 | 16/08/19 22:00:21 INFO mapreduce.Job: map 0% reduce 0% 23 | 16/08/19 22:00:32 INFO mapreduce.Job: map 60% reduce 0% 24 | 16/08/19 22:00:33 INFO mapreduce.Job: map 67% reduce 0% 25 | 16/08/19 22:18:48 INFO mapreduce.Job: map 70% reduce 0% 26 | 16/08/19 22:19:05 INFO mapreduce.Job: map 73% reduce 0% 27 | 16/08/19 22:19:08 INFO mapreduce.Job: map 77% reduce 0% 28 | 16/08/19 22:19:11 INFO mapreduce.Job: map 80% reduce 0% 29 | 16/08/19 22:19:12 INFO mapreduce.Job: map 83% reduce 0% 30 | 16/08/19 22:19:17 INFO mapreduce.Job: map 87% reduce 0% 31 | 16/08/19 22:19:19 INFO mapreduce.Job: map 90% reduce 0% 32 | 16/08/19 22:19:24 INFO mapreduce.Job: map 90% reduce 23% 33 | 16/08/19 22:19:29 INFO mapreduce.Job: map 97% reduce 23% 34 | 16/08/19 22:19:30 INFO mapreduce.Job: map 97% reduce 30% 35 | 16/08/19 22:19:37 INFO mapreduce.Job: map 100% reduce 30% 36 | 16/08/19 22:19:38 INFO mapreduce.Job: map 100% reduce 100% 37 | 16/08/19 22:19:39 INFO mapreduce.Job: Job job_1471444967480_0013 completed successfully 38 | 16/08/19 22:19:39 INFO mapreduce.Job: Counters: 49 39 | File System Counters 40 | FILE: Number of bytes read=908 41 | FILE: Number of bytes written=1439658 42 | FILE: Number of read operations=0 43 | FILE: Number of large read operations=0 44 | FILE: Number of write operations=0 45 | HDFS: Number of bytes read=2510 46 | HDFS: Number of bytes written=1048576000084 47 | HDFS: Number of read operations=43 48 | HDFS: Number of large read operations=0 49 | HDFS: Number of write operations=12 50 | Job Counters 51 | Launched map tasks=10 52 | Launched reduce tasks=1 53 | Data-local map tasks=10 54 | Total time spent by all maps in occupied slots (ms)=11333104 55 | Total time spent by all reduces in occupied slots (ms)=53028 56 | Total time spent by all map tasks (ms)=11333104 57 | Total time spent by all reduce tasks (ms)=26514 58 | Total vcore-seconds taken by all map tasks=11333104 59 | Total vcore-seconds taken by all reduce tasks=26514 60 | Total megabyte-seconds taken by all map tasks=81235689472 61 | Total megabyte-seconds taken by all reduce tasks=380104704 62 | Map-Reduce Framework 63 | Map input records=10 64 | Map output records=50 65 | Map output bytes=802 66 | Map output materialized bytes=962 67 | Input split bytes=1390 68 | Combine input records=0 69 | Combine output records=0 70 | Reduce input groups=5 71 | Reduce shuffle bytes=962 72 | Reduce input records=50 73 | Reduce output records=5 74 | Spilled Records=100 75 | Shuffled Maps =10 76 | Failed Shuffles=0 77 | Merged Map outputs=10 78 | GC time elapsed (ms)=72450 79 | CPU time spent (ms)=3875080 80 | Physical memory (bytes) snapshot=30719840256 81 | Virtual memory (bytes) snapshot=97018732544 82 | Total committed heap usage (bytes)=32197574656 83 | Shuffle Errors 84 | BAD_ID=0 85 | CONNECTION=0 86 | IO_ERROR=0 87 | WRONG_LENGTH=0 88 | WRONG_MAP=0 89 | WRONG_REDUCE=0 90 | File Input Format Counters 91 | Bytes Read=1120 92 | File Output Format Counters 93 | Bytes Written=84 94 | 16/08/19 22:19:39 INFO fs.TestDFSIO: ----- TestDFSIO ----- : write 95 | 16/08/19 22:19:39 INFO fs.TestDFSIO: Date & time: Fri Aug 19 22:19:39 EDT 2016 96 | 16/08/19 22:19:39 INFO fs.TestDFSIO: Number of files: 10 97 | 16/08/19 22:19:39 INFO fs.TestDFSIO: Total MBytes processed: 1000000.0 98 | 16/08/19 22:19:39 INFO fs.TestDFSIO: Throughput mb/sec: 88.5604202581607 99 | 16/08/19 22:19:39 INFO fs.TestDFSIO: Average IO rate mb/sec: 88.5728759765625 100 | 16/08/19 22:19:39 INFO fs.TestDFSIO: IO rate std deviation: 1.0516154937654552 101 | 16/08/19 22:19:39 INFO fs.TestDFSIO: Test exec time sec: 1166.865 102 | 16/08/19 22:19:39 INFO fs.TestDFSIO: 103 | ----- TestDFSIO ----- : write 104 | Date & time: Fri Aug 19 23:03:41 EDT 2016 105 | Number of files: 10 106 | Total MBytes processed: 1000000.0 107 | Throughput mb/sec: 82.1379388404193 108 | Average IO rate mb/sec: 82.14105987548828 109 | IO rate std deviation: 0.5063978489775659 110 | Test exec time sec: 1259.389 111 | 112 | -------------------------------------------------------------------------------- /performance Test Results/dfsio_write_results5n.txt: -------------------------------------------------------------------------------- 1 | ----- TestDFSIO ----- : write 2 | Date & time: Fri Aug 19 23:49:58 EDT 2016 3 | Number of files: 10 4 | Total MBytes processed: 1000000.0 5 | Throughput mb/sec: 135.0146983751386 6 | Average IO rate mb/sec: 135.31039428710938 7 | IO rate std deviation: 6.446408476198092 8 | Test exec time sec: 785.865 9 | 10 | -------------------------------------------------------------------------------- /performance Test Results/teragen 3 node.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sunileman/MapReduce-Performance_Testing/3e6101b1b52886b047b56c0e18ad698876474ce7/performance Test Results/teragen 3 node.jpg -------------------------------------------------------------------------------- /performance Test Results/teragen 5 node.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sunileman/MapReduce-Performance_Testing/3e6101b1b52886b047b56c0e18ad698876474ce7/performance Test Results/teragen 5 node.jpg -------------------------------------------------------------------------------- /performance Test Results/teragen_results_2016-08-22:16:43:22: -------------------------------------------------------------------------------- 1 | WARNING: Use "yarn jar" to launch YARN applications. 2 | 16/08/22 16:43:27 INFO impl.TimelineClientImpl: Timeline service address: http://instance-21736.bigstep.io:8188/ws/v1/timeline/ 3 | 16/08/22 16:43:27 INFO client.RMProxy: Connecting to ResourceManager at instance-21735.bigstep.io/176.223.251.138:8050 4 | 16/08/22 16:43:28 INFO terasort.TeraSort: Generating 10000000000 using 92 5 | 16/08/22 16:43:28 INFO mapreduce.JobSubmitter: number of splits:92 6 | 16/08/22 16:43:28 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1471444967480_0024 7 | 16/08/22 16:43:28 INFO impl.YarnClientImpl: Submitted application application_1471444967480_0024 8 | 16/08/22 16:43:28 INFO mapreduce.Job: The url to track the job: http://instance-21735.bigstep.io:8088/proxy/application_1471444967480_0024/ 9 | 16/08/22 16:43:28 INFO mapreduce.Job: Running job: job_1471444967480_0024 10 | 16/08/22 16:43:34 INFO mapreduce.Job: Job job_1471444967480_0024 running in uber mode : false 11 | 16/08/22 16:43:34 INFO mapreduce.Job: map 0% reduce 0% 12 | 16/08/22 16:43:46 INFO mapreduce.Job: map 1% reduce 0% 13 | 16/08/22 16:43:51 INFO mapreduce.Job: map 2% reduce 0% 14 | 16/08/22 16:43:55 INFO mapreduce.Job: map 3% reduce 0% 15 | 16/08/22 16:43:58 INFO mapreduce.Job: map 4% reduce 0% 16 | 16/08/22 16:44:01 INFO mapreduce.Job: map 5% reduce 0% 17 | 16/08/22 16:44:04 INFO mapreduce.Job: map 6% reduce 0% 18 | 16/08/22 16:44:08 INFO mapreduce.Job: map 7% reduce 0% 19 | 16/08/22 16:44:11 INFO mapreduce.Job: map 8% reduce 0% 20 | 16/08/22 16:44:14 INFO mapreduce.Job: map 9% reduce 0% 21 | 16/08/22 16:44:16 INFO mapreduce.Job: map 10% reduce 0% 22 | 16/08/22 16:44:20 INFO mapreduce.Job: map 11% reduce 0% 23 | 16/08/22 16:44:23 INFO mapreduce.Job: map 12% reduce 0% 24 | 16/08/22 16:44:26 INFO mapreduce.Job: map 13% reduce 0% 25 | 16/08/22 16:44:30 INFO mapreduce.Job: map 14% reduce 0% 26 | 16/08/22 16:44:33 INFO mapreduce.Job: map 15% reduce 0% 27 | 16/08/22 16:44:36 INFO mapreduce.Job: map 16% reduce 0% 28 | 16/08/22 16:44:39 INFO mapreduce.Job: map 17% reduce 0% 29 | 16/08/22 16:44:42 INFO mapreduce.Job: map 18% reduce 0% 30 | 16/08/22 16:44:45 INFO mapreduce.Job: map 19% reduce 0% 31 | 16/08/22 16:44:48 INFO mapreduce.Job: map 20% reduce 0% 32 | 16/08/22 16:44:51 INFO mapreduce.Job: map 21% reduce 0% 33 | 16/08/22 16:44:55 INFO mapreduce.Job: map 22% reduce 0% 34 | 16/08/22 16:44:58 INFO mapreduce.Job: map 23% reduce 0% 35 | 16/08/22 16:45:01 INFO mapreduce.Job: map 24% reduce 0% 36 | 16/08/22 16:45:04 INFO mapreduce.Job: map 25% reduce 0% 37 | 16/08/22 16:45:07 INFO mapreduce.Job: map 26% reduce 0% 38 | 16/08/22 16:45:10 INFO mapreduce.Job: map 27% reduce 0% 39 | 16/08/22 16:45:13 INFO mapreduce.Job: map 28% reduce 0% 40 | 16/08/22 16:45:16 INFO mapreduce.Job: map 29% reduce 0% 41 | 16/08/22 16:45:20 INFO mapreduce.Job: map 30% reduce 0% 42 | 16/08/22 16:45:23 INFO mapreduce.Job: map 31% reduce 0% 43 | 16/08/22 16:45:26 INFO mapreduce.Job: map 32% reduce 0% 44 | 16/08/22 16:45:29 INFO mapreduce.Job: map 33% reduce 0% 45 | 16/08/22 16:45:32 INFO mapreduce.Job: map 34% reduce 0% 46 | 16/08/22 16:45:35 INFO mapreduce.Job: map 35% reduce 0% 47 | 16/08/22 16:45:38 INFO mapreduce.Job: map 36% reduce 0% 48 | 16/08/22 16:45:41 INFO mapreduce.Job: map 37% reduce 0% 49 | 16/08/22 16:45:45 INFO mapreduce.Job: map 38% reduce 0% 50 | 16/08/22 16:45:48 INFO mapreduce.Job: map 39% reduce 0% 51 | 16/08/22 16:45:52 INFO mapreduce.Job: map 40% reduce 0% 52 | 16/08/22 16:45:55 INFO mapreduce.Job: map 41% reduce 0% 53 | 16/08/22 16:45:58 INFO mapreduce.Job: map 42% reduce 0% 54 | 16/08/22 16:46:01 INFO mapreduce.Job: map 43% reduce 0% 55 | 16/08/22 16:46:03 INFO mapreduce.Job: map 44% reduce 0% 56 | 16/08/22 16:46:06 INFO mapreduce.Job: map 45% reduce 0% 57 | 16/08/22 16:46:10 INFO mapreduce.Job: map 46% reduce 0% 58 | 16/08/22 16:46:13 INFO mapreduce.Job: map 47% reduce 0% 59 | 16/08/22 16:46:16 INFO mapreduce.Job: map 48% reduce 0% 60 | 16/08/22 16:46:20 INFO mapreduce.Job: map 49% reduce 0% 61 | 16/08/22 16:46:23 INFO mapreduce.Job: map 50% reduce 0% 62 | 16/08/22 16:46:26 INFO mapreduce.Job: map 51% reduce 0% 63 | 16/08/22 16:46:29 INFO mapreduce.Job: map 52% reduce 0% 64 | 16/08/22 16:46:33 INFO mapreduce.Job: map 53% reduce 0% 65 | 16/08/22 16:46:36 INFO mapreduce.Job: map 54% reduce 0% 66 | 16/08/22 16:46:40 INFO mapreduce.Job: map 55% reduce 0% 67 | 16/08/22 16:46:43 INFO mapreduce.Job: map 56% reduce 0% 68 | 16/08/22 16:46:46 INFO mapreduce.Job: map 57% reduce 0% 69 | 16/08/22 16:46:50 INFO mapreduce.Job: map 58% reduce 0% 70 | 16/08/22 16:46:53 INFO mapreduce.Job: map 59% reduce 0% 71 | 16/08/22 16:46:57 INFO mapreduce.Job: map 60% reduce 0% 72 | 16/08/22 16:47:00 INFO mapreduce.Job: map 61% reduce 0% 73 | 16/08/22 16:47:03 INFO mapreduce.Job: map 62% reduce 0% 74 | 16/08/22 16:47:06 INFO mapreduce.Job: map 63% reduce 0% 75 | 16/08/22 16:47:10 INFO mapreduce.Job: map 64% reduce 0% 76 | 16/08/22 16:47:13 INFO mapreduce.Job: map 65% reduce 0% 77 | 16/08/22 16:47:17 INFO mapreduce.Job: map 66% reduce 0% 78 | 16/08/22 16:47:20 INFO mapreduce.Job: map 67% reduce 0% 79 | 16/08/22 16:47:23 INFO mapreduce.Job: map 68% reduce 0% 80 | 16/08/22 16:47:27 INFO mapreduce.Job: map 69% reduce 0% 81 | 16/08/22 16:47:30 INFO mapreduce.Job: map 70% reduce 0% 82 | 16/08/22 16:47:33 INFO mapreduce.Job: map 71% reduce 0% 83 | 16/08/22 16:47:37 INFO mapreduce.Job: map 72% reduce 0% 84 | 16/08/22 16:47:40 INFO mapreduce.Job: map 73% reduce 0% 85 | 16/08/22 16:47:43 INFO mapreduce.Job: map 74% reduce 0% 86 | 16/08/22 16:47:46 INFO mapreduce.Job: map 75% reduce 0% 87 | 16/08/22 16:47:49 INFO mapreduce.Job: map 76% reduce 0% 88 | 16/08/22 16:47:53 INFO mapreduce.Job: map 77% reduce 0% 89 | 16/08/22 16:47:56 INFO mapreduce.Job: map 78% reduce 0% 90 | 16/08/22 16:47:59 INFO mapreduce.Job: map 79% reduce 0% 91 | 16/08/22 16:48:02 INFO mapreduce.Job: map 80% reduce 0% 92 | 16/08/22 16:48:06 INFO mapreduce.Job: map 81% reduce 0% 93 | 16/08/22 16:48:09 INFO mapreduce.Job: map 82% reduce 0% 94 | 16/08/22 16:48:12 INFO mapreduce.Job: map 83% reduce 0% 95 | 16/08/22 16:48:16 INFO mapreduce.Job: map 84% reduce 0% 96 | 16/08/22 16:48:19 INFO mapreduce.Job: map 85% reduce 0% 97 | 16/08/22 16:48:22 INFO mapreduce.Job: map 86% reduce 0% 98 | 16/08/22 16:48:26 INFO mapreduce.Job: map 87% reduce 0% 99 | 16/08/22 16:48:29 INFO mapreduce.Job: map 88% reduce 0% 100 | 16/08/22 16:48:33 INFO mapreduce.Job: map 89% reduce 0% 101 | 16/08/22 16:48:36 INFO mapreduce.Job: map 90% reduce 0% 102 | 16/08/22 16:48:40 INFO mapreduce.Job: map 91% reduce 0% 103 | 16/08/22 16:48:43 INFO mapreduce.Job: map 92% reduce 0% 104 | 16/08/22 16:48:46 INFO mapreduce.Job: map 93% reduce 0% 105 | 16/08/22 16:48:49 INFO mapreduce.Job: map 94% reduce 0% 106 | 16/08/22 16:48:53 INFO mapreduce.Job: map 95% reduce 0% 107 | 16/08/22 16:48:57 INFO mapreduce.Job: map 96% reduce 0% 108 | 16/08/22 16:49:01 INFO mapreduce.Job: map 97% reduce 0% 109 | 16/08/22 16:49:05 INFO mapreduce.Job: map 98% reduce 0% 110 | 16/08/22 16:49:11 INFO mapreduce.Job: map 99% reduce 0% 111 | 16/08/22 16:49:18 INFO mapreduce.Job: map 100% reduce 0% 112 | 16/08/22 16:49:30 INFO mapreduce.Job: Job job_1471444967480_0024 completed successfully 113 | 16/08/22 16:49:30 INFO mapreduce.Job: Counters: 31 114 | File System Counters 115 | FILE: Number of bytes read=0 116 | FILE: Number of bytes written=11982070 117 | FILE: Number of read operations=0 118 | FILE: Number of large read operations=0 119 | FILE: Number of write operations=0 120 | HDFS: Number of bytes read=8052 121 | HDFS: Number of bytes written=1000000000000 122 | HDFS: Number of read operations=368 123 | HDFS: Number of large read operations=0 124 | HDFS: Number of write operations=184 125 | Job Counters 126 | Launched map tasks=92 127 | Other local map tasks=92 128 | Total time spent by all maps in occupied slots (ms)=28506615 129 | Total time spent by all reduces in occupied slots (ms)=0 130 | Total time spent by all map tasks (ms)=28506615 131 | Total vcore-seconds taken by all map tasks=28506615 132 | Total megabyte-seconds taken by all map tasks=58381547520 133 | Map-Reduce Framework 134 | Map input records=10000000000 135 | Map output records=10000000000 136 | Input split bytes=8052 137 | Spilled Records=0 138 | Failed Shuffles=0 139 | Merged Map outputs=0 140 | GC time elapsed (ms)=142649 141 | CPU time spent (ms)=15319920 142 | Physical memory (bytes) snapshot=22814720000 143 | Virtual memory (bytes) snapshot=338591690752 144 | Total committed heap usage (bytes)=24905252864 145 | org.apache.hadoop.examples.terasort.TeraGen$Counters 146 | CHECKSUM=3028416809717741100 147 | File Input Format Counters 148 | Bytes Read=0 149 | File Output Format Counters 150 | Bytes Written=1000000000000 151 | -------------------------------------------------------------------------------- /performance Test Results/terasort 3 node.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sunileman/MapReduce-Performance_Testing/3e6101b1b52886b047b56c0e18ad698876474ce7/performance Test Results/terasort 3 node.jpg -------------------------------------------------------------------------------- /performance Test Results/terasort 5 node.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sunileman/MapReduce-Performance_Testing/3e6101b1b52886b047b56c0e18ad698876474ce7/performance Test Results/terasort 5 node.jpg -------------------------------------------------------------------------------- /performance Test Results/terasort.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sunileman/MapReduce-Performance_Testing/3e6101b1b52886b047b56c0e18ad698876474ce7/performance Test Results/terasort.jpg -------------------------------------------------------------------------------- /performance Test Results/terasort_results_2016-08-22:16:52:42: -------------------------------------------------------------------------------- 1 | WARNING: Use "yarn jar" to launch YARN applications. 2 | 16/08/22 16:52:46 INFO terasort.TeraSort: starting 3 | 16/08/22 16:52:47 INFO input.FileInputFormat: Total input paths to process : 92 4 | Spent 309ms computing base-splits. 5 | Spent 32ms computing TeraScheduler splits. 6 | Computing input splits took 341ms 7 | Sampling 10 splits of 7452 8 | Making 92 from 100000 sampled records 9 | Computing parititions took 396ms 10 | Spent 740ms computing partitions. 11 | 16/08/22 16:52:48 INFO impl.TimelineClientImpl: Timeline service address: http://instance-21736.bigstep.io:8188/ws/v1/timeline/ 12 | 16/08/22 16:52:48 INFO client.RMProxy: Connecting to ResourceManager at instance-21735.bigstep.io/176.223.251.138:8050 13 | 16/08/22 16:52:48 INFO mapreduce.JobSubmitter: number of splits:7452 14 | 16/08/22 16:52:49 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1471444967480_0025 15 | 16/08/22 16:52:49 INFO impl.YarnClientImpl: Submitted application application_1471444967480_0025 16 | 16/08/22 16:52:49 INFO mapreduce.Job: The url to track the job: http://instance-21735.bigstep.io:8088/proxy/application_1471444967480_0025/ 17 | 16/08/22 16:52:49 INFO mapreduce.Job: Running job: job_1471444967480_0025 18 | 16/08/22 16:52:54 INFO mapreduce.Job: Job job_1471444967480_0025 running in uber mode : false 19 | 16/08/22 16:52:54 INFO mapreduce.Job: map 0% reduce 0% 20 | 16/08/22 16:53:21 INFO mapreduce.Job: map 1% reduce 0% 21 | 16/08/22 16:53:24 INFO mapreduce.Job: map 2% reduce 0% 22 | 16/08/22 16:53:32 INFO mapreduce.Job: map 3% reduce 0% 23 | 16/08/22 16:53:43 INFO mapreduce.Job: map 4% reduce 0% 24 | 16/08/22 16:53:50 INFO mapreduce.Job: map 5% reduce 0% 25 | 16/08/22 16:54:01 INFO mapreduce.Job: map 6% reduce 0% 26 | 16/08/22 16:54:08 INFO mapreduce.Job: map 7% reduce 0% 27 | 16/08/22 16:54:17 INFO mapreduce.Job: map 8% reduce 0% 28 | 16/08/22 16:54:26 INFO mapreduce.Job: map 9% reduce 0% 29 | 16/08/22 16:54:36 INFO mapreduce.Job: map 10% reduce 0% 30 | 16/08/22 16:54:44 INFO mapreduce.Job: map 11% reduce 0% 31 | 16/08/22 16:54:53 INFO mapreduce.Job: map 12% reduce 0% 32 | 16/08/22 16:55:02 INFO mapreduce.Job: map 13% reduce 0% 33 | 16/08/22 16:55:11 INFO mapreduce.Job: map 14% reduce 0% 34 | 16/08/22 16:55:19 INFO mapreduce.Job: map 15% reduce 0% 35 | 16/08/22 16:55:28 INFO mapreduce.Job: map 16% reduce 0% 36 | 16/08/22 16:55:38 INFO mapreduce.Job: map 17% reduce 0% 37 | 16/08/22 16:55:47 INFO mapreduce.Job: map 18% reduce 0% 38 | 16/08/22 16:55:55 INFO mapreduce.Job: map 19% reduce 0% 39 | 16/08/22 16:56:04 INFO mapreduce.Job: map 20% reduce 0% 40 | 16/08/22 16:56:14 INFO mapreduce.Job: map 21% reduce 0% 41 | 16/08/22 16:56:23 INFO mapreduce.Job: map 22% reduce 0% 42 | 16/08/22 16:56:31 INFO mapreduce.Job: map 23% reduce 0% 43 | 16/08/22 16:56:40 INFO mapreduce.Job: map 24% reduce 0% 44 | 16/08/22 16:56:49 INFO mapreduce.Job: map 25% reduce 0% 45 | 16/08/22 16:56:58 INFO mapreduce.Job: map 26% reduce 0% 46 | 16/08/22 16:57:07 INFO mapreduce.Job: map 27% reduce 0% 47 | 16/08/22 16:57:17 INFO mapreduce.Job: map 28% reduce 0% 48 | 16/08/22 16:57:25 INFO mapreduce.Job: map 29% reduce 0% 49 | 16/08/22 16:57:33 INFO mapreduce.Job: map 30% reduce 0% 50 | 16/08/22 16:57:43 INFO mapreduce.Job: map 31% reduce 0% 51 | 16/08/22 16:57:51 INFO mapreduce.Job: map 32% reduce 0% 52 | 16/08/22 16:58:04 INFO mapreduce.Job: map 32% reduce 1% 53 | 16/08/22 16:58:05 INFO mapreduce.Job: map 33% reduce 1% 54 | 16/08/22 16:58:25 INFO mapreduce.Job: map 34% reduce 1% 55 | 16/08/22 16:58:26 INFO mapreduce.Job: map 34% reduce 2% 56 | 16/08/22 16:58:43 INFO mapreduce.Job: map 35% reduce 2% 57 | 16/08/22 16:58:49 INFO mapreduce.Job: map 35% reduce 3% 58 | 16/08/22 16:59:00 INFO mapreduce.Job: map 36% reduce 3% 59 | 16/08/22 16:59:09 INFO mapreduce.Job: map 36% reduce 4% 60 | 16/08/22 16:59:17 INFO mapreduce.Job: map 37% reduce 4% 61 | 16/08/22 16:59:31 INFO mapreduce.Job: map 37% reduce 5% 62 | 16/08/22 16:59:34 INFO mapreduce.Job: map 38% reduce 5% 63 | 16/08/22 16:59:49 INFO mapreduce.Job: map 38% reduce 6% 64 | 16/08/22 16:59:51 INFO mapreduce.Job: map 39% reduce 6% 65 | 16/08/22 17:00:09 INFO mapreduce.Job: map 40% reduce 6% 66 | 16/08/22 17:00:10 INFO mapreduce.Job: map 40% reduce 7% 67 | 16/08/22 17:00:25 INFO mapreduce.Job: map 41% reduce 7% 68 | 16/08/22 17:00:28 INFO mapreduce.Job: map 41% reduce 8% 69 | 16/08/22 17:00:43 INFO mapreduce.Job: map 42% reduce 8% 70 | 16/08/22 17:00:47 INFO mapreduce.Job: map 42% reduce 9% 71 | 16/08/22 17:01:01 INFO mapreduce.Job: map 43% reduce 9% 72 | 16/08/22 17:01:08 INFO mapreduce.Job: map 43% reduce 10% 73 | 16/08/22 17:01:18 INFO mapreduce.Job: map 44% reduce 10% 74 | 16/08/22 17:01:24 INFO mapreduce.Job: map 44% reduce 11% 75 | 16/08/22 17:01:34 INFO mapreduce.Job: map 45% reduce 11% 76 | 16/08/22 17:01:50 INFO mapreduce.Job: map 45% reduce 12% 77 | 16/08/22 17:01:51 INFO mapreduce.Job: map 46% reduce 12% 78 | 16/08/22 17:02:08 INFO mapreduce.Job: map 47% reduce 12% 79 | 16/08/22 17:02:14 INFO mapreduce.Job: map 47% reduce 13% 80 | 16/08/22 17:02:23 INFO mapreduce.Job: map 48% reduce 13% 81 | 16/08/22 17:02:37 INFO mapreduce.Job: map 48% reduce 14% 82 | 16/08/22 17:02:39 INFO mapreduce.Job: map 49% reduce 14% 83 | 16/08/22 17:02:53 INFO mapreduce.Job: map 50% reduce 14% 84 | 16/08/22 17:03:00 INFO mapreduce.Job: map 50% reduce 15% 85 | 16/08/22 17:03:09 INFO mapreduce.Job: map 51% reduce 15% 86 | 16/08/22 17:03:24 INFO mapreduce.Job: map 52% reduce 15% 87 | 16/08/22 17:03:26 INFO mapreduce.Job: map 52% reduce 16% 88 | 16/08/22 17:03:40 INFO mapreduce.Job: map 53% reduce 16% 89 | 16/08/22 17:03:54 INFO mapreduce.Job: map 53% reduce 17% 90 | 16/08/22 17:03:55 INFO mapreduce.Job: map 54% reduce 17% 91 | 16/08/22 17:04:11 INFO mapreduce.Job: map 55% reduce 17% 92 | 16/08/22 17:04:21 INFO mapreduce.Job: map 55% reduce 18% 93 | 16/08/22 17:04:25 INFO mapreduce.Job: map 56% reduce 18% 94 | 16/08/22 17:04:38 INFO mapreduce.Job: map 57% reduce 18% 95 | 16/08/22 17:04:52 INFO mapreduce.Job: map 57% reduce 19% 96 | 16/08/22 17:04:54 INFO mapreduce.Job: map 58% reduce 19% 97 | 16/08/22 17:05:08 INFO mapreduce.Job: map 59% reduce 19% 98 | 16/08/22 17:05:22 INFO mapreduce.Job: map 60% reduce 19% 99 | 16/08/22 17:05:24 INFO mapreduce.Job: map 60% reduce 20% 100 | 16/08/22 17:05:36 INFO mapreduce.Job: map 61% reduce 20% 101 | 16/08/22 17:05:50 INFO mapreduce.Job: map 62% reduce 20% 102 | 16/08/22 17:05:58 INFO mapreduce.Job: map 62% reduce 21% 103 | 16/08/22 17:06:04 INFO mapreduce.Job: map 63% reduce 21% 104 | 16/08/22 17:06:17 INFO mapreduce.Job: map 64% reduce 21% 105 | 16/08/22 17:06:31 INFO mapreduce.Job: map 65% reduce 21% 106 | 16/08/22 17:06:37 INFO mapreduce.Job: map 65% reduce 22% 107 | 16/08/22 17:06:43 INFO mapreduce.Job: map 66% reduce 22% 108 | 16/08/22 17:06:57 INFO mapreduce.Job: map 67% reduce 22% 109 | 16/08/22 17:07:11 INFO mapreduce.Job: map 68% reduce 22% 110 | 16/08/22 17:07:18 INFO mapreduce.Job: map 68% reduce 23% 111 | 16/08/22 17:07:25 INFO mapreduce.Job: map 69% reduce 23% 112 | 16/08/22 17:07:38 INFO mapreduce.Job: map 70% reduce 23% 113 | 16/08/22 17:07:51 INFO mapreduce.Job: map 71% reduce 23% 114 | 16/08/22 17:07:57 INFO mapreduce.Job: map 71% reduce 24% 115 | 16/08/22 17:08:04 INFO mapreduce.Job: map 72% reduce 24% 116 | 16/08/22 17:08:20 INFO mapreduce.Job: map 73% reduce 24% 117 | 16/08/22 17:08:33 INFO mapreduce.Job: map 74% reduce 24% 118 | 16/08/22 17:08:39 INFO mapreduce.Job: map 74% reduce 25% 119 | 16/08/22 17:08:46 INFO mapreduce.Job: map 75% reduce 25% 120 | 16/08/22 17:09:00 INFO mapreduce.Job: map 76% reduce 25% 121 | 16/08/22 17:09:14 INFO mapreduce.Job: map 77% reduce 25% 122 | 16/08/22 17:09:19 INFO mapreduce.Job: map 77% reduce 26% 123 | 16/08/22 17:09:26 INFO mapreduce.Job: map 78% reduce 26% 124 | 16/08/22 17:09:39 INFO mapreduce.Job: map 79% reduce 26% 125 | 16/08/22 17:09:53 INFO mapreduce.Job: map 80% reduce 26% 126 | 16/08/22 17:09:59 INFO mapreduce.Job: map 80% reduce 27% 127 | 16/08/22 17:10:06 INFO mapreduce.Job: map 81% reduce 27% 128 | 16/08/22 17:10:19 INFO mapreduce.Job: map 82% reduce 27% 129 | 16/08/22 17:10:32 INFO mapreduce.Job: map 83% reduce 27% 130 | 16/08/22 17:10:38 INFO mapreduce.Job: map 83% reduce 28% 131 | 16/08/22 17:10:45 INFO mapreduce.Job: map 84% reduce 28% 132 | 16/08/22 17:10:57 INFO mapreduce.Job: map 85% reduce 28% 133 | 16/08/22 17:11:12 INFO mapreduce.Job: map 86% reduce 28% 134 | 16/08/22 17:11:17 INFO mapreduce.Job: map 86% reduce 29% 135 | 16/08/22 17:11:25 INFO mapreduce.Job: map 87% reduce 29% 136 | 16/08/22 17:11:39 INFO mapreduce.Job: map 88% reduce 29% 137 | 16/08/22 17:11:52 INFO mapreduce.Job: map 89% reduce 29% 138 | 16/08/22 17:11:57 INFO mapreduce.Job: map 89% reduce 30% 139 | 16/08/22 17:12:07 INFO mapreduce.Job: map 90% reduce 30% 140 | 16/08/22 17:12:19 INFO mapreduce.Job: map 91% reduce 30% 141 | 16/08/22 17:12:32 INFO mapreduce.Job: map 92% reduce 30% 142 | 16/08/22 17:12:37 INFO mapreduce.Job: map 92% reduce 31% 143 | 16/08/22 17:12:45 INFO mapreduce.Job: map 93% reduce 31% 144 | 16/08/22 17:12:59 INFO mapreduce.Job: map 94% reduce 31% 145 | 16/08/22 17:13:12 INFO mapreduce.Job: map 95% reduce 31% 146 | 16/08/22 17:13:17 INFO mapreduce.Job: map 95% reduce 32% 147 | 16/08/22 17:13:24 INFO mapreduce.Job: map 96% reduce 32% 148 | 16/08/22 17:13:37 INFO mapreduce.Job: map 97% reduce 32% 149 | 16/08/22 17:13:51 INFO mapreduce.Job: map 98% reduce 32% 150 | 16/08/22 17:13:56 INFO mapreduce.Job: map 98% reduce 33% 151 | 16/08/22 17:14:04 INFO mapreduce.Job: map 99% reduce 33% 152 | 16/08/22 17:14:17 INFO mapreduce.Job: map 100% reduce 33% 153 | 16/08/22 17:14:27 INFO mapreduce.Job: map 100% reduce 35% 154 | 16/08/22 17:14:28 INFO mapreduce.Job: map 100% reduce 40% 155 | 16/08/22 17:14:29 INFO mapreduce.Job: map 100% reduce 44% 156 | 16/08/22 17:14:30 INFO mapreduce.Job: map 100% reduce 50% 157 | 16/08/22 17:14:31 INFO mapreduce.Job: map 100% reduce 55% 158 | 16/08/22 17:14:32 INFO mapreduce.Job: map 100% reduce 57% 159 | 16/08/22 17:14:33 INFO mapreduce.Job: map 100% reduce 60% 160 | 16/08/22 17:14:34 INFO mapreduce.Job: map 100% reduce 61% 161 | 16/08/22 17:14:35 INFO mapreduce.Job: map 100% reduce 63% 162 | 16/08/22 17:14:36 INFO mapreduce.Job: map 100% reduce 64% 163 | 16/08/22 17:14:37 INFO mapreduce.Job: map 100% reduce 65% 164 | 16/08/22 17:14:38 INFO mapreduce.Job: map 100% reduce 66% 165 | 16/08/22 17:14:39 INFO mapreduce.Job: map 100% reduce 67% 166 | 16/08/22 17:14:41 INFO mapreduce.Job: map 100% reduce 68% 167 | 16/08/22 17:14:48 INFO mapreduce.Job: map 100% reduce 69% 168 | 16/08/22 17:14:57 INFO mapreduce.Job: map 100% reduce 70% 169 | 16/08/22 17:15:08 INFO mapreduce.Job: map 100% reduce 71% 170 | 16/08/22 17:15:19 INFO mapreduce.Job: map 100% reduce 72% 171 | 16/08/22 17:15:29 INFO mapreduce.Job: map 100% reduce 73% 172 | 16/08/22 17:15:40 INFO mapreduce.Job: map 100% reduce 74% 173 | 16/08/22 17:15:50 INFO mapreduce.Job: map 100% reduce 75% 174 | 16/08/22 17:16:02 INFO mapreduce.Job: map 100% reduce 76% 175 | 16/08/22 17:16:13 INFO mapreduce.Job: map 100% reduce 77% 176 | 16/08/22 17:16:23 INFO mapreduce.Job: map 100% reduce 78% 177 | 16/08/22 17:16:37 INFO mapreduce.Job: map 100% reduce 79% 178 | 16/08/22 17:16:46 INFO mapreduce.Job: map 100% reduce 80% 179 | 16/08/22 17:16:58 INFO mapreduce.Job: map 100% reduce 81% 180 | 16/08/22 17:17:09 INFO mapreduce.Job: map 100% reduce 82% 181 | 16/08/22 17:17:21 INFO mapreduce.Job: map 100% reduce 83% 182 | 16/08/22 17:17:30 INFO mapreduce.Job: map 100% reduce 84% 183 | 16/08/22 17:17:45 INFO mapreduce.Job: map 100% reduce 85% 184 | 16/08/22 17:17:56 INFO mapreduce.Job: map 100% reduce 86% 185 | 16/08/22 17:18:06 INFO mapreduce.Job: map 100% reduce 87% 186 | 16/08/22 17:18:19 INFO mapreduce.Job: map 100% reduce 88% 187 | 16/08/22 17:18:29 INFO mapreduce.Job: map 100% reduce 89% 188 | 16/08/22 17:18:41 INFO mapreduce.Job: map 100% reduce 90% 189 | 16/08/22 17:18:54 INFO mapreduce.Job: map 100% reduce 91% 190 | 16/08/22 17:19:07 INFO mapreduce.Job: map 100% reduce 92% 191 | 16/08/22 17:19:22 INFO mapreduce.Job: map 100% reduce 93% 192 | 16/08/22 17:19:34 INFO mapreduce.Job: map 100% reduce 94% 193 | 16/08/22 17:19:50 INFO mapreduce.Job: map 100% reduce 95% 194 | 16/08/22 17:20:05 INFO mapreduce.Job: map 100% reduce 96% 195 | 16/08/22 17:20:20 INFO mapreduce.Job: map 100% reduce 97% 196 | 16/08/22 17:20:39 INFO mapreduce.Job: map 100% reduce 98% 197 | 16/08/22 17:21:00 INFO mapreduce.Job: map 100% reduce 99% 198 | 16/08/22 17:21:34 INFO mapreduce.Job: map 100% reduce 100% 199 | 16/08/22 17:22:17 INFO mapreduce.Job: Job job_1471444967480_0025 completed successfully 200 | 16/08/22 17:22:17 INFO mapreduce.Job: Counters: 49 201 | File System Counters 202 | FILE: Number of bytes read=564152211723 203 | FILE: Number of bytes written=1115563243027 204 | FILE: Number of read operations=0 205 | FILE: Number of large read operations=0 206 | FILE: Number of write operations=0 207 | HDFS: Number of bytes read=1000001177416 208 | HDFS: Number of bytes written=1000000000000 209 | HDFS: Number of read operations=22632 210 | HDFS: Number of large read operations=0 211 | HDFS: Number of write operations=184 212 | Job Counters 213 | Launched map tasks=7452 214 | Launched reduce tasks=92 215 | Data-local map tasks=7452 216 | Total time spent by all maps in occupied slots (ms)=142197503 217 | Total time spent by all reduces in occupied slots (ms)=121151267 218 | Total time spent by all map tasks (ms)=142197503 219 | Total time spent by all reduce tasks (ms)=121151267 220 | Total vcore-seconds taken by all map tasks=142197503 221 | Total vcore-seconds taken by all reduce tasks=121151267 222 | Total megabyte-seconds taken by all map tasks=291220486144 223 | Total megabyte-seconds taken by all reduce tasks=248117794816 224 | Map-Reduce Framework 225 | Map input records=10000000000 226 | Map output records=10000000000 227 | Map output bytes=1020000000000 228 | Map output materialized bytes=550407605904 229 | Input split bytes=1177416 230 | Combine input records=0 231 | Combine output records=0 232 | Reduce input groups=10000000000 233 | Reduce shuffle bytes=550407605904 234 | Reduce input records=10000000000 235 | Reduce output records=10000000000 236 | Spilled Records=20000000000 237 | Shuffled Maps =685584 238 | Failed Shuffles=0 239 | Merged Map outputs=685584 240 | GC time elapsed (ms)=8867553 241 | CPU time spent (ms)=147147440 242 | Physical memory (bytes) snapshot=5267242160128 243 | Virtual memory (bytes) snapshot=27682020782080 244 | Total committed heap usage (bytes)=5684962263040 245 | Shuffle Errors 246 | BAD_ID=0 247 | CONNECTION=0 248 | IO_ERROR=0 249 | WRONG_LENGTH=0 250 | WRONG_MAP=0 251 | WRONG_REDUCE=0 252 | File Input Format Counters 253 | Bytes Read=1000000000000 254 | File Output Format Counters 255 | Bytes Written=1000000000000 256 | 16/08/22 17:22:17 INFO terasort.TeraSort: done 257 | -------------------------------------------------------------------------------- /performance Test Results/teravalidate 3 node.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sunileman/MapReduce-Performance_Testing/3e6101b1b52886b047b56c0e18ad698876474ce7/performance Test Results/teravalidate 3 node.jpg -------------------------------------------------------------------------------- /performance Test Results/teravalidate 5 node.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sunileman/MapReduce-Performance_Testing/3e6101b1b52886b047b56c0e18ad698876474ce7/performance Test Results/teravalidate 5 node.jpg -------------------------------------------------------------------------------- /performance Test Results/teravalidate.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sunileman/MapReduce-Performance_Testing/3e6101b1b52886b047b56c0e18ad698876474ce7/performance Test Results/teravalidate.jpg -------------------------------------------------------------------------------- /performance Test Results/teravalidate_results_2016-08-22:22:14:17: -------------------------------------------------------------------------------- 1 | WARNING: Use "yarn jar" to launch YARN applications. 2 | 16/08/22 22:14:22 INFO impl.TimelineClientImpl: Timeline service address: http://instance-21736.bigstep.io:8188/ws/v1/timeline/ 3 | 16/08/22 22:14:22 INFO client.RMProxy: Connecting to ResourceManager at instance-21735.bigstep.io/176.223.251.138:8050 4 | 16/08/22 22:14:23 INFO input.FileInputFormat: Total input paths to process : 92 5 | Spent 129ms computing base-splits. 6 | Spent 2ms computing TeraScheduler splits. 7 | 16/08/22 22:14:23 INFO mapreduce.JobSubmitter: number of splits:92 8 | 16/08/22 22:14:23 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1471444967480_0026 9 | 16/08/22 22:14:23 INFO impl.YarnClientImpl: Submitted application application_1471444967480_0026 10 | 16/08/22 22:14:23 INFO mapreduce.Job: The url to track the job: http://instance-21735.bigstep.io:8088/proxy/application_1471444967480_0026/ 11 | 16/08/22 22:14:23 INFO mapreduce.Job: Running job: job_1471444967480_0026 12 | 16/08/22 22:14:29 INFO mapreduce.Job: Job job_1471444967480_0026 running in uber mode : false 13 | 16/08/22 22:14:29 INFO mapreduce.Job: map 0% reduce 0% 14 | 16/08/22 22:14:40 INFO mapreduce.Job: map 1% reduce 0% 15 | 16/08/22 22:14:42 INFO mapreduce.Job: map 2% reduce 0% 16 | 16/08/22 22:14:43 INFO mapreduce.Job: map 3% reduce 0% 17 | 16/08/22 22:14:45 INFO mapreduce.Job: map 4% reduce 0% 18 | 16/08/22 22:14:46 INFO mapreduce.Job: map 5% reduce 0% 19 | 16/08/22 22:14:49 INFO mapreduce.Job: map 6% reduce 0% 20 | 16/08/22 22:14:51 INFO mapreduce.Job: map 7% reduce 0% 21 | 16/08/22 22:14:52 INFO mapreduce.Job: map 8% reduce 0% 22 | 16/08/22 22:14:55 INFO mapreduce.Job: map 9% reduce 0% 23 | 16/08/22 22:14:58 INFO mapreduce.Job: map 11% reduce 0% 24 | 16/08/22 22:15:01 INFO mapreduce.Job: map 12% reduce 0% 25 | 16/08/22 22:15:04 INFO mapreduce.Job: map 14% reduce 0% 26 | 16/08/22 22:15:07 INFO mapreduce.Job: map 15% reduce 0% 27 | 16/08/22 22:15:10 INFO mapreduce.Job: map 16% reduce 0% 28 | 16/08/22 22:15:12 INFO mapreduce.Job: map 17% reduce 0% 29 | 16/08/22 22:15:13 INFO mapreduce.Job: map 18% reduce 0% 30 | 16/08/22 22:15:16 INFO mapreduce.Job: map 19% reduce 0% 31 | 16/08/22 22:15:18 INFO mapreduce.Job: map 20% reduce 0% 32 | 16/08/22 22:15:20 INFO mapreduce.Job: map 21% reduce 0% 33 | 16/08/22 22:15:22 INFO mapreduce.Job: map 22% reduce 0% 34 | 16/08/22 22:15:24 INFO mapreduce.Job: map 23% reduce 0% 35 | 16/08/22 22:15:25 INFO mapreduce.Job: map 24% reduce 0% 36 | 16/08/22 22:15:26 INFO mapreduce.Job: map 25% reduce 0% 37 | 16/08/22 22:15:29 INFO mapreduce.Job: map 27% reduce 0% 38 | 16/08/22 22:15:30 INFO mapreduce.Job: map 28% reduce 0% 39 | 16/08/22 22:15:31 INFO mapreduce.Job: map 29% reduce 0% 40 | 16/08/22 22:15:33 INFO mapreduce.Job: map 30% reduce 0% 41 | 16/08/22 22:15:34 INFO mapreduce.Job: map 31% reduce 0% 42 | 16/08/22 22:15:36 INFO mapreduce.Job: map 32% reduce 0% 43 | 16/08/22 22:15:38 INFO mapreduce.Job: map 33% reduce 0% 44 | 16/08/22 22:15:40 INFO mapreduce.Job: map 34% reduce 0% 45 | 16/08/22 22:15:43 INFO mapreduce.Job: map 35% reduce 0% 46 | 16/08/22 22:15:45 INFO mapreduce.Job: map 36% reduce 0% 47 | 16/08/22 22:15:47 INFO mapreduce.Job: map 37% reduce 0% 48 | 16/08/22 22:15:50 INFO mapreduce.Job: map 38% reduce 0% 49 | 16/08/22 22:15:52 INFO mapreduce.Job: map 39% reduce 0% 50 | 16/08/22 22:15:53 INFO mapreduce.Job: map 40% reduce 0% 51 | 16/08/22 22:15:55 INFO mapreduce.Job: map 41% reduce 0% 52 | 16/08/22 22:15:57 INFO mapreduce.Job: map 42% reduce 0% 53 | 16/08/22 22:15:58 INFO mapreduce.Job: map 43% reduce 0% 54 | 16/08/22 22:15:59 INFO mapreduce.Job: map 44% reduce 0% 55 | 16/08/22 22:16:02 INFO mapreduce.Job: map 45% reduce 0% 56 | 16/08/22 22:16:03 INFO mapreduce.Job: map 46% reduce 0% 57 | 16/08/22 22:16:04 INFO mapreduce.Job: map 47% reduce 0% 58 | 16/08/22 22:16:05 INFO mapreduce.Job: map 48% reduce 0% 59 | 16/08/22 22:16:06 INFO mapreduce.Job: map 49% reduce 0% 60 | 16/08/22 22:16:07 INFO mapreduce.Job: map 50% reduce 0% 61 | 16/08/22 22:16:08 INFO mapreduce.Job: map 51% reduce 0% 62 | 16/08/22 22:16:09 INFO mapreduce.Job: map 53% reduce 0% 63 | 16/08/22 22:16:10 INFO mapreduce.Job: map 54% reduce 0% 64 | 16/08/22 22:16:11 INFO mapreduce.Job: map 56% reduce 0% 65 | 16/08/22 22:16:12 INFO mapreduce.Job: map 57% reduce 0% 66 | 16/08/22 22:16:13 INFO mapreduce.Job: map 58% reduce 0% 67 | 16/08/22 22:16:14 INFO mapreduce.Job: map 60% reduce 0% 68 | 16/08/22 22:16:16 INFO mapreduce.Job: map 61% reduce 0% 69 | 16/08/22 22:16:17 INFO mapreduce.Job: map 62% reduce 0% 70 | 16/08/22 22:16:18 INFO mapreduce.Job: map 63% reduce 0% 71 | 16/08/22 22:16:19 INFO mapreduce.Job: map 64% reduce 0% 72 | 16/08/22 22:16:20 INFO mapreduce.Job: map 66% reduce 0% 73 | 16/08/22 22:16:21 INFO mapreduce.Job: map 67% reduce 0% 74 | 16/08/22 22:16:22 INFO mapreduce.Job: map 68% reduce 0% 75 | 16/08/22 22:16:23 INFO mapreduce.Job: map 70% reduce 0% 76 | 16/08/22 22:16:24 INFO mapreduce.Job: map 72% reduce 0% 77 | 16/08/22 22:16:25 INFO mapreduce.Job: map 74% reduce 0% 78 | 16/08/22 22:16:26 INFO mapreduce.Job: map 76% reduce 0% 79 | 16/08/22 22:16:27 INFO mapreduce.Job: map 77% reduce 0% 80 | 16/08/22 22:16:28 INFO mapreduce.Job: map 79% reduce 0% 81 | 16/08/22 22:16:29 INFO mapreduce.Job: map 81% reduce 0% 82 | 16/08/22 22:16:30 INFO mapreduce.Job: map 84% reduce 0% 83 | 16/08/22 22:16:31 INFO mapreduce.Job: map 86% reduce 0% 84 | 16/08/22 22:16:32 INFO mapreduce.Job: map 88% reduce 23% 85 | 16/08/22 22:16:33 INFO mapreduce.Job: map 89% reduce 23% 86 | 16/08/22 22:16:34 INFO mapreduce.Job: map 90% reduce 23% 87 | 16/08/22 22:16:35 INFO mapreduce.Job: map 91% reduce 25% 88 | 16/08/22 22:16:36 INFO mapreduce.Job: map 93% reduce 25% 89 | 16/08/22 22:16:38 INFO mapreduce.Job: map 94% reduce 28% 90 | 16/08/22 22:16:39 INFO mapreduce.Job: map 95% reduce 28% 91 | 16/08/22 22:16:40 INFO mapreduce.Job: map 96% reduce 28% 92 | 16/08/22 22:16:41 INFO mapreduce.Job: map 98% reduce 30% 93 | 16/08/22 22:16:42 INFO mapreduce.Job: map 99% reduce 30% 94 | 16/08/22 22:16:44 INFO mapreduce.Job: map 99% reduce 32% 95 | 16/08/22 22:16:45 INFO mapreduce.Job: map 100% reduce 32% 96 | 16/08/22 22:16:47 INFO mapreduce.Job: map 100% reduce 67% 97 | 16/08/22 22:16:48 INFO mapreduce.Job: map 100% reduce 100% 98 | 16/08/22 22:16:48 INFO mapreduce.Job: Job job_1471444967480_0026 completed successfully 99 | 16/08/22 22:16:48 INFO mapreduce.Job: Counters: 49 100 | File System Counters 101 | FILE: Number of bytes read=8194 102 | FILE: Number of bytes written=12151787 103 | FILE: Number of read operations=0 104 | FILE: Number of large read operations=0 105 | FILE: Number of write operations=0 106 | HDFS: Number of bytes read=1000000014628 107 | HDFS: Number of bytes written=27 108 | HDFS: Number of read operations=279 109 | HDFS: Number of large read operations=0 110 | HDFS: Number of write operations=2 111 | Job Counters 112 | Launched map tasks=92 113 | Launched reduce tasks=1 114 | Data-local map tasks=92 115 | Total time spent by all maps in occupied slots (ms)=10122417 116 | Total time spent by all reduces in occupied slots (ms)=23202 117 | Total time spent by all map tasks (ms)=10122417 118 | Total time spent by all reduce tasks (ms)=23202 119 | Total vcore-seconds taken by all map tasks=10122417 120 | Total vcore-seconds taken by all reduce tasks=23202 121 | Total megabyte-seconds taken by all map tasks=20730710016 122 | Total megabyte-seconds taken by all reduce tasks=47517696 123 | Map-Reduce Framework 124 | Map input records=10000000000 125 | Map output records=276 126 | Map output bytes=7636 127 | Map output materialized bytes=8740 128 | Input split bytes=14628 129 | Combine input records=0 130 | Combine output records=0 131 | Reduce input groups=185 132 | Reduce shuffle bytes=8740 133 | Reduce input records=276 134 | Reduce output records=1 135 | Spilled Records=552 136 | Shuffled Maps =92 137 | Failed Shuffles=0 138 | Merged Map outputs=92 139 | GC time elapsed (ms)=39980 140 | CPU time spent (ms)=6023020 141 | Physical memory (bytes) snapshot=29278613504 142 | Virtual memory (bytes) snapshot=341074534400 143 | Total committed heap usage (bytes)=34940125184 144 | Shuffle Errors 145 | BAD_ID=0 146 | CONNECTION=0 147 | IO_ERROR=0 148 | WRONG_LENGTH=0 149 | WRONG_MAP=0 150 | WRONG_REDUCE=0 151 | File Input Format Counters 152 | Bytes Read=1000000000000 153 | File Output Format Counters 154 | Bytes Written=27 155 | -------------------------------------------------------------------------------- /teragen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | trap "" HUP 4 | 5 | #if [ $EUID -eq 0 ]; then 6 | # echo "this script must not be run as root. su to hdfs user to run" 7 | # exit 1 8 | #fi 9 | 10 | #MR_EXAMPLES_JAR=/usr/hdp/2.2.0.0-2041/hadoop-mapreduce/hadoop-mapreduce-examples.jar 11 | MR_EXAMPLES_JAR=/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-examples-3.1.1.7.2.2.2-1.jar 12 | 13 | 14 | #SIZE=500G 15 | #ROWS=5000000000 16 | 17 | #SIZE=100G 18 | #ROWS=1000000000 19 | 20 | SIZE=1T 21 | ROWS=10000000000 22 | 23 | # SIZE=10G 24 | # ROWS=100000000 25 | 26 | # SIZE=1G 27 | # ROWS=10000000 28 | 29 | 30 | LOGDIR=logs 31 | 32 | if [ ! -d "$LOGDIR" ] 33 | then 34 | mkdir ./$LOGDIR 35 | fi 36 | 37 | DATE=`date +%Y-%m-%d:%H:%M:%S` 38 | 39 | RESULTSFILE="./$LOGDIR/teragen_results_$DATE" 40 | 41 | 42 | OUTPUT=/user/sunilemanjee/data/sandbox/poc/teragen/${SIZE}-terasort-input 43 | 44 | # teragen.sh 45 | # Kill any running MapReduce jobs 46 | mapred job -list | grep job_ | awk ' { system("mapred job -kill " $1) } ' 47 | # Delete the output directory 48 | hadoop fs -rm -r -f -skipTrash ${OUTPUT} 49 | 50 | # Run teragen 51 | time hadoop jar $MR_EXAMPLES_JAR teragen \ 52 | -Dmapreduce.map.log.level=INFO \ 53 | -Dmapreduce.reduce.log.level=INFO \ 54 | -Dyarn.app.mapreduce.am.log.level=INFO \ 55 | -Dio.file.buffer.size=131072 \ 56 | -Dmapreduce.map.cpu.vcores=1 \ 57 | -Dmapreduce.map.java.opts=-Xmx1536m \ 58 | -Dmapreduce.map.maxattempts=1 \ 59 | -Dmapreduce.map.memory.mb=2048 \ 60 | -Dmapreduce.map.output.compress=true \ 61 | -Dmapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.Lz4Codec \ 62 | -Dmapreduce.reduce.cpu.vcores=1 \ 63 | -Dmapreduce.reduce.java.opts=-Xmx1536m \ 64 | -Dmapreduce.reduce.maxattempts=1 \ 65 | -Dmapreduce.reduce.memory.mb=2048 \ 66 | -Dmapreduce.task.io.sort.factor=100 \ 67 | -Dmapreduce.task.io.sort.mb=384 \ 68 | -Dyarn.app.mapreduce.am.command.opts=-Xmx768m \ 69 | -Dyarn.app.mapreduce.am.resource.mb=1024 \ 70 | -Dmapred.map.tasks=92 \ 71 | ${ROWS} ${OUTPUT} >> $RESULTSFILE 2>&1 72 | 73 | #-Dmapreduce.map.log.level=TRACE \ 74 | #-Dmapreduce.reduce.log.level=TRACE \ 75 | #-Dyarn.app.mapreqduce.am.log.level=TRACE \ 76 | -------------------------------------------------------------------------------- /terasort.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | trap "" HUP 4 | 5 | #if [ $EUID -eq 0 ]; then 6 | # echo "this script must not be run as root. su to hdfs user to run" 7 | # exit 1 8 | #fi 9 | 10 | MR_EXAMPLES_JAR=/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-examples-3.1.1.7.2.2.2-1.jar 11 | 12 | 13 | #SIZE=500G 14 | #SIZE=100G 15 | SIZE=1T 16 | #SIZE=1G 17 | #SIZE=10G 18 | #INPUT=/${SIZE}-terasort-input 19 | #OUTPUT=/${SIZE}-terasort-output 20 | 21 | 22 | LOGDIR=logs 23 | 24 | if [ ! -d "$LOGDIR" ] 25 | then 26 | mkdir ./$LOGDIR 27 | fi 28 | 29 | DATE=`date +%Y-%m-%d:%H:%M:%S` 30 | 31 | RESULTSFILE="./$LOGDIR/terasort_results_$DATE" 32 | 33 | 34 | INPUT=/user/sunilemanjee/data/sandbox/poc/teragen/${SIZE}-terasort-input 35 | OUTPUT=/user/sunilemanjee/data/sandbox/poc/teragen/${SIZE}-terasort-output 36 | 37 | # terasort.sh 38 | # Kill any running MapReduce jobs 39 | mapred job -list | grep job_ | awk ' { system("mapred job -kill " $1) } ' 40 | # Delete the output directory 41 | hadoop fs -rm -r -f -skipTrash ${OUTPUT} 42 | 43 | # Run terasort 44 | time hadoop jar $MR_EXAMPLES_JAR terasort \ 45 | -Dmapreduce.map.log.level=INFO \ 46 | -Dmapreduce.reduce.log.level=INFO \ 47 | -Dyarn.app.mapreduce.am.log.level=INFO \ 48 | -Dio.file.buffer.size=131072 \ 49 | -Dmapreduce.map.cpu.vcores=1 \ 50 | -Dmapreduce.map.java.opts=-Xmx1536m \ 51 | -Dmapreduce.map.maxattempts=1 \ 52 | -Dmapreduce.map.memory.mb=2048 \ 53 | -Dmapreduce.map.output.compress=true \ 54 | -Dmapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.Lz4Codec \ 55 | -Dmapreduce.reduce.cpu.vcores=1 \ 56 | -Dmapreduce.reduce.java.opts=-Xmx1536m \ 57 | -Dmapreduce.reduce.maxattempts=1 \ 58 | -Dmapreduce.reduce.memory.mb=2048 \ 59 | -Dmapreduce.task.io.sort.factor=300 \ 60 | -Dmapreduce.task.io.sort.mb=384 \ 61 | -Dyarn.app.mapreduce.am.command.opts=-Xmx768m \ 62 | -Dyarn.app.mapreduce.am.resource.mb=1024 \ 63 | -Dmapred.reduce.tasks=92 \ 64 | -Dmapreduce.terasort.output.replication=1 \ 65 | ${INPUT} ${OUTPUT} >> $RESULTSFILE 2>&1 66 | -------------------------------------------------------------------------------- /validate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | trap "" HUP 4 | 5 | #if [ $EUID -eq 0 ]; then 6 | # echo "this script must not be run as root. su to hdfs user to run" 7 | # exit 1 8 | #fi 9 | 10 | #MR_EXAMPLES_JAR=/usr/hdp/2.3.2.0-2950/hadoop-mapreduce/hadoop-mapreduce-examples.jar 11 | MR_EXAMPLES_JAR=/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-examples-3.1.1.7.2.2.2-1.jar 12 | 13 | #SIZE=500G 14 | #SIZE=100G 15 | SIZE=1T 16 | #SIZE=1G 17 | #SIZE=10G 18 | 19 | 20 | LOGDIR=logs 21 | 22 | if [ ! -d "$LOGDIR" ] 23 | then 24 | mkdir ./$LOGDIR 25 | fi 26 | 27 | DATE=`date +%Y-%m-%d:%H:%M:%S` 28 | 29 | RESULTSFILE="./$LOGDIR/teravalidate_results_$DATE" 30 | 31 | 32 | OUTPUT=/user/sunilemanjee/data/sandbox/poc/teragen/${SIZE}-terasort-output 33 | REPORT=/user/sunilemanjee/data/sandbox/poc/teragen/${SIZE}-terasort-report 34 | 35 | 36 | # teravalidate.sh 37 | # Kill any running MapReduce jobs 38 | mapred job -list | grep job_ | awk ' { system("mapred job -kill " $1) } ' 39 | # Delete the output directory 40 | hadoop fs -rm -r -f -skipTrash ${REPORT} 41 | # Run teravalidate 42 | time hadoop jar $MR_EXAMPLES_JAR teravalidate \ 43 | -Ddfs.blocksize=256M \ 44 | -Dio.file.buffer.size=131072 \ 45 | -Dmapreduce.map.memory.mb=2048 \ 46 | -Dmapreduce.map.java.opts=-Xmx1536m \ 47 | -Dmapreduce.reduce.memory.mb=2048 \ 48 | -Dmapreduce.reduce.java.opts=-Xmx1536m \ 49 | -Dyarn.app.mapreduce.am.resource.mb=1024 \ 50 | -Dyarn.app.mapreduce.am.command-opts=-Xmx768m \ 51 | -Dmapreduce.task.io.sort.mb=1 \ 52 | -Dmapred.map.tasks=185 \ 53 | -Dmapred.reduce.tasks=185 \ 54 | ${OUTPUT} ${REPORT} >> $RESULTSFILE 2>&1 55 | -------------------------------------------------------------------------------- /yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | hadoop.registry.rm.enabled 5 | false 6 | 7 | 8 | 9 | hadoop.registry.zk.quorum 10 | ip-10-0-73-146.ec2.internal:2181 11 | 12 | 13 | 14 | yarn.acl.enable 15 | false 16 | 17 | 18 | 19 | yarn.admin.acl 20 | yarn 21 | 22 | 23 | 24 | yarn.application.classpath 25 | $HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/* 26 | 27 | 28 | 29 | yarn.client.nodemanager-connect.max-wait-ms 30 | 60000 31 | 32 | 33 | 34 | yarn.client.nodemanager-connect.retry-interval-ms 35 | 10000 36 | 37 | 38 | 39 | yarn.http.policy 40 | HTTP_ONLY 41 | 42 | 43 | 44 | yarn.log-aggregation-enable 45 | true 46 | 47 | 48 | 49 | yarn.log-aggregation.retain-seconds 50 | 2592000 51 | 52 | 53 | 54 | yarn.log.server.url 55 | http://ip-10-0-73-146.ec2.internal:19888/jobhistory/logs 56 | 57 | 58 | 59 | yarn.node-labels.enabled 60 | false 61 | 62 | 63 | 64 | yarn.node-labels.fs-store.retry-policy-spec 65 | 2000, 500 66 | 67 | 68 | 69 | yarn.node-labels.fs-store.root-dir 70 | /system/yarn/node-labels 71 | 72 | 73 | 74 | yarn.nodemanager.address 75 | 0.0.0.0:45454 76 | 77 | 78 | 79 | yarn.nodemanager.admin-env 80 | MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX 81 | 82 | 83 | 84 | yarn.nodemanager.aux-services 85 | mapreduce_shuffle,spark_shuffle 86 | 87 | 88 | 89 | yarn.nodemanager.aux-services.mapreduce_shuffle.class 90 | org.apache.hadoop.mapred.ShuffleHandler 91 | 92 | 93 | 94 | yarn.nodemanager.aux-services.spark_shuffle.class 95 | org.apache.spark.network.yarn.YarnShuffleService 96 | 97 | 98 | 99 | yarn.nodemanager.bind-host 100 | 0.0.0.0 101 | 102 | 103 | 104 | yarn.nodemanager.container-executor.class 105 | org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor 106 | 107 | 108 | 109 | yarn.nodemanager.container-monitor.interval-ms 110 | 3000 111 | 112 | 113 | 114 | yarn.nodemanager.delete.debug-delay-sec 115 | 0 116 | 117 | 118 | 119 | yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage 120 | 90 121 | 122 | 123 | 124 | yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb 125 | 1000 126 | 127 | 128 | 129 | yarn.nodemanager.disk-health-checker.min-healthy-disks 130 | 0.25 131 | 132 | 133 | 134 | yarn.nodemanager.health-checker.interval-ms 135 | 135000 136 | 137 | 138 | 139 | yarn.nodemanager.health-checker.script.timeout-ms 140 | 60000 141 | 142 | 143 | 144 | yarn.nodemanager.linux-container-executor.cgroups.hierarchy 145 | hadoop-yarn 146 | 147 | 148 | 149 | yarn.nodemanager.linux-container-executor.cgroups.mount 150 | false 151 | 152 | 153 | 154 | yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage 155 | false 156 | 157 | 158 | 159 | yarn.nodemanager.linux-container-executor.group 160 | hadoop 161 | 162 | 163 | 164 | yarn.nodemanager.linux-container-executor.resources-handler.class 165 | org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler 166 | 167 | 168 | 169 | yarn.nodemanager.local-dirs 170 | /hadoopfs/fs1/yarn/nodemanager,/hadoopfs/fs2/yarn/nodemanager,/hadoopfs/fs3/yarn/nodemanager 171 | 172 | 173 | 174 | yarn.nodemanager.log-aggregation.compression-type 175 | gz 176 | 177 | 178 | 179 | yarn.nodemanager.log-aggregation.debug-enabled 180 | false 181 | 182 | 183 | 184 | yarn.nodemanager.log-aggregation.num-log-files-per-app 185 | 30 186 | 187 | 188 | 189 | yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds 190 | -1 191 | 192 | 193 | 194 | yarn.nodemanager.log-dirs 195 | /hadoopfs/fs1/yarn/nodemanager/log,/hadoopfs/fs2/yarn/nodemanager/log,/hadoopfs/fs3/yarn/nodemanager/log 196 | 197 | 198 | 199 | yarn.nodemanager.log.retain-second 200 | 604800 201 | 202 | 203 | 204 | yarn.nodemanager.recovery.dir 205 | /var/log/hadoop-yarn/nodemanager/recovery-state 206 | 207 | 208 | 209 | yarn.nodemanager.recovery.enabled 210 | true 211 | 212 | 213 | 214 | yarn.nodemanager.remote-app-log-dir 215 | /app-logs 216 | 217 | 218 | 219 | yarn.nodemanager.remote-app-log-dir-suffix 220 | logs 221 | 222 | 223 | 224 | yarn.nodemanager.resource.cpu-vcores 225 | 12 226 | 227 | 228 | 229 | yarn.nodemanager.resource.memory-mb 230 | 101888 231 | 232 | 233 | 234 | yarn.nodemanager.resource.percentage-physical-cpu-limit 235 | 80 236 | 237 | 238 | 239 | yarn.nodemanager.vmem-check-enabled 240 | false 241 | 242 | 243 | 244 | yarn.nodemanager.vmem-pmem-ratio 245 | 2.1 246 | 247 | 248 | 249 | yarn.resourcemanager.address 250 | ip-10-0-73-146.ec2.internal:8050 251 | 252 | 253 | 254 | yarn.resourcemanager.admin.address 255 | ip-10-0-73-146.ec2.internal:8141 256 | 257 | 258 | 259 | yarn.resourcemanager.am.max-attempts 260 | 2 261 | 262 | 263 | 264 | yarn.resourcemanager.bind-host 265 | 0.0.0.0 266 | 267 | 268 | 269 | yarn.resourcemanager.connect.max-wait.ms 270 | 900000 271 | 272 | 273 | 274 | yarn.resourcemanager.connect.retry-interval.ms 275 | 30000 276 | 277 | 278 | 279 | yarn.resourcemanager.fs.state-store.retry-policy-spec 280 | 2000, 500 281 | 282 | 283 | 284 | yarn.resourcemanager.fs.state-store.uri 285 | 286 | 287 | 288 | 289 | yarn.resourcemanager.ha.enabled 290 | false 291 | 292 | 293 | 294 | yarn.resourcemanager.hostname 295 | ip-10-0-73-146.ec2.internal 296 | 297 | 298 | 299 | yarn.resourcemanager.nodes.exclude-path 300 | /etc/hadoop/conf/yarn.exclude 301 | 302 | 303 | 304 | yarn.resourcemanager.recovery.enabled 305 | true 306 | 307 | 308 | 309 | yarn.resourcemanager.resource-tracker.address 310 | ip-10-0-73-146.ec2.internal:8025 311 | 312 | 313 | 314 | yarn.resourcemanager.scheduler.address 315 | ip-10-0-73-146.ec2.internal:8030 316 | 317 | 318 | 319 | yarn.resourcemanager.scheduler.class 320 | org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler 321 | 322 | 323 | 324 | yarn.resourcemanager.scheduler.monitor.enable 325 | false 326 | 327 | 328 | 329 | yarn.resourcemanager.state-store.max-completed-applications 330 | ${yarn.resourcemanager.max-completed-applications} 331 | 332 | 333 | 334 | yarn.resourcemanager.store.class 335 | org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore 336 | 337 | 338 | 339 | yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size 340 | 10 341 | 342 | 343 | 344 | yarn.resourcemanager.system-metrics-publisher.enabled 345 | true 346 | 347 | 348 | 349 | yarn.resourcemanager.webapp.address 350 | ip-10-0-73-146.ec2.internal:8088 351 | 352 | 353 | 354 | yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled 355 | false 356 | 357 | 358 | 359 | yarn.resourcemanager.webapp.https.address 360 | ip-10-0-73-146.ec2.internal:8090 361 | 362 | 363 | 364 | yarn.resourcemanager.work-preserving-recovery.enabled 365 | true 366 | 367 | 368 | 369 | yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms 370 | 10000 371 | 372 | 373 | 374 | yarn.resourcemanager.zk-acl 375 | world:anyone:rwcda 376 | 377 | 378 | 379 | yarn.resourcemanager.zk-address 380 | ip-10-0-73-146.ec2.internal:2181 381 | 382 | 383 | 384 | yarn.resourcemanager.zk-num-retries 385 | 1000 386 | 387 | 388 | 389 | yarn.resourcemanager.zk-retry-interval-ms 390 | 1000 391 | 392 | 393 | 394 | yarn.resourcemanager.zk-state-store.parent-path 395 | /rmstore 396 | 397 | 398 | 399 | yarn.resourcemanager.zk-timeout-ms 400 | 10000 401 | 402 | 403 | 404 | yarn.scheduler.maximum-allocation-mb 405 | 101888 406 | 407 | 408 | 409 | yarn.scheduler.maximum-allocation-vcores 410 | 12 411 | 412 | 413 | 414 | yarn.scheduler.minimum-allocation-mb 415 | 768 416 | 417 | 418 | 419 | yarn.scheduler.minimum-allocation-vcores 420 | 1 421 | 422 | 423 | 424 | yarn.timeline-service.address 425 | ip-10-0-73-146.ec2.internal:10200 426 | 427 | 428 | 429 | yarn.timeline-service.bind-host 430 | 0.0.0.0 431 | 432 | 433 | 434 | yarn.timeline-service.client.max-retries 435 | 30 436 | 437 | 438 | 439 | yarn.timeline-service.client.retry-interval-ms 440 | 1000 441 | 442 | 443 | 444 | yarn.timeline-service.enabled 445 | true 446 | 447 | 448 | 449 | yarn.timeline-service.entity-group-fs-store.active-dir 450 | /ats/active/ 451 | 452 | 453 | 454 | yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds 455 | 3600 456 | 457 | 458 | 459 | yarn.timeline-service.entity-group-fs-store.done-dir 460 | /ats/done/ 461 | 462 | 463 | 464 | yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes 465 | org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl 466 | 467 | 468 | 469 | yarn.timeline-service.entity-group-fs-store.retain-seconds 470 | 604800 471 | 472 | 473 | 474 | yarn.timeline-service.entity-group-fs-store.scan-interval-seconds 475 | 60 476 | 477 | 478 | 479 | yarn.timeline-service.entity-group-fs-store.summary-store 480 | org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore 481 | 482 | 483 | 484 | yarn.timeline-service.generic-application-history.store-class 485 | org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore 486 | 487 | 488 | 489 | yarn.timeline-service.http-authentication.simple.anonymous.allowed 490 | true 491 | 492 | 493 | 494 | yarn.timeline-service.http-authentication.type 495 | simple 496 | 497 | 498 | 499 | yarn.timeline-service.leveldb-state-store.path 500 | /hadoop/yarn/timeline 501 | 502 | 503 | 504 | yarn.timeline-service.leveldb-timeline-store.path 505 | /hadoopfs/fs1/yarn/timeline 506 | 507 | 508 | 509 | yarn.timeline-service.leveldb-timeline-store.read-cache-size 510 | 104857600 511 | 512 | 513 | 514 | yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size 515 | 10000 516 | 517 | 518 | 519 | yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size 520 | 10000 521 | 522 | 523 | 524 | yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms 525 | 300000 526 | 527 | 528 | 529 | yarn.timeline-service.recovery.enabled 530 | true 531 | 532 | 533 | 534 | yarn.timeline-service.state-store-class 535 | org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore 536 | 537 | 538 | 539 | yarn.timeline-service.store-class 540 | org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore 541 | 542 | 543 | 544 | yarn.timeline-service.ttl-enable 545 | true 546 | 547 | 548 | 549 | yarn.timeline-service.ttl-ms 550 | 2678400000 551 | 552 | 553 | 554 | yarn.timeline-service.version 555 | 1.5 556 | 557 | 558 | 559 | yarn.timeline-service.webapp.address 560 | ip-10-0-73-146.ec2.internal:8188 561 | 562 | 563 | 564 | yarn.timeline-service.webapp.https.address 565 | ip-10-0-73-146.ec2.internal:8190 566 | --------------------------------------------------------------------------------