├── workflow_stats_parser ├── sample_onestage_output │ ├── output_io_reads_per_sec.png │ ├── output_committed_mem_plot.png │ ├── output_io_writes_per_sec.png │ ├── output_io_waittime_per_phase_plot.png │ ├── output_average_cpu_utilization_plot.png │ ├── 2014-05-12_21.50.41_iostat.csv │ ├── 2014-05-12_21.50.41_sar_reads.csv │ ├── 2014-05-12_21.50.41_sar.csv │ ├── 2014-05-12_21.50.41_sar_writes.csv │ └── 2014-05-12_21.50.41_active_mem.csv ├── sample_multistage_output │ ├── output_io_reads_per_sec.png │ ├── output_committed_mem_plot.png │ ├── output_io_writes_per_sec.png │ ├── output_io_waittime_per_phase_plot.png │ ├── output_average_cpu_utilization_plot.png │ ├── 2014-05-12_21.49.51_iostat.csv │ ├── 2014-05-12_21.49.51_sar_reads.csv │ ├── 2014-05-12_21.49.51_sar.csv │ ├── 2014-05-12_21.49.51_sar_writes.csv │ └── 2014-05-12_21.49.51_active_mem.csv ├── sample_multistage_input │ ├── run.sample..stage2.1u │ │ ├── C097FACXX111207.1.AGTTGCTT16..MarkDuplicates.1u.30sec_sampling.sar.data │ │ └── C097FACXX111207.1.AGTTGCTT16..MarkDuplicates.1u.30sec_sampling.iostat │ ├── run.sample..stage3.1u │ │ ├── C097FACXX111207.1.AGTTGCTT16..IndelRealigner.1u.30sec_sampling.sar.data │ │ └── C097FACXX111207.1.AGTTGCTT16..IndelRealigner.1u.30sec_sampling.iostat │ └── run.sample..stage1.1u │ │ ├── C097FACXX111207.1.AGTTGCTT16..MergeBamAlignment.1u.30sec_sampling.sar.data │ │ └── C097FACXX111207.1.AGTTGCTT16..MergeBamAlignment.1u.30sec_sampling.iostat ├── sample_onestage_input │ └── run.sample..stage1.1u │ │ ├── C097FACXX111207.1.AGTTGCTT16..MergeBamAlignment.1u.30sec_sampling.sar.data │ │ └── C097FACXX111207.1.AGTTGCTT16..MergeBamAlignment.1u.30sec_sampling.iostat ├── workflow_dictionaries.py └── plot_templates │ ├── template_iostat.plt │ ├── template_sar_reads.plt │ ├── template_sar_writes.plt │ ├── template_sar.plt │ ├── template_total_core_mpstat.plt │ ├── template_active_core_mpstat.plt │ ├── template_committed_mem.plt │ └── template_many_cores.plt ├── LICENSE ├── kill_scripts ├── kill-net-stat-script.sh ├── kill-free-script.sh ├── kill-mp-stat-script.sh ├── kill-sar-script.sh └── kill-iostat-script.sh ├── data_collection_workflow_template.pl ├── collect_stats.ksh ├── README.md └── workflow_profiler.py /workflow_stats_parser/sample_onestage_output/output_io_reads_per_sec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/workflow-profiler/HEAD/workflow_stats_parser/sample_onestage_output/output_io_reads_per_sec.png -------------------------------------------------------------------------------- /workflow_stats_parser/sample_multistage_output/output_io_reads_per_sec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/workflow-profiler/HEAD/workflow_stats_parser/sample_multistage_output/output_io_reads_per_sec.png -------------------------------------------------------------------------------- /workflow_stats_parser/sample_onestage_output/output_committed_mem_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/workflow-profiler/HEAD/workflow_stats_parser/sample_onestage_output/output_committed_mem_plot.png -------------------------------------------------------------------------------- /workflow_stats_parser/sample_onestage_output/output_io_writes_per_sec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/workflow-profiler/HEAD/workflow_stats_parser/sample_onestage_output/output_io_writes_per_sec.png -------------------------------------------------------------------------------- /workflow_stats_parser/sample_multistage_output/output_committed_mem_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/workflow-profiler/HEAD/workflow_stats_parser/sample_multistage_output/output_committed_mem_plot.png -------------------------------------------------------------------------------- /workflow_stats_parser/sample_multistage_output/output_io_writes_per_sec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/workflow-profiler/HEAD/workflow_stats_parser/sample_multistage_output/output_io_writes_per_sec.png -------------------------------------------------------------------------------- /workflow_stats_parser/sample_onestage_output/output_io_waittime_per_phase_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/workflow-profiler/HEAD/workflow_stats_parser/sample_onestage_output/output_io_waittime_per_phase_plot.png -------------------------------------------------------------------------------- /workflow_stats_parser/sample_multistage_output/output_io_waittime_per_phase_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/workflow-profiler/HEAD/workflow_stats_parser/sample_multistage_output/output_io_waittime_per_phase_plot.png -------------------------------------------------------------------------------- /workflow_stats_parser/sample_onestage_output/output_average_cpu_utilization_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/workflow-profiler/HEAD/workflow_stats_parser/sample_onestage_output/output_average_cpu_utilization_plot.png -------------------------------------------------------------------------------- /workflow_stats_parser/sample_multistage_output/output_average_cpu_utilization_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/workflow-profiler/HEAD/workflow_stats_parser/sample_multistage_output/output_average_cpu_utilization_plot.png -------------------------------------------------------------------------------- /workflow_stats_parser/sample_multistage_input/run.sample..stage2.1u/C097FACXX111207.1.AGTTGCTT16..MarkDuplicates.1u.30sec_sampling.sar.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/workflow-profiler/HEAD/workflow_stats_parser/sample_multistage_input/run.sample..stage2.1u/C097FACXX111207.1.AGTTGCTT16..MarkDuplicates.1u.30sec_sampling.sar.data -------------------------------------------------------------------------------- /workflow_stats_parser/sample_multistage_input/run.sample..stage3.1u/C097FACXX111207.1.AGTTGCTT16..IndelRealigner.1u.30sec_sampling.sar.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/workflow-profiler/HEAD/workflow_stats_parser/sample_multistage_input/run.sample..stage3.1u/C097FACXX111207.1.AGTTGCTT16..IndelRealigner.1u.30sec_sampling.sar.data -------------------------------------------------------------------------------- /workflow_stats_parser/sample_multistage_input/run.sample..stage1.1u/C097FACXX111207.1.AGTTGCTT16..MergeBamAlignment.1u.30sec_sampling.sar.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/workflow-profiler/HEAD/workflow_stats_parser/sample_multistage_input/run.sample..stage1.1u/C097FACXX111207.1.AGTTGCTT16..MergeBamAlignment.1u.30sec_sampling.sar.data -------------------------------------------------------------------------------- /workflow_stats_parser/sample_onestage_input/run.sample..stage1.1u/C097FACXX111207.1.AGTTGCTT16..MergeBamAlignment.1u.30sec_sampling.sar.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/workflow-profiler/HEAD/workflow_stats_parser/sample_onestage_input/run.sample..stage1.1u/C097FACXX111207.1.AGTTGCTT16..MergeBamAlignment.1u.30sec_sampling.sar.data -------------------------------------------------------------------------------- /workflow_stats_parser/sample_onestage_output/2014-05-12_21.50.41_iostat.csv: -------------------------------------------------------------------------------- 1 | IOSTAT: time waiting on io 2 | Stage1 3 | time in interval: 100s,average time waiting on io per 30.0s 4 | 2014-04-15 22:27:59,19 5 | 2014-04-15 22:28:29,11 6 | 2014-04-15 22:28:59,16 7 | 2014-04-15 22:29:29,33 8 | 2014-04-15 22:29:59,29 9 | 2014-04-15 22:30:29,28 10 | 2014-04-15 22:30:59,29 11 | 2014-04-15 22:31:29,20 12 | 2014-04-15 22:31:59,26 13 | 2014-04-15 22:32:29,28 14 | 2014-04-15 22:32:59,24 15 | 2014-04-15 22:33:29,34 16 | 2014-04-15 22:33:59,9 17 | 2014-04-15 22:34:29,15 18 | 2014-04-15 22:34:59,15 19 | 2014-04-15 22:35:29,11 20 | 2014-04-15 22:35:59,15 21 | 2014-04-15 22:36:29,14 22 | 2014-04-15 22:36:59,16 23 | 2014-04-15 22:37:29,12 24 | 2014-04-15 22:37:59,19 25 | 2014-04-15 22:38:29,14 26 | 2014-04-15 22:38:59,23 27 | 2014-04-15 22:39:29,24 28 | 2014-04-15 22:39:59,22 29 | 2014-04-15 22:40:29,23 30 | 2014-04-15 22:40:59,16 31 | 2014-04-15 22:41:29,18 32 | 2014-04-15 22:41:59,22 33 | 2014-04-15 22:42:29,15 34 | 2014-04-15 22:42:59,19 35 | 2014-04-15 22:43:29,17 36 | 2014-04-15 22:43:59,14 37 | 2014-04-15 22:44:29,22 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Intel Corporation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /workflow_stats_parser/sample_onestage_output/2014-05-12_21.50.41_sar_reads.csv: -------------------------------------------------------------------------------- 1 | SAR_READS: io reads in mb/sec 2 | Stage1 3 | time in interval: 100s,average io reads in mb/sec per 30.0s 4 | 2014-04-15 22:28:29,0.100361328125 5 | 2014-04-15 22:28:59,0.0 6 | 2014-04-15 22:29:29,0.0 7 | 2014-04-15 22:29:59,0.0 8 | 2014-04-15 22:30:29,0.0 9 | 2014-04-15 22:30:59,0.0 10 | 2014-04-15 22:31:29,0.0 11 | 2014-04-15 22:31:59,0.0 12 | 2014-04-15 22:32:29,0.0 13 | 2014-04-15 22:32:59,0.0 14 | 2014-04-15 22:33:29,0.0 15 | 2014-04-15 22:33:59,0.0 16 | 2014-04-15 22:34:29,0.0 17 | 2014-04-15 22:34:59,0.0 18 | 2014-04-15 22:35:29,0.0 19 | 2014-04-15 22:35:59,0.0 20 | 2014-04-15 22:36:29,0.0 21 | 2014-04-15 22:36:59,0.0 22 | 2014-04-15 22:37:29,0.0013037109375 23 | 2014-04-15 22:37:59,0.0 24 | 2014-04-15 22:38:29,0.0 25 | 2014-04-15 22:38:59,0.0 26 | 2014-04-15 22:39:29,0.0 27 | 2014-04-15 22:39:59,0.0 28 | 2014-04-15 22:40:29,0.0 29 | 2014-04-15 22:40:59,0.0 30 | 2014-04-15 22:41:29,0.0 31 | 2014-04-15 22:41:59,0.0 32 | 2014-04-15 22:42:29,0.0 33 | 2014-04-15 22:42:59,0.0 34 | 2014-04-15 22:43:29,0.0 35 | 2014-04-15 22:43:59,0.0 36 | 2014-04-15 22:44:29,0.0 37 | -------------------------------------------------------------------------------- /workflow_stats_parser/sample_onestage_output/2014-05-12_21.50.41_sar.csv: -------------------------------------------------------------------------------- 1 | SAR: cpu load (all cores) 2 | Stage1 3 | time in interval: 100s,average cpu load (all cores) per 30.0s 4 | 2014-04-15 22:28:29,13.31 5 | 2014-04-15 22:28:59,15.07 6 | 2014-04-15 22:29:29,12.87 7 | 2014-04-15 22:29:59,15.66 8 | 2014-04-15 22:30:29,10.22 9 | 2014-04-15 22:30:59,13.8 10 | 2014-04-15 22:31:29,16.68 11 | 2014-04-15 22:31:59,14.44 12 | 2014-04-15 22:32:29,11.4 13 | 2014-04-15 22:32:59,12.74 14 | 2014-04-15 22:33:29,11.43 15 | 2014-04-15 22:33:59,13.79 16 | 2014-04-15 22:34:29,9.95 17 | 2014-04-15 22:34:59,13.95 18 | 2014-04-15 22:35:29,15.21 19 | 2014-04-15 22:35:59,10.51 20 | 2014-04-15 22:36:29,15.25 21 | 2014-04-15 22:36:59,12.85 22 | 2014-04-15 22:37:29,11.89 23 | 2014-04-15 22:37:59,4.41 24 | 2014-04-15 22:38:29,4.14 25 | 2014-04-15 22:38:59,4.3 26 | 2014-04-15 22:39:29,4.26 27 | 2014-04-15 22:39:59,4.41 28 | 2014-04-15 22:40:29,4.43 29 | 2014-04-15 22:40:59,4.6 30 | 2014-04-15 22:41:29,4.56 31 | 2014-04-15 22:41:59,4.47 32 | 2014-04-15 22:42:29,4.39 33 | 2014-04-15 22:42:59,4.23 34 | 2014-04-15 22:43:29,4.1 35 | 2014-04-15 22:43:59,4.03 36 | 2014-04-15 22:44:29,3.95 37 | -------------------------------------------------------------------------------- /workflow_stats_parser/sample_onestage_output/2014-05-12_21.50.41_sar_writes.csv: -------------------------------------------------------------------------------- 1 | SAR_WRITES: io writes in mb/sec 2 | Stage1 3 | time in interval: 100s,average io writes in mb/sec per 30.0s 4 | 2014-04-15 22:28:29,6.040546875 5 | 2014-04-15 22:28:59,7.9431005859375 6 | 2014-04-15 22:29:29,8.686162109375 7 | 2014-04-15 22:29:59,7.6938916015625 8 | 2014-04-15 22:30:29,8.5798046875 9 | 2014-04-15 22:30:59,7.900537109375 10 | 2014-04-15 22:31:29,7.7779833984375 11 | 2014-04-15 22:31:59,7.80728515625 12 | 2014-04-15 22:32:29,7.8005126953125 13 | 2014-04-15 22:32:59,8.0955615234375 14 | 2014-04-15 22:33:29,9.1703662109375 15 | 2014-04-15 22:33:59,8.4146630859375 16 | 2014-04-15 22:34:29,7.8230615234375 17 | 2014-04-15 22:34:59,7.7148779296875 18 | 2014-04-15 22:35:29,7.7256640625 19 | 2014-04-15 22:35:59,8.2668603515625 20 | 2014-04-15 22:36:29,7.8008935546875 21 | 2014-04-15 22:36:59,8.61263671875 22 | 2014-04-15 22:37:29,9.041591796875 23 | 2014-04-15 22:37:59,7.5833935546875 24 | 2014-04-15 22:38:29,7.4734912109375 25 | 2014-04-15 22:38:59,7.6162744140625 26 | 2014-04-15 22:39:29,7.5460009765625 27 | 2014-04-15 22:39:59,7.28931640625 28 | 2014-04-15 22:40:29,7.5596435546875 29 | 2014-04-15 22:40:59,7.61119140625 30 | 2014-04-15 22:41:29,7.4699853515625 31 | 2014-04-15 22:41:59,7.6385693359375 32 | 2014-04-15 22:42:29,7.531787109375 33 | 2014-04-15 22:42:59,7.487216796875 34 | 2014-04-15 22:43:29,7.633876953125 35 | 2014-04-15 22:43:59,7.5304833984375 36 | 2014-04-15 22:44:29,7.55291015625 37 | -------------------------------------------------------------------------------- /workflow_stats_parser/sample_onestage_output/2014-05-12_21.50.41_active_mem.csv: -------------------------------------------------------------------------------- 1 | ACTIVE_MEM: committed memory (gb) 2 | Stage1 3 | time in interval: 100s,average committed memory (gb) per 30.0s 4 | 2014-04-15 22:28:29,3.5888900756835938 5 | 2014-04-15 22:28:59,3.6545562744140625 6 | 2014-04-15 22:29:29,3.6648597717285156 7 | 2014-04-15 22:29:59,4.042400360107422 8 | 2014-04-15 22:30:29,4.042549133300781 9 | 2014-04-15 22:30:59,3.997783660888672 10 | 2014-04-15 22:31:29,4.195831298828125 11 | 2014-04-15 22:31:59,4.147026062011719 12 | 2014-04-15 22:32:29,4.143947601318359 13 | 2014-04-15 22:32:59,4.1931610107421875 14 | 2014-04-15 22:33:29,4.187732696533203 15 | 2014-04-15 22:33:59,4.241264343261719 16 | 2014-04-15 22:34:29,4.2400970458984375 17 | 2014-04-15 22:34:59,4.220516204833984 18 | 2014-04-15 22:35:29,4.366199493408203 19 | 2014-04-15 22:35:59,4.365776062011719 20 | 2014-04-15 22:36:29,4.5504608154296875 21 | 2014-04-15 22:36:59,4.545635223388672 22 | 2014-04-15 22:37:29,4.4858856201171875 23 | 2014-04-15 22:37:59,4.485889434814453 24 | 2014-04-15 22:38:29,4.4720001220703125 25 | 2014-04-15 22:38:59,4.48583984375 26 | 2014-04-15 22:39:29,4.573696136474609 27 | 2014-04-15 22:39:59,4.547172546386719 28 | 2014-04-15 22:40:29,4.573863983154297 29 | 2014-04-15 22:40:59,4.573722839355469 30 | 2014-04-15 22:41:29,4.573764801025391 31 | 2014-04-15 22:41:59,4.573772430419922 32 | 2014-04-15 22:42:29,4.56280517578125 33 | 2014-04-15 22:42:59,4.564159393310547 34 | 2014-04-15 22:43:29,4.5294342041015625 35 | 2014-04-15 22:43:59,4.560081481933594 36 | 2014-04-15 22:44:29,4.573619842529297 37 | -------------------------------------------------------------------------------- /workflow_stats_parser/sample_multistage_output/2014-05-12_21.49.51_iostat.csv: -------------------------------------------------------------------------------- 1 | IOSTAT: time waiting on io 2 | Stage1,,Stage2,,Stage3 3 | time in interval: 100s,average time waiting on io per 30.0s,time in interval: 100s,average time waiting on io per 30.0s,time in interval: 100s,average time waiting on io per 30.0s 4 | 2014-04-15 22:27:59,19,2014-04-15 22:45:44,19,2014-04-15 22:57:40,19 5 | 2014-04-15 22:28:29,11,2014-04-15 22:46:14,5,2014-04-15 22:58:10,6 6 | 2014-04-15 22:28:59,16,2014-04-15 22:46:44,16,2014-04-15 22:58:40,13 7 | 2014-04-15 22:29:29,33,2014-04-15 22:47:14,7,2014-04-15 22:59:10,22 8 | 2014-04-15 22:29:59,29,2014-04-15 22:47:44,17,2014-04-15 22:59:40,36 9 | 2014-04-15 22:30:29,28,2014-04-15 22:48:14,10,2014-04-15 23:00:10,26 10 | 2014-04-15 22:30:59,29,2014-04-15 22:48:44,19,2014-04-15 23:00:40,18 11 | 2014-04-15 22:31:29,20,2014-04-15 22:49:14,23,2014-04-15 23:01:10,18 12 | 2014-04-15 22:31:59,26,2014-04-15 22:49:44,17,2014-04-15 23:01:40,31 13 | 2014-04-15 22:32:29,28,2014-04-15 22:50:14,13,2014-04-15 23:02:10,12 14 | 2014-04-15 22:32:59,24,2014-04-15 22:50:44,16,2014-04-15 23:02:40,19 15 | 2014-04-15 22:33:29,34,2014-04-15 22:51:14,17,2014-04-15 23:03:10,7 16 | 2014-04-15 22:33:59,9,2014-04-15 22:51:44,10,2014-04-15 23:03:40,21 17 | 2014-04-15 22:34:29,15,2014-04-15 22:52:14,16,2014-04-15 23:04:10,16 18 | 2014-04-15 22:34:59,15,2014-04-15 22:52:44,18,2014-04-15 23:04:40,16 19 | 2014-04-15 22:35:29,11,2014-04-15 22:53:14,24,2014-04-15 23:05:10,14 20 | 2014-04-15 22:35:59,15,2014-04-15 22:53:44,30,2014-04-15 23:05:40,15 21 | 2014-04-15 22:36:29,14,2014-04-15 22:54:14,19,2014-04-15 23:06:10,20 22 | 2014-04-15 22:36:59,16,2014-04-15 22:54:44,16,2014-04-15 23:06:40,15 23 | 2014-04-15 22:37:29,12,2014-04-15 22:55:14,13,2014-04-15 23:07:10,14 24 | 2014-04-15 22:37:59,19,2014-04-15 22:55:44,20,2014-04-15 23:07:40,19 25 | 2014-04-15 22:38:29,14,2014-04-15 22:56:14,18,2014-04-15 23:08:10,14 26 | 2014-04-15 22:38:59,23,,,2014-04-15 23:08:40,15 27 | 2014-04-15 22:39:29,24,,,2014-04-15 23:09:10,12 28 | 2014-04-15 22:39:59,22,,,2014-04-15 23:09:40,21 29 | 2014-04-15 22:40:29,23,,,2014-04-15 23:10:10,14 30 | 2014-04-15 22:40:59,16,,,2014-04-15 23:10:40,11 31 | 2014-04-15 22:41:29,18,,,2014-04-15 23:11:10,19 32 | 2014-04-15 22:41:59,22,,,2014-04-15 23:11:40,23 33 | 2014-04-15 22:42:29,15,,,2014-04-15 23:12:10,10 34 | 2014-04-15 22:42:59,19,,,2014-04-15 23:12:40,13 35 | 2014-04-15 22:43:29,17 36 | 2014-04-15 22:43:59,14 37 | 2014-04-15 22:44:29,22 38 | -------------------------------------------------------------------------------- /workflow_stats_parser/sample_multistage_output/2014-05-12_21.49.51_sar_reads.csv: -------------------------------------------------------------------------------- 1 | SAR_READS: io reads in mb/sec 2 | Stage1,,Stage2,,Stage3 3 | time in interval: 100s,average io reads in mb/sec per 30.0s,time in interval: 100s,average io reads in mb/sec per 30.0s,time in interval: 100s,average io reads in mb/sec per 30.0s 4 | 2014-04-15 22:28:29,0.100361328125,2014-04-15 22:46:14,0.0192822265625,2014-04-15 22:58:10,0.659130859375 5 | 2014-04-15 22:28:59,0.0,2014-04-15 22:46:44,0.0,2014-04-15 22:58:40,0.0 6 | 2014-04-15 22:29:29,0.0,2014-04-15 22:47:14,0.0,2014-04-15 22:59:10,0.0 7 | 2014-04-15 22:29:59,0.0,2014-04-15 22:47:44,0.0131689453125,2014-04-15 22:59:40,0.0 8 | 2014-04-15 22:30:29,0.0,2014-04-15 22:48:14,0.0,2014-04-15 23:00:10,0.0 9 | 2014-04-15 22:30:59,0.0,2014-04-15 22:48:44,0.0,2014-04-15 23:00:40,0.0 10 | 2014-04-15 22:31:29,0.0,2014-04-15 22:49:14,0.0,2014-04-15 23:01:10,0.0 11 | 2014-04-15 22:31:59,0.0,2014-04-15 22:49:44,0.002216796875,2014-04-15 23:01:40,0.0 12 | 2014-04-15 22:32:29,0.0,2014-04-15 22:50:14,0.0,2014-04-15 23:02:10,0.0 13 | 2014-04-15 22:32:59,0.0,2014-04-15 22:50:44,0.0,2014-04-15 23:02:40,0.0 14 | 2014-04-15 22:33:29,0.0,2014-04-15 22:51:14,0.0,2014-04-15 23:03:10,0.0 15 | 2014-04-15 22:33:59,0.0,2014-04-15 22:51:44,0.0,2014-04-15 23:03:40,0.0 16 | 2014-04-15 22:34:29,0.0,2014-04-15 22:52:14,0.0,2014-04-15 23:04:10,0.0 17 | 2014-04-15 22:34:59,0.0,2014-04-15 22:52:44,0.0,2014-04-15 23:04:40,0.0 18 | 2014-04-15 22:35:29,0.0,2014-04-15 22:53:14,0.0,2014-04-15 23:05:10,0.0 19 | 2014-04-15 22:35:59,0.0,2014-04-15 22:53:44,0.0,2014-04-15 23:05:40,0.0 20 | 2014-04-15 22:36:29,0.0,2014-04-15 22:54:14,0.0,2014-04-15 23:06:10,0.0 21 | 2014-04-15 22:36:59,0.0,2014-04-15 22:54:44,0.0,2014-04-15 23:06:40,0.0 22 | 2014-04-15 22:37:29,0.0013037109375,2014-04-15 22:55:14,0.0,2014-04-15 23:07:10,0.0 23 | 2014-04-15 22:37:59,0.0,2014-04-15 22:55:44,0.0,2014-04-15 23:07:40,0.0 24 | 2014-04-15 22:38:29,0.0,2014-04-15 22:56:14,0.0,2014-04-15 23:08:10,0.0 25 | 2014-04-15 22:38:59,0.0,,,2014-04-15 23:08:40,0.0 26 | 2014-04-15 22:39:29,0.0,,,2014-04-15 23:09:10,0.0 27 | 2014-04-15 22:39:59,0.0,,,2014-04-15 23:09:40,0.0 28 | 2014-04-15 22:40:29,0.0,,,2014-04-15 23:10:10,0.0 29 | 2014-04-15 22:40:59,0.0,,,2014-04-15 23:10:40,0.0 30 | 2014-04-15 22:41:29,0.0,,,2014-04-15 23:11:10,0.0 31 | 2014-04-15 22:41:59,0.0,,,2014-04-15 23:11:40,0.0 32 | 2014-04-15 22:42:29,0.0,,,2014-04-15 23:12:10,0.0142138671875 33 | 2014-04-15 22:42:59,0.0,,,2014-04-15 23:12:40,0.002998046875 34 | 2014-04-15 22:43:29,0.0 35 | 2014-04-15 22:43:59,0.0 36 | 2014-04-15 22:44:29,0.0 37 | -------------------------------------------------------------------------------- /workflow_stats_parser/sample_multistage_output/2014-05-12_21.49.51_sar.csv: -------------------------------------------------------------------------------- 1 | SAR: cpu load (all cores) 2 | Stage1,,Stage2,,Stage3 3 | time in interval: 100s,average cpu load (all cores) per 30.0s,time in interval: 100s,average cpu load (all cores) per 30.0s,time in interval: 100s,average cpu load (all cores) per 30.0s 4 | 2014-04-15 22:28:29,13.31,2014-04-15 22:46:14,9.54,2014-04-15 22:58:10,11.48 5 | 2014-04-15 22:28:59,15.07,2014-04-15 22:46:44,10.68,2014-04-15 22:58:40,10.25 6 | 2014-04-15 22:29:29,12.87,2014-04-15 22:47:14,10.04,2014-04-15 22:59:10,11.76 7 | 2014-04-15 22:29:59,15.66,2014-04-15 22:47:44,9.32,2014-04-15 22:59:40,9.79 8 | 2014-04-15 22:30:29,10.22,2014-04-15 22:48:14,10.94,2014-04-15 23:00:10,11.01 9 | 2014-04-15 22:30:59,13.8,2014-04-15 22:48:44,13.16,2014-04-15 23:00:40,10.76 10 | 2014-04-15 22:31:29,16.68,2014-04-15 22:49:14,15.57,2014-04-15 23:01:10,11.61 11 | 2014-04-15 22:31:59,14.44,2014-04-15 22:49:44,4.31,2014-04-15 23:01:40,10.89 12 | 2014-04-15 22:32:29,11.4,2014-04-15 22:50:14,3.94,2014-04-15 23:02:10,10.45 13 | 2014-04-15 22:32:59,12.74,2014-04-15 22:50:44,3.92,2014-04-15 23:02:40,11.27 14 | 2014-04-15 22:33:29,11.43,2014-04-15 22:51:14,3.94,2014-04-15 23:03:10,10.97 15 | 2014-04-15 22:33:59,13.79,2014-04-15 22:51:44,3.83,2014-04-15 23:03:40,10.07 16 | 2014-04-15 22:34:29,9.95,2014-04-15 22:52:14,3.94,2014-04-15 23:04:10,10.6 17 | 2014-04-15 22:34:59,13.95,2014-04-15 22:52:44,3.95,2014-04-15 23:04:40,10.74 18 | 2014-04-15 22:35:29,15.21,2014-04-15 22:53:14,3.87,2014-04-15 23:05:10,10.22 19 | 2014-04-15 22:35:59,10.51,2014-04-15 22:53:44,3.91,2014-04-15 23:05:40,10.11 20 | 2014-04-15 22:36:29,15.25,2014-04-15 22:54:14,3.88,2014-04-15 23:06:10,10.16 21 | 2014-04-15 22:36:59,12.85,2014-04-15 22:54:44,3.88,2014-04-15 23:06:40,9.34 22 | 2014-04-15 22:37:29,11.89,2014-04-15 22:55:14,3.9,2014-04-15 23:07:10,10.66 23 | 2014-04-15 22:37:59,4.41,2014-04-15 22:55:44,3.9,2014-04-15 23:07:40,10.03 24 | 2014-04-15 22:38:29,4.14,2014-04-15 22:56:14,3.92,2014-04-15 23:08:10,11.13 25 | 2014-04-15 22:38:59,4.3,,,2014-04-15 23:08:40,9.78 26 | 2014-04-15 22:39:29,4.26,,,2014-04-15 23:09:10,9.96 27 | 2014-04-15 22:39:59,4.41,,,2014-04-15 23:09:40,10.64 28 | 2014-04-15 22:40:29,4.43,,,2014-04-15 23:10:10,10.54 29 | 2014-04-15 22:40:59,4.6,,,2014-04-15 23:10:40,9.66 30 | 2014-04-15 22:41:29,4.56,,,2014-04-15 23:11:10,10.48 31 | 2014-04-15 22:41:59,4.47,,,2014-04-15 23:11:40,9.59 32 | 2014-04-15 22:42:29,4.39,,,2014-04-15 23:12:10,7.82 33 | 2014-04-15 22:42:59,4.23,,,2014-04-15 23:12:40,0.0 34 | 2014-04-15 22:43:29,4.1 35 | 2014-04-15 22:43:59,4.03 36 | 2014-04-15 22:44:29,3.95 37 | -------------------------------------------------------------------------------- /kill_scripts/kill-net-stat-script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################################################# 3 | # The MIT License (MIT) # 4 | # # 5 | # Copyright (c) 2014 Intel Corporation # 6 | # # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 8 | # of this software and associated documentation files (the "Software"), to deal # 9 | # in the Software without restriction, including without limitation the rights # 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 11 | # copies of the Software, and to permit persons to whom the Software is # 12 | # furnished to do so, subject to the following conditions: # 13 | # # 14 | # The above copyright notice and this permission notice shall be included in # 15 | # all copies or substantial portions of the Software. # 16 | # # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 23 | # THE SOFTWARE. # 24 | ################################################################################# 25 | 26 | unset me 27 | unset mpstat 28 | 29 | export me=`whoami` 30 | 31 | # Capture and record the PID of the correct emon 'counter' process: 32 | ps -aef | grep ${me} | grep 'netstat ' | grep -v 'grep' | grep -v 'kill' | awk '{printf "%8d", $2}' > ./dum_PID_netstat 33 | sort ./dum_PID_netstat > ./dum_PID_netstat.sorted 34 | netstat_PID=`tail -1 ./dum_PID_netstat.sorted | awk '{printf "%8d", $1}'` 35 | echo "netstat_PID: "${netstat_PID} 36 | while read line; do 37 | kill -s SIGUSR1 ${line} 38 | done < dum_PID_netstat.sorted 39 | # Now kill that process 40 | rm dum_PID_netstat* 41 | # perl ./sar_parser.pl -f ${SARTSV} -o ${SAROUT}.csv && 42 | 43 | -------------------------------------------------------------------------------- /kill_scripts/kill-free-script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################################################# 3 | # The MIT License (MIT) # 4 | # # 5 | # Copyright (c) 2014 Intel Corporation # 6 | # # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 8 | # of this software and associated documentation files (the "Software"), to deal # 9 | # in the Software without restriction, including without limitation the rights # 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 11 | # copies of the Software, and to permit persons to whom the Software is # 12 | # furnished to do so, subject to the following conditions: # 13 | # # 14 | # The above copyright notice and this permission notice shall be included in # 15 | # all copies or substantial portions of the Software. # 16 | # # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 23 | # THE SOFTWARE. # 24 | ################################################################################# 25 | 26 | unset me 27 | unset free 28 | 29 | export me=`whoami` 30 | 31 | # Capture and record the PID of the correct emon 'counter' process: 32 | ps -aef | grep ${me} | grep 'free -' | grep -v 'grep' | grep -v 'kill' | awk '{printf "%8d", $2}' > ./dum_PID_free 33 | sort ./dum_PID_free > ./dum_PID_free.sorted 34 | free_PID=`tail -1 ./dum_PID_free.sorted | awk '{printf "%8d", $1}'` 35 | echo "free_PID: "${free_PID} 36 | while read line; do 37 | kill -s SIGUSR1 ${line} 38 | done < dum_PID_free.sorted 39 | 40 | # Now kill that process 41 | #if [[ -z "${free_PID}" ]]; then 42 | # kill -s SIGUSR1 ${free_PID} 43 | #fi 44 | rm dum_PID_free* 45 | # perl ./sar_parser.pl -f ${SARTSV} -o ${SAROUT}.csv && 46 | -------------------------------------------------------------------------------- /kill_scripts/kill-mp-stat-script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################################################# 3 | # The MIT License (MIT) # 4 | # # 5 | # Copyright (c) 2014 Intel Corporation # 6 | # # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 8 | # of this software and associated documentation files (the "Software"), to deal # 9 | # in the Software without restriction, including without limitation the rights # 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 11 | # copies of the Software, and to permit persons to whom the Software is # 12 | # furnished to do so, subject to the following conditions: # 13 | # # 14 | # The above copyright notice and this permission notice shall be included in # 15 | # all copies or substantial portions of the Software. # 16 | # # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 23 | # THE SOFTWARE. # 24 | ################################################################################# 25 | 26 | unset me 27 | unset mpstat 28 | 29 | export me=`whoami` 30 | 31 | # Capture and record the PID of the correct emon 'counter' process: 32 | ps -aef | grep ${me} | awk '{print $2,$8}' | grep 'mpstat' | grep -v 'grep' | grep -v 'kill' | awk '{printf "%8d", $1}' > ./dum_PID_mpstat 33 | sort ./dum_PID_mpstat > ./dum_PID_mpstat.sorted 34 | mpstat_PID_mpstat=`tail -1 ./dum_PID_mpstat.sorted | awk '{printf "%8d", $1}'` 35 | echo "mpstat_PID_mpstat: "${mpstat_PID_mpstat} 36 | # Now kill that process 37 | while read line; do 38 | #echo $line 39 | kill -s SIGUSR1 ${line} 40 | done < dum_PID_mpstat.sorted 41 | #kill -s SIGUSR1 ${mpstat_PID_mpstat} 42 | rm dum_PID_mpstat* 43 | # perl ./sar_parser.pl -f ${SARTSV} -o ${SAROUT}.csv && 44 | 45 | -------------------------------------------------------------------------------- /kill_scripts/kill-sar-script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################################################# 3 | # The MIT License (MIT) # 4 | # # 5 | # Copyright (c) 2014 Intel Corporation # 6 | # # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 8 | # of this software and associated documentation files (the "Software"), to deal # 9 | # in the Software without restriction, including without limitation the rights # 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 11 | # copies of the Software, and to permit persons to whom the Software is # 12 | # furnished to do so, subject to the following conditions: # 13 | # # 14 | # The above copyright notice and this permission notice shall be included in # 15 | # all copies or substantial portions of the Software. # 16 | # # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 23 | # THE SOFTWARE. # 24 | ################################################################################# 25 | 26 | unset me 27 | unset sar_PID 28 | 29 | export me=`whoami` 30 | #echo $SARDATA 31 | # Capture and record the PID of the correct sar 'counter' process: 32 | #ps -aef | grep ${me} | grep 'sar ' | grep -v 'grep' | grep -v 'kill' | awk '{printf "%8d", $2}' > ./dum_PID_sar 33 | ps -aef | grep ${me} | awk '{print $2,$8}' | grep 'sar' | grep -v 'grep' | grep -v 'kill' | awk '{printf "%8d", $1}' > ./dum_PID_sar 34 | 35 | sort ./dum_PID_sar > ./dum_PID_sar.sorted 36 | sar_PID=`tail -1 ./dum_PID_sar.sorted | awk '{printf "%8d", $1}'` 37 | echo "sar_PID: "${sar_PID} 38 | while read line; do 39 | #echo $line 40 | kill -s SIGUSR1 ${line} 41 | done < dum_PID_sar.sorted 42 | 43 | # Now kill that process 44 | #kill -s SIGUSR1 ${sar_PID} 45 | rm dum_PID_sar* 46 | #sadf -p ${SARDATA} -- -A > ${SARTSV} 47 | #perl /opt/sar_parser.pl -f ${SARTSV} -o ${SAROUT}.csv 48 | -------------------------------------------------------------------------------- /kill_scripts/kill-iostat-script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################################################# 3 | # The MIT License (MIT) # 4 | # # 5 | # Copyright (c) 2014 Intel Corporation # 6 | # # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 8 | # of this software and associated documentation files (the "Software"), to deal # 9 | # in the Software without restriction, including without limitation the rights # 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 11 | # copies of the Software, and to permit persons to whom the Software is # 12 | # furnished to do so, subject to the following conditions: # 13 | # # 14 | # The above copyright notice and this permission notice shall be included in # 15 | # all copies or substantial portions of the Software. # 16 | # # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 23 | # THE SOFTWARE. # 24 | ################################################################################# 25 | 26 | unset me 27 | unset iostat_PID 28 | 29 | export me=`whoami` 30 | 31 | # Capture and record the PID of the correct emon 'counter' process: 32 | #ps -aef | grep ${me} | grep 'iostat ' | grep -v 'grep' | grep -v 'kill' | awk '{printf "%8d", $2}' > ./dum_PID_iostat 33 | ps -aef | grep ${me} | awk '{print $2,$8}' | grep 'iostat' | grep -v 'grep' | grep -v 'kill' | awk '{printf "%8d", $1}' > ./dum_PID_iostat 34 | sort ./dum_PID_iostat > ./dum_PID_iostat.sorted 35 | iostat_PID=`tail -1 ./dum_PID_iostat.sorted | awk '{printf "%8d", $1}'` 36 | echo "iostat_PID: "${iostat_PID} 37 | while read line; do 38 | #echo $line 39 | kill -s SIGUSR1 ${line} 40 | done < dum_PID_iostat.sorted 41 | 42 | # Now kill that process 43 | #if [[ -z "${iostat_PID}" ]]; then 44 | # kill -s SIGUSR1 ${iostat_PID} 45 | #fi 46 | rm dum_PID_iostat* 47 | # perl ./sar_parser.pl -f ${SARTSV} -o ${SAROUT}.csv && 48 | 49 | -------------------------------------------------------------------------------- /workflow_stats_parser/workflow_dictionaries.py: -------------------------------------------------------------------------------- 1 | ################################################################################# 2 | # The MIT License (MIT) # 3 | # # 4 | # Copyright (c) 2014 Intel Corporation # 5 | # # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 7 | # of this software and associated documentation files (the "Software"), to deal # 8 | # in the Software without restriction, including without limitation the rights # 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 10 | # copies of the Software, and to permit persons to whom the Software is # 11 | # furnished to do so, subject to the following conditions: # 12 | # # 13 | # The above copyright notice and this permission notice shall be included in # 14 | # all copies or substantial portions of the Software. # 15 | # # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 22 | # THE SOFTWARE. # 23 | ################################################################################# 24 | 25 | from collections import OrderedDict 26 | 27 | """ 28 | The user needs to add their own dictionary here. 29 | 30 | The step names for the dictionary should match the stage names in the 31 | workflow script. 32 | 33 | For example: if there exists run..stage1.* , run..stage2.* and run..stage3.* 34 | folders, the dictionary would be constructed as in the example provided 35 | below for sample data. 36 | """ 37 | 38 | # This dictionary is used to test the existing sample data 39 | sample_dict = OrderedDict([('Stage1', 'stage1'), 40 | ('Stage2','stage2'), 41 | ('Stage3','stage3')]) 42 | 43 | """ 44 | unordered dictionary for parsing workflow input parameter to one of the above 45 | workflow dictionaries 46 | 47 | Format is: 48 | 'workflowName': 'name_of_dictionary' 49 | 50 | The workflowName will be what is entered on the command line with the -N 51 | option,the name_of_dictionary is the name of the OrderedDict you have defined 52 | above 53 | """ 54 | 55 | # If you add a new dictionary above, be sure to update this dictionary based on 56 | # the example format provided. 57 | workflow_parse_dict = { 58 | 'sample': 'sample_dict', 59 | } 60 | -------------------------------------------------------------------------------- /workflow_stats_parser/sample_multistage_output/2014-05-12_21.49.51_sar_writes.csv: -------------------------------------------------------------------------------- 1 | SAR_WRITES: io writes in mb/sec 2 | Stage1,,Stage2,,Stage3 3 | time in interval: 100s,average io writes in mb/sec per 30.0s,time in interval: 100s,average io writes in mb/sec per 30.0s,time in interval: 100s,average io writes in mb/sec per 30.0s 4 | 2014-04-15 22:28:29,6.040546875,2014-04-15 22:46:14,0.5696484375,2014-04-15 22:58:10,2.3776708984375 5 | 2014-04-15 22:28:59,7.9431005859375,2014-04-15 22:46:44,0.772490234375,2014-04-15 22:58:40,2.4538427734375 6 | 2014-04-15 22:29:29,8.686162109375,2014-04-15 22:47:14,0.7234521484375,2014-04-15 22:59:10,4.302119140625 7 | 2014-04-15 22:29:59,7.6938916015625,2014-04-15 22:47:44,9.2573974609375,2014-04-15 22:59:40,3.9580615234375 8 | 2014-04-15 22:30:29,8.5798046875,2014-04-15 22:48:14,0.5477587890625,2014-04-15 23:00:10,3.9109716796875 9 | 2014-04-15 22:30:59,7.900537109375,2014-04-15 22:48:44,0.4334912109375,2014-04-15 23:00:40,3.806826171875 10 | 2014-04-15 22:31:29,7.7779833984375,2014-04-15 22:49:14,7.4459716796875,2014-04-15 23:01:10,3.94271484375 11 | 2014-04-15 22:31:59,7.80728515625,2014-04-15 22:49:44,6.2608203125,2014-04-15 23:01:40,3.7524365234375 12 | 2014-04-15 22:32:29,7.8005126953125,2014-04-15 22:50:14,7.45470703125,2014-04-15 23:02:10,3.586982421875 13 | 2014-04-15 22:32:59,8.0955615234375,2014-04-15 22:50:44,7.522138671875,2014-04-15 23:02:40,3.928828125 14 | 2014-04-15 22:33:29,9.1703662109375,2014-04-15 22:51:14,7.5289404296875,2014-04-15 23:03:10,3.81765625 15 | 2014-04-15 22:33:59,8.4146630859375,2014-04-15 22:51:44,7.5195947265625,2014-04-15 23:03:40,3.9178955078125 16 | 2014-04-15 22:34:29,7.8230615234375,2014-04-15 22:52:14,7.3706689453125,2014-04-15 23:04:10,3.764658203125 17 | 2014-04-15 22:34:59,7.7148779296875,2014-04-15 22:52:44,7.54787109375,2014-04-15 23:04:40,3.8847900390625 18 | 2014-04-15 22:35:29,7.7256640625,2014-04-15 22:53:14,7.4969775390625,2014-04-15 23:05:10,3.783017578125 19 | 2014-04-15 22:35:59,8.2668603515625,2014-04-15 22:53:44,7.618623046875,2014-04-15 23:05:40,4.062236328125 20 | 2014-04-15 22:36:29,7.8008935546875,2014-04-15 22:54:14,7.5059130859375,2014-04-15 23:06:10,3.7312255859375 21 | 2014-04-15 22:36:59,8.61263671875,2014-04-15 22:54:44,7.579169921875,2014-04-15 23:06:40,3.9348681640625 22 | 2014-04-15 22:37:29,9.041591796875,2014-04-15 22:55:14,7.4845703125,2014-04-15 23:07:10,4.0273681640625 23 | 2014-04-15 22:37:59,7.5833935546875,2014-04-15 22:55:44,7.547041015625,2014-04-15 23:07:40,3.8166796875 24 | 2014-04-15 22:38:29,7.4734912109375,2014-04-15 22:56:14,7.6662255859375,2014-04-15 23:08:10,3.8003955078125 25 | 2014-04-15 22:38:59,7.6162744140625,,,2014-04-15 23:08:40,4.03650390625 26 | 2014-04-15 22:39:29,7.5460009765625,,,2014-04-15 23:09:10,3.9588330078125 27 | 2014-04-15 22:39:59,7.28931640625,,,2014-04-15 23:09:40,4.133544921875 28 | 2014-04-15 22:40:29,7.5596435546875,,,2014-04-15 23:10:10,3.949580078125 29 | 2014-04-15 22:40:59,7.61119140625,,,2014-04-15 23:10:40,3.9732958984375 30 | 2014-04-15 22:41:29,7.4699853515625,,,2014-04-15 23:11:10,4.19525390625 31 | 2014-04-15 22:41:59,7.6385693359375,,,2014-04-15 23:11:40,4.4763427734375 32 | 2014-04-15 22:42:29,7.531787109375,,,2014-04-15 23:12:10,6.3166259765625 33 | 2014-04-15 22:42:59,7.487216796875,,,2014-04-15 23:12:40,0.36271484375 34 | 2014-04-15 22:43:29,7.633876953125 35 | 2014-04-15 22:43:59,7.5304833984375 36 | 2014-04-15 22:44:29,7.55291015625 37 | -------------------------------------------------------------------------------- /workflow_stats_parser/plot_templates/template_iostat.plt: -------------------------------------------------------------------------------- 1 | ################################################################################# 2 | # The MIT License (MIT) # 3 | # # 4 | # Copyright (c) 2014 Intel Corporation # 5 | # # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 7 | # of this software and associated documentation files (the "Software"), to deal # 8 | # in the Software without restriction, including without limitation the rights # 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 10 | # copies of the Software, and to permit persons to whom the Software is # 11 | # furnished to do so, subject to the following conditions: # 12 | # # 13 | # The above copyright notice and this permission notice shall be included in # 14 | # all copies or substantial portions of the Software. # 15 | # # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 22 | # THE SOFTWARE. # 23 | ################################################################################# 24 | 25 | clear 26 | reset 27 | 28 | print "cpu time spent waiting on read and write" 29 | set terminal pngcairo transparent enhanced font "arial,25" fontscale 1.0 size 1920, 1080 30 | set key outside bottom center box title "Workflow phase(s)" enhanced 31 | set key maxrows 4 32 | set key font ",25" spacing 1 samplen 2.9 width 2 height 1 33 | set xlabel "Time (hours)" font ",25" 34 | set ylabel "milliseconds" font ",25" 35 | 36 | set output "/post_processed_stats/output_io_waittime_per_phase_plot.png" 37 | set title "IO Wait per Phase (ms)\n{/*0.5 }" font ",35" 38 | set datafile separator "," 39 | set timefmt "%Y-%m-%d %H:%M:%S" 40 | set ytics font ",25" 41 | 42 | set style line 1 lt 1 lc rgb "red" lw 4 43 | set style line 2 lt 1 lc rgb "orange" lw 4 44 | set style line 3 lt 1 lc rgb "brown" lw 4 45 | set style line 4 lt 1 lc rgb "green" lw 4 46 | set style line 5 lt 1 lc rgb "cyan" lw 4 47 | set style line 6 lt 1 lc rgb "blue" lw 4 48 | set style line 7 lt 1 lc rgb "violet" lw 4 49 | set style line 8 lt 1 lc rgb "brown" lw 4 50 | set style line 9 lt 1 lc rgb "green" lw 4 51 | set style line 10 lt 1 lc rgb "cyan" lw 4 52 | set style line 11 lt 1 lc rgb "red" lw 4 53 | set style line 12 lt 1 lc rgb "violet" lw 4 54 | show style line 55 | starting_time = 37794 56 | 57 | offset = 0 58 | t0(x)=(offset=($0==0) ? x : offset, x - offset) 59 | 60 | plot "/post_processed_stats/2014-03-03_13.29.19_iostat.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 1 t "bwa aln 1" with lines, \ 61 | '' using ((timecolumn(3)-offset)/3600):4 every ::3 ls 2 t "bwa aln 2" with lines, \ 62 | '' using ((timecolumn(5)-offset)/3600):6 every ::3 ls 3 t "sampe" with lines 63 | -------------------------------------------------------------------------------- /workflow_stats_parser/sample_multistage_output/2014-05-12_21.49.51_active_mem.csv: -------------------------------------------------------------------------------- 1 | ACTIVE_MEM: committed memory (gb) 2 | Stage1,,Stage2,,Stage3 3 | time in interval: 100s,average committed memory (gb) per 30.0s,time in interval: 100s,average committed memory (gb) per 30.0s,time in interval: 100s,average committed memory (gb) per 30.0s 4 | 2014-04-15 22:28:29,3.5888900756835938,2014-04-15 22:46:14,3.168811798095703,2014-04-15 22:58:10,4.664253234863281 5 | 2014-04-15 22:28:59,3.6545562744140625,2014-04-15 22:46:44,3.163318634033203,2014-04-15 22:58:40,4.633354187011719 6 | 2014-04-15 22:29:29,3.6648597717285156,2014-04-15 22:47:14,3.1729049682617188,2014-04-15 22:59:10,4.638999938964844 7 | 2014-04-15 22:29:59,4.042400360107422,2014-04-15 22:47:44,3.828441619873047,2014-04-15 22:59:40,4.823448181152344 8 | 2014-04-15 22:30:29,4.042549133300781,2014-04-15 22:48:14,3.8303909301757812,2014-04-15 23:00:10,5.20880126953125 9 | 2014-04-15 22:30:59,3.997783660888672,2014-04-15 22:48:44,3.871917724609375,2014-04-15 23:00:40,5.375083923339844 10 | 2014-04-15 22:31:29,4.195831298828125,2014-04-15 22:49:14,4.060962677001953,2014-04-15 23:01:10,5.457305908203125 11 | 2014-04-15 22:31:59,4.147026062011719,2014-04-15 22:49:44,4.060787200927734,2014-04-15 23:01:40,5.499294281005859 12 | 2014-04-15 22:32:29,4.143947601318359,2014-04-15 22:50:14,4.060955047607422,2014-04-15 23:02:10,5.489288330078125 13 | 2014-04-15 22:32:59,4.1931610107421875,2014-04-15 22:50:44,4.054172515869141,2014-04-15 23:02:40,5.494609832763672 14 | 2014-04-15 22:33:29,4.187732696533203,2014-04-15 22:51:14,4.052825927734375,2014-04-15 23:03:10,5.4899139404296875 15 | 2014-04-15 22:33:59,4.241264343261719,2014-04-15 22:51:44,4.053741455078125,2014-04-15 23:03:40,5.496978759765625 16 | 2014-04-15 22:34:29,4.2400970458984375,2014-04-15 22:52:14,4.048801422119141,2014-04-15 23:04:10,5.482975006103516 17 | 2014-04-15 22:34:59,4.220516204833984,2014-04-15 22:52:44,4.049461364746094,2014-04-15 23:04:40,5.472286224365234 18 | 2014-04-15 22:35:29,4.366199493408203,2014-04-15 22:53:14,4.043872833251953,2014-04-15 23:05:10,5.497001647949219 19 | 2014-04-15 22:35:59,4.365776062011719,2014-04-15 22:53:44,4.042400360107422,2014-04-15 23:05:40,5.482921600341797 20 | 2014-04-15 22:36:29,4.5504608154296875,2014-04-15 22:54:14,4.04364013671875,2014-04-15 23:06:10,5.474765777587891 21 | 2014-04-15 22:36:59,4.545635223388672,2014-04-15 22:54:44,4.040348052978516,2014-04-15 23:06:40,5.496997833251953 22 | 2014-04-15 22:37:29,4.4858856201171875,2014-04-15 22:55:14,4.04437255859375,2014-04-15 23:07:10,5.496860504150391 23 | 2014-04-15 22:37:59,4.485889434814453,2014-04-15 22:55:44,4.044445037841797,2014-04-15 23:07:40,5.49688720703125 24 | 2014-04-15 22:38:29,4.4720001220703125,2014-04-15 22:56:14,4.0461578369140625,2014-04-15 23:08:10,5.496990203857422 25 | 2014-04-15 22:38:59,4.48583984375,,,2014-04-15 23:08:40,5.4969635009765625 26 | 2014-04-15 22:39:29,4.573696136474609,,,2014-04-15 23:09:10,5.486644744873047 27 | 2014-04-15 22:39:59,4.547172546386719,,,2014-04-15 23:09:40,5.483894348144531 28 | 2014-04-15 22:40:29,4.573863983154297,,,2014-04-15 23:10:10,5.497032165527344 29 | 2014-04-15 22:40:59,4.573722839355469,,,2014-04-15 23:10:40,5.489109039306641 30 | 2014-04-15 22:41:29,4.573764801025391,,,2014-04-15 23:11:10,5.480415344238281 31 | 2014-04-15 22:41:59,4.573772430419922,,,2014-04-15 23:11:40,5.497200012207031 32 | 2014-04-15 22:42:29,4.56280517578125,,,2014-04-15 23:12:10,5.498371124267578 33 | 2014-04-15 22:42:59,4.564159393310547,,,2014-04-15 23:12:40,5.498371124267578 34 | 2014-04-15 22:43:29,4.5294342041015625 35 | 2014-04-15 22:43:59,4.560081481933594 36 | 2014-04-15 22:44:29,4.573619842529297 37 | -------------------------------------------------------------------------------- /workflow_stats_parser/plot_templates/template_sar_reads.plt: -------------------------------------------------------------------------------- 1 | ################################################################################# 2 | # The MIT License (MIT) # 3 | # # 4 | # Copyright (c) 2014 Intel Corporation # 5 | # # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 7 | # of this software and associated documentation files (the "Software"), to deal # 8 | # in the Software without restriction, including without limitation the rights # 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 10 | # copies of the Software, and to permit persons to whom the Software is # 11 | # furnished to do so, subject to the following conditions: # 12 | # # 13 | # The above copyright notice and this permission notice shall be included in # 14 | # all copies or substantial portions of the Software. # 15 | # # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 22 | # THE SOFTWARE. # 23 | ################################################################################# 24 | 25 | clear 26 | reset 27 | print "io reads in MB/sec" 28 | set terminal pngcairo transparent enhanced font "arial,25" fontscale 1.0 size 1920, 1080 29 | set key outside bottom center box title "Workflow phase" enhanced 30 | set key maxrows 4 31 | set key font ",25" spacing 1 samplen 2.9 width 2 height 1 32 | set xlabel "Time (hours)" font ",25" 33 | set ylabel "MBs/sec" font ",25" 34 | 35 | set output "/post_processed_stats/output_io_reads_per_sec.png" 36 | set title "Average IO reads (MBs/s)\n{/*0.5 }" font ",35" 37 | set datafile separator "," 38 | #set xdata time 39 | set timefmt "%Y-%m-%d %H:%M:%S" 40 | #set xtics format "%d:%H:%M" font ",25" 41 | set ytics font ",25" 42 | 43 | set style line 1 lt 1 lc rgb "red" lw 4 44 | set style line 2 lt 1 lc rgb "orange" lw 4 45 | set style line 3 lt 1 lc rgb "brown" lw 4 46 | set style line 4 lt 1 lc rgb "green" lw 4 47 | set style line 5 lt 1 lc rgb "cyan" lw 4 48 | set style line 6 lt 1 lc rgb "blue" lw 4 49 | set style line 7 lt 1 lc rgb "violet" lw 4 50 | set style line 8 lt 1 lc rgb "yellow" lw 4 51 | set style line 9 lt 1 lc rgb "green" lw 4 52 | set style line 10 lt 1 lc rgb "cyan" lw 4 53 | set style line 11 lt 1 lc rgb "blue" lw 4 54 | set style line 12 lt 1 lc rgb "violet" lw 4 55 | show style line 56 | 57 | offset = 0 58 | starting_time = 37824 59 | t0(x)=(offset=($0==0) ? x : offset, x - offset) 60 | 61 | plot "/post_processed_stats/2014-03-03_13.29.19_sar_reads.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 1 t "bwa aln 1" with lines, \ 62 | '' using ((timecolumn(3)-offset)/3600):4 every ::3 ls 2 t "bwa aln 2" with lines, \ 63 | '' using ((timecolumn(5)-offset)/3600):6 every ::3 ls 3 t "sampe" with lines 64 | -------------------------------------------------------------------------------- /workflow_stats_parser/plot_templates/template_sar_writes.plt: -------------------------------------------------------------------------------- 1 | ################################################################################# 2 | # The MIT License (MIT) # 3 | # # 4 | # Copyright (c) 2014 Intel Corporation # 5 | # # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 7 | # of this software and associated documentation files (the "Software"), to deal # 8 | # in the Software without restriction, including without limitation the rights # 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 10 | # copies of the Software, and to permit persons to whom the Software is # 11 | # furnished to do so, subject to the following conditions: # 12 | # # 13 | # The above copyright notice and this permission notice shall be included in # 14 | # all copies or substantial portions of the Software. # 15 | # # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 22 | # THE SOFTWARE. # 23 | ################################################################################# 24 | 25 | clear 26 | reset 27 | print "io writes in MB/sec" 28 | set terminal pngcairo transparent enhanced font "arial,25" fontscale 1.0 size 1920, 1080 29 | set key outside bottom center box title "Workflow phase" enhanced 30 | set key maxrows 4 31 | set key font ",25" spacing 1 samplen 2.9 width 2 height 1 32 | set xlabel "Time (hours)" font ",25" 33 | set ylabel "MBs/sec" font ",25" 34 | 35 | set output "/post_processed_stats/output_io_writes_per_sec.png" 36 | set title "Average IO writes (MBs/s)\n{/*0.5 }" font ",35" 37 | set datafile separator "," 38 | #set xdata time 39 | set timefmt "%Y-%m-%d %H:%M:%S" 40 | #set xtics format "%d:%H:%M" font ",25" 41 | set ytics font ",25" 42 | 43 | set style line 1 lt 1 lc rgb "red" lw 4 44 | set style line 2 lt 1 lc rgb "orange" lw 4 45 | set style line 3 lt 1 lc rgb "brown" lw 4 46 | set style line 4 lt 1 lc rgb "green" lw 4 47 | set style line 5 lt 1 lc rgb "cyan" lw 4 48 | set style line 6 lt 1 lc rgb "blue" lw 4 49 | set style line 7 lt 1 lc rgb "violet" lw 4 50 | set style line 8 lt 1 lc rgb "yellow" lw 4 51 | set style line 9 lt 1 lc rgb "green" lw 4 52 | set style line 10 lt 1 lc rgb "cyan" lw 4 53 | set style line 11 lt 1 lc rgb "blue" lw 4 54 | set style line 12 lt 1 lc rgb "violet" lw 4 55 | show style line 56 | 57 | offset = 0 58 | starting_time = 37824 59 | t0(x)=(offset=($0==0) ? x : offset, x - offset) 60 | 61 | plot "/post_processed_stats/2014-03-03_13.29.19_sar_writes.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 1 t "bwa aln 1" with lines, \ 62 | '' using ((timecolumn(3)-offset)/3600):4 every ::3 ls 2 t "bwa aln 2" with lines, \ 63 | '' using ((timecolumn(5)-offset)/3600):6 every ::3 ls 3 t "sampe" with lines 64 | -------------------------------------------------------------------------------- /workflow_stats_parser/plot_templates/template_sar.plt: -------------------------------------------------------------------------------- 1 | ################################################################################# 2 | # The MIT License (MIT) # 3 | # # 4 | # Copyright (c) 2014 Intel Corporation # 5 | # # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 7 | # of this software and associated documentation files (the "Software"), to deal # 8 | # in the Software without restriction, including without limitation the rights # 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 10 | # copies of the Software, and to permit persons to whom the Software is # 11 | # furnished to do so, subject to the following conditions: # 12 | # # 13 | # The above copyright notice and this permission notice shall be included in # 14 | # all copies or substantial portions of the Software. # 15 | # # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 22 | # THE SOFTWARE. # 23 | ################################################################################# 24 | 25 | clear 26 | reset 27 | print "averagie cpu utilization" 28 | set terminal pngcairo transparent enhanced font "arial,25" fontscale 1.0 size 1920, 1080 29 | set key outside bottom center box title "Workflow Phase(s)" enhanced 30 | set key maxrows 4 31 | set key font ",25" spacing 1 samplen 2.9 width 2 height 1 32 | set xlabel "Time (hours)" font ",25" 33 | set ylabel "Utilization (%)" font ",25" 34 | 35 | set output "/post_processed_stats/output_average_cpu_utilization_plot.png" 36 | set title "Average CPU Utilization (%) per Phase\n{/*0.5 }" font ",35" 37 | set datafile separator "," 38 | #set xdata time 39 | set timefmt "%Y-%m-%d %H:%M:%S" 40 | #set xtics format "%d:%H:%M" font ",25" 41 | set ytics font ",25" 42 | 43 | set style line 1 lt 1 lc rgb "red" lw 4 44 | set style line 2 lt 1 lc rgb "orange" lw 4 45 | set style line 3 lt 1 lc rgb "brown" lw 4 46 | set style line 4 lt 1 lc rgb "green" lw 4 47 | set style line 5 lt 1 lc rgb "cyan" lw 4 48 | set style line 6 lt 1 lc rgb "blue" lw 4 49 | set style line 7 lt 1 lc rgb "violet" lw 4 50 | set style line 8 lt 1 lc rgb "yellow" lw 4 51 | set style line 9 lt 1 lc rgb "green" lw 4 52 | set style line 10 lt 1 lc rgb "cyan" lw 4 53 | set style line 11 lt 1 lc rgb "blue" lw 4 54 | set style line 12 lt 1 lc rgb "violet" lw 4 55 | show style line 56 | 57 | offset = 0 58 | starting_time = 37824 59 | t0(x)=(offset=($0==0) ? x : offset, x - offset) 60 | 61 | plot "/post_processed_stats/2014-03-03_13.29.19_sar.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 1 t "bwa aln 1" with lines, \ 62 | '' using ((timecolumn(3)-offset)/3600):4 every ::3 ls 2 t "bwa aln 2" with lines, \ 63 | '' using ((timecolumn(5)-offset)/3600):6 every ::3 ls 3 t "sampe" with lines 64 | -------------------------------------------------------------------------------- /workflow_stats_parser/plot_templates/template_total_core_mpstat.plt: -------------------------------------------------------------------------------- 1 | ################################################################################# 2 | # The MIT License (MIT) # 3 | # # 4 | # Copyright (c) 2014 Intel Corporation # 5 | # # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 7 | # of this software and associated documentation files (the "Software"), to deal # 8 | # in the Software without restriction, including without limitation the rights # 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 10 | # copies of the Software, and to permit persons to whom the Software is # 11 | # furnished to do so, subject to the following conditions: # 12 | # # 13 | # The above copyright notice and this permission notice shall be included in # 14 | # all copies or substantial portions of the Software. # 15 | # # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 22 | # THE SOFTWARE. # 23 | ################################################################################# 24 | 25 | clear 26 | reset 27 | print "total core" 28 | set terminal pngcairo transparent enhanced font "arial,25" fontscale 1.0 size 1920, 1080 29 | set key outside bottom center box title "Workflow Phase(s)" enhanced 30 | set key maxrows 4 31 | set key font ",25" spacing 1 samplen 2.9 width 2 height 1 32 | set xlabel "Time (hours)" offset font ",25" 33 | set ylabel "Utilization (%)" font ",25" 34 | 35 | set output "/post_processed_stats/output_total_core_plot.png" 36 | set title "Total Core Utilization per Phase (%)\n{/*0.5 }" font ",35" 37 | set datafile separator "," 38 | #set xdata time 39 | set timefmt "%Y-%m-%d %H:%M:%S" 40 | #set xtics format "%d:%H:%M" font ",25" 41 | set ytics font ",25" 42 | 43 | set style line 1 lt 1 lc rgb "red" lw 4 44 | set style line 2 lt 1 lc rgb "orange" lw 4 45 | set style line 3 lt 1 lc rgb "brown" lw 4 46 | set style line 4 lt 1 lc rgb "green" lw 4 47 | set style line 5 lt 1 lc rgb "cyan" lw 4 48 | set style line 6 lt 1 lc rgb "blue" lw 4 49 | set style line 7 lt 1 lc rgb "violet" lw 4 50 | set style line 8 lt 1 lc rgb "yellow" lw 4 51 | set style line 9 lt 1 lc rgb "green" lw 4 52 | set style line 10 lt 1 lc rgb "cyan" lw 4 53 | set style line 11 lt 1 lc rgb "blue" lw 4 54 | set style line 12 lt 1 lc rgb "violet" lw 4 55 | show style line 56 | 57 | offset = 0 58 | starting_time = 41792 59 | t0(x)=(offset=($0==0) ? x : offset, x - offset) 60 | 61 | plot "/post_processed_stats/2014-03-03_13.29.19_mpstat_total_core.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 1 t "bwa aln 1" with lines, \ 62 | '' using ((timecolumn(3)-offset)/3600):4 every ::3 ls 2 t "bwa aln 2" with lines, \ 63 | '' using ((timecolumn(5)-offset)/3600):6 every ::3 ls 3 t "sampe" with lines 64 | -------------------------------------------------------------------------------- /workflow_stats_parser/plot_templates/template_active_core_mpstat.plt: -------------------------------------------------------------------------------- 1 | ################################################################################# 2 | # The MIT License (MIT) # 3 | # # 4 | # Copyright (c) 2014 Intel Corporation # 5 | # # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 7 | # of this software and associated documentation files (the "Software"), to deal # 8 | # in the Software without restriction, including without limitation the rights # 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 10 | # copies of the Software, and to permit persons to whom the Software is # 11 | # furnished to do so, subject to the following conditions: # 12 | # # 13 | # The above copyright notice and this permission notice shall be included in # 14 | # all copies or substantial portions of the Software. # 15 | # # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 22 | # THE SOFTWARE. # 23 | ################################################################################# 24 | 25 | clear 26 | reset 27 | 28 | print "most active core" 29 | set terminal pngcairo transparent enhanced font "arial,25" fontscale 1.0 size 1920, 1080 30 | set key outside bottom center box title "Workflow phase(s)" enhanced 31 | set key maxrows 4 32 | set key font ",25" spacing 1 samplen 2.9 width 2 height 1 33 | set xlabel "Time (hours)" font ",25" 34 | set ylabel "Utilization (%)" font ",25" 35 | 36 | set output "/post_processed_stats/output_active_core_plot.png" 37 | set title "Core with Highest Utilization (%) per Phase (%)\n{/*0.5 }" font ",35" 38 | set datafile separator "," 39 | #set xdata time 40 | set timefmt "%Y-%m-%d %H:%M:%S" 41 | #set xtics format "%d:%H:%M" font ",25" 42 | set ytics font ",25" 43 | 44 | set style line 1 lt 1 lc rgb "red" lw 4 45 | set style line 2 lt 1 lc rgb "orange" lw 4 46 | set style line 3 lt 1 lc rgb "brown" lw 4 47 | set style line 4 lt 1 lc rgb "green" lw 4 48 | set style line 5 lt 1 lc rgb "cyan" lw 4 49 | set style line 6 lt 1 lc rgb "blue" lw 4 50 | set style line 7 lt 1 lc rgb "violet" lw 4 51 | set style line 8 lt 1 lc rgb "yellow" lw 4 52 | set style line 9 lt 1 lc rgb "green" lw 4 53 | set style line 10 lt 1 lc rgb "cyan" lw 4 54 | set style line 11 lt 1 lc rgb "blue" lw 4 55 | set style line 12 lt 1 lc rgb "violet" lw 4 56 | show style line 57 | starting_time = 37824 58 | 59 | offset = 0 60 | t0(x)=(offset=($0==0) ? x : offset, x - offset) 61 | 62 | plot "/post_processed_stats/2014-03-03_13.29.19_mpstat_active_core.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 1 t "bwa aln 1" with lines, \ 63 | '' using ((timecolumn(3)-offset)/3600):4 every ::3 ls 2 t "bwa aln 2" with lines, \ 64 | '' using ((timecolumn(5)-offset)/3600):6 every ::3 ls 3 t "sampe" with lines 65 | -------------------------------------------------------------------------------- /workflow_stats_parser/plot_templates/template_committed_mem.plt: -------------------------------------------------------------------------------- 1 | ################################################################################# 2 | # The MIT License (MIT) # 3 | # # 4 | # Copyright (c) 2014 Intel Corporation # 5 | # # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 7 | # of this software and associated documentation files (the "Software"), to deal # 8 | # in the Software without restriction, including without limitation the rights # 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 10 | # copies of the Software, and to permit persons to whom the Software is # 11 | # furnished to do so, subject to the following conditions: # 12 | # # 13 | # The above copyright notice and this permission notice shall be included in # 14 | # all copies or substantial portions of the Software. # 15 | # # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 22 | # THE SOFTWARE. # 23 | ################################################################################# 24 | 25 | clear 26 | reset 27 | print "committed memory" 28 | set terminal pngcairo transparent enhanced font "arial,25" fontscale 1.0 size 1920, 1080 29 | set key outside bottom center box title "Workflow phase(s)" enhanced 30 | set key maxrows 4 31 | set key font ",25" spacing 1 samplen 2.9 width 2 height 1 32 | set xlabel "Time (hours)" font ",25" 33 | set ylabel "GBs" font ",25" 34 | 35 | set output "/post_processed_stats/output_committed_mem_plot.png" 36 | #set title "Total committed memory in gigabytes per phase sampled at 100 second intervals\n{/*0.7 }" font ",35" 37 | set title "Total Committed Memory (GBs) per Phase\n{/*0.5 }" font ",35" 38 | set datafile separator "," 39 | #set xdata time 40 | set timefmt "%Y-%m-%d %H:%M:%S" 41 | #set xtics format "%d:%H:%M" font ",25" 42 | set ytics font ",25" 43 | 44 | set style line 1 lt 1 lc rgb "red" lw 4 45 | set style line 2 lt 1 lc rgb "orange" lw 4 46 | set style line 3 lt 1 lc rgb "brown" lw 4 47 | set style line 4 lt 1 lc rgb "green" lw 4 48 | set style line 5 lt 1 lc rgb "cyan" lw 4 49 | set style line 6 lt 1 lc rgb "blue" lw 4 50 | set style line 7 lt 1 lc rgb "violet" lw 4 51 | set style line 8 lt 1 lc rgb "yellow" lw 4 52 | set style line 9 lt 1 lc rgb "green" lw 4 53 | set style line 10 lt 1 lc rgb "cyan" lw 4 54 | set style line 11 lt 1 lc rgb "blue" lw 4 55 | set style line 12 lt 1 lc rgb "violet" lw 4 56 | show style line 57 | starting_time = 37824 58 | 59 | offset = 0 60 | t0(x)=(offset=($0==0) ? x : offset, x - offset) 61 | 62 | plot "/post_processed_stats/2014-03-03_13.29.19_active_mem.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 1 t "bwa aln 1" with lines, \ 63 | '' using ((timecolumn(3)-offset)/3600):4 every ::3 ls 2 t "bwa aln 2" with lines, \ 64 | '' using ((timecolumn(5)-offset)/3600):6 every ::3 ls 3 t "sampe" with lines 65 | -------------------------------------------------------------------------------- /workflow_stats_parser/plot_templates/template_many_cores.plt: -------------------------------------------------------------------------------- 1 | ################################################################################# 2 | # The MIT License (MIT) # 3 | # # 4 | # Copyright (c) 2014 Intel Corporation # 5 | # # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 7 | # of this software and associated documentation files (the "Software"), to deal # 8 | # in the Software without restriction, including without limitation the rights # 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 10 | # copies of the Software, and to permit persons to whom the Software is # 11 | # furnished to do so, subject to the following conditions: # 12 | # # 13 | # The above copyright notice and this permission notice shall be included in # 14 | # all copies or substantial portions of the Software. # 15 | # # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 22 | # THE SOFTWARE. # 23 | ################################################################################# 24 | 25 | clear 26 | reset 27 | print "average utilization across many cores" 28 | set terminal pngcairo transparent enhanced font "arial,25" fontscale 1.0 size 1920, 1080 29 | set key outside bottom center box title "Workflow Phase(s)" enhanced 30 | set key maxrows 4 31 | set key font ",25" spacing 1 samplen 2.9 width 2 height 1 32 | set xlabel "Time (hours)" font ",25" 33 | set ylabel "Utilization (%)" font ",25" 34 | 35 | set output "/post_processed_stats/output_many_cores_utilization_plot.png" 36 | set title "Average CPU Utilization (%) Across All Cores per Phase\n{/*0.5 }" font ",35" 37 | set datafile separator "," 38 | #set xdata time 39 | set timefmt "%Y-%m-%d %H:%M:%S" 40 | #set xtics format "%d:%H:%M" font ",25" 41 | set ytics font ",25" 42 | 43 | set style line 1 lt 1 lc rgb "red" lw 4 44 | set style line 2 lt 1 lc rgb "orange" lw 4 45 | set style line 3 lt 1 lc rgb "brown" lw 4 46 | set style line 4 lt 1 lc rgb "green" lw 4 47 | set style line 5 lt 1 lc rgb "cyan" lw 4 48 | set style line 6 lt 1 lc rgb "blue" lw 4 49 | set style line 7 lt 1 lc rgb "violet" lw 4 50 | set style line 8 lt 1 lc rgb "yellow" lw 4 51 | set style line 9 lt 1 lc rgb "green" lw 4 52 | set style line 10 lt 1 lc rgb "cyan" lw 4 53 | set style line 11 lt 1 lc rgb "red" lw 4 54 | set style line 12 lt 1 lc rgb "violet" lw 4 55 | 56 | show style line 57 | 58 | offset = 0 59 | starting_time = 56309 60 | t0(x)=(offset=($0==0) ? x : offset, x - offset) 61 | 62 | plot "/tmp/tmpAe1agT/0TkGIar.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 1 t "core 0" with lines,\ 63 | "/tmp/tmpAe1agT/1v8mw8f.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 2 t "core 1" with lines,\ 64 | "/tmp/tmpAe1agT/2Swc4Sc.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 3 t "core 2" with lines,\ 65 | "/tmp/tmpAe1agT/3qR_gQy.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 4 t "core 3" with lines,\ 66 | "/tmp/tmpAe1agT/40rZ_Ic.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 5 t "core 4" with lines,\ 67 | "/tmp/tmpAe1agT/5FTDVgU.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 6 t "core 5" with lines,\ 68 | "/tmp/tmpAe1agT/6F6F_zO.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 7 t "core 6" with lines,\ 69 | "/tmp/tmpAe1agT/7asjTV4.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 8 t "core 7" with lines,\ 70 | "/tmp/tmpAe1agT/8855lJ8.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 9 t "core 8" with lines,\ 71 | "/tmp/tmpAe1agT/9fqP1w1.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 10 t "core 9" with lines,\ 72 | "/tmp/tmpAe1agT/1090Xii2.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 11 t "core 10" with lines,\ 73 | "/tmp/tmpAe1agT/11PFh16_.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 12 t "core 11" with lines,\ 74 | "/tmp/tmpAe1agT/126wQeVe.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 13 t "core 12" with lines,\ 75 | "/tmp/tmpAe1agT/13fqpjE9.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 14 t "core 13" with lines,\ 76 | "/tmp/tmpAe1agT/14KGGIBO.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 15 t "core 14" with lines,\ 77 | "/tmp/tmpAe1agT/15Sx9Drj.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 16 t "core 15" with lines,\ 78 | "/tmp/tmpAe1agT/164oMqDB.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 17 t "core 16" with lines,\ 79 | "/tmp/tmpAe1agT/17BIAj2P.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 18 t "core 17" with lines,\ 80 | "/tmp/tmpAe1agT/18TsOhwO.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 19 t "core 18" with lines,\ 81 | "/tmp/tmpAe1agT/19WZ68jG.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 20 t "core 19" with lines,\ 82 | "/tmp/tmpAe1agT/206mkNTa.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 21 t "core 20" with lines,\ 83 | "/tmp/tmpAe1agT/21b7fxzo.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 22 t "core 21" with lines,\ 84 | "/tmp/tmpAe1agT/229mBLIJ.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 23 t "core 22" with lines,\ 85 | "/tmp/tmpAe1agT/23iWPP89.csv" using (t0(timecolumn(1))/3600):2 every ::3 ls 24 t "core 23" with line 86 | -------------------------------------------------------------------------------- /data_collection_workflow_template.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | ################################################################################# 3 | # The MIT License (MIT) # 4 | # # 5 | # Copyright (c) 2014 Intel Corporation # 6 | # # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 8 | # of this software and associated documentation files (the "Software"), to deal # 9 | # in the Software without restriction, including without limitation the rights # 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 11 | # copies of the Software, and to permit persons to whom the Software is # 12 | # furnished to do so, subject to the following conditions: # 13 | # # 14 | # The above copyright notice and this permission notice shall be included in # 15 | # all copies or substantial portions of the Software. # 16 | # # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 23 | # THE SOFTWARE. # 24 | ################################################################################# 25 | 26 | ############################################################################### 27 | # # 28 | #-------------- Data Collection Script Template for Pipeline -----------------# 29 | # # 30 | # PURPOSE: Serves as a template for the user to create pipeline scripts # 31 | # around the embedded calls to enable profiling # 32 | # # 33 | # # 34 | ############################################################################### 35 | 36 | # Inputs parsed from global script 37 | if (scalar(@ARGV) < 5) { 38 | die("Usage: SampleName NumThreads InputDataDirectory TempOutputDirectory profiling \n[if profiling is enabled, then the following is required]: collectstatspath interval stats \n"); 39 | } 40 | my $sample = $ARGV[0]; # sample_name passed in from global script 41 | my $numThreads = $ARGV[1]; # no_of_threads passed in from global script 42 | my $inDir = $ARGV[2]; # input_directory passed in from global script 43 | my $outDir = "".$ARGV[3]; # output_directory passed in from global script 44 | 45 | # For collect_stats script 46 | my $profiling = $ARGV[4]; # profiling is ON by default 47 | my $collectstatspath = $ARGV[5]; # location of collect_stats script and kill_scripts folder. 48 | my $interval = $ARGV[6]; # interval passed in from the global script. Default is 30, or whatever the user specifies. 49 | my $stats = $ARGV[7]; # tools passed in from the global script 50 | my $sampleprefix = $sample.'_'.$numThreads.'T'; 51 | 52 | ################################################################ 53 | # Enter code for all the variables and setup for your pipeline # 54 | ################################################################ 55 | 56 | # Subroutine Setup for profiling script 57 | 58 | sub Start_profiling { 59 | my ($tag) = @_; 60 | if ($profiling) { 61 | system("$collectstatspath $stats -d $interval -td $outDir -n $sampleprefix -tag $tag -l 5 -u 1 -s 600 &"); # will create a folder for each stage in the format : run.$sampleprefix..$stagetag.1u 62 | } 63 | } 64 | 65 | sub Stop_Profiling { 66 | if ($profiling) { 67 | system("$collectstatspath --kill-all"); #This will call all the scripts inside kill_stats folder and kill the tools 68 | } 69 | } 70 | 71 | ############################################################### 72 | # Code for stages of the pipeline with the following template # 73 | ############################################################### 74 | #Provide a stage name which is a unique identifier for each stage - keep the stage names short and relevant to the stage. Long stage names will affect the sar data files. 75 | # Tag below 'stage1' corresponds to first stage in the sample_dict in workflow_dictionaries.py 76 | my $stage_tag=stage1; 77 | Start_profiling($stage_tag); # profiling folder will contain this stage tag to help the parser identify the stages 78 | print "$stage_tag\n"; 79 | run_your_stage with the right parameters; # call your workflow stage 80 | # CHECK THE EXIT STATUS OF YOUR WORKFLOW STAGE, ON FAILURE TOO CALL Stop_Profiling() to avoid orphan profiling processes 81 | Stop_Profiling(); # invokes the kill scripts to stop profiling 82 | sleep(60); # ideal for killing scripts and not having any overhead for the next stage 83 | 84 | # Tag below 'stage2' corresponds to second stage in the sample_dict in workflow_dictionaries.py 85 | my $stage_tag=stage2; 86 | Start_profiling($stage_tag); # profiling folder will contain this stage tag to help the parser identify the stages 87 | print "$stage_tag\n"; 88 | run_your_stage with the right parameters; # call your workflow stage 89 | # CHECK THE EXIT STATUS OF YOUR WORKFLOW STAGE, ON FAILURE TOO CALL Stop_Profiling() to avoid orphan profiling processes 90 | Stop_Profiling(); # invokes the kill scripts to stop profiling 91 | sleep(60); # ideal for killing scripts and not having any overhead for the next stage 92 | 93 | # Tag below 'stage3' corresponds to third stage in the sample_dict in workflow_dictionaries.py 94 | my $stage_tag=stage3; 95 | Start_profiling($stage_tag); # profiling folder will contain this stage tag to help the parser identify the stages 96 | print "$stage_tag\n"; 97 | run_your_stage with the right parameters; # call your workflow stage 98 | # CHECK THE EXIT STATUS OF YOUR WORKFLOW STAGE, ON FAILURE TOO CALL Stop_Profiling() to avoid orphan profiling processes 99 | Stop_Profiling(); # invokes the kill scripts to stop profiling 100 | sleep(60); # ideal for killing scripts and not having any overhead for the next stage 101 | 102 | # Better to write to a log file the output of each stage. We do that for our pipelines. 103 | printf LOG "#done in %02d:%02d:%02d\n",int($runningTime /3600),int(($runningTime % 3600) /60),int($runningTime %60); 104 | 105 | exit 0; 106 | -------------------------------------------------------------------------------- /workflow_stats_parser/sample_multistage_input/run.sample..stage2.1u/C097FACXX111207.1.AGTTGCTT16..MarkDuplicates.1u.30sec_sampling.iostat: -------------------------------------------------------------------------------- 1 | Linux 2.6.32-431.11.2.el6.x86_64 (c16) 04/15/2014 _x86_64_ (24 CPU) 2 | 3 | 04/15/2014 10:45:44 PM 4 | avg-cpu: %user %nice %system %iowait %steal %idle 5 | 0.75 0.00 0.04 0.03 0.00 99.18 6 | 7 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 8 | sda 0.04 0.30 0.29 0.15 58.92 3.61 143.28 0.01 12.74 8.62 0.38 9 | sdb 0.64 0.40 7.01 1.85 1907.82 317.34 251.07 0.07 7.59 1.01 0.89 10 | 11 | 04/15/2014 10:46:14 PM 12 | avg-cpu: %user %nice %system %iowait %steal %idle 13 | 9.54 0.00 1.76 0.15 0.00 88.56 14 | 15 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 16 | sda 0.00 0.33 0.93 0.50 39.47 6.67 32.19 0.01 3.72 3.30 0.47 17 | sdb 0.00 1.33 0.00 18.03 0.00 1159.20 64.28 0.05 2.56 1.70 3.07 18 | 19 | 04/15/2014 10:46:44 PM 20 | avg-cpu: %user %nice %system %iowait %steal %idle 21 | 10.69 0.00 1.41 0.15 0.00 87.75 22 | 23 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 24 | sda 0.00 0.53 0.00 0.33 0.00 6.93 20.80 0.00 14.80 14.80 0.49 25 | sdb 0.00 2.60 0.00 17.23 0.00 1545.07 89.66 0.05 2.66 1.63 2.80 26 | 27 | 04/15/2014 10:47:14 PM 28 | avg-cpu: %user %nice %system %iowait %steal %idle 29 | 10.04 0.00 1.40 0.11 0.00 88.45 30 | 31 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 32 | sda 0.00 0.13 0.00 0.27 0.00 3.20 12.00 0.00 5.38 5.38 0.14 33 | sdb 0.00 2.10 0.00 19.40 0.00 1475.47 76.05 0.04 2.27 1.26 2.45 34 | 35 | 04/15/2014 10:47:44 PM 36 | avg-cpu: %user %nice %system %iowait %steal %idle 37 | 9.32 0.00 1.02 0.39 0.00 89.27 38 | 39 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 40 | sda 0.00 0.33 0.17 0.43 26.93 6.13 55.11 0.01 11.78 7.22 0.43 41 | sdb 0.00 15.93 0.00 71.97 0.00 18927.73 263.01 0.49 6.86 1.22 8.80 42 | 43 | 04/15/2014 10:48:14 PM 44 | avg-cpu: %user %nice %system %iowait %steal %idle 45 | 10.94 0.00 1.28 0.11 0.00 87.68 46 | 47 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 48 | sda 0.00 0.13 0.00 0.20 0.00 2.67 13.33 0.00 8.17 7.00 0.14 49 | sdb 0.00 1.70 0.00 16.40 0.00 1118.40 68.20 0.04 2.16 1.44 2.36 50 | 51 | 04/15/2014 10:48:44 PM 52 | avg-cpu: %user %nice %system %iowait %steal %idle 53 | 13.17 0.00 0.94 0.11 0.00 85.79 54 | 55 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 56 | sda 0.00 0.10 0.00 0.13 0.00 1.87 14.00 0.00 18.50 17.75 0.24 57 | sdb 0.00 1.53 0.00 16.90 0.00 885.33 52.39 0.03 1.91 1.36 2.30 58 | 59 | 04/15/2014 10:49:14 PM 60 | avg-cpu: %user %nice %system %iowait %steal %idle 61 | 15.57 0.00 0.66 0.23 0.00 83.54 62 | 63 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 64 | sda 0.00 0.20 0.00 0.30 0.00 4.00 13.33 0.01 17.22 17.22 0.52 65 | sdb 0.00 11.97 0.00 62.13 0.00 15240.27 245.28 0.42 6.73 1.16 7.20 66 | 67 | 04/15/2014 10:49:44 PM 68 | avg-cpu: %user %nice %system %iowait %steal %idle 69 | 4.30 0.00 0.42 0.29 0.00 94.99 70 | 71 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 72 | sda 0.00 0.27 0.07 0.40 4.53 5.33 21.14 0.01 11.86 11.64 0.54 73 | sdb 0.00 14.73 0.00 48.37 0.00 12804.00 264.73 0.31 6.40 1.33 6.41 74 | 75 | 04/15/2014 10:50:14 PM 76 | avg-cpu: %user %nice %system %iowait %steal %idle 77 | 3.94 0.00 0.44 0.33 0.00 95.29 78 | 79 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 80 | sda 0.00 0.43 0.00 0.90 0.00 10.67 11.85 0.01 7.59 6.63 0.60 81 | sdb 0.00 16.30 0.00 56.30 0.00 15246.40 270.81 0.38 6.73 1.36 7.65 82 | 83 | 04/15/2014 10:50:44 PM 84 | avg-cpu: %user %nice %system %iowait %steal %idle 85 | 3.92 0.00 0.45 0.31 0.00 95.32 86 | 87 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 88 | sda 0.00 0.20 0.00 0.30 0.00 4.00 13.33 0.00 10.89 10.22 0.31 89 | sdb 0.00 15.33 0.00 57.67 0.00 15380.80 266.72 0.37 6.47 1.19 6.87 90 | 91 | 04/15/2014 10:51:14 PM 92 | avg-cpu: %user %nice %system %iowait %steal %idle 93 | 3.94 0.00 0.41 0.33 0.00 95.32 94 | 95 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 96 | sda 0.00 0.40 0.00 0.27 0.00 5.33 20.00 0.00 11.62 11.50 0.31 97 | sdb 0.00 20.57 0.00 52.67 0.00 15408.80 292.57 0.35 6.61 1.35 7.11 98 | 99 | 04/15/2014 10:51:44 PM 100 | avg-cpu: %user %nice %system %iowait %steal %idle 101 | 3.83 0.00 0.50 0.32 0.00 95.35 102 | 103 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 104 | sda 0.00 0.10 0.00 0.50 0.00 4.80 9.60 0.00 4.47 3.87 0.19 105 | sdb 0.00 16.33 0.00 56.83 0.00 15385.07 270.70 0.38 6.77 1.27 7.22 106 | 107 | 04/15/2014 10:52:14 PM 108 | avg-cpu: %user %nice %system %iowait %steal %idle 109 | 3.94 0.00 0.40 0.31 0.00 95.35 110 | 111 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 112 | sda 0.00 0.10 0.00 0.27 0.00 2.93 11.00 0.00 10.50 10.12 0.27 113 | sdb 0.00 14.57 0.00 57.47 0.00 15082.13 262.45 0.39 6.77 1.27 7.28 114 | 115 | 04/15/2014 10:52:44 PM 116 | avg-cpu: %user %nice %system %iowait %steal %idle 117 | 3.95 0.00 0.38 0.31 0.00 95.36 118 | 119 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 120 | sda 0.00 0.13 0.00 0.30 0.00 3.47 11.56 0.00 12.56 11.89 0.36 121 | sdb 0.00 14.43 0.00 59.13 0.00 15444.27 261.18 0.39 6.65 1.25 7.39 122 | 123 | 04/15/2014 10:53:14 PM 124 | avg-cpu: %user %nice %system %iowait %steal %idle 125 | 3.87 0.00 0.47 0.31 0.00 95.34 126 | 127 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 128 | sda 0.00 0.13 0.00 0.40 0.00 4.27 10.67 0.01 18.08 11.58 0.46 129 | sdb 0.00 12.10 0.00 60.50 0.00 15329.07 253.37 0.42 6.87 1.18 7.14 130 | 131 | 04/15/2014 10:53:44 PM 132 | avg-cpu: %user %nice %system %iowait %steal %idle 133 | 3.91 0.00 0.41 0.33 0.00 95.35 134 | 135 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 136 | sda 0.00 0.13 0.00 0.30 0.00 3.47 11.56 0.01 24.44 24.11 0.72 137 | sdb 0.00 7.97 0.00 67.03 0.00 15578.67 232.40 0.44 6.52 1.07 7.17 138 | 139 | 04/15/2014 10:54:14 PM 140 | avg-cpu: %user %nice %system %iowait %steal %idle 141 | 3.88 0.00 0.48 0.31 0.00 95.33 142 | 143 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 144 | sda 0.00 0.13 0.00 0.37 0.00 4.00 10.91 0.01 13.73 13.73 0.50 145 | sdb 0.00 9.97 0.00 62.73 0.00 15357.87 244.81 0.43 6.80 1.13 7.09 146 | 147 | 04/15/2014 10:54:44 PM 148 | avg-cpu: %user %nice %system %iowait %steal %idle 149 | 3.88 0.00 0.46 0.31 0.00 95.36 150 | 151 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 152 | sda 0.00 0.07 0.00 0.33 0.00 3.20 9.60 0.00 10.10 9.50 0.32 153 | sdb 0.00 11.10 0.00 61.80 0.00 15493.07 250.70 0.42 6.88 1.16 7.17 154 | 155 | 04/15/2014 10:55:14 PM 156 | avg-cpu: %user %nice %system %iowait %steal %idle 157 | 3.90 0.00 0.47 0.30 0.00 95.32 158 | 159 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 160 | sda 0.00 0.07 0.00 0.23 0.00 2.40 10.29 0.00 7.43 6.43 0.15 161 | sdb 0.00 5.57 0.00 66.23 0.00 15310.67 231.16 0.46 6.90 1.09 7.20 162 | 163 | 04/15/2014 10:55:44 PM 164 | avg-cpu: %user %nice %system %iowait %steal %idle 165 | 3.90 0.00 0.46 0.31 0.00 95.33 166 | 167 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 168 | sda 0.00 0.17 0.00 0.57 0.00 5.87 10.35 0.01 14.12 8.53 0.48 169 | sdb 0.00 5.93 0.00 67.73 0.00 15429.87 227.80 0.46 6.81 1.05 7.09 170 | 171 | 04/15/2014 10:56:14 PM 172 | avg-cpu: %user %nice %system %iowait %steal %idle 173 | 3.92 0.00 0.41 0.32 0.00 95.35 174 | 175 | Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util 176 | sda 0.00 0.13 0.00 0.47 0.00 4.80 10.29 0.01 12.21 6.57 0.31 177 | sdb 0.00 12.87 0.00 61.67 0.00 15690.40 254.44 0.41 6.60 1.21 7.44 178 | 179 | -------------------------------------------------------------------------------- /collect_stats.ksh: -------------------------------------------------------------------------------- 1 | ################################################################################# 2 | # The MIT License (MIT) # 3 | # # 4 | # Copyright (c) 2014 Intel Corporation # 5 | # # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy # 7 | # of this software and associated documentation files (the "Software"), to deal # 8 | # in the Software without restriction, including without limitation the rights # 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # 10 | # copies of the Software, and to permit persons to whom the Software is # 11 | # furnished to do so, subject to the following conditions: # 12 | # # 13 | # The above copyright notice and this permission notice shall be included in # 14 | # all copies or substantial portions of the Software. # 15 | # # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # 22 | # THE SOFTWARE. # 23 | ################################################################################# 24 | 25 | FILE_PREFIX=TEST 26 | TAG=$(date +%Y%m%d).$(date +%H%M) 27 | MY_TS=$(date +%Y%m%dD)$(date +%HH%MM%SS) 28 | TARGET_DIRECTORY=~/runs 29 | PROCESSOR=Xeon 30 | SAR_INTERVAL=30 31 | USEPACCT=0 32 | if [ -z "${HOST}" ] 33 | then 34 | echo "Reading profile..." 35 | ##source ~/.profile 36 | fi 37 | usage() { 38 | echo "Usage: collect_stats.ksh <--sar || --free || --iostat || --mpstat || --netstat || --pacct || --proc>