├── .gitignore ├── 2-iops_d1_qd_1 ├── aio.conf ├── fio.py ├── iou.conf ├── iou_c.conf ├── iou_s.conf ├── plot.py ├── psync.conf ├── run.sh └── spdk_fio.conf ├── 3-c-work_breakdown_qd1 ├── aio.conf ├── clean.sh ├── iou.conf ├── iou_c.conf ├── iou_s.conf ├── plot.py ├── process_report_graph.py ├── psync.conf ├── run.sh └── spdk_fio.conf ├── 4-micro_arch_qd1 ├── aio.conf ├── fio.py ├── iou.conf ├── iou_c.conf ├── iou_s.conf ├── plot.py ├── psync.conf ├── run.sh └── spdk_fio.conf ├── 5-a-inc_qd_dev_1 ├── aio.conf ├── fio.py ├── iou.conf ├── iou_c.conf ├── iou_s.conf ├── plot.py ├── run.sh └── spdk_fio.conf ├── 5-b-inc_qd_dev_7 ├── aio.conf ├── fio.py ├── iou.conf ├── iou_c.conf ├── iou_s.conf ├── plot.py ├── run.sh └── spdk_fio.conf ├── 5-c-work_breakdown_s_point ├── aio.conf ├── clean.sh ├── iou.conf ├── iou_c.conf ├── iou_s.conf ├── plot.py ├── process_report_graph.py ├── run.sh └── spdk_fio.conf ├── 6-micro_arch_s_point ├── aio.conf ├── eliclit_results_all.py ├── fio.py ├── iou.conf ├── iou_c.conf ├── iou_s.conf ├── plot.py ├── run.sh └── spdk_fio.conf ├── 7-multi_thread ├── aio.conf ├── fio.py ├── iou.conf ├── iou_c.conf ├── iou_s.conf ├── plot.py ├── run.sh └── spdk_fio.conf ├── 8-scheduler_multi_thread ├── aio.conf ├── conf ├── fio.py ├── iou.conf ├── iou_c.conf ├── iou_s.conf ├── plot.py └── run.sh ├── LICENSE ├── README.md ├── get_start.md └── write_device ├── write_10_times.sh └── write_all.conf /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | results 3 | results_global 4 | results_local 5 | perf_graph 6 | perf_list 7 | perf_output 8 | *pdf 9 | flamegraph 10 | fio -------------------------------------------------------------------------------- /2-iops_d1_qd_1/aio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=libaio 12 | thread=1 13 | iodepth=1 14 | 15 | [job1] 16 | 17 | filename=/dev/nvme2n1 -------------------------------------------------------------------------------- /2-iops_d1_qd_1/fio.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import json 3 | 4 | import matplotlib.pyplot as plt 5 | 6 | 7 | def select_data_from_dict(data, reterieve_key): 8 | ''' 9 | Select data from dict using given keys 10 | 11 | Args: 12 | data(dict): 13 | reterieve_key(list): 14 | Returns: 15 | ''' 16 | ret = {} 17 | 18 | for k in reterieve_key: 19 | cur_keys = k.split(':') 20 | selected_data = data 21 | for single_key in cur_keys: 22 | selected_data = selected_data[single_key] 23 | ret[k] = selected_data 24 | 25 | return ret 26 | 27 | 28 | def parse_global(data, reterieve_key): 29 | return select_data_from_dict(data, reterieve_key) 30 | 31 | 32 | def parse_one_job(data, reterieve_key): 33 | return select_data_from_dict(data, reterieve_key) 34 | 35 | 36 | ## TODO: Add job specific key 37 | def parse_experiment(filename, global_reterive_key, job_reterive_key): 38 | """ 39 | Parse outputs from one experiment 40 | 41 | Args: 42 | filename (str): _description_ 43 | global_reterive_key (list(str)): _description_ 44 | job_reterive_key (list(str)): _description_ 45 | 46 | Returns: 47 | dict: parsed global results 48 | list(dict): parsed results for each job 49 | """ 50 | f = open(filename, 'r') 51 | 52 | try: 53 | data = json.load(f) 54 | except: 55 | raise (Exception( 56 | 'File {} can not loaded by json.load()'.format(filename))) 57 | f.close() 58 | 59 | num_jobs = len(data['jobs']) 60 | 61 | global_result = parse_global(data['global options'], global_reterive_key) 62 | jobs_result = [] 63 | for job in data['jobs']: 64 | jobs_result.append(parse_one_job(job, job_reterive_key)) 65 | 66 | return global_result, jobs_result 67 | 68 | 69 | # files: {some_key: file_name} 70 | def parse_one_group(files, global_reterive_key, job_reterive_key): 71 | """ 72 | Parse the all experiments from one group 73 | 74 | Args: 75 | dir (str): Dir path to the results folder 76 | files (list(str)): Output filenames 77 | global_reterive_key (list(str)): 78 | job_reterive_key (list(str)): 79 | 80 | Returns: 81 | dict: parsed results for the group 82 | """ 83 | ret = {} 84 | for k in files.keys(): 85 | cur_file_path = files[k] 86 | parsed_output = parse_experiment(cur_file_path, global_reterive_key, 87 | job_reterive_key) 88 | ret[k] = parsed_output 89 | 90 | return ret 91 | 92 | 93 | def parse_all(files, global_reterive_key, job_reterive_key): 94 | """ 95 | Parse output from all groups 96 | 97 | Args: 98 | files (_type_): _description_ 99 | global_reterive_key (_type_): _description_ 100 | job_reterive_key (_type_): _description_ 101 | 102 | Returns: 103 | _type_: _description_ 104 | """ 105 | ret = {} 106 | for group in files.keys(): 107 | group_dir, group_files = files[group] 108 | ret[group] = parse_one_group(group_dir, group_files, 109 | global_reterive_key, job_reterive_key) 110 | 111 | return ret 112 | 113 | 114 | def plot_staked_bar(labels, bar_names, bar_value, title, ylabel, 115 | fig_save_path): 116 | 117 | fig, ax = plt.subplots() 118 | 119 | #plt.title(title) 120 | #plt.ylable(ylabel) 121 | 122 | print('--------') 123 | for cur_bar in bar_names: 124 | print(cur_bar) 125 | print(bar_value[cur_bar]) 126 | ax.bar(labels, bar_value[cur_bar], label=cur_bar) 127 | 128 | plt.legend() 129 | plt.savefig(fig_save_path) 130 | 131 | 132 | def parsed_to_array(reterieve_outputs, get_x_y_label): 133 | """ 134 | Get data from the reterieve keys 135 | 136 | Args: 137 | reterieve_outputs (dict): KV of reterieved values 138 | get_x_y_label (func()): Parse reterieve_outputs and get x, y and label 139 | 140 | Returns: 141 | list : x values 142 | list : y values 143 | list : label values 144 | """ 145 | x = [] 146 | y = [] 147 | std_dev = [] 148 | label = [] 149 | 150 | for k in reterieve_outputs: 151 | global_output = reterieve_outputs[k][0] 152 | jobs_output = reterieve_outputs[k][1] 153 | 154 | cur_x, cur_y, cur_label, cur_std_dev = get_x_y_label( 155 | global_output, jobs_output) 156 | 157 | x.append(cur_x) 158 | y.append(cur_y) 159 | std_dev.append(cur_std_dev) 160 | label.append(cur_label) 161 | 162 | return x, y, std_dev, label 163 | 164 | 165 | def get_all_data(outputs, get_x_y_label): 166 | """ 167 | Get all data from a group 168 | 169 | Args: 170 | outputs (dict): Parsed data 171 | get_x_y_label (func): 172 | 173 | Returns: 174 | dict: (x, y label) for each experiment 175 | """ 176 | ret = {} 177 | for group_name in outputs.keys(): 178 | group_output = outputs[group_name] 179 | group_data = parsed_to_array(group_output, get_x_y_label) 180 | ret[group_name] = group_data 181 | 182 | return ret 183 | 184 | 185 | def parse_spdk_perf(): 186 | pass 187 | 188 | 189 | if __name__ == '__main__': 190 | test_file = '/home/user/test_script/tmp/1/output_iodepth_1.josn' 191 | 192 | global_rk = ["iodepth"] 193 | jobs_rk = ["jobname", "read:slat_ns:mean"] 194 | 195 | gr, jr = parse_experiment(test_file, global_rk, jobs_rk) 196 | print(gr) 197 | print(jr) 198 | -------------------------------------------------------------------------------- /2-iops_d1_qd_1/iou.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | iodepth=1 14 | 15 | [job1] 16 | filename=/dev/nvme2n1 -------------------------------------------------------------------------------- /2-iops_d1_qd_1/iou_c.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | iodepth=1 14 | hipri 15 | 16 | [job1] 17 | filename=/dev/nvme2n1 -------------------------------------------------------------------------------- /2-iops_d1_qd_1/iou_s.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | iodepth=1 14 | sqthread_poll=1 15 | 16 | [job1] 17 | filename=/dev/nvme2n1 -------------------------------------------------------------------------------- /2-iops_d1_qd_1/plot.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | 5 | import fio 6 | 7 | 8 | def bar_graph(x_ticks, 9 | value, 10 | std_dev, 11 | group_list, 12 | bar_width, 13 | fig_save_path, 14 | legend_label, 15 | title=None, 16 | xlabel=None, 17 | ylabel=None, 18 | axis_tick_font_size=None, 19 | axis_label_font_size=None, 20 | legend_font_size=None, 21 | datalabel_size=None, 22 | datalabel_va=None, 23 | print_legend=True): 24 | fig, ax = plt.subplots(figsize=(12, 8)) 25 | 26 | if title: 27 | plt.title(title) 28 | if axis_label_font_size: 29 | plt.xlabel(xlabel, fontsize=axis_label_font_size) 30 | plt.ylabel(ylabel, fontsize=axis_label_font_size) 31 | else: 32 | plt.xlabel(xlabel) 33 | plt.ylabel(ylabel) 34 | plt.grid(axis='y') 35 | 36 | if axis_tick_font_size: 37 | ax.tick_params(axis='both', 38 | which='major', 39 | labelsize=axis_tick_font_size) 40 | 41 | if legend_label == None: 42 | legend_label = {} 43 | for group in group_list: 44 | legend_label[group] = group 45 | 46 | x_axis = np.arange(len(x_ticks)) 47 | 48 | # compute bar offset 49 | bar_offset = [] 50 | mid_point = (len(group_list) * bar_width) / 2 51 | 52 | for i in range(len(group_list)): 53 | bar_offset.append(bar_width * i + 0.5 * bar_width - mid_point) 54 | 55 | x_axis - 0.1 56 | for (index, group_name) in zip(range(len(group_list)), group_list): 57 | y = value[group_name] 58 | bar_pos = x_axis + bar_offset[index] 59 | plt.bar(bar_pos, y, width=bar_width, label=legend_label[group_name]) 60 | y_origin = y 61 | for (index, x, y) in zip(list(range(len(y_origin))), bar_pos, 62 | y_origin): 63 | text = '{:.1f}'.format(y) 64 | plt.text(x, 65 | y, 66 | text, 67 | size=datalabel_size, 68 | ha='center', 69 | va=datalabel_va) 70 | 71 | plt.xticks(x_axis, x_ticks) 72 | 73 | if print_legend: 74 | plt.legend(fontsize=legend_font_size) 75 | 76 | plt.savefig(fig_save_path, bbox_inches='tight') 77 | 78 | 79 | dir = './results' 80 | engines = ['psync', 'aio', 'iou', 'iou_c', 'iou_s', 'spdk_fio'] 81 | files = { 82 | 'psync': 'psync.txt', 83 | 'aio': 'aio.txt', 84 | 'iou': 'iou.txt', 85 | 'iou_s': 'iou_s.txt', 86 | 'iou_c': 'iou_c.txt', 87 | 'spdk_fio': 'spdk_fio.txt' 88 | } 89 | 90 | global_rk = [] 91 | jobs_rk = ['read:lat_ns:mean', 'read:iops', 'read:lat_ns:stddev'] 92 | 93 | iops = [] 94 | iops_stddev = [] 95 | lat = [] 96 | lat_stdev = [] 97 | 98 | for e in engines: 99 | dev1_path = os.path.join(dir, e + '.txt') 100 | _, j_res = fio.parse_experiment(dev1_path, global_rk, jobs_rk) 101 | cur_lat = j_res[0]['read:lat_ns:mean'] / 1000 # us 102 | cur_iops = j_res[0]['read:iops'] / 1000 #kiops 103 | iops.append(cur_iops) 104 | lat.append(cur_lat) 105 | 106 | # print('engines', engines) 107 | # print('throughput', '=', iops) 108 | # print('latency =', lat) 109 | 110 | x_label = 'Engines' 111 | y_label = 'Throughput(KIOPS)' 112 | bar_width = 0.6 113 | axis_label_font_size = 28 114 | axis_tick_font_size = 26 115 | legend_font_size = 26 116 | datalabel_size = 18 117 | datalabel_va = 'bottom' 118 | legend_label = None 119 | 120 | data = {'throughput': iops} 121 | group_list = ['throughput'] 122 | fig_save_path = 'iops_d1_qd1.pdf' 123 | x_ticks = ['psync', 'aio', 'iou', 'iou-c', 'iou-s', 'spdk-fio'] 124 | bar_graph(x_ticks, 125 | data, 126 | None, 127 | group_list, 128 | bar_width, 129 | fig_save_path, 130 | legend_label, 131 | xlabel=None, 132 | ylabel=y_label, 133 | axis_tick_font_size=axis_tick_font_size, 134 | axis_label_font_size=axis_label_font_size, 135 | legend_font_size=legend_font_size, 136 | datalabel_size=datalabel_size, 137 | datalabel_va=datalabel_va, 138 | print_legend=False) 139 | -------------------------------------------------------------------------------- /2-iops_d1_qd_1/psync.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=psync 12 | thread=1 13 | iodepth=1 14 | 15 | [job1] 16 | 17 | filename=/dev/nvme2n1 -------------------------------------------------------------------------------- /2-iops_d1_qd_1/run.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | SPDK_FIO_PLUGIN= 4 | SPDK_SETUP_PATH= 5 | RESULTS=results 6 | 7 | 8 | FIO_RUN_TIME=120 9 | FIO_RAMP_TIME=20 10 | 11 | $SPDK_SETUP_PATH reset; 12 | FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio psync.conf --output-format=json -o $RESULTS/psync.txt; 13 | FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio aio.conf --output-format=json -o $RESULTS/aio.txt; 14 | FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio iou.conf --output-format=json -o $RESULTS/iou.txt; 15 | FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio iou_c.conf --output-format=json -o $RESULTS/iou_c.txt; 16 | FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio iou_s.conf --output-format=json -o $RESULTS/iou_s.txt; 17 | 18 | $SPDK_SETUP_PATH; 19 | LD_PRELOAD=$SPDK_FIO_PLUGIN FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio spdk_fio.conf --output-format=json -o $RESULTS/spdk_fio.txt; 20 | $SPDK_SETUP_PATH reset; -------------------------------------------------------------------------------- /2-iops_d1_qd_1/spdk_fio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | filename=trtype=PCIe traddr=0000.b1.00.0 ns=1 12 | ioengine=spdk 13 | thread=1 14 | iodepth=1 15 | 16 | [job1] 17 | -------------------------------------------------------------------------------- /3-c-work_breakdown_qd1/aio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=libaio 12 | thread=1 13 | iodepth=1 14 | 15 | [job1] 16 | 17 | filename=/dev/nvme0n1 18 | 19 | -------------------------------------------------------------------------------- /3-c-work_breakdown_qd1/clean.sh: -------------------------------------------------------------------------------- 1 | rm -f ./fio/* ./perf_output/* ./perf_list/* ./perf_graph/* ./flamegraph/* 2 | -------------------------------------------------------------------------------- /3-c-work_breakdown_qd1/iou.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | iodepth=1 14 | 15 | [job1] 16 | filename=/dev/nvme0n1 17 | -------------------------------------------------------------------------------- /3-c-work_breakdown_qd1/iou_c.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | iodepth=1 14 | hipri 15 | 16 | [job1] 17 | filename=/dev/nvme0n1 18 | 19 | -------------------------------------------------------------------------------- /3-c-work_breakdown_qd1/iou_s.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | iodepth=1 14 | sqthread_poll=1 15 | 16 | 17 | [job1] 18 | filename=/dev/nvme0n1 19 | 20 | -------------------------------------------------------------------------------- /3-c-work_breakdown_qd1/plot.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import matplotlib.pyplot as plt 3 | 4 | from process_report_graph import parse_report_file 5 | 6 | legend_name = { 7 | 'app': 'fio', 8 | 'interface': 'I/O lib', 9 | 'block': 'block_layer', 10 | 'driver': 'nvme driver', 11 | 'sys': 'kernel' 12 | } 13 | 14 | legend_name = { 15 | 'app': 'fio', 16 | 'interface': 'I/O lib', 17 | 'block': 'block_layer', 18 | 'driver': 'nvme driver', 19 | 'sys': 'kernel' 20 | } 21 | 22 | CB_color_cycle = [ 23 | '#377eb8', '#ff7f00', '#4daf4a', '#f781bf', '#a65628', '#984ea3', 24 | '#999999', '#e41a1c', '#dede00' 25 | ] 26 | 27 | color_index = 0 28 | 29 | 30 | def get_color(): 31 | global color_index 32 | prev_c = color_index 33 | color_index += 1 34 | color_index %= len(CB_color_cycle) 35 | return CB_color_cycle[prev_c] 36 | 37 | 38 | def plot(labels, 39 | bar_names, 40 | bar_value, 41 | title, 42 | xlabel, 43 | ylabel, 44 | fig_save_path, 45 | axis_label_font_size=None, 46 | axis_tick_font_size=None, 47 | legend_font_size=None, 48 | datalabel_size=None, 49 | datalabel_va=None): 50 | 51 | fig, ax = plt.subplots() 52 | plt.grid(True) 53 | if title: 54 | plt.title(title) 55 | if axis_label_font_size: 56 | plt.xlabel(xlabel, fontsize=axis_label_font_size) 57 | plt.ylabel(ylabel, fontsize=axis_label_font_size) 58 | else: 59 | plt.xlabel(xlabel) 60 | plt.ylabel(ylabel) 61 | 62 | if axis_tick_font_size: 63 | ax.tick_params(axis='both', 64 | which='major', 65 | labelsize=axis_tick_font_size) 66 | 67 | print('--------') 68 | bottom = [0] * len(labels) 69 | for index, cur_bar in zip(range(len(bar_names)), bar_names): 70 | # print(cur_bar) 71 | # print(bar_value[cur_bar]) 72 | cur_ax = ax.bar( 73 | labels, 74 | bar_value[cur_bar], 75 | label=legend_name[cur_bar], 76 | bottom=bottom, 77 | #width=0.3, 78 | #fill=False, 79 | edgecolor='black', 80 | color=get_color() 81 | #hatch=hatch[index % len(bar_names)] 82 | ) 83 | 84 | cur_bar_label = bar_value[cur_bar] 85 | cur_bar_label = ["{:.2f}".format(x) for x in cur_bar_label] 86 | ax.bar_label(cur_ax, cur_bar_label, label_type='center', size=14) 87 | for i in range(len(bottom)): 88 | bottom[i] += bar_value[cur_bar][i] 89 | 90 | plt.legend(loc='upper center', 91 | bbox_to_anchor=(0.5, 1.22), 92 | ncol=3, 93 | fancybox=True, 94 | shadow=True, 95 | fontsize=14) 96 | 97 | #plt.legend() 98 | plt.savefig(fig_save_path, bbox_inches="tight") 99 | 100 | 101 | engines = ["psync", "aio", "iou", "iou_c", "iou_s"] #, "spdk_fio_s"] 102 | result_path = 'perf_graph' 103 | 104 | data = [] 105 | for e in engines: 106 | fname = os.path.join(result_path, 'perf_parsed_' + e + '_g.txt') 107 | res = parse_report_file(fname) 108 | res = [i * 100 for i in res] 109 | data.append(res) 110 | 111 | # TODO: PLEASE CLASSIFY THE DATA BY YOURSELF 112 | data.append([73, 0, 0, 27, 0]) 113 | 114 | labels = ["psync", "aio", "iou", "iou-c", "iou-s", "spdk-fio"] 115 | bar_names = ['app', 'interface', 'block', 'driver', 'sys'] 116 | 117 | bar_value = {'app': [], 'interface': [], 'block': [], 'driver': [], 'sys': []} 118 | 119 | for split in data: 120 | for (index, cost_class) in enumerate(bar_names): 121 | bar_value[cost_class].append(split[index]) 122 | 123 | print(bar_value) 124 | 125 | title = None 126 | xlabel = 'Engines' 127 | ylabel = 'Percentage' 128 | fig_save_path = 'work_breakdown_s_point.pdf' 129 | axis_label_font_size = 20 130 | axis_tick_font_size = 16 131 | legend_font_size = 18 132 | datalabel_size = 20 133 | datalabel_va = 'bottom' 134 | plot(labels, 135 | bar_names, 136 | bar_value, 137 | title, 138 | xlabel, 139 | ylabel, 140 | fig_save_path, 141 | axis_tick_font_size=axis_tick_font_size, 142 | axis_label_font_size=axis_label_font_size, 143 | legend_font_size=legend_font_size, 144 | datalabel_size=datalabel_size, 145 | datalabel_va=datalabel_va) 146 | -------------------------------------------------------------------------------- /3-c-work_breakdown_qd1/process_report_graph.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import sys 3 | 4 | 5 | def read_file(filename): 6 | f = open(filename, 'r') 7 | lines = f.readlines() 8 | f.close() 9 | 10 | # remove all leading spaces 11 | ret = [] 12 | for l in lines: 13 | if l[0] == '#': 14 | continue 15 | ret.append(l[:-1]) 16 | 17 | return ret 18 | 19 | 20 | def parse_overhead(overhead, format): 21 | if format == 'percent': 22 | return float(overhead[:-1]) 23 | elif format == 'number': 24 | return int(overhead) 25 | else: 26 | assert 0, 'format {} not recognized, should be percent or number'.format( 27 | format) 28 | 29 | 30 | def find_top_layer(file_contents, symbol): 31 | for l in file_contents: 32 | if l != '' and l[0] != '|': 33 | pass 34 | 35 | 36 | def elicit_children(file_contents): 37 | children = [] 38 | 39 | cur_children = [] 40 | for l in file_contents: 41 | if l[1:].strip() == '': 42 | children.append(cur_children) 43 | cur_children = [] 44 | else: 45 | cur_children.append(l) 46 | 47 | if len(cur_children) != 0: 48 | children.append(cur_children) 49 | 50 | return children 51 | 52 | 53 | def reshape_child(file_contents): 54 | line_has_content = 0 55 | for l in file_contents: 56 | if l.strip() == '|': 57 | line_has_content += 1 58 | else: 59 | break 60 | 61 | file_contents = file_contents[line_has_content:] 62 | 63 | if len(file_contents) == 0: 64 | return [] 65 | 66 | processed_child = [] 67 | 68 | num_spaces = 0 69 | for c in file_contents[0]: 70 | if c == ' ' or c == '|': 71 | num_spaces += 1 72 | else: 73 | break 74 | num_spaces -= 1 75 | for l in file_contents: 76 | processed_child.append(l[num_spaces:]) 77 | 78 | return processed_child 79 | 80 | 81 | def process_child(file_contents, overhead_format='number'): 82 | """_summary_ 83 | 84 | Args: 85 | file_contents (_type_): _description_ 86 | overhead_type (str, optional): _description_. Defaults to 'number'. percent or number 87 | 88 | Returns: 89 | _type_: _description_ 90 | """ 91 | title = file_contents[0] 92 | file_contents = file_contents[1:] 93 | 94 | # process title 95 | if title[0] == '|': 96 | title = title[1:] 97 | title = title.strip() 98 | 99 | zero_overhead = False 100 | if title[:3] == '---': 101 | zero_overhead = True 102 | 103 | title = title.split('-') 104 | 105 | title_data = [] 106 | for l in title: 107 | if l != '': 108 | title_data.append(l) 109 | 110 | if zero_overhead: 111 | title_data = [0] + title_data 112 | 113 | assert len(title_data) == 2, '{}, {}'.format(len(title_data), title_data) 114 | 115 | overhead = 0 116 | if not zero_overhead: 117 | overhead = parse_overhead(title_data[0], overhead_format) 118 | 119 | name = [title_data[1]] 120 | 121 | # TODO: process the overhead 122 | 123 | addition_name = [] 124 | children = [] 125 | for i in range(len(file_contents)): 126 | l = file_contents[i] 127 | if l[1:].strip() == '|': 128 | if len(file_contents) > i + 1: 129 | children = file_contents[i + 1:] 130 | else: 131 | children = [] 132 | break 133 | else: 134 | cur_addition_name = l[1:].strip() 135 | # If there is address in addition name, skip it 136 | if cur_addition_name[:2] == '0x': 137 | continue 138 | addition_name.append(cur_addition_name) 139 | 140 | name = name + addition_name 141 | 142 | # now clean up the child, child are in 143 | children = reshape_child(children) 144 | children = elicit_children(children) 145 | 146 | parsed_children = [] 147 | if len(children) != 0: 148 | for c in children: 149 | parsed_children.append(process_child(c)) 150 | 151 | return [overhead, name, parsed_children] 152 | 153 | 154 | def print_child(child): 155 | for c in child: 156 | print(c) 157 | 158 | 159 | def print_children(children): 160 | for i in range(len(children)): 161 | print(i) 162 | for l in children[i]: 163 | print(l) 164 | print('') 165 | 166 | 167 | def print_recursively(children, indent=0): 168 | if len(children) == 0: 169 | return 170 | # the argument is the return value of parse child 171 | overhead = children[0] 172 | names = str(children[1]) 173 | 174 | print(' ' * indent + str(overhead)) 175 | print(' ' * indent + str(names)) 176 | print(' ' * indent + "|") 177 | for c in children[2]: 178 | print_recursively(c, indent + 2) 179 | print(' ' * indent + '|') 180 | 181 | 182 | def is_first_level_symbol(l): 183 | if l.strip()[-1] == '-': 184 | return True 185 | 186 | return False 187 | 188 | 189 | def process_origin_file(filename, overhead_format="number"): 190 | overhead_format = "number" 191 | parsed_data = {} 192 | first_level_symbols = [] 193 | all_overhead_by_inst = 0 194 | 195 | f = open(filename, 'r') 196 | lines = f.readlines() 197 | f.close() 198 | 199 | splitted_input = [] 200 | cur_symbol = [] 201 | for l in lines: 202 | if l[0] == "#": 203 | pass 204 | elif l == '\n': 205 | if len(cur_symbol) != 0: 206 | splitted_input.append(cur_symbol) 207 | cur_symbol = [] 208 | elif is_first_level_symbol(l): 209 | if len(cur_symbol) != 0: 210 | splitted_input.append(cur_symbol) 211 | cur_symbol = [l[:-1]] 212 | else: 213 | cur_symbol.append(l[:-1]) 214 | if len(cur_symbol) != 0: 215 | splitted_input.append(cur_symbol) 216 | 217 | for cur_symbol in splitted_input: 218 | # Prase the title line 219 | title = cur_symbol[0] 220 | title = title.split(' ') 221 | title = [e for e in title if e != ''] 222 | overhead_all = parse_overhead(title[0], 223 | 'percent') # Children and itself 224 | overhead_self_percent = parse_overhead(title[1], 'percent') 225 | overhead_self_samples = parse_overhead(title[2], 'number') 226 | 227 | all_overhead_by_inst += overhead_self_samples 228 | 229 | # If the over head is 0, skip it 230 | if overhead_all == 0: 231 | continue 232 | 233 | name = title[4] 234 | children = reshape_child(cur_symbol[1:]) 235 | children = elicit_children(children) 236 | 237 | if len(children) == 0: 238 | # TODO: Write to the final result 239 | continue 240 | 241 | parsed_children = [] 242 | for c in children: 243 | result = process_child(c, overhead_format) 244 | parsed_children.append(result) 245 | if not name in parsed_data: 246 | total_instructions = 0 # + overhead_self_samples 247 | for c in parsed_children: 248 | total_instructions += c[0] 249 | parsed_data[name] = [total_instructions, [name], parsed_children] 250 | else: 251 | print('first level symbol {} occurs more than once'.format(name)) 252 | 253 | return parsed_data, all_overhead_by_inst 254 | 255 | 256 | def get_overhead(overhead_tree, symbol): 257 | """ 258 | Args: 259 | overhead_tree (_type_): _description_ 260 | remove_list (str): A list of symbols to remove 261 | """ 262 | if symbol in overhead_tree[1]: 263 | return overhead_tree[0] 264 | 265 | overhead = 0 266 | for c in overhead_tree[2]: 267 | overhead += get_overhead(c, symbol) 268 | 269 | return overhead 270 | 271 | 272 | def remove_symbol(overhead_tree, symbol): 273 | """" WARNING: THIS WILL CHANGE THE ARGUMENT overhead_tree 274 | 275 | Args: 276 | overhead_tree (_type_): _description_ 277 | symbol (_type_): _description_ 278 | 279 | Returns: 280 | _type_: _description_ 281 | """ 282 | 283 | # Found the target symbol, return the value, then delete all its children, mark overhead as 0 284 | if symbol in overhead_tree[1]: 285 | overhead = overhead_tree[0] 286 | overhead_tree[0] = 0 287 | overhead_tree[2] = [] 288 | 289 | return overhead 290 | 291 | overhead = 0 292 | for c in overhead_tree[2]: 293 | overhead += remove_symbol(c, symbol) 294 | 295 | overhead_tree[0] = overhead_tree[0] - overhead 296 | 297 | return overhead 298 | 299 | 300 | def parse_report_file(filename): 301 | classification = { 302 | 'app': [ 303 | 'fio_ioring_commit', '__fio_gettime', 'account_io_completion', 304 | 'add_clat_sample', 'add_lat_sample', 'fio_ioring_prep', 305 | 'fio_ioring_queue', 'get_next_rand_block', 'io_queue_event', 306 | 'io_u_mark_complete', 'io_u_mark_depth', 'lock_file', 'log_io_u', 307 | 'ntime_since', 'td_io_commit', 'td_io_getevents', 'td_io_prep', 308 | 'td_io_queue', 'utime_since_now', 'get_io_u', 'io_u_mark_submit', 309 | 'fio_ioring_getevents', 'fio_ioring_event', 'fio_libaio_getevents', 310 | 'io_u_queued_complete', 'fio_libaio_event', 'io_completed', 311 | 'ramp_time_over', 'run_threads', 'put_file' 312 | ], 313 | 'interface': [ 314 | 'io_submit_sqes', 'blkdev_read_iter', 'rw_verify_area', 315 | 'io_do_iopoll', 'io_submit_one', 'io_getevents', 'do_io_getevents' 316 | ], 317 | 'block': [ 318 | 'bio_alloc_kiocb', 'bio_associate_blkg', 'bio_iov_iter_get_pages', 319 | 'bio_set_pages_dirty', 'blk_finish_plug', 'submit_bio', 320 | 'blk_mq_end_request', 'blkdev_direct_IO', 'ret_from_fork' 321 | ], 322 | 'driver': ['nvme_queue_rq', 'nvme_irq'], 323 | 'sys': [ 324 | 'asm_common_interrupt', 325 | 'entry_SYSCALL_64', 326 | 'syscall_return_via_sysret', 327 | 'syscall', 328 | ] 329 | } 330 | symbol = [ 331 | # fio 332 | ('fio_ioring_commit', [ 333 | 'entry_SYSCALL_64', 'syscall_return_via_sysret', 334 | 'asm_common_interrupt' 335 | ]), 336 | ('__fio_gettime', ['asm_common_interrupt']), 337 | ('account_io_completion', ['asm_common_interrupt']), 338 | ('add_clat_sample', ['asm_common_interrupt']), 339 | ('add_lat_sample', ['asm_common_interrupt']), 340 | ('fio_ioring_prep', ['asm_common_interrupt']), 341 | ('fio_ioring_queue', ['asm_common_interrupt']), 342 | ('get_next_rand_block', ['asm_common_interrupt']), 343 | ('io_queue_event', ['asm_common_interrupt']), 344 | ('io_u_mark_complete', ['asm_common_interrupt']), 345 | ('io_u_mark_depth', ['asm_common_interrupt']), 346 | ('lock_file', ['asm_common_interrupt']), 347 | ('log_io_u', ['asm_common_interrupt']), 348 | ('ntime_since', ['asm_common_interrupt']), 349 | ('put_file', ['asm_common_interrupt']), 350 | ('td_io_commit', ['asm_common_interrupt']), 351 | ('td_io_getevents', ['asm_common_interrupt']), 352 | ('td_io_prep', ['asm_common_interrupt']), 353 | ('td_io_queue', ['asm_common_interrupt']), 354 | ('utime_since_now', ['asm_common_interrupt']), 355 | ('get_io_u', ['asm_common_interrupt']), 356 | ('io_u_mark_submit', ['asm_common_interrupt']), 357 | ('fio_ioring_getevents', ['asm_common_interrupt', 'entry_SYSCALL_64']), 358 | ('fio_ioring_event', ['asm_common_interrupt']), 359 | ('fio_libaio_getevents', ['asm_common_interrupt']), 360 | ('fio_libaio_event', ['asm_common_interrupt']), 361 | ('io_u_queued_complete', ['asm_common_interrupt']), 362 | ('io_completed', ['asm_common_interrupt']), 363 | ('ramp_time_over', ['asm_common_interrupt']), 364 | ('run_threads', [ 365 | 'asm_common_interrupt', 'get_io_u', 'io_queue_event', 366 | 'io_u_queued_complete', 'td_io_queue', '__fio_gettime', 367 | '__get_io_u', 'fio_gettime', 'get_file', 'get_next_rand_block', 368 | 'utime_since_now', 'fio_ioring_commit ' 369 | ]), 370 | # iouring 371 | ('io_submit_sqes', [ 372 | 'blkdev_read_iter', 'rw_verify_area', 'asm_common_interrupt', 373 | 'blk_finish_plug' 374 | ]), 375 | ('io_do_iopoll', [ 376 | 'asm_common_interrupt', 377 | ]), 378 | ('ret_from_fork', ['asm_common_interrupt', 'io_submit_sqes']), 379 | # aio 380 | ('do_io_getevents', ['asm_common_interrupt']), 381 | ('io_submit_one', ['blkdev_read_iter']), 382 | # fs 383 | ('blkdev_read_iter', ['blkdev_direct_IO', 'asm_common_interrupt']), 384 | ('rw_verify_area', ['asm_common_interrupt']), 385 | ('io_getevents', ['asm_common_interrupt']), 386 | ('blkdev_direct_IO', [ 387 | 'bio_alloc_kiocb', 'bio_associate_blkg', 'bio_iov_iter_get_pages', 388 | 'bio_set_pages_dirty', 'blk_finish_plug', 'submit_bio', 389 | 'asm_common_interrupt' 390 | ]), 391 | 392 | # blk layer 393 | ('bio_alloc_kiocb', ['asm_common_interrupt']), 394 | ('bio_associate_blkg', ['asm_common_interrupt']), 395 | ('bio_iov_iter_get_pages', ['asm_common_interrupt']), 396 | ('bio_set_pages_dirty', ['asm_common_interrupt']), 397 | ('blk_finish_plug', ['nvme_queue_rq', 'asm_common_interrupt']), 398 | ('submit_bio', ['asm_common_interrupt']), 399 | ('blk_mq_end_request', ['asm_common_interrupt']), 400 | ('nvme_queue_rq', ['asm_common_interrupt']), 401 | ('nvme_irq', ['blk_mq_end_request']), 402 | # system 403 | ('asm_common_interrupt', [ 404 | 'nvme_irq', 'bio_iov_iter_get_pages', 'blk_finish_plug', 405 | 'blk_update_request', 'blk_mq_free_request' 406 | ]), 407 | ('entry_SYSCALL_64', [ 408 | 'io_submit_sqes', 'asm_common_interrupt', 'blkdev_iopoll', 409 | 'io_do_iopoll', 'do_io_getevents', 'io_submit_one', 'vfs_read' 410 | ]), 411 | ('syscall_return_via_sysret', ['asm_common_interrupt']), 412 | ('syscall', ['entry_SYSCALL_64', 'asm_common_interrupt']) 413 | ] 414 | ## check corresponding 415 | symbols_in_classify = set() 416 | for v in classification.values(): 417 | for s in v: 418 | if not s in symbols_in_classify: 419 | symbols_in_classify.add(s) 420 | else: 421 | raise Exception( 422 | "symbol {} occurs more than one in classification".format( 423 | s)) 424 | 425 | for i in symbol: 426 | s = i[0] 427 | if not s in symbols_in_classify: 428 | raise Exception("symbol {} not occurs in classification".format(s)) 429 | 430 | parsed_data, all_overhead_by_inst = process_origin_file(filename) 431 | 432 | statics = 0 433 | all_overheads = {} 434 | 435 | for item in symbol: 436 | name = item[0] 437 | # print(name) 438 | excluded_symbol = item[1] 439 | if not name in parsed_data: 440 | print(name, "NOT FOUND") 441 | continue 442 | children = copy.deepcopy(parsed_data[name]) 443 | 444 | # Overhead not belongs to the symbol 445 | overheads = 0 446 | for s in excluded_symbol: 447 | overheads += get_overhead(children, s) 448 | remove_symbol(children, s) 449 | all_overheads[item[0]] = parsed_data[name][0] - overheads 450 | statics += parsed_data[name][0] - overheads 451 | 452 | print('') 453 | 454 | print("Selected symbols:") 455 | selected_symbols = [ 456 | 'entry_SYSCALL_64', 'asm_common_interrupt', 'io_do_iopoll' 457 | ] 458 | for ss in selected_symbols: 459 | ss_overheads = 0 460 | if ss in all_overheads: 461 | ss_overheads = all_overheads[ss] 462 | 463 | print(ss, ":", ss_overheads) 464 | 465 | print("Classified:", statics, '/', all_overhead_by_inst, 466 | statics / all_overhead_by_inst) 467 | 468 | ret_key = ['app', 'interface', 'block', 'driver', 'sys'] 469 | ret = [] 470 | 471 | print("Overheads by layer:\n") 472 | for layer in ret_key: 473 | symbols = classification[layer] 474 | cur_layer_overhead = 0 475 | for s in symbols: 476 | if s in all_overheads: 477 | cur_layer_overhead += all_overheads[s] 478 | print(layer, ':', cur_layer_overhead, 479 | cur_layer_overhead / all_overhead_by_inst) 480 | ret.append(cur_layer_overhead / all_overhead_by_inst) 481 | 482 | return ret 483 | 484 | 485 | if __name__ == '__main__': 486 | filename = sys.argv[1] 487 | parse_report_file(filename) 488 | -------------------------------------------------------------------------------- /3-c-work_breakdown_qd1/psync.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=psync 12 | thread=1 13 | iodepth=1 14 | 15 | [job1] 16 | 17 | filename=/dev/nvme0n1 -------------------------------------------------------------------------------- /3-c-work_breakdown_qd1/run.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | VMLINUX= 4 | SPDK_FIO_PLUGIN= 5 | SPDK_SETUP_PATH= 6 | 7 | FIO_RUN_TIME=120 8 | FIO_RAMP_TIME=20 9 | 10 | FIO_RES='./fio' 11 | PERF_RES='./perf_output' 12 | PERF_PARSED_LIST='./perf_list' 13 | PERF_PARSED_GRAPH='./perf_graph' 14 | FLAMEGRAPH_OUT='./flamegraph' 15 | 16 | declare -a engines=("psync" "aio" "iou" "iou_s" "iou_c") 17 | 18 | $SPDK_SETUP_PATH reset 19 | 20 | for e in "${engines[@]}" 21 | do 22 | perf record -e instructions -F 99 -o $PERF_RES/${e}.perf.out env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --thread=1 --output-format=json -o $FIO_RES/${e}_fio.txt; 23 | 24 | perf record -g -e instructions -F 99 -o $PERF_RES/${e}_g.perf.out env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --thread=1 --output-format=json -o $FIO_RES/${e}_g.txt; 25 | done 26 | 27 | e="spdk_fio" 28 | 29 | $SPDK_SETUP_PATH 30 | 31 | perf record -e instructions -F 99 -o $PERF_RES/${e}.perf.out env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME LD_PRELOAD=$SPDK_FIO_PLUGIN fio ${e}.conf --thread=1 --output-format=json -o $FIO_RES/${e}_fio.txt; 32 | 33 | perf record -g -e instructions -F 99 -o $PERF_RES/${e}_g.perf.out env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME LD_PRELOAD=$SPDK_FIO_PLUGIN fio ${e}.conf --thread=1 --output-format=json -o $FIO_RES/${e}_g.txt; 34 | 35 | $SPDK_SETUP_PATH reset; 36 | 37 | # # parse 38 | 39 | for e in "${engines[@]}" 40 | do 41 | perf report --vmlinux $VMLINUX -n -m --stdio --full-source-path --source -s symbol -i $PERF_RES/${e}.perf.out >> $PERF_PARSED_LIST/perf_parsed_${e}.txt; 42 | 43 | perf report --vmlinux $VMLINUX -n -m --stdio --full-source-path --source -s symbol --call-graph=graph,0,caller,function,count -i $PERF_RES/${e}_g.perf.out >> $PERF_PARSED_GRAPH/perf_parsed_${e}_g.txt; 44 | done 45 | -------------------------------------------------------------------------------- /3-c-work_breakdown_qd1/spdk_fio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | filename=trtype=PCIe traddr=0000.af.00.0 ns=1 12 | ioengine=spdk 13 | thread=1 14 | iodepth=1 15 | 16 | [job1] 17 | -------------------------------------------------------------------------------- /4-micro_arch_qd1/aio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=libaio 12 | thread=1 13 | iodepth=1 14 | 15 | [job1] 16 | 17 | filename=/dev/nvme2n1 -------------------------------------------------------------------------------- /4-micro_arch_qd1/fio.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import json 3 | 4 | import matplotlib.pyplot as plt 5 | 6 | 7 | def select_data_from_dict(data, reterieve_key): 8 | ''' 9 | Select data from dict using given keys 10 | 11 | Args: 12 | data(dict): 13 | reterieve_key(list): 14 | Returns: 15 | ''' 16 | ret = {} 17 | 18 | for k in reterieve_key: 19 | cur_keys = k.split(':') 20 | selected_data = data 21 | for single_key in cur_keys: 22 | selected_data = selected_data[single_key] 23 | ret[k] = selected_data 24 | 25 | return ret 26 | 27 | 28 | def parse_global(data, reterieve_key): 29 | return select_data_from_dict(data, reterieve_key) 30 | 31 | 32 | def parse_one_job(data, reterieve_key): 33 | return select_data_from_dict(data, reterieve_key) 34 | 35 | 36 | ## TODO: Add job specific key 37 | def parse_experiment(filename, global_reterive_key, job_reterive_key): 38 | """ 39 | Parse outputs from one experiment 40 | 41 | Args: 42 | filename (str): _description_ 43 | global_reterive_key (list(str)): _description_ 44 | job_reterive_key (list(str)): _description_ 45 | 46 | Returns: 47 | dict: parsed global results 48 | list(dict): parsed results for each job 49 | """ 50 | f = open(filename, 'r') 51 | 52 | try: 53 | data = json.load(f) 54 | except: 55 | raise (Exception( 56 | 'File {} can not loaded by json.load()'.format(filename))) 57 | f.close() 58 | 59 | num_jobs = len(data['jobs']) 60 | 61 | global_result = parse_global(data['global options'], global_reterive_key) 62 | jobs_result = [] 63 | for job in data['jobs']: 64 | jobs_result.append(parse_one_job(job, job_reterive_key)) 65 | 66 | return global_result, jobs_result 67 | 68 | 69 | # files: {some_key: file_name} 70 | def parse_one_group(files, global_reterive_key, job_reterive_key): 71 | """ 72 | Parse the all experiments from one group 73 | 74 | Args: 75 | dir (str): Dir path to the results folder 76 | files (list(str)): Output filenames 77 | global_reterive_key (list(str)): 78 | job_reterive_key (list(str)): 79 | 80 | Returns: 81 | dict: parsed results for the group 82 | """ 83 | ret = {} 84 | for k in files.keys(): 85 | cur_file_path = files[k] 86 | parsed_output = parse_experiment(cur_file_path, global_reterive_key, 87 | job_reterive_key) 88 | ret[k] = parsed_output 89 | 90 | return ret 91 | 92 | 93 | def parse_all(files, global_reterive_key, job_reterive_key): 94 | """ 95 | Parse output from all groups 96 | 97 | Args: 98 | files (_type_): _description_ 99 | global_reterive_key (_type_): _description_ 100 | job_reterive_key (_type_): _description_ 101 | 102 | Returns: 103 | _type_: _description_ 104 | """ 105 | ret = {} 106 | for group in files.keys(): 107 | group_dir, group_files = files[group] 108 | ret[group] = parse_one_group(group_dir, group_files, 109 | global_reterive_key, job_reterive_key) 110 | 111 | return ret 112 | 113 | 114 | def plot_staked_bar(labels, bar_names, bar_value, title, ylabel, 115 | fig_save_path): 116 | 117 | fig, ax = plt.subplots() 118 | 119 | #plt.title(title) 120 | #plt.ylable(ylabel) 121 | 122 | print('--------') 123 | for cur_bar in bar_names: 124 | print(cur_bar) 125 | print(bar_value[cur_bar]) 126 | ax.bar(labels, bar_value[cur_bar], label=cur_bar) 127 | 128 | plt.legend() 129 | plt.savefig(fig_save_path) 130 | 131 | 132 | def parsed_to_array(reterieve_outputs, get_x_y_label): 133 | """ 134 | Get data from the reterieve keys 135 | 136 | Args: 137 | reterieve_outputs (dict): KV of reterieved values 138 | get_x_y_label (func()): Parse reterieve_outputs and get x, y and label 139 | 140 | Returns: 141 | list : x values 142 | list : y values 143 | list : label values 144 | """ 145 | x = [] 146 | y = [] 147 | std_dev = [] 148 | label = [] 149 | 150 | for k in reterieve_outputs: 151 | global_output = reterieve_outputs[k][0] 152 | jobs_output = reterieve_outputs[k][1] 153 | 154 | cur_x, cur_y, cur_label, cur_std_dev = get_x_y_label( 155 | global_output, jobs_output) 156 | 157 | x.append(cur_x) 158 | y.append(cur_y) 159 | std_dev.append(cur_std_dev) 160 | label.append(cur_label) 161 | 162 | return x, y, std_dev, label 163 | 164 | 165 | def get_all_data(outputs, get_x_y_label): 166 | """ 167 | Get all data from a group 168 | 169 | Args: 170 | outputs (dict): Parsed data 171 | get_x_y_label (func): 172 | 173 | Returns: 174 | dict: (x, y label) for each experiment 175 | """ 176 | ret = {} 177 | for group_name in outputs.keys(): 178 | group_output = outputs[group_name] 179 | group_data = parsed_to_array(group_output, get_x_y_label) 180 | ret[group_name] = group_data 181 | 182 | return ret 183 | 184 | 185 | def parse_spdk_perf(): 186 | pass 187 | 188 | 189 | if __name__ == '__main__': 190 | test_file = '/home/user/test_script/tmp/1/output_iodepth_1.josn' 191 | 192 | global_rk = ["iodepth"] 193 | jobs_rk = ["jobname", "read:slat_ns:mean"] 194 | 195 | gr, jr = parse_experiment(test_file, global_rk, jobs_rk) 196 | print(gr) 197 | print(jr) 198 | -------------------------------------------------------------------------------- /4-micro_arch_qd1/iou.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | iodepth=1 14 | 15 | [job1] 16 | filename=/dev/nvme2n1 -------------------------------------------------------------------------------- /4-micro_arch_qd1/iou_c.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | iodepth=1 14 | hipri 15 | 16 | [job1] 17 | filename=/dev/nvme2n1 -------------------------------------------------------------------------------- /4-micro_arch_qd1/iou_s.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | iodepth=1 14 | sqthread_poll=1 15 | 16 | [job1] 17 | filename=/dev/nvme2n1 -------------------------------------------------------------------------------- /4-micro_arch_qd1/plot.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import numpy as np 3 | import fio 4 | 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | def parse_perf_stat(filename): 9 | """_summary_ 10 | 11 | Args: 12 | filename (_type_): _description_ 13 | 14 | Returns: 15 | dict[name]: (value, stddev) value[int], percent 16 | """ 17 | f = open(filename, 'r') 18 | contents = f.readlines() 19 | f.close() 20 | 21 | ret = {} 22 | data_start = 0 23 | for line in contents: 24 | if line[0] == '#' or line == '\n': 25 | data_start += 1 26 | else: 27 | break 28 | 29 | data_start += 1 30 | contents = contents[data_start:] 31 | data_start = 0 32 | for line in contents: 33 | if line == '\n': 34 | data_start += 1 35 | else: 36 | break 37 | contents = contents[data_start:] 38 | 39 | for line in contents: 40 | # print(line) 41 | if line == '\n': 42 | break 43 | # parse data 44 | splitted_contents = line.split(' ') 45 | splitted_contents = [x for x in splitted_contents if x != ''] 46 | value = splitted_contents[0] 47 | key = splitted_contents[1] 48 | if key[-1] == '\n': 49 | key = key[0:-1] 50 | 51 | stddev = None 52 | # check if stddev exists, if exists, get it. if not exist, return None 53 | last_elem = splitted_contents[-1] 54 | if last_elem[-1] == '\n': 55 | last_elem = last_elem[:-1] 56 | if last_elem == ')': 57 | # stddev exists 58 | stddev = splitted_contents[-2][:-1] 59 | # stddev = float(stddev) 60 | 61 | ret[key] = (value, stddev) 62 | 63 | return ret 64 | 65 | 66 | def load_inst_cycle(filename, sample_length, num_files): 67 | cycles = [] 68 | inst = [] 69 | ipc = [] 70 | 71 | for i in range(num_files): 72 | cur_filename = filename + '_' + str(i) + '.txt' 73 | perf_stat = parse_perf_stat(cur_filename) 74 | 75 | cur_cycle = int(perf_stat['cycles'][0]) / sample_length 76 | cur_inst = int(perf_stat['instructions'][0]) / sample_length 77 | 78 | cycles.append(cur_cycle) 79 | inst.append(cur_inst) 80 | ipc.append(cur_inst / cur_cycle) 81 | 82 | # TODO: Return std or the list directly 83 | cycles_std = np.std(cycles) 84 | inst_std = np.std(inst) 85 | ipc_std = np.std(ipc) 86 | 87 | cycles = np.mean(cycles) 88 | inst = np.mean(inst) 89 | ipc = np.mean(ipc) 90 | 91 | return inst, cycles, ipc, inst_std, cycles_std, ipc_std 92 | 93 | 94 | def load_cache(filename, num_files): 95 | cache_ref = [] 96 | cache_miss = [] 97 | cache_miss_rate = [] 98 | 99 | for i in range(num_files): 100 | cur_filename = filename + '_' + str(i) + '.txt' 101 | perf_stat = parse_perf_stat(cur_filename) 102 | 103 | cur_cache_ref = int(perf_stat['cache-references'][0]) 104 | cur_cache_miss = int(perf_stat['cache-misses'][0]) 105 | cur_cache_miss_rate = cur_cache_miss / cur_cache_ref 106 | 107 | cache_ref.append(cur_cache_ref) 108 | cache_miss.append(cur_cache_miss) 109 | cache_miss_rate.append(cur_cache_miss_rate) 110 | 111 | cache_miss_std = np.std(cache_miss) 112 | cache_ref_std = np.std(cache_ref) 113 | cache_miss_rate_std = np.std(cache_miss_rate) 114 | 115 | cache_ref = np.mean(cache_ref) 116 | cache_miss = np.mean(cache_miss) 117 | cache_miss_rate = np.mean(cache_miss_rate) 118 | 119 | return cache_miss, cache_ref, cache_miss_rate, cache_miss_std, cache_ref_std, cache_miss_rate_std 120 | 121 | 122 | def read_data(engines, dir, inst_sample_length, cs_sample_length, 123 | num_perf_files): 124 | all_inst_per_io = [] 125 | all_inst_per_io_std = [] 126 | all_ipc = [] 127 | all_ipc_std = [] 128 | all_cache_miss_r = [] 129 | all_cache_miss_r_std = [] 130 | 131 | global_rk = [] 132 | jobs_rk = ['read:lat_ns:mean', 'read:iops', 'read:lat_ns:stddev'] 133 | for e in engines: 134 | fname_cycles = os.path.join(dir, e + '_inst_cycle') 135 | fname_cahce = os.path.join(dir, e + '_cache') 136 | 137 | inst, cycle, ipc, inst_std, _, ipc_std = load_inst_cycle( 138 | fname_cycles, inst_sample_length, num_perf_files) 139 | cache_miss, cache_ref, cache_miss_rate, _, _, cache_miss_rate_std = load_cache( 140 | fname_cahce, num_perf_files) 141 | 142 | # multiply all rate by 100 143 | # Divide inst per io by 100 144 | all_ipc.append(ipc) 145 | all_ipc_std.append(ipc_std) 146 | all_cache_miss_r.append(cache_miss_rate * 100) 147 | all_cache_miss_r_std.append(cache_miss_rate_std * 100) 148 | 149 | fname_fio_1 = os.path.join(dir, e + '.txt') 150 | _, j_res = fio.parse_experiment(fname_fio_1, global_rk, jobs_rk) 151 | iops_1 = j_res[0]['read:iops'] 152 | all_inst_per_io.append(inst / iops_1 / 1000) 153 | all_inst_per_io_std.append(inst_std / iops_1 / 1000) 154 | 155 | ret = { 156 | 'all_ipc': all_ipc, 157 | 'all_ipc_std': all_ipc_std, 158 | 'all_cache_miss_r': all_cache_miss_r, 159 | 'all_cache_miss_r_std': all_cache_miss_r_std, 160 | 'all_inst_per_io': all_inst_per_io, 161 | 'all_inst_per_io_std': all_inst_per_io_std, 162 | } 163 | return ret 164 | 165 | 166 | global_dir = './results_global' 167 | local_dir = './results_local' 168 | engines = ['psync', 'aio', 'iou', 'iou_c', 'iou_s', 'spdk_fio'] 169 | inst_sample_length = 5 170 | cs_sample_length = 5 171 | num_perf_files = 10 172 | 173 | global_data = read_data(engines, global_dir, inst_sample_length, 174 | cs_sample_length, num_perf_files) 175 | local_data = read_data(engines, local_dir, inst_sample_length, 176 | cs_sample_length, num_perf_files) 177 | 178 | x_label = 'Engines' 179 | bar_width = 0.4 180 | axis_label_font_size = 28 181 | axis_tick_font_size = 26 182 | legend_font_size = 23 183 | datalabel_size = 24 184 | datalabel_va = 'bottom' #'bottom' 185 | 186 | 187 | def draw_inst_io(system_wide, process_specific, system_wide_std, 188 | process_specific_std, fig_save_path): 189 | group_list = ['system_wide', 'process_specific'] 190 | x_ticks = ['psync', 'aio', 'iou', 'iou-c', 'iou-s', ' spdk-fio'] 191 | value = {'system_wide': system_wide, 'process_specific': process_specific} 192 | std_dev = { 193 | 'system_wide': system_wide_std, 194 | 'process_specific': process_specific_std 195 | } 196 | # std_dev 197 | bar_width = 0.4 198 | fig_save_path = fig_save_path 199 | legend_label = { 200 | 'system_wide': 'system-wide', 201 | 'process_specific': 'process-specific' 202 | } 203 | ylabel = 'Instructions per I/O (K)' 204 | datalabel_va = 'bottom' 205 | 206 | fig, ax = plt.subplots(figsize=(12, 8)) 207 | 208 | plt.ylabel(ylabel, fontsize=axis_label_font_size) 209 | plt.grid(axis='y') 210 | 211 | ax.tick_params(axis='both', which='major', labelsize=axis_tick_font_size) 212 | 213 | if legend_label == None: 214 | legend_label = {} 215 | for group in group_list: 216 | legend_label[group] = group 217 | 218 | x_axis = np.arange(len(x_ticks)) 219 | 220 | # compute bar offset 221 | bar_offset = [] 222 | mid_point = (len(group_list) * bar_width) / 2 223 | 224 | for i in range(len(group_list)): 225 | bar_offset.append(bar_width * i + 0.5 * bar_width - mid_point) 226 | 227 | print(bar_offset) 228 | print(x_axis) 229 | x_axis - 0.1 230 | 231 | for (index, group_name) in zip(range(len(group_list)), group_list): 232 | y = value[group_name] 233 | cur_stddev = std_dev[group_name] 234 | bar_pos = x_axis + bar_offset[index] 235 | plt.bar(bar_pos, 236 | y, 237 | width=bar_width, 238 | label=legend_label[group_name], 239 | yerr=cur_stddev) 240 | 241 | # print(bar_pos, y) 242 | for (x, y, engine) in zip(bar_pos, y, x_ticks): 243 | text = '{:.1f}'.format(y) 244 | if group_name == 'system_wide': 245 | if engine == 'iou-s': 246 | x = x - 0.13 247 | else: 248 | x = x - 0.08 249 | if group_name == 'process_specific' and engine == 'iou-s': 250 | if engine == 'iou-s': 251 | x = x + 0.13 252 | else: 253 | x = x + 0.08 254 | plt.text( 255 | x, y, text, size=datalabel_size, ha='center', va=datalabel_va 256 | ) # 'bottom', 'baseline', 'center', 'center_baseline', 'top' 257 | 258 | plt.xticks(x_axis, x_ticks) 259 | 260 | plt.legend(fontsize=legend_font_size, loc='upper left') 261 | 262 | plt.savefig(fig_save_path, bbox_inches='tight') 263 | 264 | 265 | def draw_cache_miss(system_wide, process_specific, system_wide_std, 266 | process_specific_std, fig_save_path): 267 | group_list = ['system_wide', 'process_specific'] 268 | x_ticks = ['psync', 'aio', 'iou', 'iou-c', 'iou-s', ' spdk-fio'] 269 | value = {'system_wide': system_wide, 'process_specific': process_specific} 270 | std_dev = { 271 | 'system_wide': system_wide_std, 272 | 'process_specific': process_specific_std 273 | } 274 | # std_dev 275 | bar_width = 0.4 276 | fig_save_path = fig_save_path 277 | legend_label = { 278 | 'system_wide': 'system-wide', 279 | 'process_specific': 'process-specific' 280 | } 281 | ylabel = 'Cache miss rate (%)' 282 | datalabel_va = 'bottom' 283 | 284 | fig, ax = plt.subplots(figsize=(12, 8)) 285 | 286 | plt.ylabel(ylabel, fontsize=axis_label_font_size) 287 | plt.grid(axis='y') 288 | 289 | ax.tick_params(axis='both', which='major', labelsize=axis_tick_font_size) 290 | 291 | if legend_label == None: 292 | legend_label = {} 293 | for group in group_list: 294 | legend_label[group] = group 295 | 296 | x_axis = np.arange(len(x_ticks)) 297 | 298 | # compute bar offset 299 | bar_offset = [] 300 | mid_point = (len(group_list) * bar_width) / 2 301 | 302 | for i in range(len(group_list)): 303 | bar_offset.append(bar_width * i + 0.5 * bar_width - mid_point) 304 | 305 | print(bar_offset) 306 | print(x_axis) 307 | x_axis - 0.1 308 | 309 | for (index, group_name) in zip(range(len(group_list)), group_list): 310 | y = value[group_name] 311 | cur_stddev = std_dev[group_name] 312 | bar_pos = x_axis + bar_offset[index] 313 | plt.bar(bar_pos, 314 | y, 315 | width=bar_width, 316 | label=legend_label[group_name], 317 | yerr=cur_stddev) 318 | 319 | for (x, y, engine) in zip(bar_pos, y, x_ticks): 320 | text = '{:.1f}'.format(y) 321 | plt.text( 322 | x, y, text, size=datalabel_size, ha='center', va=datalabel_va 323 | ) # 'bottom', 'baseline', 'center', 'center_baseline', 'top' 324 | 325 | plt.xticks(x_axis, x_ticks) 326 | 327 | if legend_font_size: 328 | plt.legend(fontsize=legend_font_size) 329 | else: 330 | plt.legend() 331 | 332 | plt.savefig(fig_save_path, bbox_inches='tight') 333 | 334 | 335 | def draw_ipc(system_wide, process_specific, system_wide_std, 336 | process_specific_std, fig_save_path): 337 | group_list = ['system_wide', 'process_specific'] 338 | x_ticks = ['psync', 'aio', 'iou', 'iou-c', 'iou-s', ' spdk-fio'] 339 | value = {'system_wide': system_wide, 'process_specific': process_specific} 340 | std_dev = { 341 | 'system_wide': system_wide_std, 342 | 'process_specific': process_specific_std 343 | } 344 | # std_dev 345 | bar_width = 0.4 346 | fig_save_path = fig_save_path 347 | legend_label = { 348 | 'system_wide': 'system-wide', 349 | 'process_specific': 'process-specific' 350 | } 351 | ylabel = 'IPC' 352 | datalabel_va = 'bottom' 353 | 354 | fig, ax = plt.subplots(figsize=(12, 8)) 355 | 356 | plt.ylabel(ylabel, fontsize=axis_label_font_size) 357 | plt.grid(axis='y') 358 | 359 | ax.tick_params(axis='both', which='major', labelsize=axis_tick_font_size) 360 | 361 | if legend_label == None: 362 | legend_label = {} 363 | for group in group_list: 364 | legend_label[group] = group 365 | 366 | x_axis = np.arange(len(x_ticks)) 367 | 368 | # compute bar offset 369 | bar_offset = [] 370 | mid_point = (len(group_list) * bar_width) / 2 371 | 372 | for i in range(len(group_list)): 373 | bar_offset.append(bar_width * i + 0.5 * bar_width - mid_point) 374 | 375 | print(bar_offset) 376 | print(x_axis) 377 | x_axis - 0.1 378 | 379 | for (index, group_name) in zip(range(len(group_list)), group_list): 380 | y = value[group_name] 381 | cur_stddev = std_dev[group_name] 382 | bar_pos = x_axis + bar_offset[index] 383 | plt.bar(bar_pos, 384 | y, 385 | width=bar_width, 386 | label=legend_label[group_name], 387 | yerr=cur_stddev) 388 | 389 | for (x, y, engine) in zip(bar_pos, y, x_ticks): 390 | text = '{:.1f}'.format(y) 391 | if group_name == 'system_wide': 392 | x = x - 0.1 393 | if group_name == 'process_specific' and engine == 'iou-s': 394 | x = x + 0.1 395 | plt.text( 396 | x, y, text, size=datalabel_size, ha='center', va=datalabel_va 397 | ) # 'bottom', 'baseline', 'center', 'center_baseline', 'top' 398 | 399 | plt.xticks(x_axis, x_ticks) 400 | 401 | if legend_font_size: 402 | plt.legend(fontsize=legend_font_size) 403 | else: 404 | plt.legend() 405 | 406 | plt.savefig(fig_save_path, bbox_inches='tight') 407 | 408 | 409 | draw_inst_io(global_data['all_inst_per_io'], local_data['all_inst_per_io'], 410 | global_data['all_inst_per_io_std'], 411 | local_data['all_inst_per_io_std'], 'inst_per_io_d1_qd1.pdf') 412 | draw_ipc(global_data['all_ipc'], local_data['all_ipc'], 413 | global_data['all_ipc_std'], local_data['all_ipc_std'], 414 | 'ipc_d1_qd1.pdf') 415 | draw_cache_miss(global_data['all_cache_miss_r'], 416 | local_data['all_cache_miss_r'], 417 | global_data['all_cache_miss_r_std'], 418 | local_data['all_cache_miss_r_std'], 'cache_miss_d1_qd1.pdf') 419 | -------------------------------------------------------------------------------- /4-micro_arch_qd1/psync.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=psync 12 | thread=1 13 | iodepth=1 14 | 15 | [job1] 16 | 17 | filename=/dev/nvme2n1 -------------------------------------------------------------------------------- /4-micro_arch_qd1/run.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | PERF_INST_SLEEP_TIME=5 4 | PERF_INST_REPEAT_TIME=1 5 | PERF_CACHE_SLEEP_TIME=5 6 | PERF_CACHE_REPEAT_TIME=1 7 | 8 | PERF_WAIT_TIME=10 9 | FIO_RUN_TIME_ALL=200 10 | FIO_RAMP_TIME=20 11 | 12 | 13 | SPDK_FIO_PLUGIN= 14 | SPDK_SETUP_PATH= 15 | 16 | declare -a engine=("psync" "aio" "iou" "iou_s" "iou_c") 17 | declare -a perf_seq=("0" "1" "2" "3" "4" "5" "6" "7" "8" "9") 18 | 19 | # system-wide 20 | 21 | RESULT='results_global' 22 | 23 | $SPDK_SETUP_PATH reset 24 | 25 | for e in "${engine[@]}" 26 | do 27 | numactl -N 1 -m 1 env FIO_RUN_TIME=$FIO_RUN_TIME_ALL FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --output-format=json -o $RESULT/${e}.txt & 28 | sleep $PERF_WAIT_TIME; 29 | for i in "${perf_seq[@]}" 30 | do 31 | perf stat -e cycles,instructions -a -r $PERF_INST_REPEAT_TIME -o $RESULT/${e}_inst_cycle_${i}.txt -- sleep $PERF_INST_SLEEP_TIME ; 32 | perf stat -e cache-misses,cache-references -a -r $PERF_CACHE_REPEAT_TIME -o $RESULT/${e}_cache_${i}.txt -- sleep $PERF_CACHE_SLEEP_TIME; 33 | done 34 | wait; 35 | done 36 | 37 | $SPDK_SETUP_PATH 38 | e="spdk_fio" 39 | numactl -N 1 -m 1 env FIO_RUN_TIME=$FIO_RUN_TIME_ALL FIO_RAMP_TIME=$FIO_RAMP_TIME LD_PRELOAD=$SPDK_FIO_PLUGIN fio ${e}.conf --output-format=json -o $RESULT/${e}.txt & 40 | sleep $PERF_WAIT_TIME; 41 | for i in "${perf_seq[@]}" 42 | do 43 | perf stat -e cycles,instructions -a -r $PERF_INST_REPEAT_TIME -o $RESULT/${e}_inst_cycle_${i}.txt -- sleep $PERF_INST_SLEEP_TIME ; 44 | perf stat -e cache-misses,cache-references -a -r $PERF_CACHE_REPEAT_TIME -o $RESULT/${e}_cache_${i}.txt -- sleep $PERF_CACHE_SLEEP_TIME; 45 | done 46 | wait 47 | 48 | $SPDK_SETUP_PATH reset 49 | 50 | # process specific 51 | 52 | RESULT='results_local' 53 | 54 | for e in "${engine[@]}" 55 | do 56 | 57 | numactl -N 1 -m 1 env FIO_RUN_TIME=$FIO_RUN_TIME_ALL FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --output-format=json -o $RESULT/${e}.txt & 58 | sleep $PERF_WAIT_TIME; 59 | for i in "${perf_seq[@]}" 60 | do 61 | perf stat -e cycles,instructions -p `pidof fio` -r $PERF_INST_REPEAT_TIME -o $RESULT/${e}_inst_cycle_${i}.txt -- sleep $PERF_INST_SLEEP_TIME ; 62 | perf stat -e cache-misses,cache-references -p `pidof fio` -r $PERF_CACHE_REPEAT_TIME -o $RESULT/${e}_cache_${i}.txt -- sleep $PERF_CACHE_SLEEP_TIME; 63 | done 64 | wait; 65 | done 66 | 67 | $SPDK_SETUP_PATH 68 | 69 | e="spdk_fio" 70 | 71 | numactl -N 1 -m 1 env FIO_RUN_TIME=$FIO_RUN_TIME_ALL FIO_RAMP_TIME=$FIO_RAMP_TIME LD_PRELOAD=$SPDK_FIO_PLUGIN fio ${e}.conf --output-format=json -o $RESULT/${e}.txt & 72 | sleep $PERF_WAIT_TIME; 73 | for i in "${perf_seq[@]}" 74 | do 75 | perf stat -e cycles,instructions -p `pidof fio` -r $PERF_INST_REPEAT_TIME -o $RESULT/${e}_inst_cycle_${i}.txt -- sleep $PERF_INST_SLEEP_TIME ; 76 | perf stat -e cache-misses,cache-references -p `pidof fio` -r $PERF_CACHE_REPEAT_TIME -o $RESULT/${e}_cache_${i}.txt -- sleep $PERF_CACHE_SLEEP_TIME; 77 | done 78 | wait 79 | 80 | $SPDK_SETUP_PATH 81 | -------------------------------------------------------------------------------- /4-micro_arch_qd1/spdk_fio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | filename=trtype=PCIe traddr=0000.b1.00.0 ns=1 12 | ioengine=spdk 13 | thread=1 14 | iodepth=1 15 | 16 | [job1] 17 | -------------------------------------------------------------------------------- /5-a-inc_qd_dev_1/aio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=libaio 12 | thread=1 13 | 14 | [job1] 15 | 16 | filename=/dev/nvme0n1 17 | 18 | -------------------------------------------------------------------------------- /5-a-inc_qd_dev_1/fio.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import json 3 | 4 | import matplotlib.pyplot as plt 5 | 6 | 7 | def select_data_from_dict(data, reterieve_key): 8 | ''' 9 | Select data from dict using given keys 10 | 11 | Args: 12 | data(dict): 13 | reterieve_key(list): 14 | Returns: 15 | ''' 16 | ret = {} 17 | 18 | for k in reterieve_key: 19 | cur_keys = k.split(':') 20 | selected_data = data 21 | for single_key in cur_keys: 22 | selected_data = selected_data[single_key] 23 | ret[k] = selected_data 24 | 25 | return ret 26 | 27 | 28 | def parse_global(data, reterieve_key): 29 | return select_data_from_dict(data, reterieve_key) 30 | 31 | 32 | def parse_one_job(data, reterieve_key): 33 | return select_data_from_dict(data, reterieve_key) 34 | 35 | 36 | ## TODO: Add job specific key 37 | def parse_experiment(filename, global_reterive_key, job_reterive_key): 38 | """ 39 | Parse outputs from one experiment 40 | 41 | Args: 42 | filename (str): _description_ 43 | global_reterive_key (list(str)): _description_ 44 | job_reterive_key (list(str)): _description_ 45 | 46 | Returns: 47 | dict: parsed global results 48 | list(dict): parsed results for each job 49 | """ 50 | f = open(filename, 'r') 51 | 52 | try: 53 | data = json.load(f) 54 | except: 55 | raise (Exception( 56 | 'File {} can not loaded by json.load()'.format(filename))) 57 | f.close() 58 | 59 | num_jobs = len(data['jobs']) 60 | 61 | global_result = parse_global(data['global options'], global_reterive_key) 62 | jobs_result = [] 63 | for job in data['jobs']: 64 | jobs_result.append(parse_one_job(job, job_reterive_key)) 65 | 66 | return global_result, jobs_result 67 | 68 | 69 | # files: {some_key: file_name} 70 | def parse_one_group(files, global_reterive_key, job_reterive_key): 71 | """ 72 | Parse the all experiments from one group 73 | 74 | Args: 75 | dir (str): Dir path to the results folder 76 | files (list(str)): Output filenames 77 | global_reterive_key (list(str)): 78 | job_reterive_key (list(str)): 79 | 80 | Returns: 81 | dict: parsed results for the group 82 | """ 83 | ret = {} 84 | for k in files.keys(): 85 | cur_file_path = files[k] 86 | parsed_output = parse_experiment(cur_file_path, global_reterive_key, 87 | job_reterive_key) 88 | ret[k] = parsed_output 89 | 90 | return ret 91 | 92 | 93 | def parse_all(files, global_reterive_key, job_reterive_key): 94 | """ 95 | Parse output from all groups 96 | 97 | Args: 98 | files (_type_): _description_ 99 | global_reterive_key (_type_): _description_ 100 | job_reterive_key (_type_): _description_ 101 | 102 | Returns: 103 | _type_: _description_ 104 | """ 105 | ret = {} 106 | for group in files.keys(): 107 | group_dir, group_files = files[group] 108 | ret[group] = parse_one_group(group_dir, group_files, 109 | global_reterive_key, job_reterive_key) 110 | 111 | return ret 112 | 113 | 114 | def plot_staked_bar(labels, bar_names, bar_value, title, ylabel, 115 | fig_save_path): 116 | 117 | fig, ax = plt.subplots() 118 | 119 | #plt.title(title) 120 | #plt.ylable(ylabel) 121 | 122 | print('--------') 123 | for cur_bar in bar_names: 124 | print(cur_bar) 125 | print(bar_value[cur_bar]) 126 | ax.bar(labels, bar_value[cur_bar], label=cur_bar) 127 | 128 | plt.legend() 129 | plt.savefig(fig_save_path) 130 | 131 | 132 | def parsed_to_array(reterieve_outputs, get_x_y_label): 133 | """ 134 | Get data from the reterieve keys 135 | 136 | Args: 137 | reterieve_outputs (dict): KV of reterieved values 138 | get_x_y_label (func()): Parse reterieve_outputs and get x, y and label 139 | 140 | Returns: 141 | list : x values 142 | list : y values 143 | list : label values 144 | """ 145 | x = [] 146 | y = [] 147 | std_dev = [] 148 | label = [] 149 | 150 | for k in reterieve_outputs: 151 | global_output = reterieve_outputs[k][0] 152 | jobs_output = reterieve_outputs[k][1] 153 | 154 | cur_x, cur_y, cur_label, cur_std_dev = get_x_y_label( 155 | global_output, jobs_output) 156 | 157 | x.append(cur_x) 158 | y.append(cur_y) 159 | std_dev.append(cur_std_dev) 160 | label.append(cur_label) 161 | 162 | return x, y, std_dev, label 163 | 164 | 165 | def get_all_data(outputs, get_x_y_label): 166 | """ 167 | Get all data from a group 168 | 169 | Args: 170 | outputs (dict): Parsed data 171 | get_x_y_label (func): 172 | 173 | Returns: 174 | dict: (x, y label) for each experiment 175 | """ 176 | ret = {} 177 | for group_name in outputs.keys(): 178 | group_output = outputs[group_name] 179 | group_data = parsed_to_array(group_output, get_x_y_label) 180 | ret[group_name] = group_data 181 | 182 | return ret 183 | 184 | 185 | def parse_spdk_perf(): 186 | pass 187 | 188 | 189 | if __name__ == '__main__': 190 | test_file = '/home/user/test_script/tmp/1/output_iodepth_1.josn' 191 | 192 | global_rk = ["iodepth"] 193 | jobs_rk = ["jobname", "read:slat_ns:mean"] 194 | 195 | gr, jr = parse_experiment(test_file, global_rk, jobs_rk) 196 | print(gr) 197 | print(jr) 198 | -------------------------------------------------------------------------------- /5-a-inc_qd_dev_1/iou.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=10 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | 14 | [job1] 15 | filename=/dev/nvme0n1 -------------------------------------------------------------------------------- /5-a-inc_qd_dev_1/iou_c.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | hipri 14 | 15 | [job1] 16 | filename=/dev/nvme0n1 -------------------------------------------------------------------------------- /5-a-inc_qd_dev_1/iou_s.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=10 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | sqthread_poll=1 14 | 15 | 16 | [job1] 17 | filename=/dev/nvme0n1 -------------------------------------------------------------------------------- /5-a-inc_qd_dev_1/plot.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import os.path 3 | import fio 4 | 5 | matplotlib_colors = [ 6 | 'blue', 'green', 'red', 'cyan', 'magenta', 'yellw', 'white' 7 | ] 8 | 9 | dot_style = [ 10 | '+', 11 | 'X', 12 | 'o', 13 | 'v', 14 | 's', 15 | 'P', 16 | ] 17 | 18 | dot_index = 0 19 | 20 | 21 | def get_dot(): 22 | global dot_index 23 | ret_styl = dot_style[dot_index] 24 | dot_index += 1 25 | return ret_styl 26 | 27 | 28 | dir = './results' 29 | global_rk = ["iodepth"] 30 | jobs_rk = ["jobname", "read:lat_ns:mean", 'read:iops', 'read:lat_ns:stddev'] 31 | 32 | qd = [1, 2, 4, 8, 16, 32, 64, 128] 33 | 34 | iops = [] 35 | lat = [] 36 | lat_stdev = [] 37 | 38 | engines = ['aio', 'iou', 'iou_c', 'iou_s', 'spdk_fio'] 39 | iops_data = {} 40 | lat_data = {} 41 | lat_std_data = {} 42 | 43 | for e in engines: 44 | iops_data[e] = [] 45 | lat_data[e] = [] 46 | lat_std_data[e] = [] 47 | for cur_qd in qd: 48 | filename = os.path.join(dir, e + '_qd_' + str(cur_qd) + '.txt') 49 | _, j_res = fio.parse_experiment(filename, global_rk, jobs_rk) 50 | cur_lat = j_res[0]['read:lat_ns:mean'] / 1000 # us 51 | cur_iops = j_res[0]['read:iops'] / 1000 #kiops 52 | cur_lat_std = j_res[0]['read:lat_ns:stddev'] / 1000 53 | iops_data[e].append(cur_iops) 54 | lat_data[e].append(cur_lat) 55 | lat_std_data[e].append(cur_lat_std) 56 | 57 | title = None 58 | xlabel = None 59 | ylabel = 'Latency (us)' 60 | group_list = ['aio', 'iou', 'iou_p_s', 'iou_p_c', 'spdk'] 61 | fig_save_path = 'iops_lat_d1.pdf' 62 | group_label = { 63 | 'aio': 'aio', 64 | 'iou': 'iou', 65 | 'iou_s': 'iou-s', 66 | 'iou_c': 'iou-c', 67 | 'spdk_fio': 'spdk' 68 | } 69 | 70 | axis_label_font_size = 28 71 | axis_tick_font_size = 28 72 | legend_font_size = 26 73 | linewidth = 4 74 | datalabel_size = 18 75 | markersize = 15 76 | 77 | fig, ax = plt.subplots(figsize=(12, 8)) 78 | plt.xlabel(xlabel, fontsize=axis_label_font_size) 79 | plt.ylabel(ylabel, fontsize=axis_label_font_size) 80 | plt.grid(True) 81 | 82 | if axis_tick_font_size: 83 | ax.tick_params(axis='both', which='major', labelsize=axis_tick_font_size) 84 | 85 | for e in engines: 86 | x = iops_data[e] 87 | y = lat_data[e] 88 | stddev = lat_std_data[e] 89 | plt.errorbar( 90 | x, 91 | y, 92 | yerr=stddev, 93 | label=group_label[e], 94 | marker=get_dot(), 95 | linewidth=linewidth, 96 | markersize=markersize, 97 | ) 98 | for i in range(len(qd)): 99 | ax.text(x[i], y[i], qd[i], size=datalabel_size) 100 | 101 | plt.legend(fontsize=legend_font_size) 102 | 103 | plt.savefig(fig_save_path, bbox_inches='tight') 104 | -------------------------------------------------------------------------------- /5-a-inc_qd_dev_1/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # NOTE: DISABLE other cpu when running this experiment 3 | 4 | SPDK_FIO_PLUGIN= 5 | SPDK_SETUP_PATH= 6 | 7 | FIO_RUN_TIME=120 8 | FIO_RAMP_TIME=20 9 | 10 | declare -a engine=("aio" "iou" "iou_s" "iou_c") 11 | declare -a qd=("1" "2" "4" "8" "16" "32" "64" "128") 12 | 13 | RESULT='results' 14 | 15 | $SPDK_SETUP_PATH reset; 16 | 17 | for e in "${engine[@]}" 18 | do 19 | for q in "${qd[@]}" 20 | do 21 | $NUMA_PIN env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio --iodepth=$q ${e}.conf --output-format=json -o $RESULT/${e}_qd_${q}.txt; 22 | done 23 | done 24 | 25 | $SPDK_SETUP_PATH; 26 | 27 | e='spdk_fio' 28 | for q in "${qd[@]}" 29 | do 30 | $NUMA_PIN env LD_PRELOAD=$SPDK_FIO_PLUGIN FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio --iodepth=$q ${e}.conf --output-format=json -o $RESULT/${e}_qd_${q}.txt; 31 | done 32 | 33 | $SPDK_SETUP_PATH reset; -------------------------------------------------------------------------------- /5-a-inc_qd_dev_1/spdk_fio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | filename=trtype=PCIe traddr=0000.af.00.0 ns=1 12 | ioengine=spdk 13 | thread=1 14 | 15 | [job1] 16 | -------------------------------------------------------------------------------- /5-b-inc_qd_dev_7/aio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=libaio 12 | thread=1 13 | 14 | [job1] 15 | 16 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 17 | -------------------------------------------------------------------------------- /5-b-inc_qd_dev_7/fio.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import json 3 | 4 | import matplotlib.pyplot as plt 5 | 6 | 7 | def select_data_from_dict(data, reterieve_key): 8 | ''' 9 | Select data from dict using given keys 10 | 11 | Args: 12 | data(dict): 13 | reterieve_key(list): 14 | Returns: 15 | ''' 16 | ret = {} 17 | 18 | for k in reterieve_key: 19 | cur_keys = k.split(':') 20 | selected_data = data 21 | for single_key in cur_keys: 22 | selected_data = selected_data[single_key] 23 | ret[k] = selected_data 24 | 25 | return ret 26 | 27 | 28 | def parse_global(data, reterieve_key): 29 | return select_data_from_dict(data, reterieve_key) 30 | 31 | 32 | def parse_one_job(data, reterieve_key): 33 | return select_data_from_dict(data, reterieve_key) 34 | 35 | 36 | ## TODO: Add job specific key 37 | def parse_experiment(filename, global_reterive_key, job_reterive_key): 38 | """ 39 | Parse outputs from one experiment 40 | 41 | Args: 42 | filename (str): _description_ 43 | global_reterive_key (list(str)): _description_ 44 | job_reterive_key (list(str)): _description_ 45 | 46 | Returns: 47 | dict: parsed global results 48 | list(dict): parsed results for each job 49 | """ 50 | f = open(filename, 'r') 51 | 52 | try: 53 | data = json.load(f) 54 | except: 55 | raise (Exception( 56 | 'File {} can not loaded by json.load()'.format(filename))) 57 | f.close() 58 | 59 | num_jobs = len(data['jobs']) 60 | 61 | global_result = parse_global(data['global options'], global_reterive_key) 62 | jobs_result = [] 63 | for job in data['jobs']: 64 | jobs_result.append(parse_one_job(job, job_reterive_key)) 65 | 66 | return global_result, jobs_result 67 | 68 | 69 | # files: {some_key: file_name} 70 | def parse_one_group(files, global_reterive_key, job_reterive_key): 71 | """ 72 | Parse the all experiments from one group 73 | 74 | Args: 75 | dir (str): Dir path to the results folder 76 | files (list(str)): Output filenames 77 | global_reterive_key (list(str)): 78 | job_reterive_key (list(str)): 79 | 80 | Returns: 81 | dict: parsed results for the group 82 | """ 83 | ret = {} 84 | for k in files.keys(): 85 | cur_file_path = files[k] 86 | parsed_output = parse_experiment(cur_file_path, global_reterive_key, 87 | job_reterive_key) 88 | ret[k] = parsed_output 89 | 90 | return ret 91 | 92 | 93 | def parse_all(files, global_reterive_key, job_reterive_key): 94 | """ 95 | Parse output from all groups 96 | 97 | Args: 98 | files (_type_): _description_ 99 | global_reterive_key (_type_): _description_ 100 | job_reterive_key (_type_): _description_ 101 | 102 | Returns: 103 | _type_: _description_ 104 | """ 105 | ret = {} 106 | for group in files.keys(): 107 | group_dir, group_files = files[group] 108 | ret[group] = parse_one_group(group_dir, group_files, 109 | global_reterive_key, job_reterive_key) 110 | 111 | return ret 112 | 113 | 114 | def plot_staked_bar(labels, bar_names, bar_value, title, ylabel, 115 | fig_save_path): 116 | 117 | fig, ax = plt.subplots() 118 | 119 | #plt.title(title) 120 | #plt.ylable(ylabel) 121 | 122 | print('--------') 123 | for cur_bar in bar_names: 124 | print(cur_bar) 125 | print(bar_value[cur_bar]) 126 | ax.bar(labels, bar_value[cur_bar], label=cur_bar) 127 | 128 | plt.legend() 129 | plt.savefig(fig_save_path) 130 | 131 | 132 | def parsed_to_array(reterieve_outputs, get_x_y_label): 133 | """ 134 | Get data from the reterieve keys 135 | 136 | Args: 137 | reterieve_outputs (dict): KV of reterieved values 138 | get_x_y_label (func()): Parse reterieve_outputs and get x, y and label 139 | 140 | Returns: 141 | list : x values 142 | list : y values 143 | list : label values 144 | """ 145 | x = [] 146 | y = [] 147 | std_dev = [] 148 | label = [] 149 | 150 | for k in reterieve_outputs: 151 | global_output = reterieve_outputs[k][0] 152 | jobs_output = reterieve_outputs[k][1] 153 | 154 | cur_x, cur_y, cur_label, cur_std_dev = get_x_y_label( 155 | global_output, jobs_output) 156 | 157 | x.append(cur_x) 158 | y.append(cur_y) 159 | std_dev.append(cur_std_dev) 160 | label.append(cur_label) 161 | 162 | return x, y, std_dev, label 163 | 164 | 165 | def get_all_data(outputs, get_x_y_label): 166 | """ 167 | Get all data from a group 168 | 169 | Args: 170 | outputs (dict): Parsed data 171 | get_x_y_label (func): 172 | 173 | Returns: 174 | dict: (x, y label) for each experiment 175 | """ 176 | ret = {} 177 | for group_name in outputs.keys(): 178 | group_output = outputs[group_name] 179 | group_data = parsed_to_array(group_output, get_x_y_label) 180 | ret[group_name] = group_data 181 | 182 | return ret 183 | 184 | 185 | def parse_spdk_perf(): 186 | pass 187 | 188 | 189 | if __name__ == '__main__': 190 | test_file = '/home/user/test_script/tmp/1/output_iodepth_1.josn' 191 | 192 | global_rk = ["iodepth"] 193 | jobs_rk = ["jobname", "read:slat_ns:mean"] 194 | 195 | gr, jr = parse_experiment(test_file, global_rk, jobs_rk) 196 | print(gr) 197 | print(jr) 198 | -------------------------------------------------------------------------------- /5-b-inc_qd_dev_7/iou.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=10 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | 14 | [job1] 15 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 -------------------------------------------------------------------------------- /5-b-inc_qd_dev_7/iou_c.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | hipri 14 | 15 | [job1] 16 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 -------------------------------------------------------------------------------- /5-b-inc_qd_dev_7/iou_s.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=10 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | sqthread_poll=1 14 | 15 | 16 | [job1] 17 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 -------------------------------------------------------------------------------- /5-b-inc_qd_dev_7/plot.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import os.path 3 | import fio 4 | 5 | matplotlib_colors = [ 6 | 'blue', 'green', 'red', 'cyan', 'magenta', 'yellw', 'white' 7 | ] 8 | 9 | dot_style = [ 10 | '+', 11 | 'X', 12 | 'o', 13 | 'v', 14 | 's', 15 | 'P', 16 | ] 17 | 18 | dot_index = 0 19 | 20 | 21 | def get_dot(): 22 | global dot_index 23 | ret_styl = dot_style[dot_index] 24 | dot_index += 1 25 | return ret_styl 26 | 27 | 28 | dir = './results' 29 | global_rk = ["iodepth"] 30 | jobs_rk = ["jobname", "read:lat_ns:mean", 'read:iops', 'read:lat_ns:stddev'] 31 | 32 | qd = [1, 2, 4, 8, 16, 32, 64, 128] 33 | 34 | iops = [] 35 | lat = [] 36 | lat_stdev = [] 37 | 38 | engines = ['aio', 'iou', 'iou_c', 'iou_s', 'spdk_fio'] 39 | iops_data = {} 40 | lat_data = {} 41 | lat_std_data = {} 42 | 43 | for e in engines: 44 | iops_data[e] = [] 45 | lat_data[e] = [] 46 | lat_std_data[e] = [] 47 | for cur_qd in qd: 48 | filename = os.path.join(dir, e + '_qd_' + str(cur_qd) + '.txt') 49 | _, j_res = fio.parse_experiment(filename, global_rk, jobs_rk) 50 | cur_lat = j_res[0]['read:lat_ns:mean'] / 1000 # us 51 | cur_iops = j_res[0]['read:iops'] / 1000 #kiops 52 | cur_lat_std = j_res[0]['read:lat_ns:stddev'] / 1000 53 | iops_data[e].append(cur_iops) 54 | lat_data[e].append(cur_lat) 55 | lat_std_data[e].append(cur_lat_std) 56 | 57 | title = None 58 | xlabel = None 59 | ylabel = 'Latency (us)' 60 | group_list = ['aio', 'iou', 'iou_p_s', 'iou_p_c', 'spdk'] 61 | fig_save_path = 'iops_lat_d1.pdf' 62 | group_label = { 63 | 'aio': 'aio', 64 | 'iou': 'iou', 65 | 'iou_s': 'iou-s', 66 | 'iou_c': 'iou-c', 67 | 'spdk_fio': 'spdk' 68 | } 69 | 70 | axis_label_font_size = 28 71 | axis_tick_font_size = 28 72 | legend_font_size = 26 73 | linewidth = 4 74 | datalabel_size = 18 75 | markersize = 15 76 | 77 | fig, ax = plt.subplots(figsize=(12, 8)) 78 | plt.xlabel(xlabel, fontsize=axis_label_font_size) 79 | plt.ylabel(ylabel, fontsize=axis_label_font_size) 80 | plt.grid(True) 81 | 82 | if axis_tick_font_size: 83 | ax.tick_params(axis='both', which='major', labelsize=axis_tick_font_size) 84 | 85 | for e in engines: 86 | x = iops_data[e] 87 | y = lat_data[e] 88 | stddev = lat_std_data[e] 89 | plt.errorbar( 90 | x, 91 | y, 92 | yerr=stddev, 93 | label=group_label[e], 94 | marker=get_dot(), 95 | linewidth=linewidth, 96 | markersize=markersize, 97 | ) 98 | for i in range(len(qd)): 99 | ax.text(x[i], y[i], qd[i], size=datalabel_size) 100 | 101 | plt.legend(fontsize=legend_font_size) 102 | 103 | plt.savefig(fig_save_path, bbox_inches='tight') 104 | -------------------------------------------------------------------------------- /5-b-inc_qd_dev_7/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # NOTE: DISABLE other cpu when running this experiment 3 | 4 | SPDK_FIO_PLUGIN= 5 | SPDK_SETUP_PATH= 6 | 7 | FIO_RUN_TIME=120 8 | FIO_RAMP_TIME=20 9 | 10 | declare -a engine=("aio" "iou" "iou_s" "iou_c") 11 | declare -a qd=("1" "2" "4" "8" "16" "32" "64" "128") 12 | 13 | RESULT='results' 14 | 15 | $SPDK_SETUP_PATH reset; 16 | 17 | for e in "${engine[@]}" 18 | do 19 | for q in "${qd[@]}" 20 | do 21 | $NUMA_PIN env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio --iodepth=$q ${e}.conf --output-format=json -o $RESULT/${e}_qd_${q}.txt; 22 | done 23 | done 24 | 25 | $SPDK_SETUP_PATH; 26 | 27 | e='spdk_fio' 28 | for q in "${qd[@]}" 29 | do 30 | $NUMA_PIN env LD_PRELOAD=$SPDK_FIO_PLUGIN FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio --iodepth=$q ${e}.conf --output-format=json -o $RESULT/${e}_qd_${q}.txt; 31 | done 32 | 33 | $SPDK_SETUP_PATH reset; -------------------------------------------------------------------------------- /5-b-inc_qd_dev_7/spdk_fio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | filename=trtype=PCIe traddr=0000.af.00.0 ns=1:traddr=0000.b0.00.0 ns=1:traddr=0000.b1.00.0 ns=1:traddr=0000.b2.00.0 ns=1:traddr=0000.d8.00.0 ns=1:traddr=0000.da.00.0 ns=1:traddr=0000.db.00.0 ns=1 12 | ioengine=spdk 13 | thread=1 14 | 15 | [job1] 16 | -------------------------------------------------------------------------------- /5-c-work_breakdown_s_point/aio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=libaio 12 | thread=1 13 | iodepth=8 14 | 15 | [job1] 16 | 17 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 18 | 19 | -------------------------------------------------------------------------------- /5-c-work_breakdown_s_point/clean.sh: -------------------------------------------------------------------------------- 1 | rm -f ./fio/* ./perf_output/* ./perf_list/* ./perf_graph/* ./flamegraph/* 2 | -------------------------------------------------------------------------------- /5-c-work_breakdown_s_point/iou.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | iodepth=8 14 | 15 | [job1] 16 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 17 | -------------------------------------------------------------------------------- /5-c-work_breakdown_s_point/iou_c.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | iodepth=8 14 | hipri 15 | 16 | [job1] 17 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 18 | 19 | -------------------------------------------------------------------------------- /5-c-work_breakdown_s_point/iou_s.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | iodepth=64 14 | sqthread_poll=1 15 | 16 | 17 | [job1] 18 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 19 | 20 | -------------------------------------------------------------------------------- /5-c-work_breakdown_s_point/plot.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import matplotlib.pyplot as plt 3 | 4 | from process_report_graph import parse_report_file 5 | 6 | legend_name = { 7 | 'app': 'fio', 8 | 'interface': 'I/O lib', 9 | 'block': 'block_layer', 10 | 'driver': 'nvme driver', 11 | 'sys': 'kernel' 12 | } 13 | 14 | legend_name = { 15 | 'app': 'fio', 16 | 'interface': 'I/O lib', 17 | 'block': 'block_layer', 18 | 'driver': 'nvme driver', 19 | 'sys': 'kernel' 20 | } 21 | 22 | CB_color_cycle = [ 23 | '#377eb8', '#ff7f00', '#4daf4a', '#f781bf', '#a65628', '#984ea3', 24 | '#999999', '#e41a1c', '#dede00' 25 | ] 26 | 27 | color_index = 0 28 | 29 | 30 | def get_color(): 31 | global color_index 32 | prev_c = color_index 33 | color_index += 1 34 | color_index %= len(CB_color_cycle) 35 | return CB_color_cycle[prev_c] 36 | 37 | 38 | def plot(labels, 39 | bar_names, 40 | bar_value, 41 | title, 42 | xlabel, 43 | ylabel, 44 | fig_save_path, 45 | axis_label_font_size=None, 46 | axis_tick_font_size=None, 47 | legend_font_size=None, 48 | datalabel_size=None, 49 | datalabel_va=None): 50 | 51 | fig, ax = plt.subplots() 52 | plt.grid(True) 53 | if title: 54 | plt.title(title) 55 | if axis_label_font_size: 56 | plt.xlabel(xlabel, fontsize=axis_label_font_size) 57 | plt.ylabel(ylabel, fontsize=axis_label_font_size) 58 | else: 59 | plt.xlabel(xlabel) 60 | plt.ylabel(ylabel) 61 | 62 | if axis_tick_font_size: 63 | ax.tick_params(axis='both', 64 | which='major', 65 | labelsize=axis_tick_font_size) 66 | 67 | print('--------') 68 | bottom = [0] * len(labels) 69 | for index, cur_bar in zip(range(len(bar_names)), bar_names): 70 | # print(cur_bar) 71 | # print(bar_value[cur_bar]) 72 | cur_ax = ax.bar( 73 | labels, 74 | bar_value[cur_bar], 75 | label=legend_name[cur_bar], 76 | bottom=bottom, 77 | #width=0.3, 78 | #fill=False, 79 | edgecolor='black', 80 | color=get_color() 81 | #hatch=hatch[index % len(bar_names)] 82 | ) 83 | 84 | cur_bar_label = bar_value[cur_bar] 85 | cur_bar_label = ["{:.2f}".format(x) for x in cur_bar_label] 86 | ax.bar_label(cur_ax, cur_bar_label, label_type='center', size=14) 87 | for i in range(len(bottom)): 88 | bottom[i] += bar_value[cur_bar][i] 89 | 90 | plt.legend(loc='upper center', 91 | bbox_to_anchor=(0.5, 1.22), 92 | ncol=3, 93 | fancybox=True, 94 | shadow=True, 95 | fontsize=14) 96 | 97 | #plt.legend() 98 | plt.savefig(fig_save_path, bbox_inches="tight") 99 | 100 | 101 | engines = ["aio", "iou", "iou_c", "iou_s"] #, "spdk_fio_s"] 102 | result_path = 'perf_graph' 103 | 104 | data = [] 105 | for e in engines: 106 | fname = os.path.join(result_path, 'perf_parsed_' + e + '_g.txt') 107 | res = parse_report_file(fname) 108 | res = [i * 100 for i in res] 109 | data.append(res) 110 | 111 | # TODO: PLEASE CLASSIFY THE DATA BY YOURSELF 112 | data.append([73, 0, 0, 27, 0]) 113 | 114 | labels = ["aio", "iou", "iou-c", "iou-s", "spdk-fio"] 115 | bar_names = ['app', 'interface', 'block', 'driver', 'sys'] 116 | 117 | bar_value = {'app': [], 'interface': [], 'block': [], 'driver': [], 'sys': []} 118 | 119 | for split in data: 120 | for (index, cost_class) in enumerate(bar_names): 121 | bar_value[cost_class].append(split[index]) 122 | 123 | print(bar_value) 124 | 125 | title = None 126 | xlabel = 'Engines' 127 | ylabel = 'Percentage' 128 | fig_save_path = 'work_breakdown_s_point.pdf' 129 | axis_label_font_size = 20 130 | axis_tick_font_size = 16 131 | legend_font_size = 18 132 | datalabel_size = 20 133 | datalabel_va = 'bottom' 134 | plot(labels, 135 | bar_names, 136 | bar_value, 137 | title, 138 | xlabel, 139 | ylabel, 140 | fig_save_path, 141 | axis_tick_font_size=axis_tick_font_size, 142 | axis_label_font_size=axis_label_font_size, 143 | legend_font_size=legend_font_size, 144 | datalabel_size=datalabel_size, 145 | datalabel_va=datalabel_va) 146 | -------------------------------------------------------------------------------- /5-c-work_breakdown_s_point/process_report_graph.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import sys 3 | 4 | 5 | def read_file(filename): 6 | f = open(filename, 'r') 7 | lines = f.readlines() 8 | f.close() 9 | 10 | # remove all leading spaces 11 | ret = [] 12 | for l in lines: 13 | if l[0] == '#': 14 | continue 15 | ret.append(l[:-1]) 16 | 17 | return ret 18 | 19 | 20 | def parse_overhead(overhead, format): 21 | if format == 'percent': 22 | return float(overhead[:-1]) 23 | elif format == 'number': 24 | return int(overhead) 25 | else: 26 | assert 0, 'format {} not recognized, should be percent or number'.format( 27 | format) 28 | 29 | 30 | def find_top_layer(file_contents, symbol): 31 | for l in file_contents: 32 | if l != '' and l[0] != '|': 33 | pass 34 | 35 | 36 | def elicit_children(file_contents): 37 | children = [] 38 | 39 | cur_children = [] 40 | for l in file_contents: 41 | if l[1:].strip() == '': 42 | children.append(cur_children) 43 | cur_children = [] 44 | else: 45 | cur_children.append(l) 46 | 47 | if len(cur_children) != 0: 48 | children.append(cur_children) 49 | 50 | return children 51 | 52 | 53 | def reshape_child(file_contents): 54 | line_has_content = 0 55 | for l in file_contents: 56 | if l.strip() == '|': 57 | line_has_content += 1 58 | else: 59 | break 60 | 61 | file_contents = file_contents[line_has_content:] 62 | 63 | if len(file_contents) == 0: 64 | return [] 65 | 66 | processed_child = [] 67 | 68 | num_spaces = 0 69 | for c in file_contents[0]: 70 | if c == ' ' or c == '|': 71 | num_spaces += 1 72 | else: 73 | break 74 | num_spaces -= 1 75 | for l in file_contents: 76 | processed_child.append(l[num_spaces:]) 77 | 78 | return processed_child 79 | 80 | 81 | def process_child(file_contents, overhead_format='number'): 82 | """_summary_ 83 | 84 | Args: 85 | file_contents (_type_): _description_ 86 | overhead_type (str, optional): _description_. Defaults to 'number'. percent or number 87 | 88 | Returns: 89 | _type_: _description_ 90 | """ 91 | title = file_contents[0] 92 | file_contents = file_contents[1:] 93 | 94 | # process title 95 | if title[0] == '|': 96 | title = title[1:] 97 | title = title.strip() 98 | 99 | zero_overhead = False 100 | if title[:3] == '---': 101 | zero_overhead = True 102 | 103 | title = title.split('-') 104 | 105 | title_data = [] 106 | for l in title: 107 | if l != '': 108 | title_data.append(l) 109 | 110 | if zero_overhead: 111 | title_data = [0] + title_data 112 | 113 | assert len(title_data) == 2, '{}, {}'.format(len(title_data), title_data) 114 | 115 | overhead = 0 116 | if not zero_overhead: 117 | overhead = parse_overhead(title_data[0], overhead_format) 118 | 119 | name = [title_data[1]] 120 | 121 | # TODO: process the overhead 122 | 123 | addition_name = [] 124 | children = [] 125 | for i in range(len(file_contents)): 126 | l = file_contents[i] 127 | if l[1:].strip() == '|': 128 | if len(file_contents) > i + 1: 129 | children = file_contents[i + 1:] 130 | else: 131 | children = [] 132 | break 133 | else: 134 | cur_addition_name = l[1:].strip() 135 | # If there is address in addition name, skip it 136 | if cur_addition_name[:2] == '0x': 137 | continue 138 | addition_name.append(cur_addition_name) 139 | 140 | name = name + addition_name 141 | 142 | # now clean up the child, child are in 143 | children = reshape_child(children) 144 | children = elicit_children(children) 145 | 146 | parsed_children = [] 147 | if len(children) != 0: 148 | for c in children: 149 | parsed_children.append(process_child(c)) 150 | 151 | return [overhead, name, parsed_children] 152 | 153 | 154 | def print_child(child): 155 | for c in child: 156 | print(c) 157 | 158 | 159 | def print_children(children): 160 | for i in range(len(children)): 161 | print(i) 162 | for l in children[i]: 163 | print(l) 164 | print('') 165 | 166 | 167 | def print_recursively(children, indent=0): 168 | if len(children) == 0: 169 | return 170 | # the argument is the return value of parse child 171 | overhead = children[0] 172 | names = str(children[1]) 173 | 174 | print(' ' * indent + str(overhead)) 175 | print(' ' * indent + str(names)) 176 | print(' ' * indent + "|") 177 | for c in children[2]: 178 | print_recursively(c, indent + 2) 179 | print(' ' * indent + '|') 180 | 181 | 182 | def is_first_level_symbol(l): 183 | if l.strip()[-1] == '-': 184 | return True 185 | 186 | return False 187 | 188 | 189 | def process_origin_file(filename, overhead_format="number"): 190 | overhead_format = "number" 191 | parsed_data = {} 192 | first_level_symbols = [] 193 | all_overhead_by_inst = 0 194 | 195 | f = open(filename, 'r') 196 | lines = f.readlines() 197 | f.close() 198 | 199 | splitted_input = [] 200 | cur_symbol = [] 201 | for l in lines: 202 | if l[0] == "#": 203 | pass 204 | elif l == '\n': 205 | if len(cur_symbol) != 0: 206 | splitted_input.append(cur_symbol) 207 | cur_symbol = [] 208 | elif is_first_level_symbol(l): 209 | if len(cur_symbol) != 0: 210 | splitted_input.append(cur_symbol) 211 | cur_symbol = [l[:-1]] 212 | else: 213 | cur_symbol.append(l[:-1]) 214 | if len(cur_symbol) != 0: 215 | splitted_input.append(cur_symbol) 216 | 217 | for cur_symbol in splitted_input: 218 | # Prase the title line 219 | title = cur_symbol[0] 220 | title = title.split(' ') 221 | title = [e for e in title if e != ''] 222 | overhead_all = parse_overhead(title[0], 223 | 'percent') # Children and itself 224 | overhead_self_percent = parse_overhead(title[1], 'percent') 225 | overhead_self_samples = parse_overhead(title[2], 'number') 226 | 227 | all_overhead_by_inst += overhead_self_samples 228 | 229 | # If the over head is 0, skip it 230 | if overhead_all == 0: 231 | continue 232 | 233 | name = title[4] 234 | children = reshape_child(cur_symbol[1:]) 235 | children = elicit_children(children) 236 | 237 | if len(children) == 0: 238 | # TODO: Write to the final result 239 | continue 240 | 241 | parsed_children = [] 242 | for c in children: 243 | result = process_child(c, overhead_format) 244 | parsed_children.append(result) 245 | if not name in parsed_data: 246 | total_instructions = 0 # + overhead_self_samples 247 | for c in parsed_children: 248 | total_instructions += c[0] 249 | parsed_data[name] = [total_instructions, [name], parsed_children] 250 | else: 251 | print('first level symbol {} occurs more than once'.format(name)) 252 | 253 | return parsed_data, all_overhead_by_inst 254 | 255 | 256 | def get_overhead(overhead_tree, symbol): 257 | """ 258 | Args: 259 | overhead_tree (_type_): _description_ 260 | remove_list (str): A list of symbols to remove 261 | """ 262 | if symbol in overhead_tree[1]: 263 | return overhead_tree[0] 264 | 265 | overhead = 0 266 | for c in overhead_tree[2]: 267 | overhead += get_overhead(c, symbol) 268 | 269 | return overhead 270 | 271 | 272 | def remove_symbol(overhead_tree, symbol): 273 | """" WARNING: THIS WILL CHANGE THE ARGUMENT overhead_tree 274 | 275 | Args: 276 | overhead_tree (_type_): _description_ 277 | symbol (_type_): _description_ 278 | 279 | Returns: 280 | _type_: _description_ 281 | """ 282 | 283 | # Found the target symbol, return the value, then delete all its children, mark overhead as 0 284 | if symbol in overhead_tree[1]: 285 | overhead = overhead_tree[0] 286 | overhead_tree[0] = 0 287 | overhead_tree[2] = [] 288 | 289 | return overhead 290 | 291 | overhead = 0 292 | for c in overhead_tree[2]: 293 | overhead += remove_symbol(c, symbol) 294 | 295 | overhead_tree[0] = overhead_tree[0] - overhead 296 | 297 | return overhead 298 | 299 | 300 | def parse_report_file(filename): 301 | classification = { 302 | 'app': [ 303 | 'fio_ioring_commit', '__fio_gettime', 'account_io_completion', 304 | 'add_clat_sample', 'add_lat_sample', 'fio_ioring_prep', 305 | 'fio_ioring_queue', 'get_next_rand_block', 'io_queue_event', 306 | 'io_u_mark_complete', 'io_u_mark_depth', 'lock_file', 'log_io_u', 307 | 'ntime_since', 'td_io_commit', 'td_io_getevents', 'td_io_prep', 308 | 'td_io_queue', 'utime_since_now', 'get_io_u', 'io_u_mark_submit', 309 | 'fio_ioring_getevents', 'fio_ioring_event', 'fio_libaio_getevents', 310 | 'io_u_queued_complete', 'fio_libaio_event', 'io_completed', 311 | 'ramp_time_over', 'run_threads', 'put_file' 312 | ], 313 | 'interface': [ 314 | 'io_submit_sqes', 'blkdev_read_iter', 'rw_verify_area', 315 | 'io_do_iopoll', 'io_submit_one', 'io_getevents', 'do_io_getevents' 316 | ], 317 | 'block': [ 318 | 'bio_alloc_kiocb', 'bio_associate_blkg', 'bio_iov_iter_get_pages', 319 | 'bio_set_pages_dirty', 'blk_finish_plug', 'submit_bio', 320 | 'blk_mq_end_request', 'blkdev_direct_IO', 'ret_from_fork' 321 | ], 322 | 'driver': ['nvme_queue_rq', 'nvme_irq'], 323 | 'sys': [ 324 | 'asm_common_interrupt', 325 | 'entry_SYSCALL_64', 326 | 'syscall_return_via_sysret', 327 | 'syscall', 328 | ] 329 | } 330 | symbol = [ 331 | # fio 332 | ('fio_ioring_commit', [ 333 | 'entry_SYSCALL_64', 'syscall_return_via_sysret', 334 | 'asm_common_interrupt' 335 | ]), 336 | ('__fio_gettime', ['asm_common_interrupt']), 337 | ('account_io_completion', ['asm_common_interrupt']), 338 | ('add_clat_sample', ['asm_common_interrupt']), 339 | ('add_lat_sample', ['asm_common_interrupt']), 340 | ('fio_ioring_prep', ['asm_common_interrupt']), 341 | ('fio_ioring_queue', ['asm_common_interrupt']), 342 | ('get_next_rand_block', ['asm_common_interrupt']), 343 | ('io_queue_event', ['asm_common_interrupt']), 344 | ('io_u_mark_complete', ['asm_common_interrupt']), 345 | ('io_u_mark_depth', ['asm_common_interrupt']), 346 | ('lock_file', ['asm_common_interrupt']), 347 | ('log_io_u', ['asm_common_interrupt']), 348 | ('ntime_since', ['asm_common_interrupt']), 349 | ('put_file', ['asm_common_interrupt']), 350 | ('td_io_commit', ['asm_common_interrupt']), 351 | ('td_io_getevents', ['asm_common_interrupt']), 352 | ('td_io_prep', ['asm_common_interrupt']), 353 | ('td_io_queue', ['asm_common_interrupt']), 354 | ('utime_since_now', ['asm_common_interrupt']), 355 | ('get_io_u', ['asm_common_interrupt']), 356 | ('io_u_mark_submit', ['asm_common_interrupt']), 357 | ('fio_ioring_getevents', ['asm_common_interrupt', 'entry_SYSCALL_64']), 358 | ('fio_ioring_event', ['asm_common_interrupt']), 359 | ('fio_libaio_getevents', ['asm_common_interrupt']), 360 | ('fio_libaio_event', ['asm_common_interrupt']), 361 | ('io_u_queued_complete', ['asm_common_interrupt']), 362 | ('io_completed', ['asm_common_interrupt']), 363 | ('ramp_time_over', ['asm_common_interrupt']), 364 | ('run_threads', [ 365 | 'asm_common_interrupt', 'get_io_u', 'io_queue_event', 366 | 'io_u_queued_complete', 'td_io_queue', '__fio_gettime', 367 | '__get_io_u', 'fio_gettime', 'get_file', 'get_next_rand_block', 368 | 'utime_since_now', 'fio_ioring_commit ' 369 | ]), 370 | # iouring 371 | ('io_submit_sqes', [ 372 | 'blkdev_read_iter', 'rw_verify_area', 'asm_common_interrupt', 373 | 'blk_finish_plug' 374 | ]), 375 | ('io_do_iopoll', [ 376 | 'asm_common_interrupt', 377 | ]), 378 | ('ret_from_fork', ['asm_common_interrupt', 'io_submit_sqes']), 379 | # aio 380 | ('do_io_getevents', ['asm_common_interrupt']), 381 | ('io_submit_one', ['blkdev_read_iter']), 382 | # fs 383 | ('blkdev_read_iter', ['blkdev_direct_IO', 'asm_common_interrupt']), 384 | ('rw_verify_area', ['asm_common_interrupt']), 385 | ('io_getevents', ['asm_common_interrupt']), 386 | ('blkdev_direct_IO', [ 387 | 'bio_alloc_kiocb', 'bio_associate_blkg', 'bio_iov_iter_get_pages', 388 | 'bio_set_pages_dirty', 'blk_finish_plug', 'submit_bio', 389 | 'asm_common_interrupt' 390 | ]), 391 | 392 | # blk layer 393 | ('bio_alloc_kiocb', ['asm_common_interrupt']), 394 | ('bio_associate_blkg', ['asm_common_interrupt']), 395 | ('bio_iov_iter_get_pages', ['asm_common_interrupt']), 396 | ('bio_set_pages_dirty', ['asm_common_interrupt']), 397 | ('blk_finish_plug', ['nvme_queue_rq', 'asm_common_interrupt']), 398 | ('submit_bio', ['asm_common_interrupt']), 399 | ('blk_mq_end_request', ['asm_common_interrupt']), 400 | ('nvme_queue_rq', ['asm_common_interrupt']), 401 | ('nvme_irq', ['blk_mq_end_request']), 402 | # system 403 | ('asm_common_interrupt', [ 404 | 'nvme_irq', 'bio_iov_iter_get_pages', 'blk_finish_plug', 405 | 'blk_update_request', 'blk_mq_free_request' 406 | ]), 407 | ('entry_SYSCALL_64', [ 408 | 'io_submit_sqes', 'asm_common_interrupt', 'blkdev_iopoll', 409 | 'io_do_iopoll', 'do_io_getevents', 'io_submit_one', 'vfs_read' 410 | ]), 411 | ('syscall_return_via_sysret', ['asm_common_interrupt']), 412 | ('syscall', ['entry_SYSCALL_64', 'asm_common_interrupt']) 413 | ] 414 | ## check corresponding 415 | symbols_in_classify = set() 416 | for v in classification.values(): 417 | for s in v: 418 | if not s in symbols_in_classify: 419 | symbols_in_classify.add(s) 420 | else: 421 | raise Exception( 422 | "symbol {} occurs more than one in classification".format( 423 | s)) 424 | 425 | for i in symbol: 426 | s = i[0] 427 | if not s in symbols_in_classify: 428 | raise Exception("symbol {} not occurs in classification".format(s)) 429 | 430 | parsed_data, all_overhead_by_inst = process_origin_file(filename) 431 | 432 | statics = 0 433 | all_overheads = {} 434 | 435 | for item in symbol: 436 | name = item[0] 437 | # print(name) 438 | excluded_symbol = item[1] 439 | if not name in parsed_data: 440 | print(name, "NOT FOUND") 441 | continue 442 | children = copy.deepcopy(parsed_data[name]) 443 | 444 | # Overhead not belongs to the symbol 445 | overheads = 0 446 | for s in excluded_symbol: 447 | overheads += get_overhead(children, s) 448 | remove_symbol(children, s) 449 | all_overheads[item[0]] = parsed_data[name][0] - overheads 450 | statics += parsed_data[name][0] - overheads 451 | 452 | print('') 453 | 454 | print("Selected symbols:") 455 | selected_symbols = [ 456 | 'entry_SYSCALL_64', 'asm_common_interrupt', 'io_do_iopoll' 457 | ] 458 | for ss in selected_symbols: 459 | ss_overheads = 0 460 | if ss in all_overheads: 461 | ss_overheads = all_overheads[ss] 462 | 463 | print(ss, ":", ss_overheads) 464 | 465 | print("Classified:", statics, '/', all_overhead_by_inst, 466 | statics / all_overhead_by_inst) 467 | 468 | ret_key = ['app', 'interface', 'block', 'driver', 'sys'] 469 | ret = [] 470 | 471 | print("Overheads by layer:\n") 472 | for layer in ret_key: 473 | symbols = classification[layer] 474 | cur_layer_overhead = 0 475 | for s in symbols: 476 | if s in all_overheads: 477 | cur_layer_overhead += all_overheads[s] 478 | print(layer, ':', cur_layer_overhead, 479 | cur_layer_overhead / all_overhead_by_inst) 480 | ret.append(cur_layer_overhead / all_overhead_by_inst) 481 | 482 | return ret 483 | 484 | 485 | if __name__ == '__main__': 486 | filename = sys.argv[1] 487 | parse_report_file(filename) 488 | -------------------------------------------------------------------------------- /5-c-work_breakdown_s_point/run.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | VMLINUX= 4 | SPDK_FIO_PLUGIN= 5 | SPDK_SETUP_PATH= 6 | FLAMEGRAPH= 7 | 8 | FIO_RUN_TIME=120 9 | FIO_RAMP_TIME=20 10 | 11 | FIO_RES='./fio' 12 | PERF_RES='./perf_output' 13 | PERF_PARSED_LIST='./perf_list' 14 | PERF_PARSED_GRAPH='./perf_graph' 15 | FLAMEGRAPH_OUT='./flamegraph' 16 | 17 | declare -a engines=("aio" "iou" "iou_s" "iou_c") 18 | 19 | $SPDK_SETUP_PATH reset 20 | 21 | for e in "${engines[@]}" 22 | do 23 | perf record -e instructions -F 99 -o $PERF_RES/${e}.perf.out env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --thread=1 --output-format=json -o $FIO_RES/${e}_fio.txt; 24 | 25 | perf record -g -e instructions -F 99 -o $PERF_RES/${e}_g.perf.out env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --thread=1 --output-format=json -o $FIO_RES/${e}_g.txt; 26 | done 27 | 28 | e="spdk_fio" 29 | 30 | $SPDK_SETUP_PATH 31 | 32 | perf record -e instructions -F 99 -o $PERF_RES/${e}.perf.out env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME LD_PRELOAD=$SPDK_FIO_PLUGIN fio ${e}.conf --thread=1 --output-format=json -o $FIO_RES/${e}_fio.txt; 33 | 34 | perf record -g -e instructions -F 99 -o $PERF_RES/${e}_g.perf.out env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME LD_PRELOAD=$SPDK_FIO_PLUGIN fio ${e}.conf --thread=1 --output-format=json -o $FIO_RES/${e}_g.txt; 35 | 36 | $SPDK_SETUP_PATH reset 37 | 38 | # parse 39 | 40 | for e in "${engines[@]}" 41 | do 42 | perf report --vmlinux $VMLINUX -n -m --stdio --full-source-path --source -s symbol -i $PERF_RES/${e}.perf.out >> $PERF_PARSED_LIST/perf_parsed_${e}.txt; 43 | 44 | perf report --vmlinux $VMLINUX -n -m --stdio --full-source-path --source -s symbol --call-graph=graph,0,caller,function,count -i $PERF_RES/${e}_g.perf.out >> $PERF_PARSED_GRAPH/perf_parsed_${e}_g.txt; 45 | 46 | $FLAMEGRAPH $PERF_RES/${e}_g.perf.out $FLAMEGRAPH_OUT/${e}.svg; 47 | 48 | done 49 | -------------------------------------------------------------------------------- /5-c-work_breakdown_s_point/spdk_fio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | filename=trtype=PCIe traddr=0000.af.00.0 ns=1:traddr=0000.b0.00.0 ns=1:traddr=0000.b1.00.0 ns=1:traddr=0000.b2.00.0 ns=1:traddr=0000.d8.00.0 ns=1:traddr=0000.da.00.0 ns=1:traddr=0000.db.00.0 ns=1 12 | ioengine=spdk 13 | thread=1 14 | iodepth=16 15 | 16 | [job1] 17 | -------------------------------------------------------------------------------- /6-micro_arch_s_point/aio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=libaio 12 | thread=1 13 | iodepth=8 14 | 15 | [job1] 16 | 17 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 18 | 19 | -------------------------------------------------------------------------------- /6-micro_arch_s_point/eliclit_results_all.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import numpy as np 3 | import fio 4 | 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | def parse_perf_stat(filename): 9 | """_summary_ 10 | 11 | Args: 12 | filename (_type_): _description_ 13 | 14 | Returns: 15 | dict[name]: (value, stddev) value[int], percent 16 | """ 17 | f = open(filename, 'r') 18 | contents = f.readlines() 19 | f.close() 20 | 21 | ret = {} 22 | data_start = 0 23 | for line in contents: 24 | if line[0] == '#' or line == '\n': 25 | data_start += 1 26 | else: 27 | break 28 | 29 | data_start += 1 30 | contents = contents[data_start:] 31 | data_start = 0 32 | for line in contents: 33 | if line == '\n': 34 | data_start += 1 35 | else: 36 | break 37 | contents = contents[data_start:] 38 | 39 | for line in contents: 40 | # print(line) 41 | if line == '\n': 42 | break 43 | # parse data 44 | splitted_contents = line.split(' ') 45 | splitted_contents = [x for x in splitted_contents if x != ''] 46 | value = splitted_contents[0] 47 | key = splitted_contents[1] 48 | if key[-1] == '\n': 49 | key = key[0:-1] 50 | 51 | stddev = None 52 | # check if stddev exists, if exists, get it. if not exist, return None 53 | last_elem = splitted_contents[-1] 54 | if last_elem[-1] == '\n': 55 | last_elem = last_elem[:-1] 56 | if last_elem == ')': 57 | # stddev exists 58 | stddev = splitted_contents[-2][:-1] 59 | # stddev = float(stddev) 60 | 61 | ret[key] = (value, stddev) 62 | 63 | return ret 64 | 65 | 66 | def load_inst_cycle(filename, sample_length, num_files): 67 | cycles = [] 68 | inst = [] 69 | ipc = [] 70 | 71 | for i in range(num_files): 72 | cur_filename = filename + '_' + str(i) + '.txt' 73 | perf_stat = parse_perf_stat(cur_filename) 74 | 75 | cur_cycle = int(perf_stat['cycles'][0]) / sample_length 76 | cur_inst = int(perf_stat['instructions'][0]) / sample_length 77 | 78 | cycles.append(cur_cycle) 79 | inst.append(cur_inst) 80 | ipc.append(cur_inst / cur_cycle) 81 | 82 | # TODO: Return std or the list directly 83 | cycles_std = np.std(cycles) 84 | inst_std = np.std(inst) 85 | ipc_std = np.std(ipc) 86 | 87 | cycles = np.mean(cycles) 88 | inst = np.mean(inst) 89 | ipc = np.mean(ipc) 90 | 91 | return inst, cycles, ipc, inst_std, cycles_std, ipc_std 92 | 93 | 94 | def load_cache(filename, num_files): 95 | cache_ref = [] 96 | cache_miss = [] 97 | cache_miss_rate = [] 98 | 99 | for i in range(num_files): 100 | cur_filename = filename + '_' + str(i) + '.txt' 101 | perf_stat = parse_perf_stat(cur_filename) 102 | 103 | cur_cache_ref = int(perf_stat['cache-references'][0]) 104 | cur_cache_miss = int(perf_stat['cache-misses'][0]) 105 | cur_cache_miss_rate = cur_cache_miss / cur_cache_ref 106 | 107 | cache_ref.append(cur_cache_ref) 108 | cache_miss.append(cur_cache_miss) 109 | cache_miss_rate.append(cur_cache_miss_rate) 110 | 111 | cache_miss_std = np.std(cache_miss) 112 | cache_ref_std = np.std(cache_ref) 113 | cache_miss_rate_std = np.std(cache_miss_rate) 114 | 115 | cache_ref = np.mean(cache_ref) 116 | cache_miss = np.mean(cache_miss) 117 | cache_miss_rate = np.mean(cache_miss_rate) 118 | 119 | return cache_miss, cache_ref, cache_miss_rate, cache_miss_std, cache_ref_std, cache_miss_rate_std 120 | 121 | 122 | def load_llc(filename, num_files): 123 | llc_load = [] 124 | llc_load_miss = [] 125 | llc_store = [] 126 | llc_store_miss = [] 127 | llc_load_miss_rate = [] 128 | llc_store_miss_rate = [] 129 | 130 | for i in range(num_files): 131 | cur_filename = filename + '_' + str(i) + '.txt' 132 | 133 | perf_stat = parse_perf_stat(cur_filename) 134 | cur_llc_load = int(perf_stat['LLC-loads'][0]) 135 | cur_llc_load_miss = int(perf_stat['LLC-load-misses'][0]) 136 | cur_llc_store = int(perf_stat['LLC-stores'][0]) 137 | cur_llc_store_miss = int(perf_stat['LLC-store-misses'][0]) 138 | cur_llc_load_miss_rate = cur_llc_load_miss / cur_llc_load 139 | cur_llc_store_miss_rate = cur_llc_store_miss / cur_llc_store 140 | 141 | llc_load.append(cur_llc_load) 142 | llc_load_miss.append(cur_llc_load_miss) 143 | llc_store.append(cur_llc_store) 144 | llc_store_miss.append(cur_llc_store_miss) 145 | llc_load_miss_rate.append(cur_llc_load_miss_rate) 146 | llc_store_miss_rate.append(cur_llc_store_miss_rate) 147 | 148 | llc_load = np.mean(llc_load) 149 | llc_load_miss = np.mean(llc_load_miss) 150 | llc_store = np.mean(llc_store) 151 | llc_store_miss = np.mean(llc_store_miss) 152 | llc_load_miss_rate = np.mean(llc_load_miss_rate) 153 | llc_store_miss_rate = np.mean(llc_store_miss_rate) 154 | 155 | return llc_load_miss, llc_load, llc_load_miss_rate, llc_store_miss, llc_store, llc_store_miss_rate 156 | 157 | 158 | def load_dtlb(filename, num_files): 159 | dtlb_load_miss = [] 160 | dtlb_loads = [] 161 | dtlb_store_miss = [] 162 | dtlb_stores = [] 163 | 164 | for i in range(num_files): 165 | cur_filename = filename + '_' + str(i) + '.txt' 166 | perf_stat = parse_perf_stat(cur_filename) 167 | cur_dtlb_load_miss = int(perf_stat['dTLB-load-misses'][0]) 168 | cur_dtlb_loads = int(perf_stat['dTLB-loads'][0]) 169 | cur_dtlb_store_miss = int(perf_stat['dTLB-store-misses'][0]) 170 | cur_dtlb_stores = int(perf_stat['dTLB-stores'][0]) 171 | 172 | load_miss_rate = cur_dtlb_load_miss / cur_dtlb_loads 173 | store_miss_rate = cur_dtlb_store_miss / cur_dtlb_stores 174 | 175 | dtlb_load_miss.append(cur_dtlb_load_miss) 176 | dtlb_loads.append(cur_dtlb_loads) 177 | dtlb_store_miss.append(cur_dtlb_store_miss) 178 | dtlb_stores.append(cur_dtlb_stores) 179 | 180 | dtlb_load_miss = np.mean(dtlb_load_miss) 181 | dtlb_loads = np.mean(dtlb_loads) 182 | dtlb_store_miss = np.mean(dtlb_store_miss) 183 | dtlb_stores = np.mean(dtlb_stores) 184 | 185 | return dtlb_load_miss, dtlb_loads, load_miss_rate, dtlb_store_miss, dtlb_store_miss, store_miss_rate 186 | 187 | 188 | def load_itlb(filename, num_files): 189 | itlb_load_miss = [] 190 | itlb_loads = [] 191 | load_miss_rate = [] 192 | 193 | for i in range(num_files): 194 | cur_filename = filename + '_' + str(i) + '.txt' 195 | perf_stat = parse_perf_stat(cur_filename) 196 | cur_itlb_load_miss = int(perf_stat['iTLB-load-misses'][0]) 197 | cur_itlb_loads = int(perf_stat['iTLB-loads'][0]) 198 | cur_load_miss_rate = cur_itlb_load_miss / cur_itlb_loads 199 | 200 | itlb_load_miss.append(cur_itlb_load_miss) 201 | itlb_loads.append(cur_itlb_loads) 202 | load_miss_rate.append(cur_load_miss_rate) 203 | 204 | itlb_load_miss = np.mean(itlb_load_miss) 205 | itlb_loads = np.mean(itlb_loads) 206 | load_miss_rate = np.mean(load_miss_rate) 207 | 208 | return itlb_load_miss, itlb_loads, load_miss_rate 209 | 210 | 211 | def load_branches(filename, num_files): 212 | branch_misses = [] 213 | branches = [] 214 | branch_miss_rate = [] 215 | 216 | for i in range(num_files): 217 | cur_filename = filename + '_' + str(i) + '.txt' 218 | perf_stat = parse_perf_stat(cur_filename) 219 | cur_branch_misses = int(perf_stat['branch-misses'][0]) 220 | cur_branches = int(perf_stat['branches'][0]) 221 | cur_branch_miss_rate = cur_branch_misses / cur_branches 222 | 223 | branch_misses.append(cur_branch_misses) 224 | branches.append(cur_branches) 225 | branch_miss_rate.append(cur_branch_miss_rate) 226 | 227 | branch_misses_std = np.std(branch_misses) 228 | branches_std = np.std(branches) 229 | branch_miss_rate_std = np.std(branch_miss_rate) 230 | 231 | branch_misses = np.mean(branch_misses) 232 | branches = np.mean(branches) 233 | branch_miss_rate = np.mean(branch_miss_rate) 234 | 235 | return branch_misses, branches, branch_miss_rate, branch_misses_std, branches_std, branch_miss_rate_std 236 | 237 | 238 | def load_context_switch(filename, sample_length, num_files): 239 | cs = [] 240 | 241 | for i in range(num_files): 242 | cur_filename = filename + '_' + str(i) + '.txt' 243 | perf_stat = parse_perf_stat(cur_filename) 244 | cur_cs = int(perf_stat['cs'][0]) 245 | 246 | cs.append(cur_cs / sample_length) 247 | 248 | cs_std = np.std(cs) 249 | 250 | cs = np.mean(cs) 251 | 252 | return cs, cs_std 253 | 254 | 255 | def read_data(engines, dir, inst_sample_length, cs_sample_length, 256 | num_perf_files): 257 | all_inst_per_io = [] 258 | all_inst_per_io_std = [] 259 | all_ipc = [] 260 | all_ipc_std = [] 261 | all_cache_miss_r = [] 262 | all_cache_miss_r_std = [] 263 | all_llc_load_miss_r = [] 264 | all_llc_store_miss_r = [] 265 | all_dtlb_load_miss_r = [] 266 | all_dtlb_store_miss_r = [] 267 | all_itlb_miss_r = [] 268 | all_branch_miss_rate = [] 269 | all_branch_miss_rate_std = [] 270 | all_cs_per_ses = [] 271 | all_cs_per_io = [] 272 | all_cs_per_io_std = [] 273 | 274 | global_rk = [] 275 | jobs_rk = ['read:lat_ns:mean', 'read:iops', 'read:lat_ns:stddev'] 276 | for e in engines: 277 | fname_cycles = os.path.join(dir, e + '_s_inst_cycle') 278 | fname_cahce = os.path.join(dir, e + '_s_cache') 279 | fname_llc = os.path.join(dir, e + '_s_LLC') 280 | fname_dtlb = os.path.join(dir, e + '_s_DTLB') 281 | fname_itlb = os.path.join(dir, e + '_s_ITLB') 282 | fname_branch = os.path.join(dir, e + '_s_branch') 283 | fname_cs = os.path.join(dir, e + '_s_cs') 284 | 285 | inst, cycle, ipc, inst_std, _, ipc_std = load_inst_cycle( 286 | fname_cycles, inst_sample_length, num_perf_files) 287 | cache_miss, cache_ref, cache_miss_rate, _, _, cache_miss_rate_std = load_cache( 288 | fname_cahce, num_perf_files) 289 | # _, _, llc_load_miss_rate, _, _, llc_store_miss_rate = load_llc( 290 | # fname_llc,num_perf_files) 291 | # _, _, dtlb_load_miss_rate, _, _, dtlb_store_miss_rate = load_dtlb( 292 | # fname_dtlb,num_perf_files) 293 | # _, _, itlb_load_miss_rate = load_itlb(fname_itlb,num_perf_files) 294 | _, _, branch_miss_rate, _, _, branch_miss_rate_std = load_branches( 295 | fname_branch, num_perf_files) 296 | cs_per_sec, cs_per_sec_std = load_context_switch( 297 | fname_cs, cs_sample_length, num_perf_files) 298 | 299 | # multiply all rate by 100 300 | # Divide inst per io by 100 301 | all_ipc.append(ipc) 302 | all_ipc_std.append(ipc_std) 303 | all_cache_miss_r.append(cache_miss_rate * 100) 304 | all_cache_miss_r_std.append(cache_miss_rate_std * 100) 305 | # all_llc_load_miss_r.append(llc_load_miss_rate * 100) 306 | # all_llc_store_miss_r.append(llc_store_miss_rate * 100) 307 | # all_dtlb_load_miss_r.append(dtlb_load_miss_rate * 100) 308 | # all_dtlb_store_miss_r.append(dtlb_store_miss_rate * 100) 309 | # all_itlb_miss_r.append(itlb_load_miss_rate * 100) 310 | all_branch_miss_rate.append(branch_miss_rate * 100) 311 | all_branch_miss_rate_std.append(branch_miss_rate_std * 100) 312 | all_cs_per_ses.append(cs_per_sec) 313 | 314 | fname_fio_1 = os.path.join(dir, e + '_s.txt') 315 | _, j_res = fio.parse_experiment(fname_fio_1, global_rk, jobs_rk) 316 | iops_1 = j_res[0]['read:iops'] 317 | all_inst_per_io.append(inst / iops_1 / 1000) 318 | all_inst_per_io_std.append(inst_std / iops_1 / 1000) 319 | all_cs_per_io.append(cs_per_sec / iops_1) 320 | all_cs_per_io_std.append(cs_per_sec_std / iops_1) 321 | 322 | ret = { 323 | 'all_ipc': all_ipc, 324 | 'all_ipc_std': all_ipc_std, 325 | 'all_cache_miss_r': all_cache_miss_r, 326 | 'all_cache_miss_r_std': all_cache_miss_r_std, 327 | 'all_llc_load_miss_r': all_llc_load_miss_r, 328 | 'all_llc_store_miss_r': all_llc_store_miss_r, 329 | 'all_dtlb_load_miss_r': all_dtlb_load_miss_r, 330 | 'all_dtlb_store_miss_r': all_dtlb_store_miss_r, 331 | 'all_itlb_miss_r': all_itlb_miss_r, 332 | 'all_inst_per_io': all_inst_per_io, 333 | 'all_inst_per_io_std': all_inst_per_io_std, 334 | 'all_branch_miss_rate': all_branch_miss_rate, 335 | 'all_branch_miss_rate_std': all_branch_miss_rate_std, 336 | 'all_cs_per_ses': all_cs_per_ses, 337 | 'all_cs_per_io': all_cs_per_io, 338 | 'all_cs_per_io_std': all_cs_per_io_std 339 | } 340 | return ret 341 | 342 | 343 | global_dir = './results_global' 344 | local_dir = './results_local' 345 | engines = ['aio', 'iou', 'iou_c', 'iou_s', 'spdk_fio'] 346 | inst_sample_length = 5 347 | cs_sample_length = 5 348 | num_perf_files = 10 349 | # global_inst_per_io_1, global_inst_1, global_ipc_1, global_cache_miss_1 = read_data( 350 | # engines, global_dir, sample_length) 351 | # local_inst_per_io_1, local_inst_1, local_ipc_1, local_cache_miss_1 = read_data( 352 | # engines, local_dir, sample_length) 353 | 354 | global_data = read_data(engines, global_dir, inst_sample_length, 355 | cs_sample_length, num_perf_files) 356 | local_data = read_data(engines, local_dir, inst_sample_length, 357 | cs_sample_length, num_perf_files) 358 | 359 | x_label = 'Engines' 360 | bar_width = 0.4 361 | axis_label_font_size = 28 362 | axis_tick_font_size = 26 363 | legend_font_size = 23 364 | datalabel_size = 24 365 | datalabel_va = 'bottom' #'bottom' 366 | 367 | 368 | def draw_inst_io(system_wide, process_specific, system_wide_std, 369 | process_specific_std, fig_save_path): 370 | group_list = ['system_wide', 'process_specific'] 371 | x_ticks = ['aio', 'iou', 'iou-c', 'iou-s', ' spdk-fio'] 372 | value = {'system_wide': system_wide, 'process_specific': process_specific} 373 | std_dev = { 374 | 'system_wide': system_wide_std, 375 | 'process_specific': process_specific_std 376 | } 377 | # std_dev 378 | bar_width = 0.4 379 | fig_save_path = fig_save_path 380 | legend_label = { 381 | 'system_wide': 'system-wide', 382 | 'process_specific': 'process-specific' 383 | } 384 | ylabel = 'Instructions per I/O (K)' 385 | # axis_tick_font_size = axis_tick_font_size 386 | # axis_label_font_size = axis_label_font_size 387 | # legend_font_size = legend_font_size 388 | # datalabel_size = datalabel_size 389 | datalabel_va = 'bottom' 390 | 391 | fig, ax = plt.subplots(figsize=(12, 8)) 392 | 393 | # plt.title(title) 394 | # plt.xlabel(xlabel, fontsize=axis_label_font_size) 395 | plt.ylabel(ylabel, fontsize=axis_label_font_size) 396 | plt.grid(axis='y') 397 | 398 | ax.tick_params(axis='both', which='major', labelsize=axis_tick_font_size) 399 | 400 | if legend_label == None: 401 | legend_label = {} 402 | for group in group_list: 403 | legend_label[group] = group 404 | 405 | x_axis = np.arange(len(x_ticks)) 406 | 407 | # compute bar offset 408 | bar_offset = [] 409 | mid_point = (len(group_list) * bar_width) / 2 410 | 411 | for i in range(len(group_list)): 412 | bar_offset.append(bar_width * i + 0.5 * bar_width - mid_point) 413 | 414 | # print(bar_offset) 415 | # print(x_axis) 416 | x_axis - 0.1 417 | 418 | for (index, group_name) in zip(range(len(group_list)), group_list): 419 | y = value[group_name] 420 | cur_stddev = std_dev[group_name] 421 | bar_pos = x_axis + bar_offset[index] 422 | plt.bar(bar_pos, 423 | y, 424 | width=bar_width, 425 | label=legend_label[group_name], 426 | yerr=cur_stddev) 427 | 428 | # print(bar_pos, y) 429 | for (x, y, engine) in zip(bar_pos, y, x_ticks): 430 | text = '{:.1f}'.format(y) 431 | if group_name == 'system_wide': 432 | x = x - 0.1 433 | if group_name == 'process_specific' and engine == 'iou-s': 434 | x = x + 0.1 435 | plt.text( 436 | x, 437 | y, 438 | text, 439 | size=datalabel_size, 440 | ha='center', 441 | # rotation='vertical', 442 | va=datalabel_va 443 | ) # 'bottom', 'baseline', 'center', 'center_baseline', 'top' 444 | 445 | plt.xticks(x_axis, x_ticks) 446 | 447 | if legend_font_size: 448 | plt.legend(fontsize=legend_font_size) 449 | else: 450 | plt.legend() 451 | 452 | plt.savefig(fig_save_path, bbox_inches='tight') 453 | 454 | 455 | def draw_cache_miss(system_wide, process_specific, system_wide_std, 456 | process_specific_std, fig_save_path): 457 | group_list = ['system_wide', 'process_specific'] 458 | x_ticks = ['aio', 'iou', 'iou-c', 'iou-s', ' spdk-fio'] 459 | value = {'system_wide': system_wide, 'process_specific': process_specific} 460 | std_dev = { 461 | 'system_wide': system_wide_std, 462 | 'process_specific': process_specific_std 463 | } 464 | # std_dev 465 | bar_width = 0.4 466 | fig_save_path = fig_save_path 467 | legend_label = { 468 | 'system_wide': 'system-wide', 469 | 'process_specific': 'process-specific' 470 | } 471 | ylabel = 'Cache miss rate (%)' 472 | # axis_tick_font_size = axis_tick_font_size 473 | # axis_label_font_size = axis_label_font_size 474 | # legend_font_size = legend_font_size 475 | # datalabel_size = datalabel_size 476 | datalabel_va = 'bottom' 477 | 478 | fig, ax = plt.subplots(figsize=(12, 8)) 479 | 480 | # plt.title(title) 481 | # plt.xlabel(xlabel, fontsize=axis_label_font_size) 482 | plt.ylabel(ylabel, fontsize=axis_label_font_size) 483 | plt.grid(axis='y') 484 | 485 | ax.tick_params(axis='both', which='major', labelsize=axis_tick_font_size) 486 | 487 | if legend_label == None: 488 | legend_label = {} 489 | for group in group_list: 490 | legend_label[group] = group 491 | 492 | x_axis = np.arange(len(x_ticks)) 493 | 494 | # compute bar offset 495 | bar_offset = [] 496 | mid_point = (len(group_list) * bar_width) / 2 497 | 498 | for i in range(len(group_list)): 499 | bar_offset.append(bar_width * i + 0.5 * bar_width - mid_point) 500 | 501 | # print(bar_offset) 502 | # print(x_axis) 503 | x_axis - 0.1 504 | 505 | for (index, group_name) in zip(range(len(group_list)), group_list): 506 | y = value[group_name] 507 | cur_stddev = std_dev[group_name] 508 | bar_pos = x_axis + bar_offset[index] 509 | plt.bar(bar_pos, 510 | y, 511 | width=bar_width, 512 | label=legend_label[group_name], 513 | yerr=cur_stddev) 514 | 515 | # print(bar_pos, y) 516 | for (x, y, engine) in zip(bar_pos, y, x_ticks): 517 | text = '{:.1f}'.format(y) 518 | # if group_name == 'system_wide' and engine == 'iou-s': 519 | # x = x - 0.3 520 | # y = y - 25 521 | # if group_name == 'process_specific' and engine == 'iou-s': 522 | # x = x + 0.32 523 | # y = y - 10 524 | plt.text( 525 | x, 526 | y, 527 | text, 528 | size=datalabel_size, 529 | ha='center', 530 | # rotation='vertical', 531 | va=datalabel_va 532 | ) # 'bottom', 'baseline', 'center', 'center_baseline', 'top' 533 | 534 | plt.xticks(x_axis, x_ticks) 535 | 536 | if legend_font_size: 537 | plt.legend(fontsize=legend_font_size) 538 | else: 539 | plt.legend() 540 | 541 | plt.savefig(fig_save_path, bbox_inches='tight') 542 | 543 | 544 | def draw_ipc(system_wide, process_specific, system_wide_std, 545 | process_specific_std, fig_save_path): 546 | group_list = ['system_wide', 'process_specific'] 547 | x_ticks = ['aio', 'iou', 'iou-c', 'iou-s', ' spdk-fio'] 548 | value = {'system_wide': system_wide, 'process_specific': process_specific} 549 | std_dev = { 550 | 'system_wide': system_wide_std, 551 | 'process_specific': process_specific_std 552 | } 553 | # std_dev 554 | bar_width = 0.4 555 | fig_save_path = fig_save_path 556 | legend_label = { 557 | 'system_wide': 'system-wide', 558 | 'process_specific': 'process-specific' 559 | } 560 | ylabel = 'IPC' 561 | # axis_tick_font_size = axis_tick_font_size 562 | # axis_label_font_size = axis_label_font_size 563 | # legend_font_size = legend_font_size 564 | # datalabel_size = datalabel_size 565 | datalabel_va = 'bottom' 566 | 567 | fig, ax = plt.subplots(figsize=(12, 8)) 568 | 569 | # plt.title(title) 570 | # plt.xlabel(xlabel, fontsize=axis_label_font_size) 571 | plt.ylabel(ylabel, fontsize=axis_label_font_size) 572 | plt.grid(axis='y') 573 | 574 | ax.tick_params(axis='both', which='major', labelsize=axis_tick_font_size) 575 | 576 | if legend_label == None: 577 | legend_label = {} 578 | for group in group_list: 579 | legend_label[group] = group 580 | 581 | x_axis = np.arange(len(x_ticks)) 582 | 583 | # compute bar offset 584 | bar_offset = [] 585 | mid_point = (len(group_list) * bar_width) / 2 586 | 587 | for i in range(len(group_list)): 588 | bar_offset.append(bar_width * i + 0.5 * bar_width - mid_point) 589 | 590 | # print(bar_offset) 591 | # print(x_axis) 592 | x_axis - 0.1 593 | 594 | for (index, group_name) in zip(range(len(group_list)), group_list): 595 | y = value[group_name] 596 | cur_stddev = std_dev[group_name] 597 | bar_pos = x_axis + bar_offset[index] 598 | plt.bar(bar_pos, 599 | y, 600 | width=bar_width, 601 | label=legend_label[group_name], 602 | yerr=cur_stddev) 603 | 604 | # print(bar_pos, y) 605 | for (x, y, engine) in zip(bar_pos, y, x_ticks): 606 | text = '{:.1f}'.format(y) 607 | text = '{:.1f}'.format(y) 608 | if group_name == 'system_wide': 609 | x = x - 0.1 610 | if group_name == 'process_specific' and engine == 'iou-s': 611 | x = x + 0.1 612 | plt.text( 613 | x, 614 | y, 615 | text, 616 | size=datalabel_size, 617 | ha='center', 618 | # rotation='vertical', 619 | va=datalabel_va 620 | ) # 'bottom', 'baseline', 'center', 'center_baseline', 'top' 621 | 622 | plt.xticks(x_axis, x_ticks) 623 | 624 | if legend_font_size: 625 | plt.legend(fontsize=legend_font_size) 626 | else: 627 | plt.legend() 628 | 629 | plt.savefig(fig_save_path, bbox_inches='tight') 630 | 631 | 632 | def draw_branch_miss(system_wide, process_specific, system_wide_std, 633 | process_specific_std, fig_save_path): 634 | group_list = ['system_wide', 'process_specific'] 635 | x_ticks = ['aio', 'iou', 'iou-c', 'iou-s', ' spdk-fio'] 636 | value = {'system_wide': system_wide, 'process_specific': process_specific} 637 | std_dev = { 638 | 'system_wide': system_wide_std, 639 | 'process_specific': process_specific_std 640 | } 641 | # std_dev 642 | bar_width = 0.4 643 | fig_save_path = fig_save_path 644 | legend_label = { 645 | 'system_wide': 'system-wide', 646 | 'process_specific': 'process-specific' 647 | } 648 | ylabel = 'Branch miss rate (%)' 649 | # axis_tick_font_size = axis_tick_font_size 650 | # axis_label_font_size = axis_label_font_size 651 | # legend_font_size = legend_font_size 652 | # datalabel_size = datalabel_size 653 | datalabel_va = 'bottom' 654 | 655 | fig, ax = plt.subplots(figsize=(12, 8)) 656 | 657 | # plt.title(title) 658 | # plt.xlabel(xlabel, fontsize=axis_label_font_size) 659 | plt.ylabel(ylabel, fontsize=axis_label_font_size) 660 | plt.grid(axis='y') 661 | 662 | ax.tick_params(axis='both', which='major', labelsize=axis_tick_font_size) 663 | 664 | if legend_label == None: 665 | legend_label = {} 666 | for group in group_list: 667 | legend_label[group] = group 668 | 669 | x_axis = np.arange(len(x_ticks)) 670 | 671 | # compute bar offset 672 | bar_offset = [] 673 | mid_point = (len(group_list) * bar_width) / 2 674 | 675 | for i in range(len(group_list)): 676 | bar_offset.append(bar_width * i + 0.5 * bar_width - mid_point) 677 | 678 | # print(bar_offset) 679 | # print(x_axis) 680 | x_axis - 0.1 681 | 682 | for (index, group_name) in zip(range(len(group_list)), group_list): 683 | y = value[group_name] 684 | cur_stddev = std_dev[group_name] 685 | bar_pos = x_axis + bar_offset[index] 686 | plt.bar(bar_pos, 687 | y, 688 | width=bar_width, 689 | label=legend_label[group_name], 690 | yerr=cur_stddev) 691 | 692 | # print(bar_pos, y) 693 | for (x, y, engine) in zip(bar_pos, y, x_ticks): 694 | text = '{:.2f}'.format(y) 695 | if group_name == 'system_wide': 696 | x = x - 0.1 697 | if group_name == 'process_specific' and engine == 'iou-s': 698 | x = x + 0.1 699 | plt.text( 700 | x, 701 | y, 702 | text, 703 | size=datalabel_size, 704 | ha='center', 705 | # rotation='vertical', 706 | va=datalabel_va 707 | ) # 'bottom', 'baseline', 'center', 'center_baseline', 'top' 708 | 709 | plt.xticks(x_axis, x_ticks) 710 | 711 | if legend_font_size: 712 | plt.legend(fontsize=legend_font_size) 713 | else: 714 | plt.legend() 715 | 716 | plt.savefig(fig_save_path, bbox_inches='tight') 717 | 718 | 719 | def draw_cs_per_sec_io(system_wide, process_specific, system_wide_std, 720 | process_specific_std, fig_save_path): 721 | group_list = ['system_wide', 'process_specific'] 722 | x_ticks = ['aio', 'iou', 'iou-c', 'iou-s', ' spdk-fio'] 723 | value = {'system_wide': system_wide, 'process_specific': process_specific} 724 | std_dev = { 725 | 'system_wide': system_wide_std, 726 | 'process_specific': process_specific_std 727 | } 728 | # std_dev 729 | bar_width = 0.4 730 | fig_save_path = fig_save_path 731 | legend_label = { 732 | 'system_wide': 'system-wide', 733 | 'process_specific': 'process-specific' 734 | } 735 | ylabel = None 736 | # axis_tick_font_size = axis_tick_font_size 737 | # axis_label_font_size = axis_label_font_size 738 | # legend_font_size = legend_font_size 739 | # datalabel_size = datalabel_size 740 | datalabel_va = 'bottom' 741 | 742 | fig, ax = plt.subplots(figsize=(12, 8)) 743 | 744 | # plt.title(title) 745 | # plt.xlabel(xlabel, fontsize=axis_label_font_size) 746 | # plt.ylabel(ylabel, fontsize=axis_label_font_size) 747 | plt.grid(axis='y') 748 | 749 | ax.tick_params(axis='both', which='major', labelsize=axis_tick_font_size) 750 | 751 | if legend_label == None: 752 | legend_label = {} 753 | for group in group_list: 754 | legend_label[group] = group 755 | 756 | x_axis = np.arange(len(x_ticks)) 757 | 758 | # compute bar offset 759 | bar_offset = [] 760 | mid_point = (len(group_list) * bar_width) / 2 761 | 762 | for i in range(len(group_list)): 763 | bar_offset.append(bar_width * i + 0.5 * bar_width - mid_point) 764 | 765 | # print(bar_offset) 766 | # print(x_axis) 767 | x_axis - 0.1 768 | 769 | for (index, group_name) in zip(range(len(group_list)), group_list): 770 | y = value[group_name] 771 | cur_stddev = std_dev[group_name] 772 | bar_pos = x_axis + bar_offset[index] 773 | plt.bar(bar_pos, 774 | y, 775 | width=bar_width, 776 | label=legend_label[group_name], 777 | yerr=cur_stddev) 778 | 779 | # print(bar_pos, y) 780 | for (x, y, engine) in zip(bar_pos, y, x_ticks): 781 | text = '{:.1f}'.format(y) 782 | # if group_name == 'system_wide' and engine == 'psync': 783 | # x = x + 0.32 784 | # y = y - 0.12 785 | # if group_name == 'system_wide' and engine == 'aio': 786 | # x = x + 0.32 787 | # y = y - 0.18 788 | # if group_name == 'system_wide' and engine == 'iou': 789 | # x = x + 0.32 790 | # y = y - 0.18 791 | plt.text( 792 | x, 793 | y, 794 | text, 795 | size=datalabel_size, 796 | ha='center', 797 | # rotation='vertical', 798 | va=datalabel_va 799 | ) # 'bottom', 'baseline', 'center', 'center_baseline', 'top' 800 | 801 | plt.xticks(x_axis, x_ticks) 802 | 803 | if legend_font_size: 804 | plt.legend(fontsize=legend_font_size) 805 | else: 806 | plt.legend() 807 | 808 | plt.savefig(fig_save_path, bbox_inches='tight') 809 | 810 | 811 | draw_inst_io(global_data['all_inst_per_io'], local_data['all_inst_per_io'], 812 | global_data['all_inst_per_io_std'], 813 | local_data['all_inst_per_io_std'], 'inst_per_io_d7_qds.pdf') 814 | draw_ipc(global_data['all_ipc'], local_data['all_ipc'], 815 | global_data['all_ipc_std'], local_data['all_ipc_std'], 816 | 'ipc_d7_qds.pdf') 817 | draw_branch_miss(global_data['all_branch_miss_rate'], 818 | local_data['all_branch_miss_rate'], 819 | global_data['all_branch_miss_rate_std'], 820 | local_data['all_branch_miss_rate_std'], 'branch_d7_qds.pdf') 821 | draw_cs_per_sec_io(global_data['all_cs_per_io'], local_data['all_cs_per_io'], 822 | global_data['all_cs_per_io_std'], 823 | local_data['all_cs_per_io_std'], 'cs_per_io_d7_qds.pdf') 824 | draw_cache_miss(global_data['all_cache_miss_r'], 825 | local_data['all_cache_miss_r'], 826 | global_data['all_cache_miss_r_std'], 827 | local_data['all_cache_miss_r_std'], 'cache_miss_d7_qds.pdf') 828 | 829 | print(global_data['all_cs_per_io_std']) 830 | -------------------------------------------------------------------------------- /6-micro_arch_s_point/fio.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import json 3 | 4 | import matplotlib.pyplot as plt 5 | 6 | 7 | def select_data_from_dict(data, reterieve_key): 8 | ''' 9 | Select data from dict using given keys 10 | 11 | Args: 12 | data(dict): 13 | reterieve_key(list): 14 | Returns: 15 | ''' 16 | ret = {} 17 | 18 | for k in reterieve_key: 19 | cur_keys = k.split(':') 20 | selected_data = data 21 | for single_key in cur_keys: 22 | selected_data = selected_data[single_key] 23 | ret[k] = selected_data 24 | 25 | return ret 26 | 27 | 28 | def parse_global(data, reterieve_key): 29 | return select_data_from_dict(data, reterieve_key) 30 | 31 | 32 | def parse_one_job(data, reterieve_key): 33 | return select_data_from_dict(data, reterieve_key) 34 | 35 | 36 | ## TODO: Add job specific key 37 | def parse_experiment(filename, global_reterive_key, job_reterive_key): 38 | """ 39 | Parse outputs from one experiment 40 | 41 | Args: 42 | filename (str): _description_ 43 | global_reterive_key (list(str)): _description_ 44 | job_reterive_key (list(str)): _description_ 45 | 46 | Returns: 47 | dict: parsed global results 48 | list(dict): parsed results for each job 49 | """ 50 | f = open(filename, 'r') 51 | 52 | try: 53 | data = json.load(f) 54 | except: 55 | raise (Exception( 56 | 'File {} can not loaded by json.load()'.format(filename))) 57 | f.close() 58 | 59 | num_jobs = len(data['jobs']) 60 | 61 | global_result = parse_global(data['global options'], global_reterive_key) 62 | jobs_result = [] 63 | for job in data['jobs']: 64 | jobs_result.append(parse_one_job(job, job_reterive_key)) 65 | 66 | return global_result, jobs_result 67 | 68 | 69 | # files: {some_key: file_name} 70 | def parse_one_group(files, global_reterive_key, job_reterive_key): 71 | """ 72 | Parse the all experiments from one group 73 | 74 | Args: 75 | dir (str): Dir path to the results folder 76 | files (list(str)): Output filenames 77 | global_reterive_key (list(str)): 78 | job_reterive_key (list(str)): 79 | 80 | Returns: 81 | dict: parsed results for the group 82 | """ 83 | ret = {} 84 | for k in files.keys(): 85 | cur_file_path = files[k] 86 | parsed_output = parse_experiment(cur_file_path, global_reterive_key, 87 | job_reterive_key) 88 | ret[k] = parsed_output 89 | 90 | return ret 91 | 92 | 93 | def parse_all(files, global_reterive_key, job_reterive_key): 94 | """ 95 | Parse output from all groups 96 | 97 | Args: 98 | files (_type_): _description_ 99 | global_reterive_key (_type_): _description_ 100 | job_reterive_key (_type_): _description_ 101 | 102 | Returns: 103 | _type_: _description_ 104 | """ 105 | ret = {} 106 | for group in files.keys(): 107 | group_dir, group_files = files[group] 108 | ret[group] = parse_one_group(group_dir, group_files, 109 | global_reterive_key, job_reterive_key) 110 | 111 | return ret 112 | 113 | 114 | def plot_staked_bar(labels, bar_names, bar_value, title, ylabel, 115 | fig_save_path): 116 | 117 | fig, ax = plt.subplots() 118 | 119 | #plt.title(title) 120 | #plt.ylable(ylabel) 121 | 122 | print('--------') 123 | for cur_bar in bar_names: 124 | print(cur_bar) 125 | print(bar_value[cur_bar]) 126 | ax.bar(labels, bar_value[cur_bar], label=cur_bar) 127 | 128 | plt.legend() 129 | plt.savefig(fig_save_path) 130 | 131 | 132 | def parsed_to_array(reterieve_outputs, get_x_y_label): 133 | """ 134 | Get data from the reterieve keys 135 | 136 | Args: 137 | reterieve_outputs (dict): KV of reterieved values 138 | get_x_y_label (func()): Parse reterieve_outputs and get x, y and label 139 | 140 | Returns: 141 | list : x values 142 | list : y values 143 | list : label values 144 | """ 145 | x = [] 146 | y = [] 147 | std_dev = [] 148 | label = [] 149 | 150 | for k in reterieve_outputs: 151 | global_output = reterieve_outputs[k][0] 152 | jobs_output = reterieve_outputs[k][1] 153 | 154 | cur_x, cur_y, cur_label, cur_std_dev = get_x_y_label( 155 | global_output, jobs_output) 156 | 157 | x.append(cur_x) 158 | y.append(cur_y) 159 | std_dev.append(cur_std_dev) 160 | label.append(cur_label) 161 | 162 | return x, y, std_dev, label 163 | 164 | 165 | def get_all_data(outputs, get_x_y_label): 166 | """ 167 | Get all data from a group 168 | 169 | Args: 170 | outputs (dict): Parsed data 171 | get_x_y_label (func): 172 | 173 | Returns: 174 | dict: (x, y label) for each experiment 175 | """ 176 | ret = {} 177 | for group_name in outputs.keys(): 178 | group_output = outputs[group_name] 179 | group_data = parsed_to_array(group_output, get_x_y_label) 180 | ret[group_name] = group_data 181 | 182 | return ret 183 | 184 | 185 | def parse_spdk_perf(): 186 | pass 187 | 188 | 189 | if __name__ == '__main__': 190 | test_file = '/home/user/test_script/tmp/1/output_iodepth_1.josn' 191 | 192 | global_rk = ["iodepth"] 193 | jobs_rk = ["jobname", "read:slat_ns:mean"] 194 | 195 | gr, jr = parse_experiment(test_file, global_rk, jobs_rk) 196 | print(gr) 197 | print(jr) 198 | -------------------------------------------------------------------------------- /6-micro_arch_s_point/iou.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | iodepth=8 14 | 15 | [job1] 16 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 17 | -------------------------------------------------------------------------------- /6-micro_arch_s_point/iou_c.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | iodepth=8 14 | hipri 15 | 16 | [job1] 17 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 18 | 19 | -------------------------------------------------------------------------------- /6-micro_arch_s_point/iou_s.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | thread=1 13 | iodepth=64 14 | sqthread_poll=1 15 | 16 | 17 | [job1] 18 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 19 | 20 | -------------------------------------------------------------------------------- /6-micro_arch_s_point/plot.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import numpy as np 3 | import fio 4 | 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | def parse_perf_stat(filename): 9 | """_summary_ 10 | 11 | Args: 12 | filename (_type_): _description_ 13 | 14 | Returns: 15 | dict[name]: (value, stddev) value[int], percent 16 | """ 17 | f = open(filename, 'r') 18 | contents = f.readlines() 19 | f.close() 20 | 21 | ret = {} 22 | data_start = 0 23 | for line in contents: 24 | if line[0] == '#' or line == '\n': 25 | data_start += 1 26 | else: 27 | break 28 | 29 | data_start += 1 30 | contents = contents[data_start:] 31 | data_start = 0 32 | for line in contents: 33 | if line == '\n': 34 | data_start += 1 35 | else: 36 | break 37 | contents = contents[data_start:] 38 | 39 | for line in contents: 40 | # print(line) 41 | if line == '\n': 42 | break 43 | # parse data 44 | splitted_contents = line.split(' ') 45 | splitted_contents = [x for x in splitted_contents if x != ''] 46 | value = splitted_contents[0] 47 | key = splitted_contents[1] 48 | if key[-1] == '\n': 49 | key = key[0:-1] 50 | 51 | stddev = None 52 | # check if stddev exists, if exists, get it. if not exist, return None 53 | last_elem = splitted_contents[-1] 54 | if last_elem[-1] == '\n': 55 | last_elem = last_elem[:-1] 56 | if last_elem == ')': 57 | # stddev exists 58 | stddev = splitted_contents[-2][:-1] 59 | # stddev = float(stddev) 60 | 61 | ret[key] = (value, stddev) 62 | 63 | return ret 64 | 65 | 66 | def load_inst_cycle(filename, sample_length, num_files): 67 | cycles = [] 68 | inst = [] 69 | ipc = [] 70 | 71 | for i in range(num_files): 72 | cur_filename = filename + '_' + str(i) + '.txt' 73 | perf_stat = parse_perf_stat(cur_filename) 74 | 75 | cur_cycle = int(perf_stat['cycles'][0]) / sample_length 76 | cur_inst = int(perf_stat['instructions'][0]) / sample_length 77 | 78 | cycles.append(cur_cycle) 79 | inst.append(cur_inst) 80 | ipc.append(cur_inst / cur_cycle) 81 | 82 | # TODO: Return std or the list directly 83 | cycles_std = np.std(cycles) 84 | inst_std = np.std(inst) 85 | ipc_std = np.std(ipc) 86 | 87 | cycles = np.mean(cycles) 88 | inst = np.mean(inst) 89 | ipc = np.mean(ipc) 90 | 91 | return inst, cycles, ipc, inst_std, cycles_std, ipc_std 92 | 93 | 94 | def load_cache(filename, num_files): 95 | cache_ref = [] 96 | cache_miss = [] 97 | cache_miss_rate = [] 98 | 99 | for i in range(num_files): 100 | cur_filename = filename + '_' + str(i) + '.txt' 101 | perf_stat = parse_perf_stat(cur_filename) 102 | 103 | cur_cache_ref = int(perf_stat['cache-references'][0]) 104 | cur_cache_miss = int(perf_stat['cache-misses'][0]) 105 | cur_cache_miss_rate = cur_cache_miss / cur_cache_ref 106 | 107 | cache_ref.append(cur_cache_ref) 108 | cache_miss.append(cur_cache_miss) 109 | cache_miss_rate.append(cur_cache_miss_rate) 110 | 111 | cache_miss_std = np.std(cache_miss) 112 | cache_ref_std = np.std(cache_ref) 113 | cache_miss_rate_std = np.std(cache_miss_rate) 114 | 115 | cache_ref = np.mean(cache_ref) 116 | cache_miss = np.mean(cache_miss) 117 | cache_miss_rate = np.mean(cache_miss_rate) 118 | 119 | return cache_miss, cache_ref, cache_miss_rate, cache_miss_std, cache_ref_std, cache_miss_rate_std 120 | 121 | 122 | def read_data(engines, dir, inst_sample_length, cs_sample_length, 123 | num_perf_files): 124 | all_inst_per_io = [] 125 | all_inst_per_io_std = [] 126 | all_ipc = [] 127 | all_ipc_std = [] 128 | all_cache_miss_r = [] 129 | all_cache_miss_r_std = [] 130 | 131 | global_rk = [] 132 | jobs_rk = ['read:lat_ns:mean', 'read:iops', 'read:lat_ns:stddev'] 133 | for e in engines: 134 | fname_cycles = os.path.join(dir, e + '_inst_cycle') 135 | fname_cahce = os.path.join(dir, e + '_cache') 136 | 137 | inst, cycle, ipc, inst_std, _, ipc_std = load_inst_cycle( 138 | fname_cycles, inst_sample_length, num_perf_files) 139 | cache_miss, cache_ref, cache_miss_rate, _, _, cache_miss_rate_std = load_cache( 140 | fname_cahce, num_perf_files) 141 | 142 | # multiply all rate by 100 143 | # Divide inst per io by 100 144 | all_ipc.append(ipc) 145 | all_ipc_std.append(ipc_std) 146 | all_cache_miss_r.append(cache_miss_rate * 100) 147 | all_cache_miss_r_std.append(cache_miss_rate_std * 100) 148 | 149 | fname_fio_1 = os.path.join(dir, e + '.txt') 150 | _, j_res = fio.parse_experiment(fname_fio_1, global_rk, jobs_rk) 151 | iops_1 = j_res[0]['read:iops'] 152 | all_inst_per_io.append(inst / iops_1 / 1000) 153 | all_inst_per_io_std.append(inst_std / iops_1 / 1000) 154 | 155 | ret = { 156 | 'all_ipc': all_ipc, 157 | 'all_ipc_std': all_ipc_std, 158 | 'all_cache_miss_r': all_cache_miss_r, 159 | 'all_cache_miss_r_std': all_cache_miss_r_std, 160 | 'all_inst_per_io': all_inst_per_io, 161 | 'all_inst_per_io_std': all_inst_per_io_std, 162 | } 163 | return ret 164 | 165 | 166 | global_dir = './results_global' 167 | local_dir = './results_local' 168 | engines = ['aio', 'iou', 'iou_c', 'iou_s', 'spdk_fio'] 169 | inst_sample_length = 5 170 | cs_sample_length = 5 171 | num_perf_files = 10 172 | 173 | global_data = read_data(engines, global_dir, inst_sample_length, 174 | cs_sample_length, num_perf_files) 175 | local_data = read_data(engines, local_dir, inst_sample_length, 176 | cs_sample_length, num_perf_files) 177 | 178 | x_label = 'Engines' 179 | bar_width = 0.4 180 | axis_label_font_size = 28 181 | axis_tick_font_size = 26 182 | legend_font_size = 23 183 | datalabel_size = 24 184 | datalabel_va = 'bottom' #'bottom' 185 | 186 | 187 | def draw_inst_io(system_wide, process_specific, system_wide_std, 188 | process_specific_std, fig_save_path): 189 | group_list = ['system_wide', 'process_specific'] 190 | x_ticks = ['aio', 'iou', 'iou-c', 'iou-s', ' spdk-fio'] 191 | value = {'system_wide': system_wide, 'process_specific': process_specific} 192 | std_dev = { 193 | 'system_wide': system_wide_std, 194 | 'process_specific': process_specific_std 195 | } 196 | # std_dev 197 | bar_width = 0.4 198 | fig_save_path = fig_save_path 199 | legend_label = { 200 | 'system_wide': 'system-wide', 201 | 'process_specific': 'process-specific' 202 | } 203 | ylabel = 'Instructions per I/O (K)' 204 | datalabel_va = 'bottom' 205 | 206 | fig, ax = plt.subplots(figsize=(12, 8)) 207 | 208 | plt.ylabel(ylabel, fontsize=axis_label_font_size) 209 | plt.grid(axis='y') 210 | 211 | ax.tick_params(axis='both', which='major', labelsize=axis_tick_font_size) 212 | 213 | if legend_label == None: 214 | legend_label = {} 215 | for group in group_list: 216 | legend_label[group] = group 217 | 218 | x_axis = np.arange(len(x_ticks)) 219 | 220 | # compute bar offset 221 | bar_offset = [] 222 | mid_point = (len(group_list) * bar_width) / 2 223 | 224 | for i in range(len(group_list)): 225 | bar_offset.append(bar_width * i + 0.5 * bar_width - mid_point) 226 | 227 | print(bar_offset) 228 | print(x_axis) 229 | x_axis - 0.1 230 | 231 | for (index, group_name) in zip(range(len(group_list)), group_list): 232 | y = value[group_name] 233 | cur_stddev = std_dev[group_name] 234 | bar_pos = x_axis + bar_offset[index] 235 | plt.bar(bar_pos, 236 | y, 237 | width=bar_width, 238 | label=legend_label[group_name], 239 | yerr=cur_stddev) 240 | 241 | # print(bar_pos, y) 242 | for (x, y, engine) in zip(bar_pos, y, x_ticks): 243 | text = '{:.1f}'.format(y) 244 | if group_name == 'system_wide': 245 | if engine == 'iou-s': 246 | x = x - 0.13 247 | else: 248 | x = x - 0.08 249 | if group_name == 'process_specific' and engine == 'iou-s': 250 | if engine == 'iou-s': 251 | x = x + 0.13 252 | else: 253 | x = x + 0.08 254 | plt.text( 255 | x, y, text, size=datalabel_size, ha='center', va=datalabel_va 256 | ) # 'bottom', 'baseline', 'center', 'center_baseline', 'top' 257 | 258 | plt.xticks(x_axis, x_ticks) 259 | 260 | plt.legend(fontsize=legend_font_size, loc='upper left') 261 | 262 | plt.savefig(fig_save_path, bbox_inches='tight') 263 | 264 | 265 | def draw_cache_miss(system_wide, process_specific, system_wide_std, 266 | process_specific_std, fig_save_path): 267 | group_list = ['system_wide', 'process_specific'] 268 | x_ticks = ['aio', 'iou', 'iou-c', 'iou-s', ' spdk-fio'] 269 | value = {'system_wide': system_wide, 'process_specific': process_specific} 270 | std_dev = { 271 | 'system_wide': system_wide_std, 272 | 'process_specific': process_specific_std 273 | } 274 | # std_dev 275 | bar_width = 0.4 276 | fig_save_path = fig_save_path 277 | legend_label = { 278 | 'system_wide': 'system-wide', 279 | 'process_specific': 'process-specific' 280 | } 281 | ylabel = 'Cache miss rate (%)' 282 | datalabel_va = 'bottom' 283 | 284 | fig, ax = plt.subplots(figsize=(12, 8)) 285 | 286 | plt.ylabel(ylabel, fontsize=axis_label_font_size) 287 | plt.grid(axis='y') 288 | 289 | ax.tick_params(axis='both', which='major', labelsize=axis_tick_font_size) 290 | 291 | if legend_label == None: 292 | legend_label = {} 293 | for group in group_list: 294 | legend_label[group] = group 295 | 296 | x_axis = np.arange(len(x_ticks)) 297 | 298 | # compute bar offset 299 | bar_offset = [] 300 | mid_point = (len(group_list) * bar_width) / 2 301 | 302 | for i in range(len(group_list)): 303 | bar_offset.append(bar_width * i + 0.5 * bar_width - mid_point) 304 | 305 | print(bar_offset) 306 | print(x_axis) 307 | x_axis - 0.1 308 | 309 | for (index, group_name) in zip(range(len(group_list)), group_list): 310 | y = value[group_name] 311 | cur_stddev = std_dev[group_name] 312 | bar_pos = x_axis + bar_offset[index] 313 | plt.bar(bar_pos, 314 | y, 315 | width=bar_width, 316 | label=legend_label[group_name], 317 | yerr=cur_stddev) 318 | 319 | for (x, y, engine) in zip(bar_pos, y, x_ticks): 320 | text = '{:.1f}'.format(y) 321 | plt.text( 322 | x, y, text, size=datalabel_size, ha='center', va=datalabel_va 323 | ) # 'bottom', 'baseline', 'center', 'center_baseline', 'top' 324 | 325 | plt.xticks(x_axis, x_ticks) 326 | 327 | if legend_font_size: 328 | plt.legend(fontsize=legend_font_size) 329 | else: 330 | plt.legend() 331 | 332 | plt.savefig(fig_save_path, bbox_inches='tight') 333 | 334 | 335 | def draw_ipc(system_wide, process_specific, system_wide_std, 336 | process_specific_std, fig_save_path): 337 | group_list = ['system_wide', 'process_specific'] 338 | x_ticks = ['aio', 'iou', 'iou-c', 'iou-s', ' spdk-fio'] 339 | value = {'system_wide': system_wide, 'process_specific': process_specific} 340 | std_dev = { 341 | 'system_wide': system_wide_std, 342 | 'process_specific': process_specific_std 343 | } 344 | # std_dev 345 | bar_width = 0.4 346 | fig_save_path = fig_save_path 347 | legend_label = { 348 | 'system_wide': 'system-wide', 349 | 'process_specific': 'process-specific' 350 | } 351 | ylabel = 'IPC' 352 | datalabel_va = 'bottom' 353 | 354 | fig, ax = plt.subplots(figsize=(12, 8)) 355 | 356 | plt.ylabel(ylabel, fontsize=axis_label_font_size) 357 | plt.grid(axis='y') 358 | 359 | ax.tick_params(axis='both', which='major', labelsize=axis_tick_font_size) 360 | 361 | if legend_label == None: 362 | legend_label = {} 363 | for group in group_list: 364 | legend_label[group] = group 365 | 366 | x_axis = np.arange(len(x_ticks)) 367 | 368 | # compute bar offset 369 | bar_offset = [] 370 | mid_point = (len(group_list) * bar_width) / 2 371 | 372 | for i in range(len(group_list)): 373 | bar_offset.append(bar_width * i + 0.5 * bar_width - mid_point) 374 | 375 | print(bar_offset) 376 | print(x_axis) 377 | x_axis - 0.1 378 | 379 | for (index, group_name) in zip(range(len(group_list)), group_list): 380 | y = value[group_name] 381 | cur_stddev = std_dev[group_name] 382 | bar_pos = x_axis + bar_offset[index] 383 | plt.bar(bar_pos, 384 | y, 385 | width=bar_width, 386 | label=legend_label[group_name], 387 | yerr=cur_stddev) 388 | 389 | for (x, y, engine) in zip(bar_pos, y, x_ticks): 390 | text = '{:.1f}'.format(y) 391 | if group_name == 'system_wide': 392 | x = x - 0.1 393 | if group_name == 'process_specific' and engine == 'iou-s': 394 | x = x + 0.1 395 | plt.text( 396 | x, y, text, size=datalabel_size, ha='center', va=datalabel_va 397 | ) # 'bottom', 'baseline', 'center', 'center_baseline', 'top' 398 | 399 | plt.xticks(x_axis, x_ticks) 400 | 401 | if legend_font_size: 402 | plt.legend(fontsize=legend_font_size) 403 | else: 404 | plt.legend() 405 | 406 | plt.savefig(fig_save_path, bbox_inches='tight') 407 | 408 | 409 | draw_inst_io(global_data['all_inst_per_io'], local_data['all_inst_per_io'], 410 | global_data['all_inst_per_io_std'], 411 | local_data['all_inst_per_io_std'], 'inst_per_io_d1_qd1.pdf') 412 | draw_ipc(global_data['all_ipc'], local_data['all_ipc'], 413 | global_data['all_ipc_std'], local_data['all_ipc_std'], 414 | 'ipc_d1_qd1.pdf') 415 | draw_cache_miss(global_data['all_cache_miss_r'], 416 | local_data['all_cache_miss_r'], 417 | global_data['all_cache_miss_r_std'], 418 | local_data['all_cache_miss_r_std'], 'cache_miss_d1_qd1.pdf') 419 | -------------------------------------------------------------------------------- /6-micro_arch_s_point/run.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | PERF_INST_SLEEP_TIME=5 4 | PERF_INST_REPEAT_TIME=1 5 | PERF_CACHE_SLEEP_TIME=5 6 | PERF_CACHE_REPEAT_TIME=1 7 | 8 | PERF_WAIT_TIME=20 9 | FIO_RUN_TIME_ALL=200 10 | FIO_RAMP_TIME=20 11 | 12 | 13 | SPDK_FIO_PLUGIN= 14 | SPDK_SETUP_PATH= 15 | 16 | declare -a engine=("aio" "iou" "iou_s" "iou_c") 17 | declare -a perf_seq=("0" "1" "2" "3" "4" "5" "6" "7" "8" "9") 18 | 19 | # system-wide 20 | 21 | RESULT='results_global' 22 | 23 | $SPDK_SETUP_PATH reset 24 | 25 | for e in "${engine[@]}" 26 | do 27 | numactl -N 1 -m 1 env FIO_RUN_TIME=$FIO_RUN_TIME_ALL FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --output-format=json -o $RESULT/${e}.txt & 28 | sleep $PERF_WAIT_TIME; 29 | for i in "${perf_seq[@]}" 30 | do 31 | perf stat -e cycles,instructions -a -r $PERF_INST_REPEAT_TIME -o $RESULT/${e}_inst_cycle_${i}.txt -- sleep $PERF_INST_SLEEP_TIME ; 32 | perf stat -e cache-misses,cache-references -a -r $PERF_CACHE_REPEAT_TIME -o $RESULT/${e}_cache_${i}.txt -- sleep $PERF_CACHE_SLEEP_TIME; 33 | done 34 | wait; 35 | done 36 | 37 | $SPDK_SETUP_PATH 38 | e="spdk_fio" 39 | numactl -N 1 -m 1 env FIO_RUN_TIME=$FIO_RUN_TIME_ALL FIO_RAMP_TIME=$FIO_RAMP_TIME LD_PRELOAD=$SPDK_FIO_PLUGIN fio ${e}.conf --output-format=json -o $RESULT/${e}.txt & 40 | sleep $PERF_WAIT_TIME; 41 | for i in "${perf_seq[@]}" 42 | do 43 | perf stat -e cycles,instructions -a -r $PERF_INST_REPEAT_TIME -o $RESULT/${e}_inst_cycle_${i}.txt -- sleep $PERF_INST_SLEEP_TIME ; 44 | perf stat -e cache-misses,cache-references -a -r $PERF_CACHE_REPEAT_TIME -o $RESULT/${e}_cache_${i}.txt -- sleep $PERF_CACHE_SLEEP_TIME; 45 | done 46 | wait 47 | 48 | $SPDK_SETUP_PATH reset 49 | 50 | # process specific 51 | 52 | RESULT='results_local' 53 | 54 | for e in "${engine[@]}" 55 | do 56 | 57 | numactl -N 1 -m 1 env FIO_RUN_TIME=$FIO_RUN_TIME_ALL FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --output-format=json -o $RESULT/${e}.txt & 58 | sleep $PERF_WAIT_TIME; 59 | for i in "${perf_seq[@]}" 60 | do 61 | perf stat -e cycles,instructions -p `pidof fio` -r $PERF_INST_REPEAT_TIME -o $RESULT/${e}_inst_cycle_${i}.txt -- sleep $PERF_INST_SLEEP_TIME ; 62 | perf stat -e cache-misses,cache-references -p `pidof fio` -r $PERF_CACHE_REPEAT_TIME -o $RESULT/${e}_cache_${i}.txt -- sleep $PERF_CACHE_SLEEP_TIME; 63 | done 64 | wait; 65 | done 66 | 67 | $SPDK_SETUP_PATH 68 | 69 | e="spdk_fio" 70 | 71 | numactl -N 1 -m 1 env FIO_RUN_TIME=$FIO_RUN_TIME_ALL FIO_RAMP_TIME=$FIO_RAMP_TIME LD_PRELOAD=$SPDK_FIO_PLUGIN fio ${e}.conf --output-format=json -o $RESULT/${e}.txt & 72 | sleep $PERF_WAIT_TIME; 73 | for i in "${perf_seq[@]}" 74 | do 75 | perf stat -e cycles,instructions -p `pidof fio` -r $PERF_INST_REPEAT_TIME -o $RESULT/${e}_inst_cycle_${i}.txt -- sleep $PERF_INST_SLEEP_TIME ; 76 | perf stat -e cache-misses,cache-references -p `pidof fio` -r $PERF_CACHE_REPEAT_TIME -o $RESULT/${e}_cache_${i}.txt -- sleep $PERF_CACHE_SLEEP_TIME; 77 | done 78 | wait 79 | 80 | $SPDK_SETUP_PATH 81 | -------------------------------------------------------------------------------- /6-micro_arch_s_point/spdk_fio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | filename=trtype=PCIe traddr=0000.af.00.0 ns=1:traddr=0000.b0.00.0 ns=1:traddr=0000.b1.00.0 ns=1:traddr=0000.b2.00.0 ns=1:traddr=0000.d8.00.0 ns=1:traddr=0000.da.00.0 ns=1:traddr=0000.db.00.0 ns=1 12 | ioengine=spdk 13 | thread=1 14 | iodepth=16 15 | 16 | [job1] 17 | -------------------------------------------------------------------------------- /7-multi_thread/aio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=libaio 12 | iodepth=128 13 | 14 | [job1] 15 | 16 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 17 | 18 | -------------------------------------------------------------------------------- /7-multi_thread/fio.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import json 3 | 4 | import matplotlib.pyplot as plt 5 | 6 | 7 | def select_data_from_dict(data, reterieve_key): 8 | ''' 9 | Select data from dict using given keys 10 | 11 | Args: 12 | data(dict): 13 | reterieve_key(list): 14 | Returns: 15 | ''' 16 | ret = {} 17 | 18 | for k in reterieve_key: 19 | cur_keys = k.split(':') 20 | selected_data = data 21 | for single_key in cur_keys: 22 | selected_data = selected_data[single_key] 23 | ret[k] = selected_data 24 | 25 | return ret 26 | 27 | 28 | def parse_global(data, reterieve_key): 29 | return select_data_from_dict(data, reterieve_key) 30 | 31 | 32 | def parse_one_job(data, reterieve_key): 33 | return select_data_from_dict(data, reterieve_key) 34 | 35 | 36 | ## TODO: Add job specific key 37 | def parse_experiment(filename, global_reterive_key, job_reterive_key): 38 | """ 39 | Parse outputs from one experiment 40 | 41 | Args: 42 | filename (str): _description_ 43 | global_reterive_key (list(str)): _description_ 44 | job_reterive_key (list(str)): _description_ 45 | 46 | Returns: 47 | dict: parsed global results 48 | list(dict): parsed results for each job 49 | """ 50 | f = open(filename, 'r') 51 | 52 | try: 53 | data = json.load(f) 54 | except: 55 | raise (Exception( 56 | 'File {} can not loaded by json.load()'.format(filename))) 57 | f.close() 58 | 59 | num_jobs = len(data['jobs']) 60 | 61 | global_result = parse_global(data['global options'], global_reterive_key) 62 | jobs_result = [] 63 | for job in data['jobs']: 64 | jobs_result.append(parse_one_job(job, job_reterive_key)) 65 | 66 | return global_result, jobs_result 67 | 68 | 69 | # files: {some_key: file_name} 70 | def parse_one_group(files, global_reterive_key, job_reterive_key): 71 | """ 72 | Parse the all experiments from one group 73 | 74 | Args: 75 | dir (str): Dir path to the results folder 76 | files (list(str)): Output filenames 77 | global_reterive_key (list(str)): 78 | job_reterive_key (list(str)): 79 | 80 | Returns: 81 | dict: parsed results for the group 82 | """ 83 | ret = {} 84 | for k in files.keys(): 85 | cur_file_path = files[k] 86 | parsed_output = parse_experiment(cur_file_path, global_reterive_key, 87 | job_reterive_key) 88 | ret[k] = parsed_output 89 | 90 | return ret 91 | 92 | 93 | def parse_all(files, global_reterive_key, job_reterive_key): 94 | """ 95 | Parse output from all groups 96 | 97 | Args: 98 | files (_type_): _description_ 99 | global_reterive_key (_type_): _description_ 100 | job_reterive_key (_type_): _description_ 101 | 102 | Returns: 103 | _type_: _description_ 104 | """ 105 | ret = {} 106 | for group in files.keys(): 107 | group_dir, group_files = files[group] 108 | ret[group] = parse_one_group(group_dir, group_files, 109 | global_reterive_key, job_reterive_key) 110 | 111 | return ret 112 | 113 | 114 | def plot_staked_bar(labels, bar_names, bar_value, title, ylabel, 115 | fig_save_path): 116 | 117 | fig, ax = plt.subplots() 118 | 119 | #plt.title(title) 120 | #plt.ylable(ylabel) 121 | 122 | print('--------') 123 | for cur_bar in bar_names: 124 | print(cur_bar) 125 | print(bar_value[cur_bar]) 126 | ax.bar(labels, bar_value[cur_bar], label=cur_bar) 127 | 128 | plt.legend() 129 | plt.savefig(fig_save_path) 130 | 131 | 132 | def parsed_to_array(reterieve_outputs, get_x_y_label): 133 | """ 134 | Get data from the reterieve keys 135 | 136 | Args: 137 | reterieve_outputs (dict): KV of reterieved values 138 | get_x_y_label (func()): Parse reterieve_outputs and get x, y and label 139 | 140 | Returns: 141 | list : x values 142 | list : y values 143 | list : label values 144 | """ 145 | x = [] 146 | y = [] 147 | std_dev = [] 148 | label = [] 149 | 150 | for k in reterieve_outputs: 151 | global_output = reterieve_outputs[k][0] 152 | jobs_output = reterieve_outputs[k][1] 153 | 154 | cur_x, cur_y, cur_label, cur_std_dev = get_x_y_label( 155 | global_output, jobs_output) 156 | 157 | x.append(cur_x) 158 | y.append(cur_y) 159 | std_dev.append(cur_std_dev) 160 | label.append(cur_label) 161 | 162 | return x, y, std_dev, label 163 | 164 | 165 | def get_all_data(outputs, get_x_y_label): 166 | """ 167 | Get all data from a group 168 | 169 | Args: 170 | outputs (dict): Parsed data 171 | get_x_y_label (func): 172 | 173 | Returns: 174 | dict: (x, y label) for each experiment 175 | """ 176 | ret = {} 177 | for group_name in outputs.keys(): 178 | group_output = outputs[group_name] 179 | group_data = parsed_to_array(group_output, get_x_y_label) 180 | ret[group_name] = group_data 181 | 182 | return ret 183 | 184 | 185 | def parse_spdk_perf(): 186 | pass 187 | 188 | 189 | if __name__ == '__main__': 190 | test_file = '/home/user/test_script/tmp/1/output_iodepth_1.josn' 191 | 192 | global_rk = ["iodepth"] 193 | jobs_rk = ["jobname", "read:slat_ns:mean"] 194 | 195 | gr, jr = parse_experiment(test_file, global_rk, jobs_rk) 196 | print(gr) 197 | print(jr) 198 | -------------------------------------------------------------------------------- /7-multi_thread/iou.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | iodepth=128 13 | 14 | [job1] 15 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 16 | -------------------------------------------------------------------------------- /7-multi_thread/iou_c.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | iodepth=128 13 | hipri 14 | 15 | [job1] 16 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 17 | 18 | -------------------------------------------------------------------------------- /7-multi_thread/iou_s.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | iodepth=128 13 | sqthread_poll=1 14 | 15 | [job1] 16 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 17 | 18 | -------------------------------------------------------------------------------- /7-multi_thread/plot.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import numpy as np 3 | import fio 4 | 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | def parse_spdk_perf(filename): 9 | f = open(filename) 10 | contents = f.readlines() 11 | f.close() 12 | 13 | contents.reverse() 14 | 15 | last_line = '' 16 | for l in contents: 17 | if l != '\n': 18 | last_line = l 19 | break 20 | data = l.split(' ') 21 | 22 | data = [x for x in data if x != ''] 23 | iops = float(data[2]) 24 | latency = float(data[4]) 25 | 26 | return iops, latency 27 | 28 | 29 | def read_data(engines, num_processes, dir): 30 | ret = {} 31 | for e in engines: 32 | ret[e] = [] 33 | ret['spdk_perf'] = [] 34 | 35 | global_rk = [] 36 | jobs_rk = ['read:lat_ns:mean', 'read:iops', 'read:lat_ns:stddev'] 37 | for e in engines: 38 | for num_p in range(1, num_processes + 1): 39 | # qd = 1 40 | cur_filename = e + '_t_' + str(num_p) + '.txt' 41 | cur_filename = os.path.join(dir, cur_filename) 42 | 43 | _, j_res = fio.parse_experiment(cur_filename, global_rk, jobs_rk) 44 | all_iops = [] 45 | for single_job_res in j_res: 46 | all_iops.append(single_job_res['read:iops']) 47 | 48 | iops = np.sum(all_iops) 49 | iops = iops / 1000 50 | ret[e].append(iops) 51 | 52 | for num_p in range(1, num_processes + 1): 53 | cur_filename = 'spdk_perf_t_' + str(num_p) + '.txt' 54 | cur_filename = os.path.join(dir, cur_filename) 55 | cur_iops, cur_latency = parse_spdk_perf(cur_filename) 56 | ret['spdk_perf'].append(cur_iops / 1000) 57 | return ret 58 | 59 | 60 | dir = 'results' 61 | engines = ['aio', 'iou', 'iou_s', 'iou_c', 'spdk_fio', 'spdk_perf'] 62 | num_process = 20 63 | 64 | ret = read_data(engines[:-1], num_process, dir) 65 | 66 | x_label = 'Engines' 67 | bar_width = 0.4 68 | axis_label_font_size = 28 69 | axis_tick_font_size = 26 70 | legend_font_size = 26 71 | datalabel_size = 18 72 | datalabel_va = 'bottom' #'bottom' 73 | linewidth = 4 74 | markersize = 15 75 | 76 | matplotlib_colors = [ 77 | 'blue', 'green', 'red', 'cyan', 'magenta', 'yellw', 'white' 78 | ] 79 | 80 | dot_style = [ 81 | '+', 82 | 'X', 83 | 'o', 84 | 'v', 85 | 's', 86 | 'P', 87 | ] 88 | 89 | 90 | def draw_graph( 91 | data, 92 | # xlabel, 93 | # ylabel, 94 | group_list, 95 | fig_save_path, 96 | ): 97 | # group_label=None, 98 | # axis_tick_font_size=None, 99 | # axis_label_font_size=None, 100 | # legend_font_size=None, 101 | # linewidth=None, 102 | # datalabel_size=None, 103 | # markersize=None 104 | 105 | fig, ax = plt.subplots(figsize=(12, 8)) 106 | 107 | # plt.xlabel('xlabel', fontsize=axis_label_font_size) 108 | plt.ylabel('MIOPS', fontsize=axis_label_font_size) 109 | plt.grid(True) 110 | 111 | ax.tick_params(axis='both', which='major', labelsize=axis_tick_font_size) 112 | ax.xaxis.set_ticks([1, 4, 8, 12, 16, 20]) 113 | ax.set_ylim([0, 5]) 114 | 115 | group_label = { 116 | 'aio': 'aio', 117 | 'iou': 'iou', 118 | 'iou_s': 'iou-s', 119 | 'iou_c': 'iou-c', 120 | 'spdk_fio': 'spdk_fio', 121 | 'spdk_perf': 'spdk_perf' 122 | } 123 | 124 | for (index, group_name) in zip(range(len(group_list)), group_list): 125 | # x, y, std_dev, data_label = data[group_name] 126 | x = range(1, 21) 127 | y = data[group_name] 128 | y = [i / 1000 for i in y] 129 | if group_name == 'iou_s': 130 | x = list(range(2, 21, 2)) 131 | y = y[:10] 132 | 133 | plt.errorbar( 134 | x, 135 | y, 136 | # yerr=std_dev, 137 | label=group_label[group_name], 138 | marker=dot_style[index % len(dot_style)], 139 | linewidth=linewidth, 140 | markersize=markersize, 141 | ) # TODO: Add more options, may be passing from the arguments 142 | # for i in range(len(data_label)): 143 | # ax.text(x[i], y[i], data_label[i], size=datalabel_size) 144 | 145 | plt.legend(fontsize=legend_font_size) 146 | 147 | plt.savefig(fig_save_path, bbox_inches='tight') 148 | 149 | 150 | draw_graph(ret, engines, 'inc_process.pdf') -------------------------------------------------------------------------------- /7-multi_thread/run.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | FIO_RUN_TIME=5 3 | FIO_RAMP_TIME=3 4 | 5 | SPDK_FIO_PLUGIN= 6 | SPDK_SETUP_PATH= 7 | SPDK_PERF_PATH= 8 | 9 | declare -a engine=("aio" "iou" "iou_c" "iou_s") 10 | declare -a num_threads_socket1=("1" "2" "3" "4" "5" "6" "7" "8" "9" "10") 11 | declare -a num_threads_socket2=("11" "12" "13" "14" "15" "16" "17" "18" "19" "20") 12 | 13 | # system-wide 14 | RESULT='results' 15 | 16 | $SPDK_SETUP_PATH reset; 17 | 18 | # on socket 1 19 | for e in "${engine[@]}" 20 | do 21 | for t in "${num_threads_socket1[@]}" 22 | do 23 | numactl -C $((20-${t}))-19 -m 1 env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --thread=1 --output-format=json --numjobs=${t} -o $RESULT/${e}_t_${t}.txt; 24 | done 25 | done 26 | 27 | # both socket 1 and socket 2 28 | for e in "${engine[@]}" 29 | do 30 | for t in "${num_threads_socket2[@]}" 31 | do 32 | numactl -C $((20-${t}))-19 env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --thread=1 --output-format=json --numjobs=${t} -o $RESULT/${e}_t_${t}.txt; 33 | done 34 | done 35 | 36 | declare -a num_threads=("1" "2" "3" "4" "5") 37 | 38 | chcpu -d 1-9; 39 | 40 | e='iou_s' 41 | 42 | for t in "${num_threads[@]}" 43 | do 44 | FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --numjobs=${t} --thread=1 --output-format=json -o $RESULT/${e}_t_${t}.txt; 45 | done 46 | 47 | declare -a num_threads=("6" "7" "8" "9" "10") 48 | 49 | chcpu -e 1-9; 50 | 51 | for t in "${num_threads[@]}" 52 | do 53 | FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --numjobs=${t} --thread=1 --output-format=json -o $RESULT/${e}_t_${t}.txt; 54 | done 55 | 56 | 57 | $SPDK_SETUP_PATH; 58 | 59 | # on socket 1 60 | e='spdk_fio' 61 | declare -a spdk_mask=("palceholder" "80000" "c0000" "e0000" "f0000" "f8000" "fc000" "fe000" "ff000" "ff800" "ffc00" "ffe00" "fff00" "fff80" "fffc0" "fffe0" "ffff0" "ffff8" "ffffc" "ffffe" "fffff") 62 | 63 | for t in "${num_threads_socket1[@]}" 64 | do 65 | numactl -C $((20-${t}))-19 -m 1 env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME LD_PRELOAD=$SPDK_FIO_PLUGIN fio ${e}.conf --thread=1 --output-format=json --numjobs=${t} -o $RESULT/${e}_t_${t}.txt; 66 | numactl -C $((20-${t}))-19 -m 1 $SPDK_PERF_PATH -q 128 -o 4096 -w randread -r 'trtype:PCIe traddr:0000:af:00.0' -r 'trtype:PCIe traddr:0000:b0:00.0' -r 'trtype:PCIe traddr:0000:b1:00.0' -r 'trtype:PCIe traddr:0000:b2:00.0' -r 'trtype:PCIe traddr:0000:d8:00.0' -r 'trtype:PCIe traddr:0000:da:00.0' -r 'trtype:PCIe traddr:0000:db:00.0' -t $FIO_RUN_TIME -a $FIO_RAMP_TIME -c ${spdk_mask[$t]} > $RESULT/spdk_perf_t_${t}.txt; 67 | done 68 | 69 | # # both socket 1 and socket 2 70 | 71 | for t in "${num_threads_socket2[@]}" 72 | do 73 | numactl -C $((20-${t}))-19 env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME LD_PRELOAD=$SPDK_FIO_PLUGIN fio ${e}.conf --thread=1 --output-format=json --numjobs=${t} -o $RESULT/${e}_t_${t}.txt; 74 | numactl -C $((20-${t}))-19 $SPDK_PERF_PATH -q 128 -o 4096 -w randread -r 'trtype:PCIe traddr:0000:af:00.0' -r 'trtype:PCIe traddr:0000:b0:00.0' -r 'trtype:PCIe traddr:0000:b1:00.0' -r 'trtype:PCIe traddr:0000:b2:00.0' -r 'trtype:PCIe traddr:0000:d8:00.0' -r 'trtype:PCIe traddr:0000:da:00.0' -r 'trtype:PCIe traddr:0000:db:00.0' -t $FIO_RUN_TIME -a $FIO_RAMP_TIME -c ${spdk_mask[$t]} > $RESULT/spdk_perf_t_${t}.txt; 75 | done 76 | 77 | ## SPDK PERF 78 | 79 | $SPDK_SETUP_PATH reset; 80 | -------------------------------------------------------------------------------- /7-multi_thread/spdk_fio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | filename=trtype=PCIe traddr=0000.af.00.0 ns=1:traddr=0000.b0.00.0 ns=1:traddr=0000.b1.00.0 ns=1:traddr=0000.b2.00.0 ns=1:traddr=0000.d8.00.0 ns=1:traddr=0000.da.00.0 ns=1:traddr=0000.db.00.0 ns=1 12 | ioengine=spdk 13 | thread=1 14 | iodepth=128 15 | 16 | [job1] 17 | -------------------------------------------------------------------------------- /8-scheduler_multi_thread/aio.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=libaio 12 | iodepth=128 13 | 14 | [job1] 15 | 16 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 17 | 18 | -------------------------------------------------------------------------------- /8-scheduler_multi_thread/conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/atlarge-research/Performance-Characterization-Storage-Stacks/c2abd38f92c9fc3e9d9278f3ccdce675c207a59f/8-scheduler_multi_thread/conf -------------------------------------------------------------------------------- /8-scheduler_multi_thread/fio.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import json 3 | 4 | import matplotlib.pyplot as plt 5 | 6 | 7 | def select_data_from_dict(data, reterieve_key): 8 | ''' 9 | Select data from dict using given keys 10 | 11 | Args: 12 | data(dict): 13 | reterieve_key(list): 14 | Returns: 15 | ''' 16 | ret = {} 17 | 18 | for k in reterieve_key: 19 | cur_keys = k.split(':') 20 | selected_data = data 21 | for single_key in cur_keys: 22 | selected_data = selected_data[single_key] 23 | ret[k] = selected_data 24 | 25 | return ret 26 | 27 | 28 | def parse_global(data, reterieve_key): 29 | return select_data_from_dict(data, reterieve_key) 30 | 31 | 32 | def parse_one_job(data, reterieve_key): 33 | return select_data_from_dict(data, reterieve_key) 34 | 35 | 36 | ## TODO: Add job specific key 37 | def parse_experiment(filename, global_reterive_key, job_reterive_key): 38 | """ 39 | Parse outputs from one experiment 40 | 41 | Args: 42 | filename (str): _description_ 43 | global_reterive_key (list(str)): _description_ 44 | job_reterive_key (list(str)): _description_ 45 | 46 | Returns: 47 | dict: parsed global results 48 | list(dict): parsed results for each job 49 | """ 50 | f = open(filename, 'r') 51 | 52 | try: 53 | data = json.load(f) 54 | except: 55 | raise (Exception( 56 | 'File {} can not loaded by json.load()'.format(filename))) 57 | f.close() 58 | 59 | num_jobs = len(data['jobs']) 60 | 61 | global_result = parse_global(data['global options'], global_reterive_key) 62 | jobs_result = [] 63 | for job in data['jobs']: 64 | jobs_result.append(parse_one_job(job, job_reterive_key)) 65 | 66 | return global_result, jobs_result 67 | 68 | 69 | # files: {some_key: file_name} 70 | def parse_one_group(files, global_reterive_key, job_reterive_key): 71 | """ 72 | Parse the all experiments from one group 73 | 74 | Args: 75 | dir (str): Dir path to the results folder 76 | files (list(str)): Output filenames 77 | global_reterive_key (list(str)): 78 | job_reterive_key (list(str)): 79 | 80 | Returns: 81 | dict: parsed results for the group 82 | """ 83 | ret = {} 84 | for k in files.keys(): 85 | cur_file_path = files[k] 86 | parsed_output = parse_experiment(cur_file_path, global_reterive_key, 87 | job_reterive_key) 88 | ret[k] = parsed_output 89 | 90 | return ret 91 | 92 | 93 | def parse_all(files, global_reterive_key, job_reterive_key): 94 | """ 95 | Parse output from all groups 96 | 97 | Args: 98 | files (_type_): _description_ 99 | global_reterive_key (_type_): _description_ 100 | job_reterive_key (_type_): _description_ 101 | 102 | Returns: 103 | _type_: _description_ 104 | """ 105 | ret = {} 106 | for group in files.keys(): 107 | group_dir, group_files = files[group] 108 | ret[group] = parse_one_group(group_dir, group_files, 109 | global_reterive_key, job_reterive_key) 110 | 111 | return ret 112 | 113 | 114 | def plot_staked_bar(labels, bar_names, bar_value, title, ylabel, 115 | fig_save_path): 116 | 117 | fig, ax = plt.subplots() 118 | 119 | #plt.title(title) 120 | #plt.ylable(ylabel) 121 | 122 | print('--------') 123 | for cur_bar in bar_names: 124 | print(cur_bar) 125 | print(bar_value[cur_bar]) 126 | ax.bar(labels, bar_value[cur_bar], label=cur_bar) 127 | 128 | plt.legend() 129 | plt.savefig(fig_save_path) 130 | 131 | 132 | def parsed_to_array(reterieve_outputs, get_x_y_label): 133 | """ 134 | Get data from the reterieve keys 135 | 136 | Args: 137 | reterieve_outputs (dict): KV of reterieved values 138 | get_x_y_label (func()): Parse reterieve_outputs and get x, y and label 139 | 140 | Returns: 141 | list : x values 142 | list : y values 143 | list : label values 144 | """ 145 | x = [] 146 | y = [] 147 | std_dev = [] 148 | label = [] 149 | 150 | for k in reterieve_outputs: 151 | global_output = reterieve_outputs[k][0] 152 | jobs_output = reterieve_outputs[k][1] 153 | 154 | cur_x, cur_y, cur_label, cur_std_dev = get_x_y_label( 155 | global_output, jobs_output) 156 | 157 | x.append(cur_x) 158 | y.append(cur_y) 159 | std_dev.append(cur_std_dev) 160 | label.append(cur_label) 161 | 162 | return x, y, std_dev, label 163 | 164 | 165 | def get_all_data(outputs, get_x_y_label): 166 | """ 167 | Get all data from a group 168 | 169 | Args: 170 | outputs (dict): Parsed data 171 | get_x_y_label (func): 172 | 173 | Returns: 174 | dict: (x, y label) for each experiment 175 | """ 176 | ret = {} 177 | for group_name in outputs.keys(): 178 | group_output = outputs[group_name] 179 | group_data = parsed_to_array(group_output, get_x_y_label) 180 | ret[group_name] = group_data 181 | 182 | return ret 183 | 184 | 185 | def parse_spdk_perf(): 186 | pass 187 | 188 | 189 | if __name__ == '__main__': 190 | test_file = '/home/user/test_script/tmp/1/output_iodepth_1.josn' 191 | 192 | global_rk = ["iodepth"] 193 | jobs_rk = ["jobname", "read:slat_ns:mean"] 194 | 195 | gr, jr = parse_experiment(test_file, global_rk, jobs_rk) 196 | print(gr) 197 | print(jr) 198 | -------------------------------------------------------------------------------- /8-scheduler_multi_thread/iou.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | iodepth=128 13 | 14 | [job1] 15 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 16 | -------------------------------------------------------------------------------- /8-scheduler_multi_thread/iou_c.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | iodepth=128 13 | hipri 14 | 15 | [job1] 16 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 17 | 18 | -------------------------------------------------------------------------------- /8-scheduler_multi_thread/iou_s.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | time_based=1 5 | runtime=${FIO_RUN_TIME} 6 | ramp_time=${FIO_RAMP_TIME} 7 | allow_file_create=0 8 | rw=randread 9 | bs=4k 10 | norandommap=1 11 | ioengine=io_uring 12 | iodepth=128 13 | sqthread_poll=1 14 | 15 | [job1] 16 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 17 | 18 | -------------------------------------------------------------------------------- /8-scheduler_multi_thread/plot.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import numpy as np 3 | import fio 4 | 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | def read_data(engines, schedulers, num_processes, dir): 9 | ret = {} 10 | for e in engines: 11 | ret[e] = {} 12 | for s in schedulers: 13 | ret[e][s] = [] 14 | 15 | global_rk = [] 16 | jobs_rk = ['read:lat_ns:mean', 'read:iops', 'read:lat_ns:stddev'] 17 | for e in engines: 18 | for s in schedulers: 19 | for num_t in range(1, num_processes + 1): 20 | if e == 'iou_s' and num_t > 10: 21 | break 22 | cur_filename = e + '_{}_threads_{}.txt'.format(s, num_t) 23 | cur_filename = os.path.join(dir, cur_filename) 24 | 25 | _, j_res = fio.parse_experiment(cur_filename, global_rk, 26 | jobs_rk) 27 | all_iops = [] 28 | for single_job_res in j_res: 29 | all_iops.append(single_job_res['read:iops']) 30 | 31 | iops = np.sum(all_iops) 32 | iops = iops / 1000 33 | ret[e][s].append(iops) 34 | 35 | return ret 36 | 37 | 38 | dir = 'results' 39 | engines = ['aio', 'iou', 'iou_s', 'iou_c'] 40 | schedulers = ['none', "bfq", "kyber", "mq-deadline"] 41 | num_process = 20 42 | 43 | ret = read_data(engines, schedulers, num_process, dir) 44 | 45 | x_label = 'Engines' 46 | bar_width = 0.4 47 | axis_label_font_size = 28 48 | axis_tick_font_size = 26 49 | legend_font_size = 26 50 | datalabel_size = 18 51 | datalabel_va = 'bottom' #'bottom' 52 | linewidth = 4 53 | markersize = 15 54 | 55 | matplotlib_colors = [ 56 | 'blue', 'green', 'red', 'cyan', 'magenta', 'yellw', 'white' 57 | ] 58 | 59 | dot_style = [ 60 | '+', 61 | 'X', 62 | 'o', 63 | 'v', 64 | 's', 65 | 'P', 66 | ] 67 | 68 | line_style = [] 69 | 70 | 71 | def draw_graph( 72 | data, 73 | # xlabel, 74 | # ylabel, 75 | group_list, 76 | fig_save_path, 77 | ): 78 | fig, ax = plt.subplots(figsize=(12, 8)) 79 | 80 | # plt.xlabel('xlabel', fontsize=axis_label_font_size) 81 | plt.ylabel('MIOPS', fontsize=axis_label_font_size) 82 | plt.grid(True) 83 | 84 | ax.tick_params(axis='both', which='major', labelsize=axis_tick_font_size) 85 | ax.xaxis.set_ticks([1, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20]) 86 | ax.set_ylim([0, 5]) 87 | 88 | group_label = { 89 | 'none': 'none', 90 | "bfq": 'bfq', 91 | "kyber": "kyber", 92 | "mq-deadline": "mq-deadline" 93 | } 94 | 95 | for (index, group_name) in zip(range(len(group_list)), group_list): 96 | # x, y, std_dev, data_label = data[group_name] 97 | x = range(1, len(data[group_name]) + 1) 98 | y = data[group_name] 99 | y = [i / 1000 for i in y] 100 | if group_name == 'iou_s': 101 | x = list(range(2, 21, 2)) 102 | y = y[:10] 103 | 104 | plt.errorbar( 105 | x, 106 | y, 107 | # yerr=std_dev, 108 | label=group_label[group_name], 109 | marker=dot_style[index % len(dot_style)], 110 | linewidth=linewidth, 111 | markersize=markersize, 112 | ) # TODO: Add more options, may be passing from the arguments 113 | # for i in range(len(data_label)): 114 | # ax.text(x[i], y[i], data_label[i], size=datalabel_size) 115 | 116 | plt.legend(fontsize=legend_font_size) 117 | 118 | plt.savefig(fig_save_path, bbox_inches='tight') 119 | 120 | 121 | draw_graph(ret['aio'], schedulers, 'sched_inc_thread_aio.pdf') 122 | draw_graph(ret['iou'], schedulers, 'sched_inc_thread_iou.pdf') 123 | draw_graph(ret['iou_s'], schedulers, 'sched_inc_thread_iou_s.pdf') 124 | draw_graph(ret['iou_c'], schedulers, 'sched_inc_thread_iou_c.pdf') -------------------------------------------------------------------------------- /8-scheduler_multi_thread/run.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | FIO_RUN_TIME=120 3 | FIO_RAMP_TIME=20 4 | 5 | declare -a engine=("aio" "iou" "iou_c") 6 | declare -a sched=("none" "bfq" "kyber" "mq-deadline") 7 | declare -a num_threads=("1" "2" "3" "4" "5" "6" "7" "8" "9" "10") 8 | 9 | 10 | RESULT='results' 11 | 12 | SPDK_SETUP_PATH= 13 | 14 | $SPDK_SETUP_PATH reset 15 | 16 | for e in "${engine[@]}" 17 | do 18 | for s in "${sched[@]}" 19 | do 20 | for t in "${num_threads[@]}" 21 | do 22 | numactl -C $((20-${t}))-19 -m 1 env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --ioscheduler=${s} --numjobs=${t} --thread=1 --output-format=json -o $RESULT/${e}_${s}_threads_${t}.txt; 23 | done 24 | done 25 | done 26 | 27 | declare -a num_threads=("11" "12" "13" "14" "15" "16" "17" "18" "19" "20") 28 | 29 | for e in "${engine[@]}" 30 | do 31 | for s in "${sched[@]}" 32 | do 33 | for t in "${num_threads[@]}" 34 | do 35 | numactl -C $((20-${t}))-19 -m 1 env FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --ioscheduler=${s} --numjobs=${t} --thread=1 --output-format=json -o $RESULT/${e}_${s}_threads_${t}.txt; 36 | done 37 | done 38 | done 39 | 40 | declare -a num_threads=("1" "2" "3" "4" "5") 41 | 42 | chcpu -d 1-9 43 | 44 | e='iou_s' 45 | for s in "${sched[@]}" 46 | do 47 | for t in "${num_threads[@]}" 48 | do 49 | FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --ioscheduler=${s} --numjobs=${t} --thread=1 --output-format=json -o $RESULT/${e}_${s}_threads_${t}.txt; 50 | done 51 | done 52 | 53 | declare -a num_threads=("6" "7" "8" "9" "10") 54 | 55 | chcpu -e 1-9 56 | 57 | for s in "${sched[@]}" 58 | do 59 | for t in "${num_threads[@]}" 60 | do 61 | FIO_RUN_TIME=$FIO_RUN_TIME FIO_RAMP_TIME=$FIO_RAMP_TIME fio ${e}.conf --ioscheduler=${s} --numjobs=${t} --thread=1 --output-format=json -o $RESULT/${e}_${s}_threads_${t}.txt; 62 | done 63 | done -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 @Large Research 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Instructions 2 | 3 | ## IMPORTANT 4 | 5 | There are predefined devices in the fio configuration files. PLEASE REPLACE THEM WITH YOUR OWN DEVICE. 6 | 7 | ## Environment 8 | 9 | ### Install fio 10 | 11 | ```bash 12 | git clone https://github.com/axboe/fio.git 13 | git checkout 77c758db876d93022e8f2bb4fd4c1acbbf7e76ac 14 | cd fio 15 | ./configure 16 | make 17 | ``` 18 | 19 | NOTE: Please add fio to $PATH 20 | 21 | ### Install SPDK 22 | 23 | [SPDK setup](https://spdk.io/doc/getting_started.html) 24 | [fio spdk setup manual](https://github.com/spdk/spdk/blob/master/examples/nvme/fio_plugin/README.md) 25 | 26 | ```bash 27 | git clone https://github.com/spdk/spdk.git 28 | cd spdk 29 | git checkout aed4ece93c659195d4b56399a181f41e00a7a25e 30 | git submodule update --init 31 | sudo scripts/pkgdep.sh 32 | 33 | # Configure with fio 34 | ./configure --with-fio=/path/to/fio/repo 35 | 36 | make 37 | 38 | # bind and unbind device 39 | 40 | sudo scripts/setup.sh 41 | 42 | sudo scripts/setup.sh reset 43 | 44 | ``` 45 | 46 | Make sure there are huge pages on the node where SPDK will be run on. For example: 47 | 48 | ```bash 49 | sudo HUGENODE="nodes_hp[0]=4096,nodes_hp[1]=4096" $SPDK/scripts/setup.sh 50 | ``` 51 | 52 | 53 | 54 | ### Prepare 55 | 56 | Fully write the devices 10 times before the experiment 57 | ```bash 58 | cd write_device 59 | sudo ./run.sh 60 | ``` 61 | 62 | ### NOTE 63 | 64 | * Set the 'SPDK_FIO_PLUGIN' and 'SPDK_SETUP_PATH' before run the script 65 | * Fill your own devices in the fio configurations file before running. 66 | * In fig 7 and fig 8, the script will automatically turn down the CPU 0 using chcpu. Replace it with your cpu id in socket 1, or delete them if there is only one NUMA node on your machine. 67 | 68 | ## Figure 2 69 | 70 | Before running the script: 71 | 72 | * Set the 'SPDK_SETUP_PATH' environment variable. 73 | * Disable the CPU core that are not in the same socket with the device 74 | 75 | Run the script and plot: 76 | 77 | ```bash 78 | cd 2-iops_d1_qd_1 79 | mkdir results 80 | sudo ./run.sh 81 | python3 plot.py 82 | ``` 83 | 84 | ## Figure 3 85 | 86 | * Set the 'SPDK_SETUP_PATH' environment variable. 87 | * Disable the CPU core that are not in the same socket with the device 88 | 89 | Run the script and plot: 90 | 91 | ```bash 92 | cd 2-iops_d1_qd_1 93 | mkdir fio perf_graph perf_list perf_output 94 | sudo ./run.sh 95 | python3 plot.py 96 | ``` 97 | 98 | NOTE: 99 | 100 | The work breakdown of iou-s might not be parsed correctly by the script. The work breakdown needs to be calculated manually for the perf report. 101 | 102 | ### For spdk 103 | 104 | * Parse the perf file with srcline(set your vmlinux path): 105 | 106 | ```bash 107 | perf report --vmlinux $VMLINUX -n -m --stdio --full-source-path --source -s symbol,srcline -i perf_output/spdk_fio.perf.out >> spdk_srcline.txt; 108 | 109 | ``` 110 | 111 | * Delete all the lines that does not contain the actual data 112 | * Get the number of instructions and instructions taken by SPDK 113 | 114 | ```bash 115 | # all 116 | cat spdk_srcline.txt | tr -s ' ' | cut -d ' ' -f 3 | awk '{ sum += $1 } END { print sum }' 117 | # spdk 118 | grep $YOUR_SPDK_PATH spdk_srcline.txt | tr -s ' ' | cut -d ' ' -f 3 | awk '{ sum += $1 } END { print sum }' 119 | ``` 120 | 121 | * fill the overhead of spdk and fio in plot.py:112 122 | 123 | 124 | ## Figure 4 125 | 126 | Before running the script: 127 | 128 | * Set up the 'SPDK_FIO_PLUGIN', 'SPDK_SETUP_PATH' 129 | * Disable the CPU core that are not in the same socket with the device 130 | 131 | Run the script and plot: 132 | 133 | ```bash 134 | cd 4-micro_arch_qd1 135 | mkdir results_global results_local 136 | sudo ./run.sh 137 | python3 plot.py 138 | ``` 139 | 140 | ## Figure 5-a-b 141 | 142 | Before running the script: 143 | 144 | * Set the 'SPDK_SETUP_PATH' and 'SPDK_FIO_PLUGIN' environment variable. 145 | * Disable the CPU core that are not in the same socket with the device 146 | 147 | Run the script and plot: 148 | 149 | ```bash 150 | sudo ./run.sh 151 | mkdir results 152 | python3 plot.py 153 | ``` 154 | 155 | ## Figure 5-c 156 | 157 | see figure 3 158 | 159 | ## Figure 6 160 | 161 | see figure 4 162 | 163 | ## Figure 7 164 | 165 | Before running the script: 166 | 167 | * Set the 'SPDK_SETUP_PATH', 'SPDK_FIO_PLUGIN' environment variable. 168 | 169 | Run the script and plot: 170 | 171 | ```bash 172 | cd 7-multi_thread 173 | sudo ./run.sh 174 | mkdir results 175 | python3 plot.py 176 | ``` 177 | 178 | ## Figure 8 179 | 180 | Before running the script: 181 | 182 | * Set the 'SPDK_SETUP_PATH' environment variable. 183 | 184 | Run the script and plot: 185 | 186 | ```bash 187 | cd 8-scheduler_multi_thread 188 | mkdir results 189 | sudo ./run.sh 190 | python3 plot.py 191 | ``` 192 | 193 | # License 194 | This code and artifact is distributed under the MIT license. 195 | 196 | ``` 197 | MIT License 198 | 199 | Copyright (c) 2022 @Large Research 200 | 201 | Permission is hereby granted, free of charge, to any person obtaining a copy 202 | of this software and associated documentation files (the "Software"), to deal 203 | in the Software without restriction, including without limitation the rights 204 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 205 | copies of the Software, and to permit persons to whom the Software is 206 | furnished to do so, subject to the following conditions: 207 | 208 | The above copyright notice and this permission notice shall be included in all 209 | copies or substantial portions of the Software. 210 | 211 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 212 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 213 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 214 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 215 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 216 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 217 | SOFTWARE. 218 | ``` 219 | -------------------------------------------------------------------------------- /get_start.md: -------------------------------------------------------------------------------- 1 | # Instructions 2 | 3 | ## IMPORTANT 4 | 5 | There are predefined devices in the fio configuration files. PLEASE REPLACE THEM WITH YOUR OWN DEVICE. 6 | 7 | ## Environment 8 | 9 | ### Install fio 10 | 11 | ```bash 12 | git clone https://github.com/axboe/fio.git 13 | git checkout 77c758db876d93022e8f2bb4fd4c1acbbf7e76ac 14 | cd fio 15 | ./configure 16 | make 17 | ``` 18 | 19 | NOTE: Please add fio to $PATH 20 | 21 | ### Install SPDK 22 | 23 | [SPDK setup](https://spdk.io/doc/getting_started.html) 24 | [fio spdk setup manual](https://github.com/spdk/spdk/blob/master/examples/nvme/fio_plugin/README.md) 25 | 26 | ```bash 27 | git clone https://github.com/spdk/spdk.git 28 | cd spdk 29 | git checkout aed4ece93c659195d4b56399a181f41e00a7a25e 30 | git submodule update --init 31 | sudo scripts/pkgdep.sh 32 | 33 | # Configure with fio 34 | ./configure --with-fio=/path/to/fio/repo 35 | 36 | make 37 | 38 | # bind and unbind device 39 | 40 | sudo scripts/setup.sh 41 | 42 | sudo scripts/setup.sh reset 43 | 44 | ``` 45 | 46 | Make sure there are huge pages on the node where SPDK will be run on. For example: 47 | 48 | ```bash 49 | sudo HUGENODE="nodes_hp[0]=4096,nodes_hp[1]=4096" $SPDK/scripts/setup.sh 50 | ``` 51 | 52 | 53 | 54 | ### Prepare 55 | 56 | Fully write the devices 10 times before the experiment 57 | ```bash 58 | cd write_device 59 | sudo ./run.sh 60 | ``` 61 | 62 | ### NOTE 63 | 64 | * Set the 'SPDK_FIO_PLUGIN' and 'SPDK_SETUP_PATH' before run the script 65 | * Fill your own devices in the fio configurations file before running. 66 | * In fig 7 and fig 8, the script will automatically turn down the CPU 0 using chcpu. Replace it with your cpu id in socket 1, or delete them if there is only one NUMA node on your machine. 67 | 68 | ## Figure 2 69 | 70 | Before running the script: 71 | 72 | * Set the 'SPDK_SETUP_PATH' environment variable. 73 | * Disable the CPU core that are not in the same socket with the device 74 | 75 | Run the script and plot: 76 | 77 | ```bash 78 | cd 2-iops_d1_qd_1 79 | mkdir results 80 | sudo ./run.sh 81 | python3 plot.py 82 | ``` 83 | 84 | ## Figure 3 85 | 86 | * Set the 'SPDK_SETUP_PATH' environment variable. 87 | * Disable the CPU core that are not in the same socket with the device 88 | 89 | Run the script and plot: 90 | 91 | ```bash 92 | cd 2-iops_d1_qd_1 93 | mkdir fio perf_graph perf_list perf_output 94 | sudo ./run.sh 95 | python3 plot.py 96 | ``` 97 | 98 | NOTE: 99 | 100 | The work breakdown of iou-s might not be parsed correctly by the script. The work breakdown needs to be calculated manually for the perf report. 101 | 102 | ### For spdk 103 | 104 | * Parse the perf file with srcline(set your vmlinux path): 105 | 106 | ```bash 107 | perf report --vmlinux $VMLINUX -n -m --stdio --full-source-path --source -s symbol,srcline -i perf_output/spdk_fio.perf.out >> spdk_srcline.txt; 108 | 109 | ``` 110 | 111 | * Delete all the lines that does not contain the actual data 112 | * Get the number of instructions and instructions taken by SPDK 113 | 114 | ```bash 115 | # all 116 | cat spdk_srcline.txt | tr -s ' ' | cut -d ' ' -f 3 | awk '{ sum += $1 } END { print sum }' 117 | # spdk 118 | grep $YOUR_SPDK_PATH spdk_srcline.txt | tr -s ' ' | cut -d ' ' -f 3 | awk '{ sum += $1 } END { print sum }' 119 | ``` 120 | 121 | * fill the overhead of spdk and fio in plot.py:112 122 | 123 | 124 | ## Figure 4 125 | 126 | Before running the script: 127 | 128 | * Set up the 'SPDK_FIO_PLUGIN', 'SPDK_SETUP_PATH' 129 | * Disable the CPU core that are not in the same socket with the device 130 | 131 | Run the script and plot: 132 | 133 | ```bash 134 | cd 4-micro_arch_qd1 135 | mkdir results_global results_local 136 | sudo ./run.sh 137 | python3 plot.py 138 | ``` 139 | 140 | ## Figure 5-a-b 141 | 142 | Before running the script: 143 | 144 | * Set the 'SPDK_SETUP_PATH' and 'SPDK_FIO_PLUGIN' environment variable. 145 | * Disable the CPU core that are not in the same socket with the device 146 | 147 | Run the script and plot: 148 | 149 | ```bash 150 | sudo ./run.sh 151 | mkdir results 152 | python3 plot.py 153 | ``` 154 | 155 | ## Figure 5-c 156 | 157 | see figure 3 158 | 159 | ## Figure 6 160 | 161 | see figure 4 162 | 163 | ## Figure 7 164 | 165 | Before running the script: 166 | 167 | * Set the 'SPDK_SETUP_PATH', 'SPDK_FIO_PLUGIN' environment variable. 168 | 169 | Run the script and plot: 170 | 171 | ```bash 172 | cd 7-multi_thread 173 | sudo ./run.sh 174 | mkdir results 175 | python3 plot.py 176 | ``` 177 | 178 | ## Figure 8 179 | 180 | Before running the script: 181 | 182 | * Set the 'SPDK_SETUP_PATH' environment variable. 183 | 184 | Run the script and plot: 185 | 186 | ```bash 187 | cd 8-scheduler_multi_thread 188 | mkdir results 189 | sudo ./run.sh 190 | python3 plot.py 191 | ``` -------------------------------------------------------------------------------- /write_device/write_10_times.sh: -------------------------------------------------------------------------------- 1 | fio write_all.conf -o log_0.txt; 2 | fio write_all.conf -o log_1.txt; 3 | fio write_all.conf -o log_2.txt; 4 | fio write_all.conf -o log_3.txt; 5 | fio write_all.conf -o log_4.txt; 6 | fio write_all.conf -o log_5.txt; 7 | fio write_all.conf -o log_6.txt; 8 | fio write_all.conf -o log_7.txt; 9 | fio write_all.conf -o log_8.txt; 10 | fio write_all.conf -o log_9.txt; -------------------------------------------------------------------------------- /write_device/write_all.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | direct=1 3 | size=100% 4 | allow_file_create=0 5 | rw=write 6 | bs=1M 7 | ioengine=io_uring 8 | thread=1 9 | iodepth=64 10 | hipri 11 | numjobs=15 12 | 13 | [job1] 14 | filename=/dev/nvme0n1:/dev/nvme1n1:/dev/nvme2n1:/dev/nvme3n1:/dev/nvme4n1:/dev/nvme6n1:/dev/nvme7n1 15 | --------------------------------------------------------------------------------