├── README.md ├── extractor.ksh ├── fio-parser.py ├── fio-run-repeat.ksh ├── fio-run.ksh ├── fio-wrapper.sh ├── fio.ini ├── fio.service ├── fio_stats.py ├── how-to-get-hosts-ready-for-fio ├── how-to-run-fio └── start-all /README.md: -------------------------------------------------------------------------------- 1 | #Note: Only tested with fio version 3.x 2 | 3 | # fio-parser.py 4 | This repository contains code used to parse fio normal output (as in non json) 5 | As well as instructions for installing and running fio in both single and multi instance mode 6 | 7 | usage: fio-parser.py [-h] [--directory DIRECTORY] 8 | 9 | optional arguments: 10 | -h, --help show this help message and exit 11 | --directory DIRECTORY, -d DIRECTORY 12 | Specify the directory with fio output files to parse. 13 | If none if provided, ./ is used 14 | 15 | Output looks something like this 16 | 17 | fio_file,reads,read_bw(MiB/s),read_lat(ms),writes,write_bw(MIB/s),write_lat(ms) 18 | output-vm32-iodepth200-filesize8G-bs64-rwmix100,69900,4372,352.57964 19 | output-vm32-iodepth200-filesize8G-bs64-rwmix75,38600,2416,478.70303,12900,809,494.06208 20 | output-vm32-iodepth200-filesize8G-bs64-rwmix50,21000,1316,604.07159,21000,1316,610.97722 21 | output-vm32-iodepth200-filesize8G-bs64-rwmix25,8636,540,695.76001,25800,1617,738.59605 22 | output-vm32-iodepth200-filesize8G-bs64-rwmix0,,,,28300,1770,881.72075 23 | output-vm32-iodepth100-filesize8G-bs64-rwmix100,65400,4089,191.61536 24 | output-vm32-iodepth100-filesize8G-bs64-rwmix75,36100,2256,256.18691,12100,755,276.21681 25 | output-vm32-iodepth100-filesize8G-bs64-rwmix50,22100,1380,284.67851,22100,1381,294.03005 26 | output-vm32-iodepth100-filesize8G-bs64-rwmix0,,,,27300,1708,462.75325 27 | output-vm32-iodepth100-filesize8G-bs64-rwmix25,8201,513,352.77054,24500,1534,403.34769 28 | 29 | -------------------------------------------------------------------------------- /extractor.ksh: -------------------------------------------------------------------------------- 1 | for bs in 64 4; do 2 | for mix in 100 0; do 3 | for run in 1 2 3; 4 | do ./fio-parser.py -d $1/run${run}/$2vms/1TBWorkingset/output/ | grep bs${bs}-rwmix${mix} | while read line; do 5 | depth=`echo $line | sed "s/-iodepth/ /g" | sed "s/-filesize/ filesize/g" | awk '{print $2}'` 6 | shortline=`echo $line | cut -d, -f2,3,4,5,6,7,8` 7 | dir=`echo $1 | sed "s#/# #g" | wc -w` 8 | (( dir = dir + 1 )) 9 | dir=`echo $1 | cut -d/ -f$dir` 10 | echo $dir,$run,$depth,$shortline 11 | done 12 | done 13 | done 14 | done 15 | -------------------------------------------------------------------------------- /fio-parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | from os import listdir, makedirs 3 | from os.path import isfile, join, exists, isdir 4 | import argparse 5 | import json 6 | import os 7 | import sys 8 | 9 | def command_line(): 10 | parser = argparse.ArgumentParser(prog='fio-parser.py',description='%(prog)s is used to parse fio output files') 11 | parser.add_argument('--directory','-d',type=str,help='Specify the directory with fio output files to parse. If none if provided, ./ is used') 12 | #parser.add_argument('--output','-rod',type=str,help='Specify the output file to dump results. If none, output is to screen') 13 | arg = vars(parser.parse_args()) 14 | 15 | if not arg['directory']: 16 | directory=os.getcwd() 17 | else: 18 | directory=arg['directory'] 19 | ''' 20 | if arg['output']: 21 | if '/' in arg['output']: 22 | dir = '/'.join(arg['output'].split('/')[:-1]) 23 | output_file = arg['output'].split('/')[-1] 24 | if ! output_file: 25 | print('--output %s missing file name, exiting' % (arg['output']) 26 | exit() 27 | else: 28 | dir='./' 29 | output_file = arg['output'] 30 | 31 | if isdir(dir): 32 | ''' 33 | 34 | file_list = get_file_list(directory) 35 | return file_list 36 | 37 | def get_file_list(directory): 38 | ''' 39 | extract the contents of the current file if present to only overwrite the section in play 40 | ''' 41 | if os.path.exists(directory): 42 | file_list=[] 43 | onlyfiles = [ files for files in listdir(directory) if isfile(join(directory,files))] 44 | #Check for non binary files and make a list of them 45 | for files in onlyfiles: 46 | abs_path = ('%s/%s' % (directory,files)) 47 | #with open( abs_path,'rb') as fh: 48 | with open( abs_path,'r') as fh: 49 | if 'fio-parser.py' not in files: 50 | if 'IO depths' in fh.read(): 51 | file_list.append('%s/%s' % (directory,files)) 52 | if (len(file_list)) > 0: 53 | file_list 54 | return file_list 55 | else: 56 | print('No text files found') 57 | exit() 58 | else: 59 | print('Directory %s invalid' % (directory)) 60 | exit() 61 | 62 | def parse_files(file_list): 63 | total_output_list = ['fio_file,reads,read_bw(MiB/s),read_lat(ms),writes,write_bw(MIB/s),write_lat(ms)'] 64 | for working_file in file_list: 65 | with open( working_file,'r') as fh: 66 | file_content = fh.readlines() 67 | #extract_content(file_content, working_file, total_output_list) 68 | parsed_content = extract_content(file_content) 69 | total_output(parsed_content, total_output_list, working_file) 70 | return total_output_list 71 | 72 | def bandwidth_conversion(line): 73 | bandwidth = (line.split(','))[1].split(' ')[1].split('=')[1] 74 | if 'Mi' in bandwidth: 75 | bandwidth = bandwidth.split('M')[0] 76 | elif 'Ki' in bandwidth: 77 | bandwidth = int(bandwidth.split('K')[0]) / 2**10 78 | elif 'Gi' in bandwidth: 79 | bandwidth = int(bandwidth.split('K')[0]) * 2**20 80 | elif 'Ti' in bandwidth: 81 | bandwidth = int(bandwidth.split('K')[0]) * 2**30 82 | return bandwidth 83 | 84 | def io_conversion(line): 85 | io = line.split(',')[0].split('=')[1] 86 | if 'k' in io: 87 | io = int( float(io[0:-1]) * 10**3) 88 | elif 'm' in io: 89 | io = int( float(io[0:-1]) * 10**6) 90 | return io 91 | 92 | 93 | def lat_conversion(line): 94 | lat = ((float(line.split(',')[2].split('=')[1]))) 95 | if line[5] == 'u': 96 | lat = lat / 10**3 97 | elif line[5] == 'n': 98 | lat = lat / 10**6 99 | elif line[5] == 's': 100 | lat = lat * 10**3 101 | return lat 102 | 103 | 104 | #look for 'All clients in file', if found, this file contains 105 | #The contents ofd a multi host job 106 | def single_or_multi_job(content): 107 | for line in content: 108 | if 'All clients:' in line: 109 | return True 110 | return False 111 | 112 | def search(parsed_content,line): 113 | if 'read: IOP' in line: 114 | parsed_content['read_iop']= io_conversion(line) 115 | parsed_content['read_bw'] = bandwidth_conversion(line) 116 | if 'read_iop' in parsed_content.keys(): 117 | if line[0:3] == 'lat' and 'read_lat' not in parsed_content.keys(): 118 | parsed_content['read_lat'] = lat_conversion(line) 119 | 120 | if 'write: IOP' in line: 121 | parsed_content['write_iop']= io_conversion(line) 122 | parsed_content['write_bw'] = bandwidth_conversion(line) 123 | if 'write_iop' in parsed_content.keys(): 124 | if line[0:3] == 'lat' and 'write_lat' not in parsed_content.keys(): 125 | parsed_content['write_lat'] = lat_conversion(line) 126 | 127 | 128 | def extract_content(content): 129 | multihost = single_or_multi_job(content) 130 | parsed_content={} 131 | if multihost == True: 132 | begin_search = False 133 | else: 134 | begin_search = True 135 | 136 | for line in content: 137 | line=line.strip() 138 | if multihost == True: 139 | if 'All clients:' in line: 140 | begin_search = True 141 | if begin_search == True: 142 | search(parsed_content,line) 143 | #Exit search here 144 | if begin_search == True and 'IO depths' in line: 145 | return parsed_content 146 | 147 | def total_output(parsed_content, total_output_list, working_file): 148 | output = working_file.split('/')[-1] 149 | if 'read_iop' in parsed_content.keys(): 150 | output += ((',%s,%s') % (parsed_content['read_iop'],parsed_content['read_bw'])) 151 | if 'read_lat' in parsed_content.keys(): 152 | output += ((',%s') % (parsed_content['read_lat'])) 153 | if 'write_iop' in parsed_content.keys() and 'read_iop' not in parsed_content.keys(): 154 | output += ((',,,,%s,%s') % (parsed_content['write_iop'],parsed_content['write_bw'])) 155 | elif 'write_iop' in parsed_content.keys(): 156 | output += ((',%s,%s') % (parsed_content['write_iop'],parsed_content['write_bw'])) 157 | if 'write_lat' in parsed_content.keys(): 158 | output += ((',%s') % (parsed_content['write_lat'])) 159 | total_output_list.append(output) 160 | 161 | file_list = command_line() 162 | output = parse_files(file_list) 163 | for line in output: 164 | print(line) 165 | 166 | -------------------------------------------------------------------------------- /fio-run-repeat.ksh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ksh 2 | #Change the diredtory=< > to your directory where in I/O is written 3 | #When you run, ./fio-run /mnt/cvs/output 64 <-- outstanding I/O 4 | #for try in {1..6}; do for depth in 200 100 64 32 16 8 4 2 1; do ./fio-run /mnt/cvs/output/point$try $depth ; done; done 5 | if [[ -z $1 ]]; then 6 | echo "Please enter a base directory for output and config files" 7 | exit 8 | else 9 | base_dir=$1 10 | fi 11 | 12 | workingset=1TBWorkingset 13 | size=crap 14 | iodepth=crap 15 | bs=crap 16 | rw=crap 17 | rwmixread=crap 18 | max=crap 19 | job=crap 20 | #./ip-collect $1 21 | #for count in 32 16 8 4 2 1; do 22 | #for count in 32 1 ; do 23 | #for count in 5 1 ; do 24 | for count in 1 ; do 25 | #Create Directories, exit on error" 26 | dir=${base_dir}/${count}vms/$workingset 27 | config_dir=${dir}/config 28 | output_dir=${dir}/output 29 | mkdir -p $config_dir >/dev/null 2>&1 30 | mkdir -p $output_dir >/dev/null 2>&1 31 | if [[ -z `ls $dir` ]]; then 32 | echo "The specified directory either does not exist or could not be accessed/created: $base_dir" 33 | echo "exiting" 34 | exit 35 | fi 36 | 37 | if [[ $count == 64 ]]; then 38 | #max=15 39 | size=1G 40 | elif [[ $count == 32 ]]; then 41 | size=8G 42 | elif [[ $count == 16 ]]; then 43 | #size=10M 44 | size=16G 45 | elif [[ $count == 8 ]]; then 46 | #size=10M 47 | size=32G 48 | elif [[ $count == 5 ]]; then 49 | #size=10M 50 | size=52G 51 | elif [[ $count == 4 ]]; then 52 | #size=10M 53 | size=64G 54 | elif [[ $count == 2 ]]; then 55 | #size=10M 56 | size=128G 57 | elif [[ $count == 1 ]]; then 58 | #size=10M 59 | size=100G 60 | fi 61 | for iosize in 64; do 62 | if [[ $iosize -le 16 ]]; then 63 | rw=randrw 64 | else 65 | rw=rw 66 | fi 67 | for mix in 100 0; do 68 | if [[ $mix == 0 ]]; then 69 | #max= 15 70 | max=64 71 | elif [[ $mix = 100 ]]; then 72 | if [[ $rw == "randrw" ]]; then 73 | #max=40 74 | max=64 75 | else 76 | #max=15 77 | max=64 78 | fi 79 | else 80 | #max=15 81 | #max=10 82 | max=64 83 | fi 84 | max=$2 85 | i=$max 86 | min=0 87 | (( min = max - 1 )) 88 | #while [[ $i -gt $min ]]; do 89 | for i in 450 400 300 200 100 64 32 16; do 90 | echo "[global]" > config-$count 91 | echo "name=fio-test" >> config-$count 92 | echo "directory=/mnt" >> config-$count 93 | echo "ioengine=libaio" >> config-$count 94 | echo "direct=1" >> config-$count 95 | echo "numjobs=1" >> config-$count 96 | echo "nrfiles=100" >> config-$count 97 | echo "runtime=30" >> config-$count 98 | echo "group_reporting=1" >> config-$count 99 | echo "time_based" >> config-$count 100 | echo "stonewall" >> config-$count 101 | echo "bs=${iosize}K" >> config-$count 102 | echo "rw=${rw}" >> config-$count 103 | echo "rwmixread=${mix}" >> config-$count 104 | echo "iodepth=${i}" >> config-$count 105 | echo "size=${size}" >> config-$count 106 | echo "ramp_time=20" >> config-$count 107 | echo "[rw]" >> config-$count 108 | echo "[randrw]" >> config-$count 109 | file="vm$count-iodepth$i-filesize$size-bs$iosize-rwmix$mix" 110 | 111 | #Run Stuff Here 112 | echo fio --output-format=normal --output=../output/output-${file} --group_reporting --section=$rw config-${file} > ${config_dir}/command-${file} 113 | cat config-$count > ${config_dir}/config-${file} 114 | fio --output-format=normal --output=${output_dir}/output-$file --group_reporting --section=$rw config-$count 115 | done 116 | done 117 | done 118 | #/mnt/fio/fio-shutdown-post-$count 119 | #/mnt/fio/fio-stop 120 | done 121 | -------------------------------------------------------------------------------- /fio-run.ksh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ksh 2 | #Change the diredtory=< > to your directory where in I/O is written 3 | #When you run, ./fio-run /mnt/cvs/output 64 <-- outstanding I/O 4 | #for try in {1..6}; do for depth in 200 100 64 32 16 8 4 2 1; do ./fio-run /mnt/cvs/output/point$try $depth ; done; done 5 | if [[ -z $1 ]]; then 6 | echo "Please enter a base directory for output and config files" 7 | exit 8 | else 9 | base_dir=$1 10 | fi 11 | 12 | workingset=1TBWorkingset 13 | size=crap 14 | iodepth=crap 15 | bs=crap 16 | rw=crap 17 | rwmixread=crap 18 | max=crap 19 | job=crap 20 | #./ip-collect $1 21 | #for count in 32 16 8 4 2 1; do 22 | #for count in 32 1 ; do 23 | #for count in 5 1 ; do 24 | for count in 1 ; do 25 | #Create Directories, exit on error" 26 | dir=${base_dir}/${count}vms/$workingset 27 | config_dir=${dir}/config 28 | output_dir=${dir}/output 29 | mkdir -p $config_dir >/dev/null 2>&1 30 | mkdir -p $output_dir >/dev/null 2>&1 31 | if [[ -z `ls $dir` ]]; then 32 | echo "The specified directory either does not exist or could not be accessed/created: $base_dir" 33 | echo "exiting" 34 | exit 35 | fi 36 | 37 | if [[ $count == 64 ]]; then 38 | #max=15 39 | size=1G 40 | elif [[ $count == 32 ]]; then 41 | size=8G 42 | elif [[ $count == 16 ]]; then 43 | #size=10M 44 | size=16G 45 | elif [[ $count == 6 ]]; then 46 | #size=10M 47 | size=42G 48 | elif [[ $count == 5 ]]; then 49 | #size=10M 50 | size=52G 51 | elif [[ $count == 4 ]]; then 52 | #size=10M 53 | size=64G 54 | elif [[ $count == 2 ]]; then 55 | #size=10M 56 | size=128G 57 | elif [[ $count == 1 ]]; then 58 | #size=10M 59 | size=256G 60 | fi 61 | for iosize in 64 4; do 62 | if [[ $iosize -le 16 ]]; then 63 | rw=randrw 64 | else 65 | rw=randrw 66 | fi 67 | for mix in 100 0; do 68 | if [[ $mix == 0 ]]; then 69 | #max= 15 70 | max=64 71 | elif [[ $mix = 100 ]]; then 72 | if [[ $rw == "randrw" ]]; then 73 | #max=40 74 | max=64 75 | else 76 | #max=15 77 | max=64 78 | fi 79 | else 80 | #max=15 81 | #max=10 82 | max=64 83 | fi 84 | max=$2 85 | i=$max 86 | min=0 87 | (( min = max - 1 )) 88 | while [[ $i -gt $min ]]; do 89 | echo "[global]" > config-$count 90 | echo "name=fio-test" >> config-$count 91 | echo "directory=/mnt/fio" >> config-$count 92 | echo "ioengine=libaio" >> config-$count 93 | echo "direct=1" >> config-$count 94 | echo "numjobs=4" >> config-$count 95 | echo "nrfiles=100" >> config-$count 96 | echo "runtime=600" >> config-$count 97 | echo "group_reporting=1" >> config-$count 98 | echo "time_based" >> config-$count 99 | echo "stonewall" >> config-$count 100 | echo "bs=${iosize}K" >> config-$count 101 | echo "rw=${rw}" >> config-$count 102 | echo "rwmixread=${mix}" >> config-$count 103 | echo "iodepth=${i}" >> config-$count 104 | echo "size=${size}" >> config-$count 105 | echo "ramp_time=20" >> config-$count 106 | echo "[rw]" >> config-$count 107 | echo "[randrw]" >> config-$count 108 | file="vm$count-iodepth$i-filesize$size-bs$iosize-rwmix$mix" 109 | 110 | #Run Stuff Here 111 | echo fio --client=/mnt/fio/fio-hosts-$count --output-format=normal --output=../output/output-${file} --group_reporting --section=$rw config-${file} > ${config_dir}/command-${file} 112 | cat config-$count > ${config_dir}/config-${file} 113 | fio --client=/mnt/fio/fio-hosts-$count --output-format=normal --output=${output_dir}/output-$file --group_reporting --section=$rw config-$count 114 | (( i = i - 1 )) 115 | done 116 | done 117 | done 118 | #/mnt/fio/fio-shutdown-post-$count 119 | #/mnt/fio/fio-stop 120 | done 121 | -------------------------------------------------------------------------------- /fio-wrapper.sh: -------------------------------------------------------------------------------- 1 | for run in 1; do 2 | for i in 200 100 80 60 48 44 40 36 32 28 24 20 16 12 8 4 1; do 3 | /opt/fio-parser/fio-run.ksh /mnt/fio/output/$1/run${run}/ $i 4 | done 5 | done 6 | -------------------------------------------------------------------------------- /fio.ini: -------------------------------------------------------------------------------- 1 | [global] 2 | name=fio-test 3 | directory=/mnt/nfsv41 #This is the directory where files are written 4 | ioengine=libaio #Async threads, jobs turned over to asynch threads and core moves on 5 | direct=1 #Use directio, if you use libaio and NFS this must be set to 1 enabling directio 6 | numjobs=1 #To match how many users on the system 7 | nrfiles=16 #Num files per job 8 | runtime=30 #If time_based is set, run for this amount of time 9 | group_reporting #This is used to aggregate the job results, otherwise you have lots of data to parse 10 | time_based #This setting says run the jobs until this much time has elapsed 11 | stonewall 12 | bs=8K 13 | rw=randrw #choose rw if sequential io, choose randrw for random io 14 | rwmixread=100 #<-- Modify to get different i/o distributions 15 | #iodepth=15 #<-- Modify this to get the i/o they want (latency * target op count) 16 | size=1000G #Aggregate file size per job (if nrfiles = 4, files=2.5GiB) 17 | ramp_time=20 #Warm up 18 | norandommap 19 | randrepeat=0 20 | dedupe_percentage=0 21 | buffer_compress_percentage=50 22 | buffer_compress_chunk=1024 23 | buffer_pattern="aaaa" 24 | create_serialize=0 #Setting this to zero allows fio to create all of the files in parallel 25 | 26 | [rw] 27 | -------------------------------------------------------------------------------- /fio.service: -------------------------------------------------------------------------------- 1 | #/etc/systemd/system/fio.service 2 | [Unit] 3 | Description=fio service 4 | 5 | [Service] 6 | ExecStart=/bin/bash /usr/bin/start-all.sh 7 | 8 | [Install] 9 | WantedBy=multi-user.target 10 | -------------------------------------------------------------------------------- /fio_stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3.6 2 | import sys 3 | import string 4 | import re 5 | import pprint 6 | import array as arr 7 | import pandas as pd 8 | import csv 9 | import matplotlib.pyplot as plt 10 | import xlsxwriter as xc 11 | from os import listdir 12 | from os.path import isfile, join 13 | import math 14 | 15 | 16 | #pd.set_option('display.max_columns', None) 17 | 18 | 19 | #file_name=sys.argv[1] 20 | dir=sys.argv[1] 21 | 22 | def bandwidth_conversion(line): 23 | bandwidth = (line.split(','))[1].split(' ')[1].split('=')[1] 24 | if 'Mi' in bandwidth: 25 | bandwidth = bandwidth.split('M')[0] 26 | elif 'Ki' in bandwidth: 27 | bandwidth = float(bandwidth.split('K')[0]) / 2**10 28 | elif 'Gi' in bandwidth: 29 | bandwidth = float(bandwidth.split('K')[0]) * 2**20 30 | elif 'Ti' in bandwidth: 31 | bandwidth = float(bandwidth.split('K')[0]) * 2**30 32 | return bandwidth 33 | 34 | def io_conversion(line): 35 | io = line.split(',')[0].split('=')[1] 36 | if 'k' in io: 37 | io = int( float(io[0:-1]) * 10**3) 38 | elif 'm' in io: 39 | io = int( float(io[0:-1]) * 10**6) 40 | return io 41 | 42 | 43 | def lat_conversion(line): 44 | lat = ((float(line.split(',')[2].split('=')[1]))) 45 | if line[5] == 'u': 46 | lat = lat / 10**3 47 | elif line[5] == 'n': 48 | lat = lat / 10**6 49 | elif line[5] == 's': 50 | lat = lat * 10**3 51 | return lat 52 | 53 | 54 | def read_data(dir): 55 | p=0 56 | reads=0 57 | rd_latency=0 58 | rd_bandwidth=0 59 | writes=0 60 | wrt_bandwidth=0 61 | wrt_latency=0 62 | r=0 63 | w=0 64 | parsed_line=[] 65 | fio_data=[] 66 | fio_files = [f for f in listdir(dir) if isfile(join(dir, f))] 67 | for file in fio_files: 68 | abs_path = ('%s/%s' % (dir,file)) 69 | rdf=open(abs_path,'r') 70 | reads=0 71 | rd_bandwidth=0 72 | rd_latency=0 73 | writes=0 74 | wrt_bandwidth=0 75 | wrt_latency=0 76 | for line in rdf: 77 | line=line.strip() 78 | if ('IO depths' in line): 79 | break 80 | if ('iodepth=' in line): 81 | if (p == 0): 82 | tmp=line.split('iodepth=') 83 | p+=1 84 | iodepth=tmp[1] 85 | else: 86 | p=0 87 | if ('read: IOPS' in line): 88 | rd_bandwidth=bandwidth_conversion(line) 89 | reads=io_conversion(line) 90 | if (line[0:3] == 'lat' and reads != 0 and r == 0 and 'percentiles' not in line and '%' not in line): 91 | rd_latency=lat_conversion(line) 92 | r+=1 93 | if ('write: IOPS' in line): 94 | wrt_bandwidth=bandwidth_conversion(line) 95 | writes=int(io_conversion(line)) 96 | if (line[0:3] == 'lat' and writes != 0 and 'percentiles' not in line and '%' not in line): 97 | wrt_latency=lat_conversion(line) 98 | r=0 99 | if('run' in file): 100 | tmp=file.split("run") 101 | tmp2=tmp[1].split("-") 102 | run_number=tmp2[1] 103 | total=int(reads)+int(writes) 104 | tmp=int(reads)/total 105 | read_percent=math.ceil(tmp*100) 106 | fio_data=[str(read_percent),str(iodepth),str(run_number),str(reads),str(rd_bandwidth),str(rd_latency),str(writes),str(wrt_bandwidth),str(wrt_latency)] 107 | print(fio_data) 108 | parsed_line.append(fio_data) 109 | fio_df=pd.DataFrame(parsed_line,columns=['read_percent','iodepth','run_number','reads','reads_bw_MiBs','read_lat_ms','writes','writes_bw_MiBs','write_lat_ms']) 110 | #print(fio_df) 111 | return fio_df 112 | 113 | 114 | 115 | 116 | 117 | def create_excel_spreadsheet(): 118 | cell_loc=['A','M'] 119 | iodepth=arr.array('L',[1,16,32,48,64,80,96,112,128,256,512]) 120 | icount=len(iodepth) 121 | i=0 122 | a=2 123 | r=0 124 | z=0 125 | c=0 126 | workbook=xc.Workbook('baseline__no_nconnect.xlsx') 127 | worksheet=workbook.add_worksheet() 128 | #u=int(icount/2) 129 | #while (z <= 11): 130 | while(r < 125): 131 | while (i < icount): 132 | cell_location=str(cell_loc[c]) + str(a) 133 | #print("Cell" + str(cell_location) + "Read%" + str(r) + "Iodepth" + str(iodepth[i]) + "\n") 134 | worksheet.insert_image(cell_location,'read_percent_{}_iodepth_{}.csv.png'.format(r,iodepth[i]),{'x_scale': 0.5, 'y_scale': 0.5}) 135 | c=c+1 136 | i=i+1 137 | if(i !=11): 138 | cell_location=str(cell_loc[c]) + str(a) 139 | #print("Cell" + str(cell_location) + "Read%" + str(r) + "Iodepth" + str(iodepth[i]) + "\n") 140 | worksheet.insert_image(cell_location,'read_percent_{}_iodepth_{}.csv.png'.format(r,iodepth[i]),{'x_scale': 0.5, 'y_scale': 0.5}) 141 | c=0 142 | a=a+20 143 | i=i+1 144 | i=0 145 | r=r+25 146 | workbook.close() 147 | 148 | 149 | 150 | def graph_data(file,r,j): 151 | 152 | i=j 153 | print(file,r,j) 154 | fio_t=pd.read_csv(file) 155 | fig,ax = plt.subplots(figsize=(15,6),dpi=80) 156 | plt.title("Throughput and Latency\n Reads={}% Iodpeth={}".format(r,i)) 157 | ax2=ax.twinx() 158 | if r == 0: 159 | l1=ax.plot(fio_t.run_number,fio_t.writes_bw_MiBs, label='Write Throughput',color='blue') 160 | l2=ax2.plot(fio_t.run_number,fio_t.write_lat_ms,label='write_latency',color='red') 161 | lgs=l1 + l2 162 | elif r == 100: 163 | l1=ax.plot(fio_t.run_number,fio_t.reads_bw_MiBs, label='Read Throughput',color='green') 164 | l2=ax2.plot(fio_t.run_number,fio_t.read_lat_ms,label='read_latency',color='orange') 165 | lgs=l1 + l2 166 | else: 167 | l1=ax.plot(fio_t.run_number,fio_t.reads_bw_MiBs, label='Read Throughput',color='green') 168 | l2=ax2.plot(fio_t.run_number,fio_t.read_lat_ms,label='read_latency',color='orange') 169 | l3=ax.plot(fio_t.run_number,fio_t.writes_bw_MiBs, label='Write Throughput',color='blue') 170 | l4=ax2.plot(fio_t.run_number,fio_t.write_lat_ms,label='write_latency',color='red') 171 | lgs=l1 + l2 + l3 + l4 172 | lns=[l.get_label() for l in lgs] 173 | ax.legend(lgs,lns,loc=0) 174 | ax.set_xlabel('runs') 175 | ax.set_ylabel('Throughput MB/s',color='blue') 176 | ax2.set_ylabel('Latency ms',color='red') 177 | ax2.set_ylim(0,30) 178 | plt.xticks(fio_t.run_number) 179 | ax.grid() 180 | fig.savefig('{}.png'.format(file)) 181 | plt.close() 182 | #plt.show() 183 | 184 | 185 | 186 | 187 | def parse_data(fio_df): 188 | x=0 189 | r=0 190 | iodepth=arr.array('L',[1,16,32,48,64,80,96,112,128,256,512]) 191 | icount=len(iodepth) 192 | i=0 193 | run=1 194 | y=60 195 | z=0 196 | row="" 197 | data=[] 198 | fio_df.to_csv("fio_converted_data.csv", encoding="utf-8",index=False) 199 | fioData=pd.read_csv("fio_converted_data.csv") 200 | d=fioData.sort_values(by=['read_percent','iodepth','run_number']) 201 | d.to_csv("fio_data_sorted.csv",encoding='utf-8',index=False) 202 | count=len(d.index) 203 | 204 | while (r < 125): 205 | while ( i < icount): 206 | g=d.iloc[z:y,0:9] 207 | file="read_percent_{}_iodepth_{}.csv".format(r,iodepth[i]) 208 | g.to_csv(file,encoding='utf-8',index=False) 209 | j=iodepth[i] 210 | graph_data(file,r,j) 211 | z=y 212 | y=y+60 213 | i=i+1 214 | i=0 215 | r=r+25 216 | 217 | 218 | def convert_csv(file_name): 219 | fd=open(file_name,"r") 220 | fdr=fd.readlines() 221 | x=1 222 | y=0 223 | fio_data=[] 224 | fd=[] 225 | count=len(fdr) 226 | while (x < count): 227 | replace_dashes=fdr[x].replace("-",',') 228 | fill_empty_fields=replace_dashes.replace(',,',',0,') 229 | temp1=fill_empty_fields.replace(',,',',0,') 230 | temp2=temp1.split("\n") 231 | missing=temp2[0].count(',') 232 | if (missing == 12): 233 | temp4=str(temp2[0]) + "," + "," + "," 234 | #temp5=temp4.replace(',,',',0,') 235 | temp3=temp4.split(",") 236 | else: 237 | temp3=temp2[0].split(",") 238 | fio_data.append(temp3) 239 | x=x+1 240 | fio_df=pd.DataFrame(fio_data,columns=['fio1','fio2','fio3','fio4','fio5','read_percent','iodepth','iodepth_number','run','run_number','reads','reads_bw_MiBs','read_lat_ms','writes','writes_bw_MiBs','write_lat_ms']) 241 | fio_df['total throughput MiBs']=fio_df['reads_bw_MiBs'] + fio_df['writes_bw_MiBs'] 242 | d=fio_df.to_csv("fio_converted_data.csv", encoding='utf-8',index=False) 243 | return 244 | 245 | 246 | def main(): 247 | fio_df=read_data(dir) 248 | #convert_csv(file_name) 249 | parse_data(fio_df) 250 | create_excel_spreadsheet() 251 | 252 | main() 253 | 254 | 255 | -------------------------------------------------------------------------------- /how-to-get-hosts-ready-for-fio: -------------------------------------------------------------------------------- 1 | #######FIO on SLES15 ####### 2 | add key 3 | create root key 4 | cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys 5 | ssh localhost 6 | 7 | sudo zypper update 8 | sudo zypper install -y git gcc vim nfs-utils fio net-tools-deprecated 9 | sudo git clone https://github.com/mchad1/fio-parser.git 10 | 11 | sudo cp fio.service /etc/systemd/system/fio.service 12 | sudo cp start-all /usr/bin/start-all.sh 13 | sudo systemctl enable fio.service 14 | sudo systemctl start fio.service 15 | 16 | modify /etc/fstab: 17 | Storage IP:/volume /mnt/point nfs nconnect=16,noatime 18 | 19 | golden image 20 | Clone - clone- clone 21 | 22 | (Open firewalls to port 8765 for fio) 23 | create an image from this machine and create a scaling group to quickly create and tear down fio instances 24 | 25 | #From the master: 26 | fio --clients=[ || ip addresses] configfile 27 | 28 | ####### FIO on UBUNTU ####### 29 | add key 30 | create root key 31 | cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys 32 | ssh localhost 33 | 34 | sudo apt-get update 35 | sudo apt-get install -y git gcc vim nfs-utils fio 36 | sudo git clone https://github.com/mchad1/fio-parser.git 37 | 38 | sudo cp fio.service /lib/systemd/system/fio.service 39 | sudo cp start-all /usr/bin/start-all.sh 40 | sudo systemctl enable fio.service 41 | sudo systemctl start fio.service 42 | 43 | modify /etc/fstab: 44 | Storage IP:/volume /mnt/point nfs noatime 45 | 46 | golden image 47 | Clone - clone- clone 48 | -------------------------------------------------------------------------------- /how-to-run-fio: -------------------------------------------------------------------------------- 1 | #Sample FIO Config File 2 | % vim /home/user/configfile 3 | [global] 4 | name=fio-test 5 | directory=/mnt/nfs-fio-1 #This is the directory where files are written 6 | ioengine=libaio #Async threads, jobs turned over to asynch threads and core moves on 7 | direct=1 #Use directio, if you use libaio and NFS this must be set to 1 enabling directio 8 | numjobs=1 #To match how many users on the system 9 | nrfiles=4 #Num files per job 10 | runtime=30 #If time_based is set, run for this amount of time 11 | group_reporting #This is used to aggregate the job results, otherwise you have lots of data to parse 12 | time_based #This setting says run the jobs until this much time has elapsed 13 | stonewall 14 | bs=64K 15 | rw=rw||randrw #choose rw if sequential io, choose randrw for random io 16 | rwmixread=100 #<-- Modify to get different i/o distributions 17 | iodepth=15 #<-- Modify this to get the i/o they want (latency * target op count) 18 | size=10G #Aggregate file size per job (if nrfiles = 4, files=2.5GiB) 19 | ramp_time=20 #Warm up 20 | [rw] 21 | [randrw] 22 | 23 | 24 | #Note: to figure out iodpepth for a single instance test: 25 | use a ping of the storage to find the latency. 26 | Then multiply target io count like this (io count * (latency/1000)) 27 | What ever the number is, use that for iodepth IF numjobs=1. Otherwise divide number by numjobs 28 | and set iodepth to the result. 29 | 30 | #Note: to figure out iodepth for multiple instance test: 31 | use a ping of the storage to find the latency. 32 | Then multiply target io count like this (iocount * (latency/1000)) 33 | What ever that number is, divide it by the produce of numjobs * num scale out instances 34 | Set iodepth to the result of that math. iodepth = (iocount * (latency/1000)) / (numbjobs * num scale out client) 35 | 36 | 37 | #Sample run Single instance: 38 | % fio configfile 39 | 40 | #Sample multi instance run 41 | % cat /home/users/hosts 42 | 10.10.10.11 43 | 10.10.10.12 44 | 10.10.10.13 45 | 10.10.10.14 46 | 47 | % fio --client=/home/user/hosts configfile --output=/file 48 | 49 | Note: To do this latter test, make sure that fio --server is running on all machines. For help doing this see 50 | https://github.com/mchad1/fio-parser/blob/master/get-fio-hosts-ready 51 | 52 | #To parse files, used fio-parser.py found in this repo. 53 | 54 | -------------------------------------------------------------------------------- /start-all: -------------------------------------------------------------------------------- 1 | #/usr/bin/start-all.sh 2 | fio --server 3 | --------------------------------------------------------------------------------