├── collect.sh ├── .gitmodules ├── parallel.sh ├── .gitignore ├── exec_perf.sh ├── exec_pin.sh ├── DiffOutput.sh ├── exec.sh ├── comp.sh ├── instrument.sh ├── vars.sh ├── run.sh ├── benchs.sh └── README.md /collect.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ $ANALYZE -eq 1 ]]; then 4 | # this is left as an example for the user to collect their results. 5 | : 6 | fi 7 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "Benchmarks"] 2 | ignore = dirty 3 | path = Benchs 4 | url = https://github.com/lac-dcc/Benchmarks 5 | branch = master 6 | -------------------------------------------------------------------------------- /parallel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ $EXEC -eq 1 ]]; then 4 | echo 'STARTING EXECUTION' ; 5 | parallel --tty --verbose --joblog run.log --jobs $JOBS < /tmp/run.txt ; 6 | fi 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.txt 2 | *.log 3 | *.ibc 4 | *.rbc 5 | *.exe 6 | *.bc 7 | *.o 8 | *.out 9 | table.csv 10 | count.csv 11 | stores.csv 12 | loads.csv 13 | br.csv 14 | binops.csv 15 | feat.dat 16 | prof.dat 17 | .DS_Store 18 | Benchs/sim/Output/sim.res 19 | -------------------------------------------------------------------------------- /exec_perf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function execute() { 4 | 5 | exe=$exe_name ; 6 | 7 | if [[ -n $INSTRUMENT && $INSTRUMENT -eq 1 ]]; then 8 | exe=INS_$exe_name ; 9 | fi 10 | 11 | cmd="$TIMEOUT --signal=TERM ${RUNTIME} \ 12 | $PERF_BIN stat -e $PERF_TOOL:$PERF_TYPE \ 13 | ./$exe $RUN_OPTIONS < $STDIN > $STDOUT" ; 14 | 15 | echo "$cmd" 16 | echo "cd $(pwd) && $cmd" >> $BASEDIR/run.txt ; 17 | 18 | } 19 | -------------------------------------------------------------------------------- /exec_pin.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function execute() { 4 | 5 | exe=$exe_name ; 6 | 7 | if [[ -n $INSTRUMENT && $INSTRUMENT -eq 1 ]]; then 8 | exe=INS_$exe_name ; 9 | fi 10 | 11 | cmd="$TIMEOUT --signal=TERM ${RUNTIME} \ 12 | $PIN_PATH/pin -t $PIN_LIB/obj-intel64/${PIN_TOOL}.${suffix} \ 13 | $PIN_FLAGS \ 14 | -- ./$exe $RUN_OPTIONS < $STDIN > $STDOUT" ; 15 | 16 | echo "$cmd" 17 | echo "cd $(pwd) && $cmd" >> /tmp/run.txt ; 18 | 19 | } 20 | -------------------------------------------------------------------------------- /DiffOutput.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Command line parameters: 4 | DIFF="diff" 5 | PROG=$1 6 | GOODOUTPUT=$2 7 | TESTOUTPUT=$3 8 | # Output filename: 9 | DIFFOUTPUT=/tmp/${PROG}.diff 10 | 11 | # Diff the two files. 12 | $DIFF $GOODOUTPUT $TESTOUTPUT > $DIFFOUTPUT 2>&1 13 | 14 | if [[ $? -eq 0 ]]; then 15 | echo "" 16 | echo "******************** TEST '$PROG' SUCCEED! ********************" 17 | else 18 | # They are different! 19 | echo "" 20 | echo "******************** TEST '$PROG' FAILED! ********************" 21 | echo "Execution Context Diff:" 22 | head -n 10 $DIFFOUTPUT | cat -v 23 | # rm $DIFFOUTPUT 24 | exit 1 25 | fi 26 | -------------------------------------------------------------------------------- /exec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function execute() { 4 | 5 | exe=$exe_name ; 6 | 7 | if [[ -n $INSTRUMENT && $INSTRUMENT -eq 1 ]]; then 8 | exe=INS_$exe_name 9 | fi 10 | 11 | if [[ $(pwd) =~ "cBench" ]]; then 12 | for i in $(seq 1 1); do # this must be changed 13 | cmd="./__run $i $exe_name" 14 | echo "cd $(pwd) && $cmd" >> /tmp/run.txt 15 | done 16 | return 17 | fi 18 | 19 | if [[ $DIFF -eq 1 ]]; then 20 | cmd="$TIMEOUT --signal=TERM ${RUNTIME} ./$exe $RUN_OPTIONS < $STDIN &> $bench_name.output && \ 21 | $BASEDIR/DiffOutput.sh $bench_name $bench_name.reference_output $bench_name.output" 22 | else 23 | cmd="$TIMEOUT --signal=TERM ${RUNTIME} ./$exe $RUN_OPTIONS < $STDIN &> $STDOUT" 24 | fi 25 | 26 | echo "$cmd" 27 | echo "cd $(pwd) && $cmd" >> /tmp/run.txt 28 | 29 | } 30 | -------------------------------------------------------------------------------- /comp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # this is left as an example 4 | function compile() { 5 | 6 | if [[ -n $CPU2006 && $CPU2006 -eq 1 ]]; then 7 | # rbc -> lnk 8 | $LLVM_PATH/opt -S $rbc_name -o $lnk_name 9 | else 10 | # source_files is the variable with all the files we're gonna compile 11 | parallel --tty --jobs=${JOBS} $LLVM_PATH/$COMPILER $COMPILE_FLAGS \ 12 | -Xclang -disable-O0-optnone \ 13 | -S -c -emit-llvm {} -o {.}.bc ::: "${source_files[@]}" 14 | 15 | parallel --tty --jobs=${JOBS} $LLVM_PATH/opt -S {.}.bc -o {.}.rbc ::: "${source_files[@]}" 16 | 17 | #Generate all the bcs into a big bc: 18 | $LLVM_PATH/llvm-link -S *.rbc -o $lnk_name 19 | fi 20 | 21 | # optimizations 22 | $LLVM_PATH/opt -S ${OPT} $lnk_name -o $prf_name 23 | # Compile our instrumented file, in IR format, to x86: 24 | $LLVM_PATH/llc -filetype=obj $prf_name -o $obj_name ; 25 | # Compile everything now, producing a final executable file: 26 | $LLVM_PATH/$COMPILER -lm $obj_name -o $exe_name ; 27 | 28 | } 29 | -------------------------------------------------------------------------------- /instrument.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # this is left as an example 4 | function compile() { 5 | 6 | if [[ -n $CPU2006 && $CPU2006 -eq 1 ]]; then 7 | # rbc -> lnk 8 | $LLVM_PATH/opt -S $rbc_name -o $lnk_name 9 | else 10 | # source_files is the variable with all the files we're gonna compile 11 | parallel --tty --jobs=${JOBS} $LLVM_PATH/$COMPILER $COMPILE_FLAGS \ 12 | -Xclang -disable-O0-optnone \ 13 | -S -c -emit-llvm {} -o {.}.bc ::: "${source_files[@]}" 14 | 15 | parallel --tty --jobs=${JOBS} $LLVM_PATH/opt -S {.}.bc -o {.}.rbc ::: "${source_files[@]}" 16 | 17 | #Generate all the bcs into a big bc: 18 | $LLVM_PATH/llvm-link -S *.rbc -o $lnk_name 19 | fi 20 | 21 | # Optimizations 22 | $LLVM_PATH/opt -S -load $PASS_PATH -${PASS} $lnk_name -o $prf_name 23 | 24 | # Generate obj file 25 | $LLVM_PATH/llc -filetype=obj $prf_name -o $obj_name ; 26 | 27 | # Compile everything now, producing a final executable file: 28 | $LLVM_PATH/$COMPILER -lm $obj_name -o INS_$exe_name ; 29 | } 30 | -------------------------------------------------------------------------------- /vars.sh: -------------------------------------------------------------------------------- 1 | # if 0, redirect benchmark output to /dev/null 2 | # if 1, print benchmark output to stdout 3 | [[ -n $DEBUG ]] || DEBUG=0 4 | 5 | # Specify the timeout. Default is INF(0) 6 | [[ -n $RUNTIME ]] || RUNTIME=0 7 | 8 | # Execute the benchmark 9 | [[ -n $EXEC ]] || EXEC=1 10 | 11 | # Compile 12 | [[ -n $COMPILE ]] || COMPILE=1 13 | 14 | # Instrument 15 | [[ -n $INSTRUMENT ]] || INSTRUMENT=0 16 | 17 | # JOBS 18 | [[ -n $JOBS ]] || JOBS=1 19 | 20 | # ANALYZE 21 | [[ -n $ANALYZE ]] || ANALYZE=0 22 | 23 | # INSTRUMENT 24 | [[ -n $INSTRUMENT ]] || INSTRUMENT=0 25 | 26 | # DIFF 27 | [[ -n $DIFF ]] || DIFF=0 28 | 29 | [[ $DIFF -eq 1 && $DEBUG -eq 1 ]] && { 30 | echo "Can't use DIFF=1 & DEBUG=1 at the same time" 31 | exit 1 32 | } 33 | 34 | # Remove all temp files 35 | [[ -n CLEAN ]] || CLEAN=0 36 | 37 | # Set the lib suffix. 38 | suffix="dylib" 39 | if [[ $(uname -s) == "Linux" ]]; then 40 | suffix="so" 41 | fi 42 | 43 | # if we're on osx, we must use `gtimeout` instead of `timeout` 44 | # `gtimeout` can be download from homebrew 45 | TIMEOUT=timeout 46 | if [[ $(uname -s) == "Darwin" ]]; then 47 | TIMEOUT=gtimeout 48 | fi 49 | 50 | 51 | # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- 52 | 53 | # LLVM_PATH => The place where I have all the LLVM tools 54 | LLVM_PATH="" 55 | 56 | [[ -d "${LLVM_PATH}" ]] || { 57 | echo "One must define LLVM_PATH before running tf" 58 | exit 1 59 | } 60 | 61 | # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- 62 | 63 | # THIS PART IS LEFT AS AN EXAMPLE FOR THE PEOPLE WORKING WITH PIN! 64 | 65 | # PIN 66 | [[ -n $PIN ]] || PIN=0 67 | 68 | if [[ $PIN -eq 1 ]]; then 69 | # PIN_PATH => The place where I keep the pin source code 70 | PIN_PATH="" 71 | [[ -n $PIN_PATH ]] || { 72 | echo "One must define the PIN before when PIN=1" 73 | exit 1 74 | } 75 | 76 | # PIN_LIB => The place where I keep the Pin lib implemented. 77 | PIN_LIB="" 78 | [[ -n $PIN_LIB ]] || { 79 | echo "One must define PIN_LIB when PIN=1" 80 | exit 1 81 | } 82 | 83 | # PIN_TOOL => The tool used 84 | [[ -z $PIN_TOOL ]] || { 85 | echo "You must define a PIN_TOOL variable before using tf with PIN" 86 | exit 1 87 | } 88 | 89 | # PIN_FLAGS => Flags to pass to PIN 90 | [[ -n $PIN_FLAGS ]] || PIN_FLAGS=" " 91 | 92 | echo "PIN_PATH is set to $PIN_PATH" 93 | echo "PIN_LIB is set to $PIN_LIB" 94 | echo "PIN_TOOL is set to $PIN_TOOL" 95 | 96 | echo "Compiling PIN TOOLS" 97 | PIN_ROOT=$PIN_PATH make -C $PIN_LIB || { 98 | echo "Error compiling PIN TOOLS" 99 | exit 1 100 | } 101 | 102 | fi 103 | 104 | # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- 105 | 106 | # perf 107 | [[ -n $OCPERF ]] || OCPERF=0 108 | 109 | if [[ $OCPERF -eq 1 ]]; then 110 | #PERF EVENT 111 | [[ -n $PERF_TOOL ]] || PERF_TOOL="mem_uops_retired.all_loads" # mem-stores 112 | 113 | #USER OR KERNEL SPACE 114 | [[ -n $PERF_TYPE ]] || PERF_TYPE="u" 115 | 116 | #OUTPUT FILE 117 | [[ -n $PERF_FILE ]] || PERF_FILE="perf_${PERF_TOOL}_${PERF_TYPE}.out" 118 | 119 | PERF_BIN="" 120 | [[ -n $PERF_BIN ]] || { 121 | echo "One must define PERF_BIN when PERF=1" 122 | exit 1 123 | } 124 | 125 | echo "PERF_BIN is set to $PERF_BIN" 126 | echo "PERF_TOOL is set to $PERF_TOOL" 127 | echo "PERF_TYPE is set to $PERF_TYPE" 128 | fi 129 | 130 | # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- 131 | 132 | BASEDIR="$(pwd)" 133 | 134 | BENCHSDIR="$BASEDIR/Benchs/" 135 | 136 | # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- 137 | 138 | echo "#########################" 139 | echo "DEBUG is set to $DEBUG" 140 | echo "RUNTIME is set to $RUNTIME" 141 | echo "CLEAN is set to $CLEAN" 142 | echo "PIN is set to $PIN" 143 | echo "EXEC is set to $EXEC" 144 | echo "COMPILE is set to $COMPILE" 145 | echo "INSTRUMENT is set to $INSTRUMENT" 146 | echo "suffix is set to $suffix" # .so or .dylib 147 | echo "BASEDIR is set to $BASEDIR" 148 | echo "Benchmarks dir is set to $BENCHSDIR" 149 | echo "PASS is set to $PASS" 150 | echo "DIFF is set to $DIFF" 151 | echo "#########################" 152 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | trap 'echo "Killing build_exec.sh script" ; exit' INT TERM 4 | 5 | # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- 6 | 7 | function cleanup() { 8 | rm -f *.bc 9 | rm -f *.rbc 10 | rm -f *.ibc 11 | rm -f *.o 12 | } 13 | 14 | function cleanup_all() { 15 | rm -f *.bc 16 | rm -f *.rbc 17 | rm -f *.ibc 18 | rm -f *.o 19 | rm -f *.exe 20 | rm -f *.txt 21 | } 22 | 23 | function unset_vars() { 24 | unset COMPILER 25 | unset STDIN 26 | unset STDOUT 27 | unset RUN_OPTIONS 28 | 29 | unset CPU2006 30 | } 31 | 32 | function set_vars(){ 33 | source info.sh 34 | 35 | # Let's set the variables that are unset 36 | 37 | # sometimes we need to use clang++ 38 | [[ -n $COMPILER ]] || COMPILER=clang 39 | # We can specify STDIN to something other than /dev/stdin 40 | [[ -n $STDIN ]] || STDIN=/dev/null 41 | # And STDOUT default is /dev/null. 42 | [[ -n $STDOUT ]] || STDOUT=/dev/null 43 | # But if we set DEBUG=1, than we ignore the previous definition of STDOUT 44 | if [[ $DEBUG == 1 ]]; then 45 | STDOUT=/dev/stdout ; 46 | fi 47 | 48 | if [[ $(pwd) =~ "cpu2006" ]]; then 49 | echo "Setting CPU2006=1" 50 | CPU2006=1 51 | fi 52 | 53 | # Common files used by comp.sh and instrument.sh 54 | if [[ -n $CPU2006 && $CPU2006 -eq 1 ]]; then 55 | if [[ $(uname -s) == "Linux" ]]; then 56 | rbc_name="$bench_name.linux" 57 | else 58 | rbc_name="$bench_name.llvm" 59 | fi 60 | fi 61 | 62 | lnk_name="$bench_name.rbc" 63 | prf_name="$bench_name.ibc" 64 | obj_name="$bench_name.o" 65 | exe_name="$bench_name.exe" 66 | 67 | # options for exe name 68 | if [[ -n $INSTRUMENT && $INSTRUMENT -eq 1 ]]; then 69 | exe_name=INS_$exe_name 70 | fi 71 | 72 | if [[ -n $ASAN && $ASAN -eq 1 ]]; then 73 | exe_name=ASAN_$exe_name 74 | fi 75 | 76 | if [[ $SSA -eq 0 ]]; then 77 | exe_name=NO_SSA_$exe_name ; 78 | fi 79 | 80 | } 81 | 82 | # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- 83 | 84 | function walk() { 85 | 86 | if [[ $# == 0 ]]; then 87 | echo "Error, you must specify the directories this script must compile" 88 | echo 'ex: walk $( ls -d */ )' 89 | exit 90 | else 91 | dirs=("$@") 92 | fi 93 | 94 | parent_dir=$(pwd) 95 | 96 | for dir in "${dirs[@]}"; do 97 | cd "$parent_dir"/"$dir" ; 98 | 99 | d=$(basename $(pwd)) 100 | echo "Sourcing info.sh from $(pwd)" ; 101 | 102 | if [[ -n $CLEAN && $CLEAN -eq 1 ]]; then 103 | cleanup_all ; 104 | continue ; 105 | fi 106 | 107 | set_vars ; 108 | cleanup ; 109 | 110 | if [[ $COMPILE -eq 1 ]]; then 111 | compile ; 112 | fi 113 | 114 | execute ; 115 | 116 | unset_vars ; 117 | 118 | echo 119 | echo "###############" 120 | echo 121 | 122 | cd "$parent_dir" 123 | 124 | done 125 | } 126 | 127 | # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- # -- 128 | 129 | source "vars.sh" 130 | source "benchs.sh" 131 | source "comp.sh" 132 | source "exec.sh" 133 | 134 | if [[ -n $CLEAN && $CLEAN -eq 1 ]]; then 135 | echo "REMOVING ALL TEMP FILES!" 136 | 137 | for bench in "${benchs[@]}"; do 138 | cd $BENCHSDIR 139 | echo "Removing from $bench" ; 140 | cd $bench ; 141 | $bench ; 142 | echo "" ; 143 | done 144 | 145 | exit 0 146 | fi 147 | 148 | if [[ -n $PIN && $PIN -eq 1 ]]; then 149 | # replace the function `execute` 150 | source "exec_pin.sh" 151 | fi 152 | 153 | if [[ -n $OCPERF && $OCPERF -eq 1 ]]; then 154 | # replace the function `execute` 155 | source "exec_perf.sh" 156 | fi 157 | 158 | if [[ -n $INSTRUMENT && $INSTRUMENT -eq 1 ]]; then 159 | # replace the compile function 160 | source "instrument.sh" 161 | fi 162 | 163 | if [[ -n $SANITIZE && $SANITIZE -eq 1 ]]; then 164 | # replace the compile function 165 | source "sanitize.sh" 166 | fi 167 | 168 | rm -f /tmp/run.txt 169 | touch /tmp/run.txt 170 | 171 | if [[ "$#" -ne 0 ]]; then 172 | # check if the input is a file 173 | if [[ -f "$@" ]]; then 174 | echo "Reading input file..." 175 | # Read the content into "${benchs[@]}" array 176 | IFS=$'\n' read -d '' -r -a benchs < "$@" 177 | else 178 | benchs=( "$@" ) 179 | fi 180 | 181 | walk "${benchs[@]}" ; 182 | 183 | else 184 | for bench in "${benchs[@]}"; do 185 | cd $BENCHSDIR 186 | echo "Starting $bench" ; 187 | cd $bench ; 188 | $bench ; 189 | done 190 | fi 191 | 192 | cd $BASEDIR ; 193 | 194 | source "parallel.sh" 195 | source "collect.sh" 196 | -------------------------------------------------------------------------------- /benchs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function ASC_Sequoia() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 4 | function BenchmarkGame() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 5 | function BitBench() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 6 | function CoyoteBench() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 7 | function Dhrystone() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 8 | function DOE_ProxyApps_C() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 9 | function McGill() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 10 | function MiBench() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 11 | function Misc() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 12 | function Shootout() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 13 | function Stanford() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 14 | function Ptrdist() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 15 | function Trimaran() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 16 | function TSVC() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 17 | function NPB-serial() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 18 | function VersaBench() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 19 | function FreeBench() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 20 | function MallocBench() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 21 | function McCat() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 22 | function Olden() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 23 | function Prolangs-C() { dirs=($( ls -d */ )); walk "${dirs[@]}" ; } 24 | 25 | function Fhourstones() { walk "." ; } 26 | function Fhourstones_31() { walk "." ; } 27 | function Linpack() { walk "." ; } 28 | function ASCI_Purple() { walk "." ; } 29 | function SciMark2-C() { walk "." ; } 30 | function sim() { walk "." ; } 31 | function mafft() { walk "." ; } 32 | function tramp3d-v4() { walk "." ; } 33 | function llubenchmark() { walk "." ; } 34 | function nbench() { walk "." ; } 35 | function PAQ8p() { walk "." ; } 36 | 37 | function mediabench() { 38 | dirs=("adpcm/rawcaudio" "adpcm/rawdaudio" "g721/g721encode" "jpeg/jpeg-6a" 39 | "gsm/toast" "mpeg2/mpeg2dec") 40 | walk "${dirs[@]}" 41 | } 42 | 43 | function PolyBench(){ 44 | dirs=("linear-algebra/kernels/2mm" "linear-algebra/kernels/3mm" "linear-algebra/kernels/atax" 45 | "linear-algebra/kernels/bicg" "linear-algebra/kernels/doitgen" "linear-algebra/kernels/mvt" 46 | "linear-algebra/solvers/cholesky" "linear-algebra/solvers/durbin" "linear-algebra/solvers/gramschmidt" 47 | "linear-algebra/solvers/lu" "linear-algebra/solvers/ludcmp" "linear-algebra/solvers/trisolv" 48 | "linear-algebra/blas/gemm" "linear-algebra/blas/gemver" "linear-algebra/blas/gesummv" 49 | "linear-algebra/blas/symm" "linear-algebra/blas/syr2k" "linear-algebra/blas/syrk" 50 | "linear-algebra/blas/trmm" 51 | "datamining/correlation" "datamining/covariance" 52 | "medley/floyd-warshall" "medley/deriche" "medley/nussinov" 53 | "stencils/adi" "stencils/fdtd-2d" "stencils/heat-3d" "stencils/jacobi-1d" "stencils/jacobi-2d" 54 | "stencils/seidel-2d") 55 | walk "${dirs[@]}" 56 | } 57 | 58 | function cBench(){ 59 | dirs=("security_rijndael_d/src" "automotive_susan_c/src" 60 | "security_rijndael_e/src" "consumer_jpeg_c/src" "consumer_lame/src" 61 | "bzip2e/src" "telecom_adpcm_d/src" "bzip2d/src" 62 | "network_dijkstra/src" "office_stringsearch1/src" 63 | "consumer_tiffdither/src" "automotive_qsort1/src" 64 | "consumer_jpeg_d/src" "automotive_susan_e/src" 65 | "automotive_bitcount/src" "security_blowfish_e/src" "consumer_tiff2bw/src" 66 | "telecom_CRC32/src" "security_sha/src" "consumer_tiffmedian/src" 67 | "automotive_susan_s/src" "network_patricia/src" "telecom_adpcm_c/src" 68 | "consumer_tiff2rgba/src" "security_blowfish_d/src") 69 | 70 | cant=("consumer_mad/src" "office_ispell/src" "office_ghostscript/src" "office_rsynth/src" 71 | "security_pgp_d/src" "security_pgp_e/src" "telecom_gsm/src") 72 | 73 | walk "${dirs[@]}" 74 | } 75 | 76 | function cpu2006(){ 77 | dirs=("400.perlbench/rbc/" "401.bzip2/rbc/" "403.gcc/rbc/" "429.mcf/rbc/" 78 | "433.milc/rbc/" "444.namd/rbc/" "445.gobmk/rbc/" 79 | "456.hmmer/rbc/" "458.sjeng/rbc/" "462.libquantum/rbc/" 80 | "464.h264ref/rbc/" "470.lbm/rbc/") 81 | walk "${dirs[@]}" 82 | } 83 | 84 | benchs=( "ASC_Sequoia" "BenchmarkGame" "BitBench" "CoyoteBench" "DOE_ProxyApps_C" 85 | "Dhrystone" "McGill" "MiBench" "Misc" "Shootout" "Stanford" "Fhourstones" 86 | "Linpack" "mediabench" "ASCI_Purple" "Fhourstones_31" "PolyBench" 87 | "SciMark2-C" "sim" "mafft" "tramp3d-v4" "llubenchmark" "nbench" "Ptrdist" 88 | "Trimaran" "TSVC" "PAQ8p" "NPB-serial" "VersaBench" "FreeBench" 89 | "MallocBench" "McCat" "Olden" "Prolangs-C" "cpu2006") 90 | 91 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tf 2 | Easily run llvm test-suite benchmarks. This is a simple test framework built using bash. 3 | 4 | ## What tf can do for you? 5 | - Run programs with [PIN](https://software.intel.com/en-us/articles/pin-a-dynamic-binary-instrumentation-tool) 6 | - Compile and run benchmarks with your own llvm pass 7 | - Parallel execution 8 | - Easily collect statistics 9 | - Run with a time limit 10 | 11 | ## Benchmarks 12 | 13 | Here is a list of benchmarks available in this repo: 14 | 15 | - ASCI_Purple 16 | - ASC_Sequoia 17 | - BenchmarkGame 18 | - BitBench 19 | - CoyoteBench 20 | - Dhrystone 21 | - DOE_ProxyApps_C 22 | - Fhourstones 23 | - Fhourstones_31 24 | - FreeBench 25 | - Linpack 26 | - llubenchmark 27 | - mafft 28 | - MallocBench 29 | - McCat 30 | - McGill 31 | - mediabench 32 | - MiBench 33 | - Misc 34 | - nbench 35 | - NPB-serial 36 | - Olden 37 | - PAQ8p (**C++**) 38 | - Prolangs-C 39 | - Ptrdist 40 | - SciMark2-C 41 | - Shootout 42 | - sim 43 | - Stanford 44 | - tramp3d-v4 (**C++**) 45 | - Trimaran 46 | - TSVC 47 | - VersaBench 48 | - PolyBench 49 | 50 | Benchmarks are stored in a different [repo](https://github.com/guilhermeleobas/Benchmarks). 51 | 52 | ## Requirements 53 | - timeout or [gtimeout](https://stackoverflow.com/questions/3504945/timeout-command-on-mac-os-x) if you're on OS X. 54 | - [gnu-parallel](http://brewformulas.org/Parallel) 55 | - Any version of LLVM. 56 | 57 | ## Getting tf 58 | Simply clone this repository **recursively** 59 | ```bash 60 | git clone --recursive git@github.com:guilhermeleobas/tf.git 61 | ``` 62 | 63 | ## Building LLVM (if you gonna use your pass) 64 | 65 | Below is a set of instructions to build LLVM 3.8 from source. Remember the path you build LLVM because you gonna need later. 66 | 67 | ```bash 68 | svn co http://llvm.org/svn/llvm-project/llvm/tags/RELEASE_380/final llvm38 69 | cd llvm38/tools 70 | svn co http://llvm.org/svn/llvm-project/cfe/tags/RELEASE_380/final clang 71 | cd .. 72 | mkdir build 73 | cd build 74 | ../configure 75 | make -j8 76 | ``` 77 | 78 | ## Usage 79 | 80 | The first thing you need to do is select which benchmarks you want to execute. Open `benchs.sh` and add the benchmark you want to run into the variable `benchs`. 81 | 82 | Then, go to the file `vars.sh` and set the variable `$LLVM_PATH` to where you build LLVM. This path is something like `/path/to/llvm/build/Release+Asserts/bin` or `/path/to/llvm/build/bin` in newer versions. 83 | 84 | ### Compiling Benchmarks 85 | 86 | Simply type `./run.sh`. If you only want to compile, set the flag `EXEC=0` before calling `run.sh`. 87 | 88 | Tip: You can skip compilation by setting `COMPILE=0` before calling `run.sh`. 89 | 90 | ### Running with a time limit 91 | 92 | `RUNTIME=8m ./run.sh` or change the file `vars.sh`. 93 | 94 | `RUNTIME` receives a number followed by an optional unit: 95 | - `s` for seconds (the default) 96 | - `m` for minutes 97 | - `h` for hours 98 | - `d` for days 99 | 100 | After the specified time interval, **timeout** will send a `TERM` signal to the benchmark process. 101 | 102 | Tip: Set `RUNTIME=0` to run indefinitely. 103 | 104 | ### Compare output 105 | 106 | Just run with `DIFF=1` and **tf** will compare the output produced by the binary with a reference output. 107 | 108 | ### Parallel execution 109 | 110 | We use gnu-parallel to run the benchmarks, even if you're running things sequentially. To run in parallel, change the variable `$JOBS` in `vars.sh` or call `JOBS=njobs ./run.sh` from the command line. 111 | 112 | ### Using with Intel PIN 113 | 114 | You need to set a few variables before. Go to the file `vars.sh` and change: 115 | - `PIN_PATH=/path/to/pin/` 116 | - `PIN_LIB=/path/to/pintool/` 117 | 118 | The later must point to where your Pintool **source code** is, this way we can easily build your Pintool for you. 119 | 120 | Now, call `PIN=1 ./run.sh` 121 | 122 | ### Collecting stats 123 | 124 | **gnu-parallel** creates a logfile called `run.log`. This log contains the job sequence, which host the job was run on, the start time and run time, how much data was transferred, the exit value, the signal that killed the job, and finally the command being run. 125 | 126 | You can easily parse the logfile to a csv using python and [pandas](https://pandas.pydata.org/): 127 | 128 | ```python 129 | import pandas as pd 130 | pd.read_csv('run.log', sep='\t').to_csv('run.csv') 131 | ``` 132 | 133 | You can also add your own code in the file `collect.sh`. **tf** will execute this file after all benchmarks have finished executing. 134 | 135 | ### Compiling benchmarks with your own LLVM pass 136 | 137 | See `instrument.sh`file. You can control how each benchmark is compiled there. 138 | 139 | Add the path of your pass to the variable `pass_path` at the beginning of the `instrument.sh` file. Then, call **tf** with `INSTRUMENT=1 PASS=YourPassNameHere` 140 | 141 | ```bash 142 | COMPILE=1 INSTRUMENT=1 PASS=YourPassNameHere EXEC=1 ./run.sh 143 | ``` 144 | 145 | ------------ 146 | 147 | ### Adding more benchmarks 148 | 149 | 1) For each folder that contains .c files, i.e., the folder that will 150 | contain the executable file that you are creating, add the following 151 | `info.sh` file there: 152 | 153 | ```bash 154 | bench_name="XX" 155 | 156 | source_files=( "foo.c" "bar.c" "baz.c" "..." ) 157 | COMPILE_FLAGS=" -lm " 158 | COMPILER="clang" # or clang++ for C++ programs 159 | RUN_OPTIONS=" irsmk_input " 160 | STDIN=" file.in " 161 | ``` 162 | 163 | The last two variables are used when `tf` creates the command that will be executed: 164 | 165 | ```bash 166 | timeout -signal=TERM ${RUNTIME} ./${bench_name}.exe ${RUN_OPTIONS} < ${STDIN} > /dev/null 167 | ``` 168 | 169 | 2) Add a function into `benchs.sh`, for the new benchmark. 170 | 171 | If the benchmark does not contain subfolders, add: 172 | ```bash 173 | function Fhourstones() { 174 | walk "." ; 175 | } 176 | ``` 177 | otherwise, add: 178 | ```bash 179 | function Misc() { 180 | dirs=($( ls -d */ )) ; # list every folder inside Misc/ 181 | walk "${dirs[@]}" ; 182 | } 183 | ``` 184 | 185 | -------- 186 | 187 | ### TIPS 188 | Compiling all benchmarks takes a **considerable** amount of time. Is a good idea to compile them first and execute later: 189 | 190 | ```bash 191 | COMPILE=1 EXEC=0 ./run.sh # To compile 192 | COMPILE=0 EXEC=1 ./run.sh # To execute 193 | ``` 194 | 195 | Also, run benchmarks in parallel whenever you can. Running 220 benchmarks sequentially with a time limit of 8 minutes takes 29 hours to complete. 196 | --------------------------------------------------------------------------------