├── scripts ├── build_phoenix.sh ├── plot.sh ├── build_benchmarks.sh ├── build_parsec.sh ├── build.sh ├── build_qemu.sh ├── run_benchmarks.sh ├── run_benchmarks_test.sh ├── run_math.sh ├── run_sqlite.sh ├── run_cas.sh ├── run_phoenix.sh ├── run_parsec.sh ├── run_openssl.sh └── download_benchmarks.sh ├── sourceme ├── configs ├── nlib │ ├── math.nmi │ ├── sqlite.nmi │ └── openssl.nmi ├── qemu.config ├── risotto.config ├── tcg-tso.config ├── no-fences.config └── native.config ├── .gitmodules ├── qemu.nix ├── default.nix ├── plots ├── fig15.ipynb ├── fig14.ipynb ├── fig12.ipynb └── fig13.ipynb └── README.md /scripts/build_phoenix.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | cd ${RISOTTO_ROOT}/phoenix/phoenix-2.0 4 | 5 | make 6 | 7 | cd - 8 | -------------------------------------------------------------------------------- /scripts/plot.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | nix-shell --run 'jupyter nbconvert --to notebook --inplace --execute plots/fig*.ipynb' ${RISOTTO_ROOT}/default.nix 5 | -------------------------------------------------------------------------------- /sourceme: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export RISOTTO_ROOT=${PWD} 4 | 5 | git submodule init 6 | git submodule update --recursive 7 | 8 | mkdir -p ${RISOTTO_ROOT}/results 9 | -------------------------------------------------------------------------------- /scripts/build_benchmarks.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ${RISOTTO_ROOT}/scripts/build_parsec.sh 4 | ${RISOTTO_ROOT}/scripts/build_phoenix.sh 5 | ${RISOTTO_ROOT}/scripts/build_sqlite.sh 6 | ${RISOTTO_ROOT}/scripts/build_cas.sh 7 | -------------------------------------------------------------------------------- /configs/nlib/math.nmi: -------------------------------------------------------------------------------- 1 | # ------------------------ 2 | library "libm.so.6"; 3 | 4 | # Maths 5 | f64 exp(f64 v); 6 | f64 sqrt(f64 v); 7 | f64 log(f64 v); 8 | 9 | f64 sin(f64 v); 10 | f64 cos(f64 v); 11 | f64 tan(f64 v); 12 | f64 asin(f64 v); 13 | f64 acos(f64 v); 14 | f64 atan(f64 v); 15 | -------------------------------------------------------------------------------- /scripts/build_parsec.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | cd ${RISOTTO_ROOT}/benchmarks/parsec-benchmark 4 | 5 | ./get-inputs 6 | 7 | source env.sh 8 | parsecmgmt -a build -p blackscholes bodytrack canneal facesim ferret fluidanimate freqmine streamcluster swaptions vips 9 | 10 | cd - 11 | -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Build qemu 4 | nix-shell --run ${RISOTTO_ROOT}/scripts/build_qemu.sh ${RISOTTO_ROOT}/qemu.nix 5 | 6 | if [ "$1" == "build" ] ; then 7 | # Build benchmarks binaries 8 | nix-shell --run ${RISOTTO_ROOT}/scripts/build_benchmarks.sh ${RISOTTO_ROOT}/default.nix 9 | else 10 | # Download benchmarks binaries 11 | nix-shell --run ${RISOTTO_ROOT}/scripts/download_benchmarks.sh ${RISOTTO_ROOT}/default.nix 12 | fi 13 | -------------------------------------------------------------------------------- /configs/nlib/sqlite.nmi: -------------------------------------------------------------------------------- 1 | # ------------------------ 2 | # library "libsqlite3.so.0"; 3 | # library "/nix/store/wlmvw1djy5rwxl431p74hlqj2w0jd05l-sqlite-3.35.5/lib/libsqlite3.so.0.8.6"; 4 | library "/nix/store/gsss4560f68n8gfn0zx5gb7mgc0gp49n-sqlite-3.35.5/lib/libsqlite3.so.0"; 5 | 6 | i32 sqlite3_exec(ptr db, const ptr sql, ptr callback, ptr args, ptr err); 7 | i32 sqlite3_open(const ptr filename, ptr db); 8 | i32 sqlite3_close(ptr db); 9 | const ptr sqlite3_errmsg(ptr db); 10 | -------------------------------------------------------------------------------- /configs/qemu.config: -------------------------------------------------------------------------------- 1 | # PARSEC 2 | PARSEC_DIR=build/parsec-bins 3 | 4 | # PHOENIX 5 | PHOENIX_DIR=build/phoenix-x86 6 | 7 | # Openssl 8 | OPENSSL_BIN=/nix/store/10jzrky6qp59qm75zwxxjysyzphywznj-openssl-1.1.1l-bin/bin/openssl 9 | 10 | # Math 11 | MATH_BIN=build/math-bins/math.x86_64 12 | 13 | # SQlite microbench 14 | SQLITE_BENCH_BIN=build/sqlite-bins/sqlite-bench.x86_64 15 | 16 | # cas benchmark 17 | CAS_BENCH_BIN=build/cas/cas.x86_64 18 | 19 | # QEMU 20 | QEMU_PATH=build/qemu/master-6.1.0 21 | -------------------------------------------------------------------------------- /configs/risotto.config: -------------------------------------------------------------------------------- 1 | # PARSEC 2 | PARSEC_DIR=build/parsec-bins 3 | 4 | # PHOENIX 5 | PHOENIX_DIR=build/phoenix-x86 6 | 7 | # Openssl 8 | OPENSSL_BIN=/nix/store/10jzrky6qp59qm75zwxxjysyzphywznj-openssl-1.1.1l-bin/bin/openssl 9 | 10 | # Math 11 | MATH_BIN=build/math-bins/math.x86_64 12 | 13 | # SQlite microbench 14 | SQLITE_BENCH_BIN=build/sqlite-bins/sqlite-bench.x86_64 15 | 16 | # cas benchmark 17 | CAS_BENCH_BIN=build/cas/cas.x86_64 18 | 19 | # QEMU 20 | QEMU_PATH=build/qemu/risotto 21 | -------------------------------------------------------------------------------- /configs/tcg-tso.config: -------------------------------------------------------------------------------- 1 | # PARSEC 2 | PARSEC_DIR=build/parsec-bins 3 | 4 | # PHOENIX 5 | PHOENIX_DIR=build/phoenix-x86 6 | 7 | # Openssl 8 | OPENSSL_BIN=/nix/store/10jzrky6qp59qm75zwxxjysyzphywznj-openssl-1.1.1l-bin/bin/openssl 9 | 10 | # Math 11 | MATH_BIN=build/math-bins/math.x86_64 12 | 13 | # SQlite microbench 14 | SQLITE_BENCH_BIN=build/sqlite-bins/sqlite-bench.x86_64 15 | 16 | # cas benchmark 17 | CAS_BENCH_BIN=build/cas/cas.x86_64 18 | 19 | # QEMU 20 | QEMU_PATH=build/qemu/tcg-tso 21 | -------------------------------------------------------------------------------- /configs/no-fences.config: -------------------------------------------------------------------------------- 1 | # PARSEC 2 | PARSEC_DIR=build/parsec-bins 3 | 4 | # PHOENIX 5 | PHOENIX_DIR=build/phoenix-x86 6 | 7 | # Openssl 8 | OPENSSL_BIN=/nix/store/10jzrky6qp59qm75zwxxjysyzphywznj-openssl-1.1.1l-bin/bin/openssl 9 | 10 | # Math 11 | MATH_BIN=build/math-bins/math.x86_64 12 | 13 | # SQlite microbench 14 | SQLITE_BENCH_BIN=build/sqlite-bins/sqlite-bench.x86_64 15 | 16 | # cas benchmark 17 | CAS_BENCH_BIN=build/cas/cas.x86_64 18 | 19 | # QEMU 20 | QEMU_PATH=build/qemu/no-fences 21 | -------------------------------------------------------------------------------- /configs/native.config: -------------------------------------------------------------------------------- 1 | # PARSEC 2 | PARSEC_DIR=build/parsec-bins 3 | 4 | # PHOENIX 5 | PHOENIX_DIR=build/phoenix-aarch64 6 | 7 | # Openssl 8 | OPENSSL_BIN=/nix/store/42c9xn9ikr7nbaw76vva6h8yc1vyymv4-openssl-1.1.1l-bin/bin/openssl 9 | 10 | # Math 11 | MATH_BIN=build/math-bins/math.aarch64 12 | 13 | # SQlite microbench 14 | SQLITE_BENCH_BIN=build/sqlite-bins/sqlite-bench.aarch64 15 | 16 | # cas benchmark 17 | CAS_BENCH_BIN=build/cas/cas.aarch64 18 | 19 | # QEMU 20 | QEMU_PATH=build/qemu/master-6.1.0 21 | -------------------------------------------------------------------------------- /scripts/build_qemu.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | for p in master-6.1.0 no-fences tcg-tso risotto; do 4 | # switch the qemu repo to the proper branch 5 | cd ${RISOTTO_ROOT}/qemu 6 | git switch ${p} 7 | 8 | # create the build directory 9 | mkdir -p ${RISOTTO_ROOT}/build/qemu/${p} 10 | cd ${RISOTTO_ROOT}/build/qemu/${p} 11 | 12 | # build qemu 13 | ${RISOTTO_ROOT}/qemu/configure --target-list=x86_64-linux-user 14 | make -j $(nproc) 15 | 16 | cd ${RISOTTO_ROOT} 17 | done 18 | -------------------------------------------------------------------------------- /scripts/run_benchmarks.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | taskset -p -c 0-111 $$ 4 | 5 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_parsec.sh ${RISOTTO_ROOT}/default.nix 6 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_phoenix.sh ${RISOTTO_ROOT}/default.nix 7 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_math.sh ${RISOTTO_ROOT}/default.nix 8 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_openssl.sh ${RISOTTO_ROOT}/default.nix 9 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_sqlite.sh ${RISOTTO_ROOT}/default.nix 10 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_cas.sh ${RISOTTO_ROOT}/default.nix 11 | -------------------------------------------------------------------------------- /scripts/run_benchmarks_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export NR_RUNS=3 4 | 5 | taskset -p -c 0-111 $$ 6 | 7 | DATASET=simsmall nix-shell --run ${RISOTTO_ROOT}/scripts/run_parsec.sh ${RISOTTO_ROOT}/default.nix 8 | DATASET=small nix-shell --run ${RISOTTO_ROOT}/scripts/run_phoenix.sh ${RISOTTO_ROOT}/default.nix 9 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_math.sh ${RISOTTO_ROOT}/default.nix 10 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_openssl.sh ${RISOTTO_ROOT}/default.nix 11 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_sqlite.sh ${RISOTTO_ROOT}/default.nix 12 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_cas.sh ${RISOTTO_ROOT}/default.nix 13 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "qemu"] 2 | path = qemu 3 | url = https://github.com/rgouicem/qemu.git 4 | [submodule "benchmarks/phoenix"] 5 | path = benchmarks/phoenix 6 | url = https://github.com/rgouicem/phoenix.git 7 | [submodule "a2a-benchmarks"] 8 | path = a2a-benchmarks 9 | url = https://github.com/rgouicem/a2a-benchmarks.git 10 | [submodule "proofs"] 11 | path = proofs 12 | url = https://github.com/binary-translation/risotto-proofs.git 13 | [submodule "benchmarks/sqlite-bench"] 14 | path = benchmarks/sqlite-bench 15 | url = https://github.com/rgouicem/sqlite-bench 16 | [submodule "benchmarks/cas-benchmark"] 17 | path = benchmarks/cas 18 | url = https://github.com/rgouicem/cas-benchmark 19 | [submodule "benchmarks/math-bench"] 20 | path = benchmarks/math 21 | url = https://github.com/rgouicem/math-bench 22 | -------------------------------------------------------------------------------- /qemu.nix: -------------------------------------------------------------------------------- 1 | let 2 | tarball = "https://github.com/NixOS/nixpkgs/archive/f0869b1a2c0b150aac26e10bb5c2364ffb2e804f.tar.gz"; 3 | 4 | pkgs = import (fetchTarball tarball) {}; 5 | 6 | pypkgs = [ pkgs.python3Packages.pandas 7 | pkgs.python3Packages.tqdm 8 | pkgs.python3Packages.matplotlib ]; 9 | 10 | deps = p: [ 11 | p.gcc10 12 | p.zlib 13 | p.curl 14 | p.getopt 15 | p.flex 16 | p.binutils 17 | p.bison 18 | p.bc 19 | p.pkg-config 20 | p.m4 21 | p.which 22 | p.gnumake 23 | p.cmake 24 | p.gettext 25 | p.autoconf 26 | p.sqlite 27 | p.openssl 28 | p.glib 29 | p.yasm 30 | p.ninja 31 | p.pv 32 | ]; 33 | 34 | in 35 | 36 | with import (fetchTarball tarball) {}; 37 | 38 | gcc10Stdenv.mkDerivation { 39 | RISOTTO_ROOT = builtins.getEnv "RISOTTO_ROOT"; 40 | 41 | name = "risotto-native"; 42 | nativeBuildInputs = [ 43 | bashInteractive 44 | ]; 45 | buildInputs = pypkgs ++ deps pkgs; 46 | runScript = "bash"; 47 | } 48 | -------------------------------------------------------------------------------- /scripts/run_math.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | NR_RUNS=${NR_RUNS:=10} 4 | OUTPUT=${OUTPUT:=results/math.csv} 5 | 6 | #################################### 7 | 8 | # Reset scaling governor to schedutil on exit 9 | trap "echo schedutil | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor" EXIT 10 | 11 | # Set scaling governor to 'performance' (max frequency all the time) 12 | echo performance | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor 13 | 14 | 15 | # Run all benchmarks natively (aarch64) and with some QEMU variants 16 | # native 17 | ./a2a-benchmarks/bench.py -b micro.math -d native -r native -o ${OUTPUT} -a aarch64 -n $(nproc) -i ${NR_RUNS} -t native -c configs/native.config -vvv 18 | 19 | # QEMUs 20 | ./a2a-benchmarks/bench.py -b micro.math -d native -r qemu -o ${OUTPUT} -a x86_64 -n $(nproc) -i ${NR_RUNS} -t qemu -c configs/qemu.config -vvv 21 | ./a2a-benchmarks/bench.py -b micro.math -d native -r qemu --run-opt='-nlib configs/nlib/math.nmi' -o ${OUTPUT} -a x86_64 -n $(nproc) -i ${NR_RUNS} -t risotto -c configs/risotto.config -vvv 22 | -------------------------------------------------------------------------------- /scripts/run_sqlite.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | NR_RUNS=${NR_RUNS:=10} 4 | OUTPUT=${OUTPUT:=results/sqlite.csv} 5 | 6 | #################################### 7 | 8 | # Reset scaling governor to schedutil on exit 9 | trap "echo schedutil | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor" EXIT 10 | 11 | # Set scaling governor to 'performance' (max frequency all the time) 12 | echo performance | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor 13 | 14 | 15 | # Run all benchmarks natively (aarch64) and with some QEMU variants 16 | # native 17 | ./a2a-benchmarks/bench.py -b micro.sqlite -d native -r native -o ${OUTPUT} -a aarch64 -n $(nproc) -i ${NR_RUNS} -t native -c configs/native.config -vvv 18 | 19 | # QEMUs 20 | ./a2a-benchmarks/bench.py -b micro.sqlite -d native -r qemu -o ${OUTPUT} -a x86_64 -n $(nproc) -i ${NR_RUNS} -t qemu -c configs/qemu.config -vvv 21 | ./a2a-benchmarks/bench.py -b micro.sqlite -d native -r qemu --run-opt='-nlib configs/nlib/sqlite.nmi' -o ${OUTPUT} -a x86_64 -n $(nproc) -i ${NR_RUNS} -t risotto -c configs/risotto.config -vvv 22 | 23 | -------------------------------------------------------------------------------- /scripts/run_cas.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | NR_RUNS=${NR_RUNS:=10} 4 | OUTPUT=${OUTPUT:=results/cas.csv} 5 | 6 | #################################### 7 | 8 | # Reset scaling governor to schedutil on exit 9 | trap "echo schedutil | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor" EXIT 10 | 11 | # Set scaling governor to 'performance' (max frequency all the time) 12 | echo performance | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor 13 | 14 | 15 | # Run all benchmarks natively (aarch64) and with some QEMU variants 16 | configs=(1-1 4-1 4-2 4-4 8-1 8-4 8-8 16-1 16-8 16-16) 17 | for c in ${configs[@]}; do 18 | # native 19 | ./a2a-benchmarks/bench.py -b micro.cas -d $c -r native -o ${OUTPUT} -a aarch64 -n $(nproc) -i ${NR_RUNS} -t native -c configs/native.config -vvv 20 | 21 | # QEMUs 22 | ./a2a-benchmarks/bench.py -b micro.cas -d $c -r qemu -o ${OUTPUT} -a x86_64 -n $(nproc) -i ${NR_RUNS} -t qemu -c configs/qemu.config -vvv 23 | ./a2a-benchmarks/bench.py -b micro.cas -d $c -r qemu -o ${OUTPUT} -a x86_64 -n $(nproc) -i ${NR_RUNS} -t risotto -c configs/risotto.config -vvv 24 | done 25 | -------------------------------------------------------------------------------- /scripts/run_phoenix.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | NR_RUNS=${NR_RUNS:=10} 4 | DATASET=${DATASET:=large} 5 | OUTPUT=${OUTPUT:=results/parsec-phoenix.csv} 6 | 7 | #################################### 8 | 9 | # Reset scaling governor to schedutil on exit 10 | trap "echo schedutil | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor" EXIT 11 | 12 | # Set scaling governor to 'performance' (max frequency all the time) 13 | echo performance | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor 14 | 15 | 16 | # Run all benchmarks natively (aarch64) and with all QEMU variants 17 | benchmarks=(histogram kmeans linearregression matrixmultiply pca stringmatch wordcount) 18 | for b in ${benchmarks[@]}; do 19 | # native 20 | ./a2a-benchmarks/bench.py -b phoenix.$b -d ${DATASET} -r native -o ${OUTPUT} -a aarch64 -n $(nproc) -i ${NR_RUNS} -t native -c configs/native.config -vvv 21 | 22 | # QEMUs 23 | for q in qemu no-fences tcg-tso risotto; do 24 | ./a2a-benchmarks/bench.py -b phoenix.$b -d ${DATASET} -r qemu -o ${OUTPUT} -a x86_64 -n $(nproc) -i ${NR_RUNS} -t $q -c configs/${q}.config -vvv 25 | done 26 | done 27 | 28 | -------------------------------------------------------------------------------- /scripts/run_parsec.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | NR_RUNS=${NR_RUNS:=10} 4 | DATASET=${DATASET:=native} 5 | OUTPUT=${OUTPUT:=results/parsec-phoenix.csv} 6 | 7 | #################################### 8 | 9 | # Reset scaling governor to schedutil on exit 10 | trap "echo schedutil | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor" EXIT 11 | 12 | # Set scaling governor to 'performance' (max frequency all the time) 13 | echo performance | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor 14 | 15 | 16 | # Run all benchmarks natively (aarch64) and with all QEMU variants 17 | benchmarks=(blackscholes bodytrack canneal facesim fluidanimate freqmine streamcluster swaptions vips) 18 | for b in ${benchmarks[@]}; do 19 | # native 20 | ./a2a-benchmarks/bench.py -b parsec.$b -d ${DATASET} -r native -o ${OUTPUT} -a aarch64 -n $(nproc) -i ${NR_RUNS} -t native -c configs/native.config -vvv 21 | 22 | # QEMUs 23 | for q in qemu no-fences tcg-tso risotto; do 24 | ./a2a-benchmarks/bench.py -b parsec.$b -d ${DATASET} -r qemu -o ${OUTPUT} -a x86_64 -n $(nproc) -i ${NR_RUNS} -t $q -c configs/${q}.config -vvv 25 | done 26 | done 27 | -------------------------------------------------------------------------------- /scripts/run_openssl.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | NR_RUNS=${NR_RUNS:=10} 4 | OUTPUT=${OUTPUT:=results/openssl.csv} 5 | 6 | #################################### 7 | 8 | # Reset scaling governor to schedutil on exit 9 | trap "echo schedutil | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor" EXIT 10 | 11 | # Set scaling governor to 'performance' (max frequency all the time) 12 | echo performance | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor 13 | 14 | 15 | # Run all benchmarks natively (aarch64) and with some QEMU variants 16 | benchmarks=(md5 sha1 sha256 rsa) 17 | for b in ${benchmarks[@]}; do 18 | # native 19 | ./a2a-benchmarks/bench.py -b openssl.${b} -d native -r native -o ${OUTPUT} -a aarch64 -n $(nproc) -i ${NR_RUNS} -t native -c configs/native.config -vvv 20 | 21 | # QEMUs 22 | ./a2a-benchmarks/bench.py -b openssl.${b} -d native -r qemu -o ${OUTPUT} -a x86_64 -n $(nproc) -i ${NR_RUNS} -t qemu -c configs/qemu.config -vvv 23 | ./a2a-benchmarks/bench.py -b openssl.${b} -d native -r qemu --run-opt='-nlib configs/nlib/openssl.nmi' -o ${OUTPUT} -a x86_64 -n $(nproc) -i ${NR_RUNS} -t risotto -c configs/risotto.config -vvv 24 | done 25 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | let 2 | tarball = "https://github.com/NixOS/nixpkgs/archive/f0869b1a2c0b150aac26e10bb5c2364ffb2e804f.tar.gz"; 3 | 4 | x86pkgs = import (fetchTarball tarball) { 5 | localSystem = { system = "x86_64-linux"; }; 6 | }; 7 | 8 | pkgs = import (fetchTarball tarball) {}; 9 | 10 | pypkgs = [ pkgs.python3Packages.pandas 11 | pkgs.python3Packages.tqdm 12 | pkgs.python3Packages.matplotlib 13 | pkgs.python3Packages.seaborn 14 | pkgs.python3Packages.numpy ]; 15 | 16 | deps = p: [ 17 | p.gcc10 18 | p.zlib 19 | p.curl 20 | p.getopt 21 | p.flex 22 | p.binutils 23 | p.bison 24 | p.bc 25 | p.pkg-config-unwrapped 26 | p.m4 27 | p.which 28 | p.gnumake 29 | p.cmake 30 | p.gettext 31 | p.autoconf 32 | p.sqlite 33 | p.openssl 34 | p.yasm 35 | p.ninja 36 | p.pv 37 | p.jupyter 38 | ]; 39 | 40 | in 41 | 42 | with import (fetchTarball tarball) {}; 43 | 44 | gcc10Stdenv.mkDerivation { 45 | X86-DEPS = buildEnv { 46 | name = "risotto-x86"; 47 | paths = deps x86pkgs; 48 | pathsToLink = [ "/include" "/lib" "/lib64" "/bin" "/usr" "/usr/lib" ]; 49 | extraOutputsToInstall = [ "out" "bin" "dev" ]; 50 | }; 51 | 52 | RISOTTO_ROOT = builtins.getEnv "RISOTTO_ROOT"; 53 | 54 | name = "risotto-native"; 55 | nativeBuildInputs = [ 56 | bashInteractive 57 | ]; 58 | buildInputs = pypkgs ++ deps pkgs; 59 | runScript = "bash"; 60 | } 61 | -------------------------------------------------------------------------------- /scripts/download_benchmarks.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | mkdir -p ${RISOTTO_ROOT}/build 4 | cd ${RISOTTO_ROOT}/build 5 | 6 | # parsec 7 | echo "Downloading PARSEC pre-built binaries archive..." 8 | wget -O parsec-bins.tar.xz -q --show-progress https://nextcloud.in.tum.de/index.php/s/AFBrjRbsHkcgF7b/download/parsec-bins.tar.xz 9 | echo "Extracting archive..." 10 | pv parsec-bins.tar.xz | tar xJ 11 | 12 | # phoenix 13 | echo "Downloading Phoenix pre-built binaries archive..." 14 | wget -O phoenix-bins.tar.xz -q --show-progress https://nextcloud.in.tum.de/index.php/s/bZiqieCkr782QWG/download/phoenix-bins.tar.xz 15 | echo "Extracting archive..." 16 | pv phoenix-bins.tar.xz | tar xJ 17 | 18 | # sqlite 19 | echo "Downloading sqlite pre-built binaries archive..." 20 | wget -O sqlite-bins.tar.xz -q --show-progress https://nextcloud.in.tum.de/index.php/s/W8FMraHb2rxySW2/download/sqlite-bins.tar.xz 21 | echo "Extracting archive..." 22 | pv sqlite-bins.tar.xz | tar xJ 23 | 24 | # math 25 | echo "Downloading math pre-built binaries archive..." 26 | wget -O math-bins.tar.xz -q --show-progress https://nextcloud.in.tum.de/index.php/s/fyyWKg6FyECTynk/download/math-bins.tar.xz 27 | echo "Extracting archive..." 28 | pv math-bins.tar.xz | tar xJ 29 | 30 | # cas 31 | echo "Downloading CAS pre-built binaries archive..." 32 | wget -O cas.tar.xz -q --show-progress https://nextcloud.in.tum.de/index.php/s/KqAHQ7siJHABBE9/download/cas.tar.xz 33 | echo "Extracting archive..." 34 | pv cas.tar.xz | tar xJ 35 | 36 | cd ${RISOTTO_ROOT} 37 | -------------------------------------------------------------------------------- /configs/nlib/openssl.nmi: -------------------------------------------------------------------------------- 1 | library "/nix/store/fxghvs3di1vjcaa3fhgr1xp7lpnf1jvs-openssl-1.1.1l/lib/libssl.so.1.1"; 2 | # library "/nix/store/gc6x3qdhssgj50rrcdpr13sg8ly74q9r-openssl-1.1.1k/lib/libssl.so.1.1"; 3 | # library "libssl.so.1.1"; 4 | 5 | i32 OPENSSL_init_ssl(u64 opts, const ptr settings); 6 | 7 | # ptr OSSL_STORE_LOADER_new(ptr e, const string scheme); 8 | 9 | 10 | ptr CRYPTO_malloc(ulong sz, ptr f, i32 l); 11 | ptr CRYPTO_zalloc(ulong sz, ptr f, i32 l); 12 | ptr CRYPTO_realloc(ptr a, ulong sz, ptr f, i32 l); 13 | ptr CRYPTO_clear_realloc(ptr a, ulong old, ulong sz, ptr f, i32 l); 14 | 15 | void CRYPTO_clear_free(ptr p, ulong sz, ptr f, i32 l); 16 | void CRYPTO_free(ptr p, ptr f, i32 l); 17 | 18 | i32 ENGINE_init(ptr e); 19 | i32 ENGINE_register_all_complete(); 20 | ptr ENGINE_by_id(const string id); 21 | i32 ENGINE_ctrl(ptr e, i32 cmd, i64 i, ptr p, ptr f); 22 | i32 ENGINE_ctrl_cmd(ptr e, string cmd_name, i64 i, ptr p, ptr f, i32 opt); 23 | i32 ENGINE_free(ptr e); 24 | i32 ENGINE_finish(ptr e); 25 | 26 | ptr ENGINE_get_pkey_asn1_meth_str(ptr e, const string str, i32 len); 27 | 28 | ptr BN_new(); 29 | #void BN_init(ptr p); 30 | void BN_clear(ptr p); 31 | void BN_free(ptr p); 32 | void BN_clear_free(ptr p); 33 | 34 | const ptr BN_value_one(); 35 | i32 BN_set_word(ptr a, ulong w); 36 | ulong BN_get_word(ptr a); 37 | 38 | ptr EVP_PKEY_CTX_new_id(i32 id, ptr e); 39 | ptr EVP_PKEY_CTX_dup(ptr ctx); 40 | void EVP_PKEY_CTX_free(ptr ctx); 41 | 42 | i32 EVP_PKEY_keygen_init(ptr ctx); 43 | i32 EVP_PKEY_keygen(ptr ctx, ptr ppkey); 44 | i32 EVP_PKEY_paramgen_init(ptr ctx); 45 | i32 EVP_PKEY_paramgen(ptr ctx, ptr ppkey); 46 | 47 | ptr EVP_PKEY_asn1_find_str(ptr e, const string str, i32 len); 48 | i32 EVP_PKEY_asn1_get0_info(ptr pp, ptr baseid, ptr flags, ptr pinfo, ptr str, ptr ameth); 49 | 50 | #i32 EVP_PKEY_CTX_set_rsa_keygen_bits(ptr ctx, i32 mbits); 51 | #i32 EVP_PKEY_CTX_set1_rsa_keygen_pubexp(ptr ctx, ptr pubexp); 52 | #i32 EVP_PKEY_CTX_set_rsa_keygen_primes(ptr ctx, i32 primes); 53 | 54 | 55 | i32 RAND_bytes(ptr buf, i32 num); 56 | 57 | # const ptr EVP_MD_CTX_md(const ptr ctx); 58 | 59 | library "/nix/store/fxghvs3di1vjcaa3fhgr1xp7lpnf1jvs-openssl-1.1.1l/lib/libcrypto.so.1.1"; 60 | # library "/nix/store/gc6x3qdhssgj50rrcdpr13sg8ly74q9r-openssl-1.1.1k/lib/libcrypto.so.1.1"; 61 | # library "libcrypto.so.3"; 62 | 63 | ptr EVP_PKEY_CTX_new(ptr pkey, ptr e); 64 | 65 | i32 RSA_pkey_ctx_ctrl(ptr ctx, i32 optype, i32 cmd, i32 p1, ptr p2); 66 | ptr d2i_PrivateKey(i32 type, ptr a, const ptr pp, ilong length); 67 | i32 EVP_PKEY_sign_init(ptr ctx); 68 | i32 EVP_PKEY_sign(ptr ctx, ptr sig, ptr siglen, const ptr tbs, ulong tbslen); 69 | 70 | i32 EVP_PKEY_verify_init(ptr ctx); 71 | i32 EVP_PKEY_verify(ptr ctx, ptr sig, ulong siglen, const ptr tbs, ulong tbslen); 72 | 73 | i32 EVP_Digest(const ptr data, ulong count, ptr md, ptr size, ptr type, ptr impl); 74 | # void EVP_MD_free(ptr md); 75 | ptr EVP_MD_CTX_new(); 76 | const ptr EVP_MD_CTX_md(const ptr ctx); 77 | ptr EVP_MD_CTX_md_data(const ptr ctx); 78 | void EVP_MD_CTX_free(ptr ctx); 79 | #ptr EVP_MD_fetch(ptr ctx, const string algorithm, ptr properties); 80 | #i32 EVP_Digest_MD2_loop(ptr args); 81 | ptr MD5_Transform(ptr ctx, const ptr block); 82 | #i32 MD5_loop(ptr args); 83 | ptr MD5(const ptr d, ulong n, ptr md); 84 | ptr SHA1(const ptr d, ulong n, ptr md); 85 | ptr SHA256(const ptr d, ulong n, ptr md); 86 | ptr RSA_new(); 87 | void RSA_free(ptr rsa); 88 | ptr d2i_RSAPrivateKey(ptr a, ptr b, i32 len); 89 | i32 RSA_generate_multi_prime_key(ptr rsa, i32 bits, i32 primes, ptr e_value, ptr cb); 90 | i32 RSA_sign(i32 type, const ptr m, u32 m_len, ptr sigret, ptr siglen, ptr rsa); 91 | i32 RSA_verify(i32 type, const ptr m, u32 m_len, const ptr sigbuf, u32 siglen, ptr rsa); 92 | -------------------------------------------------------------------------------- /plots/fig15.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "aa92dc16", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import pandas as pd\n", 11 | "import numpy as np\n", 12 | "import seaborn as sbs\n", 13 | "import matplotlib.pyplot as plt\n", 14 | "import matplotlib.ticker" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "id": "9ae0ee68", 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "input_csv = \"../results/cas.csv\"\n", 25 | "baseline = \"mode,threads,vars,value\"\n", 26 | "base_mode, base_threads, base_vars, base_time = baseline.split(',')\n", 27 | "df = pd.read_csv(input_csv, sep=';')\n", 28 | "df" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "id": "7c43747f", 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "# change value from time to throughput\n", 39 | "NUM_ITERATIONS = 10000000\n", 40 | "df['value'] = (df['threads'] * NUM_ITERATIONS) / df['value']\n", 41 | "df['unit'] = 'ops/s'\n", 42 | "df" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "id": "ad6e7278", 49 | "metadata": {}, 50 | "outputs": [], 51 | "source": [ 52 | "fig = plt.figure(figsize=(6,2.5), dpi=500)\n", 53 | "sbs.set(style=\"whitegrid\")\n", 54 | "palette = {\n", 55 | " 'orange': '#faa200',\n", 56 | " 'sky blue': '#00b7ec',\n", 57 | " 'bluish green': '#00a077',\n", 58 | " 'yellow': '#f5e636',\n", 59 | " 'blue': '#0077b8',\n", 60 | " 'vermillion': '#f3640d',\n", 61 | " 'reddish purple': '#e47ead'\n", 62 | "}\n", 63 | "order = [ '' ]\n", 64 | "ax = sbs.barplot(data=df, x='dataset', y='value', hue='tag',\n", 65 | " hue_order=['qemu', 'risotto', 'native'])\n", 66 | "#plt.xticks(ticks=range(0, len(configs)), labels=configs, rotation=0, ha=\"center\", fontsize='xx-small')\n", 67 | "plt.xlabel(\"(#threads - #vars) configurations\", fontsize='x-small')\n", 68 | "plt.ylabel(\"CAS throughput (ops/s)\")\n", 69 | "ax.yaxis.set_major_formatter(matplotlib.ticker.EngFormatter(unit=''))\n", 70 | "\n", 71 | "# Set color + hatch\n", 72 | "style = {\n", 73 | " 'fill': [ True, True, True ],\n", 74 | " 'color': [ palette['vermillion'], palette['bluish green'], palette['orange'] ],\n", 75 | " 'label': [ 'QEMU', 'risotto', 'native'],\n", 76 | " 'hatch': [ '///', '', ''],\n", 77 | " 'edgecolor': [ 'black', 'black', 'black' ]\n", 78 | "}\n", 79 | "configs = set(df['dataset'].values)\n", 80 | "for idx, bar in enumerate(ax.patches):\n", 81 | " bar_nr = int(idx / int(len(configs)))\n", 82 | " bar.set(color=style['color'][bar_nr], fill=style['fill'][bar_nr],\n", 83 | " hatch=style['hatch'][bar_nr], edgecolor=style['edgecolor'][bar_nr])\n", 84 | " #bar.set_label(style['label'][bar_nr])\n", 85 | "\n", 86 | "handles, labels = ax.get_legend_handles_labels()\n", 87 | "ax.legend(labels=style['label'], handles=handles, loc='upper center', bbox_to_anchor=(.5, 1.15),\n", 88 | " borderaxespad=0, ncol=3, fontsize='x-small')" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "id": "ecfb69af", 95 | "metadata": {}, 96 | "outputs": [], 97 | "source": [ 98 | "fig.savefig(\"fig15.pdf\", dpi=500, bbox_inches='tight')" 99 | ] 100 | } 101 | ], 102 | "metadata": { 103 | "kernelspec": { 104 | "display_name": "Python 3 (ipykernel)", 105 | "language": "python", 106 | "name": "python3" 107 | }, 108 | "language_info": { 109 | "codemirror_mode": { 110 | "name": "ipython", 111 | "version": 3 112 | }, 113 | "file_extension": ".py", 114 | "mimetype": "text/x-python", 115 | "name": "python", 116 | "nbconvert_exporter": "python", 117 | "pygments_lexer": "ipython3", 118 | "version": "3.10.7" 119 | } 120 | }, 121 | "nbformat": 4, 122 | "nbformat_minor": 5 123 | } 124 | -------------------------------------------------------------------------------- /plots/fig14.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "aa92dc16", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import pandas as pd\n", 11 | "import numpy as np\n", 12 | "import seaborn as sbs\n", 13 | "import matplotlib.pyplot as plt" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "id": "9ae0ee68", 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "input_csv = \"../results/math.csv\"\n", 24 | "baseline = \"x86_64,qemu,qemu\"\n", 25 | "base_arch, base_runtime, base_tag = baseline.split(',')\n", 26 | "df = pd.read_csv(input_csv, sep=';')\n", 27 | "df" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "id": "7f4a1715", 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "base_df = df.loc[df['tag'] == 'qemu']\n", 38 | "base_df" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "id": "efe644ad", 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "# Get the mean for each baseline benchmark\n", 49 | "base_means = {}\n", 50 | "for b in set(base_df['bench']):\n", 51 | " base_means[b] = np.array(base_df.loc[base_df['bench'] == b]['value'].values, dtype=np.float32).mean()\n", 52 | "base_means" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": null, 58 | "id": "7c3ce4f1", 59 | "metadata": {}, 60 | "outputs": [], 61 | "source": [ 62 | "# Print the mean of every benchmark for each runtime\n", 63 | "mean_df = pd.DataFrame()\n", 64 | "for b in sorted(set(df['bench'])):\n", 65 | " df_b = df.loc[df['bench'] == b]\n", 66 | " tmp_dict = { 'bench': b }\n", 67 | " for t in set(df_b['tag']):\n", 68 | " df_b_t = df_b.loc[df_b['tag'] == t]\n", 69 | " tmp_dict[t] = np.mean(df_b_t['value'])\n", 70 | " mean_df = mean_df.append(tmp_dict, ignore_index=True)\n", 71 | "mean_df" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": null, 77 | "id": "b2d4f8f0", 78 | "metadata": {}, 79 | "outputs": [], 80 | "source": [ 81 | "# Normalize all results from original df to these means\n", 82 | "df_norm = pd.DataFrame(columns=['arch', 'bench', 'dataset', 'threads', 'unit', 'value', 'runtime',\n", 83 | " 'tag', 'norm', 'label'])\n", 84 | "norm_vals = []\n", 85 | "for row in df.itertuples():\n", 86 | " try:\n", 87 | " if row.arch == base_arch and row.runtime == base_runtime and row.tag == base_tag:\n", 88 | " continue\n", 89 | " # norm = base_means[row.bench] / float(row.value) # speedup\n", 90 | " norm = float(row.value) / base_means[row.bench] # relative perf\n", 91 | " \n", 92 | " # norm = 100 * (base_means[row.bench] - float(row.value)) / base_means[row.bench]\n", 93 | " dct = row._asdict()\n", 94 | " dct['norm'] = norm\n", 95 | " dct['label'] = f\"{dct['tag']}\"\n", 96 | " dct['bench'] = dct['bench'][11:]\n", 97 | " # dct['label'] = f\"{dct['runtime']}-{dct['tag']}\"\n", 98 | " del dct['Index']\n", 99 | " del dct['cmdline']\n", 100 | " norm_vals.append(dct)\n", 101 | " except KeyError:\n", 102 | " pass\n", 103 | "df_norm = df_norm.append(norm_vals, ignore_index=True)\n", 104 | "df_norm" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": null, 110 | "id": "4f4bc8d4", 111 | "metadata": {}, 112 | "outputs": [], 113 | "source": [ 114 | "import re \n", 115 | "\n", 116 | "def sorted_nicely( l ): \n", 117 | " \"\"\" Sort the given iterable in the way that humans expect.\"\"\" \n", 118 | " convert = lambda text: int(text) if text.isdigit() else text \n", 119 | " alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ] \n", 120 | " return sorted(l, key = alphanum_key)" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": null, 126 | "id": "91ee07c5", 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [ 130 | "max(df_norm['norm'].values)\n", 131 | "df_norm.loc[df_norm['bench'] == 'log']['norm'].values.mean()" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "id": "a99becbd", 138 | "metadata": {}, 139 | "outputs": [], 140 | "source": [ 141 | "fig = plt.figure(figsize=(5,2.5), dpi=500)\n", 142 | "sbs.set(style=\"whitegrid\")\n", 143 | "palette = {\n", 144 | " 'orange': '#faa200',\n", 145 | " 'sky blue': '#00b7ec',\n", 146 | " 'bluish green': '#00a077',\n", 147 | " 'yellow': '#f5e636',\n", 148 | " 'blue': '#0077b8',\n", 149 | " 'vermillion': '#f3640d',\n", 150 | " 'reddish purple': '#e47ead'\n", 151 | "}\n", 152 | "order = [ 'sqrt', 'exp', 'log', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan' ]\n", 153 | "ax = sbs.barplot(data=df_norm, x='bench', y='norm', hue='label',\n", 154 | " hue_order=['risotto', 'native'], order=order)\n", 155 | "plt.grid(visible=True, axis='y')\n", 156 | "plt.xticks(rotation=0, ha=\"center\", fontsize='xx-small')\n", 157 | "plt.xlabel(\"Function\", fontsize='x-small')\n", 158 | "plt.ylabel(\"Speedup w.r.t. QEMU\")\n", 159 | "max_val = max(df_norm['norm'].values)\n", 160 | "plt.ylim(0, max_val*1.1)\n", 161 | "plt.axhline(y=1, xmin=0, xmax=1, color='tomato', linewidth=2.5)\n", 162 | "# Annotate the raw value of the baseline\n", 163 | "for idx, value in enumerate(order):\n", 164 | " bm = base_means[f\"micro.math-{value}\"]\n", 165 | " if bm > 1000000:\n", 166 | " v = f\"{bm / 1000000:.0f}M\"\n", 167 | " elif bm > 1000:\n", 168 | " v = f\"{bm / 1000:.1f}k\"\n", 169 | " else:\n", 170 | " v = f\"{bm:.0f}\"\n", 171 | " #tmp = max(mean_df.loc[mean_df['bench'] == value].values[0][1:]) / base_means[value]\n", 172 | " tmp = max(df_norm.loc[df_norm['bench'] == value]['norm'].values)\n", 173 | " plt.text(idx, tmp+.5, f\"{v}\", fontsize='xx-small', color='tomato', ha='center')\n", 174 | "#plt.vlines(range(0, len(order)), ymin=0, ymax=max_val, linestyle='dashed', colors='grey', linewidth=.25, zorder=0)\n", 175 | "\n", 176 | "# Set color + hatch\n", 177 | "style = {\n", 178 | " 'fill': [ True, True ],\n", 179 | " 'color': [ palette['bluish green'], palette['orange'] ],\n", 180 | " 'hatch': [ '', '', ''],\n", 181 | " 'edgecolor': [ 'black', 'black', 'black' ]\n", 182 | "}\n", 183 | "for idx, bar in enumerate(ax.patches):\n", 184 | " bar_nr = int(idx / int(len(order)))\n", 185 | " bar.set(color=style['color'][bar_nr], fill=style['fill'][bar_nr],\n", 186 | " hatch=style['hatch'][bar_nr], edgecolor=style['edgecolor'][bar_nr])\n", 187 | "\n", 188 | "plt.legend(loc='upper center', bbox_to_anchor=(.5, 1.15), borderaxespad=0, ncol=2, fontsize='x-small')" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": null, 194 | "id": "41550396", 195 | "metadata": {}, 196 | "outputs": [], 197 | "source": [ 198 | "fig.savefig(\"fig14.pdf\", dpi=500, bbox_inches='tight')" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": null, 204 | "id": "5cd6f91a", 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [ 208 | "for i in sorted_nicely(base_means):\n", 209 | " print(f\"{i:25}: {base_means[i]:>20.1f}\")" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": null, 215 | "id": "8062f960", 216 | "metadata": {}, 217 | "outputs": [], 218 | "source": [ 219 | "print(f\"bench{20*' '} risotto native\")\n", 220 | "for b in sorted_nicely(base_means):\n", 221 | " ris = df_norm.loc[(df_norm['bench'] == b[11:]) & (df_norm['tag'] == 'risotto')]\n", 222 | " nat = df_norm.loc[(df_norm['bench'] == b[11:]) & (df_norm['tag'] == 'native')]\n", 223 | " print(f\"{b:25} {np.mean(ris['norm'].values):>10.4f} {np.mean(nat['norm'].values):>10.2f}\")" 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": null, 229 | "id": "84217928", 230 | "metadata": {}, 231 | "outputs": [], 232 | "source": [] 233 | } 234 | ], 235 | "metadata": { 236 | "kernelspec": { 237 | "display_name": "Python 3 (ipykernel)", 238 | "language": "python", 239 | "name": "python3" 240 | }, 241 | "language_info": { 242 | "codemirror_mode": { 243 | "name": "ipython", 244 | "version": 3 245 | }, 246 | "file_extension": ".py", 247 | "mimetype": "text/x-python", 248 | "name": "python", 249 | "nbconvert_exporter": "python", 250 | "pygments_lexer": "ipython3", 251 | "version": "3.10.7" 252 | } 253 | }, 254 | "nbformat": 4, 255 | "nbformat_minor": 5 256 | } 257 | -------------------------------------------------------------------------------- /plots/fig12.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "53ddb186", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import pandas as pd\n", 11 | "import numpy as np\n", 12 | "import matplotlib.pyplot as plt\n", 13 | "import matplotlib\n", 14 | "import seaborn as sbs\n", 15 | "import matplotlib.ticker as mtick" 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": null, 21 | "id": "f37ad79a", 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "# Script parameters\n", 26 | "input_csv = \"../results/parsec-phoenix.csv\"\n", 27 | "baseline = \"x86_64,qemu,qemu\"" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "id": "d4d3595c", 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "# Parsing baseline argument\n", 38 | "base_arch, base_runtime, base_tag = baseline.split(',')" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "id": "aef9702f", 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "# Read input file\n", 49 | "df = pd.read_csv(input_csv, sep=';')\n", 50 | "df" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": null, 56 | "id": "f6b68ca5", 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "# Extract baseline from dataframe\n", 61 | "base_df = df.loc[(df['arch'] == base_arch) & (df['runtime'] == base_runtime) & (df['tag'] == base_tag)]\n", 62 | "base_df" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "id": "617134c1", 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "# Get the mean for each baseline benchmark\n", 73 | "base_means = {}\n", 74 | "for b in set(base_df['bench']):\n", 75 | " base_means[b] = np.array(base_df.loc[base_df['bench'] == b]['value'].values, dtype=np.float32).mean()" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": null, 81 | "id": "3e3734a9", 82 | "metadata": {}, 83 | "outputs": [], 84 | "source": [ 85 | "# Print the mean of every benchmark for each runtime\n", 86 | "mean_df = pd.DataFrame()\n", 87 | "for b in sorted(set(df['bench'])):\n", 88 | " df_b = df.loc[df['bench'] == b]\n", 89 | " tmp_dict = { 'bench': b }\n", 90 | " for t in set(df_b['tag']):\n", 91 | " df_b_t = df_b.loc[df_b['tag'] == t]\n", 92 | " tmp_dict[t] = np.mean(df_b_t['value'])\n", 93 | " mean_df = mean_df.append(tmp_dict, ignore_index=True)\n", 94 | "mean_df" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": null, 100 | "id": "b0a2ded4", 101 | "metadata": {}, 102 | "outputs": [], 103 | "source": [ 104 | "# Normalize all results from original df to these means\n", 105 | "df_norm = pd.DataFrame(columns=['arch', 'bench', 'dataset', 'threads', 'unit', 'value', 'runtime',\n", 106 | " 'tag', 'norm', 'label'])\n", 107 | "norm_vals = []\n", 108 | "for row in df.itertuples():\n", 109 | " try:\n", 110 | " if row.arch == base_arch and row.runtime == base_runtime and row.tag == base_tag:\n", 111 | " continue\n", 112 | " # norm = base_means[row.bench] / float(row.value) # speedup\n", 113 | " norm = 100 * float(row.value) / base_means[row.bench] # relative perf\n", 114 | " \n", 115 | " # norm = 100 * (base_means[row.bench] - float(row.value)) / base_means[row.bench]\n", 116 | " dct = row._asdict()\n", 117 | " dct['norm'] = norm\n", 118 | " dct['label'] = f\"{dct['tag']}\"\n", 119 | " # dct['label'] = f\"{dct['runtime']}-{dct['tag']}\"\n", 120 | " del dct['Index']\n", 121 | " del dct['cmdline']\n", 122 | " norm_vals.append(dct)\n", 123 | " except KeyError:\n", 124 | " pass\n", 125 | "df_norm = df_norm.append(norm_vals, ignore_index=True)\n", 126 | "df_norm" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "id": "9bd9c1bd", 133 | "metadata": {}, 134 | "outputs": [], 135 | "source": [ 136 | "# refactor xticks to remove benchmark suite prefix\n", 137 | "xlabels = []\n", 138 | "xticks = []\n", 139 | "for idx, b in enumerate(sorted(set(df_norm['bench']))):\n", 140 | " if b.startswith(\"parsec.\"):\n", 141 | " xlabels.append(b[7:])\n", 142 | " xticks.append(idx)\n", 143 | " else:\n", 144 | " xlabels.append(b[8:])\n", 145 | " xticks.append(idx)\n", 146 | "\n", 147 | "# Plot\n", 148 | "fig = plt.figure(figsize=(10, 3), dpi=500)\n", 149 | "sbs.set(style=\"whitegrid\")\n", 150 | "palette = {\n", 151 | " 'orange': '#faa200',\n", 152 | " 'sky blue': '#00b7ec',\n", 153 | " 'bluish green': '#00a077',\n", 154 | " 'yellow': '#f5e636',\n", 155 | " 'blue': '#0077b8',\n", 156 | " 'vermillion': '#f3640d',\n", 157 | " 'reddish purple': '#e47ead'\n", 158 | "}\n", 159 | "ax = sbs.barplot(data=df_norm, ci='sd',\n", 160 | " x='bench', y='norm',\n", 161 | " hue='label',# palette=palette,\n", 162 | " order=sorted(set(df_norm['bench'])), hue_order=['no-fences', 'tcg-tso', 'risotto', 'native'])\n", 163 | "plt.grid(visible=True, axis='y')\n", 164 | "plt.xticks(ticks=xticks, labels=xlabels, rotation=30, ha=\"right\", fontsize='xx-small')\n", 165 | "ax.set_axisbelow(True)\n", 166 | "plt.xlabel(\"\")\n", 167 | "max_val = max(df_norm['norm'].values)\n", 168 | "plt.ylim(0, max_val*1.05)\n", 169 | "ax.yaxis.set_major_formatter(mtick.PercentFormatter())\n", 170 | "plt.ylabel(\"Run time w.r.t. QEMU\")\n", 171 | "plt.axhline(y=100, xmin=0, xmax=1, color='tomato', linewidth=2.5)\n", 172 | "# Annotate the raw value of the baseline\n", 173 | "for idx, value in enumerate(sorted(set(base_means))):\n", 174 | " tmp = df_norm.loc[df_norm['bench'] == value]['norm'].values.mean()\n", 175 | " v = base_means[value]\n", 176 | " if v < 10:\n", 177 | " v_str = f\"{v:.1f}\"\n", 178 | " else:\n", 179 | " v_str = f\"{v:.0f}\"\n", 180 | " plt.text(idx, max(tmp, max_val), v_str, fontsize='xx-small', color='tomato', ha='center')\n", 181 | " \n", 182 | "# Set color + hatch\n", 183 | "style = {\n", 184 | " 'fill': [ True, True, True, True ],\n", 185 | " 'color': [ palette['vermillion'], palette['sky blue'], palette['bluish green'], palette['orange'] ],\n", 186 | " 'hatch': [ '', '///', '', ''],\n", 187 | " 'label': ['no-fences [incorrect]', 'tcg-ver', 'risotto', 'native'],\n", 188 | " 'edgecolor': [ 'black', 'black', 'black', 'black' ]\n", 189 | "}\n", 190 | "for idx, bar in enumerate(ax.patches):\n", 191 | " bar_nr = int(idx / len(base_means))\n", 192 | " bar.set(color=style['color'][bar_nr], fill=style['fill'][bar_nr],\n", 193 | " hatch=style['hatch'][bar_nr], edgecolor=style['edgecolor'][bar_nr])\n", 194 | " \n", 195 | "# parsec / phoenix separation\n", 196 | "#plt.vlines([ 8.5 ], ymin=0, ymax=1.2, linestyle='solid', colors='black', linewidth=2.5, zorder=10)\n", 197 | "#matplotlib.text(.4, .1, \"parsec\", xycoords='axes points')\n", 198 | "\n", 199 | "handles, labels = ax.get_legend_handles_labels()\n", 200 | "plt.legend(labels=style['label'], handles=handles, loc='upper center', bbox_to_anchor=(0.5, 1.15),\n", 201 | " borderaxespad=0, ncol=4, fontsize='x-small')" 202 | ] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "execution_count": null, 207 | "id": "aea34e01", 208 | "metadata": {}, 209 | "outputs": [], 210 | "source": [ 211 | "fig.savefig(\"fig12.pdf\", dpi=500, bbox_inches='tight')" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": null, 217 | "id": "2095965c", 218 | "metadata": {}, 219 | "outputs": [], 220 | "source": [ 221 | "# Compute fence cost\n", 222 | "fence_cost = 100 * (1 - mean_df['no-fences'] / mean_df['qemu'])\n", 223 | "print(f\"Average time spent on fences: {fence_cost.mean():.2f}%\")\n", 224 | "fence_cost" 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": null, 230 | "id": "8f83d335", 231 | "metadata": {}, 232 | "outputs": [], 233 | "source": [ 234 | "# tcg-tso gain\n", 235 | "for b in sorted(set(df_norm['bench'])):\n", 236 | " d = df_norm.loc[df_norm['bench'] == b]\n", 237 | " d = d.loc[d['tag'] == 'tcg-tso']\n", 238 | " print(f\"{b}: {d['value'].values.mean():.2f}\")" 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "id": "f8934693", 245 | "metadata": {}, 246 | "outputs": [], 247 | "source": [ 248 | "mean_df['tcg-tso-pct'] = 100 * (1 - mean_df['tcg-tso'] / mean_df['qemu'])\n", 249 | "mean_df" 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": null, 255 | "id": "e71c15f4", 256 | "metadata": {}, 257 | "outputs": [], 258 | "source": [ 259 | "mean_df['tcg-tso-pct'].values.mean()" 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": null, 265 | "id": "04dd8df3", 266 | "metadata": {}, 267 | "outputs": [], 268 | "source": [] 269 | } 270 | ], 271 | "metadata": { 272 | "kernelspec": { 273 | "display_name": "Python 3 (ipykernel)", 274 | "language": "python", 275 | "name": "python3" 276 | }, 277 | "language_info": { 278 | "codemirror_mode": { 279 | "name": "ipython", 280 | "version": 3 281 | }, 282 | "file_extension": ".py", 283 | "mimetype": "text/x-python", 284 | "name": "python", 285 | "nbconvert_exporter": "python", 286 | "pygments_lexer": "ipython3", 287 | "version": "3.10.7" 288 | } 289 | }, 290 | "nbformat": 4, 291 | "nbformat_minor": 5 292 | } 293 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Risotto: A Dynamic Binary Translator for Weak Memory Architectures --- Artifact 2 | 3 | This repository contains all the code and instructions to reproduce the results from the following paper: 4 | 5 | ``` 6 | Risotto: A Dynamic Binary Translator for Weak Memory Architectures 7 | Redha Gouicem (TU Munich), Dennis Sprokholt (TU Delft), Jasper Ruehl (TU Munich), Rodrigo C. O. Rocha (University of Edinburgh), Tom Spink (University of St Andrews), Soham Chakraborty (TU Delft), Pramod Bhatotia (TU Munich) 8 | International Conference on Architectural Support for Programming Languages and Operating Systems (ASPLOS), 2023 9 | ``` 10 | 11 | ## Content 12 | 13 | This repository contains multiple submodules: 14 | * `qemu`: contains our modified QEMU (modified mappings for memory operations, support for native shared libraries, native CAS translation) 15 | * `benchmarks`: contains the benchmarks used in the paper 16 | * `proofs`: contains the formal proof regarding our memory mappings and optimisations 17 | 18 | ## Requirements 19 | 20 | ### Hardware 21 | 22 | To run these experiments, you will need an arm64 machine with at least ARMv8.2. 23 | If you want to easily build the x86_64 versions of the benchmarks, you will also need an x86 machine. 24 | You can also cross-compile them on the ARM machine, but we do not provide instructions for this. You can adapt the building scripts to use a cross-compiler. 25 | 26 | ### Software 27 | 28 | We rely on NixOS to have a unified execution environment. 29 | You can either install NixOS as a full operating system, or just as a package manager (Nix) on top of your Linux installation. 30 | You can find out more information on the [Nix download page](https://nixos.org/download.html). 31 | The easy installation consists of using Nix as a package manager, installing it this way: 32 | ```sh 33 | sh <(curl -L https://nixos.org/nix/install) --daemon 34 | ``` 35 | 36 | Required packages to reproduce the plots are available in the Nix environment, but you can also reproduce them on you local OS if you prefer. You will need the following packages: 37 | ```sh 38 | apt install python3-pip # install pip 39 | pip install notebook pandas matplotlib seaborn 40 | ``` 41 | 42 | ## Quick result reproduction 43 | 44 | This section details how to quickly reproduce the results of the paper in a fully automated way. 45 | If you have encounter any issue or only want to run a specific part of the evaluation, you can check the [Detailed instructions](#detailed-instructions) below. 46 | 47 | From the root directory of the repository, run these commands: 48 | ```sh 49 | source sourceme 50 | ./scripts/build.sh 51 | ./scripts/run_benchmarks.sh 52 | ./scripts/plot.sh 53 | ``` 54 | This will produce the figures from the paper (12 to 15) in the `plots/` directory, as pdf files. 55 | 56 | ## Detailed instructions 57 | 58 | ### Building the software 59 | Before running the experiments, we need to build the software and the benchmarks. 60 | You can try to build everything at once or separately. 61 | 62 | #### Building everything 63 | You can build everything by executing: 64 | 65 | ```sh 66 | source sourceme 67 | ./scripts/build.sh 68 | ``` 69 | 70 | If everything works fine, you can skip to [Running the benchmarks](#running-the-benchmarks). 71 | 72 | Note that this script downloads the pre-built binaries for the benchmarks and does not build them. If you want to build them, check the following sections. 73 | 74 | #### Building QEMU 75 | You need to build our modified QEMU as well as the baselines for the evaluation to work. It is located in the `qemu` submodule and can be built by running: 76 | ```sh 77 | source sourceme 78 | nix-shell --run scripts/build_qemu.sh qemu.nix 79 | ``` 80 | 81 | Since QEMU will be executed on the ARM machine, it has to be built for ARM. To simplify this, we recommend to build this directly on the ARM machine you will use to reproduce the results of the paper. 82 | 83 | After running this script, you should have four versions of QEMU built, located in `build/qemu/`: 84 | * `master-6.1.0`: this is the official QEMU 6.1.0, unmodified (used as a baseline) 85 | * `no-fences`: QEMU 6.1.0 but no fence is generated for any memory access. This version may create incorrect executions (used as a comparison) 86 | * `tcg-tso`: QEMU 6.1.0 + our memory mappings (used to evaluate the performance of our mappings) 87 | * `risotto`: QEMU 6.1.0 + our memory mappings + native CAS + native shared libraries 88 | 89 | #### Building the benchmarks 90 | The benchmarks need to be built twice: once for arm64 and once for x86_64. You can do this in different ways: 91 | * Download pre-built binaries (recommended) 92 | * Build the x86_64 binaries on an x86_64 machine, and the arm64 binaries on an arm64 machine 93 | * Build everything on the same machine with cross-compilation for the non-native version. 94 | 95 | ##### Downloading pre-built binaries 96 | We provide an archive with all benchmarks packaged for both x86_64 and arm64 at [this address](https://nextcloud.in.tum.de/index.php/s/XenwcXiHy4TTZMC). 97 | You can just download and extract the archives wherever you want with: 98 | ```sh 99 | tar xf 100 | ``` 101 | 102 | These archives contain the binaries and input data for each benchmark. 103 | 104 | ##### Build binaries on x86_64 and arm64 machines 105 | 106 | On each machine, you can run the same commands to build all the benchmarks: 107 | ```sh 108 | source sourceme 109 | nix-shell --run scripts/build_benchmarks.sh default.nix 110 | ``` 111 | 112 | Note that you will need to check the paths specified in the configuration files available in `config/*.config` and set them up properly. 113 | 114 | ### Running the benchmarks 115 | 116 | This section shows how to run the benchmarks and plot each figure available in the paper. 117 | Note that the provided scripts pin the benchmarks to a subset of cores, tailored for our test machine (cores 0 to 111). 118 | If you want to use your own machine, change the `taskset` command in the `scripts/run_benchmarks.sh` script to match your configuration. 119 | If you have a machine with multiple NUMA nodes, we suggest to pin the benchmarks to a single node. 120 | You can get the list of cores of each node with the `lscpu` command. 121 | 122 | #### Figure 12: PARSEC and Phoenix 123 | 124 | You can run these benchmarks by executing these two commands from the root of this repository (after running `source sourceme`): 125 | ```sh 126 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_parsec.sh ${RISOTTO_ROOT}/default.nix 127 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_phoenix.sh ${RISOTTO_ROOT}/default.nix 128 | ``` 129 | The results will be available in `results/parsec-phoenix.csv`. Note that if you run the scripts multiple times, the results are appended to the csv file. If you want to start again from scratch, delete or rename the csv before running the benchmarks. 130 | 131 | To plot the figure, start the jupyter notebook server from the root of the repository (if it's not already started): 132 | ```sh 133 | jupyter notebook 134 | ``` 135 | In the browser window, open the `plots/fig12.ipynb` and run all the cells in order. The plot should be available in the 9th cell. 136 | 137 | #### Figure 13: openssl and sqlite 138 | 139 | You can run these benchmarks by executing these two commands from the root of this repository (after running `source sourceme`): 140 | ```sh 141 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_openssl.sh ${RISOTTO_ROOT}/default.nix 142 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_sqlite.sh ${RISOTTO_ROOT}/default.nix 143 | ``` 144 | The results will be available in `results/openssl.csv` and `results/sqlite.csv`. Note that if you run the scripts multiple times, the results are appended to the csv files. If you want to start again from scratch, delete or rename the csv before running the benchmarks. 145 | 146 | To plot the figure, start the jupyter notebook server from the root of the repository (if it's not already started): 147 | ```sh 148 | jupyter notebook 149 | ``` 150 | In the browser window, open the `plots/fig13.ipynb` and run all the cells in order. The plot should be available in the 11th cell. 151 | 152 | #### Figure 14: math 153 | 154 | You can run these benchmarks by executing this command from the root of this repository (after running `source sourceme`): 155 | ```sh 156 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_math.sh ${RISOTTO_ROOT}/default.nix 157 | ``` 158 | The results will be available in `results/math.csv`. Note that if you run the scripts multiple times, the results are appended to the csv file. If you want to start again from scratch, delete or rename the csv before running the benchmarks. 159 | 160 | To plot the figure, start the jupyter notebook server from the root of the repository (if it's not already started): 161 | ```sh 162 | jupyter notebook 163 | ``` 164 | In the browser window, open the `plots/fig14.ipynb` and run all the cells in order. The plot should be available in the 9th cell. 165 | 166 | #### Figure 15: cas 167 | 168 | You can run these benchmarks by executing this command from the root of this repository (after running `source sourceme`): 169 | ```sh 170 | nix-shell --run ${RISOTTO_ROOT}/scripts/run_cas.sh ${RISOTTO_ROOT}/default.nix 171 | ``` 172 | The results will be available in `results/cas.csv`. Note that if you run the scripts multiple times, the results are appended to the csv file. If you want to start again from scratch, delete or rename the csv before running the benchmarks. 173 | 174 | To plot the figure, start the jupyter notebook server from the root of the repository (if it's not already started): 175 | ```sh 176 | jupyter notebook 177 | ``` 178 | In the browser window, open the `plots/fig15.ipynb` and run all the cells in order. The plot should be available in the 4th cell. 179 | 180 | ## Proofs 181 | 182 | The proofs have their own `README.md` (inside `proofs/`), with explanations. For evaluation, it's easiest to use the pre-build Docker image. Check the proofs with: 183 | 184 | ```sh 185 | docker run -it --rm sourcedennis/risotto-proofs:latest agda src/Main.agda --safe 186 | ``` 187 | 188 | Generate HTML-rendered Agda with (into local directory `html/`): 189 | 190 | ```sh 191 | docker run -it --rm -v "$PWD/html:/proofs/html" sourcedennis/risotto-proofs:latest agda --html --html-dir=html src/Main.agda 192 | ``` 193 | 194 | Or, build the Docker images yourself with the contained `proofs/Dockerfile`. 195 | -------------------------------------------------------------------------------- /plots/fig13.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "a3d65ecd", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import pandas as pd\n", 11 | "import numpy as np\n", 12 | "import seaborn as sbs\n", 13 | "import matplotlib.pyplot as plt" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "id": "28f7a306", 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "input_csv = \"../results/openssl.csv\"\n", 24 | "input_sqlite_csv = \"../results/sqlite.csv\"\n", 25 | "output_pdf = \"../results/fig13.pdf\"\n", 26 | "baseline = \"x86_64,qemu,qemu\"\n", 27 | "base_arch, base_runtime, base_tag = baseline.split(',')\n", 28 | "df = pd.read_csv(input_csv, sep=';')\n", 29 | "df" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "id": "9bd6dc73", 36 | "metadata": {}, 37 | "outputs": [], 38 | "source": [ 39 | "# Select which ciphers to plot\n", 40 | "ciphers = [ #'openssl.md5-64', 'openssl.md5-256',\n", 41 | " 'openssl.md5-1024', 'openssl.md5-8192',\n", 42 | " #'openssl.rsa512-sign', 'openssl.rsa512-verify',\n", 43 | " 'openssl.rsa1024-sign', 'openssl.rsa1024-verify',\n", 44 | " 'openssl.rsa2048-sign', 'openssl.rsa2048-verify',\n", 45 | " #'openssl.sha1-64', 'openssl.sha1-256',\n", 46 | " 'openssl.sha1-1024', 'openssl.sha1-8192',\n", 47 | " #'openssl.sha256-64', 'openssl.sha256-256',\n", 48 | " 'openssl.sha256-1024', 'openssl.sha256-8192'\n", 49 | " ]\n", 50 | "\n", 51 | "df = df.loc[df['bench'].isin(ciphers)]\n", 52 | "df" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": null, 58 | "id": "8c28a92a", 59 | "metadata": {}, 60 | "outputs": [], 61 | "source": [ 62 | "# Insert the sqlite results\n", 63 | "df_sqlite = pd.read_csv(input_sqlite_csv, sep=';')\n", 64 | "df_sqlite = df_sqlite.loc[df_sqlite['bench'] == \"micro.sqlite-total-multi\"]\n", 65 | "df_sqlite['bench'] = \"xxxxxxx.sqlite\"\n", 66 | "# convert to ops/s\n", 67 | "df_sqlite['value'] = 269236 / (df_sqlite['value'] / 1000)\n", 68 | "df_sqlite['unit'] = \"ops/s\"\n", 69 | "\n", 70 | "df_sqlite" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": null, 76 | "id": "5157dc80", 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "df = pd.concat([ df, df_sqlite] , ignore_index=True)\n", 81 | "df" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "id": "cb8c9622", 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "base_df = df.loc[df['tag'] == 'qemu']\n", 92 | "base_df" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": null, 98 | "id": "f1ca3a33", 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "# Get the mean for each baseline benchmark\n", 103 | "base_means = {}\n", 104 | "for b in set(base_df['bench']):\n", 105 | " base_means[b] = np.array(base_df.loc[base_df['bench'] == b]['value'].values, dtype=np.float32).mean()" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "id": "c5e11e81", 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "# Print the mean of every benchmark for each runtime\n", 116 | "mean_df = pd.DataFrame()\n", 117 | "for b in sorted(set(df['bench'])):\n", 118 | " df_b = df.loc[df['bench'] == b]\n", 119 | " tmp_dict = { 'bench': b }\n", 120 | " for t in set(df_b['tag']):\n", 121 | " df_b_t = df_b.loc[df_b['tag'] == t]\n", 122 | " tmp_dict[t] = np.mean(df_b_t['value'])\n", 123 | " mean_df = mean_df.append(tmp_dict, ignore_index=True)\n", 124 | "mean_df" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "id": "0a7ec17d", 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [ 134 | "# Normalize all results from original df to these means\n", 135 | "df_norm = pd.DataFrame(columns=['arch', 'bench', 'dataset', 'threads', 'unit', 'value', 'runtime',\n", 136 | " 'tag', 'norm', 'label'])\n", 137 | "norm_vals = []\n", 138 | "for row in df.itertuples():\n", 139 | " try:\n", 140 | " if row.arch == base_arch and row.runtime == base_runtime and row.tag == base_tag:\n", 141 | " continue\n", 142 | " #if row.bench == \"sqlite.speedtest1\":\n", 143 | " # norm = base_means[row.bench] / float(row.value) # speedup\n", 144 | " #else:\n", 145 | " norm = float(row.value) / base_means[row.bench] # relative perf\n", 146 | " \n", 147 | " # norm = 100 * (base_means[row.bench] - float(row.value)) / base_means[row.bench]\n", 148 | " dct = row._asdict()\n", 149 | " dct['norm'] = norm\n", 150 | " dct['label'] = f\"{dct['tag']}\"\n", 151 | " # dct['label'] = f\"{dct['runtime']}-{dct['tag']}\"\n", 152 | " del dct['Index']\n", 153 | " del dct['cmdline']\n", 154 | " norm_vals.append(dct)\n", 155 | " except KeyError:\n", 156 | " pass\n", 157 | "df_norm = df_norm.append(norm_vals, ignore_index=True)\n", 158 | "df_norm" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": null, 164 | "id": "eb4172c0", 165 | "metadata": {}, 166 | "outputs": [], 167 | "source": [ 168 | "import re \n", 169 | "\n", 170 | "def sorted_nicely( l ): \n", 171 | " \"\"\" Sort the given iterable in the way that humans expect.\"\"\" \n", 172 | " convert = lambda text: int(text) if text.isdigit() else text \n", 173 | " alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ] \n", 174 | " return sorted(l, key = alphanum_key)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": null, 180 | "id": "161ce109", 181 | "metadata": {}, 182 | "outputs": [], 183 | "source": [ 184 | "fig = plt.figure(figsize=(5,2.5), dpi=500)\n", 185 | "sbs.set(style=\"whitegrid\")\n", 186 | "palette = {\n", 187 | " 'orange': '#faa200',\n", 188 | " 'sky blue': '#00b7ec',\n", 189 | " 'bluish green': '#00a077',\n", 190 | " 'yellow': '#f5e636',\n", 191 | " 'blue': '#0077b8',\n", 192 | " 'vermillion': '#f3640d',\n", 193 | " 'reddish purple': '#e47ead'\n", 194 | "}\n", 195 | "ax = sbs.barplot(data=df_norm, x='bench', y='norm', hue='label',\n", 196 | " hue_order=['risotto', 'native'], order=sorted_nicely(base_means))\n", 197 | "plt.grid(visible=True, axis='y')\n", 198 | "plt.xticks(ticks=range(0, len(set(df_norm['bench']))),\n", 199 | " labels=[ l[8:] for l in sorted_nicely(base_means) ], # remove openssl. from the x ticks\n", 200 | " rotation=45, ha=\"right\", fontsize='xx-small')\n", 201 | "plt.xlabel(\"\")\n", 202 | "plt.ylabel(\"Speedup w.r.t. QEMU\")\n", 203 | "plt.ylim((0, 27))\n", 204 | "plt.axhline(y=1, xmin=0, xmax=1, color='tomato', linewidth=2.5)\n", 205 | "# Annotate the raw value of the baseline\n", 206 | "for idx, value in enumerate(sorted_nicely(set(base_means))):\n", 207 | " if base_means[value] > 1000000:\n", 208 | " v = f\"{base_means[value] / 1000000:.0f}M\"\n", 209 | " elif base_means[value] > 1000:\n", 210 | " v = f\"{base_means[value] / 1000:.0f}k\"\n", 211 | " else:\n", 212 | " v = f\"{base_means[value]:.0f}\"\n", 213 | " tmp = max(mean_df.loc[mean_df['bench'] == value].values[0][1:]) / base_means[value]\n", 214 | " plt.text(idx, tmp+1, f\"{v}\", fontsize='xx-small', color='tomato', rotation=90, ha='center')\n", 215 | " #plt.text(idx, max(tmp+.3, 21), f\"{v}\", fontsize='xx-small', color='tomato', rotation=45, ha='center')\n", 216 | "#plt.vlines(range(0, len(set(base_means))), ymin=0, ymax=20.8, linestyle='dashed', colors='grey', linewidth=.5, zorder=0)\n", 217 | "\n", 218 | "# Set color + hatch\n", 219 | "style = {\n", 220 | " 'fill': [ True, True ],\n", 221 | " 'color': [ palette['bluish green'], palette['orange'] ],\n", 222 | " 'hatch': [ '', '', ''],\n", 223 | " 'edgecolor': [ 'black', 'black', 'black' ]\n", 224 | "}\n", 225 | "for idx, bar in enumerate(ax.patches):\n", 226 | " bar_nr = int(idx / int(len(base_means)))\n", 227 | " bar.set(color=style['color'][bar_nr], fill=style['fill'][bar_nr],\n", 228 | " hatch=style['hatch'][bar_nr], edgecolor=style['edgecolor'][bar_nr])\n", 229 | "\n", 230 | "plt.legend(loc='upper center', bbox_to_anchor=(.5, 1.15), borderaxespad=0, ncol=2, fontsize='x-small')" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "id": "e67b6f0d", 237 | "metadata": {}, 238 | "outputs": [], 239 | "source": [ 240 | "fig.savefig(\"fig13.pdf\", dpi=500, bbox_inches='tight')" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": null, 246 | "id": "b06ee3f4", 247 | "metadata": {}, 248 | "outputs": [], 249 | "source": [ 250 | "for i in sorted_nicely(base_means):\n", 251 | " print(f\"{i:25}: {base_means[i]:>20.1f}\")" 252 | ] 253 | }, 254 | { 255 | "cell_type": "code", 256 | "execution_count": null, 257 | "id": "a38b65a9", 258 | "metadata": {}, 259 | "outputs": [], 260 | "source": [ 261 | "print(value)\n", 262 | "max(mean_df.loc[mean_df['bench'] == value].values[0][1:]) / base_means[value]" 263 | ] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": null, 268 | "id": "b5effd31", 269 | "metadata": {}, 270 | "outputs": [], 271 | "source": [ 272 | "mean_df['risotto-speedup'] = mean_df['risotto'] / mean_df['qemu']\n", 273 | "mean_df" 274 | ] 275 | } 276 | ], 277 | "metadata": { 278 | "kernelspec": { 279 | "display_name": "Python 3 (ipykernel)", 280 | "language": "python", 281 | "name": "python3" 282 | }, 283 | "language_info": { 284 | "codemirror_mode": { 285 | "name": "ipython", 286 | "version": 3 287 | }, 288 | "file_extension": ".py", 289 | "mimetype": "text/x-python", 290 | "name": "python", 291 | "nbconvert_exporter": "python", 292 | "pygments_lexer": "ipython3", 293 | "version": "3.10.7" 294 | } 295 | }, 296 | "nbformat": 4, 297 | "nbformat_minor": 5 298 | } 299 | --------------------------------------------------------------------------------