├── graphs ├── 07-rocksdb-dram.csv ├── 07-rocksdb-optane.csv ├── 16-nova_ni_read.csv ├── 16-nova_ni_write.csv ├── 14-pangolin.csv ├── 02-best_case_latency.csv ├── 11-nova_datalog.csv ├── 12-flush_instr_lat.csv ├── 12-flush_instr_bw.csv ├── 17-numa.csv ├── 15-alloc_load.csv ├── 06-bandwidth_thmix.csv ├── 15-alloc_store.csv ├── 18-pmemkv.csv ├── 05-bandwidth_accesssize_dram.csv ├── 04-bandwidth_thcount_optane_ni.csv ├── 04-bandwidth_thcount_optane.csv ├── 04-bandwidth_thcount_dram.csv ├── 05-bandwidth_accesssize_optane_ni.csv ├── 13-sfence_interval.csv ├── 09-infer_xpbuffer.csv ├── 03-tail_latency.csv ├── 05-bandwidth_accesssize_optane.csv ├── 06-bandwidth_seqwrite_latency.csv ├── 08-store.csv ├── 08-storeclwb.csv └── 08-ntstore.csv ├── src ├── testscript │ ├── subtests │ │ ├── bin │ │ │ └── aeprun │ │ ├── 70_straight.sh │ │ ├── 50_repeat.sh │ │ ├── 20_stridelat.sh │ │ ├── 60_raw.sh │ │ ├── 80_align.sh │ │ ├── 110_probeimc.sh │ │ ├── 31_seq.sh │ │ ├── 90_flushfence.sh │ │ ├── 30_stridebw.sh │ │ ├── 40_sizebw.sh │ │ └── 41_sizebw_unalign.sh │ ├── umount.sh │ ├── config.example.json │ ├── parsing │ │ ├── 50_repeat_cloud.py │ │ ├── 100_probe.sh │ │ ├── 10_parse_basic.py │ │ └── repeat.py │ ├── perf_mount.sh │ ├── run.example.sh │ ├── mount.sh │ ├── perf_stridelat.sh │ └── parse_bw.py ├── data │ ├── parse.R │ ├── example.delete.py │ ├── example.query.py │ └── example.update.py ├── kernel │ ├── Makefile │ ├── debug.c │ ├── support.h │ ├── gen_tasks.c │ ├── perf_util.h │ ├── misc.c │ ├── rep.c │ ├── perf_util.c │ ├── lattester.h │ ├── proc.c │ ├── memaccess.h │ └── lat.c ├── plot │ ├── plot_timeseries.py │ ├── plot_cdf.py │ └── merge_plots.py └── README.md ├── README.md └── data └── README.md /graphs/07-rocksdb-dram.csv: -------------------------------------------------------------------------------- 1 | access,Ext4-DAX 2 | WAL\POSIX,120144 3 | WAL\FLEX,461005 4 | Persistent\memtable,521479 5 | -------------------------------------------------------------------------------- /graphs/07-rocksdb-optane.csv: -------------------------------------------------------------------------------- 1 | access,Ext4-DAX 2 | WAL\POSIX,122655 3 | WAL\FLEX,435454 4 | Persistent\memtable,391469 5 | -------------------------------------------------------------------------------- /src/testscript/subtests/bin/aeprun: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # aeprun is a wrapper we used to collect system info and logs 3 | $* 4 | -------------------------------------------------------------------------------- /graphs/16-nova_ni_read.csv: -------------------------------------------------------------------------------- 1 | ,"I,sync","NI,sync","I,async","NI,async" 2 | Read\Sequential,30,32.8,25.5859375,26.26953125 3 | Read\Random,19.1,25.7,22.16796875,25.48828125 4 | -------------------------------------------------------------------------------- /graphs/16-nova_ni_write.csv: -------------------------------------------------------------------------------- 1 | ,"I,sync","NI,sync","I,async","NI,async" 2 | Write\Sequential,8.56,9.24,6.7109375,8.241210938 3 | Write\Random,7.03,8.53,6.021484375,7.452148438 4 | 5 | -------------------------------------------------------------------------------- /graphs/14-pangolin.csv: -------------------------------------------------------------------------------- 1 | ,PGL-NT,PGL-CLWB 2 | 64,2.583,2.478 3 | 128,2.337,2.113 4 | 256,2.564,2.228 5 | 512,2.869,2.44 6 | 1024,3.125,3.025 7 | 2048,3.912,4.349 8 | 4096,5.999,6.972 9 | 8192,10.887,11.981 -------------------------------------------------------------------------------- /graphs/02-best_case_latency.csv: -------------------------------------------------------------------------------- 1 | ,DRAM,DRAM-Err,Optane,Optane-Err 2 | Read\Sequential,81.4,27.6,169.4,67.6 3 | Read\Random,101.5,27.6,305.0,27.9 4 | Write\(ntstore),86.9,6.8,90.4,8.16 5 | Write\(clwb),57.3,3.7,62.3,5.5 -------------------------------------------------------------------------------- /graphs/11-nova_datalog.csv: -------------------------------------------------------------------------------- 1 | access,XFS-DAX-sync,XFS-DAX,Ext4-DAX-sync,Ext4-DAX,NOVA,NOVA-relaxed,NOVA-datalog 2 | Overwrite\64B,40.23,2.644,3.27,2.484,8.95,2.89,1.274 3 | Overwrite\256B,56.834,3.098,3.626,3.029,9.542,3.498,1.467 4 | Read\4KB,0,1.514,0,1.568,1.362,1.325,1.610 5 | -------------------------------------------------------------------------------- /src/testscript/umount.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [[ $EUID -ne 0 ]]; then 3 | echo "Please run as root" 4 | exit 1 5 | fi 6 | 7 | umount /mnt/latency 8 | umount /mnt/report 9 | 10 | rmmod latfs 11 | rmmod repfs 12 | 13 | echo 1 > /proc/sys/kernel/soft_watchdog 14 | -------------------------------------------------------------------------------- /graphs/12-flush_instr_lat.csv: -------------------------------------------------------------------------------- 1 | ,ntstore,store+clwb 2 | 64,124.6945412,116.4267059 3 | 128,134.6914005,133.7120085 4 | 256,211.9761356,192.4693859 5 | 512,304.6760902,327.8956378 6 | 1024,483.9663861,601.0082197 7 | 2048,853.497982,1015.167211 8 | 4096,1550.553082,1774.245648 -------------------------------------------------------------------------------- /src/testscript/config.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "media": "lpmem", 3 | "interleaving": "1", 4 | "local": "1", 5 | "avgpower": "15000", 6 | "peakpower": "20000", 7 | "emon": "0", 8 | "aepwatch": "1", 9 | "fence": "0", 10 | "baddata": "0", 11 | "prefetching": "0", 12 | "rerun": "0" 13 | } 14 | -------------------------------------------------------------------------------- /graphs/12-flush_instr_bw.csv: -------------------------------------------------------------------------------- 1 | ,ntstore,store+clwb,store 2 | 64,1.831835938,2.756445313,2.463085938 3 | 128,3.551757813,4.976757813,2.832617188 4 | 256,6.836328125,6.63203125,3.17265625 5 | 512,13.225,6.672265625,3.111328125 6 | 1024,13.42421875,6.6875,3.209375 7 | 2048,13.36875,6.66875,3.2875 8 | 4096,13.45,6.890625,3.240625 -------------------------------------------------------------------------------- /graphs/17-numa.csv: -------------------------------------------------------------------------------- 1 | ,Optane-1,Optane-Remote-1,Optane-4,Optane-Remote-4 2 | R,4.700488281,4.31796875,14.61962891,13.30830078 3 | R:W (4:1),5.786914063,5.017382813,13.84111328,4.773242188 4 | R:W (3:1),5.953027344,4.973925781,12.74951172,2.18984375 5 | R:W (2:1),5.868261719,4.647070313,10.40605469,1.742871094 6 | R:W (1:1),6.259667969,4.832324219,6.898730469,1.951269531 7 | W,5.225195313,2.135058594,13.50947266,5.901269531 -------------------------------------------------------------------------------- /graphs/15-alloc_load.csv: -------------------------------------------------------------------------------- 1 | ,6 Threads,3 Threads,2 Threads,1 Thread 2 | 64,6.740039063,8.930078125,9.43515625,8.753515625 3 | 128,18.31015625,18.96972656,18.36660156,19.86347656 4 | 256,34.32109375,34.43222656,33.03945313,35.58183594 5 | 512,34.78203125,35.97304688,36.35703125,38.778125 6 | 1024,32.97578125,34.840625,35.8140625,39.0796875 7 | 2048,28.9375,32.490625,34.4578125,38.865625 8 | 4096,26.86875,29.803125,32.93125,38.440625 -------------------------------------------------------------------------------- /graphs/06-bandwidth_thmix.csv: -------------------------------------------------------------------------------- 1 | Read-Write,PM-LDRAM,PM-RDRAM,PM-Optane,PM-PMEP 2 | 0-16,67872,29490.2,3892.9,2378.3 3 | 2-14,69878.4,32583.8,4698.7,1822.3 4 | 4-12,71289.4,36207.2,5706.4,1949.4 5 | 6-10,73426.3,39473.1,7011.4,2093.4 6 | 8-8,75004.5,41556.5,7740.3,2215.8 7 | 10-6,77135.1,41188.7,7856.2,2359.2 8 | 12-4,79368.3,38783.1,8298.5,2600.3 9 | 14-2,80706.4,36429.1,8325.9,3426.1 10 | 16-0,84629.4,34302.7,10714.6,2676.5 11 | -------------------------------------------------------------------------------- /graphs/15-alloc_store.csv: -------------------------------------------------------------------------------- 1 | ,6 Threads,3 Threads,2 Threads,1 Thread 2 | 64,3.153320313,3.2890625,3.265625,3.34765625 3 | 128,5.526367188,5.666015625,6.048828125,6.416992188 4 | 256,12.05078125,12.703125,12.80859375,13.29296875 5 | 512,11.77050781,12.26757813,12.71679688,13.29785156 6 | 1024,11.36523438,11.60449219,12.51367188,13.28320313 7 | 2048,10.61523438,10.85449219,12.1484375,13.38085938 8 | 4096,9.706054688,9.931640625,11.48730469,13.39648438 9 | -------------------------------------------------------------------------------- /src/data/parse.R: -------------------------------------------------------------------------------- 1 | df<-read.csv("aepsweep.csv") 2 | df<-df[df$sccess_size == 0 || df$access_size == df$stride_size,] 3 | df<-df[df$rerun==0,] 4 | #df<-df[df$media == 'lpmem'] 5 | df<-df[df$avgpower == 15000,] 6 | #df<-df[df$task_type=="RandBW",] 7 | #vars = c("access_size", "media", "operation", "prefetching", "stride_size", "task_type", "threads", "throughput", "time") 8 | #df1<-df[vars] 9 | rpivotTable::rpivotTable(data = df, header=FALSE) 10 | 11 | -------------------------------------------------------------------------------- /graphs/18-pmemkv.csv: -------------------------------------------------------------------------------- 1 | overwrite,DRAM,DRAM-Remote,Optane,Optane-Remote 2 | 1,2.231054688,1.776855469,0.9965820313,0.8059570313 3 | 2,3.859375,3.112207031,1.530566406,1.135839844 4 | 3,5.524121094,4.331445313,2.108984375,1.089355469 5 | 4,6.705664063,5.551269531,2.372558594,0.891796875 6 | 6,8.717871094,7.097460938,2.631835938,0.66875 7 | 8,10.17705078,8.902832031,2.611523438,0.6234375 8 | 10,10.54599609,9.804785156,2.6296875,0.59453125 9 | 12,10.16142578,9.883886719,2.581445313,0.5811523438 -------------------------------------------------------------------------------- /src/testscript/parsing/50_repeat_cloud.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import repeat 3 | import os 4 | 5 | stride_array=[0, 64, 128, 192, 256, 320, 384, 448, 512, 768, 1024, 1280, 1536, 2048] 6 | tag_array=['ldram', 'lpmem', 'ldram_emon', 'lpmem_emon', 'lpmem_18', 'ldram_18'] 7 | 8 | 9 | for sizebit in range(24, 30): 10 | size = 2**sizebit; 11 | for tag in tag_array: 12 | tag = 'AEP2-5-256-{0}-{1}'.format(size, tag) 13 | repeat.download(tag) 14 | repeat.parse(tag) 15 | 16 | -------------------------------------------------------------------------------- /src/testscript/parsing/100_probe.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | gsutil cat gs://gcp-bucket/AEP2-10-256-1-probe/stdout.txt | \ 4 | grep -A12 'DIMM 0$' | awk '{print $1}' | awk -F, '{print $1}' > output.txt 5 | 6 | stride=$1 7 | for i in `seq 0 12`; do 8 | count=`echo "2^$i" | bc` 9 | gsutil cat gs://gcp-bucket/AEP2-10-$stride-$count-probe/stdout.txt | \ 10 | grep -A12 'DIMM 0$' | awk '{print $2}' > one.txt 11 | paste output.txt one.txt -d, > temp.txt 12 | mv temp.txt output.txt 13 | done 14 | 15 | rm one.txt 16 | cat output.txt 17 | -------------------------------------------------------------------------------- /src/data/example.delete.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from google.cloud import datastore 4 | 5 | client = datastore.Client() 6 | query = client.query(kind='aepsweep') 7 | query.add_filter('avgpower', '=', 12500) 8 | query.add_filter('machine', '=', 'AEP1') 9 | query.add_filter('baddata', '=', '1') 10 | 11 | l = [] 12 | result = list(query.fetch()) 13 | print(len(result)) 14 | for item in result: 15 | l.append(item.key) 16 | if len(l) >= 500: 17 | client.delete_multi(l) 18 | l = [] 19 | client.delete_multi(l) 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /graphs/05-bandwidth_accesssize_dram.csv: -------------------------------------------------------------------------------- 1 | AccessSize,Read,Write(ntstore),Write(clwb) 2 | 64,93.35,74.7625,26.3125 3 | 128,94.4125,74.25,25.8125 4 | 256,96.475,76.7875,26.025 5 | 512,96.0875,76.9,26.125 6 | 1024,96.6625,77.325,25.8125 7 | 2048,98.0875,76.75,26.1125 8 | 4096,98.725,76.625,26.025 9 | 8192,99.2625,77.35,25.9875 10 | 16384,98.825,77.3625,26.08554688 11 | 32768,99.525,77.1,26.125 12 | 65536,100.0375,77.2875,26.0875 13 | 131072,100.1,77.0625,26.125 14 | 262144,100.125,78.3375,26.3625 15 | 524288,100.6625,77.15,25.9375 16 | 1048576,101.2625,77.125,26.1 17 | 2097152,101.45,76.84,26.3 -------------------------------------------------------------------------------- /graphs/04-bandwidth_thcount_optane_ni.csv: -------------------------------------------------------------------------------- 1 | Threads,Read,Write(ntstore),Write(clwb) 2 | 1,2.92,2.22,1.44 3 | 2,5.66,2.21,1.79 4 | 3,6.45,2.21,1.79 5 | 4,6.52,2.20,1.71 6 | 5,6.51,2.12,1.64 7 | 6,6.50,1.96,1.55 8 | 7,6.50,1.73,1.48 9 | 8,6.46,1.53,1.40 10 | 9,6.39,1.37,1.30 11 | 10,6.29,1.25,1.21 12 | 11,6.14,1.16,1.12 13 | 12,5.91,1.12,1.08 14 | 13,5.60,1.02,1.06 15 | 14,5.25,0.98,0.98 16 | 15,4.97,0.94,0.96 17 | 16,4.73,0.94,0.88 18 | 17,4.59,0.93,0.90 19 | 18,4.30,0.84,0.84 20 | 19,4.16,0.89,0.89 21 | 20,4.10,0.86,0.85 22 | 21,4.02,0.84,0.82 23 | 22,3.97,0.78,0.84 24 | 23,3.91,0.81,0.81 25 | 24,3.89,0.80,0.82 -------------------------------------------------------------------------------- /src/testscript/perf_mount.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | 4 | if [ $# -ne 2 ]; then 5 | echo './mount.sh [DRAM_DEV] [PMEM_DEV]' 6 | echo 'e.g., ./mount.sh /dev/pmem0 /dev/pmem14' 7 | exit 1 8 | fi 9 | 10 | cd ../kernel 11 | 12 | DRAM_DEV=$1 13 | PMEM_DEV=$2 14 | 15 | 16 | echo 0 > /proc/sys/kernel/soft_watchdog 17 | # Hard lock watchdog at nmi_watchdog 18 | 19 | make clean 2>&1 > /dev/null 20 | make -j 2 PERF=1>&1 21 | 22 | mkdir -p /mnt/latency 23 | mkdir -p /mnt/report 24 | 25 | insmod repfs.ko 26 | insmod latfs.ko 27 | 28 | mount -t ReportFS $DRAM_DEV /mnt/report 29 | mount -t LatencyFS $PMEM_DEV /mnt/latency 30 | 31 | -------------------------------------------------------------------------------- /graphs/04-bandwidth_thcount_optane.csv: -------------------------------------------------------------------------------- 1 | Threads,Read,Write(clwb),Write(ntstore) 2 | 1,2.88,4.13,1.42 3 | 2,5.75,8.19,2.85 4 | 3,8.94,11.81,4.33 5 | 4,11.63,12.50,5.75 6 | 5,14.31,12.50,7.23 7 | 6,16.63,12.31,8.50 8 | 7,18.94,12.25,9.63 9 | 8,21.13,12.00,9.98 10 | 9,23.25,11.81,9.90 11 | 10,24.94,11.63,9.75 12 | 11,26.88,11.38,9.63 13 | 12,28.75,10.69,9.50 14 | 13,30.56,10.56,9.21 15 | 14,32.13,10.00,8.85 16 | 15,33.69,9.38,8.75 17 | 16,34.88,10.00,8.67 18 | 17,35.69,9.56,8.25 19 | 18,36.56,9.00,8.25 20 | 19,37.06,9.13,8.31 21 | 20,37.19,8.75,7.92 22 | 21,37.25,7.88,7.88 23 | 22,37.63,8.25,7.79 24 | 23,37.56,8.19,7.67 25 | 24,37.56,7.50,7.50 -------------------------------------------------------------------------------- /graphs/04-bandwidth_thcount_dram.csv: -------------------------------------------------------------------------------- 1 | Threads,Read,Write(ntstore),Write(clwb) 2 | 1,7.38,4.31,1.44 3 | 2,14.50,8.69,3.00 4 | 3,21.88,13.00,4.44 5 | 4,29.31,17.25,6.00 6 | 5,37.63,21.56,7.50 7 | 6,46.88,25.75,9.00 8 | 7,56.25,29.88,10.31 9 | 8,65.38,33.81,11.75 10 | 9,73.63,38.13,13.31 11 | 10,80.19,42.13,14.75 12 | 11,86.44,46.00,16.31 13 | 12,91.88,49.56,17.75 14 | 13,96.31,52.19,19.19 15 | 14,99.88,56.63,20.75 16 | 15,101.56,61.44,22.19 17 | 16,103.56,64.19,23.75 18 | 17,104.88,67.31,25.00 19 | 18,105.31,69.63,26.75 20 | 19,105.56,71.44,28.13 21 | 20,105.38,73.56,29.63 22 | 21,105.44,74.63,31.13 23 | 22,105.31,75.94,32.56 24 | 23,104.81,76.38,34.00 25 | 24,104.56,76.81,35.56 -------------------------------------------------------------------------------- /graphs/05-bandwidth_accesssize_optane_ni.csv: -------------------------------------------------------------------------------- 1 | AccessSize,Read,Write(ntstore),Write(clwb) 2 | 64,1.675,0.5546875,0.51328125 3 | 128,3.34375,1.0984375,0.92734375 4 | 256,6.5046875,2.20703125,1.7546875 5 | 512,6.53046875,2.209375,1.7609375 6 | 1024,6.51875,2.2234375,1.7765625 7 | 2048,6.49921875,2.22734375,1.7859375 8 | 4096,6.559375,2.23125,1.77421875 9 | 8192,6.51640625,2.23671875,1.778125 10 | 16384,6.50234375,2.2328125,1.775 11 | 32768,6.5140625,2.22109375,1.7875 12 | 65536,6.5578125,2.21640625,1.77578125 13 | 131072,6.5046875,2.21796875,1.771875 14 | 262144,6.51953125,2.2171875,1.7765625 15 | 524288,6.56875,2.21953125,1.771875 16 | 1048576,6.56171875,2.21015625,1.7765625 17 | 2097152,6.510742188,2.21015625,1.771875 -------------------------------------------------------------------------------- /graphs/13-sfence_interval.csv: -------------------------------------------------------------------------------- 1 | ,clwb(every 64B),clwb(write size),ntstore 2 | 64,0.575,0.571875,0.3375 3 | 128,1.05,0.990625,0.65 4 | 256,1.290625,1.30625,1.2625 5 | 512,1.278125,1.21875,2.2125 6 | 1024,1.275,1.228125,2.2375 7 | 2048,1.140625,1.134375,2.225 8 | 4096,1.103125,1.10625,2.2375 9 | 8192,1.1125,1.103125,2.2375 10 | 16384,1.075,1.109375,2.2125 11 | 32768,1.090625,1.0875,2.225 12 | 65536,1.084375,1.065625,2.225 13 | 131072,1.090625,1.053125,2.225 14 | 262144,1.084375,1.05,2.2375 15 | 524288,1.0875,1.059375,2.2375 16 | 1048576,1.08125,1.0625,2.225 17 | 2097152,1.084375,1.05625,2.2375 18 | 4194304,1.0875,1.065625,2.2375 19 | 8388608,1.08125,1.01875,2.225 20 | 16777216,1.1,0.975,2.225 21 | 33554432,1.1,0.9625,2.225 22 | 67108864,1.1,0.8,2.225 -------------------------------------------------------------------------------- /src/testscript/subtests/70_straight.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #`sed 's/^#define SIZEBT_MACRO.*$/TEST/g'` 4 | 5 | 6 | cd ../kernel 7 | 8 | rm -f ../kernel/output.txt > /dev/null 9 | cp -f memaccess.c memaccess.c.bak 10 | 11 | hostname=`hostname -s` 12 | TAG=${TAG:-`date +%Y%m%d%H%M%S`} 13 | 14 | GS_Bucket="gcp-bucket" 15 | 16 | 17 | for writesize in `seq 4096 4096 49152`; do 18 | msg="$hostname-7-$writesize-$TAG" 19 | echomsg="task=7,write_size=$writesize,write_start=204795904" # 4999x4096 20 | echo $echomsg 21 | export TaskID=$msg 22 | echo Report $1 Test $2 23 | ../testscript/umount.sh 24 | ../testscript/mount.sh $1 $2 25 | echo "sleep 5" > /tmp/aeprun.sh 26 | echo "echo $echomsg > /proc/lattester" >> /tmp/aeprun.sh 27 | echo "sleep 30" >> /tmp/aeprun.sh 28 | chmod +x /tmp/aeprun.sh 29 | aeprun /tmp/aeprun.sh 30 | done 31 | 32 | -------------------------------------------------------------------------------- /graphs/09-infer_xpbuffer.csv: -------------------------------------------------------------------------------- 1 | RegionSize,WriteAmp 2 | 256,0.7983534336 3 | 512,0.9125254154 4 | 1024,0.9702842236 5 | 2048,1.036476374 6 | 4096,1.034444809 7 | 8192,1.035977364 8 | 16384,1.036271095 9 | 18432,1.038828373 10 | 20480,1.358539343 11 | 22528,1.503426075 12 | 24576,1.608360767 13 | 26624,1.681452274 14 | 28672,1.736249924 15 | 30720,1.786276579 16 | 32768,1.819602013 17 | 34816,1.856842041 18 | 36864,1.876713514 19 | 38912,1.893642426 20 | 40960,1.917868853 21 | 43008,1.936532259 22 | 45056,1.954580069 23 | 47104,1.970157623 24 | 49152,1.984173536 25 | 51200,1.996604919 26 | 53248,2.007486582 27 | 55296,2.015207052 28 | 57344,2.022521019 29 | 59392,2.028713226 30 | 61440,2.034002066 31 | 63488,2.040292978 32 | 65536,2.043026686 33 | 131072,2.076053858 34 | 262144,2.076242447 35 | 524288,2.076369524 36 | 1048576,2.075001478 37 | 2097152,2.074094296 38 | -------------------------------------------------------------------------------- /graphs/03-tail_latency.csv: -------------------------------------------------------------------------------- 1 | HotspotRegion,99.99%,99.999%,Max 2 | 256,0.32,52.73784243,56.8 3 | 512,0.3863636364,52.42506345,56.48727273 4 | 1024,0.3836363636,52.2,59.93454545 5 | 2048,0.3845454545,51.00503496,56.65454545 6 | 4096,0.3772727273,50.97534576,59.08272727 7 | 8192,0.3818181818,49.49370629,63.33 8 | 16384,0.3781818182,47.64149758,53.44636364 9 | 32768,0.3772727273,48.43600878,55.39636364 10 | 65536,0.3736363636,18.53914935,32.72 11 | 131072,0.3745454545,0.3945454545,30.54181818 12 | 262144,0.3736363636,0.3881818182,17.49363636 13 | 524288,0.3723361818,0.3918181818,17.07818182 14 | 1048576,0.2481818182,0.2518181818,16.28181818 15 | 2097152,0.25,0.2545454545,15.70545455 16 | 4194304,0.3663636364,0.38,16.96272727 17 | 8388608,0.3745454545,0.3863636364,15.54818182 18 | 16777216,0.3736363636,0.3927790636,15.58454545 19 | 33554432,0.3727272727,0.3863636364,16.12636364 20 | 67108864,0.3754545455,0.3927790636,15.78090909 21 | -------------------------------------------------------------------------------- /src/data/example.query.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | import csv 3 | import os 4 | import pandas as pd 5 | from google.cloud import datastore 6 | from pivottablejs import pivot_ui 7 | import subprocess 8 | 9 | client = datastore.Client() 10 | query = client.query(kind='aepsweep') 11 | query.add_filter('baddata', '=', 0) 12 | query.add_filter('task', '=', 'Rand') 13 | #query.add_filter('avgpower', '=', 15000) 14 | #query.add_filter('interleaving', '=', 1) 15 | #query.add_filter('task_type', '=', 'RandBW') 16 | query.add_filter('operation', '=', 'Store+Flush') 17 | query.add_filter('tag', '=', 'storeflush') 18 | data = [] 19 | results = list(query.fetch()) 20 | 21 | df = pd.DataFrame(results) 22 | df = df.loc[:, (df != df.iloc[0]).any()] 23 | #df = df.drop(columns=['tag', 'date', 'avgpower', 'peakpower']) 24 | pivot_ui(df) 25 | 26 | with open('output.csv', 'w') as writeFile: 27 | writer = csv.writer(writeFile) 28 | writer.writerows(data) 29 | -------------------------------------------------------------------------------- /src/testscript/subtests/50_repeat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #`sed 's/^#define SIZEBT_MACRO.*$/TEST/g'` 4 | 5 | 6 | cd ../kernel 7 | 8 | rm -f ../kernel/output.txt > /dev/null 9 | cp -f memaccess.c memaccess.c.bak 10 | 11 | stride_array=( 0) 12 | stride_array=( 0 64 128 192 256 320 384 448 512 768 1024 1280 1536 2048 ) 13 | hostname=`hostname -s` 14 | TAG=${TAG:-`date +%Y%m%d%H%M%S`} 15 | 16 | GS_Bucket="gcp-bucket" 17 | 18 | #- StrideBW: AEP1-5-0 19 | 20 | for stridesize in ${stride_array[@]}; do 21 | msg="$hostname-5-$stridesize-$TAG" 22 | echomsg="task=5,stride_size=$stridesize" 23 | echo $echomsg 24 | export TaskID=$msg 25 | echo Report $1 Test $2 26 | ../testscript/umount.sh 27 | ../testscript/mount.sh $1 $2 28 | echo "sleep 5" > /tmp/aeprun.sh 29 | echo "echo $echomsg > /proc/lattester" >> /tmp/aeprun.sh 30 | echo "sleep 5" >> /tmp/aeprun.sh 31 | chmod +x /tmp/aeprun.sh 32 | aeprun /tmp/aeprun.sh 33 | sleep 5 34 | dd if=$1 of=/tmp/dump bs=8M count=2 35 | gsutil cp /tmp/dump gs://$GS_Bucket/$TaskID/dump 36 | done 37 | 38 | -------------------------------------------------------------------------------- /graphs/05-bandwidth_accesssize_optane.csv: -------------------------------------------------------------------------------- 1 | AccessSize,Read,Write(ntstore),Write(clwb) 2 | 64,9.67,3.166015625,2.75 3 | 128,18.75,5.833007813,4.25 4 | 256,33.65,12.1875,9.478515625 5 | 512,34.23,11.875,9.3125 6 | 1024,32.29,11.35351563,8.916015625 7 | 2048,29.33,10.52050781,8.25 8 | 4096,25.98,9.4375,7.6875 9 | 8192,27.08,9.958007813,7.978515625 10 | 12288,28.39,10.56152344,8.249023438 11 | 16384,29.33,10.95800781,8.6875 12 | 20480,30.26,11.47558594,8.872070313 13 | 24576,30.85,12.22460938,9.225585938 14 | 28672,31.64,11.70507813,9.225585938 15 | 32768,32.10,11.6875,9.270507813 16 | 36864,32.39,11.74707031,9.413085938 17 | 40960,32.64,11.85058594,9.434570313 18 | 45056,32.63,12.01660156,9.413085938 19 | 49152,32.82,12.32910156,9.475585938 20 | 53248,32.99,12.07910156,9.497070313 21 | 57344,33.18,11.99609375,9.622070313 22 | 61440,33.57,12.03808594,9.497070313 23 | 65536,33.79,12.16601563,9.5 24 | 131072,35.33,12.375,9.75 25 | 262144,36.52,12.58300781,9.93 26 | 524288,37.40,12.625,10.0625 27 | 1048576,37.75,12.63,10.04101563 28 | 2097152,37.93,12.65,10.06 -------------------------------------------------------------------------------- /src/testscript/run.example.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Sample run script 4 | 5 | # Disable CPU Prefetching 6 | wrmsr -a 0x1a4 0xf 7 | 8 | export GOOGLE_APPLICATION_CREDENTIALS=dbuser.json 9 | export AEPWatch=1 10 | export EMon=0 11 | 12 | export PATH=$PATH:`pwd`/subtests/bin 13 | testapp=./subtests/30_stridebw.sh 14 | repdev=`mount | grep ReportFS | awk {'print \$1'}` 15 | testdev=`mount | grep LatencyFS | awk {'print \$1'}` 16 | runtime=20 17 | 18 | if [ -z $repdev ] || [ -z $testdev ]; then 19 | echo "Please run mount.sh first" 20 | exit 1 21 | fi 22 | 23 | for i in `seq $runtime`; do 24 | export TAG=repeatr$i 25 | echo =====Workload description==== 26 | echo Run: $testapp 27 | echo Tag: $TAG 28 | echo LatFS on: $testdev 29 | echo RepFS on: $repdev 30 | echo Config: 31 | cat config.json 32 | echo Press any key to continue... 33 | read 34 | 35 | $testapp $repdev $testdev 36 | ./parse_bw.py output.txt $runtime 1 > $TAG.summary.txt 37 | mv output.txt ./$TAG.output.txt 38 | 39 | done 40 | 41 | wrmsr -a 0x1a4 0x0 42 | 43 | aeprelease 44 | -------------------------------------------------------------------------------- /src/kernel/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Makefile for the Latency and Report psudo file systems. 3 | # 4 | 5 | obj-m += repfs.o latfs.o 6 | 7 | ccflags-y+=-mmmx -msse2 -mavx -mpreferred-stack-boundary=4 8 | repfs-y := rep.o 9 | latfs-y := lat.o proc.o misc.o memaccess.o tasks.o 10 | 11 | avx_supported := $(call as-instr,vpxor %xmm0$(comma)%xmm0$(comma)%xmm0,yes,no) 12 | avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\ 13 | $(comma)4)$(comma)%ymm2,yes,no) 14 | 15 | AEP := $(shell cat /proc/cpuinfo | grep clwb | wc -l) 16 | 17 | ifeq ($(AEP), 0) 18 | ccflags-y+= -DNOAVX512 19 | else 20 | ccflags-y+= -DAEP_SUPPORTED 21 | endif 22 | 23 | ifeq ($(PERF), 1) 24 | ccflags-y+=-DUSE_PERF 25 | latfs-y+=perf_util.o 26 | endif 27 | 28 | ccflags-y+= -DFENCE 29 | 30 | all: 31 | make -C /lib/modules/$(shell uname -r)/build M=`pwd` 32 | gcc -O2 -o gen_tasks gen_tasks.c $(ccflags-y) 33 | 34 | debug: 35 | gcc -O0 -o debug -mavx -g debug.c memaccess.c $(ccflags-y) 36 | 37 | clean: 38 | make -C /lib/modules/$(shell uname -r)/build M=`pwd` clean 39 | rm -f tasks.py gen_tasks debug *.pyc -f 40 | 41 | -------------------------------------------------------------------------------- /src/plot/plot_timeseries.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | import numpy as np 4 | import matplotlib 5 | matplotlib.use('Agg') 6 | import matplotlib.pyplot as plt 7 | from matplotlib.backends.backend_pdf import PdfPages 8 | 9 | ymax=1E6 10 | ymin=1E1 11 | pp = PdfPages('timeseries.pdf') 12 | 13 | for i in range(1, len(sys.argv)): 14 | fileName=sys.argv[i] 15 | chartTitle='' 16 | x=[] 17 | times=[] 18 | with open(fileName, mode='r') as f: 19 | for line in f: 20 | line = line.strip('\n').strip('\r') 21 | try: 22 | line = int(line) 23 | times.append(min(ymax, line)) 24 | except ValueError: 25 | chartTitle = line 26 | #plt.yticks(np.arange(ymin, ymax, 25)) 27 | #plt.xticks(np.arange(0, len(times) - 1, 100000)) 28 | #plt.xticks(rotation='vertical') 29 | plt.title(chartTitle, fontsize=10) 30 | plt.ylim(ymin, ymax) 31 | plt.yscale('log') 32 | plt.plot(times) 33 | plt.grid(True) 34 | plt.ylabel('Latency (cycles)') 35 | plt.savefig(pp, format='pdf') 36 | 37 | pp.close() 38 | -------------------------------------------------------------------------------- /src/plot/plot_cdf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | import numpy as np 4 | import matplotlib 5 | matplotlib.use('Agg') 6 | import matplotlib.pyplot as plt 7 | from matplotlib.backends.backend_pdf import PdfPages 8 | 9 | pp = PdfPages('cdf.pdf') 10 | 11 | for i in range(1, len(sys.argv)): 12 | fileName=sys.argv[i] 13 | chartTitle='' 14 | x=[] 15 | times=[] 16 | with open(fileName, mode='r') as f: 17 | for line in f: 18 | line = line.strip('\n').strip('\r') 19 | try: 20 | line = int(line) 21 | times.append(line) 22 | x.append(len(x)) 23 | except ValueError: 24 | chartTitle = line 25 | chartSize = len(x) 26 | x[:] = [float(item + 1) / chartSize for item in x] 27 | times.sort() 28 | plt.title(chartTitle, fontsize=10) 29 | plt.xticks(np.arange(0, 1.01, 0.05), rotation='vertical') 30 | plt.yscale('log') 31 | plt.ylim(1E1, 1E6) 32 | plt.plot(x, times) 33 | plt.grid(True) 34 | plt.ylabel('Latency (cycles)') 35 | plt.savefig(pp, format='pdf') 36 | 37 | pp.close() 38 | -------------------------------------------------------------------------------- /src/testscript/subtests/20_stridelat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | stride_array=$(seq -s ' ' 0 64 512) 4 | access_64b_4k=$(seq -s ' ' 64 64 4096) 5 | access_4k_16k=$(seq -s ' ' 4352 256 16384) 6 | access_array=("${access_64b_4k[@]}" "${access_4k_16k[@]}") 7 | 8 | rm -f output.txt 9 | for accesssize in ${access_array[@]}; do 10 | for stridesize in ${stride_array[@]}; do 11 | stridesize=`expr $accesssize + $stridesize` 12 | msg="AEP2-2-$accesssize-$stridesize" 13 | echo "task=2,access_size=$accesssize,stride_size=$stridesize,message=$msg" | tee -a output.txt 14 | echo "task=2,access_size=$accesssize,stride_size=$stridesize,message=$msg" > /proc/lattester 15 | if [[ $accesssize > $stridesize ]]; then 16 | continue 17 | fi 18 | while true; do 19 | x=`dmesg | tail -n 3 | grep 'LATTester_LAT_END'` 20 | if [[ $x ]]; then 21 | break; 22 | fi 23 | sleep 1 24 | done 25 | msgstart=`dmesg | grep -n 'LATTester_START' | tail -1 | awk -F':' {' print $1 '}` 26 | msgend=`dmesg | grep -n 'LATTester_LAT_END' | tail -1 | awk -F':' {' print $1 '}` 27 | msgend+="p" 28 | dmesg | sed -n "$msgstart,$msgend" | grep "average" | tee -a output.txt 29 | done 30 | done 31 | -------------------------------------------------------------------------------- /src/data/example.update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | import csv 3 | import os 4 | import pandas as pd 5 | from google.cloud import datastore 6 | from pivottablejs import pivot_ui 7 | import subprocess 8 | 9 | client = datastore.Client() 10 | query = client.query(kind='aepsweep') 11 | #query.add_filter('baddata', '=', 0) 12 | #query.add_filter('task', '=', 'Rand') 13 | #query.add_filter('avgpower', '=', 15000) 14 | #query.add_filter('interleaving', '=', 1) 15 | #query.add_filter('task_type', '=', 'RandBW') 16 | #query.add_filter('operation', '=', 'Store+Flush') 17 | query.add_filter('tag', '=', 'storeflush') 18 | data = [] 19 | results = list(query.fetch()) 20 | 21 | count = 0 22 | for item in results: 23 | # if item['stride_size'] == item['access_size']: 24 | throughput = item['throughput'] 25 | tag = item.key.name 26 | line = subprocess.check_output('gsutil cat gs://gcp-bucket/'+tag+'/stdout.txt | grep "media\:imc.write" | awk \'{print $2}\'', shell=True) 27 | ewr = 1.0/float(line) 28 | data.append([tag, ewr, throughput]) 29 | count += 1 30 | print('[{0}/{1}]'.format(count, len(results)), ewr, throughput, tag) 31 | item.update({'ewr': ewr}) 32 | client.put(item) 33 | 34 | -------------------------------------------------------------------------------- /src/testscript/mount.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Mount file systems 3 | 4 | if [[ $# -ne 2 ]]; then 5 | echo './mount.sh [DRAM_DEV] [PMEM_DEV]' 6 | echo 'e.g., ./mount.sh /dev/pmem0 /dev/pmem14' 7 | exit 1 8 | fi 9 | 10 | if [[ $EUID -ne 0 ]]; then 11 | echo "Please run as root" 12 | exit 1 13 | fi 14 | 15 | cd ../kernel 16 | 17 | DRAM_DEV=$1 18 | PMEM_DEV=$2 19 | 20 | 21 | repdev=`mount | grep ReportFS | awk {'print \$1'}` 22 | testdev=`mount | grep LatencyFS | awk {'print \$1'}` 23 | 24 | 25 | if [ ! -z $repdev ] || [ ! -z $testdev ]; then 26 | echo "Please run umount.sh first" 27 | exit 1 28 | fi 29 | 30 | 31 | echo 0 > /proc/sys/kernel/soft_watchdog 32 | # Hard lock watchdog at nmi_watchdog 33 | 34 | make clean 2>&1 > /dev/null 35 | make -j 2>&1 36 | 37 | mkdir -p /mnt/latency 38 | mkdir -p /mnt/report 39 | 40 | REPFS=`lsmod | grep repfs` 41 | LATFS=`lsmod | grep latfs` 42 | 43 | if [ ! -z "$REPFS" ]; then 44 | echo Unmounting existing ReportFS 45 | ../testscript/umount.sh 46 | elif [ ! -z "$LATFS" ]; then 47 | echo Unmounting existing LatencyFS 48 | ../testscript/umount.sh 49 | fi 50 | 51 | insmod repfs.ko 52 | insmod latfs.ko 53 | 54 | mount -t ReportFS $DRAM_DEV /mnt/report 55 | mount -t LatencyFS $PMEM_DEV /mnt/latency 56 | -------------------------------------------------------------------------------- /src/testscript/subtests/60_raw.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #stride_array=$(seq -s ' ' 0 64 512) 4 | #stride_array=$(seq -s ' ' 0 64 256) 5 | stride_array=( 0 ) 6 | 7 | access_64b_4k=$(seq -s ' ' 64 64 4096) 8 | access_4k_16k=$(seq -s ' ' 4352 256 16384) 9 | access_16k_72k=$(seq -s ' ' 17408 1024 73728) 10 | #access_array=$(seq -s ' ' 5120 256 16384) 11 | access_array=("${access_64b_4k[@]}" "${access_4k_16k[@]}") 12 | #access_array=("${access_16k_72k[@]}") 13 | 14 | rm -f output.txt 15 | for accesssize in ${access_array[@]}; do 16 | for stridesize in ${stride_array[@]}; do 17 | stridesize=`expr $accesssize + $stridesize` 18 | msg="AEP2-6-$accesssize-$stridesize" 19 | echo "task=6,access_size=$accesssize,stride_size=$stridesize,message=$msg" | tee -a output.txt 20 | echo "task=6,access_size=$accesssize,stride_size=$stridesize,message=$msg" > /proc/lattester 21 | while true; do 22 | sleep 2 23 | x=`dmesg | tail -n 4 | grep 'LATTester_LAT_END'` 24 | if [[ $x ]]; then 25 | break; 26 | fi 27 | done 28 | msgstart=`dmesg | grep -n 'LATTester_START' | tail -1 | awk -F':' {' print $1 '}` 29 | msgend=`dmesg | grep -n 'LATTester_LAT_END' | tail -1 | awk -F':' {' print $1 '}` 30 | msgend+="p" 31 | dmesg | sed -n "$msgstart,$msgend" | grep "average" | tee -a output.txt 32 | done 33 | done 34 | -------------------------------------------------------------------------------- /src/testscript/perf_stridelat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #stride_array=( 64 ) 4 | #access_array=( 64 128 192 256 320 384 448 512 ) 5 | 6 | stride_array=( 0 64 128 192 256 320 384 448 512 ) 7 | access_array=( 0 64 128 192 256 320 384 448 512 576 640 704 768 832 896 960 1024 1088 1152 1216 1280 1344 1408 1472 1536 1600 1664 1728 1792 1856 1920 1984 2048 ) 8 | 9 | output_file=`date +%Y%m%d-%H%M%S`-output.txt 10 | 11 | wrmsr -a 0x1a4 0xf 12 | 13 | for accesssize in ${access_array[@]}; do 14 | for stridesize in ${stride_array[@]}; do 15 | msg="AEP2-2-$accesssize-$stridesize" 16 | echo "===================task=2,access_size=$accesssize,stride_size=$stridesize,message=$msg======================" | tee -a $output_file 17 | echo "task=2,access_size=$accesssize,stride_size=$stridesize,message=$msg" > /proc/lattester 18 | while true; do 19 | sleep 1 20 | x=`dmesg | tail -n 5 | grep 'LATTester_LAT_END'` 21 | if [[ $x ]]; then 22 | break; 23 | fi 24 | done 25 | msgstart=`dmesg | grep -n 'LATTester_START' | tail -1 | awk -F':' {' print $1 '}` 26 | msgend=`dmesg | grep -n 'LATTester_LAT_END' | tail -1 | awk -F':' {' print $1 '}` 27 | msgend+="p" 28 | dmesg | sed -n "$msgstart,$msgend" | grep "average" | tee -a $output_file 29 | dmesg | sed -n "$msgstart,$msgend" | grep "Perf counter" | tee -a $output_file 30 | done 31 | done 32 | 33 | wrmsr -a 0x1a4 0x0 34 | -------------------------------------------------------------------------------- /src/testscript/subtests/80_align.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #`sed 's/^#define SIZEBT_MACRO.*$/TEST/g'` 4 | 5 | 6 | cd ../kernel 7 | 8 | rm -f ../kernel/output.txt > /dev/null 9 | cp -f memaccess.c memaccess.c.bak 10 | 11 | hostname=`hostname -s` 12 | TAG=${TAG:-`date +%Y%m%d%H%M%S`} 13 | 14 | GS_Bucket="gcp-bucket" 15 | op_array=( 0 1 2 3 ) 16 | 17 | for parallel in `seq 18 18`; do 18 | for bwsize_bit in `seq 6 12`; do 19 | for op in ${op_array[@]}; do 20 | for align_mode in `seq 5 6`; do 21 | msg="$hostname-8-$bwsize_bit-$parallel-$op-$align_mode-$TAG" 22 | echomsg="task=8,parallel=$parallel,bwsize_bit=$bwsize_bit,op=$op,align_mode=$align_mode,message=$msg,runtime=5" 23 | echo $echomsg 24 | export TaskID=$msg 25 | echo "sleep 3" > /tmp/aeprun.sh 26 | echo "echo $echomsg > /proc/lattester" >> /tmp/aeprun.sh 27 | echo "sleep 1" >> /tmp/aeprun.sh 28 | echo "sleep 1" >> /tmp/aeprun.sh 29 | echo "sleep 1" >> /tmp/aeprun.sh 30 | echo "sleep 1" >> /tmp/aeprun.sh 31 | echo "sleep 1" >> /tmp/aeprun.sh 32 | echo "sleep 1" >> /tmp/aeprun.sh 33 | echo "sleep 1" >> /tmp/aeprun.sh 34 | echo "sleep 1" >> /tmp/aeprun.sh 35 | chmod +x /tmp/aeprun.sh 36 | aeprun /tmp/aeprun.sh 37 | sleep 1 38 | msgstart=`dmesg | grep -n 'LATTester_START' | tail -1 | awk -F':' {' print $1 '}` 39 | msgend=`dmesg | grep -n 'LATTester_END' | tail -1 | awk -F':' {' print $1 '}` 40 | msgend+="p" 41 | dmesg | sed -n "$msgstart,$msgend" | tee -a output.txt 42 | done 43 | done 44 | done 45 | done 46 | 47 | -------------------------------------------------------------------------------- /src/plot/merge_plots.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | from PyPDF2 import PdfFileReader, PdfFileWriter 4 | from PyPDF2.pdf import PageObject 5 | import StringIO 6 | from reportlab.pdfgen import canvas 7 | 8 | rows = 2 9 | cols = (len(sys.argv) - 1) / rows 10 | if rows * cols < (len(sys.argv) - 1): 11 | cols = cols + 1 12 | 13 | if len(sys.argv) < 2: 14 | sys.exit() 15 | 16 | reader = PdfFileReader(open(sys.argv[1],'rb')) 17 | pages = reader.getNumPages() 18 | pageWidth = reader.getPage(0).mediaBox.getWidth() 19 | pageHeight = reader.getPage(0).mediaBox.getHeight() 20 | print("Found a total of " + str(pages) + " pages.") 21 | 22 | writer = PdfFileWriter() 23 | for p in range(0, pages): 24 | page = PageObject.createBlankPage(None, pageWidth * cols, pageHeight * rows) 25 | for i in range(1, len(sys.argv)): 26 | myCol = (i - 1) % cols 27 | myRow = (i - 1) / cols 28 | 29 | reader = PdfFileReader(open(sys.argv[i],'rb')) 30 | temp = reader.getPage(p) 31 | page.mergeTranslatedPage(temp, myCol * pageWidth, myRow * pageHeight) 32 | 33 | packet = StringIO.StringIO() 34 | can = canvas.Canvas(packet) 35 | can.setPageSize(size=(pageWidth, pageHeight)) 36 | can.drawString(65, 285, sys.argv[i]) 37 | can.save() 38 | packet.seek(0) 39 | newPdf = PdfFileReader(packet) 40 | page.mergeTranslatedPage(newPdf.getPage(0), myCol * pageWidth, myRow * pageHeight) 41 | writer.addPage(page) 42 | print("Finished creating page " + str(p + 1)) 43 | 44 | with open('merged.pdf', 'wb') as f: 45 | writer.write(f) 46 | -------------------------------------------------------------------------------- /src/testscript/subtests/110_probeimc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #`sed 's/^#define SIZEBT_MACRO.*$/TEST/g'` 4 | 5 | 6 | cd ../kernel 7 | 8 | rm -f ../kernel/output.txt > /dev/null 9 | cp -f memaccess.c memaccess.c.bak 10 | 11 | hostname=`hostname -s` 12 | TAG=${TAG:-`date +%Y%m%d%H%M%S`} 13 | 14 | GS_Bucket="gcp-bucket" 15 | 16 | hostname=`hostname -s` 17 | TAG=${TAG:-`date +%Y%m%d%H%M%S`} 18 | 19 | function run() { 20 | msg="$hostname-11-$count-$parallel-$TAG" 21 | echomsg="task=11,probe_count=$count,parallel=$parallel,runtime=25" 22 | echo $echomsg 23 | export TaskID=$msg 24 | echo "sleep 1" > /tmp/aeprun.sh 25 | echo "sleep 1" >> /tmp/aeprun.sh 26 | echo "echo $echomsg > /proc/lattester" >> /tmp/aeprun.sh 27 | echo "sleep 1" >> /tmp/aeprun.sh 28 | echo "sleep 1" >> /tmp/aeprun.sh 29 | echo "sleep 1" >> /tmp/aeprun.sh 30 | echo "sleep 1" >> /tmp/aeprun.sh 31 | echo "sleep 1" >> /tmp/aeprun.sh 32 | echo "sleep 1" >> /tmp/aeprun.sh 33 | echo "sleep 1" >> /tmp/aeprun.sh 34 | echo "sleep 1" >> /tmp/aeprun.sh 35 | echo "sleep 1" >> /tmp/aeprun.sh 36 | echo "sleep 1" >> /tmp/aeprun.sh 37 | echo "sleep 1" >> /tmp/aeprun.sh 38 | echo "sleep 1" >> /tmp/aeprun.sh 39 | echo "sleep 1" >> /tmp/aeprun.sh 40 | echo "sleep 1" >> /tmp/aeprun.sh 41 | echo "sleep 1" >> /tmp/aeprun.sh 42 | echo "sleep 1" >> /tmp/aeprun.sh 43 | echo "sleep 1" >> /tmp/aeprun.sh 44 | chmod +x /tmp/aeprun.sh 45 | aeprun /tmp/aeprun.sh 46 | } 47 | 48 | for parallel in `seq 1 12`; do 49 | for count in `seq 4 5`; do 50 | run 51 | done 52 | done 53 | 54 | #count=5 55 | #run 56 | #count=7 57 | #run 58 | #count=9 59 | #run 60 | #count=10 61 | #run 62 | 63 | -------------------------------------------------------------------------------- /src/kernel/debug.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | #include 3 | #include 4 | #include "memaccess.h" 5 | #include "lat.h" 6 | 7 | #define LFS_PERMRAND_ENTRIES 0x1000 8 | uint32_t *lfs_random_array; 9 | 10 | 11 | void latencyfs_prealloc_global_permutation_array(int size) 12 | { 13 | int i; 14 | uint32_t temp, j; 15 | lfs_random_array = malloc(sizeof(uint32_t) * LFS_PERMRAND_ENTRIES); 16 | 17 | 18 | // initial range of numbers 19 | for (i = 0; i < LFS_PERMRAND_ENTRIES - 1; i++) { 20 | lfs_random_array[i] = i + 1; 21 | } 22 | 23 | lfs_random_array[LFS_PERMRAND_ENTRIES - 1] = 0; 24 | 25 | for (i = LFS_PERMRAND_ENTRIES - 1; i >= 0; i--) 26 | { 27 | //generate a random number [0, n-1] 28 | j = rand(); 29 | j = j % LFS_PERMRAND_ENTRIES; 30 | 31 | //swap the last element with element at random index 32 | temp = lfs_random_array[i]; 33 | lfs_random_array[i] = lfs_random_array[j]; 34 | lfs_random_array[j] = temp; 35 | } 36 | for (i = 0; i < LFS_PERMRAND_ENTRIES; i++) { 37 | lfs_random_array[i] *= size; 38 | } 39 | } 40 | 41 | 42 | int main() { 43 | char *buf; 44 | // int i; 45 | uint64_t seed=0xdeadbeef12345678; 46 | uint64_t a=0xfc0; 47 | 48 | buf = malloc(4096*1024); 49 | // for (i = 0; i < 4096*1024; i++) 50 | // { 51 | // buf[i]='a'; 52 | // } 53 | 54 | // latencyfs_prealloc_global_permutation_array(64); 55 | sizebw_nt(buf, 64, 128, &seed, a); 56 | return 0; 57 | } 58 | -------------------------------------------------------------------------------- /graphs/06-bandwidth_seqwrite_latency.csv: -------------------------------------------------------------------------------- 1 | ,DRAM,DRAM,DRAM-Remote,DRAM-Remote,Optane,Optane,PMEP,PMEP 2 | M0,121.41,52.271875,183.53,24.68095703,259.96,11.50751953,2767.44,2.754296875 3 | M50,92.94,29.21132813,182.45,24.65947266,261.15,11.48769531,2706.82,2.753320313 4 | M100,88.76,18.59160156,168.47,18.38037109,265.67,12.03955078,1830.92,2.713476563 5 | M200,87.15,11.44638672,144.88,10.26210938,218.04,11.01289063,922.85,2.429785156 6 | M300,91.29,8.159570313,143.06,7.910546875,188.22,7.805371094,765.71,2.286914063 7 | M400,89.68,6.433691406,139.77,6.183105469,177.91,5.895507813,629.52,2.212011719 8 | M500,89.03,5.31171875,148.43,5.075390625,173.16,4.692578125,475.79,2.109667969 9 | M700,89.11,3.695800781,141.18,3.822753906,189.67,3.406445313,439.32,1.625097656 10 | M850,88.32,3.498144531,137.24,2.924023438,190.32,3.129882813,421.95,1.387597656 11 | M1000,88.29,3.104296875,139.01,2.821386719,173.92,2.755273438,399.38,1.221679688 12 | M1150,88.16,2.800878906,139.29,2.540527344,170.32,2.45859375,400.8,1.089746094 13 | M1300,88.11,2.565527344,138.75,2.306542969,169.79,2.195507813,400.7,0.9873046875 14 | M1500,88.43,2.212109375,138.37,2.065332031,168.78,1.795410156,394.45,0.8833984375 15 | M1700,88.49,2.058984375,136.25,1.78046875,167.79,1.798925781,389.04,0.8034179688 16 | M2500,87.99,1.686328125,134.3,1.373144531,166.8,1.350683594,397.45,0.5995117188 17 | M3500,87.96,1.41015625,137.24,1.154980469,166.12,1.07578125,393.54,0.476953125 18 | M5000,87.92,1.201953125,137.06,0.9469726563,165.45,0.86875,396.56,0.3815429688 19 | M20000,87.87,0.8342773438,136.42,0.5814453125,164.64,0.4904296875,401.91,0.2120117188 20 | M40000,88.53,0.7609375,136.16,0.5194335938,164.47,0.4385742188,409.86,0.1807617188 21 | M80000,87.93,0.74140625,133.1,0.49765625,163.76,0.4125,419.29,0.1631835938 -------------------------------------------------------------------------------- /src/kernel/support.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | #ifndef LATTESTER_SUPPORT_H 3 | #define LATTESTER_SUPPORT_H 4 | 5 | #define EAX_IDX 0 6 | #define EBX_IDX 1 7 | #define ECX_IDX 2 8 | #define EDX_IDX 3 9 | 10 | #define TIME_SERIES_PATH "/tmp/" 11 | #define POOL_SIZE (off_t)16 << 30 12 | #define OPS_COUNT 1E7 13 | #define CORE_AFFINITY 0 14 | 15 | /* 16 | * Source: https://stackoverflow.com/questions/6121792/how-to-check-if-a-cpu-supports-the-sse3-instruction-set 17 | */ 18 | int avx_support() { 19 | int support = 0; 20 | 21 | int info[4]; 22 | __cpuid_count(0, 0, info[0], info[1], info[2], info[3]); 23 | int nIds = info[0]; 24 | 25 | __cpuid_count(0x80000000, 0, info[0], info[1], info[2], info[3]); 26 | unsigned nExIds = info[0]; 27 | 28 | if (nIds >= 0x00000001) { 29 | __cpuid_count(0x00000001, 0, info[0], info[1], info[2], info[3]); 30 | support = (info[2] & ((int)1 << 28)) != 0 ? 1 : 0; 31 | } 32 | if (nIds >= 0x00000007) { 33 | __cpuid_count(0x00000007, 0, info[0], info[1], info[2], info[3]); 34 | support = (info[1] & ((int)1 << 5)) != 0 ? 2 : support; 35 | } 36 | return support; 37 | } 38 | 39 | int clwb_support() { 40 | int clwb_support = 0; 41 | unsigned cpuinfo[4] = { 0 }; 42 | __cpuid_count(0x0, 0x0, cpuinfo[EAX_IDX], cpuinfo[EBX_IDX], 43 | cpuinfo[ECX_IDX], cpuinfo[EDX_IDX]); 44 | if (cpuinfo[EAX_IDX] >= 0x0) { 45 | __cpuid_count(0x7, 0x0, cpuinfo[EAX_IDX], cpuinfo[EBX_IDX], 46 | cpuinfo[ECX_IDX], cpuinfo[EDX_IDX]); 47 | clwb_support = (cpuinfo[EBX_IDX] & (1 << 24)) != 0; 48 | } 49 | return clwb_support; 50 | } 51 | 52 | #endif // LATTESTER_SUPPORT_H 53 | -------------------------------------------------------------------------------- /src/kernel/gen_tasks.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "common.h" 12 | #include "lattester.h" 13 | #include "memaccess.h" 14 | 15 | int main(int argc, char **argv) 16 | { 17 | int i; 18 | char buf[200]; 19 | FILE *fp; 20 | int task = atoi(argv[2]); 21 | int task_count = (task == 1) ? BASIC_OPS_TASK_COUNT : BENCH_SIZE_TASK_COUNT; 22 | int ops_mb = LATENCY_OPS_COUNT * 8 / 1048576; 23 | int ops_count = LATENCY_OPS_COUNT; 24 | 25 | if (argc != 3) 26 | { 27 | printf("%s \n", argv[0]); 28 | return 0; 29 | } 30 | 31 | fp = fopen("../testscript/parsing/tasks.py", "w"); 32 | if (!fp) 33 | { 34 | printf("fopen error %d\n", errno); 35 | } 36 | fprintf(fp, "#!/usr/bin/python\n"); 37 | 38 | if (task == 1) 39 | { 40 | fprintf(fp, "tasks = [\n"); 41 | for (i = 0; i < BASIC_OPS_TASK_COUNT; i++) 42 | fprintf(fp, "\t'%s-seq',\n", latency_tasks_str[i]); 43 | for (i = 0; i < BASIC_OPS_TASK_COUNT; i++) 44 | fprintf(fp, "\t'%s-rand',\n", latency_tasks_str[i]); 45 | 46 | fprintf(fp, "]\n"); 47 | fprintf(fp, "dd='dd if=%s of=/tmp/dump bs=%dM skip=1 count=%d'\n", argv[1], ops_mb, BASIC_OPS_TASK_COUNT * 2); 48 | }else if (task == 2) 49 | { 50 | fprintf(fp, "tasks = [\n"); 51 | for (i = 0; i < BENCH_SIZE_TASK_COUNT; i++) 52 | fprintf(fp, "\t'%s',\n", bench_size_map[i]); 53 | fprintf(fp, "]\n"); 54 | fprintf(fp, "dd='dd if=%s of=/tmp/dump bs=%dM count=%d'\n", argv[1], ops_mb, BENCH_SIZE_TASK_COUNT); 55 | }else if (task == 5) 56 | { 57 | fprintf(fp, "dd='dd if=%s of=/tmp/dump bs=%dM count=1'\n", argv[1], ops_mb*2); 58 | ops_count *= 2; 59 | } 60 | 61 | fprintf(fp, "ops_count=%d\n", ops_count); 62 | fclose(fp); 63 | return 0; 64 | } 65 | -------------------------------------------------------------------------------- /src/testscript/subtests/31_seq.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #`sed 's/^#define SIZEBT_MACRO.*$/TEST/g'` 4 | 5 | 6 | cd ../kernel 7 | rm -f output.txt > /dev/null 8 | 9 | size_array=( 64 128 256 512 1024 2048 4096 8192 12288 16384 20480 24576 32768 65536 131072 262144 524288 1048576 ) 10 | #parallel_array=( 10 12 14 16 18 20 22 23 24 ) 11 | parallel_array=( 1 ) 12 | op_array=( 0 ) 13 | hostname=`hostname -s` 14 | delay=0 15 | TAG=${TAG:-`date +%Y%m%d%H%M%S`} 16 | 17 | 18 | #- StrideBW: AEP1-3-TAG-64-256-1-300-2 19 | # 3: Strided Bandwidth Test [access_size] [stride_size] [delay] [parallel] [runtime] [global] 20 | 21 | for accesssize in ${size_array[@]}; do 22 | stridesize=$accesssize 23 | for parallel in ${parallel_array[@]}; do 24 | for op in ${op_array[@]}; do 25 | msg="$hostname-3-$TAG-$accesssize-$stridesize-$parallel-$delay-$op" 26 | echomsg="task=3,parallel=$parallel,runtime=5,access_size=$accesssize,stride_size=$stridesize,delay=$delay,message=$msg,op=$op" 27 | echo $echomsg 28 | export TaskID=$msg 29 | echo "sleep 3" > /tmp/aeprun.sh 30 | echo "echo $echomsg > /proc/lattester" >> /tmp/aeprun.sh 31 | echo "sleep 1" >> /tmp/aeprun.sh 32 | echo "sleep 1" >> /tmp/aeprun.sh 33 | echo "sleep 1" >> /tmp/aeprun.sh 34 | echo "sleep 1" >> /tmp/aeprun.sh 35 | echo "sleep 1" >> /tmp/aeprun.sh 36 | echo "sleep 1" >> /tmp/aeprun.sh 37 | echo "sleep 1" >> /tmp/aeprun.sh 38 | echo "sleep 1" >> /tmp/aeprun.sh 39 | echo "sleep 1" >> /tmp/aeprun.sh 40 | echo "sleep 1" >> /tmp/aeprun.sh 41 | chmod +x /tmp/aeprun.sh 42 | aeprun /tmp/aeprun.sh 43 | sleep 1 44 | msgstart=`dmesg | grep -n 'LATTester_START' | tail -1 | awk -F':' {' print $1 '}` 45 | msgend=`dmesg | grep -n 'LATTester_END' | tail -1 | awk -F':' {' print $1 '}` 46 | msgend+="p" 47 | dmesg | sed -n "$msgstart,$msgend" | tee -a output.txt 48 | done 49 | done 50 | done 51 | cp -f memaccess.c.bak memaccess.c 52 | mv output.txt ../testscript/ 53 | -------------------------------------------------------------------------------- /src/testscript/subtests/90_flushfence.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | cd ../kernel 5 | 6 | rm -f ../kernel/output.txt > /dev/null 7 | cp -f memaccess.c memaccess.c.bak 8 | 9 | hostname=`hostname -s` 10 | TAG=${TAG:-`date +%Y%m%d%H%M%S`} 11 | 12 | function run() { 13 | msg="$hostname-9-$TAG-$accesssize-$clwb_rate-$fence_rate-$parallel-$op" 14 | echomsg="task=9,parallel=$parallel,runtime=5,access_size=$accesssize,clwb_rate=$clwb_rate,fence_rate=$fence_rate,message=$msg,op=$op" 15 | echo $echomsg 16 | export TaskID=$msg 17 | echo "sleep 1" > /tmp/aeprun.sh 18 | echo "echo $echomsg > /proc/lattester" >> /tmp/aeprun.sh 19 | echo "sleep 1" >> /tmp/aeprun.sh 20 | echo "sleep 1" >> /tmp/aeprun.sh 21 | echo "sleep 1" >> /tmp/aeprun.sh 22 | echo "sleep 1" >> /tmp/aeprun.sh 23 | echo "sleep 1" >> /tmp/aeprun.sh 24 | echo "sleep 1" >> /tmp/aeprun.sh 25 | echo "sleep 1" >> /tmp/aeprun.sh 26 | chmod +x /tmp/aeprun.sh 27 | aeprun /tmp/aeprun.sh 28 | sleep 1 29 | msgstart=`dmesg | grep -n 'LATTester_START' | tail -1 | awk -F':' {' print $1 '}` 30 | msgend=`dmesg | grep -n 'LATTester_END' | tail -1 | awk -F':' {' print $1 '}` 31 | msgend+="p" 32 | dmesg | sed -n "$msgstart,$msgend" | tee -a output.txt 33 | 34 | } 35 | 36 | 37 | 38 | for sizebit in `seq 6 22`; do 39 | accesssize=`echo 2^$sizebit | bc` 40 | for fencebit in `seq $sizebit $sizebit`; do # Fencing per accesssize 41 | fence_rate=`echo 2^$fencebit | bc` 42 | for clwbbit in `seq 6 $fencebit`; do 43 | clwb_rate=`echo 2^$clwbbit | bc` 44 | for parallel in `seq 1 4`; do 45 | for op in `seq 0 1`; do 46 | run 47 | done 48 | done 49 | done 50 | done 51 | done 52 | 53 | for sizebit in `seq 8 22`; do 54 | accesssize=`echo 2^$sizebit | bc` 55 | for fencebit in `seq 8 8`; do # Fencing per aepline 56 | fence_rate=`echo 2^$fencebit | bc` 57 | for clwbbit in `seq 8 8`; do 58 | clwb_rate=`echo 2^$clwbbit | bc` 59 | for parallel in `seq 1 4`; do 60 | for op in `seq 0 1`; do 61 | run 62 | done 63 | done 64 | done 65 | done 66 | done 67 | 68 | 69 | cp -f memaccess.c.bak memaccess.c 70 | mv output.txt ../testscript/ 71 | -------------------------------------------------------------------------------- /src/testscript/subtests/30_stridebw.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #`sed 's/^#define SIZEBT_MACRO.*$/TEST/g'` 4 | 5 | 6 | cd ../kernel 7 | 8 | rm -f ../kernel/output.txt > /dev/null 9 | cp -f memaccess.c memaccess.c.bak 10 | 11 | access_array=( 64 128 192 256 512 1024 2048 4096 8192 16384 ) 12 | #parallel_array=( 1 2 4 8 ) 13 | parallel_array=( 1 2 3 4 6 8 ) 14 | stride_array=( 0 64 128 192 256 512 1024 4096 16384 ) 15 | op_array=( 3 ) 16 | hostname=`hostname -s` 17 | delay=0 18 | TAG=${TAG:-`date +%Y%m%d%H%M%S`} 19 | 20 | 21 | #- StrideBW: AEP1-3-TAG-64-256-1-300-2 22 | # 3: Strided Bandwidth Test [access_size] [stride_size] [delay] [parallel] [runtime] [global] 23 | 24 | for accesssize in ${access_array[@]}; do 25 | for stridesize in ${stride_array[@]}; do 26 | for parallel in ${parallel_array[@]}; do 27 | for op in ${op_array[@]}; do 28 | msg="$hostname-3-$TAG-$accesssize-$stridesize-$parallel-$delay-$op" 29 | echomsg="task=3,parallel=$parallel,runtime=5,access_size=$accesssize,stride_size=$stridesize,delay=$delay,message=$msg,op=$op" 30 | echo $echomsg 31 | if [[ $accesssize > $stridesize ]]; then 32 | continue 33 | fi 34 | export TaskID=$msg 35 | echo "sleep 3" > /tmp/aeprun.sh 36 | echo "echo $echomsg > /proc/lattester" >> /tmp/aeprun.sh 37 | echo "sleep 1" >> /tmp/aeprun.sh 38 | echo "sleep 1" >> /tmp/aeprun.sh 39 | echo "sleep 1" >> /tmp/aeprun.sh 40 | echo "sleep 1" >> /tmp/aeprun.sh 41 | echo "sleep 1" >> /tmp/aeprun.sh 42 | echo "sleep 1" >> /tmp/aeprun.sh 43 | echo "sleep 1" >> /tmp/aeprun.sh 44 | echo "sleep 1" >> /tmp/aeprun.sh 45 | echo "sleep 1" >> /tmp/aeprun.sh 46 | echo "sleep 1" >> /tmp/aeprun.sh 47 | chmod +x /tmp/aeprun.sh 48 | aeprun /tmp/aeprun.sh 49 | sleep 1 50 | msgstart=`dmesg | grep -n 'LATTester_START' | tail -1 | awk -F':' {' print $1 '}` 51 | msgend=`dmesg | grep -n 'LATTester_END' | tail -1 | awk -F':' {' print $1 '}` 52 | msgend+="p" 53 | dmesg | sed -n "$msgstart,$msgend" | tee -a output.txt 54 | done 55 | done 56 | done 57 | done 58 | cp -f memaccess.c.bak memaccess.c 59 | mv output.txt ../testscript/ 60 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # An Empirical Guide to the Behavior and Use of Scalable Persistent Memory 2 | 3 | Performance characterization and empirical study of Intel's Optane persistent memory. This repo contains the data and source code for our [USENIX FAST paper](https://www.usenix.org/conference/fast20/presentation/yang). 4 | 5 | 6 | ## Table of Content 7 | 8 | Directory | Description 9 | ----------|------------ 10 | [data/](https://github.com/NVSL/OptaneStudy/tree/master/data) | Raw data (in csv format) of the sweep test 11 | [graphs/](https://github.com/NVSL/OptaneStudy/tree/master/graphs) | Data (in csv format) used in the paper graphs 12 | [src/](https://github.com/NVSL/OptaneStudy/tree/master/src) | Source code of lattester tool 13 | 14 | 15 | ## Configuration 16 | 17 | #### Hardware 18 | 19 | Item | Description 20 | ---|--- 21 | CPUs | 2 22 | CPU | Intel Xeon Platinum 8260 ES (Cascade Lake SP) 23 | CPUFreq. | 24 Cores at 2.2 Ghz base clock 24 | CPU L1 | 32 KB i-Cache, 32 KB d-cache 25 | CPU L2 | 1 MB 26 | CPU L3 | 33 MB (shared) 27 | DRAM | 2x6x32 GB Micron DDR4 2666 MHz (36ASF4G72PZ) 28 | PM | 2x6x256 GB Intel Optane DC 2666 MHz QS (NMA1XXD256GQS) 29 | 30 | #### Software 31 | 32 | Item | Description 33 | ---|--- 34 | GNU/Linux Distro | Fedora 27 35 | Linux Kernel | 4.13.0 36 | CPU Governor | Performance 37 | HyperThreading (SMP) | Disabled 38 | NVDIMM Firmware | 01.01.00.5253 39 | KASLR | Disabled 40 | CPU mitigations | Off 41 | 42 | ## Bibliography 43 | 44 | ``` 45 | @inproceedings {OptaneStudy, 46 | author = {Jian Yang and Juno Kim and Morteza Hoseinzadeh and Joseph Izraelevitz and Steve Swanson}, 47 | title = {An Empirical Guide to the Behavior and Use of Scalable Persistent Memory}, 48 | booktitle = {18th {USENIX} Conference on File and Storage Technologies ({FAST} 20)}, 49 | year = {2020}, 50 | isbn = {978-1-939133-12-0}, 51 | address = {Santa Clara, CA}, 52 | pages = {169--182}, 53 | url = {https://www.usenix.org/conference/fast20/presentation/yang}, 54 | publisher = {{USENIX} Association}, 55 | month = feb, 56 | } 57 | ``` 58 | 59 | ## License 60 | ``` 61 | # SPDX-License-Identifier: GPL-2.0 62 | 63 | Copyright 2019 Regents of the University of California, UCSD Non-Volatile Systems Lab 64 | ``` 65 | -------------------------------------------------------------------------------- /src/testscript/subtests/40_sizebw.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #`sed 's/^#define SIZEBT_MACRO.*$/TEST/g'` 4 | 5 | 6 | cd ../kernel 7 | 8 | rm -f ../kernel/output.txt > /dev/null 9 | cp -f memaccess.c memaccess.c.bak 10 | 11 | #bwsizebit_array=( 6 7 8 ) 12 | bwsizebit_array=( 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 ) 13 | #parallel_array=( 1 ) 14 | parallel_array=( 1 2 3 4 6 8 ) 15 | #op_array=( 0 ) 16 | op_array=( 3 ) 17 | hostname=`hostname -s` 18 | TAG=${TAG:-`date +%Y%m%d%H%M%S`} 19 | 20 | 21 | for bwsizebit in ${bwsizebit_array[@]}; do 22 | accesssize=`echo 2^$bwsizebit | bc` 23 | if (( "$bwsizebit" <= "9" )); then 24 | sed -i 's/^#define SIZEBTNT_MACRO.*$/#define SIZEBTNT_MACRO SIZEBTNT_'"$accesssize"'_AVX512/g' memaccess.c 25 | sed -i 's/^#define SIZEBTLD_MACRO.*$/#define SIZEBTLD_MACRO SIZEBTLD_'"$accesssize"'_AVX512/g' memaccess.c 26 | sed -i 's/^#define SIZEBTST_MACRO.*$/#define SIZEBTST_MACRO SIZEBTST_'"$accesssize"'_AVX512/g' memaccess.c 27 | sed -i 's/^#define SIZEBTSTFLUSH_MACRO.*$/#define SIZEBTSTFLUSH_MACRO SIZEBTSTFLUSH_'"$accesssize"'_AVX512/g' memaccess.c 28 | ../testscript/umount.sh 29 | ../testscript/mount.sh $1 $2 30 | fi 31 | for parallel in ${parallel_array[@]}; do 32 | for op in ${op_array[@]}; do 33 | msg="$hostname-4-$TAG-$bwsizebit-$accesssize-$parallel-$op" 34 | echomsg="task=4,parallel=$parallel,runtime=5,bwsize_bit=$bwsizebit,access_size=$accesssize,message=$msg,op=$op" 35 | echo $echomsg 36 | export TaskID=$msg 37 | echo "sleep 5" > /tmp/aeprun.sh 38 | echo "echo $echomsg > /proc/lattester" >> /tmp/aeprun.sh 39 | echo "sleep 3" >> /tmp/aeprun.sh 40 | echo "sleep 3" >> /tmp/aeprun.sh 41 | echo "sleep 3" >> /tmp/aeprun.sh 42 | echo "sleep 3" >> /tmp/aeprun.sh 43 | echo "sleep 3" >> /tmp/aeprun.sh 44 | chmod +x /tmp/aeprun.sh 45 | aeprun /tmp/aeprun.sh 46 | sleep 1 47 | msgstart=`dmesg | grep -n 'LATTester_START' | tail -1 | awk -F':' {' print $1 '}` 48 | msgend=`dmesg | grep -n 'LATTester_END' | tail -1 | awk -F':' {' print $1 '}` 49 | msgend+="p" 50 | dmesg | sed -n "$msgstart,$msgend" | tee -a output.txt 51 | done 52 | done 53 | done 54 | cp -f memaccess.c.bak memaccess.c 55 | mv output.txt ../testscript/ 56 | -------------------------------------------------------------------------------- /src/testscript/parsing/10_parse_basic.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | import os 4 | import struct 5 | import numpy as np 6 | 7 | 8 | # For Basic Latency Test (task = 1) 9 | # tasks.py can be created via ./gen_tasks 10 | # Alternatively, defines the following and remove the tasks import 11 | # tasks = [ 12 | # 'load-fence-64-seq', 13 | # 'ntload-fence-64-seq', 14 | # ... 15 | # ] 16 | # dd='dd if=/dev/pmem1 of=/tmp/dump bs=8M skip=1 count=50' 17 | # ops_count=1048576 18 | from tasks import * 19 | 20 | insane = 1e18 # when rdtscp produces a negative number 21 | 22 | freqstr = os.popen('lscpu | grep MHz | head -n 1 | awk {\'print $3\'}').read() 23 | cpufreq = float(freqstr)/1000 24 | 25 | def stat(series): 26 | print 'CPU Freq: ', cpufreq 27 | print 'item,min,p25, p50, p75, p95, p99, p99.9, max, avg, stddev' 28 | #print 'item,avg,min,stddev,p50,p95,p99,p99.9,max' 29 | for fn in series: 30 | times = [] 31 | with open('/tmp/'+fn, mode='r') as f: 32 | for line in f: 33 | line = line.strip('\n').strip('\r') 34 | try: 35 | line = int(line) 36 | if line < insane: 37 | # times.append(line) 38 | times.append(line/float(cpufreq)) 39 | except ValueError: 40 | continue 41 | f.close() 42 | a = np.array(times) 43 | list = [fn, np.amin(a), np.percentile(a, 25), 44 | np.percentile(a, 50), np.percentile(a, 75), 45 | np.percentile(a, 95), np.percentile(a, 99), 46 | np.percentile(a, 99.9), np.amax(a), np.average(a), np.std(a)] 47 | print ','.join(str(x) for x in list) 48 | 49 | parsed = [] 50 | 51 | os.system(dd) 52 | 53 | f = open('/tmp/dump', 'rb') 54 | try: 55 | for i in range(0, len(tasks)): 56 | parsed.append(tasks[i]) 57 | txt = open('/tmp/'+tasks[i], 'w') 58 | for j in range(0, ops_count): 59 | item = struct.unpack('L', f.read(8)) 60 | txt.write(str(item[0])+"\n") 61 | txt.write(str(tasks[i])) 62 | txt.close() 63 | 64 | finally: 65 | f.close() 66 | 67 | stat(parsed) 68 | 69 | #cmd = " ".join('/tmp/'+str(e) for e in parsed) 70 | #os.system('python stat.py ' + cmd) 71 | -------------------------------------------------------------------------------- /src/testscript/parsing/repeat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | import os 4 | import struct 5 | import matplotlib.pyplot as plt 6 | import numpy as np 7 | 8 | 9 | OPS_COUNT = 1048576 * 16 10 | INSANE = 1e18 # when rdtscp produces a negative number 11 | THRESHOLD = 22000 12 | 13 | def stat(times): 14 | # print('min,p25, p50, p75, p95, p99, p99.9, max, avg, stddev') 15 | a = np.array(times) 16 | # list = [np.amin(a), np.percentile(a, 25), 17 | # np.percentile(a, 50), np.percentile(a, 75), 18 | # np.percentile(a, 95), np.percentile(a, 99), 19 | # np.percentile(a, 99.9), np.amax(a), np.average(a), np.std(a)] 20 | # print(','.join(str(x) for x in list)) 21 | list = ['RESULT', np.percentile(a, 99), np.percentile(a, 99.9), np.percentile(a, 99.99), np.percentile(a, 99.999), np.percentile(a, 99.9999)] 22 | print(','.join(str(x) for x in list)) 23 | 24 | def download(tag): 25 | os.system('gsutil cp gs://gcp-bucket/{0}/dump /tmp/dump'.format(tag)) 26 | 27 | def parse(tag): 28 | times = [] 29 | spikes = [] 30 | insanes = [] 31 | print("Tag=", tag) 32 | with open('/tmp/dump', mode='rb') as f: 33 | for j in range(0, OPS_COUNT): 34 | item = struct.unpack('L', f.read(8)) 35 | try: 36 | line = int(item[0]) 37 | if line < INSANE: 38 | if line > THRESHOLD: 39 | spikes.append(line) 40 | # line = THRESHOLD 41 | times.append(line) 42 | else: 43 | insanes.append(line); 44 | except ValueError: 45 | continue 46 | f.close() 47 | 48 | print("Ignored=", len(insanes)) 49 | if insanes: 50 | print(insanes) 51 | 52 | print("Spike=", len(spikes)) 53 | if spikes: 54 | print("AvgSpike=", sum(spikes)/len(spikes)) 55 | 56 | stat(times) 57 | 58 | # plt.plot(times) 59 | # ax = plt.axes() 60 | # ax.set_ylim(0, THRESHOLD) 61 | # plt.savefig(tag + ".pdf") 62 | # plt.clf() 63 | # plt.cla() 64 | # plt.close() 65 | 66 | 67 | if __name__ == "__main__": 68 | if len(sys.argv) > 1: 69 | download(sys.argv[1]) 70 | parse(sys.argv[1]) 71 | else: 72 | parse('LOCAL') 73 | -------------------------------------------------------------------------------- /src/testscript/subtests/41_sizebw_unalign.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #`sed 's/^#define SIZEBT_MACRO.*$/TEST/g'` 4 | 5 | ########################################### 6 | # New SizeBW Test # 7 | # Use 4K align for access_size over 4K # 8 | ########################################### 9 | 10 | 11 | cd ../kernel 12 | 13 | rm -f ../kernel/output.txt > /dev/null 14 | cp -f memaccess.c memaccess.c.bak 15 | 16 | alignbit_array=( 12 12 12 12 12 12 12 12 ) 17 | #alignbit_array=( 6 7 8 9 10 11 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 12 ) 18 | size_array=( 45056 49152 65536 131072 262144 524288 1048576 2097152 ) 19 | #size_array=( 64 128 256 512 1024 2048 4096 8192 12288 16384 20480 24576 28672 32768 36864 40960 45056 49152 65536 131072 262144 524288 1048576 2097152 ) 20 | #parallel_array=( 23 24 ) # DRAM 21 | parallel_array=( 4 1 2 ) # NT 22 | #parallel_array=( 16 4 12 ) # PMEM 23 | op_array=( 0 1 3 ) 24 | hostname=`hostname -s` 25 | TAG=${TAG:-`date +%Y%m%d%H%M%S`} 26 | 27 | len=${#size_array[@]} 28 | len1=${#op_array[@]} 29 | for (( i=0; i<$len; i++ )); do 30 | bwsizebit=${alignbit_array[i]} 31 | accesssize=${size_array[i]} 32 | if (( "$bwsizebit" <= "9" )); then 33 | sed -i 's/^#define SIZEBTNT_MACRO.*$/#define SIZEBTNT_MACRO SIZEBTNT_'"$accesssize"'_AVX512/g' memaccess.c 34 | sed -i 's/^#define SIZEBTLD_MACRO.*$/#define SIZEBTLD_MACRO SIZEBTLD_'"$accesssize"'_AVX512/g' memaccess.c 35 | sed -i 's/^#define SIZEBTST_MACRO.*$/#define SIZEBTST_MACRO SIZEBTST_'"$accesssize"'_AVX512/g' memaccess.c 36 | sed -i 's/^#define SIZEBTSTFLUSH_MACRO.*$/#define SIZEBTSTFLUSH_MACRO SIZEBTSTFLUSH_'"$accesssize"'_AVX512/g' memaccess.c 37 | ../testscript/umount.sh 38 | ../testscript/mount.sh $1 $2 39 | fi 40 | 41 | for (( j=0; j<$len1; j++ )); do 42 | parallel=${parallel_array[j]} 43 | op=${op_array[j]} 44 | msg="$hostname-4-$TAG-$bwsizebit-$accesssize-$parallel-$op" 45 | echomsg="task=4,parallel=$parallel,runtime=5,bwsize_bit=$bwsizebit,access_size=$accesssize,message=$msg,op=$op" 46 | echo $echomsg 47 | export TaskID=$msg 48 | echo "sleep 2" > /tmp/aeprun.sh 49 | echo "echo $echomsg > /proc/lattester" >> /tmp/aeprun.sh 50 | echo "sleep 1" >> /tmp/aeprun.sh 51 | echo "sleep 1" >> /tmp/aeprun.sh 52 | echo "sleep 1" >> /tmp/aeprun.sh 53 | echo "sleep 1" >> /tmp/aeprun.sh 54 | echo "sleep 1" >> /tmp/aeprun.sh 55 | echo "sleep 1" >> /tmp/aeprun.sh 56 | chmod +x /tmp/aeprun.sh 57 | aeprun /tmp/aeprun.sh 58 | sleep 1 59 | msgstart=`dmesg | grep -n 'LATTester_START' | tail -1 | awk -F':' {' print $1 '}` 60 | msgend=`dmesg | grep -n 'LATTester_END' | tail -1 | awk -F':' {' print $1 '}` 61 | msgend+="p" 62 | dmesg | sed -n "$msgstart,$msgend" | tee -a output.txt 63 | done 64 | done 65 | cp -f memaccess.c.bak memaccess.c 66 | mv output.txt ../testscript/ 67 | -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- 1 | # Raw data of the sweep test 2 | 3 | ## Files 4 | 5 | * `bandwidth` : Measure bandwidth with different strided or random accesses. 6 | * `allocation.csv` : Measure bandwidth by allocating buffers from different DIMMs (alloc_type). 7 | * `flush.csv` : Measure bandwidth with sequential accesses with different write and persistent patterns. 8 | 9 | ## Table fields 10 | 11 | Name | Description 12 | ---|--- 13 | `__key__` | SHA1 of the primary key (for locating output logs) (internal use) 14 | `access_size` | Access size (bytes) 15 | `aepwatch` | If AEPWatch is enabled 16 | `alloc_type` | Allocation type (explained below) 17 | `avgpower` | Average power budget (watts) 18 | `clwb_size` | The write window between issuing `clwb` insturctions 19 | `date` | Date of the test 20 | `emon` | If EMon is enabled 21 | `err` | Effective read ratio 22 | `ewr` | Effective write ratio 23 | `fence_size` | The write window between issuing `sfence` insturctions 24 | `interleaving` | If 1, test was running on 6 DIMMs interleaved, otherwise test was running on 1 DIMM 25 | `local` | If 1, the test was running on the local NUMA node 26 | `machine` | Hostname the test was running on 27 | `media` | `(l=local|r=remote)(dram|pmem)` 28 | `operation` | Possible values: (`Load|Store|Store+Flush|NTStore`), an `lfence` or `sfence` was issued after operation 29 | `peakpower` | Peak power budget (watts) 30 | `prefetching` | If 0, cache prefetching is disabled 31 | `tag` | Tests with the same tag were run together (sequentially with 10s delay) 32 | `task_type` | Possible values: (`StrideBW|RandBW|AlignBW|ClwbFence`) (internal use) 33 | `threads` | Number of threads used for the test (mapped to CPU 1..N). 34 | `throughput` | Throughput value (average) 35 | `version` | Head commit of the LATTester repo (internal use) 36 | 37 | 38 | ## `alloc_type` 39 | 40 | This field defines how a thread writes to certain DIMM(s) (Figure 15 in paper) 41 | 42 | ``` 43 | +---------+ +---------+ 44 | | DIMM0 +--+ +--+ DIMM3 | 45 | +---------+ | | +---------+ 46 | | | 47 | +-------+------------+------+ 48 | +---------+ | | | | +---------+ 49 | | DIMM1 +--+ iMC0 | CPU | iMC1 +--+ DIMM4 | 50 | +---------+ | | | | +---------+ 51 | +-------+------------+------+ 52 | | | 53 | +---------+ | | +---------+ 54 | | DIMM2 +--+ +--+ DIMM5 | 55 | +---------+ +---------+ 56 | ``` 57 | `alloc_type` | Interleaving | Buffer | DIMM Binding | Th 0 writes to 58 | ---|---|---|---|--- 59 | 0 | Interleaved| Global| no restriction| DIMM0-5 60 | 1 | Interleaved| Thread-local| no restriction| DIMM0-5 61 | 2 | Interleave| Thread-local| each thread bind to a DIMM| DIMM0 62 | 3 | Interleaved| Thread-local| each thread bind to 3 DIMMs belong to an iMC| DIMM0-2 63 | 4 | Interleaved| Thread-local| each thread bind to 2 DIMMs belong to two iMCs| DIMM0, DIMM3 64 | 5 | Non-Interleaved| Global| no restriction| DIMM0-5 65 | 6 | Non-Interleaved| Thread-local| each thread bind to a DIMM| DIMM0 66 | 67 | ## AEPWatch and EMon 68 | AEPWatch and EMon are performance monitoring tools developed by Intel. These tools read from the on-DIMM counter (AEPWatch) and PMU counters (EMon) periodically. 69 | 70 | The data presented in this repo do not use data generated by AEPWatch and EMon. We report whether they are enabled on each run because they might impact throughput slightly. -------------------------------------------------------------------------------- /src/kernel/perf_util.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | #ifndef _PERF_UTIL_H_ 3 | #define _PERF_UTIL_H_ 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | /* There is a `perf_event` in `perf_event.h` 10 | * So use this name to prevent conflict 11 | */ 12 | struct __perf_event { 13 | struct perf_event_attr attr; 14 | u64 value; 15 | u64 enabled; 16 | u64 running; 17 | const char *name; 18 | }; 19 | 20 | #define PERF_EVENT_READ(e, evt) \ 21 | e->value = perf_event_read_value(evt, &e->enabled, &e->running); 22 | 23 | #define PERF_EVENT_PRINT(e) \ 24 | pr_info("\tPerf counter [%s]\t: %lld\n", e.name, e.value); 25 | 26 | #define PERF_EVENT_HW(cfg) \ 27 | { \ 28 | .attr = \ 29 | { \ 30 | .type = PERF_TYPE_HARDWARE, \ 31 | .config = PERF_COUNT_HW_##cfg, \ 32 | .size = sizeof(struct perf_event_attr), \ 33 | .pinned = 1, \ 34 | .disabled = 1, \ 35 | .exclude_user = 1, \ 36 | .exclude_hv = 1, \ 37 | .exclude_idle = 1, \ 38 | }, \ 39 | .name = "HW." #cfg, .value = 0xffff, .enabled = 0xffff, \ 40 | .running = 0xffff, \ 41 | } 42 | 43 | #define DEFINE_PERF_EVENT_HW(event, cfg) \ 44 | struct __perf_event event = PERF_EVENT_HW(cfg); 45 | 46 | #define PERF_EVENT_CACHE(level, op, result) \ 47 | { \ 48 | .attr = \ 49 | { \ 50 | .type = PERF_TYPE_HW_CACHE, \ 51 | .config = (PERF_COUNT_HW_CACHE_##level) \ 52 | | (PERF_COUNT_HW_CACHE_OP_##op << 8) \ 53 | | (PERF_COUNT_HW_CACHE_RESULT_##result << 16), \ 54 | .size = sizeof(struct perf_event_attr), \ 55 | .pinned = 1, \ 56 | .disabled = 1, \ 57 | .exclude_user = 1, \ 58 | .exclude_hv = 1, \ 59 | .exclude_idle = 1, \ 60 | }, \ 61 | .name = "CACHE." #level "." #op "." #result, .value = 0xffff, \ 62 | .enabled = 0xffff, .running = 0xffff, \ 63 | } 64 | 65 | #define DEFINE_PERF_EVENT_CACHE(event, level, op, result) \ 66 | struct __perf_event event = PERF_EVENT_CACHE(level, op, result); 67 | 68 | int lattest_perf_event_create(void); 69 | void lattest_perf_event_read(void); 70 | void lattest_perf_event_enable(void); 71 | void lattest_perf_event_disable(void); 72 | void lattest_perf_event_reset(void); 73 | void lattest_perf_event_release(void); 74 | void lattest_perf_event_print(void); 75 | 76 | #endif /* _PERF_UTIL_H_ */ 77 | -------------------------------------------------------------------------------- /src/kernel/misc.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | #include 3 | #include "lattester.h" 4 | 5 | extern uint32_t *lfs_random_array; 6 | 7 | int latencyfs_getopt(const char *caller, char **options, 8 | const struct latency_option *opts, char **optopt, char **optarg, 9 | unsigned long *value) 10 | { 11 | char *token; 12 | char *val; 13 | 14 | do { 15 | if ((token = strsep(options, ",")) == NULL) 16 | return 0; 17 | } while (*token == '\0'); 18 | if (optopt) 19 | *optopt = token; 20 | 21 | if ((val = strchr(token, '=')) != NULL) { 22 | *val++ = 0; 23 | } 24 | *optarg = val; 25 | for (; opts->name; opts++) 26 | { 27 | if (!strcmp(opts->name, token)) 28 | { 29 | if (!val) { 30 | if (opts->has_arg & OPT_NOPARAM) { 31 | return opts->val; 32 | } 33 | pr_info("%s: the %s option requires an argument\n", 34 | caller, token); 35 | return -EINVAL; 36 | } 37 | if (opts->has_arg & OPT_INT) { 38 | char *v; 39 | 40 | *value = simple_strtoul(val, &v, 0); 41 | if (!*v) 42 | { 43 | return opts->val; 44 | } 45 | pr_info("%s: invalid numeric value in %s=%s\n", 46 | caller, token, val); 47 | return -EDOM; 48 | } 49 | if (opts->has_arg & OPT_STRING) { 50 | return opts->val; 51 | } 52 | pr_info("%s: unexpected argument %s to the %s option\n", 53 | caller, val, token); 54 | return -EINVAL; 55 | } 56 | } 57 | pr_info("%s: Unrecognized option %s\n", caller, token); 58 | return -EOPNOTSUPP; 59 | } 60 | 61 | 62 | /* 63 | * Generates a 16 KB (4K * sizeof int) global random array. 64 | * It seems such configuration is enough to prevent CPU from establishing 65 | * a pattern. 66 | * This array is almost guaranteed to be cached in L2. 67 | * We used to have a much larger random pool. 68 | */ 69 | 70 | /* pre-alloc RNG with a small permuation array */ 71 | inline void latencyfs_prealloc_global_permutation_array(int size) 72 | { 73 | int i; 74 | uint32_t temp, j; 75 | lfs_random_array = kmalloc(sizeof(uint32_t) * LFS_PERMRAND_ENTRIES, GFP_KERNEL); 76 | 77 | 78 | // initial range of numbers 79 | for (i = 0; i < LFS_PERMRAND_ENTRIES - 1; i++) { 80 | lfs_random_array[i] = i + 1; 81 | } 82 | 83 | lfs_random_array[LFS_PERMRAND_ENTRIES - 1] = 0; 84 | 85 | for (i = LFS_PERMRAND_ENTRIES - 1; i >= 0; i--) 86 | { 87 | //generate a random number [0, n-1] 88 | get_random_bytes(&j, sizeof(uint32_t)); 89 | j = j % LFS_PERMRAND_ENTRIES; 90 | 91 | //swap the last element with element at random index 92 | temp = lfs_random_array[i]; 93 | lfs_random_array[i] = lfs_random_array[j]; 94 | lfs_random_array[j] = temp; 95 | } 96 | /* TODO: currently we allocate 2GB per thread so we store shifted values 97 | * For 4GB and larger regions, we need to do `mul` or `shl` at runtime 98 | */ 99 | for (i = 0; i < LFS_PERMRAND_ENTRIES; i++) { 100 | lfs_random_array[i] *= size; 101 | } 102 | } 103 | 104 | /* 105 | * pre-alloc RNG, Large array approach, the array should be 2^n size 106 | * Address = rand_long & BIT_MASK 107 | */ 108 | inline void latencyfs_prealloc_large(uint8_t *addr, uint64_t size, uint64_t bit_mask) 109 | { 110 | int i; 111 | uint32_t *buf; 112 | get_random_bytes(addr, size); 113 | buf = (uint32_t *)addr; 114 | for (i = 0; i < size >> 2; i++) { 115 | buf[i] &= bit_mask; 116 | } 117 | } 118 | 119 | /* in-flight fast pseudo-RNG */ 120 | inline unsigned long fastrand(unsigned long *x, unsigned long *y, unsigned long *z, uint64_t bit_mask) 121 | { 122 | unsigned long t; 123 | 124 | *x ^= *x << 16; 125 | *x ^= *x >> 5; 126 | *x ^= *x << 1; 127 | 128 | t = *x; 129 | *x = *y; 130 | *y = *z; 131 | *z = t ^ *x ^ *y; 132 | 133 | return *z & bit_mask; 134 | } 135 | -------------------------------------------------------------------------------- /src/README.md: -------------------------------------------------------------------------------- 1 | # LatTester 2 | 3 | LatTester is designed to measure memory latency on AEP platfrom. 4 | 5 | It contains two modules, each of them need to be mounted on a pmem device (in `fsdax` mode): 6 | 7 | **LatencyFS:** LatencyFS (LatFS) is the *working area* of the tester. A series of memory accesses will be issued to locations within this area. 8 | 9 | **ReportFS:** ReportFS (RepFS) is used to store access latency of individual requests. It reports the CPU cycles of individual access requests as contigious 8-byte values. User can gather the result using `dd if=/dev/pmemX of=result bs= count=` 10 | 11 | 12 | ## Requirement 13 | - Linux 4.13 kernel (with devel) 14 | - CPU with clwb/avx-512 support (i.e., support Optane) 15 | - Two real or emulated PMem regions 16 | - LatTester will overwrite content of the PMem devices 17 | 18 | ## Tasks 19 | It supports following tasks: 20 | 1. Random and sequential latency of flush instructions 21 | 2. Strided access latency 22 | 3. Strided access bandwidth 23 | 4. Random Size-Bandwidth Test 24 | 5. Overwrite Test 25 | 6. Read-after-write Test 26 | 7. Straight Write Test 27 | 8. Writer Thread Test 28 | 9. Flush/Fence BW Test 29 | 10. APC Cache Probe Test 30 | 11. iMC Probe Test 31 | 32 | ## Configuration 33 | LatTester can update results in GCP cloud datastore. Following fields are manually entered (`testscript/config.example.json`). 34 | ``` 35 | { 36 | "media": "lpmem", 37 | "interleaving": "1", 38 | "local": "1", 39 | "avgpower": "15000", 40 | "peakpower": "20000", 41 | "emon": "1", 42 | "aepwatch": "1", 43 | "fence": "0", 44 | "baddata": "0", 45 | "prefetching": "0", 46 | "rerun": "0" 47 | } 48 | ``` 49 | 50 | Also see macros defined in `kernel/lattester.h` 51 | 52 | ## Usage (in testscripts) 53 | 54 | #### Load modules 55 | 56 | `/mount.sh [RepFS] [LatFS]` 57 | 58 | - `RepFS` is an emulated PMem device that stores per-access latency. 59 | - `LatFS` is a PMem device used for latency measurement. (LATTester will allocate from CPU #0.) 60 | 61 | #### Usage 62 | 63 | `cat /proc/lattester` 64 | 65 | ``` 66 | LATTester task= op= [options] 67 | Tasks: 68 | 1: Basic Latency Test (not configurable) 69 | 2: Strided Latency Test [access_size] [stride_size] 70 | 3: Strided Bandwidth Test [access_size] [stride_size] [delay] [parallel] [runtime] [global] 71 | 4: Random Size-Bandwidth Test [bwtest_bit] [access_size] [parallel] [runtime] [align_mode] 72 | 5: Overwrite Test [stride_size] 73 | 6: Read-after-write Test [stride_size] [access_size] 74 | 7: Straight Write Test [start_addr] [size] 75 | 8: Writer Thread Test [bwsize_bit] [parallel] [align_mode] 76 | 9: Flush/Fence BW Test [acccess_size] [fence_rate] [flush_rate] [op (1=random)] 77 | 10: APC Cache Probe Test [probe_count], [strided_size] 78 | 11: iMC Probe Test [probe_count] 79 | 12: Sequencial Test [access_size], [parallel] 80 | Options: 81 | [op|o] Operation: 0=Load, 1=Write(NT) 2=Write(clwb), 3=Write+Flush(clwb+flush) default=0 82 | [access_size|a] Access size (bytes), default=64 83 | [align_mode|l] Align mode: 0 = Global (anywhere), 1 = Per-Thread pool (default) 2 = Per-DIMM pool 3=Per-iMC pool 84 | 4 = Per-Channel Group, 5 = [NIOnly]: Global, 6 = [NIOnly]: Per-DIMM 85 | [stride_size|s] Stride size (bytes), default=64 86 | [delay|d] Delay (cycles), default=0 87 | [parallel|p] Concurrent kthreads, default=1 88 | [bwsize_bit|b] Aligned size (2^b bytes), default=6 89 | [runtime|t] Runtime (seconds), default=10 90 | [write_addr|w] For Straight Write: access location, default=0 91 | [write_size|z] For Straight Write: access size, default=0 92 | [fence_rate|f] Access size before issuing a sfence instruction 93 | [clwb_rate|c] Access size before issuing a clwb instruction 94 | ``` 95 | 96 | #### Run a test 97 | 98 | `echo "task= op= [options] > /proc/lattester"` 99 | 100 | `dmesg | tail` to find 101 | 102 | #### Sample script (testscript/run.example.sh) 103 | 104 | -------------------------------------------------------------------------------- /src/kernel/rep.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #include "lattester.h" 27 | /* 28 | * BRIEF DESCRIPTION 29 | * 30 | * Report psudo file system 31 | * 32 | * Copyright 2019 Regents of the University of California, 33 | * UCSD Non-Volatile Systems Lab 34 | */ 35 | 36 | 37 | int support_clwb = 0; 38 | static struct report_sbi *g_report_sbi = NULL; 39 | 40 | #ifndef X86_FEATURE_CLFLUSHOPT 41 | #define X86_FEATURE_CLFLUSHOPT (9 * 32 + 23) /* CLFLUSHOPT instruction */ 42 | #endif 43 | 44 | #ifndef X86_FEATURE_CLWB 45 | #define X86_FEATURE_CLWB (9 * 32 + 24) /* CLWB instruction */ 46 | #endif 47 | 48 | static inline bool arch_has_clwb(void) { 49 | return static_cpu_has(X86_FEATURE_CLWB); 50 | } 51 | 52 | static struct report_sbi *reportfs_get_sbi(void) { return g_report_sbi; } 53 | EXPORT_SYMBOL(reportfs_get_sbi); 54 | 55 | static int reportfs_fill_super(struct super_block *sb, void *data, int silent) { 56 | struct report_sbi *sbi; 57 | void *virt_addr = NULL; 58 | struct inode *root; 59 | struct dax_device *dax_dev; 60 | pfn_t __pfn_t; 61 | long size; 62 | int ret; 63 | 64 | sbi = kzalloc(sizeof(struct report_sbi), GFP_KERNEL); 65 | if (!sbi) 66 | return -ENOMEM; 67 | 68 | if (g_report_sbi) { 69 | pr_err("Another ReportFS already at VA %p PA %llx\n", 70 | g_report_sbi->virt_addr, g_report_sbi->phys_addr); 71 | return -EEXIST; 72 | } else { 73 | g_report_sbi = sbi; 74 | } 75 | 76 | sb->s_fs_info = sbi; 77 | sbi->sb = sb; 78 | 79 | #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 6) 80 | ret = bdev_dax_supported(sb, PAGE_SIZE); 81 | #else 82 | ret = bdev_dax_supported(sb->s_bdev, PAGE_SIZE); 83 | #endif 84 | pr_info("%s: dax_supported = %d; bdev->super=0x%p", __func__, ret, 85 | sb->s_bdev->bd_super); 86 | #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 6) 87 | if (ret) { 88 | #else 89 | if (!ret) { 90 | #endif 91 | pr_err("device does not support DAX\n"); 92 | return ret; 93 | } 94 | 95 | sbi->s_bdev = sb->s_bdev; 96 | 97 | dax_dev = fs_dax_get_by_host(sb->s_bdev->bd_disk->disk_name); 98 | if (!dax_dev) { 99 | pr_err("Couldn't retrieve DAX device.\n"); 100 | return -EINVAL; 101 | } 102 | 103 | size = dax_direct_access(dax_dev, 0, LONG_MAX / PAGE_SIZE, &virt_addr, 104 | &__pfn_t) * 105 | PAGE_SIZE; 106 | if (size <= 0) { 107 | pr_err("direct_access failed\n"); 108 | return -EINVAL; 109 | } 110 | 111 | sbi->virt_addr = virt_addr; 112 | sbi->phys_addr = pfn_t_to_pfn(__pfn_t) << PAGE_SHIFT; 113 | sbi->initsize = size; 114 | 115 | pr_info("%s: dev %s, phys_addr 0x%llx, virt_addr %p, size %ld\n", __func__, 116 | sbi->s_bdev->bd_disk->disk_name, sbi->phys_addr, sbi->virt_addr, 117 | sbi->initsize); 118 | 119 | root = new_inode(sb); 120 | if (!root) { 121 | pr_err("root inode allocation failed\n"); 122 | return -ENOMEM; 123 | } 124 | 125 | root->i_ino = 0; 126 | root->i_sb = sb; 127 | #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 20, 0) 128 | root->i_atime = root->i_mtime = root->i_ctime = current_kernel_time(); 129 | #else 130 | root->i_atime = root->i_mtime = root->i_ctime = ktime_to_timespec64(ktime_get_real()); 131 | #endif 132 | inode_init_owner(root, NULL, S_IFDIR); 133 | 134 | sb->s_root = d_make_root(root); 135 | if (!sb->s_root) { 136 | pr_err("d_make_root failed\n"); 137 | return -ENOMEM; 138 | } 139 | 140 | return 0; 141 | } 142 | 143 | static struct dentry *reportfs_mount(struct file_system_type *fs_type, 144 | int flags, const char *dev_name, 145 | void *data) { 146 | return mount_bdev(fs_type, flags, dev_name, data, reportfs_fill_super); 147 | } 148 | 149 | static struct file_system_type reportfs_fs_type = { 150 | .owner = THIS_MODULE, 151 | .name = "ReportFS", 152 | .mount = reportfs_mount, 153 | .kill_sb = kill_block_super, 154 | }; 155 | 156 | static int __init init_reportfs(void) { 157 | int rc = 0; 158 | 159 | pr_info("%s: %d cpus online\n", __func__, num_online_cpus()); 160 | if (arch_has_clwb()) 161 | support_clwb = 1; 162 | 163 | pr_info("Arch new instructions support: CLWB %s\n", 164 | support_clwb ? "YES" : "NO"); 165 | 166 | rc = register_filesystem(&reportfs_fs_type); 167 | if (rc) 168 | return rc; 169 | 170 | return 0; 171 | } 172 | 173 | static void __exit exit_reportfs(void) { 174 | unregister_filesystem(&reportfs_fs_type); 175 | } 176 | 177 | MODULE_LICENSE("GPL"); 178 | 179 | module_init(init_reportfs); 180 | module_exit(exit_reportfs); 181 | -------------------------------------------------------------------------------- /src/kernel/perf_util.c: -------------------------------------------------------------------------------- 1 | #include "perf_util.h" 2 | #include 3 | #include 4 | 5 | #define NUM_PERF_EVENTS 13 6 | static DEFINE_PER_CPU(struct perf_event **, lattest_perf_event_list); 7 | 8 | #define for_each_lattest_perf_event(id) for (id = 0; id < NUM_PERF_EVENTS; ++id) 9 | 10 | static struct __perf_event perf_counters[] = { 11 | PERF_EVENT_CACHE(L1D, READ, MISS), 12 | PERF_EVENT_CACHE(L1D, READ, ACCESS), 13 | PERF_EVENT_CACHE(L1D, WRITE, ACCESS), 14 | PERF_EVENT_CACHE(LL, READ, MISS), 15 | PERF_EVENT_CACHE(LL, WRITE, MISS), 16 | PERF_EVENT_CACHE(LL, READ, ACCESS), 17 | PERF_EVENT_CACHE(LL, WRITE, ACCESS), 18 | // PERF_EVENT_CACHE(NODE, READ, MISS), 19 | // PERF_EVENT_CACHE(NODE, WRITE, MISS), 20 | // PERF_EVENT_CACHE(NODE, READ, ACCESS), 21 | // PERF_EVENT_CACHE(NODE, WRITE, ACCESS), 22 | PERF_EVENT_CACHE(DTLB, READ, MISS), 23 | PERF_EVENT_CACHE(DTLB, WRITE, MISS), 24 | PERF_EVENT_CACHE(DTLB, READ, ACCESS), 25 | PERF_EVENT_CACHE(DTLB, WRITE, ACCESS), 26 | // PERF_EVENT_HW(INSTRUCTIONS), 27 | PERF_EVENT_HW(CACHE_REFERENCES), 28 | PERF_EVENT_HW(CACHE_MISSES), 29 | }; 30 | 31 | static void lattest_perf_overflow_callback(struct perf_event *event, 32 | struct perf_sample_data *data, 33 | struct pt_regs *regs) 34 | { 35 | event->hw.interrupts = 0; 36 | pr_err("Overflow detected for perf counter: %d - %lld\n", 37 | event->attr.type, event->attr.config); 38 | // TODO: call `lattest_perf_exit()` ? 39 | } 40 | 41 | void lattest_perf_exit(void) 42 | { 43 | int cpu, id; 44 | struct perf_event *event; 45 | for_each_possible_cpu(cpu) 46 | { 47 | for_each_lattest_perf_event(id) 48 | { 49 | event = per_cpu(lattest_perf_event_list, cpu)[id]; 50 | if (event) 51 | perf_event_release_kernel(event); 52 | } 53 | kfree(per_cpu(lattest_perf_event_list, cpu)); 54 | } 55 | } 56 | 57 | int lattest_perf_init(void) 58 | { 59 | int cpu, ret = 0; 60 | for_each_possible_cpu(cpu) 61 | { 62 | per_cpu(lattest_perf_event_list, cpu) = 63 | kcalloc(NUM_PERF_EVENTS, sizeof(struct perf_event *), 64 | GFP_KERNEL); 65 | if (!per_cpu(lattest_perf_event_list, cpu)) { 66 | pr_err("lattest_perf_init: failed to allocate %d perf events for cpu %d\n", 67 | NUM_PERF_EVENTS, cpu); 68 | ret = -ENOMEM; 69 | goto out; 70 | } 71 | } 72 | out: 73 | if (ret) 74 | lattest_perf_exit(); 75 | return ret; 76 | } 77 | 78 | int _lattest_perf_event_create(int cpu, int event_id) 79 | { 80 | struct perf_event *evt; 81 | struct perf_event_attr *attr; 82 | 83 | if (event_id >= NUM_PERF_EVENTS) { 84 | pr_err("_lattest_perf_event_create: event_id %d out of range %d\n", 85 | event_id, NUM_PERF_EVENTS); 86 | return -EINVAL; 87 | } 88 | 89 | /* If already created */ 90 | if (per_cpu(lattest_perf_event_list, cpu)[event_id]) 91 | return 0; 92 | 93 | // TODO: find a proper place to memset attr to 0? 94 | attr = &perf_counters[event_id].attr; 95 | 96 | evt = perf_event_create_kernel_counter( 97 | attr, cpu, NULL, lattest_perf_overflow_callback, NULL); 98 | if (IS_ERR(evt)) { 99 | pr_err("Perf event create on CPU %d failed with %ld\n", cpu, 100 | PTR_ERR(evt)); 101 | return PTR_ERR(evt); 102 | } 103 | 104 | per_cpu(lattest_perf_event_list, cpu)[event_id] = evt; 105 | 106 | return 0; 107 | } 108 | 109 | int lattest_perf_event_create(void) 110 | { 111 | int event_id, ret = 0; 112 | 113 | lattest_perf_init(); 114 | for_each_lattest_perf_event(event_id) 115 | { 116 | ret = _lattest_perf_event_create(smp_processor_id(), event_id); 117 | if (ret) 118 | return ret; 119 | } 120 | return ret; 121 | } 122 | 123 | void lattest_perf_event_read(void) 124 | { 125 | struct perf_event *evt; 126 | struct __perf_event *counter; 127 | int event_id = 0; 128 | for_each_lattest_perf_event(event_id) 129 | { 130 | evt = this_cpu_read(lattest_perf_event_list)[event_id]; 131 | counter = &perf_counters[event_id]; 132 | PERF_EVENT_READ(counter, evt); 133 | } 134 | } 135 | 136 | void lattest_perf_event_enable(void) 137 | { 138 | int event_id = 0; 139 | for_each_lattest_perf_event(event_id) 140 | { 141 | perf_event_enable( 142 | this_cpu_read(lattest_perf_event_list)[event_id]); 143 | } 144 | } 145 | 146 | void lattest_perf_event_disable(void) 147 | { 148 | struct perf_event *evt; 149 | int event_id = 0; 150 | for_each_lattest_perf_event(event_id) 151 | { 152 | evt = this_cpu_read(lattest_perf_event_list)[event_id]; 153 | if (evt) { 154 | perf_event_disable(evt); 155 | } 156 | } 157 | } 158 | 159 | void lattest_perf_event_reset(void) 160 | { 161 | struct perf_event *evt; 162 | int event_id = 0; 163 | for_each_lattest_perf_event(event_id) 164 | { 165 | evt = this_cpu_read(lattest_perf_event_list)[event_id]; 166 | if (evt) { 167 | local64_set(&evt->count, 0); 168 | } 169 | } 170 | } 171 | 172 | void _lattest_perf_event_release(int cpu) 173 | { 174 | struct perf_event *evt; 175 | int event_id = 0; 176 | for_each_lattest_perf_event(event_id) 177 | { 178 | evt = this_cpu_read(lattest_perf_event_list)[event_id]; 179 | if (evt) { 180 | perf_event_release_kernel(evt); 181 | } 182 | per_cpu(lattest_perf_event_list, cpu)[event_id] = NULL; 183 | } 184 | } 185 | 186 | void lattest_perf_event_release(void) 187 | { 188 | int cpu; 189 | 190 | for_each_possible_cpu(cpu) 191 | { 192 | _lattest_perf_event_release(cpu); 193 | } 194 | 195 | lattest_perf_exit(); 196 | } 197 | 198 | void lattest_perf_event_print(void) 199 | { 200 | int event_id = 0; 201 | for_each_lattest_perf_event(event_id) 202 | { 203 | PERF_EVENT_PRINT(perf_counters[event_id]); 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /src/kernel/lattester.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | #ifndef LATTESTER_H 3 | #define LATTESTER_H 4 | /* 5 | * BRIEF DESCRIPTION 6 | * 7 | * Header for latfs 8 | * 9 | * Copyright 2019 Regents of the University of California, 10 | * UCSD Non-Volatile Systems Lab 11 | */ 12 | 13 | #if __KERNEL__ 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #endif 24 | 25 | #define CACHELINE_BITS (6) 26 | #define CACHELINE_SIZE (64) 27 | #define LATENCY_OPS_COUNT 1048576L 28 | 29 | /* 30 | * Content on ReportFS 31 | * 32 | * Task 1: For random latency tests 33 | * | OP_COUNT * 8 | 34 | * +-----------------+---------------+---------------+-----+---------------+ 35 | * | Random Pool | Report Task 0 | Report Task 1 | ... | Report Task N | 36 | * +-----------------+---------------+---------------+-----+---------------+ 37 | * 38 | * Task 2: Strided latency 39 | * | OP_COUNT * 8 | 40 | * +---------------+---------------+-----+---------------+ 41 | * | Report Task 0 | Report Task 1 | ... | Report Task N | 42 | * +---------------+---------------+-----+---------------+ 43 | * 44 | * Task 5:Overwrite latency 45 | * | 2 * OP_COUNT * 8 | 46 | * +--------------------+ 47 | * | Per-access latency | 48 | * +--------------------+ 49 | * 50 | * 51 | */ 52 | 53 | #define OPT_NOPARAM 1 54 | #define OPT_INT 2 55 | #define OPT_STRING 4 56 | 57 | /* LATENCY_OPS_COUNT = Subtasks (seq/rand) */ 58 | #define BASIC_OP_POOL_BITS 30 59 | #define BASIC_OP_POOL_SIZE (1L << POOL_BITS) /* Size of the test region */ 60 | #define BASIC_OP_POOL_PAGE_BIT (BASIC_OP_POOL_BITS - PAGE_SHIFT) 61 | #define BASIC_OP_POOL_PAGES (1L << BASIC_OP_POOL_PAGE_BIT) /* # Pages */ 62 | 63 | #define BASIC_OP_POOL_LINE_BITS (BASIC_OP_POOL_BITS - CACHELINE_BITS) 64 | #define BASIC_OP_MASK 0x3FFFFFC0 /*0b1{24, POOL_LINE_BITS}0{6, CACHELINE_BITS} */ 65 | 66 | #define PRETHREAD_BIT 31 /* 2GB/Thread */ 67 | //#define PRETHREAD_BIT 33 /* 8GB/Thread */ 68 | #define GLOBAL_BIT 34 /* 16GB/Global */ 69 | //#define GLOBAL_BIT 38 /* 256GB/Global */ 70 | #define GLOBAL_BIT_NI 32 /* 4GB/DIMM, 24 GB with software interleaving */ 71 | //#define PRETHREAD_BIT 28 /* 256MB/Thread */ 72 | 73 | #define GLOBAL_WORKSET (1L << GLOBAL_BIT) 74 | #define PERTHREAD_WORKSET (1L << PRETHREAD_BIT) 75 | #define PERDIMMGROUP_WORKSET (PERTHREAD_WORKSET * 6) 76 | 77 | //#define PERTHREAD_MASK 0xFFFFFC0 //256M 78 | #define PERTHREAD_MASK 0x7FFFFFC0 //2G 79 | 80 | #define PERTHREAD_CHECKSTOP (1L << 25) /* yield after wrtting 32MB */ 81 | 82 | #define MB 0x100000 83 | #define GB 0x40000000 84 | 85 | #define DIMM_SIZE (256UL * GB) 86 | #define GLOBAL_WORKSET_NI (1UL << GLOBAL_BIT_NI) 87 | 88 | 89 | #define LFS_THREAD_MAX 48 90 | #define LFS_ACCESS_MAX_BITS 22 91 | #define LFS_ACCESS_MAX (1 << LFS_ACCESS_MAX_BITS) // 512KB * 4096 (rng_array) = 2G 92 | #define LFS_RUNTIME_MAX 600 93 | #define LFS_DELAY_MAX 1000000 94 | 95 | #define LFS_PERMRAND_ENTRIES 0x1000 96 | #define LFS_PERMRAND_SIZE LFS_PERMRAND_ENTRIES * 4 97 | #define LFS_PERMRAND_SIZE_IMM "$0x4000," 98 | 99 | 100 | 101 | #define LAT_ASSERT(x) \ 102 | if (!(x)) { \ 103 | dump_stack(); \ 104 | printk(KERN_WARNING "assertion failed %s:%d: %s\n", \ 105 | __FILE__, __LINE__, #x); \ 106 | } 107 | 108 | 109 | #define rdtscp(low,high,aux) \ 110 | __asm__ __volatile__ (".byte 0x0f,0x01,0xf9" : "=a" (low), "=d" (high), "=c" (aux)) 111 | 112 | 113 | /* since Skylake, Intel uses non-inclusive last-level cache, drop cache before testing */ 114 | static inline void drop_cache(void) 115 | { 116 | asm volatile("wbinvd": : :"memory"); 117 | asm volatile("mfence": : :"memory"); 118 | } 119 | 120 | typedef enum { 121 | TASK_INVALID = 0, 122 | TASK_BASIC_OP, 123 | TASK_STRIDED_LAT, 124 | TASK_STRIDED_BW, 125 | TASK_SIZE_BW, 126 | TASK_OVERWRITE, 127 | TASK_READ_AFTER_WRITE, 128 | TASK_STRAIGHT_WRITE, 129 | TASK_THBIND, 130 | TASK_FLUSHFENCE, 131 | TASK_CACHEPROBE, 132 | TASK_IMCPROBE, 133 | TASK_SEQ, 134 | 135 | TASK_COUNT 136 | } lattest_task; 137 | 138 | typedef enum { 139 | ALIGN_GLOBAL = 0, /* All threads writes to entire dataset, interleaved */ 140 | ALIGN_PERTHREAD, /* Thread-local dataset (6 DIMMs), interleaved */ 141 | ALIGN_PERDIMM, /* Per-DIMM dataset, interleaved */ 142 | ALIGN_PERIMC, /* Per-iMC (3 DIMMs) dataset, interleaved */ 143 | ALIGN_PERCHANNELGROUP, /* Two DIMMs on two iMCs, interleaved */ 144 | ALIGN_NI_GLOBAL, /* ALIGN_GLOBAL in non-interleaved mode */ 145 | ALIGN_NI, /* ALIGN_PERTHREAD in non-interleaved mode */ 146 | 147 | ALIGN_INVALID 148 | } align_mode; 149 | 150 | typedef int(bench_func_t)(void *); 151 | 152 | /* Fine if the struct is not cacheline aligned */ 153 | struct latencyfs_timing { 154 | uint64_t v; 155 | char padding[56]; /* 64 - 8 */ 156 | }__attribute((__packed__)); 157 | 158 | struct report_sbi { 159 | struct super_block *sb; 160 | struct block_device *s_bdev; 161 | void *virt_addr; 162 | unsigned long initsize; 163 | uint64_t phys_addr; 164 | }; 165 | 166 | struct latency_sbi 167 | { 168 | struct super_block *sb; 169 | struct block_device *s_bdev; 170 | void *virt_addr; 171 | unsigned long initsize; 172 | uint64_t phys_addr; 173 | uint64_t write_start; 174 | uint64_t write_size; /* TODO: write_size cannot be over 4GB */ 175 | uint64_t probe_count; 176 | uint64_t access_size; 177 | uint64_t strided_size; 178 | int align; 179 | int clwb_rate; 180 | int fence_rate; 181 | int worker_cnt; 182 | int runtime; 183 | int bwsize_bit; 184 | int delay; 185 | int op; 186 | 187 | struct latencyfs_timing *timing; 188 | struct report_sbi *rep; 189 | struct task_struct **workers; 190 | struct latencyfs_worker_ctx *ctx; 191 | }; 192 | 193 | struct latencyfs_worker_ctx { 194 | int core_id; 195 | uint8_t *addr; 196 | uint8_t *seed_addr; /* for multi-threaded tasks only */ 197 | struct latency_sbi *sbi; 198 | }; 199 | 200 | int latency_job(void *arg); 201 | int strided_latjob(void *arg); 202 | int strided_bwjob(void *arg); 203 | int sizebw_job(void *arg); 204 | int latencyfs_proc_init(void); 205 | int overwrite_job(void *arg); 206 | int read_after_write_job(void *arg); 207 | int straight_write_job(void *arg); 208 | int align_bwjob(void *arg); 209 | int cachefence_bwjob(void *arg); 210 | int cacheprobe_job(void *arg); 211 | int imcprobe_job(void *arg); 212 | int seq_bwjob(void *arg); 213 | 214 | struct latency_option { 215 | const char *name; 216 | unsigned int has_arg; 217 | int val; 218 | }; 219 | 220 | #if __KERNEL__ 221 | /* misc */ 222 | inline void latencyfs_prealloc_large(uint8_t *addr, uint64_t size, uint64_t bit_mask); 223 | inline unsigned long fastrand(unsigned long *x, unsigned long *y, unsigned long *z, uint64_t bit_mask); 224 | inline void latencyfs_prealloc_global_permutation_array(int size); 225 | 226 | int latencyfs_getopt(const char *caller, char **options, const struct latency_option *opts, 227 | char **optopt, char **optarg, unsigned long *value); 228 | void latencyfs_start_task(struct latency_sbi *sbi, int task, int threads); 229 | 230 | int latencyfs_print_help(struct seq_file *seq, void *v); 231 | #endif 232 | 233 | #endif // LATTESTER_H 234 | -------------------------------------------------------------------------------- /src/kernel/proc.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "lattester.h" 7 | 8 | struct proc_dir_entry *latencyfs_proc; 9 | extern struct latency_sbi *global_sbi; 10 | 11 | 12 | static const struct latency_option lattest_opts[] = 13 | {{"task", OPT_INT, 'T'}, 14 | {"op", OPT_INT, 'o'}, 15 | {"parallel", OPT_INT, 'p'}, 16 | {"runtime", OPT_INT, 't'}, 17 | {"message", OPT_STRING, 'm'}, 18 | {"access_size", OPT_INT, 'a'}, 19 | {"bwsize_bit", OPT_INT, 'b'}, 20 | {"delay", OPT_INT, 'd'}, 21 | {"stride_size", OPT_INT, 's'}, 22 | {"write_start", OPT_INT, 'w'}, 23 | {"write_size", OPT_INT, 'z'}, 24 | {"align_mode", OPT_INT, 'l'}, 25 | {"fence_rate", OPT_INT, 'f'}, 26 | {"clwb_rate", OPT_INT, 'c'}, 27 | {"probe_count", OPT_INT, 'r'}, 28 | {NULL, 0, 0}}; 29 | 30 | 31 | 32 | void latencyfs_parse_cmd(struct latency_sbi *sbi, char *cmd) 33 | { 34 | int op; 35 | int ret = 0; 36 | char *optarg; 37 | char *message = NULL; 38 | unsigned long optint; 39 | int threads = 1; 40 | int task = 0; 41 | 42 | /* Default args */ 43 | sbi->op = 0; 44 | sbi->access_size = 64; 45 | sbi->strided_size = 64; 46 | sbi->runtime = 10; 47 | sbi->bwsize_bit = 6; 48 | sbi->delay = 0; 49 | sbi->write_start = 0; 50 | sbi->write_size = 64; 51 | sbi->clwb_rate = 64; 52 | sbi->fence_rate = 64; 53 | sbi->probe_count = LATENCY_OPS_COUNT; 54 | sbi->align = ALIGN_PERTHREAD; 55 | 56 | while ((op = latencyfs_getopt("lattester", &cmd, lattest_opts, NULL, &optarg, 57 | &optint)) != 0) 58 | { 59 | switch (op) 60 | { 61 | case 'T': 62 | if (optint > 0 && optint < TASK_COUNT) 63 | task = optint; 64 | else 65 | pr_info("Ignore parameter: task = %ld\n", optint); 66 | break; 67 | case 'o': 68 | if (optint >= 0 && optint < 4) 69 | sbi->op = optint; 70 | else 71 | pr_info("Ignore parameter: op = %ld\n", optint); 72 | break; 73 | case 'a': 74 | if (optint > 0 && optint <= LFS_ACCESS_MAX) 75 | sbi->access_size = optint; 76 | else 77 | pr_info("Ignore parameter: access_size = %ld\n", optint); 78 | break; 79 | case 'd': 80 | if (optint >= 0 && optint <= LFS_DELAY_MAX) 81 | sbi->delay = optint; 82 | else 83 | pr_info("Ignore parameter: delay = %ld\n", optint); 84 | break; 85 | case 's': 86 | if (optint >= 0 && optint <= LFS_ACCESS_MAX) 87 | sbi->strided_size = optint; 88 | else 89 | pr_info("Ignore parameter: stride_size = %ld\n", optint); 90 | break; 91 | case 'p': 92 | if (optint > 0 && optint < LFS_THREAD_MAX) 93 | threads = optint; 94 | else 95 | pr_info("Ignore parameter: parallel = %ld\n", optint); 96 | break; 97 | case 't': 98 | if (optint > 0 && optint < LFS_RUNTIME_MAX) 99 | sbi->runtime = optint; 100 | else 101 | pr_info("Ignore parameter: runtime = %ld\n", optint); 102 | break; 103 | case 'w': 104 | if (optint > 0 && optint < GLOBAL_WORKSET) 105 | sbi->write_start = optint; 106 | else 107 | pr_info("Ignore parameter: write_start = %ld\n", optint); 108 | break; 109 | case 'l': 110 | if (optint >= 0 && optint < ALIGN_INVALID) 111 | sbi->align = optint; 112 | else 113 | pr_info("Ignore parameter: align_mode = %ld\n", optint); 114 | break; 115 | case 'z': 116 | if (optint > 0 && optint < LFS_ACCESS_MAX) 117 | sbi->write_size = optint; 118 | else 119 | pr_info("Ignore parameter: write_size = %ld\n", optint); 120 | break; 121 | case 'm': 122 | message = optarg; 123 | break; 124 | case 'b': 125 | if (optint >= CACHELINE_BITS && optint <= LFS_ACCESS_MAX_BITS) 126 | sbi->bwsize_bit = optint; 127 | else 128 | pr_info("Ignore parameter: bwsize_bit = %ld\n", optint); 129 | break; 130 | case 'f': 131 | if (optint > 0 && optint <= LFS_ACCESS_MAX) 132 | sbi->fence_rate = optint; 133 | else 134 | pr_info("Ignore parameter: fence_rate = %ld\n", optint); 135 | break; 136 | case 'c': 137 | if (optint > 0 && optint <= LFS_ACCESS_MAX) 138 | sbi->clwb_rate = optint; 139 | else 140 | pr_info("Ignore parameter: clwb_rate = %ld\n", optint); 141 | break; 142 | case 'r': 143 | if (optint > 0 && optint <= (1ULL << 28)) // 64GB (access size is 28) 144 | sbi->probe_count = optint; 145 | else 146 | pr_info("Ignore parameter: probe_count = %ld\n", optint); 147 | break; 148 | default: 149 | pr_info("unknown opt %s\n", optarg); 150 | ret = -EINVAL; 151 | break; 152 | } 153 | } 154 | if (ret) 155 | goto out; 156 | 157 | if (task == 0) { 158 | pr_info("Task not assigned\n"); 159 | ret = -EINVAL; 160 | goto out; 161 | } 162 | 163 | /* Sanity checks */ 164 | if (task == TASK_STRIDED_LAT || task == TASK_STRIDED_BW) { 165 | if (sbi->strided_size < sbi->access_size) { 166 | pr_info("Error: stride size is smaller than access size.\n"); 167 | ret = -EINVAL; 168 | goto out; 169 | } 170 | } 171 | 172 | if (task == TASK_READ_AFTER_WRITE) { 173 | if ((sbi->strided_size != 0) && (sbi->strided_size < sbi->access_size)) { 174 | pr_info("Error: stride size (not 0) is smaller than access size.\n"); 175 | ret = -EINVAL; 176 | goto out; 177 | } 178 | } 179 | 180 | if (task == TASK_SIZE_BW) { 181 | if (!sbi->bwsize_bit) { 182 | pr_info("SizeBW task requires bwsize_bit.\n"); 183 | ret = -EINVAL; 184 | goto out; 185 | } 186 | } 187 | 188 | if (task == TASK_THBIND) { 189 | sbi->access_size = 1 << sbi->bwsize_bit; 190 | if (sbi->bwsize_bit > PAGE_SHIFT) { 191 | pr_info("sizebw_bit > 12 is not supported for this test.\n"); 192 | ret = -EINVAL; 193 | goto out; 194 | } 195 | } 196 | 197 | if (task == TASK_FLUSHFENCE) { 198 | if (sbi->access_size % sbi->fence_rate || sbi->access_size % sbi->clwb_rate 199 | || sbi->fence_rate % sbi->clwb_rate) { 200 | pr_info("FlushFence requirement: \n"); 201 | pr_info("access_size %% fence_rate == 0\n"); 202 | pr_info("access_size %% clwb_rate == 0\n"); 203 | pr_info("fence_rate %% clwb_rate == 0: \n"); 204 | ret = -EINVAL; 205 | goto out; 206 | } 207 | } 208 | 209 | if (task == TASK_IMCPROBE) { 210 | sbi->align = ALIGN_NI; 211 | pr_info("Warning: iMC test uses per-DIMM alignment, please ensure AppDirectNotInterleaved is used\n"); 212 | } 213 | if (message) pr_info("LATTester_START: %s\n", message); 214 | latencyfs_start_task(sbi, task, threads); 215 | if (message) pr_info("LATTester_END: %s\n", message); 216 | 217 | return; 218 | 219 | out: 220 | pr_info("parse error: %d\n", ret); 221 | } 222 | 223 | static ssize_t latencyfs_proc_write(struct file *file, const char __user *buffer, 224 | size_t count, loff_t *ppos) 225 | { 226 | char *cmdline; 227 | int ret = 0; 228 | 229 | if (!try_module_get(THIS_MODULE)) 230 | return -ENODEV; 231 | 232 | cmdline = kmalloc(count, GFP_KERNEL); 233 | if (!cmdline) 234 | { 235 | pr_err("kmalloc failure\n"); 236 | return -ENOMEM; 237 | } 238 | if (copy_from_user(cmdline, buffer, count)) 239 | { 240 | return -EFAULT; 241 | } 242 | 243 | cmdline[count - 1] = 0; 244 | pr_info("Command [%s]\n", cmdline); 245 | 246 | if (!global_sbi) { 247 | pr_err("LatencyFS not mounted."); 248 | kfree(cmdline); 249 | return count; 250 | } 251 | 252 | latencyfs_parse_cmd(global_sbi, cmdline); 253 | 254 | kfree(cmdline); 255 | 256 | module_put(THIS_MODULE); 257 | 258 | if (ret) 259 | return ret; 260 | else 261 | return (int)count; 262 | } 263 | 264 | 265 | static int latencyfs_proc_open(struct inode *inode, struct file *file) 266 | { 267 | return single_open(file, latencyfs_print_help, inode->i_private); 268 | } 269 | 270 | static struct file_operations latencyfs_ops = { 271 | .owner = THIS_MODULE, 272 | .open = latencyfs_proc_open, 273 | .read = seq_read, 274 | .llseek = seq_lseek, 275 | .release = single_release, 276 | .write = latencyfs_proc_write, 277 | }; 278 | 279 | int latencyfs_proc_init(void) 280 | { 281 | 282 | latencyfs_proc = proc_create("lattester", 0666, NULL, &latencyfs_ops); 283 | if (!latencyfs_proc) 284 | { 285 | pr_err("/proc/lattester creation failed\n"); 286 | return -ENOMEM; 287 | } 288 | 289 | return 0; 290 | } 291 | -------------------------------------------------------------------------------- /src/testscript/parse_bw.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | import re 4 | import sys 5 | import datetime 6 | import json 7 | import math 8 | from google.cloud import datastore 9 | 10 | # Parse output and upload to Google Cloud Datastore 11 | # ./parse_bw.py 12 | # E.g. ./parse_bw.py tag 5 0 13 | 14 | kind = 'aepsweep' 15 | kind_align = 'aepalign' 16 | kind_flush = 'aepflush' 17 | upqueue = [] 18 | 19 | if len(sys.argv) >= 3: 20 | timer_stop = int(sys.argv[2]) 21 | else: 22 | timer_stop = 5 23 | 24 | if len(sys.argv) == 4: 25 | updatedb = 1 26 | datastore_client = datastore.Client() 27 | with open('config.json') as f: 28 | config = json.load(f) 29 | else: 30 | updatedb = 0 31 | 32 | if len(sys.argv) < 2 or len(sys.argv) > 4: 33 | print('{0} [output.txt] '.format(sys.argv[0])) 34 | exit(0) 35 | 36 | # Predefined data format (see config.example.json) 37 | ''' 38 | { 39 | "media": "lpmem", 40 | "interleaving": "1", 41 | "local": "1", 42 | "avgpower": "15000", 43 | "peakpower": "20000", 44 | "emon": "1", 45 | "aepwatch": "1", 46 | "fence": "1", 47 | "baddata": "0", 48 | "rerun": "0" 49 | } 50 | ''' 51 | 52 | operation_dict = ['Load', 'NTStore', 'Store', 'Store+Flush'] 53 | task_dict = [None, None, None, 'StrideBW', 54 | 'RandBW', None, None, None, 'AlignBW', 'ClwbFence'] 55 | version = os.popen('git rev-parse HEAD').read() 56 | 57 | 58 | def upload_datastore(client, parallel, tasktype, access_size, align_size, stride_size, delay, 59 | op, throughput, tag, aeprun, machine): 60 | global upqueue 61 | 62 | key = datastore_client.key(kind, aeprun) 63 | task = datastore.Entity(key=key) 64 | 65 | for k in config: 66 | task[k] = _decode(config[k]) 67 | 68 | task['access_size'] = int(access_size) 69 | task['threads'] = int(parallel) 70 | task['task_type'] = task_dict[int(tasktype)] 71 | task['access_size'] = int(access_size) 72 | task['stride_size'] = int(stride_size) 73 | task['align_size'] = int(align_size) 74 | task['operation'] = operation_dict[int(op)] 75 | task['date'] = str(datetime.date.today()) 76 | task['version'] = version 77 | task['throughput'] = float(throughput) 78 | task['tag'] = tag 79 | task['machine'] = machine 80 | upqueue.append(task) 81 | if len(upqueue) >= 500: 82 | datastore_client.put_multi(upqueue) 83 | upqueue = [] 84 | 85 | 86 | def update_flush(datastore_client, parallel, tasktype, access_size, 87 | fencesize, flushsize, op, throughput, tag, aeprun, machine): 88 | 89 | global upqueue 90 | 91 | key = datastore_client.key(kind_flush, aeprun) 92 | task = datastore.Entity(key=key) 93 | 94 | for k in config: 95 | task[k] = _decode(config[k]) 96 | 97 | task['access_size'] = int(access_size) 98 | task['threads'] = int(parallel) 99 | task['task_type'] = task_dict[int(tasktype)] 100 | task['fence_size'] = int(fencesize) 101 | task['clwb_size'] = int(flushsize) 102 | task['op'] = int(op) 103 | task['date'] = str(datetime.date.today()) 104 | task['version'] = version 105 | task['throughput'] = float(throughput) 106 | task['tag'] = tag 107 | task['machine'] = machine 108 | repeat = False 109 | for t in upqueue: 110 | if t.key.name == aeprun: 111 | repeat = True 112 | if not repeat: 113 | upqueue.append(task) 114 | if len(upqueue) >= 500: 115 | datastore_client.put_multi(upqueue) 116 | upqueue = [] 117 | 118 | 119 | def update_align(datastore_client, parallel, tasktype, access_size, 120 | op, alloc_type, throughput, tag, aeprun, machine): 121 | 122 | global upqueue 123 | 124 | key = datastore_client.key(kind_align, aeprun) 125 | task = datastore.Entity(key=key) 126 | 127 | for k in config: 128 | task[k] = _decode(config[k]) 129 | 130 | task['access_size'] = int(access_size) 131 | task['threads'] = int(parallel) 132 | task['task_type'] = task_dict[int(tasktype)] 133 | task['access_size'] = int(access_size) 134 | task['alloc_type'] = int(alloc_type) 135 | task['operation'] = operation_dict[int(op)] 136 | task['date'] = str(datetime.date.today()) 137 | task['version'] = version 138 | task['throughput'] = float(throughput) 139 | task['tag'] = tag 140 | task['machine'] = machine 141 | upqueue.append(task) 142 | if len(upqueue) >= 500: 143 | datastore_client.put_multi(upqueue) 144 | upqueue = [] 145 | 146 | 147 | def _decode(o): 148 | if isinstance(o, str) or isinstance(o, unicode): 149 | try: 150 | return int(o) 151 | except ValueError: 152 | return o 153 | elif isinstance(o, dict): 154 | return {k: _decode(v) for k, v in o.items()} 155 | elif isinstance(o, list): 156 | return [_decode(v) for v in o] 157 | else: 158 | return o 159 | 160 | 161 | timer = 0 162 | with open(sys.argv[1], 'r') as f: 163 | task = "" 164 | throughput = [] 165 | for line in f: 166 | line = re.sub('\[.*\] ', '', line) 167 | parts = line.split(' ') 168 | if parts[0] == 'LATTester_START:': 169 | assert timer == 0 170 | task = parts[1].strip('\n').strip('\r') 171 | throughput = [] 172 | elif parts[0] == 'LATTester_END:': 173 | assert timer == timer_stop 174 | timer = 0 175 | taskparts = task.split('-') 176 | machine = taskparts[0] 177 | tasktype = taskparts[1] 178 | avg = sum(throughput) / float(len(throughput)) 179 | if tasktype == '3': 180 | # - StrideBW: AEP1-3-TAG-64-256-1-300-2 181 | tag = taskparts[2] 182 | size = taskparts[3] 183 | stride = taskparts[4] 184 | parallel = taskparts[5] 185 | delay = taskparts[6] 186 | op = taskparts[7] 187 | a = [tag, op, parallel, size, stride, delay, avg] 188 | if updatedb == 1: 189 | upload_datastore(datastore_client, parallel, tasktype, size, 0, stride, 190 | delay, op, avg, tag, task, machine) 191 | print(*a, sep=',') 192 | elif tasktype == '4': 193 | # - RandBW: AEP1-4-TAG-64-1-2 194 | stride = 0 195 | delay = 0 196 | tag = taskparts[2] 197 | align_size = int(math.pow(2, int(taskparts[3]))) 198 | # AEP1-4-unalign-7-128-2-0 199 | size = taskparts[4] 200 | parallel = taskparts[5] 201 | op = taskparts[6] 202 | a = [tag, op, parallel, size, align_size, avg] 203 | if updatedb == 1: 204 | upload_datastore(datastore_client, parallel, tasktype, size, align_size, stride, 205 | delay, op, avg, tag, task, machine) 206 | print(*a, sep=',') 207 | elif tasktype == '8': 208 | # Align: AEP1-8-9-6-2-2-TAG 209 | size = int(math.pow(2, int(taskparts[2]))) 210 | parallel = taskparts[3] 211 | op = taskparts[4] 212 | alloc_type = taskparts[5] 213 | tag = taskparts[6] 214 | a = [tag, op, parallel, size, alloc_type, avg] 215 | if updatedb == 1: 216 | update_align(datastore_client, parallel, tasktype, size, 217 | op, alloc_type, avg, tag, task, machine) 218 | print(*a, sep=',') 219 | elif tasktype == '9': 220 | # AEP2-9-flushfence-65536-64-8192-3-0 221 | tag = taskparts[2] 222 | size = taskparts[3] 223 | flush = taskparts[4] 224 | fence = taskparts[5] 225 | parallel = taskparts[6] 226 | op = taskparts[7] 227 | a = [task, parallel, size, flush, fence, avg] 228 | if updatedb: 229 | update_flush(datastore_client, parallel, tasktype, size, 230 | fence, flush, op, avg, tag, task, machine) 231 | print(*a, sep=',') 232 | 233 | else: 234 | lineparts = parts[0].strip('\n').strip('\r').split('\t') 235 | if not re.match("\d+$", lineparts[0]) or int(lineparts[0]) != timer + 1: 236 | # sys.stderr.write('ignore line:' + line) 237 | continue 238 | second = lineparts[0] 239 | value = lineparts[1] 240 | throughput.append(int(value)) 241 | timer += 1 242 | 243 | if updatedb: 244 | datastore_client.put_multi(upqueue) 245 | -------------------------------------------------------------------------------- /src/kernel/memaccess.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | #ifndef LATTESTER_MEMACCESS_H 3 | #define LATTESTER_MEMACCESS_H 4 | /* 5 | * BRIEF DESCRIPTION 6 | * 7 | * Header for memory access functions 8 | * 9 | * Copyright 2019 Regents of the University of California, 10 | * UCSD Non-Volatile Systems Lab 11 | */ 12 | 13 | // Size-BW Random (char *start_addr, long size, long count, long *rand_seed, long access_mask); 14 | typedef void (*lfs_test_randfunc_t)(char *, long, long, long *, long); 15 | 16 | // Size-BW Stride: latency_test(char * start_addr, long size, long stride, long bufsize) 17 | typedef void (*lfs_test_stridefunc_t)(char *start_addr, long size, long skip, long delay, long count); 18 | 19 | // Size-BW Random (char *start_addr, long size, long count, long *rand_seed, long access_mask); 20 | typedef void (*lfs_test_seq_t)(char *, char *, long); 21 | 22 | void cachefence(char *start_addr, long size, long cache, long fence); 23 | 24 | void stride_load(char *start_addr, long size, long skip, long delay, long count); 25 | void stride_store(char *start_addr, long size, long skip, long delay, long count); 26 | void stride_storeclwb(char *start_addr, long size, long skip, long delay, long count); 27 | void stride_nt(char *start_addr, long size, long skip, long delay, long count); 28 | 29 | void sizebw_load(char *start_addr, long size, long count, long *rand_seed, long access_mask); 30 | void sizebw_store(char *start_addr, long size, long count, long *rand_seed, long access_mask); 31 | void sizebw_storeclwb(char *start_addr, long size, long count, long *rand_seed, long access_mask); 32 | void sizebw_nt(char *start_addr, long size, long count, long *rand_seed, long access_mask); 33 | 34 | void seq_load(char * start_addr, char *end_addr, long size); 35 | void seq_store(char * start_addr, char *end_addr, long size); 36 | void seq_clwb(char * start_addr, char *end_addr, long size); 37 | void seq_nt(char * start_addr, char *end_addr, long size); 38 | 39 | void cacheprobe(char *start_addr, char *end_addr, long stride); 40 | void imcprobe(char *start_addr, char *end_addr, long loop); 41 | 42 | void stride_read_after_write(char *start_addr, long size, long skip, long delay, long count); 43 | // Pointer chasing tests 44 | int init_chasing_index(uint64_t *cindex, uint64_t csize); 45 | void chasing_storeclwb(char *start_addr, long size, long skip, long count, uint64_t *cindex); 46 | void chasing_loadnt(char *start_addr, long size, long skip, long count, uint64_t *cindex); 47 | 48 | #define SIZEBTNT_64_AVX512 \ 49 | "vmovntdq %%zmm0, 0x0(%%r9, %%r10) \n" \ 50 | "add $0x40, %%r10 \n" 51 | 52 | #define SIZEBTNT_128_AVX512 \ 53 | "vmovntdq %%zmm0, 0x0(%%r9, %%r10) \n" \ 54 | "vmovntdq %%zmm0, 0x40(%%r9, %%r10) \n" \ 55 | "add $0x80, %%r10 \n" 56 | 57 | #define SIZEBTNT_256_AVX512 \ 58 | "vmovntdq %%zmm0, 0x0(%%r9, %%r10) \n" \ 59 | "vmovntdq %%zmm0, 0x40(%%r9, %%r10) \n" \ 60 | "vmovntdq %%zmm0, 0x80(%%r9, %%r10) \n" \ 61 | "vmovntdq %%zmm0, 0xc0(%%r9, %%r10) \n" \ 62 | "add $0x100, %%r10 \n" 63 | 64 | #define SIZEBTNT_512_AVX512 \ 65 | "vmovntdq %%zmm0, 0x0(%%r9, %%r10) \n" \ 66 | "vmovntdq %%zmm0, 0x40(%%r9, %%r10) \n" \ 67 | "vmovntdq %%zmm0, 0x80(%%r9, %%r10) \n" \ 68 | "vmovntdq %%zmm0, 0xc0(%%r9, %%r10) \n" \ 69 | "vmovntdq %%zmm0, 0x100(%%r9, %%r10) \n" \ 70 | "vmovntdq %%zmm0, 0x140(%%r9, %%r10) \n" \ 71 | "vmovntdq %%zmm0, 0x180(%%r9, %%r10) \n" \ 72 | "vmovntdq %%zmm0, 0x1c0(%%r9, %%r10) \n" \ 73 | "add $0x200, %%r10 \n" 74 | 75 | #define SIZEBTNT_1024_AVX512 \ 76 | "vmovntdq %%zmm0, 0x0(%%r9, %%r10) \n" \ 77 | "vmovntdq %%zmm0, 0x40(%%r9, %%r10) \n" \ 78 | "vmovntdq %%zmm0, 0x80(%%r9, %%r10) \n" \ 79 | "vmovntdq %%zmm0, 0xc0(%%r9, %%r10) \n" \ 80 | "vmovntdq %%zmm0, 0x100(%%r9, %%r10) \n" \ 81 | "vmovntdq %%zmm0, 0x140(%%r9, %%r10) \n" \ 82 | "vmovntdq %%zmm0, 0x180(%%r9, %%r10) \n" \ 83 | "vmovntdq %%zmm0, 0x1c0(%%r9, %%r10) \n" \ 84 | "vmovntdq %%zmm0, 0x200(%%r9, %%r10) \n" \ 85 | "vmovntdq %%zmm0, 0x240(%%r9, %%r10) \n" \ 86 | "vmovntdq %%zmm0, 0x280(%%r9, %%r10) \n" \ 87 | "vmovntdq %%zmm0, 0x2c0(%%r9, %%r10) \n" \ 88 | "vmovntdq %%zmm0, 0x300(%%r9, %%r10) \n" \ 89 | "vmovntdq %%zmm0, 0x340(%%r9, %%r10) \n" \ 90 | "vmovntdq %%zmm0, 0x380(%%r9, %%r10) \n" \ 91 | "vmovntdq %%zmm0, 0x3c0(%%r9, %%r10) \n" \ 92 | "add $0x400, %%r10 \n" 93 | 94 | #define SIZEBTSTFLUSH_64_AVX512 \ 95 | "vmovdqa64 %%zmm0, 0x0(%%r9, %%r10) \n" \ 96 | "clwb 0x0(%%r9, %%r10) \n" \ 97 | "add $0x40, %%r10 \n" 98 | 99 | #define SIZEBTSTFLUSH_128_AVX512 \ 100 | "vmovdqa64 %%zmm0, 0x0(%%r9, %%r10) \n" \ 101 | "clwb 0x0(%%r9, %%r10) \n" \ 102 | "vmovdqa64 %%zmm0, 0x40(%%r9, %%r10) \n" \ 103 | "clwb 0x40(%%r9, %%r10) \n" \ 104 | "add $0x80, %%r10 \n" 105 | 106 | #define SIZEBTSTFLUSH_256_AVX512 \ 107 | "vmovdqa64 %%zmm0, 0x0(%%r9, %%r10) \n" \ 108 | "clwb 0x0(%%r9, %%r10) \n" \ 109 | "vmovdqa64 %%zmm0, 0x40(%%r9, %%r10) \n" \ 110 | "clwb 0x40(%%r9, %%r10) \n" \ 111 | "vmovdqa64 %%zmm0, 0x80(%%r9, %%r10) \n" \ 112 | "clwb 0x80(%%r9, %%r10) \n" \ 113 | "vmovdqa64 %%zmm0, 0xc0(%%r9, %%r10) \n" \ 114 | "clwb 0xc0(%%r9, %%r10) \n" \ 115 | "add $0x100, %%r10 \n" 116 | 117 | #define SIZEBTSTFLUSH_512_AVX512 \ 118 | "vmovdqa64 %%zmm0, 0x0(%%r9, %%r10) \n" \ 119 | "clwb 0x0(%%r9, %%r10) \n" \ 120 | "vmovdqa64 %%zmm0, 0x40(%%r9, %%r10) \n" \ 121 | "clwb 0x40(%%r9, %%r10) \n" \ 122 | "vmovdqa64 %%zmm0, 0x80(%%r9, %%r10) \n" \ 123 | "clwb 0x80(%%r9, %%r10) \n" \ 124 | "vmovdqa64 %%zmm0, 0xc0(%%r9, %%r10) \n" \ 125 | "clwb 0xc0(%%r9, %%r10) \n" \ 126 | "vmovdqa64 %%zmm0, 0x100(%%r9, %%r10) \n" \ 127 | "clwb 0x100(%%r9, %%r10) \n" \ 128 | "vmovdqa64 %%zmm0, 0x140(%%r9, %%r10) \n" \ 129 | "clwb 0x140(%%r9, %%r10) \n" \ 130 | "vmovdqa64 %%zmm0, 0x180(%%r9, %%r10) \n" \ 131 | "clwb 0x180(%%r9, %%r10) \n" \ 132 | "vmovdqa64 %%zmm0, 0x1c0(%%r9, %%r10) \n" \ 133 | "clwb 0x1c0(%%r9, %%r10) \n" \ 134 | "add $0x200, %%r10 \n" 135 | 136 | #define SIZEBTSTFLUSH_1024_AVX512 \ 137 | "vmovdqa64 %%zmm0, 0x0(%%r9, %%r10) \n" \ 138 | "clwb 0x0(%%r9, %%r10) \n" \ 139 | "vmovdqa64 %%zmm0, 0x40(%%r9, %%r10) \n" \ 140 | "clwb 0x40(%%r9, %%r10) \n" \ 141 | "vmovdqa64 %%zmm0, 0x80(%%r9, %%r10) \n" \ 142 | "clwb 0x80(%%r9, %%r10) \n" \ 143 | "vmovdqa64 %%zmm0, 0xc0(%%r9, %%r10) \n" \ 144 | "clwb 0xc0(%%r9, %%r10) \n" \ 145 | "vmovdqa64 %%zmm0, 0x100(%%r9, %%r10) \n" \ 146 | "clwb 0x100(%%r9, %%r10) \n" \ 147 | "vmovdqa64 %%zmm0, 0x140(%%r9, %%r10) \n" \ 148 | "clwb 0x140(%%r9, %%r10) \n" \ 149 | "vmovdqa64 %%zmm0, 0x180(%%r9, %%r10) \n" \ 150 | "clwb 0x180(%%r9, %%r10) \n" \ 151 | "vmovdqa64 %%zmm0, 0x1c0(%%r9, %%r10) \n" \ 152 | "clwb 0x1c0(%%r9, %%r10) \n" \ 153 | "vmovdqa64 %%zmm0, 0x200(%%r9, %%r10) \n" \ 154 | "clwb 0x200(%%r9, %%r10) \n" \ 155 | "vmovdqa64 %%zmm0, 0x240(%%r9, %%r10) \n" \ 156 | "clwb 0x240(%%r9, %%r10) \n" \ 157 | "vmovdqa64 %%zmm0, 0x280(%%r9, %%r10) \n" \ 158 | "clwb 0x280(%%r9, %%r10) \n" \ 159 | "vmovdqa64 %%zmm0, 0x2c0(%%r9, %%r10) \n" \ 160 | "clwb 0x2c0(%%r9, %%r10) \n" \ 161 | "vmovdqa64 %%zmm0, 0x300(%%r9, %%r10) \n" \ 162 | "clwb 0x300(%%r9, %%r10) \n" \ 163 | "vmovdqa64 %%zmm0, 0x340(%%r9, %%r10) \n" \ 164 | "clwb 0x340(%%r9, %%r10) \n" \ 165 | "vmovdqa64 %%zmm0, 0x380(%%r9, %%r10) \n" \ 166 | "clwb 0x380(%%r9, %%r10) \n" \ 167 | "vmovdqa64 %%zmm0, 0x3c0(%%r9, %%r10) \n" \ 168 | "clwb 0x3c0(%%r9, %%r10) \n" \ 169 | "add $0x400, %%r10 \n" 170 | 171 | #define SIZEBTST_64_AVX512 \ 172 | "vmovdqa64 %%zmm0, 0x0(%%r9, %%r10) \n" \ 173 | "add $0x40, %%r10 \n" 174 | 175 | #define SIZEBTST_128_AVX512 \ 176 | "vmovdqa64 %%zmm0, 0x0(%%r9, %%r10) \n" \ 177 | "vmovdqa64 %%zmm0, 0x40(%%r9, %%r10) \n" \ 178 | "add $0x80, %%r10 \n" 179 | 180 | #define SIZEBTST_256_AVX512 \ 181 | "vmovdqa64 %%zmm0, 0x0(%%r9, %%r10) \n" \ 182 | "vmovdqa64 %%zmm0, 0x40(%%r9, %%r10) \n" \ 183 | "vmovdqa64 %%zmm0, 0x80(%%r9, %%r10) \n" \ 184 | "vmovdqa64 %%zmm0, 0xc0(%%r9, %%r10) \n" \ 185 | "add $0x100, %%r10 \n" 186 | 187 | #define SIZEBTST_512_AVX512 \ 188 | "vmovdqa64 %%zmm0, 0x0(%%r9, %%r10) \n" \ 189 | "vmovdqa64 %%zmm0, 0x40(%%r9, %%r10) \n" \ 190 | "vmovdqa64 %%zmm0, 0x80(%%r9, %%r10) \n" \ 191 | "vmovdqa64 %%zmm0, 0xc0(%%r9, %%r10) \n" \ 192 | "vmovdqa64 %%zmm0, 0x100(%%r9, %%r10) \n" \ 193 | "vmovdqa64 %%zmm0, 0x140(%%r9, %%r10) \n" \ 194 | "vmovdqa64 %%zmm0, 0x180(%%r9, %%r10) \n" \ 195 | "vmovdqa64 %%zmm0, 0x1c0(%%r9, %%r10) \n" \ 196 | "add $0x200, %%r10 \n" 197 | 198 | #define SIZEBTST_1024_AVX512 \ 199 | "vmovdqa64 %%zmm0, 0x0(%%r9, %%r10) \n" \ 200 | "vmovdqa64 %%zmm0, 0x40(%%r9, %%r10) \n" \ 201 | "vmovdqa64 %%zmm0, 0x80(%%r9, %%r10) \n" \ 202 | "vmovdqa64 %%zmm0, 0xc0(%%r9, %%r10) \n" \ 203 | "vmovdqa64 %%zmm0, 0x100(%%r9, %%r10) \n" \ 204 | "vmovdqa64 %%zmm0, 0x140(%%r9, %%r10) \n" \ 205 | "vmovdqa64 %%zmm0, 0x180(%%r9, %%r10) \n" \ 206 | "vmovdqa64 %%zmm0, 0x1c0(%%r9, %%r10) \n" \ 207 | "vmovdqa64 %%zmm0, 0x200(%%r9, %%r10) \n" \ 208 | "vmovdqa64 %%zmm0, 0x240(%%r9, %%r10) \n" \ 209 | "vmovdqa64 %%zmm0, 0x280(%%r9, %%r10) \n" \ 210 | "vmovdqa64 %%zmm0, 0x2c0(%%r9, %%r10) \n" \ 211 | "vmovdqa64 %%zmm0, 0x300(%%r9, %%r10) \n" \ 212 | "vmovdqa64 %%zmm0, 0x340(%%r9, %%r10) \n" \ 213 | "vmovdqa64 %%zmm0, 0x380(%%r9, %%r10) \n" \ 214 | "vmovdqa64 %%zmm0, 0x3c0(%%r9, %%r10) \n" \ 215 | "add $0x400, %%r10 \n" 216 | 217 | #define SIZEBTLD_64_AVX512 \ 218 | "vmovntdqa 0x0(%%r9, %%r10), %%zmm0 \n" \ 219 | "add $0x40, %%r10 \n" 220 | 221 | #define SIZEBTLD_128_AVX512 \ 222 | "vmovntdqa 0x0(%%r9, %%r10), %%zmm0 \n" \ 223 | "vmovntdqa 0x40(%%r9, %%r10), %%zmm1 \n" \ 224 | "add $0x80, %%r10 \n" 225 | 226 | #define SIZEBTLD_256_AVX512 \ 227 | "vmovntdqa 0x0(%%r9, %%r10), %%zmm0 \n" \ 228 | "vmovntdqa 0x40(%%r9, %%r10), %%zmm1 \n" \ 229 | "vmovntdqa 0x80(%%r9, %%r10), %%zmm2 \n" \ 230 | "vmovntdqa 0xc0(%%r9, %%r10), %%zmm3 \n" \ 231 | "add $0x100, %%r10 \n" 232 | 233 | #define SIZEBTLD_512_AVX512 \ 234 | "vmovntdqa 0x0(%%r9, %%r10), %%zmm0 \n" \ 235 | "vmovntdqa 0x40(%%r9, %%r10), %%zmm1 \n" \ 236 | "vmovntdqa 0x80(%%r9, %%r10), %%zmm2 \n" \ 237 | "vmovntdqa 0xc0(%%r9, %%r10), %%zmm3 \n" \ 238 | "vmovntdqa 0x100(%%r9, %%r10), %%zmm4 \n" \ 239 | "vmovntdqa 0x140(%%r9, %%r10), %%zmm5 \n" \ 240 | "vmovntdqa 0x180(%%r9, %%r10), %%zmm6 \n" \ 241 | "vmovntdqa 0x1c0(%%r9, %%r10), %%zmm7 \n" \ 242 | "add $0x200, %%r10 \n" 243 | 244 | #define SIZEBTLD_1024_AVX512 \ 245 | "vmovntdqa 0x0(%%r9, %%r10), %%zmm0 \n" \ 246 | "vmovntdqa 0x40(%%r9, %%r10), %%zmm1 \n" \ 247 | "vmovntdqa 0x80(%%r9, %%r10), %%zmm2 \n" \ 248 | "vmovntdqa 0xc0(%%r9, %%r10), %%zmm3 \n" \ 249 | "vmovntdqa 0x100(%%r9, %%r10), %%zmm4 \n" \ 250 | "vmovntdqa 0x140(%%r9, %%r10), %%zmm5 \n" \ 251 | "vmovntdqa 0x180(%%r9, %%r10), %%zmm6 \n" \ 252 | "vmovntdqa 0x1c0(%%r9, %%r10), %%zmm7 \n" \ 253 | "vmovntdqa 0x200(%%r9, %%r10), %%zmm8 \n" \ 254 | "vmovntdqa 0x240(%%r9, %%r10), %%zmm9 \n" \ 255 | "vmovntdqa 0x280(%%r9, %%r10), %%zmm10 \n" \ 256 | "vmovntdqa 0x2c0(%%r9, %%r10), %%zmm11 \n" \ 257 | "vmovntdqa 0x300(%%r9, %%r10), %%zmm12 \n" \ 258 | "vmovntdqa 0x340(%%r9, %%r10), %%zmm13 \n" \ 259 | "vmovntdqa 0x380(%%r9, %%r10), %%zmm14 \n" \ 260 | "vmovntdqa 0x3c0(%%r9, %%r10), %%zmm15 \n" \ 261 | "add $0x400, %%r10 \n" 262 | 263 | #define SIZEBT_NT_64 "movnti %[random], 0x0(%%r9, %%r10) \n" \ 264 | "movnti %[random], 0x8(%%r9, %%r10) \n" \ 265 | "movnti %[random], 0x10(%%r9, %%r10) \n" \ 266 | "movnti %[random], 0x18(%%r9, %%r10) \n" \ 267 | "movnti %[random], 0x20(%%r9, %%r10) \n" \ 268 | "movnti %[random], 0x28(%%r9, %%r10) \n" \ 269 | "movnti %[random], 0x30(%%r9, %%r10) \n" \ 270 | "movnti %[random], 0x38(%%r9, %%r10) \n" \ 271 | "add $0x40, %%r10 \n" 272 | 273 | #define SIZEBT_LOAD_64 "mov 0x0(%%r9, %%r10), %%r13 \n" \ 274 | "mov 0x8(%%r9, %%r10), %%r13 \n" \ 275 | "mov 0x10(%%r9, %%r10), %%r13 \n" \ 276 | "mov 0x18(%%r9, %%r10), %%r13 \n" \ 277 | "mov 0x20(%%r9, %%r10), %%r13 \n" \ 278 | "mov 0x28(%%r9, %%r10), %%r13 \n" \ 279 | "mov 0x30(%%r9, %%r10), %%r13 \n" \ 280 | "mov 0x38(%%r9, %%r10), %%r13 \n" \ 281 | 282 | // Benchmark functions map 283 | static const char *bench_size_map[] = { 284 | "load", 285 | "non-temporal", 286 | "store", 287 | #ifdef AEP_SUPPORTED 288 | "store-flush" 289 | #endif 290 | }; 291 | 292 | static lfs_test_stridefunc_t lfs_stride_bw[] = { 293 | &stride_load, 294 | &stride_nt, 295 | &stride_store, 296 | #ifdef AEP_SUPPORTED 297 | &stride_storeclwb, 298 | #endif 299 | }; 300 | 301 | static lfs_test_randfunc_t lfs_size_bw[] = { 302 | &sizebw_load, 303 | &sizebw_nt, 304 | &sizebw_store, 305 | #ifdef AEP_SUPPORTED 306 | &sizebw_storeclwb, 307 | #endif 308 | }; 309 | 310 | static lfs_test_seq_t lfs_seq_bw[] = { 311 | &seq_load, 312 | &seq_nt, 313 | &seq_store, 314 | #ifdef AEP_SUPPORTED 315 | &seq_clwb 316 | #endif 317 | }; 318 | 319 | enum bench_size_tasks 320 | { 321 | bench_load, 322 | bench_store_nt, 323 | bench_store, 324 | #ifdef AEP_SUPPORTED 325 | bench_store_flush, 326 | #endif 327 | BENCH_SIZE_TASK_COUNT 328 | }; 329 | 330 | #endif 331 | -------------------------------------------------------------------------------- /src/kernel/lat.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "lattester.h" 20 | 21 | static const char* lattest_task_desc[] = { 22 | "Invalid", 23 | "Basic Latency Test", 24 | "Strided Latency Test ", 25 | "Strided Bandwidth Test", 26 | "Random Size-Bandwidth Test", 27 | "Overwrite Test", 28 | "Read-after-write Test", 29 | "Straight Write Test", 30 | "Writer Thread Test", 31 | "Flush/Fence BW Test", 32 | "APC Cache Probe Test", 33 | "iMC Probe Test", 34 | "Sequencial Test" 35 | }; 36 | 37 | static const char* lattest_task_desc_op[] = { 38 | "", 39 | "(not configurable)", 40 | "[access_size] [stride_size]", 41 | "[access_size] [stride_size] [delay] [parallel] [runtime] [global]", 42 | "[bwtest_bit] [access_size] [parallel] [runtime] [align_mode]", 43 | "[stride_size]", 44 | "[stride_size] [access_size]", 45 | "[start_addr] [size]", 46 | "[bwsize_bit] [parallel] [align_mode]", 47 | "[acccess_size] [fence_rate] [flush_rate] [op (1=random)]", 48 | "[probe_count], [strided_size]", 49 | "[probe_count]", 50 | "[access_size], [parallel]" 51 | }; 52 | 53 | struct latency_sbi *global_sbi = NULL; 54 | 55 | uint32_t *lfs_random_array = NULL; 56 | 57 | struct mutex latencyfs_lock; 58 | 59 | int support_clwb = 0; 60 | 61 | /* Module parameters handled by procfs */ 62 | /* 63 | int task = 0; 64 | int op = 0; 65 | int access_size = 64; 66 | int stride_size = 64; 67 | int threads = 1; 68 | int runtime = 60; 69 | 70 | module_param(task, int, 0444); 71 | MODULE_PARM_DESC(task, "Task"); 72 | 73 | module_param(op, int, 0444); 74 | MODULE_PARM_DESC(op, "Operation? 0=Load, 1=Store (require clwb), 2=StoreNT"); 75 | 76 | module_param(access_size, int, 0444); 77 | MODULE_PARM_DESC(access_size, "Access Size"); 78 | 79 | module_param(stride_size, int, 0444); 80 | MODULE_PARM_DESC(stride_size, "Strided Size"); 81 | 82 | module_param(threads, int, 0444); 83 | MODULE_PARM_DESC(threads, "#threads"); 84 | 85 | module_param(runtime, int, 0444); 86 | MODULE_PARM_DESC(runtime, "runtime"); 87 | */ 88 | 89 | static struct report_sbi *rep_sbi; 90 | 91 | #ifndef X86_FEATURE_CLFLUSHOPT 92 | #define X86_FEATURE_CLFLUSHOPT (9*32+23) /* CLFLUSHOPT instruction */ 93 | #endif 94 | #ifndef X86_FEATURE_CLWB 95 | #define X86_FEATURE_CLWB (9*32+24) /* CLWB instruction */ 96 | #endif 97 | 98 | #define PAGENR_2_ADDR (ctx, pagenr, dimm) \ 99 | ctx->addr + (dimm * pagenr << PAGE_SHIFT) 100 | 101 | 102 | static inline bool arch_has_clwb(void) 103 | { 104 | return static_cpu_has(X86_FEATURE_CLWB); 105 | } 106 | 107 | extern struct report_sbi* reportfs_get_sbi(void); 108 | 109 | int latencyfs_print_help(struct seq_file *seq, void *v) 110 | { 111 | int i; 112 | seq_printf(seq, "LATTester task= op= [options]\n"); 113 | #ifndef AEP_SUPPORTED 114 | seq_printf(seq, "!! Warning: clwb not supported on current platform.\n"); 115 | #endif 116 | seq_printf(seq, "Tasks:\n"); 117 | for (i = 1; i < TASK_COUNT; i++) 118 | seq_printf(seq, "\t%d: %s %s\n", i, lattest_task_desc[i], lattest_task_desc_op[i]); 119 | seq_printf(seq, "Options:\n"); 120 | seq_printf(seq, "\t [op|o]\tOperation: 0=Load, 1=Write(NT) 2=Write(clwb), 3=Write+Flush(clwb+flush) default=0\n"); 121 | seq_printf(seq, "\t [access_size|a] Access size (bytes), default=64\n"); 122 | seq_printf(seq, "\t [align_mode|l] Align mode: 0 = Global (anywhere), 1 = Per-Thread pool (default) 2 = Per-DIMM pool 3=Per-iMC pool\n"); 123 | seq_printf(seq, "\t 4 = Per-Channel Group, 5 = [NIOnly]: Global, 6 = [NIOnly]: Per-DIMM\n"); 124 | seq_printf(seq, "\t [stride_size|s] Stride size (bytes), default=64\n"); 125 | seq_printf(seq, "\t [delay|d] Delay (cycles), default=0\n"); 126 | seq_printf(seq, "\t [parallel|p] Concurrent kthreads, default=1\n"); 127 | seq_printf(seq, "\t [bwsize_bit|b] Aligned size (2^b bytes), default=6\n"); 128 | seq_printf(seq, "\t [runtime|t] Runtime (seconds), default=10\n"); 129 | seq_printf(seq, "\t [write_addr|w] For Straight Write: access location, default=0 \n"); 130 | seq_printf(seq, "\t [write_size|z] For Straight Write: access size, default=0 \n"); 131 | seq_printf(seq, "\t [fence_rate|f] Access size before issuing a sfence instruction \n"); 132 | seq_printf(seq, "\t [clwb_rate|c] Access size before issuing a clwb instruction \n"); 133 | return 0; 134 | } 135 | 136 | inline void latencyfs_cleanup(struct latency_sbi *sbi) 137 | { 138 | kfree(sbi->workers); 139 | kfree(sbi->timing); 140 | kfree(sbi->ctx); 141 | } 142 | 143 | inline void latencyfs_new_threads(struct latency_sbi *sbi, bench_func_t func) 144 | { 145 | int i; 146 | struct latencyfs_worker_ctx *ctx; 147 | int cnt = sbi->worker_cnt; 148 | uint64_t dimmgroup; 149 | char kthread_name[20]; 150 | 151 | if (sbi->align >= ALIGN_PERDIMM) { 152 | pr_info("Warning: PerDIMM/PerIMC/PerChannel alignment mode selected, it is only supported on certain tasks\n"); 153 | if (sbi->access_size > 4096) 154 | pr_info("Warning: PerDIMM alignment with 4K+ accesses will write across DIMMs\n"); 155 | if (sbi->worker_cnt > 6) 156 | pr_info("Warning: PerDIMM alignment with 6+ threads\n"); 157 | } 158 | 159 | sbi->workers = kmalloc(sizeof(struct task_struct *) * cnt, GFP_KERNEL); 160 | sbi->ctx = kmalloc(sizeof(struct latencyfs_worker_ctx) * cnt, GFP_KERNEL); 161 | sbi->timing = kmalloc(sizeof(struct latencyfs_timing) * cnt, GFP_KERNEL); 162 | for (i = 0; i < cnt; i++) { 163 | ctx = &sbi->ctx[i]; 164 | ctx->sbi = sbi; 165 | ctx->core_id = i; 166 | switch (sbi->align) { 167 | case ALIGN_GLOBAL: 168 | case ALIGN_PERIMC: 169 | case ALIGN_NI_GLOBAL: 170 | ctx->addr = (u8 *)sbi->virt_addr; 171 | break; 172 | case ALIGN_PERTHREAD: 173 | ctx->addr = (u8 *)sbi->virt_addr + (i * PERTHREAD_WORKSET); 174 | break; 175 | case ALIGN_PERDIMM: 176 | case ALIGN_PERCHANNELGROUP: 177 | dimmgroup = i / 6; 178 | ctx->addr = (u8 *)sbi->virt_addr + (dimmgroup * PERDIMMGROUP_WORKSET); 179 | break; 180 | case ALIGN_NI: 181 | ctx->addr = (u8 *)sbi->virt_addr + ((i % 6) *6* 4096); 182 | pr_info("dimm %d, addr %p", i, ctx->addr); 183 | break; 184 | default: 185 | pr_err("Undefined align mode %d\n", sbi->align); 186 | } 187 | sbi->timing[i].v = 0; 188 | ctx->seed_addr = (u8 *)sbi->rep->virt_addr + (i * PERTHREAD_WORKSET); 189 | sprintf(kthread_name, "lattester%d", ctx->core_id); 190 | sbi->workers[i] = kthread_create(func, (void *)ctx, kthread_name); 191 | kthread_bind(sbi->workers[i], ctx->core_id); 192 | } 193 | } 194 | 195 | void latencyfs_timer_callback(unsigned long data) 196 | { 197 | //TODO 198 | } 199 | 200 | inline void latencyfs_create_seed(struct latency_sbi *sbi) 201 | { 202 | latencyfs_prealloc_large(sbi->rep->virt_addr, sbi->worker_cnt * PERTHREAD_WORKSET, PERTHREAD_MASK); 203 | pr_info("Random buffer created."); 204 | drop_cache(); 205 | } 206 | 207 | inline void latencyfs_stop_threads(struct latency_sbi *sbi) 208 | { 209 | int i; 210 | int cnt = sbi->worker_cnt; 211 | 212 | for (i = 0; i < cnt; i++) { 213 | kthread_stop(sbi->workers[i]); 214 | } 215 | } 216 | 217 | 218 | inline void latencyfs_monitor_threads(struct latency_sbi *sbi) 219 | { 220 | int i; 221 | int elapsed = 0; 222 | int runtime = sbi->runtime; 223 | u64 last = 0, total; 224 | 225 | latencyfs_create_seed(sbi); 226 | 227 | /* Wake up workers */ 228 | for (i = 0; i < sbi->worker_cnt; i++) { 229 | wake_up_process(sbi->workers[i]); 230 | } 231 | 232 | /* Print result every second 233 | * Use a timer if accuracy is a concern 234 | */ 235 | while (elapsed < runtime) { 236 | msleep(1000); 237 | total = 0; 238 | for (i = 0; i < sbi->worker_cnt; i++) { 239 | total += sbi->timing[i].v; 240 | } 241 | printk(KERN_ALERT "%d\t%lld (%d)\n", elapsed + 1, (total - last) / MB, smp_processor_id()); 242 | 243 | last = total; 244 | elapsed ++; 245 | } 246 | latencyfs_stop_threads(sbi); 247 | } 248 | 249 | int setup_singlethread_op(struct latency_sbi *sbi, int op) 250 | { 251 | switch (op) { 252 | case TASK_BASIC_OP: // 1 253 | latencyfs_new_threads(sbi, &latency_job); 254 | break; 255 | case TASK_OVERWRITE: // 5 256 | latencyfs_new_threads(sbi, &overwrite_job); 257 | break; 258 | case TASK_READ_AFTER_WRITE: // 6 259 | latencyfs_new_threads(sbi, &read_after_write_job); 260 | break; 261 | case TASK_STRAIGHT_WRITE: // 7 262 | latencyfs_new_threads(sbi, &straight_write_job); 263 | break; 264 | case TASK_STRIDED_LAT: // 2 265 | latencyfs_new_threads(sbi, &strided_latjob); 266 | break; 267 | case TASK_CACHEPROBE: //10 268 | latencyfs_new_threads(sbi, &cacheprobe_job); 269 | break; 270 | 271 | default: 272 | pr_err("Single thread op %d not expected\n", op); 273 | return -EINVAL; 274 | } 275 | wake_up_process(sbi->workers[0]); 276 | return 0; 277 | } 278 | 279 | int setup_multithread_op(struct latency_sbi *sbi, int op) { 280 | unsigned long delta = 0; 281 | switch (op) { 282 | case TASK_STRIDED_BW: // 3 283 | latencyfs_new_threads(sbi, &strided_bwjob); 284 | break; 285 | case TASK_SIZE_BW: // 4 286 | //latencyfs_prealloc_global_permutation_array(sbi->access_size); 287 | delta = PERTHREAD_CHECKSTOP % sbi->access_size; 288 | latencyfs_new_threads(sbi, &sizebw_job); 289 | break; 290 | case TASK_THBIND: // 8 291 | latencyfs_new_threads(sbi, &align_bwjob); 292 | break; 293 | case TASK_FLUSHFENCE: // 9 294 | latencyfs_new_threads(sbi, &cachefence_bwjob); 295 | break; 296 | case TASK_IMCPROBE: //11 297 | latencyfs_new_threads(sbi, &imcprobe_job); 298 | break; 299 | case TASK_SEQ: //12 300 | latencyfs_new_threads(sbi, &seq_bwjob); 301 | break; 302 | 303 | default: 304 | pr_err("Single thread op %d not expected\n", op); 305 | return -EINVAL; 306 | } 307 | latencyfs_monitor_threads(sbi); 308 | return 0; 309 | } 310 | 311 | 312 | void latencyfs_start_task(struct latency_sbi *sbi, int task, int threads) 313 | { 314 | int i, ret = 0; 315 | 316 | switch (task) { 317 | case TASK_BASIC_OP: // 1 318 | case TASK_STRIDED_LAT: // 2 319 | case TASK_OVERWRITE: // 5 320 | case TASK_READ_AFTER_WRITE: //6 321 | case TASK_STRAIGHT_WRITE: // 7 322 | case TASK_CACHEPROBE: // 10 323 | sbi->worker_cnt = 1; 324 | ret = setup_singlethread_op(sbi, task); 325 | break; 326 | 327 | case TASK_STRIDED_BW: // 2 328 | case TASK_SIZE_BW: // 3 329 | case TASK_THBIND: // 8 330 | case TASK_FLUSHFENCE: // 9 331 | case TASK_IMCPROBE: // 11 332 | case TASK_SEQ: 333 | sbi->worker_cnt = threads; 334 | ret = setup_multithread_op(sbi, task); 335 | break; 336 | 337 | case TASK_INVALID: 338 | pr_err("Remount using task=, tasks:\n"); 339 | for (i = 1; i < TASK_COUNT; i++) { 340 | pr_err("%d: %s\n", i, lattest_task_desc[i]); 341 | } 342 | break; 343 | 344 | default: 345 | pr_err("Invalid Task\n"); 346 | return; 347 | } 348 | 349 | if (ret) { 350 | pr_info("Warning: start task returned %d", ret); 351 | } 352 | } 353 | 354 | 355 | static int latencyfs_fill_super(struct super_block *sb, void *data, int silent) 356 | { 357 | struct latency_sbi *sbi; 358 | void *virt_addr = NULL; 359 | struct inode *root; 360 | struct dax_device *dax_dev; 361 | pfn_t __pfn_t; 362 | long size; 363 | int ret = 0; 364 | 365 | if (rep_sbi) { 366 | pr_err("Already mounted\n"); 367 | return -EEXIST; 368 | } 369 | 370 | rep_sbi = reportfs_get_sbi(); 371 | 372 | if (!rep_sbi) { 373 | pr_err("Mount ReportFS first\n"); 374 | return -EINVAL; 375 | } 376 | 377 | sbi = kzalloc(sizeof(struct latency_sbi), GFP_KERNEL); 378 | if (!sbi) 379 | return -ENOMEM; 380 | sb->s_fs_info = sbi; 381 | sbi->sb = sb; 382 | sbi->rep = rep_sbi; 383 | 384 | #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 6) 385 | ret = bdev_dax_supported(sb, PAGE_SIZE); 386 | #else 387 | ret = bdev_dax_supported(sb->s_bdev, PAGE_SIZE); 388 | #endif 389 | pr_info("%s: dax_supported = %d; bdev->super=0x%p", 390 | __func__, ret, sb->s_bdev->bd_super); 391 | #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 6) 392 | if (ret) 393 | #else 394 | if (!ret) 395 | #endif 396 | { 397 | pr_err("device does not support DAX\n"); 398 | return ret; 399 | } 400 | 401 | sbi->s_bdev = sb->s_bdev; 402 | 403 | dax_dev = fs_dax_get_by_host(sb->s_bdev->bd_disk->disk_name); 404 | if (!dax_dev) 405 | { 406 | pr_err("Couldn't retrieve DAX device.\n"); 407 | return -EINVAL; 408 | } 409 | 410 | size = dax_direct_access(dax_dev, 0, LONG_MAX / PAGE_SIZE, 411 | &virt_addr, &__pfn_t) * PAGE_SIZE; 412 | if (size <= 0) 413 | { 414 | pr_err("direct_access failed\n"); 415 | return -EINVAL; 416 | } 417 | 418 | sbi->virt_addr = virt_addr; 419 | sbi->phys_addr = pfn_t_to_pfn(__pfn_t) << PAGE_SHIFT; 420 | sbi->initsize = size; 421 | 422 | pr_info("%s: dev %s, phys_addr 0x%llx, virt_addr %p, size %ld\n", 423 | __func__, sbi->s_bdev->bd_disk->disk_name, 424 | sbi->phys_addr, sbi->virt_addr, sbi->initsize); 425 | 426 | root = new_inode(sb); 427 | if (!root) 428 | { 429 | pr_err("root inode allocation failed\n"); 430 | return -ENOMEM; 431 | } 432 | 433 | root->i_ino = 0; 434 | root->i_sb = sb; 435 | #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 20, 0) 436 | root->i_atime = root->i_mtime = root->i_ctime = current_kernel_time(); 437 | #else 438 | root->i_atime = root->i_mtime = root->i_ctime = ktime_to_timespec64(ktime_get_real()); 439 | #endif 440 | inode_init_owner(root, NULL, S_IFDIR); 441 | 442 | sb->s_root = d_make_root(root); 443 | if (!sb->s_root) { 444 | pr_err("d_make_root failed\n"); 445 | return -ENOMEM; 446 | } 447 | 448 | global_sbi = sbi; 449 | 450 | #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 6) 451 | return ret; 452 | #else 453 | return !ret; 454 | #endif 455 | } 456 | 457 | static struct dentry *latencyfs_mount(struct file_system_type *fs_type, 458 | int flags, const char *dev_name, void *data) 459 | { 460 | return mount_bdev(fs_type, flags, dev_name, data, latencyfs_fill_super); 461 | } 462 | 463 | static struct file_system_type latencyfs_fs_type = { 464 | .owner = THIS_MODULE, 465 | .name = "LatencyFS", 466 | .mount = latencyfs_mount, 467 | .kill_sb = kill_block_super, 468 | }; 469 | 470 | static int __init init_latencyfs(void) 471 | { 472 | int rc = 0; 473 | struct file *fp; 474 | 475 | mutex_init(&latencyfs_lock); 476 | 477 | pr_info("%s: %d cpus online\n", __func__, num_online_cpus()); 478 | if (arch_has_clwb()) 479 | support_clwb = 1; 480 | 481 | pr_info("Arch new instructions support: CLWB %s\n", 482 | support_clwb ? "YES" : "NO"); 483 | 484 | fp = filp_open("/proc/lattester", O_RDONLY, 0); 485 | if (!PTR_ERR(fp)) 486 | { 487 | pr_err("LATTester running, exiting.\n"); 488 | return 0; 489 | } 490 | 491 | rc = latencyfs_proc_init(); 492 | if (rc) 493 | return rc; 494 | 495 | rc = register_filesystem(&latencyfs_fs_type); 496 | if (rc) 497 | return rc; 498 | 499 | return 0; 500 | } 501 | 502 | static void __exit exit_latencyfs(void) 503 | { 504 | if (lfs_random_array) 505 | kfree(lfs_random_array); 506 | remove_proc_entry("lattester", NULL); 507 | unregister_filesystem(&latencyfs_fs_type); 508 | } 509 | 510 | MODULE_LICENSE("GPL"); 511 | 512 | module_init(init_latencyfs) 513 | module_exit(exit_latencyfs) 514 | -------------------------------------------------------------------------------- /graphs/08-store.csv: -------------------------------------------------------------------------------- 1 | 628201170d11c490f236ad7e1801c9373c72e314,0.4674768412985938,896.0 2 | 37bc5d95e0cc8b40944b964b0acf1c1e30d7cb32,0.22182766840493057,588.8 3 | f8ff13216ed0000fa0a8585f82e2f5b60e49432c,0.2530880332428538,576.0 4 | f19e5825e112a73baff00ce6c504f6dd8d43a315,0.24983756862590012,563.2 5 | 5ba9e5675a27dbf261685fb0f5df897256db745c,0.255758485299334,537.6 6 | 595fc0394f8f328f6afe25d35730696106886171,0.26694329450823284,512.0 7 | a3b4979bf8c76ec08c96ee96d5c474c8862aac89,0.29317682845913073,601.6 8 | ca4d91c4a667ebbdf01e44b2bc855ecec0439819,0.2376997641166795,563.2 9 | 5dcae28b72d618dfce07a43a1625d7e9332a3068,0.2665024679532239,537.6 10 | 4a5bad661ecf726333c4c8a32337343b47717ea0,0.23010426111707774,512.0 11 | 4623de91d13d0fa5e4ae610d1f3276bfa488bcc5,0.25067100581070884,473.6 12 | 4bd55ff8d0279899749a1e87263369200d1853f6,0.26176890930298496,512.0 13 | 678ff87d1d420cd5859dde379c649add3fbee164,0.2633104632635858,537.6 14 | 382c17ad56c242d059ebd51db5a2beae4077aed0,0.2170263738928509,512.0 15 | 35207bb501ae619e7f54a2d71364c47f81f58c53,0.25286455538319214,499.2 16 | 5987ce5474d70e2d0f1b091a440bccc59cff9c1a,0.23706321942363165,512.0 17 | 2d9e2c354f5c9d89375433d4c2d7edeb782e79bc,0.252948297306144,460.8 18 | ba62c5e73f8a9b88145679a0ce7a851d76122d65,0.24594161842841128,409.6 19 | 6cdbd838dd6710d843c9b00fd0c135248054d42a,0.2522804593883677,486.4 20 | 3800ca79bfde5cb8d414a6289dea3b43e4f583ba,0.2087173258906092,460.8 21 | e2b075b27c3e864c05a244a21423d3d3a86ebe91,0.24592007008855513,460.8 22 | 33e5aa2b50d0a735005a6fbee966dc9756c98eac,0.21461081131636447,460.8 23 | be4ed1af71686e8f3bd4795de04824ffce7628c8,0.24664774690266236,396.8 24 | 8b1831daad9e29daad31105f94047e0f4444ab46,0.24764834808418365,409.6 25 | ba4c1648ddfe2d5c43f669822e6b8cc32dcd9376,0.48030165749636455,921.6 26 | 9dd62757f26262f1511318cf8dc8fa7c3b8f2025,0.21094825107938991,588.8 27 | 05c229855889b0f22f756b8059a09e7ccf5ba031,0.25369470139886235,576.0 28 | 899be8f32d621928eee851f801d0e3991dc998b3,0.22583177659307238,563.2 29 | 3d59a4ee340c4e000f3ed2cedba881026c331f10,0.26929354775878284,537.6 30 | 810c2329a231b8761e81821b76fcbc9980df98bf,0.2675553443093565,512.0 31 | 958b793f45984bd80a735182c2501fa4311e4381,0.25786984915339717,460.8 32 | 52e4fb0fcf10c9b22afa8adc79f69f533b4090c1,0.2517162989168483,451.2 33 | 39b09bd2e094552d4d39ea212c56759a90535f96,0.25196659775017943,451.2 34 | dad270f65cf18a10e3ccc217fe53c7b033b30d86,0.25128930603033944,460.8 35 | f20d8f08ae02c830d5974ef5c1953fe14aa1ec0c,0.2065778572489771,284.8 36 | 258d6f9744a63d069e8eed4f1334042c5c10e35e,0.24620093170076382,204.8 37 | 4fa2b8b98605294bb5cdbe9eb2d749e6f94c0eeb,0.2549661521239616,524.8 38 | 25dca42e2c144122eb88ffad87aafd46f914c69f,0.19782789244097987,512.0 39 | 62003f86e0596f0abfbce88d66f937236f8b0f62,0.2355434708985269,499.2 40 | b87ed29fe0a9f0297b0a79fa9476eef53f2e3dfd,0.2508019401465599,512.0 41 | 18ce479794b88e8211d7b611974cb86cba7aa9b7,0.23868207786216844,460.8 42 | c99ae187563fb1d808b56d8157f344b5ebb1da52,0.24172332425503965,512.0 43 | 0c4f5890d48b33240b2a8f57fdef96559d372213,0.2594957395624578,512.0 44 | 3486f68da841994de56ed4d72dd9c885980f1d08,0.21153019114913016,486.4 45 | 1d9aea38d203fd989ae164045f6baf966005af7a,0.252525384544179,499.2 46 | 94fbae1e0b9940d6db8351743ec417dfcd6ed615,0.25162673751910214,460.8 47 | ae8e1d37ebaeb49b14363f78d41a4ffa586ee937,0.25190820699384636,460.8 48 | eb893fe5358221f065d8791439a3ac1d4ebf02b4,0.25218424307141885,409.6 49 | 0ab11553b701db2f1435a6fa33a9481444747c6f,0.26091638814159857,460.8 50 | 69421e0809214973332bb9c2e8b378612e14bce3,0.22473364425538353,435.2 51 | c5b3f97a4c666bbb3b9b6e169bad130b2960cb9a,0.2510480412087677,422.4 52 | a6fda65808f0561eb0f11342d791c044681f7711,0.24709213000063093,358.4 53 | 39dba0b006c610f8c56c28ed4f1807ef25f67098,0.24601812767806977,204.8 54 | 67351c3d2c1884869e363b641ca4293966e93a3a,0.23979086629534604,102.4 55 | 66ef2a7fcb424d85621d7c9ea288c7726beb478b,0.2514605744914822,486.4 56 | 29732662cbff6681bfdf5e5dbf08e276a7e28cd9,0.20451010468520017,486.4 57 | 9eb78e41c6f1c9578dbdf240dfdb26f0e3df9bbe,0.246705473719031,460.8 58 | 386da4c81579395fbf9965d062a0e786702cc25d,0.24650369918620263,460.8 59 | cd34a18aa25607ef11c8eb9322501393bbfba2d8,0.2477141732360791,460.8 60 | 941475857faf13ee24b6a6a5ea08f00f22bb50f8,0.24025783459028843,409.6 61 | 0f47a11a4551f9189453da64580f8e337784bfe9,0.4749640913386113,921.6 62 | 3da7c013a221f6e443c9d59f474adb49bad829c7,0.2125043469957851,588.8 63 | bd93f2b2dedc9fe4d7d098d609c8d229a57f6123,0.27219935276939333,576.0 64 | 57d40c2b70e2a397bc2a163e21da4a4c8382b553,0.2721717983934928,563.2 65 | 848485bef64fa048c6ca05fa14492aaa79b18609,0.26888292545075293,537.6 66 | bf01fa10f9defbfff3fe1bc798c345648c9d84d8,0.265662563376179,512.0 67 | d757bdbf61973a24c2fa14b02d85b507a41cba0d,0.2568700890299995,511.0 68 | f6e79fbfb9bd5807c9a49ad76b21f84305d44132,0.25087775789286837,498.2 69 | 03d78588572152afabfc17752c04037394ea18f3,0.23692521331011518,498.2 70 | 0d227f84320d800817540f08e2702ad0a138add7,0.24746407311855168,459.8 71 | 7be4d49dc84e5e5315e3e90d8876f318f2af94a7,0.23815828192661348,459.8 72 | 322b82dba101a4adddb19209ff83a97573ced23b,0.2416911898252631,408.8 73 | 3440cfd1f1888f3d4221bb096672779493fced68,0.3017644200450908,552.0 74 | 7b6b607449a7fd2e3c7d7f3f73e9a3b8b9130396,0.18517431995198955,513.6 75 | 87b93db8ad8308b76aab2e58af2aa2ca9e98a77c,0.2741750371908749,499.2 76 | 4202be19416b026b966b48599acfab5279f704c2,0.22770160525001507,494.4 77 | eb822ed46a22795508b480ff17f82e09297e2803,0.23034742729001653,364.8 78 | 00a187206f0e0619454917067ec718d040ee2481,0.2580723256617433,264.0 79 | 84bdfd695749ce4d75d52b29b7712c01ff899212,0.4865902695752112,933.4 80 | 335789e6b47e4a316cb4e3cdb55d5f121c7fcf0c,0.2139962692286652,587.8 81 | 6fe66285357c2c6832ffd3e33c07c93cfa81384e,0.27301410455389497,575.0 82 | 92604b66bba0c6a19489b3d24559e8ec50b37b2a,0.2731170463783209,562.2 83 | c0672321cc3ae64e0d79d71683113910060494cb,0.254963712794813,536.6 84 | 3933bfde78ca3d388ff1fbec478f69e5001e47dd,0.26749319530592447,511.0 85 | 3302128d562fe58ba7f6b529a5e6b9b999eeea2e,0.2837314110452261,575.0 86 | 2dafbf286376dbe0cc91e6572f752e35f85d00c1,0.2686904483063708,536.6 87 | d7d05f47d5a37c9505c589129ca63e18e4b707cd,0.24947311113462087,536.6 88 | f1bab2ade80e8a789296f735b9e907e44dd5e73d,0.26372644737207385,511.0 89 | d4ea4db7b65f054354fa95c76774f303fd5e121b,0.23598143487009837,523.8 90 | 5cf6b808c1b04143263059fcae35eecbe5abb4f0,0.2524907676968214,511.0 91 | e99b5e6e3594649b98a363c302dd168ffb67fa6f,0.2795359190273089,511.0 92 | dfd22454426b603ccd41ecd2ecc3c1161d3e4650,0.24062390874977133,485.4 93 | c69070dee0ed1dc585f8ce68fd1a44c02cfa465a,0.25686946500806096,459.8 94 | 77fbb16d034ed26cb205becd9262d0f51ef71759,0.2529989918974921,408.6 95 | 404173d622af69ee9c754dd5489f5648901597b5,0.2501367019198092,229.6 96 | 9c34fdd37a21b1baef7514e1323e58d1f63c7180,0.24417725204645416,102.2 97 | 465ab078f7a5cbaec36d6281e7b501aba16e9fd4,0.31986395372740756,626.2 98 | 08d25b81285568922d9059630f7e75b29d7b5dde,0.2234526584237037,536.6 99 | 7081a3405034bda63759e58b1884946eea1f3e55,0.2622718887630283,536.6 100 | 7a7fe8528ab6b2415ef78665388dd849111b6e36,0.2598320437602109,511.0 101 | 96a0bb5834432738bf9d9b052ab77d66425e1064,0.23364981447431976,472.6 102 | 4282306b6dbb405ba200b875a90449f0a66a5e8c,0.2591460031139658,511.0 103 | dd31fd94c9ce52ee9ea55c3567d9f8eaed7f311d,0.5589953795939242,1011.2 104 | 3033a24942f83eec099cc1646d01948859406da4,0.24381353901324057,588.8 105 | 5821dd51cc1be5b11e8829e556bfaac20460cfe0,0.2814012112237636,576.0 106 | afa4cbaca21de0d041953d6e688b02ff50ba97a5,0.27522900336724215,563.2 107 | f3f3e49c48e6cb9dc3687b052c94a7e01b713a76,0.25646602235883376,537.6 108 | fafc5ca575747e886984ab216a56e1218c13f64a,0.26894173464432886,512.0 109 | 9158a4827269a7e9527ea6f2e7e6f289cbcdd4ce,0.49930963601002,947.2 110 | 1f6e30153962a5daff8be928458afbc73b870157,0.21795464695388114,563.2 111 | a9ac3fc20bb40828325005eed07a8f7506344d43,0.27162348821682614,563.2 112 | 1d84187e59d5999a766dcf41f43fee1e909da3ba,0.2700416958693141,563.2 113 | 1cfbbd9ea795e8432437071dd5f11620e84e7977,0.23914652251926277,537.6 114 | 9afbee40603ee377d96460714888b120319b0a45,0.25463256650891875,512.0 115 | a16b8d2d073cbd9a78f5375e6f3fd3a551a005b5,0.2650573225289587,550.4 116 | f46ad6af7dfdc7a4ad47c3938c9b14a52d2606eb,0.2176977521390651,512.0 117 | 003e064ad196362cb14a4f59f79e585d1ca5b04a,0.24119066709001885,499.2 118 | 2fa8df50b03e5bb1bff7fc2e9101b9669d5c364d,0.23789488523244687,512.0 119 | 31521dd41ea3a277a6bd92df8c9e557e965335e0,0.24305624963503444,460.8 120 | c2abafe38f677ec63b36a74160dba5bc4d8e59bb,0.2468188030864927,409.6 121 | 02e9f4e7fc41a4fd8f57d888f8bbf10523c5d8ec,0.4513316983004081,787.2 122 | 86744171f655acdb99ee51ca4fbb30be520dbecb,0.3067379306377101,576.0 123 | f933b3df035e2cc7aec5c3abe40b47a6a85bcd4e,0.23744418323736044,550.4 124 | 5f0dc369bca15742d789edaf2f70fb819dc7fab2,0.28320522612632004,512.0 125 | d35987c1c150243e476cf9ee2682ddf571a57a51,0.24867460097711339,416.0 126 | f1c72806c8a4bd15d1221b7944c84470b295631a,0.25035371019124986,307.2 127 | 061bbe39feba48bb9c12b891d63fd433b053e10e,0.4837668699967239,934.4 128 | d0ef772110d1ad3dbe2d4b3344938f9c4c90b02b,0.21255568373750383,588.8 129 | 4fbe81c673910545805c186578527ce56e468da0,0.2558645359352851,576.0 130 | 44cb6c27bbaca2974c47a6ff1485ed97eecec445,0.27276848115434715,563.2 131 | af59935aaa0934f7fa76d22a79304a8922404d17,0.25460898985857233,537.6 132 | dea2d79b3a99ecd093cc2fade99cb8bb742e3aec,0.2694536076047477,512.0 133 | 2d5b93062f03d6c5716d5acb1b442a4ab054744b,0.2996358813606819,576.0 134 | 712ff92a1c424552418dbc0e9ec8f01e2db1b8f3,0.2559601733351168,524.8 135 | a5e6b08547ca6dcbae5a5029ac3ba4680cfbc21b,0.2657537943547798,499.2 136 | ad30a2873be3d6d997693861fcf817a31a44b8fa,0.2537758416470117,422.4 137 | 4d9a37bb610e43696370ade122972879177bf4b1,0.23128928792884235,230.4 138 | 2f00af85ff8f0ee949626cacdec741144cf979ff,0.24449315661673832,204.8 139 | 7c0cb9fa72cd6ef1c8c74d2675d3754f9011cd33,0.4985814651361702,934.4 140 | 2a5bacddbc4ce92ac9328c9434740787dd25ff06,0.27959698304399705,563.2 141 | 7354a028abcc4ce381c1ba1e47d53ba2e6696f9e,0.23692176031352336,537.6 142 | f5c61c08d658d83bbbb772bfa1fc5ba4f774da79,0.27098759015656,563.2 143 | 5bc4b8e3c7f25ae221d5f379d3ad1e5b8a27aacd,0.23886256385361596,537.6 144 | 71fc1ced5185599cae92f91036b6c781f64a863b,0.25437782563929046,512.0 145 | 627748f94367308af434194606fef708150cfc11,0.5404169441468192,1011.2 146 | 34c4585596318b5d9a4a5788d2a533faad09b872,0.23049930461950624,563.2 147 | 4f7d75136f970ccb381b71b18b3ab6d1dd0d714d,0.23642267154493532,537.6 148 | 1a8122dd9ee4336deb648aada07e009af8a30a4f,0.2492580067697125,563.2 149 | c04be9ebb92353a5708a26b6ad60fae6245eed28,0.2682727546987533,537.6 150 | 9234d8a19bd2949df3e1295fbaf14c70b4a6fc1f,0.26591048698409164,512.0 151 | c9ed253e028af9e36e0845b420e18edea7de503b,0.47194172941384355,908.8 152 | 685b38c48be7173de6dfc08a20aba0ea282d6ded,0.21243914574485825,588.8 153 | 5ea83292708311ccf1b1b3410db1fcdabfcb4ab5,0.2539338292754569,576.0 154 | 49421d47179fe6733011dc5a885ddb65e99bd052,0.27101011872936237,563.2 155 | ed7334ec2dc3868cc1a155ec33b789e73309f16e,0.25448506052405684,537.6 156 | 35d331f0a2ee8c4e4912cd1b3a962581476e87ea,0.2688413315698562,512.0 157 | a0a6db2ca0ca2c3b1792fa9e88b82d3d9d7abf32,0.5030697545851168,947.2 158 | 9a5e106fd5bd7b0a3f2b861670e2d1c3431a8afb,0.22872370022604765,576.0 159 | 04b0640d21ed6559221ba6bcaca8a19113592802,0.2773376105815715,576.0 160 | 99e27a7147902ba1a8a8c248b1da874a0aa11988,0.2742222106680211,563.2 161 | 9be80fe04edf4da4659a6938a5e11d0e24eacd07,0.23951270933596638,537.6 162 | 918d9f621b9a87412d2e085ceb6cc09b6b3043e8,0.2563716071551641,512.0 163 | 112bd80b0c05b7e5a1f36ee466520c2625f39bcf,0.300107631035548,588.8 164 | 2c9bb9be4c893082ef43de3fda68d1a8233eb941,0.24884837859035583,537.6 165 | a20ea6837754a30fd388a438958a3e058c5725f4,0.2728534741871369,537.6 166 | 85b27cab64721f1a30f0815e728751c13fae6f2b,0.27103102815632973,512.0 167 | d88154070de7dcba6521fa47627d04884d519cfe,0.26242608294489234,460.8 168 | f9fd8f6f23ffa873bcf6e6fea3925034baea37ca,0.26320803046181723,409.6 169 | c90e1af2309c4dc0c38ba5076a48c3e12218fca8,0.26503264098853346,537.6 170 | 5a6d1c4a23971edfe02f7368141df7867959a84c,0.22771404221257802,512.0 171 | 50c5082e34c40dc9a7fbe1bbaf30214c48444807,0.24204777990898552,499.2 172 | e15ea3253723fde282a773e02891454b1eda9d23,0.24886336319457852,460.8 173 | 81db9463ae254c7294d038ec2003a01d828d377f,0.24161430075861867,384.0 174 | cbbe81d82b2b92752cb8d92f907b2626da4adf87,0.2450349087353001,307.2 175 | 1373db2d3953bac158ce41f6ffe96841b3383f60,0.48493079063702393,947.2 176 | 575c62d8cf24f11e1396e819b9d94668eb8c0c03,0.22385756866474055,588.8 177 | 2275f084c0f771a6e2c127a766d600287f5a49d8,0.27485357439778874,576.0 178 | 1278b794c4edba31499389be3bd349b761c647aa,0.2704063009741653,563.2 179 | 9efcf1ebebeb9d8ec0c9745346f4e89f905dcb70,0.2546229424128441,537.6 180 | dc1faf51ac363764e174ff90d53c9ad25a578f36,0.256668508822025,512.0 181 | 0d1e8a700a357619c60e3b866dd23c9c398e424f,0.24427095241561242,422.4 182 | 447d51472bfee5322b8b404f196bd8295448c043,0.20718975298863856,409.6 183 | 76ef2cab7c61eaf3ada9811f2d12e971683f2003,0.21465415234116955,422.4 184 | 00347e2de9102b4c8ea83436e4c0925a76e74529,0.24268330897604717,409.6 185 | a9fc6a411dad6ac74b9515757f1082763f3df311,0.2335888999506269,384.0 186 | 307dbd15b9db6651dce064ee258f97aff9104814,0.2443184621704935,307.2 187 | 78a081b2c53d0d92ae3a15ed545965c3437b76fd,0.26538419695362725,524.8 188 | bcd821a4da08d9938428250d650a60b461f1ab24,0.21332235425317403,499.2 189 | 633dc55a9fc0bcbcdf0ac198efd6d475b7e9ceca,0.23906420148465365,499.2 190 | ff18007721968867e59164fce8297050c213ae99,0.2544650406625127,486.4 191 | d48ead57165a4564b60565f68f443301f7869beb,0.2414548099720488,460.8 192 | 88db4e28540364985522e705e97fe184aa308813,0.24445261898937085,409.6 193 | 7d063352a1fed64f7c3c14cb297583525c91173a,0.23555466998089042,384.0 194 | 70e024e67a1bb442ee0e88582f3a557144fd36bb,0.23627610076712877,382.4 195 | 2261eadc3c4c1273924b9f8e97b30b5c20acee8e,0.23468488589546121,382.4 196 | 9236b3653d1856326f373fce6d8a9d743a52e7ae,0.23694576326900182,395.2 197 | 757ef659b68d7b3094924635d627794b56719897,0.17323110491300492,256.0 198 | b57f64618ccb0b34a05299b8e9ec14ac0b818d81,0.2361318294397831,206.4 199 | f5c9425b196d2028a34c1ee868ac3a2cd3932e16,0.2525651578990357,473.6 200 | a38f29fc92b62f4aa256b5c16be3dcb51a936d62,0.20335860356177182,460.8 201 | 891e860982473a101bec60c002ffe9e8dd7a6d73,0.24811124278128674,460.8 202 | 6e358237499f52f6fd9d2ed9b6a225f49d426e9e,0.2482113992293257,460.8 203 | d43895152fe07c75040787e9da6f0c316aa0caf9,0.2365631291189663,460.8 204 | 26255dfd25a2d21ba65827599fe016f2ff619ad1,0.239684683022638,409.6 205 | f0b4847475d2b642a7bc689fe4c23a00457ba0be,0.24490656582635664,435.2 206 | a932a339b85f948cdd6645bfb6ca6f145dd9bc0c,0.2015776543408393,435.2 207 | d636b7c99828127021972388742251494aea8b98,0.24318319943344666,422.4 208 | 9218eaf1876b799a50249231513da9ea3696625e,0.22822358608091967,409.6 209 | 1e761747c1921b8944002851556b2432aecb3e77,0.2337326740173291,384.0 210 | afeaf6304645922826dfdaf17429d9ff45c871ee,0.2371421179615666,409.6 211 | c149065db16d3ce87515d4f21fe1644d23c8df37,0.2429965627595917,422.4 212 | 6a79381cbe8b273c1a6af458d05c807ed828e664,0.17410227931682862,390.4 213 | 989b7d648699a5dc6c8ae80e9c94151cd1547b2e,0.1929554925553976,390.4 214 | 46ad8543902acf1b3e7f9b1917d9dc7d61de6d71,0.1862960169610279,326.4 215 | e13e792412842c128699e705dc8b45dc03194586,0.23723757928770636,192.0 216 | 0136c19d159f4bb848fcce84c6299cb085b23202,0.2097438856394434,153.6 217 | 2f1d0712bb83eb04e8e0e796fbd44b810c33b6f6,0.24475957355058794,422.4 218 | d5c4815cd34e3770c739cc9cda391ac93bb75889,0.20208064226163275,422.4 219 | d5ccc637bc9a41d552e9ca397bdafbe30c775a9b,0.2430156158745932,422.4 220 | 5335ce6834c7a46775b8dac630b8ff83f7d892f7,0.22754554045179884,409.6 221 | 2bb0f4440cbd7e52751be1a74641dee99dbf8e07,0.22102867196678364,384.0 222 | c1e80c0e5e3a72767d3ebf320384fcfb0f73bbd5,0.23681444112916047,409.6 223 | d125781dcf6d31e0901cc714c06cea5ba31dbedf,0.4852809575293893,921.6 224 | 07d202d206bc7a68a5dd04243881842d407c76b7,0.22371889733129208,588.8 225 | 51d5baca3f874782d8401501589f1710bbf254c0,0.2543464964916706,576.0 226 | 102caeb35ba978190b841ab8c668ee471375269b,0.227380026331344,563.2 227 | 3c7dcb09924aaa37c4d3819410c2d9332c744a2a,0.2715300526645588,537.6 228 | f2d49c842452ed1db11fc4854f3cd886e7ec1ded,0.27080892310632515,512.0 229 | 2be621e0b2a3e621ff378bb40a276206da288ee9,0.5053160354358195,960.0 230 | 29cdf632f030e22da1b3ab2cb96ad3ef801271e3,0.22033175681390038,576.0 231 | a5ae25c575ae622b317ab627e6001971f008535e,0.2720737551761796,576.0 232 | d77c14c6d345bbb2cad98320a357a93312027514,0.24964176446480396,563.2 233 | f978edf3eb3337c846913ff8102224668ea43bbf,0.25380350460498846,537.6 234 | c704c09547ad1580e0cb9172b560942092baff0a,0.2556967561335763,512.0 235 | 161de0ce168c4b0f3b28f28acd19f7244d2c7907,0.2586590865244379,550.4 236 | 1700851b84a828ea27037064a8eaa74ae0f92ea4,0.21625315185774777,537.6 237 | 98e58c2221e943ee22a376e04100cc561cbd2c7e,0.25669224403351437,537.6 238 | 76c52de73db10ad3a547b4f2dfb75b8f0212578f,0.24117449869493732,512.0 239 | 93e13a9d64700e579f1da37e6aaa9c416c603420,0.2583543346808218,537.6 240 | da5104029ce51308c5a433ca2ce01ee8248462bf,0.25879646403196654,512.0 241 | 4af9e275d4aca413d387de384e8a55420de0be91,0.2584247980864998,563.2 242 | 45be1c74cd32c730fa415bd846fd531b5f91f3b6,0.2163345084644121,537.6 243 | c79dde251813c444310ae10e8f11a1a000648898,0.25690488282786417,537.6 244 | 8c692a1cdd2c026b4a3b661be60b11f2a1b55f44,0.25766100010177634,512.0 245 | 24f4343bf74f8749d722dea569f87d12fcb423d3,0.25914203236902367,537.6 246 | 26ea2c2d2e947b4bbbd0846289bf585da84f7328,0.2591436480815445,512.0 247 | 5384f9e75f5e82de0417167606b182b3b4f63b16,0.2595556796515839,563.2 248 | 108025dedb79102c9481cdf5342b124e4d44d93f,0.21551247552652592,550.4 249 | 19d39d0c95a3f2df854b56a9861418c94bcffa65,0.25682586581440675,537.6 250 | cb38a7a4143fe52f7976554e4e9adac54aa0a2e7,0.25715669669477675,512.0 251 | e43b327147a53dd9d6f64003a7e8e0411e46bffc,0.2594174429698347,537.6 252 | a082ed170a2ef233e32d027a51d48d848b8344d1,0.25240016281845934,512.0 253 | 13aeb9e4cd716444bedfe1d20a4e6d47dc70e7d2,0.2611268469633141,563.2 254 | fb696c19a8ef5288f8393ef46e12451a59ac208a,0.21637937208891117,537.6 255 | c02f7101dd7e2bae2173fea48926b56d2088301b,0.25675068124989914,537.6 256 | 77495ed4eaf790c53d37ad2bfba49bb0572d6a66,0.25738434863002835,512.0 257 | 6f7ae8f69ab79e0814570bde54af4bdd44771ed4,0.25902223686165093,537.6 258 | 978b222fa661988958ab0c4725bfd064d58240f0,0.25963737999101155,512.0 259 | 2962f31067d8deeb4e2667da3f4bc0dbbccf53b6,0.26542400412436695,576.0 260 | 48c081b232253be257dd0973eebc4bc2b9ccaf28,0.2563798711950395,537.6 261 | b29154e5ba49ac0969be02be4f0fa780a21d6aac,0.2562647847514633,537.6 262 | cc6508c0822b18f0c4e452db23a2f85079ce6da1,0.25678006944516046,512.0 263 | 7faf1fe4d4179efb132cae9e4d7ce620fb61d3ef,0.23387146350459695,537.6 264 | 23616aa59f7188d84bd224e602c0b6c8cc72b4f4,0.25262791965539527,512.0 265 | a901c34c9449693a0be54fa8ba109e10ff7d2b22,0.2643255449813132,576.0 266 | 2f54a31ec06a43bc0f214c2034435461f9e6ca2c,0.21383460530799564,550.4 267 | 7c34575e12374ea0b6bb350249433819f869b378,0.25740684908831873,537.6 268 | db29fd5a18e003fe7b9caf240da8c6da49aa39c6,0.25792100072360447,512.0 269 | 3e4f434dd299b900b7156c3240d012ff51f9de13,0.24920443572479098,537.6 270 | a463cb4a6949efeb0e947a078d99798396b4f262,0.2531301436081932,512.0 271 | 11eecdcd9e71b31b9f11e7622a7815d82684512a,0.2685495994608061,576.0 272 | dfe2e987d7d3a9a2c171e681b4a41b0678e13e64,0.2598109460422807,563.2 273 | cc630bedb6a08cdec78cef7bca0b2d38d0f98f42,0.2601072415725271,537.6 274 | 129d520e2958bd424b764f9688eb55d60181d040,0.25950080588105434,550.4 275 | 3c503182726b2589c73c272089b720b46b893d47,0.26103743069713664,537.6 276 | 4079f5e2c310e228bdbce79f17a5c40ca84c2c8f,0.2610967523151043,512.0 277 | c4c9542ec76eec6877e8695c0fa811cf8357c0ee,0.27277345142772813,588.8 278 | 31f3f4f20d96fe6d4325bb465aaeecd2e6e8a39d,0.26149479238043033,563.2 279 | a5514cdd3a1f959d5c59700e5089dd8def55c59c,0.2612414420586473,537.6 280 | 9a38d0b1417334c21437f59ba64be8cb5701a1ba,0.2236130751778137,550.4 281 | c65d9ad363aa0d9e616e61919384030b12168000,0.2614307380378259,537.6 282 | 96c5f294aa7b5739081db019235f3be738d02a00,0.25373225023651913,512.0 283 | 010bb01ce42f20d7721a928a8b6524a5d97bb8bf,0.2783895315540885,588.8 284 | 99d3b27109bf327925836aa3091fc82c5e7edbbb,0.2643459282950886,563.2 285 | 8995fab39a9987605af6a642dd8be0279cd398c2,0.262720576647573,550.4 286 | efd3cb7f6823765bf88c43b88b4c9ca64307519b,0.24601258147156235,563.2 287 | 9a529fde5b4871fca19e9b89309b2b0c0f148f93,0.25106998384845214,537.6 288 | cbd83c00ae6f364ec456d0d785cbf91c96d0f003,0.2536883155641772,512.0 289 | f37663247200c1757c87ee87c4e903bceafcba7b,0.2933798498350055,614.4 290 | 0df915b01a21316c83cf0a2e8c6a59e77814433e,0.26609780237580094,563.2 291 | 686d047a742120a2d73da8781c65a5492094bf13,0.2636797760854397,537.6 292 | aabf704626c24b6d72e5ca18b6a3def0151f3bd2,0.24586470966836765,563.2 293 | 9b227d5f99532bc97214b877ffda5584e8aec9a3,0.26271303309882243,537.6 294 | c77274cf6c5ac12cd3ca46aef5ada9177e5626ee,0.26169595360511183,512.0 295 | abf853bb5b526c25d02eea24435fb58fa63c8572,0.30989592270360955,652.8 296 | 1bcde8fe2ecc9c1d8af2d0762b38e26f99abb810,0.2672320743829817,563.2 297 | d56a4e9966b98c98c9df1dd7192676ac0732772d,0.2641567747846402,537.6 298 | 919717ddaa8584a116282b0b8eb23eb2a4bb7a29,0.2467608385449762,563.2 299 | c4dbdf4abc2995cb58a5a5866a01e26bcdd631ab,0.2523084879195905,537.6 300 | 3fb98931ad01e59d29a0d04f772743ecff5b903d,0.2628546178908632,512.0 301 | 5b785194242170006679c9a5aac211e9fc1741c1,0.32736152940022756,678.4 302 | a1f03e0ce648bb9d4c3cfb719f8b769e3088ca51,0.2675688251213649,563.2 303 | 50d3db3a62ba53ddef8f57f669f6c41ccc46c1fb,0.26569974004982716,563.2 304 | 4c567d6e7a8b5a2ff5138931f0926eaa42710f8c,0.24774125545777548,550.4 305 | fc604bd429327b7607cc747abfa4aa04e3d10adf,0.25270807545042145,537.6 306 | 7f0441827ffc4012a81e7d500b85a8372dc5786e,0.25465722730879015,512.0 307 | 47058534b622ab130df4f9b6b05a68bb31d90216,0.24545742906183682,473.6 308 | 7758888d1e7a3f1cb8f3c46f0b554868f2c9d35b,0.24394743057688423,473.6 309 | a55aaded761a37e8ed5f3015528ab67ecf99ae4e,0.24404642796992473,460.8 310 | 4ae6093f9371170e9381ab7946c65396600a45a2,0.24418840965451816,460.8 311 | 2da02c6cf8d9297f6618ed4b8da05877648f372b,0.23411304853423784,460.8 312 | 7e8721f7b18bbcc65ae7e8e1d8e4202ee3e959f3,0.2449464368437493,409.6 313 | 02b16500a2943995094ad7446394e968aaf139e9,0.24600230333484077,422.4 314 | 80c915b13e8e41586599bc9c5ce218b3227571fc,0.2443276423099727,486.4 315 | 0d52c0b3f09c677aed40759ac4a7d94a44bbfde5,0.24428681455926696,460.8 316 | 4e63c39bf394b465a261bc087fb94cd15c799826,0.2445920427101091,460.8 317 | 08db92988715df2d98bb3c61d05d2669ee3f14c0,0.22254630136301662,460.8 318 | 15e26a5e6174c08d02719c027815f68de2041b55,0.23812016611141723,409.6 319 | 1526da0aa0915b33170e5993c234c3ad6d5c0a0d,0.27751414635951266,563.2 320 | 79c5f5bab816786e0abf9107a410ae16bbf9a5c5,0.25479242821485276,537.6 321 | 8266ad830a45c0e12477ec74f6ed0851f2e7f426,0.2549718972249718,537.6 322 | 78f865d7041568e515bc2b861bbd64c92e575c50,0.2545007525986691,512.0 323 | dbcd9d00c9b8d09ec9a2fe718051a34f59ca5c83,0.2554191387271097,537.6 324 | eedb880d387c652efdffb79365d42a20723ae708,0.25522836222110035,512.0 325 | ff940a03c99679e5d8002179502fc1fb7f156f6e,0.25477865784802317,550.4 326 | 6343b4169caa7894df811677cfd2c5305b29d923,0.25209083457885995,537.6 327 | ac2561fea0a78dcb0b1be597038cbbd699b65c93,0.2522313314352736,537.6 328 | 35ea7488080f00ee353ec7660631eb12717c262e,0.2524643579720148,512.0 329 | 08170aec3a966fb7b31839dccb7b3ad935f19cdc,0.25326149570208556,537.6 330 | b040a1f5e1ebbd21e7654465cd751ae234cb0da0,0.24744284489688573,512.0 331 | -------------------------------------------------------------------------------- /graphs/08-storeclwb.csv: -------------------------------------------------------------------------------- 1 | 843698438b6dc337dc76b794f943a6e53eac7c1c,0.9687454965462674,1126.4 2 | ab39af48b8b5c0afb4eb464d216edffbf7193d2f,0.9724040436647311,1792.0 3 | ae972904c6d65959a24323302e76612262e88b2c,0.9549021063046171,1792.0 4 | 263e03cab03d719ccd6ea57b1f3b472472925a21,0.9095394561717396,1740.8 5 | 71ddfa64b38450337c5f7017f2d4db8826bb928d,0.7778174449527647,1536.0 6 | 672db51d150f11b7f865bf2b1085d5c34ace2ec7,0.6011578130413295,1331.2 7 | 64215f7e805158ecd69e467b1bdcb005259c5359,0.9690692260511333,1139.2 8 | 44ef9d8b05798c43ac2ba4f0e8b95c06f7cefaf6,0.9659688243384573,1830.4 9 | cf06e7a489fc5bc2ed42eca753782b48361ee801,0.95498354179403,1804.8 10 | 50d2e5cd9e51ed514388bc68a1084b5eb05ce084,0.9129066595913119,1740.8 11 | 0952a0d46a160982ca29f16220e2128c71e350df,0.6881699877429829,1536.0 12 | 0247944d2a1e40a95db90b6ea3fcb3305ab72711,0.6859776271623527,1331.2 13 | 70ee58fea856c24aac4b046ce6ce625cf4ddcd97,0.9750843112234039,1139.2 14 | fdb5ef9c55ffe2aac49717260f9b0e704a32ad67,0.9649997872494928,1804.8 15 | 0c46eae4149519cb70677b8c8fcf57d24dd8739a,0.9538447068467802,1804.8 16 | 386edae30f7e79758f988bbc1c09de9ef3fea759,0.9116043199167633,1740.8 17 | a30afbef1fe01446125dbb85c1b8e1809785cde7,0.7916272376967217,1612.8 18 | 4c0d04942ce96c78d1c630b50ac04d64be06a10d,0.6849698135306043,1433.6 19 | 9ef21ccc73696adb44cb4d13103ae6819a32be15,0.9753099019442135,1139.2 20 | dee8889367c8cd42a836724d896fc9b09149ffaf,0.9717729731969353,1817.6 21 | 5bd9319fbf7428316dcf4e607c2501e86aa309ad,0.9542327589024571,1804.8 22 | cbe17132be196d1e78bb8a464d00b553a3c48aab,0.9127201410999614,1740.8 23 | 61370263d672a922e570db6f98725f4618fa08ae,0.7995819640060228,1612.8 24 | 987be5a0c60ebb366ff553f04b9afd4a11e563c1,0.6933020983650265,1433.6 25 | b0d824f33038a7f71d382f9e0d9a8b25412c05d9,0.968788856515509,1126.4 26 | 0fd1e413ce4508fe45e3221c9f2e0a60f74713e0,0.9650485582069639,1817.6 27 | 6b45b7820b42d43df4642b6b2e8bc74879fcf046,0.9540842084456417,1804.8 28 | 8c60bdfbacb4b5ff09c912801fb7c82ab80b5ab9,0.9134683404793876,1740.8 29 | 34f228a531c3b2722def833d947557afa3154ed9,0.7985760239143502,1587.2 30 | 4da1157d8da45507b672763877c32b7ee8994136,0.687861731694182,1433.6 31 | da65affca95460c0954e882b8fc81474c30f4c99,0.9686947405488525,1126.4 32 | 2a14cb664f937ce966f600da328c07c07c21a25f,0.964949906534455,1830.4 33 | 13d0544c0bbfe016e2b1cf4b762c4e47173335f7,0.9544905876523122,1804.8 34 | 490a3e563b71120ec53d11a9a7e7212cdd3e7705,0.9129661216821906,1740.8 35 | 7263ff93941917f0098e569b49be277ce3d7ff9a,0.7984301606425512,1612.8 36 | b1847e4f3269b06214af8496ecdfd9033a51bc9d,0.6959230003512654,1420.8 37 | a160e570cdb3047ba7d055394750ba885a303ab5,0.9748491180102996,1139.2 38 | 518ae72a7d40bd3c22a26328a5b0958f9816f650,0.9649778317056682,1817.6 39 | f01d2bf302f1bb2daea0caf02dcc9c9720e3ff91,0.9540680285002745,1804.8 40 | b5a851fe3e9b4a53ab6bf714961611e6a51ff72c,0.9132792585835544,1740.8 41 | 9ca09e9b3ebe0f52b7a5bc35a478ec578cf9ab3f,0.6941336106135785,1612.8 42 | 3a0550ce6edb4ab83fa77587927e0e2bdad53f1d,0.6212324582707208,1433.6 43 | f539e0260711812bc88723ecd3eb34ff150ee6e4,0.9745635579793064,1139.2 44 | c6a386ac648fdeb03f0a8820dbbeb11bb2f91b12,0.9721301110529892,1817.6 45 | 428702a95812c0d223448ab25693c29821355e3a,0.9541985229924265,1804.8 46 | 688791902eaf095fdd59e764ed73de8297e1203a,0.9140881429204066,1740.8 47 | a0e33e86c947e0d4e0879053a6d0a4205843f27d,0.6946620793851002,1612.8 48 | 481f45693eb66c291ee3d19c0a2d58addabc4705,0.619796151291299,1433.6 49 | 707208e59b9f79a808719508b0f729c1843ff7fd,0.968213064764795,1126.4 50 | a85a8401e9154d89813187c3dda376a7cca74d6a,0.9645173587605479,1804.8 51 | e1cc0f65b925e377c98aa1f7cded8138e891a11c,0.9539614907694549,1804.8 52 | e9c47d89dc90d6f612d0ef42904213870a4e27c5,0.9141765366936585,1740.8 53 | dfc0256ef8bf686c4b13fe362ae29e48d40f8760,0.8002715931703697,1612.8 54 | 1c47ba1d1fc84436e0d180e24866e77c3062231f,0.6213760748690881,1433.6 55 | c62abbae627d478443856d9dc6e08d2193808140,0.9684100207184816,1139.2 56 | 41ef14b8c4773dc8aafaec054fc097aa3e4d9f2f,0.9711117012986464,1804.8 57 | b9732f64e181e6194a17cc00f06e4bea09ec34bc,0.9540364496621883,1804.8 58 | 9d1967cd3694af436a002b2fa65a1be0cc19df33,0.914382452086272,1740.8 59 | 86d375040148115c3b45a6bf1f11fe7348462246,0.6975761018973078,1612.8 60 | bfd483fc4a3eb59f0ada4a3db76a7dbe23917174,0.6209777200763286,1433.6 61 | 9d22d97254c6e26b24979d9446f6a02ddf0b1197,0.9741589074862426,1126.4 62 | f6d89a183a4d94199f00facea91fa8e1c28a80d7,0.9645305865117588,1817.6 63 | aa9f9f8a14554439b76b0166a1fdf566f620c9bd,0.9542501258829292,1804.8 64 | 148abefe8fed5dc305227f71ebe6d98fbe238e2d,0.9146877574922415,1740.8 65 | d7a03dff58d622ff8b7646c31bf5fb0cd7505da5,0.8036325489110707,1612.8 66 | 95d4603f4516d3c4d1c71e3196dfdb7fd0d40c40,0.6994688120468416,1433.6 67 | 54f6a85bb9c80f2471625121cd606eac331f999c,0.9681559370014077,1126.4 68 | 820c1771b3cc4836f98fe370d07bb39a1979e797,0.9647072705520661,1817.6 69 | 7de0d0599da68e2e2dd2f5b670460c3631a9deb6,0.9541610057135795,1804.8 70 | ec3516cd0a0c7016d4b6597cef94c4d1e21c528f,0.9149825103283111,1792.0 71 | 3f6c9fbc976da2404faa740fe2b34704a883f15e,0.8020154638693225,1612.8 72 | 47d574f3ee47b178724215c2c0058f249d4c13e0,0.6890710449577482,1433.6 73 | f81d92f249353cbd7d02455dbb69dac3ccdbd72b,0.2456510408558853,486.4 74 | 8cf8c404a6dd6045dd35536dff1cff616608191c,0.24433532237976197,512.0 75 | 2f3b5bf6c8580e2cf4a75c115a37702ec583b19c,0.24475412151682577,537.6 76 | b3f2774c772c155c05169e4a13efef2f6d3a6b2a,0.2292983528481604,512.0 77 | bdfc8faa586dcabb0a4d53391689971210b68d16,0.23467129214401905,537.6 78 | 4873c8c75678e81c594103a3541e489530b152dd,0.23769696883211341,512.0 79 | cc7706fe717fa5e2f7d54c0b0f117a31f99fef8c,0.4827471087814852,908.8 80 | 04ac7e3fb63e3e65d10665799d9b08a476a6f05c,0.4789532807524896,947.2 81 | ae2fbb867f2ee6758cdb2243bba78cf933dc5b3d,0.4534803787002015,921.6 82 | f4df2b4eeb56eef219bf720df30e9e8fc14246b0,0.4183201186710995,870.4 83 | d8939ebd4fed68e353376f0128d467ceb3faa1b3,0.37012556315830486,768.0 84 | be1586cfb9e007930d13091c0ddcf59c9ab60330,0.3162212611107707,716.8 85 | abca6e822329a7d7184d976375bd2241862e9121,0.9685970642987288,1126.4 86 | 620e6d69469e0ddf05ce7440edd280ad6ec8a60f,0.9680148415120168,1792.0 87 | a999d4f0f9912c7b431ab95a686c921c7629e5d9,0.9563115900688103,1804.8 88 | 64c4c1f1e4423651a4138eb48755c6a72157e3d5,0.9089176565836183,1740.8 89 | fbb15755df0257d500dfa87fb6715197e5cf3ba8,0.7888792311150046,1536.0 90 | 7b6b5770bcee1ef47b656528d9703ec295d9851e,0.6932516085753414,1433.6 91 | e0f2f1aa300a53ec1737e71b1e06c81a62e1b2eb,0.9685224029043932,1126.4 92 | 0a163a9c6da0d80ce0f4f5dea6e5ecb215536947,0.9662098277178602,1792.0 93 | 4e4721252e8be1e41c37312d4c148e0c48daf164,0.9560769122061987,1792.0 94 | 12e8afd5547ca492b898d873807b62ae181753aa,0.9106835216129477,1740.8 95 | 3da05456f6d5580dbcb09d90f368d1e1baf65331,0.7764421654015125,1689.6 96 | 9b4f2ec04f72412e737f6de86b3bd7c4382629e2,0.609243765066985,1331.2 97 | e6d8fca35880cf28478dd3e7ec0629d9a5cf5f0e,0.9737916218720258,1139.2 98 | 64152c7c63c89e86df85ae1092ca65203d5f9d5d,0.9628287430440247,1804.8 99 | 5a7f89ea2b40e42dd7a3f6e820ffba0637e7cf5e,0.7794564496858216,1804.8 100 | 977e704555d9c87587ecb7869dfd40aa5051658a,0.9146410798184987,1740.8 101 | b9eddbb3e31e2d48b4c7f30fb5c4dbba08ae5b97,0.8029394120782055,1612.8 102 | b8d9eb318b62f5b6b45f92b15f7498f96157ae8e,0.6978165811762839,1433.6 103 | ca4079278ffae9fb154c640a75ddad9756e2fc50,0.9676812169977804,1139.2 104 | fcfb41c2b0b56d1d877e0220e31a52a67b5adf0a,0.9656149913050754,1766.4 105 | 00652b2c74f2b15bce76a12e8be1e4666918de68,0.9486906361961288,1766.4 106 | 7220e21a1f3d8a182aed9f429e9517d978b0e646,0.9106474653931711,1689.6 107 | 8ccbe585a2bf031f61f2ca5551a28cb1500f6da7,0.792554828837624,1536.0 108 | c9ca096cb586d8f6008245f4f7422c14bbdfcbcd,0.6754396882784675,1331.2 109 | 5285b4497333ef207e4436c0436c302970b7bf32,0.9679865703925008,1152.0 110 | 14759d65f7017acc359454d02172e09b7d41607d,0.9552906546268334,1766.4 111 | f1678ea2f32129249daccc892973a179a6254660,0.7825239260961926,1766.4 112 | 7deaeea17290c75f376a3cf9c0832f187801e0f2,0.9010481281237731,1689.6 113 | f9b3e5fde4fad04e7af0d272915bd8d8da74f148,0.6613638170547644,1459.2 114 | a02c1d5c429d026970e11d9061844d8d030a99bc,0.5595620999632952,1228.8 115 | 2ede464b1dcbafce7ce8298742c79fc2b5a4014c,0.4802643554295876,896.0 116 | eca637c06be28900d182b2f5bc5ebfa4f7e6ec00,0.3510011512224839,921.6 117 | 940a09ffe38b16ce369bd2f3fc3c297a39ece829,0.4141757625390195,921.6 118 | ae033a3d516701f2154688beff1d07b558351538,0.4280128514242255,870.4 119 | 1ecb20247f7b5f2cdaf9885c0497fc2af3ba29c0,0.3288727230325123,691.2 120 | af2b406751405453f5d4bd475f2886d218fb5422,0.30267831498860137,614.4 121 | fc17cd95bf95b5b135c1738eab25a3770b6ae91a,0.9730449234896574,1113.6 122 | 6df608ecf3ab0d6b8e0506c400487518e79ac66a,0.9599608433939808,1804.8 123 | a19b9b11c9b2f5365537e63705e06b47f2c540e6,0.954036069047923,1804.8 124 | f35e70e44b0b1c7f2a3057d9586f693e52cdc3b6,0.916316086975955,1740.8 125 | 243700e32fba778e69c9019a7ccdd98db56da479,0.8054048514840475,1612.8 126 | 46fb0d9b545c8821fa7c72b24a90bc5a3b204ba0,0.6931557516940959,1433.6 127 | f48c581bd16cdc1c99593c95071629c84aae10a4,0.46661087258460776,796.8 128 | 9a54c5fda274c8e07e938b28bededf9576a6b3e6,0.46708119952755045,848.0 129 | 6ba0ccb5efbfe4f243bb543dd35ac81141e85cf6,0.446398990161235,854.4 130 | 90ee1e5c376f15d6bcad76bbdac786e58ea2e615,0.4110754777510258,806.4 131 | b2eac193ece9c223c81e055bab31b068097496ee,0.28506238332345646,441.6 132 | 8f08b01afec9165e7dc5f99124bb61501f56fad8,0.23123345406071857,281.6 133 | 009aa6601a820a73702754aee1c8b450a52c21d4,0.6527802021223664,1100.8 134 | d90db3e236a52996e09320f8e42cb56639097a13,0.648039538416357,1241.6 135 | 38b66a1fc8e4ad895e2cf5e8571a1cf6bb919143,0.54028988961024,1190.4 136 | 670c728c1a749e3619a2a9254706d443db488a2a,0.5664904223880645,1126.4 137 | b4fc4533e64ffd5da1cb1b91c724042d6f9f183a,0.42515392438840344,960.0 138 | 2c696ed88c39799271563442e0a27264db9ab3bd,0.4172296565796013,819.2 139 | 0e0b5ccf898ca0bf2ce3a8a9c0a0fd03ddbf87cf,0.4830364800846978,921.6 140 | 517b69f4718f2c5ee03062d3c385134281ac6369,0.35257598511715366,947.2 141 | 0da7bb0f1785a33bc9d1fcce9d9deaf16d06265c,0.41376525775125483,921.6 142 | 6af34c3ff160ee19ca8999f4ccfc818087b194f3,0.43117779593264677,870.4 143 | c1e6ad8fda79cd4bc70a8fb5dc98a38b1ea5464d,0.3776154214592698,768.0 144 | 24051c530c45e51b691458d5ef7007cf0050256b,0.3131502605122042,716.8 145 | 5e3f87ae69fc83b3377098a8923ac0df45851882,0.4693214177019497,806.4 146 | ba9600595b1522b4902b1e27d8ce6b8999507fc5,0.362588135694467,844.8 147 | 9212642e87ea6ae64f68e2eb1e33fae877af2652,0.44650845124107863,844.8 148 | ace687d6d61727bdda396839c72e03146c1d91cd,0.3709919189465504,665.6 149 | f24495f1fd59b59d188de94d2df9fd272cddfe60,0.2839888518711008,307.2 150 | 2ee7163a5c61399d133170506cac574d20706436,0.2722028558229826,204.8 151 | ad7a05fe4492a0ef17191b55c12c23c3001bc955,0.48234465157199086,908.8 152 | 61e3748b9086a09632640fcf8cc059e3b0198790,0.3533873881221638,934.4 153 | 035d3eab709806ec1ce0ac6051dd0ed4b1c43c16,0.4663258681076772,960.0 154 | 7b0015ec2caf89a4f08a6412b9c29dc455ff1d9d,0.43217764428941624,870.4 155 | 2445ce448d9a45ed6a20a3442cb086b23a6ac226,0.36850109081965915,768.0 156 | 402259820a797adfc5f8df8ae93945e079ac3e9b,0.3086075820847144,614.4 157 | 77124f68a7c2580e3c085b8e898d6fbdd1374c45,0.9731938741302717,1152.0 158 | eebca61956101483ac90db70b4a78f9eb02e9c42,0.9713694709192018,1804.8 159 | a015dea3dcd569c0a4b42c4ccbe19c62bdbf3524,0.9535527048398224,1804.8 160 | ace293dd5edc1ca46f1302bafa00adf06dd893d6,0.9139431833880783,1740.8 161 | a2a30caf376659fb2b0c089399b0827417b0d0aa,0.8051245039547233,1612.8 162 | a99c35ac70a1d4853559a402855f46344b29d282,0.6217573922862648,1433.6 163 | 1c0b1b3c87d8281c6f4adf617ff1919a6eba2cb0,0.727940985066669,1112.6 164 | fc813679e66880af7b2d2804da0811e4c8d8c0a4,0.7250811009605148,1343.0 165 | d6f792dd41181e2f3ddbf30266bd0e4ab2986fa3,0.699532767809898,1304.6 166 | d6ba71ae23d1119d077fe591756ce875a48b869d,0.6294398043618015,1227.8 167 | 437c3a4510608b54027f6025b8c03cca4d81ccbe,0.48276192100431176,920.6 168 | 7c7b066c1a7a3ede6050052a7bf211dc698b6856,0.3781351991290986,715.8 169 | cc9024eb352d21d656f548632609873d6f1f86ad,0.7058280446088268,1065.6 170 | 281f6b3b61987c97e494d3a6bfb51586d3802843,0.7002523156662179,1238.4 171 | 4c3d082dab08f50c772dd2802f62ad81cc8fbdd7,0.6819336726201296,1224.0 172 | a38909c0c78c5854e8a2a9d894424d567eb97b11,0.6307760633196884,1171.2 173 | bb0dfd20a6cfb58d72b7c3ad563ff9dad54b82e9,0.3751934461965514,633.6 174 | 4516b1301c31f8343c80813ccfe09b90991cef0c,0.3115334597990182,408.0 175 | ba547212a31c3b3e7472d07f023c7af00213676b,0.9738886849661136,1138.2 176 | beacab552444bee4cba08dbb445ce512e6f70b00,0.962368076568587,1816.6 177 | 77dd2ad53d168bb0013c874f39443595e525d91b,0.9538568034170885,1803.8 178 | b6289956d02fdd748fc4395b9585b48441a3b5c9,0.915572250339228,1739.8 179 | c10c14521438526aae7354201884b8d8d2afa9e6,0.6915699012315614,1611.8 180 | 366aad489c5c5c17b44fa05e78dfb9c57a8fc30b,0.6945399822121869,1432.6 181 | b650d35db28fc3abba04ba90470cce6532bda91e,0.7336491299263127,1125.4 182 | f7cfc06942641585b189a8825589c34a5f7c4495,0.7311574623063268,1394.2 183 | 12a27e3e57f32bf7b04f04a960c45ef342968758,0.7078818141837545,1343.0 184 | 055347ffde273a6e2cbf4dc5484edc80f69260f2,0.657335156620335,1279.0 185 | 99e1ce8c7c91262417e08f6173f2febf65cbcb2c,0.5400111922472595,1074.2 186 | 41fd7bb63b696688d37f53a1d7d1d914e5f499d7,0.4647429052086828,920.6 187 | 3920be5b72d23d3dd4b490537d94cfa0f6f192f9,0.7069754594940315,1074.2 188 | 4d14d8bc8ab6de0685e645f92f585a2e33e68a3f,0.4597446566394051,1227.8 189 | 2fe77ec9d32095d1743c14427d7df0eefa9427dd,0.5936458374616188,1227.8 190 | b7c2d81e22a36d3bc9005548975c46ca8908237d,0.4610116623299816,920.6 191 | adf670df5dd5f5c1323fe6f57c844e1468d717bd,0.34074817520730044,383.0 192 | 6230c2ee64fb4a3938d3766b416cc03b48966d2a,0.291487960218455,204.4 193 | c902c26efd891aca7e1a9055fb8e6244ea32b643,0.7317450368748833,1112.6 194 | 181a492268b755ce8e6a464a17e3784ef528bbdb,0.7292322950634315,1355.8 195 | d51ddf3790af7d77c91f82044bb21bb4ee3ccad2,0.6086893305009994,1343.0 196 | 52383f50135cc9e028ceee11258d2294096131d4,0.5743880833037093,1279.0 197 | 7ae37298ad3772cb0997f29e6e156a4dba0fe308,0.4734712798796307,1074.2 198 | 485fc9900862125f9a1ff421365a64d4aa3cf52c,0.4486781435683303,920.6 199 | e10f29f623eda42dadb31ba94cc88a5247d6061e,0.9708598984668287,1152.0 200 | a5edff6b95090983d7f2d2e118a8a3b9557ac96e,0.9685284084250235,1792.0 201 | 5a38da72d91b5b73cc9626d01e0245e76f6fb97e,0.9511656081485329,1792.0 202 | 6a09e217a32f8fefe918fabce1944e3f60b0b98d,0.9109765198684444,1728.0 203 | a0609a0b6467df6292e8bca3aa67d5e7cd5285c5,0.786418824809676,1536.0 204 | b37ab199b4221249ef2de2c08d7d14734be207a5,0.6071196995211103,1331.2 205 | 2cf422d7cd6478cd603c7062809d12f3f6daf729,0.9714887153283256,1152.0 206 | 7a3536da89537f0a2099ff86570a6fa6502f5106,0.9601991259316714,1804.8 207 | 80daaab4e0dd9767fa81b56d9e2c39a6380c8e85,0.952272286232261,1779.2 208 | 42a39a5baa5a8bf896261d9ea3ee5d9678497837,0.914035886801751,1740.8 209 | 29195fcc0d1c2f0c0ee9e2e204ca10ba321bd298,0.7154663334867655,1536.0 210 | d585e6426951ba82c9eb6edfaefde2975d455060,0.685085020087955,1331.2 211 | 80777bc95efc221e6f2d68089f258fb73413fc09,0.9671920898228825,1126.4 212 | f50df145688d38286c3c2ecce2ae71060e7b6035,0.9588748294887441,1779.2 213 | f778848370d06bdc0967b61fc7658f85b7ff91f2,0.9478233094735167,1753.6 214 | 8895aa0597243e8aa57aa24c823989e09140a177,0.721875807341037,1689.6 215 | 1af17c4b2e218883422d709fd37f473bb36c0b17,0.6581245147852828,1382.4 216 | a6211ed3d87ca77fa6399a0baacb2a39d63bcfdc,0.5999996760954379,1228.8 217 | 73c71129ca4c4d81ecabac67d351757c3195d888,0.9420402362111606,1145.6 218 | 8e4da57b3e8dbf5b16813ccc295ebbd975e589ba,0.9445480994034511,1612.8 219 | 073c5b02280a5c0d8be7ad6bbc40d816c5474407,0.9244079342618814,1612.8 220 | 4185a105c67b98d3a14ec3abb43156e2fb1fba10,0.871965983524486,1484.8 221 | 3c27a4681a313d2837d88e68831406e51f39c7fb,0.5928672446777479,998.4 222 | 7ff35daf4271e4dab9fb61e19b928e0087e68b44,0.3647348104623027,614.4 223 | 71caff1f32fefbdc7f4ccc874a734d726eaf384f,0.9739227707555518,1139.2 224 | a46dc60132293c0a8575953e58db9822a6fce647,0.9624120429982003,1817.6 225 | 82be7110333cc0a678698211c9b180e5f9d00740,0.774865497346905,1804.8 226 | fd0903e8a61b8dd7a918705bc175e7ddc7c4325b,0.9155813683624989,1740.8 227 | 4dadee1b80193c6a8e4eaf5003615308d45ac30a,0.691577510875047,1612.8 228 | deda6a723747e329ea2171f48e1510173e790bd4,0.6975875628711891,1433.6 229 | 9c11455e17cc08b4cc4e61b950f3e18ca31c5660,0.9519344187788492,1126.4 230 | 49b71fb221373c225cd4a12193dfc7e5093197fa,0.9385991639422127,1638.4 231 | a3c7bb39ea2eb78b3cde29a5c53ffb9c1f601602,0.7743208733405077,1612.8 232 | 06fc0afbc7f225303eb62df0d3d580c85e606818,0.8169806917390066,1382.4 233 | 2fde48f9d6e4315d130588e0f924fb5be44baebe,0.4221740158046025,614.4 234 | 79ce1d747789b3dcfb1de7ebc867ae091d2a1dec,0.3528755571331984,409.6 235 | 0e6f25137e43891c5eb6b7cc4e152c5b9aafd747,0.9716324159428164,1139.2 236 | 98fc05cefd8d9ce11290c51a5b7cec9b0b0d183d,0.9601575105944578,1804.8 237 | f24ef6cd6af7eb3174047f5094b87139d75f6149,0.7762944788182474,1779.2 238 | dc3df0baa33fdcb906a733efaafe05c19d841559,0.912069251398848,1715.2 239 | dce7076638ee9a180ef047c6ac760c4ebda87486,0.6791827681924564,1536.0 240 | 4f3af83c94fa34affa6b860674e478ef732fca88,0.6775659301694681,1331.2 241 | 52cf1dcd8528a71c2fa649dc3d635e868adf2eff,0.9738125339736818,1152.0 242 | 5842c0e721bf86a9b2786ca306eb4aebae570da2,0.9618269411979253,1804.8 243 | e57f80d7d2fc1684fc0407f0e44eb69a5897273e,0.7759862699144485,1804.8 244 | 771698a05636fe7c21230118ccfc4dbe4d1e9ad9,0.9133472443340043,1740.8 245 | 988b94ca6862894e9acac2fc40fb6f7b5d8d8ebc,0.6905377068741473,1612.8 246 | ded5afcfdb7c794407e8040f4f1ce25313b88148,0.5558565560427716,1433.6 247 | c3443283c903e5dcd02ef825d289ed2fae1d205b,0.9735377438141021,1152.0 248 | 67bae8b30c1f945c76eb61292682b600cbd68a4d,0.9714187966521965,1817.6 249 | 3df464eaa717438b743ac04fd52ce1dc2ca55052,0.7757209300946813,1804.8 250 | 2cd67890d8e64e7043dbbc92083ea02e78dcb058,0.9142134924121037,1740.8 251 | a7d8f823fbe206b7ce5bd55e240a7cc8024d70af,0.8028882843943483,1612.8 252 | 7058dfa4c78fb00f08c5472912382b5439f2cdc2,0.6211840574978983,1433.6 253 | 1da092fa28220b1762fa04ea17607392d88b4941,0.9728334038031433,1139.2 254 | efcfafc61ed438cbcd1a8db1e8f63c360b3733e8,0.9606123446843546,1792.0 255 | 13abd4aa0fe3ed9e03a4c45dd7050ee9a4b611bf,0.7772953046810808,1804.8 256 | 9d55750afed161582aeed219bb65762be7c986b2,0.9135764890637469,1740.8 257 | 726577caf7b30a0c054abfab20d04265bd2886de,0.780667890712452,1536.0 258 | 9aadcb61bbfa6181a65604ccfba3bdf8f922a873,0.6652210554051202,1331.2 259 | b4130485626f4bc30ec1a57bd3527b5555e19c53,0.9583531949115368,1139.2 260 | 61b73114275f4e34c320359d7bd81456ee649592,0.950680008719562,1715.2 261 | d0697b4f7aab95586569fe073b885f8ce09c2bbb,0.7672070838963188,1715.2 262 | e9d06b5579d9cc5b4519b8d10aaf86c1b66c5429,0.9045995758508566,1638.4 263 | eb891da1bada52cad8aacfd1445e5b49d6ff0686,0.6554594693507168,1382.4 264 | 6932591aebca748d01e3486004065482370de36b,0.5241390203813814,1126.4 265 | 019b8f0897df80fef83114720220ed2eac1b1d37,0.9623780966808035,1139.2 266 | c9c10c9af7f135d39b4875d02e68452a5c0d0d06,0.9495468629301819,1715.2 267 | 44bffc44b12e162408024035c8a12584af18f973,0.9390680626383525,1728.0 268 | 9397166af8004ebb319811947beac07efb26b1fc,0.7635568524807156,1587.2 269 | 846a230ba3337c29149edc494f31bd88cc9c4a10,0.5931008666844592,1152.0 270 | cea0b86e0cad687c89c4022d30f9fcb409a4b57f,0.45908632906100577,819.2 271 | 162a48c3b83eba621d72e8d0b062afeba6375451,0.9746845168925002,1139.2 272 | 4396c1c8e7343109f4cbf39923a70511d238335c,0.9712306269428959,1804.8 273 | e18020b4ead1ad04d9d5e9af9b29da1fa17048e4,0.9542188864446222,1804.8 274 | ad254e549946453995413cc722b8c9f52bcdbfd6,0.9149012101452445,1740.8 275 | 44b072d1a8e1100bbb9e7b652096ba5f94a3dc0b,0.8030369771716812,1612.8 276 | ea59f9a492511c2b38707c75a88627ad88b55d13,0.6216536433884168,1433.6 277 | 3b3972aa08cbd48149c3410f52bae1789fc0907a,0.24451765057307256,486.4 278 | a10e8be82ff355ce6417f47d7ef7154c44517937,0.21106436303460732,512.0 279 | f678693ba71bbb1489345679f970fa133d87ffc2,0.2429315701810777,499.2 280 | 769e272616c0d2a260fd3f5a661675ecda49d958,0.24325819453852984,512.0 281 | 6b06c77b2e9a192c65ee5cf21bd5817be41eac83,0.2441001337862802,460.8 282 | c26805d8abc8081258999b5da8fd60529df9ab41,0.2448026023230319,409.6 283 | 3ba053ba7473648a045010e6fe8e7ba318815ee4,0.48329932177129614,921.6 284 | 8fd9a838a305ffb65e93d736fbc36dce00dab317,0.3524580668766717,947.2 285 | 002e38ba9d742db1ead8b4c50f9605e3e74f2448,0.475608614687273,960.0 286 | 027a9a5b826b4b922459b4384154841c34de3dfc,0.41130394215389354,972.8 287 | 7ee3264a4f0f442e4ac628c44c06d7f4b79037f0,0.4462678085139044,921.6 288 | 7bddf2406730685ec948f7d5a0a2f449c9faa8f3,0.39374118570211514,819.2 289 | 70bd060bb417293c578638331ef24f4da8da5dbd,0.23581058722074125,478.4 290 | f3bb543cd5ee80ad98e4784827fc6541d146287e,0.23713562870149382,478.4 291 | 9c497576e16bae81d2887da74c7168a1571b6007,0.23594851170545153,480.0 292 | 09e10237c8aaab71c2e228f107f93ee1da923bbe,0.23817187283092842,499.2 293 | 8fe4ad5d333302f596bdbaa6a0f85f598da635a2,0.2376989335101072,307.2 294 | 091f8da12db310f94e3e4a7d3285b80447804625,0.18752191679973823,230.4 295 | 8308a8a1684b58691058e389905bb597edf8a41f,0.32476945316142125,640.0 296 | daa8b13c1aceac3549b2c5a0694cd579cc43ba44,0.24717373632783676,665.6 297 | e241c0f391a29e0952d6224ac1960c5972ed22ca,0.3106975904541512,652.8 298 | d941dc4f5dda5c3dbb9a56908fbb22a38da8fac6,0.3004508432440348,614.4 299 | 238d8014d4c40a39ead7f2cfd45c21a5fc1d6aaa,0.287618062946663,614.4 300 | f5ba9586b7a51d7d5a02ec34416cbb77c584663c,0.2657367363243663,614.4 301 | 54cf7e6006bbb772bcc332813ea6d0acd2f84aba,0.24526306394891254,486.4 302 | 3920a5c5add372ad8130041c7ae019c92da5bbc3,0.20443953537673762,512.0 303 | 8d6af1ea1c4b271f0c8a852cdbcc1537c42184de,0.23085805007504995,537.6 304 | fe27a65b545abd5752a96f700559fe2c524460b6,0.24441211776248123,512.0 305 | 3acacbdd96cb5cb3efac57348ee5e019c242f8cd,0.2453421936266552,537.6 306 | 362290e11661a2b8817b59c8cd975f31eccd9d0a,0.23730455880201784,512.0 307 | 24de2478e2ec92e68485fdee811778608a4b692e,0.24126013341110902,460.8 308 | b3806cc38573bcfe8a175cac7f07af6184ba1dc4,0.17471429345176842,460.8 309 | 731120b5cd12fe2f12ada982b589926d74cb9091,0.23790515919154576,473.6 310 | fbc291f74f3dad33c6993ef2cfabc74a49f11803,0.2384687538380924,409.6 311 | 602b0e39754786818f9dcb227285fc4edc522be6,0.2182564439416373,230.4 312 | 1c6ddc937b52212ad8131cf3e8c1023b28ecc26d,0.2249722882526691,153.6 313 | 6d4efe3cdb848627bf6bb3eff2378229fe477da8,0.2451815868047356,486.4 314 | 13827117d3c5ecb2a714cb237023a75f61d9a7a4,0.2038677092734002,512.0 315 | 4757e881d3ffd7ff5c21b649bd6c348a447ebedb,0.23069369980661908,499.2 316 | 54d752f305b84774c3c641361bfdb4e59e9e1028,0.2284101570360459,512.0 317 | b573a3cc157c9071c5642e4cc67710affffffc1b,0.24501455201652184,537.6 318 | 8b79469a97066b78cc64a54ccbb65097f09b37ba,0.23734623868818075,512.0 319 | 2d94e6b2db2acc08d8f791844f9b4d7ffe8bcb55,0.9716536595857997,1100.8 320 | 4ecc0d23ed50b3b6afb5f93d81b99936ba07cf3e,0.9696708574507629,1804.8 321 | 5f14d0b7cf65b3fe5f973fd54bea6c37b6484005,0.9529057132854756,1804.8 322 | c91e46defefffbe7212410a74e00b1c2a125b54f,0.915665440836971,1740.8 323 | 0df879152680cd17695d5d268349a6f3ee69c66d,0.8020546408820848,1600.0 324 | 08ec94f9ba889af693e6722c10adbb7c38dee99d,0.5526531347222788,1433.6 325 | 5998836b0ae0b6daf36c5ec60b50627518ea8fe2,0.9729605342469531,1152.0 326 | a0cb4de4fe6ae2d945bc98f0584ba43de52c5529,0.9601915802295087,1817.6 327 | 027c55f47ed8cb785edf9a7a3de8078622bf6b5e,0.9528924713943329,1792.0 328 | e08229513630ea4d880e82ed28355017a79b7839,0.9143114808218209,1740.8 329 | 79e5cf15defd563b91bc7c686e13c5c80bdb51f1,0.8016142154581783,1612.8 330 | 9d55aa9ad627ad588740858b19ae54e3c944c28f,0.6958260864087997,1433.6 331 | -------------------------------------------------------------------------------- /graphs/08-ntstore.csv: -------------------------------------------------------------------------------- 1 | 6161033c3ec677a39070474e3f3c0e658d3502fe,0.983978363819189,2278.4 2 | 649edfab4066d19be9e588fad35a18a08367effa,0.9829467897121643,2278.4 3 | bbff7ea6578d491b39c5210faabb39756c081676,0.9740633780198156,2265.6 4 | 1be1499f05c6e1c380c0e91cca20f5e58c76873f,0.9630761972518475,2201.6 5 | 77998ea53a57a5730b133f010faf2f9bcaecf69e,0.7418612581445236,1996.8 6 | 10a50ca1a0df294284314a32e1e7cefc660644c7,0.6988956314090032,1536.0 7 | e8e4511f39e137a7baf9cf12de96ddedae99ac90,0.9730037661195641,2201.6 8 | 593ac8dfa9ddd034b4e19350c8567ca4cc599136,0.9760386012756762,2188.8 9 | c80e04879d36538479fa2f019366292daa2c4701,0.9579219961466777,2150.4 10 | da815bdd6c963e17e094d950a9b58f46a8f66d27,0.9302004579315777,2099.2 11 | 9f8f48f1387a15c179ff13e4e42833d8c8a87de8,0.8043004049039932,1766.4 12 | 442a40336053c51cd3be4cb1d5c0db1cd23ffb00,0.669162505830079,1433.6 13 | 68cdff29f08b2a573902dc93649008e1c5a37a34,0.9744978701826799,2201.6 14 | d790ad1c8b1f7d034aaad57607839cb2ff0990e2,0.9687436706470847,2201.6 15 | 84772149a0fe35429302a4642b3f60d6ffcb0a19,0.9580353677053195,2150.4 16 | cf17e802082bc67c01873b77ac7269babf5f4316,0.921129874896495,2048.0 17 | 7f12f9aab657bc83ffc6e957682cfc8ae558d403,0.7622232517657488,1689.6 18 | 0daf810b03c74172ef1e7b4606bcae3ee8b51c9a,0.558180192614448,1228.8 19 | a4a99bd4a91a1ce6eaea3b7c50f447874b07a2bf,0.4872426709044289,1088.0 20 | b02c69e30850175dd81316138380ef007febef87,0.4820695197714953,1075.2 21 | 46dac6451b37e0bdeaa348c14934a29f02518b85,0.415905592299408,1036.8 22 | 7cec9f84dffc49391d9cdd8270be15fdfb470b31,0.4269190949699214,921.6 23 | ac498cb8d42ad37bd8f0df071c580d58dfbe9f66,0.32850582087762853,691.2 24 | 3ba0f4475bf6bccbc5156cfe1be324c468658d41,0.3019476930662959,614.4 25 | 3c94481038bcb2c101695e28b01b715b90619713,0.9806203188873425,2291.2 26 | 2e3fed364b2d049bfdcf994be0561d8532e7b358,0.9824830518563344,2252.8 27 | 3dcee53c34df99a3946a0b1a66a6978eec417816,0.9732151368320746,2240.0 28 | 2747d962411006ae8dc373ec0bb63f5723daf0c1,0.9622650837651399,2214.4 29 | e16471e55fa7ce807d5ea79552b53c4ace2517c8,0.8725224400133532,1996.8 30 | 4bbfddeaedeb785b399db515b65b74776b11aac2,0.617685256021645,1536.0 31 | 884a5b2b502606b0ec8e0760ecb7cb898b5a72b4,0.47197005730540437,960.0 32 | 7fe5da0315391af8bdc831838ec8c8f8927e5e4b,0.46267214150398095,947.2 33 | ed9040cde9921556401d2fe2f3c0faa5d0d69a41,0.441415740700401,915.2 34 | 877054204a62114903e28b6c15f708df1b3769d7,0.31286099019040475,867.2 35 | 5e9effc7de8659463b0e93e60cb693a31f0fd3c5,0.28384569940997084,441.6 36 | cc81420929e42a84a26176a54cb7b6b0aed14849,0.26990833608965414,281.6 37 | 623d91c564a8c3d68e3b0fa629537f0a13c83219,0.6549424204711293,1497.6 38 | 4872551487cd0cf35749930ed09e58cf31ed657c,0.646091525422444,1484.8 39 | 624ffa6fb40ab0b70783215a9651dfbe61848ab4,0.6165637658997087,1382.4 40 | c8612e50807686851624829fc751f0a674fc7c63,0.5591494745014189,1228.8 41 | b13f3abef2a9c49d867cbb419e3f594065b3a463,0.42284927516818965,998.4 42 | ad8f2929a46adbb51ab9dea386b46df4493398ed,0.4063329797424944,819.2 43 | 53ef30b285372709d2a7995efdefe7e7199531c2,0.49033071351078567,1139.2 44 | 3648553d1d5ab5dd71219a95284f8cac064dd9d4,0.48458037508135926,1100.8 45 | 6341aa0cd0b32f52566eccbbec1b12c2b3ddfc08,0.41216025421670066,1036.8 46 | 2f772e81ff165d970e765b8d0d334e0cfd10957e,0.3816431228712792,972.8 47 | e192b53288a7611a61b055b0ee7a81ae6d63d0e9,0.34546403575436163,844.8 48 | 395d3ab92a8f6e7d3c49e495b0c4c566c39a296c,0.3141673504295768,716.8 49 | 1e8d862ac5830aacc866a0e0eab4faa69d3b8c8a,0.4755728886065984,972.8 50 | 1477bd88a68581314d5dafc8717f4ace36c646fe,0.3462061715315003,947.2 51 | bfdcbe44dbf48353dffd9739359b4af10750a790,0.4417973097848964,883.2 52 | 8dac1f6c5691dfc95c57b3f47765990912b2f150,0.3692960112563177,665.6 53 | 6f622105c871614a459948c10ca952e7c5e69ca9,0.29180619417265136,307.2 54 | c82bf4809ab2152c247bfce4c70c2ebdc72e005a,0.2597259958686199,204.8 55 | 80d888a00fdd4f76dd97481d2d661e925a655f4d,0.4892195592561527,1113.6 56 | fcf1a984ad52f8280830cb01a7b6422e7fdbb574,0.4855598584322094,1100.8 57 | 8d5d5ce29ab06597f6a643c24cff5834fc654298,0.41996233283009066,1036.8 58 | 8acc2c9b3326b47464dd9a8b4ba2d738def152b5,0.3830859597938097,972.8 59 | 6971d51c4c1c7d780529543c9b583e734f2c9e6b,0.36927850987009836,768.0 60 | 52151d6f946ea4e7fa5de7bcc1a35f39ca298b97,0.33327729558714286,716.8 61 | a8cdd4bc5cba3645c27e71cec2579ea02031442c,0.9828948414052402,2278.4 62 | b28c3c126d1bd24603e8cd59e23c4499641e6d52,0.9740028481129596,2265.6 63 | d83ac32e75f937db530df2e69f6bfbdf4ada52d4,0.9726841258308473,2252.8 64 | 26fac27cb935a64a1bf0987c7a0a57cdc86b1999,0.9625207433461042,2201.6 65 | d04204d6c5e7866198af5a5ed47b37ab0e974fe1,0.7422054784920827,1984.0 66 | 0ad874216330759b3eeec7bf7bfef166c3112399,0.6173750109630163,1536.0 67 | 81845cb28f17a2c5a7241821ebacf2904cdfa540,0.729446332085712,1624.6 68 | 5359b13082a1c08f808488e3ecd3ebf8fe706a2f,0.7195462542563632,1611.8 69 | 7bdfdae46337e65afb95369ff1ef92aa77bef5ab,0.6911796258732247,1535.0 70 | 9cd91bea6bd17fdd4f828fa3f7949fbbd2050e0b,0.5272471960964856,1330.2 71 | c47231f9c33b4318ddb6b6a255ecf02ad51584ff,0.47542076146381845,997.4 72 | 18826bc74b7c5d38c42e3739e8d3653ba97eed63,0.3750024800013584,818.2 73 | 5d71e74a14af38d396da92022606cdd897ea12b2,0.7092037926193526,1440.0 74 | 0703415a90c0fcca1314395bce02a97cd0eb3608,0.7047082672028087,1420.8 75 | f119cb4e70d7228117e113853d54fde12696d5b2,0.6795277453048285,1382.4 76 | d545a08ac95b3d2ebbebabb47f5114121e7fc58c,0.6234502219648681,1286.4 77 | daf8eef13d81be521d44b3d69715b58d8bdfbacd,0.31066169724169385,633.6 78 | 722a5ccee5a606d24fdea4318c51ed07e4ecf9bd,0.2740159845672949,384.0 79 | 6dfa2db9ac922034cbbdb7589b9aee47d34e5abb,0.980970941429981,2290.2 80 | 19e0daa86d4da781c8b379aecc914b93ece1c6cb,0.9751842969004288,2264.6 81 | 5afda3ed9c734cb10ff7b79f6a8a2b1e0a217e54,0.9742273021746095,2239.0 82 | e491b1d82d3463200ac8c78859a5843df1458629,0.9638436650735122,2200.6 83 | 63aa15bb7000b71144dd50ee62249a2527c7becc,0.7417149661104045,1995.8 84 | 41198048b1f7d59838f210aeeb18432a690e7e40,0.6996753560178202,1535.0 85 | a8377bb8210bf9ffedaa68c593ae612994859c32,0.7373505426554866,1663.0 86 | 8be4e98eb1886d1c13cb2814a6b0ae442e82f184,0.7299968054021405,1663.0 87 | a9288bb015de26f8ed7bfee36c9a4720d41a6b71,0.6057281255842912,1599.0 88 | e737a378566898c02974f8652340950aed2e6207,0.6467831353522202,1432.6 89 | 1160673de0bd02eee13fa11c36fa893c53652d4d,0.5330817403304785,1151.0 90 | e853fa383f837048d4974521b87792e3a6305d28,0.4213306656257471,1023.0 91 | 2ec084066ac1daa1aa338d7932d02311fe0bd3b7,0.7096655125244663,1432.6 92 | 5d9a7e150e6c0224061f78ffaa1bc85563ee1b98,0.7062593890102056,1407.0 93 | aedfc90bd29c22f0ec0ba32d67618b7796bfabb6,0.6756253635256148,1381.4 94 | a1f4b8ffa558fecd1eaad5d141eb651f67e728f1,0.5274581901611397,971.8 95 | 14cc60c9f1c98069017609f024b0ee51d1abf2d4,0.3377897359430664,383.0 96 | 24bdc94cee9566aad5793e9e6c7a47a54bea2310,0.29081343480529515,255.4 97 | 59bafd47a2e406e678af4f1e5eaa5a928950c6b1,0.7355674678257983,1650.2 98 | 89e6551b2c312ef82c300f0cd7477fa745c2fbfb,0.724181313304994,1624.6 99 | 5a4887d04d7ef12e16a05fa46332846909f55af4,0.700155915483268,1573.4 100 | 2066ad8469bd4751770986868a89031cb3f26879,0.636324085507543,1394.2 101 | ee326900632f142c8cd3286d1ac6eb25b9a9c231,0.521916113711485,1151.0 102 | df147e5fc2751c558e9c3323e430bc1cef78bf55,0.41187100952224076,920.6 103 | f8fa6ed64b8fc88d741343feb5c0199bab8f5384,0.977975504843297,2240.0 104 | e08458f2f741c7e0df091db5912bd32d6603b0f7,0.9729232564142831,2252.8 105 | 9b3eba74dee070a6a0bbfc3a2186c1008affd2ef,0.9617036931550926,2188.8 106 | 27ae29b6e48cc03232510790ed3c35a6028a3698,0.9369473718324667,2150.4 107 | 5360540cfe979add4989dcbd938cf9010a1dbe50,0.8137240377914445,1843.2 108 | 5dc4e08a5b3c19ca78d279f1ec5da1a970a6dac0,0.6675633626257867,1433.6 109 | bb45eb5b035d0d7fb7935a63a93de6ffc3c191a7,0.9811150609290995,2252.8 110 | 31149e067314b698d7fa8cfef05c6a4a141a612c,0.9727827377428114,2252.8 111 | 548d85a1ba09222dc692c9b6d7777beea2e186a0,0.967292702767782,2214.4 112 | 805219dbecb2d2543191db651e81c120dbad62cc,0.9479680519881365,2150.4 113 | 21e6072a58be42b29bd002df5f0ca36d612bdc16,0.8412316787922562,1920.0 114 | 2953fe2bec1b6bdb1fe70f501cb75cddd210aaec,0.678459697419539,1433.6 115 | 8e45df593f8d971b89379fd993bc3996fb5795d8,0.977043225214375,2188.8 116 | 5485c0957d23ef7b6def891e925f9abb3fd2b8b4,0.975664682506389,2188.8 117 | 4300f0204f53821898b84c748ea607fbd549833c,0.9598596252907796,2150.4 118 | 577adc51da82e853451599320096850e20591d2e,0.9256459369150253,2048.0 119 | 0260db8268f4a2f79350dc4b369e02d09efd1173,0.7465286548977242,1612.8 120 | 9b9f6546b1e3ca29ec786d9c0078399e163b72a1,0.5597625961959245,1228.8 121 | 855f9d778d6708b02da1f9cc8ac8f6b375d2849d,0.947612591563369,1907.2 122 | 16533bf11e48ea0a942d7c009e3194021f089dbe,0.9511233580469036,1945.6 123 | 30a9818ff0f95a0eae219c6525c32b1e04f69ee8,0.9290538486976416,1900.8 124 | 32ad02357b7941ad39d5c447a5b667e38104348d,0.8852494073823108,1766.4 125 | 8a084d27b87efb81c63444fef018ce2cc579b7de,0.46030028709717313,998.4 126 | bc5bd0b781263ab5fe6533c11a65a1f9e7e52577,0.3574811152232579,614.4 127 | c7e84e5ccfd3cddce952063da1a7d89c42e6acbb,0.9839680432662359,2291.2 128 | 1b8448b481ed1c4aa70794b0bab834e6c44e64b0,0.9827063929411466,2265.6 129 | a042f1ef4ec905352254ef9b5fd43b6c65bc94fb,0.9746024770559844,2265.6 130 | d146e71eee3dafdc8856a5d9cbfd8fe102ed046d,0.9623958994429899,2227.2 131 | 152430299e13e63b9c538b5e250ce43de20b1650,0.8704388239040938,1996.8 132 | 0d3b7757549ce92767a85ad298c7c043133b9688,0.6982331479743669,1536.0 133 | c4f3f009479d585f07c7bf2cabf615648ef1733b,0.9527573984603223,1971.2 134 | cdd043b1bbb4a184e93215525c7928e4fc24b190,0.9482567126315206,1984.0 135 | 9ab70c362f56be04d276d763f53a7f47eb15eaa6,0.7696392073986049,1958.4 136 | c9a1d7161b1b6e5cefd12d6aabcdc4ddfcd4126b,0.8262268582197815,1536.0 137 | 2e65631005c3dda73132d1d80cb497875c889598,0.466742008154296,691.2 138 | ca1f76fcc43cea399a43346ff2271252fe186166,0.3538263281219498,409.6 139 | d32013fbb3e41097f3d65fd3c201e88962443bb0,0.9814851407744825,2252.8 140 | 6fce6a7e9435cd6e5d7be2799332ed25441c9118,0.9732490851373011,2240.0 141 | 9578cfea7c7d12e76fdd56d93d7e283d46458e6a,0.9685331378750309,2227.2 142 | 92aa45eebb20d17b578e7484441e146c5565e778,0.9458141144687094,2150.4 143 | 5127a36b99b182a62755a4b23d6db1661ebb7357,0.8236645107654663,1843.2 144 | bfcd7184df2dba192d32a31921564e924f9459de,0.6762879409348815,1433.6 145 | ea15f1a7851b500f04783b6c98f4dcf1f9f77f2c,0.9808025475270465,2265.6 146 | 0440d0d22c5970f5a1d41fa5d74a389855182c31,0.9824996728546858,2252.8 147 | 2e3f739544ec375dc131a1ed274edd106a0525a4,0.7878083150245566,2252.8 148 | 37fb39c751f02857a77ed88cbda77b9c06800b13,0.9471671503996468,2201.6 149 | 3635dec3c13df9cd305847b8dc14746fa43f7f0c,0.7350293811350013,1856.0 150 | b9e630c50482971ad434144c51d3195b2a3718e2,0.6108364006481433,1536.0 151 | 460a5e20d8abdfdb6c208668563ab23c21d3da69,0.9801364758409739,2265.6 152 | a0973c4cbc3d6436e16bb8c223ed658937170e1b,0.9822252897771188,2265.6 153 | 0d8eac430e78fd9e5fcc7c2f6fce04809a6218ac,0.9733671841563848,2265.6 154 | 11e25286606048c29b8e455a59942812b8063643,0.9633373443695757,2252.8 155 | 0b38d73de4c588fd038fd8c7915fac461b9f4db0,0.8667689464084494,1996.8 156 | 8cdac284ee539a730c27b71abd07bafb7744173c,0.7013434105305181,1536.0 157 | ec9749e630a0bdb3d78f6d2f90811855d0211055,0.9830326523246021,2278.4 158 | 5d70c6559f823764917937476bd8074294b0b9ec,0.9736654782253726,2252.8 159 | 9b58ac102c066e67228906c919716e1e53b2905f,0.9728086296563895,2227.2 160 | 03e33ece2912dd28a5d5f2f6f0628403d25e1d99,0.9468885250516166,2150.4 161 | bf1a7f3a13021e0b7bec5f8547ec6e2220339691,0.717234798915024,1843.2 162 | d35f42d3636844f16c572c24110bba2891503661,0.5914664597235385,1433.6 163 | 3e8a58d5926bff190a3a4db034263e810bfde71b,0.9658235761590115,2099.2 164 | 4318fa49a2fade1ba3ab1eab7cd847bde910d33d,0.9664446725718177,2086.4 165 | 6186a9945f08d05124ff25a04e8b7fc90c72019c,0.775663668699262,2073.6 166 | 4ba9ee8034a1c1258976515ef2c9d9494a1c4a69,0.9186917018947058,1996.8 167 | 1459b3438b27729193a9837ecdfcc727cd83b444,0.7555248925900222,1536.0 168 | 18889145ae17d1e337d32bd4a32ca1376891b128,0.5126796285037268,1126.4 169 | c8f86ca6dc72c902bda69476f1f41a9addc83c7f,0.971992693481291,2137.6 170 | a529f1845c974c0c457d5ebfe6fdab7d0ed241f7,0.9590227006841129,2112.0 171 | b516ea6eaba760ff4b3b7e67d99f034615bf6750,0.9453436179699385,2073.6 172 | ac494e978088180d2eb134ba0b08d4bdbf64d925,0.8913080597924351,1843.2 173 | 5f1a4bc8efd4dd2e66192f92042bc4c5b74c59aa,0.5978022599171126,1228.8 174 | 75f6e96909eecd918498ab16b922dc3e27cbd902,0.49167874765335035,819.2 175 | 111241c5a4e826e650bcd80da882d2d195db8fa0,0.9818359533624641,2304.0 176 | c28c8e620550fcdda6b45e885abc5ade97feee22,0.9770694899407371,2291.2 177 | 63226893593fdf8a53339c07388bb5e08673eb37,0.9741666904246596,2252.8 178 | 5084c90571ffb3c504a44dface3bfc8bb2552113,0.963525893902498,2240.0 179 | 302813aa0c9f009ad0fe25f631031ee38be574f1,0.7435413686549668,1971.2 180 | d5af4954e71991e91fb3ea3520194d2fbf3cc0a2,0.6159459052614329,1536.0 181 | f182d460c7fafef649583cf6821ab88c8eb961ee,0.24532346606420366,563.2 182 | b7fbb019b5089dcad7cd3aa9de1a7068fb00cecf,0.23064035311039552,550.4 183 | 437cb3dc2b4ddab42045923b77684c59633e6fdc,0.21559680855948432,537.6 184 | 1106d97f17e58c8acac699dd9ac187736e9ef610,0.24338267284847995,512.0 185 | f6ff85c93f79066dc9f29b21343dc0ebae60731d,0.24412964973735057,460.8 186 | b5c5134321b338bddd055ad24a3ebecb240fc422,0.2365428665612929,409.6 187 | 40956a54f25005d336ddc9648cca840ceb7605df,0.49008098750639123,1126.4 188 | 072c06e24465865657eea3c2721ead9f340c5ef1,0.4875444074736593,1126.4 189 | 12bf95ae783000a2a4f9a09304d3534950463486,0.4307330914338039,1075.2 190 | ae01b2474ad3dd4ad5700947e1187a282be2198d,0.4139908313449301,1075.2 191 | bef7358c939b068471d6ea84bcfb853036b25756,0.4473231964578471,998.4 192 | 3391e38b8e463286c13e158998f554df25aaa93f,0.42389415295812105,921.6 193 | 987776527295cb96db79101f5f2dc508af064792,0.23718901854413307,491.2 194 | ef41d599f3e052d477e21dd78ff0a1a372935e16,0.23700617213279498,481.6 195 | 109cdb4cd515d7b06b7ccb8a9627d96489dde4ae,0.23580618159291428,478.4 196 | d9789b40a8e81a348a7708caad33628b16df69a2,0.23765181651388786,499.2 197 | 1c87ef80663bf37e96a9526ae20b96536d657b61,0.17559857420027572,307.2 198 | 09a9a622eef437e43448a71d29fed23c00f5f3c6,0.23695257749843684,230.4 199 | bb363b84321ffca99c857a0431442c17fea08abc,0.32498398193076494,742.4 200 | bd719ec677399401d7eb5f3a6d4a37afccb5a578,0.2581226995485039,729.6 201 | 5057c0e76b245357ed4b448e2949d7496e30e7d2,0.31001832932458395,691.2 202 | cac3b97bc159386fa9c5d82bc817b3a40f517ac1,0.2996332157950631,665.6 203 | 199b8dee04a6102cc90b8a4246d77dc265478738,0.2705579957447134,614.4 204 | 12df124ee35dc02d5792d17c13f9c82555a05952,0.265598349679863,614.4 205 | 0d27386a8eb982f9f7d4929c5530e1b3d13999a6,0.24565925126884977,563.2 206 | 2a3f38e74425dc30b36843493d0b9c556ffe52be,0.20458557985358036,563.2 207 | efbfa03082c946e11756db451599d0bc86963efe,0.23037583678711213,537.6 208 | c3988b302e5665646eed7a084edf0072127607c1,0.2448877594169398,563.2 209 | 74517aec68f62452f42ba6e8031cd40e274158a0,0.24522065258046494,537.6 210 | 08ee1732d20152d3d3f3793f59f2722f9f8644d4,0.24559014435088783,512.0 211 | aa84a6559e38722e18491ae7d9d0f775b706dae9,0.24507165457711697,563.2 212 | 8f32449efe51e703d88eef2848e03e21315785a1,0.1762909369839351,512.0 213 | 2fd0ee03864d128296f99279fe64515aec69c8b5,0.21412419525237272,499.2 214 | b83f77e89fe4d4c75e32e331a4acef08ff80ba2d,0.23844472114611098,409.6 215 | 63cba9a51edc237aee06ed2d3661504ade23a4fa,0.21776105412904537,230.4 216 | eed00116f467f6abb1c48f9cf939cf7934ce23e6,0.20519968858156476,153.6 217 | 71c5fdd0b8434e9ad5dba93e4eb4d14c05cc1337,0.24568458809544444,563.2 218 | b910cd5e6ebe12b393ecbf592eb61c3227284f11,0.21160642207361888,563.2 219 | 2899a93f3197f767f00469260b9fa706ac871c48,0.23054231009638046,537.6 220 | 6606c1a2b51a0012f0b9500372537e643dbac3a7,0.24438553916326344,563.2 221 | 35552528a247abc7ee3e9331c17a30da16e75fbd,0.2345035290967474,537.6 222 | dcad5fb0dce74c26fd9e2d06d7bda0205c744d64,0.2372642300178356,512.0 223 | 8fc8d33469cf8e1c382d50292f32d295b5b715a9,0.98225027978014,2265.6 224 | 590542ac63f34bd1030ec94bd84b8594f6b66b6f,0.9816466604200774,2252.8 225 | bcb0e9ffddc8d75a7f33414a01329a7842cbed6d,0.9718377834252797,2265.6 226 | 553975ac3eda56d421085211c8fcbf5fa9a1a56b,0.962037600900833,2201.6 227 | 0230631a4edd5215bd68fe673680490f59b2c651,0.743843058768782,1996.8 228 | 8ef8c3f45a8be3c68cde9497d8bf801f95063232,0.6916857037261266,1536.0 229 | 3915ca93a60f1be57bcd0192546aea843f1e1ff8,0.9828789913213457,2291.2 230 | f01e7389c765177d8ce078685b71af8cf0e6ae14,0.9820485094168021,2278.4 231 | 30c6e16aea1d05f13f62747b912890e38a6a983d,0.9688561028483287,2252.8 232 | 889daebae8a8c3ef9acbad9dd5422ebc43879b0f,0.9525460175341701,2201.6 233 | 907663fbc8996f9017430a0c370dac60f047a656,0.8592537489335934,1920.0 234 | 81eff808789704701ca6f56136f718e85f5fed8b,0.6171121891986322,1536.0 235 | e65e39d1b56db0d8e9a56d3a54d8105db80e5a0f,0.9810870252539807,2265.6 236 | 8e9bf9011b8a4d78a5285835ba8ab450a810a936,0.9775045941080341,2278.4 237 | 62892136aab789697b6fa2550dc679cf903d9b21,0.9695341512903021,2252.8 238 | 7bbc8ea1c0e8210b946b182f7b148d4fff261c9b,0.9428593469832472,2150.4 239 | 42988f992a6e84ffed5e35b9a7fa6dafe57ebe30,0.7986200613961032,1766.4 240 | c1e0609ae654b5dc03b71c7ce82e770efc23c44d,0.5975336122944528,1433.6 241 | 6093fb8830d396368ebe3aed7280b494c64703ee,0.9812276595794341,2265.6 242 | a7e468bec247355ee7c3c60ab94426d46442c880,0.9776456558838711,2265.6 243 | 8acff69bf05de6cf51214175ed69547b7b25eca0,0.9712202630055556,2252.8 244 | 74451f0311980457a81ff9d0b1286f44da30ba04,0.9530057915383608,2201.6 245 | 83ba508a389ae174141e004205afd7b7bd53bc5a,0.8324730773056095,1843.2 246 | 9600fb26a41375e4c23e75f958ffed5fc0004911,0.6819887576557888,1638.4 247 | a3137fe8b664685ff540c7f7e2bf1eee2eef7937,0.9814035930073304,2278.4 248 | 3d7a1d93c47ce5f0a941e2b506be8f0b133ddceb,0.9831825601117996,2252.8 249 | f6bbf16fa14a8f1902aabc248e49e80fa70ac8e4,0.9723484075061715,2240.0 250 | fc9a6e4a396cbf6a4fb08ce94b8bc17d808e6bdb,0.95413957040152,2201.6 251 | f1684bcfa011e006e5f84e550166d80b3ecf847e,0.7323726902272404,1920.0 252 | 9ae52a762c42bd1d972d7702d7ffa704c04919d1,0.6792306758418771,1433.6 253 | 36c1c7621990ebb0ff76974f22f660d0b14501f8,0.9815896806523924,2291.2 254 | 0ad3a1c0129e4ff2af9e8bf747ca2dd5382907fc,0.9832107288176324,2265.6 255 | 5713bc85a14b60c1999499423763b4ae5fc4349a,0.9721712004776069,2240.0 256 | 0e7efbc5cce74761c8ff11e0afce44d8591f2109,0.9554557864201286,2201.6 257 | 42bc76d7ef64f4975f17468d40afbd1292fcffa9,0.8479308294160478,1920.0 258 | 84eb62ad71fc1ad7f95b001038a4400a7142e9eb,0.6200014676403736,1536.0 259 | 07636118211ac1521652ce242fa1d54302676d88,0.9813238598740649,2278.4 260 | e15bcdfc762baa6ea37152688d59e92ed12b4123,0.9837594775863264,2265.6 261 | 140094b325f3e7eedd1f1b0aeaac193408e88e86,0.9722831434065099,2265.6 262 | 6a51c80549f757eb899081280f313a62ce88df06,0.9570366767376951,2201.6 263 | d1750aa94090c574e166b30cd76227bcf68ad588,0.7439383803787923,1920.0 264 | 342a218ac3bdf65dc10841fddf9cf9ebafac6648,0.6156740332794499,1536.0 265 | 2520b0b1209a0178c17549aa36548108ef6c8ca2,0.9811693355763595,2278.4 266 | 2823163753b6d4b57ff54447de2fa46a71a317e8,0.9831770259128771,2291.2 267 | 6d0743f308cc270ae16c0612f86a615209fc81fa,0.9725347871410654,2265.6 268 | feded60314013c479805c12156f3c52442e55022,0.9556314106927738,2201.6 269 | f4c2b92c3936bc931100d78c3b932ae4d4885950,0.8501852612115651,1920.0 270 | a1d7f3d9fa5126e9237a726761c25226679cb585,0.621199212681995,1536.0 271 | 145a398dbf8b9adc33fb04c5b4e9e0238bad617c,0.9809723526122166,2265.6 272 | be00ea8b3aa3e410da6ed5bca10d40bfc0b7a9e1,0.9830486224864514,2252.8 273 | 5bd41255645bea6e2a25de8e130fe16719eae456,0.9724028285961636,2240.0 274 | e9dfcdf3c6af6cf19e7dc2f5efaf38a29dff47eb,0.9567981623258879,2201.6 275 | 8ae29ed6d2c96d1a85f3841d733c80fcb831099f,0.8472192628866694,1920.0 276 | ab899eca32eba6b65ffac2fe8a6f23f66ffa6a1c,0.6209704528668339,1536.0 277 | d4ca3d30c5be3e87b4d9312df262199243a21aff,0.9807109332418469,2265.6 278 | acedb26e9bc5d278d7effe0bd21d38877fa07b4b,0.9833263574963219,2278.4 279 | eea042f51cf9169e1aaaf2e4c59c13336c478dab,0.9728693309022406,2265.6 280 | 3b2524142b902eb71da718864ad878c55e0ce0cc,0.9573278903089273,2201.6 281 | c895299c55317e54b7a028e3569e0909391f1424,0.8510417188249944,1920.0 282 | ab45beec1064febbcf60d541ed30027a84ef2a61,0.6932859315364809,1536.0 283 | 386fb515d708da9b00e53a71d1290375a1cbdc60,0.9806899387045914,2291.2 284 | 3d0de1add0245618f2c8a31918dc6df1bb1f00a7,0.9773535602718022,2265.6 285 | 2922296c86e1f328c3586a8e288e49cba0725814,0.9727737150335046,2240.0 286 | 551b38041215de51f60ee17a42d32daaaf197f31,0.9578208184722501,2201.6 287 | 7329d55631d89de2f3769db8a13b3bae78ca8772,0.7449716476624425,1920.0 288 | b9be494fbe7599be2cebb00c13f31c9afd1536f7,0.6200692912023773,1536.0 289 | 35aac7fcba104b7889f41353d366c37c4b5069af,0.9806155532969941,2278.4 290 | 190ce95a0fac078c2c3d73e53694e6932d3efbc0,0.9772342481810428,2265.6 291 | 969b5270499607d72b4ff276195c14fcb999e102,0.9725223490166663,2265.6 292 | b32a54c3d7572468064fb0736753a7fbc9f2bf9c,0.9583167344748338,2201.6 293 | 2723a313a062c012689eebd5d7a687dd0448560a,0.74689135119229,1920.0 294 | 93a22e124ea271eae48bde91273d0fb63fbd5d51,0.6187853733751278,1536.0 295 | fa0980ca9f7fac3a25f6f83523d2dcd2b43c555c,0.9807155382853996,2291.2 296 | f222e7efd1ef5b7c9d13a2db57c894ad462d456e,0.977395583099358,2278.4 297 | ab6d86b1c0892a95ddb294ebba0ada698278af86,0.9725154370768319,2252.8 298 | ae9bfd28d22013a3dd0945cdacd4c1fd6c664346,0.9575631414935558,2201.6 299 | bef8c680904351d3e5d7433c53d3fe50b2a29608,0.8515553815564176,1920.0 300 | af7b77444c56e08be36d941a5fc182618b44867c,0.6181406939968134,1536.0 301 | 76e0ceafa7411b370117f15d33ba053ec2824648,0.9806894087910822,2291.2 302 | 58c764cf0debf74c50a5f4c5cc47a6f9b3738903,0.9827054162896316,2252.8 303 | db70b815049e2ea448b6f9aa2defa0910aa549b5,0.9726419575328829,2252.8 304 | c1ff0887a787969c6cb66a862cfa2f63f31a39a8,0.9572258809775904,2201.6 305 | 4e2b6ca0f7884be7423cf79b4aa542010091dbef,0.8547338797493189,1920.0 306 | 673cd9762e9459467f6c845a05cdd6ce58c27753,0.6925361990953569,1536.0 307 | e045464663629f3c69d3ecef7a4c1024d172e8a7,0.24600525481621813,563.2 308 | 575fc753b344e7c4ef506d1010156082177d6c3c,0.24459322312220333,563.2 309 | e816e4da3f8d25172fd702446774faa494ac9741,0.24476288434988025,537.6 310 | 5886b4871ea65c75791a935e0ab6b6abde5b2259,0.22937149895041656,563.2 311 | 3827283729c1fb5e5107e3c88dc3d300d83197f2,0.23482940293082483,537.6 312 | 2fec75a91a2c9671054963a87a3e60faa40ad0fa,0.23775461314275304,512.0 313 | f5a13eeb68987460b487a5f93b0e5dbeb1c8e8d8,0.48877350312414886,1126.4 314 | 035c00f4ddfb4f918d40311a820d3712a329b1e1,0.4784550129525197,1100.8 315 | 9e3cfe851e0ec827d9a1ac0de4d1b94f69e91c08,0.44968665491969323,998.4 316 | 76835783060db8c7dde7ed47f9895a06920afd55,0.4152459917622986,921.6 317 | 0045382b84b161a7f01d0e383d0194967ba0ce73,0.36626332598819417,768.0 318 | e4460524f90346d54710c96f29091dd7d0a0789b,0.31650503411266034,716.8 319 | be1e737ebb6e52b4b92143ad3a5b4d508954ecaa,0.9807702970202968,2252.8 320 | f62f248cbad5284c6e608d8882d7ede8e66f5578,0.9827166331717698,2265.6 321 | f283c995c5bb02a26f17fc077d338033a0401018,0.9658800529118128,2227.2 322 | 6c5ef1abee9f988f695b3299adf5434b00cf7c8a,0.9314622725096875,2112.0 323 | 270a6585dfad8d525bf366826339531b0aa2256e,0.695624259404621,1766.4 324 | cf1f9050df939deccbc136a3f7eb58f7468314ca,0.6168484890207883,1523.2 325 | e88a2f0074d471f37a5c3ea20026aa58f0a94439,0.981101209330254,2265.6 326 | 67e4f6d0f28771b6b344dd505d96f7b35e6ab4a2,0.9830864097835638,2252.8 327 | 64f24e38fa14db6f84d867383cfc08249172b2cd,0.9687856471601614,2227.2 328 | 23320d93aa6c10746a3a9465ff01dd6b55f3f8d2,0.9370678351940256,2150.4 329 | 6469f2b01d879569e015c8728c7cef51f98672c6,0.7977637286216162,1766.4 330 | fca7da96a4a8fa4217ced397b7c3957aa9225803,0.6084743079421713,1523.2 331 | --------------------------------------------------------------------------------