├── .github └── CODEOWNERS ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README-sc20.txt ├── README.md ├── config-debug-run.ini ├── config-minimal.ini ├── config-scc.ini ├── contrib └── s3 │ ├── README.md │ ├── config-s3.ini │ └── prepare-s3.sh ├── find ├── sfind.ini └── sfind.sh ├── include ├── io500-debug.h ├── io500-opt.h ├── io500-phase.h └── io500-util.h ├── io500.sh ├── prepare.sh ├── src ├── ini-parse.c ├── main.c ├── phase-definitions.h ├── phase_dbg.c ├── phase_find.c ├── phase_find.h ├── phase_find_easy.c ├── phase_find_hard.c ├── phase_ior.c ├── phase_ior.h ├── phase_ior_easy.c ├── phase_ior_easy_read.c ├── phase_ior_easy_write.c ├── phase_ior_hard.c ├── phase_ior_hard_read.c ├── phase_ior_hard_write.c ├── phase_ior_rnd1MB.c ├── phase_ior_rnd4K.c ├── phase_ior_rnd_read1MB.c ├── phase_ior_rnd_read4K.c ├── phase_ior_rnd_read4k-easywrite.c ├── phase_ior_rnd_write1MB.c ├── phase_ior_rnd_write4K.c ├── phase_mdtest.c ├── phase_mdtest.h ├── phase_mdtest_easy.c ├── phase_mdtest_easy_delete.c ├── phase_mdtest_easy_stat.c ├── phase_mdtest_easy_write.c ├── phase_mdtest_hard.c ├── phase_mdtest_hard_delete.c ├── phase_mdtest_hard_read.c ├── phase_mdtest_hard_stat.c ├── phase_mdtest_hard_write.c ├── phase_mdworkbench.c ├── phase_mdworkbench.h ├── phase_mdworkbench_bench.c ├── phase_mdworkbench_create.c ├── phase_mdworkbench_delete.c ├── phase_opt.c ├── phase_timestamp.c ├── test │ ├── CMakeFiles │ │ ├── CMakeDirectoryInformation.cmake │ │ ├── metatest.dir │ │ │ ├── DependInfo.cmake │ │ │ ├── build.make │ │ │ ├── cmake_clean.cmake │ │ │ ├── depend.make │ │ │ ├── flags.make │ │ │ ├── link.txt │ │ │ └── progress.make │ │ └── progress.marks │ ├── CMakeLists.txt │ ├── CTestTestfile.cmake │ ├── Makefile │ ├── cmake_install.cmake │ └── metatest.c ├── util.c └── verifier.c └── test └── ini-test.c /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # This is a comment. 2 | # Each line is a file pattern followed by one or more owners. 3 | # The pattern is followed by one or more GitHub usernames or team 4 | # names using the standard @username or @org/team-name format. 5 | # You can also refer to a user by an email address that has been 6 | # added to their GitHub Enterprise account, e.g. user@example.com. 7 | # These owners will be the default owners for everything in all 8 | # repos in the io500 org, unless a later match takes precedence, 9 | # they will be requested for review when someone opens a pull request. 10 | * @io500/board 11 | 12 | # Order is important; the last matching pattern takes the most 13 | # precedence. When someone opens a pull request that only 14 | # modifies JS files, only @js-owner and not the global 15 | # owner(s) will be requested for a review. 16 | #*.js @js-owner 17 | 18 | # You can also use email addresses if you prefer. They'll be 19 | # used to look up users just like we do for commit author 20 | # emails. 21 | #*.go docs@example.com 22 | 23 | # In this example, @doctocat owns any files in the build/logs 24 | # directory at the root of the repository and any of its 25 | # subdirectories. 26 | #/build/logs/ @doctocat 27 | 28 | # The `docs/*` pattern will match files like 29 | # `docs/getting-started.md` but not further nested files like 30 | # `docs/build-app/troubleshooting.md`. 31 | #docs/* docs@example.com 32 | 33 | # In this example, @octocat owns any file in an apps directory 34 | # anywhere in your repository. 35 | #apps/ @octocat 36 | 37 | # In this example, @doctocat owns any file in the `/docs` 38 | # directory in the root of your repository. 39 | #/docs/ @doctocat 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | results/ 2 | build/ 3 | out/ 4 | bin/ 5 | core 6 | vgcore.* 7 | io500 8 | io500-verify 9 | system-information.txt 10 | 11 | # Temporary files 12 | *.orig 13 | *.tmp 14 | *.swo 15 | *.swp 16 | *.diff 17 | *.patch 18 | *~ 19 | tags 20 | TAGS 21 | 22 | # Prerequisites 23 | *.d 24 | 25 | # Object files 26 | *.o 27 | *.ko 28 | *.obj 29 | *.elf 30 | 31 | # Linker output 32 | *.ilk 33 | *.map 34 | *.exp 35 | 36 | # Precompiled Headers 37 | *.gch 38 | *.pch 39 | 40 | # Libraries 41 | *.lib 42 | *.a 43 | *.la 44 | *.lo 45 | 46 | # Shared objects (inc. Windows DLLs) 47 | *.dll 48 | *.so 49 | *.so.* 50 | *.dylib 51 | 52 | # Executables 53 | *.exe 54 | *.out 55 | *.app 56 | *.i*86 57 | *.x86_64 58 | *.hex 59 | 60 | # Debug files 61 | *.dSYM/ 62 | *.su 63 | *.idb 64 | *.pdb 65 | 66 | # Kernel Module Compile Results 67 | *.mod* 68 | *.cmd 69 | .tmp_versions/ 70 | modules.order 71 | Module.symvers 72 | Mkfile.old 73 | dkms.conf 74 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | # SC-20 4 | * Remove use of older bash script, only use C-app run for submission 5 | 6 | # ISC-20 7 | * Combined script/C-app runs via io500.sh to validate continuity of results 8 | * Support hashing of configuration option and optained result to allow verification of results. 9 | * Pfind provides additional options for parallelizing large directories (see pfind directory) and diagnostics output. 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 The Virtual Institute for I/O 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CC = mpicc 2 | CFLAGS += -std=gnu99 -Wall -Wempty-body -Werror -Wstrict-prototypes -Werror=maybe-uninitialized -Warray-bounds 3 | 4 | IORCFLAGS = $(shell grep CFLAGS ./build/ior/src/build.conf | cut -d "=" -f 2-) 5 | CFLAGS += -g3 -lefence -I./include/ -I./src/ -I./build/pfind/src/ -I./build/ior/src/ 6 | IORLIBS = $(shell grep LDFLAGS ./build/ior/src/build.conf | cut -d "=" -f 2-) 7 | LDFLAGS += -lm $(IORCFLAGS) $(IORLIBS) # -lgpfs # may need some additional flags as provided to IOR 8 | 9 | VERSION_GIT=$(shell git describe --always --abbrev=12) 10 | VERSION_TREE=$(shell git diff src | wc -l | sed -e 's/ *//g' -e 's/^0//' | sed "s/\([0-9]\)/-\1/") 11 | VERSION=$(VERSION_GIT)$(VERSION_TREE) 12 | CFLAGS += -DVERSION="\"$(VERSION)\"" 13 | PROGRAM = io500 14 | VERIFIER = io500-verify 15 | SEARCHPATH += src 16 | SEARCHPATH += include 17 | SEARCHPATH += test 18 | vpath %.c $(SEARCHPATH) 19 | vpath %.h $(SEARCHPATH) 20 | .SUFFIXES: 21 | 22 | DEPS += io500-util.h io500-debug.h io500-opt.h 23 | OBJSO += util.o 24 | OBJSO += ini-parse.o phase_dbg.o phase_opt.o phase_timestamp.o 25 | OBJSO += phase_find.o phase_find_easy.o phase_find_hard.o phase_ior_easy.o phase_ior_easy_read.o phase_mdtest.o phase_ior.o phase_ior_easy_write.o phase_ior_hard.o phase_ior_hard_read.o phase_ior_hard_write.o phase_mdtest_easy.o phase_mdtest_easy_delete.o phase_mdtest_easy_stat.o phase_mdtest_easy_write.o phase_mdtest_hard.o phase_mdtest_hard_delete.o phase_mdtest_hard_read.o phase_mdtest_hard_stat.o phase_mdtest_hard_write.o phase_ior_rnd1MB.o phase_ior_rnd4K.o phase_ior_rnd_write4K.o phase_ior_rnd_read4K.o phase_ior_rnd_write1MB.o phase_ior_rnd_read1MB.o phase_mdworkbench.o phase_mdworkbench_create.o phase_mdworkbench_delete.o phase_mdworkbench_bench.o phase_ior_rnd_read4k-easywrite.o 26 | 27 | OBJS = $(patsubst %,./build/%,$(OBJSO)) 28 | 29 | TESTS += ini-test 30 | TESTSEXE = $(patsubst %,./build/%.exe,$(TESTS)) 31 | 32 | all: $(VERIFIER) $(PROGRAM) $(TESTSEXE) 33 | 34 | clean: 35 | @echo CLEAN 36 | @$(RM) ./build/*.o ./build/io500.a *.exe $(PROGRAM) 37 | 38 | ./build/io500.a: $(OBJS) 39 | @echo AR $@ 40 | ar rcsT $@ $(OBJS) 41 | 42 | $(VERIFIER): ./build/verifier.o ./build/io500.a 43 | @echo LD $@ 44 | $(CC) -o $@ ./build/verifier.o ./build/io500.a $(LDFLAGS) 45 | 46 | $(PROGRAM): ./build/io500.a ./build/main.o 47 | @echo LD $@ 48 | $(CC) -o $@ ./build/main.o $(LDFLAGS) ./build/io500.a ./build/pfind/pfind.a ./build/ior/src/libaiori.a $(LDFLAGS) 49 | 50 | .PHONY: ./build/main.o 51 | ./build/main.o: main.c $(DEPS) 52 | @echo CC $@ 53 | $(CC) $(CFLAGS) -c -o $@ $< 54 | 55 | ./build/%.o: %.c $(DEPS) 56 | @echo CC $@ 57 | $(CC) $(CFLAGS) -c -o $@ $< 58 | 59 | ./build/%.exe: ./build/%.o $(DEPS) ./build/io500.a 60 | @echo LD $@ 61 | $(CC) -o $@ $< $(LDFLAGS) ./build/io500.a 62 | -------------------------------------------------------------------------------- /README-sc20.txt: -------------------------------------------------------------------------------- 1 | In the SC20 list, only the new C-application benchmark will be used. 2 | 3 | The io500.sh script can optionally be used to prepare the output 4 | directories, launch the io500 application, and collect the test results 5 | and logs for submission, or the io500 binary can be run directly. 6 | 7 | The test results will be put into a separate directory and tarred up for 8 | submission. At a minimum, the io500.sh script needs the io500_mpirun and 9 | io500_mpiargs values set for your system in the setup_paths() function 10 | at the start. The paths to the installed IOR and mdtest binaries are 11 | also needed if they were installed separately. The script must be run 12 | from within the Git checkout tree. 13 | 14 | In detail, to setup the run: 15 | 1) Create a system information -- this can be done later after the 16 | run as well but please ensure it is consistent. 17 | Visit the page: https://vi4io.org/io500-info-creator/ 18 | and store the output into "system-information.txt" 19 | 20 | 2) Setup a configuration, the minimal configuration is provided in 21 | config-minimal.ini describing the data directory. 22 | To see available options, either see the "config-full.ini" file or run 23 | $ ./io500 --list 24 | Change the MPI options in io500.sh 25 | io500_mpiargs='-np 2' 26 | io500_mpirun="mpiexec" 27 | 28 | It may also be desirable to update the setup_directories() function 29 | to create the top-level output directories for the test files and set, 30 | e.g., striping parameters for Lustre. These directories are created 31 | before the run, and may hold directory and file layout parameters. 32 | 33 | 3) Create a batch submission file which executes the benchmark script. 34 | This in turn runs the io500 application using: 35 | $ ./io500.sh 36 | Example: 37 | $ ./io500.sh config-minimal.ini 38 | This will then automatically create a .tar file for upload. 39 | 40 | 4) Submit the .tgz file together with system-information.txt and the 41 | batch file via the submission page. 42 | 43 | You can find examples for running the scripts and the generated output 44 | * for Lustre: http://vi4io.org/assets/io500/2020-06/isc-20-dkrz-10nodes.tgz 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # io500 2 | 3 | This is the C version of the IO500 benchmark. 4 | 5 | ## Preparation 6 | 7 | The program interfaces directly with IOR/MDtest and pfind. 8 | To retrieve the required packages and compile the library version, run 9 | 10 | $ ./prepare.sh 11 | 12 | Then you can compile the io500 application running make. 13 | 14 | ## Usage 15 | 16 | The benchmark requires a .ini file containing the options. 17 | The .ini file is structured in sections depending on the phase. 18 | 19 | Detailed help for the available options is provided when running: 20 | 21 | $ ./io500 -h 22 | Synopsis: ./io500 [-v=] [--dry-run] 23 | 24 | The benchmark output the commands it would run (equivalence of command line invocations of ior/mdtest). Use --dry-run to not invoke any command. 25 | 26 | In order to create a new INI file with all the options, you can execute: 27 | 28 | $ ./io500 --list > config-all.ini 29 | 30 | The config-some illustrates the setting of various options. For more details, run ./io500 -h. 31 | 32 | To see the currently active options, run: 33 | 34 | $ ./io500 -h 35 | 36 | 37 | ### Integrity check 38 | 39 | After a run is completed, the score obtained and the configuration file can be verified to ensure that it wasn't accidentially modified. 40 | 41 | You can either use the full-featured io500 application: 42 | 43 | $ ./io500 config-test-run.ini --verify result.txt 44 | config-hash = 1065C0D 45 | score-hash = C97CC873 46 | [OK] But this is an invalid run! 47 | 48 | Or the lightweight verification tool which has fewer dependencies: 49 | 50 | $ ./io500-verify config-test-run.ini result.txt 51 | 52 | ## Output 53 | 54 | - The benchmark will output a default set of information in the INI format to simplify parsing. When setting verbosity to 5, you will receive more information. 55 | - It also stores the output files from IOR and MDTest in the results/ subdirectory with the timestamp of the run. 56 | 57 | ### Example output on the command line 58 | 59 | The following is the minimal output when setting verbosity to 0 60 | 61 | $ mpiexec -np 2 ./io500 config-minimal.ini 62 | [RESULT] ior-easy-write 0.186620 GiB/s : time 0.027 seconds 63 | [RESULT] mdtest-easy-write 103.300821 kIOPS : time 1.121 seconds 64 | [RESULT] ior-hard-write 0.001313 GiB/s : time 0.067 seconds 65 | [RESULT] mdtest-hard-write 58.939081 kIOPS : time 1.021 seconds 66 | [RESULT] find 1486.435084 kIOPS : time 0.118 seconds 67 | [RESULT] ior-easy-read 1.575557 GiB/s : time 0.005 seconds 68 | [RESULT] mdtest-easy-stat 839.392805 kIOPS : time 0.138 seconds 69 | [RESULT] ior-hard-read 2.272671 GiB/s : time 0.000 seconds 70 | [RESULT] mdtest-hard-stat 1212.558124 kIOPS : time 0.050 seconds 71 | [RESULT] mdtest-easy-delete 160.765642 kIOPS : time 0.753 seconds 72 | [RESULT] mdtest-hard-read 275.011939 kIOPS : time 0.219 seconds 73 | [RESULT] mdtest-hard-delete 132.015851 kIOPS : time 0.474 seconds 74 | [SCORE INVALID] Bandwidth 0.172092 GB/s : IOPS 292.625029 kiops : TOTAL 7.096374 75 | 76 | This information is also saved in the file result_summary.txt in the respective results directory. 77 | 78 | In the same directory, you will also find the result.txt file that contains more information and is stored using the INI file format. 79 | 80 | version = SC20-testing 81 | config-hash = 25C33C96 82 | result-dir = ./results/ 83 | ; START 2020-01-06 10:23:49 84 | ; ERROR INVALID stonewall-time != 300 85 | 86 | 87 | [ior-easy-write] 88 | t_start = 2020-01-06 10:23:49 89 | exe = ./ior -C -Q 1 -g -G 271 -k -e -o ./out//ior-easy/ior_file_easy -O stoneWallingStatusFile=./out//ior-easy/stonewall -O stoneWallingWearOut=1 -t 2m -b 2m -F -w -D 1 -a POSIX 90 | ; ERROR INVALID Write phase needed 0.020932s instead of stonewall 1s. Stonewall was hit at 0.0s 91 | throughput-stonewall = 0.37 92 | score = 0.186620 93 | ; ERROR INVALID Runtime of phase (0.027088) is below stonewall time. This shouldn't happen! 94 | t_delta = 0.0271 95 | t_end = 2020-01-06 10:23:49 96 | 97 | [mdtest-easy-write] 98 | t_start = 2020-01-06 10:23:49 99 | exe = ./mdtest -n 1000000 -u -L -F -N 1 -d ./out//mdtest-easy -x ./out//mdtest-easy-stonewall -C -W 1 -a POSIX 100 | rate-stonewall = 109.492799 101 | score = 103.300821 102 | t_delta = 1.1207 103 | t_end = 2020-01-06 10:23:50 104 | 105 | [timestamp] 106 | t_start = 2020-01-06 10:23:50 107 | t_delta = 0.0000 108 | t_end = 2020-01-06 10:23:50 109 | 110 | [ior-hard-write] 111 | t_start = 2020-01-06 10:23:50 112 | exe = ./ior -C -Q 1 -g -G 27 -k -e -o ./out//ior-hard/file -O stoneWallingStatusFile=./out//ior-hard/stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 1 -w -D 1 -a POSIX 113 | ; ERROR INVALID Write phase needed 0.066709s instead of stonewall 1s. Stonewall was hit at 0.0s 114 | throughput-stonewall = 0.00 115 | score = 0.001313 116 | ; ERROR INVALID Runtime of phase (0.067146) is below stonewall time. This shouldn't happen! 117 | t_delta = 0.0671 118 | t_end = 2020-01-06 10:23:50 119 | 120 | [mdtest-hard-write] 121 | t_start = 2020-01-06 10:23:50 122 | exe = ./mdtest -n 1000000 -t -w 3901 -e 3901 -N 1 -F -d ./out//mdtest-hard -x ./out//mdtest-hard-stonewall -C -W 1 -a POSIX 123 | rate-stonewall = 59.263301 124 | score = 58.939081 125 | t_delta = 1.0207 126 | t_end = 2020-01-06 10:23:51 127 | 128 | [find] 129 | t_start = 2020-01-06 10:23:51 130 | exe = ./pfind ./out/ -newer ./results//timestampfile -size 3901c -name *01* -C -H 1 -q 10000 131 | found = 1596 132 | total-files = 175761 133 | score = 1486.435084 134 | t_delta = 0.1184 135 | t_end = 2020-01-06 10:23:51 136 | 137 | [ior-easy-read] 138 | t_start = 2020-01-06 10:23:51 139 | exe = ./ior -C -Q 1 -g -G 271 -k -e -o ./out//ior-easy/ior_file_easy -O stoneWallingStatusFile=./out//ior-easy/stonewall -O stoneWallingWearOut=1 -t 2m -b 2m -F -r -R -a POSIX 140 | score = 1.575557 141 | t_delta = 0.0054 142 | t_end = 2020-01-06 10:23:51 143 | 144 | [mdtest-easy-stat] 145 | t_start = 2020-01-06 10:23:51 146 | exe = ./mdtest -n 1000000 -u -L -F -N 1 -d ./out//mdtest-easy -x ./out//mdtest-easy-stonewall -T -a POSIX 147 | score = 839.392805 148 | t_delta = 0.1381 149 | t_end = 2020-01-06 10:23:51 150 | 151 | [ior-hard-read] 152 | t_start = 2020-01-06 10:23:51 153 | exe = ./ior -C -Q 1 -g -G 27 -k -e -o ./out//ior-hard/file -O stoneWallingStatusFile=./out//ior-hard/stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 1 -r -R -a POSIX 154 | score = 2.272671 155 | t_delta = 0.0004 156 | t_end = 2020-01-06 10:23:51 157 | 158 | [mdtest-hard-stat] 159 | t_start = 2020-01-06 10:23:51 160 | exe = ./mdtest -n 1000000 -t -w 3901 -e 3901 -N 1 -F -d ./out//mdtest-hard -x ./out//mdtest-hard-stonewall -T -a POSIX 161 | score = 1212.558124 162 | t_delta = 0.0499 163 | t_end = 2020-01-06 10:23:51 164 | 165 | [mdtest-easy-delete] 166 | t_start = 2020-01-06 10:23:51 167 | exe = ./mdtest -n 1000000 -u -L -F -N 1 -d ./out//mdtest-easy -x ./out//mdtest-easy-stonewall -r -a POSIX 168 | score = 160.765642 169 | t_delta = 0.7534 170 | t_end = 2020-01-06 10:23:52 171 | 172 | [mdtest-hard-read] 173 | t_start = 2020-01-06 10:23:52 174 | exe = ./mdtest -n 1000000 -t -w 3901 -e 3901 -N 1 -F -d ./out//mdtest-hard -x ./out//mdtest-hard-stonewall -E -X -a POSIX 175 | score = 275.011939 176 | t_delta = 0.2191 177 | t_end = 2020-01-06 10:23:52 178 | 179 | [mdtest-hard-delete] 180 | t_start = 2020-01-06 10:23:52 181 | exe = ./mdtest -n 1000000 -t -w 3901 -e 3901 -N 1 -F -d ./out//mdtest-hard -x ./out//mdtest-hard-stonewall -r -a POSIX 182 | score = 132.015851 183 | t_delta = 0.4742 184 | t_end = 2020-01-06 10:23:53 185 | 186 | [SCORE] 187 | MD = 292.625029 188 | BW = 0.172092 189 | SCORE = 7.096374 [INVALID] 190 | hash = 884F29B 191 | ; END 2020-01-06 10:23:53 192 | -------------------------------------------------------------------------------- /config-debug-run.ini: -------------------------------------------------------------------------------- 1 | [global] 2 | datadir = ./datafiles 3 | timestamp-datadir = FALSE 4 | timestamp-resultdir = FALSE 5 | 6 | [debug] 7 | stonewall-time = 1 8 | pause-dir = ./pause/ 9 | 10 | -------------------------------------------------------------------------------- /config-minimal.ini: -------------------------------------------------------------------------------- 1 | [global] 2 | datadir = ./datafiles 3 | -------------------------------------------------------------------------------- /config-scc.ini: -------------------------------------------------------------------------------- 1 | [global] 2 | datadir = ./datafiles 3 | timestamp-datadir = TRUE 4 | resultdir = ./results 5 | timestamp-resultdir = TRUE 6 | api = POSIX 7 | drop-caches = FALSE 8 | drop-caches-cmd = sudo -n bash -c "echo 3 > /proc/sys/vm/drop_caches" 9 | io-buffers-on-gpu = FALSE 10 | verbosity = 1 11 | scc = TRUE 12 | dataPacketType = timestamp 13 | 14 | [debug] 15 | stonewall-time = 30 16 | 17 | [ior-easy] 18 | API = 19 | transferSize = 2m 20 | blockSize = 9920000m 21 | filePerProc = TRUE 22 | uniqueDir = FALSE 23 | run = TRUE 24 | verbosity = 25 | 26 | [ior-easy-write] 27 | API = 28 | run = TRUE 29 | 30 | [mdtest-easy] 31 | API = 32 | n = 1000000 33 | run = TRUE 34 | 35 | [mdtest-easy-write] 36 | API = 37 | run = TRUE 38 | 39 | [timestamp] 40 | 41 | [find-easy] 42 | external-script = 43 | external-mpi-args = 44 | external-extra-args = 45 | nproc = 46 | run = TRUE 47 | pfind-queue-length = 10000 48 | pfind-steal-next = FALSE 49 | pfind-parallelize-single-dir-access-using-hashing = FALSE 50 | 51 | [ior-hard] 52 | API = 53 | segmentCount = 10000000 54 | collective = 55 | run = TRUE 56 | verbosity = 57 | 58 | [ior-hard-write] 59 | API = 60 | collective = 61 | run = TRUE 62 | 63 | [mdtest-hard] 64 | API = 65 | n = 1000000 66 | files-per-dir = 67 | run = TRUE 68 | 69 | [mdtest-hard-write] 70 | API = 71 | run = TRUE 72 | 73 | [find] 74 | external-script = 75 | external-mpi-args = 76 | external-extra-args = 77 | nproc = 78 | run = TRUE 79 | pfind-queue-length = 10000 80 | pfind-steal-next = FALSE 81 | pfind-parallelize-single-dir-access-using-hashing = FALSE 82 | 83 | [find-hard] 84 | external-script = 85 | external-mpi-args = 86 | external-extra-args = 87 | nproc = 88 | run = FALSE 89 | pfind-queue-length = 10000 90 | pfind-steal-next = FALSE 91 | pfind-parallelize-single-dir-access-using-hashing = FALSE 92 | 93 | [mdworkbench-bench] 94 | run = FALSE 95 | 96 | [ior-easy-read] 97 | API = 98 | run = TRUE 99 | 100 | [mdtest-easy-stat] 101 | API = 102 | run = TRUE 103 | 104 | [ior-hard-read] 105 | API = 106 | collective = 107 | run = TRUE 108 | 109 | [mdtest-hard-stat] 110 | API = 111 | run = TRUE 112 | 113 | [mdtest-easy-delete] 114 | API = 115 | run = TRUE 116 | 117 | [mdtest-hard-read] 118 | API = 119 | run = TRUE 120 | 121 | [mdtest-hard-delete] 122 | API = 123 | run = TRUE 124 | 125 | -------------------------------------------------------------------------------- /contrib/s3/README.md: -------------------------------------------------------------------------------- 1 | # io500-s3 2 | 3 | The following explains how to use the IO500 to benchmark S3 compatible storage. 4 | S3 runs are not fully compliant at the time of writing because find is not yet supported, but this might be useful for testing and comparing different S3 implementations. Some people might find it helpful since it is relatively complicated to set up. (Hopefully, in the long run, IO500 compliant runs would be possible with S3) 5 | 6 | ## Preparation 7 | 8 | To retrieve the required packages and compile all needed software, including libS3, the library used to communicate with the S3 interface, please run: 9 | 10 | ```console 11 | ./contrib/s3/prepare-s3.sh 12 | ``` 13 | 14 | ## Usage 15 | 16 | The benchmark requires a .ini file containing the options. 17 | The libS3 library should be dynamically linked before launching the benchmark. 18 | A sample config-s3.ini is provided, please replace the variables with the corresponding S3 storage access info. 19 | 20 | ```console 21 | $ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/build/libs3/build/lib 22 | ./io500 ./contrib/s3/config-s3.ini 23 | ``` 24 | 25 | Detailed help for the available options for the S3 IOR/MDtest benchmark is provided when running: 26 | 27 | ```console 28 | ./bin/ior -a=S3-libs3 -h 29 | ``` 30 | 31 | ### Example output on the command line 32 | 33 | ```console 34 | mpiexec -np 2 ./io500 config-s3.ini 35 | IO500 version io500-isc22_v1 (standard) 36 | [RESULT] ior-easy-write 0.002189 GiB/s : time 303.523 seconds 37 | [RESULT] mdtest-easy-write 0.005847 kIOPS : time 302.666 seconds 38 | [ ] timestamp 0.000000 kIOPS : time 0.001 seconds 39 | [RESULT] ior-hard-write 0.000130 GiB/s : time 301.501 seconds 40 | [RESULT] mdtest-hard-write 0.001975 kIOPS : time 302.595 seconds 41 | [RESULT] find 0.000000 kIOPS : time 0.000 seconds [INVALID] 42 | [RESULT] ior-easy-read 0.010056 GiB/s : time 66.307 seconds 43 | [RESULT] mdtest-easy-stat 0.006040 kIOPS : time 292.609 seconds 44 | [RESULT] ior-hard-read 0.000262 GiB/s : time 150.044 seconds 45 | [RESULT] mdtest-hard-stat 0.006040 kIOPS : time 100.022 seconds 46 | [RESULT] mdtest-easy-delete 0.002928 kIOPS : time 601.830 seconds 47 | [RESULT] mdtest-hard-read 0.003012 kIOPS : time 198.905 seconds 48 | [RESULT] mdtest-hard-delete 0.002956 kIOPS : time 203.265 seconds 49 | [SCORE ] Bandwidth 0.000931 GiB/s : IOPS 0.003778 kiops : TOTAL 0.001875 [INVALID] 50 | ``` 51 | -------------------------------------------------------------------------------- /contrib/s3/config-s3.ini: -------------------------------------------------------------------------------- 1 | [global] 2 | datadir = ./datafiles 3 | timestamp-datadir = False 4 | resultdir = ./results 5 | timestamp-resultdir = True 6 | api = S3-libs3 --S3-libs3.host=${S3_ENDPOINT} --S3-libs3.access-key=${ACCESS_KEY} --S3-libs3.secret-key=${SECRET_KEY} --S3-libs3.s3-compatible --S3-libs3.use-ssl 7 | [debug] 8 | # Stonewall time must be 300 for a valid result, can be smaller for testing 9 | stonewall-time = 300 10 | [find] 11 | run = False 12 | [find-easy] 13 | run = False 14 | [find-hard] 15 | run = False -------------------------------------------------------------------------------- /contrib/s3/prepare-s3.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | echo This script downloads some code needed for the benchmarks 6 | echo It will also attempt to build the benchmarks 7 | echo It will output OK at the end if builds succeed 8 | echo 9 | 10 | LIBS3_HASH=287e4bee6fd430 11 | # tested with the following mpi implementation 12 | # MPILib="openmpi/2.0.2p2_hpcx-nag62" 13 | 14 | # shellcheck source=/dev/null 15 | source "$(dirname "$0")/../../prepare.sh" 16 | function main_s3 { 17 | # listed here, easier to spot and run if something fails 18 | setup 19 | # Prerequistics 20 | echo "Please make sure that you have the required packages: curl libxml2 openssl libiconv gcc automake autoconf openmpi(or another mpi implementation) cmake" 21 | # module load curl libxml2 openssl libiconv gcc automake autoconf $MPILib cmake 22 | echo "Tested on Ubuntu: apt install -y gcc cmake autoconf openssl libssl-dev libcurl4-openssl-dev libxml2-dev openmpi-bin libopenmpi-dev" 23 | echo "Tested on Centos/Fedora: dnf install -y git gcc cmake automake autoconf openssl libxml2 libxml2-devel libcurl-devel openssl-devel openmpi" 24 | sleep 1 25 | get_schema_tools 26 | get_libs3 27 | get_ior 28 | get_pfind 29 | build_ior_s3 30 | build_pfind 31 | build_io500 32 | echo 33 | echo "OK: All required software packages are now prepared" 34 | ls "$BIN" 35 | } 36 | 37 | ###### GET FUNCTION 38 | function get_libs3 { 39 | echo "Getting libs3" 40 | git_co https://github.com/bji/libs3.git libs3 $LIBS3_HASH 41 | pushd "$BUILD"/libs3 42 | $MAKE clean 43 | DESTDIR=$INSTALL_DIR $MAKE install 44 | popd 45 | } 46 | ###### BUILD FUNCTION 47 | function build_ior_s3 { 48 | pushd "$BUILD"/ior 49 | ./bootstrap 50 | ./configure --prefix="$INSTALL_DIR" --with-S3-libs3 CPPFLAGS=-I"$INSTALL_DIR"/include LDFLAGS=-L"$INSTALL_DIR"/lib 51 | cd src 52 | $MAKE clean 53 | $MAKE install 54 | echo "IOR: OK" 55 | echo 56 | popd 57 | } 58 | ###### CALL MAIN 59 | main_s3 60 | -------------------------------------------------------------------------------- /find/sfind.ini: -------------------------------------------------------------------------------- 1 | [global] 2 | datadir = ./test 3 | resultdir = ./results 4 | timestamp-resultdir = FALSE 5 | API = POSIX 6 | 7 | [find] 8 | external-script = ./find/sfind.sh 9 | 10 | [ior-hard] 11 | noRun = True 12 | 13 | [ior-easy] 14 | noRun = True 15 | 16 | [debug] 17 | stonewall-time = 1 18 | -------------------------------------------------------------------------------- /find/sfind.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # this script takes arguments just like the normal GNU find command 4 | # in fact, it just passes them directly on 5 | 6 | # you can replace this with whatever you want. You will almost surely want to because this will be really slow for large number of files. 7 | # Whatever you do, you just need to accept something that looks like normal find parameters: 8 | # -name *01* -size 3901c -newer some_file 9 | # When you are done pring 'x/y' where x is matched files and y is total files searched. 10 | # There is a parallel version also install in bin that might be better 11 | 12 | function parse_rates { 13 | #find -D rates gives a weird thing like this: 14 | #Predicate success rates after completion 15 | # ( ( -name *01* [0.8] [602/30415=0.0197929] -a [0.008] [308/30415=0.0101266] [call stat] [need type] -newer /Users/jbent/io-500-dev/datafiles/io500.2017.10.21-21.05.56/timestampfile [0.01] [308/602=0.511628] ) -a [8e-05] [308/30415=0.0101266] [call stat] -size 3900 [0.01] [308/308=1] ) -a [8e-05] [308/30415=0.0101266] -print [1] [308/308=1] 16 | # so if we parse out the first 602/30415, we can get 30415 as the total number of files searched 17 | # and if we parse out the final 308/308, we can get 308 as the total number of files matched 18 | echo $rates | tr " " "\n" | grep '/' | $1 -1 | cut -d \/ -f 2 | cut -d = -f 1 19 | } 20 | 21 | rates=`find -D rates $* 2>&1 | grep -A1 Predicate | tail -1` 22 | total_files=$(parse_rates 'head') 23 | match_files=$(parse_rates 'tail') 24 | echo "MATCHED $match_files/$total_files" 25 | -------------------------------------------------------------------------------- /include/io500-debug.h: -------------------------------------------------------------------------------- 1 | #ifndef IO500_DEBUG_H 2 | #define IO500_DEBUG_H 3 | 4 | #include 5 | 6 | #include 7 | 8 | extern FILE* file_out; 9 | 10 | #define FATAL(...) do{ printf("FATAL ("__FILE__":%d) ", __LINE__); printf(__VA_ARGS__); exit(1); }while(0); 11 | 12 | #define DEBUG_ALL(...) do{ if(opt.verbosity > 5 && opt.rank == 0){ printf("; [D] "__VA_ARGS__);} }while(0); 13 | #define DEBUG_INFO(...) do{ if(opt.verbosity > 4 && opt.rank == 0){ printf("; [I] "__VA_ARGS__);} }while(0); 14 | 15 | #define PRINT_PAIR(key, format, ...) do{ if(opt.rank == 0){ fprintf(file_out, "%-15s = "format, key, __VA_ARGS__); fflush(file_out); } }while(0); 16 | #define PRINT_PAIR_HEADER(key) do{ if(opt.rank == 0){ fprintf(file_out, "%-15s = ", key); } }while(0); 17 | #define INFO_PAIR(key, format, ...) do{ if(opt.verbosity > 1){ PRINT_PAIR(key, format, __VA_ARGS__);} }while(0); 18 | 19 | #define ERROR(...) do{ fprintf(file_out, "; ERROR "__VA_ARGS__); printf("ERROR "__VA_ARGS__); fflush(file_out); }while(0); 20 | #define WARNING(...) do{ fprintf(file_out, "; WARNING "__VA_ARGS__); printf("WARNING "__VA_ARGS__); fflush(file_out); }while(0); 21 | 22 | #define INVALID(...) do{ if (opt.rank == 0){fprintf(file_out, "; ERROR INVALID "__VA_ARGS__); printf("ERROR INVALID (%s:%d) ", __FILE__, __LINE__); printf(__VA_ARGS__); fflush(file_out); opt.is_valid_phase = 0; } }while(0); 23 | 24 | 25 | #define r0printf(...) do{ if(opt.rank == 0){ printf(__VA_ARGS__); fflush(stdout); } }while(0); 26 | 27 | 28 | #define UMPI_CHECK(MPI_STATUS) do { \ 29 | char resultString[MPI_MAX_ERROR_STRING]; \ 30 | int resultLength; \ 31 | \ 32 | if (MPI_STATUS != MPI_SUCCESS) { \ 33 | MPI_Error_string(MPI_STATUS, resultString, &resultLength); \ 34 | FATAL("MPI ERROR: MPI %s, (%s:%d)\n", \ 35 | resultString, __FILE__, __LINE__); \ 36 | } \ 37 | } while(0) 38 | 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /include/io500-opt.h: -------------------------------------------------------------------------------- 1 | #ifndef IO500_OPT_H 2 | #define IO500_OPT_H 3 | 4 | #include 5 | #include 6 | 7 | typedef enum{ 8 | IO500_MODE_STANDARD = 1, 9 | IO500_MODE_EXTENDED, 10 | } io500_mode; 11 | 12 | typedef struct{ 13 | int drop_caches; 14 | char * drop_caches_cmd; 15 | 16 | int stonewall; 17 | char * pause_dir; // if set check for files of the phase 18 | 19 | char * datadir; 20 | char * resdir; 21 | 22 | int dry_run; 23 | int verbosity; 24 | int is_valid_phase; /* set to 0 during a phase, if invalid */ 25 | int is_valid_run; /* set to 0 the first occurence it is invalid */ 26 | int is_valid_extended_run; 27 | 28 | int rank; 29 | int mpi_size; 30 | int io_buffers_on_gpu; /* are the I/O buffers to be allocated on a GPU */ 31 | 32 | char * api; 33 | char * apiArgs; // for IOR and mdtest 34 | char * dataPacketType; /* how test data is created */ 35 | char * timestamp; 36 | int timestamp_resdir; 37 | int timestamp_datadir; 38 | 39 | int scc; 40 | int minwrite; 41 | io500_mode mode; 42 | 43 | aiori_xfer_hint_t backend_hints; 44 | aiori_mod_opt_t * backend_opt; 45 | ior_aiori_t const * aiori; 46 | } io500_opt_t; 47 | 48 | extern io500_opt_t opt; 49 | 50 | #endif 51 | -------------------------------------------------------------------------------- /include/io500-phase.h: -------------------------------------------------------------------------------- 1 | #ifndef IO500_PHASE_H 2 | #define IO500_PHASE_H 3 | 4 | #include 5 | 6 | #include 7 | 8 | #define DEBUG_OPTION "INVALIDATES RUN; FOR DEBUGGING." 9 | 10 | typedef enum { 11 | IO500_NO_SCORE, 12 | IO500_SCORE_MD, 13 | IO500_SCORE_BW, 14 | IO500_SCORE_LAST 15 | } io500_phase_score_group; 16 | 17 | // bitfield of the type 18 | typedef enum { 19 | IO500_PHASE_DUMMY = 1, 20 | IO500_PHASE_WRITE = 2, 21 | IO500_PHASE_READ = 4, 22 | IO500_PHASE_UPDATE = 8, 23 | IO500_PHASE_REMOVE = 16, 24 | IO500_PHASE_FLAG_OPTIONAL = 32 25 | } io500_phase_type; 26 | 27 | typedef struct{ 28 | char const * name; 29 | io500_phase_type type; 30 | ini_option_t * options; 31 | void (*validate)(void); // check options 32 | double (*run)(void); // returns the score 33 | bool verify_stonewall; // double check that runtime meets stonewall? 34 | void (*cleanup)(void); // remove generated files/directories if possible 35 | 36 | double score; // the measured score 37 | io500_phase_score_group group; 38 | } u_phase_t; 39 | 40 | #define IO500_PHASES (2 + 1 + 2*3 + 3 + 3 + 4 + 5 + 3 + 4 + 1) 41 | 42 | extern u_phase_t p_opt; 43 | extern u_phase_t p_debug; 44 | 45 | extern u_phase_t p_timestamp; 46 | 47 | extern u_phase_t p_find; 48 | extern u_phase_t p_find_easy; 49 | extern u_phase_t p_find_hard; 50 | 51 | extern u_phase_t p_ior_hard; 52 | extern u_phase_t p_ior_hard_write; 53 | extern u_phase_t p_ior_hard_read; 54 | 55 | extern u_phase_t p_ior_rnd4K; 56 | extern u_phase_t p_ior_rnd4K_write; 57 | extern u_phase_t p_ior_rnd4K_read; 58 | 59 | extern u_phase_t p_ior_rnd1MB; 60 | extern u_phase_t p_ior_rnd1MB_write; 61 | extern u_phase_t p_ior_rnd1MB_read; 62 | 63 | extern u_phase_t p_mdworkbench; 64 | extern u_phase_t p_mdworkbench_create; 65 | extern u_phase_t p_mdworkbench_bench; 66 | extern u_phase_t p_mdworkbench_delete; 67 | 68 | extern u_phase_t p_ior_easy; 69 | extern u_phase_t p_ior_easy_write; 70 | extern u_phase_t p_ior_easy_read; 71 | extern u_phase_t p_ior_rnd4K_read_easywrite; 72 | 73 | extern u_phase_t p_mdtest_easy; 74 | extern u_phase_t p_mdtest_easy_write; 75 | extern u_phase_t p_mdtest_easy_stat; 76 | extern u_phase_t p_mdtest_easy_delete; 77 | 78 | extern u_phase_t p_mdtest_hard; 79 | extern u_phase_t p_mdtest_hard_write; 80 | extern u_phase_t p_mdtest_hard_stat; 81 | extern u_phase_t p_mdtest_hard_read; 82 | extern u_phase_t p_mdtest_hard_delete; 83 | 84 | #endif 85 | -------------------------------------------------------------------------------- /include/io500-util.h: -------------------------------------------------------------------------------- 1 | #ifndef IO500_UTIL_H 2 | #define IO500_UTIL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | #define INI_UNSET_STRING NULL 13 | #define INI_UNSET_INT (-2147483648) 14 | #define INI_UNSET_UINT (unsigned)(-1) 15 | #define INI_UNSET_UINT64 (uint64_t)(-1) 16 | #define INI_UNSET_BOOL 2 17 | #define INI_UNSET_FLOAT 1e38 18 | 19 | typedef enum{ 20 | INI_STRING, 21 | INI_INT, 22 | INI_UINT, 23 | INI_UINT64, 24 | INI_BOOL, 25 | INI_FLOAT 26 | } ini_var_type_e; 27 | 28 | typedef struct { 29 | char const * name; // the flag, e.g., testArg 30 | char const * help; // help text provided to understand what this flag does 31 | 32 | bool mandatory; 33 | ini_var_type_e type; // for checking during the parsing 34 | 35 | char * default_val; // the default value, NULL if no value set 36 | void * var; // the pointer to the variable to fill of the given type 37 | } ini_option_t; 38 | 39 | typedef struct { 40 | char const * name; // the section name 41 | ini_option_t * option; 42 | } ini_section_t; 43 | 44 | typedef void(*ini_call_back_f)(bool is_section, char const * key, char const * val); 45 | 46 | /** 47 | Parse the ini file in data according to the expected specification 48 | @Return 0 if parsing is successfull 49 | */ 50 | int u_parse_ini(char const * data, ini_section_t ** specification, ini_call_back_f func); 51 | 52 | 53 | void u_ini_parse_file(char const * file, ini_section_t** out_cfg, ini_call_back_f func, char ** out_buffer); 54 | 55 | /** 56 | Compute a hash based on the current values 57 | */ 58 | uint32_t u_ini_gen_hash(ini_section_t ** sections); 59 | void u_ini_print_values(FILE * fd, ini_section_t ** sections, bool show_help); 60 | 61 | /** 62 | * Hash functions to increase integrity 63 | */ 64 | uint32_t u_hash_update(uint32_t hash, char const * str); 65 | void u_hash_update_key_val(uint32_t * hash, char const * key, char const * val); 66 | void u_hash_update_key_val_dbl(uint32_t * hash, char const * key, double val); 67 | void u_hash_print(FILE * file, uint32_t hash); 68 | void u_verify_result_files(ini_section_t ** cfg, char const * result_file); 69 | 70 | // imported from IOR 71 | double GetTimeStamp(void); 72 | void u_create_datadir(char const * dir); 73 | void u_purge_datadir(char const * dir); 74 | void u_purge_file(char const * file); 75 | 76 | void u_create_dir_recursive(char const * dir, ior_aiori_t const * api, aiori_mod_opt_t * module_options); 77 | 78 | // invoke an external shell command 79 | void u_call_cmd(char const * command); 80 | 81 | void u_print_timestamp(FILE * out); 82 | void * u_malloc(int size); 83 | 84 | 85 | FILE * u_res_file_prep(char const * name); 86 | void u_res_file_close(FILE * out); 87 | 88 | uint32_t u_phase_unique_random_number(char const * phase_name); 89 | 90 | /** 91 | * Functions to handle the argument vectors for invoking other APIs 92 | */ 93 | typedef struct{ 94 | int size; 95 | char ** vector; 96 | } u_argv_t; 97 | 98 | u_argv_t * u_argv_create(void); 99 | void u_argv_free(u_argv_t * argv); 100 | void u_argv_push(u_argv_t * argv, char const * str); 101 | 102 | void u_argv_push_default_if_set(u_argv_t * argv, char * const arg, char const * dflt, char const * var); 103 | /* if the first argument is the API, subsequent options provided with --api.option will be permitted*/ 104 | void u_argv_push_default_if_set_api_options(u_argv_t * argv, char * const arg, char const * dflt, char const * var); 105 | void u_argv_push_default_if_set_bool(u_argv_t * argv, char * const arg, int dflt, int var); 106 | 107 | void u_argv_push_printf(u_argv_t * argv, char const * format, ...) 108 | __attribute__ ((format (printf, 2, 3))); 109 | char * u_flatten_argv(u_argv_t * argv); 110 | 111 | #endif 112 | -------------------------------------------------------------------------------- /io500.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #SBATCH --nodes=10 --ntasks-per-node=6 -p compute -A ku0598 3 | 4 | # INSTRUCTIONS: 5 | # 6 | # The only parts of the script that may need to be modified are: 7 | # - setup() to configure the binary locations and MPI parameters 8 | # Please visit https://vi4io.org/io500-info-creator/ to help generate the 9 | # "system-information.txt" file, by pasting the output of the info-creator. 10 | # This file contains details of your system hardware for your submission. 11 | 12 | # This script takes its parameters from the same .ini file as io500 binary. 13 | io500_ini="$1" # You can set the ini file here 14 | io500_mpirun="mpiexec" 15 | io500_mpiargs="-np 2" 16 | 17 | function setup(){ 18 | local workdir="$1" 19 | local resultdir="$2" 20 | mkdir -p $workdir $resultdir 21 | 22 | # Example commands to create output directories for Lustre. Creating 23 | # top-level directories is allowed, but not the whole directory tree. 24 | #if (( $(lfs df $workdir | grep -c MDT) > 1 )); then 25 | # lfs setdirstripe -D -c -1 $workdir 26 | #fi 27 | #lfs setstripe -c 1 $workdir 28 | #mkdir $workdir/ior-easy $workdir/ior-hard 29 | #mkdir $workdir/mdtest-easy $workdir/mdtest-hard 30 | #local osts=$(lfs df $workdir | grep -c OST) 31 | # Try overstriping for ior-hard to improve scaling, or use wide striping 32 | #lfs setstripe -C $((osts * 4)) $workdir/ior-hard || 33 | # lfs setstripe -c -1 $workdir/ior-hard 34 | # Try to use DoM if available, otherwise use default for small files 35 | #lfs setstripe -E 64k -L mdt $workdir/mdtest-easy || true #DoM? 36 | #lfs setstripe -E 64k -L mdt $workdir/mdtest-hard || true #DoM? 37 | #lfs setstripe -E 64k -L mdt $workdir/mdtest-rnd 38 | } 39 | 40 | # ***** YOU SHOULD NOT EDIT ANYTHING BELOW THIS LINE ***** 41 | set -eo pipefail # better error handling 42 | 43 | if [[ -z "$io500_ini" ]]; then 44 | echo "error: ini file must be specified. usage: $0 " 45 | exit 1 46 | fi 47 | if [[ ! -s "$io500_ini" ]]; then 48 | echo "error: ini file '$io500_ini' not found or empty" 49 | exit 2 50 | fi 51 | 52 | function get_ini_section_param() { 53 | local section="$1" 54 | local param="$2" 55 | local inside=false 56 | 57 | while read LINE; do 58 | LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE) 59 | $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break 60 | [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue 61 | ! $inside && continue 62 | #echo $LINE | awk -F = "/^$param/ { print \$2 }" 63 | if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then 64 | # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging 65 | echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/" 66 | return 67 | fi 68 | done < $io500_ini 69 | echo "" 70 | } 71 | 72 | function get_ini_global_param() { 73 | local param="$1" 74 | local default="$2" 75 | local val 76 | 77 | val=$(get_ini_section_param global $param | 78 | sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/') 79 | 80 | echo "${val:-$default}" 81 | } 82 | 83 | function run_benchmarks { 84 | $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp 85 | } 86 | 87 | create_tarball() { 88 | local sourcedir=$(dirname $io500_resultdir) 89 | local fname=$(basename ${io500_resultdir}) 90 | local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz 91 | 92 | cp -v $0 $io500_ini $io500_resultdir 93 | tar czf $tarball -C $sourcedir $fname 94 | echo "Created result tarball $tarball" 95 | } 96 | 97 | function main { 98 | # These commands extract the 'datadir' and 'resultdir' from .ini file 99 | timestamp=$(date +%Y.%m.%d-%H.%M.%S) # create a uniquifier 100 | [ $(get_ini_global_param timestamp-datadir True) != "False" ] && 101 | ts="$timestamp" || ts="io500" 102 | # working directory where the test files will be created 103 | export io500_workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts 104 | [ $(get_ini_global_param timestamp-resultdir True) != "False" ] && 105 | ts="$timestamp" || ts="io500" 106 | # the directory where the output results will be kept 107 | export io500_resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts 108 | 109 | setup $io500_workdir $io500_resultdir 110 | run_benchmarks 111 | 112 | if [[ ! -s "system-information.txt" ]]; then 113 | echo "Warning: please create a 'system-information.txt' description by" 114 | echo "copying the information from https://vi4io.org/io500-info-creator/" 115 | else 116 | cp "system-information.txt" $io500_resultdir 117 | fi 118 | 119 | create_tarball 120 | } 121 | 122 | main 123 | -------------------------------------------------------------------------------- /prepare.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | set -e 4 | 5 | echo This script downloads the code for the benchmarks 6 | echo It will also attempt to build the benchmarks 7 | echo It will output OK at the end if builds succeed 8 | echo 9 | 10 | IOR_HASH=348572ccfe8963 11 | PFIND_HASH=aaba722a178 12 | 13 | INSTALL_DIR=$PWD 14 | BIN=$INSTALL_DIR/bin 15 | BUILD=$PWD/build 16 | MAKE="make -j${NPROC:-$(nproc 2> /dev/null || echo 4)}" # handle missing nproc 17 | 18 | function main { 19 | # listed here, easier to spot and run if something fails 20 | setup 21 | 22 | get_schema_tools 23 | get_ior 24 | get_pfind 25 | 26 | build_ior 27 | build_pfind 28 | build_io500 29 | 30 | echo 31 | echo "OK: All required software packages are now prepared" 32 | ls "$BIN" 33 | } 34 | 35 | function setup { 36 | #rm -rf $BUILD $BIN 37 | mkdir -p "$BUILD" "$BIN" 38 | #cp utilities/find/mmfind.sh $BIN 39 | } 40 | 41 | function git_co { 42 | local repo=$1 43 | local dir=$2 44 | local tag=$3 45 | 46 | pushd "$BUILD" 47 | [ -d "$dir" ] || git clone "$repo" "$dir" 48 | cd "$dir" 49 | git fetch 50 | if [ -n "$tag" ]; then git checkout "$tag"; fi 51 | popd 52 | } 53 | 54 | ###### GET FUNCTIONS 55 | function get_ior { 56 | local ior_dir="ior" 57 | if [ -d "$ior_dir" ]; then 58 | echo "IOR already exists. Skipping download." 59 | else 60 | echo "Getting IOR and mdtest" 61 | git_co https://github.com/hpc/ior.git "$ior_dir" $IOR_HASH 62 | fi 63 | } 64 | 65 | function get_pfind { 66 | local pfind_dir="pfind" 67 | if [ -d "$pfind_dir" ]; then 68 | echo "Parallel find already exists. Skipping download." 69 | else 70 | echo "Preparing parallel find" 71 | git_co https://github.com/VI4IO/pfind.git "$pfind_dir" $PFIND_HASH 72 | fi 73 | } 74 | 75 | function get_schema_tools { 76 | local schema_build_dir="build/cdcl-schema-tools" 77 | local schema_dir="schema-tools" 78 | if [ -d "$schema_build_dir" ] && [ -d "$schema_dir" ]; then 79 | echo "Schema tools already exist. Skipping download." 80 | else 81 | echo "Downloading supplementary schema tools" 82 | git_co https://github.com/VI4IO/cdcl-schema-tools.git cdcl-schema-tools 83 | [ -d "$dir" ] || ln -sf "$PWD"/build/cdcl-schema-tools schema-tools 84 | fi 85 | } 86 | 87 | ###### BUILD FUNCTIONS 88 | function build_ior { 89 | pushd "$BUILD"/ior 90 | ./bootstrap 91 | # Add here extra flags 92 | ./configure --prefix="$INSTALL_DIR" 93 | cd src 94 | $MAKE clean 95 | $MAKE install 96 | echo "IOR: OK" 97 | echo 98 | popd 99 | } 100 | 101 | function build_pfind { 102 | pushd "$BUILD"/pfind 103 | ./prepare.sh 104 | ./compile.sh 105 | ln -sf "$BUILD"/pfind/pfind "$BIN"/pfind 106 | echo "Pfind: OK" 107 | echo 108 | popd 109 | } 110 | 111 | function build_io500 { 112 | $MAKE 113 | echo "io500: OK" 114 | echo 115 | } 116 | 117 | ###### CALL MAIN 118 | main 119 | -------------------------------------------------------------------------------- /src/ini-parse.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | 8 | uint32_t u_ini_gen_hash(ini_section_t ** sections){ 9 | uint32_t value = 0; 10 | for( ini_section_t ** ps = sections ; *ps != NULL; ps++){ 11 | ini_section_t * s = *ps; 12 | for( ini_option_t * o = s->option ; o->name != NULL; o++){ 13 | if(o->default_val){ 14 | // compute a hash for each option individually to make it invariant to reordered options 15 | u_hash_update_key_val(& value, o->name, o->default_val); 16 | } 17 | } 18 | } 19 | return value; 20 | } 21 | 22 | void u_ini_parse_file(char const * file, ini_section_t** cfg, ini_call_back_f func, char ** out_data){ 23 | struct stat statbuf; 24 | int ret = stat(file, & statbuf); 25 | if(ret != 0){ 26 | FATAL("Cannot open config file %s\n", file); 27 | } 28 | 29 | char * buff = ""; 30 | if(statbuf.st_size > 0){ 31 | buff = malloc(statbuf.st_size + 1); 32 | if(! buff){ 33 | FATAL("Cannot malloc();") 34 | } 35 | 36 | FILE * f = fopen(file, "r"); 37 | if(ret != 0){ 38 | FATAL("Cannot open config file %s\n", file); 39 | } 40 | ret = fread(buff, statbuf.st_size, 1, f); 41 | fclose(f); 42 | if( ret != 1 ){ 43 | FATAL("Couldn't read config file %s\n", file); 44 | } 45 | buff[statbuf.st_size] = '\0'; 46 | } 47 | 48 | ret = u_parse_ini(buff, cfg, func); 49 | if (ret != 0){ 50 | FATAL("Couldn't parse config file %s\n", file); 51 | } 52 | 53 | if (out_data){ 54 | *out_data = strdup(buff); 55 | } 56 | free(buff); 57 | } 58 | 59 | int u_parse_ini(char const * data, ini_section_t ** sections, ini_call_back_f cb_func){ 60 | int reti; 61 | // prepare regexes 62 | regex_t r_section, r_int, r_uint, r_str, r_float, r_empty; 63 | reti = regcomp(& r_section, "^[[:space:]]*\\[[[:space:]]*([0-9a-zA-Z_-]+)[[:space:]]*\\][[:space:]]*([[:space:]][#;].*)?$", REG_EXTENDED); 64 | if (reti){ 65 | FATAL("Could not compile regex\n"); 66 | } 67 | reti = regcomp(& r_str, "^[[:space:]]*([0-9a-zA-Z_.-]+)[[:space:]]*=[[:space:]]*([^#;]*)[[:space:]]*([[:space:]][#;].*)?$", REG_EXTENDED); 68 | if (reti){ 69 | FATAL("Could not compile regex\n"); 70 | } 71 | reti = regcomp(& r_int, "^[-]?[0-9]+$", REG_EXTENDED); 72 | if (reti){ 73 | FATAL("Could not compile regex\n"); 74 | } 75 | reti = regcomp(& r_uint, "^[0-9]+$", REG_EXTENDED); 76 | if (reti){ 77 | FATAL("Could not compile regex\n"); 78 | } 79 | reti = regcomp(& r_empty, "^[[:space:]]*([#;].*)?$", REG_EXTENDED); 80 | if (reti){ 81 | FATAL("Could not compile regex\n"); 82 | } 83 | reti = regcomp(& r_float, "^([0-9e.+-]+)$", REG_EXTENDED); 84 | if (reti){ 85 | FATAL("Could not compile regex\n"); 86 | } 87 | 88 | char * copy = strdup(data); 89 | char * token; 90 | char * saveptr; 91 | token = strtok_r(copy, "\n", &saveptr); 92 | 93 | ini_section_t * section = NULL; 94 | int line = 0; 95 | 96 | char * lastsaveptr = saveptr; 97 | 98 | // parse each line 99 | while(token){ 100 | line += 1 + ((token - lastsaveptr) > 0 ? (token - lastsaveptr) : 0); // strtok skips whole lines 101 | lastsaveptr = saveptr; 102 | 103 | regmatch_t match[3]; 104 | DEBUG_INFO("Parsing: \"%s\"\n", token); 105 | 106 | reti = regexec(&r_section, token, 2, match, 0); 107 | if (reti == 0) { 108 | char * sname = token + match[1].rm_so; 109 | token[match[1].rm_eo] = '\0'; 110 | DEBUG_INFO("Section: \"%s\"\n", sname); 111 | 112 | if(cb_func){ // callback handler for sections 113 | cb_func(1, sname, NULL); 114 | } 115 | 116 | if( sections ){ 117 | section = NULL; 118 | for( ini_section_t ** ps = sections ; *ps != NULL; ps++){ 119 | if(strcasecmp((*ps)->name, sname) == 0){ 120 | section = *ps; 121 | break; 122 | } 123 | } 124 | if(section == NULL){ 125 | ERROR("Parsing error in line %d, unknown section %s\n", line, sname); 126 | return 1; 127 | } 128 | } 129 | token = strtok_r(NULL, "\n", &saveptr); 130 | continue; 131 | } 132 | 133 | reti = regexec(&r_str, token, 3, match, 0); 134 | if (reti == 0) { 135 | char * var = token + match[1].rm_so; 136 | char * val = token + match[2].rm_so; 137 | token[match[1].rm_eo] = '\0'; 138 | 139 | // trim whitespace from the right of the string value 140 | for(int p = match[2].rm_eo; p >= match[2].rm_so; p--){ 141 | switch(token[p]){ 142 | case ' ': 143 | case '\t': 144 | case '\r': 145 | case '\0': 146 | token[p] = '\0'; 147 | break; 148 | default: 149 | goto end_loop; 150 | } 151 | } 152 | end_loop: 153 | 154 | DEBUG_INFO("Var: \"%s\"=\"%s\"\n", var, val); 155 | 156 | if(*val == '\0'){ 157 | // no argument 158 | token = strtok_r(NULL, "\n", &saveptr); 159 | continue; 160 | } 161 | 162 | if(cb_func){ // callback handler for variables 163 | cb_func(0, var, val); 164 | } 165 | 166 | if( ! sections ) continue; 167 | 168 | if(section == NULL){ 169 | ERROR("Parsing error in line %d, variable assigned outside section\n", line); 170 | return 1; 171 | } 172 | 173 | ini_option_t * option = section->option; 174 | for( ; option->name != NULL; option++){ 175 | if(strcasecmp(option->name, var) == 0){ 176 | break; 177 | } 178 | } 179 | 180 | if(option->name == NULL){ 181 | ERROR("Parsing error in section %s line %d, unknown option \"%s\" with value \"%s\"\n", section->name, line, var, val); 182 | return 1; 183 | } 184 | 185 | if(option->type == INI_INT){ 186 | reti = regexec(& r_int, val, 0, NULL, 0); 187 | if(reti != 0){ 188 | ERROR("Parsing error in section %s line %d, option %s expects integer, received \"%s\"\n", section->name, line, var, val); 189 | return 1; 190 | } 191 | }else if(option->type == INI_UINT || option->type == INI_UINT64){ 192 | reti = regexec(& r_uint, val, 0, NULL, 0); 193 | if(reti != 0){ 194 | ERROR("Parsing error in section %s line %d, option %s expects integer >= 0, received \"%s\"\n", section->name, line, var, val); 195 | return 1; 196 | } 197 | }else if(option->type == INI_BOOL){ 198 | if(strcasecmp(val, "true") == 0 || strcmp(val, "1") == 0){ 199 | val = "TRUE"; 200 | }else if(strcasecmp(val, "false") == 0 || strcmp(val, "0") == 0){ 201 | val = "FALSE"; 202 | }else{ 203 | ERROR("Parsing error in section %s line %d, option %s expects bool (true|false), received \"%s\"\n", section->name, line, var, val); 204 | return 1; 205 | } 206 | }else if(option->type == INI_FLOAT){ 207 | reti = regexec(& r_float, val, 0, NULL, 0); 208 | if(reti != 0){ 209 | ERROR("Parsing error in section %s line %d, option %s expects float, received \"%s\"\n", section->name, line, var, val); 210 | return 1; 211 | } 212 | } 213 | // assign new value 214 | option->default_val = strdup(val); 215 | 216 | token = strtok_r(NULL, "\n", &saveptr); 217 | continue; 218 | } 219 | 220 | if(sections){ 221 | // must be the empty line 222 | reti = regexec(&r_empty, token, 0, NULL, 0); 223 | if (reti != 0) { 224 | ERROR("Parsing error in section %s line %d, unexpected content: \"%s\"\n", section ? section->name : "no_section", line, token); 225 | return 1; 226 | } 227 | } 228 | 229 | token = strtok_r(NULL, "\n", &saveptr); 230 | } 231 | 232 | // check for mandatory options and assign values 233 | int error = 0; 234 | if(sections){ 235 | for( ini_section_t ** ps = sections ; *ps != NULL; ps++){ 236 | ini_section_t * s = *ps; 237 | 238 | for( ini_option_t * o = s->option ; o->name != NULL; o++){ 239 | if( o->mandatory && o->default_val == NULL ){ 240 | ERROR("[%s]: The mandatory option \"%s\" is not set\n", s->name, o->name); 241 | error = 1; 242 | } 243 | if(! o->var) continue; 244 | // assing a value to the configuration 245 | if(o->default_val){ 246 | switch(o->type){ 247 | case(INI_INT):{ 248 | *(int*) o->var = atoi(o->default_val); 249 | break; 250 | }case(INI_UINT):{ 251 | *(unsigned*) o->var = atoi(o->default_val); 252 | break; 253 | }case(INI_UINT64):{ 254 | *(uint64_t*) o->var = (uint64_t) atoll(o->default_val); 255 | break; 256 | }case(INI_BOOL):{ 257 | *(int*) o->var = o->default_val[0] == 'T'; 258 | break; 259 | }case(INI_STRING):{ 260 | *(char**) o->var = o->default_val; 261 | break; 262 | }case(INI_FLOAT):{ 263 | *(float*) o->var = atof(o->default_val); 264 | break; 265 | } 266 | } 267 | }else{ 268 | switch(o->type){ 269 | case(INI_INT):{ 270 | *(int*) o->var = INI_UNSET_INT; 271 | break; 272 | }case(INI_UINT):{ 273 | *(unsigned*) o->var = INI_UNSET_UINT; 274 | break; 275 | }case(INI_UINT64):{ 276 | *(uint64_t*) o->var = INI_UNSET_UINT64; 277 | break; 278 | }case(INI_BOOL):{ 279 | *(int*) o->var = INI_UNSET_BOOL; 280 | break; 281 | }case(INI_STRING):{ 282 | *(char**) o->var = INI_UNSET_STRING; 283 | break; 284 | }case(INI_FLOAT):{ 285 | *(float*) o->var = INI_UNSET_FLOAT; 286 | break; 287 | } 288 | } 289 | } 290 | } 291 | } 292 | } 293 | // cleanup 294 | regfree(&r_section); 295 | regfree(&r_int); 296 | regfree(&r_uint); 297 | regfree(&r_str); 298 | free(copy); 299 | return error; 300 | } 301 | 302 | void u_ini_print_values(FILE * fd, ini_section_t ** sections, bool show_help){ 303 | for( ini_section_t ** ps = sections ; *ps != NULL; ps++){ 304 | ini_section_t * s = *ps; 305 | fprintf(fd, "[%s]\n", s->name); 306 | 307 | for( ini_option_t * o = s->option ; o->name != NULL; o++){ 308 | if (o->help && show_help){ 309 | fprintf(fd, "# %s\n", o->help); 310 | } 311 | fprintf(fd, "%s = %s\n", o->name, o->default_val ? o->default_val : ""); 312 | } 313 | fprintf(fd, "\n"); 314 | } 315 | } 316 | -------------------------------------------------------------------------------- /src/main.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | 14 | #include 15 | 16 | #define INVALID_RUN(...) do{ if (opt.rank == 0){fprintf(file_out, "; ERROR INVALID "__VA_ARGS__); printf("ERROR INVALID (%s:%d) ", __FILE__, __LINE__); printf(__VA_ARGS__); fflush(file_out); opt.is_valid_run = 0; } }while(0); 17 | 18 | #define RUN_PHASE(phase) ( ! (phase->type & IO500_PHASE_FLAG_OPTIONAL && opt.mode == IO500_MODE_STANDARD) ) 19 | 20 | FILE* file_out = NULL; 21 | 22 | static char const * io500_phase_str[IO500_SCORE_LAST] = { 23 | "NO SCORE", 24 | "MD", 25 | "BW"}; 26 | 27 | extern int rank; 28 | 29 | static void prepare_aiori(void){ 30 | // check selected API, might be followed by API options 31 | char * api = strdup(opt.api); 32 | char * token = strstr(api, " "); 33 | 34 | option_help options [] = { 35 | LAST_OPTION 36 | }; 37 | options_all_t * global_options = airoi_create_all_module_options(options); 38 | if(token){ 39 | *token = '\0'; 40 | opt.apiArgs = strdup(opt.api); 41 | opt.api = api; 42 | 43 | // parse the API options, a bit cumbersome at the moment 44 | // find the next token for all the APIs 45 | token++; 46 | for(char * p = token ; ; p++){ 47 | if(*p == ' '){ 48 | *p = '\0'; 49 | if( p - token > 1 && *token != ' '){ 50 | DEBUG_INFO("API token: \"%s\"\n", token); 51 | if( option_parse_str(token, global_options) ){ 52 | FATAL("Couldn't parse API option: %s\n", token); 53 | } 54 | } 55 | token = p + 1; 56 | }else if(*p == '\0'){ 57 | break; 58 | }else if(p[0] == '\\' && p[1] == ' '){ 59 | // skip escaped whitespace by moving it forward 60 | for(char * c = p ; *c != '\0' ; c++){ 61 | c[0] = c[1]; 62 | } 63 | p++; 64 | } 65 | } 66 | DEBUG_INFO("API token: \"%s\"\n", token); 67 | if( option_parse_str(token, global_options) ){ 68 | FATAL("Coudln't parse API option: %s\n", token); 69 | } 70 | } 71 | opt.aiori = aiori_select(opt.api); 72 | opt.backend_opt = airoi_update_module_options(opt.aiori, global_options); 73 | if(opt.aiori == NULL){ 74 | FATAL("Could not load AIORI backend for %s with options: %s\n", opt.api, opt.apiArgs); 75 | } 76 | if (opt.aiori->xfer_hints){ 77 | memset(& opt.backend_hints, 0, sizeof(opt.backend_hints)); 78 | opt.aiori->xfer_hints(& opt.backend_hints); 79 | } 80 | if(opt.aiori->check_params){ 81 | opt.aiori->check_params(opt.backend_opt); 82 | } 83 | if (opt.aiori->initialize){ 84 | opt.aiori->initialize(opt.backend_opt); 85 | } 86 | 87 | if(opt.timestamp == NULL){ 88 | opt.timestamp = malloc(30); 89 | if(opt.rank == 0){ 90 | struct tm* tm_info; 91 | time_t timer; 92 | time(&timer); 93 | tm_info = localtime(&timer); 94 | strftime(opt.timestamp, 30, "%Y.%m.%d-%H.%M.%S", tm_info); 95 | } 96 | UMPI_CHECK(MPI_Bcast(opt.timestamp, 30, MPI_CHAR, 0, MPI_COMM_WORLD)); 97 | } 98 | 99 | if(opt.timestamp_resdir){ 100 | char resdir[PATH_MAX]; 101 | sprintf(resdir, "%s/%s", opt.resdir, opt.timestamp); 102 | opt.resdir = strdup(resdir); 103 | } 104 | 105 | if(opt.timestamp_datadir){ 106 | char resdir[PATH_MAX]; 107 | sprintf(resdir, "%s/%s", opt.datadir, opt.timestamp); 108 | opt.datadir = strdup(resdir); 109 | } 110 | 111 | if(opt.rank == 0){ 112 | ior_aiori_t const * posix = aiori_select("POSIX"); 113 | u_create_dir_recursive(opt.resdir, posix, NULL); 114 | } 115 | } 116 | 117 | static void print_cfg_hash(FILE * out, ini_section_t ** cfg){ 118 | if(opt.rank == 0){ 119 | PRINT_PAIR_HEADER("config-hash"); 120 | uint32_t hash = u_ini_gen_hash(cfg); 121 | u_hash_print(out, hash); 122 | fprintf(out, "\n"); 123 | } 124 | } 125 | 126 | #define dupprintf(...) do{ if(opt.rank == 0) { fprintf(res_summary, __VA_ARGS__); printf(__VA_ARGS__); } }while(0); 127 | 128 | static double calc_score(double scores[IO500_SCORE_LAST], int extended, uint32_t * hash){ 129 | double overall_score = 1; 130 | for(io500_phase_score_group g=1; g < IO500_SCORE_LAST; g++){ 131 | char score_string[2048]; 132 | char *p = score_string; 133 | double score = 1; 134 | int numbers = 0; 135 | p += sprintf(p, " %s = (", io500_phase_str[g]); 136 | for(int i=0; i < IO500_PHASES; i++){ 137 | if(phases[i]->group == g && (extended || ! (phases[i]->type & IO500_PHASE_FLAG_OPTIONAL)) ){ 138 | double t = phases[i]->score; 139 | if(t <= 0){ 140 | continue; 141 | } 142 | score *= t; 143 | p += sprintf(p, " * "); 144 | numbers++; 145 | p += sprintf(p, "%.8f", t); 146 | } 147 | } 148 | if(numbers == 0){ 149 | score = 0; 150 | }else{ 151 | DEBUG_INFO("%s)^%f\n", score_string, 1.0/numbers); 152 | score = pow(score, 1.0/numbers); 153 | } 154 | scores[g] = score; 155 | PRINT_PAIR(io500_phase_str[g], "%f\n", score); 156 | u_hash_update_key_val_dbl(hash, io500_phase_str[g], score); 157 | 158 | overall_score *= score; 159 | } 160 | return sqrt(overall_score); 161 | } 162 | 163 | static const char * io500_mode_str(io500_mode mode){ 164 | switch(mode){ 165 | case (IO500_MODE_STANDARD): return "standard"; 166 | case (IO500_MODE_EXTENDED): return "extended"; 167 | default: 168 | return "unknown"; 169 | } 170 | } 171 | 172 | int main(int argc, char ** argv){ 173 | int mpi_init = 0; 174 | file_out = stdout; 175 | opt = (io500_opt_t) { 176 | .mode = IO500_MODE_STANDARD, 177 | .is_valid_run = 1, 178 | .is_valid_extended_run = 1 179 | }; 180 | 181 | ini_section_t ** cfg = u_options(); 182 | 183 | if (argc < 2 || strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0){ 184 | help: 185 | r0printf("Synopsis: %s [-v=] [--dry-run] [--cleanup] [--config-hash] [--timestamp ]\n\n", argv[0]); 186 | r0printf("--config-hash Compute the configuration hash\n"); 187 | r0printf("--cleanup will run the delete phases of the benchmark useful to get rid of a partially executed benchmark\n"); 188 | r0printf("--dry-run will show the executed IO benchmark arguments but not run them (It will run drop caches, though, if enabled)\n"); 189 | r0printf("--list list available options for the .ini file\n"); 190 | r0printf("--mode=standard|extended define the mode to run the benchmark\n"); 191 | r0printf("--timestamp use for the output directory\n"); 192 | r0printf("--verify to verify that the output hasn't been modified accidentially; call like: io500 test.ini --verify test.out\n\n"); 193 | 194 | goto out; 195 | } 196 | if (argc < 2 || strcmp(argv[1], "-l") == 0 || strcmp(argv[1], "--list") == 0){ 197 | if (opt.rank == 0){ 198 | /* print this as a comment, in case it is saved into the .ini file */ 199 | r0printf("# Supported and current values of the ini file:\n"); 200 | u_ini_print_values(stdout, cfg, TRUE); 201 | } 202 | goto out; 203 | } 204 | 205 | mpi_init = 1; 206 | MPI_Init(& argc, & argv); 207 | MPI_Comm_rank(MPI_COMM_WORLD, & opt.rank); 208 | MPI_Comm_size(MPI_COMM_WORLD, & opt.mpi_size); 209 | rank = opt.rank; 210 | 211 | int verbosity_override = -1; 212 | int print_help = 0; 213 | 214 | int config_hash_only = 0; 215 | int cleanup_only = 0; 216 | int verify_only = 0; 217 | if(argc > 2){ 218 | for(int i = 2; i < argc; i++){ 219 | if(strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0 ){ 220 | print_help = 1; 221 | }else if(strncmp(argv[i], "-v=", 3) == 0){ 222 | verbosity_override = atoi(argv[i]+3); 223 | opt.verbosity = verbosity_override; 224 | }else if(strcmp(argv[i], "--mode=standard") == 0){ 225 | opt.mode = IO500_MODE_STANDARD; 226 | }else if(strcmp(argv[i], "--mode=extended") == 0){ 227 | opt.mode = IO500_MODE_EXTENDED; 228 | }else if(strcmp(argv[i], "--cleanup") == 0 ){ 229 | cleanup_only = 1; 230 | }else if(strcmp(argv[i], "--config-hash") == 0 ){ 231 | config_hash_only = 1; 232 | }else if(strcmp(argv[i], "--dry-run") == 0 ){ 233 | opt.dry_run = 1; 234 | }else if(strncmp(argv[i], "--timestamp", sizeof("--timestamp") - 1) == 0){ 235 | if (strchr(argv[i], '=') != NULL) 236 | opt.timestamp = strdup(strchr(argv[i], '=') + 1); 237 | else if (argv[i + 1] != NULL && argv[i + 1][0] != '\0') 238 | opt.timestamp = strdup(argv[++i]); 239 | else 240 | FATAL("Missing timestamp argument\n"); 241 | }else if (strcmp(argv[i], "-l") == 0 || strcmp(argv[i], "--list") == 0){ 242 | r0printf("# Supported and current values of the ini file:\n"); 243 | u_ini_print_values(stdout, cfg, TRUE); 244 | exit(1); 245 | }else if(strcmp(argv[i], "--verify") == 0 ){ 246 | verify_only = 1; 247 | break; 248 | }else{ 249 | FATAL("Unknown option: %s\n", argv[i]); 250 | } 251 | } 252 | } 253 | 254 | 255 | char * ini_data = NULL; 256 | { 257 | int ini_len = 0; 258 | if( opt.rank == 0){ 259 | u_ini_parse_file(argv[1], cfg, NULL, & ini_data); 260 | ini_len = strlen(ini_data); 261 | UMPI_CHECK(MPI_Bcast(& ini_len, 1, MPI_INT, 0, MPI_COMM_WORLD)); 262 | UMPI_CHECK(MPI_Bcast(ini_data, ini_len, MPI_CHAR, 0, MPI_COMM_WORLD)); 263 | }else{ 264 | UMPI_CHECK(MPI_Bcast(& ini_len, 1, MPI_INT, 0, MPI_COMM_WORLD)); 265 | ini_data = u_malloc(ini_len + 1); 266 | ini_data[ini_len] = 0; 267 | UMPI_CHECK(MPI_Bcast(ini_data, ini_len, MPI_CHAR, 0, MPI_COMM_WORLD)); 268 | int ret = u_parse_ini(ini_data, cfg, NULL); 269 | if (ret != 0){ 270 | FATAL("Couldn't parse config file on rank %d\n", opt.rank); 271 | } 272 | } 273 | } 274 | if(opt.scc){ 275 | // student cluster competition 276 | opt.minwrite = 30; 277 | }else{ 278 | opt.minwrite = 300; 279 | } 280 | if(verbosity_override > -1){ 281 | opt.verbosity = verbosity_override; 282 | } 283 | if(print_help){ 284 | goto help; 285 | } 286 | 287 | if(verify_only){ 288 | if(argc == 3){ 289 | FATAL("--verify option requires the output file as last parameter!"); 290 | } 291 | if(verbosity_override == -1){ 292 | opt.verbosity = 0; 293 | } 294 | u_verify_result_files(cfg, argv[argc-1]); 295 | goto out; 296 | } 297 | 298 | if(config_hash_only){ 299 | print_cfg_hash(stdout, cfg); 300 | goto out; 301 | } 302 | 303 | prepare_aiori(); 304 | 305 | FILE * res_summary = NULL; 306 | if(opt.rank == 0){ 307 | char file[PATH_MAX]; 308 | sprintf(file, "%s/result_summary.txt", opt.resdir); 309 | res_summary = fopen(file, "w"); 310 | if(! res_summary){ 311 | FATAL("Could not open \"%s\" for writing (%s)\n", file, strerror(errno)); 312 | } 313 | sprintf(file, "%s/result.txt", opt.resdir); 314 | file_out = fopen(file, "w"); 315 | if(! file_out){ 316 | FATAL("Could not open \"%s\" for writing (%s)\n", file, strerror(errno)); 317 | } 318 | fprintf(file_out, "[run]\n"); 319 | PRINT_PAIR("procs", "%d\n", opt.mpi_size); 320 | sprintf(file, "%s/config-orig.ini", opt.resdir); 321 | FILE * fd = fopen(file, "w"); 322 | fwrite(ini_data, strlen(ini_data), 1, fd); 323 | fclose(fd); 324 | } 325 | 326 | PRINT_PAIR("version", "%s\n", VERSION); 327 | print_cfg_hash(file_out, cfg); 328 | PRINT_PAIR("result-dir", "%s\n", opt.resdir); 329 | PRINT_PAIR("mode", "%s\n", io500_mode_str(opt.mode)); 330 | 331 | if(opt.rank == 0){ 332 | // create configuration in result directory to ensure it is preserved 333 | char file[PATH_MAX]; 334 | sprintf(file, "%s/config.ini", opt.resdir); 335 | FILE * fd = fopen(file, "w"); 336 | if(! fd){ 337 | FATAL("Couldn't write the configuration file to the result directory: %s (%s)\n", file, strerror(errno)); 338 | } 339 | u_ini_print_values(fd, cfg, 0); 340 | fclose(fd); 341 | } 342 | 343 | if(opt.dry_run){ 344 | INVALID_RUN("DRY RUN MODE ACTIVATED\n"); 345 | } 346 | if(opt.pause_dir){ 347 | INVALID_RUN("PAUSING BETWEEN RUNS ACTIVATED\n"); 348 | } 349 | 350 | MPI_Barrier(MPI_COMM_WORLD); 351 | if(opt.verbosity > 0 && opt.rank == 0){ 352 | fprintf(file_out, "; START "); 353 | u_print_timestamp(file_out); 354 | fprintf(file_out, "\n"); 355 | } 356 | 357 | // manage a hash for the scores 358 | uint32_t score_hash = 0; 359 | uint32_t score_extended_hash = 0; 360 | u_hash_update_key_val(& score_hash, "version", VERSION); 361 | 362 | dupprintf("IO500 version %s (%s)\n", VERSION, io500_mode_str(opt.mode)); 363 | 364 | for(int i=0; i < IO500_PHASES; i++){ 365 | if(RUN_PHASE(phases[i]) && phases[i]->validate){ 366 | phases[i]->validate(); 367 | } 368 | } 369 | if(opt.rank == 0){ 370 | fprintf(file_out, "\n"); 371 | } 372 | 373 | for(int i=0; i < IO500_PHASES; i++){ 374 | u_phase_t * phase = phases[i]; 375 | if(! phase->run) continue; 376 | if( cleanup_only && ! (phase->type & IO500_PHASE_REMOVE) ) continue; 377 | if(! RUN_PHASE(phase)){ 378 | continue; 379 | } 380 | 381 | if(opt.drop_caches && ! (phase->type & IO500_PHASE_DUMMY) ){ 382 | DEBUG_INFO("Dropping cache\n"); 383 | if(opt.rank == 0) 384 | u_call_cmd("LANG=C free -m"); 385 | u_call_cmd(opt.drop_caches_cmd); 386 | if(opt.rank == 0) 387 | u_call_cmd("LANG=C free -m"); 388 | } 389 | 390 | MPI_Barrier(MPI_COMM_WORLD); 391 | if(opt.rank == 0){ 392 | fprintf(file_out, "\n[%s]\n", phase->name); 393 | 394 | if(opt.pause_dir){ 395 | // if the file exists 396 | char path[PATH_MAX]; 397 | int ret; 398 | struct stat statbuf; 399 | if(opt.verbosity > 0){ 400 | PRINT_PAIR_HEADER("t_pause"); 401 | u_print_timestamp(file_out); 402 | fprintf(file_out, "\n"); 403 | } 404 | sprintf(path, "%s/%s", opt.pause_dir, phase->name); 405 | fprintf(file_out, "; Checking for pause file %s\n", path); 406 | fflush(file_out); 407 | while(1){ 408 | ret = stat(path, & statbuf); 409 | if(ret != 0){ 410 | break; 411 | } 412 | sleep(1); 413 | } 414 | MPI_Barrier(MPI_COMM_WORLD); 415 | } 416 | 417 | if(opt.verbosity > 0){ 418 | PRINT_PAIR_HEADER("t_start"); 419 | u_print_timestamp(file_out); 420 | fprintf(file_out, "\n"); 421 | } 422 | } 423 | 424 | if(opt.pause_dir && opt.rank != 0){ 425 | MPI_Barrier(MPI_COMM_WORLD); 426 | } 427 | 428 | double start = GetTimeStamp(); 429 | opt.is_valid_phase = 1; 430 | double score = phase->run(); 431 | double runtime = GetTimeStamp() - start; 432 | 433 | if(opt.rank == 0){ 434 | // This is an additional sanity check 435 | if(! opt.dry_run){ 436 | if( phases[i]->verify_stonewall && runtime < opt.stonewall ){ 437 | INVALID("Runtime of phase (%f) is below stonewall time. This shouldn't happen!\n", runtime); 438 | }else if(score == 0.0 && ! (phase->type & IO500_PHASE_DUMMY)){ 439 | INVALID("Resulting score shouldn't be 0.0\n"); 440 | } 441 | } 442 | if(runtime < opt.minwrite && phases[i]->verify_stonewall){ 443 | INVALID("Runtime is smaller than expected minimum runtime\n"); 444 | } 445 | if(! opt.is_valid_phase){ 446 | opt.is_valid_extended_run = 0; 447 | if(! (phase->type & IO500_PHASE_FLAG_OPTIONAL)){ 448 | opt.is_valid_run = 0; 449 | } 450 | } 451 | if(! (phase->type & IO500_PHASE_DUMMY)){ 452 | PRINT_PAIR("score", "%f\n", score); 453 | } 454 | char * valid_str = opt.is_valid_phase ? "" : " [INVALID]"; 455 | char score_str[40]; 456 | sprintf(score_str, "%f", score); 457 | if(phase->group > IO500_NO_SCORE){ 458 | dupprintf("[RESULT]"); 459 | }else{ 460 | dupprintf("[ ]"); 461 | } 462 | dupprintf(" %20s %15s %s : time %.3f seconds%s\n", phase->name, score_str, phase->name[0] == 'i' ? "GiB/s" : "kIOPS", runtime, valid_str); 463 | } 464 | if(phase->group > IO500_NO_SCORE){ 465 | if(phase->type & IO500_PHASE_FLAG_OPTIONAL){ 466 | u_hash_update_key_val_dbl(& score_extended_hash, phase->name, score); 467 | }else{ 468 | u_hash_update_key_val_dbl(& score_hash, phase->name, score); 469 | } 470 | } 471 | phases[i]->score = score; 472 | 473 | 474 | if(opt.verbosity > 0 && opt.rank == 0){ 475 | PRINT_PAIR("t_delta", "%.4f\n", runtime); 476 | PRINT_PAIR_HEADER("t_end"); 477 | u_print_timestamp(file_out); 478 | fprintf(file_out, "\n"); 479 | } 480 | } 481 | 482 | MPI_Barrier(MPI_COMM_WORLD); 483 | if(opt.rank == 0){ 484 | char * valid_str = opt.is_valid_run ? "" : " [INVALID]"; 485 | // compute the overall score 486 | fprintf(file_out, "\n[SCORE]\n"); 487 | double scores[IO500_SCORE_LAST]; 488 | double overall_score = calc_score(scores, 0, & score_hash); 489 | PRINT_PAIR("SCORE", "%f%s\n", overall_score, valid_str); 490 | u_hash_update_key_val_dbl(& score_hash, "SCORE", overall_score); 491 | if( ! opt.is_valid_run ){ 492 | u_hash_update_key_val(& score_hash, "valid", "NO"); 493 | } 494 | PRINT_PAIR("hash", "%X\n", (int) score_hash); 495 | dupprintf("[SCORE ] Bandwidth %f GiB/s : IOPS %f kiops : TOTAL %f%s\n", 496 | scores[IO500_SCORE_BW], scores[IO500_SCORE_MD], overall_score, valid_str); 497 | 498 | // extended run 499 | if(opt.mode == IO500_MODE_EXTENDED){ 500 | valid_str = opt.is_valid_extended_run ? "" : " [INVALID]"; 501 | fprintf(file_out, "\n[SCOREX]\n"); 502 | double overall_extended_score = calc_score(scores, 1, & score_extended_hash); 503 | u_hash_update_key_val_dbl(& score_extended_hash, "SCORE", overall_extended_score); 504 | if( ! opt.is_valid_extended_run ){ 505 | u_hash_update_key_val(& score_extended_hash, "valid", "NO"); 506 | } 507 | PRINT_PAIR("SCORE", "%f%s\n", overall_extended_score, valid_str); 508 | PRINT_PAIR("hash", "%X\n", (int) score_extended_hash); 509 | dupprintf("[SCOREX] Bandwidth %f GiB/s : IOPS %f kiops : TOTAL %f%s\n", scores[IO500_SCORE_BW], scores[IO500_SCORE_MD], overall_extended_score, valid_str); 510 | } 511 | 512 | printf("\nThe result files are stored in the directory: %s\n", opt.resdir); 513 | } 514 | 515 | for(int i=0; i < IO500_PHASES; i++){ 516 | if(RUN_PHASE(phases[i]) && phases[i]->cleanup){ 517 | phases[i]->cleanup(); 518 | } 519 | } 520 | 521 | if(opt.rank == 0){ 522 | fclose(res_summary); 523 | u_purge_datadir(""); 524 | } 525 | 526 | if(opt.rank == 0 && opt.verbosity > 0){ 527 | fprintf(file_out, "; END "); 528 | u_print_timestamp(file_out); 529 | fprintf(file_out, "\n"); 530 | } 531 | 532 | if (opt.aiori->finalize){ 533 | opt.aiori->finalize(opt.backend_opt); 534 | } 535 | 536 | fclose(file_out); 537 | out: 538 | if (mpi_init) 539 | MPI_Finalize(); 540 | return 0; 541 | } 542 | -------------------------------------------------------------------------------- /src/phase-definitions.h: -------------------------------------------------------------------------------- 1 | static u_phase_t * phases[IO500_PHASES] = { 2 | & p_opt, 3 | & p_debug, 4 | 5 | & p_ior_easy, 6 | & p_ior_easy_write, 7 | 8 | & p_ior_rnd4K, 9 | & p_ior_rnd4K_write, 10 | 11 | & p_mdtest_easy, 12 | & p_mdtest_easy_write, 13 | 14 | & p_ior_rnd1MB, 15 | & p_ior_rnd1MB_write, 16 | 17 | & p_mdworkbench, 18 | & p_mdworkbench_create, 19 | 20 | & p_timestamp, 21 | 22 | & p_find_easy, 23 | 24 | & p_ior_hard, 25 | & p_ior_hard_write, 26 | 27 | & p_mdtest_hard, 28 | & p_mdtest_hard_write, 29 | 30 | & p_find, 31 | 32 | & p_ior_rnd4K_read, 33 | & p_ior_rnd1MB_read, 34 | 35 | & p_find_hard, 36 | 37 | & p_mdworkbench_bench, 38 | 39 | & p_ior_easy_read, 40 | & p_mdtest_easy_stat, 41 | 42 | & p_ior_hard_read, 43 | & p_mdtest_hard_stat, 44 | 45 | & p_mdworkbench_delete, 46 | 47 | & p_mdtest_easy_delete, 48 | & p_mdtest_hard_read, 49 | & p_mdtest_hard_delete, 50 | & p_ior_rnd4K_read_easywrite 51 | }; 52 | 53 | static ini_section_t ** u_options(void){ 54 | ini_section_t ** ini_section = u_malloc(sizeof(ini_section_t*) * (IO500_PHASES + 1)); 55 | for(int i=0; i < IO500_PHASES; i++){ 56 | ini_section[i] = u_malloc(sizeof(ini_section_t)); 57 | ini_section[i]->name = phases[i]->name; 58 | ini_section[i]->option = phases[i]->options; 59 | } 60 | ini_section[IO500_PHASES] = NULL; 61 | return ini_section; 62 | } 63 | -------------------------------------------------------------------------------- /src/phase_dbg.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #define STRINGIFY(x) #x 4 | #define TOSTRING(x) STRINGIFY(x) 5 | 6 | static ini_option_t option[] = { 7 | {"stonewall-time", "For a valid result, the stonewall timer must be set to the value according to the rules. If smaller "DEBUG_OPTION, 0, INI_INT, "300", & opt.stonewall}, 8 | {"pause-dir", "Pause between phases while in this directory lies a file with the phase name, e.g., easy-create. This can be useful for performance testing, e.g., of tiered storage. At the moment it "DEBUG_OPTION, 0, INI_STRING, NULL, & opt.pause_dir}, 9 | 10 | {NULL} }; 11 | 12 | 13 | static void validate(void){ 14 | if(opt.stonewall != opt.minwrite && opt.rank == 0){ 15 | INVALID("stonewall-time < %us\n", opt.minwrite); 16 | } 17 | } 18 | 19 | u_phase_t p_debug = { 20 | "debug", 21 | IO500_PHASE_DUMMY, 22 | option, 23 | validate, 24 | NULL 25 | }; 26 | -------------------------------------------------------------------------------- /src/phase_find.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include "phase_find.h" 10 | 11 | static opt_find of; 12 | 13 | static double run(void){ 14 | if(of.run == 0) return 0.0; 15 | 16 | if(opt.rank == 0){ 17 | // check the existance of the timestamp file just for correctness 18 | char timestamp_file[PATH_MAX]; 19 | sprintf(timestamp_file, "%s/timestampfile", opt.resdir); 20 | FILE * f = fopen(timestamp_file, "r"); 21 | if(! f){ 22 | FATAL("Couldn't open timestampfile: %s\n", timestamp_file); 23 | } 24 | fclose(f); 25 | } 26 | 27 | return run_find("find", & of); 28 | } 29 | 30 | double run_find(const char * phase_name, opt_find * of){ 31 | int ret = 0; 32 | PRINT_PAIR("exe", "%s\n", of->command); 33 | if(of->nproc != INI_UNSET_UINT){ 34 | PRINT_PAIR("nproc", "%d\n", of->nproc); 35 | } 36 | 37 | if(opt.dry_run){ 38 | return 0; 39 | } 40 | if(! of->ext_find){ 41 | if(of->pfind_com == MPI_COMM_NULL){ 42 | return 0; 43 | } 44 | 45 | int rank; 46 | MPI_Comm_rank(of->pfind_com, & rank); 47 | 48 | // pfind supports stonewalling timer -s, but ignore for now 49 | pfind_find_results_t * res = pfind_find(of->pfind_o); 50 | if(! res){ 51 | INVALID("PFind returned with an error, this is invalid.\n") 52 | return 0.0; 53 | } 54 | of->pfind_res = pfind_aggregrate_results(res); 55 | 56 | if(rank == 0){ 57 | char res_file[PATH_MAX]; 58 | sprintf(res_file, "%s/%s.csv", opt.resdir, phase_name); 59 | FILE * fd = fopen(res_file, "w"); 60 | fprintf(fd, "runtime: %f rate: %f\n", of->pfind_res->runtime, of->pfind_res->rate); 61 | fprintf(fd, "rank, errors, unknown, found, total, checked, job steal msgs received, work items send, job steal msgs send, work items stolen, time spend in job stealing in s, number of completion tokens send\n"); 62 | fprintf(fd, "0, %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64, res->errors, res->unknown_file, res->found_files, res->total_files, res->checked_dirents); 63 | fprintf(fd, ", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %.3fs, %"PRIu64"\n", res->monitor.job_steal_inbound, res->monitor.work_send, res->monitor.job_steal_tries, res->monitor.work_stolen, res->monitor.job_steal_mpitime_us / 1000000.0, res->monitor.completion_tokens_send); 64 | for(int i=1; i < of->nproc; i++){ 65 | MPI_Recv(& res->errors, 5, MPI_LONG_LONG_INT, i, 4712, of->pfind_com, MPI_STATUS_IGNORE); 66 | fprintf(fd, "%d, %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64, i, res->errors, res->unknown_file, res->found_files, res->total_files, res->checked_dirents); 67 | MPI_Recv(& res->monitor, 6, MPI_LONG_LONG_INT, i, 4713, of->pfind_com, MPI_STATUS_IGNORE); 68 | fprintf(fd, ", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %.3fs, %"PRIu64"\n", res->monitor.job_steal_inbound, res->monitor.work_send, res->monitor.job_steal_tries, res->monitor.work_stolen, res->monitor.job_steal_mpitime_us / 1000000.0, res->monitor.completion_tokens_send); 69 | } 70 | fclose(fd); 71 | }else{ 72 | MPI_Send(& res->errors, 5, MPI_LONG_LONG_INT, 0, 4712, of->pfind_com); 73 | MPI_Send(& res->monitor, 6, MPI_LONG_LONG_INT, 0, 4713, of->pfind_com); 74 | } 75 | free(res); 76 | of->found_files = of->pfind_res->found_files; 77 | of->runtime = of->pfind_res->runtime; 78 | 79 | if( of->found_files == 0 ){ 80 | INVALID("Find didn't find anything, this is likely invalid.\n") 81 | } 82 | PRINT_PAIR("found", "%"PRIu64"\n", of->found_files); 83 | PRINT_PAIR("total-files", "%"PRIu64"\n", of->pfind_res->total_files); 84 | return of->pfind_res->total_files / of->runtime / 1000; 85 | } 86 | 87 | //WARNING("Running the external script with nproc=%d\n", ); 88 | // only one process runs the external find 89 | if(opt.rank != 0){ 90 | MPI_Barrier(MPI_COMM_WORLD); 91 | return 0; 92 | } 93 | 94 | double performance; 95 | double start = GetTimeStamp(); 96 | FILE * fp = popen(of->command, "r"); 97 | if (fp == NULL) { 98 | ERROR("Failed to run find command: \"%s\" error: %s\n", of->command, strerror(errno)); 99 | return -1; 100 | } 101 | char line[1024]; 102 | uint64_t hits = 0; 103 | *line = '\0'; 104 | while (fgets(line, sizeof(line), fp) != NULL) { 105 | DEBUG_ALL("Found: %s", line); 106 | hits++; 107 | } 108 | ret = pclose(fp); 109 | if(ret != 0){ 110 | INVALID("Exit code %d != 0 from find command: \"%s\"\nPossible error: %s\n", ret, of->command, strerror(ret)); 111 | } 112 | 113 | double runtime = GetTimeStamp() - start; 114 | 115 | // support two semantics, the count only semantics 116 | if(*line != '\0'){ 117 | line[strlen(line) - 1] = 0; 118 | } 119 | PRINT_PAIR("last-output", "\"%s\"\n", line); 120 | if(strstr(line, "MATCHED ") == line){ 121 | // the script is supposed to output the number of files in this line 122 | char * ptr = line + 8; 123 | char * left = strtok(ptr, "/"); 124 | char * right = strtok(NULL, "/"); 125 | if(left == NULL || right == NULL || *left == 0 || * right == 0){ 126 | FATAL("Invalid output from the external script, expected: MATCHED /\n"); 127 | } 128 | hits = atoll(left); 129 | long long totals = atoll(right); 130 | PRINT_PAIR("total-files", "%lld\n", totals); 131 | performance = totals / runtime / 1000; 132 | }else{ 133 | WARNING("Couldn't find \"MATCHED x/y\" as last output, badly approximate using hits.\n"); 134 | performance = hits / runtime / 1000; 135 | } 136 | 137 | of->found_files = hits; 138 | of->runtime = runtime; 139 | 140 | if( of->found_files == 0 ){ 141 | INVALID("Find didn't find anything, this is likely invalid.\n") 142 | } 143 | 144 | PRINT_PAIR("found", "%"PRIu64"\n", of->found_files); 145 | 146 | MPI_Barrier(MPI_COMM_WORLD); 147 | return performance; 148 | } 149 | 150 | static ini_option_t option[] = { 151 | {"external-script", "Set to an external script to perform the find phase", 0, INI_STRING, NULL, & of.ext_find}, 152 | {"external-mpi-args", "Startup arguments for external scripts, some MPI's may not support this!", 0, INI_STRING, "", & of.ext_mpi}, 153 | {"external-extra-args", "Extra arguments for the external scripts", 0, INI_STRING, "", & of.ext_args}, 154 | {"nproc", "Set the number of processes for pfind/the external script", 0, INI_UINT, NULL, & of.nproc}, 155 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & of.run}, 156 | {"pfind-queue-length", "Pfind queue length", 0, INI_INT, "10000", & of.pfind_queue_length}, 157 | {"pfind-steal-next", "Pfind Steal from next", 0, INI_BOOL, "FALSE", & of.pfind_steal_from_next}, 158 | {"pfind-parallelize-single-dir-access-using-hashing", "Parallelize the readdir by using hashing. Your system must support this!", 0, INI_BOOL, "FALSE", & of.pfind_par_single_dir_access_hash}, 159 | {NULL} }; 160 | 161 | static void validate(void){ 162 | if(of.run == 0) return; 163 | if(of.ext_find){ 164 | char args[PATH_MAX]; 165 | sprintf(args, "%s -newer %s/timestampfile -size 3901c -name \"*01*\"", opt.datadir, opt.resdir); 166 | external_find_prepare_arguments(args, & of); 167 | }else{ 168 | u_argv_t * argv = u_argv_create(); 169 | u_argv_push(argv, "./pfind"); 170 | u_argv_push(argv, opt.datadir); 171 | u_argv_push(argv, "-newer"); 172 | u_argv_push_printf(argv, "%s/timestampfile", opt.resdir); 173 | u_argv_push(argv, "-size"); 174 | u_argv_push(argv, "3901c"); 175 | u_argv_push(argv, "-name"); 176 | u_argv_push(argv, "*01*"); 177 | u_argv_push(argv, "-C"); 178 | if(of.pfind_steal_from_next){ 179 | u_argv_push(argv, "-N"); 180 | } 181 | if(of.pfind_par_single_dir_access_hash){ 182 | u_argv_push(argv, "-H"); 183 | u_argv_push(argv, "1"); 184 | } 185 | u_argv_push(argv, "-q"); 186 | u_argv_push_printf(argv, "%d", of.pfind_queue_length); 187 | 188 | pfind_prepare_arguments(argv, & of); 189 | } 190 | } 191 | 192 | void external_find_prepare_arguments(char * args, opt_find * of){ 193 | struct stat sb; 194 | int ret = stat(of->ext_find, & sb); 195 | if(ret != 0){ 196 | FATAL("Cannot check external-script %s\n", of->ext_find); 197 | } 198 | if(! (sb.st_mode & S_IXUSR) ){ 199 | FATAL("The external-script must be a executable file %s\n", of->ext_find); 200 | } 201 | char command[PATH_MAX]; 202 | sprintf(command, "%s %s %s %s", of->ext_mpi, of->ext_find, of->ext_args, args); 203 | of->command = strdup(command); 204 | 205 | if(of->nproc != opt.mpi_size && of->nproc != 1 && of->nproc != INI_UNSET_UINT){ 206 | WARNING("An external-script will always be run with nproc=1, note that some MPI implementations do not support to run MPI programs from an MPI program\n"); 207 | } 208 | } 209 | 210 | void pfind_prepare_arguments(u_argv_t * argv, opt_find * of){ 211 | if(*of->ext_args != '\0' || *of->ext_mpi != '\0' ){ 212 | WARNING("Using internal pfind, will ignore any arguments to the external script\n"); 213 | } 214 | 215 | MPI_Comm com = MPI_COMM_WORLD; 216 | if(of->nproc != INI_UNSET_UINT){ 217 | int color = opt.rank < of->nproc; 218 | int ret = MPI_Comm_split(MPI_COMM_WORLD, color, opt.rank, & com); 219 | MPI_Comm_size(com, & ret); 220 | DEBUG_INFO("Configuring pfind to run with %d procs\n", ret); 221 | if(color && of->nproc != ret){ 222 | FATAL("Couldn't split rank for find into %d procs (got %d procs)\n", of->nproc, ret); 223 | } 224 | if(color == 0){ 225 | MPI_Comm_free(& com); 226 | com = MPI_COMM_NULL; 227 | } 228 | } 229 | 230 | of->command = u_flatten_argv(argv); 231 | of->pfind_com = com; 232 | if(com != MPI_COMM_NULL) { 233 | of->pfind_o = pfind_parse_args(argv->size, argv->vector, 0, com); 234 | MPI_Comm_size(of->pfind_com, & of->nproc); 235 | } 236 | u_argv_free(argv); 237 | } 238 | 239 | u_phase_t p_find = { 240 | "find", 241 | IO500_PHASE_READ, 242 | option, 243 | validate, 244 | run, 245 | 0, 246 | .group = IO500_SCORE_MD 247 | }; 248 | -------------------------------------------------------------------------------- /src/phase_find.h: -------------------------------------------------------------------------------- 1 | #ifndef IO500_PHASE_FIND_H 2 | #define IO500_PHASE_FIND_H 3 | 4 | #include 5 | #include 6 | 7 | typedef struct{ 8 | char * ext_find; 9 | char * ext_args; 10 | char * ext_mpi; 11 | int nproc; 12 | int run; 13 | char * command; 14 | 15 | pfind_options_t * pfind_o; 16 | pfind_find_results_t * pfind_res; 17 | MPI_Comm pfind_com; 18 | int pfind_queue_length; 19 | int pfind_steal_from_next; 20 | int pfind_par_single_dir_access_hash; 21 | 22 | uint64_t found_files; 23 | double runtime; 24 | } opt_find; 25 | 26 | double run_find(const char * phase_name, opt_find * of); 27 | 28 | void pfind_prepare_arguments(u_argv_t * argv, opt_find * of); 29 | void external_find_prepare_arguments(char * args, opt_find * of); 30 | #endif 31 | -------------------------------------------------------------------------------- /src/phase_find_easy.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include "phase_find.h" 10 | 11 | static opt_find of; 12 | 13 | static double run(void){ 14 | if(of.run == 0) return 0.0; 15 | return run_find("find-easy", & of); 16 | } 17 | 18 | static ini_option_t option[] = { 19 | {"external-script", "Set to an external script to perform the find phase", 0, INI_STRING, NULL, & of.ext_find}, 20 | {"external-mpi-args", "Startup arguments for external scripts, some MPI's may not support this!", 0, INI_STRING, "", & of.ext_mpi}, 21 | {"external-extra-args", "Extra arguments for the external scripts", 0, INI_STRING, "", & of.ext_args}, 22 | {"nproc", "Set the number of processes for pfind/the external script", 0, INI_UINT, NULL, & of.nproc}, 23 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & of.run}, 24 | {"pfind-queue-length", "Pfind queue length", 0, INI_INT, "10000", & of.pfind_queue_length}, 25 | {"pfind-steal-next", "Pfind Steal from next", 0, INI_BOOL, "FALSE", & of.pfind_steal_from_next}, 26 | {"pfind-parallelize-single-dir-access-using-hashing", "Parallelize the readdir by using hashing. Your system must support this!", 0, INI_BOOL, "FALSE", & of.pfind_par_single_dir_access_hash}, 27 | {NULL} }; 28 | 29 | static void validate(void){ 30 | if(of.run == 0) return; 31 | if(of.ext_find){ 32 | char args[1024]; 33 | sprintf(args, "%s/mdtest-easy/ -name \"*01*\"", opt.datadir); 34 | external_find_prepare_arguments(args, & of); 35 | }else{ 36 | u_argv_t * argv = u_argv_create(); 37 | u_argv_push(argv, "./pfind"); 38 | u_argv_push_printf(argv, "%s/mdtest-easy/", opt.datadir); 39 | u_argv_push(argv, "-name"); 40 | u_argv_push(argv, "*01*"); 41 | u_argv_push(argv, "-C"); 42 | if(of.pfind_steal_from_next){ 43 | u_argv_push(argv, "-N"); 44 | } 45 | if(of.pfind_par_single_dir_access_hash){ 46 | u_argv_push(argv, "-H"); 47 | u_argv_push(argv, "1"); 48 | } 49 | u_argv_push(argv, "-q"); 50 | u_argv_push_printf(argv, "%d", of.pfind_queue_length); 51 | 52 | pfind_prepare_arguments(argv, & of); 53 | } 54 | } 55 | 56 | u_phase_t p_find_easy = { 57 | "find-easy", 58 | IO500_PHASE_READ | IO500_PHASE_FLAG_OPTIONAL, 59 | option, 60 | validate, 61 | run, 62 | 0, 63 | .group = IO500_SCORE_MD 64 | }; 65 | -------------------------------------------------------------------------------- /src/phase_find_hard.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include "phase_find.h" 10 | 11 | static opt_find of; 12 | 13 | static double run(void){ 14 | if(of.run == 0) return 0.0; 15 | return run_find("find-hard", & of);; 16 | } 17 | 18 | static ini_option_t option[] = { 19 | {"external-script", "Set to an external script to perform the find phase", 0, INI_STRING, NULL, & of.ext_find}, 20 | {"external-mpi-args", "Startup arguments for external scripts, some MPI's may not support this!", 0, INI_STRING, "", & of.ext_mpi}, 21 | {"external-extra-args", "Extra arguments for the external scripts", 0, INI_STRING, "", & of.ext_args}, 22 | {"nproc", "Set the number of processes for pfind/the external script", 0, INI_UINT, NULL, & of.nproc}, 23 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & of.run}, 24 | {"pfind-queue-length", "Pfind queue length", 0, INI_INT, "10000", & of.pfind_queue_length}, 25 | {"pfind-steal-next", "Pfind Steal from next", 0, INI_BOOL, "FALSE", & of.pfind_steal_from_next}, 26 | {"pfind-parallelize-single-dir-access-using-hashing", "Parallelize the readdir by using hashing. Your system must support this!", 0, INI_BOOL, "FALSE", & of.pfind_par_single_dir_access_hash}, 27 | {NULL} }; 28 | 29 | static void validate(void){ 30 | if(of.run == 0) return; 31 | if(of.ext_find){ 32 | char args[1024]; 33 | sprintf(args, "%s/mdtest-hard/ -newer %s/timestampfile -size 3901c -name \"*01*\"", opt.datadir, opt.resdir); 34 | external_find_prepare_arguments(args, & of); 35 | }else{ 36 | u_argv_t * argv = u_argv_create(); 37 | u_argv_push(argv, "./pfind"); 38 | u_argv_push_printf(argv, "%s/mdtest-hard/", opt.datadir); 39 | u_argv_push(argv, "-newer"); 40 | u_argv_push_printf(argv, "%s/timestampfile", opt.resdir); 41 | u_argv_push(argv, "-size"); 42 | u_argv_push(argv, "3901c"); 43 | u_argv_push(argv, "-name"); 44 | u_argv_push(argv, "*01*"); 45 | u_argv_push(argv, "-C"); 46 | if(of.pfind_steal_from_next){ 47 | u_argv_push(argv, "-N"); 48 | } 49 | if(of.pfind_par_single_dir_access_hash){ 50 | u_argv_push(argv, "-H"); 51 | u_argv_push(argv, "1"); 52 | } 53 | u_argv_push(argv, "-q"); 54 | u_argv_push_printf(argv, "%d", of.pfind_queue_length); 55 | 56 | pfind_prepare_arguments(argv, & of); 57 | } 58 | } 59 | 60 | u_phase_t p_find_hard = { 61 | "find-hard", 62 | IO500_PHASE_READ | IO500_PHASE_FLAG_OPTIONAL, 63 | option, 64 | validate, 65 | run, 66 | 0, 67 | .group = IO500_SCORE_MD 68 | }; 69 | -------------------------------------------------------------------------------- /src/phase_ior.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | 5 | double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out){ 6 | IOR_test_t * test = ior_run(argv->size, argv->vector, MPI_COMM_WORLD, out); 7 | assert(test); 8 | IOR_results_t * res = test->results; 9 | assert(res); 10 | u_res_file_close(out); 11 | u_argv_free(argv); 12 | 13 | if(opt.rank != 0){ 14 | return 0; 15 | } 16 | 17 | IOR_point_t * p = & res->write; 18 | *res_out = p; 19 | 20 | if(res->errors){ 21 | INVALID("Errors (%d) occured during phase in IOR. This invalidates your run.\n", res->errors); 22 | } 23 | if( p->time < opt.stonewall ){ 24 | INVALID("Write phase needed %fs instead of stonewall %ds. Stonewall was hit at %.1fs\n", p->time, opt.stonewall, p->stonewall_time); 25 | } 26 | INFO_PAIR("accessed-pairs", "%zu\n", p->pairs_accessed); 27 | 28 | PRINT_PAIR("throughput-stonewall","%.2f\n", p->stonewall_avg_data_accessed * opt.mpi_size / p->stonewall_time / GIBIBYTE); 29 | double tp = p->aggFileSizeForBW / p->time / GIBIBYTE; 30 | 31 | return tp; 32 | } 33 | 34 | double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out){ 35 | IOR_results_t * res = ior_run(argv->size, argv->vector, MPI_COMM_WORLD, out)->results; 36 | u_res_file_close(out); 37 | u_argv_free(argv); 38 | 39 | IOR_point_t * p = & res->read; 40 | *res_out = p; 41 | 42 | if(res->errors){ 43 | INVALID("Errors (%d) occured during phase in IOR. This invalidates your run.\n", res->errors); 44 | } 45 | double tp = p->aggFileSizeForBW / p->time / GIBIBYTE; 46 | return tp; 47 | } 48 | -------------------------------------------------------------------------------- /src/phase_ior.h: -------------------------------------------------------------------------------- 1 | #ifndef IO500_PHASE_IOR_H 2 | #define IO500_PHASE_IOR_H 3 | 4 | #include 5 | #include 6 | 7 | typedef struct{ 8 | int run; 9 | char * api; 10 | 11 | int filePerProc; 12 | int uniqueDir; 13 | char * transferSize; 14 | char * blockSize; 15 | int verbosity; 16 | } opt_ior_easy; 17 | 18 | extern opt_ior_easy ior_easy_o; 19 | 20 | typedef struct{ 21 | int run; 22 | int collective; 23 | char * api; 24 | 25 | int segments; 26 | int verbosity; 27 | } opt_ior_hard; 28 | 29 | extern opt_ior_hard ior_hard_o; 30 | 31 | typedef struct{ 32 | int run; 33 | char * api; 34 | 35 | int random_prefill_bytes; 36 | uint64_t block_size; 37 | int verbosity; 38 | } opt_ior_rnd; 39 | 40 | extern opt_ior_rnd ior_rnd4K_o; 41 | extern opt_ior_rnd ior_rnd1MB_o; 42 | 43 | 44 | void ior_easy_add_params(u_argv_t * argv, int addStdFlags); 45 | void ior_hard_add_params(u_argv_t * argv); 46 | void ior_rnd4K_add_params(u_argv_t * argv); 47 | void ior_rnd1MB_add_params(u_argv_t * argv); 48 | 49 | // Generic helpers 50 | double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out); 51 | double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out); 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /src/phase_ior_easy.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | opt_ior_easy ior_easy_o; 8 | 9 | static ini_option_t option[] = { 10 | {"API", "The API to be used", 0, INI_STRING, NULL, & ior_easy_o.api}, 11 | {"transferSize", "Transfer size", 0, INI_STRING, "2m", & ior_easy_o.transferSize}, 12 | {"blockSize", "Block size; must be a multiple of transferSize", 0, INI_STRING, "9920000m", & ior_easy_o.blockSize}, 13 | {"filePerProc", "Create one file per process", 0, INI_BOOL, "TRUE", & ior_easy_o.filePerProc}, 14 | {"uniqueDir", "Use unique directory per file per process", 0, INI_BOOL, "FALSE", & ior_easy_o.uniqueDir}, 15 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & ior_easy_o.run}, 16 | {"verbosity", "The verbosity level", 0, INI_INT, 0, & ior_easy_o.verbosity}, 17 | {NULL} }; 18 | 19 | static void validate(void){ 20 | u_create_datadir("ior-easy"); 21 | } 22 | 23 | static void cleanup(void){ 24 | if (opt.dry_run) return; 25 | 26 | if(opt.rank == 0){ 27 | char filename[PATH_MAX]; 28 | sprintf(filename, "%s/ior-easy.stonewall", opt.resdir); 29 | unlink(filename); 30 | 31 | if(! ior_easy_o.filePerProc){ 32 | u_purge_file("ior-easy/ior_file_easy"); 33 | } 34 | } 35 | if(ior_easy_o.filePerProc){ 36 | char filename[PATH_MAX]; 37 | sprintf(filename, "ior-easy/ior_file_easy.%08d", opt.rank); 38 | u_purge_file(filename); 39 | } 40 | if(opt.rank == 0){ 41 | u_purge_datadir("ior-easy"); 42 | } 43 | } 44 | 45 | void ior_easy_add_params(u_argv_t * argv, int addStdFlags){ 46 | opt_ior_easy d = ior_easy_o; 47 | 48 | u_argv_push(argv, "./ior"); 49 | u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType); 50 | for(int i=0; i < ior_easy_o.verbosity; i++){ 51 | u_argv_push(argv, "-v"); /* verbose */ 52 | } 53 | if(opt.io_buffers_on_gpu){ 54 | u_argv_push(argv, "-O"); 55 | u_argv_push(argv, "allocateBufferOnGPU=1"); 56 | } 57 | u_argv_push(argv, "-C"); /* reorder tasks in constant order for read */ 58 | u_argv_push(argv, "-Q"); /* task per node offset */ 59 | u_argv_push(argv, "1"); 60 | u_argv_push(argv, "-g"); /* barriers between open, read, write, close */ 61 | u_argv_push(argv, "-G"); /* use fixed timestamp signature */ 62 | int hash = u_phase_unique_random_number("ior-easy"); 63 | u_argv_push_printf(argv, "%d", hash); 64 | // u_argv_push(argv, "271"); 65 | u_argv_push(argv, "-k"); /* keep file after program exit */ 66 | u_argv_push(argv, "-e"); /* fsync upon write close */ 67 | u_argv_push(argv, "-o"); /* filename for output file */ 68 | u_argv_push_printf(argv, "%s/ior-easy/ior_file_easy", opt.datadir); 69 | if(addStdFlags){ 70 | u_argv_push(argv, "-O"); /* additional IOR options */ 71 | u_argv_push_printf(argv, "stoneWallingStatusFile=%s/ior-easy.stonewall", opt.resdir); 72 | u_argv_push(argv, "-t"); /* transfer size */ 73 | u_argv_push(argv, d.transferSize); 74 | u_argv_push(argv, "-b"); /* blocksize in bytes */ 75 | u_argv_push(argv, d.blockSize); 76 | } 77 | 78 | if(ior_easy_o.uniqueDir){ 79 | u_argv_push(argv, "-u"); /* unique directory for each output file */ 80 | } 81 | 82 | if(ior_easy_o.filePerProc){ 83 | u_argv_push(argv, "-F"); /* write a separate file per process */ 84 | } 85 | } 86 | 87 | u_phase_t p_ior_easy = { 88 | "ior-easy", 89 | IO500_PHASE_DUMMY, 90 | option, 91 | validate, 92 | NULL, 93 | .cleanup = cleanup, 94 | }; 95 | -------------------------------------------------------------------------------- /src/phase_ior_easy_read.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | typedef struct{ 9 | int run; 10 | char * api; 11 | 12 | char * command; 13 | IOR_point_t * res; 14 | } opt_ior_easy_read; 15 | 16 | static opt_ior_easy_read o; 17 | 18 | static ini_option_t option[] = { 19 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.api}, 20 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, 21 | {NULL} }; 22 | 23 | 24 | static void validate(void){ 25 | 26 | } 27 | 28 | 29 | static double run(void){ 30 | opt_ior_easy d = ior_easy_o; 31 | 32 | u_argv_t * argv = u_argv_create(); 33 | ior_easy_add_params(argv, 1); 34 | u_argv_push(argv, "-r"); /* read existing file */ 35 | u_argv_push(argv, "-R"); /* verify data read */ 36 | u_argv_push_default_if_set_api_options(argv, "-a", d.api, o.api); 37 | u_argv_push(argv, "-O"); 38 | u_argv_push_printf(argv, "saveRankPerformanceDetailsCSV=%s/ior-easy-read.csv", opt.resdir); 39 | // u_argv_push_default_if_set(argv, "-U", /* use hints file */ 40 | // d.hintsFileName, o.hintsFileName); 41 | // u_argv_push_default_if_set_api_options(argv, "-a", /* backend API */ 42 | // d.api, o.api); 43 | // u_argv_push_default_if_set_bool(argv, "--posix.odirect", d.odirect, o.odirect); 44 | 45 | o.command = u_flatten_argv(argv); 46 | 47 | PRINT_PAIR("exe", "%s\n", o.command); 48 | if(opt.dry_run || d.run == 0 || o.run == 0){ 49 | u_argv_free(argv); 50 | return 0; 51 | } 52 | FILE * out = u_res_file_prep(p_ior_easy_read.name); 53 | return ior_process_read(argv, out, & o.res); 54 | } 55 | 56 | 57 | u_phase_t p_ior_easy_read = { 58 | "ior-easy-read", 59 | IO500_PHASE_READ, 60 | option, 61 | validate, 62 | run, 63 | 0, 64 | .group = IO500_SCORE_BW, 65 | }; 66 | -------------------------------------------------------------------------------- /src/phase_ior_easy_write.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | typedef struct{ 10 | int run; 11 | char * api; 12 | 13 | char * command; 14 | IOR_point_t * res; 15 | } opt_ior_easy_write; 16 | 17 | static opt_ior_easy_write o; 18 | 19 | static ini_option_t option[] = { 20 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.api}, 21 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, 22 | {NULL} }; 23 | 24 | static void validate(void){ 25 | 26 | } 27 | 28 | static double run(void){ 29 | opt_ior_easy d = ior_easy_o; 30 | 31 | u_argv_t * argv = u_argv_create(); 32 | ior_easy_add_params(argv, 1); 33 | u_argv_push(argv, "-w"); /* write file */ 34 | u_argv_push(argv, "-D"); /* deadline for stonewall in seconds */ 35 | u_argv_push_printf(argv, "%d", opt.stonewall); 36 | u_argv_push(argv, "-O"); /* additional IOR options */ 37 | u_argv_push(argv, "stoneWallingWearOut=1"); 38 | u_argv_push_default_if_set_api_options(argv, "-a", d.api, o.api); 39 | u_argv_push(argv, "-O"); 40 | u_argv_push_printf(argv, "saveRankPerformanceDetailsCSV=%s/ior-easy-write.csv", opt.resdir); 41 | // u_argv_push_default_if_set(argv, "-U", /* use hints file */ 42 | // d.hintsFileName, o.hintsFileName); 43 | // u_argv_push_default_if_set_api_options(argv, "-a", /* backend API */ 44 | // d.api, o.api); 45 | // u_argv_push_default_if_set_bool(argv, "--posix.odirect", d.odirect, o.odirect); 46 | 47 | o.command = u_flatten_argv(argv); 48 | 49 | PRINT_PAIR("exe", "%s\n", o.command); 50 | if(opt.dry_run || d.run == 0 || o.run == 0){ 51 | u_argv_free(argv); 52 | return 0; 53 | } 54 | FILE * out = u_res_file_prep(p_ior_easy_write.name); 55 | return ior_process_write(argv, out, & o.res); 56 | } 57 | 58 | u_phase_t p_ior_easy_write = { 59 | "ior-easy-write", 60 | IO500_PHASE_WRITE, 61 | option, 62 | validate, 63 | run, 64 | .verify_stonewall = 1, 65 | .group = IO500_SCORE_BW, 66 | }; 67 | -------------------------------------------------------------------------------- /src/phase_ior_hard.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | opt_ior_hard ior_hard_o; 8 | 9 | static ini_option_t option[] = { 10 | {"API", "The API to be used", 0, INI_STRING, NULL, & ior_hard_o.api}, 11 | {"segmentCount", "Number of segments", 0, INI_INT, "10000000", & ior_hard_o.segments}, 12 | {"collective", "Collective operation (for supported backends)", 0, INI_BOOL, NULL, & ior_hard_o.collective}, 13 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & ior_hard_o.run}, 14 | {"verbosity", "The verbosity level", 0, INI_INT, 0, & ior_hard_o.verbosity}, 15 | {NULL} }; 16 | 17 | 18 | 19 | static void validate(void){ 20 | u_create_datadir("ior-hard"); 21 | } 22 | 23 | static void cleanup(void){ 24 | if (opt.dry_run) return; 25 | 26 | if( opt.rank == 0){ 27 | char filename[PATH_MAX]; 28 | sprintf(filename, "%s/ior-hard.stonewall", opt.resdir); 29 | unlink(filename); 30 | u_purge_file("ior-hard/file"); 31 | u_purge_datadir("ior-hard"); 32 | } 33 | } 34 | 35 | void ior_hard_add_params(u_argv_t * argv){ 36 | opt_ior_hard d = ior_hard_o; 37 | 38 | u_argv_push(argv, "./ior"); 39 | u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType); 40 | for(int i=0; i < ior_hard_o.verbosity; i++){ 41 | u_argv_push(argv, "-v"); /* verbose */ 42 | } 43 | if(opt.io_buffers_on_gpu){ 44 | u_argv_push(argv, "-O"); 45 | u_argv_push(argv, "allocateBufferOnGPU=1"); 46 | } 47 | u_argv_push(argv, "-C"); /* reorder tasks in constant order for read */ 48 | u_argv_push(argv, "-Q"); /* task per node offset */ 49 | u_argv_push(argv, "1"); 50 | u_argv_push(argv, "-g"); /* barriers between open, read, write, close */ 51 | int hash = u_phase_unique_random_number("ior-hard"); 52 | u_argv_push_printf(argv, "-G=%d", hash); 53 | // u_argv_push(argv, "27"); 54 | u_argv_push(argv, "-k"); /* keep file after program exit */ 55 | u_argv_push(argv, "-e"); /* fsync upon write close */ 56 | u_argv_push(argv, "-o"); /* filename for output file */ 57 | u_argv_push_printf(argv, "%s/ior-hard/file", opt.datadir); 58 | u_argv_push(argv, "-O"); /* additional IOR directives */ 59 | u_argv_push_printf(argv, "stoneWallingStatusFile=%s/ior-hard.stonewall", opt.resdir ); 60 | // u_argv_push(argv, "stoneWallingWearOut=1"); 61 | u_argv_push(argv, "-t"); /* transfer size */ 62 | u_argv_push(argv, "47008"); 63 | u_argv_push(argv, "-b"); /* blocksize in bytes */ 64 | u_argv_push(argv, "47008"); 65 | u_argv_push(argv, "-s"); /* number of segments */ 66 | u_argv_push_printf(argv, "%d", d.segments); 67 | } 68 | 69 | u_phase_t p_ior_hard = { 70 | "ior-hard", 71 | IO500_PHASE_DUMMY, 72 | option, 73 | validate, 74 | NULL, 75 | .cleanup = cleanup, 76 | }; 77 | -------------------------------------------------------------------------------- /src/phase_ior_hard_read.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | typedef struct{ 9 | int run; 10 | char * api; 11 | 12 | char * command; 13 | IOR_point_t * res; 14 | int collective; 15 | } opt_ior_hard_read; 16 | 17 | static opt_ior_hard_read o; 18 | 19 | static ini_option_t option[] = { 20 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.api}, 21 | {"collective", "Collective operation (for supported backends)", 0, INI_BOOL, NULL, & o.collective}, 22 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, 23 | {NULL} }; 24 | 25 | 26 | static void validate(void){ 27 | 28 | } 29 | 30 | 31 | static double run(void){ 32 | opt_ior_hard d = ior_hard_o; 33 | 34 | u_argv_t * argv = u_argv_create(); 35 | ior_hard_add_params(argv); 36 | u_argv_push(argv, "-r"); /* read file */ 37 | u_argv_push(argv, "-R"); /* verify data read */ 38 | u_argv_push_default_if_set_bool(argv, "-c", d.collective, o.collective); 39 | u_argv_push_default_if_set_api_options(argv, "-a", /* backend API */ 40 | d.api, o.api); 41 | u_argv_push(argv, "-O"); 42 | u_argv_push_printf(argv, "saveRankPerformanceDetailsCSV=%s/ior-hard-read.csv", opt.resdir); 43 | // u_argv_push_default_if_set(argv, "-U", /* use hints file */ 44 | // d.hintsFileName, o.hintsFileName); 45 | // u_argv_push_default_if_set_bool(argv, "--posix.odirect", d.odirect, o.odirect); 46 | 47 | o.command = u_flatten_argv(argv); 48 | 49 | PRINT_PAIR("exe", "%s\n", o.command); 50 | if(opt.dry_run || d.run == 0 || o.run == 0){ 51 | u_argv_free(argv); 52 | return 0; 53 | } 54 | FILE * out = u_res_file_prep(p_ior_hard_read.name); 55 | return ior_process_read(argv, out, & o.res); 56 | } 57 | 58 | 59 | u_phase_t p_ior_hard_read = { 60 | "ior-hard-read", 61 | IO500_PHASE_READ, 62 | option, 63 | validate, 64 | run, 65 | 0, 66 | .group = IO500_SCORE_BW, 67 | }; 68 | -------------------------------------------------------------------------------- /src/phase_ior_hard_write.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | typedef struct{ 10 | int run; 11 | char * api; 12 | 13 | char * command; 14 | IOR_point_t * res; 15 | int collective; 16 | } opt_ior_hard_write; 17 | 18 | static opt_ior_hard_write o; 19 | 20 | static ini_option_t option[] = { 21 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.api}, 22 | {"collective", "Collective operation (for supported backends)", 0, INI_BOOL, NULL, & o.collective}, 23 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, 24 | {NULL} }; 25 | 26 | static void validate(void){ 27 | 28 | } 29 | 30 | static double run(void){ 31 | opt_ior_hard d = ior_hard_o; 32 | 33 | u_argv_t * argv = u_argv_create(); 34 | ior_hard_add_params(argv); 35 | u_argv_push(argv, "-w"); /* write file */ 36 | u_argv_push_default_if_set_bool(argv, "-c", d.collective, o.collective); 37 | u_argv_push(argv, "-D"); /* deadline for stonewall in seconds */ 38 | u_argv_push_printf(argv, "%d", opt.stonewall); 39 | u_argv_push_default_if_set_api_options(argv, "-a", /* backend API */ 40 | d.api, o.api); 41 | u_argv_push(argv, "-O"); 42 | u_argv_push_printf(argv, "saveRankPerformanceDetailsCSV=%s/ior-hard-write.csv", opt.resdir); 43 | u_argv_push(argv, "-O"); 44 | u_argv_push(argv, "stoneWallingWearOut=1"); 45 | // u_argv_push_default_if_set(argv, "-U", /* use hints file */ 46 | // d.hintsFileName, o.hintsFileName); 47 | 48 | o.command = u_flatten_argv(argv); 49 | 50 | PRINT_PAIR("exe", "%s\n", o.command); 51 | if(opt.dry_run || d.run == 0 || o.run == 0){ 52 | u_argv_free(argv); 53 | return 0; 54 | } 55 | FILE * out = u_res_file_prep(p_ior_hard_write.name); 56 | return ior_process_write(argv, out, & o.res); 57 | } 58 | 59 | u_phase_t p_ior_hard_write = { 60 | "ior-hard-write", 61 | IO500_PHASE_WRITE, 62 | option, 63 | validate, 64 | run, 65 | .verify_stonewall = 1, 66 | .group = IO500_SCORE_BW, 67 | }; 68 | -------------------------------------------------------------------------------- /src/phase_ior_rnd1MB.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | opt_ior_rnd ior_rnd1MB_o; 8 | 9 | static ini_option_t option[] = { 10 | {"API", "The API to be used", 0, INI_STRING, NULL, & ior_rnd1MB_o.api}, 11 | {"blockSize", "Size of a random block, change only if explicitly allowed", 0, INI_UINT64, "1073741824", & ior_rnd1MB_o.block_size}, 12 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & ior_rnd1MB_o.run}, 13 | {"verbosity", "The verbosity level", 0, INI_INT, 0, & ior_rnd1MB_o.verbosity}, 14 | {"randomPrefill", "Prefill the file with this blocksize in bytes, e.g., 2097152", 0, INI_INT, "0", & ior_rnd1MB_o.random_prefill_bytes}, 15 | {NULL} }; 16 | 17 | 18 | static void validate(void){ 19 | opt_ior_rnd d = ior_rnd1MB_o; 20 | if(d.block_size < 1024){ 21 | FATAL("Random blocksize must be larger than 1024\n"); 22 | } 23 | if(d.random_prefill_bytes > 0 && (d.block_size % d.random_prefill_bytes) != 0){ 24 | FATAL("Random prefill bytes must divide blocksize\n"); 25 | } 26 | if(d.random_prefill_bytes > 0 && d.block_size < d.random_prefill_bytes){ 27 | FATAL("Random prefill bytes must be < blocksize\n"); 28 | } 29 | } 30 | 31 | 32 | void ior_rnd1MB_add_params(u_argv_t * argv){ 33 | opt_ior_rnd d = ior_rnd1MB_o; 34 | 35 | u_argv_push(argv, "./ior"); 36 | u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType); 37 | for(int i=0; i < ior_rnd1MB_o.verbosity; i++){ 38 | u_argv_push(argv, "-v"); 39 | } 40 | if(opt.io_buffers_on_gpu){ 41 | u_argv_push(argv, "-O"); 42 | u_argv_push(argv, "allocateBufferOnGPU=1"); 43 | } 44 | u_argv_push(argv, "-Q=1"); 45 | //u_argv_push(argv, "-F"); 46 | u_argv_push(argv, "-g"); 47 | int hash = u_phase_unique_random_number("ior-random1MB"); 48 | u_argv_push_printf(argv, "-G=%d", hash); 49 | u_argv_push(argv, "-z"); 50 | u_argv_push(argv, "--random-offset-seed=11"); 51 | u_argv_push(argv, "-e"); 52 | u_argv_push_printf(argv, "-o=%s/ior-rnd1MB/file", opt.datadir); 53 | u_argv_push(argv, "-O"); 54 | u_argv_push_printf(argv, "stoneWallingStatusFile=%s/ior-rnd1MB.stonewall", opt.resdir ); 55 | u_argv_push(argv, "-k"); 56 | u_argv_push(argv, "-t=1048576"); 57 | u_argv_push_printf(argv, "-b=%ld", d.block_size); 58 | u_argv_push_printf(argv, "-s=%d", 10000000); 59 | } 60 | 61 | u_phase_t p_ior_rnd1MB = { 62 | "ior-rnd1MB", 63 | IO500_PHASE_DUMMY, 64 | option, 65 | validate, 66 | NULL, 67 | NULL 68 | }; 69 | -------------------------------------------------------------------------------- /src/phase_ior_rnd4K.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | opt_ior_rnd ior_rnd4K_o; 8 | 9 | static ini_option_t option[] = { 10 | {"API", "The API to be used", 0, INI_STRING, NULL, & ior_rnd4K_o.api}, 11 | {"blockSize", "Size of a random block, change only if explicitly allowed", 0, INI_UINT64, "1073741824", & ior_rnd4K_o.block_size}, 12 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & ior_rnd4K_o.run}, 13 | {"verbosity", "The verbosity level", 0, INI_INT, 0, & ior_rnd4K_o.verbosity}, 14 | {"randomPrefill", "Prefill the file with this blocksize in bytes, e.g., 2097152", 0, INI_INT, "0", & ior_rnd4K_o.random_prefill_bytes}, 15 | {NULL} }; 16 | 17 | 18 | static void validate(void){ 19 | opt_ior_rnd d = ior_rnd4K_o; 20 | if(d.block_size < 1024){ 21 | FATAL("Random blocksize must be larger than 1024\n"); 22 | } 23 | if(d.random_prefill_bytes > 0 && (d.block_size % d.random_prefill_bytes) != 0){ 24 | FATAL("Random prefill bytes must divide blocksize\n"); 25 | } 26 | if(d.random_prefill_bytes > 0 && d.block_size < d.random_prefill_bytes){ 27 | FATAL("Random prefill bytes must be < blocksize\n"); 28 | } 29 | } 30 | 31 | 32 | void ior_rnd4K_add_params(u_argv_t * argv){ 33 | opt_ior_rnd d = ior_rnd4K_o; 34 | 35 | u_argv_push(argv, "./ior"); 36 | u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType); 37 | for(int i=0; i < ior_rnd4K_o.verbosity; i++){ 38 | u_argv_push(argv, "-v"); 39 | } 40 | if(opt.io_buffers_on_gpu){ 41 | u_argv_push(argv, "-O"); 42 | u_argv_push(argv, "allocateBufferOnGPU=1"); 43 | } 44 | u_argv_push(argv, "-Q=1"); 45 | //u_argv_push(argv, "-F"); 46 | u_argv_push(argv, "-g"); 47 | int hash = u_phase_unique_random_number("ior-random4K"); 48 | u_argv_push_printf(argv, "-G=%d", hash); 49 | u_argv_push(argv, "-z"); 50 | u_argv_push(argv, "--random-offset-seed=123"); 51 | u_argv_push(argv, "-e"); 52 | u_argv_push_printf(argv, "-o=%s/ior-rnd4K/file", opt.datadir); 53 | u_argv_push(argv, "-O"); 54 | u_argv_push_printf(argv, "stoneWallingStatusFile=%s/ior-rnd4K.stonewall", opt.resdir ); 55 | u_argv_push(argv, "-k"); 56 | u_argv_push(argv, "-t=4096"); 57 | u_argv_push_printf(argv, "-b=%ld", d.block_size); 58 | u_argv_push_printf(argv, "-s=%d", 10000000); 59 | } 60 | 61 | u_phase_t p_ior_rnd4K = { 62 | "ior-rnd4K", 63 | IO500_PHASE_DUMMY, 64 | option, 65 | validate, 66 | NULL, 67 | NULL 68 | }; 69 | -------------------------------------------------------------------------------- /src/phase_ior_rnd_read1MB.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | typedef struct{ 9 | int run; 10 | char * api; 11 | 12 | char * command; 13 | IOR_point_t * res; 14 | } opt_ior_rnd_read; 15 | 16 | static opt_ior_rnd_read o; 17 | 18 | static ini_option_t option[] = { 19 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.api}, 20 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, 21 | {NULL} }; 22 | 23 | 24 | static void validate(void){ 25 | 26 | } 27 | 28 | 29 | static double run(void){ 30 | opt_ior_rnd d = ior_rnd1MB_o; 31 | 32 | u_argv_t * argv = u_argv_create(); 33 | ior_rnd1MB_add_params(argv); 34 | u_argv_push(argv, "-r"); 35 | u_argv_push(argv, "-R"); 36 | u_argv_push_default_if_set_api_options(argv, "-a", d.api, o.api); 37 | u_argv_push(argv, "-O"); 38 | u_argv_push_printf(argv, "saveRankPerformanceDetailsCSV=%s/ior-rnd1MB-read.csv", opt.resdir); 39 | 40 | o.command = u_flatten_argv(argv); 41 | 42 | PRINT_PAIR("exe", "%s\n", o.command); 43 | if(opt.dry_run || d.run == 0 || o.run == 0){ 44 | u_argv_free(argv); 45 | return 0; 46 | } 47 | FILE * out = u_res_file_prep(p_ior_rnd1MB_read.name); 48 | return ior_process_read(argv, out, & o.res); 49 | } 50 | 51 | 52 | u_phase_t p_ior_rnd1MB_read = { 53 | "ior-rnd1MB-read", 54 | IO500_PHASE_READ | IO500_PHASE_FLAG_OPTIONAL, 55 | option, 56 | validate, 57 | run, 58 | 0, 59 | .group = IO500_SCORE_BW, 60 | }; 61 | -------------------------------------------------------------------------------- /src/phase_ior_rnd_read4K.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | typedef struct{ 9 | int run; 10 | char * api; 11 | 12 | char * command; 13 | IOR_point_t * res; 14 | } opt_ior_rnd_read; 15 | 16 | static opt_ior_rnd_read o; 17 | 18 | static ini_option_t option[] = { 19 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.api}, 20 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, 21 | {NULL} }; 22 | 23 | 24 | static void validate(void){ 25 | 26 | } 27 | 28 | 29 | static double run(void){ 30 | opt_ior_rnd d = ior_rnd4K_o; 31 | 32 | u_argv_t * argv = u_argv_create(); 33 | ior_rnd4K_add_params(argv); 34 | u_argv_push(argv, "-r"); 35 | u_argv_push(argv, "-R"); 36 | u_argv_push_default_if_set_api_options(argv, "-a", d.api, o.api); 37 | u_argv_push(argv, "-O"); 38 | u_argv_push_printf(argv, "saveRankPerformanceDetailsCSV=%s/ior-rnd4K-read.csv", opt.resdir); 39 | 40 | o.command = u_flatten_argv(argv); 41 | 42 | PRINT_PAIR("exe", "%s\n", o.command); 43 | if(opt.dry_run || d.run == 0 || o.run == 0){ 44 | u_argv_free(argv); 45 | return 0; 46 | } 47 | FILE * out = u_res_file_prep(p_ior_rnd4K_read.name); 48 | return ior_process_read(argv, out, & o.res); 49 | } 50 | 51 | 52 | u_phase_t p_ior_rnd4K_read = { 53 | "ior-rnd4K-read", 54 | IO500_PHASE_READ | IO500_PHASE_FLAG_OPTIONAL, 55 | option, 56 | validate, 57 | run, 58 | 0, 59 | .group = IO500_SCORE_BW, 60 | }; 61 | -------------------------------------------------------------------------------- /src/phase_ior_rnd_read4k-easywrite.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | typedef struct{ 9 | int run; 10 | char * api; 11 | 12 | char * command; 13 | IOR_point_t * res; 14 | } opt_ior_rnd_read; 15 | 16 | static opt_ior_rnd_read o; 17 | 18 | static ini_option_t option[] = { 19 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.api}, 20 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, 21 | {NULL} }; 22 | 23 | 24 | static void validate(void){ 25 | 26 | } 27 | 28 | 29 | static double run(void){ 30 | opt_ior_rnd_read d = o; 31 | 32 | u_argv_t * argv = u_argv_create(); 33 | ior_easy_add_params(argv, 0); 34 | u_argv_push(argv, "-r"); 35 | u_argv_push(argv, "-D"); 36 | u_argv_push_printf(argv, "%d", opt.stonewall); 37 | u_argv_push(argv, "-z"); 38 | u_argv_push(argv, "-z"); 39 | u_argv_push_default_if_set_api_options(argv, "-a", d.api, o.api); 40 | u_argv_push(argv, "-O"); 41 | u_argv_push_printf(argv, "saveRankPerformanceDetailsCSV=%s/ior-rnd4K-easy-read.csv", opt.resdir); 42 | u_argv_push(argv, "-t=4096"); 43 | u_argv_push(argv, "-b=4096"); 44 | u_argv_push(argv, "-s=10000000"); 45 | u_argv_push(argv, "-O"); 46 | u_argv_push(argv, "stoneWallingWearOut=1"); 47 | 48 | o.command = u_flatten_argv(argv); 49 | 50 | PRINT_PAIR("exe", "%s\n", o.command); 51 | if(opt.dry_run || d.run == 0 || o.run == 0){ 52 | u_argv_free(argv); 53 | return 0; 54 | } 55 | FILE * out = u_res_file_prep(p_ior_rnd4K_read_easywrite.name); 56 | return ior_process_read(argv, out, & o.res); 57 | } 58 | 59 | 60 | u_phase_t p_ior_rnd4K_read_easywrite = { 61 | "ior-rnd4K-easy-read", 62 | IO500_PHASE_READ, 63 | option, 64 | validate, 65 | run, 66 | 0, 67 | .group = IO500_NO_SCORE, 68 | }; 69 | -------------------------------------------------------------------------------- /src/phase_ior_rnd_write1MB.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | typedef struct{ 10 | int run; 11 | char * api; 12 | 13 | char * command; 14 | IOR_point_t * res; 15 | } opt_ior_rnd_write; 16 | 17 | static opt_ior_rnd_write o; 18 | 19 | static ini_option_t option[] = { 20 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.api}, 21 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, 22 | {NULL} }; 23 | 24 | static void validate(void){ 25 | if( ! opt.dry_run && opt.rank == 0){ 26 | u_create_datadir("ior-rnd1MB"); 27 | } 28 | } 29 | 30 | static void cleanup(void){ 31 | if (opt.dry_run) return; 32 | 33 | if(opt.rank == 0){ 34 | char filename[PATH_MAX]; 35 | sprintf(filename, "%s/ior-rnd1MB.stonewall", opt.resdir); 36 | unlink(filename); 37 | u_purge_file("ior-rnd1MB/file"); 38 | u_purge_datadir("ior-rnd1MB"); 39 | } 40 | } 41 | 42 | static double run(void){ 43 | opt_ior_rnd d = ior_rnd1MB_o; 44 | 45 | u_argv_t * argv = u_argv_create(); 46 | ior_rnd1MB_add_params(argv); 47 | u_argv_push(argv, "-w"); 48 | u_argv_push(argv, "-D"); 49 | u_argv_push_printf(argv, "%d", opt.stonewall); 50 | u_argv_push_default_if_set_api_options(argv, "-a", d.api, o.api); 51 | if(d.random_prefill_bytes > 0){ 52 | u_argv_push_printf(argv, "--randomPrefill=%u", d.random_prefill_bytes); 53 | } 54 | u_argv_push(argv, "-O"); 55 | u_argv_push(argv, "stoneWallingWearOut=1"); 56 | u_argv_push(argv, "-O"); 57 | u_argv_push_printf(argv, "saveRankPerformanceDetailsCSV=%s/ior-rnd1MB-write.csv", opt.resdir); 58 | o.command = u_flatten_argv(argv); 59 | 60 | PRINT_PAIR("exe", "%s\n", o.command); 61 | if(opt.dry_run || d.run == 0 || o.run == 0){ 62 | u_argv_free(argv); 63 | return 0; 64 | } 65 | FILE * out = u_res_file_prep(p_ior_rnd1MB_write.name); 66 | return ior_process_write(argv, out, & o.res); 67 | } 68 | 69 | u_phase_t p_ior_rnd1MB_write = { 70 | "ior-rnd1MB-write", 71 | IO500_PHASE_WRITE | IO500_PHASE_FLAG_OPTIONAL, 72 | option, 73 | validate, 74 | run, 75 | .verify_stonewall = 1, 76 | .group = IO500_SCORE_BW, 77 | .cleanup = cleanup, 78 | }; 79 | -------------------------------------------------------------------------------- /src/phase_ior_rnd_write4K.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | typedef struct{ 10 | int run; 11 | char * api; 12 | 13 | char * command; 14 | IOR_point_t * res; 15 | } opt_ior_rnd_write; 16 | 17 | static opt_ior_rnd_write o; 18 | 19 | static ini_option_t option[] = { 20 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.api}, 21 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, 22 | {NULL} }; 23 | 24 | static void validate(void){ 25 | if( ! opt.dry_run && opt.rank == 0){ 26 | u_create_datadir("ior-rnd4K"); 27 | } 28 | } 29 | 30 | static void cleanup(void){ 31 | if (opt.dry_run) return; 32 | 33 | if(opt.rank == 0){ 34 | char filename[PATH_MAX]; 35 | sprintf(filename, "%s/ior-rnd4K.stonewall", opt.resdir); 36 | unlink(filename); 37 | u_purge_file("ior-rnd4K/file"); 38 | u_purge_datadir("ior-rnd4K"); 39 | } 40 | } 41 | 42 | static double run(void){ 43 | opt_ior_rnd d = ior_rnd4K_o; 44 | 45 | u_argv_t * argv = u_argv_create(); 46 | ior_rnd4K_add_params(argv); 47 | u_argv_push(argv, "-w"); 48 | u_argv_push(argv, "-D"); 49 | u_argv_push_printf(argv, "%d", opt.stonewall); 50 | u_argv_push_default_if_set_api_options(argv, "-a", d.api, o.api); 51 | if(d.random_prefill_bytes > 0){ 52 | u_argv_push_printf(argv, "--randomPrefill=%u", d.random_prefill_bytes); 53 | } 54 | u_argv_push(argv, "-O"); 55 | u_argv_push(argv, "stoneWallingWearOut=1"); 56 | u_argv_push(argv, "-O"); 57 | u_argv_push_printf(argv, "saveRankPerformanceDetailsCSV=%s/ior-rnd4K-write.csv", opt.resdir); 58 | o.command = u_flatten_argv(argv); 59 | 60 | PRINT_PAIR("exe", "%s\n", o.command); 61 | if(opt.dry_run || d.run == 0 || o.run == 0){ 62 | u_argv_free(argv); 63 | return 0; 64 | } 65 | FILE * out = u_res_file_prep(p_ior_rnd4K_write.name); 66 | return ior_process_write(argv, out, & o.res); 67 | } 68 | 69 | u_phase_t p_ior_rnd4K_write = { 70 | "ior-rnd4K-write", 71 | IO500_PHASE_WRITE | IO500_PHASE_FLAG_OPTIONAL, 72 | option, 73 | validate, 74 | run, 75 | .verify_stonewall = 1, 76 | .group = IO500_SCORE_BW, 77 | .cleanup = cleanup, 78 | }; 79 | -------------------------------------------------------------------------------- /src/phase_mdtest.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void p_mdtest_run(u_argv_t * argv, FILE * out, mdtest_generic_res * d, mdtest_test_num_t test){ 4 | mdtest_results_t * res = mdtest_run(argv->size, argv->vector, MPI_COMM_WORLD, out); 5 | d->time = res->time[test]; 6 | INFO_PAIR("time", "%f\n", d->time); 7 | d->items = res->items[test]; 8 | d->rate = res->rate[test] / 1000; 9 | d->rate_stonewall = res->stonewall_item_sum[test] / res->stonewall_time[test] / 1000; 10 | u_res_file_close(out); 11 | u_argv_free(argv); 12 | free(res); 13 | } 14 | 15 | void mdtest_add_generic_params(u_argv_t * argv, opt_mdtest_generic * dflt, opt_mdtest_generic * generic){ 16 | u_argv_push_default_if_set_api_options(argv, "-a", dflt->api, generic->api); 17 | 18 | generic->command = u_flatten_argv(argv); 19 | PRINT_PAIR("exe", "%s\n", generic->command); 20 | } 21 | -------------------------------------------------------------------------------- /src/phase_mdtest.h: -------------------------------------------------------------------------------- 1 | #ifndef IO500_PHASE_MDTEST_H 2 | #define IO500_PHASE_MDTEST_H 3 | 4 | #include 5 | #include 6 | 7 | typedef struct{ 8 | int run; 9 | char * api; 10 | 11 | uint64_t files_per_proc; 12 | uint64_t files_per_dir; 13 | char * command; 14 | } opt_mdtest_generic; 15 | 16 | typedef struct{ 17 | double rate; 18 | double rate_stonewall; 19 | uint64_t items; 20 | double time; 21 | } mdtest_generic_res; 22 | 23 | 24 | typedef struct{ 25 | opt_mdtest_generic g; 26 | mdtest_generic_res res; 27 | } opt_mdtest_easy; 28 | 29 | extern opt_mdtest_easy mdtest_easy_o; 30 | 31 | typedef struct{ 32 | opt_mdtest_generic g; 33 | mdtest_generic_res res; 34 | } opt_mdtest_hard; 35 | 36 | extern opt_mdtest_hard mdtest_hard_o; 37 | 38 | void mdtest_add_generic_params(u_argv_t * argv, opt_mdtest_generic * dflt, opt_mdtest_generic * generic); 39 | 40 | void mdtest_easy_add_params(u_argv_t * argv); 41 | void mdtest_hard_add_params(u_argv_t * argv); 42 | 43 | void p_mdtest_run(u_argv_t * argv, FILE * out, mdtest_generic_res * d, mdtest_test_num_t test); 44 | 45 | // retrieve results in a later stage 46 | mdtest_generic_res* mdtest_easy_write_get_result(void); 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /src/phase_mdtest_easy.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | opt_mdtest_easy mdtest_easy_o; 10 | 11 | static ini_option_t option[] = { 12 | {"API", "The API to be used", 0, INI_STRING, NULL, & mdtest_easy_o.g.api}, 13 | {"n", "Files per proc", 0, INI_UINT64, "1000000", & mdtest_easy_o.g.files_per_proc}, 14 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & mdtest_easy_o.g.run}, 15 | {NULL} }; 16 | 17 | static void validate(void){ 18 | 19 | } 20 | 21 | static void cleanup(void){ 22 | if( ! opt.dry_run && opt.rank == 0){ 23 | char filename[PATH_MAX]; 24 | sprintf(filename, "%s/mdtest-easy.stonewall", opt.resdir); 25 | unlink(filename); 26 | u_purge_datadir("mdtest-easy"); 27 | } 28 | } 29 | 30 | void mdtest_easy_add_params(u_argv_t * argv){ 31 | opt_mdtest_easy d = mdtest_easy_o; 32 | 33 | u_argv_push(argv, "./mdtest"); 34 | u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType); 35 | if(opt.io_buffers_on_gpu){ 36 | u_argv_push(argv, "--allocateBufferOnGPU"); 37 | } 38 | u_argv_push(argv, "-n"); /* number of files per process */ 39 | u_argv_push_printf(argv, "%"PRIu64, d.g.files_per_proc); 40 | u_argv_push(argv, "-u"); /* unique output directory per process */ 41 | u_argv_push(argv, "-L"); /* create files only at leaf of tree */ 42 | u_argv_push(argv, "-F"); /* create only files, not directories */ 43 | u_argv_push(argv, "-P"); /* print both creation rate and elapsed time */ 44 | u_argv_push(argv, "-G"); 45 | int hash = u_phase_unique_random_number("mdtest-easy"); 46 | u_argv_push_printf(argv, "%d", hash); 47 | u_argv_push(argv, "-N"); /* number of ranks between neighbours */ 48 | u_argv_push(argv, "1"); 49 | u_argv_push(argv, "-d"); /* output directory */ 50 | u_argv_push_printf(argv, "%s/mdtest-easy", opt.datadir); 51 | u_argv_push(argv, "-x"); /* stonewall filename */ 52 | u_argv_push_printf(argv, "%s/mdtest-easy.stonewall", opt.resdir); 53 | } 54 | 55 | u_phase_t p_mdtest_easy = { 56 | "mdtest-easy", 57 | IO500_PHASE_DUMMY, 58 | option, 59 | validate, 60 | NULL, 61 | .cleanup = cleanup, 62 | }; 63 | -------------------------------------------------------------------------------- /src/phase_mdtest_easy_delete.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | typedef struct{ 8 | opt_mdtest_generic g; 9 | mdtest_generic_res res; 10 | } opt_mdtest_easy_delete; 11 | 12 | static opt_mdtest_easy_delete o; 13 | 14 | static ini_option_t option[] = { 15 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.g.api}, 16 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.g.run}, 17 | {NULL} }; 18 | 19 | 20 | static void validate(void){ 21 | 22 | } 23 | 24 | static double run(void){ 25 | u_argv_t * argv = u_argv_create(); 26 | mdtest_easy_add_params(argv); 27 | u_argv_push(argv, "-r"); /* only remove files */ 28 | u_argv_push_printf(argv, "--saveRankPerformanceDetails=%s/mdtest-easy-delete.csv", opt.resdir); 29 | 30 | opt_mdtest_easy d = mdtest_easy_o; 31 | mdtest_add_generic_params(argv, & d.g, & o.g); 32 | 33 | if(opt.dry_run || o.g.run == 0 || mdtest_easy_o.g.run == 0){ 34 | u_argv_free(argv); 35 | return 0; 36 | } 37 | 38 | FILE * out = u_res_file_prep(p_mdtest_easy_delete.name); 39 | p_mdtest_run(argv, out, & o.res, MDTEST_FILE_REMOVE_NUM); 40 | 41 | return o.res.rate; 42 | } 43 | 44 | u_phase_t p_mdtest_easy_delete = { 45 | "mdtest-easy-delete", 46 | IO500_PHASE_REMOVE, 47 | option, 48 | validate, 49 | run, 50 | 0, 51 | .group = IO500_SCORE_MD 52 | }; 53 | -------------------------------------------------------------------------------- /src/phase_mdtest_easy_stat.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | typedef struct{ 8 | opt_mdtest_generic g; 9 | mdtest_generic_res res; 10 | } opt_mdtest_easy_stat; 11 | 12 | static opt_mdtest_easy_stat o; 13 | 14 | static ini_option_t option[] = { 15 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.g.api}, 16 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.g.run}, 17 | {NULL} }; 18 | 19 | 20 | static void validate(void){ 21 | 22 | } 23 | 24 | static double run(void){ 25 | u_argv_t * argv = u_argv_create(); 26 | mdtest_easy_add_params(argv); 27 | u_argv_push(argv, "-T"); /* only stat files */ 28 | u_argv_push_printf(argv, "--saveRankPerformanceDetails=%s/mdtest-easy-stat.csv", opt.resdir); 29 | 30 | opt_mdtest_easy d = mdtest_easy_o; 31 | mdtest_add_generic_params(argv, & d.g, & o.g); 32 | 33 | if(opt.dry_run || o.g.run == 0 || mdtest_easy_o.g.run == 0){ 34 | u_argv_free(argv); 35 | return 0; 36 | } 37 | 38 | FILE * out = u_res_file_prep(p_mdtest_easy_stat.name); 39 | p_mdtest_run(argv, out, & o.res, MDTEST_FILE_STAT_NUM); 40 | return o.res.rate; 41 | } 42 | 43 | u_phase_t p_mdtest_easy_stat = { 44 | "mdtest-easy-stat", 45 | IO500_PHASE_READ, 46 | option, 47 | validate, 48 | run, 49 | 0, 50 | .group = IO500_SCORE_MD 51 | }; 52 | -------------------------------------------------------------------------------- /src/phase_mdtest_easy_write.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | typedef struct{ 8 | opt_mdtest_generic g; 9 | mdtest_generic_res res; 10 | } opt_mdtest_easy_write; 11 | 12 | static opt_mdtest_easy_write o; 13 | 14 | static ini_option_t option[] = { 15 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.g.api}, 16 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.g.run}, 17 | {NULL} }; 18 | 19 | mdtest_generic_res * mdtest_easy_write_get_result(){ 20 | return & o.res; 21 | } 22 | 23 | 24 | static void validate(void){ 25 | 26 | } 27 | 28 | static double run(void){ 29 | u_argv_t * argv = u_argv_create(); 30 | mdtest_easy_add_params(argv); 31 | u_argv_push(argv, "-C"); /* only create files */ 32 | u_argv_push(argv, "-Y"); /* call sync command after each phase */ 33 | u_argv_push(argv, "-W"); /* deadline for stonewall in seconds */ 34 | u_argv_push_printf(argv, "%d", opt.stonewall); 35 | u_argv_push_printf(argv, "--saveRankPerformanceDetails=%s/mdtest-easy-write.csv", opt.resdir); 36 | 37 | opt_mdtest_easy d = mdtest_easy_o; 38 | mdtest_add_generic_params(argv, & d.g, & o.g); 39 | 40 | if(opt.dry_run || o.g.run == 0 || mdtest_easy_o.g.run == 0){ 41 | u_argv_free(argv); 42 | return 0; 43 | } 44 | 45 | FILE * out = u_res_file_prep(p_mdtest_easy_write.name); 46 | p_mdtest_run(argv, out, & o.res, MDTEST_FILE_CREATE_NUM); 47 | 48 | PRINT_PAIR("rate-stonewall", "%f\n", o.res.rate_stonewall); 49 | return o.res.rate; 50 | } 51 | 52 | u_phase_t p_mdtest_easy_write = { 53 | "mdtest-easy-write", 54 | IO500_PHASE_WRITE, 55 | option, 56 | validate, 57 | run, 58 | .verify_stonewall = 1, 59 | .group = IO500_SCORE_MD 60 | }; 61 | -------------------------------------------------------------------------------- /src/phase_mdtest_hard.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | opt_mdtest_hard mdtest_hard_o; 10 | 11 | static ini_option_t option[] = { 12 | {"API", "The API to be used", 0, INI_STRING, NULL, & mdtest_hard_o.g.api}, 13 | {"n", "Files per proc", 0, INI_UINT64, "1000000", & mdtest_hard_o.g.files_per_proc}, 14 | {"files-per-dir", "File limit per directory (MDTest -I flag) to overcome file system limitations "DEBUG_OPTION, 0, INI_UINT64, NULL, & mdtest_hard_o.g.files_per_dir}, 15 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & mdtest_hard_o.g.run}, 16 | {NULL} }; 17 | 18 | static void validate(void){ 19 | if(mdtest_hard_o.g.files_per_dir != INI_UNSET_UINT64 && mdtest_hard_o.g.files_per_dir > 0 && opt.rank == 0){ 20 | INVALID("files-per-dir is set\n"); 21 | } 22 | } 23 | 24 | static void cleanup(void){ 25 | if( ! opt.dry_run && opt.rank == 0){ 26 | char filename[PATH_MAX]; 27 | sprintf(filename, "%s/mdtest-hard.stonewall", opt.resdir); 28 | unlink(filename); 29 | u_purge_datadir("mdtest-hard"); 30 | } 31 | } 32 | 33 | void mdtest_hard_add_params(u_argv_t * argv){ 34 | opt_mdtest_hard d = mdtest_hard_o; 35 | 36 | u_argv_push(argv, "./mdtest"); 37 | u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType); 38 | if(opt.io_buffers_on_gpu){ 39 | u_argv_push(argv, "--allocateBufferOnGPU"); 40 | } 41 | u_argv_push(argv, "-n"); /* number of files per process */ 42 | u_argv_push_printf(argv, "%"PRIu64, d.g.files_per_proc); 43 | 44 | if(mdtest_hard_o.g.files_per_dir != INI_UNSET_UINT64 && mdtest_hard_o.g.files_per_dir > 0){ 45 | u_argv_push_printf(argv, "-I=%"PRIu64, mdtest_hard_o.g.files_per_dir); 46 | } 47 | 48 | u_argv_push(argv, "-t"); /* include subdirectory creation in timing */ 49 | u_argv_push(argv, "-w"); /* write data to each file after creation */ 50 | u_argv_push(argv, "3901"); 51 | u_argv_push(argv, "-e"); /* read data from each file */ 52 | u_argv_push(argv, "3901"); 53 | u_argv_push(argv, "-P"); /* print both creation rate and elapsed time */ 54 | int hash = u_phase_unique_random_number("mdtest-hard"); 55 | u_argv_push_printf(argv, "-G=%d", hash); 56 | u_argv_push(argv, "-N"); /* number of ranks between neighbours */ 57 | u_argv_push(argv, "1"); 58 | u_argv_push(argv, "-F"); /* create only files, not directories */ 59 | u_argv_push(argv, "-d"); /* output directory */ 60 | u_argv_push_printf(argv, "%s/mdtest-hard", opt.datadir); 61 | u_argv_push(argv, "-x"); /* stonewall synchronization filename */ 62 | u_argv_push_printf(argv, "%s/mdtest-hard.stonewall", opt.resdir); 63 | } 64 | 65 | u_phase_t p_mdtest_hard = { 66 | "mdtest-hard", 67 | IO500_PHASE_DUMMY, 68 | option, 69 | validate, 70 | NULL, 71 | .cleanup = cleanup, 72 | }; 73 | -------------------------------------------------------------------------------- /src/phase_mdtest_hard_delete.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | typedef struct{ 8 | opt_mdtest_generic g; 9 | mdtest_generic_res res; 10 | } opt_mdtest_hard_delete; 11 | 12 | static opt_mdtest_hard_delete o; 13 | 14 | static ini_option_t option[] = { 15 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.g.api}, 16 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.g.run}, 17 | {NULL} }; 18 | 19 | 20 | static void validate(void){ 21 | 22 | } 23 | 24 | static double run(void){ 25 | u_argv_t * argv = u_argv_create(); 26 | mdtest_hard_add_params(argv); 27 | u_argv_push(argv, "-r"); /* only remove files */ 28 | u_argv_push_printf(argv, "--saveRankPerformanceDetails=%s/mdtest-hard-delete.csv", opt.resdir); 29 | 30 | opt_mdtest_hard d = mdtest_hard_o; 31 | mdtest_add_generic_params(argv, & d.g, & o.g); 32 | 33 | if(opt.dry_run || o.g.run == 0 || mdtest_hard_o.g.run == 0){ 34 | u_argv_free(argv); 35 | return 0; 36 | } 37 | 38 | FILE * out = u_res_file_prep(p_mdtest_hard_delete.name); 39 | p_mdtest_run(argv, out, & o.res, MDTEST_FILE_REMOVE_NUM); 40 | 41 | return o.res.rate; 42 | } 43 | 44 | u_phase_t p_mdtest_hard_delete = { 45 | "mdtest-hard-delete", 46 | IO500_PHASE_REMOVE, 47 | option, 48 | validate, 49 | run, 50 | 0, 51 | .group = IO500_SCORE_MD 52 | }; 53 | -------------------------------------------------------------------------------- /src/phase_mdtest_hard_read.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | typedef struct{ 8 | opt_mdtest_generic g; 9 | mdtest_generic_res res; 10 | } opt_mdtest_hard_read; 11 | 12 | static opt_mdtest_hard_read o; 13 | 14 | static ini_option_t option[] = { 15 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.g.api}, 16 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.g.run}, 17 | {NULL} }; 18 | 19 | 20 | static void validate(void){ 21 | 22 | } 23 | 24 | static double run(void){ 25 | u_argv_t * argv = u_argv_create(); 26 | mdtest_hard_add_params(argv); 27 | u_argv_push(argv, "-E"); /* only read files */ 28 | u_argv_push(argv, "-X"); /* verify data read */ 29 | u_argv_push_printf(argv, "--saveRankPerformanceDetails=%s/mdtest-hard-read.csv", opt.resdir); 30 | 31 | opt_mdtest_hard d = mdtest_hard_o; 32 | mdtest_add_generic_params(argv, & d.g, & o.g); 33 | 34 | if(opt.dry_run || o.g.run == 0 || mdtest_hard_o.g.run == 0){ 35 | u_argv_free(argv); 36 | return 0; 37 | } 38 | 39 | FILE * out = u_res_file_prep(p_mdtest_hard_read.name); 40 | p_mdtest_run(argv, out, & o.res, MDTEST_FILE_READ_NUM); 41 | 42 | return o.res.rate; 43 | } 44 | 45 | u_phase_t p_mdtest_hard_read = { 46 | "mdtest-hard-read", 47 | IO500_PHASE_READ, 48 | option, 49 | validate, 50 | run, 51 | 0, 52 | .group = IO500_SCORE_MD 53 | }; 54 | -------------------------------------------------------------------------------- /src/phase_mdtest_hard_stat.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | typedef struct{ 8 | opt_mdtest_generic g; 9 | mdtest_generic_res res; 10 | } opt_mdtest_hard_stat; 11 | 12 | static opt_mdtest_hard_stat o; 13 | 14 | static ini_option_t option[] = { 15 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.g.api}, 16 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.g.run}, 17 | {NULL} }; 18 | 19 | 20 | static void validate(void){ 21 | 22 | } 23 | 24 | static double run(void){ 25 | u_argv_t * argv = u_argv_create(); 26 | mdtest_hard_add_params(argv); 27 | u_argv_push(argv, "-T"); /* only stat files */ 28 | u_argv_push_printf(argv, "--saveRankPerformanceDetails=%s/mdtest-hard-stat.csv", opt.resdir); 29 | 30 | opt_mdtest_hard d = mdtest_hard_o; 31 | mdtest_add_generic_params(argv, & d.g, & o.g); 32 | 33 | if(opt.dry_run || o.g.run == 0 || mdtest_hard_o.g.run == 0){ 34 | u_argv_free(argv); 35 | return 0; 36 | } 37 | 38 | FILE * out = u_res_file_prep(p_mdtest_hard_stat.name); 39 | p_mdtest_run(argv, out, & o.res, MDTEST_FILE_STAT_NUM); 40 | return o.res.rate; 41 | } 42 | 43 | u_phase_t p_mdtest_hard_stat = { 44 | "mdtest-hard-stat", 45 | IO500_PHASE_READ, 46 | option, 47 | validate, 48 | run, 49 | 0, 50 | .group = IO500_SCORE_MD 51 | }; 52 | -------------------------------------------------------------------------------- /src/phase_mdtest_hard_write.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | typedef struct{ 8 | opt_mdtest_generic g; 9 | mdtest_generic_res res; 10 | } opt_mdtest_hard_write; 11 | 12 | static opt_mdtest_hard_write o; 13 | 14 | static ini_option_t option[] = { 15 | {"API", "The API to be used", 0, INI_STRING, NULL, & o.g.api}, 16 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.g.run}, 17 | {NULL} }; 18 | 19 | 20 | static void validate(void){ 21 | 22 | } 23 | 24 | static double run(void){ 25 | u_argv_t * argv = u_argv_create(); 26 | mdtest_hard_add_params(argv); 27 | u_argv_push(argv, "-C"); /* only create files */ 28 | u_argv_push(argv, "-Y"); /* call sync command after each phase */ 29 | 30 | opt_mdtest_hard d = mdtest_hard_o; 31 | if((d.g.files_per_dir != INI_UNSET_UINT64 && d.g.files_per_dir > 0)){ 32 | // Must disable stonewalling for supporting this option -- for now! 33 | WARNING("stonewalling disabled in order to support -I option. Make sure your number of elements is big enough to meet the runtime limits!"); 34 | }else{ 35 | u_argv_push(argv, "-W"); /* deadline for stonewall in seconds */ 36 | u_argv_push_printf(argv, "%d", opt.stonewall); 37 | } 38 | 39 | u_argv_push_printf(argv, "--saveRankPerformanceDetails=%s/mdtest-hard-write.csv", opt.resdir); 40 | 41 | mdtest_add_generic_params(argv, & d.g, & o.g); 42 | 43 | if(opt.dry_run || o.g.run == 0 || mdtest_hard_o.g.run == 0){ 44 | u_argv_free(argv); 45 | return 0; 46 | } 47 | 48 | FILE * out = u_res_file_prep(p_mdtest_hard_write.name); 49 | p_mdtest_run(argv, out, & o.res, MDTEST_FILE_CREATE_NUM); 50 | 51 | PRINT_PAIR("rate-stonewall", "%f\n", o.res.rate_stonewall); 52 | return o.res.rate; 53 | } 54 | 55 | u_phase_t p_mdtest_hard_write = { 56 | "mdtest-hard-write", 57 | IO500_PHASE_WRITE, 58 | option, 59 | validate, 60 | run, 61 | .verify_stonewall = 1, 62 | .group = IO500_SCORE_MD 63 | }; 64 | -------------------------------------------------------------------------------- /src/phase_mdworkbench.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | /* 9 | This phase adjusts the precreate based on the rate of mdtest. 10 | It attempts to run pre-create for 60s. 11 | */ 12 | 13 | opt_mdworkbench mdworkbench_o; 14 | 15 | static ini_option_t option[] = { 16 | {"API", "The API to be used", 0, INI_STRING, NULL, & mdworkbench_o.api}, 17 | {"waitingTime", "Waiting time of an IO operation relative to runtime (1.0 is 100%%)", 0, INI_FLOAT, "0.0", & mdworkbench_o.waiting_time}, 18 | {"precreatePerSet", "Files to precreate per set (always 10 sets), this is normally dynamically determined", 0, INI_UINT64, NULL, & mdworkbench_o.precreate_per_set}, 19 | {"filesPerProc", "Files to run per iteration and set (always 10 sets), this is normally dynamically determined", 0, INI_UINT64, NULL, & mdworkbench_o.files_per_proc}, 20 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & mdworkbench_o.run}, 21 | {"verbosity", "The verbosity level", 0, INI_INT, 0, & mdworkbench_o.verbosity}, 22 | {NULL} }; 23 | 24 | static void validate(void){ 25 | opt_mdworkbench d = mdworkbench_o; 26 | 27 | if(d.precreate_per_set != INI_UNSET_UINT64 || d.files_per_proc != INI_UNSET_UINT64){ 28 | if(d.precreate_per_set == INI_UNSET_UINT64 || d.files_per_proc == INI_UNSET_UINT64){ 29 | FATAL("MDWorkbench, must set either both options or none of: filesPerProc and precreatePerSet\n"); 30 | } 31 | } 32 | } 33 | 34 | 35 | void mdworkbench_process(u_argv_t * argv, FILE * out, mdworkbench_results_t ** res_out){ 36 | mdworkbench_results_t * res = md_workbench_run(argv->size, argv->vector, MPI_COMM_WORLD, out); 37 | u_res_file_close(out); 38 | u_argv_free(argv); 39 | 40 | if(res->errors != 0){ 41 | INVALID("Errors (%d) occured during the md-workbench phase. This invalidates your run.\n", res->errors); 42 | } 43 | *res_out = res; 44 | } 45 | 46 | 47 | 48 | void mdworkbench_add_params(u_argv_t * argv, int is_create){ 49 | opt_mdworkbench * d = & mdworkbench_o; 50 | 51 | u_argv_push(argv, "./md-workbench"); 52 | u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType); 53 | for(int i=0; i < mdworkbench_o.verbosity; i++){ 54 | u_argv_push(argv, "-v"); 55 | } 56 | if(opt.io_buffers_on_gpu){ 57 | u_argv_push(argv, "--allocateBufferOnGPU"); 58 | } 59 | u_argv_push(argv, "--process-reports"); 60 | u_argv_push_default_if_set_api_options(argv, "-a", d->api, d->api); 61 | u_argv_push_printf(argv, "-o=%s/mdworkbench", opt.datadir); 62 | u_argv_push_printf(argv, "-t=%f", d->waiting_time); 63 | u_argv_push(argv, "-O=1"); 64 | u_argv_push_printf(argv, "--run-info-file=%s/mdworkbench.status", opt.resdir ); 65 | 66 | // determine the number of objects to precreate and the number of objects per iteration 67 | 68 | uint64_t precreate_per_set; 69 | uint64_t files_per_proc; 70 | 71 | if (d->precreate_per_set != INI_UNSET_UINT64 && d->files_per_proc != INI_UNSET_UINT64){ 72 | precreate_per_set = d->precreate_per_set; 73 | files_per_proc = d->files_per_proc; 74 | }else{ 75 | mdtest_generic_res* mdtest = mdtest_easy_write_get_result(); 76 | MPI_Bcast(& mdtest->rate, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); 77 | 78 | char file[PATH_MAX]; 79 | sprintf(file, "%s/mdworkbench-size", opt.resdir); 80 | if (is_create && opt.rank == 0){ 81 | // store the actual processed size, allows easy deletion 82 | FILE * f = fopen(file, "w"); 83 | if(! f){ 84 | WARNING("Couldn't open mdworkbench-file: %s\n", file); 85 | }else{ 86 | fwrite(& mdtest->rate, sizeof(mdtest->rate), 1, f); 87 | } 88 | fclose(f); 89 | } 90 | if(! is_create && mdtest->rate <= 0.0){ 91 | // read the size back as this is a deletion run 92 | if(opt.rank == 0){ 93 | FILE * f = fopen(file, "r"); 94 | if(! f){ 95 | WARNING("Couldn't open mdworkbench-file: %s\n", file); 96 | }else{ 97 | fread(& mdtest->rate, sizeof(mdtest->rate), 1, f); 98 | } 99 | fclose(f); 100 | } 101 | MPI_Bcast(& mdtest->rate, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); 102 | } 103 | if( mdtest->rate <= 0.0 ){ 104 | if(opt.rank == 0){ 105 | WARNING("MDWorkbench uses the MDTest rates to determine suitable options but MDTest didn't run, will use a low (and sane) default instead\n"); 106 | } 107 | mdtest->rate = 10.0; 108 | } 109 | // run for 60s 110 | int time = opt.stonewall < 60 ? opt.stonewall : 60; 111 | precreate_per_set = (uint64_t) (mdtest->rate * time * 1000 / opt.mpi_size) / 10; 112 | 113 | files_per_proc = precreate_per_set; 114 | } 115 | // we have 10 sets 116 | u_argv_push(argv, "-D=10"); 117 | PRINT_PAIR("filesPerProc", "%"PRIu64"\n", files_per_proc); 118 | PRINT_PAIR("precreatePerSet", "%"PRIu64"\n", precreate_per_set); 119 | int hash = u_phase_unique_random_number("md-workbench"); 120 | u_argv_push_printf(argv, "-G=%d", hash); 121 | u_argv_push_printf(argv, "-P=%"PRIu64, precreate_per_set); 122 | u_argv_push_printf(argv, "-I=%"PRIu64, files_per_proc); 123 | } 124 | 125 | u_phase_t p_mdworkbench = { 126 | "mdworkbench", 127 | IO500_PHASE_DUMMY, 128 | option, 129 | validate, 130 | NULL, 131 | NULL, 132 | }; 133 | -------------------------------------------------------------------------------- /src/phase_mdworkbench.h: -------------------------------------------------------------------------------- 1 | #ifndef IO500_PHASE_MDWORKBENCH_H 2 | #define IO500_PHASE_MDWORKBENCH_H 3 | 4 | #include 5 | #include 6 | 7 | typedef struct{ 8 | int run; 9 | char * api; 10 | int verbosity; 11 | float waiting_time; 12 | uint64_t precreate_per_set; 13 | uint64_t files_per_proc; 14 | } opt_mdworkbench; 15 | 16 | extern opt_mdworkbench mdworkbench_o; 17 | 18 | void mdworkbench_add_params(u_argv_t * argv, int is_create); 19 | void mdworkbench_process(u_argv_t * argv, FILE * out, mdworkbench_results_t ** res_out); 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /src/phase_mdworkbench_bench.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | typedef struct{ 9 | int run; 10 | char * command; 11 | mdworkbench_results_t * res; 12 | } opt_mdworkbench_bench; 13 | 14 | static opt_mdworkbench_bench o; 15 | 16 | static ini_option_t option[] = { 17 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, 18 | {NULL} }; 19 | 20 | 21 | static double run(void){ 22 | opt_mdworkbench d = mdworkbench_o; 23 | 24 | u_argv_t * argv = u_argv_create(); 25 | mdworkbench_add_params(argv, 0); 26 | u_argv_push(argv, "-2"); 27 | u_argv_push(argv, "-R=2"); 28 | u_argv_push(argv, "-X"); /* turn read verification on */ 29 | 30 | o.command = u_flatten_argv(argv); 31 | PRINT_PAIR("exe", "%s\n", o.command); 32 | if(opt.dry_run || d.run == 0){ 33 | u_argv_free(argv); 34 | return 0; 35 | } 36 | FILE * out = u_res_file_prep(p_mdworkbench_bench.name); 37 | mdworkbench_process(argv, out, & o.res); 38 | if(o.res->count != 2){ 39 | INVALID("During the md-workbench phase not two iterations are performed but %d This invalidates your run.\n", o.res->count); 40 | return 0.0; 41 | } 42 | 43 | double rate = o.res->result[1].rate; 44 | if( o.res->result[0].rate < rate * 0.5 45 | || o.res->result[0].rate * 0.5 > rate){ 46 | WARNING("The results of both md-workbench differs by 2x.\n"); 47 | } 48 | PRINT_PAIR("maxOpTime", "%f\n", o.res->result[1].max_op_time); 49 | PRINT_PAIR("scoreIteration0", "%f\n", o.res->result[0].rate / 1000.0); 50 | PRINT_PAIR("maxOpTime0", "%f\n", o.res->result[0].max_op_time); 51 | return rate / 1000.0; 52 | } 53 | 54 | 55 | u_phase_t p_mdworkbench_bench = { 56 | "mdworkbench-bench", 57 | IO500_PHASE_WRITE | IO500_PHASE_FLAG_OPTIONAL, 58 | option, 59 | NULL, 60 | run, 61 | 0, 62 | .group = IO500_SCORE_MD, 63 | }; 64 | -------------------------------------------------------------------------------- /src/phase_mdworkbench_create.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | typedef struct{ 9 | int run; 10 | char * command; 11 | mdworkbench_results_t * res; 12 | } opt_mdworkbench_create; 13 | 14 | static opt_mdworkbench_create o; 15 | 16 | static ini_option_t option[] = { 17 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, 18 | {NULL} }; 19 | 20 | 21 | static double run(void){ 22 | opt_mdworkbench d = mdworkbench_o; 23 | 24 | u_argv_t * argv = u_argv_create(); 25 | mdworkbench_add_params(argv, 1); 26 | u_argv_push(argv, "-1"); 27 | 28 | o.command = u_flatten_argv(argv); 29 | PRINT_PAIR("exe", "%s\n", o.command); 30 | if(opt.dry_run || d.run == 0){ 31 | u_argv_free(argv); 32 | return 0; 33 | } 34 | FILE * out = u_res_file_prep(p_mdworkbench_create.name); 35 | mdworkbench_process(argv, out, & o.res); 36 | PRINT_PAIR("maxOpTime", "%f\n", o.res->result[0].max_op_time); 37 | 38 | double rate = o.res->result[0].rate; 39 | return rate / 1000.0; 40 | } 41 | 42 | 43 | u_phase_t p_mdworkbench_create = { 44 | "mdworkbench-create", 45 | IO500_PHASE_WRITE | IO500_PHASE_FLAG_OPTIONAL , 46 | option, 47 | NULL, 48 | run, 49 | 0, 50 | .group = IO500_NO_SCORE, 51 | }; 52 | -------------------------------------------------------------------------------- /src/phase_mdworkbench_delete.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | typedef struct{ 9 | int run; 10 | char * command; 11 | mdworkbench_results_t * res; 12 | } opt_mdworkbench_delete; 13 | 14 | static opt_mdworkbench_delete o; 15 | 16 | static ini_option_t option[] = { 17 | {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, 18 | {NULL} }; 19 | 20 | static double run(void){ 21 | opt_mdworkbench d = mdworkbench_o; 22 | 23 | u_argv_t * argv = u_argv_create(); 24 | mdworkbench_add_params(argv, 0); 25 | u_argv_push(argv, "-3"); 26 | 27 | o.command = u_flatten_argv(argv); 28 | PRINT_PAIR("exe", "%s\n", o.command); 29 | if(opt.dry_run || d.run == 0){ 30 | u_argv_free(argv); 31 | return 0; 32 | } 33 | FILE * out = u_res_file_prep(p_mdworkbench_delete.name); 34 | mdworkbench_process(argv, out, & o.res); 35 | PRINT_PAIR("maxOpTime", "%f\n", o.res->result[0].max_op_time); 36 | 37 | double rate = o.res->result[0].rate; 38 | return rate / 1000.0; 39 | } 40 | 41 | 42 | u_phase_t p_mdworkbench_delete = { 43 | "mdworkbench-delete", 44 | IO500_PHASE_REMOVE | IO500_PHASE_FLAG_OPTIONAL, 45 | option, 46 | NULL, 47 | run, 48 | 0, 49 | .group = IO500_NO_SCORE, 50 | }; 51 | -------------------------------------------------------------------------------- /src/phase_opt.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | io500_opt_t opt; 4 | 5 | static ini_option_t option[] = { 6 | {"datadir", "The directory where the IO500 runs", 1, INI_STRING, "./datafiles", & opt.datadir}, 7 | {"timestamp-datadir", "The data directory is suffixed by a timestamp. Useful for running several IO500 tests concurrently.", 0, INI_BOOL, "TRUE", & opt.timestamp_datadir}, 8 | {"resultdir", "The result directory.", 0, INI_STRING, "./results", & opt.resdir}, 9 | {"timestamp-resultdir", "The result directory is suffixed by a timestamp. Useful for running several IO500 tests concurrently.", 0, INI_BOOL, "TRUE", & opt.timestamp_resdir}, 10 | {"api", "The general API for the tests (to create/delete the datadir, extra options will be passed to IOR/mdtest)", 0, INI_STRING, "POSIX", & opt.api}, 11 | {"drop-caches", "Purge the caches, this is useful for testing and needed for single node runs", 0, INI_BOOL, "FALSE", & opt.drop_caches}, 12 | {"drop-caches-cmd", "Cache purging command, invoked before each I/O phase", 0, INI_STRING, "sudo -n bash -c \"echo 3 > /proc/sys/vm/drop_caches\"", & opt.drop_caches_cmd}, 13 | {"io-buffers-on-gpu", "Allocate the I/O buffers on the GPU", 0, INI_BOOL, "FALSE", & opt.io_buffers_on_gpu}, 14 | {"verbosity", "The verbosity level between 1 and 10", 0, INI_UINT, "1", & opt.verbosity}, 15 | {"scc", "Use the rules for the Student Cluster Competition", 0, INI_BOOL, "FALSE", & opt.scc}, 16 | {"dataPacketType", "Type of packet that will be created [timestamp|offset|incompressible|random]", 0, INI_STRING, "timestamp", & opt.dataPacketType}, 17 | {NULL} }; 18 | 19 | static void validate(void){ 20 | 21 | } 22 | 23 | u_phase_t p_opt = { 24 | "global", 25 | IO500_PHASE_DUMMY, 26 | option, 27 | validate, 28 | NULL 29 | }; 30 | -------------------------------------------------------------------------------- /src/phase_timestamp.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | static ini_option_t option[] = { 7 | {NULL} }; 8 | 9 | 10 | static void validate(void){ 11 | 12 | } 13 | 14 | static double run(void){ 15 | if(opt.rank != 0) return 0; 16 | 17 | char timestamp_file[PATH_MAX]; 18 | sprintf(timestamp_file, "%s/timestampfile", opt.resdir); 19 | INFO_PAIR("timestamp-file", "%s\n", timestamp_file); 20 | FILE * f = fopen(timestamp_file, "w"); 21 | if(! f){ 22 | FATAL("Couldn't open timestampfile: %s\n", timestamp_file); 23 | } 24 | fclose(f); 25 | return 0; 26 | } 27 | 28 | u_phase_t p_timestamp = { 29 | "timestamp", 30 | IO500_PHASE_DUMMY, 31 | option, 32 | validate, 33 | run, 34 | 0 35 | }; 36 | -------------------------------------------------------------------------------- /src/test/CMakeFiles/CMakeDirectoryInformation.cmake: -------------------------------------------------------------------------------- 1 | # CMAKE generated file: DO NOT EDIT! 2 | # Generated by "Unix Makefiles" Generator, CMake Version 3.16 3 | 4 | # Relative path conversion top directories. 5 | set(CMAKE_RELATIVE_PATH_TOP_SOURCE "/home/kunkel/ur-git/esiwace/ESD-Middleware") 6 | set(CMAKE_RELATIVE_PATH_TOP_BINARY "/home/kunkel/ur-git/esiwace/ESD-Middleware") 7 | 8 | # Force unix paths in dependencies. 9 | set(CMAKE_FORCE_UNIX_PATHS 1) 10 | 11 | 12 | # The C and CXX include file regular expressions for this directory. 13 | set(CMAKE_C_INCLUDE_REGEX_SCAN "^.*$") 14 | set(CMAKE_C_INCLUDE_REGEX_COMPLAIN "^$") 15 | set(CMAKE_CXX_INCLUDE_REGEX_SCAN ${CMAKE_C_INCLUDE_REGEX_SCAN}) 16 | set(CMAKE_CXX_INCLUDE_REGEX_COMPLAIN ${CMAKE_C_INCLUDE_REGEX_COMPLAIN}) 17 | -------------------------------------------------------------------------------- /src/test/CMakeFiles/metatest.dir/DependInfo.cmake: -------------------------------------------------------------------------------- 1 | # The set of languages for which implicit dependencies are needed: 2 | set(CMAKE_DEPENDS_LANGUAGES 3 | "C" 4 | ) 5 | # The set of files for implicit dependencies of each language: 6 | set(CMAKE_DEPENDS_CHECK_C 7 | "/home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test/metatest.c" "/home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test/CMakeFiles/metatest.dir/metatest.c.o" 8 | ) 9 | set(CMAKE_C_COMPILER_ID "GNU") 10 | 11 | # The include file search paths: 12 | set(CMAKE_C_TARGET_INCLUDE_PATH 13 | "src" 14 | "." 15 | "src/include" 16 | "/usr/include/glib-2.0" 17 | "/usr/lib/x86_64-linux-gnu/glib-2.0/include" 18 | "deps/smd/include" 19 | "/usr/lib/x86_64-linux-gnu/openmpi/include/openmpi" 20 | "/usr/lib/x86_64-linux-gnu/openmpi/include" 21 | ) 22 | 23 | # Targets to which this target links. 24 | set(CMAKE_TARGET_LINKED_INFO_FILES 25 | "/home/kunkel/ur-git/esiwace/ESD-Middleware/src/CMakeFiles/esdm.dir/DependInfo.cmake" 26 | "/home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-data/posix/CMakeFiles/esdmposix.dir/DependInfo.cmake" 27 | "/home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-data/dummy/CMakeFiles/esdmdummy.dir/DependInfo.cmake" 28 | "/home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/CMakeFiles/esdm-mdposix.dir/DependInfo.cmake" 29 | "/home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-data/pmem/CMakeFiles/esdmpmem.dir/DependInfo.cmake" 30 | "/home/kunkel/ur-git/esiwace/ESD-Middleware/src/CMakeFiles/esdmutils.dir/DependInfo.cmake" 31 | "/home/kunkel/ur-git/esiwace/ESD-Middleware/deps/smd/src/CMakeFiles/smd.dir/DependInfo.cmake" 32 | ) 33 | 34 | # Fortran module output directory. 35 | set(CMAKE_Fortran_TARGET_MODULE_DIR "") 36 | -------------------------------------------------------------------------------- /src/test/CMakeFiles/metatest.dir/build.make: -------------------------------------------------------------------------------- 1 | # CMAKE generated file: DO NOT EDIT! 2 | # Generated by "Unix Makefiles" Generator, CMake Version 3.16 3 | 4 | # Delete rule output on recipe failure. 5 | .DELETE_ON_ERROR: 6 | 7 | 8 | #============================================================================= 9 | # Special targets provided by cmake. 10 | 11 | # Disable implicit rules so canonical targets will work. 12 | .SUFFIXES: 13 | 14 | 15 | # Remove some rules from gmake that .SUFFIXES does not remove. 16 | SUFFIXES = 17 | 18 | .SUFFIXES: .hpux_make_needs_suffix_list 19 | 20 | 21 | # Suppress display of executed commands. 22 | $(VERBOSE).SILENT: 23 | 24 | 25 | # A target that is always out of date. 26 | cmake_force: 27 | 28 | .PHONY : cmake_force 29 | 30 | #============================================================================= 31 | # Set environment variables for the build. 32 | 33 | # The shell in which to execute make rules. 34 | SHELL = /bin/sh 35 | 36 | # The CMake executable. 37 | CMAKE_COMMAND = /usr/bin/cmake 38 | 39 | # The command to remove a file. 40 | RM = /usr/bin/cmake -E remove -f 41 | 42 | # Escaping for special characters. 43 | EQUALS = = 44 | 45 | # The top-level source directory on which CMake was run. 46 | CMAKE_SOURCE_DIR = /home/kunkel/ur-git/esiwace/ESD-Middleware 47 | 48 | # The top-level build directory on which CMake was run. 49 | CMAKE_BINARY_DIR = /home/kunkel/ur-git/esiwace/ESD-Middleware 50 | 51 | # Include any dependencies generated for this target. 52 | include src/backends-metadata/posix/test/CMakeFiles/metatest.dir/depend.make 53 | 54 | # Include the progress variables for this target. 55 | include src/backends-metadata/posix/test/CMakeFiles/metatest.dir/progress.make 56 | 57 | # Include the compile flags for this target's objects. 58 | include src/backends-metadata/posix/test/CMakeFiles/metatest.dir/flags.make 59 | 60 | src/backends-metadata/posix/test/CMakeFiles/metatest.dir/metatest.c.o: src/backends-metadata/posix/test/CMakeFiles/metatest.dir/flags.make 61 | src/backends-metadata/posix/test/CMakeFiles/metatest.dir/metatest.c.o: src/backends-metadata/posix/test/metatest.c 62 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --green --progress-dir=/home/kunkel/ur-git/esiwace/ESD-Middleware/CMakeFiles --progress-num=$(CMAKE_PROGRESS_1) "Building C object src/backends-metadata/posix/test/CMakeFiles/metatest.dir/metatest.c.o" 63 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test && /usr/bin/cc $(C_DEFINES) $(C_INCLUDES) $(C_FLAGS) -o CMakeFiles/metatest.dir/metatest.c.o -c /home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test/metatest.c 64 | 65 | src/backends-metadata/posix/test/CMakeFiles/metatest.dir/metatest.c.i: cmake_force 66 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --green "Preprocessing C source to CMakeFiles/metatest.dir/metatest.c.i" 67 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test && /usr/bin/cc $(C_DEFINES) $(C_INCLUDES) $(C_FLAGS) -E /home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test/metatest.c > CMakeFiles/metatest.dir/metatest.c.i 68 | 69 | src/backends-metadata/posix/test/CMakeFiles/metatest.dir/metatest.c.s: cmake_force 70 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --green "Compiling C source to assembly CMakeFiles/metatest.dir/metatest.c.s" 71 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test && /usr/bin/cc $(C_DEFINES) $(C_INCLUDES) $(C_FLAGS) -S /home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test/metatest.c -o CMakeFiles/metatest.dir/metatest.c.s 72 | 73 | # Object files for target metatest 74 | metatest_OBJECTS = \ 75 | "CMakeFiles/metatest.dir/metatest.c.o" 76 | 77 | # External object files for target metatest 78 | metatest_EXTERNAL_OBJECTS = 79 | 80 | src/backends-metadata/posix/test/metatest: src/backends-metadata/posix/test/CMakeFiles/metatest.dir/metatest.c.o 81 | src/backends-metadata/posix/test/metatest: src/backends-metadata/posix/test/CMakeFiles/metatest.dir/build.make 82 | src/backends-metadata/posix/test/metatest: src/libesdm.so 83 | src/backends-metadata/posix/test/metatest: /usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so 84 | src/backends-metadata/posix/test/metatest: src/backends-data/posix/libesdmposix.so 85 | src/backends-metadata/posix/test/metatest: src/backends-data/dummy/libesdmdummy.so 86 | src/backends-metadata/posix/test/metatest: src/backends-metadata/posix/libesdm-mdposix.so 87 | src/backends-metadata/posix/test/metatest: src/backends-data/pmem/libesdmpmem.so 88 | src/backends-metadata/posix/test/metatest: src/libesdmutils.so 89 | src/backends-metadata/posix/test/metatest: deps/smd/src/libsmd.so 90 | src/backends-metadata/posix/test/metatest: /usr/local/lib/libpmem.so 91 | src/backends-metadata/posix/test/metatest: src/backends-metadata/posix/test/CMakeFiles/metatest.dir/link.txt 92 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --green --bold --progress-dir=/home/kunkel/ur-git/esiwace/ESD-Middleware/CMakeFiles --progress-num=$(CMAKE_PROGRESS_2) "Linking C executable metatest" 93 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test && $(CMAKE_COMMAND) -E cmake_link_script CMakeFiles/metatest.dir/link.txt --verbose=$(VERBOSE) 94 | 95 | # Rule to build all files generated by this target. 96 | src/backends-metadata/posix/test/CMakeFiles/metatest.dir/build: src/backends-metadata/posix/test/metatest 97 | 98 | .PHONY : src/backends-metadata/posix/test/CMakeFiles/metatest.dir/build 99 | 100 | src/backends-metadata/posix/test/CMakeFiles/metatest.dir/clean: 101 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test && $(CMAKE_COMMAND) -P CMakeFiles/metatest.dir/cmake_clean.cmake 102 | .PHONY : src/backends-metadata/posix/test/CMakeFiles/metatest.dir/clean 103 | 104 | src/backends-metadata/posix/test/CMakeFiles/metatest.dir/depend: 105 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware && $(CMAKE_COMMAND) -E cmake_depends "Unix Makefiles" /home/kunkel/ur-git/esiwace/ESD-Middleware /home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test /home/kunkel/ur-git/esiwace/ESD-Middleware /home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test /home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test/CMakeFiles/metatest.dir/DependInfo.cmake --color=$(COLOR) 106 | .PHONY : src/backends-metadata/posix/test/CMakeFiles/metatest.dir/depend 107 | 108 | -------------------------------------------------------------------------------- /src/test/CMakeFiles/metatest.dir/cmake_clean.cmake: -------------------------------------------------------------------------------- 1 | file(REMOVE_RECURSE 2 | "CMakeFiles/metatest.dir/metatest.c.o" 3 | "metatest" 4 | "metatest.pdb" 5 | ) 6 | 7 | # Per-language clean rules from dependency scanning. 8 | foreach(lang C) 9 | include(CMakeFiles/metatest.dir/cmake_clean_${lang}.cmake OPTIONAL) 10 | endforeach() 11 | -------------------------------------------------------------------------------- /src/test/CMakeFiles/metatest.dir/depend.make: -------------------------------------------------------------------------------- 1 | # Empty dependencies file for metatest. 2 | # This may be replaced when dependencies are built. 3 | -------------------------------------------------------------------------------- /src/test/CMakeFiles/metatest.dir/flags.make: -------------------------------------------------------------------------------- 1 | # CMAKE generated file: DO NOT EDIT! 2 | # Generated by "Unix Makefiles" Generator, CMake Version 3.16 3 | 4 | # compile C with /usr/bin/cc 5 | C_FLAGS = -std=gnu11 -pedantic -Wall -Wdouble-promotion -Wfloat-equal -flto -fdata-sections -ffunction-sections -Wl,--gc-sections -Wl,--allow-shlib-undefined -Wl,--retain-symbols-file=/home/kunkel/ur-git/esiwace/ESD-Middleware/dev/symbols.txt -pedantic-errors -DDEBUG_OFF -O3 -D__FILENAME__='"$(subst /home/kunkel/ur-git/esiwace/ESD-Middleware/,,$(abspath $<))"' 6 | 7 | C_DEFINES = 8 | 9 | C_INCLUDES = -I/home/kunkel/ur-git/esiwace/ESD-Middleware/src -isystem /home/kunkel/ur-git/esiwace/ESD-Middleware -isystem /home/kunkel/ur-git/esiwace/ESD-Middleware/src/include -isystem /usr/include/glib-2.0 -isystem /usr/lib/x86_64-linux-gnu/glib-2.0/include -isystem /home/kunkel/ur-git/esiwace/ESD-Middleware/deps/smd/include -isystem /usr/lib/x86_64-linux-gnu/openmpi/include/openmpi -isystem /usr/lib/x86_64-linux-gnu/openmpi/include 10 | 11 | -------------------------------------------------------------------------------- /src/test/CMakeFiles/metatest.dir/link.txt: -------------------------------------------------------------------------------- 1 | /usr/bin/cc -std=gnu11 -pedantic -Wall -Wdouble-promotion -Wfloat-equal -flto -fdata-sections -ffunction-sections -Wl,--gc-sections -Wl,--allow-shlib-undefined -Wl,--retain-symbols-file=/home/kunkel/ur-git/esiwace/ESD-Middleware/dev/symbols.txt -pedantic-errors -DDEBUG_OFF -O3 -D__FILENAME__='"$(subst /home/kunkel/ur-git/esiwace/ESD-Middleware/,,$(abspath $<))"' -rdynamic CMakeFiles/metatest.dir/metatest.c.o -o metatest -Wl,-rpath,/home/kunkel/ur-git/esiwace/ESD-Middleware/src:/usr/lib/x86_64-linux-gnu/openmpi/lib:/home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-data/posix:/home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-data/dummy:/home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix:/home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-data/pmem:/home/kunkel/ur-git/esiwace/ESD-Middleware/deps/smd/src:/usr/local/lib ../../../libesdm.so /usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so -lrt ../../../backends-data/posix/libesdmposix.so ../../../backends-data/dummy/libesdmdummy.so ../libesdm-mdposix.so ../../../backends-data/pmem/libesdmpmem.so ../../../libesdmutils.so -lglib-2.0 -ljansson ../../../../deps/smd/src/libsmd.so -pthread -lm -lglib-2.0 /usr/local/lib/libpmem.so 2 | -------------------------------------------------------------------------------- /src/test/CMakeFiles/metatest.dir/progress.make: -------------------------------------------------------------------------------- 1 | CMAKE_PROGRESS_1 = 63 2 | CMAKE_PROGRESS_2 = 3 | 4 | -------------------------------------------------------------------------------- /src/test/CMakeFiles/progress.marks: -------------------------------------------------------------------------------- 1 | 26 2 | -------------------------------------------------------------------------------- /src/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB TESTFILES "${CMAKE_CURRENT_SOURCE_DIR}" "*.c") 2 | foreach(TESTFILE ${TESTFILES}) 3 | if(IS_DIRECTORY ${TESTFILE} ) 4 | #message(STATUS ${TESTFILE}) 5 | else() 6 | get_filename_component(TESTNAME_C ${TESTFILE} NAME) 7 | STRING(REGEX REPLACE ".c$" "" TESTNAME ${TESTNAME_C}) 8 | 9 | # Build, link and add as test 10 | add_executable(${TESTNAME} ${TESTFILE}) 11 | target_link_libraries(${TESTNAME} esdm ${MPI_LIBRARIES} -lrt) 12 | target_include_directories(${TESTNAME} PRIVATE ${MPI_INCLUDE_PATH} ${CMAKE_BINARY_DIR} ${ESDM_INCLUDE_DIRS} ${GLIB_INCLUDE_DIRS}) 13 | 14 | add_test(${TESTNAME} ./${TESTNAME}) 15 | endif() 16 | endforeach() 17 | -------------------------------------------------------------------------------- /src/test/CTestTestfile.cmake: -------------------------------------------------------------------------------- 1 | # CMake generated Testfile for 2 | # Source directory: /home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test 3 | # Build directory: /home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test 4 | # 5 | # This file includes the relevant testing commands required for 6 | # testing this directory and lists subdirectories to be tested as well. 7 | add_test(metatest "./metatest") 8 | set_tests_properties(metatest PROPERTIES _BACKTRACE_TRIPLES "/home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test/CMakeLists.txt;14;add_test;/home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test/CMakeLists.txt;0;") 9 | -------------------------------------------------------------------------------- /src/test/Makefile: -------------------------------------------------------------------------------- 1 | # CMAKE generated file: DO NOT EDIT! 2 | # Generated by "Unix Makefiles" Generator, CMake Version 3.16 3 | 4 | # Default target executed when no arguments are given to make. 5 | default_target: all 6 | 7 | .PHONY : default_target 8 | 9 | # Allow only one "make -f Makefile2" at a time, but pass parallelism. 10 | .NOTPARALLEL: 11 | 12 | 13 | #============================================================================= 14 | # Special targets provided by cmake. 15 | 16 | # Disable implicit rules so canonical targets will work. 17 | .SUFFIXES: 18 | 19 | 20 | # Remove some rules from gmake that .SUFFIXES does not remove. 21 | SUFFIXES = 22 | 23 | .SUFFIXES: .hpux_make_needs_suffix_list 24 | 25 | 26 | # Suppress display of executed commands. 27 | $(VERBOSE).SILENT: 28 | 29 | 30 | # A target that is always out of date. 31 | cmake_force: 32 | 33 | .PHONY : cmake_force 34 | 35 | #============================================================================= 36 | # Set environment variables for the build. 37 | 38 | # The shell in which to execute make rules. 39 | SHELL = /bin/sh 40 | 41 | # The CMake executable. 42 | CMAKE_COMMAND = /usr/bin/cmake 43 | 44 | # The command to remove a file. 45 | RM = /usr/bin/cmake -E remove -f 46 | 47 | # Escaping for special characters. 48 | EQUALS = = 49 | 50 | # The top-level source directory on which CMake was run. 51 | CMAKE_SOURCE_DIR = /home/kunkel/ur-git/esiwace/ESD-Middleware 52 | 53 | # The top-level build directory on which CMake was run. 54 | CMAKE_BINARY_DIR = /home/kunkel/ur-git/esiwace/ESD-Middleware 55 | 56 | #============================================================================= 57 | # Targets provided globally by CMake. 58 | 59 | # Special rule for the target install/strip 60 | install/strip: preinstall 61 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Installing the project stripped..." 62 | /usr/bin/cmake -DCMAKE_INSTALL_DO_STRIP=1 -P cmake_install.cmake 63 | .PHONY : install/strip 64 | 65 | # Special rule for the target install/strip 66 | install/strip/fast: preinstall/fast 67 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Installing the project stripped..." 68 | /usr/bin/cmake -DCMAKE_INSTALL_DO_STRIP=1 -P cmake_install.cmake 69 | .PHONY : install/strip/fast 70 | 71 | # Special rule for the target install/local 72 | install/local: preinstall 73 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Installing only the local directory..." 74 | /usr/bin/cmake -DCMAKE_INSTALL_LOCAL_ONLY=1 -P cmake_install.cmake 75 | .PHONY : install/local 76 | 77 | # Special rule for the target install/local 78 | install/local/fast: preinstall/fast 79 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Installing only the local directory..." 80 | /usr/bin/cmake -DCMAKE_INSTALL_LOCAL_ONLY=1 -P cmake_install.cmake 81 | .PHONY : install/local/fast 82 | 83 | # Special rule for the target edit_cache 84 | edit_cache: 85 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Running CMake cache editor..." 86 | /usr/bin/ccmake -S$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) 87 | .PHONY : edit_cache 88 | 89 | # Special rule for the target edit_cache 90 | edit_cache/fast: edit_cache 91 | 92 | .PHONY : edit_cache/fast 93 | 94 | # Special rule for the target test 95 | test: 96 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Running tests..." 97 | /usr/bin/ctest --force-new-ctest-process $(ARGS) 98 | .PHONY : test 99 | 100 | # Special rule for the target test 101 | test/fast: test 102 | 103 | .PHONY : test/fast 104 | 105 | # Special rule for the target install 106 | install: preinstall 107 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Install the project..." 108 | /usr/bin/cmake -P cmake_install.cmake 109 | .PHONY : install 110 | 111 | # Special rule for the target install 112 | install/fast: preinstall/fast 113 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Install the project..." 114 | /usr/bin/cmake -P cmake_install.cmake 115 | .PHONY : install/fast 116 | 117 | # Special rule for the target list_install_components 118 | list_install_components: 119 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Available install components are: \"Unspecified\"" 120 | .PHONY : list_install_components 121 | 122 | # Special rule for the target list_install_components 123 | list_install_components/fast: list_install_components 124 | 125 | .PHONY : list_install_components/fast 126 | 127 | # Special rule for the target rebuild_cache 128 | rebuild_cache: 129 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Running CMake to regenerate build system..." 130 | /usr/bin/cmake -S$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) 131 | .PHONY : rebuild_cache 132 | 133 | # Special rule for the target rebuild_cache 134 | rebuild_cache/fast: rebuild_cache 135 | 136 | .PHONY : rebuild_cache/fast 137 | 138 | # The main all target 139 | all: cmake_check_build_system 140 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware && $(CMAKE_COMMAND) -E cmake_progress_start /home/kunkel/ur-git/esiwace/ESD-Middleware/CMakeFiles /home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test/CMakeFiles/progress.marks 141 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware && $(MAKE) -f CMakeFiles/Makefile2 src/backends-metadata/posix/test/all 142 | $(CMAKE_COMMAND) -E cmake_progress_start /home/kunkel/ur-git/esiwace/ESD-Middleware/CMakeFiles 0 143 | .PHONY : all 144 | 145 | # The main clean target 146 | clean: 147 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware && $(MAKE) -f CMakeFiles/Makefile2 src/backends-metadata/posix/test/clean 148 | .PHONY : clean 149 | 150 | # The main clean target 151 | clean/fast: clean 152 | 153 | .PHONY : clean/fast 154 | 155 | # Prepare targets for installation. 156 | preinstall: all 157 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware && $(MAKE) -f CMakeFiles/Makefile2 src/backends-metadata/posix/test/preinstall 158 | .PHONY : preinstall 159 | 160 | # Prepare targets for installation. 161 | preinstall/fast: 162 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware && $(MAKE) -f CMakeFiles/Makefile2 src/backends-metadata/posix/test/preinstall 163 | .PHONY : preinstall/fast 164 | 165 | # clear depends 166 | depend: 167 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware && $(CMAKE_COMMAND) -S$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 1 168 | .PHONY : depend 169 | 170 | # Convenience name for target. 171 | src/backends-metadata/posix/test/CMakeFiles/metatest.dir/rule: 172 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware && $(MAKE) -f CMakeFiles/Makefile2 src/backends-metadata/posix/test/CMakeFiles/metatest.dir/rule 173 | .PHONY : src/backends-metadata/posix/test/CMakeFiles/metatest.dir/rule 174 | 175 | # Convenience name for target. 176 | metatest: src/backends-metadata/posix/test/CMakeFiles/metatest.dir/rule 177 | 178 | .PHONY : metatest 179 | 180 | # fast build rule for target. 181 | metatest/fast: 182 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware && $(MAKE) -f src/backends-metadata/posix/test/CMakeFiles/metatest.dir/build.make src/backends-metadata/posix/test/CMakeFiles/metatest.dir/build 183 | .PHONY : metatest/fast 184 | 185 | metatest.o: metatest.c.o 186 | 187 | .PHONY : metatest.o 188 | 189 | # target to build an object file 190 | metatest.c.o: 191 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware && $(MAKE) -f src/backends-metadata/posix/test/CMakeFiles/metatest.dir/build.make src/backends-metadata/posix/test/CMakeFiles/metatest.dir/metatest.c.o 192 | .PHONY : metatest.c.o 193 | 194 | metatest.i: metatest.c.i 195 | 196 | .PHONY : metatest.i 197 | 198 | # target to preprocess a source file 199 | metatest.c.i: 200 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware && $(MAKE) -f src/backends-metadata/posix/test/CMakeFiles/metatest.dir/build.make src/backends-metadata/posix/test/CMakeFiles/metatest.dir/metatest.c.i 201 | .PHONY : metatest.c.i 202 | 203 | metatest.s: metatest.c.s 204 | 205 | .PHONY : metatest.s 206 | 207 | # target to generate assembly for a file 208 | metatest.c.s: 209 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware && $(MAKE) -f src/backends-metadata/posix/test/CMakeFiles/metatest.dir/build.make src/backends-metadata/posix/test/CMakeFiles/metatest.dir/metatest.c.s 210 | .PHONY : metatest.c.s 211 | 212 | # Help Target 213 | help: 214 | @echo "The following are some of the valid targets for this Makefile:" 215 | @echo "... all (the default if no target is provided)" 216 | @echo "... clean" 217 | @echo "... depend" 218 | @echo "... install/strip" 219 | @echo "... install/local" 220 | @echo "... edit_cache" 221 | @echo "... test" 222 | @echo "... install" 223 | @echo "... list_install_components" 224 | @echo "... rebuild_cache" 225 | @echo "... metatest" 226 | @echo "... metatest.o" 227 | @echo "... metatest.i" 228 | @echo "... metatest.s" 229 | .PHONY : help 230 | 231 | 232 | 233 | #============================================================================= 234 | # Special targets to cleanup operation of make. 235 | 236 | # Special rule to run CMake to check the build system integrity. 237 | # No rule that depends on this can have commands that come from listfiles 238 | # because they might be regenerated. 239 | cmake_check_build_system: 240 | cd /home/kunkel/ur-git/esiwace/ESD-Middleware && $(CMAKE_COMMAND) -S$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 0 241 | .PHONY : cmake_check_build_system 242 | 243 | -------------------------------------------------------------------------------- /src/test/cmake_install.cmake: -------------------------------------------------------------------------------- 1 | # Install script for directory: /home/kunkel/ur-git/esiwace/ESD-Middleware/src/backends-metadata/posix/test 2 | 3 | # Set the install prefix 4 | if(NOT DEFINED CMAKE_INSTALL_PREFIX) 5 | set(CMAKE_INSTALL_PREFIX "/home/kunkel/ur-git/esiwace/ESD-Middleware/install") 6 | endif() 7 | string(REGEX REPLACE "/$" "" CMAKE_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") 8 | 9 | # Set the install configuration name. 10 | if(NOT DEFINED CMAKE_INSTALL_CONFIG_NAME) 11 | if(BUILD_TYPE) 12 | string(REGEX REPLACE "^[^A-Za-z0-9_]+" "" 13 | CMAKE_INSTALL_CONFIG_NAME "${BUILD_TYPE}") 14 | else() 15 | set(CMAKE_INSTALL_CONFIG_NAME "Release") 16 | endif() 17 | message(STATUS "Install configuration: \"${CMAKE_INSTALL_CONFIG_NAME}\"") 18 | endif() 19 | 20 | # Set the component getting installed. 21 | if(NOT CMAKE_INSTALL_COMPONENT) 22 | if(COMPONENT) 23 | message(STATUS "Install component: \"${COMPONENT}\"") 24 | set(CMAKE_INSTALL_COMPONENT "${COMPONENT}") 25 | else() 26 | set(CMAKE_INSTALL_COMPONENT) 27 | endif() 28 | endif() 29 | 30 | # Install shared libraries without execute permission? 31 | if(NOT DEFINED CMAKE_INSTALL_SO_NO_EXE) 32 | set(CMAKE_INSTALL_SO_NO_EXE "1") 33 | endif() 34 | 35 | # Is this installation the result of a crosscompile? 36 | if(NOT DEFINED CMAKE_CROSSCOMPILING) 37 | set(CMAKE_CROSSCOMPILING "FALSE") 38 | endif() 39 | 40 | -------------------------------------------------------------------------------- /src/test/metatest.c: -------------------------------------------------------------------------------- 1 | /** 2 | * This test uses the generic patterns to verify the high-level interface works as intended 3 | */ 4 | 5 | 6 | #include 7 | 8 | extern esdm_instance_t esdm; 9 | 10 | int main() { 11 | esdm_status ret; 12 | char const * cfg = "{\"esdm\": {\"backends\": [" 13 | "{" 14 | "\"type\": \"DUMMY\"," 15 | "\"id\": \"p1\"," 16 | "\"target\": \"x\"" 17 | "}" 18 | "]," 19 | "\"metadata\": {" 20 | "\"type\": \"metadummy\"," 21 | "\"id\": \"md\"," 22 | "\"target\": \"./_metadummy\"}}" 23 | "}"; 24 | esdm_load_config_str(cfg); 25 | esdm_init(); 26 | //esdm_md_backend_t *b = esdm.modules->metadata_backend; 27 | 28 | ret = esdm_mkfs(ESDM_FORMAT_PURGE_RECREATE, ESDM_ACCESSIBILITY_GLOBAL); 29 | eassert(ret == ESDM_SUCCESS); 30 | ret = esdm_mkfs(ESDM_FORMAT_PURGE_RECREATE, ESDM_ACCESSIBILITY_NODELOCAL); 31 | eassert(ret == ESDM_SUCCESS); 32 | 33 | esdm_simple_dspace_t dataspace = esdm_dataspace_2d(50, 100, SMD_DTYPE_UINT64); 34 | eassert(dataspace.ptr); 35 | esdm_container_t *container; 36 | 37 | ret = esdm_container_create("testContainer", 1, &container); 38 | eassert(ret == ESDM_SUCCESS); 39 | esdm_dataset_t *dataset; 40 | 41 | ret = esdm_dataset_create(container, "testDataset", dataspace.ptr, &dataset); 42 | eassert(ret == ESDM_SUCCESS); 43 | 44 | esdm_fragment_t *f1, *f2, *f3, *f4; 45 | 46 | { 47 | esdm_dataspace_t* space; 48 | ret = esdm_dataspace_create_full(2, (int64_t [2]){25, 50}, (int64_t [2]){0, 0}, SMD_DTYPE_UINT64, &space); 49 | eassert(ret == ESDM_SUCCESS); 50 | ret = esdmI_fragment_create(dataset, space, ea_checked_malloc(esdm_dataspace_total_bytes(space)), &f1); 51 | eassert(ret == ESDM_SUCCESS); 52 | } 53 | { 54 | esdm_dataspace_t* space; 55 | ret = esdm_dataspace_create_full(2, (int64_t [2]){25, 50}, (int64_t [2]){25, 0}, SMD_DTYPE_UINT64, &space); 56 | eassert(ret == ESDM_SUCCESS); 57 | ret = esdmI_fragment_create(dataset, space, ea_checked_malloc(esdm_dataspace_total_bytes(space)), &f2); 58 | eassert(ret == ESDM_SUCCESS); 59 | } 60 | { 61 | esdm_dataspace_t* space; 62 | ret = esdm_dataspace_create_full(2, (int64_t [2]){25, 50}, (int64_t [2]){25, 50}, SMD_DTYPE_UINT64, &space); 63 | eassert(ret == ESDM_SUCCESS); 64 | ret = esdmI_fragment_create(dataset, space, ea_checked_malloc(esdm_dataspace_total_bytes(space)), &f3); 65 | eassert(ret == ESDM_SUCCESS); 66 | } 67 | { 68 | esdm_dataspace_t* space; 69 | ret = esdm_dataspace_create_full(2, (int64_t [2]){25, 50}, (int64_t [2]){0, 50}, SMD_DTYPE_UINT64, &space); 70 | eassert(ret == ESDM_SUCCESS); 71 | ret = esdmI_fragment_create(dataset, space, ea_checked_malloc(esdm_dataspace_total_bytes(space)), &f4); 72 | eassert(ret == ESDM_SUCCESS); 73 | } 74 | 75 | //ret = b->callbacks.fragment_update(b, f1); 76 | //eassert(ret == ESDM_SUCCESS); 77 | //ret = b->callbacks.fragment_update(b, f2); 78 | //eassert(ret == ESDM_SUCCESS); 79 | //ret = b->callbacks.fragment_update(b, f3); 80 | //eassert(ret == ESDM_SUCCESS); 81 | //ret = b->callbacks.fragment_update(b, f4); 82 | //eassert(ret == ESDM_SUCCESS); 83 | 84 | //{ 85 | // int64_t size[] = {30, 30}; 86 | // int64_t offset[] = {10, 10}; 87 | // esdm_dataspace_subspace(dataspace, 2, size, offset, &res); 88 | //} 89 | 90 | esdmI_container_destroy(container); 91 | 92 | esdm_finalize(); 93 | 94 | return 0; 95 | } 96 | -------------------------------------------------------------------------------- /src/util.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | 11 | #include 12 | 13 | #include 14 | #include 15 | 16 | 17 | /** 18 | * rotate the value and add the current character 19 | */ 20 | static uint32_t rotladd (uint32_t x, char c){ 21 | return ((x<<2) | (x>>(32-2))) + c; 22 | } 23 | 24 | uint32_t u_hash_update(uint32_t hash, char const * str){ 25 | for(char const * c = str; *c != 0; c++){ 26 | hash = rotladd(hash, *c); 27 | } 28 | return hash; 29 | } 30 | 31 | /* compute a unique random number based on the phase and the timestamp */ 32 | uint32_t u_phase_unique_random_number(char const * phase_name){ 33 | uint32_t hash = 0; 34 | hash = u_hash_update(hash, phase_name); 35 | hash = u_hash_update(hash, opt.timestamp); 36 | if(hash == 0 || hash == -1){ 37 | return 4711; 38 | } 39 | return hash; 40 | } 41 | 42 | 43 | void u_hash_update_key_val(uint32_t * hash, char const * key, char const * val){ 44 | uint32_t hsh = 0; 45 | hsh = u_hash_update(hsh, key); 46 | hsh = u_hash_update(hsh, val); 47 | *hash = *hash ^ hsh; 48 | DEBUG_INFO("hash current: %X updated with (%s=%s)\n", (int)*hash, key, val); 49 | } 50 | 51 | void u_hash_update_key_val_dbl(uint32_t * hash, char const * key, double val){ 52 | char str[40]; 53 | sprintf(str, "%f", val); 54 | u_hash_update_key_val(hash, key, str); 55 | } 56 | 57 | void u_hash_print(FILE * file, uint32_t hash){ 58 | fprintf(file, "%X", (int) hash); 59 | } 60 | 61 | void u_call_cmd(char const * str){ 62 | int ret = system(str); 63 | if (ret != 0) { 64 | WARNING("Calling \"%s\" returned %d\n", str, ret); 65 | } 66 | } 67 | void u_purge_datadir(char const * dir){ 68 | char d[PATH_MAX]; 69 | sprintf(d, "%s/%s", opt.datadir, dir); 70 | DEBUG_INFO("Removing dir %s\n", d); 71 | 72 | opt.aiori->rmdir(d, opt.backend_opt); 73 | } 74 | 75 | void u_purge_file(char const * file){ 76 | char f[PATH_MAX]; 77 | sprintf(f, "%s/%s", opt.datadir, file); 78 | DEBUG_INFO("Removing file %s\n", f); 79 | opt.aiori->remove(f, opt.backend_opt); 80 | } 81 | 82 | void u_create_datadir(char const * dir){ 83 | if(opt.rank != 0){ 84 | return; 85 | } 86 | char d[PATH_MAX]; 87 | sprintf(d, "%s/%s", opt.datadir, dir); 88 | u_create_dir_recursive(d, opt.aiori, opt.backend_opt); 89 | } 90 | 91 | void u_create_dir_recursive(char const * dir, ior_aiori_t const * api, aiori_mod_opt_t * module_options){ 92 | char * d = strdup(dir); 93 | char outdir[PATH_MAX]; 94 | char * wp = outdir; 95 | if (dir[0] == '/'){ 96 | wp += sprintf(wp, "/"); 97 | } 98 | 99 | char * next = strtok(d, "/"); 100 | while(next){ 101 | if(*next == '/' || *next == 0) continue; 102 | wp += sprintf(wp, "%s/", next); 103 | 104 | struct stat sb; 105 | int ret = api->stat(outdir, & sb, module_options); 106 | if(ret != 0){ 107 | DEBUG_INFO("Creating dir %s\n", outdir); 108 | ret = api->mkdir(outdir, S_IRWXU, module_options); 109 | if(ret != 0){ 110 | FATAL("Couldn't create directory %s (Error: %s)\n", outdir, strerror(errno)); 111 | } 112 | } 113 | next = strtok(NULL, "/"); 114 | } 115 | free(d); 116 | } 117 | 118 | u_argv_t * u_argv_create(void){ 119 | u_argv_t * p = u_malloc(sizeof(u_argv_t)); 120 | memset(p, 0, sizeof(u_argv_t)); 121 | return p; 122 | } 123 | 124 | void u_argv_free(u_argv_t * argv){ 125 | for(int i=0; i < argv->size; i++){ 126 | free(argv->vector[i]); 127 | } 128 | free(argv); 129 | } 130 | 131 | void u_argv_push(u_argv_t * argv, char const * str){ 132 | int size = ++argv->size; 133 | argv->vector = realloc(argv->vector, size * sizeof(void*)); 134 | argv->vector[argv->size - 1] = strdup(str); 135 | } 136 | 137 | void u_argv_push_default_if_set_bool(u_argv_t * argv, char * const arg, int dflt, int var){ 138 | if(var != INI_UNSET_BOOL){ 139 | if((int) var) 140 | u_argv_push(argv, arg); 141 | }else if(dflt != INI_UNSET_BOOL){ 142 | if((int) dflt) 143 | u_argv_push(argv, arg); 144 | } 145 | } 146 | 147 | static void push_api_args(u_argv_t * argv, char const * var){ 148 | char * str = strdup(var); 149 | char * saveptr; 150 | char * t = strtok_r(str, " ", & saveptr); 151 | u_argv_push(argv, str); // this is the API 152 | if(t){ 153 | int len = strlen(str) + 3; 154 | char buff[len + 1]; 155 | sprintf(buff, "--%s.", str); 156 | while(true){ 157 | t = strtok_r(NULL, " ", & saveptr); 158 | if(! t){ 159 | break; 160 | } 161 | if(strncasecmp(buff, t, len) != 0){ 162 | WARNING("Provided API option %s starts with a different prefix and might be not supported - don't worry if you checked!\n", t); 163 | } 164 | u_argv_push(argv, t); 165 | } 166 | } 167 | free(str); 168 | } 169 | 170 | void u_argv_push_default_if_set_api_options(u_argv_t * argv, char * const arg, char const * dflt, char const * var){ 171 | if(var != INI_UNSET_STRING){ 172 | u_argv_push(argv, arg); 173 | push_api_args(argv, var); 174 | }else if(dflt != INI_UNSET_STRING){ 175 | u_argv_push(argv, arg); 176 | push_api_args(argv, dflt); 177 | }else{ 178 | // add generic args 179 | u_argv_push(argv, arg); 180 | if(opt.apiArgs){ 181 | push_api_args(argv, opt.apiArgs); 182 | }else{ 183 | u_argv_push(argv, opt.api); 184 | } 185 | } 186 | } 187 | 188 | void u_argv_push_default_if_set(u_argv_t * argv, char * const arg, char const * dflt, char const * var){ 189 | if(var != INI_UNSET_STRING){ 190 | u_argv_push(argv, arg); 191 | u_argv_push(argv, var); 192 | }else if(dflt != INI_UNSET_STRING){ 193 | u_argv_push(argv, arg); 194 | u_argv_push(argv, dflt); 195 | } 196 | } 197 | 198 | 199 | void u_argv_push_printf(u_argv_t * argv, char const * format, ...){ 200 | char buff[PATH_MAX]; 201 | va_list args; 202 | va_start(args, format); 203 | vsprintf(buff, format, args); 204 | va_end (args); 205 | u_argv_push(argv, buff); 206 | } 207 | 208 | char * u_flatten_argv(u_argv_t * argv){ 209 | char command[PATH_MAX]; 210 | char * p = command; 211 | *p = '\0'; 212 | for(int i = 0; i < argv->size; i++){ 213 | if(i != 0) p+= sprintf(p, " "); 214 | p += sprintf(p, "%s", argv->vector[i]); 215 | } 216 | return strdup(command); 217 | } 218 | 219 | void * u_malloc(int size){ 220 | char * buff = malloc(size); 221 | if(! buff){ 222 | FATAL("Cannot malloc();") 223 | } 224 | return buff; 225 | } 226 | 227 | void u_print_timestamp(FILE * out){ 228 | char buffer[30]; 229 | struct tm* tm_info; 230 | time_t timer; 231 | 232 | time(&timer); 233 | tm_info = localtime(&timer); 234 | strftime(buffer, 30, "%Y-%m-%d %H:%M:%S", tm_info); 235 | fprintf(out, "%s", buffer); 236 | } 237 | 238 | FILE * u_res_file_prep(char const * name){ 239 | FILE * out = stdout; 240 | if(opt.rank == 0){ 241 | char fname[PATH_MAX]; 242 | sprintf(fname, "%s/%s.txt", opt.resdir, name); 243 | INFO_PAIR("result-file", "%s\n", fname); 244 | out = fopen(fname, "w"); 245 | if( out == NULL ){ 246 | FATAL("Couldn't create results file: %s (Error: %s)\n", fname, strerror(errno)); 247 | } 248 | } 249 | return out; 250 | } 251 | 252 | void u_res_file_close(FILE * out){ 253 | if(opt.rank == 0){ 254 | fclose(out); 255 | out_logfile = stdout; 256 | } 257 | } 258 | 259 | typedef struct{ 260 | uint32_t score_hash; 261 | 262 | char const * cfg_hash_read; 263 | char const * score_hash_read; 264 | } res_file_data_t ; 265 | 266 | static res_file_data_t res_data; 267 | 268 | static void hash_func(bool is_section, char const * key, char const * val){ 269 | static char const * last_section = NULL; 270 | if(is_section){ 271 | last_section = key; 272 | return; 273 | } 274 | if(strcmp(key, "version") == 0){ 275 | printf("result file ver = %s\n", val); 276 | if(strcmp(val, VERSION) != 0){ 277 | WARNING("Verify the output with the matching version of the benchmark.\n"); 278 | } 279 | u_hash_update_key_val(& res_data.score_hash, key, val); 280 | return; 281 | } 282 | if(strcmp(key, "config-hash") == 0){ 283 | res_data.cfg_hash_read = strdup(val); 284 | return; 285 | } 286 | 287 | if(! last_section) return; 288 | 289 | if(strcmp(last_section, "SCORE") == 0){ 290 | if(strcmp(key, "SCORE") == 0){ 291 | // might be followed with the info INVALID 292 | char * strippedScore = strtok((char*)val, " "); 293 | u_hash_update_key_val(& res_data.score_hash, key, strippedScore); 294 | }else if(strcmp(key, "hash") == 0){ 295 | res_data.score_hash_read = strdup(val); 296 | }else{ 297 | u_hash_update_key_val(& res_data.score_hash, key, val); 298 | } 299 | return; 300 | } 301 | if(strcmp(key, "score") == 0){ 302 | u_hash_update_key_val(& res_data.score_hash, last_section, val); 303 | return; 304 | } 305 | DEBUG_INFO("ignored: [%s] %s %s\n", last_section, key, val); 306 | } 307 | 308 | void u_verify_result_files(ini_section_t ** cfg, char const * result){ 309 | uint32_t hash = 0; 310 | int error = 0; 311 | 312 | hash = u_ini_gen_hash(cfg); 313 | u_ini_parse_file(result, NULL, hash_func, NULL); 314 | 315 | printf("[run]\n"); 316 | printf("config-hash = %s\n", res_data.cfg_hash_read); 317 | printf("score-hash = %s\n", res_data.score_hash_read); 318 | 319 | if(res_data.cfg_hash_read == NULL){ 320 | FATAL("ERROR: Empty config hash read\n"); 321 | } 322 | 323 | char hash_str[30]; 324 | sprintf(hash_str, "%X", hash); 325 | if(strcmp(hash_str, res_data.cfg_hash_read) != 0){ 326 | printf("ERROR: Configuration hash expected: %s read: %s\n", hash_str, res_data.cfg_hash_read); 327 | error = 1; 328 | } 329 | 330 | sprintf(hash_str, "%X", res_data.score_hash); 331 | // check if this is a valid run 332 | if(strcmp(hash_str, res_data.score_hash_read) != 0){ 333 | char const * shash = strdup(hash_str); 334 | u_hash_update_key_val(& res_data.score_hash, "valid", "NO"); 335 | sprintf(hash_str, "%X", res_data.score_hash); 336 | error = 1; 337 | if(strcmp(hash_str, res_data.score_hash_read) != 0){ 338 | printf("\nERROR: Score hash expected: \"%s\" read: \"%s\"\n", shash, res_data.score_hash_read); 339 | }else{ 340 | printf("\n[OK] But this is an invalid run!\n"); 341 | } 342 | } 343 | 344 | if(error == 0){ 345 | printf("\n[OK]\n"); 346 | } 347 | 348 | exit(error); 349 | } 350 | -------------------------------------------------------------------------------- /src/verifier.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #include 10 | 11 | // Dummy prototypes to satisfy need for depending modules 12 | // TODO use ifdefs to strip dependency 13 | void pfind_find(void){} 14 | void pfind_aggregrate_results(void){} 15 | double GetTimeStamp(void){ return 0; } 16 | void pfind_parse_args(void){} 17 | IOR_test_t * ior_run(int argc, char **argv, MPI_Comm world_com, FILE * out_logfile){ return NULL; } 18 | void mdtest_run(void){} 19 | void md_workbench_run(void){} 20 | 21 | FILE* out_logfile; 22 | FILE* file_out; 23 | 24 | int main(int argc, char ** argv){ 25 | file_out = stdout; 26 | ini_section_t ** cfg = u_options(); 27 | 28 | if (argc < 3){ 29 | printf("Synopsis: %s [VerbosityLevel]\n\n", argv[0]); 30 | exit(0); 31 | } 32 | opt.verbosity = 0; 33 | u_ini_parse_file(argv[1], cfg, NULL, NULL); 34 | if(argc == 4){ 35 | opt.verbosity = atoi(argv[3]); 36 | printf("Verbosity: %d\n", opt.verbosity); 37 | } 38 | 39 | PRINT_PAIR("version", "%s\n", VERSION); 40 | 41 | u_verify_result_files(cfg, argv[2]); 42 | return 0; 43 | } 44 | -------------------------------------------------------------------------------- /test/ini-test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | FILE* file_out; 7 | FILE* out_logfile; 8 | 9 | int main(void){ 10 | file_out = stdout; 11 | 12 | int ret; 13 | { 14 | ini_section_t * testsec[] = {NULL}; 15 | ret = u_parse_ini("[general] \ntest=24", testsec, NULL); 16 | assert(ret != 0); 17 | } 18 | 19 | { 20 | ini_option_t option[] = { 21 | {"--test", "test help", 1, INI_INT, NULL}, 22 | {NULL} }; 23 | ini_section_t * testsec[] = { 24 | & (ini_section_t) { .name = "general", option}, 25 | NULL 26 | }; 27 | ret = u_parse_ini(" \t[general ] \n test \t= 24 ; ignore me\ndata=test string\ntest3 =-333", testsec, NULL); 28 | assert(ret == 1); 29 | } 30 | 31 | { 32 | ini_option_t option[] = { 33 | {"--test", "test help", 1, INI_INT, NULL}, 34 | {"test", "test help2", 1, INI_INT, "123"}, 35 | {NULL} }; 36 | ini_section_t * testsec[] = { 37 | & (ini_section_t) { .name = "general", option}, 38 | NULL 39 | }; 40 | ret = u_parse_ini(" \t[general ] \n test \t= 24 ; ignore me\ndata=test string\ntest3 =-333", testsec, NULL); 41 | assert(ret == 1); 42 | } 43 | 44 | { 45 | ini_option_t option[] = { 46 | {"--test", "test help", 1, INI_INT, NULL}, 47 | {"test", "test help2", 1, INI_INT, "123"}, 48 | {"data", "test help2", 1, INI_STRING, "test"}, 49 | {"test3", "test help2", 1, INI_UINT, "test"}, 50 | {NULL} }; 51 | ini_section_t * testsec[] = { 52 | & (ini_section_t) { .name = "general", option}, 53 | NULL 54 | }; 55 | ret = u_parse_ini(" \t[general ] \n test \t= 24 ; ignore me\ndata=test string\ntest3 =-333", testsec, NULL); 56 | assert(ret == 1); 57 | uint32_t hash = u_ini_gen_hash(testsec); 58 | u_hash_print(stdout, hash); 59 | printf("\n"); 60 | } 61 | 62 | { 63 | ini_option_t option[] = { 64 | {"--test", "test help", 1, INI_INT, NULL}, 65 | {"test", "test help2", 1, INI_INT, "123"}, 66 | {"data", "test help2", 1, INI_STRING, "test"}, 67 | {"test3", "test help2", 1, INI_INT, "test"}, 68 | {NULL} }; 69 | ini_section_t * testsec[] = { 70 | & (ini_section_t) { .name = "general", option}, 71 | NULL 72 | }; 73 | ret = u_parse_ini(" \t[general ] \n test \t= 24 ; ignore me\ndata=test string\ntest3 =-333", testsec, NULL); 74 | assert(ret == 1); 75 | uint32_t hash = u_ini_gen_hash(testsec); 76 | u_hash_print(stdout, hash); 77 | printf("\n"); 78 | } 79 | 80 | { 81 | ini_option_t option[] = { 82 | {"test", "test help2", 1, INI_INT, "123"}, 83 | {"data", "test help2", 1, INI_STRING, "test"}, 84 | {"test3", "test help2", 1, INI_INT, "test"}, 85 | {NULL} }; 86 | ini_section_t * testsec[] = { 87 | & (ini_section_t) { .name = "general", option}, 88 | NULL 89 | }; 90 | ret = u_parse_ini(" \t[general ] \n test \t= 24 ; ignore me\ndata=test string\ntest3 =-333", testsec, NULL); 91 | uint32_t hash = u_ini_gen_hash(testsec); 92 | u_hash_print(stdout, hash); 93 | printf("\n"); 94 | assert(ret == 0); 95 | } 96 | 97 | printf("\nOK\n"); 98 | 99 | return 0; 100 | } 101 | --------------------------------------------------------------------------------