├── .gitignore ├── .travis.yml ├── COPYRIGHT ├── COPYRIGHT.ANL ├── INSTALL ├── Makefile.in ├── README.md ├── buildme_bgq ├── buildme_opt ├── buildme_zlib ├── configure.ac ├── cruise-config.in ├── cruise-defs.h ├── cruise-runtime-config.h.in ├── ipc_cleanup ├── maint ├── check_fns │ ├── cstdio_list.txt │ ├── fakechroot_list.txt │ ├── gnulibc_list.txt │ ├── posix_list.txt │ ├── scrmfs_check_fns │ ├── scrmfs_list.txt │ └── scrmfs_unsupported_list.txt └── config │ ├── check_arch.m4 │ ├── check_numa.m4 │ └── inttypes-pri.m4 ├── prepare ├── src ├── cruise-fixed.c ├── cruise-fixed.h ├── cruise-internal.h ├── cruise-stack.c ├── cruise-stack.h ├── cruise-stdio.c ├── cruise-stdio.h ├── cruise-sysio.c ├── cruise-sysio.h ├── cruise.c ├── cruise.h ├── uthash.h └── utlist.h └── tests ├── Makefile ├── makefile.bgq ├── test1.c ├── test_fopen.c ├── test_fprintf.c ├── test_memcpy.c ├── test_ramdisk.c ├── test_ramdisk_iter.c ├── test_readwrite.c ├── test_scanf.c ├── test_truncate.c ├── test_ungetc.c ├── test_writeread.c └── test_wscanf.c /.gitignore: -------------------------------------------------------------------------------- 1 | /Makefile 2 | /Makefile.in 3 | /aclocal.m4 4 | /autom4te.cache/ 5 | /config.* 6 | /configure 7 | /depcomp 8 | /install-sh 9 | /libtool 10 | /ltmain.sh 11 | /m4/ 12 | /missing 13 | /stamp-h? 14 | .deps/ 15 | .dirstamp 16 | .libs/ 17 | *.l[ao] 18 | *~ 19 | cscope.out 20 | 21 | # cruise-specific 22 | cruise-runtime-config.h 23 | cruise-config 24 | 25 | # Object files 26 | *.o 27 | *.po 28 | *.ko 29 | *.obj 30 | *.elf 31 | 32 | # Precompiled Headers 33 | *.gch 34 | *.pch 35 | 36 | # Libraries 37 | *.lib 38 | *.a 39 | *.la 40 | *.lo 41 | 42 | # Shared objects 43 | *.so 44 | *.so.* 45 | *.dylib 46 | 47 | # Executables 48 | *.out 49 | *.x86_64 50 | *.hex 51 | 52 | # Debug files 53 | *.dSYM/ 54 | *.su 55 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | dist: trusty 3 | compiler: clang 4 | before_install: 5 | - sudo apt-get -qq update 6 | - sudo apt-get install -y mpich libnuma-dev 7 | script: 8 | - ./prepare 9 | - ./configure --prefix=$PWD/install 10 | - make 11 | - make install 12 | - ./install/bin/cruise-config --pre-ld-flags --post-ld-flags 13 | - cd tests 14 | - make test_memcpy test_ramdisk 15 | - mpiexec -n 4 ./test_memcpy 16 | - mpiexec -n 4 ./test_ramdisk 17 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | --------------------- 2 | Copyright and BSD License 3 | --------------------- 4 | 5 | Copyright (c) 2014, Lawrence Livermore National Security, LLC. 6 | Produced at the Lawrence Livermore National Laboratory 7 | Written by 8 | Raghunath Rajachandrasekar 9 | Kathryn Mohror 10 | Adam Moody 11 | LLNL-CODE-642432. 12 | All rights reserved. 13 | This file is part of CRUISE. 14 | For details, see https://github.com/hpc/cruise 15 | 16 | Redistribution and use in source and binary forms, with or without 17 | modification, are permitted provided that the following conditions 18 | are met: 19 | 20 | - Redistributions of source code must retain the above copyright 21 | notice, this list of conditions and the disclaimer below. 22 | 23 | - Redistributions in binary form must reproduce the above copyright 24 | notice, this list of conditions and the disclaimer (as noted 25 | below) in the documentation and/or other materials provided with 26 | the distribution. 27 | 28 | - Neither the name of the LLNS/LLNL nor the names of its 29 | contributors may be used to endorse or promote products derived 30 | from this software without specific prior written permission. 31 | 32 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 33 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 34 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 35 | FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE 36 | LIVERMORE NATIONAL SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR 37 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 38 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 39 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 40 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 41 | OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 42 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 43 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 44 | 45 | --------------------- 46 | Additional BSD Notice 47 | --------------------- 48 | 49 | 1. This notice is required to be provided under our contract with the 50 | U.S. Department of Energy (DOE). This work was produced at Lawrence 51 | Livermore National Laboratory under Contract No. DE-AC52-07NA27344 52 | with the DOE. 53 | 54 | 2. Neither the United States Government nor Lawrence Livermore 55 | National Security, LLC nor any of their employees, makes any 56 | warranty, express or implied, or assumes any liability or 57 | responsibility for the accuracy, completeness, or usefulness of 58 | any information, apparatus, product, or process disclosed, or 59 | represents that its use would not infringe privately-owned rights. 60 | 61 | 3. Also, reference herein to any specific commercial products, process, 62 | or services by trade name, trademark, manufacturer or otherwise does 63 | not necessarily constitute or imply its endorsement, recommendation, 64 | or favoring by the United States Government or Lawrence Livermore 65 | National Security, LLC. The views and opinions of authors expressed 66 | herein do not necessarily state or reflect those of the United States 67 | Government or Lawrence Livermore National Security, LLC, and shall 68 | not be used for advertising or product endorsement purposes. 69 | -------------------------------------------------------------------------------- /COPYRIGHT.ANL: -------------------------------------------------------------------------------- 1 | 2 | COPYRIGHT 3 | 4 | The following is a notice of limited availability of the code, and disclaimer 5 | which must be included in the prologue of the code and in all source listings 6 | of the code. 7 | 8 | Copyright Notice 9 | + 2009 University of Chicago 10 | 11 | Permission is hereby granted to use, reproduce, prepare derivative works, and 12 | to redistribute to others. This software was authored by: 13 | 14 | Argonne National Laboratory Group 15 | P. Carns: (630) 685-1268; e-mail: carns@mcs.anl.gov 16 | Mathematics and Computer Science Division 17 | Argonne National Laboratory, Argonne IL 60439 18 | 19 | 20 | GOVERNMENT LICENSE 21 | 22 | Portions of this material resulted from work developed under a U.S. 23 | Government Contract and are subject to the following license: the Government 24 | is granted for itself and others acting on its behalf a paid-up, nonexclusive, 25 | irrevocable worldwide license in this computer software to reproduce, prepare 26 | derivative works, and perform publicly and display publicly. 27 | 28 | DISCLAIMER 29 | 30 | This computer code material was prepared, in part, as an account of work 31 | sponsored by an agency of the United States Government. Neither the United 32 | States, nor the University of Chicago, nor any of their employees, makes any 33 | warranty express or implied, or assumes any legal liability or responsibility 34 | for the accuracy, completeness, or usefulness of any information, apparatus, 35 | product, or process disclosed, or represents that its use would not infringe 36 | privately owned rights. 37 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | 1. Prepare and run 'configure': 2 | 3 | # debug 4 | use -q ic-11.1.046 5 | ./prepare 6 | ./configure \ 7 | --prefix=`pwd`/install \ 8 | CFLAGS="-g -O0" 9 | 10 | # optimized 11 | use -q ic-11.1.046 12 | ./prepare 13 | ./configure \ 14 | --prefix=`pwd`/install \ 15 | CFLAGS="-O3" 16 | 17 | --disable-ld-preload 18 | 19 | If you do not want to generate the .so library used for LD_PRELOAD. 20 | 21 | --with-numa=$PATH_TO_NUMA 22 | 23 | Needed to enable NUMA-aware memory allocation on Linux machines 24 | (set NUMA policy at runtime with CRUISE_NUMA_POLICY = local | interleaved, 25 | set NUMA node explicitly with CRUISE_USE_NUMA_BANK = ) 26 | 27 | 28 | 29 | 2. Build library 30 | 31 | make clean 32 | make 33 | make install 34 | 35 | 3. Run "./install/bin/cruise-config --pre-ld-flags" 36 | and "./install/bin/cruise-config --post-ld-flags" 37 | 38 | Copy the output of each of these commands to a separate variable,' 39 | say $(PRE_CRUISE_FLAGS) and $(POST_CRUISE_FLAGS) 40 | 41 | 4. Compile the test program 42 | 43 | If you're compiling test_ramdisk.c for instance, you would do 44 | something like this in your makefile: 45 | 46 | setenv PRE_CRUISE_FLAGS `./install/bin/cruise-config --pre-ld-flags` 47 | setenv POST_CRUISE_FLAGS `./install/bin/cruise-config --post-ld-flags` 48 | mpicc $PRE_CRUISE_FLAGS -g -O0 -o test_ramdisk test_ramdisk.c $POST_CRUISE_FLAGS 49 | 50 | If you are running test_interpose, you can do something similar in 51 | tests/Makefile. 52 | 53 | 6. If you don't want to statically build, but just preload instead, preload this shared object: 54 | 55 | LD_PRELOAD=$INSTALL_PATH/lib/libcruise.so ./test_interpose 56 | 57 | 7. Use the "ipcs" command to list the currently opened shm segments. Use 58 | "ipcrm -M key" to delete it after a run, or use the "ipc_cleanup" script 59 | in the base folder to delete all shm segements belonging to you on a node. 60 | -------------------------------------------------------------------------------- /Makefile.in: -------------------------------------------------------------------------------- 1 | all: src/libcruise-posix.a 2 | 3 | DESTDIR = 4 | srcdir = @srcdir@ 5 | prefix = @prefix@ 6 | exec_prefix = @exec_prefix@ 7 | datarootdir = $(DESTDIR)@datarootdir@ 8 | includedir = $(DESTDIR)@includedir@ 9 | mandir = $(DESTDIR)@mandir@ 10 | sbindir = $(DESTDIR)@sbindir@ 11 | bindir = $(DESTDIR)@bindir@ 12 | libdir = $(DESTDIR)@libdir@ 13 | LDFLAGS = @LDFLAGS@ 14 | CC = @CC@ 15 | LD = @LD@ 16 | 17 | NUMA_LIBS = -lnuma 18 | 19 | 20 | DISABLE_LDPRELOAD = @DISABLE_LDPRELOAD@ 21 | 22 | ifndef DISABLE_LDPRELOAD 23 | all: src/libcruise.so 24 | endif 25 | 26 | VPATH = $(srcdir) 27 | 28 | CFLAGS = -DCRUISE_CONFIG_H=\"cruise-runtime-config.h\" -I . -I ../ -I $(srcdir) -I$(srcdir)/../ @CFLAGS@ @CPPFLAGS@ -D_LARGEFILE64_SOURCE 29 | 30 | CFLAGS_SHARED = -DCRUISE_CONFIG_H=\"cruise-runtime-config.h\" -I . -I$(srcdir) -I$(srcdir)/../ @CFLAGS@ @CPPFLAGS@ -D_LARGEFILE64_SOURCE -shared -fpic -DPIC -DCRUISE_PRELOAD 31 | 32 | LIBS = @LIBBZ2@ $(NUMA_LIBS) 33 | 34 | lib:: 35 | @mkdir -p $@ 36 | 37 | HEADERS = \ 38 | src/cruise-stack.h \ 39 | src/cruise-fixed.h \ 40 | src/cruise-sysio.h \ 41 | src/cruise-stdio.h \ 42 | src/cruise-internal.h \ 43 | src/cruise.h 44 | 45 | OBJS = \ 46 | src/cruise-stack.o \ 47 | src/cruise-fixed.o \ 48 | src/cruise-sysio.o \ 49 | src/cruise-stdio.o \ 50 | src/cruise.o 51 | 52 | POBJS = \ 53 | src/cruise-stack.po \ 54 | src/cruise-fixed.po \ 55 | src/cruise-sysio.po \ 56 | src/cruise-stdio.po \ 57 | src/cruise.po 58 | 59 | src/cruise-fixed.o: src/cruise-fixed.c $(HEADERS) 60 | $(CC) $(CFLAGS) -c $< -o $@ 61 | 62 | src/cruise-fixed.po: src/cruise-fixed.c $(HEADERS) 63 | $(CC) $(CFLAGS_SHARED) -c $< -o $@ 64 | 65 | 66 | src/cruise.o: src/cruise.c $(HEADERS) 67 | $(CC) $(CFLAGS) -c src/cruise.c -o $@ 68 | 69 | src/cruise.po: src/cruise.c $(HEADERS) 70 | $(CC) $(CFLAGS_SHARED) -c src/cruise.c -o $@ 71 | 72 | 73 | src/cruise-sysio.o: src/cruise-sysio.c $(HEADERS) 74 | $(CC) $(CFLAGS) -c src/cruise-sysio.c -o $@ 75 | 76 | src/cruise-sysio.po: src/cruise-sysio.c $(HEADERS) 77 | $(CC) $(CFLAGS_SHARED) -c src/cruise-sysio.c -o $@ 78 | 79 | 80 | src/cruise-stdio.o: src/cruise-stdio.c $(HEADERS) 81 | $(CC) $(CFLAGS) -c src/cruise-stdio.c -o $@ 82 | 83 | src/cruise-stdio.po: src/cruise-stdio.c $(HEADERS) 84 | $(CC) $(CFLAGS_SHARED) -c src/cruise-stdio.c -o $@ 85 | 86 | 87 | src/cruise-stack.o: src/cruise-stack.c $(HEADERS) 88 | $(CC) $(CFLAGS) -c $< -o $@ 89 | 90 | src/cruise-stack.po: src/cruise-stack.c $(HEADERS) 91 | $(CC) $(CFLAGS_SHARED) -c $< -o $@ 92 | 93 | 94 | %.i: %.c 95 | $(CC) -E $(CFLAGS) -c $< -o $@ 96 | 97 | src/libcruise-posix.a: $(OBJS) 98 | ar rcs $@ $^ 99 | 100 | src/libcruise.so: $(POBJS) 101 | $(CC) $(CFLAGS_SHARED) $(LDFLAGS) -ldl -o $@ $^ -lpthread -lrt 102 | 103 | install:: all 104 | install -d $(libdir) 105 | install -m 755 src/libcruise-posix.a $(libdir) 106 | ifndef DISABLE_LDPRELOAD 107 | install -m 755 src/libcruise.so $(libdir) 108 | endif 109 | install -d $(bindir) 110 | install -m 755 cruise-config $(bindir) 111 | 112 | clean:: 113 | rm -f *.o *.po *.a src/*.o src/*.po src/*.a src/*.so *.in~ 114 | 115 | distclean:: clean 116 | rm -f cruise-runtime-config.h aclocal.m4 autom4te.cache/* config.status config.log Makefile 117 | rm -rf autom4te.cache 118 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | --- 2 | # CRUISE: Checkpoint-Restart In User-SpacE 3 | [![Build Status](https://travis-ci.org/LLNL/cruise.svg?branch=master)](https://travis-ci.org/LLNL/cruise) 4 | --- 5 | 6 | With the massive scale of high-performance computing systems, long-running 7 | scientific parallel applications periodically save the state of their execution 8 | to files called checkpoints to recover from system failures. Checkpoints are 9 | stored on external parallel file systems, but limited bandwidth makes this a 10 | time-consuming operation. Multilevel checkpointing systems, like the Scalable 11 | Checkpoint/Restart (SCR) library, alleviate this bottleneck by caching 12 | checkpoints in storage located close to the compute nodes. However, most large 13 | scale systems do not provide file storage on compute nodes, preventing the use 14 | of SCR. 15 | 16 | CRUISE is a novel user-space file system that stores data in main memory and 17 | transparently spills over to other storage, like local flash memory or the 18 | parallel file system, as needed. This technique extends the reach of libraries 19 | like SCR to systems where they otherwise could not be used. CRUISE also exposes 20 | file contents for Remote Direct Memory Access, allowing external tools to copy 21 | checkpoints to the parallel file system in the background with reduced CPU 22 | interruption. 23 | 24 | More information about the project, and relevant publications, can be found 25 | [HERE](http://computation.llnl.gov/projects/scalable-checkpoint-restart-for-mpi/checkpoint-file-system-research). 26 | -------------------------------------------------------------------------------- /buildme_bgq: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | ./prepare 6 | ./configure \ 7 | --prefix=`pwd`/install_bgq \ 8 | --disable-ld-preload \ 9 | CC=mpixlc CFLAGS="-g -O0 -DMACHINE_BGQ" || exit 1 10 | # CC=mpixlc CFLAGS="-O3 -DMACHINE_BGQ" || exit 1 11 | 12 | make clean || exit 1 13 | make || exit 1 14 | make install || exit 1 15 | -------------------------------------------------------------------------------- /buildme_opt: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . /usr/local/tools/dotkit/init.sh 4 | use -q ic-11.1.046 5 | 6 | set -x 7 | 8 | ./prepare 9 | ./configure \ 10 | --prefix=`pwd`/install \ 11 | --enable-ld-preload \ 12 | CC=mpiicc CFLAGS="-O3 -DENABLE_NUMA_POLICY" || exit 1 13 | 14 | make clean || exit 1 15 | make || exit 1 16 | make install || exit 1 17 | -------------------------------------------------------------------------------- /buildme_zlib: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | if [ ! -f zlib-1.2.7.tar.gz ] ; then 6 | wget zlib.net/zlib-1.2.7.tar.gz 7 | fi 8 | 9 | rm -rf zlib-1.2.7 10 | tar -zxf zlib-1.2.7.tar.gz 11 | cd zlib-1.2.7 12 | 13 | export CC=mpixlc 14 | ./configure --prefix=`pwd`/install 15 | make CC=mpixlc 16 | make install 17 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | dnl ************************************************************** 2 | dnl CRUISE AUTOCONF SCRIPT 3 | dnl 4 | dnl Process this file with autoconf to produce a configure script. 5 | dnl You may need to use autoheader as well if changing any DEFINEs 6 | 7 | dnl sanity checks, output header, location of scripts used here 8 | AC_INIT([cruise-runtime], [2.x]) 9 | AC_CONFIG_SRCDIR([configure.ac]) 10 | AC_CONFIG_HEADER(cruise-runtime-config.h) 11 | AC_CONFIG_MACRO_DIR([./maint/config/]) 12 | 13 | AC_PROG_CC([clang]) 14 | 15 | CHECK_NUMA 16 | CHECK_ARCH 17 | 18 | AC_ARG_ENABLE(cuserid, 19 | [ --disable-cuserid Disables attempted use of cuserid() at run time], 20 | [if test "x$enableval" = "xno" ; then 21 | AC_DEFINE(CRUISE_DISABLE_CUSERID, 1, Define if cuserid() should be disabled), 22 | fi] 23 | ,) 24 | 25 | AC_ARG_ENABLE(ld-preload, 26 | [ --disable-ld-preload Disables support for LD_PRELOAD library], 27 | [if test "x$enableval" = "xno" ; then 28 | DISABLE_LDPRELOAD="1" 29 | fi] 30 | ,) 31 | 32 | AC_ARG_ENABLE(st-dev-workaround, 33 | [ --enable-st-dev-workaround Gather device id from parent directory instead of file], 34 | [if test "x$enableval" = "xyes" ; then 35 | AC_DEFINE(__CP_ST_DEV_WORKAROUND, 1, Define if device id should be taken from parent directory rather than file) 36 | DISABLE_LDPRELOAD="1" 37 | fi] 38 | ,) 39 | 40 | # checks to see how we can print 64 bit values on this architecture 41 | gt_INTTYPES_PRI 42 | 43 | dnl Check byte ordering 44 | AC_C_BIGENDIAN 45 | 46 | dnl temporarily set large file flags just for this test; we don't want 47 | dnl it to propagate to the makefile because of zlib bugs 48 | old_cflags="$CFLAGS" 49 | CFLAGS="$CFLAGS -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE" 50 | AC_MSG_CHECKING(for off64_t) 51 | AC_TRY_COMPILE( 52 | [ 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | #include 60 | #include 61 | #include 62 | #include 63 | #include 64 | #include 65 | #include 66 | #include 67 | #include 68 | ], 69 | [ off64_t off; ], 70 | AC_MSG_RESULT(yes) 71 | AC_DEFINE(HAVE_OFF64_T, 1, Define if off64_t type is defined), 72 | AC_MSG_RESULT(no) 73 | ) 74 | CFLAGS="$old_cflags" 75 | 76 | AC_CHECK_HEADERS(mntent.h sys/mount.h) 77 | 78 | # libc functions wrapped by cruise 79 | 80 | CP_WRAPPERS+="-Wl,-wrap,access" 81 | CP_WRAPPERS+=",-wrap,mkdir" 82 | CP_WRAPPERS+=",-wrap,rmdir" 83 | CP_WRAPPERS+=",-wrap,unlink" 84 | CP_WRAPPERS+=",-wrap,remove" 85 | CP_WRAPPERS+=",-wrap,rename" 86 | CP_WRAPPERS+=",-wrap,truncate" 87 | CP_WRAPPERS+=",-wrap,stat" 88 | CP_WRAPPERS+=",-wrap,__lxstat" 89 | CP_WRAPPERS+=",-wrap,__lxstat64" 90 | CP_WRAPPERS+=",-wrap,__xstat" 91 | CP_WRAPPERS+=",-wrap,__xstat64" 92 | 93 | CP_WRAPPERS+=",-wrap,creat" 94 | CP_WRAPPERS+=",-wrap,creat64" 95 | CP_WRAPPERS+=",-wrap,open" 96 | CP_WRAPPERS+=",-wrap,open64" 97 | CP_WRAPPERS+=",-wrap,read" 98 | CP_WRAPPERS+=",-wrap,write" 99 | CP_WRAPPERS+=",-wrap,readv" 100 | CP_WRAPPERS+=",-wrap,writev" 101 | CP_WRAPPERS+=",-wrap,pread" 102 | CP_WRAPPERS+=",-wrap,pread64" 103 | CP_WRAPPERS+=",-wrap,pwrite" 104 | CP_WRAPPERS+=",-wrap,pwrite64" 105 | CP_WRAPPERS+=",-wrap,posix_fadvise" 106 | CP_WRAPPERS+=",-wrap,lseek" 107 | CP_WRAPPERS+=",-wrap,lseek64" 108 | CP_WRAPPERS+=",-wrap,ftruncate" 109 | CP_WRAPPERS+=",-wrap,fsync" 110 | CP_WRAPPERS+=",-wrap,fdatasync" 111 | CP_WRAPPERS+=",-wrap,flock" 112 | CP_WRAPPERS+=",-wrap,mmap" 113 | CP_WRAPPERS+=",-wrap,mmap64" 114 | CP_WRAPPERS+=",-wrap,munmap" 115 | CP_WRAPPERS+=",-wrap,msync" 116 | CP_WRAPPERS+=",-wrap,__fxstat" 117 | CP_WRAPPERS+=",-wrap,__fxstat64" 118 | CP_WRAPPERS+=",-wrap,close" 119 | 120 | # FILE* functions 121 | #CP_WRAPPERS+=",-wrap,fclose" 122 | #CP_WRAPPERS+=",-wrap,fflush" 123 | #CP_WRAPPERS+=",-wrap,fopen" 124 | #CP_WRAPPERS+=",-wrap,freopen" 125 | #CP_WRAPPERS+=",-wrap,setbuf" 126 | #CP_WRAPPERS+=",-wrap,setvbuf" 127 | # 128 | #CP_WRAPPERS+=",-wrap,fprintf" 129 | #CP_WRAPPERS+=",-wrap,fscanf" 130 | #CP_WRAPPERS+=",-wrap,vfprintf" 131 | #CP_WRAPPERS+=",-wrap,vfscanf" 132 | # 133 | #CP_WRAPPERS+=",-wrap,fgetc" 134 | #CP_WRAPPERS+=",-wrap,fgets" 135 | #CP_WRAPPERS+=",-wrap,fputc" 136 | #CP_WRAPPERS+=",-wrap,fputs" 137 | #CP_WRAPPERS+=",-wrap,getc" 138 | #CP_WRAPPERS+=",-wrap,putc" 139 | #CP_WRAPPERS+=",-wrap,ungetc" 140 | # 141 | #CP_WRAPPERS+=",-wrap,fread" 142 | #CP_WRAPPERS+=",-wrap,fwrite" 143 | # 144 | #CP_WRAPPERS+=",-wrap,fgetpos" 145 | #CP_WRAPPERS+=",-wrap,fseek" 146 | #CP_WRAPPERS+=",-wrap,fsetpos" 147 | #CP_WRAPPERS+=",-wrap,ftell" 148 | #CP_WRAPPERS+=",-wrap,rewind" 149 | # 150 | #CP_WRAPPERS+=",-wrap,clearerr" 151 | #CP_WRAPPERS+=",-wrap,feof" 152 | #CP_WRAPPERS+=",-wrap,ferror" 153 | # 154 | #CP_WRAPPERS+=",-wrap,fseeko" 155 | #CP_WRAPPERS+=",-wrap,ftello" 156 | #CP_WRAPPERS+=",-wrap,fileno" 157 | 158 | # wide character FILE* functions 159 | #CP_WRAPPERS+=",-wrap,fwprintf" 160 | #CP_WRAPPERS+=",-wrap,fwscanf" 161 | #CP_WRAPPERS+=",-wrap,vfwprintf" 162 | #CP_WRAPPERS+=",-wrap,vfwscanf" 163 | #CP_WRAPPERS+=",-wrap,fgetwc" 164 | #CP_WRAPPERS+=",-wrap,fgetws" 165 | #CP_WRAPPERS+=",-wrap,fputwc" 166 | #CP_WRAPPERS+=",-wrap,fputws" 167 | #CP_WRAPPERS+=",-wrap,fwide" 168 | #CP_WRAPPERS+=",-wrap,getwc" 169 | #CP_WRAPPERS+=",-wrap,putwc" 170 | #CP_WRAPPERS+=",-wrap,ungetwc" 171 | 172 | # ,-u,__wrap___fxstat64,-u,pthread_mutex_lock,-u,pthread_mutex_unlock 173 | 174 | # We need to know the value of the $libdir and $bindir variables so that 175 | # we can reference the correct path in the cruise compiler wrappers. 176 | # Unfortunately, those two variables are not normally evaluated by autoconf. 177 | # They are evaluated at build time using Makefile variable substitutions. 178 | # 179 | # The following logic was copied from mpich2 1.3.1 to resolve the $libdir 180 | # variable at configure time. 181 | # 182 | # Temporarily replace the default NONE value for exec_prefix 183 | # and prefix with the actual, default values. 184 | savePrefix=$prefix 185 | saveExecprefix=$exec_prefix 186 | test "x$prefix" = xNONE && prefix=$ac_default_prefix 187 | test "x$exec_prefix" = xNONE && exec_prefix=$prefix 188 | eval cruise_lib_path=$libdir 189 | eval cruise_bin_path=$bindir 190 | prefix=$savePrefix 191 | exec_prefix=$saveExecprefix 192 | 193 | CFLAGS="$old_cflags" 194 | LIBS="$old_libs" 195 | 196 | AC_SUBST(cruise_lib_path) 197 | AC_SUBST(cruise_bin_path) 198 | AC_SUBST(LDFLAGS) 199 | AC_SUBST(__CP_LOG_PATH) 200 | AC_SUBST(CP_WRAPPERS) 201 | AC_SUBST(DISABLE_LDPRELOAD) 202 | AC_OUTPUT(Makefile 203 | cruise-config 204 | ) 205 | 206 | 207 | AC_MSG_RESULT([ 208 | 209 | ========================= 210 | CRUISE Checkpoint-Restart 211 | ========================= 212 | prefix ${prefix} 213 | compiler ${CC} 214 | CFLAGS ${CFLAGS} 215 | ========================= 216 | 217 | Supported POSIX wrappers: 218 | 219 | ${CP_WRAPPERS} 220 | 221 | ]) 222 | -------------------------------------------------------------------------------- /cruise-config.in: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # (C) 2012 by Argonne National Laboratory. 5 | # See COPYRIGHT in top-level directory. 6 | # 7 | 8 | 9 | 10 | CP_WRAPPERS="@CP_WRAPPERS@" 11 | CRUISE_LIB_PATH="@cruise_lib_path@" 12 | CRUISE_LD_FLAGS="@LDFLAGS@" 13 | 14 | PRE_LD_FLAGS="-L$CRUISE_LIB_PATH $CRUISE_LD_FLAGS -lz $CP_WRAPPERS" 15 | POST_LD_FLAGS="-L$CRUISE_LIB_PATH -lcruise-posix" 16 | 17 | 18 | usage="\ 19 | Usage: cruise-config [--pre-ld-flags] [--post-ld-flags]" 20 | 21 | if test $# -eq 0; then 22 | echo "${usage}" 1>&2 23 | exit 1 24 | fi 25 | 26 | while test $# -gt 0; do 27 | case "$1" in 28 | -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;; 29 | *) optarg= ;; 30 | esac 31 | 32 | case $1 in 33 | --pre-ld-flags) 34 | echo $PRE_LD_FLAGS 35 | ;; 36 | --post-ld-flags) 37 | echo $POST_LD_FLAGS 38 | ;; 39 | *) 40 | echo "${usage}" 1>&2 41 | exit 1 42 | ;; 43 | esac 44 | shift 45 | done 46 | -------------------------------------------------------------------------------- /cruise-defs.h: -------------------------------------------------------------------------------- 1 | #define CRUISE_MAX_FILES ( 128 ) 2 | 3 | /* eventually could decouple these so there could be 4 | * more or less file descriptors than files, but for 5 | * now they're the same */ 6 | #define CRUISE_MAX_FILEDESCS ( CRUISE_MAX_FILES ) 7 | 8 | #define CRUISE_MAX_FILENAME ( 128 ) 9 | 10 | #define CRUISE_STREAM_BUFSIZE ( 1 * 1024 * 1024 ) 11 | 12 | #define CRUISE_CHUNK_BITS ( 24 ) 13 | 14 | #ifdef MACHINE_BGQ 15 | #define CRUISE_CHUNK_MEM ( 64 * 1024 * 1024 ) 16 | #else /* MACHINE_BGQ */ 17 | #define CRUISE_CHUNK_MEM ( 256 * 1024 * 1024 ) 18 | #endif /* MACHINE_BGQ */ 19 | 20 | #define CRUISE_SPILLOVER_SIZE ( 1 * 1024 * 1024 * 1024 ) 21 | 22 | #define CRUISE_SUPERBLOCK_KEY ( 4321 ) 23 | -------------------------------------------------------------------------------- /cruise-runtime-config.h.in: -------------------------------------------------------------------------------- 1 | /* cruise-runtime-config.h.in. Generated from configure.ac by autoheader. */ 2 | 3 | /* Define if building universal (internal helper macro) */ 4 | #undef AC_APPLE_UNIVERSAL_BUILD 5 | 6 | /* Define if cuserid() should be disabled */ 7 | #undef CRUISE_DISABLE_CUSERID 8 | 9 | /* Define if libnuma is available */ 10 | #undef ENABLE_NUMA_POLICY 11 | 12 | /* Define to 1 if you have the header file. */ 13 | #undef HAVE_INTTYPES_H 14 | 15 | /* Define to 1 if you have the `numa' library (-lnuma). */ 16 | #undef HAVE_LIBNUMA 17 | 18 | /* Define to 1 if you have the header file. */ 19 | #undef HAVE_MEMORY_H 20 | 21 | /* Define to 1 if you have the header file. */ 22 | #undef HAVE_MNTENT_H 23 | 24 | /* Define if off64_t type is defined */ 25 | #undef HAVE_OFF64_T 26 | 27 | /* Define to 1 if you have the header file. */ 28 | #undef HAVE_STDINT_H 29 | 30 | /* Define to 1 if you have the header file. */ 31 | #undef HAVE_STDLIB_H 32 | 33 | /* Define to 1 if you have the header file. */ 34 | #undef HAVE_STRINGS_H 35 | 36 | /* Define to 1 if you have the header file. */ 37 | #undef HAVE_STRING_H 38 | 39 | /* Define to 1 if you have the header file. */ 40 | #undef HAVE_SYS_MOUNT_H 41 | 42 | /* Define to 1 if you have the header file. */ 43 | #undef HAVE_SYS_STAT_H 44 | 45 | /* Define to 1 if you have the header file. */ 46 | #undef HAVE_SYS_TYPES_H 47 | 48 | /* Define to 1 if you have the header file. */ 49 | #undef HAVE_UNISTD_H 50 | 51 | /* Define if architecture is BG/Q */ 52 | #undef MACHINE_BGQ 53 | 54 | /* Define to the address where bug reports for this package should be sent. */ 55 | #undef PACKAGE_BUGREPORT 56 | 57 | /* Define to the full name of this package. */ 58 | #undef PACKAGE_NAME 59 | 60 | /* Define to the full name and version of this package. */ 61 | #undef PACKAGE_STRING 62 | 63 | /* Define to the one symbol short name of this package. */ 64 | #undef PACKAGE_TARNAME 65 | 66 | /* Define to the home page for this package. */ 67 | #undef PACKAGE_URL 68 | 69 | /* Define to the version of this package. */ 70 | #undef PACKAGE_VERSION 71 | 72 | /* Define if exists and defines unusable PRI* macros. */ 73 | #undef PRI_MACROS_BROKEN 74 | 75 | /* Define to 1 if you have the ANSI C header files. */ 76 | #undef STDC_HEADERS 77 | 78 | /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most 79 | significant byte first (like Motorola and SPARC, unlike Intel). */ 80 | #if defined AC_APPLE_UNIVERSAL_BUILD 81 | # if defined __BIG_ENDIAN__ 82 | # define WORDS_BIGENDIAN 1 83 | # endif 84 | #else 85 | # ifndef WORDS_BIGENDIAN 86 | # undef WORDS_BIGENDIAN 87 | # endif 88 | #endif 89 | 90 | /* Define if device id should be taken from parent directory rather than file 91 | */ 92 | #undef __CP_ST_DEV_WORKAROUND 93 | -------------------------------------------------------------------------------- /ipc_cleanup: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ########################################################################### 4 | # $URL: file:///var/svn/cfengine/llnl/admin/scripts/ipc_cleanup $ 5 | # $Author: tdhooge $ 6 | # $Date: 2011-06-27 11:06:30 -0700 (Mon, 27 Jun 2011) $ 7 | # $Rev: 295 $ 8 | ########################################################################### 9 | 10 | # Clean up shared memory after job 11 | 12 | NAME=$USER #`awk -F: -v TMP=$SLURM_UID '$3 == TMP { print $1 }' /etc/passwd` 13 | 14 | for key in $(ipcs -mc | grep -v '^(---|key)' | awk -v USER=${NAME} '$3 == USER {print $1}'); do ipcrm -m $key; done 15 | -------------------------------------------------------------------------------- /maint/check_fns/cstdio_list.txt: -------------------------------------------------------------------------------- 1 | # http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1124.pdf 2 | 3 | # 7.19.4 Operations on file 4 | 1 remove 5 | 2 rename 6 | 3 tmpfile 7 | 4 tmpnam 8 | 9 | # 7.19.5 file access functions 10 | 1 fclose 11 | 2 fflush 12 | 3 fopen 13 | 4 freopen 14 | 5 setbuf 15 | 6 setvbuf 16 | 17 | # 7.19.6 formatted input/output functions 18 | 1 fprintf 19 | 2 fscanf 20 | 3 printf 21 | 4 scanf 22 | 5 snprintf 23 | 6 sprintf 24 | 7 sscanf 25 | 8 vfprintf 26 | 9 vfscanf 27 | 10 vprintf 28 | 11 vscanf 29 | 12 vsnprintf 30 | 13 vsprintf 31 | 14 vsscanf 32 | 33 | # 7.19.7 character input/output functions 34 | 1 fgetc 35 | 2 fgets 36 | 3 fputc 37 | 4 fputs 38 | 5 getc 39 | 6 getchar 40 | 7 gets 41 | 8 putc 42 | 9 putchar 43 | 10 puts 44 | 11 ungetc 45 | 46 | # 7.19.8 direct input/output functions 47 | 1 fread 48 | 2 fwrite 49 | 50 | # 7.19.9 File positioning functions 51 | 1 fgetpos 52 | 2 fseek 53 | 3 fsetpos 54 | 4 ftell 55 | 5 rewind 56 | 57 | # 7.19.10 Error-handling functions 58 | 1 clearerr 59 | 2 feof 60 | 3 ferror 61 | 4 perror 62 | 63 | # 7.24.2 formatted wide characters 64 | 1 fwprintf 65 | 2 fwscanf 66 | 3 swprintf 67 | 4 swscanf 68 | 5 vfwprintf 69 | 6 vfwscanf 70 | 7 vswprintf 71 | 8 vswscanf 72 | 9 vwprintf 73 | 10 vwscanf 74 | 11 wprintf 75 | 12 wscanf 76 | 77 | # 7.24.3 wide character input/output 78 | 1 fgetwc 79 | 2 fgetws 80 | 3 fputwc 81 | 4 fputws 82 | 5 fwide 83 | 6 getwc 84 | 7 getwchar 85 | 8 putwc 86 | 9 putwchar 87 | 10 ungetwc 88 | -------------------------------------------------------------------------------- /maint/check_fns/fakechroot_list.txt: -------------------------------------------------------------------------------- 1 | __fxstatat.c 2 | __fxstatat64.c 3 | __getcwd_chk.c 4 | __getwd_chk.c 5 | __lxstat.c 6 | __lxstat64.c 7 | __open.c 8 | __open64.c 9 | __open64_2.c 10 | __open_2.c 11 | __openat64_2.c 12 | __openat_2.c 13 | __opendir2.c 14 | __readlink_chk.c 15 | __readlinkat_chk.c 16 | __realpath_chk.c 17 | __statfs.c 18 | __xmknod.c 19 | __xmknodat.c 20 | __xstat.c 21 | __xstat64.c 22 | _xftw.c 23 | _xftw64.c 24 | access.c 25 | acct.c 26 | bind.c 27 | bindtextdomain.c 28 | canonicalize_file_name.c 29 | chdir.c 30 | chmod.c 31 | chown.c 32 | chroot.c 33 | connect.c 34 | creat.c 35 | creat64.c 36 | dlmopen.c 37 | dlopen.c 38 | eaccess.c 39 | euidaccess.c 40 | execl.c 41 | execle.c 42 | execlp.c 43 | execv.c 44 | execve.c 45 | execvp.c 46 | faccessat.c 47 | fchmodat.c 48 | fchownat.c 49 | fopen.c 50 | fopen64.c 51 | freopen.c 52 | freopen64.c 53 | fts.c 54 | ftw.c 55 | ftw64.c 56 | futimesat.c 57 | get_current_dir_name.c 58 | getcwd.c 59 | getpeername.c 60 | getsockname.c 61 | getwd.c 62 | getxattr.c 63 | glob.c 64 | glob64.c 65 | glob_pattern_p.c 66 | inotify_add_watch.c 67 | lchmod.c 68 | lchown.c 69 | lckpwdf.c 70 | lgetxattr.c 71 | libfakechroot.c 72 | link.c 73 | linkat.c 74 | listxattr.c 75 | llistxattr.c 76 | lremovexattr.c 77 | lsetxattr.c 78 | lstat.c 79 | lstat64.c 80 | lutimes.c 81 | mkdir.c 82 | mkdirat.c 83 | mkdtemp.c 84 | mkfifo.c 85 | mkfifoat.c 86 | mknod.c 87 | mknodat.c 88 | mkstemp.c 89 | mkstemp64.c 90 | mktemp.c 91 | open.c 92 | open64.c 93 | openat.c 94 | openat64.c 95 | opendir.c 96 | pathconf.c 97 | popen.c 98 | rawmemchr.c 99 | readlink.c 100 | readlinkat.c 101 | realpath.c 102 | remove.c 103 | removexattr.c 104 | rename.c 105 | renameat.c 106 | revoke.c 107 | rmdir.c 108 | rpl_lstat.c 109 | scandir.c 110 | scandir64.c 111 | setenv.c 112 | setxattr.c 113 | stat.c 114 | stat64.c 115 | statfs.c 116 | statfs64.c 117 | statvfs.c 118 | statvfs64.c 119 | stpcpy.c 120 | strchrnul.c 121 | symlink.c 122 | symlinkat.c 123 | system.c 124 | tempnam.c 125 | tmpnam.c 126 | truncate.c 127 | truncate64.c 128 | ulckpwdf.c 129 | unlink.c 130 | unlinkat.c 131 | unsetenv.c 132 | utime.c 133 | utimensat.c 134 | utimes.c 135 | -------------------------------------------------------------------------------- /maint/check_fns/gnulibc_list.txt: -------------------------------------------------------------------------------- 1 | # http://www.gnu.org/software/libc/manual/html_node/I_002fO-Overview.html#I_002fO-Overview 2 | 3 | fopen 4 | fopen64 5 | freopen 6 | freopen64 7 | __freadable 8 | __fwritable 9 | __freading 10 | __fwriting 11 | 12 | fclose 13 | fcloseall 14 | 15 | flockfile 16 | ftrylockfile 17 | funlockfile 18 | fsetlocking 19 | 20 | fwide 21 | 22 | fputc 23 | fputwc 24 | fputc_unlocked 25 | fputwc_unlocked 26 | putc 27 | putwc 28 | putc_unlocked 29 | putwc_unlocked 30 | fputs 31 | fputws 32 | fputs_unlocked 33 | fputws_unlocked 34 | putw 35 | 36 | fgetc 37 | fgetwc 38 | fgetc_unlocked 39 | fgetwc_unlocked 40 | getc 41 | getwc 42 | getc_unlocked 43 | getwc_unlocked 44 | getw 45 | 46 | getline 47 | getdelim 48 | fgets 49 | fgetws 50 | fgets_unlocked 51 | fgetws_unlocked 52 | 53 | ungetc 54 | ungetwc 55 | 56 | fread 57 | fread_unlocked 58 | fwrite 59 | fwrite_unlocked 60 | 61 | fprintf 62 | fwprintf 63 | 64 | vfprintf 65 | vfwprintf 66 | 67 | fscanf 68 | fwscanf 69 | vfscanf 70 | vfwscanf 71 | 72 | feof 73 | feof_unlocked 74 | ferror 75 | ferror_unlocked 76 | 77 | clearerr 78 | clearerr_unlocked 79 | 80 | ftell 81 | ftello 82 | ftello64 83 | fseek 84 | fseeko 85 | fseeko64 86 | rewind 87 | 88 | fgetpos 89 | fgetpos64 90 | 91 | fflush 92 | fflush_unlocked 93 | _flushlbf 94 | __fpurge 95 | 96 | setvbuf 97 | setbuf 98 | setbuffer 99 | setlinebuf 100 | __flbf 101 | __fbufsize 102 | __fpending 103 | 104 | fmemopen 105 | open_memstream 106 | 107 | fopencookie 108 | 109 | 110 | # http://www.gnu.org/software/libc/manual/html_node/Opening-and-Closing-Files.html#Opening-and-Closing-Files 111 | open 112 | open64 113 | creat 114 | creat64 115 | close 116 | 117 | # http://www.gnu.org/software/libc/manual/html_node/I_002fO-Primitives.html#I_002fO-Primitives 118 | read 119 | pread 120 | pread64 121 | write 122 | pwrite 123 | pwrite64 124 | 125 | # http://www.gnu.org/software/libc/manual/html_node/File-Position-Primitive.html#File-Position-Primitive 126 | lseek 127 | lseek64 128 | 129 | # http://www.gnu.org/software/libc/manual/html_node/Descriptors-and-Streams.html#Descriptors-and-Streams 130 | fdopen 131 | fileno 132 | fileno_unlocked 133 | 134 | # http://www.gnu.org/software/libc/manual/html_node/Scatter_002dGather.html#Scatter_002dGather 135 | readv 136 | writev 137 | 138 | # http://www.gnu.org/software/libc/manual/html_node/Memory_002dmapped-I_002fO.html#Memory_002dmapped-I_002fO 139 | mmap 140 | mmap64 141 | munmap 142 | msync 143 | mremap 144 | 145 | # http://www.gnu.org/software/libc/manual/html_node/Waiting-for-I_002fO.html#Waiting-for-I_002fO 146 | select 147 | 148 | # http://www.gnu.org/software/libc/manual/html_node/Synchronizing-I_002fO.html#Synchronizing-I_002fO 149 | sync 150 | fsync 151 | fdatasync 152 | 153 | # http://www.gnu.org/software/libc/manual/html_node/Asynchronous-I_002fO.html#Asynchronous-I_002fO 154 | aio_read 155 | aio_read64 156 | aio_write 157 | aio_write64 158 | lio_listio 159 | lio_listio64 160 | 161 | # http://www.gnu.org/software/libc/manual/html_node/Status-of-AIO-Operations.html#Status-of-AIO-Operations 162 | aio_error 163 | aio_error64 164 | aio_return 165 | aio_return64 166 | 167 | # http://www.gnu.org/software/libc/manual/html_node/Synchronizing-AIO-Operations.html#Synchronizing-AIO-Operations 168 | aio_fsync 169 | aio_fsync64 170 | aio_suspend 171 | aio_suspend64 172 | 173 | # http://www.gnu.org/software/libc/manual/html_node/Cancel-AIO-Operations.html#Cancel-AIO-Operations 174 | aio_cancel 175 | aio_cancel64 176 | 177 | # http://www.gnu.org/software/libc/manual/html_node/Configuration-of-AIO.html#Configuration-of-AIO 178 | aio_init 179 | 180 | # http://www.gnu.org/software/libc/manual/html_node/Control-Operations.html#Control-Operations 181 | fcntl 182 | 183 | # http://www.gnu.org/software/libc/manual/html_node/Duplicating-Descriptors.html#Duplicating-Descriptors 184 | dup 185 | dup2 186 | 187 | # http://www.gnu.org/software/libc/manual/html_node/IOCTLs.html#IOCTLs 188 | ioctl 189 | 190 | # http://www.gnu.org/software/libc/manual/html_node/Working-Directory.html#Working-Directory 191 | getcwd 192 | getwd 193 | get_current_dir_name 194 | chdir 195 | fchdir 196 | 197 | # http://www.gnu.org/software/libc/manual/html_node/Opening-a-Directory.html#Opening-a-Directory 198 | opendir 199 | fdopendir 200 | dirfd 201 | 202 | # http://www.gnu.org/software/libc/manual/html_node/Reading_002fClosing-Directory.html#Reading_002fClosing-Directory 203 | readdir 204 | readdir_r 205 | readdir64 206 | readdir64_r 207 | closedir 208 | 209 | # http://www.gnu.org/software/libc/manual/html_node/Random-Access-Directory.html#Random-Access-Directory 210 | rewinddir 211 | telldir 212 | seekdir 213 | 214 | # http://www.gnu.org/software/libc/manual/html_node/Scanning-Directory-Content.html#Scanning-Directory-Content 215 | scandir 216 | scandir64 217 | 218 | # http://www.gnu.org/software/libc/manual/html_node/Working-with-Directory-Trees.html#Working-with-Directory-Trees 219 | ftw 220 | ftw64 221 | nftw 222 | nftw64 223 | 224 | # http://www.gnu.org/software/libc/manual/html_node/Hard-Links.html#Hard-Links 225 | link 226 | 227 | # http://www.gnu.org/software/libc/manual/html_node/Symbolic-Links.html#Symbolic-Links 228 | symlink 229 | readlink 230 | canonicalize_file_name 231 | realpath 232 | 233 | # http://www.gnu.org/software/libc/manual/html_node/Deleting-Files.html#Deleting-Files 234 | unlink 235 | rmdir 236 | remove 237 | 238 | # http://www.gnu.org/software/libc/manual/html_node/Renaming-Files.html#Renaming-Files 239 | rename 240 | 241 | # http://www.gnu.org/software/libc/manual/html_node/Creating-Directories.html#Creating-Directories 242 | mkdir 243 | 244 | # http://www.gnu.org/software/libc/manual/html_node/Reading-Attributes.html#Reading-Attributes 245 | stat 246 | stat64 247 | fstat 248 | fstat64 249 | lstat 250 | lstat64 251 | 252 | # http://www.gnu.org/software/libc/manual/html_node/File-Owner.html#File-Owner 253 | chown 254 | fchown 255 | 256 | # http://www.gnu.org/software/libc/manual/html_node/Setting-Permissions.html#Setting-Permissions 257 | umask 258 | getumask 259 | chmod 260 | fchmod 261 | 262 | # http://www.gnu.org/software/libc/manual/html_node/Testing-File-Access.html#Testing-File-Access 263 | access 264 | 265 | # http://www.gnu.org/software/libc/manual/html_node/File-Times.html#File-Times 266 | utime 267 | utimes 268 | lutimes 269 | futimes 270 | 271 | # http://www.gnu.org/software/libc/manual/html_node/File-Size.html#File-Size 272 | truncate 273 | truncate64 274 | ftruncate 275 | ftruncate64 276 | 277 | # http://www.gnu.org/software/libc/manual/html_node/Making-Special-Files.html#Making-Special-Files 278 | mknod 279 | 280 | # http://www.gnu.org/software/libc/manual/html_node/Temporary-Files.html#Temporary-Files 281 | tmpfile 282 | tmpfile64 283 | tmpnam 284 | tmpnam_r 285 | tempnam 286 | mktemp 287 | mkstemp 288 | mkdtemp 289 | -------------------------------------------------------------------------------- /maint/check_fns/posix_list.txt: -------------------------------------------------------------------------------- 1 | readlink 2 | readlinkat 3 | symlink 4 | symlinkat 5 | 6 | chown 7 | fchown 8 | lchown 9 | fchownat 10 | 11 | chmod 12 | fchmod 13 | fchmodat 14 | 15 | utime 16 | utimes 17 | 18 | fcntl 19 | fstatvs 20 | 21 | mknod 22 | mknodat 23 | 24 | chdir 25 | getcwd 26 | getwd 27 | get_current_dir_name 28 | 29 | opendir 30 | rewinddir 31 | readdir 32 | readdir_r 33 | closedir 34 | 35 | ftw 36 | nftw 37 | -------------------------------------------------------------------------------- /maint/check_fns/scrmfs_check_fns: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | use strict; 3 | 4 | # commands to build fakechroot list 5 | # git clone https://github.com/fakechroot/fakechroot.git fakechroot.git 6 | # cd fakechroot.git/src 7 | # ls *.c > fakechroot_list.txt 8 | # 9 | # hand-wrote gnulibc list from 10 | # http://www.gnu.org/software/libc/manual/html_node/I_002fO-Overview.html#I_002fO-Overview 11 | # 12 | # hand-wrote cstdio list from 13 | # http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1124.pdf 14 | # 15 | # hand-wrote cruise unsupported list (wrapped calls that aren't supported) 16 | # 17 | # commands to build cruise list 18 | # grep CRUISE_WRAP ../runtime/lib/*.c > cruise_list.txt 19 | 20 | 21 | my $fakechroot_list = "fakechroot_list.txt"; 22 | my $gnulibc_list = "gnulibc_list.txt"; 23 | my $cstdio_list = "cstdio_list.txt"; 24 | my $posix_list = "posix_list.txt"; 25 | my $unsupported_list = "cruise_unsupported_list.txt"; 26 | my $cruise_list = "cruise_list.txt"; 27 | 28 | if (@ARGV != 1) { 29 | print "Usage: cruise_check_fns \n"; 30 | exit 1; 31 | } 32 | my $exe = shift @ARGV; 33 | 34 | my %any_libc = (); 35 | my %unsupported = (); 36 | my %cruise = (); 37 | 38 | open(IN, $fakechroot_list) || die "failed to open $fakechroot_list\n"; 39 | while (my $line = ) { 40 | # drop trailing newline 41 | chomp $line; 42 | 43 | # chop white space 44 | $line =~ s/\s*//g; 45 | 46 | # chop trailing .c 47 | $line =~ s/\.c$//; 48 | 49 | $any_libc{$line} = 1; 50 | } 51 | close(IN); 52 | 53 | open(IN, $gnulibc_list) || die "failed to open $gnulibc_list\n"; 54 | while (my $line = ) { 55 | # drop trailing newline 56 | chomp $line; 57 | 58 | # chop white space 59 | $line =~ s/\s*//g; 60 | 61 | # skip if starts with # or is empty 62 | if ($line =~ /^#/ or not $line) { 63 | next; 64 | } 65 | 66 | $any_libc{$line} = 1; 67 | } 68 | close(IN); 69 | 70 | open(IN, $cstdio_list) || die "failed to open $cstdio_list\n"; 71 | while (my $line = ) { 72 | # drop trailing newline 73 | chomp $line; 74 | 75 | # skip if starts with # or is empty 76 | if ($line =~ /^#/ or not $line) { 77 | next; 78 | } 79 | 80 | # (space) leading integer (space) function name 81 | my $fn = undef; 82 | if ($line =~ /^\s*\d+\s*(.*?)\s*$/) { 83 | $fn = $1; 84 | } else { 85 | next; 86 | } 87 | 88 | # chop white space 89 | $fn =~ s/\s*//g; 90 | #print "cstdio: $fn\n"; 91 | 92 | $any_libc{$fn} = 1; 93 | } 94 | close(IN); 95 | 96 | open(IN, $unsupported_list) || die "failed to open $unsupported_list\n"; 97 | while (my $line = ) { 98 | # drop trailing newline 99 | chomp $line; 100 | 101 | # chop white space 102 | $line =~ s/\s*//g; 103 | 104 | # skip if starts with # or is empty 105 | if ($line =~ /^#/ or not $line) { 106 | next; 107 | } 108 | #print "unsupported: $line\n"; 109 | 110 | $unsupported{$line} = 1; 111 | } 112 | close(IN); 113 | 114 | open(IN, $cruise_list) || die "failed to open $cruise_list\n"; 115 | while (my $line = ) { 116 | # drop trailing newline 117 | chomp $line; 118 | 119 | # pick off item in parens in CRUISE_DELC(fn_name) 120 | my $fn = undef; 121 | if ($line =~ /CRUISE_WRAP\((.*?)\)/) { 122 | $fn = $1; 123 | } else { 124 | next; 125 | } 126 | 127 | # chop white space 128 | $fn =~ s/\s*//g; 129 | #print "wrapped: $fn\n"; 130 | 131 | $cruise{$fn} = 1; 132 | } 133 | close(IN); 134 | 135 | # get list of functions called by executable 136 | my $fns_str = `nm $exe`; 137 | my @fns_list = split("\n", $fns_str); 138 | foreach my $line (@fns_list) { 139 | chomp $line; 140 | # print "$line, "; 141 | 142 | my ($hex, $code, $fn) = split(/\s+/, $line); 143 | # print "$fn, "; 144 | 145 | $fn =~ s/@.*$//; 146 | # print "$fn\n"; 147 | 148 | my $used = 0; 149 | my $supported = 0; 150 | if (defined $any_libc{$fn}) { 151 | $used = 1; 152 | if (defined $cruise{$fn} and not $unsupported{$fn}) { 153 | $supported = 1; 154 | } 155 | } 156 | 157 | if ($used) { 158 | print "$fn"; 159 | if (not $supported) { 160 | print ", not supported"; 161 | } 162 | print "\n"; 163 | } 164 | } 165 | 166 | exit 0; 167 | -------------------------------------------------------------------------------- /maint/check_fns/scrmfs_list.txt: -------------------------------------------------------------------------------- 1 | ../runtime/lib/cruise-stdio.c:FILE* CRUISE_DECL(fopen)(const char *path, const char *mode) 2 | ../runtime/lib/cruise-stdio.c:FILE* CRUISE_DECL(freopen)(const char *path, const char *mode, FILE *stream) 3 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(setvbuf)(FILE* stream, char* buf, int type, size_t size) 4 | ../runtime/lib/cruise-stdio.c:void CRUISE_DECL(setbuf)(FILE* stream, char* buf) 5 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(ungetc)(int c, FILE *stream) 6 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(fgetc)(FILE *stream) 7 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(fputc)(int c, FILE *stream) 8 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(getc)(FILE *stream) 9 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(putc)(int c, FILE *stream) 10 | ../runtime/lib/cruise-stdio.c:char* CRUISE_DECL(fgets)(char* s, int n, FILE* stream) 11 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(fputs)(const char* s, FILE* stream) 12 | ../runtime/lib/cruise-stdio.c:size_t CRUISE_DECL(fread)(void *ptr, size_t size, size_t nitems, FILE *stream) 13 | ../runtime/lib/cruise-stdio.c:size_t CRUISE_DECL(fwrite)(const void *ptr, size_t size, size_t nitems, FILE *stream) 14 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(fprintf)(FILE *stream, const char* format, ...) 15 | ../runtime/lib/cruise-stdio.c: int ret = CRUISE_DECL(vfprintf)(stream, format, args); 16 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(vfprintf)(FILE *stream, const char* format, va_list ap) 17 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(fscanf)(FILE *stream, const char* format, ...) 18 | ../runtime/lib/cruise-stdio.c: int ret = CRUISE_DECL(vfscanf)(stream, format, args); 19 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(vfscanf)(FILE *stream, const char* format, va_list ap) 20 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(fseek)(FILE *stream, long offset, int whence) 21 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(fseeko)(FILE *stream, off_t offset, int whence) 22 | ../runtime/lib/cruise-stdio.c:long CRUISE_DECL(ftell)(FILE *stream) 23 | ../runtime/lib/cruise-stdio.c:off_t CRUISE_DECL(ftello)(FILE *stream) 24 | ../runtime/lib/cruise-stdio.c:void CRUISE_DECL(rewind)(FILE* stream) 25 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(fgetpos)(FILE* stream, fpos_t* pos) 26 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(fsetpos)(FILE* stream, const fpos_t* pos) 27 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(fflush)(FILE* stream) 28 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(feof)(FILE *stream) 29 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(ferror)(FILE* stream) 30 | ../runtime/lib/cruise-stdio.c:void CRUISE_DECL(clearerr)(FILE* stream) 31 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(fileno)(FILE *stream) 32 | ../runtime/lib/cruise-stdio.c:int CRUISE_DECL(fclose)(FILE *stream) 33 | ../runtime/lib/cruise-stdio.c:CRUISE_DECL2(fwprintf, int, (FILE *stream, const wchar_t* format, ...)) 34 | ../runtime/lib/cruise-stdio.c:CRUISE_DECL2(fwscanf, int, (FILE *stream, const wchar_t* format, ...)) 35 | ../runtime/lib/cruise-stdio.c:CRUISE_DECL2(vfwprintf, int, (FILE *stream, const wchar_t* format, va_list arg)) 36 | ../runtime/lib/cruise-stdio.c:CRUISE_DECL2(vfwscanf, int, (FILE *stream, const wchar_t* format, va_list arg)) 37 | ../runtime/lib/cruise-stdio.c:CRUISE_DECL2(fgetwc, wint_t, (FILE *stream)) 38 | ../runtime/lib/cruise-stdio.c:CRUISE_DECL2(fgetws, wchar_t*, (wchar_t* s, int n, FILE *stream)) 39 | ../runtime/lib/cruise-stdio.c:CRUISE_DECL2(fputwc, wint_t, (wchar_t wc, FILE *stream)) 40 | ../runtime/lib/cruise-stdio.c:CRUISE_DECL2(fputws, int, (const wchar_t* s, FILE *stream)) 41 | ../runtime/lib/cruise-stdio.c:CRUISE_DECL2(fwide, int, (FILE *stream, int mode)) 42 | ../runtime/lib/cruise-stdio.c:CRUISE_DECL2(getwc, wint_t, (FILE *stream)) 43 | ../runtime/lib/cruise-stdio.c:CRUISE_DECL2(putwc, wint_t, (wchar_t c, FILE *stream)) 44 | ../runtime/lib/cruise-stdio.c:CRUISE_DECL2(ungetwc, wint_t, (wint_t c, FILE *stream)) 45 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(access)(const char *path, int mode) 46 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(mkdir)(const char *path, mode_t mode) 47 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(rmdir)(const char *path) 48 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(rename)(const char *oldpath, const char *newpath) 49 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(truncate)(const char* path, off_t length) 50 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(unlink)(const char *path) 51 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(remove)(const char *path) 52 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(stat)( const char *path, struct stat *buf) 53 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(__xstat)(int vers, const char *path, struct stat *buf) 54 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(__xstat64)(int vers, const char *path, struct stat64 *buf) 55 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(__lxstat)(int vers, const char *path, struct stat *buf) 56 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(__lxstat64)(int vers, const char *path, struct stat64 *buf) 57 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(creat)(const char* path, mode_t mode) 58 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(creat64)(const char* path, mode_t mode) 59 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(open)(const char *path, int flags, ...) 60 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(open64)(const char* path, int flags, ...) 61 | ../runtime/lib/cruise-sysio.c:off_t CRUISE_DECL(lseek)(int fd, off_t offset, int whence) 62 | ../runtime/lib/cruise-sysio.c:off64_t CRUISE_DECL(lseek64)(int fd, off64_t offset, int whence) 63 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(posix_fadvise)(int fd, off_t offset, off_t len, int advice) 64 | ../runtime/lib/cruise-sysio.c:ssize_t CRUISE_DECL(read)(int fd, void *buf, size_t count) 65 | ../runtime/lib/cruise-sysio.c:ssize_t CRUISE_DECL(write)(int fd, const void *buf, size_t count) 66 | ../runtime/lib/cruise-sysio.c:ssize_t CRUISE_DECL(readv)(int fd, const struct iovec *iov, int iovcnt) 67 | ../runtime/lib/cruise-sysio.c:ssize_t CRUISE_DECL(writev)(int fd, const struct iovec *iov, int iovcnt) 68 | ../runtime/lib/cruise-sysio.c:ssize_t CRUISE_DECL(pread)(int fd, void *buf, size_t count, off_t offset) 69 | ../runtime/lib/cruise-sysio.c:ssize_t CRUISE_DECL(pread64)(int fd, void *buf, size_t count, off64_t offset) 70 | ../runtime/lib/cruise-sysio.c:ssize_t CRUISE_DECL(pwrite)(int fd, const void *buf, size_t count, off_t offset) 71 | ../runtime/lib/cruise-sysio.c:ssize_t CRUISE_DECL(pwrite64)(int fd, const void *buf, size_t count, off64_t offset) 72 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(ftruncate)(int fd, off_t length) 73 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(fsync)(int fd) 74 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(fdatasync)(int fd) 75 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(flock)(int fd, int operation) 76 | ../runtime/lib/cruise-sysio.c:void* CRUISE_DECL(mmap)(void *addr, size_t length, int prot, int flags, 77 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(munmap)(void *addr, size_t length) 78 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(msync)(void *addr, size_t length, int flags) 79 | ../runtime/lib/cruise-sysio.c:void* CRUISE_DECL(mmap64)(void *addr, size_t length, int prot, int flags, 80 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(__fxstat)(int vers, int fd, struct stat *buf) 81 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(__fxstat64)(int vers, int fd, struct stat64 *buf) 82 | ../runtime/lib/cruise-sysio.c:int CRUISE_DECL(close)(int fd) 83 | -------------------------------------------------------------------------------- /maint/check_fns/scrmfs_unsupported_list.txt: -------------------------------------------------------------------------------- 1 | # wrapped but not implemented 2 | 3 | freopen 4 | fwprintf 5 | fwscanf 6 | vfwprintf 7 | vfwscanf 8 | fgetwc 9 | fgetws 10 | fputwc 11 | fputws 12 | fwide 13 | getwc 14 | putwc 15 | ungetwc 16 | -------------------------------------------------------------------------------- /maint/config/check_arch.m4: -------------------------------------------------------------------------------- 1 | dnl @synopsis CHECK_ARCH() 2 | dnl 3 | dnl This macro checks for the architectur (BGQ/Linux) and defines the 4 | dnl appropriate CFLAGS in the code. BG/Q builds also disable NUMA support 5 | 6 | AC_DEFUN([CHECK_ARCH], 7 | 8 | # 9 | # Handle user hints 10 | # 11 | [AC_MSG_CHECKING(architecture type) 12 | AC_ARG_WITH([arch], 13 | [AS_HELP_STRING([--with-numa=ARCH],[specify the architecture as bgq or linux])], 14 | [if test "$withval" != no ; then 15 | AC_MSG_RESULT(yes) 16 | ARCH=$withval 17 | if test "${ARCH}" = "bgq" 18 | then 19 | AC_DEFINE([MACHINE_BGQ], [1], [Define if architecture is BG/Q]) 20 | fi 21 | else 22 | AC_MSG_RESULT(no) 23 | fi]) 24 | 25 | ]) 26 | -------------------------------------------------------------------------------- /maint/config/check_numa.m4: -------------------------------------------------------------------------------- 1 | dnl @synopsis CHECK_NUMA() 2 | dnl 3 | dnl This macro searches for an installed numa library. If nothing was 4 | dnl specified when calling configure, it searches first in /usr/local 5 | dnl and then in /usr. If the --with-numa=DIR is specified, it will try 6 | dnl to find it in DIR/include/numa.h and DIR/lib/libz.a. If 7 | dnl --without-numa is specified, the library is not searched at all. 8 | dnl 9 | dnl If either the header file (numa.h) or the library (libz) is not 10 | dnl found, the configuration exits on error, asking for a valid numa 11 | dnl installation directory or --without-numa. 12 | dnl 13 | dnl The macro defines the symbol HAVE_LIBZ if the library is found. You 14 | dnl should use autoheader to include a definition for this symbol in a 15 | dnl config.h file. Sample usage in a C/C++ source is as follows: 16 | dnl 17 | dnl #ifdef HAVE_LIBZ 18 | dnl #include 19 | dnl #endif /* HAVE_LIBZ */ 20 | dnl 21 | dnl @category InstalledPackages 22 | dnl @author Loic Dachary 23 | dnl @version 2004-09-20 24 | dnl @license GPLWithACException 25 | 26 | AC_DEFUN([CHECK_NUMA], 27 | 28 | # 29 | # Handle user hints 30 | # 31 | [AC_MSG_CHECKING(if numa is wanted ) 32 | AC_ARG_WITH([numa], 33 | [AS_HELP_STRING([--with-numa=DIR],[root directory path of libnuma installation (defaults to /usr/local or /usr if not found in /usr/local)])], 34 | [if test "$withval" != no ; then 35 | AC_MSG_RESULT(yes) 36 | if test -d "$withval" 37 | then 38 | NUMA_HOME="$withval" 39 | AC_DEFINE([ENABLE_NUMA_POLICY], [1], [Define if libnuma is available]) 40 | else 41 | AC_MSG_CHECKING([for libnuma installation in default locations]) 42 | fi 43 | else 44 | AC_MSG_RESULT(no) 45 | fi]) 46 | 47 | # 48 | # Locate numa, if wanted 49 | # 50 | if test -n "${NUMA_HOME}" 51 | then 52 | NUMA_OLD_LDFLAGS=$LDFLAGS 53 | NUMA_OLD_CPPFLAGS=$LDFLAGS 54 | LDFLAGS="$LDFLAGS -L${NUMA_HOME}/lib" 55 | CPPFLAGS="$CPPFLAGS -I${NUMA_HOME}/include" 56 | AC_LANG_SAVE 57 | AC_LANG_C 58 | AC_CHECK_LIB(numa, numa_num_possible_nodes, [numa_cv_libnuma=yes], [numa_cv_libnuma=no]) 59 | AC_CHECK_HEADER(numa.h, [numa_cv_numa_h=yes], [numa_cv_numa_h=no]) 60 | AC_LANG_RESTORE 61 | if test "$numa_cv_libnuma" = "yes" -a "$numa_cv_numa_h" = "yes" 62 | then 63 | # 64 | # If both library and header were found, use them 65 | # 66 | AC_CHECK_LIB(numa, numa_num_possible_nodes) 67 | AC_MSG_CHECKING(numa in ${NUMA_HOME}) 68 | AC_MSG_RESULT(ok) 69 | else 70 | # 71 | # If either header or library was not found, revert and bomb 72 | # 73 | AC_MSG_CHECKING(numa in ${NUMA_HOME}) 74 | LDFLAGS="$NUMA_OLD_LDFLAGS" 75 | CPPFLAGS="$NUMA_OLD_CPPFLAGS" 76 | AC_MSG_RESULT(failed) 77 | AC_MSG_ERROR(either specify a valid numa installation with --with-numa=DIR or disable numa usage with --without-numa) 78 | fi 79 | fi 80 | 81 | ]) 82 | -------------------------------------------------------------------------------- /maint/config/inttypes-pri.m4: -------------------------------------------------------------------------------- 1 | # inttypes-pri.m4 serial 4 (gettext-0.16) 2 | dnl Copyright (C) 1997-2002, 2006 Free Software Foundation, Inc. 3 | dnl This file is free software; the Free Software Foundation 4 | dnl gives unlimited permission to copy and/or distribute it, 5 | dnl with or without modifications, as long as this notice is preserved. 6 | 7 | dnl From Bruno Haible. 8 | 9 | AC_PREREQ(2.52) 10 | 11 | # Define PRI_MACROS_BROKEN if exists and defines the PRI* 12 | # macros to non-string values. This is the case on AIX 4.3.3. 13 | 14 | AC_DEFUN([gt_INTTYPES_PRI], 15 | [ 16 | AC_CHECK_HEADERS([inttypes.h]) 17 | if test $ac_cv_header_inttypes_h = yes; then 18 | AC_CACHE_CHECK([whether the inttypes.h PRIxNN macros are broken], 19 | gt_cv_inttypes_pri_broken, 20 | [ 21 | AC_TRY_COMPILE([#include 22 | #ifdef PRId32 23 | char *p = PRId32; 24 | #endif 25 | ], [], gt_cv_inttypes_pri_broken=no, gt_cv_inttypes_pri_broken=yes) 26 | ]) 27 | fi 28 | if test "$gt_cv_inttypes_pri_broken" = yes; then 29 | AC_DEFINE_UNQUOTED(PRI_MACROS_BROKEN, 1, 30 | [Define if exists and defines unusable PRI* macros.]) 31 | PRI_MACROS_BROKEN=1 32 | else 33 | PRI_MACROS_BROKEN=0 34 | fi 35 | AC_SUBST([PRI_MACROS_BROKEN]) 36 | ]) 37 | -------------------------------------------------------------------------------- /prepare: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | autoreconf -fvi 4 | -------------------------------------------------------------------------------- /src/cruise-fixed.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Written by 5 | * Raghunath Rajachandrasekar 6 | * Kathryn Mohror 7 | * Adam Moody 8 | * LLNL-CODE-642432. 9 | * All rights reserved. 10 | * This file is part of CRUISE. 11 | * For details, see https://github.com/hpc/cruise 12 | * Please also read this file COPYRIGHT 13 | */ 14 | 15 | #include "cruise-runtime-config.h" 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #define __USE_GNU 34 | #include 35 | 36 | #include "cruise-internal.h" 37 | 38 | extern int cruise_spillover_max_chunks; 39 | /* given a file id and logical chunk id, return pointer to meta data 40 | * for specified chunk, return NULL if not found */ 41 | static cruise_chunkmeta_t* cruise_get_chunkmeta(int fid, int cid) 42 | { 43 | /* lookup file meta data for specified file id */ 44 | cruise_filemeta_t* meta = cruise_get_meta_from_fid(fid); 45 | if (meta != NULL) { 46 | /* now lookup chunk meta data for specified chunk id */ 47 | if (cid >= 0 && cid < cruise_max_chunks) { 48 | cruise_chunkmeta_t* chunk_meta = &(meta->chunk_meta[cid]); 49 | return chunk_meta; 50 | } 51 | } 52 | 53 | /* failed to find file or chunk id is out of range */ 54 | return (cruise_chunkmeta_t *)NULL; 55 | } 56 | 57 | /* --------------------------------------- 58 | * Operations on file chunks 59 | * --------------------------------------- */ 60 | 61 | /* given a logical chunk id and an offset within that chunk, return the pointer 62 | * to the memory location corresponding to that location */ 63 | static inline void* cruise_compute_chunk_buf( 64 | const cruise_filemeta_t* meta, 65 | int logical_id, 66 | off_t logical_offset) 67 | { 68 | /* get pointer to chunk meta */ 69 | const cruise_chunkmeta_t* chunk_meta = &(meta->chunk_meta[logical_id]); 70 | 71 | /* identify physical chunk id */ 72 | int physical_id = chunk_meta->id; 73 | 74 | /* compute the start of the chunk */ 75 | char *start = NULL; 76 | if (physical_id < cruise_max_chunks) { 77 | /* Teng: convert data type to handle larger workload */ 78 | start = cruise_chunks + ((off_t)physical_id << cruise_chunk_bits); 79 | } else { 80 | /* chunk is in spill over */ 81 | debug("wrong chunk ID\n"); 82 | return NULL; 83 | } 84 | 85 | /* now add offset */ 86 | char* buf = start + logical_offset; 87 | return (void*)buf; 88 | } 89 | 90 | /* given a chunk id and an offset within that chunk, return the offset 91 | * in the spillover file corresponding to that location */ 92 | static inline off_t cruise_compute_spill_offset( 93 | const cruise_filemeta_t* meta, 94 | int logical_id, 95 | off_t logical_offset) 96 | { 97 | /* get pointer to chunk meta */ 98 | const cruise_chunkmeta_t* chunk_meta = &(meta->chunk_meta[logical_id]); 99 | 100 | /* identify physical chunk id */ 101 | int physical_id = chunk_meta->id; 102 | 103 | /* compute start of chunk in spill over device */ 104 | off_t start = 0; 105 | if (physical_id < cruise_max_chunks) { 106 | debug("wrong spill-chunk ID\n"); 107 | return -1; 108 | } else { 109 | /* compute buffer loc within spillover device chunk */ 110 | /* account for the cruise_max_chunks added to identify location when 111 | * grabbing this chunk */ 112 | start = ((off_t)(physical_id - cruise_max_chunks) << cruise_chunk_bits); 113 | } 114 | off_t buf = start + logical_offset; 115 | return buf; 116 | } 117 | 118 | /* allocate a new chunk for the specified file and logical chunk id */ 119 | static int cruise_chunk_alloc(int fid, cruise_filemeta_t* meta, int chunk_id) 120 | { 121 | /* get pointer to chunk meta data */ 122 | cruise_chunkmeta_t* chunk_meta = &(meta->chunk_meta[chunk_id]); 123 | 124 | /* allocate a chunk and record its location */ 125 | if (cruise_use_memfs) { 126 | /* allocate a new chunk from memory */ 127 | cruise_stack_lock(); 128 | int id = cruise_stack_pop(free_chunk_stack); 129 | cruise_stack_unlock(); 130 | 131 | /* if we got one return, otherwise try spill over */ 132 | if (id >= 0) { 133 | /* got a chunk from memory */ 134 | chunk_meta->location = CHUNK_LOCATION_MEMFS; 135 | chunk_meta->id = id; 136 | } else if (cruise_use_spillover) { 137 | /* shm segment out of space, grab a block from spill-over device */ 138 | debug("getting blocks from spill-over device\n"); 139 | 140 | /* TODO: missing lock calls? */ 141 | /* add cruise_max_chunks to identify chunk location */ 142 | cruise_stack_lock(); 143 | id = cruise_stack_pop(free_spillchunk_stack) + cruise_max_chunks; 144 | cruise_stack_unlock(); 145 | if (id < cruise_max_chunks) { 146 | debug("spill-over device out of space (%d)\n", id); 147 | return CRUISE_ERR_NOSPC; 148 | } 149 | 150 | /* got one from spill over */ 151 | chunk_meta->location = CHUNK_LOCATION_SPILLOVER; 152 | chunk_meta->id = id; 153 | } else { 154 | /* spill over isn't available, so we're out of space */ 155 | debug("memfs out of space (%d)\n", id); 156 | return CRUISE_ERR_NOSPC; 157 | } 158 | } else if (cruise_use_spillover) { 159 | /* memory file system is not enabled, but spill over is */ 160 | 161 | /* shm segment out of space, grab a block from spill-over device */ 162 | debug("getting blocks from spill-over device \n"); 163 | 164 | /* TODO: missing lock calls? */ 165 | /* add cruise_max_chunks to identify chunk location */ 166 | cruise_stack_lock(); 167 | int id = cruise_stack_pop(free_spillchunk_stack) + cruise_max_chunks; 168 | cruise_stack_unlock(); 169 | if (id < cruise_max_chunks) { 170 | debug("spill-over device out of space (%d)\n", id); 171 | return CRUISE_ERR_NOSPC; 172 | } 173 | 174 | /* got one from spill over */ 175 | chunk_meta->location = CHUNK_LOCATION_SPILLOVER; 176 | chunk_meta->id = id; 177 | } else { 178 | /* don't know how to allocate chunk */ 179 | chunk_meta->location = CHUNK_LOCATION_NULL; 180 | return CRUISE_ERR_IO; 181 | } 182 | 183 | return CRUISE_SUCCESS; 184 | } 185 | 186 | static int cruise_chunk_free(int fid, cruise_filemeta_t* meta, int chunk_id) 187 | { 188 | /* get pointer to chunk meta data */ 189 | cruise_chunkmeta_t* chunk_meta = &(meta->chunk_meta[chunk_id]); 190 | 191 | /* get physical id of chunk */ 192 | int id = chunk_meta->id; 193 | debug("free chunk %d from location %d\n", id, chunk_meta->location); 194 | 195 | /* determine location of chunk */ 196 | if (chunk_meta->location == CHUNK_LOCATION_MEMFS) { 197 | cruise_stack_lock(); 198 | cruise_stack_push(free_chunk_stack, id); 199 | cruise_stack_unlock(); 200 | } else if (chunk_meta->location == CHUNK_LOCATION_SPILLOVER) { 201 | /* TODO: free spill over chunk */ 202 | } else { 203 | /* unkwown chunk location */ 204 | debug("unknown chunk location %d\n", chunk_meta->location); 205 | return CRUISE_ERR_IO; 206 | } 207 | 208 | /* update location of chunk */ 209 | chunk_meta->location = CHUNK_LOCATION_NULL; 210 | 211 | return CRUISE_SUCCESS; 212 | } 213 | 214 | /* read data from specified chunk id, chunk offset, and count into user buffer, 215 | * count should fit within chunk starting from specified offset */ 216 | static int cruise_chunk_read( 217 | cruise_filemeta_t* meta, /* pointer to file meta data */ 218 | int chunk_id, /* logical chunk id to read data from */ 219 | off_t chunk_offset, /* logical offset within chunk to read from */ 220 | void* buf, /* buffer to store data to */ 221 | size_t count) /* number of bytes to read */ 222 | { 223 | /* get chunk meta data */ 224 | cruise_chunkmeta_t* chunk_meta = &(meta->chunk_meta[chunk_id]); 225 | 226 | /* determine location of chunk */ 227 | if (chunk_meta->location == CHUNK_LOCATION_MEMFS) { 228 | /* just need a memcpy to read data */ 229 | void* chunk_buf = cruise_compute_chunk_buf(meta, chunk_id, chunk_offset); 230 | memcpy(buf, chunk_buf, count); 231 | } else if (chunk_meta->location == CHUNK_LOCATION_SPILLOVER) { 232 | /* spill over to a file, so read from file descriptor */ 233 | //MAP_OR_FAIL(pread); 234 | off_t spill_offset = cruise_compute_spill_offset(meta, chunk_id, chunk_offset); 235 | ssize_t rc = pread(cruise_spilloverblock, buf, count, spill_offset); 236 | /* TODO: check return code for errors */ 237 | } else { 238 | /* unknown chunk type */ 239 | debug("unknown chunk type in read\n"); 240 | return CRUISE_ERR_IO; 241 | } 242 | 243 | /* assume read was successful if we get to here */ 244 | return CRUISE_SUCCESS; 245 | } 246 | 247 | /* read data from specified chunk id, chunk offset, and count into user buffer, 248 | * count should fit within chunk starting from specified offset */ 249 | static int cruise_chunk_write( 250 | cruise_filemeta_t* meta, /* pointer to file meta data */ 251 | int chunk_id, /* logical chunk id to write to */ 252 | off_t chunk_offset, /* logical offset within chunk to write to */ 253 | const void* buf, /* buffer holding data to be written */ 254 | size_t count) /* number of bytes to write */ 255 | { 256 | /* get chunk meta data */ 257 | cruise_chunkmeta_t* chunk_meta = &(meta->chunk_meta[chunk_id]); 258 | 259 | /* determine location of chunk */ 260 | if (chunk_meta->location == CHUNK_LOCATION_MEMFS) { 261 | /* just need a memcpy to write data */ 262 | void* chunk_buf = cruise_compute_chunk_buf(meta, chunk_id, chunk_offset); 263 | memcpy(chunk_buf, buf, count); 264 | // _intel_fast_memcpy(chunk_buf, buf, count); 265 | // cruise_memcpy(chunk_buf, buf, count); 266 | } else if (chunk_meta->location == CHUNK_LOCATION_SPILLOVER) { 267 | /* spill over to a file, so write to file descriptor */ 268 | //MAP_OR_FAIL(pwrite); 269 | off_t spill_offset = cruise_compute_spill_offset(meta, chunk_id, chunk_offset); 270 | ssize_t rc = pwrite(cruise_spilloverblock, buf, count, spill_offset); 271 | if (rc < 0) { 272 | perror("pwrite failed"); 273 | } 274 | /* TODO: check return code for errors */ 275 | } else { 276 | /* unknown chunk type */ 277 | debug("unknown chunk type in read\n"); 278 | return CRUISE_ERR_IO; 279 | } 280 | 281 | /* assume read was successful if we get to here */ 282 | return CRUISE_SUCCESS; 283 | } 284 | 285 | /* --------------------------------------- 286 | * Operations on file storage 287 | * --------------------------------------- */ 288 | 289 | /* if length is greater than reserved space, reserve space up to length */ 290 | int cruise_fid_store_fixed_extend(int fid, cruise_filemeta_t* meta, off_t length) 291 | { 292 | /* determine whether we need to allocate more chunks */ 293 | off_t maxsize = meta->chunks << cruise_chunk_bits; 294 | if (length > maxsize) { 295 | /* compute number of additional bytes we need */ 296 | off_t additional = length - maxsize; 297 | while (additional > 0) { 298 | /* check that we don't overrun max number of chunks for file */ 299 | if (meta->chunks == cruise_max_chunks + cruise_spillover_max_chunks) { 300 | debug("failed to allocate chunk\n"); 301 | return CRUISE_ERR_NOSPC; 302 | } 303 | 304 | /* allocate a new chunk */ 305 | int rc = cruise_chunk_alloc(fid, meta, meta->chunks); 306 | if (rc != CRUISE_SUCCESS) { 307 | debug("failed to allocate chunk\n"); 308 | return CRUISE_ERR_NOSPC; 309 | } 310 | 311 | /* increase chunk count and subtract bytes from the number we need */ 312 | meta->chunks++; 313 | additional -= cruise_chunk_size; 314 | } 315 | } 316 | 317 | return CRUISE_SUCCESS; 318 | } 319 | 320 | /* if length is shorter than reserved space, give back space down to length */ 321 | int cruise_fid_store_fixed_shrink(int fid, cruise_filemeta_t* meta, off_t length) 322 | { 323 | /* determine the number of chunks to leave after truncating */ 324 | off_t num_chunks = 0; 325 | if (length > 0) { 326 | num_chunks = (length >> cruise_chunk_bits) + 1; 327 | } 328 | 329 | /* clear off any extra chunks */ 330 | while (meta->chunks > num_chunks) { 331 | meta->chunks--; 332 | cruise_chunk_free(fid, meta, meta->chunks); 333 | } 334 | 335 | return CRUISE_SUCCESS; 336 | } 337 | 338 | /* read data from file stored as fixed-size chunks */ 339 | int cruise_fid_store_fixed_read(int fid, cruise_filemeta_t* meta, off_t pos, void* buf, size_t count) 340 | { 341 | int rc; 342 | 343 | /* get pointer to position within first chunk */ 344 | int chunk_id = pos >> cruise_chunk_bits; 345 | off_t chunk_offset = pos & cruise_chunk_mask; 346 | 347 | /* determine how many bytes remain in the current chunk */ 348 | size_t remaining = cruise_chunk_size - chunk_offset; 349 | if (count <= remaining) { 350 | /* all bytes for this read fit within the current chunk */ 351 | rc = cruise_chunk_read(meta, chunk_id, chunk_offset, buf, count); 352 | } else { 353 | /* read what's left of current chunk */ 354 | char* ptr = (char*) buf; 355 | rc = cruise_chunk_read(meta, chunk_id, chunk_offset, (void*)ptr, remaining); 356 | ptr += remaining; 357 | 358 | /* read from the next chunk */ 359 | size_t processed = remaining; 360 | while (processed < count && rc == CRUISE_SUCCESS) { 361 | /* get pointer to start of next chunk */ 362 | chunk_id++; 363 | 364 | /* compute size to read from this chunk */ 365 | size_t num = count - processed; 366 | if (num > cruise_chunk_size) { 367 | num = cruise_chunk_size; 368 | } 369 | 370 | /* read data */ 371 | rc = cruise_chunk_read(meta, chunk_id, 0, (void*)ptr, num); 372 | ptr += num; 373 | 374 | /* update number of bytes written */ 375 | processed += num; 376 | } 377 | } 378 | 379 | return rc; 380 | } 381 | 382 | /* write data to file stored as fixed-size chunks */ 383 | int cruise_fid_store_fixed_write(int fid, cruise_filemeta_t* meta, off_t pos, const void* buf, size_t count) 384 | { 385 | int rc; 386 | 387 | /* get pointer to position within first chunk */ 388 | int chunk_id = pos >> cruise_chunk_bits; 389 | off_t chunk_offset = pos & cruise_chunk_mask; 390 | 391 | /* determine how many bytes remain in the current chunk */ 392 | size_t remaining = cruise_chunk_size - chunk_offset; 393 | if (count <= remaining) { 394 | /* all bytes for this write fit within the current chunk */ 395 | rc = cruise_chunk_write(meta, chunk_id, chunk_offset, buf, count); 396 | } else { 397 | /* otherwise, fill up the remainder of the current chunk */ 398 | char* ptr = (char*) buf; 399 | rc = cruise_chunk_write(meta, chunk_id, chunk_offset, (void*)ptr, remaining); 400 | ptr += remaining; 401 | 402 | /* then write the rest of the bytes starting from beginning 403 | * of chunks */ 404 | size_t processed = remaining; 405 | while (processed < count && rc == CRUISE_SUCCESS) { 406 | /* get pointer to start of next chunk */ 407 | chunk_id++; 408 | 409 | /* compute size to write to this chunk */ 410 | size_t num = count - processed; 411 | if (num > cruise_chunk_size) { 412 | num = cruise_chunk_size; 413 | } 414 | 415 | /* write data */ 416 | rc = cruise_chunk_write(meta, chunk_id, 0, (void*)ptr, num); 417 | ptr += num; 418 | 419 | /* update number of bytes processed */ 420 | processed += num; 421 | } 422 | } 423 | 424 | return rc; 425 | } 426 | -------------------------------------------------------------------------------- /src/cruise-fixed.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Written by 5 | * Raghunath Rajachandrasekar 6 | * Kathryn Mohror 7 | * Adam Moody 8 | * LLNL-CODE-642432. 9 | * All rights reserved. 10 | * This file is part of CRUISE. 11 | * For details, see https://github.com/hpc/cruise 12 | * Please also read this file COPYRIGHT 13 | */ 14 | 15 | #ifndef CRUISE_FIXED_H 16 | #define CRUISE_FIXED_H 17 | 18 | #include "cruise-internal.h" 19 | 20 | /* if length is greater than reserved space, 21 | * reserve space up to length */ 22 | int cruise_fid_store_fixed_extend( 23 | int fid, /* file id to reserve space for */ 24 | cruise_filemeta_t* meta, /* meta data for file */ 25 | off_t length /* number of bytes to reserve for file */ 26 | ); 27 | 28 | /* if length is shorter than reserved space, 29 | * give back space down to length */ 30 | int cruise_fid_store_fixed_shrink( 31 | int fid, /* file id to free space for */ 32 | cruise_filemeta_t* meta, /* meta data for file */ 33 | off_t length /* number of bytes to reserve for file */ 34 | ); 35 | 36 | /* read data from file stored as fixed-size chunks, 37 | * returns CRUISE error code */ 38 | int cruise_fid_store_fixed_read( 39 | int fid, /* file id to read from */ 40 | cruise_filemeta_t* meta, /* meta data for file */ 41 | off_t pos, /* position within file to read from */ 42 | void* buf, /* user buffer to store data in */ 43 | size_t count /* number of bytes to read */ 44 | ); 45 | 46 | /* write data to file stored as fixed-size chunks, 47 | * returns CRUISE error code */ 48 | int cruise_fid_store_fixed_write( 49 | int fid, /* file id to write to */ 50 | cruise_filemeta_t* meta, /* meta data for file */ 51 | off_t pos, /* position within file to write to */ 52 | const void* buf, /* user buffer holding data */ 53 | size_t count /* number of bytes to write */ 54 | ); 55 | 56 | #endif /* CRUISE_FIXED_H */ 57 | -------------------------------------------------------------------------------- /src/cruise-internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Written by 5 | * Raghunath Rajachandrasekar 6 | * Kathryn Mohror 7 | * Adam Moody 8 | * LLNL-CODE-642432. 9 | * All rights reserved. 10 | * This file is part of CRUISE. 11 | * For details, see https://github.com/hpc/cruise 12 | * Please also read this file COPYRIGHT 13 | */ 14 | 15 | #ifndef CRUISE_INTERNAL_H 16 | #define CRUISE_INTERNAL_H 17 | 18 | /* this is overkill to include all of these here, but just to get things working... */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | #include "utlist.h" 38 | #include "uthash.h" 39 | 40 | /* ------------------------------- 41 | * Defines and types 42 | * ------------------------------- */ 43 | 44 | /* TODO: move common includes to another file */ 45 | #include "cruise-defs.h" 46 | 47 | //#define CRUISE_DEBUG 48 | #ifdef CRUISE_DEBUG 49 | #define debug(fmt, args... ) printf("%s: "fmt, __func__, ##args) 50 | #else 51 | #define debug(fmt, args... ) 52 | #endif 53 | 54 | /* define a macro to capture function name, file name, and line number 55 | * along with user-defined string */ 56 | #define CRUISE_UNSUPPORTED(fmt, args...) \ 57 | cruise_unsupported(__func__, __FILE__, __LINE__, fmt, ##args) 58 | 59 | #ifdef CRUISE_PRELOAD 60 | 61 | /* =================================================================== 62 | * Using LD_PRELOAD to intercept 63 | * =================================================================== 64 | * we need to use the same function names the application is calling, 65 | * and we then invoke the real library function after looking it up with 66 | * dlsym */ 67 | 68 | /* we need the dlsym function */ 69 | #define __USE_GNU 70 | #include 71 | #include 72 | 73 | /* define a static variable called __real_open to record address of 74 | * real open call and initialize it to NULL */ 75 | #define CRUISE_DECL(name,ret,args) \ 76 | static ret (*__real_ ## name)args = NULL; 77 | 78 | /* our open wrapper assumes the name of open() */ 79 | #define CRUISE_WRAP(name) name 80 | 81 | /* the address of the real open call is stored in __real_open variable */ 82 | #define CRUISE_REAL(name) __real_ ## name 83 | 84 | /* if __real_open is still NULL, call dlsym to lookup address of real 85 | * function and record it */ 86 | #define MAP_OR_FAIL(func) \ 87 | if (!(__real_ ## func)) \ 88 | { \ 89 | __real_ ## func = dlsym(RTLD_NEXT, #func); \ 90 | if(!(__real_ ## func)) { \ 91 | fprintf(stderr, "CRUISE failed to map symbol: %s\n", #func); \ 92 | exit(1); \ 93 | } \ 94 | } 95 | 96 | #else 97 | 98 | /* =================================================================== 99 | * Using ld -wrap option to intercept 100 | * =================================================================== 101 | * the linker will convert application calls from open --> __wrap_open, 102 | * so we define all of our functions as the __wrap variant and then 103 | * to call the real library, we call __real_open */ 104 | 105 | /* we don't need a variable to record the address of the real function, 106 | * just declare the existence of __real_open so the compiler knows the 107 | * prototype of this function (linker will provide it) */ 108 | #define CRUISE_DECL(name,ret,args) \ 109 | extern ret __real_ ## name args; 110 | 111 | /* we define our wrapper function as __wrap_open instead of open */ 112 | #define CRUISE_WRAP(name) __wrap_ ## name 113 | 114 | /* the linker maps the open call to __real_open() */ 115 | #define CRUISE_REAL(name) __real_ ## name 116 | 117 | /* no need to look up the address of the real function */ 118 | #define MAP_OR_FAIL(func) 119 | 120 | #endif 121 | 122 | #define CRUISE_SUCCESS 0 123 | #define CRUISE_FAILURE -1 124 | #define CRUISE_ERR_NOSPC -2 125 | #define CRUISE_ERR_IO -3 126 | #define CRUISE_ERR_NAMETOOLONG -4 127 | #define CRUISE_ERR_NOENT -5 128 | #define CRUISE_ERR_EXIST -6 129 | #define CRUISE_ERR_NOTDIR -7 130 | #define CRUISE_ERR_NFILE -8 131 | #define CRUISE_ERR_INVAL -9 132 | #define CRUISE_ERR_OVERFLOW -10 133 | #define CRUISE_ERR_FBIG -11 134 | #define CRUISE_ERR_BADF -12 135 | #define CRUISE_ERR_ISDIR -13 136 | #define CRUISE_ERR_NOMEM -14 137 | 138 | #ifndef HAVE_OFF64_T 139 | typedef int64_t off64_t; 140 | #endif 141 | 142 | /* structure to represent file descriptors */ 143 | typedef struct { 144 | off_t pos; /* current file pointer */ 145 | int read; /* whether file is opened for read */ 146 | int write; /* whether file is opened for write */ 147 | } cruise_fd_t; 148 | 149 | enum cruise_stream_orientation { 150 | CRUISE_STREAM_ORIENTATION_NULL = 0, 151 | CRUISE_STREAM_ORIENTATION_BYTE, 152 | CRUISE_STREAM_ORIENTATION_WIDE, 153 | }; 154 | 155 | /* structure to represent FILE* streams */ 156 | typedef struct { 157 | int err; /* stream error indicator flag */ 158 | int eof; /* stream end-of-file indicator flag */ 159 | int fd; /* file descriptor associated with stream */ 160 | int append; /* whether file is opened in append mode */ 161 | int orient; /* stream orientation, CRUISE_STREAM_ORIENTATION_{NULL,BYTE,WIDE} */ 162 | 163 | void* buf; /* pointer to buffer */ 164 | int buffree; /* whether we need to free buffer */ 165 | int buftype; /* _IOFBF fully buffered, _IOLBF line buffered, _IONBF unbuffered */ 166 | size_t bufsize; /* size of buffer in bytes */ 167 | off_t bufpos; /* byte offset in file corresponding to start of buffer */ 168 | size_t buflen; /* number of bytes active in buffer */ 169 | size_t bufdirty; /* whether data in buffer needs to be flushed */ 170 | 171 | unsigned char* ubuf; /* ungetc buffer (we store bytes from end) */ 172 | size_t ubufsize; /* size of ungetc buffer in bytes */ 173 | size_t ubuflen; /* number of active bytes in buffer */ 174 | 175 | unsigned char* _p; /* pointer to character in buffer */ 176 | size_t _r; /* number of bytes left at pointer */ 177 | } cruise_stream_t; 178 | 179 | enum flock_enum { 180 | UNLOCKED, 181 | EX_LOCKED, 182 | SH_LOCKED 183 | }; 184 | 185 | /* TODO: make this an enum */ 186 | #define FILE_STORAGE_NULL 0 187 | #define FILE_STORAGE_FIXED_CHUNK 1 188 | 189 | #define EXTERNAL_DATA_DIR "/tmp" 190 | 191 | /* TODO: make this an enum */ 192 | #define CHUNK_LOCATION_NULL 0 193 | #define CHUNK_LOCATION_MEMFS 1 194 | #define CHUNK_LOCATION_SPILLOVER 2 195 | 196 | typedef struct { 197 | int location; /* CHUNK_LOCATION specifies how chunk is stored */ 198 | off_t id; /* physical id of chunk in its respective storage */ 199 | } cruise_chunkmeta_t; 200 | 201 | typedef struct { 202 | off_t size; /* current file size */ 203 | int is_dir; /* is this file a directory */ 204 | pthread_spinlock_t fspinlock; /* file lock variable */ 205 | enum flock_enum flock_status; /* file lock status */ 206 | 207 | int storage; /* FILE_STORAGE specifies file data management */ 208 | 209 | off_t chunks; /* number of chunks allocated to file */ 210 | cruise_chunkmeta_t* chunk_meta; /* meta data for chunks */ 211 | 212 | } cruise_filemeta_t; 213 | 214 | /* path to fid lookup struct */ 215 | typedef struct { 216 | int in_use; /* flag incidating whether slot is in use */ 217 | const char filename[CRUISE_MAX_FILENAME]; 218 | /* full path and name of file */ 219 | } cruise_filename_t; 220 | 221 | /* ------------------------------- 222 | * Common includes 223 | * ------------------------------- */ 224 | 225 | /* TODO: move common includes to another file */ 226 | #include "cruise.h" 227 | #include "cruise-stack.h" 228 | #include "cruise-fixed.h" 229 | #include "cruise-sysio.h" 230 | #include "cruise-stdio.h" 231 | 232 | /* ------------------------------- 233 | * Global varaible declarations 234 | * ------------------------------- */ 235 | 236 | /* keep track of what we've initialized */ 237 | extern int cruise_initialized; 238 | 239 | /* list of file names */ 240 | extern cruise_filename_t* cruise_filelist; 241 | 242 | /* mount directory */ 243 | extern char* cruise_mount_prefix; 244 | extern size_t cruise_mount_prefixlen; 245 | 246 | /* array of file descriptors */ 247 | extern cruise_fd_t cruise_fds[CRUISE_MAX_FILEDESCS]; 248 | extern rlim_t cruise_fd_limit; 249 | 250 | /* array of file streams */ 251 | extern cruise_stream_t cruise_streams[CRUISE_MAX_FILEDESCS]; 252 | 253 | extern int cruise_use_memfs; 254 | extern int cruise_use_spillover; 255 | 256 | extern int cruise_max_files; /* maximum number of files to store */ 257 | extern size_t cruise_chunk_mem; /* number of bytes in memory to be used for chunk storage */ 258 | extern int cruise_chunk_bits; /* we set chunk size = 2^cruise_chunk_bits */ 259 | extern off_t cruise_chunk_size; /* chunk size in bytes */ 260 | extern off_t cruise_chunk_mask; /* mask applied to logical offset to determine physical offset within chunk */ 261 | extern int cruise_max_chunks; /* maximum number of chunks that fit in memory */ 262 | 263 | extern void* free_chunk_stack; 264 | extern void* free_spillchunk_stack; 265 | extern char* cruise_chunks; 266 | int cruise_spilloverblock; 267 | 268 | /* ------------------------------- 269 | * Common functions 270 | * ------------------------------- */ 271 | 272 | /* single function to route all unsupported wrapper calls through */ 273 | int cruise_unsupported(const char* fn_name, const char* file, int line, const char* fmt, ...); 274 | 275 | /* returns 1 if two input parameters will overflow their type when 276 | * added together */ 277 | int cruise_would_overflow_offt(off_t a, off_t b); 278 | 279 | /* returns 1 if two input parameters will overflow their type when 280 | * added together */ 281 | int cruise_would_overflow_long(long a, long b); 282 | 283 | /* given an input mode, mask it with umask and return, can specify 284 | * an input mode==0 to specify all read/write bits */ 285 | mode_t cruise_getmode(mode_t perms); 286 | 287 | int cruise_stack_lock(); 288 | 289 | int cruise_stack_unlock(); 290 | 291 | /* sets flag if the path is a special path */ 292 | int cruise_intercept_path(const char* path); 293 | 294 | /* given an fd, return 1 if we should intercept this file, 0 otherwise, 295 | * convert fd to new fd value if needed */ 296 | int cruise_intercept_fd(int* fd); 297 | 298 | /* given a FILE*, returns 1 if we should intercept this file, 299 | * 0 otherwise */ 300 | int cruise_intercept_stream(FILE* stream); 301 | 302 | /* given a path, return the file id */ 303 | int cruise_get_fid_from_path(const char* path); 304 | 305 | /* given a file descriptor, return the file id */ 306 | int cruise_get_fid_from_fd(int fd); 307 | 308 | /* return address of file descriptor structure or NULL if fd is out 309 | * of range */ 310 | cruise_fd_t* cruise_get_filedesc_from_fd(int fd); 311 | 312 | /* given a file id, return a pointer to the meta data, 313 | * otherwise return NULL */ 314 | inline cruise_filemeta_t* cruise_get_meta_from_fid(int fid); 315 | 316 | /* given an CRUISE error code, return corresponding errno code */ 317 | int cruise_err_map_to_errno(int rc); 318 | 319 | /* checks to see if fid is a directory 320 | * returns 1 for yes 321 | * returns 0 for no */ 322 | int cruise_fid_is_dir(int fid); 323 | 324 | /* checks to see if a directory is empty 325 | * assumes that check for is_dir has already been made 326 | * only checks for full path matches, does not check relative paths, 327 | * e.g. ../dirname will not work 328 | * returns 1 for yes it is empty 329 | * returns 0 for no */ 330 | int cruise_fid_is_dir_empty(const char * path); 331 | 332 | /* return current size of given file id */ 333 | off_t cruise_fid_size(int fid); 334 | 335 | /* fill in limited amount of stat information */ 336 | int cruise_fid_stat(int fid, struct stat* buf); 337 | 338 | /* allocate a file id slot for a new file 339 | * return the fid or -1 on error */ 340 | int cruise_fid_alloc(); 341 | 342 | /* return the file id back to the free pool */ 343 | int cruise_fid_free(int fid); 344 | 345 | /* add a new file and initialize metadata 346 | * returns the new fid, or negative value on error */ 347 | int cruise_fid_create_file(const char * path); 348 | 349 | /* add a new directory and initialize metadata 350 | * returns the new fid, or a negative value on error */ 351 | int cruise_fid_create_directory(const char * path); 352 | 353 | /* read count bytes from file starting from pos and store into buf, 354 | * all bytes are assumed to exist, so checks on file size should be 355 | * done before calling this routine */ 356 | int cruise_fid_read(int fid, off_t pos, void* buf, size_t count); 357 | 358 | /* write count bytes from buf into file starting at offset pos, 359 | * all bytes are assumed to be allocated to file, so file should 360 | * be extended before calling this routine */ 361 | int cruise_fid_write(int fid, off_t pos, const void* buf, size_t count); 362 | 363 | /* given a file id, write zero bytes to region of specified offset 364 | * and length, assumes space is already reserved */ 365 | int cruise_fid_write_zero(int fid, off_t pos, off_t count); 366 | 367 | /* increase size of file if length is greater than current size, 368 | * and allocate additional chunks as needed to reserve space for 369 | * length bytes */ 370 | int cruise_fid_extend(int fid, off_t length); 371 | 372 | /* truncate file id to given length, frees resources if length is 373 | * less than size and allocates and zero-fills new bytes if length 374 | * is more than size */ 375 | int cruise_fid_truncate(int fid, off_t length); 376 | 377 | /* opens a new file id with specified path, access flags, and permissions, 378 | * fills outfid with file id and outpos with position for current file pointer, 379 | * returns CRUISE error code */ 380 | int cruise_fid_open(const char* path, int flags, mode_t mode, int* outfid, off_t* outpos); 381 | 382 | int cruise_fid_close(int fid); 383 | 384 | /* delete a file id and return file its resources to free pools */ 385 | int cruise_fid_unlink(int fid); 386 | 387 | #endif /* CRUISE_INTERNAL_H */ 388 | -------------------------------------------------------------------------------- /src/cruise-stack.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Written by 5 | * Raghunath Rajachandrasekar 6 | * Kathryn Mohror 7 | * Adam Moody 8 | * LLNL-CODE-642432. 9 | * All rights reserved. 10 | * This file is part of CRUISE. 11 | * For details, see https://github.com/hpc/cruise 12 | * Please also read this file COPYRIGHT 13 | */ 14 | 15 | /* implements a fixed-size stack which stores integer values in range 16 | * of 0 to size-1, entire structure stored in an int array of size+2 17 | * int size 18 | * int last 19 | * int entries[size] 20 | * last records index within entries that points to item one past 21 | * the item at the top of the stack */ 22 | 23 | #include "cruise-stack.h" 24 | 25 | /* returns number of bytes needed to represent stack data structure */ 26 | size_t cruise_stack_bytes(int size) 27 | { 28 | size_t bytes = sizeof(cruise_stack) + size * sizeof(int); 29 | return bytes; 30 | } 31 | 32 | /* intializes stack to record all entries as being free */ 33 | void cruise_stack_init(void* start, int size) 34 | { 35 | cruise_stack* stack = (cruise_stack*) start; 36 | stack->size = size; 37 | stack->last = size; 38 | 39 | int i; 40 | int* entries = (int*) ((char*)start + sizeof(cruise_stack)); 41 | for (i = 0; i < size; i++) { 42 | /* flip the order so low numbers are at the top 43 | * to make debugging easier */ 44 | entries[i] = size-1 - i; 45 | } 46 | } 47 | 48 | /* pops one entry from stack and returns its value */ 49 | int cruise_stack_pop(void* start) 50 | { 51 | cruise_stack* stack = (cruise_stack*) start; 52 | 53 | /* check that the stack isn't empty */ 54 | if (stack->last > 0) { 55 | /* decrement the last pointer by one */ 56 | stack->last--; 57 | 58 | /* return the index into entries array, 59 | * this now points to item at top of stack */ 60 | int idx = stack->last; 61 | 62 | /* return the value of this item */ 63 | int* entries = (int*) ((char*)start + sizeof(cruise_stack)); 64 | int value = entries[idx]; 65 | return value; 66 | } else { 67 | /* out of space */ 68 | return -1; 69 | } 70 | } 71 | 72 | /* pushes item onto free stack */ 73 | void cruise_stack_push(void* start, int value) 74 | { 75 | cruise_stack* stack = (cruise_stack*) start; 76 | 77 | /* check that we have space to push item onto stack */ 78 | if (stack->last < stack->size) { 79 | /* get index of first free slot in entries array */ 80 | int idx = stack->last; 81 | 82 | /* place item on stack */ 83 | int* entries = (int*) ((char*)start + sizeof(cruise_stack)); 84 | entries[idx] = value; 85 | 86 | /* increment last pointer to point to next item */ 87 | stack->last++; 88 | } else { 89 | /* freed one too many */ 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/cruise-stack.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Written by 5 | * Raghunath Rajachandrasekar 6 | * Kathryn Mohror 7 | * Adam Moody 8 | * LLNL-CODE-642432. 9 | * All rights reserved. 10 | * This file is part of CRUISE. 11 | * For details, see https://github.com/hpc/cruise 12 | * Please also read this file COPYRIGHT 13 | */ 14 | 15 | #ifndef CRUISE_STACK_H 16 | #define CRUISE_STACK_H 17 | 18 | /* implements a fixed-size stack which stores integer values in range 19 | * of 0 to size-1, entire structure stored in an int array of size+2 20 | * int size 21 | * int last 22 | * int entries[size] 23 | * last records index within entries that points to item one past 24 | * the item at the top of the stack 25 | * 26 | * used to record which entries in a fixed-size array are free */ 27 | 28 | #include 29 | 30 | typedef struct { 31 | int size; 32 | int last; 33 | } cruise_stack; 34 | 35 | /* returns number of bytes needed to represent stack data structure */ 36 | size_t cruise_stack_bytes(int size); 37 | 38 | /* intializes stack to record all entries as being free */ 39 | void cruise_stack_init(void* start, int size); 40 | 41 | /* pops one entry from stack and returns its value */ 42 | int cruise_stack_pop(void* start); 43 | 44 | /* pushes item onto free stack */ 45 | void cruise_stack_push(void* start, int value); 46 | 47 | #endif /* CRUISE_STACK_H */ 48 | -------------------------------------------------------------------------------- /src/cruise-stdio.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Written by 5 | * Raghunath Rajachandrasekar 6 | * Kathryn Mohror 7 | * Adam Moody 8 | * LLNL-CODE-642432. 9 | * All rights reserved. 10 | * This file is part of CRUISE. 11 | * For details, see https://github.com/hpc/cruise 12 | * Please also read this file COPYRIGHT 13 | */ 14 | 15 | #ifndef CRUISE_STDIO_H 16 | #define CRUISE_STDIO_H 17 | 18 | #include "cruise-internal.h" 19 | 20 | /* TODO: declare the wrapper functions we define in cruise-stdio.c 21 | * so other routines can call them */ 22 | 23 | #endif /* CRUISE_STDIO_H */ 24 | -------------------------------------------------------------------------------- /src/cruise-sysio.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Written by 5 | * Raghunath Rajachandrasekar 6 | * Kathryn Mohror 7 | * Adam Moody 8 | * LLNL-CODE-642432. 9 | * All rights reserved. 10 | * This file is part of CRUISE. 11 | * For details, see https://github.com/hpc/cruise 12 | * Please also read this file COPYRIGHT 13 | */ 14 | 15 | #ifndef CRUISE_SYSIO_H 16 | #define CRUISE_SYSIO_H 17 | 18 | #include "cruise-internal.h" 19 | 20 | /* read count bytes info buf from file starting at offset pos, 21 | * returns number of bytes actually read in retcount, 22 | * retcount will be less than count only if an error occurs 23 | * or end of file is reached */ 24 | int cruise_fd_read(int fd, off_t pos, void* buf, size_t count, size_t* retcount); 25 | 26 | /* write count bytes from buf into file starting at offset pos, 27 | * allocates new bytes and updates file size as necessary, 28 | * fills any gaps with zeros */ 29 | int cruise_fd_write(int fd, off_t pos, const void* buf, size_t count); 30 | 31 | #endif /* CRUISE_SYSIO_H */ 32 | -------------------------------------------------------------------------------- /src/cruise.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Written by 5 | * Raghunath Rajachandrasekar 6 | * Kathryn Mohror 7 | * Adam Moody 8 | * LLNL-CODE-642432. 9 | * All rights reserved. 10 | * This file is part of CRUISE. 11 | * For details, see https://github.com/hpc/cruise 12 | * Please also read this file COPYRIGHT 13 | */ 14 | 15 | #ifndef CRUISE_H 16 | #define CRUISE_H 17 | 18 | /* TODO: namespace C */ 19 | 20 | /* linked list of chunk information given to an external library wanting 21 | * to RDMA out a file from CRUISE */ 22 | typedef struct { 23 | off_t chunk_id; 24 | int location; 25 | void *chunk_mr; 26 | off_t spillover_offset; 27 | struct chunk_list_t *next; 28 | } chunk_list_t; 29 | 30 | /* mount memfs at some prefix location */ 31 | int cruise_mount(const char prefix[], size_t size, int rank); 32 | 33 | /* get information about the chunk data region 34 | * for external async libraries to register during their init */ 35 | size_t cruise_get_data_region(void **ptr); 36 | 37 | /* get a list of chunks for a given file (useful for RDMA, etc.) */ 38 | chunk_list_t* cruise_get_chunk_list(char* path); 39 | 40 | /* debug function to print list of chunks constituting a file 41 | * and to test above function*/ 42 | void cruise_print_chunk_list(char* path); 43 | 44 | #endif /* CRUISE_H */ 45 | -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- 1 | MPICC = mpicc #mpiicc 2 | CC = gcc #icc 3 | 4 | CFLAGS = -g -O0 5 | LIBS = -lpthread 6 | 7 | NUMA_LIBS = -lnuma 8 | 9 | CRUISE_LIBS = $(LIBS) $(NUMA_LIBS) $(CONTAINER_LIBS) 10 | CRUISE_LDFLAGS = $(LDFLAGS) $(CONTAINER_LDFLAGS) 11 | 12 | PRE_CRUISE_FLAGS := $(shell echo `../install/bin/cruise-config --pre-ld-flags`) 13 | POST_CRUISE_FLAGS := $(shell echo `../install/bin/cruise-config --post-ld-flags`) 14 | 15 | all: test_writeread test_truncate test_fopen test_fprintf test_ungetc test_scanf test_wscanf test_memcpy test_ramdisk test1 16 | 17 | clean: 18 | rm -f *.o test1 test1_container test_writeread test_truncate test_fopen test_fprintf test_ungetc test_scanf test_wscanf test_memcpy test_ramdisk 19 | 20 | test1: test1.c 21 | $(CC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test1.c -o test1 $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 22 | 23 | test_writeread: test_writeread.c 24 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_writeread.c -o test_writeread $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 25 | 26 | test_truncate: test_truncate.c 27 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_truncate.c -o test_truncate $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 28 | 29 | test_fopen: test_fopen.c 30 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_fopen.c -o test_fopen $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 31 | 32 | test_fprintf: test_fprintf.c 33 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_fprintf.c -o test_fprintf $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 34 | 35 | test_ungetc: test_ungetc.c 36 | $(CC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_ungetc.c -o test_ungetc $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 37 | 38 | test_scanf: test_scanf.c 39 | $(CC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_scanf.c -o test_scanf $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 40 | 41 | test_wscanf: test_wscanf.c 42 | $(CC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_wscanf.c -o test_wscanf $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 43 | 44 | test_memcpy: test_memcpy.c 45 | $(MPICC) $(CFLAGS) $(INCLUDES) test_memcpy.c -o test_memcpy $(LDFLAGS) $(LIBS) 46 | 47 | test_ramdisk: test_ramdisk.c 48 | $(MPICC) $(CFLAGS) $(INCLUDES) test_ramdisk.c -o test_ramdisk $(LDFLAGS) $(LIBS) 49 | 50 | -------------------------------------------------------------------------------- /tests/makefile.bgq: -------------------------------------------------------------------------------- 1 | # Intel compilers 2 | #MPICC = mpiicc 3 | #CC = icc 4 | 5 | MPICC = mpixlc 6 | CC = xlc 7 | 8 | #ZLIB = -L/usr/local/tools/zlib-1.2.6/lib -lz 9 | #ZLIB = -L../zlib-1.2.7/install/lib -lz 10 | 11 | # GNU compilers 12 | #MPICC = mpicc 13 | #CC = gcc 14 | 15 | CFLAGS = -g -O0 16 | LIBS = -lpthread 17 | 18 | NUMA_HOME = 19 | NUMA_INCS = 20 | NUMA_LDFLAGS = 21 | NUMA_LIBS = 22 | #NUMA_HOME = ../numactl 23 | #NUMA_INCS = -I$(NUMA_HOME) 24 | #NUMA_LDFLAGS = -L$(NUMA_HOME) 25 | #NUMA_LIBS = -lnuma 26 | 27 | CONTAINER_HOME = 28 | CONTAINER_INCS = 29 | CONTAINER_LDFLAGS = 30 | CONTAINER_LIBS = 31 | #CONTAINER_HOME = ../container 32 | #CONTAINER_INCS = -I$(CONTAINER_HOME)/include 33 | #CONTAINER_LDFLAGS = -L$(CONTAINER_HOME)/lib 34 | #CONTAINER_LIBS = -lcontainer 35 | 36 | CRUISE_LIBS = $(LIBS) $(NUMA_LIBS) $(CONTAINER_LIBS) 37 | CRUISE_LDFLAGS = $(LDFLAGS) $(NUMA_LDFLAGS) $(CONTAINER_LDFLAGS) 38 | 39 | PRE_CRUISE_FLAGS := $(shell echo `../install_bgq/bin/cruise-config --pre-ld-flags`) 40 | POST_CRUISE_FLAGS := $(shell echo `../install_bgq/bin/cruise-config --post-ld-flags`) 41 | 42 | all: test_writeread test_readwrite test_truncate test_fopen test_fprintf test_memcpy test_ramdisk 43 | 44 | clean: 45 | rm -f *.o test1 test1_container test_writeread test_readwrite test_truncate test_fopen test_fprintf test_ungetc test_scanf test_wscanf test_memcpy test_ramdisk 46 | 47 | test1: test1.c 48 | $(CC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test1.c -o test1 $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 49 | 50 | test_writeread: test_writeread.c 51 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_writeread.c -o test_writeread $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 52 | 53 | test_readwrite: test_readwrite.c 54 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_readwrite.c -o test_readwrite $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 55 | 56 | test_truncate: test_truncate.c 57 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_truncate.c -o test_truncate $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 58 | 59 | test_fopen: test_fopen.c 60 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_fopen.c -o test_fopen $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 61 | 62 | test_fprintf: test_fprintf.c 63 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_fprintf.c -o test_fprintf $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 64 | 65 | test_ungetc: test_ungetc.c 66 | $(CC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_ungetc.c -o test_ungetc $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 67 | 68 | test_scanf: test_scanf.c 69 | $(CC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_scanf.c -o test_scanf $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 70 | 71 | test_wscanf: test_wscanf.c 72 | $(CC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_wscanf.c -o test_wscanf $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 73 | 74 | test_memcpy: test_memcpy.c 75 | $(MPICC) $(CFLAGS) $(INCLUDES) test_memcpy.c -o test_memcpy $(LDFLAGS) $(LIBS) $(ZLIB) 76 | 77 | test_ramdisk: test_ramdisk.c 78 | $(MPICC) $(CFLAGS) $(INCLUDES) test_ramdisk.c -o test_ramdisk $(LDFLAGS) $(LIBS) $(ZLIB) 79 | 80 | -------------------------------------------------------------------------------- /tests/test1.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #define UNUSEDFD 5000 10 | 11 | #define CRUISE_STATUS 12 | #ifdef CRUISE_STATUS 13 | #define status(fmt, args... ) printf("STATUS: %d %s, line %d: "fmt, testnum, \ 14 | __func__, __LINE__,##args) 15 | #else 16 | #define status(fmt, args... ) 17 | #endif 18 | 19 | #define CRUISE_ERROR 20 | #ifdef CRUISE_ERROR 21 | #define error(fmt, args... ) printf("ERROR: %d %s, line %d: "fmt, testnum, \ 22 | __func__, __LINE__, ##args) 23 | #else 24 | #define error(fmt, args... ) 25 | #endif 26 | 27 | #define CHECK(a) a; if (rc < 0) return rc; 28 | 29 | #define TEST(a) ++testnum; a; 30 | 31 | #define TESTFAIL(f,a) ++testnum; f = a; \ 32 | if (f >= 0){ error("test %d failed\n", testnum); return -1;} \ 33 | else {status("test %d succeeded\n", testnum);} 34 | 35 | #define TESTFAILERR(f,a,err) ++testnum; f = a; \ 36 | if (f >= 0 || err != errno){ error("test %d failed\n", testnum); return -1;} \ 37 | else {status("test %d succeeded\n", testnum);} 38 | 39 | #define TESTSUCC(f,a) ++testnum; f = a; \ 40 | if (f < 0) { error("test %d failed\n", testnum); return -1;} \ 41 | else {status("test %d succeeded\n", testnum);} 42 | 43 | int testnum = 0; 44 | 45 | int test1(); 46 | 47 | int main(int argc, char ** argv){ 48 | int rc; 49 | cruise_mount("/tmp", 4096 * 4096, 0); 50 | CHECK(rc = test_open()); 51 | CHECK(rc = test_close()); 52 | CHECK(rc = test_unlink()); 53 | CHECK(rc = test_mkdir()); 54 | CHECK(rc = test_rmdir()); 55 | CHECK(rc = test_stat()); 56 | CHECK(rc = test_access()); 57 | CHECK(rc = test_write()); 58 | CHECK(rc = test_read()); 59 | //seek 60 | return 0; 61 | } 62 | 63 | 64 | 65 | int test_open(){ 66 | char afile[20] = "/tmp/file1.txt"; 67 | char adir[20] = "/tmp/somewhere"; 68 | int fd; 69 | int fd1; 70 | 71 | 72 | /* open a file that does not exist without create flag 73 | * should fail */ 74 | TESTFAILERR(fd,open(afile, O_WRONLY), ENOENT); 75 | //TESTFAILERR(fd,open(afile, O_WRONLY), -1); 76 | 77 | /* open a file that does not exist with create flag 78 | * should succeed */ 79 | TESTSUCC(fd1,open(afile, O_CREAT)); 80 | 81 | /* open a file that already exists with create and exlusive flags 82 | * should fail */ 83 | TESTFAILERR(fd,open(afile, O_CREAT|O_EXCL), EEXIST); 84 | 85 | /* open a file that already exists with create flag 86 | * should succeed */ 87 | TESTSUCC(fd,open(afile, O_CREAT)); 88 | if (fd != fd1){ 89 | error("open of existing file did not return right file desc, fd=%d\n", fd); 90 | return -1; 91 | } 92 | 93 | /* test opening directory without O_DIRECTORY 94 | * should fail */ 95 | mkdir(adir,S_IRWXU); 96 | TESTFAILERR(fd, open(adir, O_WRONLY), ENOTDIR); 97 | 98 | /* test opening directory with O_DIRECTORY 99 | * should succeed */ 100 | TESTSUCC(fd, open(adir, O_WRONLY|O_DIRECTORY)); 101 | 102 | /* test opening regular file with O_DIRECTORY 103 | * should fail */ 104 | TESTFAILERR(fd, open(afile, O_WRONLY|O_DIRECTORY), ENOTDIR); 105 | 106 | close(fd); 107 | close(fd1); 108 | unlink(afile); 109 | rmdir(adir); 110 | 111 | return 1; 112 | } 113 | 114 | int test_close(){ 115 | char afile[20] = "/tmp/file1.txt"; 116 | int fd; 117 | int ret; 118 | 119 | /* close a file that does not exist 120 | * should fail */ 121 | TESTFAILERR(ret, close(UNUSEDFD), EBADF); 122 | 123 | /* test that closing an existing file works 124 | * should succeed */ 125 | TESTSUCC(fd, open(afile, O_CREAT)); 126 | TESTSUCC(ret, close(fd)); 127 | 128 | unlink(afile); 129 | 130 | /* TODO: test that close of deleted file fails 131 | * not in here now because not implemented in library */ 132 | return 1; 133 | } 134 | 135 | 136 | int test_unlink(){ 137 | char afile[20] = "/tmp/file1.txt"; 138 | char adir[20] = "/tmp/somewhere"; 139 | int fd; 140 | int ret; 141 | 142 | /* try to remove non-existent file 143 | * should fail */ 144 | TESTFAILERR(ret, unlink(afile), ENOENT); 145 | 146 | /* try to remove an existing file 147 | * we're not checking to see if the file is busy/open, so 148 | * should succeed */ 149 | fd = open(afile, O_CREAT); 150 | TESTSUCC(ret, unlink(afile)); 151 | 152 | close(fd); 153 | unlink(afile); 154 | 155 | /* try to remove a directory 156 | * should fail */ 157 | mkdir(adir, S_IRWXU); 158 | TESTFAILERR(ret, unlink(adir), EISDIR); 159 | rmdir(adir); 160 | 161 | return 1; 162 | } 163 | 164 | int test_mkdir(){ 165 | 166 | char adir[20] = "/tmp/somewhere"; 167 | int ret; 168 | 169 | /* test making a directory 170 | * should succeed */ 171 | TESTSUCC(ret,mkdir(adir,S_IRWXU)); 172 | 173 | /* try to make it again 174 | * should fail */ 175 | TESTFAILERR(ret,mkdir(adir,S_IRWXU), EEXIST); 176 | 177 | rmdir(adir); 178 | 179 | return 1; 180 | } 181 | 182 | int test_rmdir(){ 183 | 184 | char adir[20] = "/tmp/somewhere"; 185 | int ret; 186 | 187 | /* try to delete a directory that doesn't exist 188 | * should fail */ 189 | TESTFAILERR(ret,rmdir(adir), ENOENT); 190 | 191 | /* try to remove a direcotry that exists 192 | * should succeed */ 193 | mkdir(adir, S_IRWXU); 194 | TESTSUCC(ret, rmdir(adir)); 195 | 196 | return 1; 197 | } 198 | 199 | int test_stat(){ 200 | char afile[20] = "/tmp/afile"; 201 | char adir[20] = "/tmp/somewhere"; 202 | char buf[1000]; 203 | int count = 1000; 204 | int fd; 205 | int ret; 206 | struct stat statbuf; 207 | 208 | /* call stat for non-existent file 209 | * should fail */ 210 | TESTFAILERR(ret, stat(afile, &statbuf), ENOENT); 211 | 212 | /* open a file, stat it, size should be 0 213 | * should succeed */ 214 | fd = open(afile, O_CREAT); 215 | TESTSUCC(ret, stat(afile, &statbuf)); 216 | TESTSUCC(ret, statbuf.st_size == 0? 1: -1); 217 | 218 | /* write to file, stat, size should be count 219 | * should succeed */ 220 | write(fd, buf, count); 221 | TESTSUCC(ret, stat(afile, &statbuf)); 222 | TESTSUCC(ret, statbuf.st_size == count? 1: -1); 223 | 224 | /* check to see that this is a regular file */ 225 | TESTSUCC(ret, S_ISREG(statbuf.st_mode)==1? 1:-1); 226 | /* check that this is not a directory */ 227 | TESTSUCC(ret, S_ISDIR(statbuf.st_mode)==0? 1:-1); 228 | 229 | /* make a directory and make sure that ISDIR 230 | * reports is a directory */ 231 | mkdir(adir,S_IRWXU); 232 | TESTSUCC(ret, stat(adir, &statbuf)); 233 | TESTSUCC(ret, S_ISDIR(statbuf.st_mode)==1? 1:-1); 234 | TESTSUCC(ret, S_ISREG(statbuf.st_mode)==0? 1:-1); 235 | 236 | 237 | close(fd); 238 | unlink(afile); 239 | rmdir(adir); 240 | return 1; 241 | } 242 | 243 | int test_access(){ 244 | char afile[20] = "/tmp/afile"; 245 | int fd; 246 | 247 | /* test if we can access a non-existent file 248 | * should fail */ 249 | TESTFAILERR(fd, access(afile, R_OK), ENOENT); 250 | TESTFAILERR(fd, access(afile, W_OK), ENOENT); 251 | TESTFAILERR(fd, access(afile, X_OK), ENOENT); 252 | TESTFAILERR(fd, access(afile, F_OK), ENOENT); 253 | 254 | /* create the file, see if we can access it 255 | * since we are not checking permissions at this time, all these 256 | * should succeed */ 257 | fd = open(afile, O_CREAT); 258 | TESTSUCC(fd, access(afile, F_OK)); 259 | TESTSUCC(fd, access(afile, R_OK)); 260 | TESTSUCC(fd, access(afile, W_OK)); 261 | TESTSUCC(fd, access(afile, X_OK)); 262 | 263 | close(fd); 264 | unlink(afile); 265 | 266 | return 1; 267 | } 268 | 269 | 270 | int test_write(){ 271 | char afile[20] = "/tmp/writefile"; 272 | char adir[20] = "/tmp/somewhere"; 273 | char buf[1000]; 274 | int count = 1000; 275 | int fd; 276 | int ret; 277 | 278 | /* try to write to a file that doesn't exist 279 | * should fail */ 280 | TESTFAILERR(ret, write(UNUSEDFD, buf, count), EBADF); 281 | 282 | /* write to an existing file 283 | * should succeed */ 284 | TESTSUCC(fd, open(afile, O_CREAT)); 285 | TESTSUCC(ret, write(fd, buf, count)); 286 | /* should write count bytes */ 287 | TESTSUCC(ret, ret == count? 1: -1); 288 | close(fd); 289 | unlink(afile); 290 | 291 | /* try to write to a directory 292 | * should fail */ 293 | mkdir(adir,S_IRWXU); 294 | fd = open(adir, O_DIRECTORY); 295 | TESTFAILERR(ret, write(fd, buf, count), EINVAL); 296 | 297 | close(fd); 298 | rmdir(adir); 299 | 300 | return 1; 301 | } 302 | 303 | int test_read(){ 304 | char afile[20] = "/tmp/afile"; 305 | char adir[20] = "/tmp/somewhere"; 306 | char buf[1000]; 307 | int count = 1000; 308 | int fd; 309 | int ret; 310 | 311 | 312 | /* try to read a non-existent file 313 | * should fail */ 314 | TESTFAILERR(ret, read(UNUSEDFD, buf, count), EBADF); 315 | 316 | /* try to read from an existent file with 0 bytes 317 | * should succeed, should read 0 bytes */ 318 | fd = open(afile, O_CREAT); 319 | ret = read(fd, buf, count); 320 | TESTSUCC(ret, read(fd, buf, count)); 321 | TESTSUCC(ret, 0==ret? 1:-1); 322 | 323 | /* test reading from a file with bytes 324 | * should succeed */ 325 | TESTSUCC(ret, write(fd, buf, count)); 326 | /* should write count bytes */ 327 | TESTSUCC(ret, ret == count? 1: -1); 328 | TESTSUCC(ret, read(fd, buf, count)); 329 | /* should read 0 bytes since at the end of the file */ 330 | TESTSUCC(ret, 0==ret? 1:-1); 331 | /* seek to beginning and read, should get count bytes */ 332 | lseek(fd,0, SEEK_SET); 333 | TESTSUCC(ret, read(fd, buf, count)); 334 | TESTSUCC(ret, count==ret? 1:-1); 335 | 336 | /* try to read a directory 337 | * should fail */ 338 | mkdir(adir,S_IRWXU); 339 | fd = open(adir, O_DIRECTORY); 340 | TESTFAILERR(ret, read(fd, buf, count), EISDIR); 341 | 342 | close(fd); 343 | rmdir(adir); 344 | 345 | return 1; 346 | } 347 | -------------------------------------------------------------------------------- /tests/test_fopen.c: -------------------------------------------------------------------------------- 1 | // build: mpigcc -g -O3 -o test_ramdisk test_ramdisk.c 2 | // run: srun -n64 -N4 ./test_ramdisk 3 | 4 | #define _GNU_SOURCE 1 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "mpi.h" 15 | 16 | size_t filesize = 100*1024*1024; 17 | int times = 5; 18 | int seconds = 0; 19 | int rank = -1; 20 | int ranks = 0; 21 | 22 | /* reliable read from file descriptor (retries, if necessary, until hard error) */ 23 | int reliable_read(FILE* fp, void* buf, size_t size) 24 | { 25 | size_t n = 0; 26 | int retries = 10; 27 | int rank; 28 | char host[128]; 29 | while (n < size) 30 | { 31 | int rc = fread((char*) buf + n, 1, size - n, fp); 32 | if (rc > 0) { 33 | n += rc; 34 | } else { 35 | if (feof(fp)) { 36 | /* EOF */ 37 | return n; 38 | } 39 | 40 | /* got an error, check whether it was serious */ 41 | if(errno == EINTR || errno == EAGAIN) { 42 | continue; 43 | } 44 | 45 | /* something worth printing an error about */ 46 | retries--; 47 | if (retries) { 48 | /* print an error and try again */ 49 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 50 | gethostname(host, sizeof(host)); 51 | printf("%d on %s: ERROR: Error reading errno=%d @ %s:%d\n", 52 | rank, host, errno, __FILE__, __LINE__ 53 | ); 54 | } else { 55 | /* too many failed retries, give up */ 56 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 57 | gethostname(host, sizeof(host)); 58 | printf("%d on %s: ERROR: Giving up read errno=%d @ %s:%d\n", 59 | rank, host, errno, __FILE__, __LINE__ 60 | ); 61 | MPI_Abort(MPI_COMM_WORLD, 0); 62 | } 63 | } 64 | } 65 | return size; 66 | } 67 | 68 | /* reliable write to file descriptor (retries, if necessary, until hard error) */ 69 | int reliable_write(FILE* fp, const void* buf, size_t size) 70 | { 71 | size_t n = 0; 72 | int retries = 10; 73 | int rank; 74 | char host[128]; 75 | while (n < size) 76 | { 77 | size_t rc = fwrite((char*) buf + n, 1, size - n, fp); 78 | if (rc > 0) { 79 | n += rc; 80 | } else { 81 | /* got an error, check whether it was serious */ 82 | if(errno == EINTR || errno == EAGAIN) { 83 | continue; 84 | } 85 | 86 | /* something worth printing an error about */ 87 | retries--; 88 | if (retries) { 89 | /* print an error and try again */ 90 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 91 | gethostname(host, sizeof(host)); 92 | printf("%d on %s: ERROR: Error writing: errno=%d @ %s:%d\n", 93 | rank, host, errno, __FILE__, __LINE__ 94 | ); 95 | } else { 96 | /* too many failed retries, give up */ 97 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 98 | gethostname(host, sizeof(host)); 99 | printf("%d on %s: ERROR: Giving up write: errno=%d @ %s:%d\n", 100 | rank, host, errno, __FILE__, __LINE__ 101 | ); 102 | MPI_Abort(MPI_COMM_WORLD, 0); 103 | } 104 | } 105 | } 106 | return size; 107 | } 108 | 109 | /* initialize buffer with some well-known value based on rank */ 110 | int init_buffer(char* buf, size_t size, int rank, int ckpt) 111 | { 112 | size_t i; 113 | for(i=0; i < size; i++) { 114 | char c = 'a' + (char)((rank + ckpt + i) & 32); 115 | buf[i] = c; 116 | } 117 | return 0; 118 | } 119 | 120 | /* checks buffer for expected value */ 121 | int check_buffer(char* buf, size_t size, int rank, int ckpt) 122 | { 123 | size_t i; 124 | for(i=0; i < size; i++) { 125 | char c = 'a' + (char)((rank + ckpt + i) & 32); 126 | if (buf[i] != c) { 127 | return 0; 128 | } 129 | } 130 | return 1; 131 | } 132 | 133 | /* read the checkpoint data from file into buf, and return whether the read was successful */ 134 | int read_checkpoint(FILE* fp, int* rank, int* ckpt, char* buf, size_t size) 135 | { 136 | unsigned long n; 137 | char rank_buf[7]; 138 | char ckpt_buf[7]; 139 | size_t field_size = 6; 140 | 141 | /* read the rank id */ 142 | n = reliable_read(fp, rank_buf, field_size); 143 | if (n != field_size) { 144 | printf("Failed to read rank\n"); 145 | return 0; 146 | } 147 | rank_buf[6] = '\0'; 148 | 149 | /* read the checkpoint id */ 150 | n = reliable_read(fp, ckpt_buf, field_size); 151 | if (n != field_size) { 152 | printf("Failed to read timestep\n"); 153 | return 0; 154 | } 155 | ckpt_buf[6] = '\0'; 156 | 157 | /* read the checkpoint data, and check the file size */ 158 | n = reliable_read(fp, buf, size+1); 159 | if (n != size) { 160 | printf("Filesize not correct\n"); 161 | return 0; 162 | } 163 | 164 | /* if the file looks good, set the timestep and return */ 165 | sscanf(rank_buf, "%06d", rank); 166 | sscanf(ckpt_buf, "%06d", ckpt); 167 | 168 | return 0; 169 | } 170 | 171 | /* write the checkpoint data to fp, and return whether the write was successful */ 172 | int write_checkpoint(FILE* fp, int rank, int ckpt, char* buf, size_t size) 173 | { 174 | int rc; 175 | int valid = 0; 176 | char rank_buf[7]; 177 | char ckpt_buf[7]; 178 | size_t field_size = 6; 179 | 180 | /* write the rank id */ 181 | sprintf(rank_buf, "%06d", rank); 182 | rc = reliable_write(fp, rank_buf, field_size); 183 | if (rc < 0) { 184 | valid = 0; 185 | } 186 | 187 | /* write the checkpoint id (application timestep) */ 188 | sprintf(ckpt_buf, "%06d", ckpt); 189 | rc = reliable_write(fp, ckpt_buf, field_size); 190 | if (rc < 0) { 191 | valid = 0; 192 | } 193 | 194 | /* write the checkpoint data */ 195 | rc = reliable_write(fp, buf, size); 196 | if (rc < 0) { 197 | valid = 0; 198 | } 199 | 200 | return valid; 201 | } 202 | 203 | void checkdata(char* file, size_t size, int times) 204 | { 205 | char* buf = malloc(size); 206 | 207 | MPI_Barrier(MPI_COMM_WORLD); 208 | 209 | if (times > 0) { 210 | /* write the checkpoint file */ 211 | int i, j; 212 | for(i=0; i < times; i++) { 213 | int rc; 214 | int valid = 0; 215 | 216 | rc = init_buffer(buf, size, rank, i); 217 | 218 | if (rank == 0) { 219 | printf("Writing checkpoint %d.\n", i); fflush(stdout); 220 | } 221 | 222 | /* open the file and write the checkpoint */ 223 | FILE* fp = fopen(file, "w"); 224 | if (fp != NULL) { 225 | valid = 1; 226 | 227 | /* write the checkpoint data */ 228 | rc = write_checkpoint(fp, rank, i, buf, size); 229 | if (rc < 0) { 230 | valid = 0; 231 | } 232 | 233 | /* force the data to storage */ 234 | rc = fflush(fp); 235 | if (rc != 0) { 236 | valid = 0; 237 | } 238 | 239 | /* make sure the close is without error */ 240 | rc = fclose(fp); 241 | if (rc != 0) { 242 | valid = 0; 243 | } 244 | } 245 | 246 | if (rank == 0) { 247 | printf("Completed checkpoint %d.\n", i); fflush(stdout); 248 | } 249 | 250 | if (rank == 0) { 251 | printf("Reading checkpoint %d.\n", i); fflush(stdout); 252 | } 253 | 254 | memset(buf, 0, size); 255 | 256 | /* open the file and write the checkpoint */ 257 | int read_rank, read_timestep; 258 | fp = fopen(file, "r"); 259 | if (fp != NULL) { 260 | valid = 1; 261 | 262 | /* write the checkpoint data */ 263 | rc = read_checkpoint(fp, &read_rank, &read_timestep, buf, size); 264 | if (rc < 0) { 265 | valid = 0; 266 | } 267 | 268 | /* make sure the close is without error */ 269 | rc = fclose(fp); 270 | if (rc != 0) { 271 | valid = 0; 272 | } 273 | 274 | } 275 | 276 | if (read_rank != rank || read_timestep != i) { 277 | printf("INVALID HEADER on rank %d in step %d\n", rank, i); fflush(stdout); 278 | MPI_Abort(MPI_COMM_WORLD, 0); 279 | } 280 | 281 | rc = check_buffer(buf, size, rank, i); 282 | if (! rc) { 283 | printf("INVALID DATA on rank %d in step %d\n", rank, i); fflush(stdout); 284 | MPI_Abort(MPI_COMM_WORLD, 0); 285 | } 286 | 287 | if (rank == 0) { 288 | printf("Verified checkpoint %d.\n", read_timestep); fflush(stdout); 289 | } 290 | 291 | /* optionally sleep for some time */ 292 | if (seconds > 0) { 293 | if (rank == 0) { 294 | printf("Sleeping for %d seconds... \n", seconds); fflush(stdout); 295 | } 296 | sleep(seconds); 297 | } 298 | 299 | remove(file); 300 | } 301 | } 302 | 303 | MPI_Barrier(MPI_COMM_WORLD); 304 | 305 | if (buf != NULL) { 306 | free(buf); 307 | buf = NULL; 308 | } 309 | 310 | return; 311 | } 312 | 313 | int main (int argc, char* argv[]) 314 | { 315 | /* check that we got an appropriate number of arguments */ 316 | if (argc != 1 && argc != 4) { 317 | printf("Usage: test_correctness [filesize times sleep_secs]\n"); 318 | return 1; 319 | } 320 | 321 | /* read parameters from command line, if any */ 322 | if (argc > 1) { 323 | filesize = (size_t) atol(argv[1]); 324 | times = atoi(argv[2]); 325 | seconds = atoi(argv[3]); 326 | } 327 | 328 | MPI_Init(&argc, &argv); 329 | 330 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 331 | MPI_Comm_size(MPI_COMM_WORLD, &ranks); 332 | 333 | char name[256]; 334 | sprintf(name, "/tmp/rank.%d", rank); 335 | 336 | /* allocate space for the checkpoint data (make filesize a function of rank for some variation) */ 337 | filesize = filesize + rank; 338 | 339 | cruise_mount("/tmp", filesize, rank); 340 | 341 | /* verify data integrity in file */ 342 | checkdata(name, filesize, times); 343 | 344 | MPI_Finalize(); 345 | 346 | return 0; 347 | } 348 | -------------------------------------------------------------------------------- /tests/test_fprintf.c: -------------------------------------------------------------------------------- 1 | // build: mpigcc -g -O3 -o test_ramdisk test_ramdisk.c 2 | // run: srun -n64 -N4 ./test_ramdisk 3 | 4 | #define _GNU_SOURCE 1 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "mpi.h" 15 | 16 | size_t filesize = 100*1024*1024; 17 | int times = 5; 18 | int seconds = 0; 19 | int rank = -1; 20 | int ranks = 0; 21 | 22 | /* reliable read from file descriptor (retries, if necessary, until hard error) */ 23 | int reliable_read(FILE* fp, void* buf, size_t size) 24 | { 25 | size_t n = 0; 26 | int retries = 10; 27 | int rank; 28 | char host[128]; 29 | while (n < size) 30 | { 31 | int rc = fread((char*) buf + n, 1, size - n, fp); 32 | if (rc > 0) { 33 | n += rc; 34 | } else { 35 | if (feof(fp)) { 36 | /* EOF */ 37 | return n; 38 | } 39 | 40 | /* got an error, check whether it was serious */ 41 | if(errno == EINTR || errno == EAGAIN) { 42 | continue; 43 | } 44 | 45 | /* something worth printing an error about */ 46 | retries--; 47 | if (retries) { 48 | /* print an error and try again */ 49 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 50 | gethostname(host, sizeof(host)); 51 | printf("%d on %s: ERROR: Error reading errno=%d @ %s:%d\n", 52 | rank, host, errno, __FILE__, __LINE__ 53 | ); 54 | } else { 55 | /* too many failed retries, give up */ 56 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 57 | gethostname(host, sizeof(host)); 58 | printf("%d on %s: ERROR: Giving up read errno=%d @ %s:%d\n", 59 | rank, host, errno, __FILE__, __LINE__ 60 | ); 61 | MPI_Abort(MPI_COMM_WORLD, 0); 62 | } 63 | } 64 | } 65 | return size; 66 | } 67 | 68 | /* reliable write to file descriptor (retries, if necessary, until hard error) */ 69 | int reliable_write(FILE* fp, const void* buf, size_t size) 70 | { 71 | size_t n = 0; 72 | int retries = 10; 73 | int rank; 74 | char host[128]; 75 | while (n < size) 76 | { 77 | size_t rc = fwrite((char*) buf + n, 1, size - n, fp); 78 | if (rc > 0) { 79 | n += rc; 80 | } else { 81 | /* got an error, check whether it was serious */ 82 | if(errno == EINTR || errno == EAGAIN) { 83 | continue; 84 | } 85 | 86 | /* something worth printing an error about */ 87 | retries--; 88 | if (retries) { 89 | /* print an error and try again */ 90 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 91 | gethostname(host, sizeof(host)); 92 | printf("%d on %s: ERROR: Error writing: errno=%d @ %s:%d\n", 93 | rank, host, errno, __FILE__, __LINE__ 94 | ); 95 | } else { 96 | /* too many failed retries, give up */ 97 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 98 | gethostname(host, sizeof(host)); 99 | printf("%d on %s: ERROR: Giving up write: errno=%d @ %s:%d\n", 100 | rank, host, errno, __FILE__, __LINE__ 101 | ); 102 | MPI_Abort(MPI_COMM_WORLD, 0); 103 | } 104 | } 105 | } 106 | return size; 107 | } 108 | 109 | /* initialize buffer with some well-known value based on rank */ 110 | int init_buffer(char* buf, size_t size, int rank, int ckpt) 111 | { 112 | size_t i; 113 | for(i=0; i < size; i++) { 114 | char c = 'a' + (char)((rank + ckpt + i) & 32); 115 | buf[i] = c; 116 | } 117 | return 0; 118 | } 119 | 120 | /* checks buffer for expected value */ 121 | int check_buffer(char* buf, size_t size, int rank, int ckpt) 122 | { 123 | size_t i; 124 | for(i=0; i < size; i++) { 125 | char c = 'a' + (char)((rank + ckpt + i) & 32); 126 | if (buf[i] != c) { 127 | return 0; 128 | } 129 | } 130 | return 1; 131 | } 132 | 133 | /* read the checkpoint data from file into buf, and return whether the read was successful */ 134 | int read_checkpoint(FILE* fp, int* rank, int* ckpt, char* buf, size_t size) 135 | { 136 | unsigned long n; 137 | char rank_buf[7]; 138 | char ckpt_buf[7]; 139 | size_t field_size = 6; 140 | 141 | /* read the rank id */ 142 | n = reliable_read(fp, rank_buf, field_size); 143 | if (n != field_size) { 144 | printf("Failed to read rank\n"); 145 | return 0; 146 | } 147 | rank_buf[6] = '\0'; 148 | 149 | /* read the checkpoint id */ 150 | n = reliable_read(fp, ckpt_buf, field_size); 151 | if (n != field_size) { 152 | printf("Failed to read timestep\n"); 153 | return 0; 154 | } 155 | ckpt_buf[6] = '\0'; 156 | 157 | /* read the checkpoint data, and check the file size */ 158 | n = reliable_read(fp, buf, size+1); 159 | if (n != size) { 160 | printf("Filesize not correct\n"); 161 | return 0; 162 | } 163 | 164 | /* if the file looks good, set the timestep and return */ 165 | sscanf(rank_buf, "%06d", rank); 166 | sscanf(ckpt_buf, "%06d", ckpt); 167 | 168 | return 0; 169 | } 170 | 171 | /* write the checkpoint data to fp, and return whether the write was successful */ 172 | int write_checkpoint(FILE* fp, int rank, int ckpt, char* buf, size_t size) 173 | { 174 | int rc; 175 | int valid = 0; 176 | char rank_buf[7]; 177 | char ckpt_buf[7]; 178 | size_t field_size = 6; 179 | 180 | /* write the rank id */ 181 | rc = fprintf(fp, "%06d", rank); 182 | if (rc < 0) { 183 | valid = 0; 184 | } 185 | 186 | /* write the checkpoint id (application timestep) */ 187 | rc = fprintf(fp, "%06d", ckpt); 188 | if (rc < 0) { 189 | valid = 0; 190 | } 191 | 192 | /* write the checkpoint data */ 193 | rc = reliable_write(fp, buf, size); 194 | if (rc < 0) { 195 | valid = 0; 196 | } 197 | 198 | return valid; 199 | } 200 | 201 | void checkdata(char* file, size_t size, int times) 202 | { 203 | char* buf = malloc(size); 204 | 205 | MPI_Barrier(MPI_COMM_WORLD); 206 | 207 | if (times > 0) { 208 | /* write the checkpoint file */ 209 | int i, j; 210 | for(i=0; i < times; i++) { 211 | int rc; 212 | int valid = 0; 213 | 214 | rc = init_buffer(buf, size, rank, i); 215 | 216 | if (rank == 0) { 217 | printf("Writing checkpoint %d.\n", i); fflush(stdout); 218 | } 219 | 220 | /* open the file and write the checkpoint */ 221 | FILE* fp = fopen(file, "w"); 222 | if (fp != NULL) { 223 | valid = 1; 224 | 225 | /* write the checkpoint data */ 226 | rc = write_checkpoint(fp, rank, i, buf, size); 227 | if (rc < 0) { 228 | valid = 0; 229 | } 230 | 231 | /* force the data to storage */ 232 | rc = fflush(fp); 233 | if (rc != 0) { 234 | valid = 0; 235 | } 236 | 237 | /* make sure the close is without error */ 238 | rc = fclose(fp); 239 | if (rc != 0) { 240 | valid = 0; 241 | } 242 | } 243 | 244 | if (rank == 0) { 245 | printf("Completed checkpoint %d.\n", i); fflush(stdout); 246 | } 247 | 248 | if (rank == 0) { 249 | printf("Reading checkpoint %d.\n", i); fflush(stdout); 250 | } 251 | 252 | memset(buf, 0, size); 253 | 254 | /* open the file and write the checkpoint */ 255 | int read_rank, read_timestep; 256 | fp = fopen(file, "r"); 257 | if (fp != NULL) { 258 | valid = 1; 259 | 260 | /* write the checkpoint data */ 261 | rc = read_checkpoint(fp, &read_rank, &read_timestep, buf, size); 262 | if (rc < 0) { 263 | valid = 0; 264 | } 265 | 266 | /* make sure the close is without error */ 267 | rc = fclose(fp); 268 | if (rc != 0) { 269 | valid = 0; 270 | } 271 | 272 | } 273 | 274 | if (read_rank != rank || read_timestep != i) { 275 | printf("INVALID HEADER on rank %d in step %d\n", rank, i); fflush(stdout); 276 | MPI_Abort(MPI_COMM_WORLD, 0); 277 | } 278 | 279 | rc = check_buffer(buf, size, rank, i); 280 | if (! rc) { 281 | printf("INVALID DATA on rank %d in step %d\n", rank, i); fflush(stdout); 282 | MPI_Abort(MPI_COMM_WORLD, 0); 283 | } 284 | 285 | if (rank == 0) { 286 | printf("Verified checkpoint %d.\n", read_timestep); fflush(stdout); 287 | } 288 | 289 | /* optionally sleep for some time */ 290 | if (seconds > 0) { 291 | if (rank == 0) { 292 | printf("Sleeping for %d seconds... \n", seconds); fflush(stdout); 293 | } 294 | sleep(seconds); 295 | } 296 | 297 | remove(file); 298 | } 299 | } 300 | 301 | MPI_Barrier(MPI_COMM_WORLD); 302 | 303 | if (buf != NULL) { 304 | free(buf); 305 | buf = NULL; 306 | } 307 | 308 | return; 309 | } 310 | 311 | int main (int argc, char* argv[]) 312 | { 313 | /* check that we got an appropriate number of arguments */ 314 | if (argc != 1 && argc != 4) { 315 | printf("Usage: test_correctness [filesize times sleep_secs]\n"); 316 | return 1; 317 | } 318 | 319 | /* read parameters from command line, if any */ 320 | if (argc > 1) { 321 | filesize = (size_t) atol(argv[1]); 322 | times = atoi(argv[2]); 323 | seconds = atoi(argv[3]); 324 | } 325 | 326 | MPI_Init(&argc, &argv); 327 | 328 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 329 | MPI_Comm_size(MPI_COMM_WORLD, &ranks); 330 | 331 | char name[256]; 332 | sprintf(name, "/tmp/rank.%d", rank); 333 | 334 | /* allocate space for the checkpoint data (make filesize a function of rank for some variation) */ 335 | filesize = filesize + rank; 336 | 337 | cruise_mount("/tmp", filesize, rank); 338 | 339 | /* verify data integrity in file */ 340 | checkdata(name, filesize, times); 341 | 342 | MPI_Finalize(); 343 | 344 | return 0; 345 | } 346 | -------------------------------------------------------------------------------- /tests/test_memcpy.c: -------------------------------------------------------------------------------- 1 | // build: mpigcc -g -O3 -o test_ramdisk test_ramdisk.c 2 | // run: srun -n64 -N4 ./test_ramdisk 3 | 4 | #define _GNU_SOURCE 1 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "mpi.h" 14 | #include 15 | 16 | size_t filesize = 100*1024*1024; 17 | int times = 5; 18 | int seconds = 0; 19 | int rank = -1; 20 | int ranks = 0; 21 | 22 | int timestep = 0; 23 | 24 | char* memcpy_tmp = NULL; 25 | /* reliable write to file descriptor (retries, if necessary, until hard error) */ 26 | int reliable_write(int fd, const void* buf, size_t size) 27 | { 28 | size_t n = 0; 29 | int retries = 10; 30 | int rank; 31 | char host[128]; 32 | while (n < size) 33 | { 34 | //int rc = write(fd, (char*) buf + n, size - n); 35 | if ( memcpy(memcpy_tmp + n,(char*) buf + n, size-n) == NULL ) 36 | perror("memcpy failed!"); 37 | int rc = size-n; 38 | if (rc > 0) { 39 | n += rc; 40 | } else if (rc == 0) { 41 | /* something bad happened, print an error and abort */ 42 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 43 | gethostname(host, sizeof(host)); 44 | printf("%d on %s: ERROR: Error writing: write(%d, %p, %ld) returned 0 @ %s:%d\n", 45 | rank, host, fd, (char*) buf + n, size - n, __FILE__, __LINE__ 46 | ); 47 | MPI_Abort(MPI_COMM_WORLD, 0); 48 | } else { /* (rc < 0) */ 49 | /* got an error, check whether it was serious */ 50 | if(errno == EINTR || errno == EAGAIN) continue; 51 | 52 | /* something worth printing an error about */ 53 | retries--; 54 | if (retries) { 55 | /* print an error and try again */ 56 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 57 | gethostname(host, sizeof(host)); 58 | printf("%d on %s: ERROR: Error writing: write(%d, %p, %ld) errno=%d %s @ %s:%d\n", 59 | rank, host, fd, (char*) buf + n, size - n, errno, strerror(errno), __FILE__, __LINE__ 60 | ); 61 | } else { 62 | /* too many failed retries, give up */ 63 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 64 | gethostname(host, sizeof(host)); 65 | printf("%d on %s: ERROR: Giving up write: write(%d, %p, %ld) errno=%d %s @ %s:%d\n", 66 | rank, host, fd, (char*) buf + n, size - n, errno, strerror(errno), __FILE__, __LINE__ 67 | ); 68 | MPI_Abort(MPI_COMM_WORLD, 0); 69 | } 70 | } 71 | } 72 | return size; 73 | } 74 | 75 | /* initialize buffer with some well-known value based on rank */ 76 | int init_buffer(char* buf, size_t size, int rank, int ckpt) 77 | { 78 | size_t i; 79 | for(i=0; i < size; i++) { 80 | /*char c = 'a' + (rank+i) % 26;*/ 81 | char c = (char) ((size_t)rank + i) % 256; 82 | buf[i] = c; 83 | } 84 | return 0; 85 | } 86 | 87 | /* write the checkpoint data to fd, and return whether the write was successful */ 88 | int write_checkpoint(int fd, int ckpt, char* buf, size_t size) 89 | { 90 | int rc; 91 | int valid = 0; 92 | 93 | /* write the checkpoint id (application timestep) */ 94 | char ckpt_buf[7]; 95 | sprintf(ckpt_buf, "%06d", ckpt); 96 | rc = reliable_write(fd, ckpt_buf, sizeof(ckpt_buf)-1); 97 | if (rc < 0) { valid = 0; } 98 | 99 | /* write the checkpoint data */ 100 | rc = reliable_write(fd, buf, size); 101 | if (rc < 0) { valid = 0; } 102 | 103 | return valid; 104 | } 105 | 106 | double getbw(char* file, char* buf, size_t size, int times) 107 | { 108 | double bw = 0.0; 109 | 110 | MPI_Barrier(MPI_COMM_WORLD); 111 | 112 | if (times > 0) { 113 | /* start the timer */ 114 | double time_start = MPI_Wtime(); 115 | 116 | /* write the checkpoint file */ 117 | int i, count = 0; 118 | for(i=0; i < times; i++) { 119 | int rc; 120 | int valid = 0; 121 | 122 | /* open the file and write the checkpoint */ 123 | int fd_me = 1; 124 | // int fd_me = open(file, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); 125 | // if (fd_me > 0) { 126 | count++; 127 | valid = 1; 128 | 129 | /* write the checkpoint data */ 130 | rc = write_checkpoint(fd_me, timestep, buf, size); 131 | if (rc < 0) { 132 | valid = 0; 133 | } 134 | 135 | /* force the data to storage */ 136 | // rc = fsync(fd_me); 137 | if (rc < 0) { 138 | valid = 0; 139 | } 140 | 141 | /* make sure the close is without error */ 142 | // rc = close(fd_me); 143 | if (rc < 0) { 144 | valid = 0; 145 | } 146 | // } 147 | 148 | if (rank == 0) { 149 | printf("Completed checkpoint %d.\n", timestep); 150 | fflush(stdout); 151 | } 152 | 153 | /* increase the timestep counter */ 154 | timestep++; 155 | 156 | /* optionally sleep for some time */ 157 | if (seconds > 0) { 158 | if (rank == 0) { printf("Sleeping for %d seconds... \n", seconds); fflush(stdout); } 159 | sleep(seconds); 160 | } 161 | 162 | // unlink(file); 163 | } 164 | 165 | /* stop the timer and compute the bandwidth */ 166 | double time_end = MPI_Wtime(); 167 | bw = ((size * count) / (1024*1024)) / (time_end - time_start); 168 | } 169 | 170 | MPI_Barrier(MPI_COMM_WORLD); 171 | 172 | return bw; 173 | } 174 | 175 | 176 | int main (int argc, char* argv[]) 177 | { 178 | /* check that we got an appropriate number of arguments */ 179 | if (argc != 1 && argc != 4) { 180 | printf("Usage: test_correctness [filesize times sleep_secs]\n"); 181 | return 1; 182 | } 183 | 184 | /* read parameters from command line, if any */ 185 | if (argc > 1) { 186 | filesize = (size_t) atol(argv[1]); 187 | times = atoi(argv[2]); 188 | seconds = atoi(argv[3]); 189 | } 190 | 191 | MPI_Init(&argc, &argv); 192 | 193 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 194 | MPI_Comm_size(MPI_COMM_WORLD, &ranks); 195 | 196 | 197 | char name[256]; 198 | sprintf(name, "/dev/shm/rank.%d", rank); 199 | 200 | /* allocate space for the checkpoint data (make filesize a function of rank for some variation) */ 201 | filesize = filesize + rank; 202 | char* buf = (char*) malloc(filesize); 203 | 204 | memcpy_tmp = (char *) malloc(filesize); 205 | 206 | /* make up some data for the next checkpoint */ 207 | init_buffer(buf, filesize, rank, timestep); 208 | 209 | timestep++; 210 | 211 | /* prime system once before timing */ 212 | getbw(name, buf, filesize, 1); 213 | 214 | /* now compute the bandwidth and print stats */ 215 | if (times > 0) { 216 | double bw = getbw(name, buf, filesize, times); 217 | 218 | /* compute stats and print them to the screen */ 219 | double bwmin, bwmax, bwsum; 220 | MPI_Reduce(&bw, &bwmin, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD); 221 | MPI_Reduce(&bw, &bwmax, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); 222 | MPI_Reduce(&bw, &bwsum, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); 223 | if (rank == 0) { 224 | printf("FileIO: Min %7.2f MB/s\tMax %7.2f MB/s\tAvg %7.2f MB/s\tAgg %7.2f MB/s\n", 225 | bwmin, bwmax, bwsum/ranks, bwsum 226 | ); 227 | } 228 | } 229 | 230 | if (buf != NULL) { 231 | free(buf); 232 | buf = NULL; 233 | } 234 | 235 | free(memcpy_tmp); 236 | 237 | MPI_Finalize(); 238 | 239 | return 0; 240 | } 241 | -------------------------------------------------------------------------------- /tests/test_ramdisk.c: -------------------------------------------------------------------------------- 1 | // build: mpigcc -g -O3 -o test_ramdisk test_ramdisk.c 2 | // run: srun -n64 -N4 ./test_ramdisk 3 | 4 | #define _GNU_SOURCE 1 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "mpi.h" 15 | 16 | size_t filesize = 100*1024*1024; 17 | int times = 5; 18 | int seconds = 0; 19 | int rank = -1; 20 | int ranks = 0; 21 | 22 | int timestep = 0; 23 | 24 | /* reliable write to file descriptor (retries, if necessary, until hard error) */ 25 | int reliable_write(int fd, const void* buf, size_t size) 26 | { 27 | size_t n = 0; 28 | int retries = 10; 29 | int rank; 30 | char host[128]; 31 | while (n < size) 32 | { 33 | int rc = write(fd, (char*) buf + n, size - n); 34 | if (rc > 0) { 35 | n += rc; 36 | } else if (rc == 0) { 37 | /* something bad happened, print an error and abort */ 38 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 39 | gethostname(host, sizeof(host)); 40 | printf("%d on %s: ERROR: Error writing: write(%d, %p, %ld) returned 0 @ %s:%d\n", 41 | rank, host, fd, (char*) buf + n, size - n, __FILE__, __LINE__ 42 | ); 43 | MPI_Abort(MPI_COMM_WORLD, 0); 44 | } else { /* (rc < 0) */ 45 | /* got an error, check whether it was serious */ 46 | if(errno == EINTR || errno == EAGAIN) continue; 47 | 48 | /* something worth printing an error about */ 49 | retries--; 50 | if (retries) { 51 | /* print an error and try again */ 52 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 53 | gethostname(host, sizeof(host)); 54 | printf("%d on %s: ERROR: Error writing: write(%d, %p, %ld) errno=%d %s @ %s:%d\n", 55 | rank, host, fd, (char*) buf + n, size - n, errno, strerror(errno), __FILE__, __LINE__ 56 | ); 57 | } else { 58 | /* too many failed retries, give up */ 59 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 60 | gethostname(host, sizeof(host)); 61 | printf("%d on %s: ERROR: Giving up write: write(%d, %p, %ld) errno=%d %s @ %s:%d\n", 62 | rank, host, fd, (char*) buf + n, size - n, errno, strerror(errno), __FILE__, __LINE__ 63 | ); 64 | MPI_Abort(MPI_COMM_WORLD, 0); 65 | } 66 | } 67 | } 68 | return size; 69 | } 70 | 71 | /* initialize buffer with some well-known value based on rank */ 72 | int init_buffer(char* buf, size_t size, int rank, int ckpt) 73 | { 74 | size_t i; 75 | for(i=0; i < size; i++) { 76 | /*char c = 'a' + (rank+i) % 26;*/ 77 | char c = (char) ((size_t)rank + i) % 256; 78 | buf[i] = c; 79 | } 80 | return 0; 81 | } 82 | 83 | /* write the checkpoint data to fd, and return whether the write was successful */ 84 | int write_checkpoint(int fd, int ckpt, char* buf, size_t size) 85 | { 86 | int rc; 87 | int valid = 0; 88 | 89 | /* write the checkpoint id (application timestep) */ 90 | char ckpt_buf[7]; 91 | sprintf(ckpt_buf, "%06d", ckpt); 92 | rc = reliable_write(fd, ckpt_buf, sizeof(ckpt_buf)-1); 93 | if (rc < 0) { valid = 0; } 94 | 95 | /* write the checkpoint data */ 96 | rc = reliable_write(fd, buf, size); 97 | if (rc < 0) { valid = 0; } 98 | 99 | return valid; 100 | } 101 | 102 | double getbw(char* file, char* buf, size_t size, int times) 103 | { 104 | double bw = 0.0; 105 | 106 | MPI_Barrier(MPI_COMM_WORLD); 107 | 108 | if (times > 0) { 109 | /* start the timer */ 110 | double time_start = MPI_Wtime(); 111 | 112 | /* write the checkpoint file */ 113 | int i, count = 0; 114 | for(i=0; i < times; i++) { 115 | int rc; 116 | int valid = 0; 117 | 118 | /* open the file and write the checkpoint */ 119 | int fd_me = open(file, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); 120 | if (fd_me > 0) { 121 | count++; 122 | valid = 1; 123 | 124 | /* write the checkpoint data */ 125 | rc = write_checkpoint(fd_me, timestep, buf, size); 126 | if (rc < 0) { 127 | valid = 0; 128 | } 129 | 130 | /* force the data to storage */ 131 | rc = fsync(fd_me); 132 | if (rc < 0) { 133 | valid = 0; 134 | } 135 | 136 | /* make sure the close is without error */ 137 | rc = close(fd_me); 138 | if (rc < 0) { 139 | valid = 0; 140 | } 141 | } 142 | 143 | if (rank == 0) { 144 | printf("Completed checkpoint %d.\n", timestep); 145 | fflush(stdout); 146 | } 147 | 148 | /* increase the timestep counter */ 149 | timestep++; 150 | 151 | /* optionally sleep for some time */ 152 | if (seconds > 0) { 153 | if (rank == 0) { printf("Sleeping for %d seconds... \n", seconds); fflush(stdout); } 154 | sleep(seconds); 155 | } 156 | 157 | unlink(file); 158 | } 159 | 160 | /* stop the timer and compute the bandwidth */ 161 | double time_end = MPI_Wtime(); 162 | bw = ((size * count) / (1024*1024)) / (time_end - time_start); 163 | } 164 | 165 | MPI_Barrier(MPI_COMM_WORLD); 166 | 167 | return bw; 168 | } 169 | 170 | int main (int argc, char* argv[]) 171 | { 172 | /* check that we got an appropriate number of arguments */ 173 | if (argc != 1 && argc != 4) { 174 | printf("Usage: test_correctness [filesize times sleep_secs]\n"); 175 | return 1; 176 | } 177 | 178 | /* read parameters from command line, if any */ 179 | if (argc > 1) { 180 | filesize = (size_t) atol(argv[1]); 181 | times = atoi(argv[2]); 182 | seconds = atoi(argv[3]); 183 | } 184 | 185 | MPI_Init(&argc, &argv); 186 | 187 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 188 | MPI_Comm_size(MPI_COMM_WORLD, &ranks); 189 | 190 | char name[256]; 191 | sprintf(name, "/tmp/rank.%d", rank); 192 | 193 | /* allocate space for the checkpoint data (make filesize a function of rank for some variation) */ 194 | filesize = filesize + rank; 195 | char* buf = (char*) malloc(filesize); 196 | 197 | /* make up some data for the next checkpoint */ 198 | init_buffer(buf, filesize, rank, timestep); 199 | 200 | timestep++; 201 | 202 | /* prime system once before timing */ 203 | getbw(name, buf, filesize, 1); 204 | 205 | /* now compute the bandwidth and print stats */ 206 | if (times > 0) { 207 | double bw = getbw(name, buf, filesize, times); 208 | 209 | /* compute stats and print them to the screen */ 210 | double bwmin, bwmax, bwsum; 211 | MPI_Reduce(&bw, &bwmin, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD); 212 | MPI_Reduce(&bw, &bwmax, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); 213 | MPI_Reduce(&bw, &bwsum, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); 214 | if (rank == 0) { 215 | printf("FileIO: Min %7.2f MB/s\tMax %7.2f MB/s\tAvg %7.2f MB/s\tAgg %7.2f MB/s\n", 216 | bwmin, bwmax, bwsum/ranks, bwsum 217 | ); 218 | } 219 | } 220 | 221 | if (buf != NULL) { 222 | free(buf); 223 | buf = NULL; 224 | } 225 | 226 | MPI_Finalize(); 227 | 228 | return 0; 229 | } 230 | -------------------------------------------------------------------------------- /tests/test_ramdisk_iter.c: -------------------------------------------------------------------------------- 1 | // build: mpigcc -g -O3 -o test_ramdisk test_ramdisk.c 2 | // run: srun -n64 -N4 ./test_ramdisk 3 | 4 | #define _GNU_SOURCE 1 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "mpi.h" 15 | 16 | size_t filesize = 100*1024*1024; 17 | size_t write_unit = 2048; 18 | int times = 5; 19 | int seconds = 0; 20 | int rank = -1; 21 | int ranks = 0; 22 | 23 | int timestep = 0; 24 | 25 | /* reliable write to file descriptor (retries, if necessary, until hard error) */ 26 | int reliable_write(int fd, const void* buf, size_t size) 27 | { 28 | size_t n = 0; 29 | int retries = 10; 30 | int rank; 31 | char host[128]; 32 | while (n < size) 33 | { 34 | int rc = write(fd, (char*) buf + n, write_unit); 35 | if (rc > 0) { 36 | n += rc; 37 | } else if (rc == 0) { 38 | /* something bad happened, print an error and abort */ 39 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 40 | gethostname(host, sizeof(host)); 41 | printf("%d on %s: ERROR: Error writing: write(%d, %p, %ld) returned 0 @ %s:%d\n", 42 | rank, host, fd, (char*) buf + n, size - n, __FILE__, __LINE__ 43 | ); 44 | MPI_Abort(MPI_COMM_WORLD, 0); 45 | } else { /* (rc < 0) */ 46 | /* got an error, check whether it was serious */ 47 | if(errno == EINTR || errno == EAGAIN) continue; 48 | 49 | /* something worth printing an error about */ 50 | retries--; 51 | if (retries) { 52 | /* print an error and try again */ 53 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 54 | gethostname(host, sizeof(host)); 55 | printf("%d on %s: ERROR: Error writing: write(%d, %p, %ld) errno=%d %s @ %s:%d\n", 56 | rank, host, fd, (char*) buf + n, size - n, errno, strerror(errno), __FILE__, __LINE__ 57 | ); 58 | } else { 59 | /* too many failed retries, give up */ 60 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 61 | gethostname(host, sizeof(host)); 62 | printf("%d on %s: ERROR: Giving up write: write(%d, %p, %ld) errno=%d %s @ %s:%d\n", 63 | rank, host, fd, (char*) buf + n, size - n, errno, strerror(errno), __FILE__, __LINE__ 64 | ); 65 | MPI_Abort(MPI_COMM_WORLD, 0); 66 | } 67 | } 68 | } 69 | return size; 70 | } 71 | 72 | /* initialize buffer with some well-known value based on rank */ 73 | int init_buffer(char* buf, size_t size, int rank, int ckpt) 74 | { 75 | size_t i; 76 | for(i=0; i < size; i++) { 77 | /*char c = 'a' + (rank+i) % 26;*/ 78 | char c = (char) ((size_t)rank + i) % 256; 79 | buf[i] = c; 80 | } 81 | return 0; 82 | } 83 | 84 | /* write the checkpoint data to fd, and return whether the write was successful */ 85 | int write_checkpoint(int fd, int ckpt, char* buf, size_t size) 86 | { 87 | int rc; 88 | int valid = 0; 89 | 90 | /* write the checkpoint id (application timestep) */ 91 | char ckpt_buf[7]; 92 | sprintf(ckpt_buf, "%06d", ckpt); 93 | rc = reliable_write(fd, ckpt_buf, sizeof(ckpt_buf)-1); 94 | if (rc < 0) { valid = 0; } 95 | 96 | /* write the checkpoint data */ 97 | rc = reliable_write(fd, buf, size); 98 | if (rc < 0) { valid = 0; } 99 | 100 | return valid; 101 | } 102 | 103 | double getbw(char* file, char* buf, size_t size, int times) 104 | { 105 | double bw = 0.0; 106 | 107 | MPI_Barrier(MPI_COMM_WORLD); 108 | 109 | if (times > 0) { 110 | /* start the timer */ 111 | double time_start = MPI_Wtime(); 112 | 113 | /* write the checkpoint file */ 114 | int i, count = 0; 115 | for(i=0; i < times; i++) { 116 | int rc; 117 | int valid = 0; 118 | 119 | /* open the file and write the checkpoint */ 120 | int fd_me = open(file, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); 121 | if (fd_me > 0) { 122 | count++; 123 | valid = 1; 124 | 125 | /* write the checkpoint data */ 126 | rc = write_checkpoint(fd_me, timestep, buf, size); 127 | if (rc < 0) { 128 | valid = 0; 129 | } 130 | 131 | /* force the data to storage */ 132 | rc = fsync(fd_me); 133 | if (rc < 0) { 134 | valid = 0; 135 | } 136 | 137 | /* make sure the close is without error */ 138 | rc = close(fd_me); 139 | if (rc < 0) { 140 | valid = 0; 141 | } 142 | } 143 | 144 | if (rank == 0) { 145 | printf("Completed checkpoint %d.\n", timestep); 146 | fflush(stdout); 147 | } 148 | 149 | /* increase the timestep counter */ 150 | timestep++; 151 | 152 | /* optionally sleep for some time */ 153 | if (seconds > 0) { 154 | if (rank == 0) { printf("Sleeping for %d seconds... \n", seconds); fflush(stdout); } 155 | sleep(seconds); 156 | } 157 | 158 | unlink(file); 159 | } 160 | 161 | /* stop the timer and compute the bandwidth */ 162 | double time_end = MPI_Wtime(); 163 | bw = ((size * count) / (1024*1024)) / (time_end - time_start); 164 | } 165 | 166 | MPI_Barrier(MPI_COMM_WORLD); 167 | 168 | return bw; 169 | } 170 | 171 | int main (int argc, char* argv[]) 172 | { 173 | /* check that we got an appropriate number of arguments */ 174 | if (argc != 1 && argc != 4) { 175 | printf("Usage: test_correctness [filesize times sleep_secs]\n"); 176 | return 1; 177 | } 178 | 179 | /* read parameters from command line, if any */ 180 | if (argc > 1) { 181 | filesize = (size_t) atol(argv[1]); 182 | times = atoi(argv[2]); 183 | seconds = atoi(argv[3]); 184 | } 185 | 186 | MPI_Init(&argc, &argv); 187 | 188 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 189 | MPI_Comm_size(MPI_COMM_WORLD, &ranks); 190 | 191 | char name[256]; 192 | sprintf(name, "/dev/shm/rank.%d", rank); 193 | 194 | /* allocate space for the checkpoint data (make filesize a function of rank for some variation) */ 195 | filesize = filesize + rank; 196 | char* buf = (char*) malloc(filesize); 197 | 198 | /* make up some data for the next checkpoint */ 199 | init_buffer(buf, filesize, rank, timestep); 200 | 201 | timestep++; 202 | 203 | /* prime system once before timing */ 204 | getbw(name, buf, filesize, 1); 205 | 206 | /* now compute the bandwidth and print stats */ 207 | if (times > 0) { 208 | double bw = getbw(name, buf, filesize, times); 209 | 210 | /* compute stats and print them to the screen */ 211 | double bwmin, bwmax, bwsum; 212 | MPI_Reduce(&bw, &bwmin, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD); 213 | MPI_Reduce(&bw, &bwmax, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); 214 | MPI_Reduce(&bw, &bwsum, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); 215 | if (rank == 0) { 216 | printf("FileIO: Min %7.2f MB/s\tMax %7.2f MB/s\tAvg %7.2f MB/s\tAgg %7.2f MB/s\n", 217 | bwmin, bwmax, bwsum/ranks, bwsum 218 | ); 219 | } 220 | } 221 | 222 | if (buf != NULL) { 223 | free(buf); 224 | buf = NULL; 225 | } 226 | 227 | MPI_Finalize(); 228 | 229 | return 0; 230 | } 231 | -------------------------------------------------------------------------------- /tests/test_readwrite.c: -------------------------------------------------------------------------------- 1 | // build: mpigcc -g -O3 -o test_ramdisk test_ramdisk.c 2 | // run: srun -n64 -N4 ./test_ramdisk 3 | 4 | #define _GNU_SOURCE 1 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "mpi.h" 15 | 16 | //size_t filesize = 100*1024*1024; 17 | size_t filesize = 1024*1024; 18 | int times = 5; 19 | int seconds = 0; 20 | int rank = -1; 21 | int ranks = 0; 22 | 23 | /* reliable read from file descriptor (retries, if necessary, until hard error) */ 24 | int reliable_read(int fd, void* buf, size_t size) 25 | { 26 | size_t n = 0; 27 | int retries = 10; 28 | int rank; 29 | char host[128]; 30 | while (n < size) 31 | { 32 | int rc = read(fd, (char*) buf + n, size - n); 33 | if (rc > 0) { 34 | n += rc; 35 | } else if (rc == 0) { 36 | /* EOF */ 37 | return n; 38 | } else { /* (rc < 0) */ 39 | /* got an error, check whether it was serious */ 40 | if(errno == EINTR || errno == EAGAIN) { 41 | continue; 42 | } 43 | 44 | /* something worth printing an error about */ 45 | retries--; 46 | if (retries) { 47 | /* print an error and try again */ 48 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 49 | gethostname(host, sizeof(host)); 50 | printf("%d on %s: ERROR: Error reading: read(%d, %p, %ld) errno=%d %s @ %s:%d\n", 51 | rank, host, fd, (char*) buf + n, size - n, errno, strerror(errno), __FILE__, __LINE__ 52 | ); 53 | } else { 54 | /* too many failed retries, give up */ 55 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 56 | gethostname(host, sizeof(host)); 57 | printf("%d on %s: ERROR: Giving up read: read(%d, %p, %ld) errno=%d %s @ %s:%d\n", 58 | rank, host, fd, (char*) buf + n, size - n, errno, strerror(errno), __FILE__, __LINE__ 59 | ); 60 | MPI_Abort(MPI_COMM_WORLD, 0); 61 | } 62 | } 63 | } 64 | return size; 65 | } 66 | 67 | /* reliable write to file descriptor (retries, if necessary, until hard error) */ 68 | int reliable_write(int fd, const void* buf, size_t size) 69 | { 70 | size_t n = 0; 71 | int retries = 10; 72 | int rank; 73 | char host[128]; 74 | while (n < size) 75 | { 76 | int rc = write(fd, (char*) buf + n, size - n); 77 | if (rc > 0) { 78 | n += rc; 79 | } else if (rc == 0) { 80 | /* something bad happened, print an error and abort */ 81 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 82 | gethostname(host, sizeof(host)); 83 | printf("%d on %s: ERROR: Error writing: write(%d, %p, %ld) returned 0 @ %s:%d\n", 84 | rank, host, fd, (char*) buf + n, size - n, __FILE__, __LINE__ 85 | ); 86 | MPI_Abort(MPI_COMM_WORLD, 0); 87 | } else { /* (rc < 0) */ 88 | /* got an error, check whether it was serious */ 89 | if(errno == EINTR || errno == EAGAIN) { 90 | continue; 91 | } 92 | 93 | /* something worth printing an error about */ 94 | retries--; 95 | if (retries) { 96 | /* print an error and try again */ 97 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 98 | gethostname(host, sizeof(host)); 99 | printf("%d on %s: ERROR: Error writing: write(%d, %p, %ld) errno=%d %s @ %s:%d\n", 100 | rank, host, fd, (char*) buf + n, size - n, errno, strerror(errno), __FILE__, __LINE__ 101 | ); 102 | } else { 103 | /* too many failed retries, give up */ 104 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 105 | gethostname(host, sizeof(host)); 106 | printf("%d on %s: ERROR: Giving up write: write(%d, %p, %ld) errno=%d %s @ %s:%d\n", 107 | rank, host, fd, (char*) buf + n, size - n, errno, strerror(errno), __FILE__, __LINE__ 108 | ); 109 | MPI_Abort(MPI_COMM_WORLD, 0); 110 | } 111 | } 112 | } 113 | return size; 114 | } 115 | 116 | /* initialize buffer with some well-known value based on rank */ 117 | int init_buffer(char* buf, size_t size, int rank, int ckpt) 118 | { 119 | size_t i; 120 | for(i=0; i < size; i++) { 121 | char c = 'a' + (char)((rank + ckpt + i) & 32); 122 | buf[i] = c; 123 | } 124 | return 0; 125 | } 126 | 127 | /* checks buffer for expected value */ 128 | int check_buffer(char* buf, size_t size, int rank, int ckpt) 129 | { 130 | size_t i; 131 | for(i=0; i < size; i++) { 132 | char c = 'a' + (char)((rank + ckpt + i) & 32); 133 | if (buf[i] != c) { 134 | printf("check failed at byte %d, should be %c is %c\n", (int)i, c, buf[i]); 135 | return 0; 136 | } 137 | } 138 | return 1; 139 | } 140 | 141 | /* read the checkpoint data from file into buf, and return whether the read was successful */ 142 | int read_checkpoint(int fd, int* rank, int* ckpt, char* buf, size_t size) 143 | { 144 | unsigned long n; 145 | char rank_buf[7]; 146 | char ckpt_buf[7]; 147 | size_t field_size = 6; 148 | 149 | /* read the rank id */ 150 | n = reliable_read(fd, rank_buf, field_size); 151 | if (n != field_size) { 152 | printf("Failed to read rank\n"); 153 | return 0; 154 | } 155 | rank_buf[6] = '\0'; 156 | 157 | /* read the checkpoint id */ 158 | n = reliable_read(fd, ckpt_buf, field_size); 159 | if (n != field_size) { 160 | printf("Failed to read timestep\n"); 161 | return 0; 162 | } 163 | ckpt_buf[6] = '\0'; 164 | 165 | /* read the checkpoint data, and check the file size */ 166 | n = reliable_read(fd, buf, size+1); 167 | if (n != size) { 168 | printf("Filesize not correct\n"); 169 | return 0; 170 | } 171 | 172 | /* if the file looks good, set the timestep and return */ 173 | sscanf(rank_buf, "%6d", rank); 174 | sscanf(ckpt_buf, "%6d", ckpt); 175 | 176 | return 0; 177 | } 178 | 179 | /* write the checkpoint data to fd, and return whether the write was successful */ 180 | int write_checkpoint(int fd, int rank, int ckpt, char* buf, size_t size) 181 | { 182 | int rc; 183 | int valid = 0; 184 | char rank_buf[7]; 185 | char ckpt_buf[7]; 186 | size_t field_size = 6; 187 | 188 | /* write the rank id */ 189 | sprintf(rank_buf, "%06d", rank); 190 | rc = reliable_write(fd, rank_buf, field_size); 191 | if (rc < 0) { 192 | valid = 0; 193 | } 194 | 195 | /* write the checkpoint id (application timestep) */ 196 | sprintf(ckpt_buf, "%06d", ckpt); 197 | rc = reliable_write(fd, ckpt_buf, field_size); 198 | if (rc < 0) { 199 | valid = 0; 200 | } 201 | 202 | /* write the checkpoint data */ 203 | rc = reliable_write(fd, buf, size); 204 | if (rc < 0) { 205 | valid = 0; 206 | } 207 | 208 | return valid; 209 | } 210 | 211 | void checkdata(char* file, size_t size, int times) 212 | { 213 | char* buf = malloc(size); 214 | 215 | MPI_Barrier(MPI_COMM_WORLD); 216 | 217 | if (times > 0) { 218 | /* write the checkpoint file */ 219 | int i; 220 | int rc; 221 | int valid = 0; 222 | 223 | /* open the file and write the checkpoint */ 224 | int read_rank, read_timestep; 225 | int fd_me = open(file, O_RDONLY); 226 | if (fd_me >= 0) { 227 | if (rank == 0) { 228 | printf("Reading checkpoint %d.\n", i); fflush(stdout); 229 | } 230 | 231 | memset(buf, 0, size); 232 | 233 | valid = 1; 234 | 235 | /* write the checkpoint data */ 236 | rc = read_checkpoint(fd_me, &read_rank, &read_timestep, buf, size); 237 | if (rc < 0) { 238 | valid = 0; 239 | } 240 | 241 | /* make sure the close is without error */ 242 | rc = close(fd_me); 243 | if (rc < 0) { 244 | valid = 0; 245 | } 246 | 247 | if (read_rank != rank) { 248 | printf("INVALID HEADER on rank %d in step %d\n", rank, i); fflush(stdout); 249 | MPI_Abort(MPI_COMM_WORLD, 0); 250 | } 251 | 252 | rc = check_buffer(buf, size, rank, read_timestep); 253 | if (! rc) { 254 | printf("INVALID DATA on rank %d in step %d\n", rank, i); fflush(stdout); 255 | MPI_Abort(MPI_COMM_WORLD, 0); 256 | } 257 | 258 | if (rank == 0) { 259 | printf("Verified checkpoint %d.\n", read_timestep); fflush(stdout); 260 | } 261 | 262 | unlink(file); 263 | 264 | i = read_timestep + 1; 265 | } else { 266 | i = 0; 267 | } 268 | 269 | rc = init_buffer(buf, size, rank, i); 270 | 271 | if (rank == 0) { 272 | printf("Writing checkpoint %d.\n", i); fflush(stdout); 273 | } 274 | 275 | /* open the file and write the checkpoint */ 276 | fd_me = open(file, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); 277 | if (fd_me >= 0) { 278 | valid = 1; 279 | 280 | /* write the checkpoint data */ 281 | rc = write_checkpoint(fd_me, rank, i, buf, size); 282 | if (rc < 0) { 283 | valid = 0; 284 | } 285 | 286 | /* force the data to storage */ 287 | rc = fsync(fd_me); 288 | if (rc < 0) { 289 | valid = 0; 290 | } 291 | 292 | /* make sure the close is without error */ 293 | rc = close(fd_me); 294 | if (rc < 0) { 295 | valid = 0; 296 | } 297 | } 298 | 299 | if (rank == 0) { 300 | printf("Completed checkpoint %d.\n", i); fflush(stdout); 301 | } 302 | } 303 | 304 | MPI_Barrier(MPI_COMM_WORLD); 305 | 306 | if (buf != NULL) { 307 | free(buf); 308 | buf = NULL; 309 | } 310 | 311 | return; 312 | } 313 | 314 | int main (int argc, char* argv[]) 315 | { 316 | /* check that we got an appropriate number of arguments */ 317 | if (argc != 1 && argc != 4) { 318 | printf("Usage: test_correctness [filesize times sleep_secs]\n"); 319 | return 1; 320 | } 321 | 322 | /* read parameters from command line, if any */ 323 | if (argc > 1) { 324 | filesize = (size_t) atol(argv[1]); 325 | times = atoi(argv[2]); 326 | seconds = atoi(argv[3]); 327 | } 328 | 329 | MPI_Init(&argc, &argv); 330 | 331 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 332 | MPI_Comm_size(MPI_COMM_WORLD, &ranks); 333 | 334 | cruise_mount("/tmp",1024,0); 335 | 336 | char name[256]; 337 | sprintf(name, "/tmp/rank.%d", rank); 338 | 339 | /* allocate space for the checkpoint data (make filesize a function of rank for some variation) */ 340 | filesize = filesize + rank; 341 | 342 | /* verify data integrity in file */ 343 | checkdata(name, filesize, times); 344 | 345 | MPI_Finalize(); 346 | 347 | return 0; 348 | } 349 | -------------------------------------------------------------------------------- /tests/test_scanf.c: -------------------------------------------------------------------------------- 1 | // build: mpigcc -g -O3 -o test_ramdisk test_ramdisk.c 2 | // run: srun -n64 -N4 ./test_ramdisk 3 | 4 | #define _GNU_SOURCE 1 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | int rank = -1; 16 | int ranks = 0; 17 | 18 | int main (int argc, char* argv[]) 19 | { 20 | /* check that we got an appropriate number of arguments */ 21 | if (argc != 1 && argc != 4) { 22 | printf("Usage: test_correctness [filesize times sleep_secs]\n"); 23 | return 1; 24 | } 25 | 26 | char name[256]; 27 | sprintf(name, "/tmp/rank.%d", rank); 28 | 29 | cruise_mount("/tmp", 100*1024*1024, rank); 30 | 31 | FILE* fp; 32 | int c, rc; 33 | off_t pos; 34 | 35 | rc = remove(name); 36 | 37 | fp = fopen(name, "w"); 38 | if (fp == NULL) { 39 | printf("Failed to open %s\n", name); 40 | return 1; 41 | } 42 | rc = fprintf(fp, "%d %f %lf %x %u %s %s\n", 143, 25.634, 2.5e3, 345, 1, "hi", "there_world"); 43 | rc = fclose(fp); 44 | 45 | fp = fopen(name, "r"); 46 | while (1) { 47 | c = fgetc(fp); 48 | if (feof(fp)) { 49 | break; 50 | } 51 | printf("%d %c\n", c, (char)c); 52 | } 53 | fclose(fp); 54 | 55 | int i1, i2; 56 | float f1; 57 | double d1; 58 | unsigned long ul1; 59 | char s1[256], s2[256]; 60 | fp = fopen(name, "r"); 61 | rc = fscanf(fp, "%d %f %lf %x %lu %s %s\n", &i1, &f1, &d1, &i2, &ul1, s1, s2); 62 | fclose(fp); 63 | 64 | fp = fopen(name, "a+"); 65 | if (fp == NULL) { 66 | printf("Failed to open %s\n", name); 67 | return 1; 68 | } 69 | rc = fputc((int)'t', fp); 70 | pos = ftello(fp); 71 | rc = fseek(fp, 1, SEEK_SET); 72 | pos = ftello(fp); 73 | c = fgetc(fp); 74 | pos = ftello(fp); 75 | rc = fclose(fp); 76 | 77 | fp = fopen(name, "r+"); 78 | if (fp == NULL) { 79 | printf("Failed to open %s\n", name); 80 | return 1; 81 | } 82 | rc = fseek(fp, 4, SEEK_SET); 83 | rc = ungetc((int)'c', fp); 84 | rc = ungetc((int)'b', fp); 85 | rc = ungetc((int)'a', fp); 86 | while (1) { 87 | c = fgetc(fp); 88 | if (feof(fp)) { 89 | break; 90 | } 91 | if (c == EOF) { 92 | printf("ERROR: %s\n", strerror(errno)); 93 | break; 94 | } 95 | printf("%d %c\n", c, (char)c); 96 | } 97 | rc = fclose(fp); 98 | 99 | fp = fopen(name, "r"); 100 | while (1) { 101 | c = fgetc(fp); 102 | if (feof(fp)) { 103 | break; 104 | } 105 | printf("%d %c\n", c, (char)c); 106 | } 107 | fclose(fp); 108 | 109 | fp = fopen(name, "r+"); 110 | rc = fseek(fp, 5, SEEK_END); 111 | rc = fputc((int)'2', fp); 112 | rc = ungetc((int)'x', fp); 113 | rc = ungetc((int)'y', fp); 114 | rc = ungetc((int)'z', fp); 115 | while (1) { 116 | c = fgetc(fp); 117 | if (feof(fp)) { 118 | break; 119 | } 120 | if (c == EOF) { 121 | printf("ERROR: %s\n", strerror(errno)); 122 | break; 123 | } 124 | printf("%d %c\n", c, (char)c); 125 | } 126 | rc = fclose(fp); 127 | 128 | fp = fopen(name, "r"); 129 | while (1) { 130 | c = fgetc(fp); 131 | if (feof(fp)) { 132 | break; 133 | } 134 | printf("%d %c\n", c, (char)c); 135 | } 136 | fclose(fp); 137 | 138 | return 0; 139 | } 140 | -------------------------------------------------------------------------------- /tests/test_truncate.c: -------------------------------------------------------------------------------- 1 | // build: mpigcc -g -O3 -o test_ramdisk test_ramdisk.c 2 | // run: srun -n64 -N4 ./test_ramdisk 3 | 4 | #define _GNU_SOURCE 1 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "mpi.h" 15 | 16 | size_t filesize = 100*1024*1024; 17 | int times = 5; 18 | int seconds = 0; 19 | int rank = -1; 20 | int ranks = 0; 21 | 22 | /* reliable read from file descriptor (retries, if necessary, until hard error) */ 23 | int reliable_read(int fd, void* buf, size_t size) 24 | { 25 | size_t n = 0; 26 | int retries = 10; 27 | int rank; 28 | char host[128]; 29 | while (n < size) 30 | { 31 | int rc = read(fd, (char*) buf + n, size - n); 32 | if (rc > 0) { 33 | n += rc; 34 | } else if (rc == 0) { 35 | /* EOF */ 36 | return n; 37 | } else { /* (rc < 0) */ 38 | /* got an error, check whether it was serious */ 39 | if(errno == EINTR || errno == EAGAIN) { 40 | continue; 41 | } 42 | 43 | /* something worth printing an error about */ 44 | retries--; 45 | if (retries) { 46 | /* print an error and try again */ 47 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 48 | gethostname(host, sizeof(host)); 49 | printf("%d on %s: ERROR: Error reading: read(%d, %p, %ld) errno=%d @ %s:%d\n", 50 | rank, host, fd, (char*) buf + n, size - n, errno, __FILE__, __LINE__ 51 | ); 52 | } else { 53 | /* too many failed retries, give up */ 54 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 55 | gethostname(host, sizeof(host)); 56 | printf("%d on %s: ERROR: Giving up read: read(%d, %p, %ld) errno=%d @ %s:%d\n", 57 | rank, host, fd, (char*) buf + n, size - n, errno, __FILE__, __LINE__ 58 | ); 59 | MPI_Abort(MPI_COMM_WORLD, 0); 60 | } 61 | } 62 | } 63 | return size; 64 | } 65 | 66 | /* reliable write to file descriptor (retries, if necessary, until hard error) */ 67 | int reliable_write(int fd, const void* buf, size_t size) 68 | { 69 | size_t n = 0; 70 | int retries = 10; 71 | int rank; 72 | char host[128]; 73 | while (n < size) 74 | { 75 | int rc = write(fd, (char*) buf + n, size - n); 76 | if (rc > 0) { 77 | n += rc; 78 | } else if (rc == 0) { 79 | /* something bad happened, print an error and abort */ 80 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 81 | gethostname(host, sizeof(host)); 82 | printf("%d on %s: ERROR: Error writing: write(%d, %p, %ld) returned 0 @ %s:%d\n", 83 | rank, host, fd, (char*) buf + n, size - n, __FILE__, __LINE__ 84 | ); 85 | MPI_Abort(MPI_COMM_WORLD, 0); 86 | } else { /* (rc < 0) */ 87 | /* got an error, check whether it was serious */ 88 | if(errno == EINTR || errno == EAGAIN) { 89 | continue; 90 | } 91 | 92 | /* something worth printing an error about */ 93 | retries--; 94 | if (retries) { 95 | /* print an error and try again */ 96 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 97 | gethostname(host, sizeof(host)); 98 | printf("%d on %s: ERROR: Error writing: write(%d, %p, %ld) errno=%d @ %s:%d\n", 99 | rank, host, fd, (char*) buf + n, size - n, errno, __FILE__, __LINE__ 100 | ); 101 | } else { 102 | /* too many failed retries, give up */ 103 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 104 | gethostname(host, sizeof(host)); 105 | printf("%d on %s: ERROR: Giving up write: write(%d, %p, %ld) errno=%d @ %s:%d\n", 106 | rank, host, fd, (char*) buf + n, size - n, errno, __FILE__, __LINE__ 107 | ); 108 | MPI_Abort(MPI_COMM_WORLD, 0); 109 | } 110 | } 111 | } 112 | return size; 113 | } 114 | 115 | /* initialize buffer with some well-known value based on rank */ 116 | int init_buffer(char* buf, size_t size, int rank, int ckpt) 117 | { 118 | size_t i; 119 | for(i=0; i < size; i++) { 120 | char c = 'a' + (char)((rank + ckpt + i) & 32); 121 | buf[i] = c; 122 | } 123 | return 0; 124 | } 125 | 126 | /* checks buffer for expected value */ 127 | int check_buffer(char* buf, size_t size, int rank, int ckpt) 128 | { 129 | size_t i; 130 | for(i=0; i < size; i++) { 131 | char c = 'a' + (char)((rank + ckpt + i) & 32); 132 | if (buf[i] != c) { 133 | return 0; 134 | } 135 | } 136 | return 1; 137 | } 138 | 139 | /* read the checkpoint data from file into buf, and return whether the read was successful */ 140 | int read_checkpoint(int fd, int* rank, int* ckpt, char* buf, size_t size) 141 | { 142 | unsigned long n; 143 | char rank_buf[7]; 144 | char ckpt_buf[7]; 145 | size_t field_size = 6; 146 | 147 | /* read the rank id */ 148 | n = reliable_read(fd, rank_buf, field_size); 149 | if (n != field_size) { 150 | printf("Failed to read rank\n"); 151 | return 0; 152 | } 153 | rank_buf[6] = '\0'; 154 | 155 | /* read the checkpoint id */ 156 | n = reliable_read(fd, ckpt_buf, field_size); 157 | if (n != field_size) { 158 | printf("Failed to read timestep\n"); 159 | return 0; 160 | } 161 | ckpt_buf[6] = '\0'; 162 | 163 | /* read the checkpoint data, and check the file size */ 164 | n = reliable_read(fd, buf, size+1); 165 | if (n != size) { 166 | printf("Filesize not correct\n"); 167 | return 0; 168 | } 169 | 170 | /* if the file looks good, set the timestep and return */ 171 | sscanf(rank_buf, "%6d", rank); 172 | sscanf(ckpt_buf, "%6d", ckpt); 173 | 174 | return 0; 175 | } 176 | 177 | /* write the checkpoint data to fd, and return whether the write was successful */ 178 | int write_checkpoint(int fd, int rank, int ckpt, char* buf, size_t size) 179 | { 180 | int rc; 181 | int valid = 0; 182 | char rank_buf[7]; 183 | char ckpt_buf[7]; 184 | size_t field_size = 6; 185 | 186 | /* write the rank id */ 187 | sprintf(rank_buf, "%06d", rank); 188 | rc = reliable_write(fd, rank_buf, field_size); 189 | if (rc < 0) { 190 | valid = 0; 191 | } 192 | 193 | /* write the checkpoint id (application timestep) */ 194 | sprintf(ckpt_buf, "%06d", ckpt); 195 | rc = reliable_write(fd, ckpt_buf, field_size); 196 | if (rc < 0) { 197 | valid = 0; 198 | } 199 | 200 | /* write the checkpoint data */ 201 | rc = reliable_write(fd, buf, size); 202 | if (rc < 0) { 203 | valid = 0; 204 | } 205 | 206 | return valid; 207 | } 208 | 209 | void checkdata(char* file, size_t size, int times) 210 | { 211 | char* buf = malloc(size); 212 | 213 | MPI_Barrier(MPI_COMM_WORLD); 214 | 215 | if (times > 0) { 216 | /* write the checkpoint file */ 217 | int i, j; 218 | for(i=0; i < times; i++) { 219 | int rc; 220 | int valid = 0; 221 | 222 | rc = init_buffer(buf, size, rank, i); 223 | 224 | if (rank == 0) { 225 | printf("Writing checkpoint %d.\n", i); fflush(stdout); 226 | } 227 | 228 | /* open the file and write the checkpoint */ 229 | int fd_me = open(file, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); 230 | if (fd_me > 0) { 231 | valid = 1; 232 | 233 | rc = ftruncate(fd_me, size); 234 | if (rc < 0) { 235 | valid = 0; 236 | } 237 | 238 | /* write the checkpoint data */ 239 | rc = write_checkpoint(fd_me, rank, i, buf, size); 240 | if (rc < 0) { 241 | valid = 0; 242 | } 243 | 244 | /* force the data to storage */ 245 | rc = fsync(fd_me); 246 | if (rc < 0) { 247 | valid = 0; 248 | } 249 | 250 | /* make sure the close is without error */ 251 | rc = close(fd_me); 252 | if (rc < 0) { 253 | valid = 0; 254 | } 255 | } 256 | 257 | if (rank == 0) { 258 | printf("Completed checkpoint %d.\n", i); fflush(stdout); 259 | } 260 | 261 | if (rank == 0) { 262 | printf("Reading checkpoint %d.\n", i); fflush(stdout); 263 | } 264 | 265 | memset(buf, 0, size); 266 | 267 | /* open the file and write the checkpoint */ 268 | int read_rank, read_timestep; 269 | fd_me = open(file, O_RDONLY); 270 | if (fd_me > 0) { 271 | valid = 1; 272 | 273 | /* write the checkpoint data */ 274 | rc = read_checkpoint(fd_me, &read_rank, &read_timestep, buf, size); 275 | if (rc < 0) { 276 | valid = 0; 277 | } 278 | 279 | /* make sure the close is without error */ 280 | rc = close(fd_me); 281 | if (rc < 0) { 282 | valid = 0; 283 | } 284 | 285 | } 286 | 287 | if (read_rank != rank || read_timestep != i) { 288 | printf("INVALID HEADER on rank %d in step %d\n", rank, i); fflush(stdout); 289 | MPI_Abort(MPI_COMM_WORLD, 0); 290 | } 291 | 292 | rc = check_buffer(buf, size, rank, i); 293 | if (! rc) { 294 | printf("INVALID DATA on rank %d in step %d\n", rank, i); fflush(stdout); 295 | MPI_Abort(MPI_COMM_WORLD, 0); 296 | } 297 | 298 | if (rank == 0) { 299 | printf("Verified checkpoint %d.\n", read_timestep); fflush(stdout); 300 | } 301 | 302 | /* optionally sleep for some time */ 303 | if (seconds > 0) { 304 | if (rank == 0) { 305 | printf("Sleeping for %d seconds... \n", seconds); fflush(stdout); 306 | } 307 | sleep(seconds); 308 | } 309 | 310 | rc = truncate(file, 0); 311 | if (rc < 0) { 312 | valid = 0; 313 | } 314 | 315 | unlink(file); 316 | } 317 | } 318 | 319 | MPI_Barrier(MPI_COMM_WORLD); 320 | 321 | if (buf != NULL) { 322 | free(buf); 323 | buf = NULL; 324 | } 325 | 326 | return; 327 | } 328 | 329 | int main (int argc, char* argv[]) 330 | { 331 | /* check that we got an appropriate number of arguments */ 332 | if (argc != 1 && argc != 4) { 333 | printf("Usage: test_correctness [filesize times sleep_secs]\n"); 334 | return 1; 335 | } 336 | 337 | /* read parameters from command line, if any */ 338 | if (argc > 1) { 339 | filesize = (size_t) atol(argv[1]); 340 | times = atoi(argv[2]); 341 | seconds = atoi(argv[3]); 342 | } 343 | 344 | MPI_Init(&argc, &argv); 345 | 346 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 347 | MPI_Comm_size(MPI_COMM_WORLD, &ranks); 348 | 349 | char name[256]; 350 | sprintf(name, "/tmp/rank.%d", rank); 351 | 352 | /* allocate space for the checkpoint data (make filesize a function of rank for some variation) */ 353 | filesize = filesize + rank; 354 | 355 | cruise_mount("/tmp", filesize, rank); 356 | 357 | /* verify data integrity in file */ 358 | checkdata(name, filesize, times); 359 | 360 | MPI_Finalize(); 361 | 362 | return 0; 363 | } 364 | -------------------------------------------------------------------------------- /tests/test_ungetc.c: -------------------------------------------------------------------------------- 1 | // build: mpigcc -g -O3 -o test_ramdisk test_ramdisk.c 2 | // run: srun -n64 -N4 ./test_ramdisk 3 | 4 | #define _GNU_SOURCE 1 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | int rank = -1; 16 | int ranks = 0; 17 | 18 | int main (int argc, char* argv[]) 19 | { 20 | /* check that we got an appropriate number of arguments */ 21 | if (argc != 1 && argc != 4) { 22 | printf("Usage: test_correctness [filesize times sleep_secs]\n"); 23 | return 1; 24 | } 25 | 26 | char name[256]; 27 | sprintf(name, "/tmp/rank.%d", rank); 28 | 29 | cruise_mount("/tmp", 100*1024*1024, rank); 30 | 31 | FILE* fp; 32 | int c, rc; 33 | off_t pos; 34 | 35 | rc = remove(name); 36 | 37 | fp = fopen(name, "w+"); 38 | if (fp == NULL) { 39 | printf("Failed to open %s\n", name); 40 | return 1; 41 | } 42 | rc = fputc(1, fp); 43 | rc = fputc(2, fp); 44 | rc = fputc(3, fp); 45 | rc = fclose(fp); 46 | 47 | fp = fopen(name, "a+"); 48 | if (fp == NULL) { 49 | printf("Failed to open %s\n", name); 50 | return 1; 51 | } 52 | rc = fputc((int)'t', fp); 53 | pos = ftello(fp); 54 | rc = fseek(fp, 1, SEEK_SET); 55 | pos = ftello(fp); 56 | c = fgetc(fp); 57 | rc = fputc((int)'t', fp); 58 | pos = ftello(fp); 59 | rc = fclose(fp); 60 | 61 | fp = fopen(name, "w+"); 62 | if (fp == NULL) { 63 | printf("Failed to open %s\n", name); 64 | return 1; 65 | } 66 | rc = fseek(fp, 4, SEEK_SET); 67 | rc = ungetc((int)'c', fp); 68 | rc = ungetc((int)'b', fp); 69 | rc = ungetc((int)'a', fp); 70 | rc = fputc((int)'1', fp); 71 | while (1) { 72 | c = fgetc(fp); 73 | if (feof(fp)) { 74 | break; 75 | } 76 | if (c == EOF) { 77 | printf("ERROR: %s\n", strerror(errno)); 78 | break; 79 | } 80 | printf("%d %c\n", c, (char)c); 81 | } 82 | // c = fgetc(fp); 83 | // if (c != (int)'b') { 84 | // printf("Failed to open %s\n", name); 85 | // return 1; 86 | // } 87 | rc = fclose(fp); 88 | 89 | fp = fopen(name, "r"); 90 | while (1) { 91 | c = fgetc(fp); 92 | if (feof(fp)) { 93 | break; 94 | } 95 | printf("%d %c\n", c, (char)c); 96 | } 97 | fclose(fp); 98 | 99 | fp = fopen(name, "r+"); 100 | rc = fseek(fp, 5, SEEK_END); 101 | rc = fputc((int)'2', fp); 102 | rc = ungetc((int)'x', fp); 103 | rc = ungetc((int)'y', fp); 104 | rc = ungetc((int)'z', fp); 105 | while (1) { 106 | c = fgetc(fp); 107 | if (feof(fp)) { 108 | break; 109 | } 110 | if (c == EOF) { 111 | printf("ERROR: %s\n", strerror(errno)); 112 | break; 113 | } 114 | printf("%d %c\n", c, (char)c); 115 | } 116 | rc = fclose(fp); 117 | 118 | fp = fopen(name, "r"); 119 | while (1) { 120 | c = fgetc(fp); 121 | if (feof(fp)) { 122 | break; 123 | } 124 | printf("%d %c\n", c, (char)c); 125 | } 126 | fclose(fp); 127 | 128 | return 0; 129 | } 130 | -------------------------------------------------------------------------------- /tests/test_writeread.c: -------------------------------------------------------------------------------- 1 | // build: mpigcc -g -O3 -o test_ramdisk test_ramdisk.c 2 | // run: srun -n64 -N4 ./test_ramdisk 3 | 4 | #define _GNU_SOURCE 1 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "mpi.h" 15 | 16 | //size_t filesize = 100*1024*1024; 17 | size_t filesize = 1024*1024; 18 | int times = 5; 19 | int seconds = 0; 20 | int rank = -1; 21 | int ranks = 0; 22 | 23 | /* reliable read from file descriptor (retries, if necessary, until hard error) */ 24 | int reliable_read(int fd, void* buf, size_t size) 25 | { 26 | size_t n = 0; 27 | int retries = 10; 28 | int rank; 29 | char host[128]; 30 | while (n < size) 31 | { 32 | int rc = read(fd, (char*) buf + n, size - n); 33 | if (rc > 0) { 34 | n += rc; 35 | } else if (rc == 0) { 36 | /* EOF */ 37 | return n; 38 | } else { /* (rc < 0) */ 39 | /* got an error, check whether it was serious */ 40 | if(errno == EINTR || errno == EAGAIN) { 41 | continue; 42 | } 43 | 44 | /* something worth printing an error about */ 45 | retries--; 46 | if (retries) { 47 | /* print an error and try again */ 48 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 49 | gethostname(host, sizeof(host)); 50 | printf("%d on %s: ERROR: Error reading: read(%d, %p, %ld) errno=%d %s @ %s:%d\n", 51 | rank, host, fd, (char*) buf + n, size - n, errno, strerror(errno), __FILE__, __LINE__ 52 | ); 53 | } else { 54 | /* too many failed retries, give up */ 55 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 56 | gethostname(host, sizeof(host)); 57 | printf("%d on %s: ERROR: Giving up read: read(%d, %p, %ld) errno=%d %s @ %s:%d\n", 58 | rank, host, fd, (char*) buf + n, size - n, errno, strerror(errno), __FILE__, __LINE__ 59 | ); 60 | MPI_Abort(MPI_COMM_WORLD, 0); 61 | } 62 | } 63 | } 64 | return size; 65 | } 66 | 67 | /* reliable write to file descriptor (retries, if necessary, until hard error) */ 68 | int reliable_write(int fd, const void* buf, size_t size) 69 | { 70 | size_t n = 0; 71 | int retries = 10; 72 | int rank; 73 | char host[128]; 74 | while (n < size) 75 | { 76 | int rc = write(fd, (char*) buf + n, size - n); 77 | if (rc > 0) { 78 | n += rc; 79 | } else if (rc == 0) { 80 | /* something bad happened, print an error and abort */ 81 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 82 | gethostname(host, sizeof(host)); 83 | printf("%d on %s: ERROR: Error writing: write(%d, %p, %ld) returned 0 @ %s:%d\n", 84 | rank, host, fd, (char*) buf + n, size - n, __FILE__, __LINE__ 85 | ); 86 | MPI_Abort(MPI_COMM_WORLD, 0); 87 | } else { /* (rc < 0) */ 88 | /* got an error, check whether it was serious */ 89 | if(errno == EINTR || errno == EAGAIN) { 90 | continue; 91 | } 92 | 93 | /* something worth printing an error about */ 94 | retries--; 95 | if (retries) { 96 | /* print an error and try again */ 97 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 98 | gethostname(host, sizeof(host)); 99 | printf("%d on %s: ERROR: Error writing: write(%d, %p, %ld) errno=%d %s @ %s:%d\n", 100 | rank, host, fd, (char*) buf + n, size - n, errno, strerror(errno), __FILE__, __LINE__ 101 | ); 102 | } else { 103 | /* too many failed retries, give up */ 104 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 105 | gethostname(host, sizeof(host)); 106 | printf("%d on %s: ERROR: Giving up write: write(%d, %p, %ld) errno=%d %s @ %s:%d\n", 107 | rank, host, fd, (char*) buf + n, size - n, errno, strerror(errno), __FILE__, __LINE__ 108 | ); 109 | MPI_Abort(MPI_COMM_WORLD, 0); 110 | } 111 | } 112 | } 113 | return size; 114 | } 115 | 116 | /* initialize buffer with some well-known value based on rank */ 117 | int init_buffer(char* buf, size_t size, int rank, int ckpt) 118 | { 119 | size_t i; 120 | for(i=0; i < size; i++) { 121 | char c = 'a' + (char)((rank + ckpt + i) & 32); 122 | buf[i] = c; 123 | } 124 | return 0; 125 | } 126 | 127 | /* checks buffer for expected value */ 128 | int check_buffer(char* buf, size_t size, int rank, int ckpt) 129 | { 130 | size_t i; 131 | for(i=0; i < size; i++) { 132 | char c = 'a' + (char)((rank + ckpt + i) & 32); 133 | if (buf[i] != c) { 134 | printf("check failed at byte %d, should be %c is %c\n", (int)i, c, buf[i]); 135 | return 0; 136 | } 137 | } 138 | return 1; 139 | } 140 | 141 | /* read the checkpoint data from file into buf, and return whether the read was successful */ 142 | int read_checkpoint(int fd, int* rank, int* ckpt, char* buf, size_t size) 143 | { 144 | unsigned long n; 145 | char rank_buf[7]; 146 | char ckpt_buf[7]; 147 | size_t field_size = 6; 148 | 149 | /* read the rank id */ 150 | n = reliable_read(fd, rank_buf, field_size); 151 | if (n != field_size) { 152 | printf("Failed to read rank\n"); 153 | return 0; 154 | } 155 | rank_buf[6] = '\0'; 156 | 157 | /* read the checkpoint id */ 158 | n = reliable_read(fd, ckpt_buf, field_size); 159 | if (n != field_size) { 160 | printf("Failed to read timestep\n"); 161 | return 0; 162 | } 163 | ckpt_buf[6] = '\0'; 164 | 165 | /* read the checkpoint data, and check the file size */ 166 | n = reliable_read(fd, buf, size+1); 167 | if (n != size) { 168 | printf("Filesize not correct\n"); 169 | return 0; 170 | } 171 | 172 | /* if the file looks good, set the timestep and return */ 173 | sscanf(rank_buf, "%6d", rank); 174 | sscanf(ckpt_buf, "%6d", ckpt); 175 | 176 | return 0; 177 | } 178 | 179 | /* write the checkpoint data to fd, and return whether the write was successful */ 180 | int write_checkpoint(int fd, int rank, int ckpt, char* buf, size_t size) 181 | { 182 | int rc; 183 | int valid = 0; 184 | char rank_buf[7]; 185 | char ckpt_buf[7]; 186 | size_t field_size = 6; 187 | 188 | /* write the rank id */ 189 | sprintf(rank_buf, "%06d", rank); 190 | rc = reliable_write(fd, rank_buf, field_size); 191 | if (rc < 0) { 192 | valid = 0; 193 | } 194 | 195 | /* write the checkpoint id (application timestep) */ 196 | sprintf(ckpt_buf, "%06d", ckpt); 197 | rc = reliable_write(fd, ckpt_buf, field_size); 198 | if (rc < 0) { 199 | valid = 0; 200 | } 201 | 202 | /* write the checkpoint data */ 203 | rc = reliable_write(fd, buf, size); 204 | if (rc < 0) { 205 | valid = 0; 206 | } 207 | 208 | return valid; 209 | } 210 | 211 | void checkdata(char* file, size_t size, int times) 212 | { 213 | char* buf = malloc(size); 214 | 215 | MPI_Barrier(MPI_COMM_WORLD); 216 | 217 | if (times > 0) { 218 | /* write the checkpoint file */ 219 | int i, j; 220 | for(i=0; i < times; i++) { 221 | int rc; 222 | int valid = 0; 223 | 224 | rc = init_buffer(buf, size, rank, i); 225 | 226 | if (rank == 0) { 227 | printf("Writing checkpoint %d.\n", i); fflush(stdout); 228 | } 229 | 230 | /* open the file and write the checkpoint */ 231 | int fd_me = open(file, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); 232 | if (fd_me > 0) { 233 | valid = 1; 234 | 235 | /* write the checkpoint data */ 236 | rc = write_checkpoint(fd_me, rank, i, buf, size); 237 | if (rc < 0) { 238 | valid = 0; 239 | } 240 | 241 | /* force the data to storage */ 242 | rc = fsync(fd_me); 243 | if (rc < 0) { 244 | valid = 0; 245 | } 246 | 247 | /* make sure the close is without error */ 248 | rc = close(fd_me); 249 | if (rc < 0) { 250 | valid = 0; 251 | } 252 | } 253 | 254 | if (rank == 0) { 255 | printf("Completed checkpoint %d.\n", i); fflush(stdout); 256 | } 257 | 258 | if (rank == 0) { 259 | printf("Reading checkpoint %d.\n", i); fflush(stdout); 260 | } 261 | 262 | memset(buf, 0, size); 263 | 264 | /* open the file and write the checkpoint */ 265 | int read_rank, read_timestep; 266 | fd_me = open(file, O_RDONLY); 267 | if (fd_me > 0) { 268 | valid = 1; 269 | 270 | /* write the checkpoint data */ 271 | rc = read_checkpoint(fd_me, &read_rank, &read_timestep, buf, size); 272 | if (rc < 0) { 273 | valid = 0; 274 | } 275 | 276 | /* make sure the close is without error */ 277 | rc = close(fd_me); 278 | if (rc < 0) { 279 | valid = 0; 280 | } 281 | 282 | } 283 | 284 | if (read_rank != rank || read_timestep != i) { 285 | printf("INVALID HEADER on rank %d in step %d\n", rank, i); fflush(stdout); 286 | MPI_Abort(MPI_COMM_WORLD, 0); 287 | } 288 | 289 | rc = check_buffer(buf, size, rank, i); 290 | if (! rc) { 291 | printf("INVALID DATA on rank %d in step %d\n", rank, i); fflush(stdout); 292 | MPI_Abort(MPI_COMM_WORLD, 0); 293 | } 294 | 295 | if (rank == 0) { 296 | printf("Verified checkpoint %d.\n", read_timestep); fflush(stdout); 297 | } 298 | 299 | /* optionally sleep for some time */ 300 | if (seconds > 0) { 301 | if (rank == 0) { 302 | printf("Sleeping for %d seconds... \n", seconds); fflush(stdout); 303 | } 304 | sleep(seconds); 305 | } 306 | 307 | unlink(file); 308 | } 309 | } 310 | 311 | MPI_Barrier(MPI_COMM_WORLD); 312 | 313 | if (buf != NULL) { 314 | free(buf); 315 | buf = NULL; 316 | } 317 | 318 | return; 319 | } 320 | 321 | int main (int argc, char* argv[]) 322 | { 323 | /* check that we got an appropriate number of arguments */ 324 | if (argc != 1 && argc != 4) { 325 | printf("Usage: test_correctness [filesize times sleep_secs]\n"); 326 | return 1; 327 | } 328 | 329 | /* read parameters from command line, if any */ 330 | if (argc > 1) { 331 | filesize = (size_t) atol(argv[1]); 332 | times = atoi(argv[2]); 333 | seconds = atoi(argv[3]); 334 | } 335 | 336 | MPI_Init(&argc, &argv); 337 | 338 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 339 | MPI_Comm_size(MPI_COMM_WORLD, &ranks); 340 | 341 | cruise_mount("/tmp",1024,0); 342 | 343 | char name[256]; 344 | sprintf(name, "/tmp/rank.%d", rank); 345 | 346 | /* allocate space for the checkpoint data (make filesize a function of rank for some variation) */ 347 | filesize = filesize + rank; 348 | 349 | /* verify data integrity in file */ 350 | checkdata(name, filesize, times); 351 | 352 | MPI_Finalize(); 353 | 354 | return 0; 355 | } 356 | -------------------------------------------------------------------------------- /tests/test_wscanf.c: -------------------------------------------------------------------------------- 1 | // build: mpigcc -g -O3 -o test_ramdisk test_ramdisk.c 2 | // run: srun -n64 -N4 ./test_ramdisk 3 | 4 | #define _GNU_SOURCE 1 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | int rank = -1; 17 | int ranks = 0; 18 | 19 | int main (int argc, char* argv[]) 20 | { 21 | /* check that we got an appropriate number of arguments */ 22 | if (argc != 1 && argc != 4) { 23 | printf("Usage: test_correctness [filesize times sleep_secs]\n"); 24 | return 1; 25 | } 26 | 27 | char name[256]; 28 | sprintf(name, "/tmp/rank.%d", rank); 29 | 30 | cruise_mount("/tmp", 100*1024*1024, rank); 31 | 32 | FILE* fp; 33 | wint_t c; 34 | int rc; 35 | off_t pos; 36 | 37 | rc = remove(name); 38 | 39 | fp = fopen(name, "w"); 40 | if (fp == NULL) { 41 | printf("Failed to open %s\n", name); 42 | return 1; 43 | } 44 | rc = fwprintf(fp, "%d %f %lf %x %u %s %s\n", 143, 25.634, 2.5e3, 345, 1, "hi", "there_world"); 45 | rc = fclose(fp); 46 | 47 | #if 0 48 | fp = fopen(name, "r"); 49 | while (1) { 50 | c = fgetwc(fp); 51 | if (feof(fp)) { 52 | break; 53 | } 54 | printf("%d %c\n", c, (char)c); 55 | } 56 | fclose(fp); 57 | 58 | int i1, i2; 59 | float f1; 60 | double d1; 61 | unsigned long ul1; 62 | char s1[256], s2[256]; 63 | fp = fopen(name, "r"); 64 | rc = fwscanf(fp, "%d %f %lf %x %lu %s %s\n", &i1, &f1, &d1, &i2, &ul1, s1, s2); 65 | fclose(fp); 66 | 67 | fp = fopen(name, "a+"); 68 | if (fp == NULL) { 69 | printf("Failed to open %s\n", name); 70 | return 1; 71 | } 72 | rc = fputwc((wint_t)'t', fp); 73 | pos = ftello(fp); 74 | rc = fseek(fp, 1, SEEK_SET); 75 | pos = ftello(fp); 76 | c = fgetwc(fp); 77 | pos = ftello(fp); 78 | rc = fclose(fp); 79 | 80 | fp = fopen(name, "r+"); 81 | if (fp == NULL) { 82 | printf("Failed to open %s\n", name); 83 | return 1; 84 | } 85 | rc = fseek(fp, 4, SEEK_SET); 86 | rc = ungetwc((wint_t)'c', fp); 87 | rc = ungetwc((wint_t)'b', fp); 88 | rc = ungetwc((wint_t)'a', fp); 89 | while (1) { 90 | c = fgetwc(fp); 91 | if (feof(fp)) { 92 | break; 93 | } 94 | if (c == WEOF) { 95 | printf("ERROR: %s\n", strerror(errno)); 96 | break; 97 | } 98 | printf("%d %c\n", c, (char)c); 99 | } 100 | rc = fclose(fp); 101 | 102 | fp = fopen(name, "r"); 103 | while (1) { 104 | c = fgetc(fp); 105 | if (feof(fp)) { 106 | break; 107 | } 108 | printf("%d %c\n", c, (char)c); 109 | } 110 | fclose(fp); 111 | 112 | fp = fopen(name, "r+"); 113 | rc = fseek(fp, 5, SEEK_END); 114 | rc = fputc((int)'2', fp); 115 | rc = ungetc((int)'x', fp); 116 | rc = ungetc((int)'y', fp); 117 | rc = ungetc((int)'z', fp); 118 | while (1) { 119 | c = fgetc(fp); 120 | if (feof(fp)) { 121 | break; 122 | } 123 | if (c == EOF) { 124 | printf("ERROR: %s\n", strerror(errno)); 125 | break; 126 | } 127 | printf("%d %c\n", c, (char)c); 128 | } 129 | rc = fclose(fp); 130 | 131 | fp = fopen(name, "r"); 132 | while (1) { 133 | c = fgetc(fp); 134 | if (feof(fp)) { 135 | break; 136 | } 137 | printf("%d %c\n", c, (char)c); 138 | } 139 | fclose(fp); 140 | #endif 141 | 142 | return 0; 143 | } 144 | --------------------------------------------------------------------------------