├── BurstFS_Client ├── prepare ├── maint │ ├── check_fns │ │ ├── scrmfs_unsupported_list.txt │ │ ├── posix_list.txt │ │ ├── cstdio_list.txt │ │ ├── fakechroot_list.txt │ │ └── scrmfs_check_fns │ └── config │ │ ├── check_arch.m4 │ │ ├── inttypes-pri.m4 │ │ └── check_numa.m4 ├── buildme_zlib ├── buildme_bgq ├── buildme_opt ├── ipc_cleanup ├── .gitignore ├── cruise-config.in ├── cruise-defs.h ├── src │ ├── cruise-stdio.h │ ├── cruise-sysio.h │ ├── cruise-stack.h │ ├── cruise-fixed.h │ ├── cruise.h │ └── cruise-stack.c ├── LICENSE.ANL ├── LICENSE ├── LICENSE.CRUISE ├── INSTALL ├── tests │ ├── Makefile.gcc │ ├── makefile.bgq │ └── test_read.c ├── cruise-runtime-config.h.in └── Makefile.in ├── BurstFS_Meta ├── tests │ ├── tester │ │ ├── mdhimtst │ │ ├── inTestNext.txt │ │ ├── inTestAppend.txt │ │ ├── inTestRand.txt │ │ ├── inTestBulk.txt │ │ ├── inTestErr1.txt │ │ ├── inTestBasic.txt │ │ ├── runAll.sh │ │ ├── inTestBulkLarge.txt │ │ ├── inTestBulkLargeErr.txt │ │ ├── inTestOps3.txt │ │ ├── inTestOps5.txt │ │ ├── inTestOps6.txt │ │ └── inTestLarge.txt │ └── single_tests │ │ ├── .command.log.swp │ │ ├── mdhim_manifest_1_0_1 │ │ ├── range_test.sh │ │ ├── range_bget.sh │ │ ├── puts-gets.c │ │ ├── put-get.c │ │ ├── put-del.c │ │ ├── Makefile │ │ ├── put-getp.c │ │ ├── put-get_2secondary.c │ │ ├── put-getn.c │ │ ├── put-del_secondary.c │ │ ├── put-del_secondary_local.c │ │ ├── put-get_secondary_local.c │ │ ├── bput-bdel.c │ │ ├── put-get_secondary.c │ │ └── plfs-put-get.c ├── src │ ├── uthash │ │ ├── README.md │ │ └── LICENSE │ ├── client.h │ ├── local_client.h │ ├── mdhim_private.h │ ├── ds_mysql.h │ ├── Makefile │ ├── range_server.h │ ├── ds_leveldb.h │ ├── data_store.c │ ├── partitioner.h │ ├── mdhim_options.h │ └── data_store.h ├── Makefile ├── .gitignore ├── example.Makefile.cfg ├── LICENSE └── README.md ├── BurstFS_Server ├── Makefile.am ├── Makefile ├── burstfs_debug.h ├── burstfs_setup.h ├── runserver.sh ├── runrclient.sh ├── runwclient.sh ├── arraylist.h ├── burstfs_debug.c ├── burstfs_init.h ├── burstfs_sock.h ├── burstfs_cmd_handler.h ├── configure.ac ├── LICENSE ├── COPYRIGHT ├── burstfs_request_manager.h ├── log.h ├── burstfs_metadata.h ├── burstfs_const.c ├── arraylist.c ├── config.h.in ├── burstfs_service_manager.h └── burstfs_global.h ├── README └── LICENSE /BurstFS_Client/prepare: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | aclocal -I ./maint/config && autoheader && autoconf 4 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/tester/mdhimtst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/burstfs/HEAD/BurstFS_Meta/tests/tester/mdhimtst -------------------------------------------------------------------------------- /BurstFS_Meta/src/uthash/README.md: -------------------------------------------------------------------------------- 1 | 2 | Documentation for uthash is available at: 3 | 4 | http://troydhanson.github.com/uthash/ 5 | 6 | 7 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/.command.log.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/burstfs/HEAD/BurstFS_Meta/tests/single_tests/.command.log.swp -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/mdhim_manifest_1_0_1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/burstfs/HEAD/BurstFS_Meta/tests/single_tests/mdhim_manifest_1_0_1 -------------------------------------------------------------------------------- /BurstFS_Meta/Makefile: -------------------------------------------------------------------------------- 1 | include Makefile.cfg 2 | .PHONY: mdhim 3 | 4 | all: mdhim 5 | 6 | mdhim: 7 | make -C src 8 | 9 | clean: 10 | make -C src clean 11 | -------------------------------------------------------------------------------- /BurstFS_Client/maint/check_fns/scrmfs_unsupported_list.txt: -------------------------------------------------------------------------------- 1 | # wrapped but not implemented 2 | 3 | freopen 4 | fwprintf 5 | fwscanf 6 | vfwprintf 7 | vfwscanf 8 | fgetwc 9 | fgetws 10 | fputwc 11 | fputws 12 | fwide 13 | getwc 14 | putwc 15 | ungetwc 16 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/tester/inTestNext.txt: -------------------------------------------------------------------------------- 1 | # Insert small set, later used to text get OPS 2 | bput 6 nptest.txt 3 | flush 4 | # Test differ get ops: EQ (0), NEXT(1), PREV(2), FIRST(3) and LAST (4) 5 | get gt1.txt 6 | get gt2.txt 7 | get gt3.txt 8 | get gt4.txt 9 | get gt5.txt 10 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/tester/inTestAppend.txt: -------------------------------------------------------------------------------- 1 | # Put first copy of record 2 | put 08642 abcdefg 3 | 4 | # Get original copy 5 | get 08642 6 | 7 | # Append by using the -a flag in the tester program 8 | put 08642 ABCEDFG 9 | 10 | # Returned value should be concatenated 11 | get 08642 12 | -------------------------------------------------------------------------------- /BurstFS_Client/buildme_zlib: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | if [ ! -f zlib-1.2.7.tar.gz ] ; then 6 | wget zlib.net/zlib-1.2.7.tar.gz 7 | fi 8 | 9 | rm -rf zlib-1.2.7 10 | tar -zxf zlib-1.2.7.tar.gz 11 | cd zlib-1.2.7 12 | 13 | export CC=mpixlc 14 | ./configure --prefix=`pwd`/install 15 | make CC=mpixlc 16 | make install 17 | -------------------------------------------------------------------------------- /BurstFS_Client/buildme_bgq: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | ./prepare 6 | ./configure \ 7 | --prefix=`pwd`/install_bgq \ 8 | --disable-ld-preload \ 9 | CC=mpixlc CFLAGS="-g -O0 -DMACHINE_BGQ" || exit 1 10 | # CC=mpixlc CFLAGS="-O3 -DMACHINE_BGQ" || exit 1 11 | 12 | make clean || exit 1 13 | make || exit 1 14 | make install || exit 1 15 | -------------------------------------------------------------------------------- /BurstFS_Client/buildme_opt: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . /usr/local/tools/dotkit/init.sh 4 | use -q ic-11.1.046 5 | 6 | set -x 7 | 8 | ./prepare 9 | ./configure \ 10 | --prefix=`pwd`/install \ 11 | --enable-ld-preload \ 12 | CC=mpiicc CFLAGS="-O3 -DENABLE_NUMA_POLICY" || exit 1 13 | 14 | make clean || exit 1 15 | make || exit 1 16 | make install || exit 1 17 | -------------------------------------------------------------------------------- /BurstFS_Meta/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Compiled Dynamic libraries 8 | *.so 9 | *.so* 10 | *.dylib 11 | *.dll 12 | 13 | # Compiled Static libraries 14 | *.lai 15 | *.la 16 | *.a 17 | *.lib 18 | 19 | # Executables 20 | *.exe 21 | *.out 22 | *.app 23 | 24 | # Makefile and Makefile config 25 | #Makefile 26 | #Makefile.cfg 27 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/range_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ulimit -c unlimited 4 | export MPICH_MAX_THREAD_SAFETY=multiple 5 | 6 | nodes=$SLURM_NNODES 7 | BATCH_CNT=1024 8 | 9 | for BULK_NUM in ${BATCH_CNT}; do 10 | srun --clear-ssd -n${SLURM_NNODES} -N${SLURM_NNODES} ./range_test -c ${BATCH_CNT} -s 1 -t 16384 -r 1048576 -n 131072 -p /l/ssd/ -d db 11 | done 12 | 13 | -------------------------------------------------------------------------------- /BurstFS_Client/maint/check_fns/posix_list.txt: -------------------------------------------------------------------------------- 1 | readlink 2 | readlinkat 3 | symlink 4 | symlinkat 5 | 6 | chown 7 | fchown 8 | lchown 9 | fchownat 10 | 11 | chmod 12 | fchmod 13 | fchmodat 14 | 15 | utime 16 | utimes 17 | 18 | fcntl 19 | fstatvs 20 | 21 | mknod 22 | mknodat 23 | 24 | chdir 25 | getcwd 26 | getwd 27 | get_current_dir_name 28 | 29 | opendir 30 | rewinddir 31 | readdir 32 | readdir_r 33 | closedir 34 | 35 | ftw 36 | nftw 37 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/range_bget.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | ulimit -c unlimited 6 | export MPICH_MAX_THREAD_SAFETY=multiple 7 | 8 | nnodes=1 9 | nprocs=1 10 | # determine number of nodes in our allocation 11 | nodes=$SLURM_NNODES 12 | SEGNUM=1024 13 | BULKNUM=$SEGNUM 14 | srun --clear-ssd -n${nprocs} -N${nnodes} ./range_bget -c ${BULKNUM} -s 16 -t 1048576 -g 1048576 -r 1048576 -n ${SEGNUM} -p /l/ssd/ -d db 2>&1|tee range_${SLURM_NNODES}.log 15 | 16 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/tester/inTestRand.txt: -------------------------------------------------------------------------------- 1 | # Put n=40000 set of random records, key_size (for string/byte) of 128 chars 2 | # value_size of 2040 chars. Last param makes sizes to be exact. If a 0 3 | # is used instead then sizes are 1 char up to x_size. 4 | nput 40000 500 2040 1 5 | 6 | flush 7 | 8 | # get a sequence of n record starting starting with a random value or zero if 9 | # the second parameter is set to zero. The last parameter works in a similar 10 | # manner as stated above. 11 | ngetn 5000 500 1 12 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/tester/inTestBulk.txt: -------------------------------------------------------------------------------- 1 | # Put a good size numberor key/values 2 | bput 99999 56789 abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ 3 | 4 | # Get all the above key/values 5 | bget 99999 56789 6 | 7 | # Required to do next ops 8 | flush 9 | 10 | # Get a small set using the bulk get next operation 11 | bgetop 500 56789 1 12 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/tester/inTestErr1.txt: -------------------------------------------------------------------------------- 1 | # Basic test deck should be run first and DB should not be removed. 2 | 3 | # Simple bget should work 4 | bget 10 987654 5 | 6 | # Get using an op othern than EQ should fail if no flush has been done 7 | get 123456 1 8 | 9 | # This should enable the command below to work 10 | flush 11 | get 123456 1 12 | 13 | # Should fail 14 | get 13579 0 15 | 16 | # Should work as we are not using op=EQ (gets the next value in the DB after 17 | # the seed) 18 | get 13579 1 19 | 20 | # Should fail 21 | bdel 10 24680 22 | 23 | # Should fail as incorrec operator 24 | get 123456 9 25 | -------------------------------------------------------------------------------- /BurstFS_Server/Makefile.am: -------------------------------------------------------------------------------- 1 | AUTOMAKE_OPTIONS = foreign 2 | noinst_PROGRAMS=burstfsd 3 | burstfsd_SOURCES = burstfs_init.c burstfs_init.h \ 4 | burstfs_cmd_handler.c burstfs_cmd_handler.h \ 5 | burstfs_request_manager.c burstfs_request_manager.h \ 6 | burstfs_service_manager.c burstfs_service_manager.h \ 7 | burstfs_metadata.c burstfs_metadata.h \ 8 | burstfs_debug.c burstfs_debug.h \ 9 | arraylist.c arraylist.h \ 10 | burstfs_const.c burstfs_const.h \ 11 | burstfs_sock.c burstfs_sock.h 12 | burstfsd_LDADD = -lpthread -lm -lstdc++ -lrt 13 | -------------------------------------------------------------------------------- /BurstFS_Client/ipc_cleanup: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ########################################################################### 4 | # $URL: file:///var/svn/cfengine/llnl/admin/scripts/ipc_cleanup $ 5 | # $Author: tdhooge $ 6 | # $Date: 2011-06-27 11:06:30 -0700 (Mon, 27 Jun 2011) $ 7 | # $Rev: 295 $ 8 | ########################################################################### 9 | 10 | # Clean up shared memory after job 11 | 12 | NAME=$USER #`awk -F: -v TMP=$SLURM_UID '$3 == TMP { print $1 }' /etc/passwd` 13 | 14 | for key in $(ipcs -mc | grep -v '^(---|key)' | awk -v USER=${NAME} '$3 == USER {print $1}'); do ipcrm -m $key; done 15 | -------------------------------------------------------------------------------- /BurstFS_Server/Makefile: -------------------------------------------------------------------------------- 1 | #METASTORE_HOME=/p/lscratchf/wang86/MetaStore/src 2 | #LEVELDB_HOME=/g/g92/wang86/leveldb/leveldb 3 | 4 | METASTORE_HOME=../BurstFS_Meta/src 5 | 6 | all: server 7 | 8 | server: 9 | mpicc -g -O2 -Wall arraylist.c burstfs_init.c burstfs_cmd_handler.c \ 10 | burstfs_debug.c burstfs_const.c burstfs_metadata.c \ 11 | burstfs_request_manager.c burstfs_service_manager.c burstfs_sock.c \ 12 | -I${METASTORE_HOME}/Mlog/ -I${METASTORE_HOME}/uthash -I${METASTORE_HOME} \ 13 | -I${LEVELDB_HOME}/include -L${METASTORE_HOME} -DLEVELDB_SUPPORT \ 14 | -lmdhim -lm -pthread -lstdc++ -lrt -lleveldb -o server 15 | 16 | clean: 17 | rm server 18 | 19 | -------------------------------------------------------------------------------- /BurstFS_Client/maint/config/check_arch.m4: -------------------------------------------------------------------------------- 1 | dnl @synopsis CHECK_ARCH() 2 | dnl 3 | dnl This macro checks for the architectur (BGQ/Linux) and defines the 4 | dnl appropriate CFLAGS in the code. BG/Q builds also disable NUMA support 5 | 6 | AC_DEFUN([CHECK_ARCH], 7 | 8 | # 9 | # Handle user hints 10 | # 11 | [AC_MSG_CHECKING(architecture type) 12 | AC_ARG_WITH([arch], 13 | [AS_HELP_STRING([--with-numa=ARCH],[specify the architecture as bgq or linux])], 14 | [if test "$withval" != no ; then 15 | AC_MSG_RESULT(yes) 16 | ARCH=$withval 17 | if test "${ARCH}" = "bgq" 18 | then 19 | AC_DEFINE([MACHINE_BGQ], [1], [Define if architecture is BG/Q]) 20 | fi 21 | else 22 | AC_MSG_RESULT(no) 23 | fi]) 24 | 25 | ]) 26 | -------------------------------------------------------------------------------- /BurstFS_Client/.gitignore: -------------------------------------------------------------------------------- 1 | Makefile 2 | aclocal.m4 3 | autom4te.cache/ 4 | config.log 5 | config.status 6 | configure 7 | cruise-runtime-config.h 8 | install/ 9 | src/cruise-fixed.o 10 | src/cruise-fixed.po 11 | src/cruise-stack.o 12 | src/cruise-stack.po 13 | src/cruise-stdio.o 14 | src/cruise-stdio.po 15 | src/cruise-sysio.o 16 | src/cruise-sysio.po 17 | src/cruise.o 18 | src/cruise.po 19 | src/libcruise-posix.a 20 | src/libcruise.so 21 | src/lookup3.o 22 | src/lookup3.po 23 | src/lookup8.o 24 | src/lookup8.po 25 | tests/a.out 26 | tests/sizes 27 | tests/test1 28 | tests/test_fopen 29 | tests/test_fprintf 30 | tests/test_memcpy 31 | tests/test_ramdisk 32 | tests/test_scanf 33 | tests/test_truncate 34 | tests/test_ungetc 35 | tests/test_writeread 36 | tests/test_wscanf 37 | cscope.out 38 | -------------------------------------------------------------------------------- /BurstFS_Server/burstfs_debug.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | #ifndef BURSTFS_DEBUG_H 16 | #define BURSTFS_DEBUG_H 17 | #include 18 | 19 | int dbg_open(char *fname); 20 | 21 | int dbg_close(); 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /BurstFS_Server/burstfs_setup.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | #ifndef BURSTFS_SETUP_H 16 | #define BURSTFS_SETUP_H 17 | #include "burstfs_const.h" 18 | extern int *local_rank_lst; 19 | extern int rank, size; 20 | extern arraylist_t *app_config_list; 21 | extern int invert_sock_ids[MAX_NUM_CLIENTS]; 22 | #endif 23 | -------------------------------------------------------------------------------- /BurstFS_Server/runserver.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | killall srun ##kill the previous daemon if there is any remaining 4 | 5 | export BURSTFS_META_SERVER_RATIO=1 #determines the number of range servers in the key-value store 6 | export BURSTFS_META_DB_NAME=burstfs_db 7 | export CRUISE_CHUNK_MEM=0 8 | 9 | procs_per_node=1 10 | nnodes=1 11 | nprocs=`echo "${procs_per_node}*${nnodes}"|bc -l|xargs printf "%1.0f"` 12 | nlist=`scontrol show hostnames ${SLURM_NODELIST}|paste -d, -s|cut -d ',' -f1-${nnodes}` 13 | 14 | #clear all the leftovers 15 | #for ((i=1; i<=${nnodes}; i++)); do 16 | # remote_host=`echo "$nlist"|cut -d ',' -f${i}` 17 | # rsh ${remote_host} "rm -rf /tmp/burstfs*; rm /dev/shm/*; rm /tmp/mdhim*" #clear the leftovers from previous daemon 18 | #done 19 | 20 | ulimit -c unlimited 21 | srun --clear-ssd --nodelist=${nlist} --ntasks-per-node=${procs_per_node} --distribution=block -n ${nprocs} -N ${nnodes} ./burstfsd 2>&1 |tee server_${nprocs}.log & 22 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/tester/inTestBasic.txt: -------------------------------------------------------------------------------- 1 | # Key is 6_digits * rank, value is 4*30=120 chars 2 | # for string/byte 6_digit && 0rank 3 | put 123456 abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456 4 | get 123456 5 | 6 | flush 7 | 8 | # Key is 6_digits * rank + (1 to n), value is 40 chars 9 | # for string/byte 6_digit && 0rank && 0i 10 | bput 999 987654 1234ABCDEFGHIJKLMNOPQRSTUVWXYZ_+=$%*!@~ZYXWVUTSRQPONMLKJIHGFEDCBA3456 11 | bget 999 987654 12 | del 123456 13 | 14 | flush 15 | 16 | # key is 5_digits * rank + (1 to n), value is 241 17 | # for string/byte 5_digit && 0rank && 0i 18 | bput 999 56789 YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz1234_abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456i 19 | 20 | bdel 999 56789 21 | 22 | -------------------------------------------------------------------------------- /BurstFS_Client/cruise-config.in: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # (C) 2012 by Argonne National Laboratory. 5 | # See COPYRIGHT in top-level directory. 6 | # 7 | 8 | 9 | 10 | CP_WRAPPERS="@CP_WRAPPERS@" 11 | CRUISE_LIB_PATH="@cruise_lib_path@" 12 | CRUISE_LD_FLAGS="@LDFLAGS@" 13 | 14 | PRE_LD_FLAGS="-L$CRUISE_LIB_PATH $CRUISE_LD_FLAGS -lz $CP_WRAPPERS" 15 | POST_LD_FLAGS="-L$CRUISE_LIB_PATH -lcruise-posix -lcrypto -lm" 16 | 17 | 18 | usage="\ 19 | Usage: cruise-config [--pre-ld-flags] [--post-ld-flags]" 20 | 21 | if test $# -eq 0; then 22 | echo "${usage}" 1>&2 23 | exit 1 24 | fi 25 | 26 | while test $# -gt 0; do 27 | case "$1" in 28 | -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;; 29 | *) optarg= ;; 30 | esac 31 | 32 | case $1 in 33 | --pre-ld-flags) 34 | echo $PRE_LD_FLAGS 35 | ;; 36 | --post-ld-flags) 37 | echo $POST_LD_FLAGS 38 | ;; 39 | *) 40 | echo "${usage}" 1>&2 41 | exit 1 42 | ;; 43 | esac 44 | shift 45 | done 46 | -------------------------------------------------------------------------------- /BurstFS_Meta/example.Makefile.cfg: -------------------------------------------------------------------------------- 1 | #Configure your options here 2 | MPI_ROOT = /usr/local/tools/mvapich2-gnu-2.1 3 | LEVEL_DB_ROOT = /g/g92/wang86/leveldb 4 | ROCKS_DB_ROOT = /users/hng/rocksdb-2.6.fb 5 | MYSQL_DB_ROOT = /usr/include/mysql 6 | LEVELDB = 1 7 | ROCKSDB = 0 8 | MYSQLDB = 0 9 | CC = mpicc -g -O2 -Wall -fPIC 10 | 11 | #You shouldn't have to modify anything below 12 | CLIBS = -L$(MPI_ROOT)/lib 13 | CINC = -I$(MPI_ROOT)/include -IMlog/ -Iuthash 14 | CFLAGS = -lmpich -lmdhim -lm -lpthread -lstdc++ 15 | ifeq ($(LEVELDB),1) 16 | CLIBS += -L$(LEVEL_DB_ROOT) -DLEVELDB_SUPPORT 17 | CINC += -I$(LEVEL_DB_ROOT)/include 18 | CFLAGS += -L$(LEVEL_DB_ROOT) $(LEVEL_DB_ROOT)/libleveldb.a 19 | endif 20 | 21 | ifeq ($(ROCKSDB),1) 22 | CLIBS += -L$(ROCKS_DB_ROOT) -DROCKSDB_SUPPORT 23 | CINC += -I$(ROCKS_DB_ROOT)/include 24 | CFLAGS += -lsnappy -lbz2 $(ROCKS_DB_ROOT)/librocksdb.a 25 | endif 26 | 27 | 28 | ifeq ($(MYSQLDB),1) 29 | CLIBS += -L/usr/lib/mysql -DMYSQLDB_SUPPORT 30 | CINC += -I/usr/include/mysql 31 | CFLAGS += `mysql_config --cflags --libs` --std=gnu99 32 | endif 33 | -------------------------------------------------------------------------------- /BurstFS_Client/cruise-defs.h: -------------------------------------------------------------------------------- 1 | #define CRUISE_MAX_FILES ( 128 ) 2 | 3 | /* eventually could decouple these so there could be 4 | * more or less file descriptors than files, but for 5 | * now they're the same */ 6 | #define CRUISE_MAX_FILEDESCS ( CRUISE_MAX_FILES ) 7 | 8 | #define CRUISE_MAX_FILENAME ( 128 ) 9 | 10 | #define CRUISE_STREAM_BUFSIZE ( 1 * 1024 * 1024 ) 11 | 12 | #define CRUISE_CHUNK_BITS ( 24 ) 13 | 14 | #ifdef MACHINE_BGQ 15 | #define CRUISE_CHUNK_MEM ( 64 * 1024 * 1024 ) 16 | #else /* MACHINE_BGQ */ 17 | #define CRUISE_CHUNK_MEM ( 256 * 1024 * 1024 ) 18 | #endif /* MACHINE_BGQ */ 19 | 20 | #define CRUISE_SPILLOVER_SIZE ( 1 * 1024 * 1024 * 1024 ) 21 | 22 | #define CRUISE_SUPERBLOCK_KEY ( 4321 ) 23 | 24 | // const for burstfs 25 | #define GEN_STR_LEN 1024 26 | #define SOCKET_PATH "/tmp/burstfs_server_sock" 27 | #define CRUISE_DEF_REQ_SIZE 1024*1024*8*16 + 131072 28 | #define CRUISE_DEF_RECV_SIZE 1024*1024 + 131072 29 | #define BURSTFS_INDEX_BUF_SIZE (20*1024*1024) 30 | #define BURSTFS_FATTR_BUF_SIZE 1024*1024 31 | #define BURSTFS_MAX_SPLIT_CNT 1048576 32 | #define BURSTFS_MAX_READ_CNT 1048576 33 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | =============================================================================== 2 | BurstFS: A Distributed Burst Buffer File System - 0.1.1 3 | =============================================================================== 4 | 5 | Please note that the current implementation of BurstFS is not of production 6 | quality, and is for research purposes only. 7 | 8 | How to build (with CRUISE tests): 9 | 1. download and build leveldb-1.14 (wget https://github.com/google/leveldb/archive/v1.14.tar.gz). 10 | Then, change the path in BurstFS_Meta/Makefile.cfg to point to your leveldb directory 11 | 2. change directory to BurstFS_Meta 12 | make 13 | 3. change directory to BurstFS_Client 14 | ./buildme_opt 15 | cd tests 16 | ./buildme.sh 17 | 4. change directory to BurstFS_Server 18 | ./buildme_autotools (the first time you build the server only) 19 | ./buildme.sh 20 | 21 | How to run (with CRUISE tests): 22 | 1. Make sure you are in BurstFS_Server directory 23 | 2. allocate a node 24 | 3. ./runserver.sh 25 | 4. ./runwrclient.sh 26 | 27 | Note: If at Oak Ridge you will probably have to change the MPI path as well in BurstFS_Meta/Makefile.cfg in the 1st build step. 28 | -------------------------------------------------------------------------------- /BurstFS_Server/runrclient.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | export CRUISE_USE_SPILLOVER=1 3 | CRUISE_DIR=../BurstFS_Client/tests 4 | 5 | procs_per_node=1 6 | nnodes=1 7 | nprocs=`echo "${procs_per_node}*${nnodes}"|bc -l|xargs printf '%1.0f'` 8 | 9 | TO_UNMOUNT=1 10 | PAT=segmented 11 | #PAT=strided 12 | #PAT=n-n 13 | # configure the read transfer size same as the write transfer size, relaxing 14 | # this later 15 | TRAN_SZ=1048576 16 | TRAN_NUM=64 17 | if [ "$PAT" = "segmented" ]; then 18 | BLK_SZ=`echo "${TRAN_NUM}*${TRAN_SZ}"|bc -l|xargs printf '%1.0f'` 19 | SEG_CNT=1 20 | PAT=0 21 | fi 22 | 23 | if [ "$PAT" = "strided" ]; then 24 | SEG_CNT=${TRAN_NUM} 25 | BLK_SZ=${TRAN_SZ} 26 | PAT=0 27 | fi 28 | 29 | if [ "$PAT" = "n-n" ]; then 30 | BLK_SZ=`echo "${TRAN_NUM}*${TRAN_SZ}"|bc -l|xargs printf '%1.0f'` 31 | SEG_CNT=1 32 | PAT=1 33 | fi 34 | 35 | nlist=`scontrol show hostnames ${SLURM_NODELIST}|paste -d, -s|cut -d ',' -f1-${nnodes}` 36 | 37 | ulimit -c unlimited 38 | srun --drop-caches --nodelist=${nlist} --distribution=block --ntasks-per-node=${procs_per_node} -n ${nprocs} -N ${nnodes} ${CRUISE_DIR}/test_listread -b ${BLK_SZ} -s ${SEG_CNT} -t ${TRAN_SZ} -f /tmp/abc -p ${PAT} -u ${TO_UNMOUNT} 2>&1|tee rclient_${nprocs}.log 39 | -------------------------------------------------------------------------------- /BurstFS_Meta/src/uthash/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2005-2013, Troy D. Hanson http://troydhanson.github.com/uthash/ 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 10 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 11 | IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 12 | TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 13 | PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER 14 | OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 15 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 16 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 17 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 18 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 19 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 20 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 21 | 22 | -------------------------------------------------------------------------------- /BurstFS_Server/runwclient.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | export CRUISE_USE_SPILLOVER=1 3 | CRUISE_DIR=../BurstFS_Client/tests 4 | 5 | procs_per_node=1 6 | nnodes=1 7 | nprocs=`echo "${procs_per_node}*${nnodes}"|bc -l|xargs printf '%1.0f'` 8 | 9 | PAT=segmented 10 | #PAT=strided 11 | #PAT=n-n 12 | 13 | # configure the read transfer size same as the write transfer size, relaxing 14 | # this later 15 | TRAN_SZ=1048576 16 | # total number of transfers 17 | TRAN_NUM=64 18 | TO_UNMOUNT=0 19 | if [ "$PAT" = "segmented" ]; then 20 | BLK_SZ=`echo "${TRAN_NUM}*${TRAN_SZ}"|bc -l|xargs printf '%1.0f'` 21 | SEG_CNT=1 22 | PAT=0 23 | fi 24 | 25 | if [ "$PAT" = "strided" ]; then 26 | SEG_CNT=${TRAN_NUM} 27 | BLK_SZ=${TRAN_SZ} 28 | PAT=0 29 | fi 30 | 31 | if [ "$PAT" = "n-n" ]; then 32 | BLK_SZ=`echo "${TRAN_NUM}*${TRAN_SZ}"|bc -l|xargs printf '%1.0f'` 33 | SEG_CNT=1 34 | PAT=1 35 | fi 36 | 37 | ulimit -c unlimited 38 | 39 | nlist=`scontrol show hostnames ${SLURM_NODELIST}|paste -d, -s|cut -d ',' -f1-${nnodes}` 40 | 41 | srun --drop-caches --nodelist=${nlist} --distribution=block --ntasks-per-node=${procs_per_node} -n ${nprocs} -N ${nnodes} ${CRUISE_DIR}/test_pwrite -b ${BLK_SZ} -s ${SEG_CNT} -t ${TRAN_SZ} -f /tmp/abc -u ${TO_UNMOUNT} -p ${PAT} 2>&1|tee wclient_${nprocs}.log 42 | -------------------------------------------------------------------------------- /BurstFS_Server/arraylist.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | #ifndef __ARRAYLIST_H 16 | #define __ARRAYLIST_H 17 | 18 | #define DEF_ARR_CAP 1024 19 | 20 | typedef struct { 21 | long cap; 22 | long size; 23 | void **elems; 24 | }arraylist_t; 25 | 26 | arraylist_t *arraylist_create(); 27 | int arraylist_add(arraylist_t *arr, void *elem); 28 | int arraylist_reset(arraylist_t *arr); 29 | int arraylist_free(arraylist_t *arr); 30 | int arraylist_insert(arraylist_t *arr, int pos, void *elem); 31 | void * arraylist_get(arraylist_t *arr, int pos); 32 | int arraylist_capacity(arraylist_t *arr); 33 | int arraylist_size(arraylist_t *arr); 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /BurstFS_Server/burstfs_debug.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | #include 16 | #include "burstfs_debug.h" 17 | #include "burstfs_const.h" 18 | 19 | FILE *dbg_stream = NULL; 20 | char dbg_line[GEN_STR_LEN] = {0}; 21 | 22 | int dbg_open(char *fname) { 23 | dbg_stream = fopen(fname, "a"); 24 | if (dbg_stream == NULL) { 25 | dbg_stream = stderr; 26 | return ULFS_ERROR_DBG; 27 | } 28 | else { 29 | return ULFS_SUCCESS; 30 | } 31 | 32 | } 33 | 34 | int dbg_close() { 35 | if (dbg_stream == NULL) 36 | return ULFS_ERROR_DBG; 37 | else { 38 | if (fclose(dbg_stream)== 0) 39 | return ULFS_SUCCESS; 40 | return ULFS_ERROR_DBG; 41 | 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /BurstFS_Client/maint/config/inttypes-pri.m4: -------------------------------------------------------------------------------- 1 | # inttypes-pri.m4 serial 4 (gettext-0.16) 2 | dnl Copyright (C) 1997-2002, 2006 Free Software Foundation, Inc. 3 | dnl This file is free software; the Free Software Foundation 4 | dnl gives unlimited permission to copy and/or distribute it, 5 | dnl with or without modifications, as long as this notice is preserved. 6 | 7 | dnl From Bruno Haible. 8 | 9 | AC_PREREQ(2.52) 10 | 11 | # Define PRI_MACROS_BROKEN if exists and defines the PRI* 12 | # macros to non-string values. This is the case on AIX 4.3.3. 13 | 14 | AC_DEFUN([gt_INTTYPES_PRI], 15 | [ 16 | AC_CHECK_HEADERS([inttypes.h]) 17 | if test $ac_cv_header_inttypes_h = yes; then 18 | AC_CACHE_CHECK([whether the inttypes.h PRIxNN macros are broken], 19 | gt_cv_inttypes_pri_broken, 20 | [ 21 | AC_TRY_COMPILE([#include 22 | #ifdef PRId32 23 | char *p = PRId32; 24 | #endif 25 | ], [], gt_cv_inttypes_pri_broken=no, gt_cv_inttypes_pri_broken=yes) 26 | ]) 27 | fi 28 | if test "$gt_cv_inttypes_pri_broken" = yes; then 29 | AC_DEFINE_UNQUOTED(PRI_MACROS_BROKEN, 1, 30 | [Define if exists and defines unusable PRI* macros.]) 31 | PRI_MACROS_BROKEN=1 32 | else 33 | PRI_MACROS_BROKEN=0 34 | fi 35 | AC_SUBST([PRI_MACROS_BROKEN]) 36 | ]) 37 | -------------------------------------------------------------------------------- /BurstFS_Client/src/cruise-stdio.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * Copyright (c) 2013, Lawrence Livermore National Security, LLC. 17 | * Produced at the Lawrence Livermore National Laboratory. 18 | * code Written by 19 | * Raghunath Rajachandrasekar 20 | * Kathryn Mohror 21 | * Adam Moody 22 | * All rights reserved. 23 | * This file is part of CRUISE. 24 | * For details, see https://github.com/hpc/cruise 25 | * Please also read this file LICENSE.CRUISE 26 | */ 27 | 28 | #ifndef CRUISE_STDIO_H 29 | #define CRUISE_STDIO_H 30 | 31 | #include "cruise-internal.h" 32 | 33 | /* TODO: declare the wrapper functions we define in cruise-stdio.c 34 | * so other routines can call them */ 35 | 36 | #endif /* CRUISE_STDIO_H */ 37 | -------------------------------------------------------------------------------- /BurstFS_Server/burstfs_init.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | #ifndef BURSTFS_INIT_H 16 | #define BURSTFS_INIT_H 17 | #include 18 | #include "burstfs_const.h" 19 | 20 | /* 21 | * structure that records the information of 22 | * each application launched by srun. 23 | * */ 24 | typedef struct { 25 | char hostname[ULFS_MAX_FILENAME]; 26 | int rank; 27 | }name_rank_pair_t; 28 | 29 | typedef struct { 30 | pthread_t thrd; 31 | pthread_cond_t thrd_cond; 32 | }thread_ctrl_t; 33 | 34 | static int CountTasksPerNode(int rank, int numTasks); 35 | static int compare_name_rank_pair(const void *a, const void *b); 36 | static int compare_int(void *a, void *b); 37 | static int find_rank_idx(int my_rank,\ 38 | int *local_rank_lst, int local_rank_cnt); 39 | static int burstfs_exit(); 40 | #endif 41 | -------------------------------------------------------------------------------- /BurstFS_Client/maint/check_fns/cstdio_list.txt: -------------------------------------------------------------------------------- 1 | # http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1124.pdf 2 | 3 | # 7.19.4 Operations on file 4 | 1 remove 5 | 2 rename 6 | 3 tmpfile 7 | 4 tmpnam 8 | 9 | # 7.19.5 file access functions 10 | 1 fclose 11 | 2 fflush 12 | 3 fopen 13 | 4 freopen 14 | 5 setbuf 15 | 6 setvbuf 16 | 17 | # 7.19.6 formatted input/output functions 18 | 1 fprintf 19 | 2 fscanf 20 | 3 printf 21 | 4 scanf 22 | 5 snprintf 23 | 6 sprintf 24 | 7 sscanf 25 | 8 vfprintf 26 | 9 vfscanf 27 | 10 vprintf 28 | 11 vscanf 29 | 12 vsnprintf 30 | 13 vsprintf 31 | 14 vsscanf 32 | 33 | # 7.19.7 character input/output functions 34 | 1 fgetc 35 | 2 fgets 36 | 3 fputc 37 | 4 fputs 38 | 5 getc 39 | 6 getchar 40 | 7 gets 41 | 8 putc 42 | 9 putchar 43 | 10 puts 44 | 11 ungetc 45 | 46 | # 7.19.8 direct input/output functions 47 | 1 fread 48 | 2 fwrite 49 | 50 | # 7.19.9 File positioning functions 51 | 1 fgetpos 52 | 2 fseek 53 | 3 fsetpos 54 | 4 ftell 55 | 5 rewind 56 | 57 | # 7.19.10 Error-handling functions 58 | 1 clearerr 59 | 2 feof 60 | 3 ferror 61 | 4 perror 62 | 63 | # 7.24.2 formatted wide characters 64 | 1 fwprintf 65 | 2 fwscanf 66 | 3 swprintf 67 | 4 swscanf 68 | 5 vfwprintf 69 | 6 vfwscanf 70 | 7 vswprintf 71 | 8 vswscanf 72 | 9 vwprintf 73 | 10 vwscanf 74 | 11 wprintf 75 | 12 wscanf 76 | 77 | # 7.24.3 wide character input/output 78 | 1 fgetwc 79 | 2 fgetws 80 | 3 fputwc 81 | 4 fputws 82 | 5 fwide 83 | 6 getwc 84 | 7 getwchar 85 | 8 putwc 86 | 9 putwchar 87 | 10 ungetwc 88 | -------------------------------------------------------------------------------- /BurstFS_Server/burstfs_sock.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | #ifndef BURSTFS_SOCK_H 16 | #define BURSTFS_SOCK_H 17 | #include 18 | #include "burstfs_const.h" 19 | 20 | #define DEF_SOCK_PATH "/tmp/burstfs_server_sock" 21 | #define BURSTFS_SOCK_TIMEOUT 5000 22 | 23 | extern int server_sockfd; 24 | extern struct pollfd poll_set[MAX_NUM_CLIENTS]; 25 | 26 | int sock_init_server(int local_rank_idx); 27 | int sock_add(int fd); 28 | void sock_reset(); 29 | int sock_wait_cli_cmd(); 30 | char *sock_get_cmd_buf(int sock_id); 31 | int sock_handle_error(int sock_error_no); 32 | int sock_get_id(); 33 | int sock_get_error_id(); 34 | int sock_ack_cli(int sock_id, int ret_sz); 35 | int sock_sanitize(); 36 | char *sock_get_ack_buf(int sock_id); 37 | int sock_remove(int idx); 38 | int sock_notify_cli(int sock_id, int cmd); 39 | char *sock_get_cmd_buf(int sock_id); 40 | #endif 41 | -------------------------------------------------------------------------------- /BurstFS_Meta/src/client.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * 17 | * Copyright (c) 2014, Los Alamos National Laboratory 18 | * All rights reserved. 19 | * 20 | */ 21 | 22 | #ifndef __CLIENT_H 23 | #define __CLIENT_H 24 | 25 | #include "messages.h" 26 | 27 | struct mdhim_rm_t *client_put(struct mdhim_t *md, struct mdhim_putm_t *pm); 28 | struct mdhim_brm_t *client_bput(struct mdhim_t *md, struct index_t *index, 29 | struct mdhim_bputm_t **bpm_list); 30 | struct mdhim_bgetrm_t *client_bget(struct mdhim_t *md, struct index_t *index, 31 | struct mdhim_bgetm_t **bgm_list); 32 | struct mdhim_bgetrm_t *client_bget_op(struct mdhim_t *md, struct mdhim_getm_t *gm); 33 | struct mdhim_rm_t *client_delete(struct mdhim_t *md, struct mdhim_delm_t *dm); 34 | struct mdhim_brm_t *client_bdelete(struct mdhim_t *md, struct index_t *index, 35 | struct mdhim_bdelm_t **bdm_list); 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /BurstFS_Server/burstfs_cmd_handler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | #ifndef BURSTFS_CMD_HANDLER_H 16 | #define BURSTFS_CMD_HANDLER_H 17 | int delegator_handle_command(char *ptr_cmd, int sock_id); 18 | int sync_with_client(char *buf, int client_id); 19 | int open_log_file(app_config_t *app_config,\ 20 | int app_id, int client_id); 21 | int attach_to_shm(app_config_t *app_config,\ 22 | int app_id, int sock_id); 23 | int pack_ack_msg(char *ptr_cmd, int cmd,\ 24 | int rc, void *val,\ 25 | int val_len); 26 | int burstfs_broadcast_exit(int sock_id); 27 | int sync_with_client(char *cmd_buf, int sock_id); 28 | int open_log_file(app_config_t *app_config,\ 29 | int app_id, int sock_id); 30 | int attach_to_shm(app_config_t *app_config,\ 31 | int app_id, int sock_id); 32 | int pack_ack_msg(char *ptr_cmd, int cmd,\ 33 | int rc, void *val, int val_len); 34 | int burstfs_broadcast_exit(int sock_id); 35 | #endif 36 | -------------------------------------------------------------------------------- /BurstFS_Client/LICENSE.ANL: -------------------------------------------------------------------------------- 1 | 2 | COPYRIGHT 3 | 4 | The following is a notice of limited availability of the code, and disclaimer 5 | which must be included in the prologue of the code and in all source listings 6 | of the code. 7 | 8 | Copyright Notice 9 | + 2009 University of Chicago 10 | 11 | Permission is hereby granted to use, reproduce, prepare derivative works, and 12 | to redistribute to others. This software was authored by: 13 | 14 | Argonne National Laboratory Group 15 | P. Carns: (630) 685-1268; e-mail: carns@mcs.anl.gov 16 | Mathematics and Computer Science Division 17 | Argonne National Laboratory, Argonne IL 60439 18 | 19 | 20 | GOVERNMENT LICENSE 21 | 22 | Portions of this material resulted from work developed under a U.S. 23 | Government Contract and are subject to the following license: the Government 24 | is granted for itself and others acting on its behalf a paid-up, nonexclusive, 25 | irrevocable worldwide license in this computer software to reproduce, prepare 26 | derivative works, and perform publicly and display publicly. 27 | 28 | DISCLAIMER 29 | 30 | This computer code material was prepared, in part, as an account of work 31 | sponsored by an agency of the United States Government. Neither the United 32 | States, nor the University of Chicago, nor any of their employees, makes any 33 | warranty express or implied, or assumes any legal liability or responsibility 34 | for the accuracy, completeness, or usefulness of any information, apparatus, 35 | product, or process disclosed, or represents that its use would not infringe 36 | privately owned rights. 37 | -------------------------------------------------------------------------------- /BurstFS_Meta/src/local_client.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * 17 | * Copyright (c) 2014, Los Alamos National Laboratory 18 | * All rights reserved. 19 | * 20 | */ 21 | 22 | #ifndef __LOCAL_CLIENT_H 23 | #define __LOCAL_CLIENT_H 24 | 25 | #include "messages.h" 26 | 27 | struct mdhim_rm_t *local_client_put(struct mdhim_t *md, struct mdhim_putm_t *pm); 28 | struct mdhim_rm_t *local_client_bput(struct mdhim_t *md, struct mdhim_bputm_t *bpm); 29 | struct mdhim_bgetrm_t *local_client_bget(struct mdhim_t *md, struct mdhim_bgetm_t *bgm); 30 | struct mdhim_bgetrm_t *local_client_bget_op(struct mdhim_t *md, struct mdhim_getm_t *gm); 31 | struct mdhim_rm_t *local_client_commit(struct mdhim_t *md, struct mdhim_basem_t *cm); 32 | struct mdhim_rm_t *local_client_delete(struct mdhim_t *md, struct mdhim_delm_t *dm); 33 | struct mdhim_rm_t *local_client_bdelete(struct mdhim_t *md, struct mdhim_bdelm_t *dm); 34 | void local_client_close(struct mdhim_t *md, struct mdhim_basem_t *cm); 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /BurstFS_Server/configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.69]) 5 | AC_INIT([burstfsd], [0.1], [tw15g@my.fsu.edu]) 6 | AM_INIT_AUTOMAKE(burstfsd, 0, 1) 7 | AC_CONFIG_SRCDIR([burstfs_setup.h]) 8 | AC_CONFIG_HEADERS([config.h]) 9 | 10 | # Checks for programs. 11 | AC_PROG_CC([mpicc]) 12 | AC_ARG_WITH(mdhim, [--with-mdhim=PATH specify path to mdhim], [CPPFLAGS="-I${withval}/src -I${withval}/Mlog -I${withval}/src/uthash ${CPPFLAGS}"; LDFLAGS="-L${withval}/src -lmdhim ${LDFLAGS}"]) 13 | AC_ARG_WITH(leveldb, [--with-leveldb=PATH specify path to mdhim], [CPPFLAGS="-I${withval}/include ${CPPFLAGS}"; LDFLAGS="-L${withval} -lleveldb ${LDFLAGS}"]) 14 | # Checks for libraries. 15 | # FIXME: Replace `main' with a function in `-lm': 16 | AC_CHECK_LIB([m], [main]) 17 | # FIXME: Replace `main' with a function in `-lmdhim': 18 | AC_CHECK_LIB([mdhim], [main]) 19 | # FIXME: Replace `main' with a function in `-lpthread': 20 | AC_CHECK_LIB([pthread], [main]) 21 | # FIXME: Replace `main' with a function in `-lrt': 22 | AC_CHECK_LIB([rt], [main]) 23 | AC_CHECK_LIB([stdc++], [main]) 24 | 25 | # Checks for header files. 26 | AC_CHECK_HEADERS([fcntl.h limits.h stdlib.h string.h sys/socket.h sys/time.h unistd.h]) 27 | 28 | # Checks for typedefs, structures, and compiler characteristics. 29 | AC_TYPE_OFF_T 30 | AC_TYPE_SIZE_T 31 | 32 | # Checks for library functions. 33 | AC_FUNC_MALLOC 34 | AC_FUNC_MMAP 35 | AC_FUNC_REALLOC 36 | AC_CHECK_FUNCS([ftruncate getpagesize gettimeofday memset socket]) 37 | 38 | AC_CONFIG_FILES([Makefile]) 39 | AC_OUTPUT 40 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/tester/runAll.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # echo "type INT" 4 | # 5 | # rm -rf ./mdhimTst* ; mpirun -np 2 ./mdhimtst -finTestBasic.txt -t1 -q -d3 -p./ 6 | # rm mdhim.manifest 7 | # rm -rf ./mdhimTst* ; mpirun -np 2 ./mdhimtst -finTestNext.txt -t1 -q -d3 -p./ 8 | # rm mdhim.manifest 9 | # 10 | echo "type LONG" 11 | 12 | rm -rf ./mdhimTst* ; mpirun -np 2 ./mdhimtst -finTestBasic.txt -t2 -q -d3 -p./ 13 | rm mdhim.manifest 14 | rm -rf ./mdhimTst* ; mpirun -np 2 ./mdhimtst -finTestNext.txt -t2 -q -d3 -p./ 15 | rm mdhim.manifest 16 | 17 | echo "type FLOAT" 18 | 19 | rm -rf ./mdhimTst* ; mpirun -np 2 ./mdhimtst -finTestBasic.txt -t3 -q -d3 -p./ 20 | rm mdhim.manifest 21 | rm -rf ./mdhimTst* ; mpirun -np 2 ./mdhimtst -finTestNext.txt -t3 -q -d3 -p./ 22 | rm mdhim.manifest 23 | 24 | echo "type DOUBLE" 25 | 26 | rm -rf ./mdhimTst* ; mpirun -np 2 ./mdhimtst -finTestBasic.txt -t4 -q -d3 -p./ 27 | rm mdhim.manifest 28 | rm -rf ./mdhimTst* ; mpirun -np 2 ./mdhimtst -finTestNext.txt -t4 -q -d3 -p./ 29 | rm mdhim.manifest 30 | 31 | # echo "type STRING" 32 | # 33 | # rm -rf ./mdhimTst* ; mpirun -np 2 ./mdhimtst -finTestBasic.txt -t5 -q -d3 -p./ 34 | # rm mdhim.manifest 35 | # rm -rf ./mdhimTst* ; mpirun -np 2 ./mdhimtst -finTestNext.txt -t5 -q -d3 -p./ 36 | # rm mdhim.manifest 37 | # rm -rf ./mdhimTst* ; mpirun -np 2 ./mdhimtst -finTestLarge.txt -t5 -q -d3 -p./ 38 | # rm mdhim.manifest 39 | # 40 | # echo "type BYTE" 41 | # 42 | # rm -rf ./mdhimTst* ; mpirun -np 2 ./mdhimtst -finTestBasic.txt -t6 -q -d3 -p./ 43 | # rm mdhim.manifest 44 | # rm -rf ./mdhimTst* ; mpirun -np 2 ./mdhimtst -finTestNext.txt -t6 -q -d3 -p./ 45 | # rm mdhim.manifest 46 | # rm -rf ./mdhimTst* ; mpirun -np 2 ./mdhimtst -finTestLarge.txt -t6 -q -d3 -p./ 47 | # rm mdhim.manifest 48 | -------------------------------------------------------------------------------- /BurstFS_Meta/src/mdhim_private.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * 17 | * Copyright (c) 2014, Los Alamos National Laboratory 18 | * All rights reserved. 19 | * 20 | */ 21 | 22 | #include "mdhim.h" 23 | 24 | struct mdhim_rm_t *_put_record(struct mdhim_t *md, struct index_t *index, 25 | void *key, int key_len, 26 | void *value, int value_len); 27 | struct mdhim_brm_t *_create_brm(struct mdhim_rm_t *rm); 28 | void _concat_brm(struct mdhim_brm_t *head, struct mdhim_brm_t *addition); 29 | struct mdhim_brm_t *_bput_records(struct mdhim_t *md, struct index_t *index, 30 | void **keys, int *key_lens, 31 | void **values, int *value_lens, int num_records); 32 | struct mdhim_bgetrm_t *_bget_records(struct mdhim_t *md, struct index_t *index, 33 | void **keys, int *key_lens, 34 | int num_keys, int num_records, int op); 35 | 36 | struct mdhim_bgetrm_t *_bget_range_records(struct mdhim_t *md, struct index_t *index, 37 | void *start_key, void *end_key, int key_len); 38 | 39 | struct mdhim_brm_t *_bdel_records(struct mdhim_t *md, struct index_t *index, 40 | void **keys, int *key_lens, 41 | int num_records); 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | ******************** 2 | Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | Produced at the Lawrence Livermore National Laboratory. 4 | Copyright (c) 2017, Florida State University. Contributions from 5 | the Computer Architecture and Systems Research Laboratory (CASTL) 6 | at the Department of Computer Science. 7 | Written by 8 | Teng Wang tw15g@my.fsu.edu 9 | Adam Moody moody20@llnl.gov 10 | Weikuan Yu wyu3@fsu.edu 11 | Kento Sato kento@llnl.gov 12 | Kathryn Mohror kathryn@llnl.gov 13 | LLNL-CODE-728877 14 | All rights reserved. 15 | 16 | This is the license for BurstFS. For details, see https://github.com/llnl/burstfs. 17 | Permission is hereby granted, free of charge, to any person obtaining a copy 18 | of this software and associated documentation files (the "Software"), to deal 19 | in the Software without restriction, including without limitation the rights to 20 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 21 | the Software, and to permit persons to whom the Software is furnished to do so, 22 | subject to the following conditions: 23 | 24 | The above copyright notice and this permission notice shall be included in all 25 | copies or substantial portions of the Software. 26 | 27 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 28 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR 29 | PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE 30 | FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 31 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 32 | ***************** 33 | -------------------------------------------------------------------------------- /BurstFS_Client/LICENSE: -------------------------------------------------------------------------------- 1 | ******************** 2 | Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | Produced at the Lawrence Livermore National Laboratory. 4 | Copyright (c) 2017, Florida State University. Contributions from 5 | the Computer Architecture and Systems Research Laboratory (CASTL) 6 | at the Department of Computer Science. 7 | Written by 8 | Teng Wang tw15g@my.fsu.edu 9 | Adam Moody moody20@llnl.gov 10 | Weikuan Yu wyu3@fsu.edu 11 | Kento Sato kento@llnl.gov 12 | Kathryn Mohror kathryn@llnl.gov 13 | LLNL-CODE-728877 14 | All rights reserved. 15 | 16 | This is the license for BurstFS. For details, see https://github.com/llnl/burstfs. 17 | Permission is hereby granted, free of charge, to any person obtaining a copy 18 | of this software and associated documentation files (the "Software"), to deal 19 | in the Software without restriction, including without limitation the rights to 20 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 21 | the Software, and to permit persons to whom the Software is furnished to do so, 22 | subject to the following conditions: 23 | 24 | The above copyright notice and this permission notice shall be included in all 25 | copies or substantial portions of the Software. 26 | 27 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 28 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR 29 | PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE 30 | FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 31 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 32 | ***************** 33 | -------------------------------------------------------------------------------- /BurstFS_Meta/LICENSE: -------------------------------------------------------------------------------- 1 | ******************** 2 | Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | Produced at the Lawrence Livermore National Laboratory. 4 | Copyright (c) 2017, Florida State University. Contributions from 5 | the Computer Architecture and Systems Research Laboratory (CASTL) 6 | at the Department of Computer Science. 7 | Written by 8 | Teng Wang tw15g@my.fsu.edu 9 | Adam Moody moody20@llnl.gov 10 | Weikuan Yu wyu3@fsu.edu 11 | Kento Sato kento@llnl.gov 12 | Kathryn Mohror kathryn@llnl.gov 13 | LLNL-CODE-728877 14 | All rights reserved. 15 | 16 | This is the license for BurstFS. For details, see https://github.com/llnl/burstfs. 17 | Permission is hereby granted, free of charge, to any person obtaining a copy 18 | of this software and associated documentation files (the "Software"), to deal 19 | in the Software without restriction, including without limitation the rights to 20 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 21 | the Software, and to permit persons to whom the Software is furnished to do so, 22 | subject to the following conditions: 23 | 24 | The above copyright notice and this permission notice shall be included in all 25 | copies or substantial portions of the Software. 26 | 27 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 28 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR 29 | PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE 30 | FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 31 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 32 | ***************** 33 | -------------------------------------------------------------------------------- /BurstFS_Server/LICENSE: -------------------------------------------------------------------------------- 1 | ******************** 2 | Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | Produced at the Lawrence Livermore National Laboratory. 4 | Copyright (c) 2017, Florida State University. Contributions from 5 | the Computer Architecture and Systems Research Laboratory (CASTL) 6 | at the Department of Computer Science. 7 | Written by 8 | Teng Wang tw15g@my.fsu.edu 9 | Adam Moody moody20@llnl.gov 10 | Weikuan Yu wyu3@fsu.edu 11 | Kento Sato kento@llnl.gov 12 | Kathryn Mohror kathryn@llnl.gov 13 | LLNL-CODE-728877 14 | All rights reserved. 15 | 16 | This is the license for BurstFS. For details, see https://github.com/llnl/burstfs. 17 | Permission is hereby granted, free of charge, to any person obtaining a copy 18 | of this software and associated documentation files (the "Software"), to deal 19 | in the Software without restriction, including without limitation the rights to 20 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 21 | the Software, and to permit persons to whom the Software is furnished to do so, 22 | subject to the following conditions: 23 | 24 | The above copyright notice and this permission notice shall be included in all 25 | copies or substantial portions of the Software. 26 | 27 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 28 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR 29 | PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE 30 | FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 31 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 32 | ***************** 33 | -------------------------------------------------------------------------------- /BurstFS_Client/LICENSE.CRUISE: -------------------------------------------------------------------------------- 1 | --------------------- 2 | Copyright and BSD License 3 | --------------------- 4 | 5 | Copyright (c) 2017, Lawrence Livermore National Security, LLC. 6 | Produced at the Lawrence Livermore National Laboratory. 7 | 8 | Copyright (c) 2017, Florida State University. 9 | Contributions from the Computer Architecture and Systems Research Laboratory (CASTL) 10 | at the Department of Computer Science. 11 | 12 | Written by 13 | Teng Wang tw15g@my.fsu.edu 14 | Adam Moody moody20@llnl.gov 15 | Weikuan Yu wyu3@fsu.edu 16 | Kento Sato kento@llnl.gov 17 | Kathryn Mohror kathryn@llnl.gov 18 | LLNL-CODE-728877. 19 | 20 | All rights reserved. 21 | This file is part of BurstFS For details, see https://github.com/llnl/burstfs. 22 | Permission is hereby granted, free of charge, to any person obtaining a copy of this 23 | software and associated documentation files (the "Software"), 24 | to deal in the Software without restriction, including without limitation the rights to use, 25 | copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and 26 | to permit persons to whom the Software is furnished to do so, subject to the following 27 | conditions: 28 | The above copyright notice and this permission notice shall be included in all copies or substantial 29 | portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 30 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 31 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 32 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 33 | ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE 34 | OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 35 | -------------------------------------------------------------------------------- /BurstFS_Server/COPYRIGHT: -------------------------------------------------------------------------------- 1 | --------------------- 2 | Copyright and BSD License 3 | --------------------- 4 | 5 | Copyright (c) 2017, Lawrence Livermore National Security, LLC. 6 | Produced at the Lawrence Livermore National Laboratory. 7 | 8 | Copyright (c) 2017, Florida State University. 9 | Contributions from the Computer Architecture and Systems Research Laboratory (CASTL) 10 | at the Department of Computer Science. 11 | 12 | Written by 13 | Teng Wang tw15g@my.fsu.edu 14 | Adam Moody moody20@llnl.gov 15 | Weikuan Yu wyu3@fsu.edu 16 | Kento Sato kento@llnl.gov 17 | Kathryn Mohror kathryn@llnl.gov 18 | LLNL-CODE-728877. 19 | 20 | All rights reserved. 21 | This file is part of BurstFS For details, see https://github.com/llnl/burstfs. 22 | Permission is hereby granted, free of charge, to any person obtaining a copy of this 23 | software and associated documentation files (the "Software"), 24 | to deal in the Software without restriction, including without limitation the rights to use, 25 | copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and 26 | to permit persons to whom the Software is furnished to do so, subject to the following 27 | conditions: 28 | The above copyright notice and this permission notice shall be included in all copies or substantial 29 | portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 30 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 31 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 32 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 33 | ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE 34 | OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 35 | -------------------------------------------------------------------------------- /BurstFS_Client/INSTALL: -------------------------------------------------------------------------------- 1 | 1. Prepare and run 'configure': 2 | 3 | # debug 4 | use -q ic-11.1.046 5 | ./prepare 6 | ./configure \ 7 | --prefix=`pwd`/install \ 8 | CFLAGS="-g -O0" 9 | 10 | # optimized 11 | use -q ic-11.1.046 12 | ./prepare 13 | ./configure \ 14 | --prefix=`pwd`/install \ 15 | CFLAGS="-O3" 16 | 17 | --disable-ld-preload 18 | 19 | If you do not want to generate the .so library used for LD_PRELOAD. 20 | 21 | --with-numa=$PATH_TO_NUMA 22 | 23 | Needed to enable NUMA-aware memory allocation on Linux machines 24 | (set NUMA policy at runtime with CRUISE_NUMA_POLICY = local | interleaved, 25 | set NUMA node explicitly with CRUISE_USE_NUMA_BANK = ) 26 | 27 | 28 | 29 | 2. Build library 30 | 31 | make clean 32 | make 33 | make install 34 | 35 | 3. Run "./install/bin/cruise-config --pre-ld-flags" 36 | and "./install/bin/cruise-config --post-ld-flags" 37 | 38 | Copy the output of each of these commands to a separate variable,' 39 | say $(PRE_CRUISE_FLAGS) and $(POST_CRUISE_FLAGS) 40 | 41 | 4. Compile the test program 42 | 43 | If you're compiling test_ramdisk.c for instance, you would do 44 | something like this in your makefile: 45 | 46 | setenv PRE_CRUISE_FLAGS `./install/bin/cruise-config --pre-ld-flags` 47 | setenv POST_CRUISE_FLAGS `./install/bin/cruise-config --post-ld-flags` 48 | mpicc $PRE_CRUISE_FLAGS -g -O0 -o test_ramdisk test_ramdisk.c $POST_CRUISE_FLAGS 49 | 50 | If you are running test_interpose, you can do something similar in 51 | tests/Makefile. 52 | 53 | 6. If you don't want to statically build, but just preload instead, preload this shared object: 54 | 55 | LD_PRELOAD=$INSTALL_PATH/lib/libcruise.so ./test_interpose 56 | 57 | 7. Use the "ipcs" command to list the currently opened shm segments. Use 58 | "ipcrm -M key" to delete it after a run, or use the "ipc_cleanup" script 59 | in the base folder to delete all shm segements belonging to you on a node. 60 | -------------------------------------------------------------------------------- /BurstFS_Server/burstfs_request_manager.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | #ifndef BURSTFS_REQUEST_MANAGER_H 16 | #define BURSTFS_REQUEST_MANAGER_H 17 | #include "burstfs_const.h" 18 | #include "burstfs_global.h" 19 | #include "arraylist.h" 20 | 21 | typedef struct { 22 | int src_fid; 23 | long offset; 24 | long length; 25 | }shm_meta_t; /*metadata format in the shared memory*/ 26 | 27 | int rm_delegate_request(cli_signature_t *my_sig); 28 | int rm_read_remote_data(int sock_id, int num); 29 | int rm_send_remote_requests(thrd_ctrl_t *thrd_ctrl,\ 30 | int thrd_tag, long *tot_sz); 31 | int rm_pack_send_requests(char *req_msg_buf,\ 32 | send_msg_t *send_metas, int req_num,\ 33 | long *tot_sz); 34 | 35 | int compare_delegators(void *a, void *b); 36 | int rm_pack_send_msg(int rank, char *send_msg_buf,\ 37 | send_msg_t *send_metas, int meta_num,\ 38 | long *tot_sz); 39 | int rm_receive_remote_message(int app_id,\ 40 | int sock_id, long tot_sz); 41 | void print_recv_msg(int app_id,\ 42 | int cli_id, int dbg_rank, int thrd_id, shm_meta_t *msg); 43 | int rm_process_received_msg(int app_id, int sock_id,\ 44 | char *recv_msg_buf, long *ptr_tot_sz); 45 | void print_remote_del_reqs(int app_id, int cli_id, \ 46 | int dbg_rank, del_req_stat_t *del_req_stat); 47 | void print_send_msgs(send_msg_t *send_metas,\ 48 | long msg_cnt, int dbg_rank); 49 | #endif 50 | -------------------------------------------------------------------------------- /BurstFS_Client/src/cruise-sysio.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * Copyright (c) 2013, Lawrence Livermore National Security, LLC. 17 | * Produced at the Lawrence Livermore National Laboratory. 18 | * code Written by 19 | * Raghunath Rajachandrasekar 20 | * Kathryn Mohror 21 | * Adam Moody 22 | * All rights reserved. 23 | * This file is part of CRUISE. 24 | * For details, see https://github.com/hpc/cruise 25 | * Please also read this file LICENSE.CRUISE 26 | */ 27 | 28 | #ifndef CRUISE_SYSIO_H 29 | #define CRUISE_SYSIO_H 30 | 31 | #include "cruise-internal.h" 32 | 33 | /* read count bytes info buf from file starting at offset pos, 34 | * returns number of bytes actually read in retcount, 35 | * retcount will be less than count only if an error occurs 36 | * or end of file is reached */ 37 | int cruise_fd_read(int fd, off_t pos, void* buf, size_t count, size_t* retcount); 38 | 39 | /* write count bytes from buf into file starting at offset pos, 40 | * allocates new bytes and updates file size as necessary, 41 | * fills any gaps with zeros */ 42 | int cruise_fd_write(int fd, off_t pos, const void* buf, size_t count); 43 | int cruise_fd_logreadlist(read_req_t *read_req, int count); 44 | int compare_read_req(const void *a, const void *b); 45 | int compare_index_entry(const void *a, const void *b); 46 | 47 | #endif /* CRUISE_SYSIO_H */ 48 | -------------------------------------------------------------------------------- /BurstFS_Meta/src/ds_mysql.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * 17 | * Copyright (c) 2014, Los Alamos National Laboratory 18 | * All rights reserved. 19 | * 20 | */ 21 | 22 | #include 23 | #include 24 | #include "mdhim.h" 25 | #include "data_store.h" 26 | 27 | //Struct for MYSQL Handling 28 | //MDI = Mysql Database Info 29 | //Too lazy to Edit name 30 | struct MDI { 31 | MYSQL *msqdb; //Database connection 32 | int msqht; //Handle's specfication, whether it's the original or stat 33 | int msqkt; 34 | char *host; 35 | char *user; 36 | char *pswd; 37 | char *table; 38 | char *database; 39 | }; 40 | 41 | int mdhim_mysql_open(void **dbh, void **dbs, char *path, int flags, int key_type, struct mdhim_options_t *opts); 42 | int mdhim_mysql_put(void *dbh, void *key, int key_len, void *data, int32_t data_len); 43 | int mdhim_mysql_get(void *dbh, void *key, int key_len, void **data, int32_t *data_len); 44 | int mdhim_mysql_get_next(void *dbh, void **key, int *key_len, 45 | void **data, int32_t *data_len); 46 | int mdhim_mysql_get_prev(void *dbh, void **key, int *key_len, 47 | void **data, int32_t *data_len); 48 | int mdhim_mysql_close(void *dbh, void *dbs); 49 | int mdhim_mysql_del(void *dbh, void *key, int key_len); 50 | int mdhim_mysql_commit(void *dbh); 51 | int mdhim_mysql_batch_put(void *dbh, void **key, int32_t *key_lens, 52 | void **data, int32_t *data_lens, int num_record); 53 | -------------------------------------------------------------------------------- /BurstFS_Server/log.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | #ifndef __LOG_H__ 16 | #define __LOG_H__ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | extern FILE* dbg_stream; 24 | extern char dbg_line[1024]; 25 | extern int glb_rank; 26 | 27 | typedef enum { 28 | LOG_FATAL = 1, 29 | LOG_ERR = 2, 30 | LOG_WARN = 3, 31 | LOG_INFO = 4, 32 | LOG_DBG = 5 33 | } loglevel; 34 | 35 | char timestamp[256]; 36 | time_t ltime; 37 | 38 | // add and change all 39 | struct tm *ttime; 40 | struct timeval logstart, logend; 41 | double mlogtm; 42 | 43 | extern int log_print_level; 44 | #define gettid() syscall(__NR_gettid) 45 | #define LOG(level, ...) \ 46 | if(level <= log_print_level) { \ 47 | gettimeofday(&logstart, NULL); \ 48 | ltime = time(NULL); \ 49 | ttime = localtime(<ime); \ 50 | strftime(timestamp, sizeof(timestamp), \ 51 | "%Y-%m-%dT%H:%M:%S", ttime); \ 52 | fprintf(dbg_stream,"logtime:%lf rank [%d] [%s] [%ld] [%s:%d] [%s] ", \ 53 | mlogtm/1000000, glb_rank, timestamp, gettid(), \ 54 | __FILE__, __LINE__, __FUNCTION__); \ 55 | fprintf(dbg_stream, __VA_ARGS__); \ 56 | fprintf(dbg_stream, "\n"); \ 57 | fflush(dbg_stream); \ 58 | gettimeofday(&logend, NULL); \ 59 | mlogtm += 1000000*(logend.tv_sec-logstart.tv_sec)+logend.tv_usec-logstart.tv_usec; \ 60 | } 61 | #endif /* LOG_H */ 62 | -------------------------------------------------------------------------------- /BurstFS_Meta/src/Makefile: -------------------------------------------------------------------------------- 1 | include ../Makefile.cfg 2 | 3 | ifeq ($(FORTRAN),1) 4 | OBJS = mlog2.o client.o local_client.o data_store.o partitioner.o messages.o range_server.o mdhim_options.o mdhim_private.o indexes.o mdhim_fortran.o mdhim_f90_binding.o 5 | else 6 | OBJS = mlog2.o client.o local_client.o data_store.o partitioner.o messages.o range_server.o mdhim_options.o mdhim_private.o indexes.o 7 | endif 8 | 9 | ifeq ($(LEVELDB),1) 10 | OBJS += ds_leveldb.o 11 | else 12 | ifeq ($(ROCKSDB),1) 13 | OBJS += ds_leveldb.o 14 | endif 15 | endif 16 | 17 | ifeq ($(MYSQLDB),1) 18 | OBJS += ds_mysql.o 19 | endif 20 | 21 | 22 | all: mdhim.o lib 23 | 24 | mdhim.o: mdhim.c $(OBJS) 25 | $(CC) -c $< $(CINC) $(CLIBS) -lleveldb 26 | 27 | mdhim_fortran.o: mdhim_fortran.c 28 | $(CC) -c $< $(CINC) $(CLIBS) 29 | 30 | mlog2.o: Mlog2/mlog2.c 31 | $(CC) -c $^ $(CINC) $(CLIBS) -lleveldb 32 | 33 | mdhim_private.o: mdhim_private.c 34 | $(CC) -c $< $(CINC) $(CLIBS) -lleveldb 35 | 36 | client.o: client.c 37 | $(CC) -c $^ $(CINC) $(CLIBS) -lleveldb 38 | 39 | local_client.o: local_client.c 40 | $(CC) -c $^ $(CINC) $(CLIBS) -lleveldb 41 | 42 | messages.o: messages.c 43 | $(CC) -c $^ $(CINC) $(CLIBS) -lleveldb 44 | 45 | partitioner.o: partitioner.c 46 | $(CC) -c $^ $(CINC) $(CLIBS) -lleveldb 47 | 48 | indexes.o: indexes.c 49 | $(CC) -c $^ $(CINC) $(CLIBS) -lleveldb 50 | 51 | data_store.o: data_store.c 52 | $(CC) -c $^ $(CINC) $(CLIBS) -lleveldb 53 | 54 | ds_mysql.o: ds_mysql.c 55 | $(CC) -c $^ $(CINC) $(CLIBS) -lleveldb 56 | 57 | ds_leveldb.o: ds_leveldb.c 58 | $(CC) -c $^ $(CINC) $(CLIBS) -lleveldb 59 | 60 | ds_sophia.o: ds_sophia.c 61 | $(CC) -c $^ $(CINC) $(CLIBS) -lleveldb 62 | 63 | range_server.o: range_server.c 64 | $(CC) -c $^ $(CINC) $(CLIBS) -lleveldb 65 | 66 | mdhim_options.o : mdhim_options.c 67 | $(CC) -c $^ $(CINC) $(CLIBS) -lleveldb 68 | 69 | mdhim_f90_binding.o: mdhim_f90_binding.f90 70 | $(FF) -c $< $(FINC) $(FLIBS) 71 | 72 | lib: 73 | ar rvs libmdhim.a mdhim.o $(OBJS) 74 | dylib: 75 | $(CC) -shared -fPIC -o libmdhim.so.1 mdhim.o $(OBJS) -L$(LEVEL_DB_ROOT)/ -lleveldb 76 | 77 | clean: 78 | rm -rf *.o libmdhim.a libmdhim.so.1 libmdhimf90.so.1 79 | -------------------------------------------------------------------------------- /BurstFS_Client/src/cruise-stack.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * Copyright (c) 2013, Lawrence Livermore National Security, LLC. 17 | * Produced at the Lawrence Livermore National Laboratory. 18 | * code Written by 19 | * Raghunath Rajachandrasekar 20 | * Kathryn Mohror 21 | * Adam Moody 22 | * All rights reserved. 23 | * This file is part of CRUISE. 24 | * For details, see https://github.com/hpc/cruise 25 | * Please also read this file LICENSE.CRUISE 26 | */ 27 | 28 | #ifndef CRUISE_STACK_H 29 | #define CRUISE_STACK_H 30 | 31 | /* implements a fixed-size stack which stores integer values in range 32 | * of 0 to size-1, entire structure stored in an int array of size+2 33 | * int size 34 | * int last 35 | * int entries[size] 36 | * last records index within entries that points to item one past 37 | * the item at the top of the stack 38 | * 39 | * used to record which entries in a fixed-size array are free */ 40 | 41 | #include 42 | 43 | typedef struct { 44 | int size; 45 | int last; 46 | } cruise_stack; 47 | 48 | /* returns number of bytes needed to represent stack data structure */ 49 | size_t cruise_stack_bytes(int size); 50 | 51 | /* intializes stack to record all entries as being free */ 52 | void cruise_stack_init(void* start, int size); 53 | 54 | /* pops one entry from stack and returns its value */ 55 | int cruise_stack_pop(void* start); 56 | 57 | /* pushes item onto free stack */ 58 | void cruise_stack_push(void* start, int value); 59 | 60 | #endif /* CRUISE_STACK_H */ 61 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/tester/inTestBulkLarge.txt: -------------------------------------------------------------------------------- 1 | #23456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 12345 2 | 3 | # data value of 2040 4 | bput 99999 dataValueOf2040 abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz1234_abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ111112222233333abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz1234_abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ111112222233333abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz1234_abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ111112222233333abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz1234_abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ11111222223333344444555556666677777 5 | -------------------------------------------------------------------------------- /BurstFS_Client/maint/check_fns/fakechroot_list.txt: -------------------------------------------------------------------------------- 1 | __fxstatat.c 2 | __fxstatat64.c 3 | __getcwd_chk.c 4 | __getwd_chk.c 5 | __lxstat.c 6 | __lxstat64.c 7 | __open.c 8 | __open64.c 9 | __open64_2.c 10 | __open_2.c 11 | __openat64_2.c 12 | __openat_2.c 13 | __opendir2.c 14 | __readlink_chk.c 15 | __readlinkat_chk.c 16 | __realpath_chk.c 17 | __statfs.c 18 | __xmknod.c 19 | __xmknodat.c 20 | __xstat.c 21 | __xstat64.c 22 | _xftw.c 23 | _xftw64.c 24 | access.c 25 | acct.c 26 | bind.c 27 | bindtextdomain.c 28 | canonicalize_file_name.c 29 | chdir.c 30 | chmod.c 31 | chown.c 32 | chroot.c 33 | connect.c 34 | creat.c 35 | creat64.c 36 | dlmopen.c 37 | dlopen.c 38 | eaccess.c 39 | euidaccess.c 40 | execl.c 41 | execle.c 42 | execlp.c 43 | execv.c 44 | execve.c 45 | execvp.c 46 | faccessat.c 47 | fchmodat.c 48 | fchownat.c 49 | fopen.c 50 | fopen64.c 51 | freopen.c 52 | freopen64.c 53 | fts.c 54 | ftw.c 55 | ftw64.c 56 | futimesat.c 57 | get_current_dir_name.c 58 | getcwd.c 59 | getpeername.c 60 | getsockname.c 61 | getwd.c 62 | getxattr.c 63 | glob.c 64 | glob64.c 65 | glob_pattern_p.c 66 | inotify_add_watch.c 67 | lchmod.c 68 | lchown.c 69 | lckpwdf.c 70 | lgetxattr.c 71 | libfakechroot.c 72 | link.c 73 | linkat.c 74 | listxattr.c 75 | llistxattr.c 76 | lremovexattr.c 77 | lsetxattr.c 78 | lstat.c 79 | lstat64.c 80 | lutimes.c 81 | mkdir.c 82 | mkdirat.c 83 | mkdtemp.c 84 | mkfifo.c 85 | mkfifoat.c 86 | mknod.c 87 | mknodat.c 88 | mkstemp.c 89 | mkstemp64.c 90 | mktemp.c 91 | open.c 92 | open64.c 93 | openat.c 94 | openat64.c 95 | opendir.c 96 | pathconf.c 97 | popen.c 98 | rawmemchr.c 99 | readlink.c 100 | readlinkat.c 101 | realpath.c 102 | remove.c 103 | removexattr.c 104 | rename.c 105 | renameat.c 106 | revoke.c 107 | rmdir.c 108 | rpl_lstat.c 109 | scandir.c 110 | scandir64.c 111 | setenv.c 112 | setxattr.c 113 | stat.c 114 | stat64.c 115 | statfs.c 116 | statfs64.c 117 | statvfs.c 118 | statvfs64.c 119 | stpcpy.c 120 | strchrnul.c 121 | symlink.c 122 | symlinkat.c 123 | system.c 124 | tempnam.c 125 | tmpnam.c 126 | truncate.c 127 | truncate64.c 128 | ulckpwdf.c 129 | unlink.c 130 | unlinkat.c 131 | unsetenv.c 132 | utime.c 133 | utimensat.c 134 | utimes.c 135 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/puts-gets.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "mpi.h" 4 | #include "mdhim.h" 5 | #include "mdhim_options.h" 6 | 7 | int main(int argc, char **argv) { 8 | int ret; 9 | int provided = 0; 10 | struct mdhim_t *md; 11 | char *key; 12 | int value; 13 | struct mdhim_brm_t *brm; 14 | struct mdhim_bgetrm_t *bgrm; 15 | mdhim_options_t *db_opts; 16 | int i; 17 | MPI_Comm comm; 18 | 19 | ret = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); 20 | if (ret != MPI_SUCCESS) { 21 | printf("Error initializing MPI with threads\n"); 22 | exit(1); 23 | } 24 | 25 | if (provided != MPI_THREAD_MULTIPLE) { 26 | printf("Not able to enable MPI_THREAD_MULTIPLE mode\n"); 27 | exit(1); 28 | } 29 | 30 | db_opts = mdhim_options_init(); 31 | mdhim_options_set_db_path(db_opts, "./"); 32 | mdhim_options_set_db_name(db_opts, "mdhimTstDB"); 33 | mdhim_options_set_db_type(db_opts, LEVELDB); 34 | mdhim_options_set_key_type(db_opts, MDHIM_STRING_KEY); 35 | mdhim_options_set_debug_level(db_opts, MLOG_CRIT); 36 | 37 | comm = MPI_COMM_WORLD; 38 | md = mdhimInit(&comm, db_opts); 39 | if (!md) { 40 | printf("Error initializing MDHIM\n"); 41 | exit(1); 42 | } 43 | 44 | //Put the keys and values 45 | for (i = 0; i < 2; i++) { 46 | key = malloc(100); 47 | sprintf(key, "%c", (int) '0' + (md->mdhim_rank + 1) + i); 48 | value = 500 * (md->mdhim_rank + 1) + i; 49 | brm = mdhimPut(md, key, strlen(key) + 1, 50 | &value, sizeof(value), 51 | NULL, NULL); 52 | if (!brm || brm->error) { 53 | printf("Error inserting key/value into MDHIM\n"); 54 | } else { 55 | printf("Successfully inserted key/value into MDHIM\n"); 56 | } 57 | 58 | //Commit the database 59 | ret = mdhimCommit(md, md->primary_index); 60 | if (ret != MDHIM_SUCCESS) { 61 | printf("Error committing MDHIM database\n"); 62 | } else { 63 | printf("Committed MDHIM database\n"); 64 | } 65 | 66 | //Get the values 67 | value = 0; 68 | bgrm = mdhimGet(md, md->primary_index, 69 | key, strlen(key) + 1, MDHIM_GET_EQ); 70 | if (!bgrm || bgrm->error) { 71 | printf("Error getting value for key: %s from MDHIM\n", key); 72 | } else { 73 | printf("Successfully got value: %d from MDHIM for key: %s\n", *((int *) bgrm->values[0]), key); 74 | } 75 | 76 | free(key); 77 | } 78 | ret = mdhimClose(md); 79 | if (ret != MDHIM_SUCCESS) { 80 | printf("Error closing MDHIM\n"); 81 | } 82 | 83 | MPI_Barrier(MPI_COMM_WORLD); 84 | MPI_Finalize(); 85 | 86 | return 0; 87 | } 88 | -------------------------------------------------------------------------------- /BurstFS_Server/burstfs_metadata.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | #ifndef BURSTFS_METADATA_H 16 | #define BURSTFS_METADATA_H 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include "mdhim.h" 22 | #include "indexes.h" 23 | #include "arraylist.h" 24 | #include "burstfs_const.h" 25 | #include "burstfs_global.h" 26 | 27 | #define DEF_META_PATH "/l/ssd/" 28 | #define MANIFEST_FILE_NAME "mdhim_manifest_" 29 | #define DEF_DB_NAME "burstfs_db" 30 | #define DEF_SERVER_RATIO 1 31 | #define DEF_RANGE_SZ 1048576 32 | 33 | typedef struct { 34 | unsigned long fid; 35 | unsigned long offset; 36 | }burstfs_key_t; 37 | 38 | typedef struct { 39 | unsigned long delegator_id; 40 | unsigned long len; 41 | unsigned long addr; 42 | unsigned long app_rank_id; /*include both app and rank id*/ 43 | }burstfs_val_t; 44 | 45 | typedef struct { 46 | off_t file_pos; 47 | off_t mem_pos; 48 | size_t length; 49 | int fid; 50 | }burstfs_index_t; 51 | 52 | typedef struct { 53 | int fid; 54 | long offset; 55 | long length; 56 | }cli_req_t; 57 | 58 | extern arraylist_t *ulfs_keys; 59 | extern arraylist_t *ulfs_vals; 60 | extern arraylist_t *ulfs_metas; 61 | 62 | int meta_sanitize(); 63 | int meta_init_store(); 64 | void print_bget_indices(int app_id, int cli_id, \ 65 | send_msg_t *index_set, int tot_num); 66 | int meta_process_fsync (int sock_id); 67 | int meta_batch_get(int app_id, int client_id,\ 68 | int thrd_id, int dbg_rank, char *shm_reqbuf, int num,\ 69 | msg_meta_t* del_req_set); 70 | int meta_init_indices(); 71 | int meta_free_indices(); 72 | void print_fsync_indices(burstfs_key_t **burstfs_keys,\ 73 | burstfs_val_t **burstfs_vals, long num_entries); 74 | int meta_process_attr_set(char *ptr_cmd, int sock_id); 75 | int meta_process_attr_get(char *buf, int sock_id,\ 76 | burstfs_file_attr_t *ptr_attr_val); 77 | 78 | #endif 79 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/put-get.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "mpi.h" 4 | #include "mdhim.h" 5 | #include "mdhim_options.h" 6 | 7 | int main(int argc, char **argv) { 8 | int ret; 9 | int provided = 0; 10 | struct mdhim_t *md; 11 | int key; 12 | int value; 13 | struct mdhim_brm_t *brm; 14 | struct mdhim_bgetrm_t *bgrm; 15 | mdhim_options_t *db_opts; 16 | MPI_Comm comm; 17 | 18 | ret = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); 19 | if (ret != MPI_SUCCESS) { 20 | printf("Error initializing MPI with threads\n"); 21 | exit(1); 22 | } 23 | 24 | if (provided != MPI_THREAD_MULTIPLE) { 25 | printf("Not able to enable MPI_THREAD_MULTIPLE mode\n"); 26 | exit(1); 27 | } 28 | 29 | db_opts = mdhim_options_init(); 30 | mdhim_options_set_db_path(db_opts, "./"); 31 | mdhim_options_set_db_name(db_opts, "mdhim"); 32 | mdhim_options_set_db_type(db_opts, LEVELDB); 33 | mdhim_options_set_key_type(db_opts, MDHIM_INT_KEY); //Key_type = 1 (int) 34 | mdhim_options_set_debug_level(db_opts, MLOG_CRIT); 35 | mdhim_options_set_server_factor(db_opts, 1); 36 | 37 | comm = MPI_COMM_WORLD; 38 | md = mdhimInit(&comm, db_opts); 39 | if (!md) { 40 | printf("Error initializing MDHIM\n"); 41 | exit(1); 42 | } 43 | 44 | //Put the keys and values 45 | key = 100 * (md->mdhim_rank + 1); 46 | value = 500 * (md->mdhim_rank + 1); 47 | brm = mdhimPut(md, &key, sizeof(key), 48 | &value, sizeof(value), 49 | NULL, NULL); 50 | if (!brm || brm->error) { 51 | printf("Error inserting key/value into MDHIM\n"); 52 | } else { 53 | printf("Successfully inserted key/value into MDHIM\n"); 54 | } 55 | 56 | mdhim_full_release_msg(brm); 57 | //Commit the database 58 | ret = mdhimCommit(md, md->primary_index); 59 | if (ret != MDHIM_SUCCESS) { 60 | printf("Error committing MDHIM database\n"); 61 | } else { 62 | printf("Committed MDHIM database\n"); 63 | } 64 | 65 | //Get the values 66 | value = 0; 67 | bgrm = mdhimGet(md, md->primary_index, &key, sizeof(key), MDHIM_GET_EQ); 68 | if (!bgrm || bgrm->error) { 69 | printf("Error getting value for key: %d from MDHIM\n", key); 70 | } else { 71 | printf("Successfully got value: %d from MDHIM\n", *((int *) bgrm->values[0])); 72 | } 73 | 74 | mdhim_full_release_msg(bgrm); 75 | ret = mdhimClose(md); 76 | mdhim_options_destroy(db_opts); 77 | if (ret != MDHIM_SUCCESS) { 78 | printf("Error closing MDHIM\n"); 79 | } 80 | 81 | MPI_Barrier(MPI_COMM_WORLD); 82 | MPI_Finalize(); 83 | 84 | return 0; 85 | } 86 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/put-del.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "mpi.h" 4 | #include "mdhim.h" 5 | 6 | int main(int argc, char **argv) { 7 | int ret; 8 | int provided = 0; 9 | struct mdhim_t *md; 10 | int key; 11 | int value; 12 | struct mdhim_brm_t *brm; 13 | struct mdhim_bgetrm_t *bgrm; 14 | char *db_path = "./"; 15 | char *db_name = "mdhimTstDB"; 16 | int dbug = MLOG_CRIT; 17 | mdhim_options_t *db_opts; // Local variable for db create options to be passed 18 | int db_type = LEVELDB; // (data_store.h) 19 | MPI_Comm comm; 20 | 21 | // Create options for DB initialization 22 | db_opts = mdhim_options_init(); 23 | mdhim_options_set_db_path(db_opts, db_path); 24 | mdhim_options_set_db_name(db_opts, db_name); 25 | mdhim_options_set_db_type(db_opts, db_type); 26 | mdhim_options_set_key_type(db_opts, MDHIM_INT_KEY); 27 | mdhim_options_set_debug_level(db_opts, dbug); 28 | ret = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); 29 | if (ret != MPI_SUCCESS) { 30 | printf("Error initializing MPI with threads\n"); 31 | exit(1); 32 | } 33 | 34 | if (provided != MPI_THREAD_MULTIPLE) { 35 | printf("Not able to enable MPI_THREAD_MULTIPLE mode\n"); 36 | exit(1); 37 | } 38 | 39 | comm = MPI_COMM_WORLD; 40 | md = mdhimInit(&comm, db_opts); 41 | if (!md) { 42 | printf("Error initializing MDHIM\n"); 43 | exit(1); 44 | } 45 | 46 | //Put the keys and values 47 | key = 20 * (md->mdhim_rank + 1); 48 | value = 1000 * (md->mdhim_rank + 1); 49 | brm = mdhimPut(md, &key, sizeof(key), 50 | &value, sizeof(value), 51 | NULL, NULL); 52 | if (!brm || brm->error) { 53 | printf("Error inserting key/value into MDHIM\n"); 54 | } else { 55 | printf("Successfully inserted key/value into MDHIM\n"); 56 | } 57 | 58 | brm = mdhimDelete(md, md->primary_index, &key, sizeof(key)); 59 | if (!brm || brm->error) { 60 | printf("Error deleting key/value from MDHIM\n"); 61 | } else { 62 | printf("Successfully deleted key/value into MDHIM\n"); 63 | } 64 | 65 | //Get the values 66 | value = 0; 67 | bgrm = mdhimGet(md, md->primary_index, &key, sizeof(key), MDHIM_GET_EQ); 68 | if (!bgrm || bgrm->error) { 69 | printf("Error getting value for key: %d from MDHIM\n", key); 70 | } else if (bgrm->value_lens[0]) { 71 | printf("Successfully got value: %d from MDHIM\n", *((int *) bgrm->values[0])); 72 | } 73 | 74 | ret = mdhimClose(md); 75 | if (ret != MDHIM_SUCCESS) { 76 | printf("Error closing MDHIM\n"); 77 | } 78 | 79 | MPI_Barrier(MPI_COMM_WORLD); 80 | MPI_Finalize(); 81 | 82 | return 0; 83 | } 84 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/tester/inTestBulkLargeErr.txt: -------------------------------------------------------------------------------- 1 | #23456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 12345 2 | 3 | # data value of 2040 4 | bput 999999 dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040dataValueOf2040 abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz1234_abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ111112222233333abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz1234_abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ111112222233333abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz1234_abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ111112222233333abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz1234_abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ11111222223333344444555556666677777 5 | 6 | #Error: bulk put message too large. Bput is over Maximum size allowed of 2147483647. 7 | #Error: Packing message failed before sending. 8 | # 9 | # at src/messages.h MAX_BULK_OPS set to 1000000 10 | -------------------------------------------------------------------------------- /BurstFS_Meta/src/range_server.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * 17 | * Copyright (c) 2014, Los Alamos National Laboratory 18 | * All rights reserved. 19 | * 20 | */ 21 | 22 | #ifndef __RANGESRV_H 23 | #define __RANGESRV_H 24 | 25 | #include 26 | #include 27 | #include "data_store.h" 28 | #include "messages.h" 29 | #include "indexes.h" 30 | 31 | struct mdhim_t; 32 | 33 | typedef struct work_item work_item; 34 | struct work_item { 35 | work_item *next; 36 | work_item *prev; 37 | void *message; 38 | int source; 39 | }; 40 | 41 | typedef struct work_queue_t { 42 | work_item *head; 43 | work_item *tail; 44 | } work_queue_t; 45 | 46 | /* Outstanding requests (i.e., MPI_Req) that need to be freed later */ 47 | typedef struct out_req out_req; 48 | struct out_req { 49 | out_req *next; 50 | out_req *prev; 51 | void *req; 52 | MPI_Request *message; 53 | }; 54 | 55 | /* Range server specific data */ 56 | typedef struct mdhim_rs_t { 57 | work_queue_t *work_queue; 58 | pthread_mutex_t *work_queue_mutex; 59 | pthread_cond_t *work_ready_cv; 60 | pthread_t listener; 61 | pthread_t **workers; 62 | struct index *indexes; /* A linked list of remote indexes that is served 63 | (partially for fully) by this range server */ 64 | //Records seconds spent on putting records 65 | long double put_time; 66 | //Records seconds spend on getting records 67 | long double get_time; 68 | long num_put; 69 | long num_get; 70 | out_req *out_req_list; 71 | pthread_mutex_t *out_req_mutex; 72 | } mdhim_rs_t; 73 | 74 | int range_server_add_work(struct mdhim_t *md, work_item *item); 75 | int range_server_init(struct mdhim_t *md); 76 | int range_server_init_comm(struct mdhim_t *md); 77 | int range_server_stop(struct mdhim_t *md); 78 | int range_server_add_oreq(struct mdhim_t *md, MPI_Request *req, void *msg); //Add an outstanding request 79 | int range_server_clean_oreqs(struct mdhim_t *md); //Clean outstanding reqs 80 | int levedb_batch_ranges(void *dbh, char **key, int *key_len,\ 81 | char ***out_key, int **out_key_len,\ 82 | char ***out_val, int **out_val_len,\ 83 | int tot_records, int *out_records_cnt); 84 | 85 | #endif 86 | -------------------------------------------------------------------------------- /BurstFS_Client/src/cruise-fixed.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * Copyright (c) 2013, Lawrence Livermore National Security, LLC. 17 | * Produced at the Lawrence Livermore National Laboratory. 18 | * code Written by 19 | * Raghunath Rajachandrasekar 20 | * Kathryn Mohror 21 | * Adam Moody 22 | * All rights reserved. 23 | * This file is part of CRUISE. 24 | * For details, see https://github.com/hpc/cruise 25 | * Please also read this file LICENSE.CRUISE 26 | */ 27 | 28 | #ifndef CRUISE_FIXED_H 29 | #define CRUISE_FIXED_H 30 | 31 | #include "cruise-internal.h" 32 | 33 | /* if length is greater than reserved space, 34 | * reserve space up to length */ 35 | int cruise_fid_store_fixed_extend( 36 | int fid, /* file id to reserve space for */ 37 | cruise_filemeta_t* meta, /* meta data for file */ 38 | off_t length /* number of bytes to reserve for file */ 39 | ); 40 | 41 | /* if length is shorter than reserved space, 42 | * give back space down to length */ 43 | int cruise_fid_store_fixed_shrink( 44 | int fid, /* file id to free space for */ 45 | cruise_filemeta_t* meta, /* meta data for file */ 46 | off_t length /* number of bytes to reserve for file */ 47 | ); 48 | 49 | /* read data from file stored as fixed-size chunks, 50 | * returns CRUISE error code */ 51 | int cruise_fid_store_fixed_read( 52 | int fid, /* file id to read from */ 53 | cruise_filemeta_t* meta, /* meta data for file */ 54 | off_t pos, /* position within file to read from */ 55 | void* buf, /* user buffer to store data in */ 56 | size_t count /* number of bytes to read */ 57 | ); 58 | 59 | /* write data to file stored as fixed-size chunks, 60 | * returns CRUISE error code */ 61 | int cruise_fid_store_fixed_write( 62 | int fid, /* file id to write to */ 63 | cruise_filemeta_t* meta, /* meta data for file */ 64 | off_t pos, /* position within file to write to */ 65 | const void* buf, /* user buffer holding data */ 66 | size_t count /* number of bytes to write */ 67 | ); 68 | 69 | #endif /* CRUISE_FIXED_H */ 70 | -------------------------------------------------------------------------------- /BurstFS_Client/tests/Makefile.gcc: -------------------------------------------------------------------------------- 1 | # Intel compilers 2 | #MPICC = mpiicc 3 | #CC = icc 4 | 5 | #MPICC = mpixlc 6 | #CC = xlc 7 | 8 | #ZLIB = -L/usr/local/tools/zlib-1.2.6/lib -lz 9 | #ZLIB = -L../zlib-1.2.7/install/lib -lz 10 | 11 | # GNU compilers 12 | MPICC = mpicc 13 | CC = mpicc 14 | 15 | CFLAGS = -ggdb3 -O0 16 | LIBS = -lpthread 17 | 18 | NUMA_HOME = 19 | NUMA_INCS = 20 | NUMA_LDFLAGS = 21 | NUMA_LIBS = -lnuma 22 | #NUMA_HOME = ../numactl 23 | #NUMA_INCS = -I$(NUMA_HOME) 24 | #NUMA_LDFLAGS = -L$(NUMA_HOME) 25 | #NUMA_LIBS = -lnuma 26 | 27 | CONTAINER_HOME = 28 | CONTAINER_INCS = 29 | CONTAINER_LDFLAGS = 30 | CONTAINER_LIBS = 31 | #CONTAINER_HOME = ../container 32 | #CONTAINER_INCS = -I$(CONTAINER_HOME)/include 33 | #CONTAINER_LDFLAGS = -L$(CONTAINER_HOME)/lib 34 | #CONTAINER_LIBS = -lcontainer 35 | 36 | MPI_CFLAGS = $(shell pkg-config --cflags mpich | sed 's/-O2 -g//') 37 | MPI_LDFLAGS = $(shell pkg-config --libs mpich) 38 | 39 | CFLAGS += $(MPI_CFLAGS) 40 | LDFLAGS += $(MPI_LDFLAGS) 41 | 42 | CRUISE_LIBS = $(LIBS) $(NUMA_LIBS) $(CONTAINER_LIBS) 43 | CRUISE_LDFLAGS = $(LDFLAGS) $(NUMA_LDFLAGS) $(CONTAINER_LDFLAGS) -lrt 44 | 45 | PRE_CRUISE_FLAGS := $(shell echo `../cruise-config --pre-ld-flags`) 46 | POST_CRUISE_FLAGS := $(shell echo `../cruise-config --post-ld-flags`) 47 | 48 | TARGETS = test_write test_read test_pwrite test_listread test_pread test_writeread test_btio test_tileio 49 | 50 | all: $(TARGETS) 51 | 52 | clean: 53 | rm -f *.o $(TARGETS) 54 | 55 | test_read: test_read.c 56 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_read.c -o test_read $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 57 | 58 | test_write: test_write.c 59 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_write.c -o test_write $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 60 | 61 | test_pread: test_pread.c 62 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_pread.c -o test_pread $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 63 | 64 | test_pwrite: test_pwrite.c 65 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_pwrite.c -o test_pwrite $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 66 | 67 | test_listread: test_listread.c 68 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_listread.c -o test_listread $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 69 | 70 | test_writeread: test_writeread.c 71 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_writeread.c -o test_writeread $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 72 | 73 | test_btio: test_btio.c 74 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_btio.c -o test_btio $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 75 | 76 | test_tileio: test_tileio.c 77 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_tileio.c -o test_tileio $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) 78 | -------------------------------------------------------------------------------- /BurstFS_Client/cruise-runtime-config.h.in: -------------------------------------------------------------------------------- 1 | /* cruise-runtime-config.h.in. Generated from configure.ac by autoheader. */ 2 | 3 | /* Define if building universal (internal helper macro) */ 4 | #undef AC_APPLE_UNIVERSAL_BUILD 5 | 6 | /* Define if cuserid() should be disabled */ 7 | #undef CRUISE_DISABLE_CUSERID 8 | 9 | /* Define if libnuma is available */ 10 | #undef ENABLE_NUMA_POLICY 11 | 12 | /* Define to 1 if you have the header file. */ 13 | #undef HAVE_INTTYPES_H 14 | 15 | /* Define to 1 if you have the `numa' library (-lnuma). */ 16 | #undef HAVE_LIBNUMA 17 | 18 | /* Define to 1 if you have the header file. */ 19 | #undef HAVE_MEMORY_H 20 | 21 | /* Define to 1 if you have the header file. */ 22 | #undef HAVE_MNTENT_H 23 | 24 | /* Define if off64_t type is defined */ 25 | #undef HAVE_OFF64_T 26 | 27 | /* Define to 1 if you have the header file. */ 28 | #undef HAVE_STDINT_H 29 | 30 | /* Define to 1 if you have the header file. */ 31 | #undef HAVE_STDLIB_H 32 | 33 | /* Define to 1 if you have the header file. */ 34 | #undef HAVE_STRINGS_H 35 | 36 | /* Define to 1 if you have the header file. */ 37 | #undef HAVE_STRING_H 38 | 39 | /* Define to 1 if you have the header file. */ 40 | #undef HAVE_SYS_MOUNT_H 41 | 42 | /* Define to 1 if you have the header file. */ 43 | #undef HAVE_SYS_STAT_H 44 | 45 | /* Define to 1 if you have the header file. */ 46 | #undef HAVE_SYS_TYPES_H 47 | 48 | /* Define to 1 if you have the header file. */ 49 | #undef HAVE_UNISTD_H 50 | 51 | /* Define if architecture is BG/Q */ 52 | #undef MACHINE_BGQ 53 | 54 | /* Define to the address where bug reports for this package should be sent. */ 55 | #undef PACKAGE_BUGREPORT 56 | 57 | /* Define to the full name of this package. */ 58 | #undef PACKAGE_NAME 59 | 60 | /* Define to the full name and version of this package. */ 61 | #undef PACKAGE_STRING 62 | 63 | /* Define to the one symbol short name of this package. */ 64 | #undef PACKAGE_TARNAME 65 | 66 | /* Define to the home page for this package. */ 67 | #undef PACKAGE_URL 68 | 69 | /* Define to the version of this package. */ 70 | #undef PACKAGE_VERSION 71 | 72 | /* Define if exists and defines unusable PRI* macros. */ 73 | #undef PRI_MACROS_BROKEN 74 | 75 | /* Define to 1 if you have the ANSI C header files. */ 76 | #undef STDC_HEADERS 77 | 78 | /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most 79 | significant byte first (like Motorola and SPARC, unlike Intel). */ 80 | #if defined AC_APPLE_UNIVERSAL_BUILD 81 | # if defined __BIG_ENDIAN__ 82 | # define WORDS_BIGENDIAN 1 83 | # endif 84 | #else 85 | # ifndef WORDS_BIGENDIAN 86 | # undef WORDS_BIGENDIAN 87 | # endif 88 | #endif 89 | 90 | /* Define if device id should be taken from parent directory rather than file 91 | */ 92 | #undef __CP_ST_DEV_WORKAROUND 93 | -------------------------------------------------------------------------------- /BurstFS_Server/burstfs_const.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | #include "burstfs_const.h" 16 | 17 | const char * ULFS_str_errno(int rc) 18 | { 19 | switch(rc) { 20 | case ULFS_ERROR_DBG: return ULFS_STR_ERROR_DBG; 21 | case ULFS_ERROR_MDHIM: return ULFS_STR_ERROR_MDHIM; 22 | case ULFS_ERROR_THRD: return ULFS_STR_ERROR_THRDINIT; 23 | case ULFS_ERROR_GENERAL: return ULFS_STR_ERROR_GENERAL; 24 | case ULFS_ERROR_NOENV: return ULFS_STR_ERROR_NOENV; 25 | case ULFS_ERROR_NOMEM: return ULFS_STR_ERROR_NOMEM; 26 | case ULFS_ERROR_TIMEOUT: return ULFS_STR_ERROR_TIMEOUT; 27 | case ULFS_ERROR_EXIT: return ULFS_STR_ERROR_EXIT; 28 | case ULFS_ERROR_POLL: return ULFS_STR_ERROR_POLL; 29 | case ULFS_ERROR_SHMEM: return ULFS_STR_ERROR_SHMEM; 30 | case ULFS_ERROR_ROUTE: return ULFS_STR_ERROR_ROUTE; 31 | case ULFS_ERROR_EVENT_UNKNOWN: return ULFS_STR_ERROR_EVENT_UNKNOWN; 32 | case ULFS_ERROR_CONTEXT: return ULFS_STR_ERROR_CONTEXT; 33 | case ULFS_ERROR_QP: return ULFS_STR_ERROR_QP; 34 | case ULFS_ERROR_REGMEM: return ULFS_STR_ERROR_REGMEM; 35 | case ULFS_ERROR_PD: return ULFS_STR_ERROR_PD; 36 | case ULFS_ERROR_CHANNEL: return ULFS_STR_ERROR_CHANNEL; 37 | case ULFS_ERROR_POSTSEND: return ULFS_STR_ERROR_POSTSEND ; 38 | case ULFS_ERROR_ACCEPT: return ULFS_STR_ERROR_ACCEPT; 39 | case ULFS_ERROR_POSTRECV: return ULFS_STR_ERROR_POSTRECV; 40 | case ULFS_ERROR_CQ: return ULFS_STR_ERROR_CQ; 41 | case ULFS_ERROR_MDINIT: return ULFS_STR_ERROR_MDINIT; 42 | case ULFS_ERROR_THRDINIT: return ULFS_STR_ERROR_THRDINIT; 43 | case ULFS_ERROR_FILE: return ULFS_STR_ERROR_FILE; 44 | // case ULFS_ERROR_META: return ULFS_STR_ERROR_META; 45 | 46 | case ULFS_ERROR_SOCKET_FD_EXCEED: return ULFS_STR_ERROR_SOCKET_FD_EXCEED; 47 | case ULFS_ERROR_SOCK_DISCONNECT: return ULFS_STR_ERROR_SOCK_DISCONNECT; 48 | case ULFS_ERROR_SOCK_CMD: return ULFS_STR_ERROR_SOCK_CMD; 49 | case ULFS_ERROR_SOCK_LISTEN: return ULFS_STR_ERROR_SOCK_LISTEN; 50 | case ULFS_ERROR_APPCONFIG: return ULFS_STR_ERROR_APPCONFIG; 51 | case ULFS_ERROR_ARRAY_EXCEED: return ULFS_STR_ERROR_ARRAY_EXCEED; 52 | case ULFS_ERROR_RM_INIT: return ULFS_STR_ERROR_RM_INIT; 53 | case ULFS_ERROR_READ: return ULFS_STR_ERROR_READ; 54 | case ULFS_ERROR_SEND: return ULFS_STR_ERROR_SEND; 55 | case ULFS_ERROR_WRITE: return ULFS_STR_ERROR_WRITE; 56 | default: return ULFS_STR_ERROR_DEFAULT; 57 | 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/Makefile: -------------------------------------------------------------------------------- 1 | include ../../Makefile.cfg 2 | CLIBS += -L../../src 3 | CINC += -I../../src -I../../src/uthash 4 | all: put-get bput-bget put-del bput-bdel put-getn \ 5 | put-getp puts-gets bput-bgetn bput-bgetp \ 6 | bput-bget_secondary put-get_secondary \ 7 | put-get_secondary_local bput-bget_secondary_local \ 8 | put-getn_secondary put-getn_secondary_local \ 9 | put-del_secondary put-getp_secondary put-get_2secondary_local \ 10 | put-del_secondary_local plfs-put-get index_name range_test \ 11 | range_bget 12 | 13 | put-get: put-get.c 14 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 15 | 16 | plfs-put-get: plfs-put-get.c 17 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 18 | 19 | put-get_secondary: put-get_secondary.c 20 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 21 | 22 | put-get_secondary_local: put-get_secondary_local.c 23 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 24 | 25 | put-get_2secondary_local: put-get_2secondary_local.c 26 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 27 | 28 | puts-gets: puts-gets.c 29 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 30 | 31 | put-getn: put-getn.c 32 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 33 | 34 | put-getn_secondary: put-getn_secondary.c 35 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 36 | 37 | put-getn_secondary_local: put-getn_secondary_local.c 38 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 39 | 40 | put-getp_secondary: put-getp_secondary.c 41 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 42 | 43 | 44 | put-getp: put-getp.c 45 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 46 | 47 | put-del: put-del.c 48 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 49 | 50 | put-del_secondary: put-del_secondary.c 51 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 52 | 53 | put-del_secondary_local: put-del_secondary_local.c 54 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 55 | 56 | bput-bget: bput-bget.c 57 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 58 | 59 | bput-bget_secondary: bput-bget_secondary.c 60 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 61 | 62 | bput-bget_secondary_local: bput-bget_secondary_local.c 63 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 64 | 65 | bput-bgetn: bput-bgetn.c 66 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 67 | 68 | bput-bgetp: bput-bgetp.c 69 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 70 | 71 | range_test: range_test.c 72 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 73 | 74 | range_bget: range_bget.c 75 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 76 | 77 | bput-bdel: bput-bdel.c 78 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 79 | 80 | index_name: index_name.c 81 | $(CC) $< $(CINC) $(CLIBS) $(CFLAGS) -o $@ 82 | 83 | clean: 84 | rm -rf put-get bput-bget put-del bput-bdel\ 85 | put-getn put-getp range_test\ 86 | puts-gets bput-bgetn \ 87 | bput-bgetp bput-bget_secondary \ 88 | put-get_secondary put-get_secondary_local \ 89 | bput-bget_secondary_local put-getn_secondary_local \ 90 | put-getn_secondary put-del_secondary put-getp_secondary \ 91 | put-get_2secondary_local put-del_secondary_local plfs-put-get index_name \ 92 | range_test range_bget 93 | 94 | -------------------------------------------------------------------------------- /BurstFS_Client/maint/config/check_numa.m4: -------------------------------------------------------------------------------- 1 | dnl @synopsis CHECK_NUMA() 2 | dnl 3 | dnl This macro searches for an installed numa library. If nothing was 4 | dnl specified when calling configure, it searches first in /usr/local 5 | dnl and then in /usr. If the --with-numa=DIR is specified, it will try 6 | dnl to find it in DIR/include/numa.h and DIR/lib/libz.a. If 7 | dnl --without-numa is specified, the library is not searched at all. 8 | dnl 9 | dnl If either the header file (numa.h) or the library (libz) is not 10 | dnl found, the configuration exits on error, asking for a valid numa 11 | dnl installation directory or --without-numa. 12 | dnl 13 | dnl The macro defines the symbol HAVE_LIBZ if the library is found. You 14 | dnl should use autoheader to include a definition for this symbol in a 15 | dnl config.h file. Sample usage in a C/C++ source is as follows: 16 | dnl 17 | dnl #ifdef HAVE_LIBZ 18 | dnl #include 19 | dnl #endif /* HAVE_LIBZ */ 20 | dnl 21 | dnl @category InstalledPackages 22 | dnl @author Loic Dachary 23 | dnl @version 2004-09-20 24 | dnl @license GPLWithACException 25 | 26 | AC_DEFUN([CHECK_NUMA], 27 | 28 | # 29 | # Handle user hints 30 | # 31 | [AC_MSG_CHECKING(if numa is wanted ) 32 | AC_ARG_WITH([numa], 33 | [AS_HELP_STRING([--with-numa=DIR],[root directory path of libnuma installation (defaults to /usr/local or /usr if not found in /usr/local)])], 34 | [if test "$withval" != no ; then 35 | AC_MSG_RESULT(yes) 36 | if test -d "$withval" 37 | then 38 | NUMA_HOME="$withval" 39 | AC_DEFINE([ENABLE_NUMA_POLICY], [1], [Define if libnuma is available]) 40 | else 41 | AC_MSG_CHECKING([for libnuma installation in default locations]) 42 | fi 43 | else 44 | AC_MSG_RESULT(no) 45 | fi]) 46 | 47 | # 48 | # Locate numa, if wanted 49 | # 50 | if test -n "${NUMA_HOME}" 51 | then 52 | NUMA_OLD_LDFLAGS=$LDFLAGS 53 | NUMA_OLD_CPPFLAGS=$LDFLAGS 54 | LDFLAGS="$LDFLAGS -L${NUMA_HOME}/lib" 55 | CPPFLAGS="$CPPFLAGS -I${NUMA_HOME}/include" 56 | AC_LANG_SAVE 57 | AC_LANG_C 58 | AC_CHECK_LIB(numa, numa_num_possible_nodes, [numa_cv_libnuma=yes], [numa_cv_libnuma=no]) 59 | AC_CHECK_HEADER(numa.h, [numa_cv_numa_h=yes], [numa_cv_numa_h=no]) 60 | AC_LANG_RESTORE 61 | if test "$numa_cv_libnuma" = "yes" -a "$numa_cv_numa_h" = "yes" 62 | then 63 | # 64 | # If both library and header were found, use them 65 | # 66 | AC_CHECK_LIB(numa, numa_num_possible_nodes) 67 | AC_MSG_CHECKING(numa in ${NUMA_HOME}) 68 | AC_MSG_RESULT(ok) 69 | else 70 | # 71 | # If either header or library was not found, revert and bomb 72 | # 73 | AC_MSG_CHECKING(numa in ${NUMA_HOME}) 74 | LDFLAGS="$NUMA_OLD_LDFLAGS" 75 | CPPFLAGS="$NUMA_OLD_CPPFLAGS" 76 | AC_MSG_RESULT(failed) 77 | AC_MSG_ERROR(either specify a valid numa installation with --with-numa=DIR or disable numa usage with --without-numa) 78 | fi 79 | fi 80 | 81 | ]) 82 | -------------------------------------------------------------------------------- /BurstFS_Server/arraylist.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | #include "arraylist.h" 16 | #include 17 | #include 18 | #include 19 | 20 | arraylist_t * arraylist_create() { 21 | arraylist_t *arr = (arraylist_t *)malloc(sizeof(arraylist_t)); 22 | if (!arr) 23 | return NULL; 24 | 25 | arr->cap = DEF_ARR_CAP; 26 | arr->size = 0; 27 | arr->elems = (void **)malloc(arr->cap * sizeof(void *)); 28 | 29 | if (!arr->elems) 30 | return NULL; 31 | 32 | int i; 33 | for (i = 0; i < arr->cap; i++) { 34 | arr->elems[i] = NULL; 35 | } 36 | 37 | return arr; 38 | } 39 | 40 | int arraylist_capacity(arraylist_t *arr) { 41 | return arr->cap; 42 | } 43 | 44 | int arraylist_size(arraylist_t *arr) { 45 | return arr->size; 46 | } 47 | 48 | void * arraylist_get(arraylist_t *arr, int pos) { 49 | if (pos >= arr->size) 50 | return NULL; 51 | return arr->elems[pos]; 52 | }; 53 | 54 | int arraylist_insert(arraylist_t *arr, int pos, void *elem) { 55 | if (pos >= arr->cap){ 56 | arr->elems = (void **)realloc(arr->elems, \ 57 | 2 * pos * sizeof(void *)); 58 | if (!arr->elems) 59 | return -1; 60 | 61 | long i; 62 | for (i = arr->cap; i < 2 * pos; i++) { 63 | arr->elems[i] = NULL; 64 | } 65 | arr->cap = 2 * pos; 66 | } 67 | 68 | if (arr->elems[pos] != NULL) 69 | free(arr->elems[pos]); 70 | arr->elems[pos] = elem; 71 | if (pos + 1 > arr->size) 72 | arr->size = pos + 1; 73 | return 0; 74 | 75 | } 76 | 77 | 78 | int arraylist_add(arraylist_t *arr, void *elem) { 79 | if (arr->size == arr->cap){ 80 | arr->elems = (void **)realloc(arr->elems, 2 * arr->cap * sizeof(void *)); 81 | if (!arr->elems) 82 | return -1; 83 | 84 | long i; 85 | for (i = arr->cap; i < 2 * arr->cap; i++) { 86 | arr->elems[i] = NULL; 87 | } 88 | arr->cap = arr->cap * 2; 89 | } 90 | 91 | if (arr->elems[arr->size] != NULL) { 92 | free(arr->elems[arr->size]); 93 | } 94 | arr->elems[arr->size] = elem; 95 | arr->size = arr->size + 1; 96 | return 0; 97 | 98 | } 99 | 100 | int arraylist_reset(arraylist_t *arr) { 101 | if (!arr) 102 | return -1; 103 | 104 | arr->size = 0; 105 | return 0; 106 | } 107 | 108 | int arraylist_free(arraylist_t *arr) { 109 | long i; 110 | for (i = 0; i < arr->cap; i++) { 111 | if (arr->elems[i] != NULL) 112 | free(arr->elems[i]); 113 | } 114 | 115 | free(arr); 116 | return 0; 117 | } 118 | -------------------------------------------------------------------------------- /BurstFS_Client/tests/makefile.bgq: -------------------------------------------------------------------------------- 1 | # Intel compilers 2 | #MPICC = mpiicc 3 | #CC = icc 4 | 5 | MPICC = mpixlc 6 | CC = xlc 7 | 8 | #ZLIB = -L/usr/local/tools/zlib-1.2.6/lib -lz 9 | #ZLIB = -L../zlib-1.2.7/install/lib -lz 10 | 11 | # GNU compilers 12 | #MPICC = mpicc 13 | #CC = gcc 14 | 15 | CFLAGS = -g -O0 16 | LIBS = -lpthread 17 | 18 | NUMA_HOME = 19 | NUMA_INCS = 20 | NUMA_LDFLAGS = 21 | NUMA_LIBS = 22 | #NUMA_HOME = ../numactl 23 | #NUMA_INCS = -I$(NUMA_HOME) 24 | #NUMA_LDFLAGS = -L$(NUMA_HOME) 25 | #NUMA_LIBS = -lnuma 26 | 27 | CONTAINER_HOME = 28 | CONTAINER_INCS = 29 | CONTAINER_LDFLAGS = 30 | CONTAINER_LIBS = 31 | #CONTAINER_HOME = ../container 32 | #CONTAINER_INCS = -I$(CONTAINER_HOME)/include 33 | #CONTAINER_LDFLAGS = -L$(CONTAINER_HOME)/lib 34 | #CONTAINER_LIBS = -lcontainer 35 | 36 | CRUISE_LIBS = $(LIBS) $(NUMA_LIBS) $(CONTAINER_LIBS) 37 | CRUISE_LDFLAGS = $(LDFLAGS) $(NUMA_LDFLAGS) $(CONTAINER_LDFLAGS) 38 | 39 | PRE_CRUISE_FLAGS := $(shell echo `../install_bgq/bin/cruise-config --pre-ld-flags`) 40 | POST_CRUISE_FLAGS := $(shell echo `../install_bgq/bin/cruise-config --post-ld-flags`) 41 | 42 | all: test_writeread test_readwrite test_truncate test_fopen test_fprintf test_memcpy test_ramdisk 43 | 44 | clean: 45 | rm -f *.o test1 test1_container test_writeread test_readwrite test_truncate test_fopen test_fprintf test_ungetc test_scanf test_wscanf test_memcpy test_ramdisk 46 | 47 | test1: test1.c 48 | $(CC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test1.c -o test1 $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 49 | 50 | test_writeread: test_writeread.c 51 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_writeread.c -o test_writeread $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 52 | 53 | test_readwrite: test_readwrite.c 54 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_readwrite.c -o test_readwrite $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 55 | 56 | test_truncate: test_truncate.c 57 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_truncate.c -o test_truncate $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 58 | 59 | test_fopen: test_fopen.c 60 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_fopen.c -o test_fopen $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 61 | 62 | test_fprintf: test_fprintf.c 63 | $(MPICC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_fprintf.c -o test_fprintf $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 64 | 65 | test_ungetc: test_ungetc.c 66 | $(CC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_ungetc.c -o test_ungetc $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 67 | 68 | test_scanf: test_scanf.c 69 | $(CC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_scanf.c -o test_scanf $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 70 | 71 | test_wscanf: test_wscanf.c 72 | $(CC) $(CFLAGS) $(INCLUDES) $(PRE_CRUISE_FLAGS) test_wscanf.c -o test_wscanf $(CRUISE_LDFLAGS) $(CRUISE_LIBS) $(POST_CRUISE_FLAGS) $(ZLIB) 73 | 74 | test_memcpy: test_memcpy.c 75 | $(MPICC) $(CFLAGS) $(INCLUDES) test_memcpy.c -o test_memcpy $(LDFLAGS) $(LIBS) $(ZLIB) 76 | 77 | test_ramdisk: test_ramdisk.c 78 | $(MPICC) $(CFLAGS) $(INCLUDES) test_ramdisk.c -o test_ramdisk $(LDFLAGS) $(LIBS) $(ZLIB) 79 | 80 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/put-getp.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "mpi.h" 4 | #include "mdhim.h" 5 | 6 | int main(int argc, char **argv) { 7 | int ret; 8 | int provided = 0; 9 | struct mdhim_t *md; 10 | int key; 11 | int value; 12 | struct mdhim_brm_t *brm; 13 | struct mdhim_bgetrm_t *bgrm; 14 | int i; 15 | int keys_per_rank = 5; 16 | char *db_path = " "; 17 | char *db_name = "mdhim"; 18 | int dbug = MLOG_CRIT; 19 | mdhim_options_t *db_opts; // Local variable for db create options to be passed 20 | int db_type = LEVELDB; // (data_store.h) 21 | MPI_Comm comm; 22 | 23 | // Create options for DB initialization 24 | db_opts = mdhim_options_init(); 25 | mdhim_options_set_db_path(db_opts, db_path); 26 | mdhim_options_set_db_name(db_opts, db_name); 27 | mdhim_options_set_db_type(db_opts, db_type); 28 | mdhim_options_set_key_type(db_opts, MDHIM_INT_KEY); 29 | mdhim_options_set_debug_level(db_opts, dbug); 30 | 31 | ret = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); 32 | if (ret != MPI_SUCCESS) { 33 | printf("Error initializing MPI with threads\n"); 34 | exit(1); 35 | } 36 | 37 | if (provided != MPI_THREAD_MULTIPLE) { 38 | printf("Not able to enable MPI_THREAD_MULTIPLE mode\n"); 39 | exit(1); 40 | } 41 | 42 | comm = MPI_COMM_WORLD; 43 | md = mdhimInit(&comm, db_opts); 44 | if (!md) { 45 | printf("Error initializing MDHIM\n"); 46 | exit(1); 47 | } 48 | 49 | //Put the keys and values 50 | for (i = 0; i < keys_per_rank; i++) { 51 | key = keys_per_rank * md->mdhim_rank + i; 52 | value = md->mdhim_rank + i; 53 | brm = mdhimPut(md, &key, sizeof(key), 54 | &value, sizeof(value), 55 | NULL, NULL); 56 | if (!brm || brm->error) { 57 | printf("Error inserting key/value into MDHIM\n"); 58 | } else { 59 | printf("Rank: %d put key: %d with value: %d\n", md->mdhim_rank, key, value); 60 | } 61 | } 62 | 63 | //Commit the database 64 | ret = mdhimCommit(md, md->primary_index); 65 | if (ret != MDHIM_SUCCESS) { 66 | printf("Error committing MDHIM database\n"); 67 | } else { 68 | printf("Committed MDHIM database\n"); 69 | } 70 | 71 | //Get the stats 72 | ret = mdhimStatFlush(md, md->primary_index); 73 | if (ret != MDHIM_SUCCESS) { 74 | printf("Error getting stats\n"); 75 | } else { 76 | printf("Got stats\n"); 77 | } 78 | 79 | //Get the values using get_prev 80 | for (i = keys_per_rank; i > 0; i--) { 81 | value = 0; 82 | key = keys_per_rank * md->mdhim_rank + i; 83 | bgrm = mdhimBGetOp(md, md->primary_index, 84 | &key, sizeof(int), 1, MDHIM_GET_PREV); 85 | if (!bgrm || bgrm->error) { 86 | printf("Error getting value for key: %d from MDHIM\n", key); 87 | } else if (bgrm->keys[0] && bgrm->values[0]) { 88 | printf("Rank: %d successfully got key: %d with value: %d from MDHIM\n", 89 | md->mdhim_rank, 90 | *((int *) bgrm->keys[0]), 91 | *((int *) bgrm->values[0])); 92 | } 93 | } 94 | 95 | ret = mdhimClose(md); 96 | if (ret != MDHIM_SUCCESS) { 97 | printf("Error closing MDHIM\n"); 98 | } 99 | 100 | MPI_Barrier(MPI_COMM_WORLD); 101 | MPI_Finalize(); 102 | 103 | return 0; 104 | } 105 | -------------------------------------------------------------------------------- /BurstFS_Client/Makefile.in: -------------------------------------------------------------------------------- 1 | all: src/libcruise-posix.a 2 | 3 | DESTDIR = 4 | srcdir = @srcdir@ 5 | prefix = @prefix@ 6 | exec_prefix = @exec_prefix@ 7 | datarootdir = $(DESTDIR)@datarootdir@ 8 | includedir = $(DESTDIR)@includedir@ 9 | mandir = $(DESTDIR)@mandir@ 10 | sbindir = $(DESTDIR)@sbindir@ 11 | bindir = $(DESTDIR)@bindir@ 12 | libdir = $(DESTDIR)@libdir@ 13 | LDFLAGS = @LDFLAGS@ 14 | CC = @CC@ 15 | LD = @LD@ 16 | 17 | NUMA_LIBS = -lnuma 18 | 19 | 20 | DISABLE_LDPRELOAD = @DISABLE_LDPRELOAD@ 21 | 22 | ifndef DISABLE_LDPRELOAD 23 | all: src/libcruise.so 24 | endif 25 | 26 | VPATH = $(srcdir) 27 | 28 | CFLAGS = -DCRUISE_CONFIG_H=\"cruise-runtime-config.h\" -I . -I ../ -I $(srcdir) -I$(srcdir)/../ @CFLAGS@ @CPPFLAGS@ -D_LARGEFILE64_SOURCE 29 | 30 | CFLAGS_SHARED = -DCRUISE_CONFIG_H=\"cruise-runtime-config.h\" -I . -I$(srcdir) -I$(srcdir)/../ @CFLAGS@ @CPPFLAGS@ -D_LARGEFILE64_SOURCE -shared -fpic -DPIC -DCRUISE_PRELOAD 31 | 32 | LIBS = @LIBBZ2@ $(NUMA_LIBS) 33 | 34 | lib:: 35 | @mkdir -p $@ 36 | 37 | HEADERS = \ 38 | src/cruise-stack.h \ 39 | src/cruise-fixed.h \ 40 | src/cruise-sysio.h \ 41 | src/cruise-stdio.h \ 42 | src/cruise-internal.h \ 43 | src/cruise.h 44 | 45 | OBJS = \ 46 | src/cruise-stack.o \ 47 | src/cruise-fixed.o \ 48 | src/cruise-sysio.o \ 49 | src/cruise-stdio.o \ 50 | src/cruise.o 51 | 52 | POBJS = \ 53 | src/cruise-stack.po \ 54 | src/cruise-fixed.po \ 55 | src/cruise-sysio.po \ 56 | src/cruise-stdio.po \ 57 | src/cruise.po 58 | 59 | src/cruise-fixed.o: src/cruise-fixed.c $(HEADERS) 60 | $(CC) $(CFLAGS) -c $< -o $@ 61 | 62 | src/cruise-fixed.po: src/cruise-fixed.c $(HEADERS) 63 | $(CC) $(CFLAGS_SHARED) -c $< -o $@ 64 | 65 | 66 | src/cruise.o: src/cruise.c $(HEADERS) 67 | $(CC) $(CFLAGS) -c src/cruise.c -o $@ 68 | 69 | src/cruise.po: src/cruise.c $(HEADERS) 70 | $(CC) $(CFLAGS_SHARED) -c src/cruise.c -o $@ 71 | 72 | 73 | src/cruise-sysio.o: src/cruise-sysio.c $(HEADERS) 74 | $(CC) $(CFLAGS) -c src/cruise-sysio.c -o $@ 75 | 76 | src/cruise-sysio.po: src/cruise-sysio.c $(HEADERS) 77 | $(CC) $(CFLAGS_SHARED) -c src/cruise-sysio.c -o $@ 78 | 79 | 80 | src/cruise-stdio.o: src/cruise-stdio.c $(HEADERS) 81 | $(CC) $(CFLAGS) -c src/cruise-stdio.c -o $@ 82 | 83 | src/cruise-stdio.po: src/cruise-stdio.c $(HEADERS) 84 | $(CC) $(CFLAGS_SHARED) -c src/cruise-stdio.c -o $@ 85 | 86 | 87 | src/cruise-stack.o: src/cruise-stack.c $(HEADERS) 88 | $(CC) $(CFLAGS) -c $< -o $@ 89 | 90 | src/cruise-stack.po: src/cruise-stack.c $(HEADERS) 91 | $(CC) $(CFLAGS_SHARED) -c $< -o $@ 92 | 93 | 94 | %.i: %.c 95 | $(CC) -E $(CFLAGS) -c $< -o $@ 96 | 97 | src/libcruise-posix.a: $(OBJS) 98 | ar rcs $@ $^ 99 | 100 | src/libcruise.so: $(POBJS) 101 | $(CC) $(CFLAGS_SHARED) $(LDFLAGS) -ldl -o $@ $^ -lpthread -lrt 102 | 103 | install:: all 104 | install -d $(libdir) 105 | install -m 755 src/libcruise-posix.a $(libdir) 106 | ifndef DISABLE_LDPRELOAD 107 | install -m 755 src/libcruise.so $(libdir) 108 | endif 109 | install -d $(bindir) 110 | install -m 755 cruise-config $(bindir) 111 | 112 | clean:: 113 | rm -f *.o *.po *.a src/*.o src/*.po src/*.a src/*.so *.in~ 114 | 115 | distclean:: clean 116 | rm -f cruise-runtime-config.h aclocal.m4 autom4te.cache/* config.status config.log Makefile 117 | rm -rf autom4te.cache 118 | -------------------------------------------------------------------------------- /BurstFS_Meta/README.md: -------------------------------------------------------------------------------- 1 | MDHIM - Multi-Dimensional Hashing Indexing Middleware 2 | 3 | 4 | Description 5 | --------------- 6 | This version of MDHIM package is customized to support BurstFS. 7 | In order to test these extended 8 | interfaces and implementations, two test files are provided in the directory tests/singletests 9 | (range_bget.c and range_test.c). The test scripts for these two files are 10 | range_bget.sh range_test.sh. 11 | 12 | MDHIM is a parallel key/value store framework written in MPI. 13 | Unlike other big data solutions, MDHIM has been designed for an HPC 14 | environment and to take advantage of high speed networks. 15 | 16 | 17 | Requirements 18 | --------------- 19 | LevelDB or MySQL as a database backend 20 | - Leveldb: http://code.google.com/p/leveldb/ 21 | - Mysqldb: http://www.mysql.com/ 22 | - Mysqldb C Connector: http://dev.mysql.com/downloads/connector/c/ 23 | 24 | An MPI distribution that supports MPI_THREAD_MULTIPLE and per-object locking of critical sections (this excludes OpenMPI). We have had best results with MPICH: http://www.mpich.org/. For mpich, compile with --enable-thread-cs=per-object. If you get an error compiling, see if this patch helps: http://lists.mpich.org/pipermail/discuss/2014-May/002779.html . 25 | 26 | 27 | Building the Library 28 | --------------- 29 | 1. Modify example.Makefile.cfg to point to your leveldb and MPI installations and save as Makefile.cfg 30 | 2. Type: make 31 | 3. If all went well, you have the library in src/libmdhim.a 32 | 33 | 34 | Building the Tests 35 | --------------- 36 | 1. cd tests 37 | 2. Type: make 38 | 3. If all went well, you have all the tests compiled 39 | 40 | Issues 41 | --------------- 42 | 43 | With mysql you must remove the manifest file before running mdhim with a new set of paramters. 44 | 45 | 46 | mdhimtst (mdhimtst.c) 47 | --------------- 48 | Typical setup to run file: ./mdhimtst -ftest.txt -q -t5 -d3 -r0~2 -w1 -p ./ 49 | 50 | Typical batch file setup: 51 | 52 | put single_insert.txt 53 | get single_insertget.txt 54 | del single_insertdel.txt 55 | 56 | Batch command file setup: 57 | 58 | For put get del bput bget bdel 59 | [command] [#of items in file(if necessary)] [file containg key and value (if necessary)] 60 | 61 | For nput ngetn (random numbers are generated for this function) 62 | [#number of records to put/get] [key size (only applicapble for byte/strng)] [value size (nput only)] [size correctance number] 63 | 64 | For flush (only command is used to flush) 65 | 66 | 67 | Parameters: 68 | -f (file with batch commands) 69 | -d (Type of DB to use: levelDB=1 mysql=3) 70 | -t (Type of keys: int=1, longInt=2, float=3, double=4, longDouble=5, string=6, byte=7) 71 | -p (path where DB will be created) 72 | -n (Name of DataBase file or directory) 73 | -b (MLOG_CRIT=1, MLOG_DBG=2) 74 | -a (DB store append mode. By default records with same key are overwritten. This flag turns on the option to append to existing values. 75 | -w This flag turns on the option to either allow or deny threads to do command based on if it is dividiable by the modlus of the modulus number 76 | -r ~ This flag turns on the option to either allow or deny threads to do command based on if the rank falls inclusively inbetween the rank ranges. NOTE: You must use the '~' inbetween the numbers. Example: -r0~2 77 | -q<0|1> (Quiet mode, default is verbose) 1=write out to log file 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /BurstFS_Client/src/cruise.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * Copyright (c) 2013, Lawrence Livermore National Security, LLC. 17 | * Produced at the Lawrence Livermore National Laboratory. 18 | * code Written by 19 | * Raghunath Rajachandrasekar 20 | * Kathryn Mohror 21 | * Adam Moody 22 | * All rights reserved. 23 | * This file is part of CRUISE. 24 | * For details, see https://github.com/hpc/cruise 25 | * Please also read this file LICENSE.CRUISE 26 | */ 27 | 28 | #ifndef CRUISE_H 29 | #define CRUISE_H 30 | 31 | #include 32 | #include 33 | #include 34 | /* TODO: namespace C */ 35 | 36 | /* linked list of chunk information given to an external library wanting 37 | * to RDMA out a file from CRUISE */ 38 | typedef struct { 39 | off_t chunk_id; 40 | int location; 41 | void *chunk_mr; 42 | off_t spillover_offset; 43 | struct chunk_list_t *next; 44 | } chunk_list_t; 45 | 46 | /*data structures defined for burstfs********************/ 47 | 48 | #define MMAP_OPEN_FLAG O_RDWR|O_CREAT 49 | #define MMAP_OPEN_MODE 00777 50 | 51 | typedef struct { 52 | char hostname[CRUISE_MAX_FILENAME]; 53 | int rank; 54 | }name_rank_pair_t; 55 | 56 | static int get_del_cnt(); 57 | static int compare_int(void *a, void *b); 58 | static int compare_name_rank_pair(const void *a, const void *b); 59 | static int find_rank_idx(int rank,\ 60 | int *local_rank_lst, int local_rank_cnt); 61 | static int burstfs_init_socket(int proc_id,\ 62 | int l_num_procs_per_node, \ 63 | int l_num_del_per_node); 64 | static int CountTasksPerNode(int rank, int numTasks); 65 | static int burstfs_init_req_shm(int local_rank_idx, int app_id); 66 | int burstfs_mount(const char prefix[], int rank, size_t size,\ 67 | int l_app_id, int subtype); 68 | static int burstfs_init_recv_shm(int local_rank_idx, int app_id); 69 | static int burstfs_init_req_shm(int local_rank_idx, int app_id); 70 | static int burstfs_sync_to_del(); 71 | static int cruise_get_global_fid(const char *path, int *gfid); 72 | static int get_global_file_meta(int gfid, burstfs_fattr_t **file_meta); 73 | static int set_global_file_meta(burstfs_fattr_t *f_meta); 74 | static int ins_file_meta(burstfs_fattr_buf_t *ptr_f_meta_log,\ 75 | burstfs_fattr_t *ins_fattr); 76 | int compare_fattr(void *a, void *b); 77 | 78 | 79 | /* mount memfs at some prefix location */ 80 | int cruise_mount(const char prefix[], size_t size, int rank); 81 | 82 | /* get information about the chunk data region 83 | * for external async libraries to register during their init */ 84 | size_t cruise_get_data_region(void **ptr); 85 | 86 | /* get a list of chunks for a given file (useful for RDMA, etc.) */ 87 | chunk_list_t* cruise_get_chunk_list(char* path); 88 | 89 | /* debug function to print list of chunks constituting a file 90 | * and to test above function*/ 91 | void cruise_print_chunk_list(char* path); 92 | 93 | #endif /* CRUISE_H */ 94 | -------------------------------------------------------------------------------- /BurstFS_Meta/src/ds_leveldb.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * 17 | * Copyright (c) 2014, Los Alamos National Laboratory 18 | * All rights reserved. 19 | * 20 | */ 21 | 22 | #ifndef __LEVELDB_H 23 | #define __LEVELDB_H 24 | 25 | #ifndef LEVELDB_SUPPORT 26 | #include 27 | #else 28 | #include 29 | #endif 30 | 31 | #include "mdhim.h" 32 | #include "partitioner.h" 33 | #include "data_store.h" 34 | 35 | #define BURSTFS_FID(key) *(long *)key 36 | #define BURSTFS_OFFSET(key) *((long *)key + 1) 37 | #define BURSTFS_ADDR(val) *((long *)val + 2) 38 | #define BURSTFS_LEN(val) *((long *)val + 1) 39 | 40 | /* Function pointer for comparator in C */ 41 | typedef int (*mdhim_store_cmp_fn_t)(void* arg, const char* a, size_t alen, 42 | const char* b, size_t blen); 43 | 44 | struct mdhim_leveldb_t { 45 | leveldb_t *db; 46 | leveldb_options_t *options; 47 | leveldb_comparator_t* cmp; 48 | leveldb_filterpolicy_t *filter; 49 | leveldb_cache_t *cache; 50 | leveldb_env_t *env; 51 | leveldb_writeoptions_t *write_options; 52 | leveldb_readoptions_t *read_options; 53 | mdhim_store_cmp_fn_t compare; 54 | }; 55 | int mdhim_leveldb_open(void **dbh, void **dbs, char *path, int flags, int key_type, struct mdhim_options_t *opts); 56 | int mdhim_leveldb_put(void *dbh, void *key, int key_len, void *data, int32_t data_len); 57 | int mdhim_leveldb_get(void *dbh, void *key, int key_len, void **data, int32_t *data_len); 58 | int mdhim_leveldb_get_next(void *dbh, void **key, int *key_len, 59 | void **data, int32_t *data_len); 60 | int mdhim_leveldb_get_prev(void *dbh, void **key, int *key_len, 61 | void **data, int32_t *data_len); 62 | int mdhim_leveldb_close(void *dbh, void *dbs); 63 | int mdhim_leveldb_del(void *dbh, void *key, int key_len); 64 | int mdhim_leveldb_commit(void *dbh); 65 | int mdhim_leveldb_batch_put(void *dbh, void **key, int32_t *key_lens, 66 | void **data, int32_t *data_lens, int num_records); 67 | int mdhim_levedb_batch_next(void *dbh, char **key,\ 68 | int *key_len, char **data, int32_t *data_len, \ 69 | int tot_records, int *num_records); 70 | int levedb_batch_ranges(void *dbh, char **key, int *key_len,\ 71 | char ***out_key, int **out_key_len,\ 72 | char ***out_val, int **out_val_len,\ 73 | int tot_records, int *out_records_cnt); 74 | int leveldb_process_range(leveldb_iterator_t *iter,\ 75 | char *start_key, char *end_key, \ 76 | int key_len, char ***out_key, int **out_key_len, \ 77 | char ***out_val, int **out_val_len, int *tmp_records_cnt, \ 78 | int *tmp_out_cap); 79 | int handle_next_half(leveldb_iterator_t *iter,\ 80 | char *start_key, char *end_key, \ 81 | char ***out_key, int **out_key_len, \ 82 | char ***out_val, int **out_val_len, int *tmp_records_cnt, \ 83 | int *tmp_out_cap); 84 | int add_kv(char ***out_key, int **out_key_len, char ***out_val,\ 85 | int **out_val_len, int *tmp_records_cnt, int *tmp_out_cap, \ 86 | char *ret_key, char *ret_val, int key_len, int val_len); 87 | #endif 88 | -------------------------------------------------------------------------------- /BurstFS_Meta/src/data_store.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * 17 | * Copyright (c) 2014, Los Alamos National Laboratory 18 | * All rights reserved. 19 | * 20 | */ 21 | 22 | #include 23 | #include 24 | #include "mdhim_options.h" 25 | #include "data_store.h" 26 | #ifdef LEVELDB_SUPPORT 27 | #include "ds_leveldb.h" 28 | #endif 29 | #ifdef ROCKSDB_SUPPORT 30 | #include "ds_leveldb.h" 31 | #endif 32 | #ifdef SOPHIADB_SUPPORT 33 | #include "ds_sophia.h" 34 | #endif 35 | #ifdef MYSQLDB_SUPPORT 36 | #include "ds_mysql.h" 37 | #endif 38 | 39 | 40 | /** 41 | * mdhim_db_init 42 | * Initializes mdhim_store_t structure based on type 43 | * 44 | * @param type in Database store type to use (i.e., LEVELDB, etc) 45 | * @return mdhim_store_t The mdhim storage abstraction struct 46 | */ 47 | struct mdhim_store_t *mdhim_db_init(int type) { 48 | struct mdhim_store_t *store; 49 | 50 | //Initialize the store structure 51 | store = malloc(sizeof(struct mdhim_store_t)); 52 | store->type = type; 53 | store->db_handle = NULL; 54 | store->db_stats = NULL; 55 | store->mdhim_store_stats = NULL; 56 | store->mdhim_store_stats_lock = malloc(sizeof(pthread_rwlock_t)); 57 | if (pthread_rwlock_init(store->mdhim_store_stats_lock, NULL) != 0) { 58 | free(store->mdhim_store_stats_lock); 59 | return NULL; 60 | } 61 | 62 | switch(type) { 63 | 64 | #ifdef LEVELDB_SUPPORT 65 | case LEVELDB: 66 | store->open = mdhim_leveldb_open; 67 | store->put = mdhim_leveldb_put; 68 | store->batch_put = mdhim_leveldb_batch_put; 69 | store->get = mdhim_leveldb_get; 70 | store->get_next = mdhim_leveldb_get_next; 71 | store->get_prev = mdhim_leveldb_get_prev; 72 | store->del = mdhim_leveldb_del; 73 | store->commit = mdhim_leveldb_commit; 74 | store->close = mdhim_leveldb_close; 75 | break; 76 | 77 | #endif 78 | 79 | #ifdef ROCKSDB_SUPPORT 80 | case ROCKSDB: 81 | store->open = mdhim_leveldb_open; 82 | store->put = mdhim_leveldb_put; 83 | store->batch_put = mdhim_leveldb_batch_put; 84 | store->get = mdhim_leveldb_get; 85 | store->get_next = mdhim_leveldb_get_next; 86 | store->get_prev = mdhim_leveldb_get_prev; 87 | store->del = mdhim_leveldb_del; 88 | store->commit = mdhim_leveldb_commit; 89 | store->close = mdhim_leveldb_close; 90 | break; 91 | #endif 92 | 93 | #ifdef MYSQLDB_SUPPORT 94 | case MYSQLDB: 95 | store->open = mdhim_mysql_open; 96 | store->put = mdhim_mysql_put; 97 | store->batch_put = mdhim_mysql_batch_put; 98 | store->get = mdhim_mysql_get; 99 | store->get_next = mdhim_mysql_get_next; 100 | store->get_prev = mdhim_mysql_get_prev; 101 | store->del = mdhim_mysql_del; 102 | store->commit = mdhim_mysql_commit; 103 | store->close = mdhim_mysql_close; 104 | break; 105 | #endif 106 | 107 | 108 | default: 109 | free(store); 110 | store = NULL; 111 | break; 112 | } 113 | 114 | return store; 115 | } 116 | 117 | 118 | -------------------------------------------------------------------------------- /BurstFS_Client/src/cruise-stack.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * Copyright (c) 2013, Lawrence Livermore National Security, LLC. 17 | * Produced at the Lawrence Livermore National Laboratory. 18 | * code Written by 19 | * Raghunath Rajachandrasekar 20 | * Kathryn Mohror 21 | * Adam Moody 22 | * All rights reserved. 23 | * This file is part of CRUISE. 24 | * For details, see https://github.com/hpc/cruise 25 | * Please also read this file LICENSE.CRUISE 26 | */ 27 | 28 | /* implements a fixed-size stack which stores integer values in range 29 | * of 0 to size-1, entire structure stored in an int array of size+2 30 | * int size 31 | * int last 32 | * int entries[size] 33 | * last records index within entries that points to item one past 34 | * the item at the top of the stack */ 35 | 36 | #include "cruise-stack.h" 37 | 38 | /* returns number of bytes needed to represent stack data structure */ 39 | size_t cruise_stack_bytes(int size) 40 | { 41 | size_t bytes = sizeof(cruise_stack) + size * sizeof(int); 42 | return bytes; 43 | } 44 | 45 | /* intializes stack to record all entries as being free */ 46 | void cruise_stack_init(void* start, int size) 47 | { 48 | cruise_stack* stack = (cruise_stack*) start; 49 | stack->size = size; 50 | stack->last = size; 51 | 52 | int i; 53 | int* entries = (int*) ((char*)start + sizeof(cruise_stack)); 54 | for (i = 0; i < size; i++) { 55 | /* flip the order so low numbers are at the top 56 | * to make debugging easier */ 57 | entries[i] = size-1 - i; 58 | } 59 | } 60 | 61 | /* pops one entry from stack and returns its value */ 62 | int cruise_stack_pop(void* start) 63 | { 64 | cruise_stack* stack = (cruise_stack*) start; 65 | 66 | /* check that the stack isn't empty */ 67 | if (stack->last > 0) { 68 | /* decrement the last pointer by one */ 69 | stack->last--; 70 | 71 | /* return the index into entries array, 72 | * this now points to item at top of stack */ 73 | int idx = stack->last; 74 | 75 | /* return the value of this item */ 76 | int* entries = (int*) ((char*)start + sizeof(cruise_stack)); 77 | int value = entries[idx]; 78 | return value; 79 | } else { 80 | /* out of space */ 81 | return -1; 82 | } 83 | } 84 | 85 | /* pushes item onto free stack */ 86 | void cruise_stack_push(void* start, int value) 87 | { 88 | cruise_stack* stack = (cruise_stack*) start; 89 | 90 | /* check that we have space to push item onto stack */ 91 | if (stack->last < stack->size) { 92 | /* get index of first free slot in entries array */ 93 | int idx = stack->last; 94 | 95 | /* place item on stack */ 96 | int* entries = (int*) ((char*)start + sizeof(cruise_stack)); 97 | entries[idx] = value; 98 | 99 | /* increment last pointer to point to next item */ 100 | stack->last++; 101 | } else { 102 | /* freed one too many */ 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /BurstFS_Meta/src/partitioner.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * 17 | * Copyright (c) 2014, Los Alamos National Laboratory 18 | * All rights reserved. 19 | * 20 | */ 21 | 22 | #ifndef __HASH_H 23 | #define __HASH_H 24 | 25 | #include "mdhim.h" 26 | #include "uthash.h" 27 | #include "indexes.h" 28 | 29 | /* Used to determine if a rank is a range server 30 | Works like this: 31 | if myrank % RANGE_SERVER_FACTOR == 0, then I'm a range server 32 | if all the keys haven't been covered yet 33 | 34 | if the number of ranks is less than the RANGE_SERVER_FACTOR, 35 | then the last rank will be the range server 36 | */ 37 | 38 | #ifdef __cplusplus 39 | extern "C" 40 | { 41 | #endif 42 | //#define RANGE_SERVER_FACTOR 4 // NOW a global variable in partitioner.h 43 | #define MDHIM_MAX_SLICES 2147483647 44 | //32 bit unsigned integer 45 | #define MDHIM_INT_KEY 1 46 | #define MDHIM_LONG_INT_KEY 2 47 | #define MDHIM_FLOAT_KEY 3 48 | #define MDHIM_DOUBLE_KEY 4 49 | #define MDHIM_STRING_KEY 5 50 | //An arbitrary sized key 51 | #define MDHIM_BYTE_KEY 6 52 | #define MDHIM_BURSTFS_KEY 7 53 | 54 | //Maximum length of a key 55 | #define MAX_KEY_LEN 1048576 56 | 57 | /* The exponent used for the algorithm that determines the range server 58 | 59 | This exponent, should cover the number of characters in our alphabet 60 | if 2 is raised to that power. If the exponent is 6, then, 64 characters are covered 61 | */ 62 | #define MDHIM_ALPHABET_EXPONENT 6 63 | 64 | //Used for hashing strings to the appropriate range server 65 | struct mdhim_char { 66 | int id; /* we'll use this field as the key */ 67 | int pos; 68 | UT_hash_handle hh; /* makes this structure hashable */ 69 | }; 70 | 71 | typedef struct rangesrv_list rangesrv_list; 72 | struct rangesrv_list { 73 | rangesrv_info *ri; 74 | rangesrv_list *next; 75 | }; 76 | 77 | void partitioner_init(); 78 | void partitioner_release(); 79 | rangesrv_list *get_range_servers(struct mdhim_t *md, struct index_t *index, 80 | void *key, int key_len); 81 | rangesrv_info *get_range_server_by_slice(struct mdhim_t *md, 82 | struct index_t *index, int slice); 83 | void build_alphabet(); 84 | int verify_key(struct index_t *index, void *key, int key_len, int key_type); 85 | long double get_str_num(void *key, uint32_t key_len); 86 | //long double get_byte_num(void *key, uint32_t key_len); 87 | uint64_t get_byte_num(void *key, uint32_t key_len); 88 | int get_slice_num(struct mdhim_t *md, struct index_t *index, void *key, int key_len); 89 | int is_float_key(int type); 90 | unsigned long * get_meta_pair(void *key, uint32_t key_len); 91 | rangesrv_list *get_range_servers_from_stats(struct mdhim_t *md, struct index_t *index, 92 | void *key, int key_len, int op); 93 | rangesrv_list *get_range_servers_from_range(struct mdhim_t *md, struct index_t *index, 94 | void *start_key, void *end_key, int key_len); 95 | 96 | #ifdef __cplusplus 97 | } 98 | #endif 99 | #endif 100 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/put-get_2secondary.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "mpi.h" 4 | #include "mdhim.h" 5 | #include "mdhim_options.h" 6 | 7 | #define SLICE_SIZE 100 8 | #define SECONDARY_SLICE_SIZE 5 9 | 10 | int main(int argc, char **argv) { 11 | int ret; 12 | int provided = 0; 13 | struct mdhim_t *md; 14 | uint32_t key, secondary_key; 15 | int value; 16 | struct mdhim_brm_t *brm; 17 | struct mdhim_bgetrm_t *bgrm; 18 | mdhim_options_t *db_opts; 19 | struct index_t *secondary_local_index; 20 | struct secondary_info *secondary_info; 21 | MPI_Comm comm; 22 | 23 | ret = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); 24 | if (ret != MPI_SUCCESS) { 25 | printf("Error initializing MPI with threads\n"); 26 | exit(1); 27 | } 28 | 29 | if (provided != MPI_THREAD_MULTIPLE) { 30 | printf("Not able to enable MPI_THREAD_MULTIPLE mode\n"); 31 | exit(1); 32 | } 33 | 34 | db_opts = mdhim_options_init(); 35 | mdhim_options_set_db_path(db_opts, "./"); 36 | mdhim_options_set_db_name(db_opts, "mdhimTstDB"); 37 | mdhim_options_set_db_type(db_opts, LEVELDB); 38 | mdhim_options_set_key_type(db_opts, MDHIM_INT_KEY); //Key_type = 1 (int) 39 | mdhim_options_set_debug_level(db_opts, MLOG_CRIT); 40 | 41 | comm = MPI_COMM_WORLD; 42 | md = mdhimInit(&comm, db_opts); 43 | if (!md) { 44 | printf("Error initializing MDHIM\n"); 45 | exit(1); 46 | } 47 | 48 | //Put the primary keys and values 49 | key = 100 * (md->mdhim_rank + 1); 50 | value = 500 * (md->mdhim_rank + 1); 51 | 52 | //Create a secondary index on only one range server 53 | secondary_local_index = create_local_index(md, LEVELDB, 54 | MDHIM_INT_KEY); 55 | secondary_info = mdhimCreateSecondaryInfo(NULL, NULL, 0, 56 | secondary_local_index, &secondary_key, 57 | sizeof(secondary_key)); 58 | secondary_info2 = mdhimCreateSecondaryInfo(NULL, NULL, 0, 59 | secondary_local_index, &secondary_key2, 60 | sizeof(secondary_key2)); 61 | secondary_key = md->mdhim_rank + 1; 62 | brm = mdhimPut(md, 63 | &key, sizeof(key), 64 | &value, sizeof(value), secondary_info); 65 | if (!brm || brm->error) { 66 | printf("Error inserting key/value into MDHIM\n"); 67 | } else { 68 | printf("Successfully inserted key/value into MDHIM\n"); 69 | } 70 | 71 | //Release the received message 72 | mdhim_full_release_msg(brm); 73 | 74 | //Commit the database 75 | ret = mdhimCommit(md, md->primary_index); 76 | if (ret != MDHIM_SUCCESS) { 77 | printf("Error committing MDHIM database\n"); 78 | } else { 79 | printf("Committed MDHIM database\n"); 80 | } 81 | 82 | //Get the stats for the secondary index so the client figures out who to query 83 | ret = mdhimStatFlush(md, secondary_local_index); 84 | if (ret != MDHIM_SUCCESS) { 85 | printf("Error getting stats\n"); 86 | } else { 87 | printf("Got stats\n"); 88 | } 89 | 90 | //Get the primary key values from the secondary local key 91 | value = 0; 92 | bgrm = mdhimGet(md, secondary_local_index, &secondary_key, sizeof(secondary_key), 93 | MDHIM_GET_PRIMARY_EQ); 94 | if (!bgrm || bgrm->error) { 95 | printf("Error getting value for key: %d from MDHIM\n", key); 96 | } else if (bgrm->value_lens[0]) { 97 | printf("Successfully got value: %d from MDHIM\n", *((int *) bgrm->values[0])); 98 | } 99 | 100 | mdhim_full_release_msg(bgrm); 101 | ret = mdhimClose(md); 102 | mdhim_options_destroy(db_opts); 103 | if (ret != MDHIM_SUCCESS) { 104 | printf("Error closing MDHIM\n"); 105 | } 106 | 107 | MPI_Barrier(MPI_COMM_WORLD); 108 | MPI_Finalize(); 109 | 110 | return 0; 111 | } 112 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/put-getn.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "mpi.h" 5 | #include "mdhim.h" 6 | 7 | int main(int argc, char **argv) { 8 | int ret; 9 | int provided = 0; 10 | struct mdhim_t *md; 11 | int key; 12 | int value; 13 | struct mdhim_brm_t *brm; 14 | struct mdhim_bgetrm_t *bgrm; 15 | int i; 16 | int keys_per_rank = 100; 17 | char *db_path = "./"; 18 | char *db_name = "mdhimTstDB"; 19 | int dbug = MLOG_CRIT; 20 | mdhim_options_t *db_opts; // Local variable for db create options to be passed 21 | int db_type = LEVELDB; //(data_store.h) 22 | struct timeval start_tv, end_tv; 23 | unsigned totaltime; 24 | MPI_Comm comm; 25 | 26 | // Create options for DB initialization 27 | db_opts = mdhim_options_init(); 28 | mdhim_options_set_db_path(db_opts, db_path); 29 | mdhim_options_set_db_name(db_opts, db_name); 30 | mdhim_options_set_db_type(db_opts, db_type); 31 | mdhim_options_set_key_type(db_opts, MDHIM_INT_KEY); 32 | mdhim_options_set_debug_level(db_opts, dbug); 33 | gettimeofday(&start_tv, NULL); 34 | ret = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); 35 | if (ret != MPI_SUCCESS) { 36 | printf("Error initializing MPI with threads\n"); 37 | exit(1); 38 | } 39 | 40 | if (provided != MPI_THREAD_MULTIPLE) { 41 | printf("Not able to enable MPI_THREAD_MULTIPLE mode\n"); 42 | exit(1); 43 | } 44 | 45 | comm = MPI_COMM_WORLD; 46 | md = mdhimInit(&comm, db_opts); 47 | if (!md) { 48 | printf("Error initializing MDHIM\n"); 49 | exit(1); 50 | } 51 | 52 | //Put the keys and values 53 | for (i = 0; i < keys_per_rank; i++) { 54 | key = keys_per_rank * md->mdhim_rank + i; 55 | value = md->mdhim_rank + i; 56 | brm = mdhimPut(md, &key, sizeof(key), 57 | &value, sizeof(value), 58 | NULL, NULL); 59 | if (!brm || brm->error) { 60 | printf("Error inserting key/value into MDHIM\n"); 61 | } else { 62 | printf("Rank: %d put key: %d with value: %d\n", md->mdhim_rank, key, value); 63 | } 64 | 65 | mdhim_full_release_msg(brm); 66 | } 67 | 68 | //Commit the database 69 | ret = mdhimCommit(md, md->primary_index); 70 | if (ret != MDHIM_SUCCESS) { 71 | printf("Error committing MDHIM database\n"); 72 | } else { 73 | printf("Committed MDHIM database\n"); 74 | } 75 | 76 | //Get the stats 77 | ret = mdhimStatFlush(md, md->primary_index); 78 | if (ret != MDHIM_SUCCESS) { 79 | printf("Error getting stats\n"); 80 | } else { 81 | printf("Got stats\n"); 82 | } 83 | 84 | //Get the values using get_next 85 | for (i = 0; i < keys_per_rank; i++) { 86 | value = 0; 87 | key = keys_per_rank * md->mdhim_rank + i - 1; 88 | bgrm = mdhimBGetOp(md, md->primary_index, 89 | &key, sizeof(int), 1, MDHIM_GET_NEXT); 90 | if (!bgrm || bgrm->error) { 91 | printf("Rank: %d, Error getting next key/value given key: %d from MDHIM\n", 92 | md->mdhim_rank, key); 93 | } else if (bgrm->keys[0] && bgrm->values[0]) { 94 | printf("Rank: %d successfully got key: %d with value: %d from MDHIM\n", 95 | md->mdhim_rank, 96 | *((int *) bgrm->keys[0]), 97 | *((int *) bgrm->values[0])); 98 | } 99 | 100 | mdhim_full_release_msg(bgrm); 101 | } 102 | 103 | ret = mdhimClose(md); 104 | mdhim_options_destroy(db_opts); 105 | if (ret != MDHIM_SUCCESS) { 106 | printf("Error closing MDHIM\n"); 107 | } 108 | 109 | gettimeofday(&end_tv, NULL); 110 | totaltime = end_tv.tv_sec - start_tv.tv_sec; 111 | MPI_Barrier(MPI_COMM_WORLD); 112 | MPI_Finalize(); 113 | printf("Took %u seconds to insert and retrieve %d keys/values\n", totaltime, 114 | keys_per_rank); 115 | 116 | return 0; 117 | } 118 | -------------------------------------------------------------------------------- /BurstFS_Server/config.h.in: -------------------------------------------------------------------------------- 1 | /* config.h.in. Generated from configure.ac by autoheader. */ 2 | 3 | /* Define to 1 if you have the header file. */ 4 | #undef HAVE_FCNTL_H 5 | 6 | /* Define to 1 if you have the `ftruncate' function. */ 7 | #undef HAVE_FTRUNCATE 8 | 9 | /* Define to 1 if you have the `getpagesize' function. */ 10 | #undef HAVE_GETPAGESIZE 11 | 12 | /* Define to 1 if you have the `gettimeofday' function. */ 13 | #undef HAVE_GETTIMEOFDAY 14 | 15 | /* Define to 1 if you have the header file. */ 16 | #undef HAVE_INTTYPES_H 17 | 18 | /* Define to 1 if you have the `m' library (-lm). */ 19 | #undef HAVE_LIBM 20 | 21 | /* Define to 1 if you have the `mdhim' library (-lmdhim). */ 22 | #undef HAVE_LIBMDHIM 23 | 24 | /* Define to 1 if you have the `pthread' library (-lpthread). */ 25 | #undef HAVE_LIBPTHREAD 26 | 27 | /* Define to 1 if you have the `rt' library (-lrt). */ 28 | #undef HAVE_LIBRT 29 | 30 | /* Define to 1 if you have the `stdc++' library (-lstdc++). */ 31 | #undef HAVE_LIBSTDC__ 32 | 33 | /* Define to 1 if you have the header file. */ 34 | #undef HAVE_LIMITS_H 35 | 36 | /* Define to 1 if your system has a GNU libc compatible `malloc' function, and 37 | to 0 otherwise. */ 38 | #undef HAVE_MALLOC 39 | 40 | /* Define to 1 if you have the header file. */ 41 | #undef HAVE_MEMORY_H 42 | 43 | /* Define to 1 if you have the `memset' function. */ 44 | #undef HAVE_MEMSET 45 | 46 | /* Define to 1 if you have a working `mmap' system call. */ 47 | #undef HAVE_MMAP 48 | 49 | /* Define to 1 if your system has a GNU libc compatible `realloc' function, 50 | and to 0 otherwise. */ 51 | #undef HAVE_REALLOC 52 | 53 | /* Define to 1 if you have the `socket' function. */ 54 | #undef HAVE_SOCKET 55 | 56 | /* Define to 1 if you have the header file. */ 57 | #undef HAVE_STDINT_H 58 | 59 | /* Define to 1 if you have the header file. */ 60 | #undef HAVE_STDLIB_H 61 | 62 | /* Define to 1 if you have the header file. */ 63 | #undef HAVE_STRINGS_H 64 | 65 | /* Define to 1 if you have the header file. */ 66 | #undef HAVE_STRING_H 67 | 68 | /* Define to 1 if you have the header file. */ 69 | #undef HAVE_SYS_PARAM_H 70 | 71 | /* Define to 1 if you have the header file. */ 72 | #undef HAVE_SYS_SOCKET_H 73 | 74 | /* Define to 1 if you have the header file. */ 75 | #undef HAVE_SYS_STAT_H 76 | 77 | /* Define to 1 if you have the header file. */ 78 | #undef HAVE_SYS_TIME_H 79 | 80 | /* Define to 1 if you have the header file. */ 81 | #undef HAVE_SYS_TYPES_H 82 | 83 | /* Define to 1 if you have the header file. */ 84 | #undef HAVE_UNISTD_H 85 | 86 | /* Define to the address where bug reports for this package should be sent. */ 87 | #undef PACKAGE_BUGREPORT 88 | 89 | /* Define to the full name of this package. */ 90 | #undef PACKAGE_NAME 91 | 92 | /* Define to the full name and version of this package. */ 93 | #undef PACKAGE_STRING 94 | 95 | /* Define to the one symbol short name of this package. */ 96 | #undef PACKAGE_TARNAME 97 | 98 | /* Define to the home page for this package. */ 99 | #undef PACKAGE_URL 100 | 101 | /* Define to the version of this package. */ 102 | #undef PACKAGE_VERSION 103 | 104 | /* Define to 1 if you have the ANSI C header files. */ 105 | #undef STDC_HEADERS 106 | 107 | /* Define to rpl_malloc if the replacement function should be used. */ 108 | #undef malloc 109 | 110 | /* Define to `long int' if does not define. */ 111 | #undef off_t 112 | 113 | /* Define to rpl_realloc if the replacement function should be used. */ 114 | #undef realloc 115 | 116 | /* Define to `unsigned int' if does not define. */ 117 | #undef size_t 118 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/put-del_secondary.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "mpi.h" 4 | #include "mdhim.h" 5 | #include "mdhim_options.h" 6 | 7 | #define SLICE_SIZE 100 8 | #define SECONDARY_SLICE_SIZE 5 9 | 10 | int main(int argc, char **argv) { 11 | int ret; 12 | int provided = 0; 13 | struct mdhim_t *md; 14 | uint32_t key, **secondary_keys; 15 | int value, *secondary_key_lens; 16 | struct mdhim_brm_t *brm; 17 | struct mdhim_bgetrm_t *bgrm; 18 | mdhim_options_t *db_opts; 19 | struct index_t *secondary_index; 20 | struct secondary_info *secondary_info; 21 | MPI_Comm comm; 22 | 23 | ret = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); 24 | if (ret != MPI_SUCCESS) { 25 | printf("Error initializing MPI with threads\n"); 26 | exit(1); 27 | } 28 | 29 | if (provided != MPI_THREAD_MULTIPLE) { 30 | printf("Not able to enable MPI_THREAD_MULTIPLE mode\n"); 31 | exit(1); 32 | } 33 | 34 | db_opts = mdhim_options_init(); 35 | mdhim_options_set_db_path(db_opts, "./"); 36 | mdhim_options_set_db_name(db_opts, "mdhimTstDB"); 37 | mdhim_options_set_db_type(db_opts, LEVELDB); 38 | mdhim_options_set_key_type(db_opts, MDHIM_INT_KEY); //Key_type = 1 (int) 39 | mdhim_options_set_debug_level(db_opts, MLOG_CRIT); 40 | 41 | comm = MPI_COMM_WORLD; 42 | md = mdhimInit(&comm, db_opts); 43 | if (!md) { 44 | printf("Error initializing MDHIM\n"); 45 | exit(1); 46 | } 47 | 48 | //Put the primary keys and secondary keys 49 | key = 100 * (md->mdhim_rank + 1); 50 | value = 500 * (md->mdhim_rank + 1); 51 | secondary_keys = malloc(sizeof(uint32_t *)); 52 | secondary_keys[0] = malloc(sizeof(uint32_t)); 53 | *secondary_keys[0] = md->mdhim_rank + 1; 54 | secondary_key_lens = malloc(sizeof(int)); 55 | secondary_key_lens[0] = sizeof(uint32_t); 56 | //Create a secondary index 57 | secondary_index = create_global_index(md, 2, 58 | SECONDARY_SLICE_SIZE, LEVELDB, 59 | MDHIM_INT_KEY, NULL); 60 | secondary_info = mdhimCreateSecondaryInfo(secondary_index, 61 | (void **) secondary_keys, 62 | secondary_key_lens, 1, 63 | SECONDARY_GLOBAL_INFO); 64 | brm = mdhimPut(md, &key, sizeof(key), 65 | &value, sizeof(value), 66 | secondary_info, NULL); 67 | if (!brm || brm->error) { 68 | printf("Error inserting key/value into MDHIM\n"); 69 | } else { 70 | printf("Successfully inserted key/value into MDHIM\n"); 71 | } 72 | 73 | //Release the received message 74 | mdhim_full_release_msg(brm); 75 | 76 | //Commit the database 77 | ret = mdhimCommit(md, md->primary_index); 78 | if (ret != MDHIM_SUCCESS) { 79 | printf("Error committing MDHIM database\n"); 80 | } else { 81 | printf("Committed MDHIM database\n"); 82 | } 83 | 84 | brm = mdhimDelete(md, secondary_index, secondary_keys[0], secondary_key_lens[0]); 85 | if (!brm || brm->error) { 86 | printf("Error deleting key/value from MDHIM\n"); 87 | } else { 88 | printf("Successfully deleted key/value into MDHIM\n"); 89 | } 90 | 91 | //Get the primary key values from the secondary key - this should fail 92 | value = 0; 93 | bgrm = mdhimGet(md, secondary_index, secondary_keys[0], secondary_key_lens[0], 94 | MDHIM_GET_PRIMARY_EQ); 95 | if (!bgrm || bgrm->error) { 96 | printf("Error getting value for key: %d from MDHIM\n", key); 97 | } else if (bgrm->value_lens[0]) { 98 | printf("Successfully got value: %d from MDHIM\n", *((int *) bgrm->values[0])); 99 | } 100 | 101 | mdhim_full_release_msg(bgrm); 102 | free(secondary_keys[0]); 103 | free(secondary_keys); 104 | free(secondary_key_lens); 105 | ret = mdhimClose(md); 106 | mdhim_options_destroy(db_opts); 107 | if (ret != MDHIM_SUCCESS) { 108 | printf("Error closing MDHIM\n"); 109 | } 110 | 111 | MPI_Barrier(MPI_COMM_WORLD); 112 | MPI_Finalize(); 113 | 114 | return 0; 115 | } 116 | -------------------------------------------------------------------------------- /BurstFS_Client/tests/test_read.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #define GEN_STR_LEN 1024 15 | 16 | struct timeval read_start, read_end; 17 | double read_time = 0; 18 | 19 | typedef struct { 20 | int fid; 21 | long offset; 22 | long length; 23 | char *buf; 24 | }read_req_t; 25 | 26 | int main(int argc, char *argv[]) { 27 | 28 | static const char * opts = "b:s:t:f:p:u:"; 29 | 30 | char tmpfname[GEN_STR_LEN], fname[GEN_STR_LEN]; 31 | long blk_sz, seg_num, tran_sz, num_reqs; 32 | int pat, c, rank_num, rank, fd, to_unmount = 0; 33 | 34 | MPI_Init(&argc, &argv); 35 | MPI_Comm_size(MPI_COMM_WORLD, &rank_num); 36 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 37 | 38 | while((c = getopt(argc, argv, opts)) != -1){ 39 | switch (c) { 40 | case 'b': /*size of block*/ 41 | blk_sz = atol(optarg); break; 42 | case 's': /*number of blocks each process writes*/ 43 | seg_num = atol(optarg); break; 44 | case 't': /*size of each write*/ 45 | tran_sz = atol(optarg); break; 46 | case 'f': 47 | strcpy(fname, optarg); break; 48 | case 'p': 49 | pat = atoi(optarg); break; /* 0: N-1, 1: N-N */ 50 | case 'u': 51 | to_unmount = atoi(optarg); break; 52 | } 53 | } 54 | 55 | num_reqs = blk_sz*seg_num/tran_sz; 56 | char *read_buf = malloc(blk_sz * seg_num); /*read buffer*/ 57 | struct aiocb *aiocb_list = (struct aiocb *)malloc(num_reqs\ 58 | * sizeof(struct aiocb)); 59 | struct aiocb **cb_list = (struct aiocb **)malloc (num_reqs * \ 60 | sizeof (struct aiocb *)); /*list of read requests in lio_listio*/ 61 | 62 | burstfs_mount("/tmp", rank, rank_num,\ 63 | 1, 1); 64 | 65 | if (pat == 1) { 66 | sprintf(tmpfname, "%s%d", fname, rank); 67 | } else { 68 | sprintf(tmpfname, "%s", fname); 69 | } 70 | MPI_Barrier(MPI_COMM_WORLD); 71 | 72 | fd = open(tmpfname, O_RDONLY, S_IRUSR | S_IWUSR); 73 | if (fd < 0) { 74 | printf("open file failure\n"); 75 | fflush(stdout); 76 | return -1; 77 | } 78 | 79 | 80 | gettimeofday(&read_start, NULL); 81 | 82 | long i, j, offset, rc, cursor = 0; 83 | for (i = 0; i < seg_num; i++) { 84 | for (j = 0; j < blk_sz/tran_sz; j++) { 85 | if (pat == 0) 86 | offset = i * rank_num * blk_sz +\ 87 | rank * blk_sz + j * tran_sz; 88 | else if (pat == 1) 89 | offset = i * blk_sz + j * tran_sz; 90 | cursor += tran_sz; 91 | lseek(fd, offset, SEEK_SET); 92 | rc = read(fd, read_buf + cursor, tran_sz); 93 | if (rc < 0) { 94 | printf("read failure\n"); 95 | fflush(stdout); 96 | return -1; 97 | } 98 | } 99 | } 100 | gettimeofday(&read_end, NULL); 101 | read_time = (read_end.tv_sec - read_start.tv_sec)*1000000 \ 102 | + read_end.tv_usec - read_start.tv_usec; 103 | read_time = read_time/1000000; 104 | 105 | close(fd); 106 | MPI_Barrier(MPI_COMM_WORLD); 107 | 108 | 109 | if (to_unmount) { 110 | if (rank == 0) 111 | burstfs_unmount(); 112 | } 113 | 114 | free(read_buf); 115 | 116 | double read_bw = (double)blk_sz\ 117 | * seg_num / 1048576 / read_time; 118 | double agg_read_bw; 119 | 120 | double max_read_time, min_read_bw; 121 | MPI_Reduce(&read_bw, &agg_read_bw, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); 122 | MPI_Reduce(&read_time, &max_read_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); 123 | 124 | 125 | min_read_bw=(double)blk_sz*seg_num*rank_num/1048576/max_read_time; 126 | if (rank == 0) { 127 | printf("Aggregate Read BW is %lfMB/s,\ 128 | Min Read BW is %lf\n", \ 129 | agg_read_bw, min_read_bw); 130 | fflush(stdout); 131 | } 132 | 133 | MPI_Finalize(); 134 | 135 | } 136 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/put-del_secondary_local.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "mpi.h" 4 | #include "mdhim.h" 5 | #include "mdhim_options.h" 6 | 7 | #define SLICE_SIZE 100 8 | #define SECONDARY_SLICE_SIZE 5 9 | 10 | int main(int argc, char **argv) { 11 | int ret; 12 | int provided = 0; 13 | struct mdhim_t *md; 14 | uint32_t key, **secondary_keys; 15 | int value, *secondary_key_lens; 16 | struct mdhim_brm_t *brm; 17 | struct mdhim_bgetrm_t *bgrm; 18 | mdhim_options_t *db_opts; 19 | struct index_t *secondary_local_index; 20 | struct secondary_info *secondary_local_info; 21 | MPI_Comm comm; 22 | 23 | ret = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); 24 | if (ret != MPI_SUCCESS) { 25 | printf("Error initializing MPI with threads\n"); 26 | exit(1); 27 | } 28 | 29 | if (provided != MPI_THREAD_MULTIPLE) { 30 | printf("Not able to enable MPI_THREAD_MULTIPLE mode\n"); 31 | exit(1); 32 | } 33 | 34 | db_opts = mdhim_options_init(); 35 | mdhim_options_set_db_path(db_opts, "./"); 36 | mdhim_options_set_db_name(db_opts, "mdhimTstDB"); 37 | mdhim_options_set_db_type(db_opts, LEVELDB); 38 | mdhim_options_set_key_type(db_opts, MDHIM_INT_KEY); //Key_type = 1 (int) 39 | mdhim_options_set_debug_level(db_opts, MLOG_CRIT); 40 | 41 | comm = MPI_COMM_WORLD; 42 | md = mdhimInit(&comm, db_opts); 43 | if (!md) { 44 | printf("Error initializing MDHIM\n"); 45 | exit(1); 46 | } 47 | 48 | //Put the primary keys and secondary keys 49 | key = 100 * (md->mdhim_rank + 1); 50 | value = 500 * (md->mdhim_rank + 1); 51 | 52 | //Create a secondary index 53 | secondary_local_index = create_local_index(md, MDHIM_INT_KEY, 54 | md->primary_index->id, NULL); 55 | secondary_keys = malloc(sizeof(uint32_t *)); 56 | secondary_keys[0] = malloc(sizeof(uint32_t)); 57 | *secondary_keys[0] = md->mdhim_rank + 1; 58 | secondary_key_lens = malloc(sizeof(int)); 59 | secondary_key_lens[0] = sizeof(uint32_t); 60 | secondary_local_info = mdhimCreateSecondaryInfo(secondary_local_index, 61 | (void **) secondary_keys, 62 | secondary_key_lens, 1, 63 | SECONDARY_LOCAL_INFO); 64 | brm = mdhimPut(md, &key, sizeof(key), 65 | &value, sizeof(value), 66 | NULL, secondary_local_info); 67 | if (!brm || brm->error) { 68 | printf("Error inserting key/value into MDHIM\n"); 69 | } else { 70 | printf("Successfully inserted key/value into MDHIM\n"); 71 | } 72 | 73 | //Release the received message 74 | mdhim_full_release_msg(brm); 75 | mdhimReleaseSecondaryInfo(secondary_local_info); 76 | 77 | //Commit the database 78 | ret = mdhimCommit(md, md->primary_index); 79 | if (ret != MDHIM_SUCCESS) { 80 | printf("Error committing MDHIM database\n"); 81 | } else { 82 | printf("Committed MDHIM database\n"); 83 | } 84 | 85 | brm = mdhimDelete(md, secondary_local_index, secondary_keys[0], 86 | secondary_key_lens[0]); 87 | if (!brm || brm->error) { 88 | printf("Error deleting key/value from MDHIM\n"); 89 | } else { 90 | printf("Successfully deleted key/value into MDHIM\n"); 91 | } 92 | 93 | //Get the primary key values from the secondary key - this should fail 94 | value = 0; 95 | bgrm = mdhimGet(md, secondary_local_index, 96 | secondary_keys[0], secondary_key_lens[0], 97 | MDHIM_GET_PRIMARY_EQ); 98 | if (!bgrm || bgrm->error) { 99 | printf("Error getting value for key: %d from MDHIM\n", key); 100 | } else if (bgrm->value_lens[0]) { 101 | printf("Successfully got value: %d from MDHIM\n", *((int *) bgrm->values[0])); 102 | } 103 | 104 | mdhim_full_release_msg(bgrm); 105 | free(secondary_keys[0]); 106 | free(secondary_keys); 107 | free(secondary_key_lens); 108 | ret = mdhimClose(md); 109 | mdhim_options_destroy(db_opts); 110 | if (ret != MDHIM_SUCCESS) { 111 | printf("Error closing MDHIM\n"); 112 | } 113 | 114 | MPI_Barrier(MPI_COMM_WORLD); 115 | MPI_Finalize(); 116 | 117 | return 0; 118 | } 119 | -------------------------------------------------------------------------------- /BurstFS_Server/burstfs_service_manager.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | #ifndef BURSTFS_SERVICE_MANAGER_H 16 | #define BURSTFS_SERVICE_MANAGER_H 17 | #include 18 | #include "burstfs_global.h" 19 | #include "arraylist.h" 20 | #include 21 | 22 | typedef struct { 23 | long src_fid; 24 | long src_offset; 25 | long length; 26 | char *addr; 27 | }ack_meta_t; 28 | 29 | typedef struct { 30 | MPI_Request req; 31 | MPI_Status stat; 32 | int src_rank; 33 | int src_thrd; 34 | }ack_stat_t; 35 | 36 | typedef struct { 37 | int start_idx; 38 | int end_idx; 39 | int size; 40 | int app_id; 41 | int cli_id; 42 | int arrival_time; 43 | }read_task_t; 44 | 45 | typedef struct { 46 | read_task_t *read_tasks; 47 | int num; 48 | int cap; 49 | }task_set_t; 50 | 51 | typedef struct { 52 | send_msg_t *msg; 53 | int num; 54 | int cap; 55 | } service_msgs_t; 56 | 57 | typedef struct { 58 | struct aiocb read_cb; 59 | int index; 60 | int start_pos; 61 | int end_pos; 62 | char *mem_pos; 63 | }pended_read_t; 64 | 65 | typedef struct { 66 | arraylist_t *ack_list; 67 | int rank_id; 68 | int thrd_id; 69 | int src_cli_rank; 70 | long src_sz; 71 | int start_cursor; 72 | }rank_ack_meta_t; 73 | 74 | typedef struct { 75 | rank_ack_meta_t *ack_metas; 76 | int num; 77 | }rank_ack_task_t; 78 | 79 | int insert_ack_meta(rank_ack_task_t *rank_ack_task,\ 80 | ack_meta_t *ptr_ack, int pos, int rank_id, int thrd_id, \ 81 | int src_cli_rank); 82 | int compare_rank_thrd(int src_rank, int src_thrd, \ 83 | int cmp_rank, int cmp_thrd); 84 | void *sm_service_reads(void *ctx); 85 | int sm_decode_msg(char *recv_msg_buf); 86 | int sm_classfy_reads(service_msgs_t *service_msgs); 87 | int sm_ack_reads(service_msgs_t *service_msgs); 88 | int compare_send_msg(void *a, void *b); 89 | int sm_wait_until_digested(task_set_t *read_task_set,\ 90 | service_msgs_t *service_msgs,\ 91 | rank_ack_task_t *read_ack_task); 92 | int sm_ack_remote_delegator(rank_ack_meta_t *ptr_ack_meta); 93 | int batch_ins_to_ack_lst(rank_ack_task_t *rank_ack_task, 94 | read_task_t *read_task,\ 95 | service_msgs_t *service_msgs,\ 96 | char *mem_addr, int start_offset, int end_offset); 97 | int sm_read_send_pipe(task_set_t *read_task_set,\ 98 | service_msgs_t *service_msgs, \ 99 | rank_ack_task_t *rank_ack_task); 100 | int sm_init_socket(); 101 | int ins_to_ack_lst(rank_ack_task_t *rank_ack_task, \ 102 | char *mem_addr, service_msgs_t *service_msgs,\ 103 | int index, long src_offset, long len); 104 | int find_ack_meta(rank_ack_task_t *rank_ack_task,\ 105 | int src_rank, int src_thrd); 106 | int sm_cluster_reads(task_set_t *read_task_set,\ 107 | service_msgs_t *service_msgs); 108 | void reset_read_tasks(task_set_t *read_task_set,\ 109 | service_msgs_t *service_msgs, int index); 110 | int compare_read_task(void *a, void *b); 111 | int sm_exit(); 112 | void print_task_set(task_set_t *read_task_set,\ 113 | service_msgs_t * service_msgs); 114 | void print_service_msgs(service_msgs_t *service_msgs); 115 | void reset_read_tasks(task_set_t *read_task_set,\ 116 | service_msgs_t *service_msgs, int index); 117 | void print_task_set(task_set_t *read_task_set,\ 118 | service_msgs_t * service_msgs); 119 | void print_pended_reads(arraylist_t *pended_reads); 120 | void print_pended_sends(arraylist_t *pended_sends); 121 | void print_ack_meta(rank_ack_task_t *rank_ack_tasks); 122 | int compare_ack_stat(void *a, void *b); 123 | #endif 124 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/tester/inTestOps3.txt: -------------------------------------------------------------------------------- 1 | bput 9999 199135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 2 | bput 9999 25468001 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 3 | bput 9999 567891 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 4 | 5 | bget 9999 199135790 # AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 6 | bget 9999 25468001 # abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 7 | bget 9999 567891 # ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 8 | 9 | flush 10 | 11 | bgetop 999 25468001 1 12 | bgetop 999 25469001 2 13 | 14 | get 567891 3 15 | get 199135790 3 16 | get 25468001 3 17 | 18 | bput 9999 299135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 19 | bput 9999 25246802 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 20 | bput 9999 256782 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 21 | 22 | flush 23 | get 25468001 4 24 | get 199135790 4 25 | get 567891 4 26 | 27 | bput 9999 399135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 28 | bput 9999 35246803 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 29 | bput 9999 356783 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 30 | 31 | bput 9999 499135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 32 | bput 9999 45246804 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 33 | bput 9999 456784 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 34 | 35 | bdel 9999 299135790 36 | bdel 9999 25246802 37 | bdel 9999 256782 38 | 39 | bput 9999 599135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 40 | bput 9999 55246805 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 41 | bput 9999 556785 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 42 | 43 | bput 9999 991357900 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 44 | bput 9999 55468000 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 45 | bput 9999 167890 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 46 | 47 | flush 48 | get 25468001 4 49 | get 199135790 4 50 | get 567891 4 51 | get 567891 3 52 | get 199135790 3 53 | get 25468001 3 54 | 55 | bdel 9999 499135790 56 | bdel 9999 45246804 57 | bdel 9999 456784 58 | 59 | bget 9999 199135790 # AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 60 | bget 9999 25468001 # abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 61 | bget 9999 567891 # ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 62 | 63 | bdel 9999 399135790 64 | bdel 9999 35246803 65 | bdel 9999 356783 66 | 67 | bdel 9999 599135790 68 | bdel 9999 55246805 69 | bdel 9999 556785 70 | 71 | bdel 9999 991357900 72 | bdel 9999 55468000 73 | bdel 9999 167890 74 | 75 | flush 76 | get 25468001 4 77 | get 199135790 4 78 | get 567891 4 79 | get 567891 3 80 | get 199135790 3 81 | get 25468001 3 82 | 83 | bdel 9999 199135790 84 | bdel 9999 25468001 85 | bdel 9999 567891 86 | 87 | flush 88 | get 11468001 4 89 | get 119135790 4 90 | get 117891 4 91 | get 117891 3 92 | get 119135790 3 93 | get 11468001 3 94 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/put-get_secondary_local.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "mpi.h" 4 | #include "mdhim.h" 5 | #include "mdhim_options.h" 6 | 7 | #define SLICE_SIZE 100 8 | #define SECONDARY_SLICE_SIZE 5 9 | 10 | int main(int argc, char **argv) { 11 | int ret; 12 | int provided = 0; 13 | struct mdhim_t *md; 14 | uint32_t key, **secondary_keys; 15 | int value, *secondary_key_lens; 16 | struct mdhim_brm_t *brm; 17 | struct mdhim_bgetrm_t *bgrm; 18 | mdhim_options_t *db_opts; 19 | struct index_t *secondary_local_index; 20 | struct secondary_info *secondary_info; 21 | MPI_Comm comm; 22 | 23 | ret = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); 24 | if (ret != MPI_SUCCESS) { 25 | printf("Error initializing MPI with threads\n"); 26 | exit(1); 27 | } 28 | 29 | if (provided != MPI_THREAD_MULTIPLE) { 30 | printf("Not able to enable MPI_THREAD_MULTIPLE mode\n"); 31 | exit(1); 32 | } 33 | 34 | db_opts = mdhim_options_init(); 35 | mdhim_options_set_db_path(db_opts, "./"); 36 | mdhim_options_set_db_name(db_opts, "mdhimTstDB"); 37 | mdhim_options_set_db_type(db_opts, LEVELDB); 38 | mdhim_options_set_key_type(db_opts, MDHIM_INT_KEY); //Key_type = 1 (int) 39 | mdhim_options_set_debug_level(db_opts, MLOG_CRIT); 40 | 41 | comm = MPI_COMM_WORLD; 42 | md = mdhimInit(&comm, db_opts); 43 | if (!md) { 44 | printf("Error initializing MDHIM\n"); 45 | exit(1); 46 | } 47 | 48 | //Put the primary keys and values 49 | key = 100 * (md->mdhim_rank + 1); 50 | value = 500 * (md->mdhim_rank + 1); 51 | 52 | //Create a secondary index on only one range server 53 | secondary_local_index = create_local_index(md, LEVELDB, 54 | MDHIM_INT_KEY, NULL); 55 | secondary_keys = malloc(sizeof(uint32_t *)); 56 | secondary_keys[0] = malloc(sizeof(uint32_t)); 57 | *secondary_keys[0] = md->mdhim_rank + 1; 58 | secondary_keys[1] = malloc(sizeof(uint32_t)); 59 | *secondary_keys[1] = md->mdhim_rank + 2; 60 | secondary_key_lens = malloc(sizeof(int) * 2); 61 | secondary_key_lens[0] = sizeof(uint32_t); 62 | secondary_key_lens[1] = sizeof(uint32_t); 63 | secondary_info = mdhimCreateSecondaryInfo(secondary_local_index, 64 | (void **) secondary_keys, 65 | secondary_key_lens, 2, 66 | SECONDARY_LOCAL_INFO); 67 | brm = mdhimPut(md, &key, sizeof(key), 68 | &value, sizeof(value), 69 | NULL, secondary_info); 70 | if (!brm || brm->error) { 71 | printf("Error inserting key/value into MDHIM\n"); 72 | } else { 73 | printf("Successfully inserted key/value into MDHIM\n"); 74 | } 75 | 76 | //Release the received message 77 | mdhim_full_release_msg(brm); 78 | 79 | //Commit the database 80 | ret = mdhimCommit(md, md->primary_index); 81 | if (ret != MDHIM_SUCCESS) { 82 | printf("Error committing MDHIM database\n"); 83 | } else { 84 | printf("Committed MDHIM database\n"); 85 | } 86 | 87 | //Get the stats for the secondary index so the client figures out who to query 88 | ret = mdhimStatFlush(md, secondary_local_index); 89 | if (ret != MDHIM_SUCCESS) { 90 | printf("Error getting stats\n"); 91 | } else { 92 | printf("Got stats\n"); 93 | } 94 | 95 | //Get the primary key values from the secondary local key 96 | value = 0; 97 | bgrm = mdhimGet(md, secondary_local_index, 98 | &secondary_keys[0][0], 99 | secondary_key_lens[0], 100 | MDHIM_GET_PRIMARY_EQ); 101 | if (!bgrm || bgrm->error) { 102 | printf("Error getting value for key: %d from MDHIM\n", key); 103 | } else if (bgrm->value_lens[0]) { 104 | printf("Successfully got value: %d from MDHIM\n", *((int *) bgrm->values[0])); 105 | } 106 | 107 | mdhim_full_release_msg(bgrm); 108 | ret = mdhimClose(md); 109 | mdhim_options_destroy(db_opts); 110 | mdhimReleaseSecondaryInfo(secondary_info); 111 | if (ret != MDHIM_SUCCESS) { 112 | printf("Error closing MDHIM\n"); 113 | } 114 | 115 | free(secondary_keys[0]); 116 | free(secondary_keys[1]); 117 | free(secondary_keys); 118 | free(secondary_key_lens); 119 | MPI_Barrier(MPI_COMM_WORLD); 120 | MPI_Finalize(); 121 | 122 | return 0; 123 | } 124 | -------------------------------------------------------------------------------- /BurstFS_Meta/src/mdhim_options.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * 17 | * Copyright (c) 2014, Los Alamos National Laboratory 18 | * All rights reserved. 19 | * 20 | */ 21 | 22 | #ifndef __OPTIONS_H 23 | #define __OPTIONS_H 24 | 25 | #include 26 | 27 | #ifdef __cplusplus 28 | extern "C" 29 | { 30 | #endif 31 | /* Append option */ 32 | #define MDHIM_DB_OVERWRITE 0 33 | #define MDHIM_DB_APPEND 1 34 | 35 | // Options for the database (used when opening a MDHIM dataStore) 36 | typedef struct mdhim_options_t { 37 | // ------------------- 38 | //Directory location of DBs 39 | char *db_path; 40 | 41 | //Multiple paths of DBs 42 | char **db_paths; 43 | //Number of paths in db_paths 44 | int num_paths; 45 | 46 | char *manifest_path; 47 | 48 | //Name of each DB (will be modified by adding "_" to create multiple 49 | // unique DB for each rank server. 50 | char *db_name; 51 | 52 | //Different types of dataStores 53 | //LEVELDB=1 (from data_store.h) 54 | int db_type; 55 | 56 | //Primary key type 57 | //MDHIM_INT_KEY, MDHIM_LONG_INT_KEY, MDHIM_FLOAT_KEY, MDHIM_DOUBLE_KEY 58 | //MDHIM_STRING_KEY, MDHIM_BYTE_KEY 59 | //(from partitioner.h) 60 | int db_key_type; 61 | 62 | //Force the creation of a new DB (deleting any previous versions if present) 63 | int db_create_new; 64 | 65 | //Whether to append a value to an existing key or overwrite the value 66 | //MDHIM_DB_APPEND to append or MDHIM_DB_OVERWRITE (default) 67 | int db_value_append; 68 | 69 | //DEBUG level 70 | int debug_level; 71 | 72 | //Used to determine the number of range servers which is based in 73 | //if myrank % rserver_factor == 0, then myrank is a server. 74 | // This option is used to set range_server_factor previously a defined var. 75 | int rserver_factor; 76 | 77 | //Maximum size of a slice. A ranger server may server several slices. 78 | uint64_t max_recs_per_slice; 79 | 80 | //Number of worker threads per range server 81 | int num_wthreads; 82 | 83 | //Login Credentials 84 | char *db_host; 85 | char *dbs_host; 86 | char *db_user; 87 | char *db_upswd; 88 | char *dbs_user; 89 | char *dbs_upswd; 90 | 91 | 92 | } mdhim_options_t; 93 | 94 | struct mdhim_options_t* mdhim_options_init(); 95 | void mdhim_options_set_db_path(struct mdhim_options_t* opts, char *path); 96 | void mdhim_options_set_db_paths(struct mdhim_options_t* opts, char **paths, int num_paths); 97 | void mdhim_options_set_db_name(struct mdhim_options_t* opts, char *name); 98 | void mdhim_options_set_db_type(struct mdhim_options_t* opts, int type); 99 | void mdhim_options_set_key_type(struct mdhim_options_t* opts, int key_type); 100 | void mdhim_options_set_create_new_db(struct mdhim_options_t* opts, int create_new); 101 | void mdhim_options_set_login_c(struct mdhim_options_t* opts, char* db_hl, char *db_ln, char *db_pw, char *dbs_hl, char *dbs_ln, char *dbs_pw); 102 | void mdhim_options_set_debug_level(struct mdhim_options_t* opts, int dbug); 103 | void mdhim_options_set_value_append(struct mdhim_options_t* opts, int append); 104 | void mdhim_options_set_server_factor(struct mdhim_options_t* opts, int server_factor); 105 | void mdhim_options_set_max_recs_per_slice(struct mdhim_options_t* opts, uint64_t max_recs_per_slice); 106 | void mdhim_options_set_num_worker_threads(struct mdhim_options_t* opts, int num_wthreads); 107 | void set_manifest_path(mdhim_options_t* opts, char *path); 108 | void mdhim_options_destroy(struct mdhim_options_t *opts); 109 | #ifdef __cplusplus 110 | } 111 | #endif 112 | #endif 113 | -------------------------------------------------------------------------------- /BurstFS_Client/maint/check_fns/scrmfs_check_fns: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | use strict; 3 | 4 | # commands to build fakechroot list 5 | # git clone https://github.com/fakechroot/fakechroot.git fakechroot.git 6 | # cd fakechroot.git/src 7 | # ls *.c > fakechroot_list.txt 8 | # 9 | # hand-wrote gnulibc list from 10 | # http://www.gnu.org/software/libc/manual/html_node/I_002fO-Overview.html#I_002fO-Overview 11 | # 12 | # hand-wrote cstdio list from 13 | # http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1124.pdf 14 | # 15 | # hand-wrote cruise unsupported list (wrapped calls that aren't supported) 16 | # 17 | # commands to build cruise list 18 | # grep CRUISE_WRAP ../runtime/lib/*.c > cruise_list.txt 19 | 20 | 21 | my $fakechroot_list = "fakechroot_list.txt"; 22 | my $gnulibc_list = "gnulibc_list.txt"; 23 | my $cstdio_list = "cstdio_list.txt"; 24 | my $posix_list = "posix_list.txt"; 25 | my $unsupported_list = "cruise_unsupported_list.txt"; 26 | my $cruise_list = "cruise_list.txt"; 27 | 28 | if (@ARGV != 1) { 29 | print "Usage: cruise_check_fns \n"; 30 | exit 1; 31 | } 32 | my $exe = shift @ARGV; 33 | 34 | my %any_libc = (); 35 | my %unsupported = (); 36 | my %cruise = (); 37 | 38 | open(IN, $fakechroot_list) || die "failed to open $fakechroot_list\n"; 39 | while (my $line = ) { 40 | # drop trailing newline 41 | chomp $line; 42 | 43 | # chop white space 44 | $line =~ s/\s*//g; 45 | 46 | # chop trailing .c 47 | $line =~ s/\.c$//; 48 | 49 | $any_libc{$line} = 1; 50 | } 51 | close(IN); 52 | 53 | open(IN, $gnulibc_list) || die "failed to open $gnulibc_list\n"; 54 | while (my $line = ) { 55 | # drop trailing newline 56 | chomp $line; 57 | 58 | # chop white space 59 | $line =~ s/\s*//g; 60 | 61 | # skip if starts with # or is empty 62 | if ($line =~ /^#/ or not $line) { 63 | next; 64 | } 65 | 66 | $any_libc{$line} = 1; 67 | } 68 | close(IN); 69 | 70 | open(IN, $cstdio_list) || die "failed to open $cstdio_list\n"; 71 | while (my $line = ) { 72 | # drop trailing newline 73 | chomp $line; 74 | 75 | # skip if starts with # or is empty 76 | if ($line =~ /^#/ or not $line) { 77 | next; 78 | } 79 | 80 | # (space) leading integer (space) function name 81 | my $fn = undef; 82 | if ($line =~ /^\s*\d+\s*(.*?)\s*$/) { 83 | $fn = $1; 84 | } else { 85 | next; 86 | } 87 | 88 | # chop white space 89 | $fn =~ s/\s*//g; 90 | #print "cstdio: $fn\n"; 91 | 92 | $any_libc{$fn} = 1; 93 | } 94 | close(IN); 95 | 96 | open(IN, $unsupported_list) || die "failed to open $unsupported_list\n"; 97 | while (my $line = ) { 98 | # drop trailing newline 99 | chomp $line; 100 | 101 | # chop white space 102 | $line =~ s/\s*//g; 103 | 104 | # skip if starts with # or is empty 105 | if ($line =~ /^#/ or not $line) { 106 | next; 107 | } 108 | #print "unsupported: $line\n"; 109 | 110 | $unsupported{$line} = 1; 111 | } 112 | close(IN); 113 | 114 | open(IN, $cruise_list) || die "failed to open $cruise_list\n"; 115 | while (my $line = ) { 116 | # drop trailing newline 117 | chomp $line; 118 | 119 | # pick off item in parens in CRUISE_DELC(fn_name) 120 | my $fn = undef; 121 | if ($line =~ /CRUISE_WRAP\((.*?)\)/) { 122 | $fn = $1; 123 | } else { 124 | next; 125 | } 126 | 127 | # chop white space 128 | $fn =~ s/\s*//g; 129 | #print "wrapped: $fn\n"; 130 | 131 | $cruise{$fn} = 1; 132 | } 133 | close(IN); 134 | 135 | # get list of functions called by executable 136 | my $fns_str = `nm $exe`; 137 | my @fns_list = split("\n", $fns_str); 138 | foreach my $line (@fns_list) { 139 | chomp $line; 140 | # print "$line, "; 141 | 142 | my ($hex, $code, $fn) = split(/\s+/, $line); 143 | # print "$fn, "; 144 | 145 | $fn =~ s/@.*$//; 146 | # print "$fn\n"; 147 | 148 | my $used = 0; 149 | my $supported = 0; 150 | if (defined $any_libc{$fn}) { 151 | $used = 1; 152 | if (defined $cruise{$fn} and not $unsupported{$fn}) { 153 | $supported = 1; 154 | } 155 | } 156 | 157 | if ($used) { 158 | print "$fn"; 159 | if (not $supported) { 160 | print ", not supported"; 161 | } 162 | print "\n"; 163 | } 164 | } 165 | 166 | exit 0; 167 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/tester/inTestOps5.txt: -------------------------------------------------------------------------------- 1 | bput 99999 199135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 2 | bput 99999 25468001 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 3 | bput 99999 567891 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 4 | 5 | bget 99999 199135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 6 | bget 99999 25468001 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 7 | bget 99999 567891 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 8 | 9 | flush 10 | 11 | bgetop 99999 25468001 1 12 | bgetop 99999 25468001 2 13 | 14 | get 567891 3 15 | get 199135790 3 16 | get 25468001 3 17 | 18 | bput 99999 299135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 19 | bput 99999 25246802 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 20 | bput 99999 256782 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 21 | 22 | flush 23 | get 25468001 4 24 | get 199135790 4 25 | get 567891 4 26 | 27 | bput 99999 399135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 28 | bput 99999 35246803 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 29 | bput 99999 356783 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 30 | 31 | bput 99999 499135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 32 | bput 99999 45246804 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 33 | bput 99999 456784 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 34 | 35 | bdel 99999 299135790 36 | bdel 99999 25246802 37 | bdel 99999 256782 38 | 39 | bput 99999 599135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 40 | bput 99999 55246805 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 41 | bput 99999 556785 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 42 | 43 | bput 99999 991357900 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 44 | bput 99999 55468000 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 45 | bput 99999 167890 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 46 | 47 | flush 48 | 49 | ngetn 999999 0 0 50 | 51 | get 25468001 4 52 | get 199135790 4 53 | get 567891 4 54 | get 567891 3 55 | get 199135790 3 56 | get 25468001 3 57 | 58 | bdel 99999 499135790 59 | bdel 99999 45246804 60 | bdel 99999 456784 61 | 62 | bget 99999 199135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 63 | bget 99999 25468001 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 64 | bget 99999 567891 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 65 | 66 | bdel 99999 399135790 67 | bdel 99999 35246803 68 | bdel 99999 356783 69 | 70 | bdel 99999 599135790 71 | bdel 99999 55246805 72 | bdel 99999 556785 73 | 74 | bdel 99999 991357900 75 | bdel 99999 55468000 76 | bdel 99999 167890 77 | 78 | flush 79 | get 25468001 4 80 | get 199135790 4 81 | get 567891 4 82 | get 567891 3 83 | get 199135790 3 84 | get 25468001 3 85 | 86 | bdel 99999 199135790 87 | bdel 99999 25468001 88 | bdel 99999 567891 89 | 90 | flush 91 | get 11468001 4 92 | get 119135790 4 93 | get 117891 4 94 | get 117891 3 95 | get 119135790 3 96 | get 11468001 3 97 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/tester/inTestOps6.txt: -------------------------------------------------------------------------------- 1 | bput 99999 199135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 2 | bput 99999 25468001 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 3 | bput 99999 567891 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 4 | 5 | bget 99999 199135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 6 | bget 99999 25468001 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 7 | bget 99999 567891 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 8 | 9 | flush 10 | 11 | #bgetop 99999 25468001 1 12 | #bgetop 99999 25468001 2 13 | 14 | get 567891 3 15 | get 199135790 3 16 | get 25468001 3 17 | 18 | bput 99999 299135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 19 | bput 99999 25246802 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 20 | bput 99999 256782 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 21 | 22 | flush 23 | get 25468001 4 24 | get 199135790 4 25 | get 567891 4 26 | 27 | bput 99999 399135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 28 | bput 99999 35246803 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 29 | bput 99999 356783 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 30 | 31 | bput 99999 499135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 32 | bput 99999 45246804 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 33 | bput 99999 456784 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 34 | 35 | bdel 99999 299135790 36 | bdel 99999 25246802 37 | bdel 99999 256782 38 | 39 | bput 99999 599135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 40 | bput 99999 55246805 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 41 | bput 99999 556785 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 42 | 43 | bput 99999 991357900 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 44 | bput 99999 55468000 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 45 | bput 99999 167890 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 46 | 47 | flush 48 | 49 | #ngetn 999999 0 0 50 | 51 | get 25468001 4 52 | get 199135790 4 53 | get 567891 4 54 | get 567891 3 55 | get 199135790 3 56 | get 25468001 3 57 | 58 | bdel 99999 499135790 59 | bdel 99999 45246804 60 | bdel 99999 456784 61 | 62 | bget 99999 199135790 AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz_AbCdEfGhIjKlMnOpQrStUvWxYz 63 | bget 99999 25468001 abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz 64 | bget 99999 567891 ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ_ABCDEFGHIJKLMNOPQRSTUVWXYZ 65 | 66 | bdel 99999 399135790 67 | bdel 99999 35246803 68 | bdel 99999 356783 69 | 70 | bdel 99999 599135790 71 | bdel 99999 55246805 72 | bdel 99999 556785 73 | 74 | bdel 99999 991357900 75 | bdel 99999 55468000 76 | bdel 99999 167890 77 | 78 | flush 79 | get 25468001 4 80 | get 199135790 4 81 | get 567891 4 82 | get 567891 3 83 | get 199135790 3 84 | get 25468001 3 85 | 86 | bdel 99999 199135790 87 | bdel 99999 25468001 88 | bdel 99999 567891 89 | 90 | flush 91 | get 11468001 4 92 | get 119135790 4 93 | get 117891 4 94 | get 117891 3 95 | get 119135790 3 96 | get 11468001 3 97 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/bput-bdel.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "mpi.h" 5 | #include "mdhim.h" 6 | 7 | #define KEYS 50 8 | int main(int argc, char **argv) { 9 | int ret; 10 | int provided; 11 | int i; 12 | struct mdhim_t *md; 13 | int **keys; 14 | int key_lens[KEYS]; 15 | int **values; 16 | int value_lens[KEYS]; 17 | struct mdhim_brm_t *brm, *brmp; 18 | struct mdhim_bgetrm_t *bgrm, *bgrmp; 19 | char *db_path = "./"; 20 | char *db_name = "mdhimTstDB-"; 21 | int dbug = MLOG_CRIT; 22 | mdhim_options_t *db_opts; // Local variable for db create options to be passed 23 | int db_type = LEVELDB; //(data_store.h) 24 | MPI_Comm comm; 25 | 26 | // Create options for DB initialization 27 | db_opts = mdhim_options_init(); 28 | mdhim_options_set_db_path(db_opts, db_path); 29 | mdhim_options_set_db_name(db_opts, db_name); 30 | mdhim_options_set_db_type(db_opts, db_type); 31 | mdhim_options_set_key_type(db_opts, MDHIM_INT_KEY); 32 | mdhim_options_set_debug_level(db_opts, dbug); 33 | 34 | //Initialize MPI 35 | ret = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); 36 | if (ret != MPI_SUCCESS) { 37 | printf("Error initializing MPI with threads\n"); 38 | exit(1); 39 | } 40 | 41 | //Quit if the MPI doesn't support multiple thread mode 42 | if (provided != MPI_THREAD_MULTIPLE) { 43 | printf("Not able to enable MPI_THREAD_MULTIPLE mode\n"); 44 | exit(1); 45 | } 46 | 47 | //Initialize MDHIM 48 | comm = MPI_COMM_WORLD; 49 | md = mdhimInit(&comm, db_opts); 50 | if (!md) { 51 | printf("Error initializing MDHIM\n"); 52 | MPI_Abort(MPI_COMM_WORLD, ret); 53 | exit(1); 54 | } 55 | 56 | keys = malloc(sizeof(int *) * KEYS); 57 | values = malloc(sizeof(int *) * KEYS); 58 | for (i = 0; i < KEYS; i++) { 59 | keys[i] = malloc(sizeof(int)); 60 | *keys[i] = (i + 1) * (md->mdhim_rank + 1); 61 | printf("Rank: %d - Inserting key: %d\n", md->mdhim_rank, *keys[i]); 62 | key_lens[i] = sizeof(int); 63 | values[i] = malloc(sizeof(int)); 64 | *values[i] = (i + 1) * (md->mdhim_rank + 1); 65 | value_lens[i] = sizeof(int); 66 | } 67 | 68 | //Insert the records 69 | brm = mdhimBPut(md, (void **) keys, key_lens, 70 | (void **) values, value_lens, KEYS, 71 | NULL, NULL); 72 | brmp = brm; 73 | if (!brm || brm->error) { 74 | printf("Rank - %d: Error inserting keys/values into MDHIM\n", md->mdhim_rank); 75 | } 76 | while (brmp) { 77 | if (brmp->error < 0) { 78 | printf("Rank: %d - Error inserting key/values info MDHIM\n", md->mdhim_rank); 79 | } 80 | 81 | brmp = brmp->next; 82 | //Free the message 83 | mdhim_full_release_msg(brm); 84 | brm = brmp; 85 | } 86 | 87 | //Delete the records 88 | brm = mdhimBDelete(md, md->primary_index, 89 | (void **) keys, key_lens, 90 | KEYS); 91 | brmp = brm; 92 | if (!brm || brm->error) { 93 | printf("Rank - %d: Error deleting keys/values from MDHIM\n", md->mdhim_rank); 94 | } 95 | while (brmp) { 96 | if (brmp->error < 0) { 97 | printf("Rank: %d - Error deleting keys\n", md->mdhim_rank); 98 | } 99 | 100 | brmp = brmp->next; 101 | //Free the message 102 | mdhim_full_release_msg(brm); 103 | brm = brmp; 104 | } 105 | 106 | //Try to get the records back - should fail 107 | bgrm = mdhimBGet(md, md->primary_index, 108 | (void **) keys, key_lens, 109 | KEYS, MDHIM_GET_EQ); 110 | bgrmp = bgrm; 111 | while (bgrmp) { 112 | if (bgrmp->error < 0) { 113 | printf("Rank: %d - Error retrieving values\n", md->mdhim_rank); 114 | } 115 | 116 | for (i = 0; i < bgrmp->num_keys && bgrmp->error >= 0; i++) { 117 | 118 | printf("Rank: %d - Got key: %d value: %d\n", md->mdhim_rank, 119 | *(int *)bgrmp->keys[i], *(int *)bgrmp->values[i]); 120 | } 121 | 122 | bgrmp = bgrmp->next; 123 | //Free the message 124 | mdhim_full_release_msg(bgrm); 125 | bgrm = bgrmp; 126 | } 127 | 128 | ret = mdhimClose(md); 129 | if (ret != MDHIM_SUCCESS) { 130 | printf("Error closing MDHIM\n"); 131 | } 132 | 133 | MPI_Barrier(MPI_COMM_WORLD); 134 | MPI_Finalize(); 135 | for (i = 0; i < KEYS; i++) { 136 | free(keys[i]); 137 | free(values[i]); 138 | } 139 | 140 | free(keys); 141 | free(values); 142 | 143 | return 0; 144 | } 145 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/put-get_secondary.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "mpi.h" 4 | #include "mdhim.h" 5 | #include "mdhim_options.h" 6 | 7 | #define SLICE_SIZE 100 8 | #define SECONDARY_SLICE_SIZE 5 9 | 10 | int main(int argc, char **argv) { 11 | int ret; 12 | int provided = 0; 13 | struct mdhim_t *md; 14 | uint32_t key, **secondary_keys; 15 | uint32_t skey; 16 | int value, *secondary_key_lens; 17 | struct mdhim_brm_t *brm; 18 | struct mdhim_bgetrm_t *bgrm; 19 | mdhim_options_t *db_opts; 20 | struct index_t *secondary_index; 21 | struct secondary_info *secondary_info; 22 | MPI_Comm comm; 23 | 24 | ret = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); 25 | if (ret != MPI_SUCCESS) { 26 | printf("Error initializing MPI with threads\n"); 27 | exit(1); 28 | } 29 | 30 | if (provided != MPI_THREAD_MULTIPLE) { 31 | printf("Not able to enable MPI_THREAD_MULTIPLE mode\n"); 32 | exit(1); 33 | } 34 | 35 | db_opts = mdhim_options_init(); 36 | mdhim_options_set_db_path(db_opts, "./"); 37 | mdhim_options_set_db_name(db_opts, "mdhimTstDB"); 38 | mdhim_options_set_db_type(db_opts, LEVELDB); 39 | mdhim_options_set_key_type(db_opts, MDHIM_INT_KEY); //Key_type = 1 (int) 40 | mdhim_options_set_debug_level(db_opts, MLOG_CRIT); 41 | 42 | comm = MPI_COMM_WORLD; 43 | md = mdhimInit(&comm, db_opts); 44 | if (!md) { 45 | printf("Error initializing MDHIM\n"); 46 | exit(1); 47 | } 48 | 49 | //Set the primary keys and values 50 | key = 100 * (md->mdhim_rank + 1); 51 | value = 500 * (md->mdhim_rank + 1); 52 | //Set the secondary keys and values 53 | secondary_keys = malloc(sizeof(uint32_t *)); 54 | 55 | secondary_keys[0] = malloc(sizeof(uint32_t)); 56 | *secondary_keys[0] = md->mdhim_rank + 1; 57 | secondary_key_lens = malloc(sizeof(int)); 58 | secondary_key_lens[0] = sizeof(uint32_t); 59 | 60 | //Create the secondary remote index 61 | secondary_index = create_global_index(md, 2, SECONDARY_SLICE_SIZE, LEVELDB, 62 | MDHIM_INT_KEY, NULL); 63 | //Create the secondary info struct 64 | secondary_info = mdhimCreateSecondaryInfo(secondary_index, 65 | (void **) secondary_keys, 66 | secondary_key_lens, 67 | 1, SECONDARY_GLOBAL_INFO); 68 | //Put the primary and secondary keys 69 | brm = mdhimPut(md, 70 | &key, sizeof(key), 71 | &value, sizeof(value), 72 | secondary_info, 73 | NULL); 74 | 75 | if (!brm || brm->error) { 76 | printf("Error inserting key/value into MDHIM\n"); 77 | } else { 78 | printf("Successfully inserted key/value into MDHIM\n"); 79 | } 80 | mdhim_full_release_msg(brm); 81 | 82 | //Put another secondary key 83 | skey = 2 * (md->mdhim_rank + 1); 84 | brm = mdhimPutSecondary(md, 85 | secondary_index, 86 | /*Secondary key */ 87 | &skey, sizeof(skey), 88 | /* Primary key */ 89 | &key, sizeof(key)); 90 | 91 | if (!brm || brm->error) { 92 | printf("Error inserting key/value into MDHIM\n"); 93 | } else { 94 | printf("Successfully inserted key/value into MDHIM\n"); 95 | } 96 | 97 | //Release the received message 98 | mdhim_full_release_msg(brm); 99 | 100 | //Commit the database 101 | ret = mdhimCommit(md, md->primary_index); 102 | if (ret != MDHIM_SUCCESS) { 103 | printf("Error committing MDHIM database\n"); 104 | } else { 105 | printf("Committed MDHIM database\n"); 106 | } 107 | 108 | //Get the primary key values from the secondary key 109 | value = 0; 110 | 111 | bgrm = mdhimGet(md, secondary_index, 112 | secondary_keys[0], secondary_key_lens[0], 113 | MDHIM_GET_PRIMARY_EQ); 114 | if (!bgrm || bgrm->error) { 115 | printf("Error getting value for key: %d from MDHIM\n", key); 116 | } else if (bgrm->value_lens[0]) { 117 | printf("Successfully got value: %d from MDHIM\n", *((int *) bgrm->values[0])); 118 | } 119 | 120 | mdhim_full_release_msg(bgrm); 121 | ret = mdhimClose(md); 122 | mdhim_options_destroy(db_opts); 123 | if (ret != MDHIM_SUCCESS) { 124 | printf("Error closing MDHIM\n"); 125 | } 126 | 127 | free(secondary_keys[0]); 128 | free(secondary_keys); 129 | free(secondary_key_lens); 130 | MPI_Barrier(MPI_COMM_WORLD); 131 | MPI_Finalize(); 132 | 133 | return 0; 134 | } 135 | -------------------------------------------------------------------------------- /BurstFS_Server/burstfs_global.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | #ifndef BURSTFS_GLOBAL_H 16 | #define BURSTFS_GLOBAL_H 17 | #include 18 | #include 19 | #include "burstfs_const.h" 20 | #include "arraylist.h" 21 | 22 | typedef enum { 23 | COMM_MOUNT, /*the list of addrs: appid, size of buffer, offset of data section, metadata section*/ 24 | COMM_META, 25 | COMM_READ, 26 | COMM_UNMOUNT, 27 | COMM_DIGEST, 28 | COMM_SYNC_DEL, 29 | }cmd_lst_t; 30 | 31 | typedef enum { 32 | XFER_COMM_DATA, 33 | XFER_COMM_EXIT, 34 | }service_cmd_lst_t; 35 | 36 | typedef enum { 37 | ACK_SUCCESS, 38 | ACK_FAIL, 39 | }ack_status_t; 40 | 41 | typedef struct { 42 | int dest_app_id; 43 | int dest_client_id; /*note: the id is the app-side rank on the remote node*/ 44 | long dest_offset; 45 | int dest_delegator_rank; 46 | long length; 47 | int src_delegator_rank; 48 | int src_cli_id; 49 | int src_app_id; 50 | int src_fid; 51 | long src_offset; 52 | int src_thrd; 53 | int src_dbg_rank; 54 | int arrival_time; 55 | } send_msg_t; 56 | 57 | typedef struct { 58 | long src_fid; 59 | long src_offset; 60 | long length; 61 | }recv_msg_t; 62 | 63 | 64 | typedef struct { 65 | int num; 66 | send_msg_t msg_meta[MAX_META_PER_SEND]; 67 | }msg_meta_t; 68 | 69 | typedef struct { 70 | long superblock_sz; 71 | long meta_offset; 72 | long meta_size; 73 | long fmeta_offset; 74 | long fmeta_size; 75 | long data_offset; 76 | long data_size; 77 | int req_buf_sz; 78 | int recv_buf_sz; 79 | int num_procs_per_node; 80 | int client_ranks[MAX_NUM_CLIENTS]; 81 | int thrd_idxs[MAX_NUM_CLIENTS]; 82 | int shm_superblock_fds[MAX_NUM_CLIENTS]; 83 | int shm_req_fds[MAX_NUM_CLIENTS]; 84 | int shm_recv_fds[MAX_NUM_CLIENTS]; 85 | char *shm_superblocks[MAX_NUM_CLIENTS]; 86 | char *shm_req_bufs[MAX_NUM_CLIENTS]; 87 | char *shm_recv_bufs[MAX_NUM_CLIENTS]; 88 | int spill_log_fds[MAX_NUM_CLIENTS]; 89 | int spill_index_log_fds[MAX_NUM_CLIENTS]; 90 | int dbg_ranks[MAX_NUM_CLIENTS]; 91 | char external_spill_dir[MAX_PATH_LEN]; 92 | char recv_buf_name[MAX_NUM_CLIENTS][MAX_PATH_LEN]; 93 | char req_buf_name[MAX_NUM_CLIENTS][MAX_PATH_LEN]; 94 | char super_buf_name[MAX_NUM_CLIENTS][MAX_PATH_LEN]; 95 | char spill_log_name[MAX_NUM_CLIENTS][MAX_PATH_LEN]; 96 | char spill_index_log_name[MAX_NUM_CLIENTS][MAX_PATH_LEN]; 97 | }app_config_t; 98 | 99 | typedef struct { 100 | int req_cnt; /*number of requests to this delegator*/ 101 | int del_id; /*rank of delegator*/ 102 | }per_del_stat_t; 103 | 104 | typedef struct { 105 | per_del_stat_t *req_stat; 106 | int del_cnt; /*number of unique delegators*/ 107 | }del_req_stat_t; 108 | 109 | typedef struct { 110 | pthread_t thrd; 111 | pthread_cond_t thrd_cond; 112 | pthread_mutex_t thrd_lock; 113 | 114 | int has_waiting_delegator; 115 | int has_waiting_dispatcher; 116 | /* a list of read requests to 117 | * be sent to each 118 | * delegator*/ 119 | msg_meta_t *del_req_set; 120 | 121 | /* statistics of read requests 122 | * to be sent to each delegator*/ 123 | del_req_stat_t *del_req_stat; 124 | char del_req_msg_buf[REQ_BUF_LEN]; 125 | char del_recv_msg_buf[RECV_BUF_CNT][RECV_BUF_LEN]; 126 | int exit_flag; 127 | }thrd_ctrl_t; 128 | 129 | typedef struct { 130 | int app_id; 131 | int sock_id; 132 | }cli_signature_t; 133 | 134 | typedef int fattr_key_t; 135 | 136 | typedef struct { 137 | char fname[ULFS_MAX_FILENAME]; 138 | struct stat file_attr; 139 | }fattr_val_t; 140 | 141 | typedef struct { 142 | int fid; 143 | int gfid; 144 | char filename[ULFS_MAX_FILENAME]; 145 | struct stat file_attr; 146 | }burstfs_file_attr_t; 147 | extern arraylist_t *app_config_list; 148 | extern arraylist_t *thrd_list; 149 | int invert_sock_ids[MAX_NUM_CLIENTS]; 150 | extern pthread_t data_thrd; 151 | extern int glb_rank, glb_size; 152 | extern int local_rank_idx; 153 | extern int *local_rank_lst; 154 | extern int local_rank_cnt; 155 | extern long max_recs_per_slice; 156 | #endif 157 | -------------------------------------------------------------------------------- /BurstFS_Meta/src/data_store.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | * Produced at the Lawrence Livermore National Laboratory. 4 | * Copyright (c) 2017, Florida State University. Contributions from 5 | * the Computer Architecture and Systems Research Laboratory (CASTL) 6 | * at the Department of Computer Science. 7 | * 8 | * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror 9 | * LLNL-CODE-728877. All rights reserved. 10 | * 11 | * This file is part of BurstFS. For details, see https://github.com/llnl/burstfs 12 | * Please read https://github.com/llnl/burstfs/LICENSE for full license text. 13 | */ 14 | 15 | /* 16 | * 17 | * Copyright (c) 2014, Los Alamos National Laboratory 18 | * All rights reserved. 19 | * 20 | */ 21 | 22 | #ifndef __STORE_H 23 | #define __STORE_H 24 | 25 | #include "uthash.h" 26 | #include "mdhim_options.h" 27 | 28 | /* Storage Methods */ 29 | #define LEVELDB 1 //LEVELDB storage method 30 | #define MYSQLDB 3 31 | #define ROCKSDB 4 //RocksDB 32 | /* mdhim_store_t flags */ 33 | #define MDHIM_CREATE 1 //Implies read/write 34 | #define MDHIM_RDONLY 2 35 | #define MDHIM_RDWR 3 36 | 37 | /* Keys for stats database */ 38 | #define MDHIM_MAX_STAT 1 39 | #define MDHIM_MIN_STAT 2 40 | #define MDHIM_NUM_STAT 3 41 | 42 | struct mdhim_store_t; 43 | /* Function pointers for abstracting data stores */ 44 | typedef int (*mdhim_store_open_fn_t)(void **db_handle, void **db_stats, char *path, int flags, 45 | int key_type, struct mdhim_options_t *opts); 46 | typedef int (*mdhim_store_put_fn_t)(void *db_handle, void *key, int32_t key_len, 47 | void *data, int32_t data_len); 48 | typedef int (*mdhim_store_batch_put_fn_t)(void *db_handle, void **keys, int32_t *key_lens, 49 | void **data, int32_t *data_lens, int num_records); 50 | typedef int (*mdhim_store_get_fn_t)(void *db_handle, void *key, int key_len, void **data, int32_t *data_len); 51 | typedef int (*mdhim_store_get_next_fn_t)(void *db_handle, void **key, 52 | int *key_len, void **data, 53 | int32_t *data_len); 54 | typedef int (*mdhim_store_get_prev_fn_t)(void *db_handle, void **key, 55 | int *key_len, void **data, 56 | int32_t *data_len); 57 | typedef int (*mdhim_store_del_fn_t)(void *db_handle, void *key, int key_len); 58 | typedef int (*mdhim_store_commit_fn_t)(void *db_handle); 59 | typedef int (*mdhim_store_close_fn_t)(void *db_handle, void *db_stats); 60 | 61 | //Used for storing stats in a hash table 62 | struct mdhim_stat; 63 | struct mdhim_stat { 64 | int key; //Key (slice number) 65 | void *max; //Max key 66 | void *min; //Min key 67 | int dirty; //Wether this stat was updated or a new stat 68 | uint64_t num; //Number of keys in this slice 69 | struct mdhim_stat *stats; //Used for local index stats to create a multi-level hash table 70 | UT_hash_handle hh; /* makes this structure hashable */ 71 | }; 72 | 73 | 74 | //Used for storing stats in the database 75 | struct mdhim_db_stat { 76 | int slice; 77 | uint64_t imax; 78 | uint64_t imin; 79 | long double dmax; 80 | long double dmin; 81 | uint64_t num; 82 | }; 83 | 84 | //Used for transmitting integer stats to all nodes 85 | struct mdhim_db_istat { 86 | int slice; 87 | uint64_t num; 88 | uint64_t imax; 89 | uint64_t imin; 90 | }; 91 | 92 | //Used for transmitting float stats to all nodes 93 | struct mdhim_db_fstat { 94 | int slice; 95 | uint64_t num; 96 | long double dmax; 97 | long double dmin; 98 | }; 99 | 100 | /* Generic mdhim storage object */ 101 | struct mdhim_store_t { 102 | int type; 103 | //handle to db 104 | void *db_handle; 105 | //Handle to db for stats 106 | void *db_stats; 107 | //Pointers to functions based on data store 108 | mdhim_store_open_fn_t open; 109 | mdhim_store_put_fn_t put; 110 | mdhim_store_batch_put_fn_t batch_put; 111 | mdhim_store_get_fn_t get; 112 | mdhim_store_get_next_fn_t get_next; 113 | mdhim_store_get_prev_fn_t get_prev; 114 | mdhim_store_del_fn_t del; 115 | mdhim_store_commit_fn_t commit; 116 | mdhim_store_close_fn_t close; 117 | 118 | //Login credentials 119 | char *db_user; 120 | char *db_upswd; 121 | char *dbs_user; 122 | char *dbs_upswd; 123 | char *db_host; 124 | char *dbs_host; 125 | 126 | //Hashtable for stats 127 | struct mdhim_stat *mdhim_store_stats; 128 | 129 | //Lock to allow concurrent readers and a single writer to the mdhim_store_stats 130 | pthread_rwlock_t *mdhim_store_stats_lock; 131 | }; 132 | 133 | //Initializes the data store based on the type given (i.e., LEVELDB, etc...) 134 | struct mdhim_store_t *mdhim_db_init(int db_type); 135 | #endif 136 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/tester/inTestLarge.txt: -------------------------------------------------------------------------------- 1 | #23456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 12345 2 | # Key for string/byte 505_digit && 0rank 3 | put abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz12340abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ111112222233333 This_is_the_value_for_a_long_key 4 | 5 | get abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz12340abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ111112222233333 6 | 7 | # data element of size 1015 8 | put dataElementOfSize1015 abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz1234_abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ111112222233333abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz1234_abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ11111222223333344444 9 | 10 | get dataElementOfSize1015 11 | 12 | # data value of 2040 13 | put dataValueOf2040 abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz1234_abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ111112222233333abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz1234_abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ111112222233333abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz1234_abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ111112222233333abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz1234_abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ11111222223333344444555556666677777 14 | 15 | get dataValueOf2040 16 | 17 | # data element of size 505 18 | put dataElementOfSize505 abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456YXWVUTSRQPONMLKJIHGFEDCBA3456ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012abcdefghijklmnopqrstuvwxyz12340abcdefghijklmnopqrstuvwxyz1234ABCDEFGHIJKLMNOPQRSTUVWXYZ5678zyxwvutsrqponmlkjihgfedcba9012ZYXWVUTSRQPONMLKJIHGFEDCBA3456AAAAABBBBBCCCCCDDDDDEEEEEFFFFFGGGGGHHHHHIIIIIJJJJJKKKKKLLLLLMMMMMNNNNNOOOOOPPPPPQQQQQRRRRRSSSSSTTTTTUUUUUVVVVVWWWWWXXXXXYYYYYZZZZZ111112222233333 19 | 20 | get dataElementOfSize505 21 | 22 | -------------------------------------------------------------------------------- /BurstFS_Meta/tests/single_tests/plfs-put-get.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "mpi.h" 5 | #include "mdhim.h" 6 | #include "mdhim_options.h" 7 | 8 | #define SLICE_SIZE 1 9 | 10 | struct plfs_record { 11 | unsigned long long int logical_offset; 12 | unsigned long long int size; 13 | char dropping_file[PATH_MAX]; 14 | unsigned long long int physical_offset; 15 | }; 16 | 17 | FILE *open_output(int rank) { 18 | FILE *file; 19 | char rank_str[4]; 20 | char file_str[4]; 21 | char char_str[2]; 22 | int i, j; 23 | char filename[PATH_MAX]; 24 | 25 | //Opens the file and coverts the rank to characters 26 | sprintf(rank_str, "%d", rank); 27 | memset(file_str, 0, 4); 28 | memset(file_str, 'a', 3); 29 | j = strlen(file_str) - 1; 30 | for (i = strlen(rank_str) - 1; i >= 0; i--) { 31 | sprintf(char_str, "%c", rank_str[i]); 32 | file_str[j] = strtol(char_str, NULL, 10) + 'a'; 33 | j--; 34 | } 35 | 36 | sprintf(filename, "plfs-output/plfs%s", file_str); 37 | printf("file string is: %s\n", filename); 38 | file = fopen(filename, "r"); 39 | if (!file) { 40 | printf("Error opening the input file"); 41 | } 42 | 43 | return file; 44 | } 45 | 46 | struct plfs_record *parse_input(FILE *file) { 47 | struct plfs_record *rec; 48 | int ret; 49 | 50 | rec = malloc(sizeof(struct plfs_record)); 51 | ret = fscanf(file, "%llu %llu %s %llu", &rec->logical_offset, &rec->size, 52 | rec->dropping_file, &rec->physical_offset); 53 | if (!ret || ret == EOF) { 54 | printf("Error parsing file\n"); 55 | exit(1); 56 | } 57 | 58 | printf("Parsed record with logical_offset: %llu, size: %llu, dropping_file: %s," 59 | " physical_offset: %llu\n", 60 | rec->logical_offset, rec->size, 61 | rec->dropping_file, rec->physical_offset); 62 | 63 | return rec; 64 | } 65 | 66 | long long get_key(unsigned long long int lo) { 67 | unsigned long long int ret = ((unsigned long long int) lo/SLICE_SIZE) * SLICE_SIZE; 68 | return ret; 69 | } 70 | 71 | int main(int argc, char **argv) { 72 | int ret; 73 | int provided = 0; 74 | struct mdhim_t *md; 75 | struct mdhim_brm_t *brm; 76 | struct mdhim_bgetrm_t *bgrm; 77 | mdhim_options_t *db_opts; 78 | struct plfs_record *rec = NULL; 79 | FILE *file; 80 | unsigned long long int key; 81 | MPI_Comm comm; 82 | 83 | ret = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); 84 | if (ret != MPI_SUCCESS) { 85 | printf("Error initializing MPI with threads\n"); 86 | exit(1); 87 | } 88 | 89 | if (provided != MPI_THREAD_MULTIPLE) { 90 | printf("Not able to enable MPI_THREAD_MULTIPLE mode\n"); 91 | exit(1); 92 | } 93 | 94 | //Set MDHIM options 95 | db_opts = mdhim_options_init(); 96 | mdhim_options_set_db_path(db_opts, "./"); 97 | mdhim_options_set_db_name(db_opts, "mdhimTstDB"); 98 | mdhim_options_set_db_type(db_opts, LEVELDB); 99 | mdhim_options_set_key_type(db_opts, MDHIM_LONG_INT_KEY); 100 | mdhim_options_set_debug_level(db_opts, MLOG_CRIT); 101 | mdhim_options_set_max_recs_per_slice(db_opts, SLICE_SIZE); 102 | mdhim_options_set_server_factor(db_opts, 10); 103 | mdhim_options_set_value_append(db_opts, 1); 104 | 105 | //Initialize MDHIM 106 | comm = MPI_COMM_WORLD; 107 | md = mdhimInit(&comm, db_opts); 108 | if (!md) { 109 | printf("Error initializing MDHIM\n"); 110 | exit(1); 111 | } 112 | 113 | file = open_output(md->mdhim_rank); 114 | if (!file) { 115 | printf("Error opening file\n"); 116 | goto done; 117 | } 118 | 119 | rec = parse_input(file); 120 | if (!rec) { 121 | printf("Error parsing file\n"); 122 | goto done; 123 | } 124 | key = get_key(rec->logical_offset); 125 | printf("Inserting key: %llu\n", key); 126 | brm = mdhimPut(md, &key, sizeof(key), 127 | rec, sizeof(struct plfs_record), NULL, NULL); 128 | if (!brm || brm->error) { 129 | printf("Error inserting key/value into MDHIM\n"); 130 | } else { 131 | printf("Successfully inserted key/value into MDHIM\n"); 132 | } 133 | 134 | mdhim_full_release_msg(brm); 135 | //Commit the database 136 | ret = mdhimCommit(md, md->primary_index); 137 | if (ret != MDHIM_SUCCESS) { 138 | printf("Error committing MDHIM database\n"); 139 | } else { 140 | printf("Committed MDHIM database\n"); 141 | } 142 | 143 | bgrm = mdhimGet(md, md->primary_index, &key, sizeof(key), 144 | MDHIM_GET_EQ); 145 | if (!bgrm || bgrm->error) { 146 | printf("Error getting value for key: %llu from MDHIM\n", key); 147 | } else if (bgrm->value_lens[0]) { 148 | printf("Successfully got value: %d from MDHIM\n", *((int *) bgrm->values[0])); 149 | } 150 | 151 | mdhim_full_release_msg(bgrm); 152 | 153 | done: 154 | ret = mdhimClose(md); 155 | free(rec); 156 | mdhim_options_destroy(db_opts); 157 | if (ret != MDHIM_SUCCESS) { 158 | printf("Error closing MDHIM\n"); 159 | } 160 | 161 | MPI_Barrier(MPI_COMM_WORLD); 162 | MPI_Finalize(); 163 | 164 | return 0; 165 | } 166 | --------------------------------------------------------------------------------