├── test ├── CMakeLists.txt ├── legacy │ ├── dtar_tests │ │ ├── machines │ │ ├── dir │ │ │ ├── cp.sh │ │ │ └── log.h │ │ ├── env.sh │ │ ├── run.sh │ │ └── test.sh │ ├── README.md │ ├── dtest │ │ ├── README.md │ │ └── moab.py │ └── dcp1_tests │ │ ├── test_dcp1_single_dir_to_single_dir │ │ └── test.sh │ │ ├── test_dcp1_single_dir_to_single_file │ │ └── test.sh │ │ ├── test_dcp1_many_dir_to_single_dir │ │ └── test.sh │ │ ├── test_all.sh │ │ ├── test_dcp1_many_file_to_single_dir │ │ └── test.sh │ │ ├── test_dcp1_single_file_to_single_dir │ │ └── test.sh │ │ ├── test_dcp1_many_dir_to_single_file │ │ └── test.sh │ │ ├── test_dcp1_single_file_to_single_file │ │ └── test.sh │ │ └── test_dcp1_many_file_to_single_file │ │ └── test.sh └── tests │ ├── test_dsync │ ├── utility │ │ ├── one │ │ ├── two │ │ ├── test_driver.sh │ │ └── set_funcs.sh │ ├── test_xattr.py │ ├── test_metadata.sh │ ├── test_expectfail.sh │ └── test_xattr.sh │ ├── test_dcp1 │ ├── test_all.py │ ├── test_many_dir_to_single_dir.py │ ├── test_many_dir_to_single_file.py │ ├── test_many_file_to_single_dir.py │ ├── test_many_file_to_single_file.py │ ├── test_single_dir_to_single_dir.py │ ├── test_single_dir_to_single_file.py │ ├── test_single_file_to_single_dir.py │ └── test_single_file_to_single_file.py │ ├── test_dchmod │ ├── test_octal.py │ └── test_octal.sh │ └── test_dcp │ ├── test_fiemap.py │ ├── checkfiemap.c │ └── test_fiemap.sh ├── doc ├── .pip ├── rst │ ├── dcp_sierra.png │ ├── dgrep.1.rst │ ├── dparallel.1.rst │ ├── daos-deserialize.1.rst │ ├── ddup.1.rst │ ├── daos-serialize.1.rst │ ├── dbz2.1.rst │ ├── index.rst │ ├── dfilemaker.1.rst │ ├── dbcast.1.rst │ ├── proj-design.rst │ ├── dreln.1.rst │ ├── dstripe.1.rst │ ├── drm.1.rst │ ├── dchmod.1.rst │ ├── dwalk.1.rst │ ├── libmfu.rst │ ├── dfind.1.rst │ └── dsync.1.rst ├── Makefile ├── README.md └── .readthedocs.yaml ├── src ├── dbz2 │ └── CMakeLists.txt ├── dcmp │ └── CMakeLists.txt ├── dcp │ └── CMakeLists.txt ├── dfind │ ├── CMakeLists.txt │ └── common.h ├── dgrep │ ├── CMakeLists.txt │ ├── dgrep.h │ ├── log.h │ └── dgrep.c ├── dreln │ └── CMakeLists.txt ├── drm │ └── CMakeLists.txt ├── dsh │ └── CMakeLists.txt ├── dsync │ └── CMakeLists.txt ├── dwalk │ └── CMakeLists.txt ├── dbcast │ └── CMakeLists.txt ├── dchmod │ └── CMakeLists.txt ├── dstripe │ └── CMakeLists.txt ├── daos-gen │ └── CMakeLists.txt ├── dfilemaker │ └── CMakeLists.txt ├── dparallel │ ├── CMakeLists.txt │ ├── dparallel.h │ ├── log.h │ └── dparallel.c ├── daos-serialize │ └── CMakeLists.txt ├── daos-deserialize │ └── CMakeLists.txt ├── ddup │ └── CMakeLists.txt ├── dcp1 │ ├── dcp1.h │ ├── copy.h │ ├── cleanup.h │ ├── compare.h │ ├── treewalk.h │ ├── handle_args.h │ ├── CMakeLists.txt │ ├── COPYING │ └── cleanup.c ├── dtar │ └── CMakeLists.txt ├── common │ ├── mfu_bz2.h │ ├── mfu_proc.h │ ├── mfu.h │ ├── mfu_errors.h │ ├── mfu_proc.c │ ├── CMakeLists.txt │ ├── mfu_progress.h │ ├── README.md │ ├── mfu_bz2.c │ └── strmap.h └── CMakeLists.txt ├── spack.yaml ├── man ├── README.md ├── CMakeLists.txt ├── ddup.1 ├── dbz2.1 ├── dfilemaker.1 ├── dbcast.1 ├── dreln.1 └── dstripe.1 ├── cmake ├── MFU_ADD_TOOL.cmake ├── FindLibCap.cmake ├── FindLibAttr.cmake ├── FindGPFS.cmake ├── FindDTCMP.cmake ├── FindLibCircle.cmake ├── FindLibArchive.cmake ├── FindHDF5.cmake ├── FindDAOS.cmake └── SetupMPI.cmake ├── astyle.options ├── AUTHORS ├── dist ├── README.dist ├── README.md └── builddist ├── .gitlab-ci.yml ├── mpifileutils.spec ├── mpifu_venv.sh ├── .gitignore ├── NOTICE ├── scripts └── prepmfubuild.sh ├── LICENSE ├── README.md ├── .travis.yml └── CONTRIBUTING.md /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/.pip: -------------------------------------------------------------------------------- 1 | guzzle-sphinx-theme 2 | -------------------------------------------------------------------------------- /src/dbz2/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(dbz2) 2 | -------------------------------------------------------------------------------- /src/dcmp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(dcmp) 2 | -------------------------------------------------------------------------------- /src/dcp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(dcp) 2 | -------------------------------------------------------------------------------- /src/dfind/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(dfind) 2 | -------------------------------------------------------------------------------- /src/dgrep/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(dgrep) 2 | -------------------------------------------------------------------------------- /src/dreln/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(dreln) 2 | -------------------------------------------------------------------------------- /src/drm/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(drm) 2 | -------------------------------------------------------------------------------- /src/dsh/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(dsh) 2 | -------------------------------------------------------------------------------- /src/dsync/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(dsync) 2 | -------------------------------------------------------------------------------- /src/dwalk/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(dwalk) 2 | -------------------------------------------------------------------------------- /test/legacy/dtar_tests/machines: -------------------------------------------------------------------------------- 1 | localhost 2 | -------------------------------------------------------------------------------- /src/dbcast/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(dbcast) 2 | -------------------------------------------------------------------------------- /src/dchmod/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(dchmod) 2 | -------------------------------------------------------------------------------- /src/dstripe/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(dstripe) 2 | -------------------------------------------------------------------------------- /src/daos-gen/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(daos-gen) 2 | -------------------------------------------------------------------------------- /src/dfilemaker/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(dfilemaker) 2 | -------------------------------------------------------------------------------- /src/dparallel/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(dparallel) 2 | -------------------------------------------------------------------------------- /test/tests/test_dsync/utility/one: -------------------------------------------------------------------------------- 1 | 1 2 | 2 3 | 3 4 | 4 5 | -------------------------------------------------------------------------------- /test/tests/test_dsync/utility/two: -------------------------------------------------------------------------------- 1 | 3 2 | 4 3 | 5 4 | 6 5 | -------------------------------------------------------------------------------- /src/daos-serialize/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(daos-serialize) 2 | -------------------------------------------------------------------------------- /src/daos-deserialize/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(daos-deserialize) 2 | -------------------------------------------------------------------------------- /doc/rst/dcp_sierra.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpc/mpifileutils/HEAD/doc/rst/dcp_sierra.png -------------------------------------------------------------------------------- /src/ddup/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | MFU_ADD_TOOL(ddup) 2 | TARGET_LINK_LIBRARIES(ddup ${OPENSSL_LIBRARIES}) 3 | -------------------------------------------------------------------------------- /test/legacy/dtar_tests/dir/cp.sh: -------------------------------------------------------------------------------- 1 | for i in {1..100} 2 | do 3 | cp ./log.h $i.h 4 | done 5 | 6 | -------------------------------------------------------------------------------- /test/legacy/dtar_tests/env.sh: -------------------------------------------------------------------------------- 1 | export PATH="`pwd`/../install/include:$PATH" >& /dev/null 2 | export LD_LIBRARY_PATH="`pwd`/../install/lib:$LD_LIBRARY_PATH" >& /dev/null 3 | -------------------------------------------------------------------------------- /test/tests/test_dcp1/test_all.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | from subprocess import call 3 | 4 | def test_all(): 5 | rc = call("~/mpifileutils/test/legacy/dcp1_tests/test_all.sh", shell=True) 6 | -------------------------------------------------------------------------------- /spack.yaml: -------------------------------------------------------------------------------- 1 | spack: 2 | specs: 3 | - dtcmp 4 | - libarchive 5 | - libcircle 6 | mirrors: {} 7 | modules: 8 | enable: [] 9 | repos: [] 10 | packages: {} 11 | config: {} 12 | # view: true 13 | -------------------------------------------------------------------------------- /src/dcp1/dcp1.h: -------------------------------------------------------------------------------- 1 | /* See the file "COPYING" for the full license governing this code. */ 2 | 3 | #ifndef __DCP_H_ 4 | #define __DCP_H_ 5 | 6 | #include "common.h" 7 | 8 | void DCOPY_print_usage(void); 9 | 10 | #endif /* __DCP_H_ */ 11 | -------------------------------------------------------------------------------- /src/dgrep/dgrep.h: -------------------------------------------------------------------------------- 1 | #ifndef DGREP_H 2 | #define DGREP_H 3 | 4 | #include 5 | 6 | void DGREP_start(CIRCLE_handle *handle); 7 | void DGREP_search(CIRCLE_handle *handle); 8 | 9 | void print_usage(char *prog); 10 | 11 | #endif /* DGREP_H */ 12 | -------------------------------------------------------------------------------- /man/README.md: -------------------------------------------------------------------------------- 1 | Do not modify these man pages directly. 2 | They are generated from the doc/rst files using sphinx. 3 | After building new man pages, files should be copied over. 4 | 5 | cp doc/build/man/* man/ 6 | 7 | Be sure to list new tools in man/Makefile.am 8 | -------------------------------------------------------------------------------- /test/tests/test_dcp1/test_many_dir_to_single_dir.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | from subprocess import call 3 | 4 | def test_dcp1_many_dir_to_single_dir(): 5 | rc = call("~/mpifileutils/test/legacy/dcp1_tests/test_dcp1_many_dir_to_single_dir/test.sh", shell=True) 6 | -------------------------------------------------------------------------------- /test/tests/test_dcp1/test_many_dir_to_single_file.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | from subprocess import call 3 | 4 | def test_dcp1_many_dir_to_single_file(): 5 | rc = call("~/mpifileutils/test/legacy/dcp1_tests/test_dcp1_many_dir_to_single_file/test.sh", shell=True) 6 | -------------------------------------------------------------------------------- /test/tests/test_dcp1/test_many_file_to_single_dir.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | from subprocess import call 3 | 4 | def test_dcp1_many_file_to_single_dir(): 5 | rc = call("~/mpifileutils/test/legacy/dcp1_tests/test_dcp1_many_file_to_single_dir/test.sh", shell=True) 6 | -------------------------------------------------------------------------------- /test/tests/test_dcp1/test_many_file_to_single_file.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | from subprocess import call 3 | 4 | def test_dcp1_many_file_to_single_file(): 5 | rc = call("~/mpifileutils/test/legacy/dcp1_tests/test_dcp1_many_file_to_single_file/test.sh", shell=True) 6 | -------------------------------------------------------------------------------- /test/tests/test_dcp1/test_single_dir_to_single_dir.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | from subprocess import call 3 | 4 | def test_dcp1_single_dir_to_single_dir(): 5 | rc = call("~/mpifileutils/test/legacy/dcp1_tests/test_dcp1_single_dir_to_single_dir/test.sh", shell=True) 6 | -------------------------------------------------------------------------------- /test/tests/test_dcp1/test_single_dir_to_single_file.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | from subprocess import call 3 | 4 | def test_dcp1_single_dir_to_single_file(): 5 | rc = call("~/mpifileutils/test/legacy/dcp1_tests/test_dcp1_single_dir_to_single_file/test.sh", shell=True) 6 | -------------------------------------------------------------------------------- /test/tests/test_dcp1/test_single_file_to_single_dir.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | from subprocess import call 3 | 4 | def test_dcp1_single_file_to_single_dir(): 5 | rc = call("~/mpifileutils/test/legacy/dcp1_tests/test_dcp1_single_file_to_single_dir/test.sh", shell=True) 6 | -------------------------------------------------------------------------------- /test/tests/test_dcp1/test_single_file_to_single_file.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | from subprocess import call 3 | 4 | def test_dcp1_single_file_to_single_file(): 5 | rc = call("~/mpifileutils/test/legacy/dcp1_tests/test_dcp1_single_file_to_single_file/test.sh", shell=True) 6 | -------------------------------------------------------------------------------- /src/dtar/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | INCLUDE_DIRECTORIES(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}) 2 | 3 | # MFU_ADD_TOOL(dtar) 4 | ADD_EXECUTABLE(dtar dtar.c) 5 | TARGET_LINK_LIBRARIES(dtar mfu) 6 | SET_TARGET_PROPERTIES(dtar PROPERTIES C_STANDARD 99) 7 | INSTALL(TARGETS dtar DESTINATION ${CMAKE_INSTALL_BINDIR}) 8 | -------------------------------------------------------------------------------- /cmake/MFU_ADD_TOOL.cmake: -------------------------------------------------------------------------------- 1 | FUNCTION(MFU_ADD_TOOL name) 2 | 3 | ADD_EXECUTABLE(${name} ${name}.c) 4 | TARGET_LINK_LIBRARIES(${name} mfu m) 5 | SET_TARGET_PROPERTIES(${name} PROPERTIES C_STANDARD 99) 6 | INSTALL(TARGETS ${name} DESTINATION ${CMAKE_INSTALL_BINDIR}) 7 | 8 | ENDFUNCTION(MFU_ADD_TOOL name) 9 | -------------------------------------------------------------------------------- /src/dcp1/copy.h: -------------------------------------------------------------------------------- 1 | /* See the file "COPYING" for the full license governing this code. */ 2 | 3 | #ifndef __DCP_COPY_H 4 | #define __DCP_COPY_H 5 | 6 | #include "common.h" 7 | 8 | void DCOPY_do_copy(DCOPY_operation_t* op, 9 | CIRCLE_handle* handle); 10 | 11 | #endif /* __DCP_COPY_H */ 12 | -------------------------------------------------------------------------------- /src/dcp1/cleanup.h: -------------------------------------------------------------------------------- 1 | /* See the file "COPYING" for the full license governing this code. */ 2 | 3 | #ifndef __DCP_CLEANUP_H 4 | #define __DCP_CLEANUP_H 5 | 6 | #include "common.h" 7 | 8 | void DCOPY_do_cleanup(DCOPY_operation_t* op, 9 | CIRCLE_handle* handle); 10 | 11 | #endif /* __DCP_CLEANUP_H */ 12 | -------------------------------------------------------------------------------- /src/dcp1/compare.h: -------------------------------------------------------------------------------- 1 | /* See the file "COPYING" for the full license governing this code. */ 2 | 3 | #ifndef __DCP_COMPARE_H 4 | #define __DCP_COMPARE_H 5 | 6 | #include "common.h" 7 | 8 | void DCOPY_do_compare(DCOPY_operation_t* op, 9 | CIRCLE_handle* handle); 10 | 11 | #endif /* __DCP_COMPARE_H */ 12 | -------------------------------------------------------------------------------- /man/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | LIST(APPEND man_pages 2 | dbcast.1 3 | dchmod.1 4 | dcmp.1 5 | dcp.1 6 | ddup.1 7 | dbz2.1 8 | dfilemaker.1 9 | dfind.1 10 | dreln.1 11 | drm.1 12 | dstripe.1 13 | dsync.1 14 | dtar.1 15 | dwalk.1 16 | ) 17 | 18 | INSTALL(FILES ${man_pages} DESTINATION ${X_DATADIR}/man/man1) 19 | -------------------------------------------------------------------------------- /src/dcp1/treewalk.h: -------------------------------------------------------------------------------- 1 | /* See the file "COPYING" for the full license governing this code. */ 2 | 3 | #ifndef __DCP_TREEWALK_H 4 | #define __DCP_TREEWALK_H 5 | 6 | #include "common.h" 7 | 8 | void DCOPY_do_treewalk(DCOPY_operation_t* op, 9 | CIRCLE_handle* handle); 10 | 11 | #endif /* __DCP_TREEWALK_H */ 12 | -------------------------------------------------------------------------------- /astyle.options: -------------------------------------------------------------------------------- 1 | --style=kr 2 | --indent-switches 3 | --indent-preprocessor 4 | --indent-col1-comments 5 | --pad-oper 6 | --pad-header 7 | --unpad-paren 8 | --break-closing-brackets 9 | --add-brackets 10 | --add-one-line-brackets 11 | --convert-tabs 12 | --align-pointer=type 13 | --mode=c 14 | --lineend=linux 15 | --break-one-line-headers 16 | -------------------------------------------------------------------------------- /src/dparallel/dparallel.h: -------------------------------------------------------------------------------- 1 | #ifndef _DPARALLEL_DPARALLEL_H 2 | #define _DPARALLEL_DPARALLEL_H 3 | 4 | #include 5 | 6 | #include "log.h" 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | char* DPARALLEL_readline(void); 13 | void DPARALLEL_process(CIRCLE_handle* handle); 14 | 15 | #endif /* _DPARALLEL_DPARALLEL_H */ 16 | -------------------------------------------------------------------------------- /src/dfind/common.h: -------------------------------------------------------------------------------- 1 | #ifndef _COMMON_H 2 | #define _COMMON_H 3 | 4 | struct { 5 | int maxdepth; 6 | char * root; 7 | } options; 8 | 9 | extern uint64_t now_secs; 10 | extern uint64_t now_usecs; 11 | 12 | #undef DEBUG 13 | 14 | #ifdef DEBUG 15 | #define dbprintf(...) do { fprintf(stderr, __VA_ARGS__); } while(0) 16 | #else 17 | #define dbprintf(...) do {} while (0) 18 | #endif 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /test/tests/test_dchmod/test_octal.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | import subprocess 3 | 4 | # chane your path here as necessary 5 | mpifu_path = "~/mpifileutils/test/tests/test_dchmod/test_octal.sh" 6 | dchmod_path = "~/mpifileutils/install/bin/dchmod" 7 | 8 | def test_octal(): 9 | p = subprocess.Popen(["%s %s" % (mpifu_path, dchmod_path)], shell=True, executable="/bin/bash").communicate() 10 | -------------------------------------------------------------------------------- /src/dcp1/handle_args.h: -------------------------------------------------------------------------------- 1 | /* See the file "COPYING" for the full license governing this code. */ 2 | 3 | #ifndef __DCOPY_HANDLE_ARGS_H 4 | #define __DCOPY_HANDLE_ARGS_H 5 | 6 | #include "common.h" 7 | 8 | void DCOPY_parse_path_args(char** argv, int optind, int argc); 9 | 10 | void DCOPY_free_path_args(void); 11 | 12 | void DCOPY_enqueue_work_objects(CIRCLE_handle* handle); 13 | 14 | #endif /* __DCOPY_HANDLE_ARGS_H */ 15 | -------------------------------------------------------------------------------- /test/legacy/dtar_tests/run.sh: -------------------------------------------------------------------------------- 1 | #mpirun -np 4 -machinefile ./machines hostname 2 | 3 | rm ../*.log 4 | rm test.tar 5 | 6 | exe="./dtar -c -f test.tar ../treewalk.c ../dtar.c ../copy.c ../README ./dir" 7 | 8 | #exe="./dtar -c -f test.tar ../treewalk.c ../dtar.c ../copy.c ../README " 9 | mpirun -np 5 -machinefile ./machines $exe 10 | 11 | #mpirun -np 4 -machinefile ./machines gdb ./dcp 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/common/mfu_bz2.h: -------------------------------------------------------------------------------- 1 | #ifndef MFU_BZ2_H 2 | #define MFU_BZ2_H 3 | 4 | int mfu_compress_bz2(const char* src_name, const char* dst_name, int b_size); 5 | int mfu_decompress_bz2(const char* src_name, const char* dst_name); 6 | 7 | /**************** 8 | * Private internal functions 9 | ***************/ 10 | 11 | #include "sys/types.h" 12 | 13 | int mfu_create_fully_striped(const char* name, mode_t mode); 14 | 15 | #endif /* MFU_BZ2_H */ 16 | -------------------------------------------------------------------------------- /src/common/mfu_proc.h: -------------------------------------------------------------------------------- 1 | #ifndef MFU_PROC_H 2 | #define MFU_PROC_H 3 | 4 | #include 5 | #include 6 | 7 | /* records properties about the current process */ 8 | typedef struct mfu_proc_struct { 9 | uid_t getuid; 10 | uid_t geteuid; 11 | bool cap_chown; 12 | bool cap_fowner; 13 | } mfu_proc_t; 14 | 15 | /* query and cache values for current process */ 16 | void mfu_proc_set(mfu_proc_t* proc); 17 | 18 | #endif /* MFU_PROC_H */ 19 | -------------------------------------------------------------------------------- /src/dcp1/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | INCLUDE_DIRECTORIES(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}) 2 | 3 | # MFU_ADD_TOOL(dcp1) 4 | 5 | LIST(APPEND dcp1_srcs 6 | dcp1.c 7 | cleanup.c 8 | common.c 9 | compare.c 10 | copy.c 11 | handle_args.c 12 | treewalk.c 13 | ) 14 | 15 | ADD_EXECUTABLE(dcp1 ${dcp1_srcs}) 16 | TARGET_LINK_LIBRARIES(dcp1 mfu) 17 | SET_TARGET_PROPERTIES(dcp1 PROPERTIES C_STANDARD 99) 18 | INSTALL(TARGETS dcp1 DESTINATION ${CMAKE_INSTALL_BINDIR}) 19 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Bringhurst, Jon 2 | Di Natale, Giuseppe 3 | Fuller, Douglas 4 | Gonsiorowski, Elsa 5 | Groff, Todd 6 | Gu, Zheng 7 | LaFon, Jharrod 8 | Li, Xi 9 | Moody, Adam 10 | Roy Choudhury, Ahana 11 | Sikich, Danielle 12 | Wang, Feiyi 13 | Wellington, Andrew 14 | -------------------------------------------------------------------------------- /test/tests/test_dsync/utility/test_driver.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | . set_funcs.sh 4 | 5 | #echo test union with wrong arg count 6 | #union one || echo expected failure for wrong arg count 7 | # 8 | #echo test union with non-files 9 | #union notafile1 notafile2 || echo expected failure for non-files 10 | 11 | 12 | echo union: $(union one two) 13 | echo intersection: $(intersection one two) 14 | echo sets_equal 1 and 2: $(sets_equal one two; echo $?) 15 | echo sets_equal 1 and 1: $(sets_equal one one; echo $?) 16 | -------------------------------------------------------------------------------- /doc/rst/dgrep.1.rst: -------------------------------------------------------------------------------- 1 | dgrep 2 | ===== 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | dgrep ... 8 | 9 | DESCRIPTION 10 | ----------- 11 | 12 | OPTIONS 13 | ------- 14 | 15 | .. option:: -h, --help 16 | 17 | Print a brief message listing the :manpage:`dgrep(1)` options and usage. 18 | 19 | .. option:: -v, --version 20 | 21 | Print version information and exit. 22 | 23 | Known bugs 24 | ~~~~~~~~~~ 25 | 26 | SEE ALSO 27 | -------- 28 | 29 | The mpiFileUtils source code and all documentation may be downloaded 30 | from 31 | -------------------------------------------------------------------------------- /doc/rst/dparallel.1.rst: -------------------------------------------------------------------------------- 1 | dparallel 2 | ========= 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | dparallel ... 8 | 9 | DESCRIPTION 10 | ----------- 11 | 12 | OPTIONS 13 | ------- 14 | 15 | .. option:: -h, --help 16 | 17 | Print a brief message listing the :manpage:`dparallel(1)` options and usage. 18 | 19 | .. option:: -v, --version 20 | 21 | Print version information and exit. 22 | 23 | Known bugs 24 | ~~~~~~~~~~ 25 | 26 | SEE ALSO 27 | -------- 28 | 29 | The mpiFileUtils source code and all documentation may be downloaded 30 | from 31 | -------------------------------------------------------------------------------- /test/tests/test_dsync/test_xattr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | import subprocess 3 | 4 | # change paths here for bash script as necessary 5 | mpifu_path = "~/mpifileutils/test/tests/test_dsync/test_xattr.sh" 6 | 7 | # vars in bash script 8 | dsync_test_bin = "/root/mpifileutils/install/bin/dsync" 9 | dsync_src_dir = "/mnt/lustre" 10 | dsync_dest_dir = "/mnt/lustre2" 11 | dsync_test_file = "file_test_xattr_XXX" 12 | 13 | def test_xattr(): 14 | p = subprocess.Popen(["%s %s %s %s %s" % (mpifu_path, dsync_test_bin, 15 | dsync_src_dir, dsync_dest_dir, dsync_test_file)], shell=True, 16 | executable="/bin/bash").communicate() 17 | -------------------------------------------------------------------------------- /cmake/FindLibCap.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find libcap 2 | # Once done this will define 3 | # LibCap_FOUND - System has libcap 4 | # LibCap_INCLUDE_DIRS - The libcap include directories 5 | # LibCap_LIBRARIES - The libraries needed to use libcap 6 | 7 | FIND_LIBRARY(LibCap_LIBRARIES 8 | NAMES cap 9 | ) 10 | 11 | FIND_PATH(LibCap_INCLUDE_DIRS 12 | NAMES sys/capability.h 13 | ) 14 | 15 | INCLUDE(FindPackageHandleStandardArgs) 16 | FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibCap DEFAULT_MSG 17 | LibCap_LIBRARIES 18 | LibCap_INCLUDE_DIRS 19 | ) 20 | 21 | # Hide these vars from ccmake GUI 22 | MARK_AS_ADVANCED( 23 | LibCap_LIBRARIES 24 | LibCap_INCLUDE_DIRS 25 | ) 26 | -------------------------------------------------------------------------------- /cmake/FindLibAttr.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find libattr 2 | # Once done this will define 3 | # LibAttr_FOUND - System has libattr 4 | # LibAttr_INCLUDE_DIRS - The libattr include directories 5 | # LibAttr_LIBRARIES - The libraries needed to use libattr 6 | 7 | FIND_LIBRARY(LibAttr_LIBRARIES 8 | NAMES attr 9 | ) 10 | 11 | FIND_PATH(LibAttr_INCLUDE_DIRS 12 | NAMES attr/libattr.h 13 | ) 14 | 15 | INCLUDE(FindPackageHandleStandardArgs) 16 | FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibAttr DEFAULT_MSG 17 | LibAttr_LIBRARIES 18 | LibAttr_INCLUDE_DIRS 19 | ) 20 | 21 | # Hide these vars from ccmake GUI 22 | MARK_AS_ADVANCED( 23 | LibAttr_LIBRARIES 24 | LibAttr_INCLUDE_DIRS 25 | ) 26 | -------------------------------------------------------------------------------- /test/tests/test_dcp/test_fiemap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | import subprocess 3 | 4 | # change paths here for bash script as necessary 5 | mpifu_path = "~/mpifileutils/test/tests/test_dcp/test_fiemap.sh" 6 | 7 | # vars in bash script 8 | dcp_test_bin = "/root/mpifileutils/install/bin/dcp" 9 | dcp_mpirun_bin = "mpirun" 10 | dcp_cmp_bin = "diff" 11 | dcp_src_dir = "/mnt/lustre" 12 | dcp_dest_dir = "/mnt/lustre2" 13 | dcmp_tmp_file = "file_test_fiemap_XXX" 14 | 15 | def test_fiemap(): 16 | p = subprocess.Popen(["%s %s %s %s %s %s %s" % (mpifu_path, dcp_test_bin, dcp_mpirun_bin, 17 | dcp_cmp_bin, dcp_src_dir, dcp_dest_dir, dcmp_tmp_file)], shell=True, executable="/bin/bash").communicate() 18 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = python -msphinx 7 | SPHINXPROJ = mpiFileUtils 8 | SOURCEDIR = rst 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | 22 | install: 23 | cp -f build/man/* ../man/ 24 | -------------------------------------------------------------------------------- /cmake/FindGPFS.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find libgpfs 2 | # Once done this will define 3 | # GPFS_FOUND - System has libgpfs 4 | # GPFS_INCLUDE_DIRS - The libgpfs include directories 5 | # GPFS_LIBRARIES - The libraries needed to use libgpfs 6 | 7 | FIND_PATH(WITH_GPFS_PREFIX 8 | NAMES include/gpfs.h 9 | ) 10 | 11 | FIND_LIBRARY(GPFS_LIBRARIES 12 | NAMES gpfs 13 | HINTS ${WITH_GPFS_PREFIX}/lib 14 | ) 15 | 16 | FIND_PATH(GPFS_INCLUDE_DIRS 17 | NAMES gpfs.h 18 | HINTS ${WITH_GPFS_PREFIX}/include 19 | ) 20 | 21 | INCLUDE(FindPackageHandleStandardArgs) 22 | FIND_PACKAGE_HANDLE_STANDARD_ARGS(GPFS DEFAULT_MSG 23 | GPFS_LIBRARIES 24 | GPFS_INCLUDE_DIRS 25 | ) 26 | 27 | # Hide these vars from ccmake GUI 28 | MARK_AS_ADVANCED( 29 | GPFS_LIBRARIES 30 | GPFS_INCLUDE_DIRS 31 | ) 32 | -------------------------------------------------------------------------------- /dist/README.dist: -------------------------------------------------------------------------------- 1 | ########################################### 2 | mpiFileUtils 3 | ########################################### 4 | 5 | This is a release package of mpiFileUtils. 6 | It contains source for mpiFileUtils and several of its dependencies, 7 | including LWGRP, DTCMP, and libcircle. 8 | All included source files are compiled into a single library. 9 | 10 | For documentation, refer to: 11 | 12 | https://mpifileutils.readthedocs.io 13 | 14 | For a simple build, one may run the following commands: 15 | 16 | mkdir build 17 | cd build 18 | cmake -DCMAKE_INSTALL_PREFIX=../install .. 19 | make -j install 20 | 21 | For details on available build flags, 22 | refer to the "Build" section of the documentation: 23 | 24 | https://mpifileutils.readthedocs.io/en/latest/build.html 25 | -------------------------------------------------------------------------------- /src/dgrep/log.h: -------------------------------------------------------------------------------- 1 | #ifndef LOG_H 2 | #define LOG_H 3 | 4 | #include 5 | 6 | typedef enum 7 | { 8 | DGREP_LOG_FATAL = 1, 9 | DGREP_LOG_ERR = 2, 10 | DGREP_LOG_WARN = 3, 11 | DGREP_LOG_INFO = 4, 12 | DGREP_LOG_DBG = 5 13 | } DGREP_loglevel; 14 | 15 | #define LOG(level, ...) do { \ 16 | if (level <= DGREP_debug_level) { \ 17 | fprintf(DGREP_debug_stream,"%d:%s:%d:", DGREP_global_rank, __FILE__, __LINE__); \ 18 | fprintf(DGREP_debug_stream, __VA_ARGS__); \ 19 | fprintf(DGREP_debug_stream, "\n"); \ 20 | fflush(DGREP_debug_stream); \ 21 | } \ 22 | } while (0) 23 | 24 | extern int DGREP_global_rank; 25 | extern FILE *DGREP_debug_stream; 26 | extern DGREP_loglevel DGREP_debug_level; 27 | 28 | #endif /* LOG_H */ 29 | -------------------------------------------------------------------------------- /cmake/FindDTCMP.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find libdtcmp 2 | # Once done this will define 3 | # DTCMP_FOUND - System has libdtcmp 4 | # DTCMP_INCLUDE_DIRS - The libdtcmp include directories 5 | # DTCMP_LIBRARIES - The libraries needed to use libdtcmp 6 | 7 | FIND_PATH(WITH_DTCMP_PREFIX 8 | NAMES include/dtcmp.h 9 | ) 10 | 11 | FIND_LIBRARY(DTCMP_LIBRARIES 12 | NAMES dtcmp 13 | HINTS ${WITH_DTCMP_PREFIX}/lib 14 | ) 15 | 16 | FIND_PATH(DTCMP_INCLUDE_DIRS 17 | NAMES dtcmp.h 18 | HINTS ${WITH_DTCMP_PREFIX}/include 19 | ) 20 | 21 | INCLUDE(FindPackageHandleStandardArgs) 22 | FIND_PACKAGE_HANDLE_STANDARD_ARGS(DTCMP DEFAULT_MSG 23 | DTCMP_LIBRARIES 24 | DTCMP_INCLUDE_DIRS 25 | ) 26 | 27 | # Hide these vars from ccmake GUI 28 | MARK_AS_ADVANCED( 29 | DTCMP_LIBRARIES 30 | DTCMP_INCLUDE_DIRS 31 | ) 32 | -------------------------------------------------------------------------------- /test/tests/test_dsync/utility/set_funcs.sh: -------------------------------------------------------------------------------- 1 | #set -x 2 | 3 | function union() { 4 | funcname=union 5 | if [[ $# -lt 2 ]] || [[ ! -f $1 ]] || [[ ! -f $2 ]]; then 6 | echo "$0: ${funcname}: requires 2 arguments, both files: $*" 7 | exit 8 | fi 9 | 10 | sort -u <(sort $1) <(sort $2) 11 | } 12 | 13 | function intersection() { 14 | funcname=intersection 15 | if [[ $# -lt 2 ]] || [[ ! -f $1 ]] || [[ ! -f $2 ]]; then 16 | echo "$0: ${funcname}: requires 2 arguments, both files: $*" 17 | exit 18 | fi 19 | 20 | comm -12 <(sort $1) <(sort $2) 21 | } 22 | 23 | function sets_equal() { 24 | funcname=sets_equal 25 | if [[ $# -lt 2 ]] || [[ ! -f $1 ]] || [[ ! -f $2 ]]; then 26 | echo "$0: ${funcname}: requires 2 arguments, both files: $*" 27 | exit 28 | fi 29 | 30 | diff -q <(sort $1) <(sort $2) >/dev/null 31 | } 32 | -------------------------------------------------------------------------------- /cmake/FindLibCircle.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find libcircle 2 | # Once done this will define 3 | # LibCircle_FOUND - System has libcircle 4 | # LibCircle_INCLUDE_DIRS - The libcircle include directories 5 | # LibCircle_LIBRARIES - The libraries needed to use libcircle 6 | 7 | FIND_PATH(WITH_LibCircle_PREFIX 8 | NAMES include/libcircle.h 9 | ) 10 | 11 | FIND_LIBRARY(LibCircle_LIBRARIES 12 | NAMES circle 13 | HINTS ${WITH_LibCircle_PREFIX}/lib 14 | ) 15 | 16 | FIND_PATH(LibCircle_INCLUDE_DIRS 17 | NAMES libcircle.h 18 | HINTS ${WITH_LibCircle_PREFIX}/include 19 | ) 20 | 21 | INCLUDE(FindPackageHandleStandardArgs) 22 | FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibCircle DEFAULT_MSG 23 | LibCircle_LIBRARIES 24 | LibCircle_INCLUDE_DIRS 25 | ) 26 | 27 | # Hide these vars from ccmake GUI 28 | MARK_AS_ADVANCED( 29 | LibCircle_LIBRARIES 30 | LibCircle_INCLUDE_DIRS 31 | ) 32 | -------------------------------------------------------------------------------- /cmake/FindLibArchive.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find libarchive 2 | # Once done this will define 3 | # LibArchive_FOUND - System has libarchive 4 | # LibArchive_INCLUDE_DIRS - The libarchive include directories 5 | # LibArchive_LIBRARIES - The libraries needed to use libarchive 6 | 7 | FIND_PATH(WITH_LibArchive_PREFIX 8 | NAMES include/libarchive.h 9 | ) 10 | 11 | FIND_LIBRARY(LibArchive_LIBRARIES 12 | NAMES archive 13 | HINTS ${WITH_LibArchive_PREFIX}/lib 14 | ) 15 | 16 | FIND_PATH(LibArchive_INCLUDE_DIRS 17 | NAMES archive.h 18 | HINTS ${WITH_LibArchive_PREFIX}/include 19 | ) 20 | 21 | INCLUDE(FindPackageHandleStandardArgs) 22 | FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibArchive DEFAULT_MSG 23 | LibArchive_LIBRARIES 24 | LibArchive_INCLUDE_DIRS 25 | ) 26 | 27 | # Hide these vars from ccmake GUI 28 | MARK_AS_ADVANCED( 29 | LibArchive_LIBRARIES 30 | LibArchive_INCLUDE_DIRS 31 | ) 32 | -------------------------------------------------------------------------------- /dist/README.md: -------------------------------------------------------------------------------- 1 | # mpiFileUtils release tarball 2 | The builddist script creates an mpiFileUtils release tarball. 3 | 4 | ```bash 5 | ./builddist main 6 | ``` 7 | 8 | This tarball is added as a binary attachment to the corresponding mpiFileUtils release page. 9 | This contains source for mpiFileUtils, LWGRP, DTCMP, and libcircle. 10 | It also contains a set of top-level CMake files that compiles all source files into a single libmfu library. 11 | 12 | # Steps to add a new release 13 | To add a new release: 14 | 1. Define new CMake files if needed (TODO: support multiple versions of top-level CMake files) 15 | 2. Edit builddist to define the appropriate tags for all packages 16 | 3. Run builddist for appropriate tag 17 | 4. Test build and run with resulting tarball 18 | 5. Attach tarball to github release page (attach binary) 19 | 6. Update mpiFileUtils readthedocs to describe builds from this tarball 20 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | include: [component: $CI_SERVER_FQDN/lc-components/id_tokens/id_tokens-component@main] 2 | # This workflow is written to work with the CI infrastructure at LLNL and 3 | # likely will not work at other sites that use gitlab. 4 | 5 | build-job: 6 | tags: 7 | - oslic 8 | - shell 9 | script: 10 | - echo "Starting build stage for mpifileutils" 11 | stage: build 12 | script: 13 | - pwd 14 | - module --force purge 15 | - module load openmpi-gnu 16 | - cd .. 17 | - ./mpifileutils/scripts/prepmfubuild.sh 18 | - rm -fr build 19 | - mkdir build 20 | - cd build 21 | - cmake ../mpifileutils -DWITH_DTCMP_PREFIX=../install \n 22 | -DWITH_LibCircle_PREFIX=../install \n 23 | -DWITH_LibArchive_PREFIX=../install \n 24 | -DCMAKE_INSTALL_PREFIX=../install 25 | - make -j install 26 | 27 | -------------------------------------------------------------------------------- /cmake/FindHDF5.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find hdf5 2 | # Once done this will define 3 | # HDF5_FOUND - System has hdf5 4 | # HDF5_INCLUDE_DIRS - The hdf5 include directories 5 | # HDF5_LIBRARIES - The libraries needed to use hdf5 6 | 7 | FIND_PATH(WITH_HDF5_PREFIX 8 | NAMES include/hdf5.h 9 | ) 10 | 11 | FIND_LIBRARY(HDF5_LIBRARIES 12 | NAMES hdf5 13 | HINTS ${WITH_HDF5_PREFIX}/lib 14 | ) 15 | 16 | FIND_PATH(HDF5_INCLUDE_DIRS 17 | NAMES hdf5.h 18 | HINTS ${WITH_HDF5_PREFIX}/include 19 | ) 20 | 21 | FIND_PATH(HDF5_BIN_DIR 22 | NAMES h5cc h5pcc 23 | HINTS ${WITH_HDF5_PREFIX}/bin 24 | ) 25 | 26 | INCLUDE(FindPackageHandleStandardArgs) 27 | FIND_PACKAGE_HANDLE_STANDARD_ARGS(HDF5 DEFAULT_MSG 28 | HDF5_LIBRARIES 29 | HDF5_INCLUDE_DIRS 30 | HDF5_BIN_DIR 31 | ) 32 | 33 | # Hide these vars from ccmake GUI 34 | MARK_AS_ADVANCED( 35 | HDF5_LIBRARIES 36 | HDF5_INCLUDE_DIRS 37 | HDF5_BIN_DIR 38 | ) 39 | -------------------------------------------------------------------------------- /test/tests/test_dsync/test_metadata.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ############################################################################## 4 | # Description: 5 | # 6 | # Verify dsync handles metdata 7 | # - file with differing metadata is copied 8 | # - file metadata is identical after a copy 9 | # - directory metadata is identical after a sync 10 | # 11 | # Notes: 12 | # 13 | ############################################################################## 14 | 15 | # Turn on verbose output 16 | #set -x 17 | 18 | MFU_TEST_BIN=${MFU_TEST_BIN:-${1}} 19 | DSYNC_SRC_DIR=${DSYNC_SRC_DIR:-${2}} 20 | DSYNC_DEST_DIR=${DSYNC_DEST_DIR:-${3}} 21 | DSYNC_TMP_FILE=${DSYNC_TMP_FILE:-${4}} 22 | 23 | echo "Using MFU binaries at: $MFU_TEST_BIN" 24 | echo "Using src directory at: $DSYNC_SRC_DIR" 25 | echo "Using dest directory at: $DSYNC_DEST_DIR" 26 | echo "Using directory tree: $DSYNC_TMP_FILE" 27 | 28 | # tests not yet implemented 29 | 30 | exit 0 31 | -------------------------------------------------------------------------------- /mpifileutils.spec: -------------------------------------------------------------------------------- 1 | Name: mpifileutils 2 | Version: 0.12 3 | Release: 1%{?dist} 4 | Summary: File utilities designed for scalability and performance. 5 | 6 | Group: System Environment/Libraries 7 | License: Copyright and BSD License 8 | URL: https://hpc.github.io/mpifileutils 9 | Source: %{name}-%{version}.tar.gz 10 | BuildRoot: %_topdir/BUILDROOT 11 | Requires: libcircle, lwgrp, dtcmp, libarchive, openssl, openssl-devel 12 | 13 | %description 14 | File utilities designed for scalability and performance. 15 | 16 | %prep 17 | %setup -q 18 | 19 | %build 20 | %{cmake} ./ -DWITH_DTCMP_PREFIX=${installdir} -DWITH_LibCircle_PREFIX=${installdir} -DCMAKE_INSTALL_PREFIX=%{buildroot} -DENABLE_LUSTRE=ON -DENABLE_XATTRS=ON 21 | %{cmake_build} 22 | 23 | %install 24 | rm -rf %{buildroot} 25 | %{cmake_install} 26 | 27 | 28 | %files 29 | %defattr(-,root,root,-) 30 | %{_bindir}/* 31 | %{_includedir}/* 32 | %{_libdir}/* 33 | %{_mandir}/* 34 | 35 | %changelog 36 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Common library 2 | ADD_SUBDIRECTORY(common) 3 | 4 | # tools 5 | ADD_SUBDIRECTORY(dbcast) 6 | ADD_SUBDIRECTORY(dbz2) 7 | ADD_SUBDIRECTORY(dchmod) 8 | ADD_SUBDIRECTORY(dcmp) 9 | ADD_SUBDIRECTORY(dcp) 10 | ADD_SUBDIRECTORY(dcp1) 11 | ADD_SUBDIRECTORY(ddup) 12 | ADD_SUBDIRECTORY(dfilemaker) 13 | ADD_SUBDIRECTORY(dfind) 14 | ADD_SUBDIRECTORY(dreln) 15 | ADD_SUBDIRECTORY(drm) 16 | ADD_SUBDIRECTORY(dstripe) 17 | ADD_SUBDIRECTORY(dsync) 18 | IF(ENABLE_DAOS AND ENABLE_HDF5) 19 | ADD_SUBDIRECTORY(daos-serialize) 20 | ADD_SUBDIRECTORY(daos-deserialize) 21 | ENDIF(ENABLE_DAOS AND ENABLE_HDF5) 22 | IF(ENABLE_LIBARCHIVE) 23 | ADD_SUBDIRECTORY(dtar) 24 | ENDIF(ENABLE_LIBARCHIVE) 25 | ADD_SUBDIRECTORY(dwalk) 26 | 27 | # experimental tools 28 | IF(ENABLE_EXPERIMENTAL) 29 | IF(ENABLE_DAOS) 30 | ADD_SUBDIRECTORY(daos-gen) 31 | ENDIF(ENABLE_DAOS) 32 | ADD_SUBDIRECTORY(dgrep) 33 | ADD_SUBDIRECTORY(dparallel) 34 | ADD_SUBDIRECTORY(dsh) 35 | ENDIF(ENABLE_EXPERIMENTAL) 36 | -------------------------------------------------------------------------------- /src/common/mfu.h: -------------------------------------------------------------------------------- 1 | /* enable C++ codes to include this header directly */ 2 | #ifdef __cplusplus 3 | extern "C" { 4 | #endif 5 | 6 | #ifndef MFU_H 7 | #define MFU_H 8 | 9 | #define MFU_SUCCESS (0) 10 | #define MFU_FAILURE (1) 11 | 12 | /* TODO: ugly hack until we get a configure test */ 13 | // HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC 14 | #define HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC 1 15 | // HAVE_STRUCT_STAT_ST_MTIME_N 16 | // HAVE_STRUCT_STAT_ST_UMTIME 17 | // HAVE_STRUCT_STAT_ST_MTIME_USEC 18 | 19 | #ifndef _XOPEN_SOURCE 20 | #define _XOPEN_SOURCE 500 21 | #endif 22 | #include 23 | 24 | #include "mfu_util.h" 25 | #include "mfu_path.h" 26 | #include "mfu_io.h" 27 | #include "mfu_param_path.h" 28 | #include "mfu_flist.h" 29 | #include "mfu_pred.h" 30 | #include "mfu_proc.h" 31 | #include "mfu_progress.h" 32 | #include "mfu_bz2.h" 33 | 34 | #endif /* MFU_H */ 35 | 36 | /* enable C++ codes to include this header directly */ 37 | #ifdef __cplusplus 38 | } /* extern "C" */ 39 | #endif 40 | -------------------------------------------------------------------------------- /src/common/mfu_errors.h: -------------------------------------------------------------------------------- 1 | /* Defines common error codes */ 2 | 3 | /* enable C++ codes to include this header directly */ 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #ifndef MFU_ERRORS_H 9 | #define MFU_ERRORS_H 10 | 11 | #include 12 | 13 | /* Given a system error code, set errno and return -1 on error */ 14 | static inline int mfu_errno2rc(int err) 15 | { 16 | if (err == 0) { 17 | return 0; 18 | } 19 | errno = err; 20 | return -1; 21 | } 22 | 23 | /* Generic error codes */ 24 | #define MFU_ERR 1000 25 | #define MFU_ERR_INVAL_ARG 1001 26 | 27 | /* DCP-specific error codes */ 28 | #define MFU_ERR_DCP 1100 29 | #define MFU_ERR_DCP_COPY 1101 30 | 31 | /* DAOS-specific error codes*/ 32 | #define MFU_ERR_DAOS 4000 33 | #define MFU_ERR_DAOS_INVAL_ARG 4001 34 | 35 | /* Error macros */ 36 | #define MFU_ERRF "%s(%d)" 37 | #define MFU_ERRP(rc) "MFU_ERR", rc 38 | 39 | #endif /* MFU_ERRORS_H */ 40 | 41 | /* enable C++ codes to include this header directly */ 42 | #ifdef __cplusplus 43 | } /* extern "C" */ 44 | #endif 45 | -------------------------------------------------------------------------------- /mpifu_venv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script assumes you have at least python-2.7.10 installed 4 | # Pass in the top level directory of where you have the project downloaded 5 | MPIFU_DIR=$1 6 | echo $MPIFU_DIR 7 | 8 | CONDA_BIN=${MPIFU_DIR}/miniconda2/bin 9 | ACTIVATE_VENV=${MPIFU_DIR}/miniconda2/envs/mpifu_venv 10 | CONDA_INSTALL=${MPIFU_DIR}/miniconda2 11 | 12 | # get the installer for miniconda 13 | wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh 14 | chmod +x Miniconda2-latest-Linux-x86_64.sh 15 | 16 | # run the install 17 | ./Miniconda2-latest-Linux-x86_64.sh -b -p ${CONDA_INSTALL} 18 | 19 | # create and activate the mpifu virtual env 20 | ${CONDA_BIN}/conda create -n mpifu_venv 21 | source activate $ACTIVATE_VENV 22 | 23 | # now install pypandoc (which has pandoc as well) 24 | # you can use either pypandoc or pandoc to create 25 | # the man pages from the markdown format 26 | ${CONDA_BIN}/conda config --add channels conda-forge 27 | ${CONDA_BIN}/conda install pypandoc 28 | 29 | # install nose2 via pip 30 | ${CONDA_BIN}/pip install --trusted-host pypi.python.org nose2 31 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | # Documentation 2 | 3 | Edits should be made to the `.rst` files. 4 | The documentation can be built with `make html` or `make man`. 5 | The generated files will be found in the `build` directory. 6 | 7 | ## Installing Sphinx 8 | 9 | Sphinx is used to generate man pages from the `.rst` files. 10 | If Sphinx is not installed on the system, the following may be used to install Sphinx into a virtualenv. 11 | 12 | ``` shell 13 | virtualenv --system-site-packages env 14 | source env/bin/activate 15 | 16 | pip install recommonmark 17 | pip install guzzle_sphinx_theme 18 | ``` 19 | 20 | If using this method, then source the virtualenv before running the Makefile steps below: 21 | 22 | ``` shell 23 | source env/bin/activate 24 | ``` 25 | 26 | ## Man Pages 27 | 28 | Since there is no guarantee of Sphinx on each system, the man pages for each release are committed directly to the repo. 29 | This can be done with: 30 | 31 | Be sure to update the version number in the `doc/rst/conf.py` file if needed, 32 | then run Sphinx to generate the man pages from the `.rst` files. 33 | 34 | ``` shell 35 | cd doc 36 | make man 37 | make install 38 | ``` 39 | 40 | -------------------------------------------------------------------------------- /src/dparallel/log.h: -------------------------------------------------------------------------------- 1 | #ifndef _DPARALLEL_LOG_H 2 | #define _DPARALLEL_LOG_H 3 | 4 | #include "dparallel.h" 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | typedef enum DPARALLEL_loglevel { DPARALLEL_LOG_FATAL = 1, 11 | DPARALLEL_LOG_ERR = 2, 12 | DPARALLEL_LOG_WARN = 3, 13 | DPARALLEL_LOG_INFO = 4, 14 | DPARALLEL_LOG_DBG = 5 15 | } DPARALLEL_loglevel; 16 | 17 | #define LOG(level, ...) do { \ 18 | if (level <= DPARALLEL_debug_level) { \ 19 | fprintf(DPARALLEL_debug_stream, "%d:%d:%s:%d:", (int)time(NULL), \ 20 | DPARALLEL_global_rank, __FILE__, __LINE__); \ 21 | fprintf(DPARALLEL_debug_stream, __VA_ARGS__); \ 22 | fprintf(DPARALLEL_debug_stream, "\n"); \ 23 | fflush(DPARALLEL_debug_stream); \ 24 | } \ 25 | } while (0) 26 | 27 | extern FILE* DPARALLEL_debug_stream; 28 | extern enum DPARALLEL_loglevel DPARALLEL_debug_level; 29 | extern int32_t DPARALLEL_global_rank; 30 | 31 | #endif /* _DPARALLEL_LOG_H */ 32 | -------------------------------------------------------------------------------- /doc/.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file for Sphinx projects 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Set the OS, Python version and other tools you might need 8 | build: 9 | os: ubuntu-22.04 10 | tools: 11 | python: "3.12" 12 | # You can also specify other tool versions: 13 | # nodejs: "20" 14 | # rust: "1.70" 15 | # golang: "1.20" 16 | 17 | # Build documentation in the "docs/" directory with Sphinx 18 | sphinx: 19 | configuration: doc/rst/conf.py 20 | # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs 21 | # builder: "dirhtml" 22 | # Fail on all warnings to avoid broken references 23 | # fail_on_warning: true 24 | 25 | # Optionally build your docs in additional formats such as PDF and ePub 26 | # formats: 27 | # - pdf 28 | # - epub 29 | 30 | # Optional but recommended, declare the Python requirements required 31 | # to build your documentation 32 | # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 33 | # python: 34 | # install: 35 | # - requirements: docs/requirements.txt 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Object files 2 | *.o 3 | 4 | # Libraries 5 | *.lib 6 | *.a 7 | *.la 8 | *.lo 9 | 10 | # Shared objects (inc. Windows DLLs) 11 | *.dll 12 | *.so 13 | *.so.* 14 | *.dylib 15 | 16 | # Executables 17 | *.exe 18 | *.out 19 | *.app 20 | src/dcmp/dcmp 21 | src/dcp/dcp 22 | src/dcp2/dcp2 23 | src/dfilemaker/dfilemaker 24 | src/dsh/dsh 25 | src/dwalk/dwalk 26 | 27 | .pc 28 | deps 29 | .deps 30 | install 31 | autotools 32 | 33 | # Autotool artifacts 34 | /autom4te.cache 35 | Makefile 36 | config.h.in~ 37 | config.h.in 38 | config.h 39 | config.log 40 | config.status 41 | /stamp-h1 42 | auxdir/install-sh 43 | auxdir/config.guess 44 | auxdir/config.sub 45 | auxdir/ltmain.sh 46 | config.h.in 47 | config.h.in~ 48 | m4/libtool.m4 49 | m4/ltoptions.m4 50 | m4/ltsugar.m4 51 | m4/ltversion.m4 52 | m4/lt~obsolete.m4 53 | 54 | aclocal.m4 55 | auxdir/depcomp 56 | auxdir/missing 57 | configure 58 | 59 | Makefile.in 60 | */Makefile.in 61 | */*/Makefile.in 62 | */*/*/Makefile.in 63 | 64 | # Vagrant/Ansible artifacts 65 | /devops/vagrant_ansible_inventory_* 66 | /devops/.vagrant 67 | 68 | # documentation build 69 | doc/build 70 | 71 | # spack files 72 | .spack* 73 | spack.lock 74 | 75 | # Code editors 76 | .vscode 77 | 78 | # source indexing/tagging 79 | tags 80 | -------------------------------------------------------------------------------- /doc/rst/daos-deserialize.1.rst: -------------------------------------------------------------------------------- 1 | daos-deserialize 2 | === 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | **daos-deserialize [OPTION] [ ...] || []** 8 | 9 | DESCRIPTION 10 | ----------- 11 | 12 | Parallel MPI application to deserialize HDF5 files into a DAOS container. 13 | 14 | daos-deserialize is a deserialization tool that will restore DAOS data written 15 | to an HDF5 file with daos-serialize. The tool will restore the data in the 16 | the HDF5 file into a DAOS container. 17 | 18 | OPTIONS 19 | ------- 20 | .. option:: --pool UUID 21 | 22 | Specify the pool where the restored DAOS container will be created 23 | 24 | .. option:: -v, --verbose 25 | 26 | Run in verbose mode. 27 | 28 | .. option:: -q, --quiet 29 | 30 | Run tool silently. No output is printed. 31 | 32 | .. option:: -h, --help 33 | 34 | Print a brief message listing the :manpage:`daos-deserialize(1)` options and usage. 35 | 36 | EXAMPLES 37 | -------- 38 | 39 | 1. To deserialize a DAOS container by specifying individual files: 40 | 41 | ``mpirun -np 128 daos-deserialize `` 42 | 43 | 2. To deserialize a DAOS container by specifying a directory with HDF5 files: 44 | 45 | ``mpirun -np 128 daos-deserialize /path/to/dir`` 46 | 47 | SEE ALSO 48 | -------- 49 | 50 | The mpiFileUtils source code and all documentation may be downloaded 51 | from 52 | -------------------------------------------------------------------------------- /test/legacy/README.md: -------------------------------------------------------------------------------- 1 | DTEST is a testing application that is used for benchmarking, and validating the parallel unix utilities. there is currently support for dtar and dcp, but more will be added. 2 | 3 | 4 | ### If running on titan make sure to execute '$> module load python' in order to load a new enough version of python for argparse to work. ### 5 | 6 | 7 | USAGE: 8 | 9 | ./dtest [test directory] 10 | 11 | -s --size total tree size to test DEFAULT: 1G 12 | -np --proc number of process to use with dtar DEFAULT: 3 13 | -c --count number of files to test DEFAULT: 10 14 | -bs --block block size for writing files. DEFAULT: 256M 15 | -i --iterations iterations of dtar to run to average results DEFAULT: 3 16 | -v --verbose verbose mode 17 | 18 | --genfiles this flag causes dtest to only generate a file tree without doing testing. 19 | use the same parameters that you would use to run a test, except you do not 20 | have to specify a binary to test. (e.g.) ./dtest --genfiles -s 10G -c 100 test-dir 21 | 22 | --validate run a validation test to check integrity of the output files vs. th input files. 23 | 24 | --debug shows the output of the binary subprocess, and doesn't clean up files afterwords.' 25 | 26 | --noclean prevents the program from cleaning up test files on completion 27 | 28 | --sparse write sparse files to test instead of zero files. (much faster) 29 | 30 | 31 | -------------------------------------------------------------------------------- /test/legacy/dtest/README.md: -------------------------------------------------------------------------------- 1 | DTEST is a testing application that is used for benchmarking, and validating the parallel unix utilities. there is currently support for dtar and dcp, but more will be added. 2 | 3 | 4 | ### If running on titan make sure to execute '$> module load python' in order to load a new enough version of python for argparse to work. ### 5 | 6 | 7 | USAGE: 8 | 9 | ./dtest [test directory] 10 | 11 | -s --size total tree size to test DEFAULT: 1G 12 | -np --proc number of process to use with dtar DEFAULT: 3 13 | -c --count number of files to test DEFAULT: 10 14 | -bs --block block size for writing files. DEFAULT: 256M 15 | -i --iterations iterations of dtar to run to average results DEFAULT: 3 16 | -v --verbose verbose mode 17 | 18 | --genfiles this flag causes dtest to only generate a file tree without doing testing. 19 | use the same parameters that you would use to run a test, except you do not 20 | have to specify a binary to test. (e.g.) ./dtest --genfiles -s 10G -c 100 test-dir 21 | 22 | --validate run a validation test to check integrity of the output files vs. th input files. 23 | 24 | --debug shows the output of the binary subprocess, and doesn't clean up files afterwords.' 25 | 26 | --noclean prevents the program from cleaning up test files on completion 27 | 28 | --sparse write sparse files to test instead of zero files. (much faster) 29 | 30 | 31 | -------------------------------------------------------------------------------- /doc/rst/ddup.1.rst: -------------------------------------------------------------------------------- 1 | ddup 2 | ======= 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | **ddup [OPTION] PATH** 8 | 9 | DESCRIPTION 10 | ----------- 11 | 12 | Parallel MPI application to report files under a directory tree having identical content. 13 | 14 | ddup reports path names to files having identical content (duplicate files). 15 | A top-level directory is specified, and the path name to any file that is a duplicate 16 | of another anywhere under that same directory tree is reported. 17 | The path to each file is reported, along with a final hash representing its content. 18 | Multiple sets of duplicate files can be matched using this final reported hash. 19 | 20 | OPTIONS 21 | ------- 22 | 23 | .. option:: --open-noatime 24 | 25 | Open files with O_NOATIME flag, if possible. 26 | 27 | .. option:: -d, --debug LEVEL 28 | 29 | Set verbosity level. LEVEL can be one of: fatal, err, warn, info, dbg. 30 | 31 | .. option:: -v, --verbose 32 | 33 | Run in verbose mode. 34 | 35 | .. option:: -q, --quiet 36 | 37 | Run tool silently. No output is printed. 38 | 39 | .. option:: -h, --help 40 | 41 | Print the command usage, and the list of options available. 42 | 43 | EXAMPLES 44 | -------- 45 | 46 | 1. To report any duplicate files under a directory tree: 47 | 48 | ``mpirun -np 128 ddup /path/to/haystack`` 49 | 50 | SEE ALSO 51 | -------- 52 | 53 | The mpiFileUtils source code and all documentation may be downloaded 54 | from 55 | -------------------------------------------------------------------------------- /test/legacy/dtar_tests/dir/log.h: -------------------------------------------------------------------------------- 1 | #ifndef LOG_H 2 | #define LOG_H 3 | 4 | #include 5 | #include 6 | 7 | typedef enum { 8 | DTAR_LOG_FATAL = 1, 9 | DTAR_LOG_ERR = 2, 10 | DTAR_LOG_WARN = 3, 11 | DTAR_LOG_INFO = 4, 12 | DTAR_LOG_DBG = 5 13 | } DTAR_loglevel; 14 | 15 | // fprintf(DTAR_debug_stream,"[%s] ", timestamp); 16 | #define LOG(level, ...) do { \ 17 | if (level <= DTAR_debug_level) { \ 18 | char timestamp[256]; \ 19 | time_t ltime = time(NULL); \ 20 | struct tm *ttime = localtime(<ime); \ 21 | strftime(timestamp, sizeof(timestamp), \ 22 | "%Y-%m-%dT%H:%M:%S", ttime); \ 23 | if(level == DTAR_LOG_DBG) { \ 24 | fprintf(DTAR_debug_stream,"[%s] [%d] [%s:%d] ", \ 25 | timestamp, CIRCLE_global_rank, \ 26 | __FILE__, __LINE__); \ 27 | } else { \ 28 | fprintf(DTAR_debug_stream,"[%s] [%d] [%s:%d] ", \ 29 | timestamp, CIRCLE_global_rank, \ 30 | __FILE__, __LINE__); \ 31 | } \ 32 | fprintf(DTAR_debug_stream, __VA_ARGS__); \ 33 | fprintf(DTAR_debug_stream, "\n"); \ 34 | fflush(DTAR_debug_stream); \ 35 | } \ 36 | } while (0) 37 | 38 | extern int CIRCLE_global_rank; 39 | extern FILE* DTAR_debug_stream; 40 | extern DTAR_loglevel DTAR_debug_level; 41 | 42 | #endif /* LOG_H */ 43 | -------------------------------------------------------------------------------- /doc/rst/daos-serialize.1.rst: -------------------------------------------------------------------------------- 1 | daos-serialize 2 | === 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | **daos-serialize [OPTION] //** 8 | 9 | DESCRIPTION 10 | ----------- 11 | 12 | Parallel MPI application to serialize a DAOS container to an HDF5 file. 13 | 14 | daos-serialize is a serialization tool that will allow storing any type 15 | of DAOS container on a POSIX filesystem. It allows DAOS container data to 16 | be stored outside of the DAOS system. The DAOS container is converted 17 | to a set of files in HDF5 format. The serialization format of the container is meant to 18 | be deserialized by daos-deserialize. 19 | 20 | OPTIONS 21 | ------- 22 | .. option:: -o, --output-path PATH 23 | 24 | Write the HDF5 files generated during serialization to the specified 25 | output path. 26 | 27 | .. option:: -v, --verbose 28 | 29 | Run in verbose mode. 30 | 31 | .. option:: -q, --quiet 32 | 33 | Run tool silently. No output is printed. 34 | 35 | .. option:: -h, --help 36 | 37 | Print a brief message listing the :manpage:`daos-serialize(1)` options and usage. 38 | 39 | EXAMPLES 40 | -------- 41 | 42 | 1. To serialize a DAOS container: 43 | 44 | ``mpirun -np 128 daos-serialize //`` 45 | 46 | 2. To serialize a DAOS container to a specific directory: 47 | 48 | ``mpirun -np 128 daos-serialize -o /path/to/output/dir //`` 49 | 50 | SEE ALSO 51 | -------- 52 | 53 | The mpiFileUtils source code and all documentation may be downloaded 54 | from 55 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | This work was produced under the auspices of the U.S. Department of Energy by 2 | - Lawrence Livermore National Laboratory under Contract DE-AC52-07NA27344, 3 | - Los Alamos National Laboratory under Contract DE-AC52-06NA25396, 4 | - UT-Battelle, LLC under Contract DE-AC-00OR22725 with the DOE. 5 | 6 | This work was prepared as an account of work sponsored by an agency of the 7 | United States Government. Neither the United States Government nor Lawrence 8 | Livermore National Security, LLC, nor Los Alamos National Security, LLC, nor 9 | UT-Battelle, LLC, nor any of their employees makes any warranty, 10 | expressed or implied, or assumes any legal liability or responsibility for the 11 | accuracy, completeness, or usefulness of any information, apparatus, product, or 12 | process disclosed, or represents that its use would not infringe privately owned 13 | rights. Reference herein to any specific commercial product, process, or service 14 | by trade name, trademark, manufacturer, or otherwise does not necessarily 15 | constitute or imply its endorsement, recommendation, or favoring by the United 16 | States Government, Lawrence Livermore National Security, LLC, Los Alamos Security, 17 | LLC, UT-Battelle, LLC, or DataDirect Networks, Inc. The views and 18 | opinions of authors expressed herein do not necessarily state or reflect those 19 | of the United States Government, Lawrence Livermore National Security, LLC, Los 20 | Alamos Security, LLC, UT-Battelle, LLC, or DataDirect Networks, Inc. 21 | and shall not be used for advertising or product endorsement purposes. 22 | -------------------------------------------------------------------------------- /doc/rst/dbz2.1.rst: -------------------------------------------------------------------------------- 1 | dbz2 2 | ==== 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | **dbz2 [OPTIONS] [-z|-d] FILE** 8 | 9 | DESCRIPTION 10 | ----------- 11 | 12 | Parallel MPI application to compress or decompress a file. 13 | 14 | When compressing, a new file will be created with a .dbz2 extension. 15 | When decompressing, the .dbz2 extension will be dropped from the file name. 16 | 17 | OPTIONS 18 | ------- 19 | 20 | .. option:: -z, --compress 21 | 22 | Compress the file 23 | 24 | .. option:: -d, --decompress 25 | 26 | Decompress the file 27 | 28 | .. option:: -k, --keep 29 | 30 | Keep the input file. 31 | 32 | .. option:: -f, --force 33 | 34 | Overwrite the output file, if it exists. 35 | 36 | .. option:: -b, --blocksize SIZE 37 | 38 | Set the compression block size, from 1 to 9. 39 | Where 1=100kB ... and 9=900kB. Default is 9. 40 | 41 | .. option:: -v, --verbose 42 | 43 | Verbose output (optional). 44 | 45 | .. option:: -q, --quiet 46 | 47 | Quiet output 48 | 49 | .. option:: -h, --help 50 | 51 | Print usage. 52 | 53 | EXAMPLES 54 | -------- 55 | 56 | 1. To compress a file: 57 | 58 | ``mpirun -np 128 dbz2 --compress /path/to/file`` 59 | 60 | 2. To compress a file and overwrite any existing output file: 61 | 62 | ``mpirun -np 128 dbz2 --force --compress /path/to/file`` 63 | 64 | 3. To decompress a file: 65 | 66 | ``mpirun -np 128 dbz2 --decompress /path/to/file.dbz2`` 67 | 68 | SEE ALSO 69 | -------- 70 | 71 | The mpiFileUtils source code and all documentation may be downloaded 72 | from 73 | -------------------------------------------------------------------------------- /src/dparallel/dparallel.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "dparallel.h" 4 | 5 | /** The debug stream for all logging messages. */ 6 | FILE* DPARALLEL_debug_stream; 7 | 8 | /** The current log level of library logging output. */ 9 | enum DPARALLEL_loglevel DPARALLEL_debug_level; 10 | 11 | /** The rank value of the current node. */ 12 | int32_t DPARALLEL_global_rank; 13 | 14 | char* DPARALLEL_readline() 15 | { 16 | char* buf = (char*) malloc(sizeof(char) * CIRCLE_MAX_STRING_LEN); 17 | 18 | if(fgets(buf, CIRCLE_MAX_STRING_LEN, stdin) != NULL) { 19 | return buf; 20 | } 21 | else { 22 | free(buf); 23 | return 0; 24 | } 25 | } 26 | 27 | void DPARALLEL_process(CIRCLE_handle* handle) 28 | { 29 | if(DPARALLEL_global_rank == 0) { 30 | char* new_cmd = DPARALLEL_readline(); 31 | 32 | if(new_cmd != 0) { 33 | LOG(DPARALLEL_LOG_DBG, "Enqueueing command `%s'.", new_cmd); 34 | handle->enqueue(new_cmd); 35 | free(new_cmd); 36 | return; 37 | } 38 | } 39 | 40 | char cmd[CIRCLE_MAX_STRING_LEN]; 41 | handle->dequeue(cmd); 42 | int ret = system(cmd); 43 | 44 | LOG(DPARALLEL_LOG_DBG, "Command `%s' returned `%d'.", cmd, ret); 45 | 46 | return; 47 | } 48 | 49 | int main(void) 50 | { 51 | DPARALLEL_debug_level = DPARALLEL_LOG_DBG; 52 | DPARALLEL_debug_stream = stderr; 53 | 54 | CIRCLE_init(0, NULL, CIRCLE_DEFAULT_FLAGS); 55 | CIRCLE_cb_process(&DPARALLEL_process); 56 | 57 | CIRCLE_begin(); 58 | CIRCLE_finalize(); 59 | } 60 | 61 | /* EOF */ 62 | -------------------------------------------------------------------------------- /scripts/prepmfubuild.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | mkdir install 3 | installdir=`pwd`/install 4 | 5 | mkdir deps 6 | cd deps 7 | 8 | urls=( https://github.com/hpc/libcircle/releases/download/v0.3/libcircle-0.3.0.tar.gz 9 | https://github.com/llnl/lwgrp/releases/download/v1.0.6/lwgrp-1.0.6.tar.gz 10 | https://github.com/llnl/dtcmp/releases/download/v1.1.5/dtcmp-1.1.5.tar.gz 11 | https://github.com/libarchive/libarchive/releases/download/v3.7.7/libarchive-3.7.7.tar.gz 12 | ) 13 | 14 | rc=0 15 | for url in ${urls[*]}; do 16 | if [[ rc -eq 0 ]]; then 17 | wget $url 18 | rc=$? 19 | if [[ $rc -ne 0 ]]; then 20 | echo 21 | echo FAILED getting $url 22 | echo check for releases under $(echo $url | sed 's/releases.*/releases\//') 23 | fi 24 | fi 25 | done 26 | 27 | if [[ rc -eq 0 ]]; then 28 | tar -zxf libcircle-0.3.0.tar.gz 29 | cd libcircle-0.3.0 30 | ./configure --prefix=$installdir 31 | make install 32 | cd .. 33 | 34 | tar -zxf lwgrp-1.0.6.tar.gz 35 | cd lwgrp-1.0.6 36 | ./configure --prefix=$installdir 37 | make install 38 | cd .. 39 | 40 | tar -zxf dtcmp-1.1.5.tar.gz 41 | cd dtcmp-1.1.5 42 | ./configure --prefix=$installdir --with-lwgrp=$installdir 43 | make install 44 | cd .. 45 | 46 | tar -zxf libarchive-3.7.7.tar.gz 47 | cd libarchive-3.7.7 48 | ./configure --prefix=$installdir 49 | make install 50 | cd .. 51 | fi 52 | cd .. 53 | 54 | -------------------------------------------------------------------------------- /cmake/FindDAOS.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find daos libs 2 | # Once done this will define 3 | # daos_FOUND - System has libdaos 4 | # daos_INCLUDE_DIRS - daos.h 5 | # daos_LIBRARIES - libdaos 6 | # GURT_LIBRARIES - libgurt 7 | # CART_LIBRARIES - libcart 8 | # DUNS_LIBRARIES - libduns 9 | # DFS_LIBRARIES - libdfs 10 | # COMMON_LIBRARIES - libdaos_common 11 | 12 | FIND_PATH(WITH_DAOS_PREFIX 13 | NAMES include/daos.h 14 | ) 15 | 16 | FIND_LIBRARY(DAOS_LIBRARIES 17 | NAMES daos 18 | HINTS ${WITH_DAOS_PREFIX}/lib 19 | ) 20 | 21 | FIND_LIBRARY(GURT_LIBRARIES 22 | NAMES gurt 23 | HINTS ${WITH_DAOS_PREFIX}/lib 24 | ) 25 | 26 | FIND_LIBRARY(CART_LIBRARIES 27 | NAMES cart 28 | HINTS ${WITH_DAOS_PREFIX}/lib 29 | ) 30 | 31 | FIND_LIBRARY(DUNS_LIBRARIES 32 | NAMES duns 33 | HINTS ${WITH_DAOS_PREFIX}/lib 34 | ) 35 | 36 | FIND_LIBRARY(DFS_LIBRARIES 37 | NAMES dfs 38 | HINTS ${WITH_DAOS_PREFIX}/lib 39 | ) 40 | 41 | FIND_LIBRARY(COMMON_LIBRARIES 42 | NAMES daos_common 43 | HINTS ${WITH_DAOS_PREFIX}/lib 44 | ) 45 | 46 | FIND_PATH(DAOS_INCLUDE_DIRS 47 | NAMES daos.h 48 | HINTS ${WITH_DAOS_PREFIX}/include 49 | ) 50 | 51 | INCLUDE(FindPackageHandleStandardArgs) 52 | FIND_PACKAGE_HANDLE_STANDARD_ARGS(DAOS DEFAULT_MSG 53 | DAOS_LIBRARIES 54 | CART_LIBRARIES 55 | GURT_LIBRARIES 56 | DUNS_LIBRARIES 57 | DFS_LIBRARIES 58 | COMMON_LIBRARIES 59 | DAOS_INCLUDE_DIRS 60 | ) 61 | 62 | # Hide these vars from ccmake GUI 63 | MARK_AS_ADVANCED( 64 | DAOS_LIBRARIES 65 | CART_LIBRARIES 66 | GURT_LIBRARIES 67 | DUNS_LIBRARIES 68 | DFS_LIBRARIES 69 | COMMON_LIBRARIES 70 | DAOS_INCLUDE_DIRS 71 | ) 72 | -------------------------------------------------------------------------------- /src/common/mfu_proc.c: -------------------------------------------------------------------------------- 1 | /* Defines a double linked list representing a file path. */ 2 | 3 | #include "mfu.h" 4 | 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | #ifdef HAVE_LIBCAP 11 | #include 12 | #endif 13 | 14 | /* query properties and capabilities of current process */ 15 | void mfu_proc_set(mfu_proc_t* proc) 16 | { 17 | proc->getuid = getuid(); 18 | proc->geteuid = geteuid(); 19 | 20 | proc->cap_chown = false; 21 | proc->cap_fowner = false; 22 | 23 | #ifdef HAVE_LIBCAP 24 | /* query current process capability settings */ 25 | cap_t caps = cap_get_proc(); 26 | if (caps != NULL) { 27 | int rc; 28 | cap_flag_value_t flag; 29 | 30 | /* check for CAP_CHOWN */ 31 | rc = cap_get_flag(caps, CAP_CHOWN, CAP_EFFECTIVE, &flag); 32 | if (rc == 0) { 33 | proc->cap_chown = (flag == CAP_SET); 34 | } else { 35 | MFU_LOG(MFU_LOG_ERR, "cap_get_flag(CAP_CHOWN) failed: errno=%d (%s)", errno, strerror(errno)); 36 | } 37 | 38 | /* check for CAP_FOWNER */ 39 | rc = cap_get_flag(caps, CAP_FOWNER, CAP_EFFECTIVE, &flag); 40 | if (rc == 0) { 41 | proc->cap_fowner = (flag == CAP_SET); 42 | } else { 43 | MFU_LOG(MFU_LOG_ERR, "cap_get_flag(CAP_FOWNER) failed: errno=%d (%s)", errno, strerror(errno)); 44 | } 45 | 46 | /* free capabilities structure */ 47 | rc = cap_free(caps); 48 | if (rc == -1) { 49 | MFU_LOG(MFU_LOG_ERR, "cap_free() failed: errno=%d (%s)", errno, strerror(errno)); 50 | } 51 | } else { 52 | MFU_LOG(MFU_LOG_ERR, "cap_get_proc() failed: errno=%d (%s)", errno, strerror(errno)); 53 | } 54 | #endif 55 | } 56 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, Lawrence Livermore National Security, LLC 4 | Copyright (c) 2006-2007,2011-2015, Los Alamos National Security, LLC 5 | Copyright (2013-2015) UT-Battelle, LLC 6 | Copyright (c) 2015, DataDirect Networks, Inc. 7 | 8 | All rights reserved. 9 | 10 | Redistribution and use in source and binary forms, with or without 11 | modification, are permitted provided that the following conditions are met: 12 | 13 | * Redistributions of source code must retain the above copyright notice, this 14 | list of conditions and the following disclaimer. 15 | 16 | * Redistributions in binary form must reproduce the above copyright notice, 17 | this list of conditions and the following disclaimer in the documentation 18 | and/or other materials provided with the distribution. 19 | 20 | * Neither the name of the copyright holder nor the names of its 21 | contributors may be used to endorse or promote products derived from 22 | this software without specific prior written permission. 23 | 24 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 25 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 27 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 28 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 30 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 31 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 32 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 33 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 | -------------------------------------------------------------------------------- /test/tests/test_dchmod/test_octal.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # test to make sure all permissions are being set in octal mode on all files recurseively 4 | 5 | # TODO: need to figure out how to check all of the expected results when user read and write 6 | # bits are turned off because find gets a permission denied 7 | 8 | # generate random number between 700 - 777 to use for octal 9 | OCTAL="$(shuf -i700-777 -n1)" 10 | echo "OCTAL value generated is: $OCTAL" 11 | 12 | # TODO: directories to test, maybe I should use more files/directories??? 13 | # TODO: generate longer file trees here eventually 14 | mkdir -p ~/tmp0/tmp1/tmp2 15 | 16 | # get directory where dchmod command is, set default to home directory if nothing passed in 17 | if [ "$#" -eq 0 ]; then 18 | echo "provide path to dchmod" 19 | else 20 | DCHMOD=$1 21 | fi 22 | 23 | # turn on permissions, then check all of the files/directories are set 24 | $DCHMOD -v -m $OCTAL ~/tmp0 25 | 26 | EXPECTED_RESULT=$OCTAL 27 | 28 | # total count, and passed count variables to check that they match 29 | TOTAL_FILE_COUNT=0 30 | PASSED_FILE_COUNT=0 31 | 32 | for file in $(find ~/tmp0); do 33 | RESULT="$(stat -c '%a' $file)" 34 | TOTAL_FILE_COUNT=$((TOTAL_FILE_COUNT+1)) 35 | if [ ${RESULT} == $EXPECTED_RESULT ]; then 36 | PASSED_FILE_COUNT=$((PASSED_FILE_COUNT+1)) 37 | echo "$file : PASS" 38 | echo "$file: ${RESULT} matches expected: $EXPECTED_RESULT" 39 | else 40 | echo "FAILED TO SET PERMISSION ON: $file" 41 | echo "$file: ${RESULT} doesn't match expected: $EXPECTED_RESULT" 42 | fi 43 | done 44 | 45 | echo "--------------------------------" 46 | echo "TOTAL_FILE_COUNT: $TOTAL_FILE_COUNT" 47 | echo "PASSED_FILE_COUNT: $PASSED_FILE_COUNT" 48 | echo "--------------------------------" 49 | 50 | rm -rf ~/tmp0 51 | -------------------------------------------------------------------------------- /doc/rst/index.rst: -------------------------------------------------------------------------------- 1 | .. mpiFileUtils documentation master file, created by 2 | sphinx-quickstart on Fri Apr 13 11:47:51 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | ================ 7 | **mpiFileUtils** 8 | ================ 9 | 10 | Overview 11 | ************* 12 | 13 | High-performance computing users generate large datasets using parallel applications that can run with thousands of processes. 14 | However, users are often stuck managing those datasets using traditional single-process tools like cp and rm. 15 | This mismatch in scale makes it impractical for users to work with their data. 16 | 17 | The mpiFileUtils suite solves this problem by offering MPI-based tools for basic tasks like copy, remove, and compare for such datasets, 18 | delivering orders of magnitude in performance speedup over their single-process counterparts. 19 | Furthermore, the libmfu library packages common functionality to simplify the creation of new tools, 20 | and it can even be invoked directly from within HPC applications. 21 | 22 | Video Overview: `"Scalable Management of HPC Datasets with mpiFileUtils" `_, HPCKP'20. 23 | 24 | The figure below, taken from the above presentation, illustrates the potential performance improvement that one can achieve 25 | when scaling a tool like dcp to utilize more compute resources. 26 | 27 | .. figure:: dcp_sierra.png 28 | 29 | dcp scaling performance on the Sierra cluster at LLNL using 40 processes/node. Shows the time required to copy a single directory of 200k files totaling 24.4 TiB of data. The minimum time of 93 seconds at 64 nodes is 495x faster than the 12.75 hours taken by the cp command. 30 | 31 | User Guide 32 | *************************** 33 | 34 | .. toctree:: 35 | :maxdepth: 2 36 | 37 | build.rst 38 | proj-design.rst 39 | tools.rst 40 | libmfu.rst 41 | 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mpiFileUtils 2 | mpiFileUtils provides both a library called [libmfu](src/common/README.md) and a suite of MPI-based tools to manage large datasets, which may vary from large directory trees to large files. High-performance computing users often generate large datasets with parallel applications that run with many processes (millions in some cases). However those users are then stuck with single-process tools like cp and rm to manage their datasets. This suite provides MPI-based tools to handle typical jobs like copy, remove, and compare for such datasets, providing speedups of up to 20-30x. It also provides a library that simplifies the creation of new tools or can be used in applications. 3 | 4 | Documentation is available on [ReadTheDocs](http://mpifileutils.readthedocs.io). 5 | 6 | ## DAOS Support 7 | 8 | mpiFileUtils supports a DAOS backend for dcp, dsync, and dcmp. Custom serialization and deserialization for DAOS containers to and from a POSIX filesystem is provided with daos-serialize and daos-deserialize. Details and usage examples are provided in [DAOS Support](DAOS-Support.md). 9 | 10 | ## Contributors 11 | We welcome contributions to the project. For details on how to help, see our [Contributor Guide](CONTRIBUTING.md) 12 | 13 | ### Copyrights 14 | 15 | Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. 16 | Produced at the Lawrence Livermore National Laboratory 17 | CODE-673838 18 | 19 | Copyright (c) 2006-2007,2011-2015, Los Alamos National Security, LLC. 20 | (LA-CC-06-077, LA-CC-10-066, LA-CC-14-046) 21 | 22 | Copyright (2013-2015) UT-Battelle, LLC under Contract No. 23 | DE-AC05-00OR22725 with the Department of Energy. 24 | 25 | Copyright (c) 2015, DataDirect Networks, Inc. 26 | 27 | All rights reserved. 28 | 29 | ## Build Status 30 | The current status of the mpiFileUtils master branch is [![Build Status](https://travis-ci.org/hpc/mpifileutils.png?branch=master)](https://travis-ci.org/hpc/mpifileutils). 31 | -------------------------------------------------------------------------------- /doc/rst/dfilemaker.1.rst: -------------------------------------------------------------------------------- 1 | dfilemaker 2 | ========== 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | **dfilemaker ** 8 | 9 | DESCRIPTION 10 | ----------- 11 | 12 | dfilemaker creates a number of random directory trees with subdirectories and 13 | files of various sizes, that is useful for testing. 14 | 15 | Directory trees are created under destination_path, which must be a directory. 16 | 17 | .. option:: destination_path 18 | 19 | Directory under which the new directory tree will be created. 20 | 21 | OPTIONS 22 | ------- 23 | 24 | .. option:: -d, --depth=*min*-*max* 25 | 26 | Specify the depth of the file system tree to generate. The depth 27 | will be selected at random within the bounds of min and max. The 28 | default depth is set to 10 min, 20 max. 29 | 30 | .. option:: -f, --fill=*type* 31 | 32 | Specify the fill pattern of the file. Current options available are: 33 | ``random``, ``true``, ``false``, and ``alternate``. ``random`` will 34 | fill the file using :manpage:`urandom(4)`. ``true`` will fill the file 35 | with a 0xFF pattern. ``false`` will fill the file with a 0x00 pattern. 36 | ``alternate`` will fill the file with a 0xAA pattern. The default 37 | fill is ``random``. 38 | 39 | .. option:: -i, --seed=*integer* 40 | 41 | Specify the seed to use for random number generation. This can be 42 | used to create reproducible test runs. The default is to generate a 43 | random seed. 44 | 45 | .. option:: -s, --size=*min*-*max* 46 | 47 | Specify the file sizes to generate. The file size will be chosen at 48 | random random within the bounds of min and max. The default file 49 | size is set from 1MB to 5MB. 50 | 51 | .. option:: -h, --help 52 | 53 | Print a brief message listing the *dfilemaker(1)* options and usage. 54 | 55 | .. option:: -v, --version 56 | 57 | Print version information and exit. 58 | 59 | SEE ALSO 60 | -------- 61 | 62 | The mpiFileUtils source code and all documentation may be downloaded 63 | from 64 | -------------------------------------------------------------------------------- /doc/rst/dbcast.1.rst: -------------------------------------------------------------------------------- 1 | dbcast 2 | ====== 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | **dbcast [OPTION] SRC DEST** 8 | 9 | DESCRIPTION 10 | ----------- 11 | 12 | Parallel MPI application to broadcast a single file from a 13 | global file system to node-local storage, like ramdisk or an SSD. 14 | 15 | The file is logically sliced into chunks and collectively copied from a 16 | global file system to node-local storage. The bytes of the source file 17 | are read from the global file system once and the data are forwarded 18 | through MPI to be written to storage on all compute nodes. 19 | The source file SRC must be readable by all MPI processes. 20 | The destination file DEST should be the 21 | full path of the file in node-local storage. If needed, parent 22 | directories for the destination file will be created as part of the 23 | broadcast. 24 | 25 | In the current implementation, dbcast requires at least two MPI 26 | processes per compute node, and all compute nodes must run an equal 27 | number of MPI processes. 28 | 29 | OPTIONS 30 | ------- 31 | 32 | .. option:: -s, --size SIZE 33 | 34 | The chunk size in bytes used to segment files during the broadcast. 35 | Units like "MB" and "GB" should be immediately follow the number 36 | without spaces (ex. 2MB). The default size is 1MB. It is recommended 37 | to use the stripe size of a file if this is known. 38 | 39 | .. option:: -h, --help 40 | 41 | Print the command usage, and the list of options available. 42 | 43 | EXAMPLES 44 | -------- 45 | 46 | 1. To broadcast a file to /ssd on each node: 47 | 48 | ``mpirun -np 128 dbcast /global/path/to/filenane /ssd/filename`` 49 | 50 | 2. Same thing, but slicing at 10MB chunks: 51 | 52 | ``mpirun -np 128 dbcast -s 10MB /global/path/to/filenane /ssd/filename`` 53 | 54 | 3. To read the current striping parameters of a file on Lustre: 55 | 56 | ``lfs getstripe /global/path/to/filename`` 57 | 58 | SEE ALSO 59 | -------- 60 | 61 | The mpiFileUtils source code and all documentation may be downloaded 62 | from 63 | -------------------------------------------------------------------------------- /test/tests/test_dsync/test_expectfail.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . utility/set_funcs.sh 4 | 5 | ############################################################################## 6 | # Description: 7 | # 8 | # Verify dsync fails when it should 9 | # - source does not exist 10 | # - parent of dest does not exist 11 | # 12 | # Notes: 13 | # 14 | ############################################################################## 15 | 16 | # Turn on verbose output 17 | #set -x 18 | 19 | MFU_TEST_BIN=${MFU_TEST_BIN:-${1}} 20 | DSYNC_SRC_BASE=${DSYNC_SRC_BASE:-${2}} 21 | DSYNC_DEST_BASE=${DSYNC_DEST_BASE:-${3}} 22 | DSYNC_TREE_NAME=${DSYNC_TREE_NAME:-${4}} 23 | 24 | echo "Using MFU binaries at: $MFU_TEST_BIN" 25 | echo "Using src parent directory at: $DSYNC_SRC_BASE" 26 | echo "Using dest parent directory at: $DSYNC_DEST_BASE" 27 | 28 | DSYNC_SRC_DIR=$(mktemp --directory ${DSYNC_SRC_BASE}/${DSYNC_TREE_NAME}.XXXXX) 29 | DSYNC_DEST_DIR=$(mktemp --directory ${DSYNC_DEST_BASE}/${DSYNC_TREE_NAME}.XXXXX) 30 | 31 | function sync_and_verify() 32 | { 33 | local srcdir=$1 34 | local destdir=$2 35 | local name=$3 36 | local expectation=$4 37 | 38 | local result=0 39 | local dest_type="unknown" 40 | 41 | quiet_opt="--quiet" 42 | 43 | ${MFU_TEST_BIN}/dsync $quiet_opt $delete_opt $srcdir $destdir 44 | rc=$? 45 | echo "dsync failed with rc $rc" 46 | 47 | if [[ "$rc" -eq 0 ]]; then 48 | echo "FAILED verify of option $name for $destdir type $dest_type" 49 | result=1 50 | 51 | else 52 | echo "PASSED verify of option $name for $destdir type $dest_type" 53 | result=0 54 | fi 55 | 56 | return $result 57 | } 58 | 59 | # source does not exist 60 | rm -fr $DSYNC_SRC_DIR/stuff 61 | rm -fr $DSYNC_DEST_DIR/stuff 62 | mkdir $DSYNC_DEST_DIR/stuff 63 | sync_and_verify $DSYNC_SRC_DIR/stuff $DSYNC_DEST_DIR/stuff source_missing expect_dsync_fail 64 | 65 | # parent of dest does not exist 66 | rm -fr $DSYNC_SRC_DIR/stuff 67 | rm -fr $DSYNC_DEST_DIR/stuff 68 | mkdir $DSYNC_SRC_DIR/stuff 69 | sync_and_verify $DSYNC_SRC_DIR/stuff $DSYNC_DEST_DIR/stuff/childdir dest_parent_missing expect_dsync_fail 70 | 71 | # clean up 72 | rm -fr $DSYNC_SRC_DIR/stuff 73 | rm -fr $DSYNC_DEST_DIR/stuff 74 | 75 | exit 0 76 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: xenial 3 | 4 | language: c 5 | 6 | # - sudo apt-get install -qq check mpich check bc libattr1-dev python-pip 7 | before_install: 8 | - test -n $CC && unset CC 9 | - test -n $CXX && unset CXX 10 | - sudo apt-get update -qq 11 | - sudo apt-cache search mpich 12 | - sudo apt-get install -qq check bc libattr1-dev libcap-dev python-pip 13 | - sudo pip install nose 14 | - wget https://www.mpich.org/static/downloads/3.3/mpich-3.3.tar.gz 15 | - tar -zxf mpich-3.3.tar.gz 16 | - pushd mpich-3.3 17 | - ./configure --enable-fortran=no && make -j4 && sudo make install 18 | - popd 19 | - libcircle=libcircle-0.3.0 20 | - wget https://github.com/hpc/libcircle/releases/download/v0.3/${libcircle}.tar.gz 21 | - tar -zxf ${libcircle}.tar.gz 22 | - pushd ${libcircle} 23 | - ./configure --enable-tests && make && make check && sudo make install 24 | - popd 25 | - lwgrp=lwgrp-1.0.3 26 | - wget https://github.com/llnl/lwgrp/releases/download/v1.0.3/${lwgrp}.tar.gz 27 | - tar -zxf ${lwgrp}.tar.gz 28 | - pushd ${lwgrp} 29 | - ./configure && make && sudo make install 30 | - popd 31 | - dtcmp=dtcmp-1.1.1 32 | - wget https://github.com/llnl/dtcmp/releases/download/v1.1.1/${dtcmp}.tar.gz 33 | - tar zxvf ${dtcmp}.tar.gz 34 | - pushd ${dtcmp} 35 | - ./configure --with-lwgrp=/usr/local && make && sudo make install 36 | - popd 37 | - libarchive=libarchive-3.5.1 38 | - wget https://github.com/libarchive/libarchive/releases/download/3.5.1/${libarchive}.tar.gz 39 | - tar zxvf ${libarchive}.tar.gz 40 | - pushd ${libarchive} 41 | - ./configure && make && sudo make install 42 | - popd 43 | - sudo ldconfig 44 | 45 | before_script: 46 | - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then git show --summary HEAD | grep -q "Signed-off-by:"; fi' 47 | 48 | script: 49 | - mkdir build install && cd build 50 | - cmake ../ -DCMAKE_INSTALL_PREFIX=../install -DENABLE_EXPERIMENTAL=ON 51 | - make 52 | - make test 53 | - make install 54 | 55 | #script: ./autogen.sh && export CC=mpicc && ./configure && make && make test 56 | #script: ./autogen.sh && ./configure && make && make test 57 | 58 | compiler: 59 | - clang 60 | - gcc 61 | 62 | notifications: 63 | email: false 64 | #- sikich1@llnl.gov 65 | #- moody20@llnl.gov 66 | -------------------------------------------------------------------------------- /doc/rst/proj-design.rst: -------------------------------------------------------------------------------- 1 | ========================= 2 | Project Design Principles 3 | ========================= 4 | 5 | The following principles drive design decisions in the project. 6 | 7 | ------ 8 | Scale 9 | ------ 10 | 11 | The library and tools should be designed such that running with more processes 12 | increases performance, provided there are sufficient data and parallelism 13 | available in the underlying file systems. The design of the tool should not 14 | impose performance scalability bottlenecks. 15 | 16 | ----------- 17 | Performance 18 | ----------- 19 | 20 | While it is tempting to mimic the interface, behavior, and file formats of 21 | familiar tools like cp, rm, and tar, when forced with a choice between 22 | compatibility and performance, mpiFileUtils chooses performance. For example, 23 | if an archive file format requires serialization that inhibits parallel 24 | performance, mpiFileUtils will opt to define a new file format that enables 25 | parallelism rather than being constrained to existing formats. Similarly, 26 | options in the tool command line interface may have different semantics from 27 | familiar tools in cases where performance is improved. Thus, one should be 28 | careful to learn the options of each tool. 29 | 30 | ----------- 31 | Portability 32 | ----------- 33 | 34 | The tools are intended to support common file systems used in HPC centers, like 35 | Lustre, GPFS, NFS, and HPSS. Additionally, methods in the library should be portable 36 | and efficient across multiple file systems. Tool and library users can rely on 37 | mpiFileUtils to provide portable and performant implementations. 38 | 39 | ------------- 40 | Composability 41 | ------------- 42 | 43 | While the tools do not support chaining with Unix pipes, they do support 44 | interoperability through input and output files. One tool may process a dataset 45 | and generate an output file that another tool can read as input, e.g., to walk 46 | a directory tree with one tool, filter the list of file names with another, and 47 | perhaps delete a subset of matching files with a third. Additionally, when logic 48 | is deemed to be useful across multiple tools or is anticipated to be useful in 49 | future tools or applications, it should be provided in the common library. 50 | -------------------------------------------------------------------------------- /test/legacy/dtest/moab.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | class job: 3 | 4 | id = 0 5 | 6 | def __init__(self, id): 7 | self.id = id 8 | 9 | def state(self): 10 | dict = {'Q':'queued', 11 | 'R':'running', 12 | 'C':'complete', 13 | 'H':'hold', 14 | 'E':'error'} 15 | state = subprocess.Popen(['qstat', str(self.id)], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0].split('\n')[2].split()[4] 16 | 17 | return dict[state] 18 | 19 | def id(self): 20 | return self.id 21 | 22 | def name(self): 23 | name = subprocess.Popen(['qstat', str(self.id)], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0].split('\n')[2].split()[1] 24 | return name 25 | 26 | def user(self): 27 | user = subprocess.Popen(['qstat', str(self.id)], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0].split('\n')[2].split()[2] 28 | return user 29 | 30 | def time(self): 31 | time = subprocess.Popen(['qstat', str(self.id)], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0].split('\n')[2].split()[3] 32 | return time 33 | 34 | def queue(self): 35 | queue = subprocess.Popen(['qstat', str(self.id)], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0].split('\n')[2].split()[5] 36 | return queue 37 | 38 | def delete(self): 39 | subprocess.call(['qdel', str(self.id)]) 40 | 41 | def hold(self): 42 | subprocess.call(['qhold', str(self.id)]) 43 | 44 | def release(self): 45 | subprocess.call(['qrls', str(self.id)]) 46 | 47 | def submit(job): 48 | id = subprocess.Popen(['qsub', job], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0] 49 | return int(id) 50 | def delete(id): 51 | subprocess.call(['qdel', str(id)]) 52 | 53 | def hold(id): 54 | subprocess.call(['qhold', str(id)]) 55 | 56 | def release(id): 57 | subprocess.call(['qrls', str(id)]) 58 | 59 | def getjob(id): 60 | return job(id) 61 | 62 | def getjobsbyuser(user): 63 | out = subprocess.Popen(['qstat', '-u', user], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()[0] 64 | lines = out.split('\n') 65 | jobdata = lines[5:] 66 | jobs = [] 67 | print 68 | for job in jobdata: 69 | if job: 70 | jobject = getjob(job.split()[0]) 71 | jobs.append(jobject) 72 | 73 | return jobs 74 | 75 | 76 | -------------------------------------------------------------------------------- /src/common/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Version for the shared mfu library 2 | set(MFU_VERSION_MAJOR 4) # Incompatible API changes 3 | set(MFU_VERSION_MINOR 0) # Backwards-compatible functionality 4 | set(MFU_VERSION_PATCH 0) # Backwards-compatible fixes 5 | set(MFU_VERSION ${MFU_VERSION_MAJOR}.${MFU_VERSION_MINOR}.${MFU_VERSION_PATCH}) 6 | 7 | # todo re-asses if all of these must be *installed* 8 | LIST(APPEND libmfu_install_headers 9 | mfu.h 10 | mfu_errors.h 11 | mfu_bz2.h 12 | mfu_flist.h 13 | mfu_flist_internal.h 14 | mfu_io.h 15 | mfu_param_path.h 16 | mfu_path.h 17 | mfu_pred.h 18 | mfu_proc.h 19 | mfu_progress.h 20 | mfu_util.h 21 | ) 22 | if(ENABLE_DAOS) 23 | LIST(APPEND libmfu_install_headers 24 | mfu_daos.h 25 | ) 26 | ENDIF(ENABLE_DAOS) 27 | 28 | INSTALL(FILES ${libmfu_install_headers} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) 29 | 30 | # common library 31 | LIST(APPEND libmfu_srcs 32 | mfu_bz2.c 33 | mfu_bz2_static.c 34 | mfu_compress_bz2_libcircle.c 35 | mfu_decompress_bz2_libcircle.c 36 | mfu_flist.c 37 | mfu_flist_chunk.c 38 | mfu_flist_copy.c 39 | mfu_flist_io.c 40 | mfu_flist_chmod.c 41 | mfu_flist_create.c 42 | mfu_flist_remove.c 43 | mfu_flist_sort.c 44 | mfu_flist_usrgrp.c 45 | mfu_flist_walk.c 46 | mfu_io.c 47 | mfu_param_path.c 48 | mfu_path.c 49 | mfu_pred.c 50 | mfu_proc.c 51 | mfu_progress.c 52 | mfu_util.c 53 | strmap.c 54 | ) 55 | IF(ENABLE_LIBARCHIVE) 56 | LIST(APPEND libmfu_srcs 57 | mfu_flist_archive.c 58 | ) 59 | ENDIF(ENABLE_LIBARCHIVE) 60 | IF(ENABLE_DAOS) 61 | LIST(APPEND libmfu_srcs 62 | mfu_daos.c 63 | ) 64 | ENDIF(ENABLE_DAOS) 65 | 66 | ADD_LIBRARY(mfu_o OBJECT ${libmfu_srcs}) 67 | SET_TARGET_PROPERTIES(mfu_o PROPERTIES C_STANDARD 99) 68 | 69 | ADD_LIBRARY(mfu SHARED $) 70 | TARGET_LINK_LIBRARIES(mfu LINK_PUBLIC ${MFU_EXTERNAL_LIBS}) 71 | SET_TARGET_PROPERTIES(mfu PROPERTIES VERSION ${MFU_VERSION} OUTPUT_NAME mfu CLEAN_DIRECT_OUTPUT 1) 72 | INSTALL(TARGETS mfu DESTINATION ${CMAKE_INSTALL_LIBDIR}) 73 | 74 | ADD_LIBRARY(mfu-static STATIC $) 75 | TARGET_LINK_LIBRARIES(mfu-static LINK_PUBLIC ${MFU_EXTERNAL_LIBS}) 76 | SET_TARGET_PROPERTIES(mfu-static PROPERTIES OUTPUT_NAME mfu CLEAN_DIRECT_OUTPUT 1) 77 | INSTALL(TARGETS mfu-static DESTINATION ${CMAKE_INSTALL_LIBDIR}) 78 | -------------------------------------------------------------------------------- /test/legacy/dtar_tests/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | NUM_FILES=10 4 | NUM_PROC=5 5 | FILE_SIZE=1K 6 | 7 | TMPDIR=`pwd`/tmp 8 | TMPCHCK=`pwd`/check 9 | TARGET=`pwd`/test.tar 10 | 11 | D=0 12 | L=0 13 | OPTIONS='-s -np -c -d -l' 14 | 15 | ###Check options 16 | while [[ $# -gt 0 ]]; do 17 | if [[ "$1" == "-s" ]]; then 18 | FILE_SIZE=$2 19 | shift 20 | shift 21 | fi 22 | if [[ "$1" == "-np" ]]; then 23 | NUM_PROC=$2 24 | shift 25 | shift 26 | fi 27 | if [[ "$1" == "-c" ]]; then 28 | NUM_FILES=$2 29 | shift 30 | shift 31 | fi 32 | if [[ "$1" == "-d" ]]; then 33 | D=1 34 | shift 35 | fi 36 | if [[ "$1" == "-l" ]]; then 37 | L=1 38 | shift 39 | fi 40 | if [[ "$1" == "--DEBUG" ]]; then 41 | DEBUG=1 42 | shift 43 | fi 44 | done 45 | 46 | ###Create temporary directories 47 | mkdir -p $TMPDIR 48 | mkdir -p $TMPCHCK 49 | 50 | 51 | ###Write random files and direcotories in base64 for checking 52 | cd $TMPDIR 53 | function write { 54 | for i in $(seq 1 $1); do 55 | if [ $(($RANDOM % 100 )) -lt 20 ] && [ $D -eq 1 ]; then 56 | mkdir dir${i} 57 | cd dir${i} 58 | write $(( $1/2 )) 59 | else 60 | echo -ne "\rwriting files..." 61 | head -c $FILE_SIZE /dev/urandom | base64 > file$i.tmp 62 | fi 63 | done 64 | cd .. 65 | } 66 | write $NUM_FILES 67 | 68 | ###Insert symbolic links 69 | if [[ $L -eq 1 ]]; then 70 | ln -s .$0 $TMPDIR/filelink.lnk 71 | ln -s /usr/bin $TMPDIR/dirlink 72 | fi 73 | 74 | echo 75 | 76 | ###Create the tarfile 77 | echo "creating parallel tar file..." 78 | mpirun -np $NUM_PROC ./dtar -cf $TARGET $TMPDIR 79 | ###Extract the resulting tarfile with GNU tar 80 | echo extracting tar file... 81 | tar -xf $TARGET -C $TMPCHCK 82 | 83 | ###Check extracted files against the originals 84 | GOOD=1 85 | echo checking files... 86 | diff -q -r $TMPDIR $TMPCHCK/tmp 87 | DIFF=$? 88 | if [[ $DEBUG -eq 1 ]]; then 89 | echo "diff: $DIFF" 90 | fi 91 | echo 92 | if [[ $DIFF -ne 0 ]]; then 93 | if [[ "$DIFF" == *".lnk"* ]]; then 94 | echo "symbolic link failure" 95 | else 96 | echo "resulting archive is invalid" 97 | fi 98 | else 99 | echo "tar file is good" 100 | fi 101 | if [[ $DEBUG -ne 1 ]]; then 102 | rm -rf $TMPDIR $TMPCHCK $TARGET 103 | fi 104 | -------------------------------------------------------------------------------- /doc/rst/dreln.1.rst: -------------------------------------------------------------------------------- 1 | dreln 2 | ===== 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | **dreln [OPTION] OLDPATH NEWPATH PATH ...** 8 | 9 | DESCRIPTION 10 | ----------- 11 | 12 | Parallel MPI application to recursively update symlinks within a 13 | directory. 14 | 15 | dreln walks the specified PATH and updates any symlink whose target 16 | includes an absolute path to OLDPATH and replaces that symlink 17 | with a new link whose target points to NEWPATH instead. 18 | 19 | This is useful to update symlinks after migrating a large 20 | directory from one file system to another, whose links specify 21 | absolute paths to the original file system. 22 | 23 | OPTIONS 24 | ------- 25 | 26 | .. option:: -i, --input FILE 27 | 28 | Read source list from FILE. FILE must be generated by another tool 29 | from the mpiFileUtils suite. 30 | 31 | .. option:: -p, --preserve 32 | 33 | Preserve existing modification times on links. 34 | 35 | .. option:: -r, --relative 36 | 37 | Replace links using target paths that are relative to NEWPATH. 38 | 39 | .. option:: --progress N 40 | 41 | Print progress message to stdout approximately every N seconds. 42 | The number of seconds must be a non-negative integer. 43 | A value of 0 disables progress messages. 44 | 45 | .. option:: -v, --verbose 46 | 47 | Run in verbose mode. 48 | 49 | .. option:: -q, --quiet 50 | 51 | Run tool silently. No output is printed. 52 | 53 | .. option:: -h, --help 54 | 55 | Print a brief message listing the :manpage:`drm(1)` options and usage. 56 | 57 | EXAMPLES 58 | -------- 59 | 60 | 1. To update all links under /walk/path whose targets point to /orig/path 61 | and replace them with targets that point to /new/path: 62 | 63 | ``mpirun -np 128 dreln -v /orig/path /new/path /walk/path`` 64 | 65 | 2. Same as above, but replace each link target with a relative path 66 | from the link to its new target under /new/path: 67 | 68 | ``mpirun -np 128 dreln -v --relative /orig/path /new/path /walk/path`` 69 | 70 | 3. One can preserve existing modification times on links: 71 | 72 | ``mpirun -np 128 dreln -v --preserve /orig/path /new/path /walk/path`` 73 | 74 | 4. One can specifiy multiple paths to walk: 75 | 76 | ``mpirun -np 128 dreln -v /orig/path /new/path /walk/path1 /walk/path2`` 77 | 78 | SEE ALSO 79 | -------- 80 | 81 | The mpiFileUtils source code and all documentation may be downloaded 82 | from 83 | -------------------------------------------------------------------------------- /test/legacy/dcp1_tests/test_dcp1_single_dir_to_single_dir/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ############################################################################## 4 | # Description: 5 | # 6 | # A test to check if dcp1 will copy a single directory to a single directory. 7 | # 8 | # Expected behavior: 9 | # 10 | # The source directory should be copied to the destination directory. 11 | # 12 | # Reminder: 13 | # 14 | # Lines that echo to the terminal will only be available if DEBUG is enabled 15 | # in the test runner (test_all.sh). 16 | ############################################################################## 17 | 18 | # Turn on verbose output 19 | #set -x 20 | 21 | # Print out the basic paths we'll be using. 22 | echo "Using dcp1 binary at: $DCP_TEST_BIN" 23 | echo "Using tmp directory at: $DCP_TEST_TMP" 24 | 25 | ############################################################################## 26 | # Generate the paths for: 27 | # * Two directories. 28 | # * One file which contain random data. 29 | PATH_A_DIRECTORY="$DCP_TEST_TMP/dcp1_test_many_dir_to_single_dir.$RANDOM.tmp" 30 | PATH_B_DIRECTORY="$DCP_TEST_TMP/dcp1_test_many_dir_to_single_dir.$RANDOM.tmp" 31 | PATH_C_RANDOM="$DCP_TEST_TMP/$(basename $PATH_A_DIRECTORY)/dcp1_test_many_dir_to_single_dir.$RANDOM.tmp" 32 | 33 | # Print out the generated paths to make debugging easier. 34 | echo "A_DIRECTORY path at: $PATH_A_DIRECTORY" 35 | echo "B_DIRECTORY path at: $PATH_B_DIRECTORY" 36 | echo "C_RANDOM path at: $PATH_C_RANDOM" 37 | 38 | # Create the directories. 39 | mkdir $PATH_A_DIRECTORY 40 | mkdir $PATH_B_DIRECTORY 41 | 42 | # Create the random file. 43 | dd if=/dev/urandom of=$PATH_C_RANDOM bs=3M count=2 44 | 45 | ############################################################################## 46 | # Test copying a directory to a directory. The result should be the directory 47 | # placed inside the directory. 48 | 49 | mpirun -n 3 $DCP_TEST_BIN $PATH_A_DIRECTORY $PATH_B_DIRECTORY 50 | if [[ $? -ne 0 ]]; then 51 | echo "Error returned when copying a directory to a directory (A -> B)." 52 | exit 1; 53 | fi 54 | 55 | $DCP_CMP_BIN "$PATH_B_DIRECTORY/$(basename $PATH_A_DIRECTORY)/$(basename $PATH_C_RANDOM)" $PATH_C_RANDOM 56 | if [[ $? -ne 0 ]]; then 57 | echo "CMP mismatch when copying a directory to a directory (A -> B/A/C)." 58 | exit 1 59 | fi 60 | 61 | ############################################################################## 62 | # Since we didn't find any problems, exit with success. 63 | 64 | exit 0 65 | 66 | # EOF 67 | -------------------------------------------------------------------------------- /src/dcp1/COPYING: -------------------------------------------------------------------------------- 1 | Copyright (c) 2007-2012 Los Alamos National Security, LLC. 2 | All rights reserved. 3 | 4 | This software was produced under U.S. Government contract DE-AC52-06NA25396 5 | for Los Alamos National Laboratory (LANL), which is operated by Los Alamos 6 | National Security, LLC for the U.S. Department of Energy. The U.S. Government 7 | has rights to use, reproduce, and distribute this software. NEITHER THE 8 | GOVERNMENT NOR LOS ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS 9 | OR IMPLIED, OR ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software 10 | is modified to produce derivative works, such modified software should be 11 | clearly marked, so as not to confuse it with the version available from LANL. 12 | 13 | Additionally, redistribution and use in source and binary forms, with or 14 | without modification, are permitted provided that the following conditions 15 | are met: 16 | 17 | * Redistributions of source code must retain the above copyright notice, this 18 | list of conditions and the following disclaimer. 19 | 20 | * Redistributions in binary form must reproduce the above copyright notice, 21 | this list of conditions and the following disclaimer in the documentation 22 | and/or other materials provided with the distribution. 23 | 24 | * Neither the name of Los Alamos National Security, LLC, Los Alamos National 25 | Laboratory, LANL, the U.S. Government, nor the names of its contributors may 26 | be used to endorse or promote products derived from this software without 27 | without specific prior written permission. 28 | 29 | THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS 30 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 31 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 | ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR 33 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 34 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 35 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 36 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 37 | IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 38 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 39 | POSSIBILITY OF SUCH DAMAGE. 40 | 41 | dcp is from LANL-purger/MPI_FTW (file treewalk), which can be found at: 42 | 43 | 44 | LA-CC-06-077 45 | -------------------------------------------------------------------------------- /test/legacy/dcp1_tests/test_dcp1_single_dir_to_single_file/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ############################################################################## 4 | # Description: 5 | # 6 | # A test to check if dcp1 will copy a single directory to a single file. 7 | # 8 | # Expected behavior: 9 | # 10 | # An error code should be displayed if this action is attempted. 11 | # 12 | # Reminder: 13 | # 14 | # Lines that echo to the terminal will only be available if DEBUG is enabled 15 | # in the test runner (test_all.sh). 16 | ############################################################################## 17 | 18 | # Turn on verbose output 19 | #set -x 20 | 21 | # Print out the basic paths we'll be using. 22 | echo "Using dcp1 binary at: $DCP_TEST_BIN" 23 | echo "Using tmp directory at: $DCP_TEST_TMP" 24 | 25 | ############################################################################## 26 | # Generate the paths for: 27 | # * A directory. 28 | # * A file with zero length. 29 | # * A file which contains random data. 30 | PATH_A_DIRECTORY="$DCP_TEST_TMP/dcp1_test_single_dir_to_single_file.$RANDOM.tmp" 31 | PATH_B_EMPTY="$DCP_TEST_TMP/dcp1_test_single_dir_to_single_file.$RANDOM.tmp" 32 | PATH_C_RANDOM="$DCP_TEST_TMP/dcp1_test_single_dir_to_single_file.$RANDOM.tmp" 33 | 34 | # Print out the generated paths to make debugging easier. 35 | echo "A_DIRECTORY path at: $PATH_A_DIRECTORY" 36 | echo "B_EMPTY path at: $PATH_B_EMPTY" 37 | echo "C_RANDOM path at: $PATH_C_RANDOM" 38 | 39 | # Create empty file. 40 | touch $PATH_B_EMPTY 41 | 42 | # Create the random file. 43 | dd if=/dev/urandom of=$PATH_C_RANDOM bs=4M count=5 44 | 45 | ############################################################################## 46 | # Test copying the directory to an empty file. The result should be an error. 47 | 48 | mpirun -n 3 $DCP_TEST_BIN $PATH_A_DIRECTORY $PATH_B_EMPTY 49 | if [[ $? -eq 0 ]]; then 50 | echo "Unexpectedd success when copying a directory to an empty file. (A -> B)." 51 | exit 1; 52 | fi 53 | 54 | ############################################################################## 55 | # Test copying a directory to a random file. The result should be an error. 56 | 57 | mpirun -n 3 $DCP_TEST_BIN $PATH_A_DIRECTORY $PATH_C_RANDOM 58 | if [[ $? -eq 0 ]]; then 59 | echo "Unexpected success when copying a directory to a random file (A -> C)." 60 | exit 1; 61 | fi 62 | 63 | ############################################################################## 64 | # Since we didn't find any problems, exit with success. 65 | 66 | exit 0 67 | 68 | # EOF 69 | -------------------------------------------------------------------------------- /man/ddup.1: -------------------------------------------------------------------------------- 1 | .\" Man page generated from reStructuredText. 2 | . 3 | .TH "DDUP" "1" "Jan 29, 2025" "0.12" "mpiFileUtils" 4 | .SH NAME 5 | ddup \- report files with identical content 6 | . 7 | .nr rst2man-indent-level 0 8 | . 9 | .de1 rstReportMargin 10 | \\$1 \\n[an-margin] 11 | level \\n[rst2man-indent-level] 12 | level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] 13 | - 14 | \\n[rst2man-indent0] 15 | \\n[rst2man-indent1] 16 | \\n[rst2man-indent2] 17 | .. 18 | .de1 INDENT 19 | .\" .rstReportMargin pre: 20 | . RS \\$1 21 | . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] 22 | . nr rst2man-indent-level +1 23 | .\" .rstReportMargin post: 24 | .. 25 | .de UNINDENT 26 | . RE 27 | .\" indent \\n[an-margin] 28 | .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] 29 | .nr rst2man-indent-level -1 30 | .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] 31 | .in \\n[rst2man-indent\\n[rst2man-indent-level]]u 32 | .. 33 | .SH SYNOPSIS 34 | .sp 35 | \fBddup [OPTION] PATH\fP 36 | .SH DESCRIPTION 37 | .sp 38 | Parallel MPI application to report files under a directory tree having identical content. 39 | .sp 40 | ddup reports path names to files having identical content (duplicate files). 41 | A top\-level directory is specified, and the path name to any file that is a duplicate 42 | of another anywhere under that same directory tree is reported. 43 | The path to each file is reported, along with a final hash representing its content. 44 | Multiple sets of duplicate files can be matched using this final reported hash. 45 | .SH OPTIONS 46 | .INDENT 0.0 47 | .TP 48 | .B \-\-open\-noatime 49 | Open files with O_NOATIME flag, if possible. 50 | .UNINDENT 51 | .INDENT 0.0 52 | .TP 53 | .B \-d, \-\-debug LEVEL 54 | Set verbosity level. LEVEL can be one of: fatal, err, warn, info, dbg. 55 | .UNINDENT 56 | .INDENT 0.0 57 | .TP 58 | .B \-v, \-\-verbose 59 | Run in verbose mode. 60 | .UNINDENT 61 | .INDENT 0.0 62 | .TP 63 | .B \-q, \-\-quiet 64 | Run tool silently. No output is printed. 65 | .UNINDENT 66 | .INDENT 0.0 67 | .TP 68 | .B \-h, \-\-help 69 | Print the command usage, and the list of options available. 70 | .UNINDENT 71 | .SH EXAMPLES 72 | .INDENT 0.0 73 | .IP 1. 3 74 | To report any duplicate files under a directory tree: 75 | .UNINDENT 76 | .sp 77 | \fBmpirun \-np 128 ddup /path/to/haystack\fP 78 | .SH SEE ALSO 79 | .sp 80 | The mpiFileUtils source code and all documentation may be downloaded 81 | from <\fI\%https://github.com/hpc/mpifileutils\fP> 82 | .SH AUTHOR 83 | HPC 84 | .SH COPYRIGHT 85 | 2022, LLNL/LANL/UT-Battelle/DDN 86 | .\" Generated by docutils manpage writer. 87 | . 88 | -------------------------------------------------------------------------------- /src/common/mfu_progress.h: -------------------------------------------------------------------------------- 1 | /* enable C++ codes to include this header directly */ 2 | #ifdef __cplusplus 3 | extern "C" { 4 | #endif 5 | 6 | #ifndef MFU_PROGRESS_H 7 | #define MFU_PROGRESS_H 8 | 9 | #include "mpi.h" 10 | 11 | /* function prototype for progress callback 12 | * vals - array of current values summed across ranks 13 | * count - number of elements in vals array 14 | * complete - number of ranks that are complete 15 | * ranks - number of ranks involved 16 | * secs - number of seconds since start was called */ 17 | typedef void (*mfu_progress_fn)(const uint64_t* vals, int count, int complete, int ranks, double secs); 18 | 19 | /* (opaque) struct that holds state for progress message reporting */ 20 | typedef struct { 21 | MPI_Comm comm; /* dup'ed communicator to execute bcast/reduce */ 22 | MPI_Request bcast_req; /* request for outstanding bcast */ 23 | MPI_Request reduce_req; /* request for outstanding reduce */ 24 | double time_start; /* time when start was called */ 25 | double time_last; /* time when last report was requested */ 26 | double timeout; /* number of seconds between reports */ 27 | int keep_going; /* flag indicating whether any process is still working */ 28 | int count; /* number of items in values arrays */ 29 | uint64_t* values; /* array holding contribution to global sum from local proc */ 30 | uint64_t* global_vals; /* array to hold global sum across ranks */ 31 | mfu_progress_fn progfn; /* callback function to execute to print progress message */ 32 | } mfu_progress; 33 | 34 | /* start progress timer and return newly allocated structure 35 | * to track its state 36 | * secs - IN number of seconds between progress messages 37 | * count - IN number of uint64_t values to sum in each message 38 | * comm - IN communicator to dup on which to execute ibcast/ireduce 39 | * progfn - IN callback to invoke to print progress message */ 40 | mfu_progress* mfu_progress_start(int secs, int count, MPI_Comm comm, mfu_progress_fn progfn); 41 | 42 | /* update progress across all processes in work loop, 43 | * vals - IN update contribution of this process to global sum 44 | * prg - IN pointer to struct returned in start */ 45 | void mfu_progress_update(uint64_t* vals, mfu_progress* prg); 46 | 47 | /* continue broadcasting progress until all processes have completed, 48 | * and free structure allocated in start 49 | * vals - IN array with latest contribution from this process to global sum 50 | * pprg - IN address of pointer to struct returned in start */ 51 | void mfu_progress_complete(uint64_t* vals, mfu_progress** pprg); 52 | 53 | #endif /* MFU_PROGRESS_H */ 54 | 55 | /* enable C++ codes to include this header directly */ 56 | #ifdef __cplusplus 57 | } /* extern "C" */ 58 | #endif 59 | -------------------------------------------------------------------------------- /man/dbz2.1: -------------------------------------------------------------------------------- 1 | .\" Man page generated from reStructuredText. 2 | . 3 | .TH "DBZ2" "1" "Jan 29, 2025" "0.12" "mpiFileUtils" 4 | .SH NAME 5 | dbz2 \- distributed bz2 compression 6 | . 7 | .nr rst2man-indent-level 0 8 | . 9 | .de1 rstReportMargin 10 | \\$1 \\n[an-margin] 11 | level \\n[rst2man-indent-level] 12 | level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] 13 | - 14 | \\n[rst2man-indent0] 15 | \\n[rst2man-indent1] 16 | \\n[rst2man-indent2] 17 | .. 18 | .de1 INDENT 19 | .\" .rstReportMargin pre: 20 | . RS \\$1 21 | . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] 22 | . nr rst2man-indent-level +1 23 | .\" .rstReportMargin post: 24 | .. 25 | .de UNINDENT 26 | . RE 27 | .\" indent \\n[an-margin] 28 | .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] 29 | .nr rst2man-indent-level -1 30 | .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] 31 | .in \\n[rst2man-indent\\n[rst2man-indent-level]]u 32 | .. 33 | .SH SYNOPSIS 34 | .sp 35 | \fBdbz2 [OPTIONS] [\-z|\-d] FILE\fP 36 | .SH DESCRIPTION 37 | .sp 38 | Parallel MPI application to compress or decompress a file. 39 | .sp 40 | When compressing, a new file will be created with a .dbz2 extension. 41 | When decompressing, the .dbz2 extension will be dropped from the file name. 42 | .SH OPTIONS 43 | .INDENT 0.0 44 | .TP 45 | .B \-z, \-\-compress 46 | Compress the file 47 | .UNINDENT 48 | .INDENT 0.0 49 | .TP 50 | .B \-d, \-\-decompress 51 | Decompress the file 52 | .UNINDENT 53 | .INDENT 0.0 54 | .TP 55 | .B \-k, \-\-keep 56 | Keep the input file. 57 | .UNINDENT 58 | .INDENT 0.0 59 | .TP 60 | .B \-f, \-\-force 61 | Overwrite the output file, if it exists. 62 | .UNINDENT 63 | .INDENT 0.0 64 | .TP 65 | .B \-b, \-\-blocksize SIZE 66 | Set the compression block size, from 1 to 9. 67 | Where 1=100kB ... and 9=900kB. Default is 9. 68 | .UNINDENT 69 | .INDENT 0.0 70 | .TP 71 | .B \-v, \-\-verbose 72 | Verbose output (optional). 73 | .UNINDENT 74 | .INDENT 0.0 75 | .TP 76 | .B \-q, \-\-quiet 77 | Quiet output 78 | .UNINDENT 79 | .INDENT 0.0 80 | .TP 81 | .B \-h, \-\-help 82 | Print usage. 83 | .UNINDENT 84 | .SH EXAMPLES 85 | .INDENT 0.0 86 | .IP 1. 3 87 | To compress a file: 88 | .UNINDENT 89 | .sp 90 | \fBmpirun \-np 128 dbz2 \-\-compress /path/to/file\fP 91 | .INDENT 0.0 92 | .IP 2. 3 93 | To compress a file and overwrite any existing output file: 94 | .UNINDENT 95 | .sp 96 | \fBmpirun \-np 128 dbz2 \-\-force \-\-compress /path/to/file\fP 97 | .INDENT 0.0 98 | .IP 3. 3 99 | To decompress a file: 100 | .UNINDENT 101 | .sp 102 | \fBmpirun \-np 128 dbz2 \-\-decompress /path/to/file.dbz2\fP 103 | .SH SEE ALSO 104 | .sp 105 | The mpiFileUtils source code and all documentation may be downloaded 106 | from <\fI\%https://github.com/hpc/mpifileutils\fP> 107 | .SH AUTHOR 108 | HPC 109 | .SH COPYRIGHT 110 | 2022, LLNL/LANL/UT-Battelle/DDN 111 | .\" Generated by docutils manpage writer. 112 | . 113 | -------------------------------------------------------------------------------- /doc/rst/dstripe.1.rst: -------------------------------------------------------------------------------- 1 | dstripe 2 | ======= 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | **dstripe [OPTION] PATH...** 8 | 9 | DESCRIPTION 10 | ----------- 11 | 12 | Parallel MPI application to restripe files. 13 | 14 | This tool is in active development. It currently only works on Lustre. 15 | 16 | dstripe enables one to restripe file(s) across the underlying storage 17 | devices. One must specify a list of paths. All files in those paths can 18 | be restriped. By default, stripe size is 1MB and stripe count is -1 19 | allowing dstripe to use all available stripes. 20 | 21 | OPTIONS 22 | ------- 23 | 24 | .. option:: -c, --count STRIPE_COUNT 25 | 26 | The number of stripes to use during file restriping. If STRIPE_COUNT 27 | is -1, then all available stripes are used. If STRIPE_COUNT is 0, 28 | the lustre file system default is used. The default stripe count is 29 | -1. 30 | 31 | .. option:: -s, --size STRIPE_SIZE 32 | 33 | The stripe size to use during file restriping. Units like "MB" and 34 | "GB" can immediately follow the number without spaces (ex. 2MB). The 35 | default stripe size is 1MB. 36 | 37 | .. option:: -m, --minsize SIZE 38 | 39 | The minimum size a file must be to be a candidate for restriping. 40 | Files smaller than SIZE will not be restriped. Units like "MB" and 41 | "GB" can immediately follow the number without spaces (ex. 2MB). The 42 | default minimum file size is 0MB. 43 | 44 | .. option:: -r, --report 45 | 46 | Display the file size, stripe count, and stripe size of all files 47 | found in PATH. No restriping is performed when using this option. 48 | 49 | .. option:: --progress N 50 | 51 | Print progress message to stdout approximately every N seconds. 52 | The number of seconds must be a non-negative integer. 53 | A value of 0 disables progress messages. 54 | 55 | .. option:: -v, --verbose 56 | 57 | Run in verbose mode. 58 | 59 | .. option:: -q, --quiet 60 | 61 | Run tool silently. No output is printed. 62 | 63 | .. option:: -h, --help 64 | 65 | Print the command usage, and the list of options available. 66 | 67 | EXAMPLES 68 | -------- 69 | 70 | 1. To stripe a file on all storage devices using a 1MB stripe size: 71 | 72 | ``mpirun -np 128 dstripe -s 1MB /path/to/file`` 73 | 74 | 2. To stripe a file across 20 storage devices with a 1GB stripe size: 75 | 76 | ``mpirun -np 128 dstripe -c 20 -s 1GB /path/to/file`` 77 | 78 | 3. To restripe all files in /path/to/files/ that are at least 1GB in 79 | size: 80 | 81 | ``mpirun -np 128 dstripe -m 1GB /path/to/files/`` 82 | 83 | 4. To restripe all files in /path/to/files/ across 10 storage devices 84 | with 2MB stripe size: 85 | 86 | ``mpirun -np 128 dstripe -c 10 -s 2MB /path/to/files/`` 87 | 88 | 5. To display the current stripe count and stripe size of all files in 89 | /path/to/files/: 90 | 91 | ``mpirun -np 128 dstripe -r /path/to/files/`` 92 | 93 | SEE ALSO 94 | -------- 95 | 96 | The mpiFileUtils source code and all documentation may be downloaded 97 | from 98 | -------------------------------------------------------------------------------- /man/dfilemaker.1: -------------------------------------------------------------------------------- 1 | .\" Man page generated from reStructuredText. 2 | . 3 | .TH "DFILEMAKER" "1" "Jan 29, 2025" "0.12" "mpiFileUtils" 4 | .SH NAME 5 | dfilemaker \- generate random files 6 | . 7 | .nr rst2man-indent-level 0 8 | . 9 | .de1 rstReportMargin 10 | \\$1 \\n[an-margin] 11 | level \\n[rst2man-indent-level] 12 | level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] 13 | - 14 | \\n[rst2man-indent0] 15 | \\n[rst2man-indent1] 16 | \\n[rst2man-indent2] 17 | .. 18 | .de1 INDENT 19 | .\" .rstReportMargin pre: 20 | . RS \\$1 21 | . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] 22 | . nr rst2man-indent-level +1 23 | .\" .rstReportMargin post: 24 | .. 25 | .de UNINDENT 26 | . RE 27 | .\" indent \\n[an-margin] 28 | .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] 29 | .nr rst2man-indent-level -1 30 | .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] 31 | .in \\n[rst2man-indent\\n[rst2man-indent-level]]u 32 | .. 33 | .SH SYNOPSIS 34 | .sp 35 | \fBdfilemaker \fP 36 | .SH DESCRIPTION 37 | .sp 38 | dfilemaker creates a number of random directory trees with subdirectories and 39 | files of various sizes, that is useful for testing. 40 | .sp 41 | Directory trees are created under destination_path, which must be a directory. 42 | .INDENT 0.0 43 | .TP 44 | .B destination_path 45 | Directory under which the new directory tree will be created. 46 | .UNINDENT 47 | .SH OPTIONS 48 | .INDENT 0.0 49 | .TP 50 | .B \-d, \-\-depth=*min*\-*max* 51 | Specify the depth of the file system tree to generate. The depth 52 | will be selected at random within the bounds of min and max. The 53 | default depth is set to 10 min, 20 max. 54 | .UNINDENT 55 | .INDENT 0.0 56 | .TP 57 | .B \-f, \-\-fill=*type* 58 | Specify the fill pattern of the file. Current options available are: 59 | \fBrandom\fP, \fBtrue\fP, \fBfalse\fP, and \fBalternate\fP\&. \fBrandom\fP will 60 | fill the file using \fBurandom(4)\fP\&. \fBtrue\fP will fill the file 61 | with a 0xFF pattern. \fBfalse\fP will fill the file with a 0x00 pattern. 62 | \fBalternate\fP will fill the file with a 0xAA pattern. The default 63 | fill is \fBrandom\fP\&. 64 | .UNINDENT 65 | .INDENT 0.0 66 | .TP 67 | .B \-i, \-\-seed=*integer* 68 | Specify the seed to use for random number generation. This can be 69 | used to create reproducible test runs. The default is to generate a 70 | random seed. 71 | .UNINDENT 72 | .INDENT 0.0 73 | .TP 74 | .B \-s, \-\-size=*min*\-*max* 75 | Specify the file sizes to generate. The file size will be chosen at 76 | random random within the bounds of min and max. The default file 77 | size is set from 1MB to 5MB. 78 | .UNINDENT 79 | .INDENT 0.0 80 | .TP 81 | .B \-h, \-\-help 82 | Print a brief message listing the \fIdfilemaker(1)\fP options and usage. 83 | .UNINDENT 84 | .INDENT 0.0 85 | .TP 86 | .B \-v, \-\-version 87 | Print version information and exit. 88 | .UNINDENT 89 | .SH SEE ALSO 90 | .sp 91 | The mpiFileUtils source code and all documentation may be downloaded 92 | from <\fI\%https://github.com/hpc/mpifileutils\fP> 93 | .SH AUTHOR 94 | HPC 95 | .SH COPYRIGHT 96 | 2022, LLNL/LANL/UT-Battelle/DDN 97 | .\" Generated by docutils manpage writer. 98 | . 99 | -------------------------------------------------------------------------------- /src/common/README.md: -------------------------------------------------------------------------------- 1 | # libmfu: the mpiFileUtils common library 2 | The mpiFileUtils common library defines data structures and methods on those data structures 3 | that makes it easier to develop new tools or for use within HPC applications to provide portable, performant 4 | implementations across file systems common in HPC centers. 5 | 6 | To use this library, include mfu.h. 7 | 8 | #include "mfu.h" 9 | 10 | This file includes all other necessary headers. 11 | 12 | ## mfu\_flist 13 | The key data structure in libmfu is a distributed file list called mfu\_flist. 14 | This structure represents a list of files, each with stat-like metadata, that is distributed among a set of MPI ranks. 15 | 16 | The library contains functions for creating and operating on these lists. 17 | For example, one may create a list by recursively walking an existing directory 18 | or by inserting new entries one at a time. 19 | Given a list as input, functions exist to create corresponding entries (inodes) on the file system 20 | or to delete the list of files. 21 | One may filter, sort, and remap entries. 22 | One can copy a list of entries from one location to another 23 | or compare corresponding entries across two different lists. 24 | A file list can be serialized and written to or read from a file. 25 | 26 | Each MPI rank "owns" a portion of the list, and there are routines to step through the entries owned by that process. 27 | This portion is referred to as the "local" list. 28 | Functions exist to get and set properties of the items in the local list, 29 | for example to get the path name, type, and size of a file. 30 | Functions dealing with the local list can be called by the MPI process independently of other MPI processes. 31 | 32 | Other functions operate on the global list in a collective fashion, 33 | such as deleting all items in a file list. 34 | All processes in the MPI job must invoke these functions simultaenously. 35 | 36 | For full details, see [mfu_flist.h](mfu_flist.h) and refer to its usage in existing tools. 37 | 38 | ## mfu\_path 39 | mpiFileUtils represents file paths with the [mfu_path](mfu_path.h) structure. 40 | Functions are available to manipulate paths to prepend and append entries, 41 | to slice paths into pieces, and to compute relative paths. 42 | 43 | ## mfu\_param\_path 44 | Path names provided by the user on the command line (parameters) are handled through the [mfu_param_path](mfu_param_path.h) structure. 45 | Such paths may have to be checked for existence and to determine their type (file or directory). 46 | Additionally, the user may specify many such paths through invocations involving shell wildcards, 47 | so functions are available to check long lists of paths in parallel. 48 | 49 | ## mfu\_io and mfu\_util 50 | The [mfu\_io.h](mfu_io.h) functions provide wrappers for many POSIX-IO functions. 51 | This is helpful for checking error codes in a consistent manner and automating retries on failed I/O calls. 52 | One should use the wrappers in mfu\_io if available, and if not, one should consider adding the missing wrapper. 53 | 54 | The [mfu_util.h](mfu_util.h) functions provide wrappers for error reporting and memory allocation. 55 | -------------------------------------------------------------------------------- /man/dbcast.1: -------------------------------------------------------------------------------- 1 | .\" Man page generated from reStructuredText. 2 | . 3 | .TH "DBCAST" "1" "Jan 29, 2025" "0.12" "mpiFileUtils" 4 | .SH NAME 5 | dbcast \- distributed broadcast 6 | . 7 | .nr rst2man-indent-level 0 8 | . 9 | .de1 rstReportMargin 10 | \\$1 \\n[an-margin] 11 | level \\n[rst2man-indent-level] 12 | level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] 13 | - 14 | \\n[rst2man-indent0] 15 | \\n[rst2man-indent1] 16 | \\n[rst2man-indent2] 17 | .. 18 | .de1 INDENT 19 | .\" .rstReportMargin pre: 20 | . RS \\$1 21 | . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] 22 | . nr rst2man-indent-level +1 23 | .\" .rstReportMargin post: 24 | .. 25 | .de UNINDENT 26 | . RE 27 | .\" indent \\n[an-margin] 28 | .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] 29 | .nr rst2man-indent-level -1 30 | .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] 31 | .in \\n[rst2man-indent\\n[rst2man-indent-level]]u 32 | .. 33 | .SH SYNOPSIS 34 | .sp 35 | \fBdbcast [OPTION] SRC DEST\fP 36 | .SH DESCRIPTION 37 | .sp 38 | Parallel MPI application to broadcast a single file from a 39 | global file system to node\-local storage, like ramdisk or an SSD. 40 | .sp 41 | The file is logically sliced into chunks and collectively copied from a 42 | global file system to node\-local storage. The bytes of the source file 43 | are read from the global file system once and the data are forwarded 44 | through MPI to be written to storage on all compute nodes. 45 | The source file SRC must be readable by all MPI processes. 46 | The destination file DEST should be the 47 | full path of the file in node\-local storage. If needed, parent 48 | directories for the destination file will be created as part of the 49 | broadcast. 50 | .sp 51 | In the current implementation, dbcast requires at least two MPI 52 | processes per compute node, and all compute nodes must run an equal 53 | number of MPI processes. 54 | .SH OPTIONS 55 | .INDENT 0.0 56 | .TP 57 | .B \-s, \-\-size SIZE 58 | The chunk size in bytes used to segment files during the broadcast. 59 | Units like "MB" and "GB" should be immediately follow the number 60 | without spaces (ex. 2MB). The default size is 1MB. It is recommended 61 | to use the stripe size of a file if this is known. 62 | .UNINDENT 63 | .INDENT 0.0 64 | .TP 65 | .B \-h, \-\-help 66 | Print the command usage, and the list of options available. 67 | .UNINDENT 68 | .SH EXAMPLES 69 | .INDENT 0.0 70 | .IP 1. 3 71 | To broadcast a file to /ssd on each node: 72 | .UNINDENT 73 | .sp 74 | \fBmpirun \-np 128 dbcast /global/path/to/filenane /ssd/filename\fP 75 | .INDENT 0.0 76 | .IP 2. 3 77 | Same thing, but slicing at 10MB chunks: 78 | .UNINDENT 79 | .sp 80 | \fBmpirun \-np 128 dbcast \-s 10MB /global/path/to/filenane /ssd/filename\fP 81 | .INDENT 0.0 82 | .IP 3. 3 83 | To read the current striping parameters of a file on Lustre: 84 | .UNINDENT 85 | .sp 86 | \fBlfs getstripe /global/path/to/filename\fP 87 | .SH SEE ALSO 88 | .sp 89 | The mpiFileUtils source code and all documentation may be downloaded 90 | from <\fI\%https://github.com/hpc/mpifileutils\fP> 91 | .SH AUTHOR 92 | HPC 93 | .SH COPYRIGHT 94 | 2022, LLNL/LANL/UT-Battelle/DDN 95 | .\" Generated by docutils manpage writer. 96 | . 97 | -------------------------------------------------------------------------------- /test/legacy/dcp1_tests/test_dcp1_many_dir_to_single_dir/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ############################################################################## 4 | # Description: 5 | # 6 | # A test to check if dcp1 will copy many directories to a single directory. 7 | # 8 | # Expected behavior: 9 | # 10 | # All of the source directories should be copied to the destination directory. 11 | # 12 | # Reminder: 13 | # 14 | # Lines that echo to the terminal will only be available if DEBUG is enabled 15 | # in the test runner (test_all.sh). 16 | ############################################################################## 17 | 18 | # Turn on verbose output 19 | #set -x 20 | 21 | # Print out the basic paths we'll be using. 22 | echo "Using dcp1 binary at: $DCP_TEST_BIN" 23 | echo "Using tmp directory at: $DCP_TEST_TMP" 24 | 25 | ############################################################################## 26 | # Generate the paths for: 27 | # * Four directories. 28 | # * Three files which contain random data. 29 | PATH_A_DIRECTORY="$DCP_TEST_TMP/dcp1_test_many_dir_to_single_dir.$RANDOM.tmp" 30 | PATH_B_DIRECTORY="$DCP_TEST_TMP/dcp1_test_many_dir_to_single_dir.$RANDOM.tmp" 31 | PATH_C_DIRECTORY="$DCP_TEST_TMP/dcp1_test_many_dir_to_single_dir.$RANDOM.tmp" 32 | PATH_D_DIRECTORY="$DCP_TEST_TMP/dcp1_test_many_dir_to_single_dir.$RANDOM.tmp" 33 | PATH_E_RANDOM="$DCP_TEST_TMP/$(basename $PATH_A_DIRECTORY)/dcp1_test_many_dir_to_single_dir.$RANDOM.tmp" 34 | PATH_F_RANDOM="$DCP_TEST_TMP/$(basename $PATH_B_DIRECTORY)/dcp1_test_many_dir_to_single_dir.$RANDOM.tmp" 35 | PATH_G_RANDOM="$DCP_TEST_TMP/$(basename $PATH_C_DIRECTORY)/dcp1_test_many_dir_to_single_dir.$RANDOM.tmp" 36 | 37 | # Print out the generated paths to make debugging easier. 38 | echo "A_DIRECTORY path at: $PATH_A_DIRECTORY" 39 | echo "B_DIRECTORY path at: $PATH_B_DIRECTORY" 40 | echo "C_DIRECTORY path at: $PATH_C_DIRECTORY" 41 | echo "D_DIRECTORY path at: $PATH_D_DIRECTORY" 42 | echo "E_RANDOM path at: $PATH_E_RANDOM" 43 | echo "F_RANDOM path at: $PATH_F_RANDOM" 44 | echo "G_RANDOM path at: $PATH_G_RANDOM" 45 | 46 | # Create the directories. 47 | mkdir $PATH_A_DIRECTORY 48 | mkdir $PATH_B_DIRECTORY 49 | mkdir $PATH_C_DIRECTORY 50 | mkdir $PATH_D_DIRECTORY 51 | 52 | # Create the random files. 53 | dd if=/dev/urandom of=$PATH_E_RANDOM bs=2M count=1 54 | dd if=/dev/urandom of=$PATH_F_RANDOM bs=3M count=3 55 | dd if=/dev/urandom of=$PATH_G_RANDOM bs=3M count=2 56 | 57 | ############################################################################## 58 | # Test copying several directories to a directory. The result should be the directories 59 | # placed inside the directory. 60 | 61 | mpirun -n 3 $DCP_TEST_BIN $PATH_A_DIRECTORY $PATH_B_DIRECTORY $PATH_C_DIRECTORY $PATH_D_DIRECTORY 62 | if [[ $? -ne 0 ]]; then 63 | echo "Error returned when copying an several directories to a directory (A,B,C -> D)." 64 | exit 1; 65 | fi 66 | 67 | echo "comparing $PATH_D_DIRECTORY/$(basename $PATH_A_DIRECTORY)/$(basename $PATH_E_RANDOM) to $PATH_E_RANDOM" 68 | 69 | $DCP_CMP_BIN "$PATH_D_DIRECTORY/$(basename $PATH_A_DIRECTORY)/$(basename $PATH_E_RANDOM)" $PATH_E_RANDOM 70 | if [[ $? -ne 0 ]]; then 71 | echo "CMP mismatch when copying a directory to a directory (A -> D/A/E)." 72 | exit 1 73 | fi 74 | 75 | ############################################################################## 76 | # Since we didn't find any problems, exit with success. 77 | 78 | exit 0 79 | 80 | # EOF 81 | -------------------------------------------------------------------------------- /test/legacy/dcp1_tests/test_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ############################################################################### 4 | # 5 | # A simple test runner for dcp1. 6 | # 7 | ############################################################################### 8 | 9 | # Turn this on if you want output from each test printed out. 10 | DEBUG=1 11 | 12 | # set environment variables (TEST_TMP_DIR & TEST_DCP_BIN to run tests) 13 | TEST_TMP_DIR=$1 14 | TEST_DCP_BIN=$2 15 | 16 | if [ -z $TEST_TMP_DIR ] || [ -z $TEST_DCP_BIN ]; then 17 | echo 'Please set TEST_TMP_DIR and TEST_DCP_BIN' 18 | echo 'i.e ./test_all.sh /tmp/ /path/to/dcp1/install' 19 | exit 1 20 | fi 21 | 22 | # The cmp binary to use. 23 | TEST_CMP_BIN=/usr/bin/cmp 24 | 25 | # Basic counters for summary output 26 | TESTS_RUN=0 27 | TESTS_FAILED=0 28 | TESTS_PASSED=0 29 | 30 | # Determine where the test directory is 31 | TESTS_DIR=$(dirname ${BASH_SOURCE[0]}) 32 | 33 | # If we don't find any tests, just don't run anything. 34 | shopt -s nullglob 35 | 36 | # Make sure we're in the same directory as the tests. 37 | pushd $TESTS_DIR > /dev/null 38 | 39 | # Make a temp dir for tests. 40 | mkdir $TEST_TMP_DIR 41 | 42 | echo "# ==============================================================================" 43 | echo "# Running ALL tests for DCP1." 44 | echo "# ==============================================================================" 45 | echo "# Tests started at: $(date)" 46 | echo "# ==============================================================================" 47 | 48 | # Fix up the tmp and bin paths for subshells. 49 | export DCP_TEST_BIN=$(readlink -f $TEST_DCP_BIN) 50 | export DCP_TEST_TMP=$(readlink -f $TEST_TMP_DIR) 51 | #export DCP_MPIRUN_BIN=$(readlink -f $TEST_MPIRUN_BIN) 52 | export DCP_CMP_BIN=$(readlink -f $TEST_CMP_BIN) 53 | 54 | # Tell the tests what mode we're in 55 | export DEBUG 56 | 57 | # Find and run all of the tests. 58 | for TEST in ./* 59 | do 60 | if [[ -d "$TEST" ]]; then 61 | TEST_OUT=$($TEST"/test.sh"); RETVAL=$?; 62 | 63 | if [[ $DEBUG -eq 1 ]]; then 64 | echo "$TEST_OUT" 65 | fi 66 | 67 | if [[ $RETVAL -eq 0 ]]; then 68 | echo "SUCCESS $(echo "$TEST" | sed 's/[^a-zA-Z0-9_]//g')"; 69 | TESTS_PASSED=`expr $TESTS_PASSED + 1`; 70 | fi 71 | 72 | if [[ $RETVAL -ne 0 ]]; then 73 | echo "FAILED $(echo "$TEST" | sed 's/[^a-zA-Z0-9_]//g')"; 74 | TESTS_FAILED=`expr $TESTS_FAILED + 1`; 75 | fi 76 | 77 | TESTS_RUN=`expr $TESTS_RUN + 1`; 78 | fi 79 | done 80 | 81 | echo "# ==============================================================================" 82 | echo "# DCP Test Summary:" 83 | echo "# Passed: $TESTS_PASSED" 84 | echo "# Failed: $TESTS_FAILED" 85 | echo "# ==============================================================================" 86 | echo "# Tests Run: $TESTS_RUN" 87 | echo "# Percent Passed: $(echo "scale=2; ($TESTS_PASSED*100) / $TESTS_RUN" | bc)%" 88 | echo "# ==============================================================================" 89 | echo "# Tests ended at: $(date)" 90 | echo "# ==============================================================================" 91 | 92 | # Return to the original directory where this script was run. 93 | popd > /dev/null 94 | 95 | # Return failure if any tests failed. 96 | exit $TESTS_FAILED; 97 | 98 | # EOF 99 | -------------------------------------------------------------------------------- /cmake/SetupMPI.cmake: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Copyright (c) 2017, Lawrence Livermore National Security, LLC. 3 | # 4 | # Produced at the Lawrence Livermore National Laboratory 5 | # 6 | # LLNL-CODE-725085 7 | # 8 | # All rights reserved. 9 | # 10 | # This file is part of BLT. 11 | # 12 | # For additional details, please also read BLT/LICENSE. 13 | # 14 | # Redistribution and use in source and binary forms, with or without 15 | # modification, are permitted provided that the following conditions are met: 16 | # 17 | # * Redistributions of source code must retain the above copyright notice, 18 | # this list of conditions and the disclaimer below. 19 | # 20 | # * Redistributions in binary form must reproduce the above copyright notice, 21 | # this list of conditions and the disclaimer (as noted below) in the 22 | # documentation and/or other materials provided with the distribution. 23 | # 24 | # * Neither the name of the LLNS/LLNL nor the names of its contributors may 25 | # be used to endorse or promote products derived from this software without 26 | # specific prior written permission. 27 | # 28 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 29 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 | # ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, 32 | # LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY 33 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35 | # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36 | # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 37 | # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 38 | # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 39 | # POSSIBILITY OF SUCH DAMAGE. 40 | # 41 | ############################################################################### 42 | 43 | ################################ 44 | # MPI 45 | ################################ 46 | 47 | find_package(MPI REQUIRED) 48 | message(STATUS "MPI C Compile Flags: ${MPI_C_COMPILE_FLAGS}") 49 | message(STATUS "MPI C Include Path: ${MPI_C_INCLUDE_PATH}") 50 | message(STATUS "MPI C Link Flags: ${MPI_C_LINK_FLAGS}") 51 | message(STATUS "MPI C Libraries: ${MPI_C_LIBRARIES}") 52 | 53 | message(STATUS "MPI CXX Compile Flags: ${MPI_CXX_COMPILE_FLAGS}") 54 | message(STATUS "MPI CXX Include Path: ${MPI_CXX_INCLUDE_PATH}") 55 | message(STATUS "MPI CXX Link Flags: ${MPI_CXX_LINK_FLAGS}") 56 | message(STATUS "MPI CXX Libraries: ${MPI_CXX_LIBRARIES}") 57 | 58 | message(STATUS "MPI Executable: ${MPIEXEC}") 59 | message(STATUS "MPI Num Proc Flag: ${MPIEXEC_NUMPROC_FLAG}") 60 | 61 | 62 | if (ENABLE_FORTRAN) 63 | # Determine if we should use fortran mpif.h header or fortran mpi module 64 | find_path(mpif_path 65 | NAMES "mpif.h" 66 | PATHS ${MPI_Fortran_INCLUDE_PATH} 67 | NO_DEFAULT_PATH 68 | ) 69 | 70 | if(mpif_path) 71 | set(MPI_Fortran_USE_MPIF ON CACHE PATH "") 72 | message(STATUS "Using MPI Fortran header: mpif.h") 73 | else() 74 | set(MPI_Fortran_USE_MPIF OFF CACHE PATH "") 75 | message(STATUS "Using MPI Fortran module: mpi.mod") 76 | endif() 77 | endif() 78 | -------------------------------------------------------------------------------- /test/legacy/dcp1_tests/test_dcp1_many_file_to_single_dir/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ############################################################################## 4 | # Description: 5 | # 6 | # A test to check if dcp1 will copy many files to a single directory. 7 | # 8 | # Expected behavior: 9 | # 10 | # All of the source files should be copied to the destination directory. 11 | # 12 | # Reminder: 13 | # 14 | # Lines that echo to the terminal will only be available if DEBUG is enabled 15 | # in the test runner (test_all.sh). 16 | ############################################################################## 17 | 18 | # Turn on verbose output 19 | #set -x 20 | 21 | # Print out the basic paths we'll be using. 22 | echo "Using dcp1 binary at: $DCP_TEST_BIN" 23 | echo "Using tmp directory at: $DCP_TEST_TMP" 24 | 25 | ############################################################################## 26 | # Generate the paths for: 27 | # * Two files with zero length. 28 | # * Two files which contain random data. 29 | # * A directory. 30 | PATH_A_EMPTY="$DCP_TEST_TMP/dcp1_test_many_file_to_single_dir.$RANDOM.tmp" 31 | PATH_B_EMPTY="$DCP_TEST_TMP/dcp1_test_many_file_to_single_dir.$RANDOM.tmp" 32 | PATH_C_RANDOM="$DCP_TEST_TMP/dcp1_test_many_file_to_single_dir.$RANDOM.tmp" 33 | PATH_D_RANDOM="$DCP_TEST_TMP/dcp1_test_many_file_to_single_dir.$RANDOM.tmp" 34 | PATH_E_DIRECTORY="$DCP_TEST_TMP/dcp1_test_many_file_to_single_dir.$RANDOM.tmp" 35 | 36 | # Print out the generated paths to make debugging easier. 37 | echo "A_EMPTY path at: $PATH_A_EMPTY" 38 | echo "B_EMPTY path at: $PATH_B_EMPTY" 39 | echo "C_RANDOM path at: $PATH_C_RANDOM" 40 | echo "D_RANDOM path at: $PATH_D_RANDOM" 41 | echo "E_DIRECTORY path at: $PATH_E_DIRECTORY" 42 | 43 | # Create empty files. 44 | touch $PATH_A_EMPTY 45 | touch $PATH_B_EMPTY 46 | 47 | # Create the random files. 48 | dd if=/dev/urandom of=$PATH_C_RANDOM bs=2M count=4 49 | dd if=/dev/urandom of=$PATH_D_RANDOM bs=3M count=3 50 | 51 | # Create the directory. 52 | mkdir $PATH_E_DIRECTORY 53 | 54 | ############################################################################## 55 | # Test copying several files to a directory. The result should be the files 56 | # placed inside the directory. 57 | 58 | mpirun -n 3 $DCP_TEST_BIN $PATH_A_EMPTY $PATH_C_RANDOM $PATH_B_EMPTY $PATH_D_RANDOM $PATH_E_DIRECTORY 59 | if [[ $? -ne 0 ]]; then 60 | echo "Error returned when copying an several files to a directory (A,B,C,D -> E)." 61 | exit 1; 62 | fi 63 | 64 | $DCP_CMP_BIN "$PATH_E_DIRECTORY/$(basename $PATH_A_EMPTY)" $PATH_A_EMPTY 65 | if [[ $? -ne 0 ]]; then 66 | echo "CMP mismatch when copying empty file to a directory (A -> E/B)." 67 | exit 1 68 | fi 69 | 70 | $DCP_CMP_BIN "$PATH_E_DIRECTORY/$(basename $PATH_B_EMPTY)" $PATH_B_EMPTY 71 | if [[ $? -ne 0 ]]; then 72 | echo "CMP mismatch when copying empty file to a directory (B -> E/B)." 73 | exit 1 74 | fi 75 | 76 | $DCP_CMP_BIN "$PATH_E_DIRECTORY/$(basename $PATH_C_RANDOM)" $PATH_C_RANDOM 77 | if [[ $? -ne 0 ]]; then 78 | echo "CMP mismatch when copying empty file to a directory (C -> E/C)." 79 | exit 1 80 | fi 81 | 82 | $DCP_CMP_BIN "$PATH_E_DIRECTORY/$(basename $PATH_D_RANDOM)" $PATH_D_RANDOM 83 | if [[ $? -ne 0 ]]; then 84 | echo "CMP mismatch when copying empty file to a directory (D -> E/D)." 85 | exit 1 86 | fi 87 | 88 | ############################################################################## 89 | # Since we didn't find any problems, exit with success. 90 | 91 | exit 0 92 | 93 | # EOF 94 | -------------------------------------------------------------------------------- /src/dcp1/cleanup.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file contains the logic to truncate the the destination as well as 3 | * preserve permissions and ownership. Since it would be redundant, we only 4 | * pay attention to the first chunk of each file and pass the rest along. 5 | 6 | * See the file "COPYING" for the full license governing this code. 7 | */ 8 | 9 | #include "cleanup.h" 10 | #include "dcp1.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | /** Options specified by the user. */ 22 | extern DCOPY_options_t DCOPY_user_opts; 23 | 24 | static void DCOPY_truncate_file(DCOPY_operation_t* op, \ 25 | CIRCLE_handle* handle) 26 | { 27 | char dest_path_recursive[PATH_MAX]; 28 | char dest_path_file_to_file[PATH_MAX]; 29 | 30 | if(op->dest_base_appendix == NULL) { 31 | sprintf(dest_path_recursive, "%s/%s", \ 32 | DCOPY_user_opts.dest_path, \ 33 | op->operand + op->source_base_offset + 1); 34 | 35 | strncpy(dest_path_file_to_file, DCOPY_user_opts.dest_path, PATH_MAX); 36 | } 37 | else { 38 | sprintf(dest_path_recursive, "%s/%s/%s", \ 39 | DCOPY_user_opts.dest_path, \ 40 | op->dest_base_appendix, \ 41 | op->operand + op->source_base_offset + 1); 42 | 43 | sprintf(dest_path_file_to_file, "%s/%s", \ 44 | DCOPY_user_opts.dest_path, \ 45 | op->dest_base_appendix); 46 | } 47 | 48 | MFU_LOG(MFU_LOG_DBG, "Truncating file to `%" PRId64 "'.", op->file_size); 49 | 50 | /* 51 | * Try the recursive file before file-to-file. The cast below requires us 52 | * to have a maximum file_size of 2^63, not 2^64. 53 | */ 54 | if(truncate64(dest_path_recursive, op->file_size) < 0) { 55 | if(truncate64(dest_path_file_to_file, op->file_size) < 0) { 56 | MFU_LOG(MFU_LOG_ERR, "Failed to truncate destination file: %s (errno=%d %s)", 57 | dest_path_recursive, errno, strerror(errno)); 58 | 59 | DCOPY_retry_failed_operation(COPY, handle, op); 60 | return; 61 | } 62 | } 63 | } 64 | 65 | void DCOPY_do_cleanup(DCOPY_operation_t* op, \ 66 | CIRCLE_handle* handle) 67 | { 68 | char* newop; 69 | 70 | /* 71 | * Truncate file on last chunk (synchronous mode can write past end of file) 72 | */ 73 | int64_t bytes_written = (int64_t)(op->chunk + 1) * (int64_t)DCOPY_user_opts.chunk_size; 74 | if(bytes_written >= op->file_size) { 75 | /* truncate file to appropriate size, to do this before 76 | * setting permissions in case file does not have write permission */ 77 | DCOPY_truncate_file(op, handle); 78 | 79 | /* since we still may access the file in the compare step, 80 | * delay setting permissions and timestamps until final phase */ 81 | } 82 | 83 | /* 84 | * Add work item to compare source and destination if user requested it. 85 | */ 86 | if(DCOPY_user_opts.compare) { 87 | newop = DCOPY_encode_operation(COMPARE, op->chunk, op->operand, \ 88 | op->source_base_offset, \ 89 | op->dest_base_appendix, op->file_size); 90 | 91 | handle->enqueue(newop); 92 | free(newop); 93 | } 94 | 95 | return; 96 | } 97 | 98 | /* EOF */ 99 | -------------------------------------------------------------------------------- /test/legacy/dcp1_tests/test_dcp1_single_file_to_single_dir/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ############################################################################## 4 | # Description: 5 | # 6 | # A test to check if dcp1 will copy a single file to a directory. 7 | # 8 | # Expected behavior: 9 | # 10 | # The file should be placed inside the directory. 11 | # 12 | # Reminder: 13 | # 14 | # Lines that echo to the terminal will only be available if DEBUG is enabled 15 | # in the test runner (test_all.sh). 16 | ############################################################################## 17 | 18 | # Turn on verbose output 19 | #set -x 20 | 21 | # Print out the basic paths we'll be using. 22 | echo "Using dcp1 binary at: $DCP_TEST_BIN" 23 | echo "Using cmp binary at: $DCP_CMP_BIN" 24 | echo "Using tmp directory at: $DCP_TEST_TMP" 25 | 26 | ############################################################################## 27 | # Generate the paths for: 28 | # * A file that doesn't exist at all. 29 | # * A file with zero length. 30 | # * A file which contains random data. 31 | PATH_A_NOEXIST="$DCP_TEST_TMP/dcp1_test_single_file_to_single_dir.$RANDOM.tmp" 32 | PATH_B_EMPTY="$DCP_TEST_TMP/dcp1_test_single_file_to_single_dir.$RANDOM.tmp" 33 | PATH_C_RANDOM="$DCP_TEST_TMP/dcp1_test_single_file_to_single_dir.$RANDOM.tmp" 34 | PATH_D_DIRECTORY="$DCP_TEST_TMP/dcp1_test_single_file_to_single_dir.$RANDOM.tmp" 35 | 36 | # Print out the generated paths to make debugging easier. 37 | echo "A_NOEXIST path at: $PATH_A_NOEXIST" 38 | echo "B_EMPTY path at: $PATH_B_EMPTY" 39 | echo "C_RANDOM path at: $PATH_C_RANDOM" 40 | echo "D_DIRECTORY path at: $PATH_D_DIRECTORY" 41 | 42 | # Create the empty file. 43 | touch $PATH_B_EMPTY 44 | 45 | # Create the random file. 46 | dd if=/dev/urandom of=$PATH_C_RANDOM bs=4M count=5 47 | 48 | # Create the directory. 49 | mkdir $PATH_D_DIRECTORY 50 | 51 | ############################################################################## 52 | # Test copying a no-exist file to a directory. The result should be an error. 53 | 54 | mpirun -n 3 $DCP_TEST_BIN $PATH_A_NOEXIST $PATH_D_DIRECTORY 55 | 56 | if [[ $? -eq 0 ]]; then 57 | echo "Unexpected success when copying a no-exist file to a directory (A -> D)." 58 | exit 1; 59 | fi 60 | 61 | ############################################################################## 62 | # Test copying an empty file to a directory. The result should be the empty 63 | # file placed inside the directory. 64 | 65 | mpirun -n 3 $DCP_TEST_BIN $PATH_B_EMPTY $PATH_D_DIRECTORY 66 | if [[ $? -ne 0 ]]; then 67 | echo "Error returned when copying an empty file to a directory (B -> D)." 68 | exit 1; 69 | fi 70 | 71 | $DCP_CMP_BIN "$PATH_D_DIRECTORY/$(basename $PATH_B_EMPTY)" $PATH_B_EMPTY 72 | if [[ $? -ne 0 ]]; then 73 | echo "CMP mismatch when copying empty file to a directory (B -> D/B)." 74 | exit 1 75 | fi 76 | 77 | ############################################################################## 78 | # Test copying an empty file to a directory. The result should be the empty 79 | # file placed inside the directory. 80 | 81 | mpirun -n 3 $DCP_TEST_BIN $PATH_C_RANDOM $PATH_D_DIRECTORY 82 | if [[ $? -ne 0 ]]; then 83 | echo "Error returned when copying a random file to a directory (C -> D)." 84 | exit 1; 85 | fi 86 | 87 | $DCP_CMP_BIN "$PATH_D_DIRECTORY/$(basename $PATH_C_RANDOM)" $PATH_C_RANDOM 88 | if [[ $? -ne 0 ]]; then 89 | echo "CMP mismatch when copying a random file to a directory (C -> D/C)." 90 | exit 1 91 | fi 92 | 93 | ############################################################################## 94 | # Since we didn't find any problems, exit with success. 95 | 96 | exit 0 97 | 98 | # EOF 99 | -------------------------------------------------------------------------------- /man/dreln.1: -------------------------------------------------------------------------------- 1 | .\" Man page generated from reStructuredText. 2 | . 3 | .TH "DRELN" "1" "Jan 29, 2025" "0.12" "mpiFileUtils" 4 | .SH NAME 5 | dreln \- distributed relink 6 | . 7 | .nr rst2man-indent-level 0 8 | . 9 | .de1 rstReportMargin 10 | \\$1 \\n[an-margin] 11 | level \\n[rst2man-indent-level] 12 | level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] 13 | - 14 | \\n[rst2man-indent0] 15 | \\n[rst2man-indent1] 16 | \\n[rst2man-indent2] 17 | .. 18 | .de1 INDENT 19 | .\" .rstReportMargin pre: 20 | . RS \\$1 21 | . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] 22 | . nr rst2man-indent-level +1 23 | .\" .rstReportMargin post: 24 | .. 25 | .de UNINDENT 26 | . RE 27 | .\" indent \\n[an-margin] 28 | .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] 29 | .nr rst2man-indent-level -1 30 | .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] 31 | .in \\n[rst2man-indent\\n[rst2man-indent-level]]u 32 | .. 33 | .SH SYNOPSIS 34 | .sp 35 | \fBdreln [OPTION] OLDPATH NEWPATH PATH ...\fP 36 | .SH DESCRIPTION 37 | .sp 38 | Parallel MPI application to recursively update symlinks within a 39 | directory. 40 | .sp 41 | dreln walks the specified PATH and updates any symlink whose target 42 | includes an absolute path to OLDPATH and replaces that symlink 43 | with a new link whose target points to NEWPATH instead. 44 | .sp 45 | This is useful to update symlinks after migrating a large 46 | directory from one file system to another, whose links specify 47 | absolute paths to the original file system. 48 | .SH OPTIONS 49 | .INDENT 0.0 50 | .TP 51 | .B \-i, \-\-input FILE 52 | Read source list from FILE. FILE must be generated by another tool 53 | from the mpiFileUtils suite. 54 | .UNINDENT 55 | .INDENT 0.0 56 | .TP 57 | .B \-p, \-\-preserve 58 | Preserve existing modification times on links. 59 | .UNINDENT 60 | .INDENT 0.0 61 | .TP 62 | .B \-r, \-\-relative 63 | Replace links using target paths that are relative to NEWPATH. 64 | .UNINDENT 65 | .INDENT 0.0 66 | .TP 67 | .B \-\-progress N 68 | Print progress message to stdout approximately every N seconds. 69 | The number of seconds must be a non\-negative integer. 70 | A value of 0 disables progress messages. 71 | .UNINDENT 72 | .INDENT 0.0 73 | .TP 74 | .B \-v, \-\-verbose 75 | Run in verbose mode. 76 | .UNINDENT 77 | .INDENT 0.0 78 | .TP 79 | .B \-q, \-\-quiet 80 | Run tool silently. No output is printed. 81 | .UNINDENT 82 | .INDENT 0.0 83 | .TP 84 | .B \-h, \-\-help 85 | Print a brief message listing the \fBdrm(1)\fP options and usage. 86 | .UNINDENT 87 | .SH EXAMPLES 88 | .sp 89 | 1. To update all links under /walk/path whose targets point to /orig/path 90 | and replace them with targets that point to /new/path: 91 | .sp 92 | \fBmpirun \-np 128 dreln \-v /orig/path /new/path /walk/path\fP 93 | .sp 94 | 2. Same as above, but replace each link target with a relative path 95 | from the link to its new target under /new/path: 96 | .sp 97 | \fBmpirun \-np 128 dreln \-v \-\-relative /orig/path /new/path /walk/path\fP 98 | .INDENT 0.0 99 | .IP 3. 3 100 | One can preserve existing modification times on links: 101 | .UNINDENT 102 | .sp 103 | \fBmpirun \-np 128 dreln \-v \-\-preserve /orig/path /new/path /walk/path\fP 104 | .INDENT 0.0 105 | .IP 4. 3 106 | One can specifiy multiple paths to walk: 107 | .UNINDENT 108 | .sp 109 | \fBmpirun \-np 128 dreln \-v /orig/path /new/path /walk/path1 /walk/path2\fP 110 | .SH SEE ALSO 111 | .sp 112 | The mpiFileUtils source code and all documentation may be downloaded 113 | from <\fI\%https://github.com/hpc/mpifileutils\fP> 114 | .SH AUTHOR 115 | HPC 116 | .SH COPYRIGHT 117 | 2022, LLNL/LANL/UT-Battelle/DDN 118 | .\" Generated by docutils manpage writer. 119 | . 120 | -------------------------------------------------------------------------------- /test/legacy/dcp1_tests/test_dcp1_many_dir_to_single_file/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ############################################################################## 4 | # Description: 5 | # 6 | # A test to check if dcp1 will copy many directories to a single file. 7 | # 8 | # Expected behavior: 9 | # 10 | # An error code should be displayed if this action is attempted. 11 | # 12 | # Reminder: 13 | # 14 | # Lines that echo to the terminal will only be available if DEBUG is enabled 15 | # in the test runner (test_all.sh). 16 | ############################################################################## 17 | 18 | # Turn on verbose output 19 | #set -x 20 | 21 | # Print out the basic paths we'll be using. 22 | echo "Using dcp1 binary at: $DCP_TEST_BIN" 23 | echo "Using tmp directory at: $DCP_TEST_TMP" 24 | 25 | ############################################################################## 26 | # Generate the paths for: 27 | # * A file with zero length. 28 | # * A file which contains random data. 29 | # * A file which doesn't exist. 30 | # * Four directories. 31 | PATH_A_EMPTY="$DCP_TEST_TMP/dcp1_test_many_dir_to_single_file.$RANDOM.tmp" 32 | PATH_B_RANDOM="$DCP_TEST_TMP/dcp1_test_many_dir_to_single_file.$RANDOM.tmp" 33 | PATH_C_NOEXIST="$DCP_TEST_TMP/dcp1_test_many_dir_to_single_file.$RANDOM.tmp" 34 | PATH_D_DIRECTORY="$DCP_TEST_TMP/dcp1_test_many_dir_to_single_file.$RANDOM.tmp" 35 | PATH_E_DIRECTORY="$DCP_TEST_TMP/dcp1_test_many_dir_to_single_file.$RANDOM.tmp" 36 | PATH_F_DIRECTORY="$DCP_TEST_TMP/dcp1_test_many_dir_to_single_file.$RANDOM.tmp" 37 | PATH_G_DIRECTORY="$DCP_TEST_TMP/dcp1_test_many_dir_to_single_file.$RANDOM.tmp" 38 | 39 | # Print out the generated paths to make debugging easier. 40 | echo "A_EMPTY path at: $PATH_A_EMPTY" 41 | echo "B_RANDOM path at: $PATH_B_RANDOM" 42 | echo "C_NOEXIST path at: $PATH_C_NOEXIST" 43 | echo "D_DIRECTORY path at: $PATH_D_DIRECTORY" 44 | echo "E_DIRECTORY path at: $PATH_E_DIRECTORY" 45 | echo "F_DIRECTORY path at: $PATH_F_DIRECTORY" 46 | echo "G_DIRECTORY path at: $PATH_G_DIRECTORY" 47 | 48 | # Create an empty file. 49 | touch $PATH_A_EMPTY 50 | 51 | # Create the random file. 52 | dd if=/dev/urandom of=$PATH_B_RANDOM bs=2M count=4 53 | 54 | # Create the directories. 55 | mkdir $PATH_D_DIRECTORY 56 | mkdir $PATH_E_DIRECTORY 57 | mkdir $PATH_F_DIRECTORY 58 | mkdir $PATH_G_DIRECTORY 59 | 60 | ############################################################################## 61 | # Test copying several directories into an empty file. This should result in 62 | # an error. 63 | 64 | mpirun -n 3 $DCP_TEST_BIN $PATH_D_DIRECTORY $PATH_E_DIRECTORY $PATH_F_DIRECTORY $PATH_A_EMPTY 65 | if [[ $? -eq 0 ]]; then 66 | echo "Unexpected success when copying several directories to an empty file. (D,E,F -> A)." 67 | exit 1; 68 | fi 69 | 70 | ############################################################################## 71 | # Test copying several directories into a random file. This should result in 72 | # an error. 73 | 74 | mpirun -n 3 $DCP_TEST_BIN $PATH_D_DIRECTORY $PATH_E_DIRECTORY $PATH_F_DIRECTORY $PATH_B_RANDOM 75 | if [[ $? -eq 0 ]]; then 76 | echo "Unexpected success when copying several directories to a random file. (D,E,F -> B)." 77 | exit 1; 78 | fi 79 | 80 | ############################################################################## 81 | # Test copying several directories into a no-exist file. This should result in 82 | # an error. 83 | 84 | mpirun -n 3 $DCP_TEST_BIN $PATH_D_DIRECTORY $PATH_E_DIRECTORY $PATH_F_DIRECTORY $PATH_C_NOEXIST 85 | if [[ $? -eq 0 ]]; then 86 | echo "Unexpected success when copying several directories to a no-exist file. (D,E,F -> C)." 87 | exit 1; 88 | fi 89 | 90 | ############################################################################## 91 | # Since we didn't find any problems, exit with success. 92 | 93 | exit 0 94 | 95 | # EOF 96 | -------------------------------------------------------------------------------- /doc/rst/drm.1.rst: -------------------------------------------------------------------------------- 1 | drm 2 | === 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | **drm [OPTION] PATH...** 8 | 9 | DESCRIPTION 10 | ----------- 11 | 12 | Parallel MPI application to recursively delete a directory and its 13 | contents. 14 | 15 | drm is a tool for removing files recursively in parallel. 16 | drm behaves like `rm -rf`, but it is faster. 17 | 18 | .. note:: 19 | 20 | DO NOT USE SHELL REGEX!!! 21 | The --match and --exclude options use POSIX regex syntax. Because of 22 | this make sure that the shell does not try to interpret your regex before 23 | it gets passed to the program. You can generally use quotes around your 24 | regex to prevent the shell from expanding. An example of this using the 25 | --match option with --dryrun would be: 26 | 27 | ``mpirun -np 128 drm --dryrun -v --name --match 'file_.*' /path/to/dir/*`` 28 | 29 | OPTIONS 30 | ------- 31 | 32 | .. option:: -i, --input FILE 33 | 34 | Read source list from FILE. FILE must be generated by another tool 35 | from the mpiFileUtils suite. 36 | 37 | .. option:: -o, --output FILE 38 | 39 | Write the list of items drm attempts to delete to FILE in mpiFileUtils format. 40 | Format can be changed with --text option. 41 | 42 | .. option:: -t, --text 43 | 44 | Must be used with the --output option. Write list of items drm attempts 45 | to delete to FILE in ascii text format. 46 | 47 | .. option:: -l, --lite 48 | 49 | Walk file system without stat. 50 | 51 | .. option:: --stat 52 | 53 | Walk file system with stat. 54 | 55 | .. option:: --exclude REGEX 56 | 57 | Do not remove items whose full path matches REGEX, processed by :manpage:`regexec(3)`. 58 | 59 | .. option:: --match REGEX 60 | 61 | Only remove items whose full path matches REGEX, processed by 62 | :manpage:`regexec(3)`. 63 | 64 | .. option:: --name 65 | 66 | Change --exclude and match to apply to item name rather than its 67 | full path. 68 | 69 | .. option:: --dryrun 70 | 71 | Print a list of files that **would** be deleted without deleting 72 | them. This is useful to check list of items satisfying --exclude or 73 | --match options before actually deleting anything. 74 | 75 | .. option:: --aggressive 76 | 77 | This option will delete files during the walk phase, and then 78 | delete directories by level after the walk in drm. You cannot 79 | use this option with --dryrun. 80 | 81 | .. option:: -T, --traceless 82 | 83 | Delete child items without updating the mtime on their parent directory. 84 | 85 | .. option:: --progress N 86 | 87 | Print progress message to stdout approximately every N seconds. 88 | The number of seconds must be a non-negative integer. 89 | A value of 0 disables progress messages. 90 | 91 | .. option:: -v, --verbose 92 | 93 | Run in verbose mode. 94 | 95 | .. option:: -q, --quiet 96 | 97 | Run tool silently. No output is printed. 98 | 99 | .. option:: -h, --help 100 | 101 | Print a brief message listing the :manpage:`drm(1)` options and usage. 102 | 103 | EXAMPLES 104 | -------- 105 | 106 | 1. To delete a directory and its contents: 107 | 108 | ``mpirun -np 128 drm -v /dir/to/delete`` 109 | 110 | 2. Delete all items (files and directories) ending with .core from 111 | directory tree: 112 | 113 | ``mpirun -np 128 drm --match '.core$' /dir/to/delete/from`` 114 | 115 | 3. List items that would be deleted without removing them: 116 | 117 | ``mpirun -np 128 drm --dryrun --match '.core$' /dir/to/delete/from`` 118 | 119 | 4. Delete all items named foo: 120 | 121 | ``mpirun -np 128 drm --name --match '^foo$' /dir/to/delete/from`` 122 | 123 | SEE ALSO 124 | -------- 125 | 126 | The mpiFileUtils source code and all documentation may be downloaded 127 | from 128 | -------------------------------------------------------------------------------- /doc/rst/dchmod.1.rst: -------------------------------------------------------------------------------- 1 | .. include:: 2 | dchmod 3 | ====== 4 | 5 | SYNOPSIS 6 | -------- 7 | 8 | **dchmod [OPTION] PATH ...** 9 | 10 | DESCRIPTION 11 | ----------- 12 | 13 | Parallel MPI application to recursively change permissions and/or group 14 | from a top level directory. 15 | 16 | dchmod provides functionality similar to :manpage:`chmod(1)`, :manpage:`chown(1)`, and :manpage:`chgrp(1)`. 17 | Like :manpage:`chmod(1)`, the tool supports the use of octal or symbolic mode to 18 | change the permissions. 19 | 20 | OPTIONS 21 | ------- 22 | 23 | .. option:: -i, --input FILE 24 | 25 | Read source list from FILE. FILE must be generated by another tool 26 | from the mpiFileUtils suite. 27 | 28 | .. option:: -u, --owner USER 29 | 30 | Change owner to specified USER name or numeric user id. 31 | 32 | .. option:: -g, --group GROUP 33 | 34 | Change group to specified GROUP name or numeric group id. 35 | 36 | .. option:: -m, --mode MODE 37 | 38 | The mode to apply to each item. MODE may be octal or symbolic syntax 39 | similar to :manpage:`chmod(1)`. In symbolic notation, "ugoa" are supported 40 | as are "rwxX". As with chmod, if no leading letter "ugoa" is provided, 41 | mode bits are combined with umask to determine the actual mode. 42 | 43 | .. option:: -f, --force 44 | 45 | Attempt to change every item. By default, dchmod avoids unncessary 46 | chown and chmod calls, for example trying to change the group 47 | on an item that already has the correct group, or trying to change 48 | the group on an item that is not owned by the user running the tool. 49 | With --force, dchmod executes chown/chmod calls on every item. 50 | 51 | .. option:: -s, --silent 52 | 53 | Suppress EPERM error messages, which is useful when running dchmod 54 | on large directories with files owned by other users. 55 | 56 | .. option:: --exclude REGEX 57 | 58 | Do not modify items whose full path matches REGEX, processed by 59 | :manpage:`regexec(3)`. 60 | 61 | .. option:: --match REGEX 62 | 63 | Only modify items whose full path matches REGEX, processed by 64 | :manpage:`regexec(3)`. 65 | 66 | .. option:: -n, --name 67 | 68 | Change --exclude and --match to apply to item name rather than its 69 | full path. 70 | 71 | .. option:: --progress N 72 | 73 | Print progress message to stdout approximately every N seconds. 74 | The number of seconds must be a non-negative integer. 75 | A value of 0 disables progress messages. 76 | 77 | .. option:: -v, --verbose 78 | 79 | Run in verbose mode. Prints a list of statistics including the 80 | number of files walked, the number of levels there are in the 81 | directory tree, and the number of files the command operated on, and 82 | the files/sec rate for each of those. 83 | 84 | .. option:: -q, --quiet 85 | 86 | Run tool silently. No output is printed. 87 | 88 | .. option:: -h, --help 89 | 90 | Print the command usage, and the list of options available. 91 | 92 | EXAMPLES 93 | -------- 94 | 95 | 1. Use octal mode to change permissions: 96 | 97 | ``mpirun -np 128 dchmod --mode 755 /directory`` 98 | 99 | 2. Set group and mode in a single command using symbolic mode: 100 | 101 | ``mpirun -np 128 dchmod --group mygroup --mode u+r,g+rw /directory`` 102 | 103 | 3. Set owner and group, leaving permissions the same: 104 | 105 | ``mpirun -np 128 dchmod --owner user1 --group mygroup /directory`` 106 | 107 | 4. Change permissions to u+rw on all items EXCEPT those whose name match 108 | regex: 109 | 110 | ``mpirun -np 128 dchmod --name --exclude ‘afilename’ --mode u+rw /directory`` 111 | 112 | Note: You can use --match to change file permissions on all of the 113 | files/directories that match the regex. 114 | 115 | SEE ALSO 116 | -------- 117 | 118 | The mpiFileUtils source code and all documentation may be downloaded 119 | from 120 | -------------------------------------------------------------------------------- /dist/builddist: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | print_usage() { 4 | echo "Usage: builddist " 5 | echo "" 6 | echo "Tags:" 7 | echo " main - build tarball of latest" 8 | echo " v0.12 - build tarball of v0.12" 9 | } 10 | 11 | # check that we got an argument or print usage 12 | if [ $# -ne 1 ] ; then 13 | print_usage 14 | exit 1 15 | fi 16 | 17 | # for a given release, define tags for each component 18 | if [ "$1" == "main" ] ; then 19 | # to build from latest branch of all repos 20 | ORGS=( 21 | "lwgrp" "llnl" "main" 22 | "dtcmp" "llnl" "main" 23 | "libcircle" "hpc" "master" 24 | "mpifileutils" "hpc" "master" 25 | ) 26 | elif [ "$1" == "v0.12" ] ; then 27 | # to build from latest branch of all repos 28 | ORGS=( 29 | "lwgrp" "llnl" "v1.0.6" 30 | "dtcmp" "llnl" "v1.1.5" 31 | "libcircle" "hpc" "v0.3" 32 | "mpifileutils" "hpc" "v0.12" 33 | ) 34 | else 35 | echo "Error: unknown tag: $1" 36 | echo "" 37 | print_usage 38 | exit 1 39 | fi 40 | 41 | set -x 42 | 43 | # we assume everything is hosted at github 44 | REPOHOST=https://github.com 45 | 46 | # create a temporary directory to package things up 47 | rm -rf dist 48 | mkdir dist 49 | cd dist 50 | 51 | ARCH_DIR="archive" 52 | rm -rf $ARCH_DIR 53 | mkdir -p $ARCH_DIR 54 | 55 | len=${#ORGS[@]} 56 | for (( i=0; i<${len}; i=$(($i + 3)) )); do 57 | # component name 58 | component=${ORGS[$i]} 59 | 60 | # github path to component 61 | j=$(($i + 1)) 62 | repo=$REPOHOST/${ORGS[$j]}/$component 63 | 64 | # repo tag to checkout 65 | j=$(($i + 2)) 66 | TAG=${ORGS[$j]} 67 | 68 | # clone the repo 69 | git clone --depth 1 --branch $TAG $repo 70 | 71 | # git archive the source files into a tarfile 72 | cd $component 73 | #TAG=`git describe --tags $(git rev-list --tags --max-count=1)` 74 | git archive --format=tar --prefix=$component/ $TAG | gzip > $component-$TAG.tar.gz 2> /dev/null 75 | cd .. 76 | 77 | # unpack source files for this component in a directory with other components 78 | cd $ARCH_DIR 79 | tar -zxf ../$component/$component-$TAG.tar.gz 80 | 81 | # hack out include of autotools config.h (not used anyway) 82 | if [ "$component" == "lwgrp" ] ; then 83 | sed -i 's@#include "../config/config.h"@@g' lwgrp/src/lwgrp_internal.h 84 | fi 85 | if [ "$component" == "libcircle" ] ; then 86 | sed -i 's@#include @@g' libcircle/libcircle/lib.h 87 | fi 88 | 89 | # hack out common dir for library (maybe could leave this in) 90 | if [ "$component" == "mpifileutils" ] ; then 91 | sed -i 's@ADD_SUBDIRECTORY(common)@#ADD_SUBDIRECTORY(common)@g' mpifileutils/src/CMakeLists.txt 92 | fi 93 | 94 | # remove doc and test directories for a smaller tarball 95 | rm -rf ${component}/doc 96 | rm -rf ${component}/doc-dev 97 | if [ "$component" != "mpifileutils" ] ; then 98 | rm -rf ${component}/test 99 | fi 100 | cd .. 101 | done 102 | 103 | # NOTE: last TAG is from SCR 104 | # rename archive directory to mpifileutils-TAG 105 | mv $ARCH_DIR mpifileutils-$TAG 106 | 107 | # copy in top-level CMake files 108 | cp -r ../CMakeLists.txt ../../cmake mpifileutils-$TAG 109 | 110 | # copy in README 111 | cp ../README.dist mpifileutils-$TAG/README 112 | 113 | # drop FindDTCMP and FindLWGRP cmake files 114 | #rm -f mpifileutils-$TAG/cmake/{FindDTCMP.cmake,FindLibCircle.cmake} 115 | 116 | # delete original CMakeLists to avoid confusion 117 | rm -rf mpifileutils-$TAG/mpifileutils/CMakeLists.txt mpifileutils-$TAG/mpifileutils/cmake 118 | 119 | # zip up release tarball 120 | tar -czf ../mpifileutils-${TAG}.tgz mpifileutils-$TAG 121 | 122 | # delete prep directory 123 | cd .. 124 | rm -rf dist 125 | -------------------------------------------------------------------------------- /doc/rst/dwalk.1.rst: -------------------------------------------------------------------------------- 1 | dwalk 2 | ===== 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | **dwalk [OPTION] PATH ...** 8 | 9 | DESCRIPTION 10 | ----------- 11 | 12 | Parallel MPI application to recursively walk and list contents in a 13 | directory. 14 | 15 | dwalk provides functionality similar to :manpage:`ls(1)` and :manpage:`du(1)`. Like 16 | :manpage:`du(1)`, the tool reports a summary of the total number of files and 17 | bytes. Like :manpage:`ls(1)`, the tool sorts and prints information about 18 | individual files. 19 | 20 | The output can be sorted on different fields (e.g, name, user, group, 21 | size, etc). A histogram of file sizes can be computed listing the number 22 | of files that fall into user-defined bins. 23 | 24 | OPTIONS 25 | ------- 26 | 27 | .. option:: -i, --input FILE 28 | 29 | Read source list from FILE. FILE must be generated by another tool 30 | from the mpiFileUtils suite. 31 | 32 | .. option:: -o, --output FILE 33 | 34 | Write the processed list to FILE in binary format. Format can be changed 35 | With --text option. 36 | 37 | .. option:: -t, --text 38 | 39 | Must be used with the --output option. Write processed list of files to 40 | FILE in ascii text format. 41 | 42 | .. option:: -l, --lite 43 | 44 | Walk file system without stat. 45 | 46 | .. option:: -s, --sort FIELD 47 | 48 | Sort output by comma-delimited fields (see below). 49 | 50 | .. option:: -d, --distribution size:SEPARATORS 51 | 52 | Print the distribution of file sizes. For example, specifying 53 | size:0,80,100 will report the number of files that have size 0 54 | bytes, between 1-80 bytes, between 81-99 bytes, and 100 bytes or 55 | greater. 56 | 57 | .. option:: -f, --file-histogram 58 | 59 | Creates a file histogram without requiring the user to provide 60 | the bin sizes. The bins are created dynamically based on the 61 | max file size. The first bin is always for only zero byte 62 | files, and the rest go up until the max file size is included 63 | in the very last bin. It always goes up by orders of magnitude 64 | in powers of two. So, an example of bin separators would be: 65 | 0, 2^10, 2^20, 2^30. Assuming the max file size was somewhere 66 | within the 2^20 - 2^30 range. The histogram also includes both 67 | files and directories. 68 | 69 | .. option:: -p, --print 70 | 71 | Print files to the screen. 72 | 73 | .. option:: --no-atime 74 | 75 | Must be used with --lite option. Do not update last file access time. 76 | 77 | .. option:: -L, --dereference 78 | 79 | Dereference symbolic links and walk the target file or directory 80 | that each symbolic link refers to. 81 | 82 | .. option:: --progress N 83 | 84 | Print progress message to stdout approximately every N seconds. 85 | The number of seconds must be a non-negative integer. 86 | A value of 0 disables progress messages. 87 | 88 | .. option:: -v, --verbose 89 | 90 | Run in verbose mode. 91 | 92 | .. option:: -q, --quiet 93 | 94 | Run tool silently. No output is printed. 95 | 96 | .. option:: -h, --help 97 | 98 | Print usage. 99 | 100 | SORT FIELDS 101 | ----------- 102 | 103 | By default, the list of files dwalk captures is not sorted. To sort the 104 | list, one or more fields can be specified in a comma-delimited list: 105 | 106 | name,user,group,uid,gid,atime,mtime,ctime,size 107 | 108 | A field name can be preceded with ‘-’ to sort by that field in reverse 109 | order. 110 | 111 | A lexicographic sort is executed if more than one field is given. 112 | 113 | EXAMPLES 114 | -------- 115 | 116 | 1. To print summary information for a directory: 117 | 118 | ``mpirun -np 128 dwalk -v /dir/to/walk`` 119 | 120 | 2. To print a list of files, sorted by file size, then by file name: 121 | 122 | ``mpirun -np 128 dwalk –print –sort size,name /dir/to/walk`` 123 | 124 | 3. To save the list of files: 125 | 126 | ``mpirun -np 128 dwalk –output out.dwalk /dir/to/walk`` 127 | 128 | 4. Print the file distribution for specified histogram based on the size 129 | field from the top level directory. 130 | 131 | ``mpirun -np 128 dwalk -v –print -d size:0,20,1G src/`` 132 | 133 | SEE ALSO 134 | -------- 135 | 136 | The mpiFileUtils source code and all documentation may be downloaded 137 | from 138 | -------------------------------------------------------------------------------- /src/common/mfu_bz2.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | /* for statfs */ 9 | #include 10 | 11 | /* for LL_SUPER_MAGIC */ 12 | #if LUSTRE_SUPPORT 13 | #include 14 | #endif /* LUSTRE_SUPPORT */ 15 | 16 | #include "mpi.h" 17 | #include "mfu.h" 18 | 19 | int mfu_compress_bz2_libcircle(const char* src_name, const char* dst_name, int b_size, ssize_t opts_memory); 20 | int mfu_compress_bz2_static(const char* src_name, const char* dst_name, int b_size); 21 | 22 | int mfu_decompress_bz2_libcircle(const char* src_name, const char* dst_name); 23 | int mfu_decompress_bz2_static(const char* src_name, const char* dst_name); 24 | 25 | int mfu_compress_bz2(const char* src_name, const char* dst_name, int b_size) 26 | { 27 | //return mfu_compress_bz2_libcircle(src_name, dst_name, b_size, 0); 28 | return mfu_compress_bz2_static(src_name, dst_name, b_size); 29 | } 30 | 31 | 32 | int mfu_decompress_bz2(const char* src_name, const char* dst_name) 33 | { 34 | //return mfu_decompress_bz2_libcircle(src_name, dst_name); 35 | return mfu_decompress_bz2_static(src_name, dst_name); 36 | } 37 | 38 | static int mfu_create_output(const char* name, mode_t mode) 39 | { 40 | /* get our rank */ 41 | int rank; 42 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 43 | 44 | /* The file for output is opened and options set */ 45 | int fd = -1; 46 | if (rank == 0) { 47 | /* open file */ 48 | fd = mfu_open(name, O_WRONLY | O_CREAT | O_TRUNC, mode); 49 | if (fd < 0) { 50 | MFU_LOG(MFU_LOG_ERR, "Failed to open file for writing: %s errno=%d (%s)", 51 | name, errno, strerror(errno)); 52 | } 53 | } 54 | 55 | /* wait for rank 0 to finish operations */ 56 | MPI_Barrier(MPI_COMM_WORLD); 57 | 58 | /* have rest of ranks open the file */ 59 | if (rank != 0) { 60 | fd = mfu_open(name, O_WRONLY); 61 | if (fd < 0) { 62 | MFU_LOG(MFU_LOG_ERR, "Failed to open file for writing: %s errno=%d (%s)", 63 | name, errno, strerror(errno)); 64 | } 65 | } 66 | 67 | return fd; 68 | } 69 | 70 | int mfu_create_fully_striped(const char* name, mode_t mode) 71 | { 72 | /* get our rank */ 73 | int rank; 74 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 75 | 76 | #if LUSTRE_SUPPORT 77 | /* have rank 0 check whether the file will be on Lustre */ 78 | int on_lustre = 0; 79 | if (rank == 0) { 80 | /* get directory path for file */ 81 | mfu_path* dirpath = mfu_path_from_str(name); 82 | mfu_path_dirname(dirpath); 83 | char* dirstr = mfu_path_strdup(dirpath); 84 | 85 | /* statfs the directory */ 86 | errno = 0; 87 | struct statfs fs_stat; 88 | if (statfs(dirstr, &fs_stat) == 0) { 89 | /* set to 1 if this path is on lustre, 0 otherwise */ 90 | on_lustre = (fs_stat.f_type == LL_SUPER_MAGIC); 91 | } else { 92 | MFU_LOG(MFU_LOG_ERR, "Failed to statfs: `%s' errno=%d (%s)", 93 | dirstr, errno, strerror(errno)); 94 | } 95 | 96 | /* free the directory name */ 97 | mfu_free(&dirstr); 98 | mfu_path_delete(&dirpath); 99 | } 100 | 101 | /* broadcast result from rank 0 */ 102 | MPI_Bcast(&on_lustre, 1, MPI_INT, 0, MPI_COMM_WORLD); 103 | 104 | /* if on lustre, set striping while opening file, 105 | * otherwise fallback to something basic */ 106 | int fd = -1; 107 | if (on_lustre) { 108 | /* have rank 0 create the file with striping */ 109 | if (rank == 0) { 110 | mfu_stripe_set(name, 1024*1024, -1); 111 | } 112 | 113 | /* wait for rank 0 to finish operations */ 114 | MPI_Barrier(MPI_COMM_WORLD); 115 | 116 | /* open the file */ 117 | fd = mfu_open(name, O_WRONLY); 118 | if (fd < 0) { 119 | MFU_LOG(MFU_LOG_ERR, "Failed to open file for writing: %s errno=%d (%s)", 120 | name, errno, strerror(errno)); 121 | } 122 | } else { 123 | fd = mfu_create_output(name, mode); 124 | } 125 | #else 126 | int fd = mfu_create_output(name, mode); 127 | #endif 128 | 129 | return fd; 130 | } 131 | -------------------------------------------------------------------------------- /test/legacy/dcp1_tests/test_dcp1_single_file_to_single_file/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ############################################################################## 4 | # Description: 5 | # 6 | # A test to check if dcp1 will copy a single file to another single file. 7 | # 8 | # Expected behavior: 9 | # 10 | # The first file must overwrite the second file if the second file exists. 11 | # If the second file does not exist, it should be created and the contents 12 | # of the first file should be copied in. If the first file does not exist on 13 | # disk, an error should be reported. 14 | # 15 | # Reminder: 16 | # 17 | # Lines that echo to the terminal will only be available if DEBUG is enabled 18 | # in the test runner (test_all.sh). 19 | ############################################################################## 20 | 21 | # Turn on verbose output 22 | #set -x 23 | 24 | # Print out the basic paths we'll be using. 25 | echo "Using dcp1 binary at: $DCP_TEST_BIN" 26 | echo "Using cmp binary at: $DCP_CMP_BIN" 27 | echo "Using tmp directory at: $DCP_TEST_TMP" 28 | 29 | ############################################################################## 30 | # Generate the paths for: 31 | # * A file that doesn't exist at all. 32 | # * Two files with zero length. 33 | # * Two files which contain random data. 34 | PATH_A_NOEXIST="$DCP_TEST_TMP/dcp1_test_single_file_to_single_file.$RANDOM.tmp" 35 | PATH_B_EMPTY="$DCP_TEST_TMP/dcp1_test_single_file_to_single_file.$RANDOM.tmp" 36 | PATH_C_EMPTY="$DCP_TEST_TMP/dcp1_test_single_file_to_single_file.$RANDOM.tmp" 37 | PATH_D_RANDOM="$DCP_TEST_TMP/dcp1_test_single_file_to_single_file.$RANDOM.tmp" 38 | PATH_E_RANDOM="$DCP_TEST_TMP/dcp1_test_single_file_to_single_file.$RANDOM.tmp" 39 | 40 | # Print out the generated paths to make debugging easier. 41 | echo "A_NOEXIST path at: $PATH_A_NOEXIST" 42 | echo "B_EMPTY path at: $PATH_B_EMPTY" 43 | echo "C_EMPTY path at: $PATH_C_EMPTY" 44 | echo "D_RANDOM path at: $PATH_D_RANDOM" 45 | echo "E_RANDOM path at: $PATH_E_RANDOM" 46 | 47 | # Create the two empty files. 48 | touch $PATH_B_EMPTY 49 | touch $PATH_C_EMPTY 50 | 51 | # Create the two random files. 52 | dd if=/dev/urandom of=$PATH_D_RANDOM bs=4M count=5 53 | dd if=/dev/urandom of=$PATH_E_RANDOM bs=3M count=4 54 | 55 | # Stat the two random files. 56 | stat $PATH_D_RANDOM 57 | stat $PATH_E_RANDOM 58 | 59 | ############################################################################## 60 | # Test copying an empty file to an empty file. The result should be two files 61 | # which remain empty with no error output. 62 | 63 | mpirun -n 3 $DCP_TEST_BIN $PATH_B_EMPTY $PATH_C_EMPTY 64 | 65 | if [[ $? -ne 0 ]]; then 66 | echo "Error returned when copying empty file to empty file (B -> C)." 67 | exit 1; 68 | fi 69 | 70 | $DCP_CMP_BIN $PATH_B_EMPTY $PATH_C_EMPTY 71 | if [[ $? -ne 0 ]]; then 72 | echo "CMP mismatch when copying empty file to empty file (B -> C)." 73 | exit 1 74 | fi 75 | 76 | ############################################################################## 77 | # Test copying a random file to an empty file. The result should be two files 78 | # which both contain the contents of the first random file. 79 | 80 | mpirun -n 3 $DCP_TEST_BIN $PATH_D_RANDOM $PATH_B_EMPTY 81 | if [[ $? -ne 0 ]]; then 82 | echo "Error returned when copying random file to empty file (D -> B)." 83 | exit 1; 84 | fi 85 | 86 | $DCP_CMP_BIN $PATH_D_RANDOM $PATH_B_EMPTY 87 | if [[ $? -ne 0 ]]; then 88 | echo "CMP mismatch when copying random file to empty file (D -> B)." 89 | exit 1 90 | fi 91 | 92 | ############################################################################## 93 | # Test copying a random file to another random file. The result should be two 94 | # files which both contain the contents of the first random file. 95 | 96 | mpirun -n 3 $DCP_TEST_BIN $PATH_D_RANDOM $PATH_E_RANDOM 97 | if [[ $? -ne 0 ]]; then 98 | echo "Error returned when copying random file to random file (D -> E)." 99 | exit 1; 100 | fi 101 | 102 | $DCP_CMP_BIN $PATH_D_RANDOM $PATH_E_RANDOM 103 | if [[ $? -ne 0 ]]; then 104 | echo "CMP mismatch when copying random file to random file (D -> E)." 105 | exit 1 106 | fi 107 | 108 | ############################################################################## 109 | # Since we didn't find any problems, exit with success. 110 | 111 | exit 0 112 | 113 | # EOF 114 | -------------------------------------------------------------------------------- /man/dstripe.1: -------------------------------------------------------------------------------- 1 | .\" Man page generated from reStructuredText. 2 | . 3 | .TH "DSTRIPE" "1" "Jan 29, 2025" "0.12" "mpiFileUtils" 4 | .SH NAME 5 | dstripe \- restripe files on underlying storage 6 | . 7 | .nr rst2man-indent-level 0 8 | . 9 | .de1 rstReportMargin 10 | \\$1 \\n[an-margin] 11 | level \\n[rst2man-indent-level] 12 | level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] 13 | - 14 | \\n[rst2man-indent0] 15 | \\n[rst2man-indent1] 16 | \\n[rst2man-indent2] 17 | .. 18 | .de1 INDENT 19 | .\" .rstReportMargin pre: 20 | . RS \\$1 21 | . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] 22 | . nr rst2man-indent-level +1 23 | .\" .rstReportMargin post: 24 | .. 25 | .de UNINDENT 26 | . RE 27 | .\" indent \\n[an-margin] 28 | .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] 29 | .nr rst2man-indent-level -1 30 | .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] 31 | .in \\n[rst2man-indent\\n[rst2man-indent-level]]u 32 | .. 33 | .SH SYNOPSIS 34 | .sp 35 | \fBdstripe [OPTION] PATH...\fP 36 | .SH DESCRIPTION 37 | .sp 38 | Parallel MPI application to restripe files. 39 | .sp 40 | This tool is in active development. It currently only works on Lustre. 41 | .sp 42 | dstripe enables one to restripe file(s) across the underlying storage 43 | devices. One must specify a list of paths. All files in those paths can 44 | be restriped. By default, stripe size is 1MB and stripe count is \-1 45 | allowing dstripe to use all available stripes. 46 | .SH OPTIONS 47 | .INDENT 0.0 48 | .TP 49 | .B \-c, \-\-count STRIPE_COUNT 50 | The number of stripes to use during file restriping. If STRIPE_COUNT 51 | is \-1, then all available stripes are used. If STRIPE_COUNT is 0, 52 | the lustre file system default is used. The default stripe count is 53 | \-1. 54 | .UNINDENT 55 | .INDENT 0.0 56 | .TP 57 | .B \-s, \-\-size STRIPE_SIZE 58 | The stripe size to use during file restriping. Units like "MB" and 59 | "GB" can immediately follow the number without spaces (ex. 2MB). The 60 | default stripe size is 1MB. 61 | .UNINDENT 62 | .INDENT 0.0 63 | .TP 64 | .B \-m, \-\-minsize SIZE 65 | The minimum size a file must be to be a candidate for restriping. 66 | Files smaller than SIZE will not be restriped. Units like "MB" and 67 | "GB" can immediately follow the number without spaces (ex. 2MB). The 68 | default minimum file size is 0MB. 69 | .UNINDENT 70 | .INDENT 0.0 71 | .TP 72 | .B \-r, \-\-report 73 | Display the file size, stripe count, and stripe size of all files 74 | found in PATH. No restriping is performed when using this option. 75 | .UNINDENT 76 | .INDENT 0.0 77 | .TP 78 | .B \-\-progress N 79 | Print progress message to stdout approximately every N seconds. 80 | The number of seconds must be a non\-negative integer. 81 | A value of 0 disables progress messages. 82 | .UNINDENT 83 | .INDENT 0.0 84 | .TP 85 | .B \-v, \-\-verbose 86 | Run in verbose mode. 87 | .UNINDENT 88 | .INDENT 0.0 89 | .TP 90 | .B \-q, \-\-quiet 91 | Run tool silently. No output is printed. 92 | .UNINDENT 93 | .INDENT 0.0 94 | .TP 95 | .B \-h, \-\-help 96 | Print the command usage, and the list of options available. 97 | .UNINDENT 98 | .SH EXAMPLES 99 | .INDENT 0.0 100 | .IP 1. 3 101 | To stripe a file on all storage devices using a 1MB stripe size: 102 | .UNINDENT 103 | .sp 104 | \fBmpirun \-np 128 dstripe \-s 1MB /path/to/file\fP 105 | .INDENT 0.0 106 | .IP 2. 3 107 | To stripe a file across 20 storage devices with a 1GB stripe size: 108 | .UNINDENT 109 | .sp 110 | \fBmpirun \-np 128 dstripe \-c 20 \-s 1GB /path/to/file\fP 111 | .INDENT 0.0 112 | .IP 3. 3 113 | To restripe all files in /path/to/files/ that are at least 1GB in 114 | size: 115 | .UNINDENT 116 | .sp 117 | \fBmpirun \-np 128 dstripe \-m 1GB /path/to/files/\fP 118 | .INDENT 0.0 119 | .IP 4. 3 120 | To restripe all files in /path/to/files/ across 10 storage devices 121 | with 2MB stripe size: 122 | .UNINDENT 123 | .sp 124 | \fBmpirun \-np 128 dstripe \-c 10 \-s 2MB /path/to/files/\fP 125 | .INDENT 0.0 126 | .IP 5. 3 127 | To display the current stripe count and stripe size of all files in 128 | /path/to/files/: 129 | .UNINDENT 130 | .sp 131 | \fBmpirun \-np 128 dstripe \-r /path/to/files/\fP 132 | .SH SEE ALSO 133 | .sp 134 | The mpiFileUtils source code and all documentation may be downloaded 135 | from <\fI\%https://github.com/hpc/mpifileutils\fP> 136 | .SH AUTHOR 137 | HPC 138 | .SH COPYRIGHT 139 | 2022, LLNL/LANL/UT-Battelle/DDN 140 | .\" Generated by docutils manpage writer. 141 | . 142 | -------------------------------------------------------------------------------- /doc/rst/libmfu.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | Common Library - libmfu 3 | ======================== 4 | 5 | Functionality that is common to multiple tools is moved to the common library, 6 | libmfu. This goal of this library is to make it easy to develop new tools and 7 | to provide consistent behavior across tools in the suite. The library can also 8 | be useful to end applications, e.g., to efficiently create or remove a large 9 | directory tree in a portable way across different parallel file systems. 10 | 11 | --------------------------------------- 12 | libmfu: the mpiFileUtils common library 13 | --------------------------------------- 14 | 15 | The mpiFileUtils common library defines data structures and methods on those 16 | data structures that makes it easier to develop new tools or for use within HPC 17 | applications to provide portable, performant implementations across file 18 | systems common in HPC centers. 19 | 20 | .. code-block:: C 21 | 22 | #include "mfu.h" 23 | 24 | This file includes all other necessary headers. 25 | 26 | --------------------------------------- 27 | mfu_flist 28 | --------------------------------------- 29 | 30 | The key data structure in libmfu is a distributed file list called mfu_flist. 31 | This structure represents a list of files, each with stat-like metadata, that 32 | is distributed among a set of MPI ranks. 33 | 34 | The library contains functions for creating and operating on these lists. For 35 | example, one may create a list by recursively walking an existing directory or 36 | by inserting new entries one at a time. Given a list as input, functions exist 37 | to create corresponding entries (inodes) on the file system or to delete the 38 | list of files. One may filter, sort, and remap entries. One can copy a list of 39 | entries from one location to another or compare corresponding entries across 40 | two different lists. A file list can be serialized and written to or read from 41 | a file. 42 | 43 | Each MPI rank "owns" a portion of the list, and there are routines to step 44 | through the entries owned by that process. This portion is referred to as the 45 | "local" list. Functions exist to get and set properties of the items in the 46 | local list, for example to get the path name, type, and size of a file. 47 | Functions dealing with the local list can be called by the MPI process 48 | independently of other MPI processes. 49 | 50 | Other functions operate on the global list in a collective fashion, such as 51 | deleting all items in a file list. All processes in the MPI job must invoke 52 | these functions simultaneously. 53 | 54 | For full details, see `mfu_flist.h `_ 55 | and refer to its usage in existing tools. 56 | 57 | --------------------------------------- 58 | mfu_path 59 | --------------------------------------- 60 | 61 | mpiFileUtils represents file paths with the `mfu_path `_ 62 | structure. Functions are available to manipulate paths to prepend and append 63 | entries, to slice paths into pieces, and to compute relative paths. 64 | 65 | --------------------------------------- 66 | mfu_param_path 67 | --------------------------------------- 68 | 69 | Path names provided by the user on the command line (parameters) are handled 70 | through the `mfu_param_path `_ 71 | structure. Such paths may have to be checked for existence and to determine 72 | their type (file or directory). Additionally, the user may specify many such 73 | paths through invocations involving shell wildcards, so functions are available 74 | to check long lists of paths in parallel. 75 | 76 | --------------------------------------- 77 | mfu_io 78 | --------------------------------------- 79 | 80 | The `mfu_io.h `_ 81 | functions provide wrappers for many POSIX-IO functions. This is helpful for 82 | checking error codes in a consistent manner and automating retries on failed 83 | I/O calls. One should use the wrappers in mfu_io if available, and if not, one 84 | should consider adding the missing wrapper. 85 | 86 | --------------------------------------- 87 | mfu_util 88 | --------------------------------------- 89 | 90 | The `mfu_util.h `_ 91 | functions provide wrappers for error reporting and memory allocation. 92 | 93 | -------------------------------------------------------------------------------- /test/legacy/dcp1_tests/test_dcp1_many_file_to_single_file/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ############################################################################## 4 | # Description: 5 | # 6 | # A test to check if dcp1 will copy many files to a single file. 7 | # 8 | # Expected behavior: 9 | # 10 | # An error code should be displayed if this action is attempted. 11 | # 12 | # Reminder: 13 | # 14 | # Lines that echo to the terminal will only be available if DEBUG is enabled 15 | # in the test runner (test_all.sh). 16 | ############################################################################## 17 | 18 | # Turn on verbose output 19 | #set -x 20 | 21 | # Print out the basic paths we'll be using. 22 | echo "Using dcp1 binary at: $DCP_TEST_BIN" 23 | echo "Using tmp directory at: $DCP_TEST_TMP" 24 | 25 | ############################################################################## 26 | # Generate the paths for: 27 | # * Four files with zero length. 28 | # * Four files which contain random data. 29 | # * A file that doesn't exist. 30 | PATH_A_EMPTY="$DCP_TEST_TMP/dcp1_test_many_file_to_single_file.$RANDOM.tmp" 31 | PATH_B_EMPTY="$DCP_TEST_TMP/dcp1_test_many_file_to_single_file.$RANDOM.tmp" 32 | PATH_C_EMPTY="$DCP_TEST_TMP/dcp1_test_many_file_to_single_file.$RANDOM.tmp" 33 | PATH_D_EMPTY="$DCP_TEST_TMP/dcp1_test_many_file_to_single_file.$RANDOM.tmp" 34 | PATH_E_RANDOM="$DCP_TEST_TMP/dcp1_test_many_file_to_single_file.$RANDOM.tmp" 35 | PATH_F_RANDOM="$DCP_TEST_TMP/dcp1_test_many_file_to_single_file.$RANDOM.tmp" 36 | PATH_G_RANDOM="$DCP_TEST_TMP/dcp1_test_many_file_to_single_file.$RANDOM.tmp" 37 | PATH_H_RANDOM="$DCP_TEST_TMP/dcp1_test_many_file_to_single_file.$RANDOM.tmp" 38 | PATH_I_NOEXIST="$DCP_TEST_TMP/dcp1_test_many_file_to_single_file.$RANDOM.tmp" 39 | 40 | # Print out the generated paths to make debugging easier. 41 | echo "A_EMPTY path at: $PATH_A_EMPTY" 42 | echo "B_EMPTY path at: $PATH_B_EMPTY" 43 | echo "C_EMPTY path at: $PATH_C_EMPTY" 44 | echo "D_EMPTY path at: $PATH_D_EMPTY" 45 | echo "E_RANDOM path at: $PATH_E_RANDOM" 46 | echo "F_RANDOM path at: $PATH_F_RANDOM" 47 | echo "G_RANDOM path at: $PATH_G_RANDOM" 48 | echo "H_RANDOM path at: $PATH_H_RANDOM" 49 | 50 | # Create empty files. 51 | touch $PATH_A_EMPTY 52 | touch $PATH_B_EMPTY 53 | touch $PATH_C_EMPTY 54 | touch $PATH_D_EMPTY 55 | 56 | # Create the random files. 57 | dd if=/dev/urandom of=$PATH_E_RANDOM bs=2M count=4 58 | dd if=/dev/urandom of=$PATH_F_RANDOM bs=3M count=3 59 | dd if=/dev/urandom of=$PATH_G_RANDOM bs=1M count=1 60 | dd if=/dev/urandom of=$PATH_H_RANDOM bs=6M count=2 61 | 62 | ############################################################################## 63 | # Test copying several empty files into an empty file. This should result in 64 | # an error. 65 | 66 | mpirun -n 3 $DCP_TEST_BIN $PATH_A_EMPTY $PATH_B_EMPTY $PATH_C_EMPTY $PATH_D_EMPTY 67 | if [[ $? -eq 0 ]]; then 68 | echo "Unexpected success when copying empty files to an empty file. (A,B,C -> D)." 69 | exit 1; 70 | fi 71 | 72 | ############################################################################## 73 | # Test copying several empty files into a random file. This should result in 74 | # an error. 75 | 76 | mpirun -n 3 $DCP_TEST_BIN $PATH_A_EMPTY $PATH_B_EMPTY $PATH_C_EMPTY $PATH_E_EMPTY 77 | if [[ $? -eq 0 ]]; then 78 | echo "Unexpected success when copying empty files to a random file (A,B,C -> E)." 79 | exit 1; 80 | fi 81 | 82 | ############################################################################## 83 | # Test copying several random files into an empty file. This should result in 84 | # an error. 85 | 86 | mpirun -n 3 $DCP_TEST_BIN $PATH_E_RANDOM $PATH_F_RANDOM $PATH_G_RANDOM $PATH_A_EMPTY 87 | if [[ $? -eq 0 ]]; then 88 | echo "Unexpected success when copying random files to an empty file (E,F,G -> A)." 89 | exit 1; 90 | fi 91 | 92 | ############################################################################## 93 | # Test copying several random files into a file that doesn't exist. This 94 | # should result in an error. 95 | 96 | mpirun -n 3 $DCP_TEST_BIN $PATH_E_RANDOM $PATH_F_RANDOM $PATH_G_RANDOM $PATH_I_NOEXIST 97 | if [[ $? -eq 0 ]]; then 98 | echo "Unexpected success when copying random files to a no-exist file (E,F,G -> I)." 99 | exit 1; 100 | fi 101 | 102 | ############################################################################## 103 | # Since we didn't find any problems, exit with success. 104 | 105 | exit 0 106 | 107 | # EOF 108 | -------------------------------------------------------------------------------- /doc/rst/dfind.1.rst: -------------------------------------------------------------------------------- 1 | dfind 2 | ===== 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | **dfind [OPTION] [EXPRESSION] PATH ...** 8 | 9 | DESCRIPTION 10 | ----------- 11 | 12 | Parallel MPI application to filter a list of files according to an expression. 13 | 14 | dfind provides functionality similar to :manpage:`find(1)`. 15 | 16 | The file list can be obtained by either walking one or more paths provided on the command line or through an input list. 17 | 18 | The filtered list can be written to an output file. 19 | 20 | OPTIONS 21 | ------- 22 | 23 | .. option:: -i, --input FILE 24 | 25 | Read source list from FILE. FILE must be generated by another tool 26 | from the mpiFileUtils suite. 27 | 28 | .. option:: -o, --output FILE 29 | 30 | Write the processed list to a file. 31 | 32 | .. option:: -t, --text 33 | 34 | Must be used with the --output option. Write processed list of files to 35 | FILE in ascii text format. 36 | 37 | .. option:: -v, --verbose 38 | 39 | Run in verbose mode. 40 | 41 | .. option:: -q, --quiet 42 | 43 | Run tool silently. No output is printed. 44 | 45 | .. option:: -h, --help 46 | 47 | Print a brief message listing the :manpage:`dfind(1)` options and usage. 48 | 49 | EXPRESSIONS 50 | ----------- 51 | 52 | Numeric arguments can be specified as: 53 | 54 | +----+-------------+ 55 | | +N | more than N | 56 | +----+-------------+ 57 | | -N | less than N | 58 | +----+-------------+ 59 | | N | exactly N | 60 | +----+-------------+ 61 | 62 | 63 | .. option:: --amin N 64 | 65 | File was last accessed N minutes ago. 66 | 67 | .. option:: --anewer FILE 68 | 69 | File was last accessed more recently than FILE was modified. 70 | 71 | .. option:: --atime N 72 | 73 | File was last accessed N days ago. 74 | 75 | .. option:: --cmin N 76 | 77 | File's status was last changed N minutes ago. 78 | 79 | .. option:: --cnewer FILE 80 | 81 | File's status was last changed more recently than FILE was modified. 82 | 83 | .. option:: --ctime N 84 | 85 | File's status was last changed N days ago. 86 | 87 | .. option:: --mmin N 88 | 89 | File's data was last modified N minutes ago. 90 | 91 | .. option:: --newer FILE 92 | 93 | File was modified more recently than FILE. 94 | 95 | .. option:: --mtime N 96 | 97 | File's data was last modified N days ago. 98 | 99 | .. option:: --gid N 100 | 101 | File's numeric group ID is N. 102 | 103 | .. option:: --group NAME 104 | 105 | File belongs to group NAME. 106 | 107 | .. option:: --uid N 108 | 109 | File's numeric user ID is N. 110 | 111 | .. option:: --user NAME 112 | 113 | File is owned by user NAME. 114 | 115 | .. option:: --name PATTERN 116 | 117 | Base of file name matches shell pattern PATTERN. 118 | 119 | .. option:: --path PATTERN 120 | 121 | Full path to file matches shell pattern PATTERN. 122 | 123 | .. option:: --regex REGEX 124 | 125 | Full path to file matches POSIX regular expression REGEX. Regular expressions processed by :manpage:`regexec(3)`. 126 | 127 | .. option:: --size N 128 | 129 | File size is N bytes. Units can be used like 'KB', 'MB', 'GB'. 130 | 131 | .. option:: --type C 132 | 133 | File is of type C: 134 | 135 | +---+---------------+ 136 | | b | block device | 137 | +---+---------------+ 138 | | c | char device | 139 | +---+---------------+ 140 | | d | directory | 141 | +---+---------------+ 142 | | f | regular file | 143 | +---+---------------+ 144 | | l | symbolic link | 145 | +---+---------------+ 146 | | p | pipe | 147 | +---+---------------+ 148 | | s | socket | 149 | +---+---------------+ 150 | 151 | ACTIONS 152 | ------- 153 | 154 | .. option:: --print 155 | 156 | Print file name to stdout. 157 | 158 | .. option:: --exec CMD ; 159 | 160 | Execute command CMD on file. All following arguments are taken as arguments to the command until ';' is encountered. The string '{}' is replaced by the current file name. 161 | 162 | EXAMPLES 163 | -------- 164 | 165 | 1. Print all files owner by user1 under given path: 166 | 167 | ``mpirun -np 128 dfind -v --user user1 --print /path/to/target`` 168 | 169 | 2. To find all files less than 1GB and write them to a file: 170 | 171 | ``mpirun -np 128 dfind -v -o outfile --size -1GB /path/to/target`` 172 | 173 | 3. Filter list in infile to find all regular files not changed in the past 180 days and write new list to outfile: 174 | 175 | ``mpirun -np 128 dfind -v -i infile -o outfile --type f --mtime +180`` 176 | 177 | SEE ALSO 178 | -------- 179 | 180 | The mpiFileUtils source code and all documentation may be downloaded 181 | from 182 | -------------------------------------------------------------------------------- /doc/rst/dsync.1.rst: -------------------------------------------------------------------------------- 1 | dsync 2 | ===== 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | **dsync [OPTION] SRC DEST** 8 | 9 | DESCRIPTION 10 | ----------- 11 | 12 | Parallel MPI application to synchronize two files or two directory trees. 13 | 14 | dsync makes DEST match SRC, adding missing entries from DEST, and updating 15 | existing entries in DEST as necessary so that SRC and DEST have identical 16 | content, ownership, timestamps, and permissions. 17 | 18 | dsync is similar to :manpage:`rsync(1)` archive mode for creating and 19 | then maintaining an identical copy of a source directory tree. 20 | 21 | OPTIONS 22 | ------- 23 | 24 | .. option:: --dryrun 25 | 26 | Show differences without changing anything. 27 | 28 | .. option:: -b, --batch-files N 29 | 30 | Batch files into groups of up to size N during copy operation. 31 | 32 | .. option:: --bufsize SIZE 33 | 34 | Set the I/O buffer to be SIZE bytes. Units like "MB" and "GB" may 35 | immediately follow the number without spaces (e.g. 8MB). The default 36 | bufsize is 4MB. 37 | 38 | .. option:: --chunksize SIZE 39 | 40 | Multiple processes copy a large file in parallel by dividing it into chunks. 41 | Set chunk to be at minimum SIZE bytes. Units like "MB" and 42 | "GB" can immediately follow the number without spaces (e.g. 64MB). 43 | The default chunksize is 4MB. 44 | 45 | .. option:: --xattrs WHICH 46 | 47 | Copy extended attributes ("xattrs") from source files to target files. 48 | WHICH determines which xattrs are copied. Options are to copy no xattrs, 49 | all xattrs, xattrs not excluded by /etc/xattr.conf, or all xattrs except 50 | those which have special meaning to Lustre. Certain xattrs control Lustre 51 | features on a file-by-file basis, such as how the file data is distributed 52 | across Lustre servers. Values must be in {none, all, libattr, non-lustre}. 53 | The default is non-lustre. 54 | 55 | .. option:: --daos-api API 56 | 57 | Specify the DAOS API to be used. By default, the API is automatically 58 | determined based on the container type, where POSIX containers use the 59 | DFS API, and all other containers use the DAOS object API. 60 | Values must be in {DFS, DAOS}. 61 | 62 | .. option:: -c, --contents 63 | 64 | Compare files byte-by-byte rather than checking size and mtime 65 | to determine whether file contents are different. 66 | 67 | .. option:: -D, --delete 68 | 69 | Delete extraneous files from destination. 70 | 71 | .. option:: -L, --dereference 72 | 73 | Dereference symbolic links and copy the target file or directory 74 | that each symbolic link refers to. 75 | 76 | .. option:: -P, --no-dereference 77 | 78 | Do not follow symbolic links in source paths. Effectviely allows 79 | symbolic links to be copied when the link target is not valid 80 | or there is not permission to read the link's target. 81 | 82 | .. option:: -s, --direct 83 | 84 | Use O_DIRECT to avoid caching file data. 85 | 86 | .. option:: --open-noatime 87 | 88 | Open files with O_NOATIME flag. 89 | 90 | .. option:: --link-dest DIR 91 | 92 | Create hardlink in DEST to files in DIR when file is unchanged 93 | rather than create a new file. One can use this option to conserve 94 | storage space during an incremental backup. 95 | 96 | For example in the following, any file that would be copied from 97 | /src to /src.bak.inc that is the same as the file already existing 98 | in /src.bak will instead be hardlinked to the file in /src.bak: 99 | 100 | # initial backup of /src 101 | ``dsync /src /src.bak`` 102 | 103 | # incremental backup of /src 104 | ``dsync --link-dest /src.bak /src /src.bak.inc`` 105 | 106 | .. option:: -S, --sparse 107 | 108 | Create sparse files when possible. 109 | 110 | .. option:: --progress N 111 | 112 | Print progress message to stdout approximately every N seconds. 113 | The number of seconds must be a non-negative integer. 114 | A value of 0 disables progress messages. 115 | 116 | .. option:: -v, --verbose 117 | 118 | Run in verbose mode. Prints a list of statistics/timing data for the 119 | command. Files walked, started, completed, seconds, files, bytes 120 | read, byte rate, and file rate. 121 | 122 | .. option:: -q, --quiet 123 | 124 | Run tool silently. No output is printed. 125 | 126 | .. option:: -h, --help 127 | 128 | Print the command usage, and the list of options available. 129 | 130 | EXAMPLES 131 | -------- 132 | 133 | 1. Synchronize dir2 to match dir1: 134 | 135 | ``mpirun -np 128 dsync /path/to/dir1 /path/to/dir2`` 136 | 137 | SEE ALSO 138 | -------- 139 | 140 | The mpiFileUtils source code and all documentation may be downloaded 141 | from 142 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to mpiFileUtils 2 | 3 | *Thank you for taking the time to contribute!* 4 | 5 | ## Table of Contents 6 | [How To Contribute](#how-to-contribute) 7 | * [Reporting an Issue & Feature Suggestions](#reporting-an-issue--feature-suggestions) 8 | * [Pull Requests](#pull-requests) 9 | * [License](#license) 10 | * [Contributor's Declaration](#contributors-declaration) 11 | 12 | ## How To Contribute 13 | 14 | ### Reporting an Issue & Feature Suggestions 15 | You can find a list of all outstanding issues and feature requests on our Github 16 | [issue tracker](https://github.com/hpc/mpifileutils/issues). Before creating a new 17 | issue or feature suggestion, ensure that: 18 | * The issue or feature request hasn't been addressed in a newer version of mpiFileUtils. 19 | * That the issue or feature request doesn't already exist in the issue tracker. 20 | 21 | If the issue or feature request already exists, please provide additional 22 | information if possible. 23 | 24 | When creating a new issue, please provide at least the following information: 25 | * mpiFileUtils Version 26 | * Linux Distribution & Version 27 | * Steps to reproduce the problem 28 | * Information from various system logs (i.e. call stacks, dmesg output) 29 | 30 | ### Pull Requests 31 | If you would like us to consider a change to mpiFileUtils, feel free to submit 32 | a pull request. Your pull request must meet the following criteria before 33 | it can be merged. 34 | 35 | * Your pull request must be based on the current master branch and 36 | apply without conflicts. 37 | * Please try to limit pull requests to a single commit which resolves 38 | one issue. 39 | * Make sure your commit messages have a `Signed-off-by` line. See 40 | [Contributor's Declaration](#contributors-declaration) for more information. 41 | * For large pull requests, consider structuring your changes as a stack of 42 | logically independent patches which build on each other. This makes large 43 | changes easier to review and approve which speeds up the merging process. 44 | * Try to keep pull requests simple. Simple code with comments is much easier 45 | to review and approve. 46 | * Test cases should be provided when appropriate. 47 | * Your pull request must pass automated testing. 48 | * All proposed changes must be approved by an mpiFileUtils project member. 49 | 50 | ### License 51 | mpiFileUtils is distributed under the New BSD License with a few additional notices. 52 | Note that the phrase "above copyright notice" in the license text refers to the 53 | current list of copyrights that appears in the 54 | [license](https://github.com/hpc/mpifileutils/blob/master/LICENSE) file found in 55 | the mpiFileUtils master branch. 56 | 57 | ### Contributor's Declaration 58 | mpiFileUtils has adopted the signed-off-by process as described in Section 59 | 11 of the Linux kernel document on 60 | [Submitting Patches](https://www.kernel.org/doc/html/latest/process/submitting-patches.html). 61 | Each proposed contribution to the mpiFileUtils code base must include the text 62 | "Signed-off-by:" followed by the contributor's name and email address. This is 63 | the developer's certification that they have the right to submit the patch for 64 | inclusion into the code base and indicates agreement to the Developer's 65 | Certificate of Origin: 66 | 67 | "By making a contribution to this project, I certify that: 68 | 1. The contribution was created in whole or in part by me and I have the right 69 | to submit it under the open source license indicated in the file; or 70 | 2. The contribution is based upon previous work that, to the best of my knowledge, 71 | is covered under an appropriate open source license and I have the right under 72 | that license to submit that work with modifications, whether created in whole or 73 | in part by me, under the same open source license (unless I am permitted to submit 74 | under a different license), as indicated in the file; or 75 | 3. The contribution was provided directly to me by some other person who certified 76 | (1), (2) or (3) and I have not modified it. 77 | 4. I understand and agree that this project and the contribution are public and 78 | that a record of the contribution (including all personal information I submit 79 | with it, including my sign-off) is maintained indefinitely and may be 80 | redistributed consistent with this project or the open source license(s) involved." 81 | 82 | After reading and agreeing to the Contributor's Declaration, include your 83 | Signed-off-by text like the following as the last line in your commit message: 84 | 85 | ``` 86 | Signed-off-by: Random J Developer 87 | ``` 88 | 89 | Proposed contributions failing to include a "Signed-off-by:" certification will 90 | not be accepted into mpiFileUtils. The maintainers reserve the right to revert 91 | any commit made without the required certification. 92 | -------------------------------------------------------------------------------- /test/tests/test_dsync/test_xattr.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ############################################################################## 4 | # Description: 5 | # 6 | # A test to check if dsync properly copies xattrs 7 | # 8 | ############################################################################## 9 | 10 | # Turn on verbose output 11 | #set -x 12 | 13 | MFU_TEST_BIN=${MFU_TEST_BIN:-${1}} 14 | DSYNC_SRC_DIR=${DSYNC_SRC_DIR:-${2}} 15 | DSYNC_DEST_DIR=${DSYNC_DEST_DIR:-${3}} 16 | DSYNC_TMP_FILE=${DSYNC_TMP_FILE:-${4}} 17 | 18 | echo "Using MFU binaries at: $MFU_TEST_BIN" 19 | echo "Using src directory at: $DSYNC_SRC_DIR" 20 | echo "Using dest directory at: $DSYNC_DEST_DIR" 21 | echo "Using temp file name: $DSYNC_TMP_FILE" 22 | 23 | set -a lustre_xattr_names 24 | lustre_xattr_names=("lustre.lov" "trusted.som" "trusted.lov" "trusted.lma" "trusted.lmv" "trusted.dmv" "trusted.link" "trusted.fid" "trusted.version" "trusted.hsm" "trusted.lfsck_bitmap" "trusted.dummy") 25 | 26 | set -a other_xattr_names 27 | other_xattr_names=("xattr1" "456" "testing" "sync_and_verify_test") 28 | 29 | function fs_type() 30 | { 31 | fname=$1 32 | df -T ${fname} | awk '$1 != "Filesystem" {print $2}' 33 | } 34 | 35 | # set all xattrs in other_xattr_names on file 36 | # no need for equivalent for lustre xattrs, because they cannot be set from userspace 37 | function set_other_xattrs() 38 | { 39 | fname=$1 40 | 41 | set -e 42 | for attrname in ${other_xattr_names[*]}; do 43 | attr -s $attrname -V "$attrname:1234567890" $fname 44 | done 45 | set +e 46 | } 47 | 48 | function list_all_xattrs() 49 | { 50 | fname=$1 51 | 52 | echo Listing xattrs on $fname 53 | attr -l $fname | sort 54 | } 55 | 56 | function compare_xattr_lists() 57 | { 58 | fname0=$1 59 | fname1=$2 60 | 61 | for attrname in ${lustre_xattr_names[*]} ${other_xattr_names[*]}; do 62 | in0=$(attr -g -q $attrname $fname0 2>/dev/null && echo 1) 63 | in1=$(attr -g -q $attrname $fname1 2>/dev/null && echo 1) 64 | if [ "$in0" -eq 1 -o "$in1" -eq 1 ]; then 65 | echo "$attrname $in0 $in1" 66 | fi 67 | done 68 | } 69 | 70 | function sync_and_verify() 71 | { 72 | local srcdir=$1 73 | local destdir=$2 74 | local name=$3 75 | local opt=$4 76 | 77 | local result=0 78 | local dest_type=$(fs_type $destdir) 79 | 80 | if [ $opt = "non-lustre" ]; then 81 | echo "SKIPPED verify of option $opt" 82 | return 0 83 | fi 84 | 85 | if [ $opt = "libattr" -a $(id -u) -ne 0 ]; then 86 | echo "SKIPPED verify of option $opt, need root to test" 87 | return 0 88 | fi 89 | 90 | set -e 91 | rm -f $destdir/$name 92 | 93 | if [ $opt = "libattr" ]; then 94 | echo "user.sync_and_verify_test skip" >> /etc/xattr.conf 95 | fi 96 | 97 | 98 | if [[ $opt = "no-xattr-option" ]]; then 99 | xattropt="" 100 | else 101 | xattropt="--xattrs=$opt" 102 | fi 103 | 104 | ${MFU_TEST_BIN}/dsync --quiet $xattropt $srcdir $destdir 105 | 106 | if [ $opt = "libattr" ]; then 107 | sed --in-place "/^user.sync_and_verify_test/d" /etc/xattr.conf 108 | fi 109 | 110 | srclog=$(mktemp /tmp/sync_and_verify.src.XXXXX) 111 | destlog=$(mktemp /tmp/sync_and_verify.dest.XXXXX) 112 | 113 | list_all_xattrs $srcdir/$name | awk '!/Listing xattrs on/ {print $1,$2,$3,$4,$5}' > $srclog 114 | list_all_xattrs $destdir/$name | awk '!/Listing xattrs on/ {print $1,$2,$3,$4,$5}' > $destlog 115 | 116 | case $opt in 117 | "none") 118 | result=$(wc -c < $destlog) 119 | ;; 120 | "all") 121 | diff $srclog $destlog 122 | result=$? 123 | ;; 124 | "libattr") 125 | diff <(grep -v -w sync_and_verify_test $srclog) $destlog 126 | result=$? 127 | ;; 128 | esac 129 | 130 | set +e 131 | 132 | if [ "$result" -eq 0 ]; then 133 | echo "PASSED verify of option $opt for $destdir type $dest_type" 134 | else 135 | echo "FAILED verify of option $opt for $destdir type $dest_type" 136 | echo ======================= 137 | echo "dest:" 138 | cat $destlog 139 | echo ======================= 140 | exit 1 141 | fi 142 | 143 | rm $srclog $destlog 144 | 145 | return $result 146 | } 147 | 148 | # Create the source 149 | echo Preparing Source 150 | touch $DSYNC_SRC_DIR/aaa 151 | set_other_xattrs $DSYNC_SRC_DIR/aaa 152 | 153 | # Make sure the short option is accepted; rest of tests use long option 154 | set -e 155 | ${MFU_TEST_BIN}/dsync --quiet -X all $DSYNC_SRC_DIR $DSYNC_DEST_DIR 156 | set +e 157 | 158 | # Sync and verify 159 | echo 160 | echo Testing dsync 161 | sync_and_verify $DSYNC_SRC_DIR $DSYNC_DEST_DIR aaa no-xattr-option 162 | sync_and_verify $DSYNC_SRC_DIR $DSYNC_DEST_DIR aaa none 163 | sync_and_verify $DSYNC_SRC_DIR $DSYNC_DEST_DIR aaa all 164 | sync_and_verify $DSYNC_SRC_DIR $DSYNC_DEST_DIR aaa non-lustre 165 | sync_and_verify $DSYNC_SRC_DIR $DSYNC_DEST_DIR aaa libattr 166 | 167 | # Clean up 168 | rm $DSYNC_SRC_DIR/aaa 169 | 170 | exit 0 171 | -------------------------------------------------------------------------------- /src/common/strmap.h: -------------------------------------------------------------------------------- 1 | #ifndef STRMAP_H 2 | #define STRMAP_H 3 | 4 | /* Stores a set of key/value pairs, where key and value are both 5 | * stored as strings. */ 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | #define STRMAP_SUCCESS (0) 16 | 17 | /* 18 | ========================================= 19 | Define AVL tree data structures 20 | ========================================= 21 | */ 22 | 23 | /* Even though the structure is defined here, 24 | * consider these types to be opaque and only 25 | * use functions in this file to modify them. */ 26 | 27 | /* define the structure for an element of a hash */ 28 | typedef struct strmap_node_struct { 29 | const char* key; /* pointer to key string */ 30 | size_t key_len; /* number of characters in key string (including terminating NUL) */ 31 | const char* value; /* pointer to value string */ 32 | size_t value_len; /* number of characters in value string (including terminating NUL) */ 33 | int height; /* max height of subtree rooted at this node */ 34 | struct strmap_node_struct* parent; /* pointer to parent node */ 35 | struct strmap_node_struct* left; /* pointer to left child */ 36 | struct strmap_node_struct* right; /* pointer to right child */ 37 | } strmap_node; 38 | 39 | /* structure to track root of a tree */ 40 | typedef struct strmap_struct { 41 | strmap_node* root; /* pointer to the root node in the tree */ 42 | size_t len; /* sum of characters in all key/value strings (including terminating NULs) */ 43 | uint64_t size; /* number of nodes in the tree */ 44 | } strmap; 45 | 46 | /* 47 | ========================================= 48 | Allocate and delete map objects 49 | ========================================= 50 | */ 51 | 52 | /* allocates a new map */ 53 | strmap* strmap_new(void); 54 | 55 | /* copies entries from src into dst */ 56 | void strmap_merge(strmap* dst, const strmap* src); 57 | 58 | /* frees a map */ 59 | void strmap_delete(strmap** map); 60 | 61 | /* 62 | ========================================= 63 | iterate over key/value pairs 64 | ========================================= 65 | */ 66 | 67 | /* return first node in map */ 68 | const strmap_node* strmap_node_first(const strmap* map); 69 | 70 | /* return last node in map */ 71 | const strmap_node* strmap_node_last(const strmap* map); 72 | 73 | /* get the previous node in map */ 74 | const strmap_node* strmap_node_previous(const strmap_node* node); 75 | 76 | /* the next node in map */ 77 | const strmap_node* strmap_node_next(const strmap_node* node); 78 | 79 | /* returns pointer to key string */ 80 | const char* strmap_node_key(const strmap_node* node); 81 | 82 | /* returns pointer to value string */ 83 | const char* strmap_node_value(const strmap_node* node); 84 | 85 | /* 86 | ========================================= 87 | set, get, and unset key/value pairs 88 | ========================================= 89 | */ 90 | 91 | /* return number of key/value pairs in map */ 92 | uint64_t strmap_size(const strmap* map); 93 | 94 | /* insert key/value into map, overwrites existing key */ 95 | int strmap_set(strmap* map, const char* key, const char* value); 96 | 97 | /* insert key/value into map as "key=value" with printf formatting, 98 | * overwrites existing key */ 99 | int strmap_setf(strmap* map, const char* format, ...); 100 | 101 | /* returns pointer to value string if found, NULL otherwise */ 102 | const char* strmap_get(const strmap* map, const char* key); 103 | 104 | /* returns pointer to value string if found, NULL otherwise, 105 | * key can use printf formatting */ 106 | const char* strmap_getf(strmap* map, const char* format, ...); 107 | 108 | /* deletes specified key */ 109 | int strmap_unset(strmap* map, const char* key); 110 | 111 | /* deletes specified key using printf formatting */ 112 | int strmap_unsetf(strmap* map, const char* format, ...); 113 | 114 | #define strmap_foreach(strmap, node) \ 115 | for ((node) = strmap_node_first(strmap); \ 116 | (node) != NULL; \ 117 | (node) = strmap_node_next(node)) 118 | 119 | /* 120 | ========================================= 121 | pack and unpack data structure into array of bytes 122 | ========================================= 123 | */ 124 | 125 | #if 0 126 | /* returns number of bytes needed to pack map */ 127 | size_t strmap_pack_size(const strmap* map); 128 | 129 | /* pack map into buffer starting at specified memory location */ 130 | size_t strmap_pack(void* buf, const strmap* map); 131 | 132 | /* unpack map stored at buf into tree, returns number of bytes read */ 133 | size_t strmap_unpack(const void* buf, strmap* map); 134 | 135 | /* print map to stdout for debugging */ 136 | void strmap_print(const strmap* map); 137 | #endif 138 | 139 | #ifdef __cplusplus 140 | } 141 | #endif 142 | #endif /* STRMAP_H */ 143 | -------------------------------------------------------------------------------- /test/tests/test_dcp/checkfiemap.c: -------------------------------------------------------------------------------- 1 | /* GPL HEADER START 2 | * 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License version 2 only, 7 | * as published by the Free Software Foundation. 8 | * 9 | * This program is distributed in the hope that it will be useful, but 10 | * WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * General Public License version 2 for more details (a copy is included 13 | * in the LICENSE file that accompanied this code). 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * version 2 along with this program; If not, see http://www.gnu.org/licenses 17 | * 18 | * Please visit http://www.xyratex.com/contact if you need additional 19 | * information or have any questions. 20 | * 21 | * GPL HEADER END 22 | */ 23 | 24 | /* 25 | * Copyright 2013 Xyratex Technology Limited 26 | * 27 | * Author: Artem Blagodarenko 28 | * 29 | */ 30 | 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | #ifndef HAVE_FIEMAP 43 | # include 44 | # include 45 | #endif 46 | 47 | #ifndef FS_IOC_FIEMAP 48 | # define FS_IOC_FIEMAP (_IOWR('f', 11, struct fiemap)) 49 | #endif 50 | 51 | #define ONEMB 1048576 52 | 53 | static int __check_fiemap(int fd, long long orig_size, int test) 54 | { 55 | /* This buffer is enougth for 1MB length file */ 56 | union { struct fiemap f; char c[4096]; } fiemap_buf; 57 | struct fiemap *fiemap = &fiemap_buf.f; 58 | struct fiemap_extent *fm_extents = &fiemap->fm_extents[0]; 59 | unsigned int count = (sizeof(fiemap_buf) - sizeof(*fiemap)) / 60 | sizeof(*fm_extents); 61 | unsigned int i = 0; 62 | long long file_size = 0; 63 | 64 | memset(&fiemap_buf, 0, sizeof(fiemap_buf)); 65 | 66 | fiemap->fm_start = 0; 67 | fiemap->fm_flags = FIEMAP_FLAG_SYNC; 68 | fiemap->fm_extent_count = count; 69 | fiemap->fm_length = FIEMAP_MAX_OFFSET; 70 | 71 | if (ioctl(fd, FS_IOC_FIEMAP, fiemap) < 0) { 72 | fprintf(stderr, "error while ioctl %i\n", errno); 73 | return -1; 74 | } 75 | 76 | if (test) 77 | return 0; 78 | 79 | for (i = 0; i < fiemap->fm_mapped_extents; i++) { 80 | printf("extent in " 81 | "offset %lu, length %lu\n" 82 | "flags: %x\n", 83 | (unsigned long)fm_extents[i].fe_logical, 84 | (unsigned long)fm_extents[i].fe_length, 85 | fm_extents[i].fe_flags); 86 | 87 | if (fm_extents[i].fe_flags & FIEMAP_EXTENT_UNWRITTEN) { 88 | fprintf(stderr, "Unwritten extent\n"); 89 | return -2; 90 | } else { 91 | file_size += fm_extents[i].fe_length; 92 | } 93 | } 94 | 95 | printf("file size %lli, original size %lli\n", file_size, orig_size); 96 | return file_size != orig_size; 97 | } 98 | 99 | /* This test executes fiemap ioctl and check 100 | * a) there are no file ranges marked with FIEMAP_EXTENT_UNWRITTEN 101 | * b) data ranges sizes sum is equal to given in second param */ 102 | static int check_fiemap(int fd, long long orig_size) { 103 | return __check_fiemap(fd, orig_size, 0); 104 | } 105 | 106 | static int test_fiemap_support(int fd) { 107 | return __check_fiemap(fd, 0, 1); 108 | } 109 | 110 | int main(int argc, char **argv) 111 | { 112 | int c; 113 | struct option long_opts[] = { 114 | { .name = "test", .has_arg = required_argument, .val = 't' }, 115 | { .name = NULL } 116 | }; 117 | char *filename = NULL; 118 | int fd; 119 | int rc; 120 | 121 | optind = 0; 122 | while ((c = getopt_long(argc, argv, "t", long_opts, NULL)) != -1) { 123 | switch (c) { 124 | case 't': 125 | filename = optarg; 126 | fd = open(filename, O_RDONLY); 127 | if (fd < 0) { 128 | fprintf(stderr, "cannot open %s for reading, error %i\n", 129 | argv[optind], errno); 130 | return -1; 131 | } 132 | return test_fiemap_support(fd); 133 | default: 134 | fprintf(stderr, "error: %s: option '%s' unrecognized\n", 135 | argv[0], argv[optind - 1]); 136 | return -1; 137 | } 138 | } 139 | 140 | if (optind != argc - 2) { 141 | fprintf(stderr, "Usage: %s \n", argv[0]); 142 | return -1; 143 | } 144 | 145 | fd = open(argv[optind], O_RDONLY); 146 | if (fd < 0) { 147 | fprintf(stderr, "cannot open %s for reading, error %i\n", 148 | argv[optind], errno); 149 | return -1; 150 | } 151 | 152 | fprintf(stderr, "fd: %i\n", fd); 153 | 154 | rc = check_fiemap(fd, atoll(argv[optind + 1])); 155 | 156 | if (close(fd) < 0) 157 | fprintf(stderr, "closing %s, error %i", argv[optind], errno); 158 | 159 | return rc; 160 | } 161 | -------------------------------------------------------------------------------- /src/dgrep/dgrep.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "dgrep.h" 11 | #include "log.h" 12 | 13 | int DGREP_global_rank; 14 | FILE *DGREP_debug_stream; 15 | DGREP_loglevel DGREP_debug_level; 16 | 17 | char *DGREP_PATH; 18 | char *DGREP_ARGS; 19 | 20 | void 21 | DGREP_start(CIRCLE_handle *handle) 22 | { 23 | handle->enqueue(DGREP_PATH); 24 | } 25 | 26 | void 27 | DGREP_search(CIRCLE_handle *handle) 28 | { 29 | DIR *current_dir; 30 | 31 | char temp[CIRCLE_MAX_STRING_LEN]; 32 | char stat_temp[CIRCLE_MAX_STRING_LEN]; 33 | 34 | struct dirent *current_ent; 35 | struct stat st; 36 | 37 | /* Pop an item off the queue */ 38 | handle->dequeue(temp); 39 | 40 | /* Try and stat it, checking to see if it is a link */ 41 | if(lstat(temp,&st) != EXIT_SUCCESS) 42 | { 43 | LOG(DGREP_LOG_ERR, "Error: Couldn't stat \"%s\"", temp); 44 | return; 45 | } 46 | /* Check to see if it is a directory. If so, put its children in the queue */ 47 | else if(S_ISDIR(st.st_mode) && !(S_ISLNK(st.st_mode))) 48 | { 49 | current_dir = opendir(temp); 50 | 51 | if(!current_dir) 52 | { 53 | LOG(DGREP_LOG_ERR, "Unable to open dir"); 54 | } 55 | else 56 | { 57 | /* Read in each directory entry */ 58 | while((current_ent = readdir(current_dir)) != NULL) 59 | { 60 | /* We don't care about . or .. */ 61 | if((strncmp(current_ent->d_name,".",2)) && (strncmp(current_ent->d_name,"..",3))) 62 | { 63 | strcpy(stat_temp,temp); 64 | strcat(stat_temp,"/"); 65 | strcat(stat_temp,current_ent->d_name); 66 | 67 | handle->enqueue(&stat_temp[0]); 68 | } 69 | } 70 | } 71 | closedir(current_dir); 72 | } 73 | else if(S_ISREG(st.st_mode)) { 74 | FILE *fp; 75 | char cmd[CIRCLE_MAX_STRING_LEN + 10]; 76 | char out[1035]; 77 | 78 | sprintf(cmd, "grep %s %s", DGREP_ARGS, temp); 79 | 80 | fp = popen(cmd, "r"); 81 | if (fp == NULL) { 82 | printf("Failed to run command\n" ); 83 | } 84 | 85 | while (fgets(out, sizeof(out) - 1, fp) != NULL) { 86 | printf("%s", out); 87 | } 88 | 89 | pclose(fp); 90 | } 91 | } 92 | 93 | void 94 | print_usage(char *prog) 95 | { 96 | fprintf(stdout, "Usage: %s -p -a \n", prog); 97 | } 98 | 99 | int 100 | main (int argc, char **argv) 101 | { 102 | int index; 103 | int c; 104 | 105 | int args_flag = 0; 106 | int path_flag = 0; 107 | 108 | DGREP_debug_stream = stdout; 109 | DGREP_debug_level = DGREP_LOG_DBG; 110 | 111 | opterr = 0; 112 | while((c = getopt(argc, argv, "p:a:")) != -1) 113 | { 114 | switch(c) 115 | { 116 | case 'a': 117 | DGREP_ARGS = optarg; 118 | args_flag = 1; 119 | break; 120 | case 'p': 121 | DGREP_PATH = realpath(optarg, NULL); 122 | path_flag = 1; 123 | break; 124 | case '?': 125 | if (optopt == 'p' || optopt == 'a') 126 | fprintf(stderr, "Error: Option -%c requires an argument.\n", optopt); 127 | else if (isprint (optopt)) 128 | fprintf(stderr, "Error: Unknown option `-%c'.\n", optopt); 129 | else 130 | fprintf(stderr, 131 | "Error: Unknown option character `\\x%x'.\n", 132 | optopt); 133 | 134 | exit(EXIT_FAILURE); 135 | default: 136 | abort(); 137 | } 138 | } 139 | 140 | for (index = optind; index < argc; index++) 141 | { 142 | print_usage(argv[0]); 143 | fprintf(stdout, "Error: Non-option argument %s\n", argv[index]); 144 | 145 | exit(EXIT_FAILURE); 146 | } 147 | 148 | if(path_flag == 0) 149 | { 150 | print_usage(argv[0]); 151 | fprintf(stdout, "Error: You must specify a starting path name.\n"); 152 | 153 | exit(EXIT_FAILURE); 154 | } 155 | 156 | if(args_flag == 0) 157 | { 158 | print_usage(argv[0]); 159 | fprintf(stdout, "Error: You must specify a pattern to search on.\n"); 160 | 161 | exit(EXIT_FAILURE); 162 | } 163 | 164 | DGREP_global_rank = CIRCLE_init(argc, argv, CIRCLE_DEFAULT_FLAGS); 165 | 166 | CIRCLE_cb_create (&DGREP_start); 167 | CIRCLE_cb_process(&DGREP_search); 168 | 169 | CIRCLE_begin(); 170 | CIRCLE_finalize(); 171 | 172 | exit(EXIT_SUCCESS); 173 | } 174 | 175 | /* EOF */ 176 | -------------------------------------------------------------------------------- /test/tests/test_dcp/test_fiemap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ############################################################################## 4 | # Description: 5 | # 6 | # A test to check if dcp works well with FIEMAP. 7 | # 8 | ############################################################################## 9 | 10 | # Turn on verbose output 11 | #set -x 12 | 13 | DCP_TEST_BIN=${DCP_TEST_BIN:-${1}} 14 | DCP_MPIRUN_BIN=${DCP_MPIRUN_BIN:-${2}} 15 | DCP_CMP_BIN=${DCP_CMP_BIN:-${3}} 16 | DCP_SRC_DIR=${DCP_SRC_DIR:-${4}} 17 | DCP_DEST_DIR=${DCP_DEST_DIR:-${5}} 18 | DCP_TMP_FILE=${DCP_TMP_FILE:-${6}} 19 | 20 | echo "Using dcp1 binary at: $DCP_TEST_BIN" 21 | echo "Using mpirun binary at: $DCP_MPIRUN_BIN" 22 | echo "Using cmp binary at: $DCP_CMP_BIN" 23 | echo "Using src directory at: $DCP_SRC_DIR" 24 | echo "Using dest directory at: $DCP_DEST_DIR" 25 | 26 | #build checkfiemap if not found 27 | CHECKFIEMAP=${CHECKFIEMAP:-"`dirname $0`/checkfiemap"} 28 | if [ ! -f "$CHECKFIEMAP" ]; then 29 | cc `dirname $0`/checkfiemap.c -o `dirname $0`/checkfiemap 30 | if [[ $? -ne 0 ]]; then 31 | echo "Failed to build `dirname $0`/checkfiemap.c" 32 | exit 1; 33 | fi 34 | CHECKFIEMAP=`dirname $0`/checkfiemap 35 | fi 36 | 37 | # Create the two empty files. 38 | touch $DCP_SRC_DIR/aaa 39 | touch $DCP_DEST_DIR/aaa 40 | 41 | $CHECKFIEMAP --test $DCP_SRC_DIR/aaa 42 | if [[ $? -ne 0 ]]; then 43 | echo "Source filesystem $DCP_SRC_DIR does not support FIEMAP, skip testing" 44 | exit 0 45 | fi 46 | 47 | NEED_CHECK_FIEMAP=true 48 | 49 | $CHECKFIEMAP --test $DCP_DEST_DIR/aaa 50 | if [[ $? -ne 0 ]]; then 51 | echo "Destination filesystem $DCP_DEST_DIR does not support FIEMAP, won't check fiemap" 52 | NEED_CHECK_FIEMAP=false 53 | fi 54 | 55 | rm -f $DCP_SRC_DIR/aaa 56 | rm -f $DCP_DEST_DIR/aaa 57 | 58 | rm -f $DCP_SRC_DIR/$DCP_TMP_FILE 59 | rm -f $DCP_DEST_DIR/$DCP_TMP_FILE 60 | 61 | function test_fiemap { 62 | $CHECKFIEMAP $DCP_SRC_DIR/$DCP_TMP_FILE $1 63 | if [[ $? -ne 0 ]]; then 64 | echo "Fiemap mismatch: $DCP_SRC_DIR/$DCP_TMP_FILE" 65 | rm -f $DCP_SRC_DIR/$DCP_TMP_FILE 66 | rm -f $DCP_DEST_DIR/$DCP_TMP_FILE 67 | exit 1 68 | fi 69 | 70 | $DCP_MPIRUN_BIN -np 3 $DCP_TEST_BIN -S $DCP_SRC_DIR/$DCP_TMP_FILE $DCP_DEST_DIR 71 | if [[ $? -ne 0 ]]; then 72 | echo "Fialed to run cmd: $DCP_MPIRUN_BIN -np 3 $DCP_TEST_BIN -S $DCP_SRC_DIR/$DCP_TMP_FILE $DCP_DEST_DIR" 73 | rm -f $DCP_SRC_DIR/$DCP_TMP_FILE 74 | rm -f $DCP_DEST_DIR/$DCP_TMP_FILE 75 | exit 1 76 | fi 77 | 78 | # Stat the two random files. 79 | stat $DCP_SRC_DIR/$DCP_TMP_FILE 80 | if [[ $? -ne 0 ]]; then 81 | echo "Failed to stat $DCP_SRC_DIR/$DCP_TMP_FILE" 82 | rm -f $DCP_SRC_DIR/$DCP_TMP_FILE 83 | rm -f $DCP_DEST_DIR/$DCP_TMP_FILE 84 | exit 1 85 | fi 86 | stat $DCP_DEST_DIR/$DCP_TMP_FILE 87 | if [[ $? -ne 0 ]]; then 88 | echo "Failed to stat $DCP_DEST_DIR/$DCP_TMP_FILE" 89 | rm -f $DCP_SRC_DIR/$DCP_TMP_FILE 90 | rm -f $DCP_DEST_DIR/$DCP_TMP_FILE 91 | exit 1 92 | fi 93 | 94 | $DCP_CMP_BIN $DCP_SRC_DIR/$DCP_TMP_FILE $DCP_DEST_DIR/$DCP_TMP_FILE 95 | if [[ $? -ne 0 ]]; then 96 | echo "CMP mismatch: $DCP_SRC_DIR/$DCP_TMP_FILE $DCP_DEST_DIR/$DCP_TMP_FILE." 97 | exit 1 98 | fi 99 | if [ $NEED_CHECK_FIEMAP == true ]; then 100 | $CHECKFIEMAP $DCP_DEST_DIR/$DCP_TMP_FILE $1 101 | if [[ $? -ne 0 ]]; then 102 | echo "Fiemap mismatch: $DCP_DEST_DIR/$DCP_TMP_FILE" 103 | rm -f $DCP_SRC_DIR/$DCP_TMP_FILE 104 | rm -f $DCP_DEST_DIR/$DCP_TMP_FILE 105 | exit 1 106 | fi 107 | fi 108 | 109 | rm -f $DCP_SRC_DIR/$DCP_TMP_FILE 110 | rm -f $DCP_DEST_DIR/$DCP_TMP_FILE 111 | } 112 | 113 | echo "Subtest 1, no holes." 114 | # Create source file, no holes. 115 | dd if=/dev/urandom of=$DCP_SRC_DIR/$DCP_TMP_FILE bs=4M count=1 116 | dd if=/dev/urandom of=$DCP_SRC_DIR/$DCP_TMP_FILE bs=4M seek=1 count=1 117 | 118 | test_fiemap 8388608 119 | 120 | echo "Subtest 2, front 4K hole." 121 | # Create source file, front 4K hole. 122 | dd if=/dev/urandom of=$DCP_SRC_DIR/$DCP_TMP_FILE bs=4K seek=1 count=1 123 | dd if=/dev/urandom of=$DCP_SRC_DIR/$DCP_TMP_FILE bs=4k seek=2 count=1 124 | 125 | test_fiemap 8192 126 | 127 | echo "Subtest 3, middle 1G hole." 128 | # Create source file, middle 1G hole. 129 | dd if=/dev/urandom of=$DCP_SRC_DIR/$DCP_TMP_FILE bs=1M count=1 130 | dd if=/dev/urandom of=$DCP_SRC_DIR/$DCP_TMP_FILE bs=1M seek=1025 count=1 131 | 132 | test_fiemap 2097152 133 | 134 | echo "Subtest 4, end 1G hole." 135 | # Create source file, middle 1G hole. 136 | dd if=/dev/urandom of=$DCP_SRC_DIR/$DCP_TMP_FILE bs=1M count=1 137 | dd if=/dev/urandom of=$DCP_SRC_DIR/$DCP_TMP_FILE bs=1M seek=1025 count=0 138 | 139 | test_fiemap 1048576 140 | 141 | echo "Subtest 5, front 4M, middle 1G, end 1G hole." 142 | # Create source file, middle 1G hole. 143 | dd if=/dev/urandom of=$DCP_SRC_DIR/$DCP_TMP_FILE bs=4M seek=1 count=1 144 | dd if=/dev/urandom of=$DCP_SRC_DIR/$DCP_TMP_FILE bs=4M seek=258 count=1 145 | dd if=/dev/urandom of=$DCP_SRC_DIR/$DCP_TMP_FILE bs=4M seek=515 count=0 146 | 147 | test_fiemap 8388608 148 | 149 | exit 0 150 | --------------------------------------------------------------------------------