├── COPYING ├── version.txt ├── CMake ├── Git │ ├── NOTICE │ ├── Git.cmake │ ├── GitInfo │ └── LICENSE ├── hdf5_vfd_gds-config-version.cmake.in ├── hdf5_vfd_gds.pc.in ├── hdf5_vfd_gds-config.cmake.in └── Hdf5GdsVfdDetermineVersion.cmake ├── examples ├── CMakeLists.txt └── simple_dset_write.c ├── cuda.mk.in ├── cuda.ac ├── src ├── H5FDgds.h ├── CMakeLists.txt ├── H5FDgds_err.h └── H5FDgds.c ├── test ├── CMakeLists.txt └── gds_test.c ├── .github └── workflows │ └── main.yml ├── license.txt ├── README.md └── CMakeLists.txt /COPYING: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /version.txt: -------------------------------------------------------------------------------- 1 | 1.0.2 2 | -------------------------------------------------------------------------------- /CMake/Git/NOTICE: -------------------------------------------------------------------------------- 1 | Git Support Scripts 2 | Copyright 2011 Kitware, Inc. 3 | 4 | This product includes software developed at Kitware, Inc. 5 | (http://www.kitware.com/). 6 | -------------------------------------------------------------------------------- /CMake/hdf5_vfd_gds-config-version.cmake.in: -------------------------------------------------------------------------------- 1 | #----------------------------------------------------------------------------- 2 | # Version file for install directory 3 | #----------------------------------------------------------------------------- 4 | set(PACKAGE_VERSION @HDF5_VFD_GDS_VERSION_MAJOR@.@HDF5_VFD_GDS_VERSION_MINOR@.@HDF5_VFD_GDS_VERSION_PATCH@) 5 | 6 | if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") 7 | set(PACKAGE_VERSION_COMPATIBLE FALSE) 8 | else() 9 | set(PACKAGE_VERSION_COMPATIBLE TRUE) 10 | if ("${PACKAGE_VERSION}" STREQUAL "${PACKAGE_FIND_VERSION}") 11 | set(PACKAGE_VERSION_EXACT TRUE) 12 | endif() 13 | endif() 14 | -------------------------------------------------------------------------------- /CMake/hdf5_vfd_gds.pc.in: -------------------------------------------------------------------------------- 1 | # This gives access to the HDF5 GDS VFD header files 2 | prefix=@CMAKE_INSTALL_PREFIX@ 3 | exec_prefix=@CMAKE_INSTALL_PREFIX@ 4 | libdir=@HDF5_VFD_GDS_INSTALL_LIB_DIR@ 5 | includedir=@HDF5_VFD_GDS_INSTALL_INCLUDE_DIR@ 6 | 7 | Name: @HDF5_VFD_GDS_PACKAGE@ 8 | Description: @HDF5_VFD_GDS_PACKAGE_DESCRIPTION@ 9 | Version: @HDF5_VFD_GDS_VERSION_MAJOR@.@HDF5_VFD_GDS_VERSION_MINOR@.@HDF5_VFD_GDS_VERSION_PATCH@ 10 | URL: @HDF5_VFD_GDS_PACKAGE_URL@ 11 | Requires: @HDF5_VFD_GDS_PKG_DEPENDENCIES@ 12 | Libs: -L${libdir} @HDF5_VFD_GDS_LIBRARIES@ 13 | Libs.private: @HDF5_VFD_GDS_LIB_DEPENDENCIES@ 14 | Cflags: -I${includedir} @HDF5_VFD_GDS_INCLUDE_DEPENDENCIES@ 15 | -------------------------------------------------------------------------------- /CMake/Git/Git.cmake: -------------------------------------------------------------------------------- 1 | #============================================================================= 2 | # Copyright 2011 Kitware, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | #============================================================================= 16 | 17 | # Check for a hint left by the 'GitInfo' script. 18 | if(NOT GIT_EXECUTABLE) 19 | get_filename_component(_Git_DIR ${CMAKE_CURRENT_LIST_FILE} PATH) 20 | include(${_Git_DIR}/GitInfo.cmake OPTIONAL) 21 | if(GitInfo_GIT_EXECUTABLE) 22 | if(EXISTS "${GitInfo_GIT_EXECUTABLE}") 23 | set(GIT_EXECUTABLE "${GitInfo_GIT_EXECUTABLE}") 24 | elseif(EXISTS "${GitInfo_GIT_EXECUTABLE}.exe") 25 | set(GIT_EXECUTABLE "${GitInfo_GIT_EXECUTABLE}.exe") 26 | endif() 27 | endif() 28 | endif() 29 | 30 | # Find Git. 31 | find_package(Git) 32 | -------------------------------------------------------------------------------- /examples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.12.2 FATAL_ERROR) 2 | 3 | #------------------------------------------------------------------------------ 4 | # Include source and build directories 5 | #------------------------------------------------------------------------------ 6 | include_directories( 7 | ${CMAKE_CURRENT_SOURCE_DIR} 8 | ${CMAKE_CURRENT_BINARY_DIR} 9 | ) 10 | 11 | #----------------------------------------------------------------------------- 12 | # Define Sources 13 | #----------------------------------------------------------------------------- 14 | set(examples 15 | simple_dset_write 16 | ) 17 | 18 | foreach (example ${examples}) 19 | add_executable (h5gds_${example} 20 | ${CMAKE_CURRENT_SOURCE_DIR}/${example}.c 21 | ) 22 | target_include_directories(h5gds_${example} 23 | PUBLIC ${CMAKE_SOURCE_DIR}/src 24 | ) 25 | target_include_directories(h5gds_${example} 26 | PUBLIC "$" 27 | $ 28 | ) 29 | target_include_directories(h5gds_${example} 30 | SYSTEM PUBLIC ${HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES} 31 | ) 32 | target_link_libraries(h5gds_${example} 33 | ${HDF5_VFD_GDS_EXPORTED_LIBS} 34 | ${HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES} 35 | ${HDF5_VFD_GDS_EXT_PKG_DEPENDENCIES} 36 | CUDA::cudart 37 | ) 38 | endforeach() 39 | -------------------------------------------------------------------------------- /cuda.mk.in: -------------------------------------------------------------------------------- 1 | # Turn on if outer variable is set. Otherwise, allow user to set explicitly. 2 | CUDA_DEBUG = $(DEBUG) 3 | CUDA_VERBOSE = $(VERBOSE) 4 | 5 | cuda_includes := -I@CUDA_DIR@/include 6 | #cuda_srcs := $(srcdir)/src/H5FDsec2.c 7 | cuda_libs := @CUDA_LIBS@ 8 | cuda_ldflags += -L@CUDA_DIR@/lib64 9 | cuda_objs := $(patsubst %.cu,%.o,$(subst $(srcdir),$(build_dir),$(cuda_srcs))) 10 | 11 | # For more details on the architectures, see 12 | # http://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#virtual-architecture-feature-list 13 | nvcc_flags += -arch=compute_@CUDA_LEVEL@ -code=sm_@CUDA_LEVEL@ 14 | 15 | # nvcc includes are broken when the host compiler is gcc >= 5.0 16 | nvcc_flags += -D_FORCE_INLINES 17 | 18 | ifeq (1,$(CUDA_VERBOSE)) 19 | nvcc_flags += --ptxas-options=-v 20 | endif 21 | 22 | ifeq (1,$(CUDA_DEBUG)) 23 | nvcc_flags += --device-debug 24 | else 25 | nvcc_flags += -use_fast_math 26 | endif 27 | 28 | .PHONY: show-config 29 | show-config: cuda-show-config 30 | 31 | .PHONY: cuda-show-config 32 | cuda-show-config: 33 | @ echo NVCC_FLAGS = $(nvcc_flags)"\n" 34 | @ echo CUDA_DEFINES = $(cuda_defines)"\n" 35 | @ echo CUDA_INCLUDES = $(cuda_includes)"\n" 36 | @ echo CUDA_LDFLAGS = $(cuda_ldflags)"\n" 37 | @ echo CUDA_LIBS = $(cuda_libs)"\n" 38 | 39 | .PHONY: dist-clean 40 | dist-clean: cuda-dist-clean 41 | 42 | .PHONY: cuda-dist-clean 43 | cuda-dist-clean: 44 | @ $(RM) cuda.mk 45 | 46 | $(build_dir)/%.o: $(srcdir)/src/%.cu 47 | @ echo Compiling $<... 48 | $(quiet) @NVCC_PATH@ -std=$(cxx_std) -Xcompiler "$(cxx_flags) $(depend_flags) $(depend_dir)/$*$(depend_suffix)" $(nvcc_flags) -c $< -o $@ 49 | -------------------------------------------------------------------------------- /cuda.ac: -------------------------------------------------------------------------------- 1 | # 2 | ######################################################## 3 | 4 | # Allow setting CUDA compute capability at `configure`-time. 5 | AC_ARG_WITH([cuda-level], 6 | [AS_HELP_STRING( 7 | [--with-cuda-level=LEVEL], 8 | [use CUDA compute capability LEVEL])], 9 | [CUDA_LEVEL=$withval], 10 | [CUDA_LEVEL=70]) 11 | 12 | AC_ARG_WITH([cuda], 13 | [AS_HELP_STRING([--with-cuda@<:@=PATH@:>@], 14 | [use CUDA (installed in PATH) @<:@no@:>@])], 15 | [CUDA_DIR=$withval], 16 | [CUDA_DIR="no"]) 17 | 18 | # We also support using the argument "auto" to mean "yes" 19 | case x$CUDA_DIR in 20 | xno) ENABLE_CUDA=no ;; 21 | x|xauto|xyes) ENABLE_CUDA=yes ;; 22 | *) ENABLE_CUDA=yes ;; 23 | esac 24 | 25 | if test x$ENABLE_CUDA = xyes; then 26 | AC_PATH_PROG([NVCC_PATH],nvcc,[],[$CUDA_DIR/bin]) 27 | if test -z "$NVCC_PATH" ; then 28 | AC_MSG_ERROR(['bin/nvcc' not found in $CUDA_DIR/bin]) 29 | fi 30 | 31 | AC_CONFIG_FILES([cuda.mk]) 32 | fi 33 | 34 | # Some CUDA installs only use the runtime library, so test for the presence of each 35 | if test x$ENABLE_CUDA = xyes; then 36 | if test -e $CUDA_DIR/lib64/libcuda.so; then 37 | CUDA_LIBS=-lcuda 38 | else 39 | if test -e $CUDA_DIR/lib64/libcudart.so; then 40 | CUDA_LIBS=-lcudart 41 | else 42 | >&2 echo Could not find libcuda[rt].so 43 | exit 1 44 | fi 45 | fi 46 | fi 47 | 48 | AC_ARG_WITH([cufile], 49 | [AS_HELP_STRING([--with-cufile=DIR], 50 | [Beta. Might change in the future.])], 51 | [CUFILE_DIR=$withval], 52 | [withval=no]) 53 | 54 | AC_SUBST([ENABLE_CUDA]) 55 | AC_SUBST([CUDA_LEVEL]) 56 | AC_SUBST([CUDA_DIR]) 57 | AC_SUBST([NVCC_PATH]) 58 | AC_SUBST([CUDA_LIBS]) 59 | AC_SUBST([CUFILE_DIR]) 60 | -------------------------------------------------------------------------------- /CMake/Git/GitInfo: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | #============================================================================= 3 | # Copyright 2011 Kitware, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | #============================================================================= 17 | 18 | # Path conversion function. 19 | case "$(uname)" in 20 | *CYGWIN*) 21 | native_path() { 22 | cygpath -m "$1" 23 | } 24 | ;; 25 | *MINGW*) 26 | native_path() { 27 | cmd //c echo "$1" | sed 's/^"//;s/"$//' 28 | } 29 | ;; 30 | *) 31 | native_path() { 32 | echo "$1" 33 | } 34 | ;; 35 | esac 36 | 37 | # Compute native path to "git" executable. 38 | if git="$(type -p git)"; then 39 | git="$(native_path "${git}")" 40 | else 41 | git='' 42 | fi 43 | 44 | # Compute native path to ".git" dir. 45 | if dir="$(git rev-parse --git-dir)"; then 46 | dir="$(cd "$dir"; pwd)" 47 | git_dir="$(native_path "${dir}")" 48 | else 49 | git_dir='' 50 | fi 51 | 52 | # Store the values in a CMake file next to this script. 53 | echo >"${BASH_SOURCE%/*}/GitInfo.cmake" '# Generated by GitInfo 54 | set(GitInfo 1) 55 | set(GitInfo_GIT_EXECUTABLE "'"$git"'") 56 | set(GitInfo_GIT_DIR "'"$git_dir"'") 57 | ' 58 | -------------------------------------------------------------------------------- /src/H5FDgds.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Copyright by The HDF Group. * 3 | * All rights reserved. * 4 | * * 5 | * This file is part of the HDF5 GDS Virtual File Driver. The full copyright * 6 | * notice, including terms governing use, modification, and redistribution, * 7 | * is contained in the COPYING file, which can be found at the root of the * 8 | * source code distribution tree. * 9 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 10 | 11 | /* 12 | * Programmer: John Ravi 13 | * Wednesday, July 1, 2020 14 | * 15 | * Purpose: The public header file for the CUDA GPUDirect Storage driver. 16 | */ 17 | #ifndef H5FDgds_H 18 | #define H5FDgds_H 19 | 20 | #include 21 | 22 | #define H5FD_GDS_NAME "gds" 23 | #define H5FD_GDS_VALUE ((H5FD_class_value_t)(512)) 24 | 25 | #ifdef __cplusplus 26 | extern "C" { 27 | #endif 28 | 29 | /* Default values for memory boundary, file block size, and maximal copy buffer size. 30 | * Application can set these values through the function H5Pset_fapl_gds. */ 31 | #define H5FD_GDS_MBOUNDARY_DEF 4096 32 | #define H5FD_GDS_FBSIZE_DEF 4096 33 | #define H5FD_GDS_CBSIZE_DEF (16 * 1024 * 1024) 34 | 35 | herr_t H5Pset_fapl_gds(hid_t fapl_id, size_t alignment, size_t block_size, size_t cbuf_size); 36 | herr_t H5Pget_fapl_gds(hid_t fapl_id, size_t *boundary /*out*/, size_t *block_size /*out*/, 37 | size_t *cbuf_size /*out*/); 38 | 39 | #ifdef __cplusplus 40 | } 41 | #endif 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /examples/simple_dset_write.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "hdf5.h" 4 | #include "cuda.h" 5 | #include "cuda_runtime.h" 6 | 7 | #include "H5FDgds.h" 8 | 9 | #define FILENAME "gds_simple_dset_write.h5" 10 | #define DIM_SIZE 10 11 | 12 | int 13 | main(int argc, char **argv) 14 | { 15 | hsize_t i, dims[] = { DIM_SIZE, DIM_SIZE }; 16 | hid_t file_id = H5I_INVALID_HID; 17 | hid_t fapl_id = H5I_INVALID_HID; 18 | hid_t dset_id = H5I_INVALID_HID; 19 | hid_t space_id = H5I_INVALID_HID; 20 | int data[DIM_SIZE * DIM_SIZE]; 21 | int *cuda_buf = NULL; 22 | herr_t status; 23 | 24 | fapl_id = H5Pcreate(H5P_FILE_ACCESS); 25 | assert(fapl_id > 0); 26 | 27 | status = H5Pset_fapl_gds(fapl_id, MBOUNDARY_DEF, FBSIZE_DEF, CBSIZE_DEF); 28 | assert(status >= 0); 29 | 30 | file_id = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); 31 | assert(file_id > 0); 32 | 33 | space_id = H5Screate_simple(2, dims, NULL); 34 | assert(space_id > 0); 35 | 36 | dset_id = H5Dcreate2(file_id, "dset", H5T_NATIVE_INT, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 37 | assert(dset_id > 0); 38 | 39 | for (i = 0; i < dims[0] * dims[1]; i++) 40 | data[i] = i; 41 | 42 | /* Allocate memory on CUDA device and copy data into that buffer */ 43 | cudaMalloc((void **)&cuda_buf, DIM_SIZE * DIM_SIZE * sizeof(int)); 44 | cudaMemcpy(cuda_buf, data, DIM_SIZE * DIM_SIZE * sizeof(int), cudaMemcpyHostToDevice); 45 | 46 | /* Write dataset using buffer allocated on CUDA device */ 47 | status = H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, cuda_buf); 48 | assert(status >= 0); 49 | 50 | cudaFree(&cuda_buf); 51 | 52 | status = H5Sclose(space_id); 53 | assert(status >= 0); 54 | 55 | status = H5Dclose(dset_id); 56 | assert(status >= 0); 57 | 58 | status = H5Pclose(fapl_id); 59 | assert(status >= 0); 60 | 61 | status = H5Fclose(file_id); 62 | assert(status >= 0); 63 | 64 | return 0; 65 | } 66 | 67 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Include source and build directories 3 | #------------------------------------------------------------------------------ 4 | include_directories( 5 | ${CMAKE_CURRENT_SOURCE_DIR} 6 | ${CMAKE_CURRENT_BINARY_DIR} 7 | ) 8 | 9 | #------------------------------------------------------------------------------ 10 | # External dependencies 11 | #------------------------------------------------------------------------------ 12 | find_package(MPI REQUIRED) 13 | if(MPI_C_FOUND) 14 | set(HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES 15 | ${HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES} 16 | ${MPI_C_LIBRARIES} 17 | ) 18 | include_directories( 19 | ${MPI_C_INCLUDE_DIRS} 20 | ) 21 | endif() 22 | 23 | #----------------------------------------------------------------------------- 24 | # Define Sources and tests 25 | #----------------------------------------------------------------------------- 26 | set(vfd_gds_tests 27 | gds_test 28 | ) 29 | 30 | foreach(vfd_test ${vfd_gds_tests}) 31 | add_executable(${vfd_test} 32 | ${CMAKE_CURRENT_SOURCE_DIR}/${vfd_test}.c 33 | ) 34 | target_include_directories(${vfd_test} 35 | PUBLIC "$" 36 | $ 37 | ) 38 | target_include_directories(${vfd_test} 39 | SYSTEM PUBLIC ${HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES} 40 | ) 41 | target_link_libraries(${vfd_test} 42 | ${HDF5_VFD_GDS_EXPORTED_LIBS} 43 | ${HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES} 44 | ${HDF5_VFD_GDS_EXT_PKG_DEPENDENCIES} 45 | CUDA::cudart 46 | ) 47 | endforeach() 48 | 49 | #----------------------------------------------------------------------------- 50 | # Tests 51 | #----------------------------------------------------------------------------- 52 | foreach(vfd_test ${vfd_gds_tests}) 53 | add_test(NAME "${vfd_test}" 54 | COMMAND $ 55 | ) 56 | set_tests_properties(${vfd_test} PROPERTIES 57 | ENVIRONMENT "HDF5_PLUGIN_PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" 58 | ) 59 | endforeach() 60 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Test HDF5 GPUDirect Storage VFD 2 | on: 3 | push: 4 | branches: [ master ] 5 | workflow_dispatch: 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout vfd-gds 11 | uses: actions/checkout@v3 12 | - name: Checkout HDF5 13 | uses: actions/checkout@v3 14 | with: 15 | repository: HDFGroup/hdf5 16 | path: hdf5 17 | - name: Install dependencies 18 | run: sudo apt-get install ninja-build automake autoconf libtool libtool-bin libopenmpi-dev libpthread-stubs0-dev 19 | 20 | - name: Install GDS/CUDA 21 | run: | 22 | wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb 23 | sudo dpkg -i cuda-keyring_1.0-1_all.deb 24 | sudo apt-get update 25 | sudo apt-get install -y cuda-nvcc-11-8 libcufile-dev-11-8 26 | echo "PATH=/usr/local/cuda/bin${PATH:+:${PATH}}" >> $GITHUB_ENV 27 | echo "CUDA_PATH=/usr/local/cuda" >> $GITHUB_ENV 28 | echo "LD_LIBRARY_PATH=/usr/local/lib:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV 29 | - name: Install HDF5 30 | run: | 31 | cmake --version 32 | mkdir hdf5/build 33 | cd hdf5/build 34 | cmake -DCMAKE_BUILD_TYPE=Release \ 35 | -DBUILD_TESTING:BOOL=OFF \ 36 | -DENABLE_EXTENDED_TESTS:BOOL=OFF \ 37 | -DHDF5_BUILD_EXAMPLES:BOOL=OFF \ 38 | -DHDF5_BUILD_HL_LIB:BOOL=OFF \ 39 | -DHDF5_BUILD_HL_TOOLS:BOOL=OFF \ 40 | -DCMAKE_INSTALL_PREFIX=/usr/local \ 41 | -DHDF5_ENABLE_PARALLEL:BOOL=ON \ 42 | -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ 43 | -DALLOW_UNSUPPORTED:BOOL=ON \ 44 | .. 45 | sudo make -j2 install 46 | cd .. 47 | git rev-parse HEAD > git.txt 48 | 49 | - name: Install vfd-gds 50 | env: 51 | HDF5_PLUGIN_PATH: /usr/local/lib 52 | CC: mpicc 53 | run: | 54 | mkdir build && cd $_ 55 | cmake -DHDF5_DIR=/usr/local .. 56 | sudo make -j2 install 57 | #ctest --output-on-failure -------------------------------------------------------------------------------- /CMake/hdf5_vfd_gds-config.cmake.in: -------------------------------------------------------------------------------- 1 | #----------------------------------------------------------------------------- 2 | # hdf5_vfd_gds-config.cmake - HDF5 GDS VFD CMake configuration file for external projects. 3 | #----------------------------------------------------------------------------- 4 | set(__hdf5_vfd_gds_install_tree @HDF5_VFD_GDS_CONFIG_INSTALLED@) 5 | if(__hdf5_vfd_gds_install_tree) 6 | get_filename_component(location "${CMAKE_CURRENT_LIST_FILE}" PATH) 7 | set(HDF5_VFD_GDS_CONFIG_TARGETS_FILE "${location}/@HDF5_VFD_GDS_PACKAGE@-targets.cmake") 8 | else() 9 | # This is the build location. 10 | set(HDF5_VFD_GDS_CONFIG_TARGETS_FILE "@HDF5_VFD_GDS_BINARY_DIR@/src/@HDF5_VFD_GDS_PACKAGE@-targets.cmake") 11 | endif() 12 | 13 | #----------------------------------------------------------------------------- 14 | # User Options 15 | #----------------------------------------------------------------------------- 16 | set(HDF5_VFD_GDS_USE_SYSTEM_HDF5 @HDF5_VFD_GDS_USE_SYSTEM_HDF5@) 17 | set(HDF5_VFD_GDS_USE_SYSTEM_CMAKE_HDF5 @HDF5_VFD_GDS_USE_SYSTEM_CMAKE_HDF5@) 18 | 19 | #----------------------------------------------------------------------------- 20 | # Version information for HDF5 GDS VFD 21 | #----------------------------------------------------------------------------- 22 | set(HDF5_VFD_GDS_VERSION_MAJOR @HDF5_VFD_GDS_VERSION_MAJOR@) 23 | set(HDF5_VFD_GDS_VERSION_MINOR @HDF5_VFD_GDS_VERSION_MINOR@) 24 | set(HDF5_VFD_GDS_VERSION_PATCH @HDF5_VFD_GDS_VERSION_PATCH@) 25 | set(HDF5_VFD_GDS_VERSION_FULL @HDF5_VFD_GDS_VERSION_FULL@) 26 | set(HDF5_VFD_GDS_VERSION @HDF5_VFD_GDS_VERSION@) 27 | 28 | #----------------------------------------------------------------------------- 29 | # Don't include targets if this file is being picked up by another 30 | # project which has already built the HDF5 GDS VFD as a subproject 31 | #----------------------------------------------------------------------------- 32 | if(NOT HDF5_VFD_GDS_INSTALL_SKIP_TARGETS) 33 | if(NOT TARGET "hdf5" AND HDF5_VFD_GDS_USE_SYSTEM_CMAKE_HDF5) 34 | include(@HDF5_DIR@/hdf5-config.cmake) 35 | endif() 36 | if(NOT TARGET "@HDF5_VFD_GDS_PACKAGE@") 37 | include(${HDF5_VFD_GDS_CONFIG_TARGETS_FILE}) 38 | endif() 39 | endif() 40 | 41 | # cleanup 42 | unset(__hdf5_vfd_gds_install_tree) 43 | -------------------------------------------------------------------------------- /CMake/Hdf5GdsVfdDetermineVersion.cmake: -------------------------------------------------------------------------------- 1 | #========================================================================= 2 | # 3 | # Copyright (c) Kitware, Inc. 4 | # All rights reserved. 5 | # See Copyright.txt or http://www.paraview.org/HTML/Copyright.html for details. 6 | # 7 | # This software is distributed WITHOUT ANY WARRANTY; without even 8 | # the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 9 | # PURPOSE. See the above copyright notice for more information. 10 | # 11 | #========================================================================= 12 | 13 | # Used to determine the version using "git describe", if git 14 | # is found. On success sets following variables in caller's scope: 15 | # ${var_prefix}_VERSION 16 | # ${var_prefix}_VERSION_MAJOR 17 | # ${var_prefix}_VERSION_MINOR 18 | # ${var_prefix}_VERSION_PATCH 19 | # ${var_prefix}_VERSION_PATCH_EXTRA 20 | # ${var_prefix}_VERSION_FULL 21 | # ${var_prefix}_VERSION_IS_RELEASE is true, if patch-extra is empty. 22 | # 23 | # If git is not found, or git describe cannot be run successfully, then these 24 | # variables are left unchanged and status message is printed. 25 | # 26 | # Arguments are: 27 | # source_dir : Source directory 28 | # git_command : git executable 29 | # var_prefix : prefix for variables e.g. "HDF5_VFD_GDS". 30 | function(determine_version source_dir git_command var_prefix) 31 | if ("$Format:$" STREQUAL "") 32 | # We are in an exported tarball and should use the shipped version 33 | # information. Just return here to avoid the warning message at the end of 34 | # this function. 35 | return () 36 | elseif (NOT HDF5_VFD_GDS_GIT_DESCRIBE AND 37 | EXISTS ${git_command} AND 38 | EXISTS ${source_dir}/.git) 39 | execute_process( 40 | COMMAND ${git_command} describe 41 | WORKING_DIRECTORY ${source_dir} 42 | RESULT_VARIABLE result 43 | OUTPUT_VARIABLE output 44 | ERROR_QUIET 45 | OUTPUT_STRIP_TRAILING_WHITESPACE 46 | ERROR_STRIP_TRAILING_WHITESPACE) 47 | if (NOT result EQUAL 0) 48 | # git describe failed (bad return code). 49 | set(output "") 50 | endif() 51 | else () 52 | # note, output may be set to empty if ${var_prefix}_GIT_DESCRIBE is not defined. 53 | set(output "${${var_prefix}_GIT_DESCRIBE}") 54 | endif() 55 | 56 | unset(tmp_VERSION) 57 | extract_version_components("${output}" tmp) 58 | if(DEFINED tmp_VERSION) 59 | if (NOT "${tmp_VERSION}" STREQUAL "${${var_prefix}_VERSION}") 60 | message(WARNING 61 | "Version from git (${tmp_VERSION}) disagrees with hard coded version (${${var_prefix}_VERSION}). Either update the git tags or version.txt.") 62 | endif() 63 | foreach(suffix VERSION VERSION_MAJOR VERSION_MINOR VERSION_PATCH 64 | VERSION_PATCH_EXTRA VERSION_FULL VERSION_IS_RELEASE) 65 | set(${var_prefix}_${suffix} ${tmp_${suffix}} PARENT_SCOPE) 66 | endforeach() 67 | else() 68 | message(STATUS 69 | "Could not use git to determine source version, using version ${${var_prefix}_VERSION_FULL}") 70 | endif() 71 | endfunction() 72 | 73 | # Extracts components from a version string. See determine_version() for usage. 74 | function(extract_version_components version_string var_prefix) 75 | string(REGEX MATCH "^v?(([0-9]+)\\.([0-9]+)\\.([0-9]+)-?(.*))$" 76 | version_matches "${version_string}") 77 | if(CMAKE_MATCH_0) 78 | # note, we don't use CMAKE_MATCH_0 for `full` since it may or may not have 79 | # the `v` prefix. 80 | set(full ${CMAKE_MATCH_1}) 81 | set(major ${CMAKE_MATCH_2}) 82 | set(minor ${CMAKE_MATCH_3}) 83 | set(patch ${CMAKE_MATCH_4}) 84 | set(patch_extra ${CMAKE_MATCH_5}) 85 | 86 | set(${var_prefix}_VERSION "${major}.${minor}" PARENT_SCOPE) 87 | set(${var_prefix}_VERSION_MAJOR ${major} PARENT_SCOPE) 88 | set(${var_prefix}_VERSION_MINOR ${minor} PARENT_SCOPE) 89 | set(${var_prefix}_VERSION_PATCH ${patch} PARENT_SCOPE) 90 | set(${var_prefix}_VERSION_PATCH_EXTRA ${patch_extra} PARENT_SCOPE) 91 | set(${var_prefix}_VERSION_FULL ${full} PARENT_SCOPE) 92 | if("${major}.${minor}.${patch}" VERSION_EQUAL "${full}") 93 | set(${var_prefix}_VERSION_IS_RELEASE TRUE PARENT_SCOPE) 94 | else() 95 | set(${var_prefix}_VERSION_IS_RELEASE FALSE PARENT_SCOPE) 96 | endif() 97 | endif() 98 | endfunction() 99 | -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | *** License Agreement *** 2 | 3 | The HDF5 GPUDirect Storage VFD is a Virtual File Driver (VFD) for HDF5 that 4 | can be used to interface with Nvidia's GPUDirect Storage (GDS) API. Copyright (c) 2021, 5 | The Regents of the University of California, through Lawrence Berkeley National 6 | Laboratory (subject to receipt of any required approvals from the U.S. Dept. of 7 | Energy) and North Carolina State University. All rights reserved. 8 | 9 | Redistribution and use in source and binary forms, with or without 10 | modification, are permitted provided that the following conditions are met: 11 | 12 | (1) Redistributions of source code must retain the above copyright notice, 13 | this list of conditions and the following disclaimer. 14 | 15 | (2) Redistributions in binary form must reproduce the above copyright 16 | notice, this list of conditions and the following disclaimer in the 17 | documentation and/or other materials provided with the distribution. 18 | 19 | (3) Neither the name of the University of California, Lawrence Berkeley 20 | National Laboratory, U.S. Dept. of Energy, North Carolina State University 21 | nor the names of its contributors may be used to endorse or promote 22 | products derived from this software without specific prior written permission. 23 | 24 | 25 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 26 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 29 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 | POSSIBILITY OF SUCH DAMAGE. 36 | 37 | You are under no obligation whatsoever to provide any bug fixes, patches, 38 | or upgrades to the features, functionality or performance of the source 39 | code ("Enhancements") to anyone; however, if you choose to make your 40 | Enhancements available either publicly, or directly to Lawrence Berkeley 41 | National Laboratory, without imposing a separate written license agreement 42 | for such Enhancements, then you hereby grant the following license: a 43 | non-exclusive, royalty-free perpetual license to install, use, modify, 44 | prepare derivative works, incorporate into other computer software, 45 | distribute, and sublicense such enhancements or derivative works thereof, 46 | in binary and source code form. 47 | 48 | 49 | 50 | LAWRENCE BERKELEY NATIONAL LABORATORY 51 | Software: HDF5 Nvidia GPUDirect Storage VFD 52 | Developers: John Ravi 53 | 54 | *** License Agreement *** 55 | " The HDF5 GPUDirect Storage VFD is a Virtual File Driver (VFD) for HDF5 56 | that can be used to interface with Nvidia's GPUDirect Storage (GDS) API. 57 | The driver is built as a plugin library that is external to HDF5. 58 | (c) 2021, The Regents of the University of California, through Lawrence 59 | Berkeley National Laboratory (subject to receipt of any required approvals 60 | from the U.S. Dept. of Energy). All rights reserved." 61 | 62 | Redistribution and use in source and binary forms, with or without 63 | modification, are permitted provided that the following conditions are met: 64 | 65 | (1) Redistributions of source code must retain the above copyright notice, 66 | this list of conditions and the following disclaimer. 67 | 68 | (2) Redistributions in binary form must reproduce the above copyright 69 | notice, this list of conditions and the following disclaimer in the 70 | documentation and/or other materials provided with the distribution. 71 | 72 | (3) Neither the name of the University of California, Lawrence Berkeley 73 | National Laboratory, U.S. Dept. of Energy nor the names of its contributors 74 | may be used to endorse or promote products derived from this software 75 | without specific prior written permission. 76 | 77 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 78 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 79 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 80 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 81 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 82 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 83 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 84 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 85 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 86 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 87 | POSSIBILITY OF SUCH DAMAGE. 88 | 89 | You are under no obligation whatsoever to provide any bug fixes, patches, 90 | or upgrades to the features, functionality or performance of the source 91 | code ("Enhancements") to anyone; however, if you choose to make your 92 | Enhancements available either publicly, or directly to Lawrence Berkeley 93 | National Laboratory, without imposing a separate written license agreement 94 | for such Enhancements, then you hereby grant the following license: a 95 | non-exclusive, royalty-free perpetual license to install, use, modify, 96 | prepare derivative works, incorporate into other computer software, 97 | distribute, and sublicense such enhancements or derivative works thereof, 98 | in binary and source code form. 99 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Include source and build directories 3 | #------------------------------------------------------------------------------ 4 | set(HDF5_VFD_GDS_BUILD_INCLUDE_DEPENDENCIES 5 | ${CMAKE_CURRENT_SOURCE_DIR} 6 | ${CMAKE_CURRENT_BINARY_DIR} 7 | ) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Internal dependencies (exported libs) 11 | #------------------------------------------------------------------------------ 12 | 13 | #------------------------------------------------------------------------------ 14 | # External dependencies 15 | #------------------------------------------------------------------------------ 16 | 17 | #------------------------------------------------------------------------------ 18 | # Set sources 19 | #------------------------------------------------------------------------------ 20 | set(HDF5_VFD_GDS_SRCS 21 | ${CMAKE_CURRENT_SOURCE_DIR}/H5FDgds.c 22 | ) 23 | 24 | #------------------------------------------------------------------------------ 25 | # Libraries 26 | #------------------------------------------------------------------------------ 27 | 28 | # Clean up system include path first 29 | foreach(item ${HDF5_VFD_GDS_SYSTEM_INCLUDE_PATH}) 30 | if(HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES) 31 | list(REMOVE_ITEM HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES ${item}) 32 | endif() 33 | endforeach() 34 | 35 | # HDF5 GDS VFD 36 | add_library(hdf5_vfd_gds ${HDF5_VFD_GDS_SRCS}) 37 | target_include_directories(hdf5_vfd_gds 38 | PUBLIC "$" 39 | $ 40 | ) 41 | target_include_directories(hdf5_vfd_gds 42 | SYSTEM PUBLIC ${HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES} 43 | ) 44 | target_link_libraries(hdf5_vfd_gds 45 | PUBLIC 46 | ${HDF5_VFD_GDS_EXPORTED_LIBS} 47 | ${HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES} 48 | ${HDF5_VFD_GDS_EXT_PKG_DEPENDENCIES} 49 | PRIVATE 50 | "$<$:CUDA::cudart>" 51 | "$<$:CUDA::cuFile>" 52 | "$<$:CUDA::cuda_driver>" 53 | ) 54 | hdf5_vfd_gds_set_lib_options(hdf5_vfd_gds "hdf5_vfd_gds" ${HDF5_VFD_GDS_LIBTYPE}) 55 | if(HDF5_VFD_GDS_ENABLE_COVERAGE) 56 | set_coverage_flags(hdf5_vfd_gds) 57 | endif() 58 | 59 | set(HDF5_VFD_GDS_EXPORTED_LIBS hdf5_vfd_gds ${HDF5_VFD_GDS_EXPORTED_LIBS}) 60 | set(HDF5_VFD_GDS_EXPORTED_LIBS ${HDF5_VFD_GDS_EXPORTED_LIBS} PARENT_SCOPE) 61 | 62 | #----------------------------------------------------------------------------- 63 | # Specify project header files to be installed 64 | #----------------------------------------------------------------------------- 65 | set(HDF5_VFD_GDS_HEADERS 66 | ${CMAKE_CURRENT_SOURCE_DIR}/H5FDgds.h 67 | ) 68 | 69 | #----------------------------------------------------------------------------- 70 | # Add file(s) to CMake Install 71 | #----------------------------------------------------------------------------- 72 | install( 73 | FILES 74 | ${HDF5_VFD_GDS_HEADERS} 75 | DESTINATION 76 | ${HDF5_VFD_GDS_INSTALL_INCLUDE_DIR} 77 | COMPONENT 78 | headers 79 | ) 80 | 81 | #----------------------------------------------------------------------------- 82 | # Add Target(s) to CMake Install 83 | #----------------------------------------------------------------------------- 84 | install( 85 | TARGETS 86 | hdf5_vfd_gds 87 | EXPORT 88 | ${HDF5_VFD_GDS_EXPORTED_TARGETS} 89 | LIBRARY DESTINATION ${HDF5_VFD_GDS_INSTALL_LIB_DIR} 90 | ARCHIVE DESTINATION ${HDF5_VFD_GDS_INSTALL_LIB_DIR} 91 | RUNTIME DESTINATION ${HDF5_VFD_GDS_INSTALL_BIN_DIR} 92 | ) 93 | 94 | #----------------------------------------------------------------------------- 95 | # Add Target(s) to CMake Install for import into other projects 96 | #----------------------------------------------------------------------------- 97 | install( 98 | EXPORT 99 | ${HDF5_VFD_GDS_EXPORTED_TARGETS} 100 | DESTINATION 101 | ${HDF5_VFD_GDS_INSTALL_DATA_DIR}/cmake/${HDF5_VFD_GDS_PACKAGE} 102 | FILE 103 | ${HDF5_VFD_GDS_EXPORTED_TARGETS}.cmake 104 | ) 105 | 106 | #----------------------------------------------------------------------------- 107 | # Export all exported targets to the build tree for use by parent project 108 | #----------------------------------------------------------------------------- 109 | if(NOT HDF5_VFD_GDS_EXTERNALLY_CONFIGURED) 110 | export( 111 | TARGETS 112 | ${HDF5_VFD_GDS_EXPORTED_LIBS} 113 | FILE 114 | ${HDF5_VFD_GDS_EXPORTED_TARGETS}.cmake 115 | ) 116 | # TODO There is a namespace issue with the way HDF5 currently defines targets 117 | # if(NOT HDF5_VFD_GDS_USE_SYSTEM_HDF5) 118 | # export( 119 | # TARGETS 120 | # ... 121 | # APPEND FILE 122 | # ${HDF5_VFD_GDS_EXPORTED_TARGETS}.cmake 123 | # NAMESPACE hdf5:: 124 | # ) 125 | # endif() 126 | endif() 127 | 128 | #------------------------------------------------------------------------------ 129 | # Set variables for parent scope 130 | #------------------------------------------------------------------------------ 131 | 132 | # Pkg-config configuration 133 | if(CMAKE_BUILD_TYPE) 134 | string(TOLOWER ${CMAKE_BUILD_TYPE} lower_cmake_build_type) 135 | endif() 136 | 137 | # HDF5 GDS VFD package dependencies 138 | foreach(pkg_dep ${HDF5_VFD_GDS_EXT_PKG_DEPENDENCIES}) 139 | set(HDF5_VFD_GDS_PKG_DEPENDENCIES "${HDF5_VFD_GDS_PKG_DEPENDENCIES} ${pkg_dep}") 140 | endforeach() 141 | set(HDF5_VFD_GDS_PKG_DEPENDENCIES ${HDF5_VFD_GDS_PKG_DEPENDENCIES} PARENT_SCOPE) 142 | 143 | # HDF5 GDS VFD private library dependencies 144 | foreach(exported_lib ${HDF5_VFD_GDS_EXPORTED_LIBS}) 145 | if(lower_cmake_build_type MATCHES "debug") 146 | get_target_property(HDF5_VFD_GDS_LIBRARY ${exported_lib} OUTPUT_NAME_DEBUG) 147 | else() 148 | get_target_property(HDF5_VFD_GDS_LIBRARY ${exported_lib} OUTPUT_NAME_RELEASE) 149 | endif() 150 | set(HDF5_VFD_GDS_LIBRARIES "${HDF5_VFD_GDS_LIBRARIES} -l${HDF5_VFD_GDS_LIBRARY}") 151 | endforeach() 152 | set(HDF5_VFD_GDS_LIBRARIES ${HDF5_VFD_GDS_LIBRARIES} PARENT_SCOPE) 153 | 154 | # HDF5 GDS VFD external library dependencies 155 | # Need to generate -llib if not already passed 156 | set(HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES 157 | ${HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES} 158 | ${HDF5_VFD_GDS_EXT_PKG_LIB_DEPENDENCIES} 159 | PARENT_SCOPE 160 | ) 161 | foreach(lib_dep ${HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES}) 162 | # get library name 163 | get_filename_component(lib_name ${lib_dep} NAME_WE) 164 | if(lib_name MATCHES "^-l") 165 | # lib_name found is -lxxx 166 | set(HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES_LIST ${HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES_LIST} ${lib_name}) 167 | else() 168 | # lib_name is /path/to/lib so get library path and name 169 | get_filename_component(lib_path ${lib_dep} PATH) 170 | string(REGEX REPLACE "^lib" "" lib_name ${lib_name}) 171 | set(HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES_LIST ${HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES_LIST} -L${lib_path} -l${lib_name}) 172 | endif() 173 | endforeach() 174 | if(HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES_LIST) 175 | list(REMOVE_DUPLICATES HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES_LIST) 176 | endif() 177 | foreach(lib_dep ${HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES_LIST}) 178 | set(HDF5_VFD_GDS_LIB_DEPENDENCIES "${HDF5_VFD_GDS_LIB_DEPENDENCIES} ${lib_dep}") 179 | endforeach() 180 | set(HDF5_VFD_GDS_LIB_DEPENDENCIES ${HDF5_VFD_GDS_LIB_DEPENDENCIES} PARENT_SCOPE) 181 | 182 | # External include dependencies 183 | set(HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES 184 | ${HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES} 185 | ${HDF5_VFD_GDS_EXT_PKG_INCLUDE_DEPENDENCIES} 186 | PARENT_SCOPE 187 | ) 188 | if(HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES) 189 | list(REMOVE_DUPLICATES HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES) 190 | endif() 191 | foreach(inc_dep ${HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES}) 192 | set(HDF5_VFD_GDS_INCLUDE_DEPENDENCIES "${HDF5_VFD_GDS_INCLUDE_DEPENDENCIES} -I${inc_dep}") 193 | endforeach() 194 | set(HDF5_VFD_GDS_INCLUDE_DEPENDENCIES ${HDF5_VFD_GDS_INCLUDE_DEPENDENCIES} PARENT_SCOPE) 195 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Actions Status](https://github.com/hpc-io/vfd-gds/workflows/vfd-gds/badge.svg)](https://github.com/hpc-io/vfd-gds/actions) 2 | 3 | # HDF5 Nvidia GPUDirect Storage VFD 4 | 5 | ## Table of Contents 6 | 1. [Description](#1-Description) 7 | 2. [Installation](#2-Installation) 8 | * [Prerequisites](#Prerequisites) 9 | * [Build instructions](#Build-instructions) 10 | * [CMake](#CMake-build-instructions) 11 | * [CMake options](#CMake-build-options) 12 | * [Driver options](#CMake-driver-options) 13 | 3. [Usage and Testing](#3-Usage-And-Testing) 14 | * [Linked into application](#Linked-into-application) 15 | * [Dynamically loaded by FAPL](#Dynamic-load-fapl) 16 | * [Dynamically loaded by environment variable](#Dynamic-load-variable) 17 | * [Usage](#Driver-Usage) 18 | 4. [Known Issues and Limitations](#4-Known-Issues-And-Limitations) 19 | 5. [More Information](#5-More-Information) 20 | 21 | ## 1. Description 22 | 23 | The HDF5 GPUDirect Storage VFD is a Virtual File Driver (VFD) for HDF5 that can 24 | be used to interface with Nvidia's GPUDirect Storage (GDS) API. The driver is 25 | built as a plugin library that is external to HDF5. 26 | 27 | ## 2. Installation 28 | 29 | ### Prerequisites 30 | 31 | To build the GDS VFD, the following libraries are required: 32 | 33 | + `libhdf5` - The [HDF5](https://www.hdfgroup.org/downloads/hdf5/) library. 34 | Minimum version required is 1.14.0. HDF5 must be compiled with 35 | support for parallel I/O if building the GDS VFD's tests. Shared 36 | libraries must be enabled if loading the GDS VFD as an HDF5 plugin. 37 | 38 | + `libcuda` - Nvidia [CUDA](https://developer.nvidia.com/about-cuda) library. 39 | Cufile support must also be available. 40 | 41 | + `libmpi` - MPI library. The GDS VFD's tests use MPI to test functionality 42 | of the driver. 43 | 44 | Compiled libraries must either exist in the system's library paths or must be 45 | pointed to during the GDS VFD build process. 46 | 47 | ### Build instructions 48 | 49 | The HDF5 GDS VFD is built using CMake. 50 | 51 | #### CMake 52 | 53 | CMake version 2.8.12.2 or greater is required for building the driver. 54 | 55 | After obtaining the driver's source code, you can create a build directory 56 | within the source tree and run the `ccmake` or `cmake` command from it: 57 | 58 | cd vfd-gds-X 59 | mkdir build 60 | cd build 61 | ccmake .. 62 | 63 | If using `ccmake`, type `'c'` multiple times and choose suitable options or if 64 | using `cmake`, pass these options with `-D`. Some of these options may be needed 65 | if, for example, the required components mentioned previously are not located in 66 | default paths. 67 | 68 | Setting include directory and library paths may require you to toggle to 69 | the advanced mode by typing `'t'`. Once you are done and do not see any 70 | errors, type `'g'` to generate makefiles. Once you exit the CMake 71 | configuration screen and are ready to build the targets, do: 72 | 73 | make 74 | 75 | Verbose build output can be generated by appending `VERBOSE=1` to the 76 | `make` command. 77 | 78 | Assuming that the `CMAKE_INSTALL_PREFIX` has been set and that you have 79 | write permissions to the destination directory, you can install the driver 80 | by simply doing: 81 | 82 | make install 83 | 84 | ##### CMake options 85 | 86 | * `CMAKE_INSTALL_PREFIX` - This option controls the install directory that the 87 | resulting output files are written to. The default value is `/usr/local`. 88 | * `CMAKE_BUILD_TYPE` - This option controls the type of build used for the VFD. 89 | Valid values are `Release`, `Debug`, `RelWithDebInfo`, `MinSizeRel`; the default 90 | build type is `RelWithDebInfo`. 91 | 92 | ##### Driver options 93 | 94 | * `BUILD_EXAMPLES` - This option is used to enable/disable building of the 95 | GDS VFD's HDF5 example programs. The default value is `ON`. 96 | * `BUILD_TESTING` - This option is used to enable/disable building of the 97 | GDS VFD's tests. The default value is `ON`. 98 | * `HDF5_VFD_GDS_CUFILE_DIR` - This option controls the directory used when 99 | searching for the NVIDIA GPUDirect Storage cuFile library. This option may 100 | need to be set if CMake cannot automatically resolve the path. 101 | * `HDF5_VFD_GDS_CUFILE_LIB` - This option controls the library used for 102 | NVIDIA GPUDirect Storage cuFile support. This option may need to be set if 103 | CMake cannot automatically resolve it. 104 | * `HDF5_C_COMPILER_EXECUTABLE` - This option controls the HDF5 compiler 105 | wrapper script used by the VFD build process. It should be set to the full 106 | path to the HDF5 compiler wrapper (usually `bin/h5cc`), including the name 107 | of the wrapper script. The following two options may also need to be set. 108 | * `HDF5_C_LIBRARY_hdf5` - This option controls the HDF5 library used by the 109 | VFD build process. It should be set to the full path to the HDF5 library, 110 | including the library's name (e.g., `/path/libhdf5.so`). Used in conjunction 111 | with the `HDF5_C_INCLUDE_DIR` option. 112 | * `HDF5_C_INCLUDE_DIR` - This option controls the HDF5 include directory used 113 | by the VFD build process. Used in conjunction with the `HDF5_C_LIBRARY_hdf5` 114 | variable. 115 | * `MPI_C_COMPILER` - This option controls the MPI C compiler used by the VFD 116 | build process. It should be set to the full path to the MPI C compiler, 117 | including the name of the executable. 118 | 119 | ## 3. Usage and Testing 120 | 121 | To use the HDF5 GDS VFD in an HDF5 application, the driver can either be linked 122 | into the application, or it can be dynamically loaded as a plugin. If dynamically 123 | loading the GDS VFD, users should ensure that the `HDF5_PLUGIN_PATH` environment 124 | variable points to the directory containing the built VFD library if the VFD has 125 | been installed to a non-standard location. 126 | 127 | ### Linked into application 128 | 129 | To link the GDS VFD into an HDF5 application, the application should include the 130 | `H5FDgds.h` header that gets installed on the system and should link the installed 131 | VFD library (`libhdf5_vfd_gds.so`, or similar) into the application. Once this has 132 | been done, GDS VFD access can be setup by calling `H5Pset_fapl_gds(...)` on a FAPL 133 | within the HDF5 application. Refer to the HDF5 example programs under the `examples` 134 | folder to see how this is done. 135 | 136 | ### Dynamically loaded by FAPL 137 | 138 | To explicitly load the GDS VFD inside an HDF5 application, a call to the 139 | `H5Pset_driver_by_name(...)` routine should be made to setup GDS VFD access on a 140 | FAPL. This will cause HDF5 to load the VFD as a plugin and set the VFD on the 141 | given FAPL. The string "gds" should be given for the `driver_name` parameter. NULL 142 | should be given for the `driver_config` parameter, as the driver does not currently 143 | accept configuration strings. 144 | 145 | ### Dynamically loaded by environment variable 146 | 147 | To implicitly load the GDS VFD inside an HDF5 application, the `HDF5_DRIVER` 148 | environment variable may be set to the string "gds". During library initialization, 149 | HDF5 will check this environment variable, load the GDS VFD as a plugin and set the 150 | VFD as the default file driver on File Access Property Lists. Therefore, any file 151 | access that uses `H5P_DEFAULT` for its FAPL, or which uses a FAPL that hasn't had a 152 | specific VFD set on it, will automatically use the GDS VFD for file access. 153 | 154 | ### Usage 155 | 156 | Users of the GDS VFD should be aware that using the VFD in a normal, unmodified HDF5 157 | application will cause the VFD to behave similarly to the sec2 VFD; that is, the VFD 158 | will make use of POSIX I/O API routines to do file I/O. To make proper use of 159 | GPUDirect storage-capable devices with the VFD, the HDF5 application should make use 160 | of CUDA memory management API routines (e.g., cudaMalloc) to allocate memory on the 161 | device. Then, the device-allocated memory buffer should be passed to `H5Dread`/`H5Dwrite`. 162 | For examples of this, refer to the HDF5 programs under the `examples` folder. 163 | 164 | ## 4. Known Issues and Limitations 165 | 166 | * For chunked datasets, HDF5 may attempt to cache chunks in its chunk cache 167 | under certain circumstances, which can incur overhead when copying between 168 | CUDA device memory and host memory. This caching mechanism cannot currently be 169 | properly disabled; therefore, performance may suffer for chunked datasets if: 170 | - filters are applied to the dataset's chunks 171 | - fill values need to be written to chunks 172 | 173 | ## 5. More Information 174 | 175 | [GPUDirect Storage](https://developer.nvidia.com/blog/gpudirect-storage/) 176 | 177 | [HDF5 VFD Plugins RFC](https://github.com/HDFGroup/hdf5doc/blob/master/RFCs/HDF5_Library/VFL_DriverPlugins/RFC__A_Plugin_Interface_for_HDF5_Virtual_File_Drivers.pdf) 178 | -------------------------------------------------------------------------------- /CMake/Git/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /src/H5FDgds_err.h: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Copyright by The HDF Group. * 3 | * All rights reserved. * 4 | * * 5 | * This file is part of the HDF5 GDS Virtual File Driver. The full copyright * 6 | * notice, including terms governing use, modification, and redistribution, * 7 | * is contained in the COPYING file, which can be found at the root of the * 8 | * source code distribution tree. * 9 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 10 | 11 | /* 12 | * Error handling for the GDS VFD 13 | */ 14 | 15 | #ifndef H5FDgds_err_h 16 | #define H5FDgds_err_h 17 | 18 | #ifdef __cplusplus 19 | extern "C" { 20 | #endif 21 | 22 | #include 23 | 24 | #include "H5Epublic.h" 25 | 26 | extern hid_t H5FDgds_err_stack_g; 27 | extern hid_t H5FDgds_err_class_g; 28 | 29 | #define H5FD_GDS_ERR_CLS_NAME "HDF5 GDS VFD" 30 | #define H5FD_GDS_ERR_LIB_NAME "HDF5 GDS VFD" 31 | #define H5FD_GDS_ERR_VER "1.0.0" 32 | 33 | #define SUCCEED 0 34 | #define FAIL (-1) 35 | 36 | #ifndef FALSE 37 | #define FALSE false 38 | #endif 39 | #ifndef TRUE 40 | #define TRUE true 41 | #endif 42 | 43 | /* Error macros */ 44 | 45 | #ifdef H5_NO_DEPRECATED_SYMBOLS 46 | 47 | /* 48 | * Macro to push the current function to the current error stack 49 | * and then goto the "done" label, which should appear inside the 50 | * function. (v2 errors only) 51 | */ 52 | #define H5FD_GDS_GOTO_ERROR(err_major, err_minor, ret_val, ...) \ 53 | do { \ 54 | H5E_auto2_t err_func; \ 55 | \ 56 | /* Check whether automatic error reporting has been disabled */ \ 57 | (void)H5Eget_auto2(H5E_DEFAULT, &err_func, NULL); \ 58 | if (err_func) { \ 59 | if (H5FDgds_err_stack_g >= 0 && H5FDgds_err_class_g >= 0) { \ 60 | H5Epush2(H5FDgds_err_stack_g, __FILE__, __func__, __LINE__, \ 61 | H5FDgds_err_class_g, err_major, err_minor, __VA_ARGS__); \ 62 | } \ 63 | else { \ 64 | fprintf(stderr, __VA_ARGS__); \ 65 | fprintf(stderr, "\n"); \ 66 | } \ 67 | } \ 68 | \ 69 | ret_value = ret_val; \ 70 | goto done; \ 71 | } while(0) 72 | 73 | /* 74 | * Macro to push the current function to the current error stack 75 | * without calling goto. This is used for handling the case where 76 | * an error occurs during cleanup past the "done" label inside a 77 | * function so that an infinite loop does not occur where goto 78 | * continually branches back to the label. (v2 errors only) 79 | */ 80 | #define H5FD_GDS_DONE_ERROR(err_major, err_minor, ret_val, ...) \ 81 | do { \ 82 | H5E_auto2_t err_func; \ 83 | \ 84 | /* Check whether automatic error reporting has been disabled */ \ 85 | (void)H5Eget_auto2(H5E_DEFAULT, &err_func, NULL); \ 86 | if (err_func) { \ 87 | if (H5FDgds_err_stack_g >= 0 && H5FDgds_err_class_g >= 0) \ 88 | H5Epush2(H5FDgds_err_stack_g, __FILE__, __func__, __LINE__, \ 89 | H5FDgds_err_class_g, err_major, err_minor, __VA_ARGS__); \ 90 | else { \ 91 | fprintf(stderr, __VA_ARGS__); \ 92 | fprintf(stderr, "\n"); \ 93 | } \ 94 | } \ 95 | \ 96 | ret_value = ret_val; \ 97 | } while(0) 98 | 99 | /* 100 | * Macro to print out the current error stack and then clear it 101 | * for future use. (v2 errors only) 102 | */ 103 | #define PRINT_ERROR_STACK \ 104 | do { \ 105 | H5E_auto2_t err_func; \ 106 | \ 107 | /* Check whether automatic error reporting has been disabled */ \ 108 | (void)H5Eget_auto2(H5E_DEFAULT, &err_func, NULL); \ 109 | if (err_func) { \ 110 | if ((H5FDgds_err_stack_g >= 0) && (H5Eget_num(H5FDgds_err_stack_g) > 0)) { \ 111 | H5Eprint2(H5FDgds_err_stack_g, NULL); \ 112 | H5Eclear2(H5FDgds_err_stack_g); \ 113 | } \ 114 | } \ 115 | } while(0) 116 | 117 | #else 118 | 119 | /* 120 | * Macro to push the current function to the current error stack 121 | * and then goto the "done" label, which should appear inside the 122 | * function. (compatible with v1 and v2 errors) 123 | */ 124 | #define H5FD_GDS_GOTO_ERROR(err_major, err_minor, ret_val, ...) \ 125 | do { \ 126 | unsigned is_v2_err; \ 127 | union { \ 128 | H5E_auto1_t err_func_v1; \ 129 | H5E_auto2_t err_func_v2; \ 130 | } err_func; \ 131 | \ 132 | /* Determine version of error */ \ 133 | (void)H5Eauto_is_v2(H5E_DEFAULT, &is_v2_err); \ 134 | \ 135 | if (is_v2_err) \ 136 | (void)H5Eget_auto2(H5E_DEFAULT, &err_func.err_func_v2, NULL); \ 137 | else \ 138 | (void)H5Eget_auto1(&err_func.err_func_v1, NULL); \ 139 | \ 140 | /* Check whether automatic error reporting has been disabled */ \ 141 | if ( (is_v2_err && err_func.err_func_v2) || \ 142 | (!is_v2_err && err_func.err_func_v1) ) { \ 143 | if (H5FDgds_err_stack_g >= 0 && H5FDgds_err_class_g >= 0) { \ 144 | H5Epush2(H5FDgds_err_stack_g, __FILE__, __func__, __LINE__, \ 145 | H5FDgds_err_class_g, err_major, err_minor, __VA_ARGS__); \ 146 | } \ 147 | else { \ 148 | fprintf(stderr, __VA_ARGS__); \ 149 | fprintf(stderr, "\n"); \ 150 | } \ 151 | } \ 152 | \ 153 | ret_value = ret_val; \ 154 | goto done; \ 155 | } while(0) 156 | 157 | /* 158 | * Macro to push the current function to the current error stack 159 | * without calling goto. This is used for handling the case where 160 | * an error occurs during cleanup past the "done" label inside a 161 | * function so that an infinite loop does not occur where goto 162 | * continually branches back to the label. (compatible with v1 163 | * and v2 errors) 164 | */ 165 | #define H5FD_GDS_DONE_ERROR(err_major, err_minor, ret_val, ...) \ 166 | do { \ 167 | unsigned is_v2_err; \ 168 | union { \ 169 | H5E_auto1_t err_func_v1; \ 170 | H5E_auto2_t err_func_v2; \ 171 | } err_func; \ 172 | \ 173 | /* Determine version of error */ \ 174 | (void)H5Eauto_is_v2(H5E_DEFAULT, &is_v2_err); \ 175 | \ 176 | if (is_v2_err) \ 177 | (void)H5Eget_auto2(H5E_DEFAULT, &err_func.err_func_v2, NULL); \ 178 | else \ 179 | (void)H5Eget_auto1(&err_func.err_func_v1, NULL); \ 180 | \ 181 | /* Check whether automatic error reporting has been disabled */ \ 182 | if ( (is_v2_err && err_func.err_func_v2) || \ 183 | (!is_v2_err && err_func.err_func_v1) ) { \ 184 | if (H5FDgds_err_stack_g >= 0 && H5FDgds_err_class_g >= 0) { \ 185 | H5Epush2(H5FDgds_err_stack_g, __FILE__, __func__, __LINE__, \ 186 | H5FDgds_err_class_g, err_major, err_minor, __VA_ARGS__); \ 187 | } \ 188 | else { \ 189 | fprintf(stderr, __VA_ARGS__); \ 190 | fprintf(stderr, "\n"); \ 191 | } \ 192 | } \ 193 | \ 194 | ret_value = ret_val; \ 195 | } while(0) 196 | 197 | /* 198 | * Macro to print out the current error stack and then clear it 199 | * for future use. (compatible with v1 and v2 errors) 200 | */ 201 | #define PRINT_ERROR_STACK \ 202 | do { \ 203 | unsigned is_v2_err; \ 204 | union { \ 205 | H5E_auto1_t err_func_v1; \ 206 | H5E_auto2_t err_func_v2; \ 207 | } err_func; \ 208 | \ 209 | /* Determine version of error */ \ 210 | (void)H5Eauto_is_v2(H5E_DEFAULT, &is_v2_err); \ 211 | \ 212 | if (is_v2_err) \ 213 | (void)H5Eget_auto2(H5E_DEFAULT, &err_func.err_func_v2, NULL); \ 214 | else \ 215 | (void)H5Eget_auto1(&err_func.err_func_v1, NULL); \ 216 | \ 217 | /* Check whether automatic error reporting has been disabled */ \ 218 | if ( (is_v2_err && err_func.err_func_v2) || \ 219 | (!is_v2_err && err_func.err_func_v1) ) { \ 220 | if ((H5FDgds_err_stack_g >= 0) && (H5Eget_num(H5FDgds_err_stack_g) > 0)) { \ 221 | H5Eprint2(H5FDgds_err_stack_g, NULL); \ 222 | H5Eclear2(H5FDgds_err_stack_g); \ 223 | } \ 224 | } \ 225 | } while(0) 226 | 227 | #endif 228 | 229 | #define H5FD_GDS_SYS_GOTO_ERROR(err_major, err_minor, ret_val, str) \ 230 | do { \ 231 | int myerrno = errno; \ 232 | H5FD_GDS_GOTO_ERROR(err_major, err_minor, ret_val, \ 233 | "%s, errno = %d, error message = '%s'", str, \ 234 | myerrno, strerror(myerrno)); \ 235 | } while(0) 236 | 237 | /* 238 | * Macro to simply jump to the "done" label inside the function, 239 | * setting ret_value to the given value. This is often used for 240 | * short circuiting in functions when certain conditions arise. 241 | */ 242 | #define H5FD_GDS_GOTO_DONE(ret_val) \ 243 | do { \ 244 | ret_value = ret_val; \ 245 | goto done; \ 246 | } while(0) 247 | 248 | /* 249 | * Macro to return from a top-level API function, printing 250 | * out the error stack on the way out. 251 | * It should be ensured that this macro is only called once 252 | * per HDF5 operation. If it is called multiple times per 253 | * operation (e.g. due to calling top-level API functions 254 | * internally), the error stack will be inconsistent/incoherent. 255 | */ 256 | #define H5FD_GDS_FUNC_LEAVE_API \ 257 | do { \ 258 | PRINT_ERROR_STACK; \ 259 | return ret_value; \ 260 | } while(0) 261 | 262 | /* 263 | * Macro to return from internal functions. 264 | */ 265 | #define H5FD_GDS_FUNC_LEAVE \ 266 | do { \ 267 | return ret_value; \ 268 | } while(0) 269 | 270 | #ifdef __cplusplus 271 | } 272 | #endif 273 | 274 | #endif /* H5FDgds_err_h */ 275 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.25 FATAL_ERROR) 2 | 3 | # Setup cmake policies. 4 | foreach(policy 5 | CMP0012 6 | CMP0013 7 | CMP0014 8 | CMP0022 # CMake 2.8.12 9 | CMP0025 # CMake 3.0 10 | CMP0053 # CMake 3.1 11 | CMP0054 # CMake 3.1 12 | CMP0074 # CMake 3.12 13 | CMP0075 # CMake 3.12 14 | CMP0083 # CMake 3.14 15 | CMP0093 # CMake 3.15 16 | ) 17 | if(POLICY ${policy}) 18 | cmake_policy(SET ${policy} NEW) 19 | endif() 20 | endforeach() 21 | 22 | # Set a consistent MACOSX_RPATH default across all CMake versions. 23 | # When CMake 2.8.12 is required, change this default to 1. 24 | # When CMake 3.0.0 is required, remove this block (see CMP0042). 25 | if(NOT DEFINED CMAKE_MACOSX_RPATH) 26 | set(CMAKE_MACOSX_RPATH 0) 27 | endif() 28 | 29 | project(HDF5_VFD_GDS C) 30 | 31 | #------------------------------------------------------------------------------ 32 | # Setup install and output Directories 33 | #------------------------------------------------------------------------------ 34 | include(GNUInstallDirs) 35 | 36 | if(NOT HDF5_VFD_GDS_INSTALL_BIN_DIR) 37 | set(HDF5_VFD_GDS_INSTALL_BIN_DIR ${CMAKE_INSTALL_BINDIR}) 38 | endif() 39 | if(NOT HDF5_VFD_GDS_INSTALL_LIB_DIR) 40 | set(HDF5_VFD_GDS_INSTALL_LIB_DIR ${CMAKE_INSTALL_LIBDIR}) 41 | endif() 42 | if(NOT HDF5_VFD_GDS_INSTALL_INCLUDE_DIR) 43 | # Interface include will default to prefix/include 44 | set(HDF5_VFD_GDS_INSTALL_INTERFACE include) 45 | set(HDF5_VFD_GDS_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_INCLUDEDIR}) 46 | else() 47 | set(HDF5_VFD_GDS_INSTALL_INTERFACE ${HDF5_VFD_GDS_INSTALL_INCLUDE_DIR}) 48 | endif() 49 | if(NOT HDF5_VFD_GDS_INSTALL_DATA_DIR) 50 | set(HDF5_VFD_GDS_INSTALL_DATA_DIR ${CMAKE_INSTALL_DATAROOTDIR}) 51 | endif() 52 | 53 | # Setting this ensures that "make install" will leave rpaths to external 54 | # libraries intact on "make install". This ensures that one can install a 55 | # version of the HDF5 GDS VFD on the build machine without any issues. If 56 | # thisis not desired, simply specify CMAKE_INSTALL_RPATH_USE_LINK_PATH when 57 | # configuring the HDF5 GDS VFD and "make install" will strip all rpaths, which 58 | # is default behavior. 59 | if(NOT CMAKE_INSTALL_RPATH_USE_LINK_PATH) 60 | set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) 61 | endif() 62 | 63 | #------------------------------------------------------------------------------ 64 | # Set module path 65 | #------------------------------------------------------------------------------ 66 | set(HDF5_VFD_GDS_CMAKE_MODULE_PATH "${HDF5_VFD_GDS_SOURCE_DIR}/CMake") 67 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${HDF5_VFD_GDS_CMAKE_MODULE_PATH}) 68 | 69 | #------------------------------------------------------------------------------ 70 | # Locate CUDA 71 | #------------------------------------------------------------------------------ 72 | 73 | # Try to locate cufile library 74 | find_package(CUDAToolkit REQUIRED) 75 | 76 | set(HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES 77 | ${HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES} 78 | ${CUDAToolkit_INCLUDE_DIRS} 79 | ) 80 | set(HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES 81 | ${HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES} 82 | ) 83 | 84 | #------------------------------------------------------------------------------ 85 | # Locate HDF5 86 | #------------------------------------------------------------------------------ 87 | find_package(HDF5 NO_MODULE NAMES hdf5 COMPONENTS C shared) 88 | if(HDF5_FOUND) 89 | set(HDF5_C_SHARED_LIBRARY hdf5-shared) 90 | if(NOT TARGET ${HDF5_C_SHARED_LIBRARY}) 91 | message(FATAL_ERROR "Could not find hdf5 shared target, please make " 92 | "sure that HDF5 has been compiled with shared libraries enabled.") 93 | endif() 94 | set(HDF5_VFD_GDS_EXT_PKG_DEPENDENCIES 95 | ${HDF5_VFD_GDS_EXT_PKG_DEPENDENCIES} 96 | ${HDF5_C_SHARED_LIBRARY} 97 | ) 98 | set(HDF5_VFD_GDS_USE_SYSTEM_CMAKE_HDF5 1) 99 | else() 100 | # Allow for HDF5 autotools builds 101 | find_package(HDF5 MODULE REQUIRED) 102 | set(HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES 103 | ${HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES} 104 | ${HDF5_INCLUDE_DIRS} 105 | ) 106 | set(HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES 107 | ${HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES} 108 | ${HDF5_LIBRARIES} 109 | ) 110 | set(HDF5_VFD_GDS_USE_SYSTEM_CMAKE_HDF5 0) 111 | endif() 112 | 113 | #------------------------------------------------------------------------------ 114 | # Locate MPI - currently just used for building VFD's tests 115 | #------------------------------------------------------------------------------ 116 | if(HDF5_FOUND AND HDF5_IS_PARALLEL) 117 | find_package(MPI REQUIRED COMPONENTS C) 118 | set(HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES 119 | ${HDF5_VFD_GDS_EXT_INCLUDE_DEPENDENCIES} 120 | ${MPI_C_INCLUDE_DIRS} 121 | ) 122 | set(HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES 123 | ${HDF5_VFD_GDS_EXT_LIB_DEPENDENCIES} 124 | ${MPI_C_LIBRARIES} 125 | ) 126 | endif() 127 | 128 | #------------------------------------------------------------------------------ 129 | # Version information 130 | #------------------------------------------------------------------------------ 131 | include(${HDF5_VFD_GDS_CMAKE_MODULE_PATH}/Git/Git.cmake) 132 | include(Hdf5GdsVfdDetermineVersion) 133 | # Hardcoded version variables are read-in from a separate file. This makes it 134 | # easier to have a script to update version numbers automatically. 135 | file(STRINGS version.txt version_txt) 136 | extract_version_components("${version_txt}" "${PROJECT_NAME}") 137 | determine_version(${HDF5_VFD_GDS_SOURCE_DIR} ${GIT_EXECUTABLE} "${PROJECT_NAME}") 138 | set(HDF5_VFD_GDS_PACKAGE "hdf5_vfd_gds") 139 | set(HDF5_VFD_GDS_PACKAGE_NAME "HDF5_VFD_GDS") 140 | set(HDF5_VFD_GDS_PACKAGE_DESCRIPTION "HDF5 Nvidia GPUDirect Storage VFD") 141 | set(HDF5_VFD_GDS_PACKAGE_URL "https://github.com/hpc-io/vfd-gds") 142 | set(HDF5_VFD_GDS_PACKAGE_VENDOR "The HDF Group") 143 | message(STATUS "Configuring ${HDF5_VFD_GDS_PACKAGE} v${HDF5_VFD_GDS_VERSION_FULL}") 144 | 145 | #------------------------------------------------------------------------------ 146 | # Setup CMake Environment 147 | #------------------------------------------------------------------------------ 148 | if(APPLE AND NOT HDF5_VFD_GDS_EXTERNALLY_CONFIGURED) 149 | # We are doing a unix-style install i.e. everything will be installed in 150 | # ${CMAKE_INSTALL_BINDIR} and ${CMAKE_INSTALL_LIBDIR} etc. as on other unix 151 | # platforms. We still need to setup CMAKE_INSTALL_NAME_DIR correctly so that 152 | # the binaries point to appropriate location for the libraries. 153 | 154 | # 1. Make CMAKE_INSTALL_PREFIX publicly accessible, if it was hidden in 155 | # previous pass 156 | get_property(is_internal CACHE CMAKE_INSTALL_PREFIX PROPERTY TYPE) 157 | if(is_internal STREQUAL "INTERNAL") 158 | set(CMAKE_INSTALL_PREFIX ${CACHED_CMAKE_INSTALL_PREFIX} CACHE PATH "Install prefix" FORCE) 159 | else() 160 | set(CMAKE_INSTALL_PREFIX ${CACHED_CMAKE_INSTALL_PREFIX} CACHE PATH "Install prefix") 161 | endif() 162 | unset(MACOSX_APP_INSTALL_PREFIX CACHE) 163 | 164 | set(CMAKE_INSTALL_NAME_DIR "@rpath") 165 | mark_as_advanced( 166 | CMAKE_OSX_ARCHITECTURES 167 | CMAKE_OSX_DEPLOYMENT_TARGET 168 | CMAKE_OSX_SYSROOT 169 | ) 170 | else() 171 | if(WIN32) 172 | message("The HDF5 GDS VFD is not supported on this platform." FATAL_ERROR) 173 | endif() 174 | endif() 175 | 176 | #------------------------------------------------------------------------------ 177 | if(NOT HDF5_VFD_GDS_EXTERNALLY_CONFIGURED) 178 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY 179 | ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Executables." 180 | ) 181 | set(EXECUTABLE_OUTPUT_PATH ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) 182 | set(CMAKE_LIBRARY_OUTPUT_DIRECTORY 183 | ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Libraries" 184 | ) 185 | set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY 186 | ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all static libraries." 187 | ) 188 | endif() 189 | 190 | #------------------------------------------------------------------------------ 191 | # Disallow in-source build 192 | #------------------------------------------------------------------------------ 193 | if("${HDF5_VFD_GDS_SOURCE_DIR}" STREQUAL "${HDF5_VFD_GDS_BINARY_DIR}") 194 | message(FATAL_ERROR 195 | "HDF5 GDS VFD requires an out of source Build. " 196 | "Please create a separate binary directory and run CMake there.") 197 | endif() 198 | 199 | #------------------------------------------------------------------------------ 200 | # Set a default build type if none was specified 201 | #------------------------------------------------------------------------------ 202 | if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) 203 | message(STATUS "Setting build type to 'RelWithDebInfo' as none was specified.") 204 | set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Choose the type of build." FORCE) 205 | # Set the possible values of build type for cmake-gui 206 | set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" 207 | "MinSizeRel" "RelWithDebInfo") 208 | endif() 209 | 210 | if(NOT CMAKE_C_FLAGS AND CMAKE_COMPILER_IS_GNUCC) 211 | message(STATUS "GCC detected, setting additional flags") 212 | set(CMAKE_C_FLAGS "-Wall -Wextra -Winline -Wcast-qual -std=gnu99 -Wshadow" CACHE STRING "Flags used by the compiler during all build types." FORCE) 213 | endif() 214 | 215 | #----------------------------------------------------------------------------- 216 | # Targets built within this project are exported at Install time for use 217 | # by other projects. 218 | #----------------------------------------------------------------------------- 219 | if(NOT HDF5_VFD_GDS_EXPORTED_TARGETS) 220 | set(HDF5_VFD_GDS_EXPORTED_TARGETS "${HDF5_VFD_GDS_PACKAGE}-targets") 221 | endif() 222 | 223 | #------------------------------------------------------------------------------ 224 | # Choose static or shared libraries. 225 | #------------------------------------------------------------------------------ 226 | option(BUILD_SHARED_LIBS "Build with shared libraries." ON) 227 | if(BUILD_SHARED_LIBS) 228 | set(H5FD_GDS_VFD_BUILD_SHARED_LIBS 1) 229 | set(HDF5_VFD_GDS_LIBTYPE SHARED) 230 | else() 231 | message(FATAL_ERROR 232 | "HDF5 GDS VFD requires shared libraries.") 233 | set(H5FD_GDS_VFD_BUILD_SHARED_LIBS 0) 234 | set(HDF5_VFD_GDS_LIBTYPE STATIC) 235 | endif() 236 | set(CMAKE_POSITION_INDEPENDENT_CODE ON) 237 | 238 | #------------------------------------------------------------------------------- 239 | function(hdf5_vfd_gds_set_lib_options libtarget libname libtype) 240 | if(${libtype} MATCHES "SHARED") 241 | set(LIB_RELEASE_NAME "${libname}") 242 | set(LIB_DEBUG_NAME "${libname}_debug") 243 | set(LIB_VERSION ${HDF5_VFD_GDS_VERSION}.${HDF5_VFD_GDS_VERSION_PATCH}) 244 | set(API_VERSION ${HDF5_VFD_GDS_VERSION_MAJOR}) 245 | else() 246 | # if the generator supports configuration types or if the CMAKE_BUILD_TYPE has a value 247 | if(CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE) 248 | set(LIB_RELEASE_NAME "${libname}") 249 | set(LIB_DEBUG_NAME "${libname}_debug") 250 | else() 251 | set(LIB_RELEASE_NAME "lib${libname}") 252 | set(LIB_DEBUG_NAME "lib${libname}_debug") 253 | endif() 254 | endif() 255 | 256 | set_target_properties(${libtarget} 257 | PROPERTIES 258 | OUTPUT_NAME_DEBUG ${LIB_DEBUG_NAME} 259 | OUTPUT_NAME_RELEASE ${LIB_RELEASE_NAME} 260 | OUTPUT_NAME_MINSIZEREL ${LIB_RELEASE_NAME} 261 | OUTPUT_NAME_RELWITHDEBINFO ${LIB_RELEASE_NAME} 262 | OUTPUT_NAME_ASAN ${LIB_DEBUG_NAME} 263 | OUTPUT_NAME_UBSAN ${LIB_DEBUG_NAME} 264 | VERSION ${LIB_VERSION} 265 | SOVERSION ${API_VERSION} 266 | ) 267 | endfunction() 268 | 269 | # Avoid explicitly including system include paths 270 | set(HDF5_VFD_GDS_SYSTEM_INCLUDE_PATH ${CMAKE_SYSTEM_INCLUDE_PATH} 271 | ${CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES}) 272 | set(HDF5_VFD_GDS_SYSTEM_INCLUDE_PATH /usr/include ${HDF5_VFD_GDS_SYSTEM_INCLUDE_PATH}) 273 | 274 | #----------------------------------------------------------------------------- 275 | # Coverage 276 | #----------------------------------------------------------------------------- 277 | if(NOT HDF5_VFD_GDS_EXTERNALLY_CONFIGURED) 278 | option(HDF5_VFD_GDS_ENABLE_COVERAGE "Enable coverage." OFF) 279 | if(HDF5_VFD_GDS_ENABLE_COVERAGE) 280 | set(COVERAGE_FLAGS "-fprofile-arcs -ftest-coverage" CACHE STRING 281 | "Flags to the coverage program that CTest uses to perform coverage inspection" 282 | ) 283 | mark_as_advanced(COVERAGE_FLAGS) 284 | endif() 285 | 286 | macro(set_coverage_flags target) 287 | set_target_properties(${target} 288 | PROPERTIES 289 | COMPILE_FLAGS ${COVERAGE_FLAGS} 290 | LINK_FLAGS ${COVERAGE_FLAGS} 291 | ) 292 | endmacro() 293 | endif() 294 | 295 | #----------------------------------------------------------------------------- 296 | # Source 297 | #----------------------------------------------------------------------------- 298 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src) 299 | 300 | #----------------------------------------------------------------------------- 301 | # Build doxygen documentation. 302 | #----------------------------------------------------------------------------- 303 | option(BUILD_DOCUMENTATION "Build documentation." ON) 304 | #if(BUILD_DOCUMENTATION) 305 | # add_subdirectory(Documentation/Doxygen) 306 | #endif() 307 | 308 | #----------------------------------------------------------------------------- 309 | # Examples 310 | #----------------------------------------------------------------------------- 311 | option(BUILD_EXAMPLES "Build examples." ON) 312 | if(BUILD_EXAMPLES) 313 | add_subdirectory(examples) 314 | endif() 315 | 316 | #----------------------------------------------------------------------------- 317 | # Testing 318 | #----------------------------------------------------------------------------- 319 | option(BUILD_TESTING "Build testing." ON) 320 | if(NOT HDF5_VFD_GDS_EXTERNALLY_CONFIGURED AND BUILD_TESTING) 321 | enable_testing() 322 | include(CTest) 323 | add_subdirectory(test) 324 | endif() 325 | 326 | #----------------------------------------------------------------------------- 327 | # Configure the config.cmake file for the build directory 328 | #----------------------------------------------------------------------------- 329 | set(HDF5_VFD_GDS_CONFIG_INSTALLED FALSE) 330 | configure_file( 331 | ${HDF5_VFD_GDS_SOURCE_DIR}/CMake/${HDF5_VFD_GDS_PACKAGE}-config.cmake.in 332 | ${HDF5_VFD_GDS_BINARY_DIR}/${HDF5_VFD_GDS_PACKAGE}-config.cmake @ONLY 333 | ) 334 | 335 | #----------------------------------------------------------------------------- 336 | # Configure the config.cmake file for the install directory 337 | #----------------------------------------------------------------------------- 338 | set(HDF5_VFD_GDS_CONFIG_INSTALLED TRUE) 339 | configure_file( 340 | ${HDF5_VFD_GDS_SOURCE_DIR}/CMake/${HDF5_VFD_GDS_PACKAGE}-config.cmake.in 341 | ${HDF5_VFD_GDS_BINARY_DIR}/CMakeFiles/${HDF5_VFD_GDS_PACKAGE}-config.cmake @ONLY 342 | ) 343 | 344 | install( 345 | FILES 346 | ${HDF5_VFD_GDS_BINARY_DIR}/CMakeFiles/${HDF5_VFD_GDS_PACKAGE}-config.cmake 347 | DESTINATION 348 | ${HDF5_VFD_GDS_INSTALL_DATA_DIR}/cmake/${HDF5_VFD_GDS_PACKAGE} 349 | ) 350 | 351 | #----------------------------------------------------------------------------- 352 | # Configure the config-version.cmake file for the install directory 353 | #----------------------------------------------------------------------------- 354 | configure_file( 355 | ${HDF5_VFD_GDS_SOURCE_DIR}/CMake/${HDF5_VFD_GDS_PACKAGE}-config-version.cmake.in 356 | ${HDF5_VFD_GDS_BINARY_DIR}/CMakeFiles/${HDF5_VFD_GDS_PACKAGE}-config-version.cmake @ONLY 357 | ) 358 | 359 | install( 360 | FILES 361 | ${HDF5_VFD_GDS_BINARY_DIR}/CMakeFiles/${HDF5_VFD_GDS_PACKAGE}-config-version.cmake 362 | DESTINATION 363 | ${HDF5_VFD_GDS_INSTALL_DATA_DIR}/cmake/${HDF5_VFD_GDS_PACKAGE} 364 | ) 365 | 366 | #----------------------------------------------------------------------------- 367 | # For automake compatibility, also provide a pkgconfig file 368 | #----------------------------------------------------------------------------- 369 | configure_file( 370 | ${HDF5_VFD_GDS_SOURCE_DIR}/CMake/${HDF5_VFD_GDS_PACKAGE}.pc.in 371 | ${HDF5_VFD_GDS_BINARY_DIR}/CMakeFiles/${HDF5_VFD_GDS_PACKAGE}.pc @ONLY 372 | ) 373 | 374 | install( 375 | FILES 376 | ${HDF5_VFD_GDS_BINARY_DIR}/CMakeFiles/${HDF5_VFD_GDS_PACKAGE}.pc 377 | DESTINATION 378 | ${HDF5_VFD_GDS_INSTALL_LIB_DIR}/pkgconfig 379 | ) 380 | 381 | #----------------------------------------------------------------------------- 382 | # CPack 383 | #----------------------------------------------------------------------------- 384 | if(NOT HDF5_VFD_GDS_EXTERNALLY_CONFIGURED) 385 | set(CPACK_PACKAGE_DESCRIPTION_FILE ${HDF5_VFD_GDS_SOURCE_DIR}/README.md) 386 | set(CPACK_RESOURCE_FILE_LICENSE ${HDF5_VFD_GDS_SOURCE_DIR}/COPYING) 387 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY ${HDF5_VFD_GDS_PACKAGE_DESCRIPTION}) 388 | set(CPACK_PACKAGE_NAME ${HDF5_VFD_GDS_PACKAGE_NAME}) 389 | set(CPACK_PACKAGE_VENDOR ${HDF5_VFD_GDS_PACKAGE_VENDOR}) 390 | set(CPACK_PACKAGE_VERSION_MAJOR ${HDF5_VFD_GDS_VERSION_MAJOR}) 391 | set(CPACK_PACKAGE_VERSION_MINOR ${HDF5_VFD_GDS_VERSION_MINOR}) 392 | set(CPACK_PACKAGE_VERSION_PATCH ${HDF5_VFD_GDS_VERSION_PATCH}) 393 | set(CPACK_GENERATOR "TBZ2") 394 | set(CPACK_SOURCE_PACKAGE_FILE_NAME ${HDF5_VFD_GDS_PACKAGE}-${HDF5_VFD_GDS_VERSION_FULL}) 395 | set(CPACK_SOURCE_IGNORE_FILES ".git*;/GitSetup/;/.git/;.swp$;.#;/#;.*~") 396 | set(CPACK_SOURCE_STRIP_FILES "") 397 | include(CPack) 398 | endif() 399 | -------------------------------------------------------------------------------- /src/H5FDgds.c: -------------------------------------------------------------------------------- 1 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 | * Copyright by The HDF Group. * 3 | * All rights reserved. * 4 | * * 5 | * This file is part of the HDF5 GDS Virtual File Driver. The full copyright * 6 | * notice, including terms governing use, modification, and redistribution, * 7 | * is contained in the COPYING file, which can be found at the root of the * 8 | * source code distribution tree. * 9 | * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 10 | 11 | /* 12 | * Programmer: John Ravi 13 | * Wednesday, July 1, 2020 14 | * 15 | * Purpose: Interfaces with the CUDA GPUDirect Storage API 16 | * Based on the Direct I/O file driver which forces the data to be written to 17 | * the file directly without being copied into system kernel 18 | * buffer. The main system support this feature is Linux. 19 | */ 20 | 21 | #ifndef _GNU_SOURCE 22 | #define _GNU_SOURCE /* For O_DIRECT flag */ 23 | #endif 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | #include 36 | #include 37 | #include 38 | 39 | #include 40 | 41 | #include "hdf5.h" 42 | 43 | /* HDF5 header for dynamic plugin loading */ 44 | #include "H5PLextern.h" 45 | 46 | #include "H5FDgds.h" /* cuda gds file driver */ 47 | #include "H5FDgds_err.h" /* error handling */ 48 | 49 | #define H5FD_GDS (H5FD_gds_init()) 50 | 51 | /* HDF5 doesn't currently have a driver init callback. Use 52 | * macro to initialize driver if loaded as a plugin. 53 | */ 54 | #define H5FD_GDS_INIT \ 55 | do { \ 56 | if (H5FD_GDS_g < 0) \ 57 | H5FD_GDS_g = H5FD_GDS; \ 58 | } while(0) 59 | 60 | /* #define ADVISE_OS_DISABLE_READ_CACHE */ 61 | 62 | #ifdef ADVISE_OS_DISABLE_READ_CACHE 63 | #include 64 | #endif /* ADVISE_OS_DISABLE_READ_CACHE */ 65 | 66 | static bool cu_file_driver_opened = false; 67 | 68 | /* static bool reg_once = false; */ 69 | 70 | /* The driver identification number, initialized at runtime */ 71 | static hid_t H5FD_GDS_g = H5I_INVALID_HID; 72 | 73 | /* Identifiers for HDF5's error API */ 74 | hid_t H5FDgds_err_stack_g = H5I_INVALID_HID; 75 | hid_t H5FDgds_err_class_g = H5I_INVALID_HID; 76 | 77 | /* Whether to ignore file locks when disabled (env var value) */ 78 | static htri_t ignore_disabled_file_locks_s = FAIL; 79 | 80 | /* File operations */ 81 | #define OP_UNKNOWN 0 82 | #define OP_READ 1 83 | #define OP_WRITE 2 84 | 85 | /* POSIX I/O mode used as the third parameter to open/_open 86 | * when creating a new file (O_CREAT is set). 87 | */ 88 | #if defined(H5_HAVE_WIN32_API) 89 | #define H5FD_GDS_POSIX_CREATE_MODE_RW (_S_IREAD | _S_IWRITE) 90 | #else 91 | #define H5FD_GDS_POSIX_CREATE_MODE_RW 0666 92 | #endif 93 | 94 | /* 95 | * The description of a file belonging to this driver. The `eoa' and `eof' 96 | * determine the amount of hdf5 address space in use and the high-water mark 97 | * of the file (the current size of the underlying Unix file). The `pos' 98 | * value is used to eliminate file position updates when they would be a 99 | * no-op. Unfortunately we've found systems that use separate file position 100 | * indicators for reading and writing so the lseek can only be eliminated if 101 | * the current operation is the same as the previous operation. When opening 102 | * a file the `eof' will be set to the current file size, `eoa' will be set 103 | * to zero, `pos' will be set to H5F_ADDR_UNDEF (as it is when an error 104 | * occurs), and `op' will be set to H5F_OP_UNKNOWN. 105 | */ 106 | typedef struct H5FD_gds_t { 107 | H5FD_t pub; /*public stuff, must be first */ 108 | int fd; /*the unix file */ 109 | haddr_t eoa; /*end of allocated region */ 110 | haddr_t eof; /*end of file; current file size*/ 111 | hbool_t ignore_disabled_file_locks; 112 | 113 | CUfileHandle_t cf_handle; /* cufile handle */ 114 | 115 | #ifndef H5_HAVE_WIN32_API 116 | /* 117 | * On most systems the combination of device and i-node number uniquely 118 | * identify a file. 119 | */ 120 | dev_t device; /*file device number */ 121 | ino_t inode; /*file i-node number */ 122 | #else 123 | /* 124 | * On H5_HAVE_WIN32_API the low-order word of a unique identifier associated with the 125 | * file and the volume serial number uniquely identify a file. This number 126 | * (which, both? -rpm) may change when the system is restarted or when the 127 | * file is opened. After a process opens a file, the identifier is 128 | * constant until the file is closed. An application can use this 129 | * identifier and the volume serial number to determine whether two 130 | * handles refer to the same file. 131 | */ 132 | DWORD fileindexlo; 133 | DWORD fileindexhi; 134 | #endif 135 | 136 | } H5FD_gds_t; 137 | 138 | /* 139 | * These macros check for overflow of various quantities. These macros 140 | * assume that off_t is signed and haddr_t and size_t are unsigned. 141 | * 142 | * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' 143 | * is too large to be represented by the second argument 144 | * of the file seek function. 145 | * 146 | * SIZE_OVERFLOW: Checks whether a buffer size of type `hsize_t' is too 147 | * large to be represented by the `size_t' type. 148 | * 149 | * REGION_OVERFLOW: Checks whether an address and size pair describe data 150 | * which can be addressed entirely by the second 151 | * argument of the file seek function. 152 | */ 153 | #define MAXADDR (((haddr_t)1 << (8 * sizeof(off_t) - 1)) - 1) 154 | #define ADDR_OVERFLOW(A) (HADDR_UNDEF == (A) || ((A) & ~(haddr_t)MAXADDR)) 155 | #define SIZE_OVERFLOW(Z) ((Z) & ~(hsize_t)MAXADDR) 156 | #define REGION_OVERFLOW(A, Z) \ 157 | (ADDR_OVERFLOW(A) || SIZE_OVERFLOW(Z) || HADDR_UNDEF == (A) + (Z) || (off_t)((A) + (Z)) < (off_t)(A)) 158 | 159 | 160 | #define check_cudadrivercall(fn) \ 161 | { \ 162 | CUresult res = fn; \ 163 | if (res != CUDA_SUCCESS) { \ 164 | const char *str = nullptr; \ 165 | cuGetErrorName(res, &str); \ 166 | fprintf(stderr, "cuda driver api call failed %d, %d : %s\n", fn, __LINE__, str); \ 167 | fprintf(stderr, "EXITING program!!!\n"); \ 168 | exit(1); \ 169 | } \ 170 | } 171 | 172 | #define check_cudaruntimecall(fn) \ 173 | { \ 174 | cudaError_t res = fn; \ 175 | if (res != cudaSuccess) { \ 176 | const char *str = cudaGetErrorName(res); \ 177 | fprintf(stderr, "cuda runtime api call failed %d, %d : %s\n", fn, __LINE__, str); \ 178 | fprintf(stderr, "EXITING program!!!\n"); \ 179 | exit(1); \ 180 | } \ 181 | } 182 | 183 | /* Prototypes */ 184 | static hid_t H5FD_gds_init(void); 185 | static herr_t H5FD__gds_term(void); 186 | static H5FD_t *H5FD__gds_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); 187 | static herr_t H5FD__gds_close(H5FD_t *_file); 188 | static int H5FD__gds_cmp(const H5FD_t *_f1, const H5FD_t *_f2); 189 | static herr_t H5FD__gds_query(const H5FD_t *_f1, unsigned long *flags); 190 | static haddr_t H5FD__gds_get_eoa(const H5FD_t *_file, H5FD_mem_t type); 191 | static herr_t H5FD__gds_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); 192 | static haddr_t H5FD__gds_get_eof(const H5FD_t *_file, H5FD_mem_t type); 193 | static herr_t H5FD__gds_get_handle(H5FD_t *_file, hid_t fapl, void **file_handle); 194 | static herr_t H5FD__gds_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size, 195 | void *buf); 196 | static herr_t H5FD__gds_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size, 197 | const void *buf); 198 | static herr_t H5FD__gds_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); 199 | static herr_t H5FD__gds_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); 200 | static herr_t H5FD__gds_lock(H5FD_t *_file, hbool_t rw); 201 | static herr_t H5FD__gds_unlock(H5FD_t *_file); 202 | static herr_t H5FD__gds_delete(const char *filename, hid_t fapl_id); 203 | static herr_t H5FD__gds_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input, 204 | void **output); 205 | 206 | static const H5FD_class_t H5FD_gds_g = { 207 | H5FD_CLASS_VERSION, /* struct version */ 208 | H5FD_GDS_VALUE, /* value */ 209 | H5FD_GDS_NAME, /* name */ 210 | MAXADDR, /* maxaddr */ 211 | H5F_CLOSE_WEAK, /* fc_degree */ 212 | H5FD__gds_term, /* terminate */ 213 | NULL, /* sb_size */ 214 | NULL, /* sb_encode */ 215 | NULL, /* sb_decode */ 216 | 0, /* fapl_size */ 217 | NULL, /* fapl_get */ 218 | NULL, /* fapl_copy */ 219 | NULL, /* fapl_free */ 220 | 0, /* dxpl_size */ 221 | NULL, /* dxpl_copy */ 222 | NULL, /* dxpl_free */ 223 | H5FD__gds_open, /* open */ 224 | H5FD__gds_close, /* close */ 225 | H5FD__gds_cmp, /* cmp */ 226 | H5FD__gds_query, /* query */ 227 | NULL, /* get_type_map */ 228 | NULL, /* alloc */ 229 | NULL, /* free */ 230 | H5FD__gds_get_eoa, /* get_eoa */ 231 | H5FD__gds_set_eoa, /* set_eoa */ 232 | H5FD__gds_get_eof, /* get_eof */ 233 | H5FD__gds_get_handle, /* get_handle */ 234 | H5FD__gds_read, /* read */ 235 | H5FD__gds_write, /* write */ 236 | NULL, /* read_vector */ 237 | NULL, /* write_vector */ 238 | NULL, /* read_selection */ 239 | NULL, /* write_selection */ 240 | H5FD__gds_flush, /* flush */ 241 | H5FD__gds_truncate, /* truncate */ 242 | H5FD__gds_lock, /* lock */ 243 | H5FD__gds_unlock, /* unlock */ 244 | H5FD__gds_delete, /* delete */ 245 | H5FD__gds_ctl, /* ctl */ 246 | H5FD_FLMAP_DICHOTOMY /* fl_map */ 247 | }; 248 | 249 | /*------------------------------------------------------------------------- 250 | * Function: H5FD_gds_init 251 | * 252 | * Purpose: Initialize this driver by registering the driver with the 253 | * library. 254 | * 255 | * Return: Success: The driver ID for the gds driver 256 | * Failure: H5I_INVALID_HID 257 | * 258 | * Programmer: John J Ravi 259 | * Tuesday, 06 October 2020 260 | * 261 | *------------------------------------------------------------------------- 262 | */ 263 | static hid_t 264 | H5FD_gds_init(void) 265 | { 266 | CUfileError_t status; 267 | char * lock_env_var = NULL; /* Environment variable pointer */ 268 | hid_t ret_value = H5I_INVALID_HID; /* Return value */ 269 | 270 | /* Initialize error reporting */ 271 | if ((H5FDgds_err_stack_g = H5Ecreate_stack()) < 0) 272 | H5FD_GDS_GOTO_ERROR(H5E_VFL, H5E_CANTINIT, H5I_INVALID_HID, "can't create HDF5 error stack"); 273 | if ((H5FDgds_err_class_g = H5Eregister_class(H5FD_GDS_ERR_CLS_NAME, H5FD_GDS_ERR_LIB_NAME, H5FD_GDS_ERR_VER)) < 0) 274 | H5FD_GDS_GOTO_ERROR(H5E_VFL, H5E_CANTINIT, H5I_INVALID_HID, "can't register error class with HDF5 error API"); 275 | 276 | /* Check the use disabled file locks environment variable */ 277 | lock_env_var = getenv("HDF5_USE_FILE_LOCKING"); 278 | if (lock_env_var && !strcmp(lock_env_var, "BEST_EFFORT")) 279 | ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ 280 | else if (lock_env_var && (!strcmp(lock_env_var, "TRUE") || !strcmp(lock_env_var, "1"))) 281 | ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ 282 | else 283 | ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ 284 | 285 | if (!cu_file_driver_opened) { 286 | status = cuFileDriverOpen(); 287 | 288 | if (status.err == CU_FILE_SUCCESS) { 289 | cu_file_driver_opened = true; 290 | } 291 | else { 292 | H5FD_GDS_GOTO_ERROR(H5E_INTERNAL, H5E_SYSTEM, H5I_INVALID_HID, "unable to open cufile driver"); 293 | /* TODO: get the error string once the cufile c api is ready */ 294 | /* 295 | * fprintf(stderr, "cufile driver open error: %s\n", 296 | * cuFileGetErrorString(status)); 297 | */ 298 | } 299 | } 300 | 301 | if (H5I_VFL != H5Iget_type(H5FD_GDS_g)) 302 | H5FD_GDS_g = H5FDregister(&H5FD_gds_g); 303 | 304 | /* Set return value */ 305 | ret_value = H5FD_GDS_g; 306 | 307 | done: 308 | H5FD_GDS_FUNC_LEAVE; 309 | } /* end H5FD_gds_init() */ 310 | 311 | /*--------------------------------------------------------------------------- 312 | * Function: H5FD__gds_term 313 | * 314 | * Purpose: Shut down the VFD 315 | * 316 | * Returns: Non-negative on success or negative on failure 317 | * 318 | * Programmer: John J Ravi 319 | * Tuesday, 06 October 2020 320 | * 321 | *--------------------------------------------------------------------------- 322 | */ 323 | static herr_t 324 | H5FD__gds_term(void) 325 | { 326 | herr_t ret_value = SUCCEED; /* Return value */ 327 | 328 | if (cu_file_driver_opened) { 329 | /* CUfileError_t status; */ 330 | 331 | /* FIXME: cuFileDriveClose is throwing errors with h5py and cupy */ 332 | /* 333 | * status = cuFileDriverClose(); 334 | * if (status.err == CU_FILE_SUCCESS) { 335 | * cu_file_driver_opened = false; 336 | * } 337 | * else { 338 | * H5FD_GDS_GOTO_ERROR(H5E_INTERNAL, H5E_SYSTEM, NULL, "unable to close cufile driver"); 339 | * // TODO: get the error string once the cufile c api is ready 340 | * // fprintf(stderr, "cufile driver close failed: %s\n", 341 | * // cuFileGetErrorString(status)); 342 | * } 343 | */ 344 | } 345 | 346 | /* Unregister from HDF5 error API */ 347 | if (H5FDgds_err_class_g >= 0) { 348 | if (H5Eunregister_class(H5FDgds_err_class_g) < 0) 349 | H5FD_GDS_GOTO_ERROR(H5E_VFL, H5E_CLOSEERROR, FAIL, "can't unregister error class from HDF5 error API"); 350 | 351 | /* Print the current error stack before destroying it */ 352 | PRINT_ERROR_STACK; 353 | 354 | /* Destroy the error stack */ 355 | if (H5Eclose_stack(H5FDgds_err_stack_g) < 0) { 356 | H5FD_GDS_GOTO_ERROR(H5E_VFL, H5E_CLOSEERROR, FAIL, "can't close HDF5 error stack"); 357 | PRINT_ERROR_STACK; 358 | } /* end if */ 359 | 360 | H5FDgds_err_stack_g = H5I_INVALID_HID; 361 | H5FDgds_err_class_g = H5I_INVALID_HID; 362 | } /* end if */ 363 | 364 | /* Reset VFL ID */ 365 | H5FD_GDS_g = H5I_INVALID_HID; 366 | 367 | done: 368 | H5FD_GDS_FUNC_LEAVE_API; 369 | } /* end H5FD__gds_term() */ 370 | 371 | /*------------------------------------------------------------------------- 372 | * Function: H5Pset_fapl_gds 373 | * 374 | * Purpose: Modify the file access property list to use the H5FD_GDS 375 | * driver defined in this source file. There are no driver 376 | * specific properties. 377 | * 378 | * Return: Non-negative on success/Negative on failure 379 | * 380 | *------------------------------------------------------------------------- 381 | */ 382 | herr_t 383 | H5Pset_fapl_gds(hid_t fapl_id, size_t boundary, size_t block_size, size_t cbuf_size) 384 | { 385 | herr_t ret_value; 386 | 387 | /* Silence compiler */ 388 | (void)boundary; 389 | (void)block_size; 390 | (void)cbuf_size; 391 | 392 | if (H5I_GENPROP_LST != H5Iget_type(fapl_id) || TRUE != H5Pisa_class(fapl_id, H5P_FILE_ACCESS)) 393 | H5FD_GDS_GOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list"); 394 | 395 | ret_value = H5Pset_driver(fapl_id, H5FD_GDS, NULL); 396 | 397 | done: 398 | H5FD_GDS_FUNC_LEAVE_API; 399 | } 400 | 401 | /*------------------------------------------------------------------------- 402 | * Function: H5Pget_fapl_gds 403 | * 404 | * Purpose: Returns information about the gds file access property 405 | * list though the function arguments. 406 | * 407 | * Return: Success: Non-negative 408 | * 409 | * Failure: Negative 410 | * 411 | *------------------------------------------------------------------------- 412 | */ 413 | herr_t 414 | H5Pget_fapl_gds(hid_t fapl_id, size_t *boundary /*out*/, size_t *block_size /*out*/, 415 | size_t *cbuf_size /*out*/) 416 | { 417 | herr_t ret_value = SUCCEED; /* Return value */ 418 | 419 | if (H5I_GENPROP_LST != H5Iget_type(fapl_id) || TRUE != H5Pisa_class(fapl_id, H5P_FILE_ACCESS)) 420 | H5FD_GDS_GOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list"); 421 | if (H5FD_GDS != H5Pget_driver(fapl_id)) 422 | H5FD_GDS_GOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver"); 423 | 424 | if (boundary) 425 | *boundary = 0; 426 | if (block_size) 427 | *block_size = 0; 428 | if (cbuf_size) 429 | *cbuf_size = 0; 430 | 431 | done: 432 | H5FD_GDS_FUNC_LEAVE_API; 433 | } /* end H5Pget_fapl_gds() */ 434 | 435 | /*------------------------------------------------------------------------- 436 | * Function: H5FD__gds_open 437 | * 438 | * Purpose: Create and/or opens a Unix file for direct I/O as an HDF5 file. 439 | * 440 | * Return: Success: A pointer to a new file data structure. The 441 | * public fields will be initialized by the 442 | * caller, which is always H5FD_open(). 443 | * 444 | * Failure: NULL 445 | * 446 | * Programmer: John J Ravi 447 | * Tuesday, 06 October 2020 448 | * 449 | *------------------------------------------------------------------------- 450 | */ 451 | static H5FD_t * 452 | H5FD__gds_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) 453 | { 454 | CUfileError_t status; 455 | CUfileDescr_t cf_descr; 456 | 457 | int o_flags; 458 | int fd = (-1); 459 | H5FD_gds_t * file = NULL; 460 | #ifdef H5_HAVE_WIN32_API 461 | HFILE filehandle; 462 | struct _BY_HANDLE_FILE_INFORMATION fileinfo; 463 | #endif 464 | struct stat sb; 465 | H5FD_t * ret_value = NULL; 466 | 467 | H5FD_GDS_INIT; 468 | 469 | /* Sanity check on file offsets */ 470 | assert(sizeof(off_t) >= sizeof(size_t)); 471 | 472 | /* Check arguments */ 473 | if (!name || !*name) 474 | H5FD_GDS_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name"); 475 | if (0 == maxaddr || HADDR_UNDEF == maxaddr) 476 | H5FD_GDS_GOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr"); 477 | if (ADDR_OVERFLOW(maxaddr)) 478 | H5FD_GDS_GOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr"); 479 | 480 | /* Build the open flags */ 481 | o_flags = (H5F_ACC_RDWR & flags) ? O_RDWR : O_RDONLY; 482 | if (H5F_ACC_TRUNC & flags) 483 | o_flags |= O_TRUNC; 484 | if (H5F_ACC_CREAT & flags) 485 | o_flags |= O_CREAT; 486 | if (H5F_ACC_EXCL & flags) 487 | o_flags |= O_EXCL; 488 | 489 | /* Open the file */ 490 | if ((fd = open(name, o_flags, H5FD_GDS_POSIX_CREATE_MODE_RW)) < 0) 491 | H5FD_GDS_SYS_GOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file"); 492 | 493 | if (fstat(fd, &sb) < 0) 494 | H5FD_GDS_SYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, NULL, "unable to fstat file"); 495 | 496 | #ifdef ADVISE_OS_DISABLE_READ_CACHE 497 | if (posix_fadvise(fd, 0, 0, POSIX_FADV_RANDOM) != 0) { 498 | perror("posix_fadvise"); 499 | exit(EXIT_FAILURE); 500 | } 501 | 502 | if (posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED) != 0) { 503 | perror("posix_fadvise"); 504 | exit(EXIT_FAILURE); 505 | } 506 | 507 | if (posix_fadvise(fd, 0, 0, POSIX_FADV_NOREUSE) != 0) { 508 | perror("posix_fadvise"); 509 | exit(EXIT_FAILURE); 510 | } 511 | #endif /* ADVISE_OS_DISABLE_READ_CACHE */ 512 | 513 | /* Create the new file struct */ 514 | if (NULL == (file = calloc(1, sizeof(H5FD_gds_t)))) 515 | H5FD_GDS_GOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct"); 516 | 517 | memset((void *)&cf_descr, 0, sizeof(CUfileDescr_t)); 518 | cf_descr.handle.fd = fd; 519 | cf_descr.type = CU_FILE_HANDLE_TYPE_OPAQUE_FD; 520 | status = cuFileHandleRegister(&file->cf_handle, &cf_descr); 521 | if (status.err != CU_FILE_SUCCESS) { 522 | H5FD_GDS_GOTO_ERROR(H5E_INTERNAL, H5E_SYSTEM, NULL, "unable to register file with cufile driver"); 523 | } 524 | 525 | file->fd = fd; 526 | /* FIXME: Possible overflow! */ 527 | file->eof = (haddr_t)sb.st_size; 528 | 529 | #ifdef H5_HAVE_WIN32_API 530 | filehandle = _get_osfhandle(fd); 531 | (void)GetFileInformationByHandle((HANDLE)filehandle, &fileinfo); 532 | file->fileindexhi = fileinfo.nFileIndexHigh; 533 | file->fileindexlo = fileinfo.nFileIndexLow; 534 | #else 535 | file->device = sb.st_dev; 536 | file->inode = sb.st_ino; 537 | #endif /*H5_HAVE_WIN32_API*/ 538 | 539 | /* Check the file locking flags in the fapl */ 540 | if (ignore_disabled_file_locks_s != FAIL) 541 | /* The environment variable was set, so use that preferentially */ 542 | file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; 543 | else { 544 | hbool_t unused; 545 | 546 | /* Use the value in the property list */ 547 | if (H5Pget_file_locking(fapl_id, &unused, &file->ignore_disabled_file_locks) < 0) 548 | H5FD_GDS_GOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property"); 549 | } 550 | 551 | /* Set return value */ 552 | ret_value = (H5FD_t *)file; 553 | fprintf(stderr, "%s:%u - Successfully opened file w/GDS VFD\n", __func__, __LINE__); 554 | 555 | done: 556 | if (ret_value == NULL) { 557 | if (fd >= 0) 558 | close(fd); 559 | } /* end if */ 560 | 561 | H5FD_GDS_FUNC_LEAVE_API; 562 | } 563 | 564 | /*------------------------------------------------------------------------- 565 | * Function: H5FD__gds_close 566 | * 567 | * Purpose: Closes the file. 568 | * 569 | * Return: Success: 0 570 | * 571 | * Failure: -1, file not closed. 572 | * 573 | * Programmer: John J Ravi 574 | * Tuesday, 06 October 2020 575 | * 576 | *------------------------------------------------------------------------- 577 | */ 578 | static herr_t 579 | H5FD__gds_close(H5FD_t *_file) 580 | { 581 | H5FD_gds_t *file = (H5FD_gds_t *)_file; 582 | herr_t ret_value = SUCCEED; /* Return value */ 583 | 584 | /* close file handle */ 585 | cuFileHandleDeregister(file->cf_handle); 586 | 587 | if (close(file->fd) < 0) 588 | H5FD_GDS_SYS_GOTO_ERROR(H5E_IO, H5E_CANTCLOSEFILE, FAIL, "unable to close file"); 589 | 590 | free(file); 591 | 592 | done: 593 | H5FD_GDS_FUNC_LEAVE_API; 594 | } 595 | 596 | /*------------------------------------------------------------------------- 597 | * Function: H5FD__gds_cmp 598 | * 599 | * Purpose: Compares two files belonging to this driver using an 600 | * arbitrary (but consistent) ordering. 601 | * 602 | * Return: Success: A value like strcmp() 603 | * 604 | * Failure: never fails (arguments were checked by the 605 | * caller). 606 | * 607 | *------------------------------------------------------------------------- 608 | */ 609 | static int 610 | H5FD__gds_cmp(const H5FD_t *_f1, const H5FD_t *_f2) 611 | { 612 | const H5FD_gds_t *f1 = (const H5FD_gds_t *)_f1; 613 | const H5FD_gds_t *f2 = (const H5FD_gds_t *)_f2; 614 | int ret_value = 0; 615 | 616 | #ifdef H5_HAVE_WIN32_API 617 | if (f1->fileindexhi < f2->fileindexhi) 618 | H5FD_GDS_GOTO_DONE(-1); 619 | if (f1->fileindexhi > f2->fileindexhi) 620 | H5FD_GDS_GOTO_DONE(1); 621 | 622 | if (f1->fileindexlo < f2->fileindexlo) 623 | H5FD_GDS_GOTO_DONE(-1); 624 | if (f1->fileindexlo > f2->fileindexlo) 625 | H5FD_GDS_GOTO_DONE(1); 626 | 627 | #else 628 | #ifdef H5_DEV_T_IS_SCALAR 629 | if (f1->device < f2->device) 630 | H5FD_GDS_GOTO_DONE(-1); 631 | if (f1->device > f2->device) 632 | H5FD_GDS_GOTO_DONE(1); 633 | #else /* H5_DEV_T_IS_SCALAR */ 634 | /* If dev_t isn't a scalar value on this system, just use memcmp to 635 | * determine if the values are the same or not. The actual return value 636 | * shouldn't really matter... 637 | */ 638 | if (memcmp(&(f1->device), &(f2->device), sizeof(dev_t)) < 0) 639 | H5FD_GDS_GOTO_DONE(-1); 640 | if (memcmp(&(f1->device), &(f2->device), sizeof(dev_t)) > 0) 641 | H5FD_GDS_GOTO_DONE(1); 642 | #endif /* H5_DEV_T_IS_SCALAR */ 643 | 644 | if (f1->inode < f2->inode) 645 | H5FD_GDS_GOTO_DONE(-1); 646 | if (f1->inode > f2->inode) 647 | H5FD_GDS_GOTO_DONE(1); 648 | 649 | #endif 650 | 651 | done: 652 | H5FD_GDS_FUNC_LEAVE_API; 653 | } 654 | 655 | /*------------------------------------------------------------------------- 656 | * Function: H5FD__gds_query 657 | * 658 | * Purpose: Set the flags that this VFL driver is capable of supporting. 659 | * (listed in H5FDpublic.h) 660 | * 661 | * Return: Success: non-negative 662 | * 663 | * Failure: negative 664 | * 665 | * Programmer: John J Ravi 666 | * Tuesday, 06 October 2020 667 | * 668 | *------------------------------------------------------------------------- 669 | */ 670 | static herr_t 671 | H5FD__gds_query(const H5FD_t *_f, unsigned long *flags /* out */) 672 | { 673 | herr_t ret_value = SUCCEED; 674 | 675 | /* Silence compiler */ 676 | (void)_f; 677 | 678 | /* Set the VFL feature flags that this driver supports */ 679 | if (flags) { 680 | *flags = 0; 681 | *flags |= H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */ 682 | *flags |= H5FD_FEAT_ACCUMULATE_METADATA; /* OK to accumulate metadata for faster writes */ 683 | *flags |= H5FD_FEAT_AGGREGATE_SMALLDATA; /* OK to aggregate "small" raw data allocations */ 684 | *flags |= 685 | H5FD_FEAT_SUPPORTS_SWMR_IO; /* VFD supports the single-writer/multiple-readers (SWMR) pattern */ 686 | *flags |= H5FD_FEAT_DEFAULT_VFD_COMPATIBLE; /* VFD creates a file which can be opened with the default 687 | VFD */ 688 | *flags |= H5FD_FEAT_MEMMANAGE; /* VFD uses CUDA memory management routines */ 689 | } 690 | 691 | H5FD_GDS_FUNC_LEAVE_API; 692 | } 693 | 694 | /*------------------------------------------------------------------------- 695 | * Function: H5FD__gds_get_eoa 696 | * 697 | * Purpose: Gets the end-of-address marker for the file. The EOA marker 698 | * is the first address past the last byte allocated in the 699 | * format address space. 700 | * 701 | * Return: Success: The end-of-address marker. 702 | * 703 | * Failure: HADDR_UNDEF 704 | * 705 | *------------------------------------------------------------------------- 706 | */ 707 | static haddr_t 708 | H5FD__gds_get_eoa(const H5FD_t *_file, H5FD_mem_t type) 709 | { 710 | const H5FD_gds_t *file = (const H5FD_gds_t *)_file; 711 | haddr_t ret_value = HADDR_UNDEF; 712 | 713 | assert(file); 714 | 715 | /* Silence compiler */ 716 | (void)type; 717 | 718 | ret_value = file->eoa; 719 | 720 | H5FD_GDS_FUNC_LEAVE_API; 721 | } 722 | 723 | /*------------------------------------------------------------------------- 724 | * Function: H5FD__gds_set_eoa 725 | * 726 | * Purpose: Set the end-of-address marker for the file. This function is 727 | * called shortly after an existing HDF5 file is opened in order 728 | * to tell the driver where the end of the HDF5 data is located. 729 | * 730 | * Return: Success: 0 731 | * 732 | * Failure: -1 733 | * 734 | *------------------------------------------------------------------------- 735 | */ 736 | static herr_t 737 | H5FD__gds_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) 738 | { 739 | H5FD_gds_t *file = (H5FD_gds_t *)_file; 740 | herr_t ret_value = SUCCEED; 741 | 742 | /* Silence compiler */ 743 | (void)type; 744 | 745 | file->eoa = addr; 746 | 747 | H5FD_GDS_FUNC_LEAVE_API; 748 | } 749 | 750 | /*------------------------------------------------------------------------- 751 | * Function: H5FD__gds_get_eof 752 | * 753 | * Purpose: Returns the end-of-file marker, which is the greater of 754 | * either the Unix end-of-file or the HDF5 end-of-address 755 | * markers. 756 | * 757 | * Return: Success: End of file address, the first address past 758 | * the end of the "file", either the Unix file 759 | * or the HDF5 file. 760 | * 761 | * Failure: HADDR_UNDEF 762 | * 763 | *------------------------------------------------------------------------- 764 | */ 765 | static haddr_t 766 | H5FD__gds_get_eof(const H5FD_t *_file, H5FD_mem_t type) 767 | { 768 | const H5FD_gds_t *file = (const H5FD_gds_t *)_file; 769 | haddr_t ret_value = HADDR_UNDEF; 770 | 771 | assert(file); 772 | 773 | /* Silence compiler */ 774 | (void)type; 775 | 776 | ret_value = file->eof; 777 | 778 | H5FD_GDS_FUNC_LEAVE_API; 779 | } 780 | 781 | /*------------------------------------------------------------------------- 782 | * Function: H5FD_gds_get_handle 783 | * 784 | * Purpose: Returns the file handle of gds file driver. 785 | * 786 | * Returns: Non-negative if succeed or negative if fails. 787 | * 788 | *------------------------------------------------------------------------- 789 | */ 790 | static herr_t 791 | H5FD__gds_get_handle(H5FD_t *_file, hid_t fapl, void **file_handle) 792 | { 793 | H5FD_gds_t *file = (H5FD_gds_t *)_file; 794 | herr_t ret_value = SUCCEED; 795 | 796 | /* Silence compiler */ 797 | (void)fapl; 798 | 799 | if (!file_handle) 800 | H5FD_GDS_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid"); 801 | *file_handle = &(file->fd); 802 | 803 | done: 804 | H5FD_GDS_FUNC_LEAVE_API; 805 | } 806 | 807 | bool is_device_pointer(const void *ptr); 808 | bool 809 | is_device_pointer(const void *ptr) 810 | { 811 | struct cudaPointerAttributes attributes; 812 | cudaPointerGetAttributes(&attributes, ptr); 813 | return (attributes.devicePointer != NULL); 814 | } 815 | 816 | /*------------------------------------------------------------------------- 817 | * Function: H5FD__gds_read 818 | * 819 | * Purpose: 820 | * GPU buf: 821 | * interface with NVIDIA GPUDirect Storage 822 | * 823 | * CPU buf: 824 | * Reads SIZE bytes of data from FILE beginning at address ADDR 825 | * into buffer BUF according to data transfer properties in 826 | * DXPL_ID. 827 | * 828 | * Return: Success: Zero. Result is stored in caller-supplied 829 | * buffer BUF. 830 | * 831 | * Failure: -1, Contents of buffer BUF are undefined. 832 | * 833 | * Programmer: John J Ravi 834 | * Tuesday, 06 October 2020 835 | * 836 | *------------------------------------------------------------------------- 837 | */ 838 | static herr_t 839 | H5FD__gds_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, 840 | size_t size, void *buf /*out*/) 841 | { 842 | H5FD_gds_t *file = (H5FD_gds_t *)_file; 843 | off_t offset = (off_t)addr; 844 | herr_t ret_value = SUCCEED; /* Return value */ 845 | 846 | assert(file && file->pub.cls); 847 | assert(buf); 848 | 849 | /* Silence compiler */ 850 | (void)type; 851 | (void)dxpl_id; 852 | 853 | /* Check for overflow conditions */ 854 | if (HADDR_UNDEF == addr) 855 | H5FD_GDS_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "addr undefined"); 856 | if (REGION_OVERFLOW(addr, size)) 857 | H5FD_GDS_GOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow"); 858 | 859 | /* Pass to cuFile */ 860 | if (cuFileRead(file->cf_handle, buf, size, offset, 0) < 0) 861 | H5FD_GDS_GOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, 862 | "file read failed: file descriptor = %d, " 863 | "buf = %p, total read size = %zu, offset = %llu", 864 | file->fd, buf, size, (unsigned long long)offset); 865 | 866 | done: 867 | H5FD_GDS_FUNC_LEAVE_API; 868 | } 869 | 870 | /*------------------------------------------------------------------------- 871 | * Function: H5FD__gds_write 872 | * 873 | * Purpose: 874 | * GPU buf: 875 | * interface with NVIDIA GPUDirect Storage 876 | * 877 | * CPU buf: 878 | * Writes SIZE bytes of data to FILE beginning at address ADDR 879 | * from buffer BUF according to data transfer properties in 880 | * DXPL_ID. 881 | * 882 | * Return: Success: Zero 883 | * 884 | * Failure: -1 885 | * 886 | * Programmer: John J Ravi 887 | * Tuesday, 06 October 2020 888 | * 889 | *------------------------------------------------------------------------- 890 | */ 891 | static herr_t 892 | H5FD__gds_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, 893 | size_t size, const void *buf) 894 | { 895 | H5FD_gds_t *file = (H5FD_gds_t *)_file; 896 | off_t offset = (off_t)addr; 897 | herr_t ret_value = SUCCEED; /* Return value */ 898 | 899 | assert(file && file->pub.cls); 900 | assert(buf); 901 | 902 | /* Silence compiler */ 903 | (void)type; 904 | (void)dxpl_id; 905 | 906 | /* Check for overflow conditions */ 907 | if (HADDR_UNDEF == addr) 908 | H5FD_GDS_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "addr undefined"); 909 | if (REGION_OVERFLOW(addr, size)) 910 | H5FD_GDS_GOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow"); 911 | 912 | /* Pass to cuFile */ 913 | if (cuFileWrite(file->cf_handle, buf, size, offset, 0) < 0) 914 | H5FD_GDS_GOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, 915 | "file write failed: file descriptor = %d, " 916 | "buf = %p, total write size = %zu, offset = %llu", 917 | file->fd, buf, size, (unsigned long long)offset); 918 | 919 | done: 920 | H5FD_GDS_FUNC_LEAVE_API; 921 | } 922 | 923 | /*------------------------------------------------------------------------- 924 | * Function: H5FD__gds_flush 925 | * 926 | * Purpose: Flush makes use of fsync to flush data to persistent storage. 927 | * O_DIRECT will disable the OS cache, but fsync maybe necessary on 928 | * certain file system to get data to persistant storage. 929 | * 930 | * Return: Success: Zero 931 | * 932 | * Failure: -1 933 | * 934 | * Programmer: John J Ravi 935 | * Saturday, 3 October 2020 936 | * 937 | *------------------------------------------------------------------------- 938 | */ 939 | static herr_t 940 | H5FD__gds_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing) 941 | { 942 | H5FD_gds_t *file = (H5FD_gds_t *)_file; /* VFD file struct */ 943 | herr_t ret_value = SUCCEED; /* Return value */ 944 | 945 | assert(file); 946 | 947 | /* Silence compiler */ 948 | (void)dxpl_id; 949 | (void)closing; 950 | 951 | if (fsync(file->fd) < 0) { 952 | H5FD_GDS_SYS_GOTO_ERROR(H5E_VFL, H5E_CANTFLUSH, FAIL, "unable perform fsync on file descriptor"); 953 | } 954 | 955 | done: 956 | H5FD_GDS_FUNC_LEAVE_API; 957 | } /* end H5FD__gds_flush() */ 958 | 959 | /*------------------------------------------------------------------------- 960 | * Function: H5FD__gds_truncate 961 | * 962 | * Purpose: Makes sure that the true file size is the same (or larger) 963 | * than the end-of-address. 964 | * 965 | * Return: Success: Non-negative 966 | * 967 | * Failure: Negative 968 | * 969 | *------------------------------------------------------------------------- 970 | */ 971 | static herr_t 972 | H5FD__gds_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing) 973 | { 974 | H5FD_gds_t *file = (H5FD_gds_t *)_file; 975 | herr_t ret_value = SUCCEED; /* Return value */ 976 | 977 | assert(file); 978 | 979 | /* Silence compiler */ 980 | (void)dxpl_id; 981 | (void)closing; 982 | 983 | /* Extend the file to make sure it's large enough */ 984 | if (file->eoa != file->eof) { 985 | #ifdef H5_HAVE_WIN32_API 986 | HFILE filehandle; /* Windows file handle */ 987 | LARGE_INTEGER li; /* 64-bit integer for SetFilePointer() call */ 988 | 989 | /* Map the posix file handle to a Windows file handle */ 990 | filehandle = _get_osfhandle(file->fd); 991 | 992 | /* Translate 64-bit integers into form Windows wants */ 993 | /* [This algorithm is from the Windows documentation for SetFilePointer()] */ 994 | li.QuadPart = (LONGLONG)file->eoa; 995 | (void)SetFilePointer((HANDLE)filehandle, li.LowPart, &li.HighPart, FILE_BEGIN); 996 | if (SetEndOfFile((HANDLE)filehandle) == 0) 997 | H5FD_GDS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly"); 998 | #else /* H5_HAVE_WIN32_API */ 999 | if (-1 == ftruncate(file->fd, (off_t)file->eoa)) 1000 | H5FD_GDS_SYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly"); 1001 | #endif /* H5_HAVE_WIN32_API */ 1002 | 1003 | /* Update the eof value */ 1004 | file->eof = file->eoa; 1005 | } 1006 | 1007 | done: 1008 | H5FD_GDS_FUNC_LEAVE_API; 1009 | } /* end H5FD__gds_truncate() */ 1010 | 1011 | /*------------------------------------------------------------------------- 1012 | * Function: H5FD__gds_lock 1013 | * 1014 | * Purpose: To place an advisory lock on a file. 1015 | * The lock type to apply depends on the parameter "rw": 1016 | * TRUE--opens for write: an exclusive lock 1017 | * FALSE--opens for read: a shared lock 1018 | * 1019 | * Return: SUCCEED/FAIL 1020 | * 1021 | * Programmer: Vailin Choi; May 2013 1022 | * 1023 | *------------------------------------------------------------------------- 1024 | */ 1025 | static herr_t 1026 | H5FD__gds_lock(H5FD_t *_file, hbool_t rw) 1027 | { 1028 | H5FD_gds_t *file = (H5FD_gds_t *)_file; /* VFD file struct */ 1029 | int lock_flags; /* file locking flags */ 1030 | herr_t ret_value = SUCCEED; /* Return value */ 1031 | 1032 | assert(file); 1033 | 1034 | /* Set exclusive or shared lock based on rw status */ 1035 | lock_flags = rw ? LOCK_EX : LOCK_SH; 1036 | 1037 | /* Place a non-blocking lock on the file */ 1038 | if (flock(file->fd, lock_flags | LOCK_NB) < 0) { 1039 | if (file->ignore_disabled_file_locks && ENOSYS == errno) { 1040 | /* When errno is set to ENOSYS, the file system does not support 1041 | * locking, so ignore it. 1042 | */ 1043 | errno = 0; 1044 | } 1045 | else 1046 | H5FD_GDS_SYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file"); 1047 | } 1048 | 1049 | done: 1050 | H5FD_GDS_FUNC_LEAVE_API; 1051 | } /* end H5FD__gds_lock() */ 1052 | 1053 | /*------------------------------------------------------------------------- 1054 | * Function: H5FD__gds_unlock 1055 | * 1056 | * Purpose: To remove the existing lock on the file 1057 | * 1058 | * Return: SUCCEED/FAIL 1059 | * 1060 | * Programmer: Vailin Choi; May 2013 1061 | * 1062 | *------------------------------------------------------------------------- 1063 | */ 1064 | static herr_t 1065 | H5FD__gds_unlock(H5FD_t *_file) 1066 | { 1067 | H5FD_gds_t *file = (H5FD_gds_t *)_file; /* VFD file struct */ 1068 | herr_t ret_value = SUCCEED; /* Return value */ 1069 | 1070 | assert(file); 1071 | 1072 | if (flock(file->fd, LOCK_UN) < 0) { 1073 | if (file->ignore_disabled_file_locks && ENOSYS == errno) { 1074 | /* When errno is set to ENOSYS, the file system does not support 1075 | * locking, so ignore it. 1076 | */ 1077 | errno = 0; 1078 | } 1079 | else 1080 | H5FD_GDS_SYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file"); 1081 | } 1082 | 1083 | done: 1084 | H5FD_GDS_FUNC_LEAVE_API; 1085 | } /* end H5FD__gds_unlock() */ 1086 | 1087 | /*------------------------------------------------------------------------- 1088 | * Function: H5FD__gds_delete 1089 | * 1090 | * Purpose: Delete a file 1091 | * 1092 | * Return: SUCCEED/FAIL 1093 | * 1094 | *------------------------------------------------------------------------- 1095 | */ 1096 | static herr_t 1097 | H5FD__gds_delete(const char *filename, hid_t fapl_id) 1098 | { 1099 | herr_t ret_value = SUCCEED; /* Return value */ 1100 | 1101 | H5FD_GDS_INIT; 1102 | 1103 | assert(filename); 1104 | 1105 | /* Silence compiler */ 1106 | (void)fapl_id; 1107 | 1108 | if (remove(filename) < 0) 1109 | H5FD_GDS_SYS_GOTO_ERROR(H5E_VFL, H5E_CANTDELETEFILE, FAIL, "unable to delete file"); 1110 | 1111 | done: 1112 | H5FD_GDS_FUNC_LEAVE_API; 1113 | } /* end H5FD__gds_delete() */ 1114 | 1115 | /*------------------------------------------------------------------------- 1116 | * Function: H5FD__gds_ctl 1117 | * 1118 | * Purpose: Perform an optional "ctl" operation 1119 | * 1120 | * Return: SUCCEED/FAIL 1121 | * 1122 | *------------------------------------------------------------------------- 1123 | */ 1124 | static herr_t 1125 | H5FD__gds_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input, 1126 | void **output) 1127 | { 1128 | H5FD_gds_t *file = (H5FD_gds_t *)_file; /* VFD file struct */ 1129 | herr_t ret_value = SUCCEED; /* Return value */ 1130 | 1131 | assert(file); 1132 | 1133 | /* Silence compiler */ 1134 | (void)file; 1135 | (void)output; 1136 | 1137 | switch (op_code) { 1138 | /* Driver-level memory copy */ 1139 | case H5FD_CTL_MEM_COPY: 1140 | { 1141 | const H5FD_ctl_memcpy_args_t *copy_args = (const H5FD_ctl_memcpy_args_t *)input; 1142 | enum cudaMemcpyKind cpyKind; 1143 | hbool_t src_on_device = FALSE; 1144 | hbool_t dst_on_device = FALSE; 1145 | const void *src; 1146 | void *dst; 1147 | 1148 | if (!copy_args) 1149 | H5FD_GDS_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid arguments to ctl operation"); 1150 | 1151 | /* Add offsets to source and destination buffers */ 1152 | src = ((const unsigned char *)copy_args->srcbuf) + copy_args->src_off; 1153 | dst = ((unsigned char *)copy_args->dstbuf) + copy_args->dst_off; 1154 | 1155 | /* Determine type of memory copy to perform */ 1156 | src_on_device = is_device_pointer(copy_args->srcbuf); 1157 | dst_on_device = is_device_pointer(copy_args->dstbuf); 1158 | 1159 | if (src_on_device && dst_on_device) 1160 | cpyKind = cudaMemcpyDeviceToDevice; 1161 | else if (src_on_device && !dst_on_device) 1162 | cpyKind = cudaMemcpyDeviceToHost; 1163 | else if (!src_on_device && dst_on_device) 1164 | cpyKind = cudaMemcpyHostToDevice; 1165 | else 1166 | cpyKind = cudaMemcpyHostToHost; 1167 | 1168 | check_cudaruntimecall(cudaMemcpy(dst, src, copy_args->len, cpyKind)) 1169 | 1170 | break; 1171 | } 1172 | 1173 | /* Unknown op code */ 1174 | default: 1175 | if (flags & H5FD_CTL_FAIL_IF_UNKNOWN_FLAG) 1176 | H5FD_GDS_GOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "unknown op_code and fail if unknown flag is set"); 1177 | break; 1178 | } 1179 | 1180 | done: 1181 | H5FD_GDS_FUNC_LEAVE_API; 1182 | } /* end H5FD__gds_ctl() */ 1183 | 1184 | /* 1185 | * Stub routines for dynamic plugin loading 1186 | */ 1187 | 1188 | H5PL_type_t 1189 | H5PLget_plugin_type(void) { 1190 | return H5PL_TYPE_VFD; 1191 | } 1192 | 1193 | const void* 1194 | H5PLget_plugin_info(void) { 1195 | return &H5FD_gds_g; 1196 | } 1197 | -------------------------------------------------------------------------------- /test/gds_test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "hdf5.h" 9 | 10 | #define FILENAME "gds_vfd_test.h5" 11 | #define MAINPROCESS (!mpi_rank) /* define process 0 as main process */ 12 | 13 | /* Constants definitions */ 14 | #define RANK 2 15 | #define DIM0 600 16 | #define DIM1 1200 17 | #define CHUNK_DIM0 ((DIM0 + 9) / 10) 18 | #define CHUNK_DIM1 ((DIM1 + 9) / 10) 19 | #define DATASETNAME1 "Data1" 20 | #define DATASETNAME2 "Data2" 21 | #define DATASETNAME3 "Data3" 22 | #define DATASETNAME4 "Data4" 23 | #define DATASETNAME5 "Data5" 24 | #define DATASETNAME6 "Data6" 25 | #define DATASETNAME7 "Data7" 26 | #define DATASETNAME8 "Data8" 27 | #define DATASETNAME9 "Data9" 28 | #define MAX_ERR_REPORT 10 /* Maximum number of errors reported */ 29 | 30 | /* point selection order */ 31 | #define IN_ORDER 1 32 | #define OUT_OF_ORDER 2 33 | 34 | /* Hyperslab layout styles */ 35 | #define BYROW 1 /* divide into slabs of rows */ 36 | #define BYCOL 2 /* divide into blocks of columns */ 37 | #define ZROW 3 /* same as BYCOL except process 0 gets 0 rows */ 38 | #define ZCOL 4 /* same as BYCOL except process 0 gets 0 columns */ 39 | 40 | /* I/O mode definitions */ 41 | #define NO_COLLECTIVE_SUPPORT /* HDF5 currently only supports collective I/O for 42 | * MPI-based VFDs. Since the GDS VFD is not currently 43 | * MPI-aware, disable collective I/O access in tests 44 | * for now. 45 | */ 46 | #define DXFER_COLLECTIVE_IO 0x1 /* Collective IO*/ 47 | #define DXFER_INDEPENDENT_IO 0x2 /* Independent IO collectively */ 48 | 49 | /* 50 | * VRFY: Verify if the condition val is true. 51 | * If val is not true, it prints error messages and calls MPI_Abort 52 | * to abort the program. 53 | */ 54 | #define VRFY_IMPL(val, mesg, rankvar) \ 55 | do { \ 56 | if (!val) { \ 57 | printf("Proc %d: ", rankvar); \ 58 | printf("*** Parallel ERROR ***\n"); \ 59 | printf(" VRFY (%s) failed at line %4d in %s\n", mesg, (int)__LINE__, __FILE__); \ 60 | ++nerrors; \ 61 | fflush(stdout); \ 62 | printf("aborting MPI processes\n"); \ 63 | MPI_Abort(MPI_COMM_WORLD, 1); \ 64 | } \ 65 | } while (0) 66 | 67 | #define VRFY_G(val, mesg) VRFY_IMPL(val, mesg, mpi_rank_g) 68 | #define VRFY(val, mesg) VRFY_IMPL(val, mesg, mpi_rank) 69 | 70 | #define H5FD_GDS_UNUSED(param) (void)(param) 71 | 72 | typedef int DATATYPE; 73 | 74 | static MPI_Comm comm = MPI_COMM_WORLD; 75 | static int mpi_rank; 76 | static int mpi_size; 77 | int nerrors = 0; 78 | int dxfer_coll_type = DXFER_COLLECTIVE_IO; 79 | 80 | static void extend_writeInd_cuda(void); 81 | static void dataset_writeAll_cuda(void); 82 | static void dataset_readAll_cuda(void); 83 | static void dataset_writeInd_cuda(void); 84 | static void dataset_readInd_cuda(void); 85 | static void extend_writeInd2_cuda(void); 86 | 87 | /* 88 | * Tests for the HDF5 GDS VFD. 89 | */ 90 | 91 | /* 92 | * The following are various utility routines used by the tests. 93 | */ 94 | 95 | /* 96 | * Setup the dimensions of the hyperslab. 97 | * Two modes--by rows or by columns. 98 | * Assume dimension rank is 2. 99 | * BYROW divide into slabs of rows 100 | * BYCOL divide into blocks of columns 101 | * ZROW same as BYROW except process 0 gets 0 rows 102 | * ZCOL same as BYCOL except process 0 gets 0 columns 103 | */ 104 | static void 105 | slab_set(int rank, int comm_size, hsize_t start[], hsize_t count[], hsize_t stride[], hsize_t block[], 106 | int mode) 107 | { 108 | switch (mode) { 109 | case BYROW: 110 | /* Each process takes a slabs of rows. */ 111 | block[0] = (hsize_t)(DIM0 / comm_size); 112 | block[1] = (hsize_t)DIM1; 113 | stride[0] = block[0]; 114 | stride[1] = block[1]; 115 | count[0] = 1; 116 | count[1] = 1; 117 | start[0] = (hsize_t)rank * block[0]; 118 | start[1] = 0; 119 | break; 120 | case BYCOL: 121 | /* Each process takes a block of columns. */ 122 | block[0] = (hsize_t)DIM0; 123 | block[1] = (hsize_t)(DIM1 / comm_size); 124 | stride[0] = block[0]; 125 | stride[1] = block[1]; 126 | count[0] = 1; 127 | count[1] = 1; 128 | start[0] = 0; 129 | start[1] = (hsize_t)rank * block[1]; 130 | break; 131 | case ZROW: 132 | /* Similar to BYROW except process 0 gets 0 row */ 133 | block[0] = (hsize_t)(rank ? DIM0 / comm_size : 0); 134 | block[1] = (hsize_t)DIM1; 135 | stride[0] = (rank ? block[0] : 1); /* avoid setting stride to 0 */ 136 | stride[1] = block[1]; 137 | count[0] = 1; 138 | count[1] = 1; 139 | start[0] = (rank ? (hsize_t)rank * block[0] : 0); 140 | start[1] = 0; 141 | break; 142 | case ZCOL: 143 | /* Similar to BYCOL except process 0 gets 0 column */ 144 | block[0] = (hsize_t)DIM0; 145 | block[1] = (hsize_t)(rank ? DIM1 / comm_size : 0); 146 | stride[0] = block[0]; 147 | stride[1] = (hsize_t)(rank ? block[1] : 1); /* avoid setting stride to 0 */ 148 | count[0] = 1; 149 | count[1] = 1; 150 | start[0] = 0; 151 | start[1] = (rank ? (hsize_t)rank * block[1] : 0); 152 | break; 153 | default: 154 | /* Unknown mode. Set it to cover the whole dataset. */ 155 | printf("unknown slab_set mode (%d)\n", mode); 156 | block[0] = (hsize_t)DIM0; 157 | block[1] = (hsize_t)DIM1; 158 | stride[0] = block[0]; 159 | stride[1] = block[1]; 160 | count[0] = 1; 161 | count[1] = 1; 162 | start[0] = 0; 163 | start[1] = 0; 164 | break; 165 | } 166 | } 167 | 168 | /* 169 | * Setup the coordinates for point selection. 170 | */ 171 | void 172 | point_set(hsize_t start[], hsize_t count[], hsize_t stride[], hsize_t block[], size_t num_points, 173 | hsize_t coords[], int order) 174 | { 175 | hsize_t i, j, k = 0, m, n, s1, s2; 176 | 177 | assert(RANK == 2); 178 | 179 | if (OUT_OF_ORDER == order) 180 | k = (num_points * RANK) - 1; 181 | else if (IN_ORDER == order) 182 | k = 0; 183 | 184 | s1 = start[0]; 185 | s2 = start[1]; 186 | 187 | for (i = 0; i < count[0]; i++) 188 | for (j = 0; j < count[1]; j++) 189 | for (m = 0; m < block[0]; m++) 190 | for (n = 0; n < block[1]; n++) 191 | if (OUT_OF_ORDER == order) { 192 | coords[k--] = s2 + (stride[1] * j) + n; 193 | coords[k--] = s1 + (stride[0] * i) + m; 194 | } 195 | else if (IN_ORDER == order) { 196 | coords[k++] = s1 + stride[0] * i + m; 197 | coords[k++] = s2 + stride[1] * j + n; 198 | } 199 | } 200 | 201 | /* 202 | * Fill the dataset with trivial data for testing. 203 | * Assume dimension rank is 2 and data is stored contiguous. 204 | */ 205 | static void 206 | dataset_fill(hsize_t start[], hsize_t block[], DATATYPE *dataset) 207 | { 208 | DATATYPE *dataptr = dataset; 209 | hsize_t i, j; 210 | 211 | /* put some trivial data in the data_array */ 212 | for (i = 0; i < block[0]; i++) { 213 | for (j = 0; j < block[1]; j++) { 214 | *dataptr = (DATATYPE)((i + start[0]) * 100 + (j + start[1] + 1)); 215 | dataptr++; 216 | } 217 | } 218 | } 219 | 220 | int 221 | dataset_vrfy(hsize_t start[], hsize_t count[], hsize_t stride[], hsize_t block[], DATATYPE *dataset, 222 | DATATYPE *original) 223 | { 224 | hsize_t i, j; 225 | int vrfyerrs; 226 | 227 | /* unused */ 228 | H5FD_GDS_UNUSED(count); 229 | H5FD_GDS_UNUSED(stride); 230 | 231 | vrfyerrs = 0; 232 | for (i = 0; i < block[0]; i++) { 233 | for (j = 0; j < block[1]; j++) { 234 | if (*dataset != *original) { 235 | if (vrfyerrs++ < MAX_ERR_REPORT) { 236 | printf("Dataset Verify failed at [%lu][%lu](row %lu, col %lu): expect %d, got %d\n", 237 | (unsigned long)i, (unsigned long)j, (unsigned long)(i + start[0]), 238 | (unsigned long)(j + start[1]), *(original), *(dataset)); 239 | } 240 | dataset++; 241 | original++; 242 | } 243 | } 244 | } 245 | if (vrfyerrs > MAX_ERR_REPORT) 246 | printf("[more errors ...]\n"); 247 | if (vrfyerrs) 248 | printf("%d errors found in dataset_vrfy\n", vrfyerrs); 249 | return (vrfyerrs); 250 | } 251 | 252 | /* 253 | * Example of using the parallel HDF5 library to create two datasets 254 | * in one HDF5 files with parallel MPIO access support. 255 | * The Datasets are of sizes (number-of-mpi-processes x DIM0) x DIM1. 256 | * Each process controls only a slab of size DIM0 x DIM1 within each 257 | * dataset. 258 | */ 259 | 260 | static void 261 | extend_writeInd_cuda(void) 262 | { 263 | hid_t fid; /* HDF5 file ID */ 264 | hid_t acc_tpl; /* File access templates */ 265 | hid_t sid; /* Dataspace ID */ 266 | hid_t file_dataspace; /* File dataspace ID */ 267 | hid_t mem_dataspace; /* memory dataspace ID */ 268 | hid_t dataset1, dataset2; /* Dataset ID */ 269 | hsize_t dims[RANK]; /* dataset dim sizes */ 270 | hsize_t max_dims[RANK] = {H5S_UNLIMITED, H5S_UNLIMITED}; /* dataset maximum dim sizes */ 271 | DATATYPE * data_array1 = NULL; /* data buffer */ 272 | hsize_t chunk_dims[RANK]; /* chunk sizes */ 273 | hid_t dataset_pl; /* dataset create prop. list */ 274 | 275 | hsize_t start[RANK]; /* for hyperslab setting */ 276 | hsize_t count[RANK]; /* for hyperslab setting */ 277 | hsize_t stride[RANK]; /* for hyperslab setting */ 278 | hsize_t block[RANK]; /* for hyperslab setting */ 279 | DATATYPE *cuda_buff = NULL; /* data buffer */ 280 | 281 | herr_t ret; /* Generic return value */ 282 | 283 | printf("Extend independent write test on file %s\n", FILENAME); 284 | 285 | /* setup chunk-size. Make sure sizes are > 0 */ 286 | chunk_dims[0] = CHUNK_DIM0; 287 | chunk_dims[1] = CHUNK_DIM1; 288 | 289 | /* allocate memory for data buffer */ 290 | data_array1 = (DATATYPE *)malloc(DIM0 * DIM1 * sizeof(DATATYPE)); 291 | VRFY((data_array1 != NULL), "data_array1 malloc succeeded"); 292 | 293 | /* ------------------- 294 | * START AN HDF5 FILE 295 | * -------------------*/ 296 | /* setup file access template */ 297 | acc_tpl = H5Pcreate(H5P_FILE_ACCESS); 298 | VRFY((acc_tpl >= 0), "H5Pcreate H5P_FILE_ACCESS"); 299 | ret = H5Pset_all_coll_metadata_ops(acc_tpl, true); 300 | VRFY((ret >= 0), "H5Pset_all_coll_metadata_ops"); 301 | ret = H5Pset_coll_metadata_write(acc_tpl, true); 302 | VRFY((ret >= 0), "H5Pset_coll_metadata_write"); 303 | 304 | /* Set GDS VFD on FAPL */ 305 | ret = H5Pset_driver_by_name(acc_tpl, "gds", NULL); 306 | VRFY((ret >= 0), "H5Pset_driver_by_name"); 307 | 308 | /* Reduce the number of metadata cache slots, so that there are cache 309 | * collisions during the raw data I/O on the chunked dataset. This stresses 310 | * the metadata cache and tests for cache bugs. -QAK 311 | */ 312 | { 313 | int mdc_nelmts; 314 | size_t rdcc_nelmts; 315 | size_t rdcc_nbytes; 316 | double rdcc_w0; 317 | 318 | ret = H5Pget_cache(acc_tpl, &mdc_nelmts, &rdcc_nelmts, &rdcc_nbytes, &rdcc_w0); 319 | VRFY((ret >= 0), "H5Pget_cache succeeded"); 320 | mdc_nelmts = 4; 321 | ret = H5Pset_cache(acc_tpl, mdc_nelmts, rdcc_nelmts, rdcc_nbytes, rdcc_w0); 322 | VRFY((ret >= 0), "H5Pset_cache succeeded"); 323 | } 324 | 325 | /* create the file collectively */ 326 | fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl); 327 | VRFY((fid >= 0), "H5Fcreate succeeded"); 328 | 329 | /* Release file-access template */ 330 | ret = H5Pclose(acc_tpl); 331 | VRFY((ret >= 0), ""); 332 | 333 | /* -------------------------------------------------------------- 334 | * Define the dimensions of the overall datasets and create them. 335 | * ------------------------------------------------------------- */ 336 | 337 | /* set up dataset storage chunk sizes and creation property list */ 338 | dataset_pl = H5Pcreate(H5P_DATASET_CREATE); 339 | VRFY((dataset_pl >= 0), "H5Pcreate succeeded"); 340 | ret = H5Pset_chunk(dataset_pl, RANK, chunk_dims); 341 | VRFY((ret >= 0), "H5Pset_chunk succeeded"); 342 | 343 | /* setup dimensionality object */ 344 | /* start out with no rows, extend it later. */ 345 | dims[0] = dims[1] = 0; 346 | sid = H5Screate_simple(RANK, dims, max_dims); 347 | VRFY((sid >= 0), "H5Screate_simple succeeded"); 348 | 349 | /* create an extendible dataset collectively */ 350 | dataset1 = H5Dcreate2(fid, DATASETNAME1, H5T_NATIVE_INT, sid, H5P_DEFAULT, dataset_pl, H5P_DEFAULT); 351 | VRFY((dataset1 >= 0), "H5Dcreate2 succeeded"); 352 | 353 | /* create another extendible dataset collectively */ 354 | dataset2 = H5Dcreate2(fid, DATASETNAME2, H5T_NATIVE_INT, sid, H5P_DEFAULT, dataset_pl, H5P_DEFAULT); 355 | VRFY((dataset2 >= 0), "H5Dcreate2 succeeded"); 356 | 357 | /* release resource */ 358 | H5Sclose(sid); 359 | H5Pclose(dataset_pl); 360 | 361 | /* ------------------------- 362 | * Test writing to dataset1 363 | * -------------------------*/ 364 | /* set up dimensions of the slab this process accesses */ 365 | slab_set(mpi_rank, mpi_size, start, count, stride, block, BYROW); 366 | 367 | /* put some trivial data in the data_array */ 368 | dataset_fill(start, block, data_array1); 369 | 370 | /* create a memory dataspace independently */ 371 | mem_dataspace = H5Screate_simple(RANK, block, NULL); 372 | VRFY((mem_dataspace >= 0), ""); 373 | 374 | /* Extend its current dim sizes before writing */ 375 | dims[0] = DIM0; 376 | dims[1] = DIM1; 377 | ret = H5Dset_extent(dataset1, dims); 378 | VRFY((ret >= 0), "H5Dset_extent succeeded"); 379 | 380 | /* create a file dataspace independently */ 381 | file_dataspace = H5Dget_space(dataset1); 382 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 383 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 384 | VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); 385 | 386 | cudaMalloc((void **)&cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE)); 387 | cudaMemcpy(cuda_buff, data_array1, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyHostToDevice); 388 | 389 | /* write data independently */ 390 | ret = H5Dwrite(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace, H5P_DEFAULT, cuda_buff); 391 | VRFY((ret >= 0), "H5Dwrite succeeded"); 392 | 393 | /* release resource */ 394 | H5Sclose(file_dataspace); 395 | H5Sclose(mem_dataspace); 396 | 397 | /* ------------------------- 398 | * Test writing to dataset2 399 | * -------------------------*/ 400 | /* set up dimensions of the slab this process accesses */ 401 | slab_set(mpi_rank, mpi_size, start, count, stride, block, BYCOL); 402 | 403 | /* put some trivial data in the data_array */ 404 | dataset_fill(start, block, data_array1); 405 | 406 | /* create a memory dataspace independently */ 407 | mem_dataspace = H5Screate_simple(RANK, block, NULL); 408 | VRFY((mem_dataspace >= 0), ""); 409 | 410 | /* Try write to dataset2 beyond its current dim sizes. Should fail. */ 411 | H5E_BEGIN_TRY 412 | { 413 | /* create a file dataspace independently */ 414 | file_dataspace = H5Dget_space(dataset2); 415 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 416 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 417 | VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); 418 | cudaMemcpy(cuda_buff, data_array1, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyHostToDevice); 419 | 420 | /* write data independently. Should fail. */ 421 | ret = H5Dwrite(dataset2, H5T_NATIVE_INT, mem_dataspace, file_dataspace, H5P_DEFAULT, cuda_buff); 422 | VRFY((ret < 0), "H5Dwrite failed as expected"); 423 | } 424 | H5E_END_TRY; 425 | 426 | H5Sclose(file_dataspace); 427 | 428 | /* Extend dataset2 and try again. Should succeed. */ 429 | dims[0] = DIM0; 430 | dims[1] = DIM1; 431 | ret = H5Dset_extent(dataset2, dims); 432 | VRFY((ret >= 0), "H5Dset_extent succeeded"); 433 | 434 | /* create a file dataspace independently */ 435 | file_dataspace = H5Dget_space(dataset2); 436 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 437 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 438 | VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); 439 | 440 | /* write data independently */ 441 | ret = H5Dwrite(dataset2, H5T_NATIVE_INT, mem_dataspace, file_dataspace, H5P_DEFAULT, cuda_buff); 442 | VRFY((ret >= 0), "H5Dwrite succeeded"); 443 | 444 | /* release resource */ 445 | ret = H5Sclose(file_dataspace); 446 | VRFY((ret >= 0), "H5Sclose succeeded"); 447 | ret = H5Sclose(mem_dataspace); 448 | VRFY((ret >= 0), "H5Sclose succeeded"); 449 | 450 | /* close dataset collectively */ 451 | ret = H5Dclose(dataset1); 452 | VRFY((ret >= 0), "H5Dclose1 succeeded"); 453 | ret = H5Dclose(dataset2); 454 | VRFY((ret >= 0), "H5Dclose2 succeeded"); 455 | 456 | /* close the file collectively */ 457 | H5Fclose(fid); 458 | 459 | /* release data buffers */ 460 | if (data_array1) 461 | free(data_array1); 462 | cudaFree(&cuda_buff); 463 | } 464 | 465 | static void 466 | dataset_writeAll_cuda(void) 467 | { 468 | hid_t fid; /* HDF5 file ID */ 469 | hid_t acc_tpl; /* File access templates */ 470 | hid_t xfer_plist; /* Dataset transfer properties list */ 471 | hid_t sid; /* Dataspace ID */ 472 | hid_t file_dataspace; /* File dataspace ID */ 473 | hid_t mem_dataspace; /* memory dataspace ID */ 474 | hid_t dataset1, dataset2, dataset3, dataset4; /* Dataset ID */ 475 | hid_t dataset5, dataset6, dataset7; /* Dataset ID */ 476 | hid_t datatype; /* Datatype ID */ 477 | hsize_t dims[RANK]; /* dataset dim sizes */ 478 | DATATYPE * data_array1 = NULL; /* data buffer */ 479 | 480 | DATATYPE *cuda_buff = NULL; /* data buffer */ 481 | 482 | hsize_t start[RANK]; /* for hyperslab setting */ 483 | hsize_t count[RANK], stride[RANK]; /* for hyperslab setting */ 484 | hsize_t block[RANK]; /* for hyperslab setting */ 485 | 486 | size_t num_points; /* for point selection */ 487 | hsize_t *coords = NULL; /* for point selection */ 488 | hsize_t current_dims; /* for point selection */ 489 | 490 | herr_t ret; /* Generic return value */ 491 | 492 | printf("Collective write test on file %s\n", FILENAME); 493 | 494 | /* set up the coords array selection */ 495 | num_points = DIM1; 496 | coords = (hsize_t *)malloc(DIM1 * RANK * sizeof(hsize_t)); 497 | VRFY((coords != NULL), "coords malloc succeeded"); 498 | 499 | /* allocate memory for data buffer */ 500 | data_array1 = (DATATYPE *)malloc(DIM0 * DIM1 * sizeof(DATATYPE)); 501 | VRFY((data_array1 != NULL), "data_array1 malloc succeeded"); 502 | 503 | /* ------------------- 504 | * START AN HDF5 FILE 505 | * -------------------*/ 506 | /* setup file access template */ 507 | acc_tpl = H5Pcreate(H5P_FILE_ACCESS); 508 | VRFY((acc_tpl >= 0), "H5Pcreate H5P_FILE_ACCESS"); 509 | ret = H5Pset_all_coll_metadata_ops(acc_tpl, true); 510 | VRFY((ret >= 0), "H5Pset_all_coll_metadata_ops"); 511 | ret = H5Pset_coll_metadata_write(acc_tpl, true); 512 | VRFY((ret >= 0), "H5Pset_coll_metadata_write"); 513 | 514 | /* Set GDS VFD on FAPL */ 515 | ret = H5Pset_driver_by_name(acc_tpl, "gds", NULL); 516 | VRFY((ret >= 0), "H5Pset_driver_by_name"); 517 | 518 | /* create the file collectively */ 519 | fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl); 520 | VRFY((fid >= 0), "H5Fcreate succeeded"); 521 | 522 | /* Release file-access template */ 523 | ret = H5Pclose(acc_tpl); 524 | VRFY((ret >= 0), ""); 525 | 526 | /* -------------------------- 527 | * Define the dimensions of the overall datasets 528 | * and create the dataset 529 | * ------------------------- */ 530 | /* setup 2-D dimensionality object */ 531 | dims[0] = DIM0; 532 | dims[1] = DIM1; 533 | sid = H5Screate_simple(RANK, dims, NULL); 534 | VRFY((sid >= 0), "H5Screate_simple succeeded"); 535 | 536 | /* create a dataset collectively */ 537 | dataset1 = H5Dcreate2(fid, DATASETNAME1, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 538 | VRFY((dataset1 >= 0), "H5Dcreate2 succeeded"); 539 | 540 | /* create another dataset collectively */ 541 | datatype = H5Tcopy(H5T_NATIVE_INT); 542 | ret = H5Tset_order(datatype, H5T_ORDER_LE); 543 | VRFY((ret >= 0), "H5Tset_order succeeded"); 544 | 545 | dataset2 = H5Dcreate2(fid, DATASETNAME2, datatype, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 546 | VRFY((dataset2 >= 0), "H5Dcreate2 2 succeeded"); 547 | 548 | /* create a third dataset collectively */ 549 | dataset3 = H5Dcreate2(fid, DATASETNAME3, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 550 | VRFY((dataset3 >= 0), "H5Dcreate2 succeeded"); 551 | 552 | dataset5 = H5Dcreate2(fid, DATASETNAME7, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 553 | VRFY((dataset5 >= 0), "H5Dcreate2 succeeded"); 554 | dataset6 = H5Dcreate2(fid, DATASETNAME8, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 555 | VRFY((dataset6 >= 0), "H5Dcreate2 succeeded"); 556 | dataset7 = H5Dcreate2(fid, DATASETNAME9, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 557 | VRFY((dataset7 >= 0), "H5Dcreate2 succeeded"); 558 | 559 | /* release 2-D space ID created */ 560 | H5Sclose(sid); 561 | 562 | /* setup scalar dimensionality object */ 563 | sid = H5Screate(H5S_SCALAR); 564 | VRFY((sid >= 0), "H5Screate succeeded"); 565 | 566 | /* create a fourth dataset collectively */ 567 | dataset4 = H5Dcreate2(fid, DATASETNAME4, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 568 | VRFY((dataset4 >= 0), "H5Dcreate2 succeeded"); 569 | 570 | /* release scalar space ID created */ 571 | H5Sclose(sid); 572 | 573 | /* 574 | * Set up dimensions of the slab this process accesses. 575 | */ 576 | 577 | /* Dataset1: each process takes a block of rows. */ 578 | slab_set(mpi_rank, mpi_size, start, count, stride, block, BYROW); 579 | 580 | /* create a file dataspace independently */ 581 | file_dataspace = H5Dget_space(dataset1); 582 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 583 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 584 | VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); 585 | 586 | /* create a memory dataspace independently */ 587 | mem_dataspace = H5Screate_simple(RANK, block, NULL); 588 | VRFY((mem_dataspace >= 0), ""); 589 | 590 | /* fill the local slab with some trivial data */ 591 | dataset_fill(start, block, data_array1); 592 | 593 | /* set up the collective transfer properties list */ 594 | xfer_plist = H5Pcreate(H5P_DATASET_XFER); 595 | VRFY((xfer_plist >= 0), "H5Pcreate xfer succeeded"); 596 | #ifndef NO_COLLECTIVE_SUPPORT 597 | ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); 598 | VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); 599 | #endif 600 | if (dxfer_coll_type == DXFER_INDEPENDENT_IO) { 601 | ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist, H5FD_MPIO_INDIVIDUAL_IO); 602 | VRFY((ret >= 0), "set independent IO collectively succeeded"); 603 | } 604 | 605 | cudaMalloc((void **)&cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE)); 606 | cudaMemcpy(cuda_buff, data_array1, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyHostToDevice); 607 | 608 | /* write data collectively */ 609 | ret = H5Dwrite(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 610 | VRFY((ret >= 0), "H5Dwrite dataset1 succeeded"); 611 | 612 | /* setup dimensions again to writeAll with zero rows for process 0 */ 613 | printf("writeAll by some with zero row\n"); 614 | slab_set(mpi_rank, mpi_size, start, count, stride, block, ZROW); 615 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 616 | VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); 617 | /* need to make mem_dataspace to match for process 0 */ 618 | if (MAINPROCESS) { 619 | ret = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_SET, start, stride, count, block); 620 | VRFY((ret >= 0), "H5Sset_hyperslab mem_dataspace succeeded"); 621 | } 622 | ret = H5Dwrite(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 623 | VRFY((ret >= 0), "H5Dwrite dataset1 by ZROW succeeded"); 624 | 625 | /* release all temporary handles. */ 626 | /* Could have used them for dataset2 but it is cleaner */ 627 | /* to create them again.*/ 628 | H5Sclose(file_dataspace); 629 | H5Sclose(mem_dataspace); 630 | H5Pclose(xfer_plist); 631 | 632 | /* Dataset2: each process takes a block of columns. */ 633 | slab_set(mpi_rank, mpi_size, start, count, stride, block, BYCOL); 634 | 635 | /* put some trivial data in the data_array */ 636 | dataset_fill(start, block, data_array1); 637 | 638 | /* create a file dataspace independently */ 639 | file_dataspace = H5Dget_space(dataset1); 640 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 641 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 642 | VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); 643 | 644 | /* create a memory dataspace independently */ 645 | mem_dataspace = H5Screate_simple(RANK, block, NULL); 646 | VRFY((mem_dataspace >= 0), ""); 647 | 648 | /* fill the local slab with some trivial data */ 649 | dataset_fill(start, block, data_array1); 650 | 651 | /* set up the collective transfer properties list */ 652 | xfer_plist = H5Pcreate(H5P_DATASET_XFER); 653 | VRFY((xfer_plist >= 0), ""); 654 | #ifndef NO_COLLECTIVE_SUPPORT 655 | ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); 656 | VRFY((ret >= 0), "H5Pcreate xfer succeeded"); 657 | #endif 658 | if (dxfer_coll_type == DXFER_INDEPENDENT_IO) { 659 | ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist, H5FD_MPIO_INDIVIDUAL_IO); 660 | VRFY((ret >= 0), "set independent IO collectively succeeded"); 661 | } 662 | 663 | /* Raafat: Collective write: prob 1 664 | when using cuda_buff instead of data_array1: ret < 0 665 | */ 666 | cudaMemcpy(cuda_buff, data_array1, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyHostToDevice); 667 | /* write data independently */ 668 | ret = H5Dwrite(dataset2, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 669 | VRFY((ret >= 0), "H5Dwrite dataset2 succeeded"); 670 | 671 | /* setup dimensions again to writeAll with zero columns for process 0 */ 672 | printf("writeAll by some with zero col\n"); 673 | slab_set(mpi_rank, mpi_size, start, count, stride, block, ZCOL); 674 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 675 | VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); 676 | /* need to make mem_dataspace to match for process 0 */ 677 | if (MAINPROCESS) { 678 | ret = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_SET, start, stride, count, block); 679 | VRFY((ret >= 0), "H5Sset_hyperslab mem_dataspace succeeded"); 680 | } 681 | /* Raafat: Collective write: prob 2 682 | when using cuda_buff instead of data_array1: ret < 0 683 | */ 684 | ret = H5Dwrite(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 685 | VRFY((ret >= 0), "H5Dwrite dataset1 by ZCOL succeeded"); 686 | 687 | /* release all temporary handles. */ 688 | /* Could have used them for dataset3 but it is cleaner */ 689 | /* to create them again.*/ 690 | H5Sclose(file_dataspace); 691 | H5Sclose(mem_dataspace); 692 | H5Pclose(xfer_plist); 693 | 694 | /* Dataset3: each process takes a block of rows, except process zero uses "none" selection. */ 695 | slab_set(mpi_rank, mpi_size, start, count, stride, block, BYROW); 696 | 697 | /* create a file dataspace independently */ 698 | file_dataspace = H5Dget_space(dataset3); 699 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 700 | if (MAINPROCESS) { 701 | ret = H5Sselect_none(file_dataspace); 702 | VRFY((ret >= 0), "H5Sselect_none file_dataspace succeeded"); 703 | } /* end if */ 704 | else { 705 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 706 | VRFY((ret >= 0), "H5Sselect_hyperslab succeeded"); 707 | } /* end else */ 708 | 709 | /* create a memory dataspace independently */ 710 | mem_dataspace = H5Screate_simple(RANK, block, NULL); 711 | VRFY((mem_dataspace >= 0), ""); 712 | if (MAINPROCESS) { 713 | ret = H5Sselect_none(mem_dataspace); 714 | VRFY((ret >= 0), "H5Sselect_none mem_dataspace succeeded"); 715 | } /* end if */ 716 | 717 | /* fill the local slab with some trivial data */ 718 | dataset_fill(start, block, data_array1); 719 | 720 | /* set up the collective transfer properties list */ 721 | xfer_plist = H5Pcreate(H5P_DATASET_XFER); 722 | VRFY((xfer_plist >= 0), ""); 723 | #ifndef NO_COLLECTIVE_SUPPORT 724 | ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); 725 | VRFY((ret >= 0), "H5Pcreate xfer succeeded"); 726 | #endif 727 | if (dxfer_coll_type == DXFER_INDEPENDENT_IO) { 728 | ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist, H5FD_MPIO_INDIVIDUAL_IO); 729 | VRFY((ret >= 0), "set independent IO collectively succeeded"); 730 | } 731 | 732 | cudaMemcpy(cuda_buff, data_array1, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyHostToDevice); 733 | 734 | /* write data collectively */ 735 | ret = H5Dwrite(dataset3, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 736 | VRFY((ret >= 0), "H5Dwrite dataset3 succeeded"); 737 | 738 | /* write data collectively (with datatype conversion) */ 739 | /* Raafat: Collective write: prob 3 740 | when using cuda_buff instead of data_array1: Segmentation fault 741 | change H5T_NATIVE_UCHAR to H5T_NATIVE_INT make it work 742 | */ 743 | ret = H5Dwrite(dataset3, H5T_NATIVE_UCHAR, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 744 | VRFY((ret >= 0), "H5Dwrite dataset3 succeeded"); 745 | 746 | /* release all temporary handles. */ 747 | /* Could have used them for dataset4 but it is cleaner */ 748 | /* to create them again.*/ 749 | H5Sclose(file_dataspace); 750 | H5Sclose(mem_dataspace); 751 | H5Pclose(xfer_plist); 752 | 753 | /* Dataset4: each process writes no data, except process zero uses "all" selection. */ 754 | /* Additionally, these are in a scalar dataspace */ 755 | 756 | /* create a file dataspace independently */ 757 | file_dataspace = H5Dget_space(dataset4); 758 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 759 | if (MAINPROCESS) { 760 | ret = H5Sselect_none(file_dataspace); 761 | VRFY((ret >= 0), "H5Sselect_all file_dataspace succeeded"); 762 | } /* end if */ 763 | else { 764 | ret = H5Sselect_all(file_dataspace); 765 | VRFY((ret >= 0), "H5Sselect_none succeeded"); 766 | } /* end else */ 767 | 768 | /* create a memory dataspace independently */ 769 | mem_dataspace = H5Screate(H5S_SCALAR); 770 | VRFY((mem_dataspace >= 0), ""); 771 | if (MAINPROCESS) { 772 | ret = H5Sselect_none(mem_dataspace); 773 | VRFY((ret >= 0), "H5Sselect_all mem_dataspace succeeded"); 774 | } /* end if */ 775 | else { 776 | ret = H5Sselect_all(mem_dataspace); 777 | VRFY((ret >= 0), "H5Sselect_none succeeded"); 778 | } /* end else */ 779 | 780 | /* fill the local slab with some trivial data */ 781 | dataset_fill(start, block, data_array1); 782 | 783 | /* set up the collective transfer properties list */ 784 | xfer_plist = H5Pcreate(H5P_DATASET_XFER); 785 | VRFY((xfer_plist >= 0), ""); 786 | #ifndef NO_COLLECTIVE_SUPPORT 787 | ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); 788 | VRFY((ret >= 0), "H5Pcreate xfer succeeded"); 789 | #endif 790 | if (dxfer_coll_type == DXFER_INDEPENDENT_IO) { 791 | ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist, H5FD_MPIO_INDIVIDUAL_IO); 792 | VRFY((ret >= 0), "set independent IO collectively succeeded"); 793 | } 794 | 795 | cudaMemcpy(cuda_buff, data_array1, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyHostToDevice); 796 | 797 | /* write data collectively */ 798 | ret = H5Dwrite(dataset4, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 799 | VRFY((ret >= 0), "H5Dwrite dataset4 succeeded"); 800 | 801 | /* Raafat: Collective write: prob 4 802 | when using cuda_buff instead of data_array1: Segmentation fault 803 | change H5T_NATIVE_UCHAR to H5T_NATIVE_INT make it work 804 | */ 805 | /* write data collectively (with datatype conversion) */ 806 | ret = H5Dwrite(dataset4, H5T_NATIVE_UCHAR, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 807 | VRFY((ret >= 0), "H5Dwrite dataset4 succeeded"); 808 | 809 | /* release all temporary handles. */ 810 | H5Sclose(file_dataspace); 811 | H5Sclose(mem_dataspace); 812 | H5Pclose(xfer_plist); 813 | 814 | if (data_array1) 815 | free(data_array1); 816 | data_array1 = (DATATYPE *)malloc(DIM0 * DIM1 * sizeof(DATATYPE)); 817 | VRFY((data_array1 != NULL), "data_array1 malloc succeeded"); 818 | 819 | cudaFree(&cuda_buff); 820 | cudaMalloc((void **)&cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE)); 821 | 822 | block[0] = 1; 823 | block[1] = DIM1; 824 | stride[0] = 1; 825 | stride[1] = DIM1; 826 | count[0] = 1; 827 | count[1] = 1; 828 | start[0] = DIM0 / mpi_size * mpi_rank; 829 | start[1] = 0; 830 | 831 | dataset_fill(start, block, data_array1); 832 | 833 | /* Dataset5: point selection in File - Hyperslab selection in Memory*/ 834 | /* create a file dataspace independently */ 835 | point_set(start, count, stride, block, num_points, coords, OUT_OF_ORDER); 836 | file_dataspace = H5Dget_space(dataset5); 837 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 838 | ret = H5Sselect_elements(file_dataspace, H5S_SELECT_SET, num_points, coords); 839 | VRFY((ret >= 0), "H5Sselect_elements succeeded"); 840 | 841 | start[0] = 0; 842 | start[1] = 0; 843 | mem_dataspace = H5Dget_space(dataset5); 844 | VRFY((mem_dataspace >= 0), "H5Dget_space succeeded"); 845 | ret = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_SET, start, stride, count, block); 846 | VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); 847 | 848 | /* set up the collective transfer properties list */ 849 | xfer_plist = H5Pcreate(H5P_DATASET_XFER); 850 | VRFY((xfer_plist >= 0), ""); 851 | #ifndef NO_COLLECTIVE_SUPPORT 852 | ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); 853 | VRFY((ret >= 0), "H5Pcreate xfer succeeded"); 854 | #endif 855 | if (dxfer_coll_type == DXFER_INDEPENDENT_IO) { 856 | ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist, H5FD_MPIO_INDIVIDUAL_IO); 857 | VRFY((ret >= 0), "set independent IO collectively succeeded"); 858 | } 859 | 860 | cudaMemcpy(cuda_buff, data_array1, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyHostToDevice); 861 | 862 | /* Raafat: Collective write: prob 5 863 | when using cuda_buff instead of data_array1: ret < 0 864 | */ 865 | /* write data collectively */ 866 | ret = H5Dwrite(dataset5, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 867 | VRFY((ret >= 0), "H5Dwrite dataset5 succeeded"); 868 | 869 | /* release all temporary handles. */ 870 | H5Sclose(file_dataspace); 871 | H5Sclose(mem_dataspace); 872 | H5Pclose(xfer_plist); 873 | 874 | /* Dataset6: point selection in File - Point selection in Memory*/ 875 | /* create a file dataspace independently */ 876 | start[0] = DIM0 / mpi_size * mpi_rank; 877 | start[1] = 0; 878 | point_set(start, count, stride, block, num_points, coords, OUT_OF_ORDER); 879 | file_dataspace = H5Dget_space(dataset6); 880 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 881 | ret = H5Sselect_elements(file_dataspace, H5S_SELECT_SET, num_points, coords); 882 | VRFY((ret >= 0), "H5Sselect_elements succeeded"); 883 | 884 | start[0] = 0; 885 | start[1] = 0; 886 | point_set(start, count, stride, block, num_points, coords, IN_ORDER); 887 | mem_dataspace = H5Dget_space(dataset6); 888 | VRFY((mem_dataspace >= 0), "H5Dget_space succeeded"); 889 | ret = H5Sselect_elements(mem_dataspace, H5S_SELECT_SET, num_points, coords); 890 | VRFY((ret >= 0), "H5Sselect_elements succeeded"); 891 | 892 | /* set up the collective transfer properties list */ 893 | xfer_plist = H5Pcreate(H5P_DATASET_XFER); 894 | VRFY((xfer_plist >= 0), ""); 895 | #ifndef NO_COLLECTIVE_SUPPORT 896 | ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); 897 | VRFY((ret >= 0), "H5Pcreate xfer succeeded"); 898 | #endif 899 | if (dxfer_coll_type == DXFER_INDEPENDENT_IO) { 900 | ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist, H5FD_MPIO_INDIVIDUAL_IO); 901 | VRFY((ret >= 0), "set independent IO collectively succeeded"); 902 | } 903 | 904 | /* Raafat: Collective write: prob 6 905 | when using cuda_buff instead of data_array1: ret < 0 906 | */ 907 | /* write data collectively */ 908 | ret = H5Dwrite(dataset6, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 909 | VRFY((ret >= 0), "H5Dwrite dataset6 succeeded"); 910 | 911 | /* release all temporary handles. */ 912 | H5Sclose(file_dataspace); 913 | H5Sclose(mem_dataspace); 914 | H5Pclose(xfer_plist); 915 | 916 | /* Dataset7: point selection in File - All selection in Memory*/ 917 | /* create a file dataspace independently */ 918 | start[0] = DIM0 / mpi_size * mpi_rank; 919 | start[1] = 0; 920 | point_set(start, count, stride, block, num_points, coords, IN_ORDER); 921 | file_dataspace = H5Dget_space(dataset7); 922 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 923 | ret = H5Sselect_elements(file_dataspace, H5S_SELECT_SET, num_points, coords); 924 | VRFY((ret >= 0), "H5Sselect_elements succeeded"); 925 | 926 | current_dims = num_points; 927 | mem_dataspace = H5Screate_simple(1, ¤t_dims, NULL); 928 | VRFY((mem_dataspace >= 0), "mem_dataspace create succeeded"); 929 | 930 | ret = H5Sselect_all(mem_dataspace); 931 | VRFY((ret >= 0), "H5Sselect_all succeeded"); 932 | 933 | /* set up the collective transfer properties list */ 934 | xfer_plist = H5Pcreate(H5P_DATASET_XFER); 935 | VRFY((xfer_plist >= 0), ""); 936 | #ifndef NO_COLLECTIVE_SUPPORT 937 | ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); 938 | VRFY((ret >= 0), "H5Pcreate xfer succeeded"); 939 | #endif 940 | if (dxfer_coll_type == DXFER_INDEPENDENT_IO) { 941 | ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist, H5FD_MPIO_INDIVIDUAL_IO); 942 | VRFY((ret >= 0), "set independent IO collectively succeeded"); 943 | } 944 | 945 | /* write data collectively */ 946 | ret = H5Dwrite(dataset7, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 947 | VRFY((ret >= 0), "H5Dwrite dataset7 succeeded"); 948 | 949 | /* release all temporary handles. */ 950 | H5Sclose(file_dataspace); 951 | H5Sclose(mem_dataspace); 952 | H5Pclose(xfer_plist); 953 | 954 | /* 955 | * All writes completed. Close datasets collectively 956 | */ 957 | ret = H5Dclose(dataset1); 958 | VRFY((ret >= 0), "H5Dclose1 succeeded"); 959 | ret = H5Dclose(dataset2); 960 | VRFY((ret >= 0), "H5Dclose2 succeeded"); 961 | ret = H5Dclose(dataset3); 962 | VRFY((ret >= 0), "H5Dclose3 succeeded"); 963 | ret = H5Dclose(dataset4); 964 | VRFY((ret >= 0), "H5Dclose4 succeeded"); 965 | ret = H5Dclose(dataset5); 966 | VRFY((ret >= 0), "H5Dclose5 succeeded"); 967 | ret = H5Dclose(dataset6); 968 | VRFY((ret >= 0), "H5Dclose6 succeeded"); 969 | ret = H5Dclose(dataset7); 970 | VRFY((ret >= 0), "H5Dclose7 succeeded"); 971 | 972 | /* close the file collectively */ 973 | H5Fclose(fid); 974 | 975 | /* release data buffers */ 976 | if (coords) 977 | free(coords); 978 | if (data_array1) 979 | free(data_array1); 980 | cudaFree(&cuda_buff); 981 | } 982 | 983 | static void 984 | dataset_readAll_cuda(void) 985 | { 986 | hid_t fid; /* HDF5 file ID */ 987 | hid_t acc_tpl; /* File access templates */ 988 | hid_t xfer_plist; /* Dataset transfer properties list */ 989 | hid_t file_dataspace; /* File dataspace ID */ 990 | hid_t mem_dataspace; /* memory dataspace ID */ 991 | hid_t dataset1, dataset2, dataset5, dataset6, dataset7; /* Dataset ID */ 992 | DATATYPE * data_array1 = NULL; /* data buffer */ 993 | DATATYPE * data_origin1 = NULL; /* expected data buffer */ 994 | DATATYPE * cuda_buff = NULL; /* data buffer */ 995 | 996 | hsize_t start[RANK]; /* for hyperslab setting */ 997 | hsize_t count[RANK], stride[RANK]; /* for hyperslab setting */ 998 | hsize_t block[RANK]; /* for hyperslab setting */ 999 | 1000 | size_t num_points; /* for point selection */ 1001 | hsize_t *coords = NULL; /* for point selection */ 1002 | int i, j, k; 1003 | 1004 | herr_t ret; /* Generic return value */ 1005 | 1006 | printf("Collective read test on file %s\n", FILENAME); 1007 | 1008 | /* set up the coords array selection */ 1009 | num_points = DIM1; 1010 | coords = (hsize_t *)malloc(DIM0 * DIM1 * RANK * sizeof(hsize_t)); 1011 | VRFY((coords != NULL), "coords malloc succeeded"); 1012 | 1013 | /* allocate memory for data buffer */ 1014 | data_array1 = (DATATYPE *)malloc(DIM0 * DIM1 * sizeof(DATATYPE)); 1015 | VRFY((data_array1 != NULL), "data_array1 malloc succeeded"); 1016 | data_origin1 = (DATATYPE *)malloc(DIM0 * DIM1 * sizeof(DATATYPE)); 1017 | VRFY((data_origin1 != NULL), "data_origin1 malloc succeeded"); 1018 | 1019 | /* ------------------- 1020 | * OPEN AN HDF5 FILE 1021 | * -------------------*/ 1022 | /* setup file access template */ 1023 | acc_tpl = H5Pcreate(H5P_FILE_ACCESS); 1024 | VRFY((acc_tpl >= 0), "H5Pcreate H5P_FILE_ACCESS"); 1025 | ret = H5Pset_all_coll_metadata_ops(acc_tpl, true); 1026 | VRFY((ret >= 0), "H5Pset_all_coll_metadata_ops"); 1027 | ret = H5Pset_coll_metadata_write(acc_tpl, true); 1028 | VRFY((ret >= 0), "H5Pset_coll_metadata_write"); 1029 | 1030 | /* Set GDS VFD on FAPL */ 1031 | ret = H5Pset_driver_by_name(acc_tpl, "gds", NULL); 1032 | VRFY((ret >= 0), "H5Pset_driver_by_name"); 1033 | 1034 | /* open the file collectively */ 1035 | fid = H5Fopen(FILENAME, H5F_ACC_RDONLY, acc_tpl); 1036 | VRFY((fid >= 0), "H5Fopen succeeded"); 1037 | 1038 | /* Release file-access template */ 1039 | ret = H5Pclose(acc_tpl); 1040 | VRFY((ret >= 0), ""); 1041 | 1042 | /* -------------------------- 1043 | * Open the datasets in it 1044 | * ------------------------- */ 1045 | /* open the dataset1 collectively */ 1046 | dataset1 = H5Dopen2(fid, DATASETNAME1, H5P_DEFAULT); 1047 | VRFY((dataset1 >= 0), "H5Dopen2 succeeded"); 1048 | 1049 | /* open another dataset collectively */ 1050 | dataset2 = H5Dopen2(fid, DATASETNAME2, H5P_DEFAULT); 1051 | VRFY((dataset2 >= 0), "H5Dopen2 2 succeeded"); 1052 | 1053 | /* open another dataset collectively */ 1054 | dataset5 = H5Dopen2(fid, DATASETNAME7, H5P_DEFAULT); 1055 | VRFY((dataset5 >= 0), "H5Dopen2 5 succeeded"); 1056 | dataset6 = H5Dopen2(fid, DATASETNAME8, H5P_DEFAULT); 1057 | VRFY((dataset6 >= 0), "H5Dopen2 6 succeeded"); 1058 | dataset7 = H5Dopen2(fid, DATASETNAME9, H5P_DEFAULT); 1059 | VRFY((dataset7 >= 0), "H5Dopen2 7 succeeded"); 1060 | 1061 | /* 1062 | * Set up dimensions of the slab this process accesses. 1063 | */ 1064 | 1065 | /* Dataset1: each process takes a block of columns. */ 1066 | slab_set(mpi_rank, mpi_size, start, count, stride, block, BYCOL); 1067 | 1068 | /* create a file dataspace independently */ 1069 | file_dataspace = H5Dget_space(dataset1); 1070 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 1071 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 1072 | VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); 1073 | 1074 | /* create a memory dataspace independently */ 1075 | mem_dataspace = H5Screate_simple(RANK, block, NULL); 1076 | VRFY((mem_dataspace >= 0), ""); 1077 | 1078 | /* fill dataset with test data */ 1079 | dataset_fill(start, block, data_origin1); 1080 | 1081 | /* set up the collective transfer properties list */ 1082 | xfer_plist = H5Pcreate(H5P_DATASET_XFER); 1083 | VRFY((xfer_plist >= 0), ""); 1084 | #ifndef NO_COLLECTIVE_SUPPORT 1085 | ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); 1086 | VRFY((ret >= 0), "H5Pcreate xfer succeeded"); 1087 | #endif 1088 | if (dxfer_coll_type == DXFER_INDEPENDENT_IO) { 1089 | ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist, H5FD_MPIO_INDIVIDUAL_IO); 1090 | VRFY((ret >= 0), "set independent IO collectively succeeded"); 1091 | } 1092 | 1093 | cudaMalloc((void **)&cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE)); 1094 | 1095 | /* Raafat: Collective read prob 1 1096 | read data collectively 1097 | */ 1098 | ret = H5Dread(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 1099 | VRFY((ret >= 0), "H5Dread dataset1 succeeded"); 1100 | 1101 | cudaMemcpy(data_array1, cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyDeviceToHost); 1102 | /* verify the read data with original expected data */ 1103 | ret = dataset_vrfy(start, count, stride, block, data_array1, data_origin1); 1104 | if (ret) 1105 | nerrors++; 1106 | 1107 | /* setup dimensions again to readAll with zero columns for process 0 */ 1108 | printf("readAll by some with zero col\n"); 1109 | slab_set(mpi_rank, mpi_size, start, count, stride, block, ZCOL); 1110 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 1111 | VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); 1112 | /* need to make mem_dataspace to match for process 0 */ 1113 | if (MAINPROCESS) { 1114 | ret = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_SET, start, stride, count, block); 1115 | VRFY((ret >= 0), "H5Sset_hyperslab mem_dataspace succeeded"); 1116 | } 1117 | 1118 | /* Raafat: Collective read prob 2 1119 | */ 1120 | ret = H5Dread(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 1121 | VRFY((ret >= 0), "H5Dread dataset1 by ZCOL succeeded"); 1122 | 1123 | cudaMemcpy(data_array1, cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyDeviceToHost); 1124 | /* verify the read data with original expected data */ 1125 | ret = dataset_vrfy(start, count, stride, block, data_array1, data_origin1); 1126 | if (ret) 1127 | nerrors++; 1128 | 1129 | /* release all temporary handles. */ 1130 | /* Could have used them for dataset2 but it is cleaner */ 1131 | /* to create them again.*/ 1132 | H5Sclose(file_dataspace); 1133 | H5Sclose(mem_dataspace); 1134 | H5Pclose(xfer_plist); 1135 | 1136 | /* Dataset2: each process takes a block of rows. */ 1137 | slab_set(mpi_rank, mpi_size, start, count, stride, block, BYROW); 1138 | 1139 | /* create a file dataspace independently */ 1140 | file_dataspace = H5Dget_space(dataset1); 1141 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 1142 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 1143 | VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); 1144 | 1145 | /* create a memory dataspace independently */ 1146 | mem_dataspace = H5Screate_simple(RANK, block, NULL); 1147 | VRFY((mem_dataspace >= 0), ""); 1148 | 1149 | /* fill dataset with test data */ 1150 | dataset_fill(start, block, data_origin1); 1151 | 1152 | /* set up the collective transfer properties list */ 1153 | xfer_plist = H5Pcreate(H5P_DATASET_XFER); 1154 | VRFY((xfer_plist >= 0), ""); 1155 | #ifndef NO_COLLECTIVE_SUPPORT 1156 | ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); 1157 | VRFY((ret >= 0), "H5Pcreate xfer succeeded"); 1158 | #endif 1159 | if (dxfer_coll_type == DXFER_INDEPENDENT_IO) { 1160 | ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist, H5FD_MPIO_INDIVIDUAL_IO); 1161 | VRFY((ret >= 0), "set independent IO collectively succeeded"); 1162 | } 1163 | 1164 | /* read data collectively */ 1165 | ret = H5Dread(dataset2, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 1166 | VRFY((ret >= 0), "H5Dread dataset2 succeeded"); 1167 | 1168 | cudaMemcpy(data_array1, cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyDeviceToHost); 1169 | /* verify the read data with original expected data */ 1170 | ret = dataset_vrfy(start, count, stride, block, data_array1, data_origin1); 1171 | if (ret) 1172 | nerrors++; 1173 | 1174 | /* setup dimensions again to readAll with zero rows for process 0 */ 1175 | printf("readAll by some with zero row\n"); 1176 | slab_set(mpi_rank, mpi_size, start, count, stride, block, ZROW); 1177 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 1178 | VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); 1179 | /* need to make mem_dataspace to match for process 0 */ 1180 | if (MAINPROCESS) { 1181 | ret = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_SET, start, stride, count, block); 1182 | VRFY((ret >= 0), "H5Sset_hyperslab mem_dataspace succeeded"); 1183 | } 1184 | ret = H5Dread(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 1185 | VRFY((ret >= 0), "H5Dread dataset1 by ZROW succeeded"); 1186 | 1187 | cudaMemcpy(data_array1, cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyDeviceToHost); 1188 | /* verify the read data with original expected data */ 1189 | ret = dataset_vrfy(start, count, stride, block, data_array1, data_origin1); 1190 | if (ret) 1191 | nerrors++; 1192 | 1193 | /* release all temporary handles. */ 1194 | H5Sclose(file_dataspace); 1195 | H5Sclose(mem_dataspace); 1196 | H5Pclose(xfer_plist); 1197 | 1198 | if (data_array1) 1199 | free(data_array1); 1200 | if (data_origin1) 1201 | free(data_origin1); 1202 | data_array1 = (DATATYPE *)malloc(DIM0 * DIM1 * sizeof(DATATYPE)); 1203 | VRFY((data_array1 != NULL), "data_array1 malloc succeeded"); 1204 | data_origin1 = (DATATYPE *)malloc(DIM0 * DIM1 * sizeof(DATATYPE)); 1205 | VRFY((data_origin1 != NULL), "data_origin1 malloc succeeded"); 1206 | 1207 | if (NULL != cuda_buff) 1208 | cudaFree(&cuda_buff); 1209 | cudaMalloc((void **)&cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE)); 1210 | 1211 | block[0] = 1; 1212 | block[1] = DIM1; 1213 | stride[0] = 1; 1214 | stride[1] = DIM1; 1215 | count[0] = 1; 1216 | count[1] = 1; 1217 | start[0] = DIM0 / mpi_size * mpi_rank; 1218 | start[1] = 0; 1219 | 1220 | dataset_fill(start, block, data_origin1); 1221 | 1222 | /* Dataset5: point selection in memory - Hyperslab selection in file*/ 1223 | /* create a file dataspace independently */ 1224 | file_dataspace = H5Dget_space(dataset5); 1225 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 1226 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 1227 | VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); 1228 | 1229 | start[0] = 0; 1230 | start[1] = 0; 1231 | point_set(start, count, stride, block, num_points, coords, OUT_OF_ORDER); 1232 | mem_dataspace = H5Dget_space(dataset5); 1233 | VRFY((mem_dataspace >= 0), "H5Dget_space succeeded"); 1234 | ret = H5Sselect_elements(mem_dataspace, H5S_SELECT_SET, num_points, coords); 1235 | VRFY((ret >= 0), "H5Sselect_elements succeeded"); 1236 | 1237 | /* set up the collective transfer properties list */ 1238 | xfer_plist = H5Pcreate(H5P_DATASET_XFER); 1239 | VRFY((xfer_plist >= 0), ""); 1240 | #ifndef NO_COLLECTIVE_SUPPORT 1241 | ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); 1242 | VRFY((ret >= 0), "H5Pcreate xfer succeeded"); 1243 | #endif 1244 | if (dxfer_coll_type == DXFER_INDEPENDENT_IO) { 1245 | ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist, H5FD_MPIO_INDIVIDUAL_IO); 1246 | VRFY((ret >= 0), "set independent IO collectively succeeded"); 1247 | } 1248 | 1249 | /* Raafat: Collective read prob 3 1250 | */ 1251 | /* read data collectively */ 1252 | ret = H5Dread(dataset5, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 1253 | VRFY((ret >= 0), "H5Dread dataset5 succeeded"); 1254 | 1255 | cudaMemcpy(data_array1, cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyDeviceToHost); 1256 | ret = dataset_vrfy(start, count, stride, block, data_array1, data_origin1); 1257 | if (ret) 1258 | nerrors++; 1259 | 1260 | /* release all temporary handles. */ 1261 | H5Sclose(file_dataspace); 1262 | H5Sclose(mem_dataspace); 1263 | H5Pclose(xfer_plist); 1264 | 1265 | if (data_array1) 1266 | free(data_array1); 1267 | data_array1 = (DATATYPE *)malloc(DIM0 * DIM1 * sizeof(DATATYPE)); 1268 | VRFY((data_array1 != NULL), "data_array1 malloc succeeded"); 1269 | 1270 | if (NULL != cuda_buff) 1271 | cudaFree(&cuda_buff); 1272 | cudaMalloc((void **)&cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE)); 1273 | 1274 | /* Dataset6: point selection in File - Point selection in Memory*/ 1275 | /* create a file dataspace independently */ 1276 | start[0] = DIM0 / mpi_size * mpi_rank; 1277 | start[1] = 0; 1278 | point_set(start, count, stride, block, num_points, coords, IN_ORDER); 1279 | file_dataspace = H5Dget_space(dataset6); 1280 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 1281 | ret = H5Sselect_elements(file_dataspace, H5S_SELECT_SET, num_points, coords); 1282 | VRFY((ret >= 0), "H5Sselect_elements succeeded"); 1283 | 1284 | start[0] = 0; 1285 | start[1] = 0; 1286 | point_set(start, count, stride, block, num_points, coords, OUT_OF_ORDER); 1287 | mem_dataspace = H5Dget_space(dataset6); 1288 | VRFY((mem_dataspace >= 0), "H5Dget_space succeeded"); 1289 | ret = H5Sselect_elements(mem_dataspace, H5S_SELECT_SET, num_points, coords); 1290 | VRFY((ret >= 0), "H5Sselect_elements succeeded"); 1291 | 1292 | /* set up the collective transfer properties list */ 1293 | xfer_plist = H5Pcreate(H5P_DATASET_XFER); 1294 | VRFY((xfer_plist >= 0), ""); 1295 | #ifndef NO_COLLECTIVE_SUPPORT 1296 | ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); 1297 | VRFY((ret >= 0), "H5Pcreate xfer succeeded"); 1298 | #endif 1299 | if (dxfer_coll_type == DXFER_INDEPENDENT_IO) { 1300 | ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist, H5FD_MPIO_INDIVIDUAL_IO); 1301 | VRFY((ret >= 0), "set independent IO collectively succeeded"); 1302 | } 1303 | 1304 | /* Raafat: Collective read prob 4 1305 | */ 1306 | /* read data collectively */ 1307 | ret = H5Dread(dataset6, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 1308 | VRFY((ret >= 0), "H5Dread dataset6 succeeded"); 1309 | 1310 | cudaMemcpy(data_array1, cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyDeviceToHost); 1311 | ret = dataset_vrfy(start, count, stride, block, data_array1, data_origin1); 1312 | if (ret) 1313 | nerrors++; 1314 | 1315 | /* release all temporary handles. */ 1316 | H5Sclose(file_dataspace); 1317 | H5Sclose(mem_dataspace); 1318 | H5Pclose(xfer_plist); 1319 | 1320 | if (data_array1) 1321 | free(data_array1); 1322 | data_array1 = (DATATYPE *)malloc(DIM0 * DIM1 * sizeof(DATATYPE)); 1323 | VRFY((data_array1 != NULL), "data_array1 malloc succeeded"); 1324 | 1325 | if (NULL != cuda_buff) 1326 | cudaFree(&cuda_buff); 1327 | cudaMalloc((void **)&cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE)); 1328 | 1329 | /* Dataset7: point selection in memory - All selection in file*/ 1330 | /* create a file dataspace independently */ 1331 | file_dataspace = H5Dget_space(dataset7); 1332 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 1333 | ret = H5Sselect_all(file_dataspace); 1334 | VRFY((ret >= 0), "H5Sselect_all succeeded"); 1335 | 1336 | num_points = DIM0 * DIM1; 1337 | k = 0; 1338 | for (i = 0; i < DIM0; i++) { 1339 | for (j = 0; j < DIM1; j++) { 1340 | coords[k++] = i; 1341 | coords[k++] = j; 1342 | } 1343 | } 1344 | mem_dataspace = H5Dget_space(dataset7); 1345 | VRFY((mem_dataspace >= 0), "H5Dget_space succeeded"); 1346 | ret = H5Sselect_elements(mem_dataspace, H5S_SELECT_SET, num_points, coords); 1347 | VRFY((ret >= 0), "H5Sselect_elements succeeded"); 1348 | 1349 | /* set up the collective transfer properties list */ 1350 | xfer_plist = H5Pcreate(H5P_DATASET_XFER); 1351 | VRFY((xfer_plist >= 0), ""); 1352 | #ifndef NO_COLLECTIVE_SUPPORT 1353 | ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); 1354 | VRFY((ret >= 0), "H5Pcreate xfer succeeded"); 1355 | #endif 1356 | if (dxfer_coll_type == DXFER_INDEPENDENT_IO) { 1357 | ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist, H5FD_MPIO_INDIVIDUAL_IO); 1358 | VRFY((ret >= 0), "set independent IO collectively succeeded"); 1359 | } 1360 | 1361 | /* read data collectively */ 1362 | ret = H5Dread(dataset7, H5T_NATIVE_INT, mem_dataspace, file_dataspace, xfer_plist, cuda_buff); 1363 | VRFY((ret >= 0), "H5Dread dataset7 succeeded"); 1364 | 1365 | cudaMemcpy(data_array1, cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyDeviceToHost); 1366 | 1367 | start[0] = DIM0 / mpi_size * mpi_rank; 1368 | start[1] = 0; 1369 | ret = dataset_vrfy(start, count, stride, block, data_array1 + (DIM0 / mpi_size * DIM1 * mpi_rank), 1370 | data_origin1); 1371 | if (ret) 1372 | nerrors++; 1373 | 1374 | /* release all temporary handles. */ 1375 | H5Sclose(file_dataspace); 1376 | H5Sclose(mem_dataspace); 1377 | H5Pclose(xfer_plist); 1378 | 1379 | /* 1380 | * All reads completed. Close datasets collectively 1381 | */ 1382 | ret = H5Dclose(dataset1); 1383 | VRFY((ret >= 0), "H5Dclose1 succeeded"); 1384 | ret = H5Dclose(dataset2); 1385 | VRFY((ret >= 0), "H5Dclose2 succeeded"); 1386 | ret = H5Dclose(dataset5); 1387 | VRFY((ret >= 0), "H5Dclose5 succeeded"); 1388 | ret = H5Dclose(dataset6); 1389 | VRFY((ret >= 0), "H5Dclose6 succeeded"); 1390 | ret = H5Dclose(dataset7); 1391 | VRFY((ret >= 0), "H5Dclose7 succeeded"); 1392 | 1393 | /* close the file collectively */ 1394 | H5Fclose(fid); 1395 | 1396 | /* release data buffers */ 1397 | if (coords) 1398 | free(coords); 1399 | if (data_array1) 1400 | free(data_array1); 1401 | if (data_origin1) 1402 | free(data_origin1); 1403 | if (NULL != cuda_buff) 1404 | cudaFree(&cuda_buff); 1405 | } 1406 | 1407 | static void 1408 | dataset_writeInd_cuda(void) 1409 | { 1410 | hid_t fid; /* HDF5 file ID */ 1411 | hid_t acc_tpl; /* File access templates */ 1412 | hid_t sid; /* Dataspace ID */ 1413 | hid_t file_dataspace; /* File dataspace ID */ 1414 | hid_t mem_dataspace; /* memory dataspace ID */ 1415 | hid_t dataset1, dataset2; /* Dataset ID */ 1416 | hsize_t dims[RANK]; /* dataset dim sizes */ 1417 | DATATYPE * data_array1 = NULL; /* data buffer */ 1418 | DATATYPE * cuda_buff = NULL; /* data buffer */ 1419 | 1420 | hsize_t start[RANK]; /* for hyperslab setting */ 1421 | hsize_t count[RANK], stride[RANK]; /* for hyperslab setting */ 1422 | hsize_t block[RANK]; /* for hyperslab setting */ 1423 | 1424 | herr_t ret; /* Generic return value */ 1425 | 1426 | printf("Independent write test on file %s\n", FILENAME); 1427 | 1428 | /* allocate memory for data buffer */ 1429 | data_array1 = (DATATYPE *)malloc(DIM0 * DIM1 * sizeof(DATATYPE)); 1430 | VRFY((data_array1 != NULL), "data_array1 malloc succeeded"); 1431 | 1432 | /* ---------------------------------------- 1433 | * CREATE AN HDF5 FILE WITH PARALLEL ACCESS 1434 | * ---------------------------------------*/ 1435 | /* setup file access template */ 1436 | acc_tpl = H5Pcreate(H5P_FILE_ACCESS); 1437 | VRFY((acc_tpl >= 0), "H5Pcreate H5P_FILE_ACCESS"); 1438 | ret = H5Pset_all_coll_metadata_ops(acc_tpl, true); 1439 | VRFY((ret >= 0), "H5Pset_all_coll_metadata_ops"); 1440 | ret = H5Pset_coll_metadata_write(acc_tpl, true); 1441 | VRFY((ret >= 0), "H5Pset_coll_metadata_write"); 1442 | 1443 | /* Set GDS VFD on FAPL */ 1444 | ret = H5Pset_driver_by_name(acc_tpl, "gds", NULL); 1445 | VRFY((ret >= 0), "H5Pset_driver_by_name"); 1446 | 1447 | /* create the file collectively */ 1448 | fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl); 1449 | VRFY((fid >= 0), "H5Fcreate succeeded"); 1450 | 1451 | /* Release file-access template */ 1452 | ret = H5Pclose(acc_tpl); 1453 | VRFY((ret >= 0), ""); 1454 | 1455 | /* --------------------------------------------- 1456 | * Define the dimensions of the overall datasets 1457 | * and the slabs local to the MPI process. 1458 | * ------------------------------------------- */ 1459 | /* setup dimensionality object */ 1460 | dims[0] = DIM0; 1461 | dims[1] = DIM1; 1462 | sid = H5Screate_simple(RANK, dims, NULL); 1463 | VRFY((sid >= 0), "H5Screate_simple succeeded"); 1464 | 1465 | /* create a dataset collectively */ 1466 | dataset1 = H5Dcreate2(fid, DATASETNAME1, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 1467 | VRFY((dataset1 >= 0), "H5Dcreate2 succeeded"); 1468 | 1469 | /* create another dataset collectively */ 1470 | dataset2 = H5Dcreate2(fid, DATASETNAME2, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 1471 | VRFY((dataset2 >= 0), "H5Dcreate2 succeeded"); 1472 | 1473 | /* 1474 | * To test the independent orders of writes between processes, all 1475 | * even number processes write to dataset1 first, then dataset2. 1476 | * All odd number processes write to dataset2 first, then dataset1. 1477 | */ 1478 | 1479 | /* set up dimensions of the slab this process accesses */ 1480 | slab_set(mpi_rank, mpi_size, start, count, stride, block, BYROW); 1481 | 1482 | /* put some trivial data in the data_array */ 1483 | dataset_fill(start, block, data_array1); 1484 | 1485 | cudaMalloc((void **)&cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE)); 1486 | cudaMemcpy(cuda_buff, data_array1, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyHostToDevice); 1487 | 1488 | /* create a file dataspace independently */ 1489 | file_dataspace = H5Dget_space(dataset1); 1490 | VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); 1491 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 1492 | VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); 1493 | 1494 | /* create a memory dataspace independently */ 1495 | mem_dataspace = H5Screate_simple(RANK, block, NULL); 1496 | VRFY((mem_dataspace >= 0), ""); 1497 | 1498 | /* write data independently */ 1499 | ret = H5Dwrite(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace, H5P_DEFAULT, cuda_buff); 1500 | VRFY((ret >= 0), "H5Dwrite dataset1 succeeded"); 1501 | /* write data independently */ 1502 | ret = H5Dwrite(dataset2, H5T_NATIVE_INT, mem_dataspace, file_dataspace, H5P_DEFAULT, cuda_buff); 1503 | VRFY((ret >= 0), "H5Dwrite dataset2 succeeded"); 1504 | 1505 | /* setup dimensions again to write with zero rows for process 0 */ 1506 | printf("writeInd by some with zero row\n"); 1507 | slab_set(mpi_rank, mpi_size, start, count, stride, block, ZROW); 1508 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 1509 | VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); 1510 | /* need to make mem_dataspace to match for process 0 */ 1511 | if (MAINPROCESS) { 1512 | ret = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_SET, start, stride, count, block); 1513 | VRFY((ret >= 0), "H5Sset_hyperslab mem_dataspace succeeded"); 1514 | } 1515 | if ((mpi_rank / 2) * 2 != mpi_rank) { 1516 | ret = H5Dwrite(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace, H5P_DEFAULT, cuda_buff); 1517 | VRFY((ret >= 0), "H5Dwrite dataset1 by ZROW succeeded"); 1518 | } 1519 | 1520 | /* release dataspace ID */ 1521 | H5Sclose(file_dataspace); 1522 | 1523 | /* close dataset collectively */ 1524 | ret = H5Dclose(dataset1); 1525 | VRFY((ret >= 0), "H5Dclose1 succeeded"); 1526 | ret = H5Dclose(dataset2); 1527 | VRFY((ret >= 0), "H5Dclose2 succeeded"); 1528 | 1529 | /* release all IDs created */ 1530 | H5Sclose(sid); 1531 | 1532 | /* close the file collectively */ 1533 | H5Fclose(fid); 1534 | 1535 | /* release data buffers */ 1536 | if (data_array1) 1537 | free(data_array1); 1538 | cudaFree(&cuda_buff); 1539 | } 1540 | 1541 | /* Example of using the parallel HDF5 library to read a dataset */ 1542 | static void 1543 | dataset_readInd_cuda(void) 1544 | { 1545 | hid_t fid; /* HDF5 file ID */ 1546 | hid_t acc_tpl; /* File access templates */ 1547 | hid_t file_dataspace; /* File dataspace ID */ 1548 | hid_t mem_dataspace; /* memory dataspace ID */ 1549 | hid_t dataset1, dataset2; /* Dataset ID */ 1550 | DATATYPE * data_array1 = NULL; /* data buffer */ 1551 | DATATYPE * data_origin1 = NULL; /* expected data buffer */ 1552 | DATATYPE * cuda_buff = NULL; /* data buffer */ 1553 | 1554 | hsize_t start[RANK]; /* for hyperslab setting */ 1555 | hsize_t count[RANK], stride[RANK]; /* for hyperslab setting */ 1556 | hsize_t block[RANK]; /* for hyperslab setting */ 1557 | 1558 | herr_t ret; /* Generic return value */ 1559 | 1560 | printf("Independent read test on file %s\n", FILENAME); 1561 | 1562 | /* allocate memory for data buffer */ 1563 | data_array1 = (DATATYPE *)malloc(DIM0 * DIM1 * sizeof(DATATYPE)); 1564 | VRFY((data_array1 != NULL), "data_array1 malloc succeeded"); 1565 | data_origin1 = (DATATYPE *)malloc(DIM0 * DIM1 * sizeof(DATATYPE)); 1566 | VRFY((data_origin1 != NULL), "data_origin1 malloc succeeded"); 1567 | 1568 | /* setup file access template */ 1569 | acc_tpl = H5Pcreate(H5P_FILE_ACCESS); 1570 | VRFY((acc_tpl >= 0), "H5Pcreate H5P_FILE_ACCESS"); 1571 | ret = H5Pset_all_coll_metadata_ops(acc_tpl, true); 1572 | VRFY((ret >= 0), "H5Pset_all_coll_metadata_ops"); 1573 | ret = H5Pset_coll_metadata_write(acc_tpl, true); 1574 | VRFY((ret >= 0), "H5Pset_coll_metadata_write"); 1575 | 1576 | /* Set GDS VFD on FAPL */ 1577 | ret = H5Pset_driver_by_name(acc_tpl, "gds", NULL); 1578 | VRFY((ret >= 0), "H5Pset_driver_by_name"); 1579 | 1580 | /* open the file collectively */ 1581 | fid = H5Fopen(FILENAME, H5F_ACC_RDONLY, acc_tpl); 1582 | VRFY((fid >= 0), ""); 1583 | 1584 | /* Release file-access template */ 1585 | ret = H5Pclose(acc_tpl); 1586 | VRFY((ret >= 0), ""); 1587 | 1588 | /* open the dataset1 collectively */ 1589 | dataset1 = H5Dopen2(fid, DATASETNAME1, H5P_DEFAULT); 1590 | VRFY((dataset1 >= 0), ""); 1591 | 1592 | /* open another dataset collectively */ 1593 | dataset2 = H5Dopen2(fid, DATASETNAME1, H5P_DEFAULT); 1594 | VRFY((dataset2 >= 0), ""); 1595 | 1596 | /* set up dimensions of the slab this process accesses */ 1597 | slab_set(mpi_rank, mpi_size, start, count, stride, block, BYROW); 1598 | 1599 | /* create a file dataspace independently */ 1600 | file_dataspace = H5Dget_space(dataset1); 1601 | VRFY((file_dataspace >= 0), ""); 1602 | ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); 1603 | VRFY((ret >= 0), ""); 1604 | 1605 | /* create a memory dataspace independently */ 1606 | mem_dataspace = H5Screate_simple(RANK, block, NULL); 1607 | VRFY((mem_dataspace >= 0), ""); 1608 | 1609 | /* fill dataset with test data */ 1610 | dataset_fill(start, block, data_origin1); 1611 | 1612 | cudaMalloc((void **)&cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE)); 1613 | 1614 | /* read data independently */ 1615 | ret = H5Dread(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace, H5P_DEFAULT, cuda_buff); 1616 | VRFY((ret >= 0), ""); 1617 | 1618 | cudaMemcpy(data_array1, cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyDeviceToHost); 1619 | 1620 | /* verify the read data with original expected data */ 1621 | ret = dataset_vrfy(start, count, stride, block, data_array1, data_origin1); 1622 | if (ret) 1623 | nerrors++; 1624 | 1625 | /* read data independently */ 1626 | ret = H5Dread(dataset2, H5T_NATIVE_INT, mem_dataspace, file_dataspace, H5P_DEFAULT, cuda_buff); 1627 | VRFY((ret >= 0), ""); 1628 | 1629 | cudaMemcpy(data_array1, cuda_buff, DIM0 * DIM1 * sizeof(DATATYPE), cudaMemcpyDeviceToHost); 1630 | 1631 | /* verify the read data with original expected data */ 1632 | ret = dataset_vrfy(start, count, stride, block, data_array1, data_origin1); 1633 | if (ret) 1634 | nerrors++; 1635 | 1636 | /* close dataset collectively */ 1637 | ret = H5Dclose(dataset1); 1638 | VRFY((ret >= 0), ""); 1639 | ret = H5Dclose(dataset2); 1640 | VRFY((ret >= 0), ""); 1641 | 1642 | /* release all IDs created */ 1643 | H5Sclose(file_dataspace); 1644 | 1645 | /* close the file collectively */ 1646 | H5Fclose(fid); 1647 | 1648 | /* release data buffers */ 1649 | if (data_array1) 1650 | free(data_array1); 1651 | if (data_origin1) 1652 | free(data_origin1); 1653 | cudaFree(&cuda_buff); 1654 | } 1655 | 1656 | static void 1657 | extend_writeInd2_cuda(void) 1658 | { 1659 | hid_t fid; /* HDF5 file ID */ 1660 | hid_t fapl; /* File access templates */ 1661 | hid_t fs; /* File dataspace ID */ 1662 | hid_t ms; /* Memory dataspace ID */ 1663 | hid_t dataset; /* Dataset ID */ 1664 | hsize_t orig_size = 10; /* Original dataset dim size */ 1665 | hsize_t new_size = 20; /* Extended dataset dim size */ 1666 | hsize_t one = 1; 1667 | hsize_t max_size = H5S_UNLIMITED; /* dataset maximum dim size */ 1668 | hsize_t chunk_size = 16384; /* chunk size */ 1669 | hid_t dcpl; /* dataset create prop. list */ 1670 | int written[10], /* Data to write */ 1671 | retrieved[10]; /* Data read in */ 1672 | int i; /* Local index variable */ 1673 | herr_t ret; /* Generic return value */ 1674 | int * cuda_written = NULL; /* data buffer */ 1675 | int * cuda_retrieved = NULL; /* data buffer */ 1676 | 1677 | cudaMalloc((void **)&cuda_written, 10 * sizeof(int)); 1678 | cudaMalloc((void **)&cuda_retrieved, 10 * sizeof(int)); 1679 | 1680 | printf("Extend independent write test #2 on file %s\n", FILENAME); 1681 | 1682 | /* ------------------- 1683 | * START AN HDF5 FILE 1684 | * -------------------*/ 1685 | /* setup file access template */ 1686 | fapl = H5Pcreate(H5P_FILE_ACCESS); 1687 | VRFY((fapl >= 0), "H5Pcreate H5P_FILE_ACCESS"); 1688 | ret = H5Pset_all_coll_metadata_ops(fapl, true); 1689 | VRFY((ret >= 0), "H5Pset_all_coll_metadata_ops"); 1690 | ret = H5Pset_coll_metadata_write(fapl, true); 1691 | VRFY((ret >= 0), "H5Pset_coll_metadata_write"); 1692 | 1693 | /* Set GDS VFD on FAPL */ 1694 | ret = H5Pset_driver_by_name(fapl, "gds", NULL); 1695 | VRFY((ret >= 0), "H5Pset_driver_by_name"); 1696 | 1697 | /* create the file collectively */ 1698 | fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); 1699 | VRFY((fid >= 0), "H5Fcreate succeeded"); 1700 | 1701 | /* Release file-access template */ 1702 | ret = H5Pclose(fapl); 1703 | VRFY((ret >= 0), "H5Pclose succeeded"); 1704 | 1705 | /* -------------------------------------------------------------- 1706 | * Define the dimensions of the overall datasets and create them. 1707 | * ------------------------------------------------------------- */ 1708 | 1709 | /* set up dataset storage chunk sizes and creation property list */ 1710 | dcpl = H5Pcreate(H5P_DATASET_CREATE); 1711 | VRFY((dcpl >= 0), "H5Pcreate succeeded"); 1712 | ret = H5Pset_chunk(dcpl, 1, &chunk_size); 1713 | VRFY((ret >= 0), "H5Pset_chunk succeeded"); 1714 | 1715 | /* setup dimensionality object */ 1716 | fs = H5Screate_simple(1, &orig_size, &max_size); 1717 | VRFY((fs >= 0), "H5Screate_simple succeeded"); 1718 | 1719 | /* create an extendible dataset collectively */ 1720 | dataset = H5Dcreate2(fid, DATASETNAME1, H5T_NATIVE_INT, fs, H5P_DEFAULT, dcpl, H5P_DEFAULT); 1721 | VRFY((dataset >= 0), "H5Dcreat2e succeeded"); 1722 | 1723 | /* release resource */ 1724 | ret = H5Pclose(dcpl); 1725 | VRFY((ret >= 0), "H5Pclose succeeded"); 1726 | 1727 | /* ------------------------- 1728 | * Test writing to dataset 1729 | * -------------------------*/ 1730 | /* create a memory dataspace independently */ 1731 | ms = H5Screate_simple(1, &orig_size, &max_size); 1732 | VRFY((ms >= 0), "H5Screate_simple succeeded"); 1733 | 1734 | /* put some trivial data in the data_array */ 1735 | for (i = 0; i < (int)orig_size; i++) 1736 | written[i] = i; 1737 | 1738 | cudaMemcpy(cuda_written, written, 10 * sizeof(int), cudaMemcpyHostToDevice); 1739 | 1740 | ret = H5Dwrite(dataset, H5T_NATIVE_INT, ms, fs, H5P_DEFAULT, cuda_written); 1741 | VRFY((ret >= 0), "H5Dwrite succeeded"); 1742 | 1743 | /* ------------------------- 1744 | * Read initial data from dataset. 1745 | * -------------------------*/ 1746 | ret = H5Dread(dataset, H5T_NATIVE_INT, ms, fs, H5P_DEFAULT, cuda_retrieved); 1747 | VRFY((ret >= 0), "H5Dread succeeded"); 1748 | cudaMemcpy(retrieved, cuda_retrieved, 10 * sizeof(int), cudaMemcpyDeviceToHost); 1749 | 1750 | for (i = 0; i < (int)orig_size; i++) 1751 | if (written[i] != retrieved[i]) { 1752 | printf("Line #%d: written!=retrieved: written[%d]=%d, retrieved[%d]=%d\n", __LINE__, i, 1753 | written[i], i, retrieved[i]); 1754 | nerrors++; 1755 | } 1756 | 1757 | /* ------------------------- 1758 | * Extend the dataset & retrieve new dataspace 1759 | * -------------------------*/ 1760 | ret = H5Dset_extent(dataset, &new_size); 1761 | VRFY((ret >= 0), "H5Dset_extent succeeded"); 1762 | ret = H5Sclose(fs); 1763 | VRFY((ret >= 0), "H5Sclose succeeded"); 1764 | fs = H5Dget_space(dataset); 1765 | VRFY((fs >= 0), "H5Dget_space succeeded"); 1766 | 1767 | /* ------------------------- 1768 | * Write to the second half of the dataset 1769 | * -------------------------*/ 1770 | for (i = 0; i < (int)orig_size; i++) 1771 | written[i] = orig_size + i; 1772 | 1773 | ret = H5Sselect_hyperslab(fs, H5S_SELECT_SET, &orig_size, NULL, &one, &orig_size); 1774 | VRFY((ret >= 0), "H5Sselect_hyperslab succeeded"); 1775 | cudaMemcpy(cuda_written, written, 10 * sizeof(int), cudaMemcpyHostToDevice); 1776 | ret = H5Dwrite(dataset, H5T_NATIVE_INT, ms, fs, H5P_DEFAULT, cuda_written); 1777 | VRFY((ret >= 0), "H5Dwrite succeeded"); 1778 | 1779 | /* ------------------------- 1780 | * Read the new data 1781 | * -------------------------*/ 1782 | ret = H5Dread(dataset, H5T_NATIVE_INT, ms, fs, H5P_DEFAULT, cuda_retrieved); 1783 | VRFY((ret >= 0), "H5Dread succeeded"); 1784 | cudaMemcpy(retrieved, cuda_retrieved, 10 * sizeof(int), cudaMemcpyDeviceToHost); 1785 | 1786 | for (i = 0; i < (int)orig_size; i++) 1787 | if (written[i] != retrieved[i]) { 1788 | printf("Line #%d: written!=retrieved: written[%d]=%d, retrieved[%d]=%d\n", __LINE__, i, 1789 | written[i], i, retrieved[i]); 1790 | nerrors++; 1791 | } 1792 | 1793 | /* Close dataset collectively */ 1794 | ret = H5Dclose(dataset); 1795 | VRFY((ret >= 0), "H5Dclose succeeded"); 1796 | 1797 | /* Close the file collectively */ 1798 | ret = H5Fclose(fid); 1799 | VRFY((ret >= 0), "H5Fclose succeeded"); 1800 | 1801 | cudaFree(&cuda_written); 1802 | cudaFree(&cuda_retrieved); 1803 | } 1804 | 1805 | int 1806 | main(int argc, char **argv) 1807 | { 1808 | /* Initialize MPI */ 1809 | MPI_Init(&argc, &argv); 1810 | MPI_Comm_size(comm, &mpi_size); 1811 | MPI_Comm_rank(comm, &mpi_rank); 1812 | 1813 | H5open(); 1814 | 1815 | if (MAINPROCESS) { 1816 | printf("==========================\n"); 1817 | printf("GDS VFD tests\n"); 1818 | printf("==========================\n\n"); 1819 | } 1820 | 1821 | extend_writeInd_cuda(); 1822 | dataset_writeAll_cuda(); 1823 | dataset_readAll_cuda(); 1824 | dataset_writeInd_cuda(); 1825 | dataset_readInd_cuda(); 1826 | extend_writeInd2_cuda(); 1827 | 1828 | if (nerrors) 1829 | goto exit; 1830 | 1831 | if (MAINPROCESS) 1832 | puts("All GDS VFD tests passed\n"); 1833 | 1834 | exit: 1835 | if (nerrors) 1836 | if (MAINPROCESS) 1837 | printf("*** %d TEST ERROR%s OCCURRED ***\n", nerrors, nerrors > 1 ? "S" : ""); 1838 | 1839 | H5E_BEGIN_TRY 1840 | { 1841 | H5Fdelete(FILENAME, H5P_DEFAULT); 1842 | } 1843 | H5E_END_TRY; 1844 | 1845 | H5close(); 1846 | 1847 | MPI_Finalize(); 1848 | 1849 | exit((nerrors ? EXIT_FAILURE : EXIT_SUCCESS)); 1850 | } 1851 | --------------------------------------------------------------------------------