├── docs ├── requirements.txt ├── index.rst ├── direct.rst ├── endian_issues.rst ├── cd_vals.rst ├── tests.rst ├── h5repack.rst ├── Makefile ├── conf.py ├── interfaces.rst ├── hdf5_chunking.rst └── installation.rst ├── test ├── mesh.h5 ├── bigendian.h5 ├── test_zfp_050.h5 ├── test_zfp_052.h5 ├── test_zfp_054.h5 ├── test_zfp_be.h5 ├── test_zfp_le.h5 ├── test_zfp_030040.h5 ├── test_zfp_030235.h5 ├── test_zfp_110050.h5 ├── test_zfp_110xxx.h5 ├── h5dump-rate.bsh ├── h5repack-filesizes.bsh ├── h5repack-filesizes.cmake ├── h5repack_parse.patch ├── h5dump-rate.cmake ├── print_h5repack_farg.c ├── test_common.h ├── test_read.c ├── test_error.c └── test_rw_fortran.F90 ├── src ├── H5Zzfp.h ├── H5Zzfp_lib.h ├── H5Zzfp_props_private.h ├── H5Zzfp_version.h ├── H5Zzfp_props.h ├── Makefile ├── H5Zzfp_plugin.h ├── CMakeLists.txt ├── H5Zzfp_props.c └── H5Zzfp_props_f.F90 ├── .gitignore ├── .github ├── codeql-config.yml ├── workflows │ ├── codespell.yml │ ├── codeql.yml │ └── main.yml └── dependabot.yml ├── cmake ├── h5z_zfp-config-version.cmake.in ├── h5z_zfp-config.cmake.in └── HDFMacros.cmake ├── .readthedocs.yaml ├── .travis.yml ├── README.md ├── Makefile ├── LICENSE ├── CMakeLists.txt └── config.make /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx>=7.0,<9.0 2 | sphinx_rtd_theme>=2.0 3 | -------------------------------------------------------------------------------- /test/mesh.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/H5Z-ZFP/HEAD/test/mesh.h5 -------------------------------------------------------------------------------- /test/bigendian.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/H5Z-ZFP/HEAD/test/bigendian.h5 -------------------------------------------------------------------------------- /test/test_zfp_050.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/H5Z-ZFP/HEAD/test/test_zfp_050.h5 -------------------------------------------------------------------------------- /test/test_zfp_052.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/H5Z-ZFP/HEAD/test/test_zfp_052.h5 -------------------------------------------------------------------------------- /test/test_zfp_054.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/H5Z-ZFP/HEAD/test/test_zfp_054.h5 -------------------------------------------------------------------------------- /test/test_zfp_be.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/H5Z-ZFP/HEAD/test/test_zfp_be.h5 -------------------------------------------------------------------------------- /test/test_zfp_le.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/H5Z-ZFP/HEAD/test/test_zfp_le.h5 -------------------------------------------------------------------------------- /test/test_zfp_030040.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/H5Z-ZFP/HEAD/test/test_zfp_030040.h5 -------------------------------------------------------------------------------- /test/test_zfp_030235.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/H5Z-ZFP/HEAD/test/test_zfp_030235.h5 -------------------------------------------------------------------------------- /test/test_zfp_110050.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/H5Z-ZFP/HEAD/test/test_zfp_110050.h5 -------------------------------------------------------------------------------- /test/test_zfp_110xxx.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/H5Z-ZFP/HEAD/test/test_zfp_110xxx.h5 -------------------------------------------------------------------------------- /src/H5Zzfp.h: -------------------------------------------------------------------------------- 1 | #ifndef H5Z_ZFP_H 2 | #define H5Z_ZFP_H 3 | 4 | #include "H5Zzfp_lib.h" 5 | #include "H5Zzfp_plugin.h" 6 | #include "H5Zzfp_props.h" 7 | 8 | #endif 9 | -------------------------------------------------------------------------------- /src/H5Zzfp_lib.h: -------------------------------------------------------------------------------- 1 | #ifndef H5Z_ZFP_LIB_H 2 | #define H5Z_ZFP_LIB_H 3 | 4 | #include "H5Zzfp_plugin.h" 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | 10 | extern int H5Z_zfp_initialize(void); 11 | extern int H5Z_zfp_finalize(void); 12 | 13 | #ifdef __cplusplus 14 | } 15 | #endif 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Object files 2 | *.o 3 | *.ko 4 | *.obj 5 | *.elf 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Libraries 12 | *.lib 13 | *.a 14 | *.la 15 | *.lo 16 | 17 | # Shared objects (inc. Windows DLLs) 18 | *.dll 19 | *.so 20 | *.so.* 21 | *.dylib 22 | 23 | # Executables 24 | *.exe 25 | *.out 26 | *.app 27 | *.i*86 28 | *.x86_64 29 | *.hex 30 | 31 | # Debug files 32 | *.dSYM/ 33 | *.su 34 | -------------------------------------------------------------------------------- /.github/codeql-config.yml: -------------------------------------------------------------------------------- 1 | query-filters: 2 | - exclude: 3 | # See: https://codeql.github.com/codeql-query-help/cpp/cpp-short-global-name/ 4 | id: cpp/short-global-name 5 | - exclude: 6 | # See: https://codeql.github.com/codeql-query-help/cpp/cpp-guarded-free/ 7 | id: cpp/guarded-free 8 | - exclude: 9 | # See: https://codeql.github.com/codeql-query-help/cpp/cpp-unused-static-function/ 10 | id: cpp/unused-static-function 11 | 12 | -------------------------------------------------------------------------------- /src/H5Zzfp_props_private.h: -------------------------------------------------------------------------------- 1 | #ifndef H5Z_ZFP_PROPS_PRIVATE_H 2 | #define H5Z_ZFP_PROPS_PRIVATE_H 3 | 4 | typedef struct _h5z_zfp_controls_t { 5 | unsigned int mode; 6 | union { 7 | double rate; 8 | double acc; 9 | unsigned int prec; 10 | struct expert_ { 11 | unsigned int minbits; 12 | unsigned int maxbits; 13 | unsigned int maxprec; 14 | int minexp; 15 | } expert; 16 | } details; 17 | } h5z_zfp_controls_t; 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /src/H5Zzfp_version.h: -------------------------------------------------------------------------------- 1 | #ifndef H5Z_ZFP_VERSION_H 2 | #define H5Z_ZFP_VERSION_H 3 | 4 | #define H5Z_FILTER_ZFP 32013 5 | 6 | #define H5Z_FILTER_ZFP_VERSION_MAJOR 1 7 | #define H5Z_FILTER_ZFP_VERSION_MINOR 1 8 | #define H5Z_FILTER_ZFP_VERSION_PATCH 1 9 | 10 | #define H5Z_ZFP_MODE_RATE 1 11 | #define H5Z_ZFP_MODE_PRECISION 2 12 | #define H5Z_ZFP_MODE_ACCURACY 3 13 | #define H5Z_ZFP_MODE_EXPERT 4 14 | #define H5Z_ZFP_MODE_REVERSIBLE 5 15 | 16 | #define H5Z_ZFP_CD_NELMTS_MEM 6 17 | #define H5Z_ZFP_CD_NELMTS_MAX 6 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /.github/workflows/codespell.yml: -------------------------------------------------------------------------------- 1 | # GitHub Action to automate the identification of common misspellings in text files 2 | # https://github.com/codespell-project/codespell 3 | # https://github.com/codespell-project/actions-codespell 4 | name: codespell 5 | on: [push, pull_request] 6 | jobs: 7 | codespell: 8 | name: Check for spelling errors 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v6 12 | - uses: codespell-project/actions-codespell@v2 13 | with: 14 | ignore_words_list: siz, textin 15 | -------------------------------------------------------------------------------- /cmake/h5z_zfp-config-version.cmake.in: -------------------------------------------------------------------------------- 1 | set(PACKAGE_VERSION_MAJOR @PROJECT_VERSION_MAJOR@) 2 | set(PACKAGE_VERSION_MINOR @PROJECT_VERSION_MINOR@) 3 | set(PACKAGE_VERSION_PATCH @PROJECT_VERSION_PATCH@) 4 | set(PACKAGE_VERSION @PROJECT_VERSION@) 5 | 6 | # Check whether the requested PACKAGE_FIND_VERSION is compatible 7 | if(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION OR 8 | PACKAGE_VERSION_MAJOR GREATER PACKAGE_FIND_VERSION_MAJOR) 9 | set(PACKAGE_VERSION_COMPATIBLE FALSE) 10 | else() 11 | set(PACKAGE_VERSION_COMPATIBLE TRUE) 12 | if(PACKAGE_VERSION VERSION_EQUAL PACKAGE_FIND_VERSION) 13 | set(PACKAGE_VERSION_EXACT TRUE) 14 | endif() 15 | endif() 16 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.11" 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/conf.py 17 | 18 | # We recommend specifying your dependencies to enable reproducible builds: 19 | # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 20 | python: 21 | install: 22 | - requirements: docs/requirements.txt 23 | -------------------------------------------------------------------------------- /test/h5dump-rate.bsh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | H5DUMP=$1 3 | FILE=$2 4 | DSET=$3 5 | RATE=$4 6 | 7 | # Checks 8 | if [[ ! -f ${H5DUMP} ]]; then 9 | echo "*** ERROR: The executable h5dump \"${H5DUMP}\" does not exist." 10 | exit 1 11 | fi 12 | 13 | if [[ ! -f ${FILE} ]]; then 14 | echo "*** ERROR: File \"${FILE}\" does not exist." 15 | exit 1 16 | fi 17 | 18 | 19 | EXPECTED_RATIO=$(expr 64 / ${RATE}) 20 | ACTUAL_RATIO=`${H5DUMP} -H -d ${DSET} -p ${FILE} | grep COMPRESSION | cut -d':' -f1 | cut -d'(' -f2 | cut -d'.' -f1` 21 | echo "Compression ratio" 22 | echo " h5dump: ${H5DUMP}" 23 | echo " File: ${FILE}" 24 | echo " Dataset: ${DSET}" 25 | echo " Expected: ${EXPECTED_RATIO}" 26 | echo " Actual: ${ACTUAL_RATIO}" 27 | if [[ ${ACTUAL_RATIO} != ${EXPECTED_RATIO} ]]; then 28 | exit 1 29 | fi 30 | 31 | exit 0 32 | -------------------------------------------------------------------------------- /test/h5repack-filesizes.bsh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | FILE_ORIGINAL=$1 3 | FILE_REPACK=$2 4 | 5 | # Checks 6 | if [[ ! -f ${FILE_ORIGINAL} ]]; then 7 | echo "*** ERROR: Original file \"${FILE_ORIGINAL}\" does not exist." 8 | exit 1 9 | fi 10 | 11 | if [[ ! -f ${FILE_REPACK} ]]; then 12 | echo "*** ERROR: Repack file \"${FILE_REPACK}\" does not exist." 13 | exit 1 14 | fi 15 | 16 | ORIG_SIZE=$(ls -l ${FILE_ORIGINAL} | tr -s ' ' | cut -d' ' -f5) 17 | NEW_SIZE=$(ls -l ${FILE_REPACK} | tr -s ' ' | cut -d' ' -f5) 18 | RATIO=`perl -e "printf int($ORIG_SIZE*100/$NEW_SIZE)"` 19 | 20 | echo "Original file" 21 | echo " Name: ${FILE_ORIGINAL}" 22 | echo " Size: ${ORIG_SIZE}" 23 | echo "Repack file" 24 | echo " Name: ${FILE_REPACK}" 25 | echo " Size: ${NEW_SIZE}" 26 | echo " Ratio of the file sizes: ${RATIO}" 27 | if [[ $RATIO -lt 200 ]]; then 28 | exit 1 29 | fi 30 | 31 | exit 0 32 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Dependabot configuration for automatic dependency updates 2 | # This will automatically create pull requests to update GitHub Actions to their latest versions 3 | # See: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file 4 | 5 | version: 2 6 | updates: 7 | # Maintain dependencies for GitHub Actions 8 | - package-ecosystem: "github-actions" 9 | directory: "/" 10 | schedule: 11 | # Check for updates to GitHub Actions every week 12 | interval: "weekly" 13 | day: "monday" 14 | time: "09:00" 15 | # Automatically merge minor and patch updates 16 | open-pull-requests-limit: 10 17 | # Add labels to PRs 18 | labels: 19 | - "dependencies" 20 | - "github-actions" 21 | # Group all GitHub Actions updates into a single PR when possible 22 | groups: 23 | github-actions: 24 | patterns: 25 | - "*" 26 | # Commit message prefix 27 | commit-message: 28 | prefix: "ci" 29 | include: "scope" 30 | -------------------------------------------------------------------------------- /test/h5repack-filesizes.cmake: -------------------------------------------------------------------------------- 1 | #h5repack-filesizes.cmake 2 | cmake_policy(SET CMP0007 NEW) 3 | 4 | # arguments checking 5 | if (NOT FILE_ORIGINAL) 6 | message (FATAL_ERROR "Require FILE_ORIGINAL to be defined") 7 | endif () 8 | if (NOT FILE_REPACK) 9 | message (FATAL_ERROR "Require FILE_REPACK to be defined") 10 | endif () 11 | if (NOT RATIO_LIMIT) 12 | message (FATAL_ERROR "Require RATIO_LIMIT to be defined") 13 | endif () 14 | 15 | file(SIZE ${FILE_ORIGINAL} ORIG_SIZE) 16 | file(SIZE ${FILE_REPACK} NEW_SIZE) 17 | math(EXPR RATIO "(${ORIG_SIZE} * 100) / ${NEW_SIZE}") 18 | 19 | message (STATUS "Original file") 20 | message (STATUS " Name: ${FILE_ORIGINAL}") 21 | message (STATUS " Size: ${ORIG_SIZE}") 22 | message (STATUS "Repack file") 23 | message (STATUS " Name: ${FILE_REPACK}") 24 | message (STATUS " Size: ${NEW_SIZE}") 25 | message (STATUS " Ratio of the file sizes: ${RATIO}") 26 | 27 | if (${RATIO} LESS ${RATIO_LIMIT}) 28 | message (FATAL_ERROR "Failed: The RATIO was LESS ${RATIO_LIMIT}") 29 | endif () 30 | 31 | # everything went fine... 32 | message (STATUS "RATIO Passed") 33 | 34 | -------------------------------------------------------------------------------- /src/H5Zzfp_props.h: -------------------------------------------------------------------------------- 1 | #ifndef H5Z_ZFP_PROPS_H 2 | #define H5Z_ZFP_PROPS_H 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | extern herr_t H5Pset_zfp_rate(hid_t plist, double rate); 9 | extern herr_t H5Pset_zfp_precision(hid_t plist, unsigned int prec); 10 | extern herr_t H5Pset_zfp_accuracy(hid_t plist, double acc); 11 | extern herr_t H5Pset_zfp_expert(hid_t plist, unsigned int minbits, unsigned int maxbits, 12 | unsigned int maxprec, int minexp); 13 | extern herr_t H5Pset_zfp_reversible(hid_t plist); 14 | 15 | extern void H5Pset_zfp_rate_cdata_f(double rate, size_t *cd_nelmts, unsigned int *cd_values); 16 | extern void H5Pset_zfp_precision_cdata_f(unsigned int prec, size_t *cd_nelmts, unsigned int *cd_values); 17 | extern void H5Pset_zfp_accuracy_cdata_f(double acc, size_t *cd_nelmts, unsigned int *cd_values); 18 | extern void H5Pset_zfp_expert_cdata_f(unsigned int minbits, unsigned int maxbits, unsigned int maxprec, 19 | int minexp, size_t *cd_nelmts, unsigned int *cd_values); 20 | extern void H5Pset_zfp_reversible_cdata_f(size_t *cd_nelmts, unsigned int *cd_values); 21 | 22 | #ifdef __cplusplus 23 | } 24 | #endif 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ================== 2 | Welcome to H5Z-ZFP 3 | ================== 4 | 5 | H5Z-ZFP_ is a compression filter for HDF5_ using the ZFP_ compression library, supporting *lossy* and *lossless* compression of floating point and integer data to meet `bitrate `_, `accuracy `_, and/or `precision `_ targets. 6 | The filter uses the `registered /dev/null 2>&1 21 | - popd 22 | - popd 23 | 24 | install: true 25 | 26 | script: 27 | - make FCFLAGS="-O0 -fPIC --coverage -fprofile-arcs -ftest-coverage" \ 28 | CFLAGS="-O0 -fPIC --coverage" \ 29 | LDFLAGS="--coverage -lm" \ 30 | FC=gfortran CC=gcc \ 31 | ZFP_HOME=/tmp/zfp-0.5.5 \ 32 | HDF5_HOME=/tmp/hdf5-1.8.14/my_install check 33 | - pushd src; gcov -a H5Zzfp_lib H5Zzfp_plugin H5Zzfp_props; popd 34 | - pushd test; gcov -a test_read_lib test_write_lib test_write_plugin; popd 35 | 36 | after_success: 37 | - bash <(curl -s https://codecov.io/bash) 38 | 39 | -------------------------------------------------------------------------------- /test/h5repack_parse.patch: -------------------------------------------------------------------------------- 1 | --- h5repack_parse.c 2014-10-13 13:33:26.000000000 +0200 2 | +++ h5repack_parse_fixed.c 2014-12-03 10:57:21.910457000 +0100 3 | @@ -258,7 +258,7 @@ 4 | l=-1; /* filter number index check */ 5 | p=-1; /* CD_VAL count check */ 6 | r=-1; /* CD_VAL check */ 7 | - for ( m=0,q=0,u=i+1; ucd_values[j++]=atoi(stype); 18 | r=0; 19 | + } 20 | q=0; 21 | - u++; /* skip ',' */ 22 | - } 23 | - c = str[u]; 24 | - if (!isdigit(c) && l==-1) 25 | - { 26 | - if (obj_list) HDfree(obj_list); 27 | - error_msg("filter number parameter is not a digit in <%s>\n",str); 28 | - HDexit(EXIT_FAILURE); 29 | - } 30 | - stype[q]=c; 31 | - if (l==0 && p==0) 32 | - { 33 | - if (r==0) 34 | - filt->cd_values[j++]=atoi(stype); 35 | + } else { 36 | + c = str[u]; 37 | + if (!isdigit(c) && l==-1) 38 | + { 39 | + if (obj_list) HDfree(obj_list); 40 | + error_msg("filter number parameter is not a digit in <%s>\n",str); 41 | + HDexit(EXIT_FAILURE); 42 | + } 43 | + stype[q++]=c; 44 | } 45 | 46 | } /* u */ 47 | -------------------------------------------------------------------------------- /test/h5dump-rate.cmake: -------------------------------------------------------------------------------- 1 | #h5dump-rate.cmake 2 | cmake_policy(SET CMP0007 NEW) 3 | 4 | # arguments checking 5 | if (NOT TEST_PROGRAM) 6 | message (FATAL_ERROR "Require TEST_PROGRAM to be defined") 7 | endif () 8 | if (NOT TEST_FILE) 9 | message (FATAL_ERROR "Require TEST_FILE to be defined") 10 | endif () 11 | if (NOT TEST_DSET) 12 | message (FATAL_ERROR "Require TEST_DSET to be defined") 13 | endif () 14 | if (NOT TEST_RATE) 15 | message (FATAL_ERROR "Require TEST_RATE to be defined") 16 | endif () 17 | if (NOT RATE_START) 18 | message (FATAL_ERROR "Require RATE_START to be defined") 19 | endif () 20 | 21 | math (EXPR EXPECTED_RATIO "64 / ${TEST_RATE}") 22 | # run the test program, capture the stdout/stderr and the result var 23 | execute_process ( 24 | COMMAND ${TEST_PROGRAM} -H -d ${TEST_DSET} -p ${TEST_FILE} 25 | RESULT_VARIABLE TEST_RESULT 26 | OUTPUT_VARIABLE TEST_OUT 27 | ERROR_VARIABLE TEST_ERROR 28 | ) 29 | message (STATUS "dump: ${TEST_OUT}") 30 | 31 | # SIZE [0-9]* ([.0-9]*:1 COMPRESSION) 32 | string (REGEX MATCH "SIZE [0-9]* \\(([0-9]*).[0-9]*:1 COMPRESSION\\)" ACTUAL_COMPRESS ${TEST_OUT}) 33 | set (ACTUAL_RATIO ${CMAKE_MATCH_1}) 34 | message (STATUS "Compression ratio") 35 | message (STATUS " File: ${TEST_FILE}") 36 | message (STATUS " Dataset: ${TEST_DSET}") 37 | message (STATUS " Expected: ${EXPECTED_RATIO}") 38 | message (STATUS " Actual: ${ACTUAL_RATIO}") 39 | 40 | if (NOT ${ACTUAL_RATIO} EQUAL ${EXPECTED_RATIO}) 41 | message (FATAL_ERROR "Failed: The ACTUAL_RATIO was DIFFERENT to Expected: ${EXPECTED_RATIO}") 42 | endif () 43 | 44 | # everything went fine... 45 | message (STATUS "RATIO Passed") 46 | 47 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # H5Z-ZFP 2 | 3 | A highly flexible floating point and integer 4 | compression plugin for the HDF5 library using ZFP compression. 5 | 6 | [![Build Status](https://travis-ci.com/LLNL/H5Z-ZFP.svg?branch=master)](https://travis-ci.com/LLNL/H5Z-ZFP) 7 | [![Documentation Status](https://readthedocs.org/projects/h5z-zfp/badge/?version=latest)](http://h5z-zfp.readthedocs.io) 8 | [![codecov](https://codecov.io/gh/LLNL/H5Z-ZFP/branch/master/graph/badge.svg)](https://codecov.io/gh/LLNL/H5Z-ZFP) 9 | 10 | For information about ZFP compression and the BSD-Licensed ZFP 11 | library, see... 12 | 13 | - https://computing.llnl.gov/projects/zfp 14 | - https://github.com/LLNL/zfp 15 | 16 | For information about HDF5 filter plugins, see... 17 | 18 | - https://support.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters 19 | 20 | This H5Z-ZFP plugin supports ZFP versions 0.5.0 and newer. 21 | 22 | This plugin uses the [*registered*](https://portal.hdfgroup.org/documentation/hdf5-docs/registered_filter_plugins.html) 23 | HDF5 plugin filter id 32013 24 | 25 | The HDF5 filter plugin code here is also part of the Silo library. 26 | However, we have made an effort to also support it as a stand-alone 27 | package due to the likely broad appeal and utility of the ZFP 28 | compression library. 29 | 30 | This plugin supports all modes of the ZFP compression library, *rate*, 31 | *accuracy*, *precision*, *expert* and *reversible*. It supports 1, 2, 3 and 32 | 4 dimensional datasets (for ZFP version 0.5.5 and newer) of single and double 33 | precision integer and floating point data. It can be applied to HDF5 datasets 34 | of more than 3 dimensions (or 4 dimensions for ZFP versions 0.5.5 and newer) 35 | as long as no more than 3 (or 4) dimensions of the HDF5 dataset *chunking* are 36 | of size greater than 1. 37 | 38 | [**Full documentation**](http://h5z-zfp.readthedocs.io) 39 | -------------------------------------------------------------------------------- /docs/direct.rst: -------------------------------------------------------------------------------- 1 | ========================= 2 | Direct Writes (and Reads) 3 | ========================= 4 | The purpose of direct_ writes 5 | is to enable an application to write data that is already compressed in memory *directly* 6 | to an HDF5 file without first uncompressing it so the filter can then turn around and 7 | compress it during write. However, once data is written to the file with a *direct* write, 8 | consumers must still be able to read it without concern for how the producer wrote it. 9 | 10 | Doing this requires the use of an advanced HDF5 function for direct_ writes. 11 | 12 | At present, we demonstrate only minimal functionality here using *single chunking*, where 13 | the chunk size is chosen to match the size of the entire dataset. To see an example of 14 | code that does this, have a look at... 15 | 16 | .. literalinclude:: ../test/test_write.c 17 | :language: c 18 | :linenos: 19 | :start-after: ZFP Array Example 20 | :end-before: End of ZFP Array Example 21 | 22 | In particular, look for the line using ``H5Dchunk_write`` in place of ``H5Dwrite``. In all 23 | other respects, the code looks the same. 24 | 25 | 26 | The test case for this code writes uncompressed data as a dataset named ``zfparr_original``, 27 | the compressed dataset named ``zfparr_compressed`` using the filter and then the compressed 28 | data a second time named ``zfparr_direct`` using a direct_ write. Then, the ``h5diff`` tool 29 | is used to compare the data in the original and the direct_ write datasets. 30 | 31 | Note that in order for consumers to work as normal, the producer must set dataset *creation* 32 | properties as it ordinarily would using the H5Z-ZFP_ filter. In the call to ``H5Dchunk_write``, 33 | the caller indicates to the HDF5 library not to invoke the filter via the ``filters`` mask 34 | argument. 35 | 36 | .. _direct: https://docs.hdfgroup.org/hdf5/develop/group___h5_d.html#title38 37 | -------------------------------------------------------------------------------- /cmake/h5z_zfp-config.cmake.in: -------------------------------------------------------------------------------- 1 | # h5z_zfp-config.cmake 2 | # -------------------- 3 | # 4 | # Finds the H5Z_ZFP library, specify the starting search path in H5Z_ZFP_ROOT 5 | # 6 | # Static vs. shared 7 | # ----------------- 8 | # To make use of the static library instead of the shared one, one needs 9 | # to set the variable H5Z_ZFP_USE_STATIC_LIBS to ON before calling find_package. 10 | # Example: 11 | # set(H5Z_ZFP_USE_STATIC_LIBS ON) 12 | # find_package(H5Z_ZFP REQUIRED CONFIG) 13 | # 14 | # This will define the following variables: 15 | # 16 | # H5Z_ZFP_FOUND - True if the system has the H5Z_ZFP library. 17 | # H5Z_ZFP_WITH_OPENMP - True if the zfp library has been built with OpenMP support. 18 | # 19 | # and the following imported targets: 20 | # 21 | # h5z_zfp::h5z_zfp - The H5Z_ZFP library. 22 | 23 | find_path(H5Z_ZFP_INCLUDE_DIR NAMES H5Zzfp.h DOC "H5Z_ZFP include directory") 24 | if(H5Z_ZFP_USE_STATIC_LIBS) 25 | find_library(H5Z_ZFP_LIBRARY NAMES libh5zzfp.a DOC "H5Z_ZFP library") 26 | else() 27 | find_library(H5Z_ZFP_LIBRARY NAMES libh5zzfp.so HINTS $ENV{H5Z_ZFP_ROOT}/plugin DOC "H5Z_ZFP library") 28 | endif() 29 | 30 | include(FindPackageHandleStandardArgs) 31 | set(${CMAKE_FIND_PACKAGE_NAME}_CONFIG "${CMAKE_CURRENT_LIST_FILE}") 32 | find_package_handle_standard_args(H5Z_ZFP 33 | FOUND_VAR H5Z_ZFP_FOUND 34 | REQUIRED_VARS H5Z_ZFP_LIBRARY H5Z_ZFP_INCLUDE_DIR 35 | CONFIG_MODE 36 | ) 37 | 38 | if(H5Z_ZFP_FOUND) 39 | set(HDF5_USE_STATIC_LIBRARIES ${H5Z_ZFP_USE_STATIC_LIBS}) 40 | find_package(HDF5 MODULE REQUIRED COMPONENTS C) 41 | find_package(ZFP REQUIRED CONFIG) 42 | if(H5Z_ZFP_USE_STATIC_LIBS) 43 | add_library(h5z_zfp::h5z_zfp STATIC IMPORTED) 44 | else() 45 | add_library(h5z_zfp::h5z_zfp SHARED IMPORTED) 46 | endif() 47 | set_target_properties(h5z_zfp::h5z_zfp PROPERTIES 48 | IMPORTED_LOCATION "${H5Z_ZFP_LIBRARY}" 49 | INTERFACE_INCLUDE_DIRECTORIES "${H5Z_ZFP_INCLUDE_DIR}" 50 | INTERFACE_LINK_LIBRARIES "zfp::zfp;${HDF5_LIBRARIES}" 51 | LINK_LIBRARIES "zfp::zfp;${HDF5_LIBRARIES}" 52 | ) 53 | set(H5Z_ZFP_WITH_OPENMP ${ZFP_WITH_OPENMP}) 54 | endif() 55 | 56 | mark_as_advanced( 57 | H5Z_ZFP_INCLUDE_DIR 58 | H5Z_ZFP_LIBRARY 59 | ) 60 | -------------------------------------------------------------------------------- /src/Makefile: -------------------------------------------------------------------------------- 1 | ifneq ($(strip $(MAKECMDGOALS)),clean) 2 | include ../config.make 3 | endif 4 | 5 | .PHONY: all lib clean 6 | 7 | # The filter compiled to be used as a plugin 8 | H5Zzfp_plugin.o: H5Zzfp.c 9 | $(CC) -c $< -o $@ $(CFLAGS) -I$(H5Z_ZFP_BASE) -I$(ZFP_INC) -I$(HDF5_INC) 10 | 11 | # The filter compiled to be used as a library 12 | H5Zzfp_lib.o: H5Zzfp.c 13 | $(CC) -c $< -o $@ -DH5Z_ZFP_AS_LIB $(CFLAGS) -I$(H5Z_ZFP_BASE) -I$(ZFP_INC) -I$(HDF5_INC) 14 | 15 | # The plugin shared lib (HDF5 expects it to be named "lib*.{so,dylib}" 16 | plugin/libh5zzfp.$(SOEXT): H5Zzfp_plugin.o 17 | rm -rf plugin 18 | mkdir plugin 19 | $(CC) $< $(SHFLAG) -o plugin/libh5zzfp.$(SOEXT) \ 20 | $(PREPATH)$(HDF5_LIB) $(PREPATH)$(ZFP_LIB) \ 21 | -L$(ZFP_LIB) -L$(HDF5_LIB) -lhdf5 -lzfp $(LDFLAGS) 22 | 23 | # Alias target for filter plugin 24 | plugin: plugin/libh5zzfp.$(SOEXT) 25 | 26 | # C language properties interface 27 | H5Zzfp_props.o: H5Zzfp_props.c 28 | $(CC) -c $< -o $@ $(CFLAGS) -I$(H5Z_ZFP_BASE) -I$(ZFP_INC) -I$(HDF5_INC) 29 | 30 | # Fortran language properties interface 31 | H5Zzfp_props_f.o H5Zzfp_props_f.mod: H5Zzfp_props_f.F90 32 | $(FC) -c $< -o $@ $(FCFLAGS) -I$(H5Z_ZFP_BASE) -I$(ZFP_INC) -I$(HDF5_INC) 33 | 34 | # The filter member of the filter library 35 | libh5zzfp.a(H5Zzfp_lib.o): H5Zzfp_lib.o 36 | $(AR) cr libh5zzfp.a $< 37 | 38 | # The C properties interface member of the filter library 39 | libh5zzfp.a(H5Zzfp_props.o): H5Zzfp_props.o 40 | $(AR) cr libh5zzfp.a $< 41 | 42 | # The Fortran properties interface member of the filter library 43 | libh5zzfp.a(H5Zzfp_props_f.o): H5Zzfp_props_f.o 44 | $(AR) cr libh5zzfp.a $< 45 | 46 | # Alias target for filter library, conditionally includes Fortran 47 | LIBOBJ = libh5zzfp.a(H5Zzfp_lib.o) libh5zzfp.a(H5Zzfp_props.o) 48 | ifneq ($(FC),) 49 | LIBOBJ += libh5zzfp.a(H5Zzfp_props_f.o) 50 | endif 51 | lib: $(LIBOBJ) 52 | 53 | all: lib plugin 54 | 55 | install: all 56 | $(INSTALL) -d $(DESTDIR)$(PREFIX)/{plugin,include,lib} 57 | $(INSTALL) plugin/libh5zzfp.$(SOEXT) $(DESTDIR)$(PREFIX)/plugin 58 | $(INSTALL) libh5zzfp.a $(DESTDIR)$(PREFIX)/lib 59 | $(INSTALL) -m 644 H5Zzfp.h H5Zzfp_lib.h H5Zzfp_plugin.h H5Zzfp_version.h H5Zzfp_props.h $(DESTDIR)$(PREFIX)/include 60 | ifneq ($(FC),) 61 | $(INSTALL) -m 644 *.[mM][oO][dD] $(DESTDIR)$(PREFIX)/include 62 | endif 63 | 64 | clean: 65 | rm -rf plugin 66 | rm -f libh5zzfp.a *.o *.[mM][oO][dD] 67 | rm -f *.gcno *.gcda *.gcov 68 | rm -f H5Zzfp_props_f.mod 69 | -------------------------------------------------------------------------------- /docs/endian_issues.rst: -------------------------------------------------------------------------------- 1 | .. _endian-issues: 2 | 3 | ============= 4 | Endian Issues 5 | ============= 6 | 7 | This section describes some issues related to `endianness `__ of producers and consumers of the data processed by H5Z-ZFP_. 8 | This is likely less of an issue than it once was because almost all modern CPUs are `little-endian `__. 9 | 10 | That being said, the ZFP_ library writes an endian-independent stream. 11 | 12 | There is an unavoidable inefficiency when reading ZFP_ compressed data on a machine with a different endianness than the writer (e.g. a *mixed* endian context). Upon reading data from storage and decompressing the read stream with ZFP_, the correct endianness is returned in the result from ZFP_ before the buffer is handed back to HDF5_ from the decompression filter. 13 | This happens regardless of reader and writer endianness incompatibility. 14 | However, the HDF5_ library expects to get from H5Z-ZFP_ the endianness of the data as it was stored to the file on the writer machine and expects to have to byte-swap that buffer before returning to it an endian-incompatible caller. 15 | 16 | This means that in the H5Z-ZFP_ plugin, we wind up having to un-byte-swap an already correct result read in a cross-endian context. 17 | That way, when HDF5_ gets the data and byte-swaps it as it is expecting to, it will produce the correct final result. 18 | There is an endianness test in the Makefile and two ZFP_ compressed example datasets for big-endian and little-endian machines to test that cross-endian reads/writes work correctly. 19 | 20 | Again, because most CPUs are now little-endian and because ZFP_ became available only after the industry mostly moved away from big-endian, it is highly unlikely that this inefficiency will be triggered. 21 | 22 | Finally, *endian-targeting*, which is setting the file datatype for an endianness that is possibly different than the native endianness of the writer, is explicitly disallowed. 23 | For example, data may be produced on a big-endian system, but most consumers will be little-endian. 24 | Therefore, to alleviate downstream consumers from having to always byte-swap, it is desirable to byte-swap to little-endian when the data is written. 25 | However, the juxtaposition of HDF5_'s type conversion and filter operations in a pipeline makes this impractical for the H5Z-ZFP_ filter. 26 | The H5Z-ZFP_ filter will explicitly catch this condition, fail the compression and issue an error message. 27 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Include config.make only if we're not looking for help 2 | ifeq ($(findstring help, $(strip $(MAKECMDGOALS))),) 3 | ifeq ($(findstring tools, $(strip $(MAKECMDGOALS))),) 4 | ifeq ($(findstring clean, $(strip $(MAKECMDGOALS))),) 5 | ifeq ($(findstring dist, $(strip $(MAKECMDGOALS))),) 6 | include config.make 7 | endif 8 | endif 9 | endif 10 | endif 11 | 12 | # Get version string from H5Z-ZFP 13 | H5Z_ZFP_VERSINFO := $(shell grep '^\#define H5Z_FILTER_ZFP_VERSION_[MP]' src/H5Zzfp_version.h | cut -d' ' -f3 | tr '\n' '.' | cut -d'.' -f-3 2>/dev/null) 14 | 15 | .PHONY: help all clean dist install 16 | 17 | help: 18 | @echo "" 19 | @echo "" 20 | @echo "" 21 | @echo " This is H5Z-ZFP version $(H5Z_ZFP_VERSINFO)." 22 | @echo "See http://h5z-zfp.readthedocs.io/en/latest/ file for more info." 23 | @echo "" 24 | @echo "Typical make command is..." 25 | @echo "" 26 | @echo " make CC= HDF5_HOME= ZFP_HOME= PREFIX= all" 27 | @echo "" 28 | @echo "where is a dir whose children are include/lib/bin subdirs." 29 | @echo "HDF5_HOME can also be set using an INC,LIB,BIN triplet specifying" 30 | @echo "HDF5 include, library and binary dirs separated by commas." 31 | @echo "Standard make variables (e.g. CFLAGS, LD, etc.) can be set as usual." 32 | @echo "Optionally, add FC= to include Fortran support and tests." 33 | @echo "" 34 | @echo "Available make targets are..." 35 | @echo " all - build everything needed for H5Z-ZFP plugin/lib" 36 | @echo " check - all + run tests" 37 | @echo " tools - build tools (currently just print_h5repack_farg)" 38 | @echo " install - install plugin/lib" 39 | @echo " clean - clean away all derived targets" 40 | @echo " dist - create distribution tarfile" 41 | @echo " help - this help message" 42 | 43 | all: 44 | cd src; $(MAKE) $(MAKEVARS) $@ 45 | 46 | check: all 47 | cd test; $(MAKE) $(MAKEVARS) $@ 48 | 49 | tools: 50 | cd test; $(MAKE) $(MAKEVARS) print_h5repack_farg 51 | 52 | install: all 53 | cd src; $(MAKE) $(MAKEVARS) $@ 54 | 55 | clean: 56 | rm -f H5Z-ZFP-$(H5Z_ZFP_VERSINFO).tar.gz 57 | cd src; $(MAKE) $(MAKEVARS) $@ 58 | cd test; $(MAKE) $(MAKEVARS) $@ 59 | 60 | dist: clean 61 | rm -rf H5Z-ZFP-$(H5Z_ZFP_VERSINFO) H5Z-ZFP-$(H5Z_ZFP_VERSINFO).tar.gz; \ 62 | mkdir H5Z-ZFP-$(H5Z_ZFP_VERSINFO); \ 63 | tar cf - --exclude ".git*" --exclude H5Z-ZFP-$(H5Z_ZFP_VERSINFO) . | tar xf - -C H5Z-ZFP-$(H5Z_ZFP_VERSINFO); \ 64 | tar cvf - H5Z-ZFP-$(H5Z_ZFP_VERSINFO) | gzip --best > H5Z-ZFP-$(H5Z_ZFP_VERSINFO).tar.gz; \ 65 | rm -rf H5Z-ZFP-$(H5Z_ZFP_VERSINFO); 66 | -------------------------------------------------------------------------------- /src/H5Zzfp_plugin.h: -------------------------------------------------------------------------------- 1 | #ifndef H5Z_ZFP_PLUGIN_H 2 | #define H5Z_ZFP_PLUGIN_H 3 | 4 | #include "H5Zzfp_version.h" 5 | 6 | /* HDF5 generic cd_vals[] memory layout (6 unsigned ints) for 7 | controlling H5Z-ZFP behavior as a plugin. NOTE: These cd_vals 8 | used to pass properties in-memory from caller to filter via HDF5 9 | generic interface are NOT THE SAME AS the cd_vals[] that 10 | ultimately get stored to the file for the filter "header" data. 11 | 12 | cd_vals 0 1 2 3 4 5 13 | ---------------------------------------------------------------- 14 | rate: 1 unused rateA rateB unused unused 15 | precision: 2 unused prec unused unused unused 16 | accuracy: 3 unused accA accB unused unused 17 | expert: 4 unused minbits maxbits maxprec minexp 18 | 19 | A/B are high/low words of a double. 20 | */ 21 | 22 | #define H5Pset_zfp_rate_cdata(R, N, CD) \ 23 | do { if (N>=4) {double *p = (double *) &CD[2]; \ 24 | CD[0]=CD[1]=CD[2]=CD[3]=0; \ 25 | CD[0]=H5Z_ZFP_MODE_RATE; *p=R; N=4;}} while(0) 26 | 27 | #define H5Pget_zfp_rate_cdata(N, CD) \ 28 | ((double)(((N>=4)&&(CD[0]==H5Z_ZFP_MODE_RATE))?*((double *) &CD[2]):0)) 29 | 30 | #define H5Pset_zfp_precision_cdata(P, N, CD) \ 31 | do { if (N>=3) {CD[0]=H5Z_ZFP_MODE_PRECISION; \ 32 | CD[1]=0; CD[2]=P; N=3;}} while(0) 33 | 34 | #define H5Pget_zfp_precision_cdata(N, CD) \ 35 | ((double)(((N>=3)&&(CD[0]==H5Z_ZFP_MODE_PRECISION))?CD[2]:0)) 36 | 37 | #define H5Pset_zfp_accuracy_cdata(A, N, CD) \ 38 | do { if (N>=4) {double *p = (double *) &CD[2]; \ 39 | CD[0]=CD[1]=CD[2]=CD[3]=0; \ 40 | CD[0]=H5Z_ZFP_MODE_ACCURACY; *p=A; N=4;}} while(0) 41 | 42 | #define H5Pget_zfp_accuracy_cdata(N, CD) \ 43 | ((double)(((N>=4)&&(CD[0]==H5Z_ZFP_MODE_ACCURACY))?*((double *) &CD[2]):0)) 44 | 45 | #define H5Pset_zfp_expert_cdata(MiB, MaB, MaP, MiE, N, CD) \ 46 | do { if (N>=6) { CD[0]=CD[1]=CD[2]=CD[3]=CD[4]=CD[5]=0; \ 47 | CD[0]=H5Z_ZFP_MODE_EXPERT; \ 48 | CD[2]=MiB; CD[3]=MaB; CD[4]=MaP; \ 49 | CD[5]=(unsigned int)MiE; N=6;}} while(0) 50 | 51 | #define H5Pget_zfp_expert_cdata(N, CD, MiB, MaB, MaP, MiE) \ 52 | do { \ 53 | if ((N>=6)&&(CD[0] == H5Z_ZFP_MODE_EXPERT)) \ 54 | { \ 55 | unsigned int *p; int *q; \ 56 | p = &MiB; *p = CD[2]; \ 57 | p = &MaB; *p = CD[3]; \ 58 | p = &MaP; *p = CD[4]; \ 59 | q = &MiE; *q = (int) CD[5]; \ 60 | } \ 61 | } while(0) 62 | 63 | #define H5Pset_zfp_reversible_cdata(N, CD) \ 64 | do { if (N>=1) { \ 65 | CD[0]=H5Z_ZFP_MODE_REVERSIBLE; N=1;}} while(0) 66 | 67 | #define H5Pget_zfp_reversible_cdata(N, CD) \ 68 | ((int)(((N>=1)&&(CD[0]==H5Z_ZFP_MODE_REVERSIBLE))?1:0)) 69 | 70 | #endif 71 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Define headers. 2 | set(h5z_zfp_headers 3 | H5Zzfp.h H5Zzfp_lib.h H5Zzfp_plugin.h H5Zzfp_props.h H5Zzfp_version.h) 4 | 5 | #------------------------------------------------------------------------------# 6 | # Static library 7 | #------------------------------------------------------------------------------# 8 | # Define source 9 | set(h5z_zfp_lib_source 10 | ${h5z_zfp_headers} 11 | H5Zzfp_props_private.h 12 | H5Zzfp.c H5Zzfp_props.c) 13 | 14 | if (FORTRAN_INTERFACE) 15 | list(APPEND h5z_zfp_lib_source H5Zzfp_props_f.F90) 16 | endif () 17 | 18 | add_library(h5z_zfp_static STATIC ${h5z_zfp_lib_source}) 19 | target_compile_definitions(h5z_zfp_static PRIVATE H5Z_ZFP_AS_LIB) 20 | target_include_directories(h5z_zfp_static 21 | PUBLIC 22 | "$" 23 | "$" 24 | "$" 25 | ${HDF5_INCLUDE_DIRS} 26 | "$<$:${HDF5_FORTRAN_INCLUDE_DIRS}>" 27 | ) 28 | target_link_libraries(h5z_zfp_static PUBLIC zfp::zfp ${HDF5_LIBRARIES} $<$:${HDF5_FORTRAN_LIBRARIES}>) 29 | set_target_properties(h5z_zfp_static PROPERTIES POSITION_INDEPENDENT_CODE 1) 30 | set_target_properties(h5z_zfp_static PROPERTIES 31 | OUTPUT_NAME h5zzfp 32 | RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin 33 | ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib 34 | LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) 35 | install(TARGETS h5z_zfp_static 36 | RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} 37 | ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} 38 | LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) 39 | 40 | #------------------------------------------------------------------------------# 41 | # Shared library 42 | #------------------------------------------------------------------------------# 43 | # Define source 44 | set(h5z_zfp_plugin_source 45 | ${h5z_zfp_headers} 46 | H5Zzfp.c) 47 | 48 | add_library(h5z_zfp_shared SHARED ${h5z_zfp_plugin_source}) 49 | target_include_directories(h5z_zfp_shared 50 | PUBLIC 51 | $ 52 | $ 53 | $ 54 | ${HDF5_INCLUDE_DIRS}) 55 | target_link_libraries(h5z_zfp_shared PUBLIC zfp::zfp ${HDF5_LIBRARIES}) 56 | set_target_properties(h5z_zfp_shared PROPERTIES 57 | OUTPUT_NAME h5zzfp 58 | RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/plugin 59 | ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/plugin 60 | LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/plugin) 61 | install(TARGETS h5z_zfp_shared 62 | RUNTIME DESTINATION plugin 63 | ARCHIVE DESTINATION plugin 64 | LIBRARY DESTINATION plugin) 65 | 66 | #------------------------------------------------------------------------------# 67 | # Install header and module files. 68 | #------------------------------------------------------------------------------# 69 | install(FILES ${h5z_zfp_headers} 70 | DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) 71 | 72 | if (FORTRAN_INTERFACE) 73 | install(FILES ${CMAKE_BINARY_DIR}/bin/h5zzfp_props_f.mod 74 | DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) 75 | endif () 76 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Lawrence Livermore National Security, LLC. 2 | Produced at the Lawrence Livermore National Laboratory 3 | Written by Mark C. Miller, miller86@llnl.gov 4 | LLNL-CODE-707197 All rights reserved. 5 | 6 | This file is part of H5Z-ZFP. For details, see 7 | https://github.com/LLNL/H5Z-ZFP. Please also read the Additional 8 | BSD Notice. 9 | 10 | Redistribution and use in source and binary forms, with or without 11 | modification, are permitted provided that the following conditions are 12 | met: 13 | 14 | * Redistributions of source code must retain the above copyright 15 | notice, this list of conditions and the disclaimer below. 16 | 17 | * Redistributions in binary form must reproduce the above copyright 18 | notice, this list of conditions and the disclaimer (as noted below) 19 | in the documentation and/or other materials provided with the 20 | distribution. 21 | 22 | * Neither the name of the LLNS/LLNL nor the names of its contributors 23 | may be used to endorse or promote products derived from this 24 | software without specific prior written permission. 25 | 26 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE 30 | LIVERMORE NATIONAL SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR 31 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 32 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 33 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 34 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 35 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 36 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 37 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38 | 39 | Additional BSD Notice 40 | 41 | 1. This notice is required to be provided under our contract with the 42 | U.S. Department of Energy (DOE). This work was produced at Lawrence 43 | Livermore National Laboratory under Contract No. DE-AC52-07NA27344 44 | with the DOE. 45 | 46 | 2. Neither the United States Government nor Lawrence Livermore 47 | National Security, LLC nor any of their employees, makes any warranty, 48 | express or implied, or assumes any liability or responsibility for the 49 | accuracy, completeness, or usefulness of any information, apparatus, 50 | product, or process disclosed, or represents that its use would not 51 | infringe privately-owned rights. 52 | 53 | 3. Also, reference herein to any specific commercial products, 54 | process, or services by trade name, trademark, manufacturer or 55 | otherwise does not necessarily constitute or imply its endorsement, 56 | recommendation, or favoring by the United States Government or 57 | Lawrence Livermore National Security, LLC. The views and opinions of 58 | authors expressed herein do not necessarily state or reflect those of 59 | the United States Government or Lawrence Livermore National Security, 60 | LLC, and shall not be used for advertising or product endorsement 61 | purposes. 62 | -------------------------------------------------------------------------------- /docs/cd_vals.rst: -------------------------------------------------------------------------------- 1 | ======================================= 2 | H5Z-ZFP and the HDF5 filter's cd_values 3 | ======================================= 4 | 5 | .. note:: 6 | The details described here are likely relevant only to *developers* of the H5Z-ZFP_ filter. 7 | If you just want to *use* the filter, you can ignore this material. 8 | 9 | The HDF5_ library uses an array of values, named ``cd_values`` in formal arguments documenting various API functions, for managing *auxiliary data* for a filter. 10 | Instances of this ``cd_values`` array are used in two subtly different ways within HDF5. 11 | 12 | The first use is in *passing* auxiliary data for a filter from the caller to the library when initially creating a dataset. 13 | This happens *directly* in an ``H5Pset_filter()`` (`see here `_) call. 14 | 15 | The second use is in *persisting* auxiliary data for a filter to the dataset's object *header* in a file. 16 | This happens *indirectly* as part of an ``H5Dcreate()`` call. 17 | 18 | When a dataset creation property list includes a filter, the filter's ``set_local()`` method is called (see `H5Zregister() `_) as part of the ``H5Dcreate`` call. 19 | In the filter's ``set_local()`` method, the ``cd_values`` that were *passed* by the caller (in ``H5Pset_filter()``) are often modified (via ``H5Pmodify_filter()`` (`see here `__) before they are *persisted* to the dataset's object header in a file. 20 | 21 | Among other things, this design allows a filter to be generally configured for *any* dataset in a file and then adjusted as necessary to handle such things as data type and/or dimensions when it is applied to a specific dataset. 22 | Long story short, the data stored in ``cd_values`` of the dataset object's header in the file are often not the same values passed by the caller when the dataset was created. 23 | 24 | To make matters a tad more complex, the ``cd_values`` data is treated by HDF5_ as an array of C typed, 4-byte, ``unsigned integer`` values. 25 | Furthermore, regardless of `endianness `__ of the data producer, the persisted values are always stored in little-endian format in the dataset object header in the file. 26 | Nonetheless, if the persisted ``cd_values`` data is ever retrieved (e.g. via ``H5Pget_filter_by_id()`` (`see here `__), the HDF5_ library ensures the data is returned to callers with proper endianness. 27 | When command-line tools like ``h5ls`` and ``h5dump`` print ``cd_values``, the data will be displayed correctly. 28 | 29 | Handling double precision auxiliary data via ``cd_values`` is still more complicated because a single double precision value will span multiple entries in ``cd_values`` in almost all cases. 30 | Setting aside the possibility of differing floating point formats between the producer and consumers, any endianness handling the HDF5_ library does for the 4-byte entries in ``cd_values`` will certainly not ensure proper endianness handling of larger values. 31 | It is impossible for command-line tools like ``h5ls`` and ``h5dump`` to display such data correctly. 32 | 33 | Fortunately, the ZFP_ library has already been designed to handle these issues as part of the ZFP_'s *native* stream header. 34 | But, the ZFP_ library handles these issues in an endian-agnostic way. 35 | Consequently, the H5Z-ZFP_ filter uses the ``cd_values`` that is persisted to a dataset's object header to store ZFP_'s stream header. 36 | ZFP_'s stream header is stored starting at ``&cd_values[1]``. 37 | ``cd_values[0]`` is used to stored H5Z-ZFP_ filter and ZFP_ library and ZFP_ encoder version information. 38 | 39 | This also means that H5Z-ZFP_ avoids the overhead of duplicating the ZFP_ stream header in each dataset chunk. 40 | For larger chunks, these savings are probably not too terribly significant. 41 | -------------------------------------------------------------------------------- /docs/tests.rst: -------------------------------------------------------------------------------- 1 | ================== 2 | Tests and Examples 3 | ================== 4 | 5 | The tests directory contains a few simple tests of the H5Z-ZFP_ filter some 6 | of which also serve as decent examples. 7 | 8 | The test client, `test_write.c `_ 9 | is compiled a couple of different ways. 10 | One target is ``test_write_plugin`` which demonstrates the use of this filter as 11 | a standalone plugin. The other target, ``test_write_lib``, demonstrates the use 12 | of the filter as an explicitly linked library. By default, these test a simple 13 | 1D array with and without ZFP_ compression using either the :ref:`generic-interface` (for plugin) 14 | or the :ref:`properties-interface` (for library). You can use the code there as an 15 | example of using the ZFP_ filter either as a plugin or as a library. However, these 16 | also include some advanced usages for 4D and 6D, time-varying (e.g. *extendable*) 17 | datasets. The command ``test_write_lib help`` or ``test_write_plugin help`` will print a 18 | list of the example's options and how to use them. 19 | 20 | ------------------ 21 | Write Test Options 22 | ------------------ 23 | 24 | :: 25 | 26 | ./test/test_write_lib --help 27 | ifile="" set input filename 28 | ofile="test_zfp.h5" set output filename 29 | 30 | 1D dataset generation arguments... 31 | npoints=1024 set number of points for 1D dataset 32 | noise=0.001 set amount of random noise in 1D dataset 33 | amp=17.7 set amplitude of sinusoid in 1D dataset 34 | chunk=256 set chunk size for 1D dataset 35 | doint=0 also do integer 1D data 36 | 37 | ZFP compression parameters... 38 | zfpmode=3 (1=rate,2=prec,3=acc,4=expert,5=reversible) 39 | rate=4 set rate for rate mode of filter 40 | acc=0 set accuracy for accuracy mode of filter 41 | prec=11 set precision for precision mode of zfp filter 42 | minbits=0 set minbits for expert mode of zfp filter 43 | maxbits=4171 set maxbits for expert mode of zfp filter 44 | maxprec=64 set maxprec for expert mode of zfp filter 45 | minexp=-1074 set minexp for expert mode of zfp filter 46 | 47 | Advanced cases... 48 | highd=0 run 4D case 49 | sixd=0 run 6D extendable case (requires ZFP>=0.5.4) 50 | help=0 this help message 51 | 52 | The test normally just tests compression of 1D array of integer 53 | and double precision data of a sinusoidal array with a small 54 | amount of additive random noise. The ``highd`` test runs a test 55 | on a 4D dataset where two of the 4 dimensions are not correlated. 56 | This tests the plugin's ability to properly set chunking for 57 | HDF5 such that chunks span **only** correlated dimensions and 58 | have non-unity sizes in 3 or fewer dimensions. The ``sixd`` 59 | test runs a test on a 6D, extendable dataset representing an 60 | example of using ZFP_ for compression along the *time* axis. 61 | 62 | There is a companion, `test_read.c `_ 63 | which is compiled into ``test_read_plugin`` 64 | and ``test_read_lib`` which demonstrates use of the filter reading data as a 65 | plugin or library. Also, the commands ``test_read_lib help`` and 66 | ``test_read_plugin help`` will print a list of the command line options. 67 | 68 | To use the plugin examples, you need to tell the HDF5_ library where to find the 69 | H5Z-ZFP_ plugin with the ``HDF5_PLUGIN_PATH`` environment variable. The value you 70 | pass is the path to the directory containing the plugin shared library. 71 | 72 | Finally, there is a Fortran test example, 73 | `test_rw_fortran.F90 `_. 74 | The Fortran test writes and reads a 2D dataset. However, the Fortran test is designed to 75 | use the filter **only** as a library and not as a plugin. The reason for this is 76 | that the filter controls involve passing combinations of integer and floating 77 | point data from Fortran callers and this can be done only through the 78 | :ref:`properties-interface`, which by its nature requires any Fortran application 79 | to have to link with an implementation of that interface. Since we need to link 80 | extra code for Fortran, we may as well also link to the filter itself alleviating 81 | the need to use the filter as a plugin. Also, if you want to use Fortran support, 82 | the HDF5_ library must have, of course, been configured and built with Fortran support 83 | as well. 84 | 85 | In addition, a number tests are performed in the Makefile which test the plugin 86 | by using some of the HDF5_ tools such as ``h5dump`` and ``h5repack``. Again, to 87 | use these tools to read data compressed with the H5Z-ZFP_ filter, you will need 88 | to inform the HDF5_ library where to find the filter plugin. For example.. 89 | 90 | :: 91 | 92 | env HDF5_PLUGIN_PATH= h5ls test_zfp.h5 93 | 94 | Where ```` is the relative or absolute path to a directory containing the 95 | filter plugin shared library. 96 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | 3 | # Fail immediately if not using an out-of-source build 4 | if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) 5 | message(FATAL_ERROR 6 | "In-source builds are not supported. Please create a build directory " 7 | "separate from the source directory") 8 | endif () 9 | 10 | if (POLICY CMP0074) 11 | # find_package() uses _ROOT variables. 12 | cmake_policy (SET CMP0074 NEW) 13 | endif () 14 | 15 | if (POLICY CMP0083) 16 | # To control generation of Position Independent Executable (PIE) or not, 17 | # some flags are required at link time. 18 | cmake_policy (SET CMP0083 NEW) 19 | endif () 20 | 21 | # Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24: 22 | if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") 23 | cmake_policy(SET CMP0135 NEW) 24 | endif() 25 | 26 | get_property(_isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) 27 | 28 | #------------------------------------------------------------------------------# 29 | # Parse version number from H5Zzfp_version.h 30 | #------------------------------------------------------------------------------# 31 | file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/src/H5Zzfp_version.h" H5Z_ZFP_H REGEX "^\#define H5Z_FILTER_ZFP_VERSION_MAJOR") 32 | string(REGEX REPLACE "^.*MAJOR " "" H5Z_ZFP_VERSION_MAJOR "${H5Z_ZFP_H}") 33 | file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/src/H5Zzfp_version.h" H5Z_ZFP_H REGEX "^\#define H5Z_FILTER_ZFP_VERSION_MINOR") 34 | string(REGEX REPLACE "^.*MINOR " "" H5Z_ZFP_VERSION_MINOR "${H5Z_ZFP_H}") 35 | file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/src/H5Zzfp_version.h" H5Z_ZFP_H REGEX "^\#define H5Z_FILTER_ZFP_VERSION_PATCH") 36 | string(REGEX REPLACE "^.*PATCH " "" H5Z_ZFP_VERSION_PATCH "${H5Z_ZFP_H}") 37 | set(H5Z_ZFP_VERSION "${H5Z_ZFP_VERSION_MAJOR}.${H5Z_ZFP_VERSION_MINOR}.${H5Z_ZFP_VERSION_PATCH}") 38 | 39 | project(H5Z_ZFP VERSION ${H5Z_ZFP_VERSION} LANGUAGES C) 40 | 41 | # setup common output folders 42 | include (cmake/HDFMacros.cmake) 43 | SET_HDF_BUILD_TYPE() 44 | DEFAULT_FOLDERS() 45 | 46 | #------------------------------------------------------------------------------# 47 | # Compile options. 48 | #------------------------------------------------------------------------------# 49 | option(FORTRAN_INTERFACE "Enable the Fortran interface" ON) 50 | if (FORTRAN_INTERFACE) 51 | enable_language(Fortran) 52 | endif () 53 | #------------------------------------------------------------------------------# 54 | # Some boilerplate to setup nice output directories 55 | #------------------------------------------------------------------------------# 56 | set(CMAKE_INSTALL_BINDIR bin) 57 | set(CMAKE_INSTALL_LIBDIR lib) 58 | set(CMAKE_INSTALL_INCLUDEDIR include) 59 | set(CMAKE_INSTALL_CMAKEDIR lib/cmake/h5z_zfp) 60 | 61 | #------------------------------------------------------------------------------# 62 | # Required packages 63 | #------------------------------------------------------------------------------# 64 | # Find HDF5, relies on HDF5_DIR or HDF5_ROOT being set in environment. 65 | HDF5_SUPPORT () 66 | if (NOT HDF5_FOUND) 67 | message (FATAL_ERROR "HDF5 is Required for h5z_zfp") 68 | endif () 69 | 70 | # Find MPI depending on if HDF5 needs MPI. 71 | if ((HDF5_IS_PARALLEL OR HDF5_ENABLE_PARALLEL) AND ("${HDF5_VERSION}" VERSION_EQUAL "1.14.0")) 72 | find_package(MPI REQUIRED COMPONENTS C) 73 | endif() 74 | 75 | # Find ZFP, relies on ZFP_DIR or ZFP_ROOT being set in environment. 76 | find_package(ZFP REQUIRED CONFIG) 77 | if (NOT ZFP_FOUND) 78 | message (FATAL_ERROR "ZFP is Required for h5z_zfp") 79 | endif () 80 | 81 | #------------------------------------------------------------------------------# 82 | # Add source 83 | #------------------------------------------------------------------------------# 84 | add_subdirectory(${CMAKE_SOURCE_DIR}/src) 85 | 86 | #----------------------------------------------------------------------------- 87 | # Dashboard and Testing Settings 88 | #----------------------------------------------------------------------------- 89 | option (BUILD_TESTING "Build h5z-zfp Unit Testing" OFF) 90 | if (BUILD_TESTING) 91 | set (DART_TESTING_TIMEOUT 1200 CACHE STRING 92 | "Timeout in seconds for each test (default 1200=20minutes)") 93 | enable_testing () 94 | include (CTest) 95 | 96 | add_subdirectory(${CMAKE_SOURCE_DIR}/test) 97 | endif () 98 | 99 | #------------------------------------------------------------------------------# 100 | # Packaging 101 | #------------------------------------------------------------------------------# 102 | # Install h5z_zfp-config.cmake and h5z_zfp-config-version.cmake 103 | set(H5Z_ZFP_CONFIG_IN ${CMAKE_CURRENT_SOURCE_DIR}/cmake/h5z_zfp-config.cmake.in) 104 | set(H5Z_ZFP_CONFIG_OUT ${CMAKE_CURRENT_BINARY_DIR}/cmake/h5z_zfp-config.cmake) 105 | configure_file(${H5Z_ZFP_CONFIG_IN} ${H5Z_ZFP_CONFIG_OUT} @ONLY) 106 | set(H5Z_ZFP_CONFIG_VERSION_IN ${CMAKE_CURRENT_SOURCE_DIR}/cmake/h5z_zfp-config-version.cmake.in) 107 | set(H5Z_ZFP_CONFIG_VERSION_OUT ${CMAKE_CURRENT_BINARY_DIR}/cmake/h5z_zfp-config-version.cmake) 108 | configure_file(${H5Z_ZFP_CONFIG_VERSION_IN} ${H5Z_ZFP_CONFIG_VERSION_OUT} @ONLY) 109 | install(FILES ${H5Z_ZFP_CONFIG_OUT} 110 | DESTINATION ${CMAKE_INSTALL_CMAKEDIR}) 111 | install(FILES ${H5Z_ZFP_CONFIG_VERSION_OUT} 112 | DESTINATION ${CMAKE_INSTALL_CMAKEDIR}) 113 | -------------------------------------------------------------------------------- /test/print_h5repack_farg.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2016, Lawrence Livermore National Security, LLC. 3 | Produced at the Lawrence Livermore National Laboratory 4 | Written by Mark C. Miller, miller86@llnl.gov 5 | LLNL-CODE-707197. All rights reserved. 6 | 7 | This file is part of H5Z-ZFP. Please also read the BSD license 8 | https://raw.githubusercontent.com/LLNL/H5Z-ZFP/master/LICENSE 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #if defined(_WIN32) || defined(_WIN64) 16 | #define strncasecmp _strnicmp 17 | #define strcasecmp _stricmp 18 | #endif 19 | 20 | typedef unsigned int uint; 21 | 22 | #include "H5Zzfp_plugin.h" 23 | 24 | #define NAME_LEN 256 25 | 26 | /* convenience macro to handle command-line args and help */ 27 | #define HANDLE_SEP(SEPSTR) \ 28 | { \ 29 | char tmpstr[64]; \ 30 | int len = snprintf(tmpstr, sizeof(tmpstr), "\n%s...", #SEPSTR);\ 31 | printf(" %*s\n",60-len,tmpstr); \ 32 | } 33 | 34 | #define HANDLE_ARG(A,PARSEA,PRINTA,HELPSTR) \ 35 | { \ 36 | int i; \ 37 | char tmpstr[64]; \ 38 | int len; \ 39 | int len2 = strlen(#A)+1; \ 40 | for (i = 0; i < argc; i++) \ 41 | { \ 42 | if (!strncmp(argv[i], #A"=", len2)) \ 43 | { \ 44 | A = PARSEA; \ 45 | break; \ 46 | } \ 47 | else if (!strncasecmp(argv[i], "help", 4)) \ 48 | { \ 49 | return 0; \ 50 | } \ 51 | } \ 52 | len = snprintf(tmpstr, sizeof(tmpstr), "%s=" PRINTA, #A, A);\ 53 | printf(" %s%*s\n",tmpstr,60-len,#HELPSTR); \ 54 | } 55 | 56 | static void print_cdvals(int zfpmode, double rate, double acc, uint prec, 57 | uint minbits, uint maxbits, uint maxprec, int minexp) 58 | { 59 | unsigned int cd_values[10] = {0,0,0,0,0,0,0,0,0,0}; 60 | int i, cd_nelmts = 10; 61 | 62 | /* setup zfp filter via generic (cd_values) interface */ 63 | if (zfpmode == H5Z_ZFP_MODE_RATE) 64 | H5Pset_zfp_rate_cdata(rate, cd_nelmts, cd_values); 65 | else if (zfpmode == H5Z_ZFP_MODE_PRECISION) 66 | H5Pset_zfp_precision_cdata(prec, cd_nelmts, cd_values); 67 | else if (zfpmode == H5Z_ZFP_MODE_ACCURACY) 68 | H5Pset_zfp_accuracy_cdata(acc, cd_nelmts, cd_values); 69 | else if (zfpmode == H5Z_ZFP_MODE_EXPERT) 70 | H5Pset_zfp_expert_cdata(minbits, maxbits, maxprec, minexp, cd_nelmts, cd_values); 71 | else if (zfpmode == H5Z_ZFP_MODE_REVERSIBLE) 72 | H5Pset_zfp_reversible_cdata(cd_nelmts, cd_values); 73 | else 74 | return; 75 | 76 | /* h5repack -f argument format... 77 | 78 | h5repack -f UD=32013,0,6,3,0,3539053052,1062232653,0,0 79 | */ 80 | printf("\nh5repack -f argument...\n"); 81 | printf(" -f UD=%u,0,%u", H5Z_FILTER_ZFP, cd_nelmts); 82 | for (i = 0; i < cd_nelmts; i++) 83 | printf(",%u", cd_values[i]); 84 | printf("\n"); 85 | } 86 | 87 | int main(int argc, char **argv) 88 | { 89 | /* compression parameters (defaults taken from ZFP header) */ 90 | int zfpmode = 1; 91 | double rate = 3.5; 92 | double acc = 0; 93 | uint prec = 0; 94 | uint minbits = 0; 95 | uint maxbits = 0; 96 | uint maxprec = 0; 97 | int minexp = 0; 98 | int help = 0; 99 | 100 | /* ZFP filter arguments */ 101 | HANDLE_SEP(Print cdvals for set of ZFP compression parameters) 102 | HANDLE_ARG(zfpmode,(int) strtol(argv[i]+len2,0,10),"%d",set zfp mode (1=rate,2=prec,3=acc,4=expert,5=rev)); 103 | HANDLE_ARG(rate,(double) strtod(argv[i]+len2,0),"%g",set rate for rate mode of filter); 104 | HANDLE_ARG(acc,(double) strtod(argv[i]+len2,0),"%g",set accuracy for accuracy mode of filter); 105 | HANDLE_ARG(prec,(uint) strtol(argv[i]+len2,0,10),"%u",set precision for precision mode of zfp filter); 106 | HANDLE_ARG(minbits,(uint) strtol(argv[i]+len2,0,10),"%u",set minbits for expert mode of zfp filter); 107 | HANDLE_ARG(maxbits,(uint) strtol(argv[i]+len2,0,10),"%u",set maxbits for expert mode of zfp filter); 108 | HANDLE_ARG(maxprec,(uint) strtol(argv[i]+len2,0,10),"%u",set maxprec for expert mode of zfp filter); 109 | HANDLE_ARG(minexp,(int) strtol(argv[i]+len2,0,10),"%d",set minexp for expert mode of zfp filter); 110 | HANDLE_ARG(help,(int)strtol(argv[i]+len2,0,10),"%d",this help message); /* must be last */ 111 | 112 | if (!help) 113 | print_cdvals(zfpmode, rate, acc, prec, minbits, maxbits, maxprec, minexp); 114 | 115 | return 0; 116 | } 117 | -------------------------------------------------------------------------------- /test/test_common.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2016, Lawrence Livermore National Security, LLC. 3 | Produced at the Lawrence Livermore National Laboratory 4 | Written by Mark C. Miller, miller86@llnl.gov 5 | LLNL-CODE-707197. All rights reserved. 6 | 7 | This file is part of H5Z-ZFP. Please also read the BSD license 8 | https://raw.githubusercontent.com/LLNL/H5Z-ZFP/master/LICENSE 9 | */ 10 | 11 | #ifndef test_common_H 12 | #define test_common_H 13 | 14 | #ifndef _GNU_SOURCE 15 | #define _GNU_SOURCE /* ahead of ALL headers to take proper effect */ 16 | #endif 17 | 18 | #ifdef _MSC_VER 19 | #define _CRT_SECURE_NO_DEPRECATE 20 | #define _USE_MATH_DEFINES 21 | #include 22 | #define j0 _j0 23 | #include 24 | #define strncasecmp _strnicmp 25 | #define strcasecmp _stricmp 26 | #include 27 | #pragma comment(lib, "Shlwapi.lib") 28 | #define strcasestr StrStrIA 29 | #define srandom(X) srand(X) 30 | #define random rand 31 | #define read _read 32 | #define open _open 33 | #define close _close 34 | // strndup() is not available on Windows 35 | char *strndup( const char *s1, size_t n) 36 | { 37 | size_t len = strlen(s1); 38 | size_t copy_len = len < n ? len : n; 39 | char *copy = (char*)malloc(copy_len + 1); 40 | memcpy(copy, s1, copy_len); 41 | copy[copy_len] = 0; 42 | return copy; 43 | }; 44 | #else 45 | #include 46 | #include 47 | #endif 48 | 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include "hdf5.h" 56 | 57 | #define NAME_LEN 256 58 | 59 | /* convenience macro to handle command-line args and help */ 60 | #define HANDLE_SEP(SEPSTR) \ 61 | { \ 62 | char tmpstr[64]; \ 63 | int len = snprintf(tmpstr, sizeof(tmpstr), "\n%s...", #SEPSTR);\ 64 | printf(" %*s\n",60-len,tmpstr); \ 65 | } 66 | 67 | #define HANDLE_ARG(A,PARSEA,PRINTA,HELPSTR) \ 68 | { \ 69 | int i; \ 70 | char tmpstr[64]; \ 71 | int len; \ 72 | int len2 = strlen(#A)+1; \ 73 | for (i = 0; i < argc; i++) \ 74 | { \ 75 | if (!strncasecmp(#A,"help",4) && strstr(argv[i],"help"))\ 76 | { \ 77 | return 0; \ 78 | } \ 79 | else if (!strncmp(argv[i], #A"=", len2)) \ 80 | { \ 81 | A = PARSEA; \ 82 | break; \ 83 | } \ 84 | } \ 85 | len = snprintf(tmpstr, sizeof(tmpstr), "%s=" PRINTA, #A, A);\ 86 | printf(" %s%*s\n",tmpstr,60-len,#HELPSTR); \ 87 | } 88 | 89 | 90 | /* convenience macro to handle errors */ 91 | #ifdef _MSC_VER 92 | 93 | #define SET_ERROR(FNAME) \ 94 | do { \ 95 | size_t errmsglen = 94; \ 96 | char errmsg[errmsglen]; \ 97 | strerror_s(errmsg, errmsglen, errno); \ 98 | fprintf(stderr, #FNAME " failed at line %d, errno=%d (%s)\n", \ 99 | __LINE__, errno, errno?errmsg:"ok"); \ 100 | return 1; \ 101 | } while(0) 102 | 103 | #else 104 | 105 | #define SET_ERROR(FNAME) \ 106 | do { \ 107 | int _errno = errno; \ 108 | fprintf(stderr, #FNAME " failed at line %d, errno=%d (%s)\n", \ 109 | __LINE__, _errno, _errno?strerror(_errno):"ok"); \ 110 | return 1; \ 111 | } while(0) 112 | 113 | #endif 114 | 115 | /* Generate a simple, 1D sinusioidal data array with some noise */ 116 | #define TYPINT 1 117 | #define TYPDBL 2 118 | static int gen_data(size_t npoints, double noise, double amp, void **_buf, int typ) 119 | { 120 | size_t i; 121 | double *pdbl = 0; 122 | int *pint = 0; 123 | 124 | /* create data buffer to write */ 125 | if (typ == TYPINT) 126 | pint = (int *) malloc(npoints * sizeof(int)); 127 | else 128 | pdbl = (double *) malloc(npoints * sizeof(double)); 129 | srandom(0xDeadBeef); 130 | for (i = 0; i < npoints; i++) 131 | { 132 | double x = 2 * M_PI * (double) i / (double) (npoints-1); 133 | double n = noise * ((double) random() / ((double)(1<<31)-1) - 0.5); 134 | if (typ == TYPINT) 135 | pint[i] = (int) (amp * (1 + sin(x)) + n); 136 | else 137 | pdbl[i] = (double) (amp * (1 + sin(x)) + n); 138 | } 139 | if (typ == TYPINT) 140 | *_buf = pint; 141 | else 142 | *_buf = pdbl; 143 | return 0; 144 | } 145 | #endif /* test_common_H */ 146 | -------------------------------------------------------------------------------- /src/H5Zzfp_props.c: -------------------------------------------------------------------------------- 1 | #include "H5Zzfp_plugin.h" 2 | #include "H5Zzfp_props_private.h" 3 | 4 | #include "hdf5.h" 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #define H5Z_ZFP_PUSH_AND_GOTO(MAJ, MIN, RET, MSG) \ 11 | do \ 12 | { \ 13 | H5Epush(H5E_DEFAULT,__FILE__,_funcname_,__LINE__, \ 14 | H5E_ERR_CLS_g,MAJ,MIN,MSG); \ 15 | retval = RET; \ 16 | goto done; \ 17 | } while(0) 18 | 19 | static herr_t H5Pset_zfp(hid_t plist, int mode, ...) 20 | { 21 | static char const *_funcname_ = "H5Pset_zfp"; 22 | static size_t const ctrls_sz = sizeof(h5z_zfp_controls_t); 23 | unsigned int flags; 24 | size_t cd_nelmts = 0; 25 | unsigned int cd_values[1]; 26 | h5z_zfp_controls_t *ctrls_p = 0; 27 | int i; 28 | va_list ap; 29 | herr_t retval = 0; 30 | 31 | if (0 >= H5Pisa_class(plist, H5P_DATASET_CREATE)) 32 | H5Z_ZFP_PUSH_AND_GOTO(H5E_ARGS, H5E_BADTYPE, -1, "not a dataset creation property list class"); 33 | 34 | ctrls_p = (h5z_zfp_controls_t *) malloc(ctrls_sz); 35 | if (0 == ctrls_p) 36 | H5Z_ZFP_PUSH_AND_GOTO(H5E_RESOURCE, H5E_NOSPACE, -1, "allocation failed for ZFP controls"); 37 | 38 | va_start(ap, mode); 39 | ctrls_p->mode = mode; 40 | switch (mode) 41 | { 42 | case H5Z_ZFP_MODE_RATE: 43 | { 44 | ctrls_p->details.rate = va_arg(ap, double); 45 | if (0 > ctrls_p->details.rate) 46 | H5Z_ZFP_PUSH_AND_GOTO(H5E_ARGS, H5E_BADVALUE, -1, "rate out of range."); 47 | break; 48 | } 49 | case H5Z_ZFP_MODE_ACCURACY: 50 | { 51 | ctrls_p->details.acc = va_arg(ap, double); 52 | if (0 > ctrls_p->details.acc) 53 | H5Z_ZFP_PUSH_AND_GOTO(H5E_ARGS, H5E_BADVALUE, -1, "accuracy out of range."); 54 | break; 55 | } 56 | case H5Z_ZFP_MODE_PRECISION: 57 | { 58 | ctrls_p->details.prec = va_arg(ap, unsigned int); 59 | break; 60 | } 61 | case H5Z_ZFP_MODE_EXPERT: 62 | { 63 | ctrls_p->details.expert.minbits = va_arg(ap, unsigned int); 64 | ctrls_p->details.expert.maxbits = va_arg(ap, unsigned int); 65 | ctrls_p->details.expert.maxprec = va_arg(ap, unsigned int); 66 | ctrls_p->details.expert.minexp = va_arg(ap, int); 67 | break; 68 | } 69 | case H5Z_ZFP_MODE_REVERSIBLE: 70 | { 71 | break; 72 | } 73 | default: 74 | { 75 | H5Z_ZFP_PUSH_AND_GOTO(H5E_ARGS, H5E_BADVALUE, -1, "bad ZFP mode."); 76 | break; 77 | } 78 | } 79 | va_end(ap); 80 | 81 | for (i = 0; i < H5Pget_nfilters(plist); i++) 82 | { 83 | H5Z_filter_t fid; 84 | if (0 <= (fid = H5Pget_filter(plist, i, &flags, &cd_nelmts, cd_values, 0, 0, 0)) 85 | && fid == H5Z_FILTER_ZFP) 86 | { 87 | if (0 > H5Premove_filter(plist, H5Z_FILTER_ZFP)) 88 | H5Z_ZFP_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, -1, "Unable to remove old ZFP filter from pipeline."); 89 | break; 90 | } 91 | } 92 | 93 | if (0 > H5Pset_filter(plist, H5Z_FILTER_ZFP, H5Z_FLAG_MANDATORY, 0, 0)) 94 | H5Z_ZFP_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, -1, "Unable to put ZFP filter in pipeline."); 95 | 96 | if (0 == H5Pexist(plist, "zfp_controls")) 97 | { 98 | retval = H5Pinsert2(plist, "zfp_controls", ctrls_sz, ctrls_p, 0, 0, 0, 0, 0, 0); 99 | } 100 | else 101 | { 102 | retval = H5Pset(plist, "zfp_controls", ctrls_p); 103 | } 104 | 105 | /* HDF5 copies the memory we gave it */ 106 | free(ctrls_p); 107 | 108 | return retval; 109 | 110 | done: 111 | 112 | if (ctrls_p) 113 | free(ctrls_p); 114 | 115 | return retval; 116 | } 117 | 118 | herr_t H5Pset_zfp_rate(hid_t plist, double rate) 119 | { 120 | return H5Pset_zfp(plist, H5Z_ZFP_MODE_RATE, rate); 121 | } 122 | 123 | herr_t H5Pset_zfp_precision(hid_t plist, unsigned int prec) 124 | { 125 | return H5Pset_zfp(plist, H5Z_ZFP_MODE_PRECISION, prec); 126 | } 127 | 128 | herr_t H5Pset_zfp_accuracy(hid_t plist, double acc) 129 | { 130 | return H5Pset_zfp(plist, H5Z_ZFP_MODE_ACCURACY, acc); 131 | } 132 | 133 | herr_t H5Pset_zfp_expert(hid_t plist, unsigned int minbits, unsigned int maxbits, 134 | unsigned int maxprec, int minexp) 135 | { 136 | return H5Pset_zfp(plist, H5Z_ZFP_MODE_EXPERT, minbits, maxbits, maxprec, minexp); 137 | } 138 | 139 | herr_t H5Pset_zfp_reversible(hid_t plist) 140 | { 141 | return H5Pset_zfp(plist, H5Z_ZFP_MODE_REVERSIBLE); 142 | } 143 | 144 | 145 | /* Used only for Fortran wrappers */ 146 | 147 | void H5Pset_zfp_rate_cdata_f(double rate, size_t *cd_nelmts_f, unsigned int *cd_values) { 148 | size_t cd_nelmts = *cd_nelmts_f; 149 | H5Pset_zfp_rate_cdata(rate, cd_nelmts, cd_values); 150 | *cd_nelmts_f = cd_nelmts; 151 | } 152 | 153 | void H5Pset_zfp_precision_cdata_f(unsigned int prec, size_t *cd_nelmts_f, unsigned int *cd_values) { 154 | size_t cd_nelmts = *cd_nelmts_f; 155 | H5Pset_zfp_precision_cdata(prec, cd_nelmts, cd_values); 156 | *cd_nelmts_f = cd_nelmts; 157 | } 158 | 159 | void H5Pset_zfp_accuracy_cdata_f(double acc, size_t *cd_nelmts_f, unsigned int *cd_values) { 160 | size_t cd_nelmts = *cd_nelmts_f; 161 | H5Pset_zfp_accuracy_cdata(acc, cd_nelmts, cd_values); 162 | *cd_nelmts_f = cd_nelmts; 163 | } 164 | 165 | void H5Pset_zfp_expert_cdata_f(unsigned int minbits, unsigned int maxbits, unsigned int maxprec, 166 | int minexp, size_t *cd_nelmts_f, unsigned int *cd_values) { 167 | size_t cd_nelmts = *cd_nelmts_f; 168 | H5Pset_zfp_expert_cdata(minbits, maxbits, maxprec, minexp, cd_nelmts, cd_values); 169 | *cd_nelmts_f = cd_nelmts; 170 | } 171 | 172 | void H5Pset_zfp_reversible_cdata_f(size_t *cd_nelmts_f, unsigned int *cd_values) { 173 | size_t cd_nelmts = *cd_nelmts_f; 174 | H5Pset_zfp_reversible_cdata(cd_nelmts, cd_values); 175 | *cd_nelmts_f = cd_nelmts; 176 | } 177 | 178 | -------------------------------------------------------------------------------- /docs/h5repack.rst: -------------------------------------------------------------------------------- 1 | ================================== 2 | Using H5Z-ZFP Plugin with H5Repack 3 | ================================== 4 | A convenient way to use and play with the ZFP_ filter is as a *plugin* with the HDF5_ `h5repack `__ utility using the ``-f`` command-line argument to apply ZFP to existing data in a file. 5 | 6 | ----------------- 7 | Patching h5repack 8 | ----------------- 9 | 10 | .. warning:: 11 | 12 | Versions of HDF5_'s ``h5repack`` utility prior to 1.10.4 contain a bug that prevents proper parsing of the ``-f`` argument's option. 13 | In order to use ``h5repack`` with ``-f`` argument as described here, you need to apply the patch from `h5repack_parse.patch `_. 14 | To do so, after you've downloaded and untar'd HDF5_ but before you've built it, do something like the following using HDF5-1.8.14 as an example:: 15 | 16 | gunzip < hdf5-1.8.14.tar.gz | tar xvf - 17 | cd hdf5-1.8.14 18 | patch ./tools/h5repack/h5repack_parse.c /h5repack_parse.patch 19 | 20 | ------------------------------------- 21 | Constructing an HDF5_ cd_values array 22 | ------------------------------------- 23 | HDF5_'s ``h5repack`` utility uses only the *generic* interface to HDF5_ filters. 24 | Another challenge in using ``h5repack`` as described here is constructing the set ``unsigned int cd_values`` as is used in `H5Pset_filter() `__ required by the *generic* HDF5_ filter interface, especially because of the type-punning (doubles as unsigned int) which may be involved. 25 | 26 | .. note:: 27 | 28 | Querying an existing dataset using ``h5dump`` or ``h5ls`` to obtain the ``cd_values`` *stored* with a ZFP_ compressed dataset will not provide the correct ``cd_values`` needed to invoke the filter. 29 | This is because the ``cd_values`` stored in the file are different from those needed in the *generic* interface to invoke the ZFP_ filter. 30 | 31 | To facilitate constructing a valid ``-f`` argument to ``h5repack``, there is a utility program, ``print_h5repack_farg``, which is presently in the ``test`` directory and is built when tests are built. 32 | This program was originally written simply to facilitate testing of H5Z-ZFP filter. 33 | It should eventually be made a first class *tool* installed with H5Z-ZFP. 34 | However, it is presently made only as part of *testing* H5Z-ZFP. 35 | 36 | The ``print_h5repack_farg`` utility can be used to read a command-line consisting of ZFP_ filter parameters you wish to use and output part of the command-line needed for the ``-f`` argument to ``h5repack``. 37 | 38 | -------- 39 | Examples 40 | -------- 41 | 42 | .. note:: 43 | 44 | The examples below assume H5Z-ZFP has been configured to run tests by either a preceding ``make check`` (if using a vanilla ``gmake`` build) or ``make test`` (if using a ``cmake`` build). 45 | 46 | In the examples below, we use ``h5repack`` with the example data file, ``mesh.h5`` in the tests directory. 47 | 48 | To use ZFP_ filter in *rate* mode with a rate of ``4.5`` bits per value, first, use the ``print_h5repack_farg``:: 49 | 50 | % ./print_h5repack_farg zfpmode=1 rate=4.5 51 | 52 | Print cdvals for set of ZFP compression parameters... 53 | zfpmode=1 set zfp mode (1=rate,2=prec,3=acc,4=expert,5=rev) 54 | rate=4.5 set rate for rate mode of filter 55 | acc=0 set accuracy for accuracy mode of filter 56 | prec=0 set precision for precision mode of zfp filter 57 | minbits=0 set minbits for expert mode of zfp filter 58 | maxbits=0 set maxbits for expert mode of zfp filter 59 | maxprec=0 set maxprec for expert mode of zfp filter 60 | minexp=0 set minexp for expert mode of zfp filter 61 | help=0 this help message 62 | 63 | h5repack -f argument... 64 | -f UD=32013,0,4,1,0,0,1074921472 65 | 66 | Next, cut-n-paste the ``-f UD=32013,0,4,1,0,0,1074921472`` in a command to ``h5repack`` like so:: 67 | 68 | env LD_LIBRARY_PATH=:$(LD_LIBRARY_PATH) \ 69 | HDF5_PLUGIN_PATH= \ 70 | $(HDF5_BIN)/h5repack -f UD=32013,0,4,1,0,0,1074921472 \ 71 | -l Pressure,Pressure2,Pressure3:CHUNK=10x20x5 \ 72 | -l Velocity,Velocity2,Velocity3,VelocityZ,VelocityZ2,VelocityZ3:CHUNK=11x21x1x1 \ 73 | -l VelocityX_2D:CHUNK=21x31 \ 74 | mesh.h5 mesh_repack.h5 75 | 76 | where the ``-l`` arguments indicate the dataset(s) to be re-packed as well as their (new) chunking. 77 | 78 | To use ZFP_ filter in *accuracy* mode with an accuracy of ``0.075``, first, use the ``print_h5repack_farg``:: 79 | 80 | % ./print_h5repack_farg zfpmode=3 acc=0.075 81 | 82 | Print cdvals for set of ZFP compression parameters... 83 | zfpmode=3 set zfp mode (1=rate,2=prec,3=acc,4=expert,5=rev) 84 | rate=3.5 set rate for rate mode of filter 85 | acc=0.075 set accuracy for accuracy mode of filter 86 | prec=0 set precision for precision mode of zfp filter 87 | minbits=0 set minbits for expert mode of zfp filter 88 | maxbits=0 set maxbits for expert mode of zfp filter 89 | maxprec=0 set maxprec for expert mode of zfp filter 90 | minexp=0 set minexp for expert mode of zfp filter 91 | help=0 this help message 92 | 93 | h5repack -f argument... 94 | -f UD=32013,0,4,3,0,858993459,1068708659 95 | 96 | Next, cut-n-paste the ``-f UD=32013,0,4,3,0,858993459,1068708659`` in a command to ``h5repack`` like so:: 97 | 98 | env LD_LIBRARY_PATH=:$(LD_LIBRARY_PATH) \ 99 | HDF5_PLUGIN_PATH= \ 100 | $(HDF5_BIN)/h5repack -f UD=32013,0,4,3,0,858993459,1068708659 \ 101 | -l Pressure,Pressure2,Pressure3:CHUNK=10x20x5 \ 102 | -l Velocity,Velocity2,Velocity3,VelocityZ,VelocityZ2,VelocityZ3:CHUNK=11x21x1x1 \ 103 | -l VelocityX_2D:CHUNK=21x31 \ 104 | mesh.h5 mesh_repack.h5 105 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL Advanced" 2 | 3 | on: 4 | push: 5 | branches: [ "master" ] 6 | pull_request: 7 | branches: [ "master" ] 8 | schedule: 9 | - cron: "16 7 * * 0" 10 | 11 | jobs: 12 | analyze: 13 | name: Analyze 14 | # Runner size impacts CodeQL analysis time. To learn more, please see: 15 | # - https://gh.io/recommended-hardware-resources-for-running-codeql 16 | # - https://gh.io/supported-runners-and-hardware-resources 17 | # - https://gh.io/using-larger-runners (GitHub.com only) 18 | # Consider using larger runners or machines with greater resources for possible analysis time improvements. 19 | runs-on: ubuntu-latest 20 | permissions: 21 | # required for all workflows 22 | security-events: write 23 | 24 | # required to fetch internal or private CodeQL packs 25 | packages: read 26 | 27 | strategy: 28 | fail-fast: false 29 | matrix: 30 | include: 31 | - language: c-cpp 32 | build-mode: manual 33 | # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' 34 | # Use `c-cpp` to analyze code written in C, C++ or both 35 | # Use 'java-kotlin' to analyze code written in Java, Kotlin or both 36 | # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both 37 | # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, 38 | # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. 39 | # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how 40 | # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages 41 | steps: 42 | - name: Install Dependencies (Linux) 43 | run: | 44 | sudo gem install apt-spy2 45 | sudo apt-spy2 check 46 | sudo apt-spy2 fix --commit 47 | sudo apt update 48 | sudo apt upgrade 49 | sudo apt-get install -qq libhdf5-dev 50 | sudo apt-get install -qq hdf5-tools 51 | echo "HDF5_HOME=/usr/include/hdf5/serial,/usr/lib/x86_64-linux-gnu/hdf5/serial,/usr/bin" >> $GITHUB_ENV 52 | # Set env vars 53 | echo "CC=gcc" >> $GITHUB_ENV 54 | echo "FC=gfortran" >> $GITHUB_ENV 55 | echo "CXX=g++" >> $GITHUB_ENV 56 | 57 | - name: Checkout 58 | uses: actions/checkout@v6 59 | 60 | ################################## 61 | # INSTALL ZFP 62 | ################################## 63 | 64 | - name: install ZFP 65 | run: | 66 | git clone https://github.com/LLNL/zfp.git 67 | export HOME_DIR=$(echo ~) 68 | cd zfp 69 | mkdir build;cd build 70 | cmake -D ZFP_BIT_STREAM_WORD_SIZE=8 -D BUILD_CFP=ON -D CMAKE_INSTALL_PREFIX=$PWD/zfp -D BUILD_TESTING=OFF -D BUILD_UTILITIES=OFF .. 71 | make install 72 | echo "ZFP_HOME=$PWD/zfp" >> $GITHUB_ENV 73 | echo "ZFP_DIR=$PWD/zfp" >> $GITHUB_ENV 74 | shell: bash 75 | 76 | ################################## 77 | # BUILD AND TEST H5Z-ZFP 78 | ################################## 79 | 80 | - name: Initialize CodeQL 81 | uses: github/codeql-action/init@v4 82 | with: 83 | languages: c-cpp 84 | build-mode: manual 85 | # If you wish to specify custom queries, you can do so here or in a config file. 86 | # By default, queries listed here will override any specified in a config file. 87 | # Prefix the list here with "+" to use these queries and those in the config file. 88 | 89 | # For more details on CodeQL's query packs, refer to: 90 | #https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 91 | 92 | config: | 93 | query-filters: 94 | - exclude: 95 | # See: https://codeql.github.com/codeql-query-help/cpp/cpp-guarded-free/ 96 | id: cpp/guarded-free 97 | - exclude: 98 | # See: https://codeql.github.com/codeql-query-help/cpp/cpp-unused-static-function/ 99 | id: cpp/unused-static-function 100 | 101 | queries: +security-extended,security-and-quality 102 | 103 | - name: configure h5z-zfp 104 | run: | 105 | mkdir build; cd build 106 | cmake -D FORTRAN_INTERFACE=ON -D CMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON .. 107 | shell: bash 108 | 109 | - name: build and test 110 | run: | 111 | cd build 112 | cmake --build . 113 | ctest -C Debug . 114 | shell: bash 115 | 116 | # NOTE: C/C++ does not support path-based filtering when using the manual build mode. 117 | # The "paths" and "paths-ignore" configuration properties will have no effect for this 118 | # language. If desired, you can use the advanced-security/filter-sarif Action to rewrite 119 | # the SARIF file to exclude alerts from these paths. For more information, 120 | # see https://github.com/advanced-security/filter-sarif 121 | 122 | - name: Perform CodeQL Analysis 123 | uses: github/codeql-action/analyze@v4 124 | with: 125 | category: "/language:c-cpp" 126 | output: sarif-results 127 | upload: failure-only 128 | 129 | - name: filter-sarif 130 | uses: advanced-security/filter-sarif@v1 131 | with: 132 | patterns: | 133 | -**/* 134 | src/**/* 135 | input: sarif-results/cpp.sarif 136 | output: sarif-results/cpp.sarif 137 | 138 | - name: Upload SARIF 139 | uses: github/codeql-action/upload-sarif@v4 140 | with: 141 | sarif_file: sarif-results/cpp.sarif 142 | 143 | - name: Upload loc as a Build Artifact 144 | uses: actions/upload-artifact@v6 145 | with: 146 | name: sarif-results 147 | path: sarif-results 148 | retention-days: 1 149 | -------------------------------------------------------------------------------- /src/H5Zzfp_props_f.F90: -------------------------------------------------------------------------------- 1 | MODULE h5zzfp_props_f 2 | 3 | #include "H5Zzfp_version.h" 4 | 5 | USE ISO_C_BINDING 6 | USE HDF5 7 | IMPLICIT NONE 8 | 9 | ! First, create _F equivalent parameters for all the C interface's CPP symbols and 10 | ! capture the original C interface's CPP values. 11 | INTEGER, PARAMETER :: H5Z_FILTER_ZFP_F=H5Z_FILTER_ZFP 12 | 13 | INTEGER, PARAMETER :: H5Z_FILTER_ZFP_VERSION_MAJOR_F=H5Z_FILTER_ZFP_VERSION_MAJOR 14 | INTEGER, PARAMETER :: H5Z_FILTER_ZFP_VERSION_MINOR_F=H5Z_FILTER_ZFP_VERSION_MINOR 15 | INTEGER, PARAMETER :: H5Z_FILTER_ZFP_VERSION_PATCH_F=H5Z_FILTER_ZFP_VERSION_PATCH 16 | 17 | INTEGER(C_SIZE_T), PARAMETER :: H5Z_ZFP_CD_NELMTS_MEM_F=H5Z_ZFP_CD_NELMTS_MEM 18 | INTEGER(C_SIZE_T), PARAMETER :: H5Z_ZFP_CD_NELMTS_MAX_F=H5Z_ZFP_CD_NELMTS_MAX 19 | 20 | INTEGER, PARAMETER :: H5Z_ZFP_MODE_RATE_F = H5Z_ZFP_MODE_RATE 21 | INTEGER, PARAMETER :: H5Z_ZFP_MODE_PRECISION_F = H5Z_ZFP_MODE_PRECISION 22 | INTEGER, PARAMETER :: H5Z_ZFP_MODE_ACCURACY_F = H5Z_ZFP_MODE_ACCURACY 23 | INTEGER, PARAMETER :: H5Z_ZFP_MODE_EXPERT_F = H5Z_ZFP_MODE_EXPERT 24 | INTEGER, PARAMETER :: H5Z_ZFP_MODE_REVERSIBLE_F = H5Z_ZFP_MODE_REVERSIBLE 25 | 26 | ! Next, undef all the C interface's CPP symbols so we can declare Fortran parameters 27 | ! with the identical names 28 | #undef H5Z_FILTER_ZFP 29 | #undef H5Z_FILTER_ZFP_VERSION_MAJOR 30 | #undef H5Z_FILTER_ZFP_VERSION_MINOR 31 | #undef H5Z_FILTER_ZFP_VERSION_PATCH 32 | #undef H5Z_ZFP_CD_NELMTS_MEM 33 | #undef H5Z_ZFP_CD_NELMTS_MAX 34 | #undef H5Z_ZFP_MODE_RATE 35 | #undef H5Z_ZFP_MODE_PRECISION 36 | #undef H5Z_ZFP_MODE_ACCURACY 37 | #undef H5Z_ZFP_MODE_EXPERT 38 | #undef H5Z_ZFP_MODE_REVERSIBLE 39 | 40 | ! Define Fortran parameters using the original _F values captured above. 41 | INTEGER, PARAMETER :: H5Z_FILTER_ZFP=H5Z_FILTER_ZFP_F 42 | 43 | INTEGER, PARAMETER :: H5Z_FILTER_ZFP_VERSION_MAJOR=H5Z_FILTER_ZFP_VERSION_MAJOR_F 44 | INTEGER, PARAMETER :: H5Z_FILTER_ZFP_VERSION_MINOR=H5Z_FILTER_ZFP_VERSION_MINOR_F 45 | INTEGER, PARAMETER :: H5Z_FILTER_ZFP_VERSION_PATCH=H5Z_FILTER_ZFP_VERSION_PATCH_F 46 | 47 | INTEGER(C_SIZE_T), PARAMETER :: H5Z_ZFP_CD_NELMTS_MEM=H5Z_ZFP_CD_NELMTS_MEM_F 48 | INTEGER(C_SIZE_T), PARAMETER :: H5Z_ZFP_CD_NELMTS_MAX=H5Z_ZFP_CD_NELMTS_MAX_F 49 | 50 | INTEGER, PARAMETER :: H5Z_ZFP_MODE_RATE = H5Z_ZFP_MODE_RATE_F 51 | INTEGER, PARAMETER :: H5Z_ZFP_MODE_PRECISION = H5Z_ZFP_MODE_PRECISION_F 52 | INTEGER, PARAMETER :: H5Z_ZFP_MODE_ACCURACY = H5Z_ZFP_MODE_ACCURACY_F 53 | INTEGER, PARAMETER :: H5Z_ZFP_MODE_EXPERT = H5Z_ZFP_MODE_EXPERT_F 54 | INTEGER, PARAMETER :: H5Z_ZFP_MODE_REVERSIBLE = H5Z_ZFP_MODE_REVERSIBLE_F 55 | 56 | INTERFACE 57 | INTEGER(C_INT) FUNCTION H5Z_zfp_initialize() BIND(C, NAME='H5Z_zfp_initialize') 58 | IMPORT :: C_INT 59 | IMPLICIT NONE 60 | END FUNCTION H5Z_zfp_initialize 61 | 62 | INTEGER(C_INT) FUNCTION H5Z_zfp_finalize() BIND(C, NAME='H5Z_zfp_finalize') 63 | IMPORT :: C_INT 64 | IMPLICIT NONE 65 | END FUNCTION H5Z_zfp_finalize 66 | 67 | INTEGER(C_INT) FUNCTION H5Pset_zfp_rate(plist, rate) BIND(C, NAME='H5Pset_zfp_rate') 68 | IMPORT :: C_INT, C_DOUBLE, HID_T 69 | IMPLICIT NONE 70 | INTEGER(HID_T), VALUE :: plist 71 | REAL(C_DOUBLE), VALUE :: rate 72 | END FUNCTION H5Pset_zfp_rate 73 | 74 | INTEGER(C_INT) FUNCTION H5Pset_zfp_precision(plist, prec) BIND(C, NAME='H5Pset_zfp_precision') 75 | IMPORT :: C_INT, HID_T 76 | IMPLICIT NONE 77 | INTEGER(HID_T), VALUE :: plist 78 | INTEGER(C_INT), VALUE :: prec 79 | END FUNCTION H5Pset_zfp_precision 80 | 81 | INTEGER(C_INT) FUNCTION H5Pset_zfp_accuracy(plist, acc) BIND(C, NAME='H5Pset_zfp_accuracy') 82 | IMPORT :: C_INT, C_DOUBLE, HID_T 83 | IMPLICIT NONE 84 | INTEGER(HID_T), VALUE :: plist 85 | REAL(C_DOUBLE), VALUE :: acc 86 | END FUNCTION H5Pset_zfp_accuracy 87 | 88 | INTEGER(C_INT) FUNCTION H5Pset_zfp_expert(plist, minbits, maxbits, maxprec, minexp) & 89 | BIND(C, NAME='H5Pset_zfp_expert') 90 | IMPORT :: C_INT, HID_T 91 | IMPLICIT NONE 92 | INTEGER(HID_T), VALUE :: plist 93 | INTEGER(C_INT), VALUE :: minbits 94 | INTEGER(C_INT), VALUE :: maxbits 95 | INTEGER(C_INT), VALUE :: maxprec 96 | INTEGER(C_INT), VALUE :: minexp 97 | END FUNCTION H5Pset_zfp_expert 98 | 99 | INTEGER(C_INT) FUNCTION H5Pset_zfp_reversible(plist) BIND(C, NAME='H5Pset_zfp_reversible') 100 | IMPORT :: C_INT, HID_T 101 | IMPLICIT NONE 102 | INTEGER(HID_T), VALUE :: plist 103 | END FUNCTION H5Pset_zfp_reversible 104 | 105 | SUBROUTINE H5Pset_zfp_rate_cdata(rate, cd_nelmts, cd_values) BIND(C, NAME='H5Pset_zfp_rate_cdata_f') 106 | IMPORT :: C_DOUBLE, C_INT, C_SIZE_T 107 | IMPLICIT NONE 108 | REAL(C_DOUBLE), VALUE :: rate 109 | INTEGER(C_SIZE_T) :: cd_nelmts 110 | INTEGER(C_INT), DIMENSION(*) :: cd_values 111 | END SUBROUTINE H5Pset_zfp_rate_cdata 112 | 113 | SUBROUTINE H5Pset_zfp_accuracy_cdata(acc, cd_nelmts, cd_values) BIND(C, NAME='H5Pset_zfp_accuracy_cdata_f') 114 | IMPORT :: C_DOUBLE, C_INT, C_SIZE_T 115 | IMPLICIT NONE 116 | REAL(C_DOUBLE), VALUE :: acc 117 | INTEGER(C_SIZE_T) :: cd_nelmts 118 | INTEGER(C_INT), DIMENSION(*) :: cd_values 119 | END SUBROUTINE H5Pset_zfp_accuracy_cdata 120 | 121 | SUBROUTINE H5Pset_zfp_precision_cdata(prec, cd_nelmts, cd_values) BIND(C, NAME='H5Pset_zfp_precision_cdata_f') 122 | IMPORT :: C_INT, C_SIZE_T 123 | IMPLICIT NONE 124 | INTEGER(C_INT), VALUE :: prec 125 | INTEGER(C_SIZE_T) :: cd_nelmts 126 | INTEGER(C_INT), DIMENSION(*) :: cd_values 127 | END SUBROUTINE H5Pset_zfp_precision_cdata 128 | 129 | SUBROUTINE H5Pset_zfp_expert_cdata(minbits, maxbits, maxprec, minexp, cd_nelmts, cd_values) & 130 | BIND(C, NAME='H5Pset_zfp_expert_cdata_f') 131 | IMPORT :: C_INT, C_SIZE_T 132 | IMPLICIT NONE 133 | INTEGER(C_INT), VALUE :: minbits, maxbits, maxprec, minexp 134 | INTEGER(C_SIZE_T) :: cd_nelmts 135 | INTEGER(C_INT), DIMENSION(*) :: cd_values 136 | END SUBROUTINE H5Pset_zfp_expert_cdata 137 | 138 | SUBROUTINE H5Pset_zfp_reversible_cdata(cd_nelmts, cd_values) BIND(C, NAME='H5Pset_zfp_reversible_cdata_f') 139 | IMPORT :: C_INT, C_SIZE_T 140 | IMPLICIT NONE 141 | INTEGER(C_SIZE_T) :: cd_nelmts 142 | INTEGER(C_INT), DIMENSION(*) :: cd_values 143 | END SUBROUTINE H5Pset_zfp_reversible_cdata 144 | 145 | END INTERFACE 146 | 147 | END MODULE H5Zzfp_props_f 148 | -------------------------------------------------------------------------------- /test/test_read.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2016, Lawrence Livermore National Security, LLC. 3 | Produced at the Lawrence Livermore National Laboratory 4 | Written by Mark C. Miller, miller86@llnl.gov 5 | LLNL-CODE-707197 All rights reserved. 6 | 7 | This file is part of H5Z-ZFP. For details, see 8 | https://github.com/LLNL/H5Z-ZFP. Please also read the Additional 9 | BSD Notice. 10 | 11 | Redistribution and use in source and binary forms, with or without 12 | modification, are permitted provided that the following conditions are 13 | met: 14 | 15 | * Redistributions of source code must retain the above copyright 16 | notice, this list of conditions and the disclaimer below. 17 | 18 | * Redistributions in binary form must reproduce the above copyright 19 | notice, this list of conditions and the disclaimer (as noted below) 20 | in the documentation and/or other materials provided with the 21 | distribution. 22 | 23 | * Neither the name of the LLNS/LLNL nor the names of its contributors 24 | may be used to endorse or promote products derived from this 25 | software without specific prior written permission. 26 | 27 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 28 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 29 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 30 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE 31 | LIVERMORE NATIONAL SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR 32 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 33 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 34 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 35 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 36 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 37 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 38 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 | 40 | Additional BSD Notice 41 | 42 | 1. This notice is required to be provided under our contract with the 43 | U.S. Department of Energy (DOE). This work was produced at Lawrence 44 | Livermore National Laboratory under Contract No. DE-AC52-07NA27344 45 | with the DOE. 46 | 47 | 2. Neither the United States Government nor Lawrence Livermore 48 | National Security, LLC nor any of their employees, makes any warranty, 49 | express or implied, or assumes any liability or responsibility for the 50 | accuracy, completeness, or usefulness of any information, apparatus, 51 | product, or process disclosed, or represents that its use would not 52 | infringe privately-owned rights. 53 | 54 | 3. Also, reference herein to any specific commercial products, 55 | process, or services by trade name, trademark, manufacturer or 56 | otherwise does not necessarily constitute or imply its endorsement, 57 | recommendation, or favoring by the United States Government or 58 | Lawrence Livermore National Security, LLC. The views and opinions of 59 | authors expressed herein do not necessarily state or reflect those of 60 | the United States Government or Lawrence Livermore National Security, 61 | LLC, and shall not be used for advertising or product endorsement 62 | purposes. 63 | */ 64 | 65 | #include "test_common.h" 66 | 67 | #ifndef H5Z_ZFP_USE_PLUGIN 68 | #include "H5Zzfp_lib.h" 69 | #endif 70 | 71 | int main(int argc, char **argv) 72 | { 73 | int i, help=0; 74 | double *obuf, *cbuf; 75 | 76 | /* filename variables */ 77 | char *ifile = (char *) calloc(NAME_LEN,sizeof(char)); 78 | 79 | /* HDF5 dataset info */ 80 | hid_t fid, dsid, space_id; 81 | hsize_t npoints; 82 | 83 | /* absolute and relative differencing thresholds */ 84 | double max_absdiff = 0; 85 | double max_reldiff = 0; 86 | 87 | /* actual absolute and relative differences observed */ 88 | double actual_max_absdiff = 0; 89 | double actual_max_reldiff = 0; 90 | int num_absdiffs = 0; 91 | int num_reldiffs = 0; 92 | int doint = 0; 93 | int ret = 0; 94 | 95 | /* file arguments */ 96 | strcpy(ifile, "test_zfp.h5"); 97 | HANDLE_SEP(Usage: test_read [opt1=value1 opt2=value2]) 98 | HANDLE_ARG(ifile,strndup(argv[i]+len2,NAME_LEN), "\"%s\"",set input filename); 99 | HANDLE_ARG(max_absdiff,strtod(argv[i]+len2,0),"%g",set maximum absolute diff); 100 | HANDLE_ARG(max_reldiff,strtod(argv[i]+len2,0),"%g",set maximum relative diff); 101 | HANDLE_ARG(doint,(int) strtol(argv[i]+len2,0,10),"%d",check integer datasets instead); 102 | HANDLE_ARG(ret,(int) strtol(argv[i]+len2,0,10),"%d",return 1 if diffs (0=all,1=abs,2=rel)); 103 | HANDLE_ARG(help,(int)strtol(argv[i]+len2,0,10),"%d",this help message); 104 | 105 | #ifndef H5Z_ZFP_USE_PLUGIN 106 | H5Z_zfp_initialize(); 107 | #endif 108 | 109 | /* open the HDF5 file */ 110 | if (0 > (fid = H5Fopen(ifile, H5F_ACC_RDONLY, H5P_DEFAULT))) SET_ERROR(H5Fopen); 111 | 112 | /* read the original dataset */ 113 | if (0 > (dsid = H5Dopen(fid, doint?"int_original":"original", H5P_DEFAULT))) SET_ERROR(H5Dopen); 114 | if (0 > (space_id = H5Dget_space(dsid))) SET_ERROR(H5Dget_space); 115 | if (0 == (npoints = H5Sget_simple_extent_npoints(space_id))) SET_ERROR(H5Sget_simple_extent_npoints); 116 | if (0 > H5Sclose(space_id)) SET_ERROR(H5Sclose); 117 | if (0 == (obuf = (double *) malloc(npoints * sizeof(double)))) SET_ERROR(malloc); 118 | if (0 > H5Dread(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, obuf)) SET_ERROR(H5Dread); 119 | if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); 120 | 121 | /* read the compressed dataset */ 122 | if (0 > (dsid = H5Dopen(fid, doint?"int_compressed":"compressed", H5P_DEFAULT))) SET_ERROR(H5Dopen); 123 | if (0 == (cbuf = (double *) malloc(npoints * sizeof(double)))) SET_ERROR(malloc); 124 | if (0 > H5Dread(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, cbuf)) SET_ERROR(H5Dread); 125 | if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); 126 | 127 | /* clean up */ 128 | if (0 > H5Fclose(fid)) SET_ERROR(H5Fclose); 129 | 130 | /* compare original to compressed */ 131 | for (i = 0; i < (int) npoints; i++) 132 | { 133 | double absdiff = obuf[i] - cbuf[i]; 134 | if (absdiff < 0) absdiff = -absdiff; 135 | if (absdiff > 0) 136 | { 137 | double reldiff = 0; 138 | if (obuf[i] != 0) reldiff = absdiff / obuf[i]; 139 | 140 | if (absdiff > actual_max_absdiff) actual_max_absdiff = absdiff; 141 | if (reldiff > actual_max_reldiff) actual_max_reldiff = reldiff; 142 | if (absdiff > max_absdiff) 143 | num_absdiffs++; 144 | if (reldiff > max_reldiff) 145 | num_reldiffs++; 146 | } 147 | } 148 | printf("Absolute Diffs: %d values are different; actual-max-absdiff = %g\n", 149 | num_absdiffs, actual_max_absdiff); 150 | printf("Relative Diffs: %d values are different; actual-max-reldiff = %g\n", 151 | num_reldiffs, actual_max_reldiff); 152 | 153 | #ifndef H5Z_ZFP_USE_PLUGIN 154 | /* When filter is used as a library, we need to finalize it */ 155 | H5Z_zfp_finalize(); 156 | #endif 157 | 158 | free(obuf); 159 | free(cbuf); 160 | free(ifile); 161 | 162 | if (ret == 0) return (num_absdiffs+num_reldiffs)>0; 163 | if (ret == 1) return num_absdiffs>0; 164 | if (ret == 2) return num_reldiffs>0; 165 | return 0; 166 | } 167 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/ReadtheDocsTemplate.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/ReadtheDocsTemplate.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/ReadtheDocsTemplate" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/ReadtheDocsTemplate" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /config.make: -------------------------------------------------------------------------------- 1 | export SHELL = /bin/bash 2 | 3 | #ifeq ($(HDF5_HOME),) 4 | # $(warning WARNING: HDF5_HOME not specified) 5 | #endif 6 | # 7 | #ifeq ($(ZFP_HOME),) 8 | # $(warning WARNING: ZFP_HOME not specified) 9 | #endif 10 | 11 | # disallow relative paths in HOME variables 12 | HOME_WORDS := $(subst /, ,$(HDF5_HOME)) 13 | FIRST_WORD := $(firstword $(HOME_WORDS)) 14 | ifeq ($(FIRST_WORD),.) 15 | $(error Please use absolute path for HDF5_HOME) 16 | else ifeq ($(FIRST_WORD),..) 17 | $(error Please use absolute path for HDF5_HOME) 18 | endif 19 | HOME_WORDS := $(subst /, ,$(ZFP_HOME)) 20 | FIRST_WORD := $(firstword $(HOME_WORDS)) 21 | ifeq ($(FIRST_WORD),.) 22 | $(error Please use absolute path for ZFP_HOME) 23 | else ifeq ($(FIRST_WORD),..) 24 | $(error Please use absolute path for ZFP_HOME) 25 | endif 26 | 27 | # Construct H5Z_ZFP_BASE variable depending on what dir this config.cmake is being included from 28 | PWD_BASE = $(shell basename $$(pwd)) 29 | ifneq ($(wildcard config.make),) # we're in top-level dir 30 | H5Z_ZFP_BASE := ./src 31 | else ifneq ($(wildcard test_write.c),) # we're in test dir 32 | H5Z_ZFP_BASE := ../src 33 | else ifneq ($(wildcard H5Zzfp.c),) # we're in src dir 34 | H5Z_ZFP_BASE := . 35 | endif 36 | 37 | H5Z_ZFP_PLUGIN := $(H5Z_ZFP_BASE)/plugin 38 | ZFP_HAS_REVERSIBLE := 39 | ifneq ($(ZFP_HOME),) 40 | ZFP_HAS_REVERSIBLE := $(shell grep zfp_stream_set_reversible $(ZFP_HOME)/include/zfp.h 2>/dev/null) 41 | endif 42 | 43 | # Construct make-time knowledge of ZFP library version 44 | ZFP_LIB_VERSION := $(shell grep '^\#define ZFP_VERSION_[MPT]' $(ZFP_HOME)/include/zfp/version.h 2>/dev/null | tr ' ' '\n' | grep '[0-9]' | tr -d '\n') 45 | ifeq ($(ZFP_LIB_VERSION),) 46 | ZFP_LIB_VERSION := $(shell grep '^\#define ZFP_VERSION_[MRPT]' $(ZFP_HOME)/include/zfp.h 2>/dev/null | tr ' ' '\n' | grep '[0-9]' | tr -d '\n' 2>/dev/null) 47 | endif 48 | ifeq ($(ZFP_LIB_VERSION),) 49 | ZFP_LIB_VERSION := $(shell grep '^\#define ZFP_VERSION_[MRPT]' $(ZFP_HOME)/inc/zfp.h 2>/dev/null | tr ' ' '\n' | grep '[0-9]' | tr -d '\n' 2>/dev/null) 50 | endif 51 | ifeq ($(ZFP_LIB_VERSION),) 52 | $(warning WARNING: ZFP lib version not detected by make -- some tests may be skipped) 53 | endif 54 | 55 | # Detect system type 56 | PROCESSOR := $(shell uname -p | tr '[:upper:]' '[:lower:]') 57 | OSNAME := $(shell uname -s | tr '[:upper:]' '[:lower:]') 58 | OSTYPE := $(shell env | grep OSTYPE | cut -d'=' -f2- | tr '[:upper:]' '[:lower:]') 59 | # LLNL specific environment variable 60 | SYS_TYPE := $(shell env | grep SYS_TYPE | cut -d'=' -f2- | tr '[:upper:]' '[:lower:]') 61 | 62 | # Common C compilers 63 | HAS_GCC := $(shell basename $$(which gcc 2>/dev/null) 2>/dev/null) 64 | HAS_CLANG := $(shell basename $$(which clang 2>/dev/null) 2>/dev/null) 65 | HAS_ICC := $(shell basename $$(which icc 2>/dev/null) 2>/dev/null) 66 | HAS_PGCC := $(shell basename $$(which pgcc 2>/dev/null) 2>/dev/null) 67 | HAS_XLCR := $(shell basename $$(which xlc_r 2>/dev/null) 2>/dev/null) 68 | HAS_BGXLCR := $(shell basename $$(which bgxlc_r 2>/dev/null) 2>/dev/null) 69 | HAS_CC := $(shell basename $$(which cc 2>/dev/null) 2>/dev/null) 70 | 71 | # Common Fortran compilers 72 | HAS_GFORTRAN := $(shell basename $$(which gfortran 2>/dev/null) 2>/dev/null) 73 | HAS_IFORT := $(shell basename $$(which ifort 2>/dev/null) 2>/dev/null) 74 | HAS_XLFR := $(shell basename $$(which xlf_r 2>/dev/null) 2>/dev/null) 75 | HAS_BGXLFR := $(shell basename $$(which bgxlf_r 2>/dev/null) 2>/dev/null) 76 | 77 | # If compiler isn't set, lets try to pick it 78 | ifeq ($(CC),) 79 | ifeq ($(OSNAME),darwin) 80 | ifneq ($(strip $(HAS_CLANG)),) 81 | CC = $(HAS_CLANG) 82 | else ifneq ($(strip $(HAS_GCC)),) 83 | CC = $(HAS_GCC) 84 | else ifneq ($(strip $(HAS_CC)),) 85 | CC = $(HAS_CC) 86 | endif 87 | else ifneq ($(findstring ppc, $(PROCESSOR),),) 88 | ifneq ($(strip $(HAS_BGXLCR)),) 89 | CC = $(HAS_BGXLCR) 90 | else ifneq ($(strip $(HAS_XLCR)),) 91 | CC = $(HAS_XLCR) 92 | else ifneq ($(strip $(HAS_GCC)),) 93 | CC = $(HAS_GCC) 94 | endif 95 | else 96 | ifneq ($(strip $(HAS_GCC)),) 97 | CC = $(HAS_GCC) 98 | else ifneq ($(strip $(HAS_CLANG)),) 99 | CC = $(HAS_CLANG) 100 | else ifneq ($(strip $(HAS_ICC)),) 101 | CC = $(HAS_ICC) 102 | else ifneq ($(strip $(HAS_PGCC)),) 103 | CC = $(HAS_PGCC) 104 | endif 105 | endif 106 | endif 107 | 108 | # If we don't have a CC by now, error out 109 | ifeq ($(CC),) 110 | $(error $(CC)) 111 | endif 112 | 113 | # 114 | # Now, setup various flags based on compiler 115 | # 116 | ifneq ($(findstring gcc, $(CC)),) 117 | CFLAGS += -fPIC 118 | SOEXT ?= so 119 | SHFLAG ?= -shared 120 | PREPATH = -Wl,-rpath, 121 | else ifneq ($(findstring clang, $(CC)),) 122 | SOEXT ?= dylib 123 | SHFLAG ?= -dynamiclib 124 | PREPATH = -Wl,-rpath, 125 | else ifneq ($(findstring icc, $(CC)),) 126 | CFLAGS += -fpic 127 | SOEXT ?= so 128 | SHFLAG ?= -shared 129 | PREPATH = -Wl,-rpath, 130 | else ifneq ($(findstring pgcc, $(CC)),) 131 | CFLAGS += -fpic 132 | SOEXT ?= so 133 | SHFLAG ?= -shared 134 | PREPATH = -Wl,-rpath, 135 | else ifneq ($(findstring cc, $(CC)),) 136 | CFLAGS += -fPIC 137 | SOEXT ?= so 138 | SHFLAG ?= -shared 139 | PREPATH = -Wl,-rpath, 140 | else ifneq ($(findstring xlc_r, $(CC)),) 141 | CFLAGS += -qpic 142 | SOEXT ?= so 143 | SHFLAG ?= -qmkshrobj 144 | PREPATH = -Wl,-R, 145 | else ifneq ($(findstring bgxlc_r, $(CC)),) 146 | CFLAGS += -qpic 147 | SOEXT ?= so 148 | SHFLAG ?= -qmkshrobj 149 | PREPATH = -Wl,-R, 150 | endif 151 | 152 | ifneq ($(findstring gfortran, $(FC)),) 153 | FCFLAGS += -fPIC 154 | else ifneq ($(findstring ifort, $(FC)),) 155 | FCFLAGS += -fpic 156 | else ifneq ($(findstring pgf90, $(FC)),) 157 | FCFLAGS += -fpic 158 | else ifneq ($(findstring xlf_r, $(FC)),) 159 | FCFLAGS += -qpic 160 | else ifneq ($(findstring bgxlf_r, $(FC)),) 161 | FCFLAGS += -qpic 162 | else ifneq ($(findstring f77, $(FC)),) 163 | # some makefile versions set FC=f77 if FC is not set 164 | FC = 165 | endif 166 | 167 | ifneq ($(wildcard $(ZFP_HOME)/include),) 168 | ZFP_INC = $(ZFP_HOME)/include 169 | else ifneq ($(wildcard $(ZFP_HOME)/inc),) 170 | ZFP_INC = $(ZFP_HOME)/inc 171 | endif 172 | ifeq ($(wildcard $(ZFP_INC)/zfp.h),) # no header file 173 | $(warning "zfp.h not found") 174 | endif 175 | 176 | ifeq ($(wildcard $(ZFP_HOME)/lib),) 177 | ZFP_LIB = $(ZFP_HOME)/lib64 178 | else 179 | ZFP_LIB = $(ZFP_HOME)/lib 180 | endif 181 | 182 | # Check if ZFP has CFP 183 | ifeq ($(wildcard $(ZFP_LIB)/libcfp.*),) # no cfp lib file 184 | ZFP_HAS_CFP = 0 185 | else 186 | ifeq ($(wildcard $(ZFP_INC)/zfp/array.h),) # no 1.0.0 header file 187 | ifeq ($(wildcard $(ZFP_INC)/cfparrays.h),) # no 0.5.5 header file 188 | ZFP_HAS_CFP = 0 189 | else 190 | ZFP_HAS_CFP = 1 191 | endif 192 | else 193 | ZFP_HAS_CFP = 1 194 | endif 195 | endif 196 | 197 | # Check if specified individually the HDF5 include directory, 198 | # library directory and bin directory separated by commas, i.e. HDF5_HOME=INC,LIB,BIN 199 | FOUND_LIST=$(shell echo "$(HDF5_HOME)" | grep -q "," && echo "true") 200 | ifeq ("$(FOUND_LIST)","true") 201 | HDF5_INC = $(shell echo $(HDF5_HOME) | awk -F'[,]' '{print $$1}') 202 | HDF5_LIB = $(shell echo $(HDF5_HOME) | awk -F'[,]' '{print $$2}') 203 | HDF5_BIN = $(shell echo $(HDF5_HOME) | awk -F'[,]' '{print $$3}') 204 | MAKEVARS = 205 | else 206 | HDF5_INC = $(HDF5_HOME)/include 207 | ifeq ($(wildcard $(HDF5_HOME)/lib),) 208 | HDF5_LIB = $(HDF5_HOME)/lib64 209 | else 210 | HDF5_LIB = $(HDF5_HOME)/lib 211 | endif 212 | HDF5_BIN = $(HDF5_HOME)/bin 213 | MAKEVARS = HDF5_HOME=$(HDF5_HOME) 214 | endif 215 | 216 | HDF5_HAS_WRITE_CHUNK = 1 217 | ifeq ($(shell grep H5Dwrite_chunk $(HDF5_INC)/*.h),) 218 | HDF5_HAS_WRITE_CHUNK = 0 219 | endif 220 | 221 | ifeq ($(PREFIX),) 222 | PREFIX := $(shell pwd)/install 223 | endif 224 | INSTALL ?= install 225 | 226 | MAKEVARS += ZFP_HOME=$(ZFP_HOME) PREFIX=$(PREFIX) 227 | 228 | .SUFFIXES: 229 | .SUFFIXES: .c .F90 .h .o .mod 230 | 231 | %.o : %.c 232 | $(CC) $< -o $@ -c $(CFLAGS) -I$(H5Z_ZFP_BASE) -I$(ZFP_INC) -I$(HDF5_INC) 233 | 234 | %.o %.mod : %.F90 235 | $(FC) $< -o $@ -c $(FCFLAGS) -I$(H5Z_ZFP_BASE) -I$(ZFP_INC) -I$(HDF5_INC) 236 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: h5z-zfp 2 | 3 | # Controls when the action will run. 4 | #Triggers the workflow on push or pull requests. 5 | 6 | # TODO: REMOVE feat-cmake_tests before merging 7 | on: 8 | push: 9 | branches: [ master ] 10 | pull_request: 11 | branches: [ master ] 12 | 13 | # Using concurrency to cancel any in-progress job or run 14 | concurrency: 15 | group: ${{ github.workflow }}-${{ github.sha || github.event.pull_request.number }} 16 | cancel-in-progress: true 17 | 18 | # A workflow run is made up of one or more jobs that 19 | # can run sequentially or in parallel 20 | jobs: 21 | # This workflow contains a single job called "build" 22 | linux-build: 23 | strategy: 24 | matrix: 25 | name: ["Ubuntu Latest GCC"] 26 | include: 27 | - name: "Ubuntu Latest GCC" 28 | artifact: "Linux.tar.xz" 29 | os: ubuntu-latest 30 | 31 | name: ${{ matrix.name }} 32 | # The type of runner that the job will run on. 33 | runs-on: ${{ matrix.os }} 34 | if: "!contains(github.event.head_commit.message, 'skip-ci')" 35 | 36 | # Steps represent a sequence of tasks that will be executed 37 | # as part of the job. 38 | steps: 39 | - name: Install Dependencies (Linux) 40 | if: matrix.os == 'ubuntu-latest' 41 | run: | 42 | sudo apt-get update 43 | sudo apt-get install -qq libhdf5-dev 44 | sudo apt-get install -qq hdf5-tools 45 | echo "HDF5_HOME=/usr/include/hdf5/serial,/usr/lib/x86_64-linux-gnu/hdf5/serial,/usr/bin" >> $GITHUB_ENV 46 | # Set env vars 47 | echo "CC=gcc" >> $GITHUB_ENV 48 | echo "FC=gfortran" >> $GITHUB_ENV 49 | echo "CXX=g++" >> $GITHUB_ENV 50 | 51 | # Checks-out the repository under $GITHUB_WORKSPACE so the job can access it. 52 | - name: Get Sources 53 | uses: actions/checkout@v6 54 | 55 | ################################## 56 | # INSTALL ZFP 57 | ################################## 58 | 59 | - name: install ZFP 60 | run: | 61 | git clone https://github.com/LLNL/zfp.git 62 | export HOME_DIR=$(echo ~) 63 | cd zfp 64 | git checkout 1.0.0 65 | mkdir build;cd build 66 | cmake -D ZFP_BIT_STREAM_WORD_SIZE=8 -D BUILD_CFP=ON -D CMAKE_INSTALL_PREFIX=$PWD/zfp -D BUILD_TESTING=OFF -D BUILD_UTILITIES=OFF .. 67 | make install 68 | echo "ZFP_HOME=$PWD/zfp" >> $GITHUB_ENV 69 | echo "ZFP_DIR=$PWD/zfp" >> $GITHUB_ENV 70 | shell: bash 71 | 72 | ################################## 73 | # BUILD AND TEST H5Z-ZFP 74 | ################################## 75 | 76 | - name: build (autotools) and check 77 | run: | 78 | make all 79 | make check 80 | shell: bash 81 | 82 | - name: build (cmake) and test 83 | run: | 84 | mkdir build; cd build 85 | cmake -D FORTRAN_INTERFACE=ON -D CMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON .. 86 | sudo cmake --build . --target install 87 | ctest -C Release . 88 | shell: bash 89 | 90 | 91 | linux-hdf5-versions: 92 | strategy: 93 | matrix: 94 | hdf5_version: ["develop", "hdf5_1_14_3"] 95 | include: 96 | - hdf5_version: "develop" 97 | hdf5_branch: "develop" 98 | hdf5_name: "HDF5 develop" 99 | - hdf5_version: "hdf5_1_14_3" 100 | hdf5_branch: "hdf5-1_14_3" 101 | hdf5_name: "HDF5 1.14.3" 102 | 103 | name: Ubuntu Latest GCC with ${{ matrix.hdf5_name }} 104 | runs-on: ubuntu-latest 105 | if: "!contains(github.event.head_commit.message, 'skip-ci')" 106 | 107 | steps: 108 | - name: Install Dependencies 109 | run: | 110 | sudo apt-get update 111 | sudo apt-get install -qq cmake gfortran 112 | # Set env vars 113 | echo "CC=gcc" >> $GITHUB_ENV 114 | echo "FC=gfortran" >> $GITHUB_ENV 115 | echo "CXX=g++" >> $GITHUB_ENV 116 | 117 | - name: Get Sources 118 | uses: actions/checkout@v6 119 | 120 | - name: Build HDF5 from source 121 | run: | 122 | git clone --depth 1 --branch ${{ matrix.hdf5_branch }} https://github.com/HDFGroup/hdf5.git 123 | cd hdf5 124 | mkdir build 125 | cd build 126 | cmake -DCMAKE_BUILD_TYPE=Release \ 127 | -DHDF5_BUILD_FORTRAN=ON \ 128 | -DHDF5_BUILD_TOOLS=ON \ 129 | -DHDF5_BUILD_EXAMPLES=OFF \ 130 | -DHDF5_BUILD_HL_LIB=OFF \ 131 | -DBUILD_TESTING=OFF \ 132 | -DHDF5_BUILD_DOC=OFF \ 133 | -DBUILD_SHARED_LIBS=ON \ 134 | -DBUILD_STATIC_LIBS=ON \ 135 | -DCMAKE_INSTALL_PREFIX=$HOME/hdf5-install \ 136 | .. 137 | make -j$(nproc) 138 | make install 139 | echo "HDF5_ROOT=$HOME/hdf5-install" >> $GITHUB_ENV 140 | echo "HDF5_DIR=$HOME/hdf5-install/share/cmake" >> $GITHUB_ENV 141 | echo "$HOME/hdf5-install/bin" >> $GITHUB_PATH 142 | shell: bash 143 | 144 | - name: Install ZFP 145 | run: | 146 | git clone https://github.com/LLNL/zfp.git 147 | cd zfp 148 | git checkout 1.0.0 149 | mkdir build 150 | cd build 151 | cmake -DZFP_BIT_STREAM_WORD_SIZE=8 \ 152 | -DBUILD_CFP=ON \ 153 | -DCMAKE_INSTALL_PREFIX=$HOME/zfp-install \ 154 | -DBUILD_TESTING=OFF \ 155 | -DBUILD_UTILITIES=OFF \ 156 | .. 157 | make -j$(nproc) 158 | make install 159 | echo "ZFP_HOME=$HOME/zfp-install" >> $GITHUB_ENV 160 | echo "ZFP_DIR=$HOME/zfp-install" >> $GITHUB_ENV 161 | shell: bash 162 | 163 | - name: Build H5Z-ZFP (CMake) and test 164 | run: | 165 | mkdir build 166 | cd build 167 | cmake -DFORTRAN_INTERFACE=ON \ 168 | -DCMAKE_BUILD_TYPE=Release \ 169 | -DBUILD_TESTING=ON \ 170 | .. 171 | make -j$(nproc) 172 | ctest -C Release . --output-on-failure 173 | shell: bash 174 | 175 | windows-build: 176 | name: 'Windows MSVC/Clang' 177 | defaults: 178 | run: 179 | shell: bash 180 | runs-on: windows-2022 181 | steps: 182 | - name: checkout repository 183 | uses: actions/checkout@v6 184 | - name: install prerequisites 185 | run: | 186 | choco install \ 187 | unzip \ 188 | wget 189 | - name: setup MSVC Tools 190 | uses: ilammy/msvc-dev-cmd@v1 191 | - name: download HDF5 for Windows 192 | run: | 193 | wget --quiet \ 194 | https://support.hdfgroup.org/releases/hdf5/v1_14/v1_14_5/downloads/hdf5-1.14.5-win-vs2022_cl.msi 195 | - name: Install HDF5 for Windows 196 | run: msiexec /i "hdf5-1.14.5-win-vs2022_cl.msi" /qn 197 | shell: cmd 198 | - name: Download and install ZFP 199 | run: | 200 | git clone https://github.com/LLNL/zfp.git 201 | cd zfp 202 | git checkout 1.0.0 203 | mkdir -p build 204 | cd build 205 | cmake .. \ 206 | -G "Visual Studio 17 2022" -A "x64" \ 207 | -DCMAKE_CONFIGURATION_TYPES=Release \ 208 | -DBUILD_UTILITIES=OFF \ 209 | -DBUILD_TESTING=OFF \ 210 | -DBUILD_CFP=ON \ 211 | -DZFP_BIT_STREAM_WORD_SIZE=8 212 | cmake --build . --config Release -j 3 213 | cmake --build . --config Release --target install 214 | - name: Set build and test variables 215 | run: | 216 | echo "C:/Program Files/H5Z_ZFP/plugin" >> $GITHUB_PATH 217 | echo "C:/Program Files/ZFP/bin" >> $GITHUB_PATH 218 | echo "C:/Program Files/ZFP/lib" >> $GITHUB_PATH 219 | echo "C:/Program Files/HDF_Group/HDF5/1.14.5/bin" >> $GITHUB_PATH 220 | echo "HDF5_PLUGIN_PATH=C:/Program Files/H5Z_ZFP/plugin" >> $GITHUB_ENV 221 | echo "HDF5_DIR=C:/Program Files/HDF_Group/HDF5/1.14.5" >> $GITHUB_ENV 222 | echo "HDF5_ROOT=C:/Program Files/HDF_Group/HDF5/1.14.5" >> $GITHUB_ENV 223 | echo "ZFP_DIR=C:/Program Files/ZFP" >> $GITHUB_ENV 224 | - name: Build H5Z-ZFP (CMake) and test 225 | run: | 226 | mkdir build 227 | cd build 228 | cmake .. \ 229 | -G "Visual Studio 17 2022" -A "x64" \ 230 | -DCMAKE_CONFIGURATION_TYPES=Release \ 231 | -DBUILD_TESTING=ON \ 232 | -DFORTRAN_INTERFACE=OFF \ 233 | -TClangCL 234 | cmake --build . --config Release --target install 235 | ctest -C Release . 236 | -------------------------------------------------------------------------------- /test/test_error.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2016, Lawrence Livermore National Security, LLC. 3 | Produced at the Lawrence Livermore National Laboratory 4 | Written by Mark C. Miller, miller86@llnl.gov 5 | LLNL-CODE-707197. All rights reserved. 6 | 7 | This file is part of H5Z-ZFP. Please also read the BSD license 8 | https://raw.githubusercontent.com/LLNL/H5Z-ZFP/master/LICENSE 9 | */ 10 | 11 | #include "test_common.h" 12 | 13 | #ifdef H5Z_ZFP_USE_PLUGIN 14 | #include "H5Zzfp_plugin.h" 15 | #else 16 | #include "H5Zzfp_lib.h" 17 | #include "H5Zzfp_props.h" 18 | #endif 19 | 20 | static hid_t setup_filter(int n, hsize_t *chunk, int zfpmode, 21 | double rate, double acc, unsigned int prec, 22 | unsigned int minbits, unsigned int maxbits, unsigned int maxprec, int minexp) 23 | { 24 | hid_t cpid; 25 | 26 | /* setup dataset creation properties */ 27 | if (0 > (cpid = H5Pcreate(H5P_DATASET_CREATE))) SET_ERROR(H5Pcreate); 28 | if (0 > H5Pset_chunk(cpid, n, chunk)) SET_ERROR(H5Pset_chunk); 29 | 30 | /* When filter is used as a library, we need to init it */ 31 | H5Z_zfp_initialize(); 32 | 33 | /* Setup the filter using properties interface. These calls also add 34 | the filter to the pipeline */ 35 | if (zfpmode == H5Z_ZFP_MODE_RATE) 36 | H5Pset_zfp_rate(cpid, rate); 37 | else if (zfpmode == H5Z_ZFP_MODE_PRECISION) 38 | H5Pset_zfp_precision(cpid, prec); 39 | else if (zfpmode == H5Z_ZFP_MODE_ACCURACY) 40 | H5Pset_zfp_accuracy(cpid, acc); 41 | else if (zfpmode == H5Z_ZFP_MODE_EXPERT) 42 | H5Pset_zfp_expert(cpid, minbits, maxbits, maxprec, minexp); 43 | else if (zfpmode == H5Z_ZFP_MODE_REVERSIBLE) 44 | H5Pset_zfp_reversible(cpid); 45 | 46 | return cpid; 47 | } 48 | 49 | typedef struct client_data {char const *str; int has_str;} client_data_t; 50 | 51 | static int walk_hdf5_error_stack_cb(unsigned int n, H5E_error_t const *err_desc, void *_cd) 52 | { 53 | client_data_t *cd = (client_data_t *) _cd; 54 | if (n > 0) return 0; 55 | cd->has_str = strcasestr(err_desc->desc, cd->str) != 0; 56 | return 0; 57 | } 58 | 59 | static int check_hdf5_error_stack_for_string(char const *str) 60 | { 61 | client_data_t cd = {str, 0}; 62 | H5Ewalk(H5E_DEFAULT, H5E_WALK_UPWARD, walk_hdf5_error_stack_cb, &cd); 63 | return cd.has_str; 64 | } 65 | 66 | #define DSIZE 2048 67 | #define FNAME "test_zfp_errors.h5" 68 | 69 | int main(int argc, char **argv) 70 | { 71 | int i, ndiffs; 72 | unsigned corrupt[4] = {0xDeadBeef,0xBabeFace, 0xDeadBabe, 0xBeefFace}; 73 | double d = 1.0, *buf = 0, rbuf[DSIZE]; 74 | hsize_t chunk[] = {DSIZE,16,16,16,16}; 75 | haddr_t off; 76 | hsize_t siz; 77 | 78 | /* HDF5 related variables */ 79 | hid_t fid, tid, dsid, sid, cpid; 80 | 81 | int help = 0; 82 | 83 | /* compression parameters (defaults taken from ZFP header) */ 84 | int zfpmode = H5Z_ZFP_MODE_ACCURACY; 85 | double rate = 4; 86 | double acc = 0.1; 87 | unsigned int prec = 11; 88 | unsigned int minbits = 0; 89 | unsigned int maxbits = 4171; 90 | unsigned int maxprec = 64; 91 | int minexp = -1074; 92 | 93 | /* ZFP filter arguments */ 94 | HANDLE_SEP(Usage: test_error [opt1=value1 opt2=value2]) 95 | HANDLE_ARG(zfpmode,(int) strtol(argv[i]+len2,0,10),"%d", (1=rate,2=prec,3=acc,4=expert,5=reversible)); 96 | HANDLE_ARG(rate,(double) strtod(argv[i]+len2,0),"%g",set rate for rate mode); 97 | HANDLE_ARG(acc,(double) strtod(argv[i]+len2,0),"%g",set accuracy for accuracy mode); 98 | HANDLE_ARG(prec,(unsigned int) strtol(argv[i]+len2,0,10),"%u",set precision for precision mode); 99 | HANDLE_ARG(minbits,(unsigned int) strtol(argv[i]+len2,0,10),"%u",set minbits for expert mode); 100 | HANDLE_ARG(maxbits,(unsigned int) strtol(argv[i]+len2,0,10),"%u",set maxbits for expert mode); 101 | HANDLE_ARG(maxprec,(unsigned int) strtol(argv[i]+len2,0,10),"%u",set maxprec for expert mode); 102 | HANDLE_ARG(minexp,(int) strtol(argv[i]+len2,0,10),"%d",set minexp for expert mode); 103 | 104 | cpid = setup_filter(1, chunk, zfpmode, rate, acc, prec, minbits, maxbits, maxprec, minexp); 105 | 106 | /* Put this after setup_filter to permit printing of otherwise hard to 107 | construct cd_values to facilitate manual invocation of h5repack */ 108 | HANDLE_ARG(help,(int)strtol(argv[i]+len2,0,10),"%d",this help message); /* must be last for help to work */ 109 | 110 | gen_data(DSIZE, 0.01, 10, (void**)&buf, TYPDBL); 111 | 112 | H5Eset_auto(H5E_DEFAULT, 0, 0); 113 | 114 | /* setup the 1D data space */ 115 | if (0 > (sid = H5Screate_simple(1, chunk, 0))) SET_ERROR(H5Screate_simple); 116 | 117 | /* create HDF5 file */ 118 | if (0 > (fid = H5Fcreate(FNAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT))) SET_ERROR(H5Fcreate); 119 | 120 | /* test incorrect data type */ 121 | tid = H5Tcreate(H5T_STRING, 8); 122 | if (0 <= (dsid = H5Dcreate(fid, "bad_type", tid, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); 123 | #if defined(ZFP_LIB_VERSION) && ZFP_LIB_VERSION<=0x052 124 | assert(check_hdf5_error_stack_for_string("requires datatype class of H5T_FLOAT")); 125 | #else 126 | assert(check_hdf5_error_stack_for_string("requires datatype class of H5T_FLOAT or H5T_INTEGER")); 127 | #endif 128 | H5Tclose(tid); 129 | 130 | /* test invalid size of data type */ 131 | tid = H5Tcopy(H5T_NATIVE_DOUBLE); 132 | H5Tset_size(tid, 9); 133 | if (0 <= (dsid = H5Dcreate(fid, "bad_type_size", tid, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); 134 | assert(check_hdf5_error_stack_for_string("requires datatype size of 4 or 8")); 135 | H5Tclose(tid); 136 | 137 | /* test invalid chunking on highd data */ 138 | cpid = setup_filter(5, chunk, zfpmode, rate, acc, prec, minbits, maxbits, maxprec, minexp); 139 | if (0 <= (dsid = H5Dcreate(fid, "bad_chunking", H5T_NATIVE_FLOAT, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); 140 | #if defined(ZFP_LIB_VERSION) && ZFP_LIB_VERSION<=0x053 141 | assert(check_hdf5_error_stack_for_string("chunk must have only 1...3 non-unity dimensions")); 142 | #else 143 | assert(check_hdf5_error_stack_for_string("chunk must have only 1...4 non-unity dimensions")); 144 | #endif 145 | H5Pclose(cpid); 146 | 147 | /* write a compressed dataset to be corrupted later */ 148 | cpid = setup_filter(1, chunk, zfpmode, rate, acc, prec, minbits, maxbits, maxprec, minexp); 149 | if (0 > (dsid = H5Dcreate(fid, "corrupted_data", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); 150 | if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf)) SET_ERROR(H5Dwrite); 151 | off = 3496; // H5Dget_offset(dsid); 152 | siz = H5Dget_storage_size(dsid); 153 | if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); 154 | if (0 > H5Pclose(cpid)) SET_ERROR(H5Pclose); 155 | 156 | /* write a compressed dataset with some nans and infs */ 157 | cpid = setup_filter(1, chunk, zfpmode, rate, acc, prec, minbits, maxbits, maxprec, minexp); 158 | if (0 > (dsid = H5Dcreate(fid, "nans_and_infs", H5T_NATIVE_DOUBLE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) SET_ERROR(H5Dcreate); 159 | memcpy(rbuf, buf, sizeof(rbuf)); 160 | for (i = 7; i < 7+4; i++) 161 | rbuf[i] = d/(d-1.0); 162 | rbuf[42] = sqrt((double)-1.0); 163 | rbuf[42+1] = sqrt((double)-1.0); 164 | if (0 > H5Dwrite(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf)) SET_ERROR(H5Dwrite); 165 | if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); 166 | if (0 > H5Pclose(cpid)) SET_ERROR(H5Pclose); 167 | if (0 > H5Fclose(fid)) SET_ERROR(H5Fclose); 168 | 169 | /* Use raw file I/O to corrupt the dataset named corrupted_data */ 170 | FILE *fp; 171 | fp = fopen(FNAME, "rb+"); 172 | if(fp == NULL) SET_ERROR(fopen); 173 | fseek(fp, (off_t) off + (off_t) siz / 3, SEEK_SET); 174 | fwrite(corrupt, 1 , sizeof(corrupt), fp); 175 | fclose(fp); 176 | 177 | /* Now, open the file with the nans_and_infs and corrupted datasets and try to read them */ 178 | if (0 > (fid = H5Fopen(FNAME, H5F_ACC_RDONLY, H5P_DEFAULT))) SET_ERROR(H5Fopen); 179 | if (0 > (dsid = H5Dopen(fid, "nans_and_infs", H5P_DEFAULT))) SET_ERROR(H5Dopen); 180 | if (0 > H5Dread(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf)) SET_ERROR(H5Dread); 181 | if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); 182 | for (i = 0, ndiffs = 0; i < DSIZE; i++) 183 | { 184 | double d = fabs(rbuf[i] - buf[i]); 185 | if (d > acc) ndiffs++; 186 | } 187 | assert(ndiffs == 10); 188 | if (0 > (dsid = H5Dopen(fid, "corrupted_data", H5P_DEFAULT))) SET_ERROR(H5Dopen); 189 | if (0 > H5Dread(dsid, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf)) SET_ERROR(H5Dread); 190 | for (i = 0, ndiffs = 0; i < DSIZE; i++) 191 | { 192 | double d = fabs(rbuf[i] - buf[i]); 193 | if (d > acc) ndiffs++; 194 | } 195 | assert(ndiffs == 1408); 196 | if (0 > H5Dclose(dsid)) SET_ERROR(H5Dclose); 197 | 198 | free(buf); 199 | if (0 > H5Fclose(fid)) SET_ERROR(H5Fclose); 200 | H5close(); 201 | 202 | return 0; 203 | } 204 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Read the Docs Template documentation build configuration file, created by 4 | # sphinx-quickstart on Tue Aug 26 14:19:49 2014. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | #sys.path.insert(0, os.path.abspath('.')) 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [] 32 | 33 | # sphinx_rtd_theme is now installed via requirements.txt 34 | import sphinx_rtd_theme 35 | extensions.append('sphinx_rtd_theme') 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['_templates'] 39 | 40 | # The suffix of source filenames. 41 | source_suffix = '.rst' 42 | 43 | # The encoding of source files. 44 | #source_encoding = 'utf-8-sig' 45 | 46 | # The master toctree document. 47 | master_doc = 'index' 48 | 49 | # General information about the project. 50 | project = u'H5Z-ZFP' 51 | copyright = u'2016, LLNL-CODE-707197' 52 | 53 | rst_epilog = """ 54 | .. _ZFP: https://computing.llnl.gov/projects/zfp 55 | .. _HDF5: https://docs.hdfgroup.org/hdf5/develop/ 56 | .. _H5Z-ZFP: https://github.com/LLNL/H5Z-ZFP 57 | .. _Spack: https://spack.io 58 | .. _CMake: https://cmake.org/cmake/help/latest/ 59 | """ 60 | 61 | # The version info for the project you're documenting, acts as replacement for 62 | # |version| and |release|, also used in various other places throughout the 63 | # built documents. 64 | # 65 | # The short X.Y version. 66 | version = '1.1' 67 | # The full version, including alpha/beta/rc tags. 68 | release = '1.1.1' 69 | 70 | # The language for content autogenerated by Sphinx. Refer to documentation 71 | # for a list of supported languages. 72 | #language = None 73 | 74 | # There are two options for replacing |today|: either, you set today to some 75 | # non-false value, then it is used: 76 | #today = '' 77 | # Else, today_fmt is used as the format for a strftime call. 78 | #today_fmt = '%B %d, %Y' 79 | 80 | # List of patterns, relative to source directory, that match files and 81 | # directories to ignore when looking for source files. 82 | exclude_patterns = ['_build'] 83 | 84 | # The reST default role (used for this markup: `text`) to use for all 85 | # documents. 86 | #default_role = None 87 | 88 | # If true, '()' will be appended to :func: etc. cross-reference text. 89 | #add_function_parentheses = True 90 | 91 | # If true, the current module name will be prepended to all description 92 | # unit titles (such as .. function::). 93 | #add_module_names = True 94 | 95 | # If true, sectionauthor and moduleauthor directives will be shown in the 96 | # output. They are ignored by default. 97 | #show_authors = False 98 | 99 | # The name of the Pygments (syntax highlighting) style to use. 100 | pygments_style = 'sphinx' 101 | 102 | # A list of ignored prefixes for module index sorting. 103 | #modindex_common_prefix = [] 104 | 105 | # If true, keep warnings as "system message" paragraphs in the built documents. 106 | #keep_warnings = False 107 | 108 | 109 | # -- Options for HTML output ---------------------------------------------- 110 | 111 | # The theme to use for HTML and HTML Help pages. See the documentation for 112 | # a list of builtin themes. 113 | html_theme = "sphinx_rtd_theme" 114 | #html_theme = 'default' 115 | 116 | # Theme options are theme-specific and customize the look and feel of a theme 117 | # further. For a list of options available for each theme, see the 118 | # documentation. 119 | #html_theme_options = {} 120 | 121 | # Add any paths that contain custom themes here, relative to this directory. 122 | #html_theme_path = [] 123 | 124 | # The name for this set of Sphinx documents. If None, it defaults to 125 | # " v documentation". 126 | #html_title = None 127 | 128 | # A shorter title for the navigation bar. Default is the same as html_title. 129 | #html_short_title = None 130 | 131 | # The name of an image file (relative to this directory) to place at the top 132 | # of the sidebar. 133 | #html_logo = None 134 | 135 | # The name of an image file (within the static path) to use as favicon of the 136 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 137 | # pixels large. 138 | #html_favicon = None 139 | 140 | # Add any paths that contain custom static files (such as style sheets) here, 141 | # relative to this directory. They are copied after the builtin static files, 142 | # so a file named "default.css" will overwrite the builtin "default.css". 143 | #html_static_path = ['_static'] 144 | 145 | # Add any extra paths that contain custom files (such as robots.txt or 146 | # .htaccess) here, relative to this directory. These files are copied 147 | # directly to the root of the documentation. 148 | #html_extra_path = [] 149 | 150 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 151 | # using the given strftime format. 152 | #html_last_updated_fmt = '%b %d, %Y' 153 | 154 | # If true, SmartyPants will be used to convert quotes and dashes to 155 | # typographically correct entities. 156 | #html_use_smartypants = True 157 | 158 | # Custom sidebar templates, maps document names to template names. 159 | #html_sidebars = {} 160 | 161 | # Additional templates that should be rendered to pages, maps page names to 162 | # template names. 163 | #html_additional_pages = {} 164 | 165 | # If false, no module index is generated. 166 | #html_domain_indices = True 167 | 168 | # If false, no index is generated. 169 | #html_use_index = True 170 | 171 | # If true, the index is split into individual pages for each letter. 172 | #html_split_index = False 173 | 174 | # If true, links to the reST sources are added to the pages. 175 | #html_show_sourcelink = True 176 | 177 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 178 | #html_show_sphinx = True 179 | 180 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 181 | #html_show_copyright = True 182 | 183 | # If true, an OpenSearch description file will be output, and all pages will 184 | # contain a tag referring to it. The value of this option must be the 185 | # base URL from which the finished HTML is served. 186 | #html_use_opensearch = '' 187 | 188 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 189 | #html_file_suffix = None 190 | 191 | # Output file base name for HTML help builder. 192 | htmlhelp_basename = 'h5zzfpdoc' 193 | 194 | 195 | # -- Options for LaTeX output --------------------------------------------- 196 | 197 | latex_elements = { 198 | # The paper size ('letterpaper' or 'a4paper'). 199 | #'papersize': 'letterpaper', 200 | 201 | # The font size ('10pt', '11pt' or '12pt'). 202 | #'pointsize': '10pt', 203 | 204 | # Additional stuff for the LaTeX preamble. 205 | #'preamble': '', 206 | } 207 | 208 | # Grouping the document tree into LaTeX files. List of tuples 209 | # (source start file, target name, title, 210 | # author, documentclass [howto, manual, or own class]). 211 | latex_documents = [ 212 | ('index', 'H5Z-ZFP.tex', u'H5Z-ZFP Documentation', 213 | u'H5Z-ZFP', 'manual'), 214 | ] 215 | 216 | # The name of an image file (relative to this directory) to place at the top of 217 | # the title page. 218 | #latex_logo = None 219 | 220 | # For "manual" documents, if this is true, then toplevel headings are parts, 221 | # not chapters. 222 | #latex_use_parts = False 223 | 224 | # If true, show page references after internal links. 225 | #latex_show_pagerefs = False 226 | 227 | # If true, show URL addresses after external links. 228 | #latex_show_urls = False 229 | 230 | # Documents to append as an appendix to all manuals. 231 | #latex_appendices = [] 232 | 233 | # If false, no module index is generated. 234 | #latex_domain_indices = True 235 | 236 | 237 | # -- Options for manual page output --------------------------------------- 238 | 239 | # One entry per manual page. List of tuples 240 | # (source start file, name, description, authors, manual section). 241 | man_pages = [ 242 | ('index', 'h5zzfp', u'H5Z-ZFP Documentation', 243 | [u'H5Z-ZFP'], 1) 244 | ] 245 | 246 | # If true, show URL addresses after external links. 247 | #man_show_urls = False 248 | 249 | 250 | # -- Options for Texinfo output ------------------------------------------- 251 | 252 | # Grouping the document tree into Texinfo files. List of tuples 253 | # (source start file, target name, title, author, 254 | # dir menu entry, description, category) 255 | texinfo_documents = [ 256 | ('index', 'H5Z-ZFP', u'H5Z-ZFP Documentation', 257 | u'H5Z-ZFP', 'H5Z-ZFP', 'HDF5 floating point filter plugin.', 258 | 'Miscellaneous'), 259 | ] 260 | 261 | # Documents to append as an appendix to all manuals. 262 | #texinfo_appendices = [] 263 | 264 | # If false, no module index is generated. 265 | #texinfo_domain_indices = True 266 | 267 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 268 | #texinfo_show_urls = 'footnote' 269 | 270 | # If true, do not generate a @detailmenu in the "Top" node's menu. 271 | #texinfo_no_detailmenu = False 272 | -------------------------------------------------------------------------------- /docs/interfaces.rst: -------------------------------------------------------------------------------- 1 | ========== 2 | Interfaces 3 | ========== 4 | 5 | There are two interfaces to control the filter. 6 | One uses HDF5_'s *generic* interface via an array of ``unsigned int cd_values`` as is used in `H5Pset_filter() `__. 7 | The other uses HDF5_ `property lists `__ added to the `dataset creation property list `__ used when the dataset to be compressed is being created. 8 | You can find examples of writing HDF5_ data using both the `generic `_ and `properties `__ interfaces in `test_write.c `_. 9 | 10 | The filter itself supports either interface. The filter also supports all of the standard ZFP_ controls for affecting compression including *rate*, *precision*, *accuracy*, *expert* and *reversible* modes. 11 | For more information and details about these modes of controlling ZFP_ compression, please see the `ZFP README `_. 12 | 13 | Finally, you should *not* attempt to combine the ZFP_ filter with any other *byte order altering* filter such as, for example, HDF5_'s shuffle filter. 14 | Space-performance will be ruined. 15 | This is in contrast to HDF5_'s `deflate `__ filter which often performs *better* when used in conjunction with the `shuffle `__ filter. 16 | To understand why, see the description of :ref:`endian issues `. 17 | 18 | .. _generic-interface: 19 | 20 | ----------------- 21 | Generic Interface 22 | ----------------- 23 | 24 | The generic interface is the only means of controlling the H5Z-ZFP_ filter when it is used as a `dynamically loaded HDF5 plugin `_. 25 | 26 | For the generic interface, the following CPP macros are defined in the ``H5Zzfp_plugin.h`` header file:: 27 | 28 | H5Pset_zfp_rate_cdata(double rate, size_t cd_nelmts, unsigned int *cd_vals); 29 | H5Pset_zfp_precision_cdata(unsigned int prec, size_t cd_nelmts, unsigned int *cd_vals); 30 | H5Pset_zfp_accuracy_cdata(double acc, size_t cd_nelmts, unsigned int *cd_vals); 31 | H5Pset_zfp_expert_cdata(unsigned int minbits, unsigned int maxbits, 32 | unsigned int maxprec, int minexp, 33 | size_t cd_nelmts, unsigned int *cd_vals); 34 | H5Pset_zfp_reversible_cdata(size_t cd_nelmts, unsigned int *cd_vals); 35 | 36 | These macros utilize *type punning* to store the relevant ZFP_ parameters into a sufficiently large array (>=6) of ``unsigned int cd_values``. 37 | It is up to the caller to then call `H5Pset_filter() `__ with the array of cd_values constructed by one of these macros. 38 | 39 | Here is example code from `test_write.c `_... 40 | 41 | .. literalinclude:: ../test/test_write.c 42 | :language: c 43 | :linenos: 44 | :start-after: setup zfp filter via generic (cd_values) interface 45 | :end-before: #else 46 | 47 | However, these macros are only a convenience. 48 | You do not **need** the ``H5Zzfp_plugin.h`` header file if you want to avoid using it. 49 | But, you are then responsible for setting up the ``cd_values`` array correctly for the filter. 50 | For reference, the ``cd_values`` array for this ZFP_ filter is defined like so... 51 | 52 | +-----------+---------------------------------------------------------+ 53 | | | cd_values index | 54 | +-----------+--------+--------+---------+---------+---------+---------+ 55 | | ZFP mode | 0 | 1 | 2 | 3 | 4 | 5 | 56 | +-----------+--------+--------+---------+---------+---------+---------+ 57 | | rate | 1 | unused | rateA | rateB | unused | unused | 58 | +-----------+--------+--------+---------+---------+---------+---------+ 59 | | precision | 2 | unused | prec | unused | unused | unused | 60 | +-----------+--------+--------+---------+---------+---------+---------+ 61 | | accuracy | 3 | unused | accA | accB | unused | unused | 62 | +-----------+--------+--------+---------+---------+---------+---------+ 63 | | expert | 4 | unused | minbits| maxbits| maxprec| minexp | 64 | +-----------+--------+--------+---------+---------+---------+---------+ 65 | | reversible| 5 | unused | unused | unused | unused | unused | 66 | +-----------+--------+--------+---------+---------+---------+---------+ 67 | 68 | A/B are high/low 32-bit words of a double. 69 | 70 | Note that the cd_values used in the generic interface to ``H5Pset_filter()`` are **not the same** cd_values ultimately stored to the HDF5_ dataset header for a compressed dataset. 71 | The values are transformed in the set_local method to use ZFP_'s internal routines for 'meta' and 'mode' data. 72 | So, don't make the mistake of examining the values you find in a file and think you can use those same values, for example, in an invocation of h5repack. 73 | 74 | .. _properties-interface: 75 | 76 | -------------------- 77 | Properties Interface 78 | -------------------- 79 | 80 | For the properties interface, the following functions are defined in the ``H5Zzfp_props.h`` header file:: 81 | 82 | herr_t H5Pset_zfp_rate(hid_t dcpl_id, double rate); 83 | herr_t H5Pset_zfp_precision(hid_t dcpl_id, unsigned int prec); 84 | herr_t H5Pset_zfp_accuracy(hid_t dcpl_id, double acc); 85 | herr_t H5Pset_zfp_expert(hid_t dcpl_id, 86 | unsigned int minbits, unsigned int maxbits, 87 | unsigned int maxprec, int minexp); 88 | herr_t H5Pset_zfp_reversible(hid_t dcpl_id); 89 | 90 | These functions take a dataset creation property list, ``hid_t dcp_lid`` and create temporary HDF5_ property list entries to control the ZFP_ filter. 91 | Calling any of these functions removes the effects of any previous call to any one of these functions. 92 | In addition, calling any one of these functions also has the effect of adding the filter to the pipeline. 93 | 94 | Here is example code from `test_write.c `_... 95 | 96 | .. literalinclude:: ../test/test_write.c 97 | :language: c 98 | :linenos: 99 | :start-after: When filter is used as a library, we need to init it 100 | :end-before: #endif 101 | 102 | The properties interface is more type-safe than the generic interface. 103 | However, there is no way for the implementation of the properties interface to reside within the filter plugin itself. 104 | The properties interface requires that the caller link with with the filter as a *library*, ``libh5zzfp.a``. 105 | The generic interface does not require this. 106 | 107 | Note that either interface can be used whether the filter is used as a plugin or as a library. 108 | The difference is whether the application calls ``H5Z_zfp_initialize()`` or not. 109 | 110 | ----------------- 111 | Fortran Interface 112 | ----------------- 113 | 114 | Fortran equivalents for both the properties and generic interfaces, described above, has been added by Scot Breitenfeld of the HDF5_ group. 115 | The code that implements the Fortran interfaces is in the file ``H5Zzfp_props_f.F90``. 116 | An example of its use is in ``test/test_rw_fortran.F90``. 117 | 118 | .. _plugin-vs-library: 119 | 120 | ---------------------------- 121 | Plugin vs. Library Operation 122 | ---------------------------- 123 | 124 | The filter is designed to be compiled for use as both a standalone HDF5_ `dynamically loaded HDF5 plugin `_ and as an explicitly linked *library*. 125 | When it is used as a plugin, it is a best practice to link the ZFP_ library into the plugin dynamic/shared object as a *static* library. 126 | Why? In so doing, we ensure that all ZFP_ public namespace symbols remain *confined* to the plugin so as not to interfere with any application that may be directly explicitly linking to the ZFP_ library for other reasons. 127 | 128 | All HDF5_ applications are *required* to *find* the plugin dynamic library (named ``lib*.{so,dylib}``) in a directory specified by the environment variable, ``HDF5_PLUGIN_PATH``. 129 | Currently, the HDF5 library offers no mechanism for applications themselves to have pre-programmed paths in which to search for a plugin. 130 | Applications are then always vulnerable to an incorrectly specified or unspecified ``HDF5_PLUGIN_PATH`` environment variable. 131 | 132 | However, the plugin can also be used explicitly as a *library*. 133 | In this case, **do** **not** specify the ``HDF5_PLUGIN_PATH`` environment variable and instead have the application link to ``libH5Zzfp.a`` in the ``lib`` dir of the installation. 134 | Instead two initialization and finalization routines are defined:: 135 | 136 | int H5Z_zfp_initialize(void); 137 | int H5Z_zfp_finalize(void); 138 | 139 | These functions are defined in the ``H5Zzfp_lib.h`` header file. 140 | Any applications that wish to use the filter as a *library* are required to call the initialization routine, ``H5Z_zfp_initialize()`` before the filter can be referenced. 141 | In addition, to free up resources used by the filter, applications may call ``H5Z_zfp_finalize()`` when they are done using the filter. 142 | -------------------------------------------------------------------------------- /docs/hdf5_chunking.rst: -------------------------------------------------------------------------------- 1 | .. _hdf5_chunking: 2 | 3 | ============== 4 | HDF5_ Chunking 5 | ============== 6 | 7 | HDF5_'s dataset `chunking`_ feature is a way to optimize data layout on disk to support partial dataset reads by downstream consumers. 8 | This is all the more important when compression filters are applied to datasets as it frees a consumer from suffering the UNcompression of an entire dataset only to read a portion. 9 | 10 | ------------- 11 | ZFP Chunklets 12 | ------------- 13 | 14 | When using HDF5_ `chunking`_ with ZFP_ compression, it is important to account for the fact that ZFP_ does its work in tiny 4\ :sup:`d` chunklets of its own where `d` is the dataset dimension (*rank* in HDF5_ parlance). 15 | This means that whenever possible, the `chunking`_ dimensions you select in HDF5_ should be multiples of 4. 16 | When a chunk_ dimension is not a multiple of 4, ZFP_ will wind up with partial chunklets, which will be padded with useless data, reducing the results' overall time and space efficiency. 17 | 18 | The degree to which this may degrade performance depends on the percentage of a chunk_ that is padded. 19 | Suppose we have a 2D chunk of dimensions 27 x 101. 20 | ZFP_ will have to treat it as 28 x 104 by padding out each dimension to the next closest multiple of 4. 21 | The fraction of space that will wind up being wasted due to ZFP_ chunklet padding will be (28x104-27x101) / (28x104), which is about 6.4%. 22 | On the other hand, consider a 3D chunk that is 1024 x 1024 x 2. 23 | ZFP_ will have to treat it as a 1024 x 1024 x 4 resulting in 50% waste. 24 | 25 | The latter example is potentially very relevant when applying ZFP_ to compress data along the *time* dimension in a large, 3D, simulation. 26 | Ordinarily, a simulation advances one time step at a time and so needs to store in memory only the *current* timestep. 27 | However, in order to give ZFP_ enough *width* in the time dimension to satisfy the minimum chunklet dimension size of 4, the simulation 28 | needs to keep in memory 4 timesteps. 29 | This is demonstrated in the example below. 30 | 31 | -------------------- 32 | Partial I/O Requests 33 | -------------------- 34 | 35 | In any given H5Dwrite_ call, the caller has the option of writing (or reading) only a portion of the data in the dataset. 36 | This is a *partial I/O* request. 37 | This is handled by the ``mem_space_id`` and ``file_space_id`` arguments in an H5Dwrite_ call. 38 | 39 | An HDF5_ producer or consumer can issue partial I/O requests on *any* HDF5 dataset regardless of whether the dataset is compressed or not or whether the dataset has ``H5D_CONTIGUOUS`` layout. 40 | When combining partial I/O with compression, chunk size and shape in relation to partial I/O request size and shape will have an impact on performance. 41 | 42 | This is particularly important in *writer* scenarios if an I/O request winds up overlapping chunks only partially. 43 | Suppose the partially overlapped chunks exist in the file (from a previous write, for example). In that case, the HDF5_ library may wind up having to engage in *read-modify-write* operations for those chunks. 44 | 45 | If the partially overlapped chunks do not exist in the file, the HDF5_ library will wind up *fill-value* padding the chunks before they are written. 46 | HDF5_'s default fill value is zero (as defined by the associated datatype). 47 | Data producers can choose the desired fill value (see `H5Pset_fill_value `__) for a dataset, but this fill value can impact the space-performance of the compression filter. 48 | On the other hand, if the partial chunks in one I/O request wind up getting fully filled in another, any fill value impacts on compressor performance are resolved. 49 | 50 | Finally, HDF5_ manages a `chunk cache `__ and `data sieving buffer `__ to help alleviate some of the I/O performance issues that can be encountered in these situations. 51 | 52 | ----------------------------- 53 | More Than 3 (or 4) Dimensions 54 | ----------------------------- 55 | 56 | Versions of ZFP_ 0.5.3 and older support compression in only 1,2 or 3 57 | dimensions. Versions of ZFP_ 0.5.4 and newer also support 4 dimensions. 58 | 59 | What if you have a dataset with more dimensions than ZFP_ can compress? 60 | You can still use the H5Z-ZFP_ filter. But, in order to do so, you 61 | are *required* to chunk_ the dataset [1]_ . Furthermore, you must select a 62 | chunk_ size such that no more than 3 (or 4 for ZFP_ 0.5.4 and newer) 63 | dimensions are non-unitary (e.g. of size one). 64 | 65 | For example, what if you are using ZFP_ 0.5.3 and have a 4D HDF5 dataset 66 | you want to compress? To do this, you will need to chunk_ the dataset and 67 | when you define the chunk_ size and shape, you will need to select which 68 | of the 4 dimensions of the chunk you do *not* intend to have ZFP_ perform 69 | compression along by setting the size of the chunk_ in that dimension to 70 | unity (1). When you do this, as HDF5 processes writes and reads, it will 71 | organize the data so that all the H5Z-ZFP_ filter *sees* are chunks 72 | which have *extent* only in the non-unity dimensions of the chunk_. 73 | 74 | In the example below, we have a 4D array of shape ``int dims[] = {256,128,32,16};`` 75 | that we have intentionally constructed to be *smooth* in only 2 of its 4 dimensions 76 | (e.g. correlation is high in those dimensions). Because of that, we expect ZFP_ 77 | compression to do well along those dimensions, and we do not want ZFP_ to compress 78 | along the other 2 dimensions. The *uncorrelated* dimensions here are dimensions 79 | with indices ``1`` (``128`` in ``dims[]``) and ``3`` (``16`` in ``dims[]``). 80 | Thus, our chunk_ size and shape are chosen to set the size for those dimension 81 | indices to ``1``, ``hsize_t hchunk[] = {256,1,32,1};`` 82 | 83 | .. literalinclude:: ../test/test_write.c 84 | :language: c 85 | :linenos: 86 | :start-after: Test high dimensional (>3D) array 87 | :end-before: End of high dimensional test 88 | 89 | What analysis process should you use to select the chunk_ shape? Depending 90 | on what you expect in the way of access patterns in downstream consumers, 91 | this can be a challenging question to answer. There are potentially two 92 | competing interests. One is optimizing the chunk_ size and shape for access 93 | patterns anticipated by downstream consumers. The other is optimizing the chunk_ 94 | size and shape for compression. These two interests may not be compatible 95 | and you may have to compromise between them. We illustrate the issues and 96 | trade-offs using an example. 97 | 98 | --------------------------------------------------- 99 | Compression *Along* the *State Iteration* Dimension 100 | --------------------------------------------------- 101 | 102 | By *state iteration* dimension, we refer to the data producer's main iteration 103 | loop(s). For example, the main iteration dimension for many 104 | PDE-based simulations is *time*. But, for some *outer loop* methods, the 105 | main iteration dimension(s) might be some kind of parameter study including 106 | multiple parameters. 107 | 108 | The challenge here is to manage the data to meet ZFP_'s 109 | chunklet size and shape *minimum* requirements. In any H5Dwrite_ at least 4 110 | *samples* along a ZFP_ compression dimension are needed, or there will 111 | be wasted space due to padding. This means that data must be *buffered* 112 | along those dimensions *before* H5Dwrite_'s can be issued. 113 | 114 | For example, suppose you have a tensor-valued field (e.g. a 3x3 matrix 115 | at every *point*) over a 4D (3 spatial dimensions and 1 time dimension), 116 | regularly sampled domain? Conceptually, this is a 6 dimensional dataset 117 | in HDF5_ with one of the dimensions (the *time* dimension) *extendable*. 118 | So, you are free to define this as a 6 dimensional dataset in HDF5_. But, you 119 | will also have to chunk_ the dataset. You can select any chunk_ shape 120 | you want, except that no more than 3 (or 4 for ZFP_ versions 0.5.4 and 121 | newer) dimensions of the chunk_ can be non-unity. 122 | 123 | In the code snippet below, we demonstrate this case. A key issue to deal 124 | with is that because we will use ZFP_ to compress along the time dimension, 125 | this forces us to keep in memory a sufficient number of timesteps to match 126 | ZFP_'s chunklet size of 4. 127 | 128 | The code below iterates over 9 timesteps. Each of the first two groups of 4 129 | timesteps are buffered in memory in ``tbuf``. Once 4 timesteps have been buffered, we 130 | can issue an H5Dwrite_ call doing 131 | `hyperslab `__ 132 | can issue an H5Dwrite_ 133 | call doing `hyperslab `__ 134 | partial I/O on the 6D, `extendable `__ 135 | dataset. But, notice that the chunk_ dimensions (line 10) are such that only 4 of the 136 | 6 dimensions are non-unity. This means ZFP_ will only ever see something to 137 | compress that is essentially 4D. 138 | 139 | On the last iteration, we have only one *new* timestep. So, when we write this 140 | to ZFP_ 75% of that write will be *wasted* due to ZFP_ chunklet padding. However, 141 | if the application were to *restart* from this time and continue forward, this 142 | *waste* would ultimately get overwritten with new timesteps. 143 | 144 | .. literalinclude:: ../test/test_write.c 145 | :language: c 146 | :linenos: 147 | :start-after: 6D Example 148 | :end-before: End of 6D Example 149 | 150 | .. _chunking: https://portal.hdfgroup.org/display/HDF5/Chunking+in+HDF5 151 | .. _chunk: https://portal.hdfgroup.org/display/HDF5/Chunking+in+HDF5 152 | .. _H5Dwrite: https://docs.hdfgroup.org/hdf5/v1_14/group___h5_d.html#title37 153 | .. [1] The HDF5_ library currently requires dataset chunking anyways for 154 | any dataset that has any kind of filter applied. 155 | 156 | -------------------------------------------------------------------------------- /cmake/HDFMacros.cmake: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------- 2 | macro (DEFAULT_FOLDERS) 3 | #----------------------------------------------------------------------------- 4 | # Setup output Directories 5 | #----------------------------------------------------------------------------- 6 | if (NOT ${H5ZZFP_PACKAGE_NAME}_EXTERNALLY_CONFIGURED) 7 | set (CMAKE_RUNTIME_OUTPUT_DIRECTORY 8 | ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Executables." 9 | ) 10 | set (CMAKE_LIBRARY_OUTPUT_DIRECTORY 11 | ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Libraries" 12 | ) 13 | set (CMAKE_ARCHIVE_OUTPUT_DIRECTORY 14 | ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all static libraries." 15 | ) 16 | set (CMAKE_Fortran_MODULE_DIRECTORY 17 | ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all fortran modules." 18 | ) 19 | if(_isMultiConfig) 20 | set (CMAKE_TEST_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${CMAKE_BUILD_TYPE}) 21 | set (CMAKE_PDB_OUTPUT_DIRECTORY 22 | ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all pdb files." 23 | ) 24 | else () 25 | set (CMAKE_TEST_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) 26 | endif () 27 | else () 28 | # if we are externally configured, but the project uses old cmake scripts 29 | # this may not be set 30 | if (NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY) 31 | set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${EXECUTABLE_OUTPUT_PATH}) 32 | endif () 33 | endif () 34 | endmacro () 35 | 36 | #------------------------------------------------------------------------------- 37 | macro (HDF5_SUPPORT) 38 | if (NOT H5Z-ZFP_HDF5_HEADER) 39 | set (FIND_HDF_COMPONENTS C shared) 40 | message(STATUS "HDF5 FORTRAN_INTERFACE ${FORTRAN_INTERFACE}" ) 41 | if (FORTRAN_INTERFACE) 42 | set (FIND_HDF_COMPONENTS ${FIND_HDF_COMPONENTS} Fortran) 43 | endif () 44 | message (STATUS "HDF5 find comps: ${FIND_HDF_COMPONENTS}") 45 | set (SEARCH_PACKAGE_NAME "HDF5") 46 | 47 | find_package (HDF5 NAMES ${SEARCH_PACKAGE_NAME} COMPONENTS ${FIND_HDF_COMPONENTS}) 48 | if (HDF5_shared_C_FOUND) 49 | set(_HDF5_SHARED_C_STATUS "1") 50 | else() 51 | set(_HDF5_SHARED_C_STATUS "0") 52 | endif() 53 | message (STATUS "HDF5 C libs: shared: ${_HDF5_SHARED_C_STATUS}") 54 | if (FORTRAN_INTERFACE) 55 | if (HDF5_shared_Fortran_FOUND) 56 | set(_HDF5_SHARED_FORTRAN_STATUS "1") 57 | else() 58 | set(_HDF5_SHARED_FORTRAN_STATUS "0") 59 | endif() 60 | message (STATUS "HDF5 Fortran libs: shared: ${_HDF5_SHARED_FORTRAN_STATUS}") 61 | endif() 62 | if (HDF5_FOUND) 63 | if (HDF5_shared_C_FOUND) 64 | # Determine which tool naming convention to use by checking if targets exist 65 | # Newer HDF5 (develop, 1.14.5+) uses h5dump, h5diff, h5repack (without -shared suffix) 66 | # Older HDF5 versions used h5dump-shared, h5diff-shared, h5repack-shared 67 | 68 | if (TARGET ${HDF5_NAMESPACE}h5dump-shared) 69 | set (HDF5_DUMP_EXECUTABLE $) 70 | set (_HDF5_TOOL_SUFFIX "-shared") 71 | elseif (TARGET ${HDF5_NAMESPACE}h5dump) 72 | set (HDF5_DUMP_EXECUTABLE $) 73 | set (_HDF5_TOOL_SUFFIX "") 74 | else () 75 | # No target exists, create imported target and set its location 76 | # Prefer h5dump-shared for older HDF5, but will set actual path later 77 | add_executable (${HDF5_NAMESPACE}h5dump-shared IMPORTED) 78 | set_property (TARGET ${HDF5_NAMESPACE}h5dump-shared PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5dump-shared") 79 | set (HDF5_DUMP_EXECUTABLE $) 80 | set (_HDF5_TOOL_SUFFIX "-shared") 81 | endif () 82 | 83 | if (TARGET ${HDF5_NAMESPACE}h5diff-shared) 84 | set (HDF5_DIFF_EXECUTABLE $) 85 | elseif (TARGET ${HDF5_NAMESPACE}h5diff) 86 | set (HDF5_DIFF_EXECUTABLE $) 87 | else () 88 | # No target exists, create imported target and set its location 89 | add_executable (${HDF5_NAMESPACE}h5diff-shared IMPORTED) 90 | set_property (TARGET ${HDF5_NAMESPACE}h5diff-shared PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5diff-shared") 91 | set (HDF5_DIFF_EXECUTABLE $) 92 | endif () 93 | 94 | if (TARGET ${HDF5_NAMESPACE}h5repack-shared) 95 | set (HDF5_REPACK_EXECUTABLE $) 96 | elseif (TARGET ${HDF5_NAMESPACE}h5repack) 97 | set (HDF5_REPACK_EXECUTABLE $) 98 | else () 99 | # No target exists, create imported target and set its location 100 | add_executable (${HDF5_NAMESPACE}h5repack-shared IMPORTED) 101 | set_property (TARGET ${HDF5_NAMESPACE}h5repack-shared PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5repack-shared") 102 | set (HDF5_REPACK_EXECUTABLE $) 103 | endif () 104 | 105 | message (STATUS "HDF5 Tools configured (h5dump${_HDF5_TOOL_SUFFIX}, h5diff${_HDF5_TOOL_SUFFIX}, h5repack${_HDF5_TOOL_SUFFIX})") 106 | else () 107 | if (NOT TARGET ${HDF5_NAMESPACE}h5dump) 108 | add_executable (${HDF5_NAMESPACE}h5dump IMPORTED) 109 | endif () 110 | set (HDF5_DUMP_EXECUTABLE $) 111 | 112 | if (NOT TARGET ${HDF5_NAMESPACE}h5diff) 113 | add_executable (${HDF5_NAMESPACE}h5diff IMPORTED) 114 | endif () 115 | set (HDF5_DIFF_EXECUTABLE $) 116 | 117 | if (NOT TARGET ${HDF5_NAMESPACE}h5repack) 118 | add_executable (${HDF5_NAMESPACE}h5repack IMPORTED) 119 | endif () 120 | set (HDF5_REPACK_EXECUTABLE $) 121 | message (STATUS "HDF5 static Tools found - ${HDF5_DUMP_EXECUTABLE}") 122 | endif() 123 | 124 | if (NOT HDF5_static_C_FOUND AND NOT HDF5_shared_C_FOUND) 125 | #find library from non-dual-binary package 126 | set (FIND_HDF_COMPONENTS C) 127 | if (FORTRAN_INTERFACE) 128 | set (FIND_HDF_COMPONENTS ${FIND_HDF_COMPONENTS} Fortran) 129 | endif () 130 | message (STATUS "HDF5 find comps: ${FIND_HDF_COMPONENTS}") 131 | 132 | find_package (HDF5 NAMES ${SEARCH_PACKAGE_NAME} COMPONENTS ${FIND_HDF_COMPONENTS}) 133 | message (STATUS "HDF5 libs:${HDF5_FOUND} C:${HDF5_C_FOUND} Fortran:${HDF5_Fortran_FOUND}") 134 | if (HDF5_BUILD_SHARED_LIBS) 135 | add_definitions (-DH5_BUILT_AS_DYNAMIC_LIB) 136 | else () 137 | add_definitions (-DH5_BUILT_AS_STATIC_LIB) 138 | endif () 139 | if (FORTRAN_INTERFACE AND HDF5_BUILD_FORTRAN) 140 | if (HDF5_shared_Fortran_FOUND) 141 | set (HDF5_FORTRAN_INCLUDE_DIRS ${HDF5_INCLUDE_DIR_FORTRAN}) 142 | set (HDF5_FORTRAN_LIBRARIES ${HDF5_FORTRAN_SHARED_LIBRARY}) 143 | else () 144 | set (FORTRAN_INTERFACE OFF CACHE BOOL "Build FORTRAN support" FORCE) 145 | message (STATUS "HDF5 Fortran libs not found - disable build of Fortran support") 146 | endif () 147 | endif () 148 | if (WIN32) 149 | set_property (TARGET ${HDF5_NAMESPACE}h5dump PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5dumpdll") 150 | set_property (TARGET ${HDF5_NAMESPACE}h5diff PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5diffdll") 151 | set_property (TARGET ${HDF5_NAMESPACE}h5repack PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5repackdll") 152 | endif () 153 | set (HDF5_DUMP_EXECUTABLE $) 154 | set (HDF5_DIFF_EXECUTABLE $) 155 | set (HDF5_REPACK_EXECUTABLE $) 156 | message (STATUS "HDF5 windows Tools found - ${HDF5_DUMP_EXECUTABLE}") 157 | else () 158 | if (HDF5_shared_C_FOUND) 159 | set (HDF5_LIBRARIES ${HDF5_C_SHARED_LIBRARY}) 160 | set (HDF5_LIBRARY_PATH ${PACKAGE_PREFIX_DIR}/lib) 161 | else () 162 | set (HDF5_FOUND 0) 163 | endif () 164 | if (FORTRAN_INTERFACE AND HDF5_BUILD_FORTRAN) 165 | if (HDF5_shared_Fortran_FOUND) 166 | set (HDF5_FORTRAN_INCLUDE_DIRS ${HDF5_INCLUDE_DIR_FORTRAN}) 167 | set (HDF5_FORTRAN_LIBRARIES ${HDF5_FORTRAN_SHARED_LIBRARY}) 168 | else () 169 | set (FORTRAN_INTERFACE OFF CACHE BOOL "Build FORTRAN support" FORCE) 170 | message (STATUS "HDF5 Fortran libs not found - disable build of Fortran support") 171 | endif () 172 | else () 173 | set (FORTRAN_INTERFACE OFF CACHE BOOL "Build FORTRAN support" FORCE) 174 | message (STATUS "HDF5 Fortran libs not found - disable build of Fortran support") 175 | endif () 176 | message (STATUS "HDF5 Tools imported location - ${HDF5_TOOLS_DIR}") 177 | endif () 178 | else () 179 | if (FORTRAN_INTERFACE) 180 | set(FORTRAN_COMP "Fortran") 181 | endif() 182 | find_package (HDF5 COMPONENTS ${FORTRAN_COMP}) # Legacy find 183 | if (FORTRAN_INTERFACE AND NOT HDF5_Fortran_FOUND) 184 | set (FORTRAN_INTERFACE OFF CACHE BOOL "Build FORTRAN support" FORCE) 185 | message (STATUS "HDF5 Fortran libs not found - disable build of Fortran support") 186 | endif () 187 | #Legacy find_package does not set HDF5_TOOLS_DIR, so we set it here 188 | get_filename_component(HDF5_BIN_DIR ${HDF5_DIFF_EXECUTABLE} DIRECTORY) 189 | set(HDF5_TOOLS_DIR ${HDF5_BIN_DIR}) 190 | if (NOT TARGET hdf5::h5dump) 191 | add_executable (${HDF5_NAMESPACE}h5dump IMPORTED) 192 | set_property (TARGET ${HDF5_NAMESPACE}h5dump PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5dump") 193 | set (HDF5_DUMP_EXECUTABLE $) 194 | endif () 195 | 196 | if (NOT TARGET hdf5::h5diff) 197 | add_executable (${HDF5_NAMESPACE}h5diff IMPORTED) 198 | set_property (TARGET ${HDF5_NAMESPACE}h5diff PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5diff") 199 | set (HDF5_DIFF_EXECUTABLE $) 200 | endif () 201 | 202 | if (NOT TARGET hdf5::h5repack) 203 | add_executable (${HDF5_NAMESPACE}h5repack IMPORTED) 204 | set_property (TARGET ${HDF5_NAMESPACE}h5repack PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5repack") 205 | set (HDF5_REPACK_EXECUTABLE $) 206 | endif () 207 | message (STATUS "HDF5 legacy Tools found - ${HDF5_DUMP_EXECUTABLE}") 208 | endif () 209 | 210 | set (HDF5_PACKAGE_NAME ${SEARCH_PACKAGE_NAME}) 211 | 212 | if (HDF5_FOUND) 213 | set (HDF5_HAVE_H5PUBCONF_H 1) 214 | set (HDF5_HAVE_HDF5 1) 215 | set (H5Z-ZFP_HDF5_HEADER "h5pubconf.h") 216 | set (HDF5_INCLUDE_DIRS ${HDF5_INCLUDE_DIR}) 217 | message (STATUS "HDF5-${HDF5_VERSION_STRING} found: INC=${HDF5_INCLUDE_DIR} TOOLS=${HDF5_TOOLS_DIR}") 218 | else () 219 | message (FATAL_ERROR " HDF5 shared is required for H5Z-ZFP") 220 | endif () 221 | else () 222 | # This project is being called from within another and HDF5 is already configured 223 | set (HDF5_HAVE_H5PUBCONF_H 1) 224 | set (HDF5_HAVE_HDF5 1) 225 | set (HDF5_LIBRARIES ${HDF5_LINK_LIBS}) 226 | set (HDF5_INCLUDE_DIRS ${HDF5_INCLUDE_DIR}) 227 | endif () 228 | set (HDF5_LIBRARY_PATH ${PACKAGE_PREFIX_DIR}/lib) 229 | if (FORTRAN_INTERFACE) 230 | message (STATUS "HDF5 Fortran libs: include:${HDF5_FORTRAN_INCLUDE_DIRS} and shared:${HDF5_FORTRAN_LIBRARIES}") 231 | endif () 232 | message (STATUS "HDF5 link libs: ${HDF5_LIBRARIES} Includes: ${HDF5_INCLUDE_DIRS}") 233 | endmacro () 234 | 235 | #------------------------------------------------------------------------------- 236 | macro (SET_HDF_BUILD_TYPE) 237 | if (_isMultiConfig) 238 | # HDF_CFG_BUILD_TYPE is used in the Fortran install commands for the build location of the .mod files 239 | set (HDF_CFG_BUILD_TYPE \${CMAKE_INSTALL_CONFIG_NAME}) 240 | if (CMAKE_BUILD_TYPE) 241 | # set the default to the specified command line define 242 | set (HDF_CFG_NAME ${CMAKE_BUILD_TYPE}) 243 | else () 244 | # set the default to the MultiConfig variable 245 | set (HDF_CFG_NAME "$") 246 | endif () 247 | else () 248 | set (HDF_CFG_BUILD_TYPE ".") 249 | if (CMAKE_BUILD_TYPE) 250 | set (HDF_CFG_NAME ${CMAKE_BUILD_TYPE}) 251 | else () 252 | set (HDF_CFG_NAME "Release") 253 | endif () 254 | endif () 255 | if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) 256 | if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") 257 | message (VERBOSE "Setting build type to 'RelWithDebInfo' as none was specified.") 258 | endif() 259 | set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Choose the type of build." FORCE) 260 | # Set the possible values of build type for cmake-gui 261 | set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" 262 | "MinSizeRel" "RelWithDebInfo") 263 | endif() 264 | endmacro () 265 | 266 | #------------------------------------------------------------------------------- 267 | macro (TARGET_C_PROPERTIES wintarget libtype) 268 | target_compile_options(${wintarget} PRIVATE 269 | "$<$:${WIN_COMPILE_FLAGS}>" 270 | "$<$:${WIN_COMPILE_FLAGS}>" 271 | ) 272 | if(MSVC) 273 | set_property(TARGET ${wintarget} APPEND PROPERTY LINK_FLAGS "${WIN_LINK_FLAGS}") 274 | endif() 275 | endmacro () 276 | 277 | macro (HDFTEST_COPY_FILE src dest target) 278 | add_custom_command( 279 | OUTPUT "${dest}" 280 | COMMAND "${CMAKE_COMMAND}" 281 | ARGS -E copy_if_different "${src}" "${dest}" 282 | DEPENDS "${src}" 283 | ) 284 | list (APPEND ${target}_list "${dest}") 285 | endmacro () 286 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Installation 3 | ============ 4 | 5 | Three ways to install H5Z-ZFP_ are provided. 6 | These are 7 | 8 | * With generic (GNU) :ref:`make ` 9 | * With :ref:`CMake ` 10 | * With :ref:`Spack ` 11 | 12 | For both generic make and CMake_, you are responsible for also installing (or knowing where the installations are) the dependencies, ZFP_ and HDF5_. 13 | For Spack_ installations, Spack_ will handle installation of dependencies as well. 14 | 15 | .. _gnumake: 16 | 17 | --------------------------------- 18 | Installing via Generic (GNU) Make 19 | --------------------------------- 20 | 21 | H5Z-ZFP_ installation supports both vanilla (`GNU `__) Make (described below) as well as :ref:`CMake `. 22 | 23 | ^^^^^^^^^^^^^ 24 | Prerequisites 25 | ^^^^^^^^^^^^^ 26 | 27 | * `ZFP Library `_ (or from `Github `_) 28 | * `HDF5 Library `_ 29 | * `H5Z-ZFP filter plugin `_ 30 | 31 | .. _zfp-config: 32 | 33 | ^^^^^^^^^^^^^^ 34 | Compiling ZFP_ 35 | ^^^^^^^^^^^^^^ 36 | 37 | * There is a ``Config`` file in top-level directory of the ZFP_ distribution that holds ``make`` variables the ZFP_ Makefiles use. By default, this file is setup for a vanilla GNU compiler. 38 | If this is not the appropriate compiler, edit ``Config`` as necessary to adjust the compiler and compilation flags. 39 | * An important flag you **will** need to adjust in order to use the ZFP_ library with this HDF5_ filter is the ``BIT_STREAM_WORD_TYPE`` CPP flag. 40 | To use ZFP_ with H5Z-ZFP_, the ZFP_ library **must** be compiled with ``BIT_STREAM_WORD_TYPE`` of ``uint8``. 41 | Typically, this is achieved by including a line in ``Config`` of the form ``DEFS += -DBIT_STREAM_WORD_TYPE=uint8``. 42 | If you attempt to use this filter with a ZFP_ library compiled differently from this, the filter's ``can_apply`` method will always return false. 43 | This will result in silently ignoring an HDF5_ client's request to compress data with ZFP_. 44 | Also, be sure to see :ref:`endian-issues`. 45 | * After you have setup ``Config``, simply run ``make`` and it will build the ZFP_ library placing the library in a ``lib`` sub-directory and the necessary include files in ``inc[lude]`` sub-directory. 46 | * For more information and details, please see the `ZFP README `_. 47 | 48 | ^^^^^^^^^^^^^^^ 49 | Compiling HDF5_ 50 | ^^^^^^^^^^^^^^^ 51 | 52 | * If you want to be able to run the fortran tests for this filter, HDF5_ must be configured with *both* the ``--enable-fortran`` and ``--enable-fortran2003`` configuration switches. 53 | Otherwise, any vanilla installation of HDF5_ is acceptable. 54 | 55 | * The Fortran interface to this filter *requires* a Fortran 2003 compiler because it uses `ISO_C_BINDING `_ to define the Fortran interface. 56 | 57 | * If you are using HDF5-1.12 and wish to use the filter as a *library* (see :ref:`plugin-vs-library`), you may need configure HDF5 with ``--disable-memory-alloc-sanity-check`` to work around a memory management issue in HDF5. 58 | 59 | ^^^^^^^^^^^^^^^^^ 60 | Compiling H5Z-ZFP 61 | ^^^^^^^^^^^^^^^^^ 62 | 63 | H5Z-ZFP_ is designed to be compiled both as a standalone HDF5_ *plugin* and as a separate *library* an application can explicitly link. See :ref:`plugin-vs-library`. 64 | 65 | Once you have installed the prerequisites, you can compile H5Z-ZFP_ using a command-line... 66 | 67 | :: 68 | 69 | make [FC=] CC= \ 70 | ZFP_HOME= HDF5_HOME= \ 71 | PREFIX= 72 | 73 | where ```` is a directory containing ZFP_ ``inc[lude]`` and ``lib`` dirs and ```` is a directory containing HDF5_ ``include`` and ``lib`` dirs. 74 | If you don't specify a C compiler, it will try to guess one from your path. 75 | Fortran compilation is optional. 76 | If you do not specify a Fortran compiler, it will not attempt to build the Fortran interface. 77 | However, if the variable ``FC`` is already defined in your environment (as in Spack_ for example), then H5Z-ZFP_ will attempt to build Fortran. 78 | If this is not desired, the solution is to pass an *empty* ``FC`` on the make command line as in... 79 | 80 | :: 81 | 82 | make FC= CC= \ 83 | ZFP_HOME= HDF5_HOME= \ 84 | PREFIX= 85 | 86 | 87 | The Makefile uses GNU Make syntax and is designed to work on OSX and Linux. The filter has been tested on gcc, clang, xlc, icc and pgcc compilers and checked with valgrind. 88 | 89 | The command ``make help`` will print useful information about various make targets and variables. ``make check`` will compile everything and run a handful of tests. 90 | 91 | If you don't specify a ``PREFIX``, it will install to ``./install``. 92 | The installed package will look like... 93 | 94 | :: 95 | 96 | $(PREFIX)/include/{H5Zzfp.h,H5Zzfp_plugin.h,H5Zzfp_props.h,H5Zzfp_lib.h} 97 | $(PREFIX)/plugin/libh5zzfp.{so,dylib} 98 | $(PREFIX)/lib/libh5zzfp.a 99 | 100 | where ``$(PREFIX)`` resolves to whatever the full path of the installation is. 101 | 102 | To use the installed filter as an HDF5_ *plugin*, you would specify, for example, 103 | ``setenv HDF5_PLUGIN_PATH $(PREFIX)/plugin`` 104 | 105 | .. _ceemake: 106 | 107 | -------------------- 108 | Installing via CMake 109 | -------------------- 110 | 111 | It is possible to build the H5Z-ZFP_ filter using the CMake_ build system. 112 | To use CMake_ for H5Z-ZFP_, it is necessary to have also built ZFP_ with CMake. 113 | This is necessary to get the correct dependencies from ZFP_. 114 | For example, it is possible to build ZFP_ with OpenMP support. 115 | The resulting CMake_ config files of ZFP_ build will make sure that this OpenMP dependency is correctly propagated to the build of H5Z-ZFP_ filter. 116 | However, for HDF5_ it is not necessary to build it with its CMake_ build system but it is strongly recommended. 117 | 118 | ZFP_ must have been :ref:`configured ` with ``BIT_STREAM_WORD_TYPE`` of ``uint8`` as described above. 119 | 120 | Similar as for the Makefile installation, the CMake_ build system is designed such it compiles both the standalone HDF5_ *plugin* and a separate *library* an application can explicitly link. See :ref:`plugin-vs-library` 121 | 122 | Once both HDF5_ and ZFP_ have been installed, H5Z-ZFP_ can be compiled using a command=line... 123 | 124 | :: 125 | 126 | export HDF5_DIR= 127 | export ZFP_DIR= 128 | CC= FC= cmake -DCMAKE_INSTALL_PREFIX= 129 | 130 | where ```` is a directory containing ``zfp-config.cmake`` and ```` is a directory containing HDF5_ ``include`` and ``lib`` directories. 131 | Furthermore, ``src-dir`` is the directory where the H5Z-ZFP_ source is located and ``path-to-install`` is the directory in which the resulting *plugin* and *library* will be installed. 132 | Once ``cmake`` has finished successfully, you can build and install the filter using the command... 133 | 134 | :: 135 | 136 | make install 137 | 138 | This ``cmake`` and ``make`` combination builds both the C and Fortran interface. 139 | In the case you want to specify the ```` and ``>`` via command-line to CMake_, the command looks like this... 140 | 141 | :: 142 | 143 | CC= FC= cmake -DCMAKE_INSTALL_PREFIX= 144 | -DCMAKE_PREFIX_PATH=";" 145 | 146 | .. note:: 147 | 148 | The double quotes in the CMAKE_PREFIX_PATH expression are necessary to make sure that semicolon is interpreted as a semicolon instead of a new command. 149 | 150 | It is possible to build the filter without the Fortran interface. This is done as follows... 151 | 152 | :: 153 | 154 | export HDF5_DIR= 155 | export ZFP_DIR= 156 | CC= cmake -DCMAKE_INSTALL_PREFIX= -DFORTRAN_INTERFACE:BOOL=OFF 157 | 158 | followed by the same make command... 159 | 160 | :: 161 | 162 | make install 163 | 164 | ------------------------------------------- 165 | Including H5Z-ZFP filter in a CMake project 166 | ------------------------------------------- 167 | 168 | Suppose you have built the H5Z-ZFP_ filter using the CMake_ build system and installed it in ````. 169 | To include it in another CMake_ project is done using the following steps. First edit the ``CMakeLists.txt`` 170 | by adding the following two lines... 171 | 172 | :: 173 | 174 | cmake_policy(SET CMP0028 NEW) # Double colon in target name means ALIAS or IMPORTED target. 175 | ... 176 | set(H5Z_ZFP_USE_STATIC_LIBS OFF) 177 | find_package(H5Z_ZFP 1.0.1 CONFIG) 178 | ... 179 | target_link_libraries( h5z_zfp::h5z_zfp) 180 | ... 181 | 182 | where ```` in the target within the CMake_ project. 183 | This could be, for example, an executable or library. 184 | Furthermore, check if the ``cmake`` version is equal or greater than 3.9. 185 | Next, you need to make sure that the filter can be found by CMake_, followed by ``cmake`` itself and ``make``... 186 | 187 | :: 188 | 189 | export H5Z_ZFP_DIR= 190 | CC= cmake -DCMAKE_INSTALL_PREFIX= 191 | make install 192 | 193 | The ``cmake`` command itself could be different depending on the CMake_ project you have created. 194 | If you want to make use of the H5Z-ZFP_ *library* instead of the plugin, change cmake variable ``H5Z_ZFP_USE_STATIC_LIBS`` to ``ON`` and build the project. 195 | 196 | .. _spack2: 197 | 198 | --------------------- 199 | Installing via Spack_ 200 | --------------------- 201 | If you already have experience with Spack_, one way to install H5Z-ZFP_ is to use the command ``spack install h5z-zfp``. 202 | If you do not have Spack_ installed, it is easy to install. 203 | Assuming you are working in a Bash shell...:: 204 | 205 | git clone https://github.com/llnl/spack.git 206 | cd spack 207 | git checkout releases/v0.20 208 | . ./share/spack/setup-env.sh 209 | spack install h5z-zfp 210 | 211 | .. note:: 212 | 213 | It is important to work from a *released* branch of Spack_. 214 | The command ``git checkout releases/v0.20`` ensures this. 215 | If a newer release of Spack_ is available, by all means feel free to use it. 216 | Just change the ``v0.20`` to indicate the release of the Spack_ you want. 217 | The command ``git branch -r | grep releases`` will produce a list of the available release branches. 218 | 219 | If you are using a version of Spack_ very much older than the release of H5Z-ZFP_ you intend to use, you may have to *pin* various versions of H5Z-ZFP_, ZFP_ and/or HDF5_. 220 | This is done by using Spack_'s ``@`` modifier to specify versions. 221 | For example, to *pin* the version of the ZFP_ library to 0.5.5, the Spack_ command would look like:: 222 | 223 | spack install h5z-zfp ^zfp@0.5.5 224 | 225 | To use the ``develop`` version of H5Z-ZFP_ with version 1.10.6 of HDF5_ :: 226 | 227 | spack install h5z-zfp@develop ^hdf5@1.10.6 228 | 229 | By default, H5Z-ZFP_ will attempt to build with Fortran support which requires a Fortran compiler. 230 | If you wish to exclude support for Fortran, use the command:: 231 | 232 | spack install h5z-zfp~fortran 233 | 234 | Spack_ packages can sometimes favor the use of dependencies you may not need. 235 | For example, the HDF5_ package favors the use of MPI. 236 | Since H5Z-ZFP_ depends on HDF5_, this behavior will then create a dependency of H5Z-ZFP_ on MPI. 237 | To avoid this, you can force Spack_ to use a version of HDF5_ *without* MPI. 238 | In the example command below, we force Spack_ to not use MPI with HDF5_ and to not use OpenMP with ZFP_:: 239 | 240 | spack install h5z-zfp~fortran ^hdf5~mpi~fortran ^zfp~openmp 241 | 242 | This can have the effect of substantially reducing the number of dependencies Spack_ winds up having to build (from 35 in one case to 10) in order to install H5Z-ZFP_ which, in turn, speeds up the install process. 243 | 244 | .. note:: 245 | 246 | Spack_ will build H5Z-ZFP_ **and** all of its dependencies including the HDF5_ library *as well as a number of other dependencies you may not initially expect*. 247 | Be patient and let the build complete. 248 | It may take as much as an hour. 249 | 250 | In addition, by default, Spack_ installs packages to directory *hashes within* the cloned Spack_ repository's directory tree, ``$spack/opt/spack``. 251 | You can find the resulting installed HDF5_ library with the command ``spack find -vp hdf5`` and the resulting H5Z-ZFP_ plugin installation with the command ``spack find -vp h5z-zfp``. 252 | If you wish to exercise more control over how and where Spack_ installs, have a look at 253 | `configuring Spack `_ 254 | 255 | -------------------------------- 256 | H5Z-ZFP Source Code Organization 257 | -------------------------------- 258 | 259 | The source code is in two separate directories 260 | 261 | * ``src`` includes the ZFP_ filter and a few header files 262 | 263 | * ``H5Zzfp_plugin.h`` is an optional header file applications *may* wish to include because it contains several convenient macros for easily controlling various compression modes of the ZFP_ library (*rate*, *precision*, *accuracy*, *expert*) via the :ref:`generic-interface`. 264 | * ``H5Zzfp_props.h`` is a header file that contains functions to control the filter using *temporary* :ref:`properties-interface`. 265 | Fortran callers are *required* to use this interface. 266 | * ``H5Zzfp_lib.h`` is a header file for applications that wish to use the filter explicitly as a library rather than a plugin. 267 | * ``H5Zzfp.h`` is an *all-of-the-above* header file for applications that don't care too much about separating out the above functionalities. 268 | 269 | * ``test`` includes various tests. In particular ``test_write.c`` includes examples of using both the :ref:`generic-interface` and :ref:`properties-interface`. 270 | In addition, there is an example of how to use the filter from Fortran in ``test_rw_fortran.F90``. 271 | 272 | ---------------- 273 | Silo Integration 274 | ---------------- 275 | 276 | This filter (``H5Zzfp.c``) is also built-in to the `Silo library `_. 277 | In particular, the ZFP_ library itself is also embedded in Silo but is protected from appearing in Silo's global namespace through a struct of function pointers (see `Namespaces in C `_). 278 | If you happen to examine the source code here for H5Z-ZFP_, you will see some logic here that is specific to using this plugin within Silo and dealing with ZFP_ as an embedded library using this struct of function pointers wrapper. 279 | In the source code for H5Z-ZFP_ this manifests as something like what is shown in the code snippet below... 280 | 281 | .. literalinclude:: ../src/H5Zzfp.c 282 | :language: c 283 | :linenos: 284 | :start-after: /* set up dummy zfp field to compute meta header */ 285 | :end-before: if (!dummy_field) 286 | 287 | In the code snippet above, note the funny ``Z`` in front of calls to various methods in the ZFP_ library. 288 | When compiling H5Z-ZFP_ normally, that ``Z`` normally resolves to the empty string. 289 | But, when the code is compiled with ``-DAS_SILO_BUILTIN`` (which is supported and should be done *only* when ``H5Zzfp.c`` is being compiled *within* the Silo library and *next to* a version of ZFP_ that is embedded in Silo) that ``Z`` resolves to the name of a struct and struct-member dereferencing operator as in ``zfp.``. 290 | There is a similar ``B`` used for a similar purpose ahead of calls to ZFP_'s bitstream library. 291 | This is something to be aware of and to adhere to if you plan to contribute any code changes here. 292 | -------------------------------------------------------------------------------- /test/test_rw_fortran.F90: -------------------------------------------------------------------------------- 1 | PROGRAM main 2 | 3 | USE ISO_C_BINDING 4 | USE ISO_FORTRAN_ENV, ONLY: ERROR_UNIT, OUTPUT_UNIT 5 | USE HDF5 6 | USE h5zzfp_props_f 7 | IMPLICIT NONE 8 | 9 | INTEGER, PARAMETER :: dp = C_DOUBLE 10 | 11 | INTEGER, PARAMETER :: NAME_LEN=256 12 | INTEGER, PARAMETER :: DIM0=32 13 | INTEGER, PARAMETER :: DIM1=64 14 | INTEGER, PARAMETER :: CHUNK0=4 15 | INTEGER, PARAMETER :: CHUNK1=8 16 | 17 | INTEGER :: i 18 | INTEGER(hsize_t) :: j 19 | 20 | ! sinusoid data generation variables 21 | INTEGER(hsize_t) :: npoints 22 | 23 | ! compression parameters (defaults taken from ZFP header) 24 | INTEGER(C_INT) :: zfpmode = 3 !1=rate, 2=prec, 3=acc, 4=expert 25 | REAL(dp) :: rate = 4_c_double 26 | REAL(dp) :: acc = 0_c_double 27 | INTEGER(C_INT) :: prec = 11 28 | INTEGER(C_INT) :: dim = 0 29 | INTEGER(C_INT), PARAMETER :: minbits = 0 30 | INTEGER(C_INT), PARAMETER :: maxbits = 4171 31 | INTEGER(C_INT), PARAMETER :: maxprec = 64 32 | INTEGER(C_INT), PARAMETER :: minexp = -1074 33 | 34 | ! HDF5 related variables 35 | INTEGER(hid_t) fid, dsid, sid, cpid, dcpl_id, space_id 36 | INTEGER(C_INT), DIMENSION(1:H5Z_ZFP_CD_NELMTS_MEM) :: cd_values 37 | INTEGER(C_SIZE_T) :: cd_nelmts = H5Z_ZFP_CD_NELMTS_MEM 38 | 39 | ! compressed/uncompressed difference stat variables 40 | REAL(dp) :: max_absdiff = 1.e-8_dp 41 | REAL(dp) :: max_reldiff = 1.e-8_dp 42 | INTEGER(C_INT) :: num_diffs = 0 43 | 44 | REAL(dp) :: noise = 0.001 45 | REAL(dp) :: amp = 17.7 46 | 47 | REAL(dp), DIMENSION(1:DIM0,1:DIM1), TARGET :: wdata 48 | INTEGER(hsize_t), DIMENSION(1:2) :: dims = (/DIM0, DIM1/) 49 | INTEGER(hsize_t), DIMENSION(1:2) :: dims1; 50 | INTEGER(hsize_t), DIMENSION(1:2) :: chunk2 = (/CHUNK0, CHUNK1/) 51 | INTEGER(hsize_t), DIMENSION(1:1) :: chunk256 = (/256/) 52 | REAL(dp), DIMENSION(:), ALLOCATABLE, TARGET :: obuf, cbuf, cbuf1, cbuf2 53 | CHARACTER(LEN=180) :: ofile="test_zfp_fortran.h5" 54 | 55 | INTEGER :: status 56 | TYPE(C_PTR) :: f_ptr 57 | INTEGER, PARAMETER :: H5Z_FLAG_MANDATORY = INT(Z'00000000') 58 | REAL(dp) :: absdiff, reldiff 59 | 60 | INTERFACE 61 | LOGICAL FUNCTION real_eq(a,b,ulp) 62 | USE ISO_C_BINDING 63 | IMPLICIT NONE 64 | REAL(C_DOUBLE), INTENT (in):: a,b 65 | REAL(C_DOUBLE) :: Rel 66 | INTEGER, OPTIONAL, INTENT( IN ) :: ulp 67 | END FUNCTION real_eq 68 | END INTERFACE 69 | 70 | CHARACTER(LEN=180) :: arg 71 | INTEGER :: len 72 | LOGICAL :: write_only = .FALSE., avail 73 | INTEGER :: config_flag = 0 ! for h5zget_filter_info_f 74 | INTEGER :: config_flag_both = 0 ! for h5zget_filter_info_f 75 | INTEGER :: nerr = 0 76 | 77 | DO i = 1, COMMAND_ARGUMENT_COUNT() 78 | CALL GET_COMMAND_ARGUMENT(i,arg,len,status) 79 | IF (status .NE. 0) THEN 80 | WRITE (ERROR_UNIT,*) 'get_command_argument failed: status = ', status, ' arg = ', i 81 | STOP 1 82 | END IF 83 | IF(arg(1:len).EQ.'zfpmode') THEN 84 | CALL GET_COMMAND_ARGUMENT(i+1,arg,len,status) 85 | IF (status .NE. 0) THEN 86 | WRITE (ERROR_UNIT,*) 'get_command_argument failed: status = ', status, ' arg = ', i 87 | STOP 1 88 | END IF 89 | READ(arg(1:len), *) zfpmode 90 | ELSE IF (arg(1:len).EQ.'rate')THEN 91 | CALL GET_COMMAND_ARGUMENT(i+1,arg,len,status) 92 | IF (status .NE. 0) THEN 93 | WRITE (ERROR_UNIT,*) 'get_command_argument failed: status = ', status, ' arg = ', i 94 | STOP 1 95 | END IF 96 | READ(arg(1:len), *) rate 97 | ELSE IF (arg(1:len).EQ.'acc')THEN 98 | CALL GET_COMMAND_ARGUMENT(i+1,arg,len,status) 99 | IF (status .NE. 0) THEN 100 | WRITE (ERROR_UNIT,*) 'get_command_argument failed: status = ', status, ' arg = ', i 101 | STOP 1 102 | END IF 103 | READ(arg(1:len), *) acc 104 | ELSE IF (arg(1:len).EQ.'dim')THEN 105 | CALL GET_COMMAND_ARGUMENT(i+1,arg,len,status) 106 | IF (status .NE. 0) THEN 107 | WRITE (ERROR_UNIT,*) 'get_command_argument failed: status = ', status, ' arg = ', i 108 | STOP 1 109 | END IF 110 | READ(arg(1:len), *) dim 111 | ELSE IF (arg(1:len).EQ.'prec')THEN 112 | CALL GET_COMMAND_ARGUMENT(i+1,arg,len,status) 113 | IF (status .NE. 0) THEN 114 | WRITE (ERROR_UNIT,*) 'get_command_argument failed: status = ', status, ' arg = ', i 115 | STOP 1 116 | END IF 117 | READ(arg(1:len), *) prec 118 | ELSE IF (arg(1:len).EQ.'ofile')THEN 119 | CALL GET_COMMAND_ARGUMENT(i+1,arg,len,status) 120 | IF (status .NE. 0) THEN 121 | WRITE (ERROR_UNIT,*) 'get_command_argument failed: status = ', status, ' arg = ', i 122 | STOP 1 123 | END IF 124 | READ(arg(1:len), *) ofile 125 | ELSE IF (arg(1:len).EQ.'write')THEN 126 | write_only = .TRUE. 127 | 128 | ELSE IF (INDEX(arg(1:len),'help').NE.0)THEN 129 | PRINT*," *** USAGE *** " 130 | PRINT*,"zfpmode - 1=rate,2=prec,3=acc,4=expert,5=reversible" 131 | PRINT*,"rate - set rate for rate mode of filter" 132 | PRINT*,"acc - set accuracy for accuracy mode of filter" 133 | PRINT*,"prec - set PRECISION for PRECISION mode of zfp filter" 134 | PRINT*,"dim - set size of 1D dataset used" 135 | PRINT*,"ofile - set the output file" 136 | PRINT*,"write - only write the file" 137 | STOP 1 138 | ENDIF 139 | 140 | END DO 141 | 142 | ! create data to write if we're not reading from an existing file 143 | 144 | IF (dim .EQ. 0) THEN 145 | CALL gen_data(INT(dim1*dim0, c_size_t), noise, amp, wdata) 146 | ELSE 147 | CALL gen_data(INT(dim, c_size_t), noise, amp, wdata) 148 | END IF 149 | 150 | CALL h5open_f(status) 151 | CALL check("h5open_f", status, nerr) 152 | 153 | ! initialize the ZFP filter 154 | status = H5Z_zfp_initialize() 155 | CALL check("H5Z_zfp_initialize", status, nerr) 156 | 157 | ! create HDF5 file 158 | CALL h5fcreate_f(ofile, H5F_ACC_TRUNC_F, fid, status) 159 | CALL check("h5fcreate_f", status, nerr) 160 | 161 | ! setup dataset compression via cd_values 162 | 163 | CALL h5pcreate_f(H5P_DATASET_CREATE_F, cpid, status) 164 | CALL check("h5pcreate_f", status, nerr) 165 | IF (dim .EQ. 0) THEN 166 | CALL h5pset_chunk_f(cpid, 2, chunk2, status) 167 | CALL check("h5pset_chunk_f", status, nerr) 168 | ELSE 169 | CALL h5pset_chunk_f(cpid, 1, chunk256, status) 170 | CALL check("h5pset_chunk_f", status, nerr) 171 | END IF 172 | 173 | ! 174 | ! Check that filter is registered with the library now. 175 | ! If it is registered, retrieve filter's configuration. 176 | ! 177 | CALL H5Zfilter_avail_f(H5Z_FILTER_ZFP, avail, status) 178 | CALL check("H5Zfilter_avail_f", status, nerr) 179 | 180 | IF (avail) THEN 181 | CALL h5zget_filter_info_f(H5Z_FILTER_ZFP, config_flag, status) 182 | CALL check("h5zget_filter_info_f", status, nerr) 183 | ! 184 | ! Make sure h5zget_filter_info_f returns the right flag 185 | ! 186 | config_flag_both=IOR(H5Z_FILTER_ENCODE_ENABLED_F,H5Z_FILTER_DECODE_ENABLED_F) 187 | IF (config_flag .NE. config_flag_both) THEN 188 | IF(config_flag .NE. H5Z_FILTER_DECODE_ENABLED_F) THEN 189 | PRINT*,'h5zget_filter_info_f config_flag failed' 190 | ENDIF 191 | ENDIF 192 | ENDIF 193 | 194 | ! setup the 2D data space 195 | IF (dim .EQ. 0) THEN 196 | CALL h5screate_simple_f(2, dims, sid, status) 197 | CALL check("h5screate_simple_f", status, nerr) 198 | ELSE 199 | dims1 = (/dim, 1/) 200 | CALL h5screate_simple_f(1, dims1, sid, status) 201 | CALL check("h5screate_simple_f", status, nerr) 202 | END IF 203 | 204 | ! write the data WITHOUT compression 205 | CALL h5dcreate_f(fid, "original", H5T_NATIVE_DOUBLE, sid, dsid, status) 206 | CALL check("h5dcreate_f", status, nerr) 207 | f_ptr = C_LOC(wdata(1,1)) 208 | CALL h5dwrite_f(dsid, H5T_NATIVE_DOUBLE, f_ptr, status) 209 | CALL check("h5dwrite_f", status, nerr) 210 | CALL h5dclose_f(dsid,status) 211 | CALL check("h5dclose_f", status, nerr) 212 | 213 | ! write data using default parameters 214 | cd_nelmts = 0 215 | CALL H5Pset_filter_f(cpid, H5Z_FILTER_ZFP, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values, status) 216 | CALL check("H5Pset_filter_f", status, nerr) 217 | 218 | CALL h5dcreate_f(fid, "compressed-default", H5T_NATIVE_DOUBLE, sid, dsid, status, dcpl_id=cpid) 219 | CALL check("h5dcreate_f", status, nerr) 220 | f_ptr = C_LOC(wdata(1,1)) 221 | CALL h5dwrite_f(dsid, H5T_NATIVE_DOUBLE, f_ptr, status) 222 | CALL check("h5dwrite_f", status, nerr) 223 | IF(status.NE.0) PRINT*,"h5dwrite_f failed" 224 | CALL h5dclose_f(dsid,status) 225 | CALL check("h5dclose_f", status, nerr) 226 | 227 | ! write the data using properties 228 | CALL H5Premove_filter_f(cpid, H5Z_FILTER_ZFP, status) 229 | IF (zfpmode .EQ. H5Z_ZFP_MODE_RATE) THEN 230 | status = H5Pset_zfp_rate(cpid, rate) 231 | CALL check("H5Pset_zfp_rate", status, nerr) 232 | ELSE IF (zfpmode .EQ. H5Z_ZFP_MODE_PRECISION) THEN 233 | status = H5Pset_zfp_precision(cpid, prec) 234 | CALL check("H5Pset_zfp_precision", status, nerr) 235 | ELSE IF (zfpmode .EQ. H5Z_ZFP_MODE_ACCURACY)THEN 236 | status = H5Pset_zfp_accuracy(cpid, acc) 237 | CALL check("H5Pset_zfp_accuracy", status, nerr) 238 | ELSE IF (zfpmode .EQ. H5Z_ZFP_MODE_EXPERT) THEN 239 | status = H5Pset_zfp_expert(cpid, minbits, maxbits, maxprec, minexp) 240 | CALL check("H5Pset_zfp_expert", status, nerr) 241 | ELSE IF (zfpmode .EQ. H5Z_ZFP_MODE_REVERSIBLE) THEN 242 | status = H5Pset_zfp_reversible(cpid) 243 | CALL check("H5Pset_zfp_reversible", status, nerr) 244 | ENDIF 245 | CALL check("H5Pset_filter_f", status, nerr) 246 | CALL h5dcreate_f(fid, "compressed", H5T_NATIVE_DOUBLE, sid, dsid, status, dcpl_id=cpid) 247 | CALL check("h5dcreate_f", status, nerr) 248 | f_ptr = C_LOC(wdata(1,1)) 249 | CALL h5dwrite_f(dsid, H5T_NATIVE_DOUBLE, f_ptr, status) 250 | CALL check("h5dwrite_f", status, nerr) 251 | CALL h5dclose_f(dsid,status) 252 | CALL check("h5dclose_f", status, nerr) 253 | 254 | ! write the data using plug-in 255 | CALL H5Premove_filter_f(cpid, H5Z_FILTER_ZFP, status) 256 | cd_values = 0 257 | cd_nelmts = H5Z_ZFP_CD_NELMTS_MEM 258 | IF (zfpmode .EQ. H5Z_ZFP_MODE_RATE) THEN 259 | CALL H5Pset_zfp_rate_cdata(rate, cd_nelmts, cd_values) 260 | IF(cd_values(1).NE.1 .OR. cd_nelmts.NE.4)THEN 261 | PRINT*,'H5Pset_zfp_rate_cdata failed' 262 | STOP 1 263 | ENDIF 264 | ELSE IF (zfpmode .EQ. H5Z_ZFP_MODE_PRECISION) THEN 265 | CALL H5Pset_zfp_precision_cdata(prec, cd_nelmts, cd_values) 266 | IF(cd_values(1).NE.2 .OR. cd_nelmts.NE.3)THEN 267 | PRINT*,'H5Pset_zfp_precision_cdata failed' 268 | STOP 1 269 | ENDIF 270 | ELSE IF (zfpmode .EQ. H5Z_ZFP_MODE_ACCURACY)THEN 271 | CALL H5Pset_zfp_accuracy_cdata(0._dp, cd_nelmts, cd_values) 272 | IF(cd_values(1).NE.3 .OR. cd_nelmts.NE.4)THEN 273 | PRINT*,'H5Pset_zfp_accuracy_cdata failed' 274 | STOP 1 275 | ENDIF 276 | ELSE IF (zfpmode .EQ. H5Z_ZFP_MODE_EXPERT) THEN 277 | CALL H5Pset_zfp_expert_cdata(minbits, maxbits, maxprec, minexp, cd_nelmts, cd_values) 278 | IF(cd_values(1).NE.4 .OR. cd_nelmts.NE.6)THEN 279 | PRINT*,'H5Pset_zfp_expert_cdata failed' 280 | STOP 1 281 | ENDIF 282 | ELSE IF (zfpmode .EQ. H5Z_ZFP_MODE_REVERSIBLE) THEN 283 | CALL H5Pset_zfp_reversible_cdata(cd_nelmts, cd_values) 284 | IF(cd_values(1).NE.5 .OR. cd_nelmts.NE.1)THEN 285 | PRINT*,'H5Pset_zfp_reversible_cdata failed' 286 | STOP 1 287 | ENDIF 288 | ENDIF 289 | 290 | CALL H5Pset_filter_f(cpid, H5Z_FILTER_ZFP, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values, status) 291 | CALL check("H5Pset_filter_f", status, nerr) 292 | 293 | CALL h5dcreate_f(fid, "compressed-plugin", H5T_NATIVE_DOUBLE, sid, dsid, status, dcpl_id=cpid) 294 | CALL check("h5dcreate_f", status, nerr) 295 | f_ptr = C_LOC(wdata(1,1)) 296 | CALL h5dwrite_f(dsid, H5T_NATIVE_DOUBLE, f_ptr, status) 297 | CALL check("h5dwrite_f", status, nerr) 298 | CALL h5dclose_f(dsid,status) 299 | CALL check("h5dclose_f", status, nerr) 300 | 301 | ! clean up 302 | CALL h5pclose_f(cpid, status) 303 | CALL check("", status, nerr) 304 | CALL h5sclose_f(sid, status) 305 | CALL check("", status, nerr) 306 | CALL h5fclose_f(fid, status) 307 | CALL check("", status, nerr) 308 | 309 | IF(write_only) STOP 310 | 311 | CALL h5fopen_f(ofile, H5F_ACC_RDONLY_F, fid, status) 312 | CALL check("h5fopen_f", status, nerr) 313 | 314 | ! read the original dataset 315 | CALL h5dopen_f (fid, "original", dsid, status) 316 | CALL check("h5dopen_f", status, nerr) 317 | 318 | CALL h5dget_space_f(dsid, space_id,status) 319 | CALL check("h5dget_space_f", status, nerr) 320 | CALL H5Sget_simple_extent_npoints_f(space_id, npoints, status) 321 | CALL check("H5Sget_simple_extent_npoints_f", status, nerr) 322 | CALL H5Sclose_f(space_id, status) 323 | CALL check("H5Sclose_f", status, nerr) 324 | ALLOCATE(obuf(1:npoints)) 325 | f_ptr = C_LOC(obuf(1)) 326 | CALL H5Dread_f(dsid, H5T_NATIVE_DOUBLE, f_ptr, status) 327 | CALL check("H5Dread_f", status, nerr) 328 | CALL H5Dclose_f(dsid, status) 329 | CALL check("H5Dclose_f", status, nerr) 330 | 331 | ! read the compressed dataset 332 | CALL h5dopen_f (fid, "compressed-default", dsid, status) 333 | CALL check("", status, nerr) 334 | CALL H5Dget_create_plist_f(dsid, dcpl_id, status ) 335 | CALL check("", status, nerr) 336 | ALLOCATE(cbuf(1:npoints)) 337 | f_ptr = C_LOC(cbuf(1)) 338 | CALL H5Dread_f(dsid, H5T_NATIVE_DOUBLE, f_ptr, status) 339 | CALL check("H5Dread_f", status, nerr) 340 | CALL H5Dclose_f(dsid, status) 341 | CALL check("H5Dclose_f", status, nerr) 342 | 343 | ! read the compressed dataset 344 | CALL h5dopen_f (fid, "compressed", dsid, status) 345 | CALL check("", status, nerr) 346 | CALL H5Dget_create_plist_f(dsid, dcpl_id, status ) 347 | CALL check("", status, nerr) 348 | ALLOCATE(cbuf1(1:npoints)) 349 | f_ptr = C_LOC(cbuf1(1)) 350 | CALL H5Dread_f(dsid, H5T_NATIVE_DOUBLE, f_ptr, status) 351 | CALL check("H5Dread_f", status, nerr) 352 | CALL H5Dclose_f(dsid, status) 353 | CALL check("H5Dclose_f", status, nerr) 354 | 355 | ! read the compressed dataset (plugin) 356 | CALL h5dopen_f (fid, "compressed-plugin", dsid, status) 357 | CALL check("", status, nerr) 358 | CALL H5Dget_create_plist_f(dsid, dcpl_id, status ) 359 | CALL check("", status, nerr) 360 | ALLOCATE(cbuf2(1:npoints)) 361 | f_ptr = C_LOC(cbuf2(1)) 362 | CALL H5Dread_f(dsid, H5T_NATIVE_DOUBLE, f_ptr, status) 363 | CALL check("H5Dread_f", status, nerr) 364 | CALL H5Dclose_f(dsid, status) 365 | CALL check("H5Dclose_f", status, nerr) 366 | 367 | ! clean up 368 | CALL H5Pclose_f(dcpl_id, status) 369 | CALL check("H5Pclose_f", status, nerr) 370 | CALL H5Fclose_f(fid, status) 371 | CALL check("H5Fclose_f", status, nerr) 372 | 373 | ! compare to generated data 374 | DO j = 1, npoints 375 | absdiff = obuf(j) - cbuf(j) 376 | if(absdiff < 0) absdiff = -absdiff 377 | IF(absdiff > max_absdiff) THEN 378 | reldiff = 0 379 | IF (obuf(j) .NE. 0) reldiff = absdiff / obuf(j) 380 | 381 | IF (absdiff > max_absdiff) max_absdiff = absdiff 382 | IF (reldiff > max_reldiff) max_reldiff = reldiff 383 | IF( .NOT.real_eq(obuf(j), cbuf(j), 100) ) THEN 384 | num_diffs = num_diffs + 1 385 | ENDIF 386 | ENDIF 387 | ENDDO 388 | 389 | IF(num_diffs.NE.0)THEN 390 | WRITE(ERROR_UNIT,'(A)') "Fortran read/write test Failed" 391 | WRITE(ERROR_UNIT,'(I0," values are different; max-absdiff = ",E15.8,", max-reldiff = ",E15.8)') & 392 | num_diffs,max_absdiff, max_reldiff 393 | STOP 1 394 | ELSE IF(nerr.NE.0)THEN 395 | WRITE(ERROR_UNIT,'(A)') "Fortran read/write test Failed" 396 | STOP 1 397 | ELSE 398 | WRITE(OUTPUT_UNIT,'(A)') "Fortran read/write test Passed" 399 | ENDIF 400 | 401 | DEALLOCATE(obuf, cbuf, cbuf1, cbuf2) 402 | 403 | ! initialize the ZFP filter 404 | status = H5Z_zfp_finalize() 405 | CALL check("H5Z_zfp_finalize", status, nerr) 406 | 407 | CALL H5close_f(status) 408 | 409 | CALL EXIT(0) 410 | 411 | END PROGRAM main 412 | 413 | ! Generate a simple, 1D sinusioidal data array with some noise 414 | SUBROUTINE gen_data(npoints, noise, amp, buf) 415 | USE ISO_C_BINDING 416 | IMPLICIT NONE 417 | INTEGER(C_SIZE_T) :: npoints 418 | REAL(C_DOUBLE) :: noise 419 | REAL(C_DOUBLE) :: amp 420 | REAL(C_DOUBLE), DIMENSION(1:npoints) :: buf 421 | 422 | REAL(C_DOUBLE), PARAMETER :: PI = 3.1415926535897932384626433832795028841971_C_DOUBLE 423 | 424 | INTEGER :: size 425 | INTEGER, DIMENSION(:), ALLOCATABLE :: seed 426 | INTEGER(C_SIZE_T) :: i 427 | REAL(C_DOUBLE) :: x 428 | REAL(C_DOUBLE) :: rand 429 | 430 | ! Fixed random seed. 431 | CALL RANDOM_SEED(SIZE=size) 432 | ALLOCATE(seed(size)) 433 | seed = 123456789 434 | CALL RANDOM_SEED(PUT=seed) 435 | 436 | DO i = 1, npoints 437 | rand = REAL(i, C_DOUBLE) 438 | CALL RANDOM_NUMBER(rand) 439 | x = 2_c_double * PI * REAL(i-1, C_DOUBLE) / REAL(npoints-1, C_DOUBLE) 440 | buf(i) = amp*( 1.0_C_DOUBLE + SIN(x)) + (rand - 0.5_C_DOUBLE)*noise 441 | ENDDO 442 | 443 | IF (ALLOCATED(seed)) DEALLOCATE(seed) 444 | END SUBROUTINE gen_data 445 | 446 | LOGICAL FUNCTION real_eq(a,b,ulp) 447 | USE ISO_C_BINDING 448 | IMPLICIT NONE 449 | REAL(C_DOUBLE), INTENT (in):: a,b 450 | REAL(C_DOUBLE) :: Rel = 1.0_C_DOUBLE 451 | INTEGER, OPTIONAL, INTENT( IN ) :: ulp 452 | IF ( PRESENT( ulp ) ) Rel = REAL( ABS(ulp), C_DOUBLE) 453 | real_eq = ABS( a - b ) < ( Rel * SPACING( MAX(ABS(a),ABS(b)) ) ) 454 | END FUNCTION real_eq 455 | 456 | SUBROUTINE check(string,error,total_error) 457 | USE ISO_FORTRAN_ENV, ONLY: ERROR_UNIT 458 | CHARACTER(LEN=*) :: string 459 | INTEGER :: error, total_error 460 | IF (error .LT. 0) THEN 461 | total_error=total_error+1 462 | WRITE(ERROR_UNIT,*) string, " FAILED" 463 | ENDIF 464 | RETURN 465 | END SUBROUTINE check 466 | --------------------------------------------------------------------------------