├── .clang_format ├── .github └── workflows │ └── prgate.yaml ├── .gitignore ├── CHANGELOG.md ├── CMakeLists.txt ├── CMakePresets.json ├── CSharpWrapper ├── AssemblyInfo.cs.in ├── CMakeLists.txt ├── NumpyIONative.i └── Tensor.cs ├── LICENSE ├── README.md ├── RELEASE_NOTES ├── VERSION ├── assets └── test │ ├── float32.npy │ ├── float64.npy │ ├── int16.npy │ ├── int32.npy │ ├── int32_array.npy │ ├── int32_big.npy │ ├── int32_scalar.npy │ ├── int64.npy │ ├── int8.npy │ ├── test.npz │ ├── test_compressed.npz │ ├── uint16.npy │ ├── uint32.npy │ ├── uint64.npy │ ├── uint8.npy │ ├── uint8_fortran.npy │ └── unicode.npy ├── cmake └── npyConfig.cmake.in ├── doc ├── CMakeLists.txt └── Doxyfile.in ├── include └── npy │ ├── core.h │ ├── npy.h │ ├── npz.h │ └── tensor.h ├── nuget ├── template.nuspec.in └── template.targets.in ├── samples ├── CMakeLists.txt ├── README.md ├── display.png ├── display.py ├── images.cpp ├── images_net.cs └── requirements.txt ├── src ├── CMakeLists.txt ├── dtype.cpp ├── miniz │ ├── miniz.cpp │ └── miniz.h ├── npy.cpp ├── npz.cpp ├── tensor.cpp ├── zip.cpp └── zip.h └── test ├── CMakeLists.txt ├── CSharpTests ├── CMakeLists.txt ├── test.cs ├── test_exceptions.cs ├── test_npy_peek.cs ├── test_npy_read.cs ├── test_npy_write.cs ├── test_npz_peek.cs ├── test_npz_read.cs └── test_npz_write.cs ├── crc32.cpp ├── exceptions.cpp ├── generate_large_test.py ├── libnpy_tests.cpp ├── libnpy_tests.h ├── memstream.cpp ├── npy_peek.cpp ├── npy_read.cpp ├── npy_read.h ├── npy_write.cpp ├── npz_peek.cpp ├── npz_read.cpp ├── npz_write.cpp └── tensor.cpp /.clang_format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | # BasedOnStyle: LLVM 4 | AccessModifierOffset: -2 5 | AlignAfterOpenBracket: AlwaysBreak 6 | AlignConsecutiveAssignments: false 7 | AlignConsecutiveDeclarations: false 8 | AlignEscapedNewlines: DontAlign 9 | AlignOperands: false 10 | AlignTrailingComments: false 11 | AllowAllParametersOfDeclarationOnNextLine: true 12 | AllowShortBlocksOnASingleLine: false 13 | AllowShortCaseLabelsOnASingleLine: false 14 | AllowShortFunctionsOnASingleLine: Empty 15 | AllowShortIfStatementsOnASingleLine: false 16 | AllowShortLoopsOnASingleLine: false 17 | AlwaysBreakAfterDefinitionReturnType: None 18 | AlwaysBreakAfterReturnType: None 19 | AlwaysBreakBeforeMultilineStrings: true 20 | AlwaysBreakTemplateDeclarations: true 21 | BinPackArguments: false 22 | BinPackParameters: false 23 | BraceWrapping: 24 | AfterClass: true 25 | AfterControlStatement: true 26 | AfterEnum: true 27 | AfterFunction: true 28 | AfterNamespace: true 29 | AfterObjCDeclaration: true 30 | AfterStruct: true 31 | AfterUnion: true 32 | AfterExternBlock: true 33 | BeforeCatch: true 34 | BeforeElse: true 35 | IndentBraces: false 36 | SplitEmptyFunction: false 37 | SplitEmptyRecord: false 38 | SplitEmptyNamespace: false 39 | BreakBeforeBinaryOperators: None 40 | BreakBeforeBraces: Custom 41 | BreakBeforeInheritanceComma: false 42 | BreakBeforeTernaryOperators: false 43 | BreakConstructorInitializersBeforeComma: false 44 | BreakConstructorInitializers: AfterColon 45 | BreakAfterJavaFieldAnnotations: false 46 | BreakStringLiterals: true 47 | ColumnLimit: 80 48 | CommentPragmas: "^ IWYU pragma:" 49 | CompactNamespaces: false 50 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 51 | ConstructorInitializerIndentWidth: 2 52 | ContinuationIndentWidth: 2 53 | Cpp11BracedListStyle: true 54 | DerivePointerAlignment: false 55 | DisableFormat: false 56 | ExperimentalAutoDetectBinPacking: false 57 | FixNamespaceComments: false 58 | ForEachMacros: 59 | - FOREACH 60 | IncludeBlocks: Regroup 61 | IncludeCategories: 62 | - Regex: '^"(llvm|llvm-c|clang|clang-c)/' 63 | Priority: 2 64 | - Regex: '^(<|"(gtest|gmock|isl|json)/)' 65 | Priority: 3 66 | - Regex: ".*" 67 | Priority: 1 68 | IncludeIsMainRegex: "(Test)?$" 69 | IndentCaseLabels: true 70 | IndentPPDirectives: None 71 | IndentWidth: 2 72 | IndentWrappedFunctionNames: false 73 | JavaScriptQuotes: Leave 74 | JavaScriptWrapImports: true 75 | KeepEmptyLinesAtTheStartOfBlocks: false 76 | MacroBlockBegin: "" 77 | MacroBlockEnd: "" 78 | MaxEmptyLinesToKeep: 1 79 | NamespaceIndentation: All 80 | ObjCBlockIndentWidth: 2 81 | ObjCSpaceAfterProperty: false 82 | ObjCSpaceBeforeProtocolList: true 83 | PenaltyBreakAssignment: 2 84 | PenaltyBreakBeforeFirstCallParameter: 19 85 | PenaltyBreakComment: 300 86 | PenaltyBreakFirstLessLess: 120 87 | PenaltyBreakString: 1000 88 | PenaltyExcessCharacter: 1000000 89 | PenaltyReturnTypeOnItsOwnLine: 600 90 | PointerAlignment: Left 91 | ReflowComments: true 92 | SortIncludes: true 93 | SortUsingDeclarations: true 94 | SpaceAfterCStyleCast: false 95 | SpaceAfterTemplateKeyword: true 96 | SpaceBeforeAssignmentOperators: true 97 | SpaceBeforeParens: ControlStatements 98 | SpaceInEmptyParentheses: false 99 | SpacesBeforeTrailingComments: 1 100 | SpacesInAngles: false 101 | SpacesInContainerLiterals: false 102 | SpacesInCStyleCastParentheses: false 103 | SpacesInParentheses: false 104 | SpacesInSquareBrackets: false 105 | Standard: Cpp11 106 | TabWidth: 2 107 | UseTab: Never 108 | --- -------------------------------------------------------------------------------- /.github/workflows/prgate.yaml: -------------------------------------------------------------------------------- 1 | name: PR Gate 2 | 3 | on: 4 | pull_request: 5 | branches: ["main"] 6 | workflow_dispatch: 7 | 8 | jobs: 9 | cpp-format: 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | path: 14 | - check: src 15 | exclude: (/miniz/) 16 | - check: test 17 | exclude: '' 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v4 21 | 22 | - name: Run clang-format style check 23 | uses: jidicula/clang-format-action@v4.8.0 24 | with: 25 | clang-format-version: '18' 26 | check-path: ${{matrix.path['check']}} 27 | exclude-regex: ${{matrix.path['exclude']}} 28 | 29 | linux: 30 | runs-on: ubuntu-latest 31 | 32 | steps: 33 | - name: Checkout 34 | uses: actions/checkout@v4 35 | 36 | - name: Get dependencies 37 | run: | 38 | sudo apt-get install ninja-build 39 | 40 | - name: CMake config 41 | run: cmake -B ${{github.workspace}}/build --preset release-clang 42 | 43 | - name: CMake build 44 | working-directory: ${{github.workspace}}/build 45 | run: ninja 46 | 47 | - name: Use Python 3.11 48 | uses: actions/setup-python@v4 49 | with: 50 | python-version: "3.11" 51 | 52 | - name: Generate large NPZ files 53 | working-directory: ${{github.workspace}}/ 54 | run: | 55 | pip install numpy 56 | python ${{github.workspace}}/test/generate_large_test.py 57 | 58 | - name: CMake test 59 | working-directory: ${{github.workspace}}/build 60 | run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test 61 | 62 | linux-asan: 63 | runs-on: ubuntu-latest 64 | 65 | steps: 66 | - name: Checkout 67 | uses: actions/checkout@v4 68 | 69 | - name: Get dependencies 70 | run: | 71 | sudo apt-get install ninja-build 72 | 73 | - name: CMake config 74 | run: cmake -B ${{github.workspace}}/build --preset release-clang -DLIBNPY_SANITIZE=address 75 | 76 | - name: CMake build 77 | working-directory: ${{github.workspace}}/build 78 | run: ninja 79 | 80 | - name: Use Python 3.11 81 | uses: actions/setup-python@v4 82 | with: 83 | python-version: "3.11" 84 | 85 | - name: Generate large NPZ files 86 | working-directory: ${{github.workspace}}/ 87 | run: | 88 | pip install numpy 89 | python ${{github.workspace}}/test/generate_large_test.py 90 | 91 | - name: CMake test 92 | working-directory: ${{github.workspace}}/build 93 | run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test 94 | 95 | windows: 96 | runs-on: windows-latest 97 | 98 | steps: 99 | - name: Checkout 100 | uses: actions/checkout@v4 101 | 102 | - name: CMake config 103 | run: | 104 | cmake -B ${{github.workspace}}/build --preset release 105 | 106 | - name: CMake build 107 | working-directory: ${{github.workspace}}/build 108 | run: cmake --build . --config Release 109 | 110 | - name: Use Python 3.11 111 | uses: actions/setup-python@v4 112 | with: 113 | python-version: "3.11" 114 | 115 | - name: Generate large NPZ files 116 | working-directory: ${{github.workspace}}/ 117 | run: | 118 | pip install numpy 119 | python ${{github.workspace}}/test/generate_large_test.py 120 | 121 | - name: CMake test 122 | working-directory: ${{github.workspace}}/build 123 | run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test 124 | 125 | macos: 126 | runs-on: macos-latest 127 | 128 | steps: 129 | - name: Checkout 130 | uses: actions/checkout@v4 131 | 132 | - name: Get dependencies 133 | run: | 134 | brew update && brew install ninja 135 | 136 | - name: CMake config 137 | run: cmake -B ${{github.workspace}}/build --preset release-clang 138 | 139 | - name: CMake build 140 | working-directory: ${{github.workspace}}/build 141 | run: ninja 142 | 143 | - name: Use Python 3.11 144 | uses: actions/setup-python@v4 145 | with: 146 | python-version: "3.11" 147 | 148 | - name: Generate large NPZ files 149 | working-directory: ${{github.workspace}}/ 150 | run: | 151 | pip install numpy 152 | python ${{github.workspace}}/test/generate_large_test.py 153 | 154 | - name: CMake test 155 | working-directory: ${{github.workspace}}/build 156 | run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test 157 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | 34 | # Misc 35 | .vscode 36 | build* 37 | assets/test/*_large*.npz 38 | .cache 39 | .env -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [2024-11-01 - Version 1.5.3](https://github.com/matajoh/libnpy/releases/tag/v1.5.3) 4 | 5 | Improvements: 6 | - Increased CHUNK size as per miniz instructions 7 | - Added tests for very large arrays in NPZ files 8 | - Added some CI tests to catch issues across platforms 9 | - Removed the internal IO streams in favor of just using stringstream 10 | - NPZs can now be read from and written to memory 11 | 12 | Bugfixes: 13 | - Fixed an issue where very large arrays in NPZ files would throw an error 14 | - Fixed a bug with mac builds due to deprecated APIs 15 | 16 | ## [2021-10-05 - Version 1.5.2](https://github.com/matajoh/libnpy/releases/tag/v1.5.2) 17 | 18 | Removing `using namespace std` to simplify library use 19 | 20 | ## [2021-08-26 - Version 1.5.1](https://github.com/matajoh/libnpy/releases/tag/v1.5.1) 21 | 22 | Improvements: 23 | - CMake build now uses the highest compiler warning/error setting 24 | 25 | Bugfixes: 26 | - Fixed some bugs exposed by heightened compiler warnings 27 | 28 | ## [2021-06-09 - Version 1.5.0](https://github.com/matajoh/libnpy/releases/tag/v1.5.0) 29 | 30 | Improvements: 31 | - Added a `keys` member to `inpzstream` so it is possible to query the keys of the tensors 32 | 33 | ## [2021-05-28 - Version 1.4.1](https://github.com/matajoh/libnpy/releases/tag/v1.4.1) 34 | 35 | Bug fixes: 36 | - Fixed a bug with integer shifting 37 | 38 | ## [2021-05-28 - Version 1.4.0](https://github.com/matajoh/libnpy/releases/tag/v1.4.0) 39 | 40 | Improvements: 41 | - Further minor CMake changes to improve ease of use 42 | - NPZ streams now have `is_open` methods to check for successful file opening 43 | - Minor code style changes 44 | 45 | Bug fixes: 46 | - NPZ files will now correctly handle PKZIP versions after 2.0, both for reading and writing 47 | 48 | ## [2021-05-21 - Version 1.3.1](https://github.com/matajoh/libnpy/releases/tag/v1.3.1) 49 | 50 | Improvements: 51 | - Updated CMake integration to make the library easier to use via `FetchContent` 52 | 53 | ## [2021-02-10 - Version 1.3.0](https://github.com/matajoh/libnpy/releases/tag/v1.3.0) 54 | 55 | New Features: 56 | - Support for Unicode string tensors (npy type 'U') 57 | 58 | Breaking change: 59 | - `CopyFrom` interface for C# Tensors has been changed to use *Buffer objects 60 | 61 | ## [2021-02-09 - Version 1.2.2](https://github.com/matajoh/libnpy/releases/tag/v1.2.2) 62 | 63 | Improvements: 64 | - Bug fix for a missing comma on 1d shape 65 | 66 | ## [2021-02-08 - Version 1.2.1](https://github.com/matajoh/libnpy/releases/tag/v1.2.1) 67 | 68 | Improvements: 69 | - Bug fix for scalar tensor reading 70 | - Bug fix with memstream buffer size at initialization 71 | - ".npy" will be added to tensor names in NPZ writing if not already present 72 | 73 | ## [2021-01-19 - Version 1.2.0](https://github.com/matajoh/libnpy/releases/tag/v1.2.0) 74 | 75 | New Features: 76 | - Easier indexing (variable argument index method + negative indexes) 77 | - Easier access to shape 78 | 79 | Improvements: 80 | - Cmake upgraded to "modern" usage, i.e. you use the library by adding `npy::npy` as a link library 81 | 82 | ## [2021-01-16 - Version 1.1.1](https://github.com/matajoh/libnpy/releases/tag/v1.1.1) 83 | 84 | Improvements: 85 | - Minor cmake change 86 | 87 | ## [2021-01-16 - Version 1.1.0](https://github.com/matajoh/libnpy/releases/tag/v1.1.0) 88 | 89 | New Features: 90 | - Zip64 compatibility 91 | 92 | Improvements: 93 | - Can use `numpy` style lookup for tensors (i.e. dropping the `.npy` from the name) 94 | - Added a crc32 test 95 | 96 | ## [2021-01-15 - Version 1.0.0](https://github.com/matajoh/libnpy/releases/tag/v1.0.0) 97 | 98 | New Features: 99 | - There is no longer a dependency on `zlib` 100 | 101 | Improvements: 102 | - Better packaging (NuGet packages are now produced for C++ and C#) 103 | 104 | ## [2019-04-01 - Version 0.2.0](https://github.com/matajoh/libnpy/releases/tag/v0.2.0) 105 | 106 | Breaking changes: 107 | - Renamed `endian` => `endian_t` 108 | - Renamed `data_type` => `data_type_t` 109 | - Renamed `compression_method` => `compression_method_t` 110 | 111 | New Features: 112 | - Cleaned up exception handling. There are now tests for exceptions being correctly thrown, and the exceptions are properly wrapped for .NET- 113 | - Added peeking for NPY files to get the header information, and contains/peek functionality for `inpzstream`. 114 | 115 | Improvements: 116 | - Removed the unnecessary copies in the compression/decompression process 117 | 118 | ## [2019-03-24 - Version 0.1.0](https://github.com/matajoh/libnpy/releases/tag/v0.1.0) 119 | 120 | Initial Release -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required( VERSION 3.13...3.16 FATAL_ERROR ) 2 | 3 | # -------------------- Version -------------------------------- 4 | 5 | file( STRINGS "VERSION" LIBNPY_VERSION_FILE ) 6 | 7 | string( REPLACE "." ";" LIBNPY_VERSION_LIST ${LIBNPY_VERSION_FILE} ) 8 | 9 | list( GET LIBNPY_VERSION_LIST 0 LIBNPY_VERSION_MAJOR ) 10 | 11 | list( GET LIBNPY_VERSION_LIST 1 LIBNPY_VERSION_MINOR ) 12 | 13 | list( GET LIBNPY_VERSION_LIST 2 LIBNPY_VERSION_REVISION ) 14 | 15 | set( LIBNPY_VERSION ${LIBNPY_VERSION_MAJOR}.${LIBNPY_VERSION_MINOR}.${LIBNPY_VERSION_REVISION} ) 16 | 17 | message("Configure LIBNPY_VERSION at ${LIBNPY_VERSION}") 18 | 19 | project( libnpy VERSION ${LIBNPY_VERSION} LANGUAGES CXX) 20 | 21 | # -------------------- Options -------------------------------- 22 | 23 | option( LIBNPY_BUILD_TESTS "Specifies whether to build the tests" OFF ) 24 | option( LIBNPY_BUILD_SAMPLES "Specifies whether to build the samples" OFF ) 25 | option( LIBNPY_BUILD_DOCUMENTATION "Specifies whether to build the documentation for the API and XML" OFF ) 26 | option( LIBNPY_INCLUDE_CSHARP "Specifies whether to build libnpy with C# bindings" OFF ) 27 | set( LIBNPY_SANITIZE "" CACHE STRING "Argument to pass to sanitize (disabled by default)") 28 | 29 | set(CMAKE_CXX_STANDARD 17) 30 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 31 | 32 | # -------------------- Find packages -------------------------- 33 | 34 | if ( WIN32 AND INCLUDE_CSHARP ) 35 | enable_language( CSharp ) 36 | else() 37 | set( INCLUDE_CSHARP OFF ) 38 | endif() 39 | 40 | set( CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake ) 41 | 42 | if( INCLUDE_CSHARP ) 43 | find_package( SWIG REQUIRED ) 44 | 45 | # Select the .NET architecture 46 | set( CSHARP_PLATFORM_DESC "C# target platform: x86, x64, anycpu, or itanium" ) 47 | if( CMAKE_SIZEOF_VOID_P EQUAL 8 ) 48 | set( CSHARP_PLATFORM "x64" CACHE STRING ${CSHARP_PLATFORM_DESC}) 49 | else() 50 | set( CSHARP_PLATFORM "x86" CACHE STRING ${CSHARP_PLATFORM_DESC}) 51 | endif() 52 | endif() 53 | 54 | if( BUILD_DOCUMENTATION ) 55 | find_package( Doxygen REQUIRED ) 56 | endif() 57 | 58 | find_program(CLANG_FORMAT NAMES clang-format-10 clang-format-14 clang-format-18 ) 59 | 60 | string(COMPARE EQUAL ${CLANG_FORMAT} "CLANG_FORMAT-NOTFOUND" CLANG_FORMAT_NOT_FOUND) 61 | if(CLANG_FORMAT_NOT_FOUND) 62 | message("libnpy_format target not defined: no clang-format tool found") 63 | else() 64 | file(GLOB ALL_SOURCE_FILES CONFIGURE_DEPENDS 65 | src/*.cpp 66 | src/*.h 67 | include/libnpy/*.h 68 | test/*.cpp 69 | test/*.h 70 | examples/*.cpp 71 | ) 72 | 73 | add_custom_target(libnpy_format 74 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} 75 | COMMAND ${CLANG_FORMAT} 76 | -i 77 | ${ALL_SOURCE_FILES}) 78 | endif() 79 | 80 | # -------------------- Walk the subdirectories -------------------- 81 | 82 | add_subdirectory( src ) 83 | 84 | if( LIBNPY_BUILD_SAMPLES ) 85 | add_subdirectory( samples ) 86 | endif() 87 | 88 | if( LIBNPY_BUILD_DOCUMENTATION ) 89 | add_subdirectory( doc ) 90 | endif() 91 | 92 | if( LIBNPY_INCLUDE_CSHARP ) 93 | add_subdirectory( CSharpWrapper ) 94 | endif() 95 | 96 | target_include_directories(npy 97 | PUBLIC 98 | $ 99 | $ 100 | PRIVATE 101 | ${CMAKE_CURRENT_SOURCE_DIR}/src 102 | ) 103 | 104 | # -------------------- Testing ------------------------------------ 105 | 106 | if( LIBNPY_BUILD_TESTS ) 107 | if( MSVC ) 108 | set( LIBNPY_CSHARP_DIR ${CMAKE_BINARY_DIR}/CSharpWrapper/$ ) 109 | endif() 110 | 111 | include( CTest ) 112 | add_subdirectory( test ) 113 | endif() 114 | 115 | 116 | # -------------------- Build settings ----------------------------- 117 | 118 | # use C++11 119 | target_compile_features(npy PRIVATE cxx_std_11) 120 | 121 | # -------------------- INSTALL ------------------------------------ 122 | 123 | set(INSTALL_CONFIGDIR "cmake") 124 | 125 | install(TARGETS npy 126 | EXPORT npy-targets 127 | ARCHIVE DESTINATION "build/native/lib" 128 | LIBRARY DESTINATION "build/native/lib" 129 | ) 130 | 131 | install(DIRECTORY include/ DESTINATION "build/native/include") 132 | 133 | install(EXPORT npy-targets 134 | FILE 135 | npyTargets.cmake 136 | NAMESPACE 137 | npy:: 138 | DESTINATION 139 | ${INSTALL_CONFIGDIR} 140 | ) 141 | 142 | include(CMakePackageConfigHelpers) 143 | write_basic_package_version_file( 144 | ${CMAKE_CURRENT_BINARY_DIR}/npyConfigVersion.cmake 145 | VERSION ${PROJECT_VERSION} 146 | COMPATIBILITY AnyNewerVersion 147 | ) 148 | 149 | configure_package_config_file(${CMAKE_CURRENT_LIST_DIR}/cmake/npyConfig.cmake.in 150 | ${CMAKE_CURRENT_BINARY_DIR}/npyConfig.cmake 151 | INSTALL_DESTINATION ${INSTALL_CONFIGDIR} 152 | ) 153 | 154 | install(FILES 155 | ${CMAKE_CURRENT_BINARY_DIR}/npyConfig.cmake 156 | ${CMAKE_CURRENT_BINARY_DIR}/npyConfigVersion.cmake 157 | DESTINATION ${INSTALL_CONFIGDIR} 158 | ) 159 | 160 | export(EXPORT npy-targets 161 | FILE ${CMAKE_CURRENT_BINARY_DIR}/npyTargets.cmake 162 | NAMESPACE npy:: 163 | ) 164 | 165 | export(PACKAGE npy) 166 | 167 | # -------------------- Package ------------------------------------ 168 | 169 | set( PROJECT_FILES 170 | README.md 171 | CHANGELOG.md 172 | ) 173 | 174 | # copy these files into the root of the distribution zip 175 | install( FILES ${PROJECT_FILES} DESTINATION "." ) 176 | 177 | if( MSVC ) 178 | # NuGet files 179 | set( LIBNPY_NUGET_NAME "npy-${SYSTEM_TOOLKIT}-${SYSTEM_BITS}-${CMAKE_BUILD_TYPE}" CACHE STRING "npy NuGet Name" FORCE ) 180 | file( READ RELEASE_NOTES LIBNPY_RELEASE_NOTES ) 181 | 182 | configure_file("${CMAKE_CURRENT_SOURCE_DIR}/nuget/template.nuspec.in" "${CMAKE_CURRENT_BINARY_DIR}/nuget/${LIBNPY_NUGET_NAME}.nuspec" @ONLY ) 183 | configure_file("${CMAKE_CURRENT_SOURCE_DIR}/nuget/template.targets.in" 184 | "${CMAKE_CURRENT_BINARY_DIR}/nuget/build/native/${LIBNPY_NUGET_NAME}.targets" @ONLY ) 185 | else() 186 | set( CPACK_SYSTEM_NAME ${SYSTEM_NAME} ) 187 | set( CPACK_PACKAGE_VERSION "${LIBNPY_VERSION}" ) 188 | set( CPACK_GENERATOR "ZIP" ) 189 | set( CPACK_SOURCE_GENERATOR "ZIP" ) 190 | set( CPACK_INCLUDE_TOPLEVEL_DIRECTORY 0 ) 191 | include( CPack ) 192 | endif() 193 | -------------------------------------------------------------------------------- /CMakePresets.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 3, 3 | "configurePresets": [ 4 | { 5 | "name": "debug-clang", 6 | "displayName": "Debug Build using clang", 7 | "description": "Sets up a debug build that uses Clang++", 8 | "generator": "Ninja", 9 | "cacheVariables": { 10 | "CMAKE_BUILD_TYPE": "Debug", 11 | "CMAKE_INSTALL_PREFIX": "${sourceDir}/build/dist", 12 | "CMAKE_CXX_COMPILER": "clang++", 13 | "LIBNPY_BUILD_TESTS": "ON" 14 | } 15 | }, 16 | { 17 | "name": "debug", 18 | "displayName": "Debug Build", 19 | "description": "Sets up a debug build that uses the default compiler and generator", 20 | "cacheVariables": { 21 | "CMAKE_BUILD_TYPE": "Debug", 22 | "CMAKE_INSTALL_PREFIX": "${sourceDir}/build/dist", 23 | "LIBNPY_BUILD_TESTS": "ON" 24 | } 25 | }, 26 | { 27 | "name": "release-clang", 28 | "displayName": "Release Build using clang", 29 | "description": "Sets up a release build that uses Clang++", 30 | "generator": "Ninja", 31 | "cacheVariables": { 32 | "CMAKE_BUILD_TYPE": "Release", 33 | "CMAKE_INSTALL_PREFIX": "${sourceDir}/build/dist", 34 | "CMAKE_CXX_COMPILER": "clang++", 35 | "LIBNPY_BUILD_TESTS": "ON" 36 | } 37 | }, 38 | { 39 | "name": "release", 40 | "displayName": "Release Build", 41 | "description": "Sets up a release build that uses the default compiler and generator", 42 | "cacheVariables": { 43 | "CMAKE_BUILD_TYPE": "Release", 44 | "CMAKE_INSTALL_PREFIX": "${sourceDir}/build/dist", 45 | "LIBNPY_BUILD_TESTS": "ON" 46 | } 47 | } 48 | ] 49 | } -------------------------------------------------------------------------------- /CSharpWrapper/AssemblyInfo.cs.in: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | 4 | [assembly: AssemblyVersion("@LIBNPY_VERSION@")] 5 | [assembly: AssemblyTitle("@PROJECT_NAME@ C# @CSHARP_PLATFORM@ wrapper @LIBNPY_VERSION@")] 6 | [assembly: AssemblyCopyright("Copyright Matthew Johnson (c) 2021")] 7 | [assembly: AssemblyDescription("NumpyIO C# @CSHARP_PLATFORM@ wrapper @LIBNPY_VERSION_CSHARP_AssemblyVersion@, compiled with @CSHARP_TYPE@ @CSHARP_VERSION@")] 8 | [assembly: AssemblyCompany("")] 9 | [assembly: AssemblyFileVersion("@LIBNPY_VERSION@")] 10 | [assembly: AssemblyCulture("")] 11 | [assembly: AssemblyTrademark("NumpyIO")] 12 | [assembly: AssemblyProduct("NumpyIO")] 13 | [assembly: AssemblyInformationalVersion("@LIBNPY_VERSION_CSHARP_AssemblyVersion@")] 14 | [assembly: AssemblyConfiguration("")] 15 | [assembly: AssemblyDelaySign(false)] 16 | [assembly: AssemblyKeyName("")] 17 | [assembly: AssemblyKeyFile("")] 18 | -------------------------------------------------------------------------------- /CSharpWrapper/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include( ${SWIG_USE_FILE} ) 2 | 3 | execute_process( 4 | COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD 5 | WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} 6 | OUTPUT_VARIABLE GIT_BRANCH 7 | OUTPUT_STRIP_TRAILING_WHITESPACE 8 | ) 9 | 10 | # Get the latest abbreviated commit hash of the working branch 11 | execute_process( 12 | COMMAND ${GIT_EXECUTABLE} log -1 --format=%h 13 | WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} 14 | OUTPUT_VARIABLE GIT_COMMIT_HASH 15 | OUTPUT_STRIP_TRAILING_WHITESPACE 16 | ) 17 | 18 | set_source_files_properties ( NumpyIONative.i PROPERTIES CPLUSPLUS ON ) 19 | 20 | # CSharp version requirements: http://msdn.microsoft.com/en-us/library/system.reflection.assemblyversionattribute.aspx 21 | # major.minor[.build[.revision]] where all components are 16-bit unsigned integers 22 | 23 | set(NUMPYIO_VERSION_CSHARP_AssemblyVersion "${LIBNPY_VERSION_MAJOR}.${LIBNPY_VERSION_MINOR}.${GIT_BRANCH}.${GIT_COMMIT_HASH}") 24 | 25 | # Make sure the nested directory structure exists 26 | set(CSHARP_SOURCE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/swig CACHE INTERNAL "") 27 | file(MAKE_DIRECTORY ${CSHARP_SOURCE_DIRECTORY}) 28 | 29 | # Create swig target 30 | set(CMAKE_SWIG_OUTDIR ${CSHARP_SOURCE_DIRECTORY}) 31 | 32 | set(CMAKE_SWIG_FLAGS -I${CMAKE_CURRENT_SOURCE_DIR} -namespace \"NumpyIO\" ${CMAKE_SWIG_GLOBAL_FLAGS} ${CMAKE_SWIG_FLAGS}) 33 | 34 | SET_SOURCE_FILES_PROPERTIES(NumpyIONative.i PROPERTIES SWIG_FLAGS "-includeall") 35 | 36 | SWIG_ADD_LIBRARY( NumpyIONative 37 | LANGUAGE csharp 38 | TYPE SHARED 39 | SOURCES NumpyIONative.i 40 | ) 41 | 42 | INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) 43 | 44 | SWIG_LINK_LIBRARIES( ${SWIG_MODULE_NumpyIONative_REAL_NAME} npy ) 45 | 46 | if( UNIX ) 47 | set_target_properties(${SWIG_MODULE_NumpyIONative_REAL_NAME} PROPERTIES PREFIX "lib" SUFFIX ".so") 48 | set( NUMPYIO_NATIVE libNumpyIONative.so CACHE INTERNAL "The NumpyIO built library" ) 49 | else() 50 | set_target_properties(${SWIG_MODULE_NumpyIONative_REAL_NAME} PROPERTIES SUFFIX ".dll") 51 | set( NUMPYIO_NATIVE NumpyIONative.dll CACHE INTERNAL "The NumpyIONative built library" ) 52 | endif() 53 | 54 | # Configure AssemblyInfo.cs 55 | configure_file( 56 | ${CMAKE_CURRENT_SOURCE_DIR}/AssemblyInfo.cs.in 57 | ${CSHARP_SOURCE_DIRECTORY}/AssemblyInfo.cs 58 | @ONLY 59 | ) 60 | 61 | configure_file( 62 | ${CMAKE_CURRENT_SOURCE_DIR}/Tensor.cs 63 | ${CSHARP_SOURCE_DIRECTORY}/Tensor.cs 64 | COPYONLY 65 | ) 66 | 67 | # build the SWIG CSharp files into a wrapper library 68 | FILE(GLOB SWIG_CSHARP_SOURCES ${CSHARP_SOURCE_DIRECTORY}/*.cs) 69 | 70 | foreach( source ${SWIG_CSHARP_SOURCES} ) 71 | set_source_files_properties(${source} PROPERTIES GENERATED TRUE) 72 | endforeach(source) 73 | 74 | # Add managed wrapper 75 | add_library( 76 | NumpyIO 77 | SHARED 78 | ${SWIG_CSHARP_SOURCES} 79 | ) 80 | 81 | target_compile_options( NumpyIO PUBLIC "/unsafe" ) 82 | target_link_libraries( NumpyIO ${SWIG_MODULE_NumpyIONative_REAL_NAME} ) 83 | 84 | install( TARGETS NumpyIO ${SWIG_MODULE_NumpyIONative_REAL_NAME} RUNTIME DESTINATION "lib/net472") -------------------------------------------------------------------------------- /CSharpWrapper/NumpyIONative.i: -------------------------------------------------------------------------------- 1 | %module NumpyIO 2 | %{ 3 | #include "npy/tensor.h" 4 | #include "npy/npy.h" 5 | #include "npy/npz.h" 6 | using namespace npy; 7 | %} 8 | 9 | %include "std_vector.i" 10 | %include "std_string.i" 11 | %include "std_wstring.i" 12 | %include "stdint.i" 13 | %include "arrays_csharp.i" 14 | %include "typemaps.i" 15 | %include "attribute.i" 16 | 17 | %rename(DataType) data_type_t; 18 | %typemap(csbase) data_type_t "byte"; 19 | enum class data_type_t : char { 20 | INT8, 21 | UINT8, 22 | INT16, 23 | UINT16, 24 | INT32, 25 | UINT32, 26 | INT64, 27 | UINT64, 28 | FLOAT32, 29 | FLOAT64, 30 | UNICODE_STRING 31 | }; 32 | 33 | %rename(Endian) endian_t; 34 | %typemap(csbase) endian_t "byte"; 35 | enum class endian_t : char { 36 | NATIVE, 37 | BIG, 38 | LITTLE 39 | }; 40 | 41 | %rename(CompressionMethod) compression_method_t; 42 | %typemap(csbase) compression_method_t "ushort"; 43 | enum class compression_method_t : std::uint16_t { 44 | STORED = 0, 45 | DEFLATED = 8 46 | }; 47 | 48 | %typemap(ctype, out="void *") const wstring * "wchar_t *" 49 | %typemap(imtype, 50 | inattributes="[global::System.Runtime.InteropServices.MarshalAs(UnmanagedType.LPArray, ArraySubType=UnmanagedType.LPStr)]", 51 | outattributes="[return: global::System.Runtime.InteropServices.MarshalAs(UnmanagedType.LPArray, ArraySubType=UnmanagedType.LPStr)]" 52 | ) const wstring * "string[]" 53 | %typemap(cstype) const wstring * "string[]" 54 | 55 | %template(UInt8Buffer) std::vector; 56 | %template(Int8Buffer) std::vector; 57 | %template(UInt16Buffer) std::vector; 58 | %template(Int16Buffer) std::vector; 59 | %template(UInt32Buffer) std::vector; 60 | %template(Int32Buffer) std::vector; 61 | %template(UInt64Buffer) std::vector; 62 | %template(Int64Buffer) std::vector; 63 | %template(Float32Buffer) std::vector; 64 | %template(Float64Buffer) std::vector; 65 | %apply const std::wstring & {std::wstring &}; 66 | %template(UnicodeStringBuffer) std::vector; 67 | %template(StringList) std::vector; 68 | 69 | %template(Shape) std::vector; 70 | 71 | %rename(HeaderInfo) header_info; 72 | struct header_info 73 | { 74 | header_info(data_type_t dtype, 75 | endian_t endianness, 76 | bool fortran_order, 77 | const std::vector &shape); 78 | 79 | 80 | %rename(DataType) dtype; 81 | data_type_t dtype; 82 | 83 | %rename(Endianness) endianness; 84 | endian_t endianness; 85 | 86 | %rename(FortranOrder) fortran_order; 87 | bool fortran_order; 88 | 89 | %rename(Shape) shape; 90 | std::vector shape; 91 | }; 92 | 93 | %exception peek(const std::string& path) %{ 94 | try{ 95 | $action 96 | } catch( std::invalid_argument& e) { 97 | SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "Invalid path location", e.what()); 98 | return $null; 99 | } catch( std::logic_error& e) { 100 | SWIG_CSharpSetPendingException(SWIG_CSharpIOException, e.what()); 101 | return $null; 102 | } 103 | %} 104 | 105 | %rename(Peek) peek; 106 | header_info peek(const std::string& path); 107 | 108 | template 109 | class tensor { 110 | public: 111 | %exception tensor(const std::string& path) %{ 112 | try{ 113 | $action 114 | } catch( std::invalid_argument& e) { 115 | SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "Invalid path location", e.what()); 116 | return $null; 117 | } catch( std::logic_error& e) { 118 | SWIG_CSharpSetPendingException(SWIG_CSharpIOException, e.what()); 119 | return $null; 120 | } 121 | %} 122 | 123 | explicit tensor(const std::string& path); 124 | 125 | tensor(const std::vector& shape); 126 | 127 | tensor(const std::vector& shape, bool fortran_order); 128 | 129 | %exception save(const std::string& path, endian_t endian = endian_t::NATIVE) %{ 130 | try{ 131 | $action 132 | } catch (std::invalid_argument& e) { 133 | SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "Invalid path location", e.what()); 134 | return $null; 135 | } catch (std::logic_error& e) { 136 | SWIG_CSharpSetPendingException(SWIG_CSharpIOException, e.what()); 137 | return $null; 138 | } 139 | %} 140 | 141 | %csmethodmodifiers save "public override"; 142 | %rename(Save) save; 143 | void save(const std::string& path, endian_t endian = endian_t::NATIVE); 144 | 145 | %exception copy_from(const std::vector& source) %{ 146 | try{ 147 | $action 148 | } catch (std::invalid_argument& e){ 149 | SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "Incorrect number of items", e.what()); 150 | return $null; 151 | } 152 | %} 153 | 154 | %csmethodmodifiers copy_from "public unsafe override"; 155 | %rename(CopyFrom) copy_from; 156 | void copy_from(const std::vector& source); 157 | 158 | %csmethodmodifiers values "protected override" 159 | %rename(getValues) values; 160 | const std::vector& values() const; 161 | 162 | %csmethodmodifiers shape "protected override" 163 | %rename(getShape) shape; 164 | const std::vector shape() const; 165 | 166 | %csmethodmodifiers fortran_order "protected override" 167 | %rename(getFortranOrder) fortran_order; 168 | bool fortran_order() const; 169 | 170 | %csmethodmodifiers dtype "protected override" 171 | %rename(getDataType) dtype; 172 | data_type_t dtype() const; 173 | 174 | %csmethodmodifiers size "protected override" 175 | %rename(getSize) size; 176 | size_t size() const; 177 | 178 | %exception get(const std::vector& index) const %{ 179 | try{ 180 | $action 181 | }catch(std::invalid_argument& e){ 182 | SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "incorrect index size", e.what()); 183 | return $null; 184 | }catch(std::out_of_range& e){ 185 | SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentOutOfRangeException, "index out of range", e.what()); 186 | return $null; 187 | } 188 | %} 189 | 190 | %csmethodmodifiers get "protected override" 191 | const T& get(const std::vector& index) const; 192 | 193 | %exception set(const std::vector& index, const T& value) const %{ 194 | try{ 195 | $action 196 | }catch(std::invalid_argument& e){ 197 | SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "incorrect index size", e.what()); 198 | return $null; 199 | }catch(std::out_of_range& e){ 200 | SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentOutOfRangeException, "index out of range", e.what()); 201 | return $null; 202 | } 203 | %} 204 | 205 | %csmethodmodifiers set "protected override" 206 | void set(const std::vector& index, const T& value); 207 | }; 208 | 209 | %typemap(csbase) SWIGTYPE "Tensor"; 210 | %template(UInt8Tensor) tensor; 211 | %typemap(csbase) SWIGTYPE "Tensor"; 212 | %template(Int8Tensor) tensor; 213 | %typemap(csbase) SWIGTYPE "Tensor"; 214 | %template(UInt16Tensor) tensor; 215 | %typemap(csbase) SWIGTYPE "Tensor"; 216 | %template(Int16Tensor) tensor; 217 | %typemap(csbase) SWIGTYPE "Tensor"; 218 | %template(UInt32Tensor) tensor; 219 | %typemap(csbase) SWIGTYPE "Tensor"; 220 | %template(Int32Tensor) tensor; 221 | %typemap(csbase) SWIGTYPE "Tensor"; 222 | %template(UInt64Tensor) tensor; 223 | %typemap(csbase) SWIGTYPE "Tensor"; 224 | %template(Int64Tensor) tensor; 225 | %typemap(csbase) SWIGTYPE "Tensor"; 226 | %template(Float32Tensor) tensor; 227 | %typemap(csbase) SWIGTYPE "Tensor"; 228 | %template(Float64Tensor) tensor; 229 | %typemap(csbase) SWIGTYPE "Tensor"; 230 | %template(UnicodeStringTensor) tensor; 231 | 232 | %typemap(csbase) SWIGTYPE "" 233 | 234 | %rename(NPZOutputStream) onpzstream; 235 | class onpzstream { 236 | public: 237 | onpzstream(const std::string& path, compression_method_t compression=compression_method_t::STORED, endian_t endian=endian_t::NATIVE); 238 | 239 | %rename(Close) close; 240 | void close(); 241 | 242 | %exception write(const std::string& filename, const tensor& tensor) %{ 243 | try{ 244 | $action 245 | }catch(std::invalid_argument& e){ 246 | SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "Unsupported compression method", e.what()); 247 | return $null; 248 | }catch(std::logic_error& e){ 249 | SWIG_CSharpSetPendingException(SWIG_CSharpIOException, e.what()); 250 | return $null; 251 | } 252 | %} 253 | 254 | template 255 | void write(const std::string& filename, const tensor& tensor); 256 | }; 257 | 258 | %extend onpzstream { 259 | %template(Write) write; 260 | %template(Write) write; 261 | %template(Write) write; 262 | %template(Write) write; 263 | %template(Write) write; 264 | %template(Write) write; 265 | %template(Write) write; 266 | %template(Write) write; 267 | %template(Write) write; 268 | %template(Write) write; 269 | %template(Write) write; 270 | }; 271 | 272 | %rename(NPZInputStream) inpzstream; 273 | class inpzstream { 274 | public: 275 | %exception inpzstream(const std::string& path) %{ 276 | try{ 277 | $action 278 | }catch(std::invalid_argument& e){ 279 | SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "File does not exist", e.what()); 280 | return $null; 281 | }catch(std::logic_error& e){ 282 | SWIG_CSharpSetPendingException(SWIG_CSharpIOException, e.what()); 283 | return $null; 284 | } 285 | %} 286 | 287 | inpzstream(const std::string& path); 288 | 289 | %rename(Keys) keys; 290 | const std::vector& keys() const; 291 | 292 | %rename(Contains) contains; 293 | bool contains(const std::string& filename); 294 | 295 | %exception peek(const std::string& filename) %{ 296 | try{ 297 | $action 298 | }catch(std::invalid_argument& e){ 299 | SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "Filename does not exist in archive", e.what()); 300 | return $null; 301 | }catch(std::logic_error& e){ 302 | SWIG_CSharpSetPendingException(SWIG_CSharpIOException, e.what()); 303 | return $null; 304 | } 305 | %} 306 | 307 | %rename(Peek) peek; 308 | header_info peek(const std::string& filename); 309 | 310 | 311 | %exception read(const std::string& filename) %{ 312 | try{ 313 | $action 314 | }catch(std::invalid_argument& e){ 315 | SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "Filename does not exist in archive", e.what()); 316 | return $null; 317 | }catch(std::logic_error& e){ 318 | SWIG_CSharpSetPendingException(SWIG_CSharpIOException, e.what()); 319 | return $null; 320 | } 321 | %} 322 | 323 | template 324 | tensor read(const std::string& filename); 325 | 326 | %rename(Close) close; 327 | void close(); 328 | }; 329 | 330 | %extend inpzstream { 331 | %template(ReadUInt8) read; 332 | %template(ReadInt8) read; 333 | %template(ReadUInt16) read; 334 | %template(ReadInt16) read; 335 | %template(ReadUInt32) read; 336 | %template(ReadInt32) read; 337 | %template(ReadUInt64) read; 338 | %template(ReadInt64) read; 339 | %template(ReadFloat32) read; 340 | %template(ReadFloat64) read; 341 | %template(ReadUnicodeString) read; 342 | }; 343 | -------------------------------------------------------------------------------- /CSharpWrapper/Tensor.cs: -------------------------------------------------------------------------------- 1 | // ---------------------------------------------------------------------------- 2 | // 3 | // Tensor.cs -- abstract base class for .NET Tensor objects 4 | // 5 | // Copyright (C) 2021 Matthew Johnson 6 | // 7 | // For conditions of distribution and use, see copyright notice in LICENSE 8 | // 9 | // ---------------------------------------------------------------------------- 10 | 11 | using System; 12 | using System.Collections.Generic; 13 | 14 | namespace NumpyIO 15 | { 16 | /// 17 | /// This class is the abstract base class for all .NET tensor objects. 18 | /// The autogenerated classes will all be subclasses of this class. 19 | /// 20 | /// The data type 21 | /// The buffer type 22 | public abstract class Tensor where B : IList 23 | { 24 | /// 25 | /// Copy the data from the provided buffer. These values will 26 | /// be copied into the underlying C++ type. 27 | /// 28 | /// The source buffer 29 | public abstract void CopyFrom(B source); 30 | 31 | /// 32 | /// Save the tensor to the provided location on the disk. 33 | /// 34 | /// a valid location on the disk 35 | public abstract void Save(string path); 36 | 37 | /// 38 | /// Save the tensor to the provided location on the disk. 39 | /// 40 | /// a valid location on the disk 41 | /// the endianness with which to write the tensor 42 | public abstract void Save(string path, Endian endian); 43 | 44 | /// 45 | /// A readonly list of values. Changing values in this object 46 | /// will NOT affect the underlying Tensor. 47 | /// 48 | public IList Values 49 | { 50 | get 51 | { 52 | return this.getValues(); 53 | } 54 | } 55 | 56 | /// 57 | /// The shape of the tensor. 58 | /// 59 | public Shape Shape 60 | { 61 | get 62 | { 63 | return this.getShape(); 64 | } 65 | } 66 | 67 | /// 68 | /// Whether the data is stored in FORTRAN, or column-major, order. 69 | /// 70 | public bool FortranOrder 71 | { 72 | get 73 | { 74 | return this.getFortranOrder(); 75 | } 76 | } 77 | 78 | /// 79 | /// The data type of the tensor. 80 | /// 81 | public DataType DataType 82 | { 83 | get 84 | { 85 | return this.getDataType(); 86 | } 87 | } 88 | 89 | /// 90 | /// The number of elements in the tensor. 91 | /// 92 | public uint Size 93 | { 94 | get 95 | { 96 | return this.getSize(); 97 | } 98 | } 99 | 100 | /// 101 | /// Index operator. 102 | /// 103 | /// The index into the tensor 104 | /// The value at the provided index 105 | public T this[params int[] multiIndex] 106 | { 107 | get 108 | { 109 | if (multiIndex.Length != this.Shape.Count) 110 | { 111 | throw new ArgumentException("Incorrect number of indices"); 112 | } 113 | 114 | return this.get(new Int32Buffer(multiIndex)); 115 | } 116 | set 117 | { 118 | if (multiIndex.Length != this.Shape.Count) 119 | { 120 | throw new ArgumentException("Incorrect number of indices"); 121 | } 122 | 123 | this.set(new Int32Buffer(multiIndex), value); 124 | } 125 | } 126 | 127 | /// 128 | /// Autogenerated subclasses will implement this method. Maps to 129 | /// \link npy::tensor::values \endlink. 130 | /// 131 | /// A read-only buffer object 132 | protected abstract B getValues(); 133 | 134 | /// 135 | /// Autogenerated subclasses will implement this method. Maps to 136 | /// \link npy::tensor::shape \endlink. 137 | /// 138 | /// A Shape object 139 | protected abstract Shape getShape(); 140 | 141 | /// 142 | /// Autogenerated subclasses will implement this method. Maps to 143 | /// \link npy::tensor::fortran_order \endlink. 144 | /// 145 | /// Whether the data is stored in FORTRAN, or column-major, order 146 | protected abstract bool getFortranOrder(); 147 | 148 | /// 149 | /// Autogenerated subclasses will implement this method. Maps to 150 | /// \link npy::tensor::dtype \endlink. 151 | /// 152 | /// The data type of the tensor 153 | protected abstract DataType getDataType(); 154 | 155 | /// 156 | /// Autogenerated subclasses will implement this method. Maps to 157 | /// \link npy::tensor::size \endlink. 158 | /// 159 | /// The number of elements in the tensor 160 | protected abstract uint getSize(); 161 | 162 | /// 163 | /// Autogenerated subclasses will implement this method. Maps to 164 | /// \link npy::tensor::get \endlink 165 | protected abstract T get(Int32Buffer multiIndex); 166 | 167 | /// 168 | /// Autogenereted subclasses will implement this method. Maps to 169 | /// \link npy::tensor::set \endlink 170 | /// 171 | protected abstract void set(Int32Buffer multiIndex, T value); 172 | } 173 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Matthew A Johnson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # libnpy 2 | 3 | `libnpy` is a multi-platform C++ library for reading and writing NPY and 4 | NPZ files, with an additional .NET interface. It was built with the 5 | intention of making it easier for multi-language projects to use NPZ and 6 | NPY files for data storage, given their simplicity and support across 7 | most Python deep learning frameworks. 8 | 9 | The implementations in this library are based upon the following file 10 | format documents: 11 | - **NPY**: The NPY file format is documented by the NumPy developers 12 | in [this note](https://docs.scipy.org/doc/numpy/reference/generated/numpy.lib.format.html) 13 | - **NPZ**: While not explicitly documented, the NPZ format is a 14 | a PKZIP archive of NPY files, and thus is documented 15 | here: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT. 16 | 17 | ## Getting Started 18 | 19 | Start by installing [CMake](https://cmake.org/) in the way appropriate for your 20 | environment. 21 | 22 | ### Linux 23 | 24 | Create a build directory and initialize the cmake project: 25 | 26 | mkdir build 27 | cd build 28 | cmake .. --preset release 29 | 30 | You can then build and run the tests using: 31 | 32 | make 33 | ctest 34 | 35 | ### Windows 36 | 37 | Create a build directory and initialize the cmake project: 38 | 39 | mkdir build 40 | cd build 41 | cmake .. --preset release 42 | 43 | You can then build and run the tests using: 44 | 45 | cmake --build . --config Release 46 | ctest -C Release 47 | 48 | ## Sample code 49 | 50 | Once the library has been built and installed, you can begin to use it 51 | in your code. We have provided some 52 | [sample programs](https://github.com/matajoh/libnpy/tree/main/samples) 53 | (and naturally the [tests](https://github.com/matajoh/libnpy/tree/main/test) 54 | as well) which show how to use the library, but the basic concepts are as follows. 55 | For the purpose of this sample code we will use the built-in [tensor](src/tensor.h) 56 | class, but you should use your own tensor class as appropriate. 57 | 58 | ```C++ 59 | #include "tensor.h" 60 | #include "npy.h" 61 | #include "npz.h" 62 | 63 | ... 64 | // create a tensor object 65 | std::vector shape({32, 32, 3}); 66 | npy::tensor color(shape); 67 | 68 | // fill it with some data 69 | for (int row = 0; row < color.shape(0); ++row) 70 | { 71 | for (int col = 0; col < color.shape(1); ++col) 72 | { 73 | color(row, col, 0) = static_cast(row << 3); 74 | color(row, col, 1) = static_cast(col << 3); 75 | color(row, col, 2) = 128; 76 | } 77 | } 78 | 79 | // save it to disk as an NPY file 80 | npy::save("color.npy", color); 81 | 82 | // we can manually set the endianness to use 83 | npy::save("color.npy", color, npy::endian_t::BIG); 84 | 85 | // the built-in tensor class also has a save method 86 | color.save("color.npy"); 87 | 88 | // we can peek at the header of the file 89 | npy::header_info header = npy::peek("color.npy"); 90 | 91 | // we can load it back the same way 92 | color = npy::load("color.npy"); 93 | 94 | // let's create a second tensor as well 95 | shape = {32, 32}; 96 | npy::tensor gray(shape); 97 | 98 | for (int row = 0; row < gray.shape(0); ++row) 99 | { 100 | for (int col = 0; col < gray.shape(1); ++col) 101 | { 102 | gray(row, col) = 0.21f * color(row, col, 0) + 103 | 0.72f * color(row, col, 1) + 104 | 0.07f * color(row, col, 2); 105 | } 106 | } 107 | 108 | // we can write them to an NPZ file 109 | { 110 | npy::onpzstream output("test.npz"); 111 | output.write("color.npy", color); 112 | output.write("gray.npy", gray); 113 | } 114 | 115 | // and we can read them back out again 116 | { 117 | npy::inpzstream input("test.npz"); 118 | 119 | // we can test to see if the archive contains a file 120 | if (input.contains("color.npy")) 121 | { 122 | // and peek at its header 123 | header = input.peek("color.npy"); 124 | } 125 | 126 | color = input.read("color.npy"); 127 | gray = input.read("gray.npy"); 128 | } 129 | ``` 130 | 131 | The generated documentation contains more details on all of the functionality. 132 | We hope you find that the library fulfills your needs and is easy to use, but 133 | if you have any difficulties please create 134 | [issues](https://github.com/matajoh/libnpy/issues) so the maintainers can make 135 | the library even better. Thanks! 136 | -------------------------------------------------------------------------------- /RELEASE_NOTES: -------------------------------------------------------------------------------- 1 | Improvements: 2 | - Increased CHUNK size as per miniz instructions 3 | - Added tests for very large arrays in NPZ files 4 | - Added some CI tests to catch issues across platforms 5 | - Removed the internal IO streams in favor of just using stringstream 6 | - NPZs can now be read from and written to memory 7 | 8 | Bugfixes: 9 | - Fixed an issue where very large arrays in NPZ files would throw an error 10 | - Fixed a bug with mac builds due to deprecated APIs 11 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 1.5.3 -------------------------------------------------------------------------------- /assets/test/float32.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/float32.npy -------------------------------------------------------------------------------- /assets/test/float64.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/float64.npy -------------------------------------------------------------------------------- /assets/test/int16.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/int16.npy -------------------------------------------------------------------------------- /assets/test/int32.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/int32.npy -------------------------------------------------------------------------------- /assets/test/int32_array.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/int32_array.npy -------------------------------------------------------------------------------- /assets/test/int32_big.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/int32_big.npy -------------------------------------------------------------------------------- /assets/test/int32_scalar.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/int32_scalar.npy -------------------------------------------------------------------------------- /assets/test/int64.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/int64.npy -------------------------------------------------------------------------------- /assets/test/int8.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/int8.npy -------------------------------------------------------------------------------- /assets/test/test.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/test.npz -------------------------------------------------------------------------------- /assets/test/test_compressed.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/test_compressed.npz -------------------------------------------------------------------------------- /assets/test/uint16.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/uint16.npy -------------------------------------------------------------------------------- /assets/test/uint32.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/uint32.npy -------------------------------------------------------------------------------- /assets/test/uint64.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/uint64.npy -------------------------------------------------------------------------------- /assets/test/uint8.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/uint8.npy -------------------------------------------------------------------------------- /assets/test/uint8_fortran.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/uint8_fortran.npy -------------------------------------------------------------------------------- /assets/test/unicode.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/assets/test/unicode.npy -------------------------------------------------------------------------------- /cmake/npyConfig.cmake.in: -------------------------------------------------------------------------------- 1 | get_filename_component(NPY_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) 2 | 3 | if(NOT TARGET npy::npy) 4 | include("${NPY_CMAKE_DIR}/npyTargets.cmake") 5 | endif() 6 | 7 | set(LIBNPY_lIBRARIES npy::npy) 8 | -------------------------------------------------------------------------------- /doc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set( README_FILES 2 | ../README.md 3 | ../CHANGELOG.md 4 | ../LICENSE 5 | ) 6 | 7 | # copy these files into the root of the distribution zip 8 | install( FILES ${README_FILES} DESTINATION "." ) 9 | 10 | # build the documentation from the C++ comments 11 | configure_file( 12 | Doxyfile.in 13 | ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY 14 | ) 15 | 16 | add_custom_target( npy_doc 17 | ALL 18 | ${DOXYGEN_EXECUTABLE} 19 | ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile 20 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 21 | COMMENT "Generating API documentation with Doxygen" VERBATIM 22 | ) 23 | -------------------------------------------------------------------------------- /include/npy/core.h: -------------------------------------------------------------------------------- 1 | // ---------------------------------------------------------------------------- 2 | // 3 | // core.h -- core types, enums and functions used by the library 4 | // 5 | // Copyright (C) 2021 Matthew Johnson 6 | // 7 | // For conditions of distribution and use, see copyright notice in LICENSE 8 | // 9 | // ---------------------------------------------------------------------------- 10 | 11 | #ifndef _CORE_H_ 12 | #define _CORE_H_ 13 | 14 | #include 15 | #include 16 | #include 17 | 18 | namespace npy { 19 | /** Enumeration which represents a type of endianness */ 20 | enum class endian_t : char { 21 | /** Indicates that the native endianness should be used. Native in this case 22 | * means that of the hardware the program is currently running on. 23 | */ 24 | NATIVE, 25 | /** Indicates the use of big-endian encoding */ 26 | BIG, 27 | /** Indicates the use of little-endian encoding */ 28 | LITTLE 29 | }; 30 | 31 | /** This function will return the endianness of the current hardware. */ 32 | inline endian_t native_endian() { 33 | union { 34 | std::uint32_t i; 35 | char c[4]; 36 | } endian_test = {0x01020304}; 37 | 38 | return endian_test.c[0] == 1 ? endian_t::BIG : endian_t::LITTLE; 39 | }; 40 | 41 | /** This enum represents the different types of tensor data that can be stored. 42 | */ 43 | enum class data_type_t : char { 44 | /** 8 bit signed integer */ 45 | INT8, 46 | /** 8 bit unsigned integer */ 47 | UINT8, 48 | /** 16-bit signed integer (short) */ 49 | INT16, 50 | /** 16-bit unsigned integer (ushort) */ 51 | UINT16, 52 | /** 32-bit signed integer (int) */ 53 | INT32, 54 | /** 32-bit unsigned integer (uint) */ 55 | UINT32, 56 | /** 64-bit integer (long) */ 57 | INT64, 58 | /** 64-bit unsigned integer (long) */ 59 | UINT64, 60 | /** 32-bit floating point value (float) */ 61 | FLOAT32, 62 | /** 64-bit floating point value (double) */ 63 | FLOAT64, 64 | /** Unicode string (std::wstring) */ 65 | UNICODE_STRING 66 | }; 67 | 68 | /** Convert a data type and endianness to a NPY dtype string. 69 | * \param dtype the data type 70 | * \param endian the endianness. Defaults to the current endianness of the 71 | * caller. \return the NPY dtype string 72 | */ 73 | const std::string &to_dtype(data_type_t dtype, 74 | endian_t endian = endian_t::NATIVE); 75 | 76 | /** Converts from an NPY dtype string to a data type and endianness. 77 | * \param dtype the NPY dtype string 78 | * \return a pair of data type and endianness corresponding to the input 79 | */ 80 | const std::pair &from_dtype(const std::string &dtype); 81 | 82 | typedef std::basic_istringstream imemstream; 83 | typedef std::basic_ostringstream omemstream; 84 | 85 | std::ostream &operator<<(std::ostream &os, const endian_t &obj); 86 | std::ostream &operator<<(std::ostream &os, const data_type_t &obj); 87 | 88 | } // namespace npy 89 | 90 | #endif -------------------------------------------------------------------------------- /include/npy/npy.h: -------------------------------------------------------------------------------- 1 | // ---------------------------------------------------------------------------- 2 | // 3 | // npy.h -- methods for reading and writing the numpy lib (NPY) format. The 4 | // implementation is based upon the description available at: 5 | // https://docs.scipy.org/doc/numpy/reference/generated/numpy.lib.format.html 6 | // 7 | // Copyright (C) 2021 Matthew Johnson 8 | // 9 | // For conditions of distribution and use, see copyright notice in LICENSE 10 | // 11 | // ---------------------------------------------------------------------------- 12 | 13 | #ifndef _NPY_H_ 14 | #define _NPY_H_ 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "core.h" 26 | 27 | const int STATIC_HEADER_LENGTH = 10; 28 | 29 | namespace npy { 30 | /** Class representing the header info for an NPY file */ 31 | struct header_info { 32 | /** Constructor. 33 | * \param dictionary a Python-encoded dictionary containing the header 34 | * information 35 | */ 36 | explicit header_info(const std::string &dictionary); 37 | 38 | /** Constructor */ 39 | header_info(data_type_t dtype, npy::endian_t endianness, bool fortran_order, 40 | const std::vector &shape); 41 | 42 | /** The data type of the NPY file */ 43 | data_type_t dtype; 44 | 45 | /** The endianness of the data in the NPY file */ 46 | npy::endian_t endianness; 47 | 48 | /** Whether the values in the tensor are stored in FORTRAN, or column major, 49 | * order */ 50 | bool fortran_order; 51 | 52 | /** A vector of values indicating the shape of each dimension of the tensor. 53 | */ 54 | std::vector shape; 55 | 56 | /** Value used to indicate the maximum length of an element (used by Unicode 57 | * strings) */ 58 | std::size_t max_element_length; 59 | }; 60 | 61 | /** Writes an NPY header to the provided stream. 62 | * \param output the output stream 63 | * \param dtype the NPY-encoded dtype string (includes data type and 64 | * endianness) \param fortran_order whether the data is encoded in FORTRAN (i.e. 65 | * column major) order \param shape a sequence of values indicating the shape of 66 | * each dimension of the tensor \sa npy::to_dtype 67 | */ 68 | template 69 | void write_npy_header(std::basic_ostream &output, 70 | const std::string &dtype, bool fortran_order, 71 | const std::vector &shape) { 72 | std::ostringstream buff; 73 | buff << "{'descr': '" << dtype; 74 | buff << "', 'fortran_order': " << (fortran_order ? "True" : "False"); 75 | buff << ", 'shape': ("; 76 | for (auto dim = shape.begin(); dim < shape.end(); ++dim) { 77 | buff << *dim; 78 | if (dim < shape.end() - 1) { 79 | buff << ", "; 80 | } 81 | } 82 | 83 | if (shape.size() == 1) { 84 | buff << ","; 85 | } 86 | 87 | buff << "), }"; 88 | std::string dictionary = buff.str(); 89 | auto dict_length = dictionary.size() + 1; 90 | std::string end = "\n"; 91 | auto header_length = dict_length + STATIC_HEADER_LENGTH; 92 | if (header_length % 64 != 0) { 93 | header_length = ((header_length / 64) + 1) * 64; 94 | dict_length = header_length - STATIC_HEADER_LENGTH; 95 | end = std::string(dict_length - dictionary.length(), ' '); 96 | end.back() = '\n'; 97 | } 98 | 99 | const char header[STATIC_HEADER_LENGTH] = { 100 | static_cast(0x93), 'N', 'U', 101 | 'M', 'P', 'Y', 102 | 0x01, 0x00, static_cast(dict_length), 103 | 0x00}; 104 | output.write(header, STATIC_HEADER_LENGTH); 105 | output.write(reinterpret_cast(dictionary.data()), 106 | dictionary.length()); 107 | output.write(reinterpret_cast(end.data()), end.length()); 108 | } 109 | 110 | template 111 | void copy_to(const T *data_ptr, std::size_t num_elements, 112 | std::basic_ostream &output, npy::endian_t endianness) { 113 | if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { 114 | output.write(reinterpret_cast(data_ptr), 115 | num_elements * sizeof(T)); 116 | } else { 117 | CHAR buffer[sizeof(T)]; 118 | for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { 119 | const CHAR *start = reinterpret_cast(curr); 120 | std::reverse_copy(start, start + sizeof(T), buffer); 121 | output.write(buffer, sizeof(T)); 122 | } 123 | } 124 | } 125 | 126 | /** Saves a tensor to the provided stream. 127 | * \tparam T the data type 128 | * \tparam TENSOR the tensor type. 129 | * \param output the output stream 130 | * \param tensor the tensor 131 | * \param endianness the endianness to use in saving the tensor 132 | * \sa npy::tensor 133 | */ 134 | template class TENSOR, typename CHAR, 135 | std::enable_if_t::value, int> = 42> 136 | void save(std::basic_ostream &output, const TENSOR &tensor, 137 | endian_t endianness = npy::endian_t::NATIVE) { 138 | auto dtype = to_dtype(tensor.dtype(), endianness); 139 | write_npy_header(output, dtype, tensor.fortran_order(), tensor.shape()); 140 | copy_to(tensor.data(), tensor.size(), output, endianness); 141 | }; 142 | 143 | /** Saves a unicode string tensor to the provided stream. 144 | * \tparam TENSOR the tensor type. 145 | * \param output the output stream 146 | * \param tensor the tensor 147 | * \param endianness the endianness to use in saving the tensor 148 | * \sa npy::tensor 149 | */ 150 | template class TENSOR, typename CHAR, 151 | std::enable_if_t::value, int> = 42> 152 | void save(std::basic_ostream &output, const TENSOR &tensor, 153 | endian_t endianness = npy::endian_t::NATIVE) { 154 | std::size_t max_length = 0; 155 | for (const auto &element : tensor) { 156 | if (element.size() > max_length) { 157 | max_length = element.size(); 158 | } 159 | } 160 | 161 | if (endianness == npy::endian_t::NATIVE) { 162 | endianness = native_endian(); 163 | } 164 | 165 | std::string dtype = ">U" + std::to_string(max_length); 166 | if (endianness == npy::endian_t::LITTLE) { 167 | dtype = " unicode(tensor.size() * max_length, 0); 173 | auto word_start = unicode.begin(); 174 | for (const auto &element : tensor) { 175 | auto char_it = word_start; 176 | for (const auto &wchar : element) { 177 | *char_it = static_cast(wchar); 178 | char_it += 1; 179 | } 180 | 181 | word_start += max_length; 182 | } 183 | 184 | copy_to(unicode.data(), unicode.size(), output, endianness); 185 | }; 186 | 187 | /** Saves a tensor to the provided location on disk. 188 | * \tparam T the data type 189 | * \tparam TENSOR the tensor type. 190 | * \param path a path to a valid location on disk 191 | * \param tensor the tensor 192 | * \param endianness the endianness to use in saving the tensor 193 | * \sa npy::tensor 194 | */ 195 | template class TENSOR> 196 | void save(const std::string &path, const TENSOR &tensor, 197 | endian_t endianness = npy::endian_t::NATIVE) { 198 | std::ofstream output(path, std::ios::out | std::ios::binary); 199 | if (!output.is_open()) { 200 | throw std::invalid_argument("path"); 201 | } 202 | 203 | save(output, tensor, endianness); 204 | }; 205 | 206 | /** Read an NPY header from the provided stream. 207 | * \param input the input stream 208 | * \return the header information 209 | */ 210 | template 211 | header_info read_npy_header(std::basic_istream &input) { 212 | std::uint8_t header[STATIC_HEADER_LENGTH]; 213 | input.read(reinterpret_cast(header), STATIC_HEADER_LENGTH); 214 | assert(header[0] == 0x93); 215 | assert(header[1] == 'N'); 216 | assert(header[2] == 'U'); 217 | assert(header[3] == 'M'); 218 | assert(header[4] == 'P'); 219 | assert(header[5] == 'Y'); 220 | size_t dict_length = 0; 221 | if (header[6] == 0x01 && header[7] == 0x00) { 222 | dict_length = header[8] | (header[9] << 8); 223 | } else if (header[6] == 0x02 && header[7] == 0x00) { 224 | std::uint8_t extra[2]; 225 | input.read(reinterpret_cast(extra), 2); 226 | dict_length = 227 | header[8] | (header[9] << 8) | (extra[0] << 16) | (extra[1] << 24); 228 | } 229 | 230 | std::vector buffer(dict_length); 231 | input.read(buffer.data(), dict_length); 232 | std::string dictionary(buffer.begin(), buffer.end()); 233 | return header_info(dictionary); 234 | } 235 | 236 | template 237 | void copy_to(std::basic_istream &input, T *data_ptr, 238 | std::size_t num_elements, npy::endian_t endianness) { 239 | if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { 240 | CHAR *start = reinterpret_cast(data_ptr); 241 | input.read(start, num_elements * sizeof(T)); 242 | } else { 243 | CHAR buffer[sizeof(T)]; 244 | for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { 245 | input.read(buffer, sizeof(T)); 246 | CHAR *start = reinterpret_cast(curr); 247 | std::reverse_copy(buffer, buffer + sizeof(T), start); 248 | } 249 | } 250 | } 251 | 252 | /** Loads a tensor in NPY format from the provided stream. The type of the 253 | * tensor must match the data to be read. \tparam T the data type \tparam TENSOR 254 | * the tensor type \param input the input stream \return an object of type 255 | * TENSOR read from the stream \sa npy::tensor 256 | */ 257 | template class TENSOR, typename CHAR, 258 | std::enable_if_t::value, int> = 42> 259 | TENSOR load(std::basic_istream &input) { 260 | header_info info = read_npy_header(input); 261 | TENSOR tensor(info.shape, info.fortran_order); 262 | if (info.dtype != tensor.dtype()) { 263 | throw std::logic_error("requested dtype does not match stream's dtype"); 264 | } 265 | 266 | copy_to(input, tensor.data(), tensor.size(), info.endianness); 267 | return tensor; 268 | } 269 | 270 | /** Loads a unicode string tensor in NPY format from the provided stream. The 271 | * type of the tensor must match the data to be read. \tparam T the data type 272 | * \tparam TENSOR the tensor type 273 | * \param input the input stream 274 | * \return an object of type TENSOR read from the stream 275 | * \sa npy::tensor 276 | */ 277 | template class TENSOR, typename CHAR, 278 | std::enable_if_t::value, int> = 42> 279 | TENSOR load(std::basic_istream &input) { 280 | header_info info = read_npy_header(input); 281 | TENSOR tensor(info.shape, info.fortran_order); 282 | if (info.dtype != tensor.dtype()) { 283 | throw std::logic_error("requested dtype does not match stream's dtype"); 284 | } 285 | 286 | std::vector unicode(tensor.size() * info.max_element_length, 0); 287 | copy_to(input, unicode.data(), unicode.size(), info.endianness); 288 | 289 | auto word_start = unicode.begin(); 290 | for (auto &element : tensor) { 291 | auto char_it = word_start; 292 | for (std::size_t i = 0; i < info.max_element_length && *char_it > 0; 293 | ++i, ++char_it) { 294 | element.push_back(static_cast(*char_it)); 295 | } 296 | 297 | word_start += info.max_element_length; 298 | } 299 | 300 | return tensor; 301 | } 302 | 303 | /** Loads a tensor in NPY format from the specified location on the disk. The 304 | * type of the tensor must match the data to be read. \tparam T the data type 305 | * \tparam TENSOR the tensor type 306 | * \param path a valid location on the disk 307 | * \return an object of type TENSOR read from the stream 308 | * \sa npy::tensor 309 | */ 310 | template class TENSOR> 311 | TENSOR load(const std::string &path) { 312 | std::ifstream input(path, std::ios::in | std::ios::binary); 313 | if (!input.is_open()) { 314 | throw std::invalid_argument("path"); 315 | } 316 | 317 | return load(input); 318 | } 319 | 320 | /** Return the header information for an NPY file. 321 | * \param input the input stream containing the NPY-encoded bytes 322 | * \return the NPY header information 323 | */ 324 | template header_info peek(std::basic_istream &input) { 325 | return read_npy_header(input); 326 | } 327 | 328 | /** Return the header information for an NPY file. 329 | * \param path the path to the NPY file on disk 330 | * \return the NPY header information 331 | */ 332 | header_info peek(const std::string &path); 333 | 334 | } // namespace npy 335 | 336 | #endif -------------------------------------------------------------------------------- /include/npy/npz.h: -------------------------------------------------------------------------------- 1 | // ---------------------------------------------------------------------------- 2 | // 3 | // npz.h -- methods for reading and writing the numpy archive (NPZ) file 4 | // format. The implementation here is based upon the PKZIP Application 5 | // note: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT. 6 | // 7 | // Copyright (C) 2021 Matthew Johnson 8 | // 9 | // For conditions of distribution and use, see copyright notice in LICENSE 10 | // 11 | // ---------------------------------------------------------------------------- 12 | 13 | #ifndef _NPZ_H_ 14 | #define _NPZ_H_ 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "core.h" 25 | #include "npy.h" 26 | #include "tensor.h" 27 | 28 | namespace npy { 29 | /** Enumeration indicating the compression method to use for data in the NPZ 30 | * archive. */ 31 | enum class compression_method_t : std::uint16_t { 32 | /** Store the data with no compression */ 33 | STORED = 0, 34 | /** Use the DEFLATE algorithm to compress the data */ 35 | DEFLATED = 8 36 | }; 37 | 38 | /** Struct representing a file in the NPZ archive. */ 39 | struct file_entry { 40 | /** The name of the file */ 41 | std::string filename; 42 | /** The CRC32 checksum of the uncompressed data */ 43 | std::uint32_t crc32; 44 | /** The size of the compressed data */ 45 | std::uint64_t compressed_size; 46 | /** The size of the uncompressed data */ 47 | std::uint64_t uncompressed_size; 48 | /** The method used to compress the data */ 49 | std::uint16_t compression_method; 50 | /** The offset of the file in the archive */ 51 | std::uint64_t offset; 52 | 53 | /** Check if this entry matches another entry 54 | * \param other the other entry 55 | * \return if these entries match 56 | */ 57 | bool check(const file_entry &other) const; 58 | }; 59 | 60 | /** Class representing an output stream for an NPZ archive file */ 61 | class onpzstream { 62 | public: 63 | /** Constructor 64 | * \param output the output stream to write to 65 | * \param compression how the entries should be compressed 66 | * \param endianness the endianness to use in writing the entries 67 | */ 68 | onpzstream(const std::shared_ptr &output, 69 | compression_method_t compression = compression_method_t::STORED, 70 | endian_t endianness = npy::endian_t::NATIVE); 71 | 72 | /** Constructor. 73 | * \param path the path to the file on disk 74 | * \param compression how the entries should be compressed 75 | * \param endianness the endianness to use in writing the entries 76 | */ 77 | onpzstream(const std::string &path, 78 | compression_method_t compression = compression_method_t::STORED, 79 | endian_t endianness = npy::endian_t::NATIVE); 80 | 81 | /** Whether the underlying stream has successfully been opened. */ 82 | bool is_open() const; 83 | 84 | /** Closes this stream. This will write the directory and close 85 | * the underlying stream as well. */ 86 | void close(); 87 | 88 | /** Write a tensor to the NPZ archive. 89 | * \tparam T the data type 90 | * \tparam TENSOR the tensor type 91 | * \param filename the name of the file in the archive 92 | * \param tensor the tensor to write 93 | */ 94 | template class TENSOR> 95 | void write(const std::string &filename, const TENSOR &tensor) { 96 | if (m_closed) { 97 | throw std::logic_error("Stream is closed"); 98 | } 99 | 100 | omemstream output; 101 | save(output, tensor, m_endianness); 102 | 103 | std::string suffix = ".npy"; 104 | std::string name = filename; 105 | if (name.size() < 4 || 106 | !std::equal(suffix.rbegin(), suffix.rend(), name.rbegin())) { 107 | name += ".npy"; 108 | } 109 | 110 | write_file(name, output.str()); 111 | } 112 | 113 | /** Write a tensor to the NPZ archive. 114 | * \tparam T the data type 115 | * \param filename the name of the file in the archive 116 | * \param tensor the tensor to write 117 | */ 118 | template 119 | void write(const std::string &filename, const tensor &tensor) { 120 | write(filename, tensor); 121 | } 122 | 123 | /** Destructor. This will call 124 | * \link npy::onpzstream::close \endlink, if it has not been called already. 125 | */ 126 | ~onpzstream(); 127 | 128 | private: 129 | /** Write a file to the stream. 130 | * \param filename the name of the file 131 | * \param bytes the file data 132 | */ 133 | void write_file(const std::string &filename, std::string &&bytes); 134 | 135 | bool m_closed; 136 | std::shared_ptr m_output; 137 | compression_method_t m_compression_method; 138 | endian_t m_endianness; 139 | std::vector m_entries; 140 | }; 141 | 142 | /** Class representing an input stream from an NPZ archive file */ 143 | class inpzstream { 144 | public: 145 | /** Constructor. 146 | * \param stream the input stream to read from 147 | */ 148 | inpzstream(const std::shared_ptr &stream); 149 | 150 | /** Constructor. 151 | * \param path the path to the NPZ file on the disk 152 | */ 153 | inpzstream(const std::string &path); 154 | 155 | /** Whether the underlying stream has successfully been opened. */ 156 | bool is_open() const; 157 | 158 | /** Closes the underlying stream. */ 159 | void close(); 160 | 161 | /** The keys of the tensors in the NPZ */ 162 | const std::vector &keys() const; 163 | 164 | /** Returns whether this NPZ contains the specified tensor 165 | * \param filename the name of the tensor in the archive 166 | * \return whether the tensor is in the archive 167 | */ 168 | bool contains(const std::string &filename); 169 | 170 | /** Returns the header for a specified tensor. 171 | * \param filename the name of the tensor in the archive 172 | * \return the header for the tensor 173 | */ 174 | header_info peek(const std::string &filename); 175 | 176 | /** Read a tensor from the archive. This method will thrown an exception if 177 | * the tensor does not exist, or if the data type of the tensor does not 178 | * match the template type. \tparam T the data type \tparam TENSOR the tensor 179 | * type \param filename the name of the tensor in the archive. \return a 180 | * instance of TENSOR read from the archive. \sa npy::tensor 181 | */ 182 | template class TENSOR> 183 | TENSOR read(const std::string &filename) { 184 | imemstream stream(read_file(filename)); 185 | return load(stream); 186 | } 187 | 188 | /** Read a tensor from the archive. This method will thrown an exception if 189 | * the tensor does not exist, or if the data type of the tensor does not 190 | * match. \tparam T the data type \param filename the name of the tensor in 191 | * the archive. \return a instance of tensor read from the archive. 192 | */ 193 | template tensor read(const std::string &filename) { 194 | return read(filename); 195 | } 196 | 197 | private: 198 | /** Reads the bytes for a file from the archive. 199 | * \param filename the name of the file 200 | * \return the raw file bytes 201 | */ 202 | std::string read_file(const std::string &filename); 203 | 204 | /** Read all entries from the directory. */ 205 | void read_entries(); 206 | 207 | std::shared_ptr m_input; 208 | std::map m_entries; 209 | std::vector m_keys; 210 | }; 211 | } // namespace npy 212 | 213 | #endif -------------------------------------------------------------------------------- /include/npy/tensor.h: -------------------------------------------------------------------------------- 1 | // ---------------------------------------------------------------------------- 2 | // 3 | // tensor.h -- default tensor class for use with the library. 4 | // 5 | // Copyright (C) 2021 Matthew Johnson 6 | // 7 | // For conditions of distribution and use, see copyright notice in LICENSE 8 | // 9 | // ---------------------------------------------------------------------------- 10 | 11 | #ifndef _TENSOR_H_ 12 | #define _TENSOR_H_ 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "core.h" 22 | #include "npy.h" 23 | 24 | namespace npy { 25 | /** The default tensor class. This class can be used as a data exchange format 26 | * for the library, but the methods and classes will also work with your own 27 | * tensor implementation. The library methods require the following methods to 28 | * be present in a tensor type: 29 | * - \link data \endlink 30 | * - \link shape \endlink 31 | * - \link size \endlink 32 | * - \link dtype \endlink 33 | * - \link fortran_order \endlink 34 | * 35 | * As long as these are present and have the same semantics, the library should 36 | * handle them in the same was as this implementation. Only certain type of 37 | * tensor objects are natively supported (see \link npy::data_type_t \endlink). 38 | */ 39 | template class tensor { 40 | public: 41 | /** Constructor. 42 | * \param path the path to an NPY file on the disk 43 | */ 44 | explicit tensor(const std::string &path) 45 | : tensor(npy::load(path)) {} 46 | 47 | /** Constructor. This will allocate a data buffer of the appropriate size in 48 | * row-major order. \param shape the shape of the tensor 49 | */ 50 | tensor(const std::vector &shape) : tensor(shape, false) {} 51 | 52 | /** Constructor. This will allocate a data buffer of the appropriate size. 53 | * \param shape the shape of the tensor 54 | * \param fortran_order whether the data is stored in FORTRAN, or column 55 | * major, order 56 | */ 57 | tensor(const std::vector &shape, bool fortran_order) 58 | : m_shape(shape), 59 | m_ravel_strides(tensor::get_ravel_strides(shape, fortran_order)), 60 | m_fortran_order(fortran_order), m_dtype(tensor::get_dtype()), 61 | m_values(tensor::get_size(shape)) {} 62 | 63 | /** Copy constructor. */ 64 | tensor(const tensor &other) 65 | : m_shape(other.m_shape), m_ravel_strides(other.m_ravel_strides), 66 | m_fortran_order(other.m_fortran_order), m_dtype(other.m_dtype), 67 | m_values(other.m_values) {} 68 | 69 | /** Move constructor. */ 70 | tensor(tensor &&other) 71 | : m_shape(std::move(other.m_shape)), 72 | m_ravel_strides(std::move(other.m_ravel_strides)), 73 | m_fortran_order(other.m_fortran_order), m_dtype(other.m_dtype), 74 | m_values(std::move(other.m_values)) {} 75 | 76 | /** Variable parameter index function. 77 | * \param index an index into the tensor. Can be negative (in which case it 78 | * will work as in numpy) \return the value at the provided index 79 | */ 80 | template const T &operator()(Indices... index) const { 81 | return m_values[ravel(std::vector({index...}))]; 82 | } 83 | 84 | /** Index function. 85 | * \param multi_index the index into the tensor. 86 | * \return the value at the provided index 87 | */ 88 | const T &operator()(const std::vector &multi_index) const { 89 | return m_values[ravel(multi_index)]; 90 | } 91 | 92 | /** Variable parameter index function. 93 | * \param index an index into the tensor. Can be negative (in which case it 94 | * will work as in numpy) \return the value at the provided index 95 | */ 96 | template T &operator()(Indices... index) { 97 | return m_values[ravel(std::vector({index...}))]; 98 | } 99 | 100 | /** Index function. 101 | * \param multi_index the index into the tensor. 102 | * \return the value at the provided index 103 | */ 104 | T &operator()(const std::vector &multi_index) { 105 | return m_values[ravel(multi_index)]; 106 | } 107 | 108 | /** Iterator pointing at the beginning of the tensor in memory. */ 109 | typename std::vector::iterator begin() { return m_values.begin(); } 110 | 111 | /** Iterator pointing at the beginning of the tensor in memory. */ 112 | typename std::vector::const_iterator begin() const { 113 | return m_values.begin(); 114 | } 115 | 116 | /** Iterator pointing at the end of the tensor in memory. */ 117 | typename std::vector::iterator end() { return m_values.end(); } 118 | 119 | /** Iterator pointing at the end of the tensor in memory. */ 120 | typename std::vector::const_iterator end() const { return m_values.end(); } 121 | 122 | /** Sets the value at the provided index. 123 | * \param multi_index an index into the tensor 124 | * \param value the value to set 125 | */ 126 | void set(const std::vector &multi_index, const T &value) { 127 | m_values[ravel(multi_index)] = value; 128 | } 129 | 130 | /** Gets the value at the provided index. 131 | * \param multi_index the index into the tensor 132 | * \return the value at the provided index 133 | */ 134 | const T &get(const std::vector &multi_index) const { 135 | return m_values[ravel(multi_index)]; 136 | } 137 | 138 | /** The data type of the tensor. */ 139 | const data_type_t dtype() const { return m_dtype; } 140 | 141 | /** The underlying values buffer. */ 142 | const std::vector &values() const { return m_values; } 143 | 144 | /** Copy values from the source to this tensor. 145 | * \param source pointer to the start of the source buffer 146 | * \param nitems the number of items to copy. Should be equal to \link size 147 | * \endlink. 148 | */ 149 | void copy_from(const T *source, size_t nitems) { 150 | if (nitems != size()) { 151 | throw std::invalid_argument("nitems"); 152 | } 153 | 154 | std::copy(source, source + nitems, m_values.begin()); 155 | } 156 | 157 | /** Copy values from the provided vector. 158 | * \param source the source vector. Should have the same size as \link values 159 | * \endlink. 160 | */ 161 | void copy_from(const std::vector &source) { 162 | if (source.size() != size()) { 163 | throw std::invalid_argument("source.size"); 164 | } 165 | 166 | std::copy(source.begin(), source.end(), m_values.begin()); 167 | } 168 | 169 | /** Move values from the provided vector. 170 | * \param source the source vector. Should have the same size as \link values 171 | * \endlink. 172 | */ 173 | void move_from(std::vector &&source) { 174 | if (source.size() != size()) { 175 | throw std::invalid_argument("source.size"); 176 | } 177 | 178 | m_values = std::move(source); 179 | } 180 | 181 | /** A pointer to the start of the underlying values buffer. */ 182 | T *data() { return m_values.data(); } 183 | 184 | /** A pointer to the start of the underlying values buffer. */ 185 | const T *data() const { return m_values.data(); } 186 | 187 | /** The number of elements in the tensor. */ 188 | size_t size() const { return m_values.size(); } 189 | 190 | /** The shape of the vector. Each element is the size of the 191 | * corresponding dimension. */ 192 | const std::vector &shape() const { return m_shape; } 193 | 194 | /** Returns the dimensionality of the tensor at the specified index. 195 | * \param index index into the shape 196 | * \return the dimensionality at the index 197 | */ 198 | const size_t shape(int index) const { return m_shape[index]; } 199 | 200 | /** Whether the tensor data is stored in FORTRAN, or column-major, order. */ 201 | bool fortran_order() const { return m_fortran_order; } 202 | 203 | /** Copy assignment operator. */ 204 | tensor &operator=(const tensor &other) { 205 | m_shape = other.m_shape; 206 | m_ravel_strides = other.m_ravel_strides; 207 | m_fortran_order = other.m_fortran_order; 208 | m_dtype = other.m_dtype; 209 | m_values = other.m_values; 210 | return *this; 211 | } 212 | 213 | /** Move assignment operator. */ 214 | tensor &operator=(tensor &&other) { 215 | m_shape = std::move(other.m_shape); 216 | m_ravel_strides = std::move(other.m_ravel_strides); 217 | m_fortran_order = other.m_fortran_order; 218 | m_dtype = other.m_dtype; 219 | m_values = std::move(other.m_values); 220 | return *this; 221 | } 222 | 223 | /** Save this tensor to the provided location on disk. 224 | * \param path a valid location on disk 225 | * \param endianness the endianness to use in writing the tensor 226 | */ 227 | void save(const std::string &path, 228 | endian_t endianness = npy::endian_t::NATIVE) { 229 | npy::save(path, *this, endianness); 230 | } 231 | 232 | /** Ravels a multi-index into a single value indexing the buffer. 233 | * \tparam INDEX_IT the index iterator class 234 | * \tparam SHAPE_IT the shape iterator class 235 | * \param index the multi-index iterator 236 | * \param shape the shape iterator 237 | * \return the single value in the buffer corresponding to the multi-index 238 | */ 239 | template 240 | size_t ravel(INDEX_IT index, SHAPE_IT shape) const { 241 | std::size_t ravel = 0; 242 | for (auto stride = m_ravel_strides.begin(); stride < m_ravel_strides.end(); 243 | ++index, ++shape, ++stride) { 244 | if (*index >= *shape) { 245 | throw std::out_of_range("multi_index"); 246 | } 247 | 248 | ravel += *index * *stride; 249 | } 250 | 251 | return ravel; 252 | } 253 | 254 | /** Ravels a multi-index into a single value indexing the buffer. 255 | * \param multi_index the multi-index value 256 | * \return the single value in the buffer corresponding to the multi-index 257 | */ 258 | size_t ravel(const std::vector &multi_index) const { 259 | if (multi_index.size() != m_shape.size()) { 260 | throw std::invalid_argument("multi_index"); 261 | } 262 | 263 | std::vector abs_multi_index(multi_index.size()); 264 | std::transform(multi_index.begin(), multi_index.end(), m_shape.begin(), 265 | abs_multi_index.begin(), 266 | [](std::int32_t index, std::size_t shape) -> std::size_t { 267 | if (index < 0) { 268 | return static_cast(shape + index); 269 | } 270 | 271 | return static_cast(index); 272 | }); 273 | 274 | return ravel(abs_multi_index); 275 | } 276 | 277 | /** Ravels a multi-index into a single value indexing the buffer. 278 | * \param abs_multi_index the multi-index value 279 | * \return the single value in the buffer corresponding to the multi-index 280 | */ 281 | size_t ravel(const std::vector &abs_multi_index) const { 282 | if (m_fortran_order) { 283 | return ravel(abs_multi_index.rbegin(), m_shape.rbegin()); 284 | } 285 | 286 | return ravel(abs_multi_index.begin(), m_shape.begin()); 287 | } 288 | 289 | private: 290 | std::vector m_shape; 291 | std::vector m_ravel_strides; 292 | bool m_fortran_order; 293 | data_type_t m_dtype; 294 | std::vector m_values; 295 | 296 | /** Returns the data type for this tensor. */ 297 | static data_type_t get_dtype(); 298 | 299 | /** Gets the size of a tensor given its shape */ 300 | static size_t get_size(const std::vector &shape) { 301 | size_t size = 1; 302 | for (auto &dim : shape) { 303 | size *= dim; 304 | } 305 | 306 | return size; 307 | } 308 | 309 | /** Gets the strides for ravelling */ 310 | static std::vector get_ravel_strides(const std::vector &shape, 311 | bool fortran_order) { 312 | std::vector ravel_strides(shape.size()); 313 | size_t stride = 1; 314 | auto ravel = ravel_strides.rbegin(); 315 | if (fortran_order) { 316 | for (auto max_index = shape.begin(); max_index < shape.end(); 317 | ++max_index, ++ravel) { 318 | *ravel = stride; 319 | stride *= *max_index; 320 | } 321 | } else { 322 | for (auto max_index = shape.rbegin(); max_index < shape.rend(); 323 | ++max_index, ++ravel) { 324 | *ravel = stride; 325 | stride *= *max_index; 326 | } 327 | } 328 | 329 | return ravel_strides; 330 | } 331 | }; 332 | } // namespace npy 333 | 334 | #endif -------------------------------------------------------------------------------- /nuget/template.nuspec.in: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | @LIBNPY_NUGET_NAME@ 5 | @LIBNPY_VERSION@ 6 | @LIBNPY_NUGET_NAME@ 7 | Matthew Johnson 8 | Matthew Johnson 9 | false 10 | MIT 11 | https://github.com/matajoh/libnpy 12 | C++ library for reading and writing NPY and NPZ files. 13 | @LIBNPY_RELEASE_NOTES@ 14 | native 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /nuget/template.targets.in: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | $(MSBuildThisFileDirectory)include;%(AdditionalIncludeDirectories) 5 | 6 | 7 | $(MSBuildThisFileDirectory)lib\;%(AdditionalLibraryDirectories) 8 | @LIBNPY_NUGET_LIB@;%(AdditionalDependencies) 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /samples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable( npy_images images.cpp ) 2 | target_link_libraries( npy_images npy::npy ) 3 | 4 | if( INCLUDE_CSHARP ) 5 | add_executable( npy_images_net images_net.cs ) 6 | target_link_libraries( npy_images_net NumpyIO ${SWIG_MODULE_NumpyIONative_REAL_NAME} ) 7 | endif() -------------------------------------------------------------------------------- /samples/README.md: -------------------------------------------------------------------------------- 1 | # Sample Code 2 | 3 | The code here provides examples of how to use the library and the various functions it supports. 4 | 5 | ## Getting Started 6 | 7 | To build the sample code, start by using the dependency install guide for your platform below. 8 | 9 | ### Ubuntu 18.04 [gcc 7.3.0], Ubuntu 16.04 [gcc 5.4.0] 10 | 11 | First, install all of the necessary dependencies: 12 | 13 | sudo apt-get install git cmake build-essential python3 python3-pip 14 | 15 | You may find that `cmake` is easier to use via the curses GUI: 16 | 17 | sudo apt-get install cmake-curses-gui 18 | 19 | Then, install the python dependencies: 20 | 21 | pip3 install -r requirements.txt 22 | 23 | Once everything is in place, you can clone the repository and generate the 24 | makefiles: 25 | 26 | mkdir build 27 | cd build 28 | cmake -DCMAKE_BUILD_TYPE=Debug .. 29 | 30 | Your other build options are `Release` and `RelWithDebInfo`. 31 | 32 | ### Windows 10 [Visual Studio 2017] 33 | 34 | On Windows, you can download and install the dependencies from the following 35 | locations: 36 | 37 | #### Install CMake 38 | Download and run e.g. `v3.11/cmake-3.11.0-win64-x64.msi` from 39 | https://cmake.org/files/. 40 | 41 | #### Install git and Visual Studio. 42 | Get the latest Windows git from https://git-scm.com/downloads. Download a 43 | version of Visual Studio from https://visualstudio.microsoft.com/vs/. You 44 | will need the C++ and C# compilers. 45 | 46 | #### Download and install Python 47 | 48 | Download and run e.g. `Python 3.6.8` from https://www.python.org/downloads/windows/. 49 | 50 | #### Generate MSBuild 51 | Now that everything is ready, cmake can generate the MSBuild files necessary 52 | for the project. Run the following commands in a command prompt once you have 53 | navigated to your desired source code folder: 54 | 55 | mkdir build 56 | cd build 57 | cmake -G "Visual Studio 15 2017 Win64" .. 58 | 59 | ### Build and Run 60 | You are now able to build the test the library. Doing so is the same 61 | regardless of your platform. First, navigate to the `build` folder you 62 | created above. Then run the following commands: 63 | 64 | cmake --build . --config 65 | 66 | Where `` is one of `Release|Debug|RelWithDebInfo`. This will build 67 | the project, including the tests and (if selected) the documentation. You 68 | can then browse to the output directory and do the following: 69 | 70 | ``` 71 | ./images 72 | python display.py 73 | ``` 74 | 75 | You should see the following screen: 76 | 77 | ![Expected output of display.py](display.png) 78 | 79 | Which indicates the sample has run correctly. 80 | 81 | If you have built the .NET sample, then you can simply substitute `images_net` for `images` above and the same screen should appear. 82 | -------------------------------------------------------------------------------- /samples/display.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matajoh/libnpy/e76b1dae9fe717ddf3291adff091af3c85966200/samples/display.png -------------------------------------------------------------------------------- /samples/display.py: -------------------------------------------------------------------------------- 1 | """ Python display script to test NPY/NPZ file output 2 | 3 | This script assumes that you have the numpy and matplotlib modules installed 4 | """ 5 | 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | 9 | def _main(): 10 | plt.figure(1) 11 | 12 | # npy 13 | plt.subplot(131) 14 | plt.imshow(np.load("color.npy")) 15 | 16 | test = np.load("test.npz") 17 | 18 | # npz 19 | plt.subplot(132) 20 | plt.imshow(test["color.npy"]) 21 | 22 | plt.subplot(133) 23 | plt.imshow(test["gray.npy"], cmap="gray") 24 | 25 | plt.show() 26 | 27 | 28 | if __name__ == "__main__": 29 | _main() 30 | -------------------------------------------------------------------------------- /samples/images.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "npy/tensor.h" 4 | #include "npy/npy.h" 5 | #include "npy/npz.h" 6 | 7 | int main() 8 | { 9 | // create a tensor object 10 | std::vector shape({32, 32, 3}); 11 | npy::tensor color(shape); 12 | 13 | // fill it with some data 14 | for (int row = 0; row < color.shape(0); ++row) 15 | { 16 | for (int col = 0; col < color.shape(1); ++col) 17 | { 18 | color(row, col, 0) = static_cast(row << 3); 19 | color(row, col, 1) = static_cast(col << 3); 20 | color(row, col, 2) = 128; 21 | } 22 | } 23 | 24 | // save it to disk as an NPY file 25 | npy::save("color.npy", color); 26 | 27 | // we can manually set the endianness to use 28 | npy::save("color.npy", color, npy::endian_t::BIG); 29 | 30 | // the built-in tensor class also has a save method 31 | color.save("color.npy"); 32 | 33 | // we can peek at the header of the file 34 | npy::header_info header = npy::peek("color.npy"); 35 | 36 | // we can load it back the same way 37 | color = npy::load("color.npy"); 38 | 39 | // let's create a second tensor as well 40 | shape = {32, 32}; 41 | npy::tensor gray(shape); 42 | 43 | for (int row = 0; row < gray.shape(0); ++row) 44 | { 45 | for (int col = 0; col < gray.shape(1); ++col) 46 | { 47 | gray(row, col) = 0.21f * color(row, col, 0) + 48 | 0.72f * color(row, col, 1) + 49 | 0.07f * color(row, col, 2); 50 | } 51 | } 52 | 53 | // we can write them to an NPZ file 54 | { 55 | npy::onpzstream output("test.npz"); 56 | output.write("color.npy", color); 57 | output.write("gray.npy", gray); 58 | } 59 | 60 | // and we can read them back out again 61 | { 62 | npy::inpzstream input("test.npz"); 63 | 64 | // we can test to see if the archive contains a file 65 | if (input.contains("color.npy")) 66 | { 67 | // and peek at its header 68 | header = input.peek("color.npy"); 69 | } 70 | 71 | color = input.read("color.npy"); 72 | gray = input.read("gray.npy"); 73 | } 74 | 75 | return 0; 76 | } -------------------------------------------------------------------------------- /samples/images_net.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using NumpyIO; 3 | 4 | public unsafe class Sample 5 | { 6 | public static void Main(string[] args) 7 | { 8 | // create a tensor object 9 | Shape shape = new Shape(new uint[] { 32, 32, 3 }); 10 | UInt8Tensor color = new UInt8Tensor(shape); 11 | 12 | // fill it with some data. 13 | for (int row = 0; row < color.Shape[0]; ++row) 14 | { 15 | for (int col = 0; col < color.Shape[1]; ++col) 16 | { 17 | color[row, col, 0] = (byte)(row << 3); 18 | color[row, col, 1] = (byte)(col << 3); 19 | color[row, col, 2] = 128; 20 | } 21 | } 22 | 23 | // save it to disk as an NPY file 24 | color.Save("color.npy"); 25 | 26 | // we can manually set the endianness to use 27 | color.Save("color.npy", Endian.BIG); 28 | 29 | // we can peek at the header of a file 30 | HeaderInfo header = NumpyIO.NumpyIO.Peek("color.npy"); 31 | 32 | // we can load it using the path constructor 33 | color = new UInt8Tensor("color.npy"); 34 | 35 | // let's create a second tensor as well 36 | shape = new Shape(new uint[] { 32, 32 }); 37 | Float32Tensor gray = new Float32Tensor(shape); 38 | for (int row = 0; row < gray.Shape[0]; ++row) 39 | { 40 | for (int col = 0; col < gray.Shape[1]; ++col) 41 | { 42 | gray[row, col] = 0.21f * color[row, col, 0] + 43 | 0.72f * color[row, col, 1] + 44 | 0.07f * color[row, col, 2]; 45 | } 46 | } 47 | 48 | // we can write them to an NPZ file 49 | NPZOutputStream output = new NPZOutputStream("test.npz"); 50 | output.Write("color.npy", color); 51 | output.Write("gray.npy", gray); 52 | output.Close(); 53 | 54 | // and we can read them back out again 55 | NPZInputStream input = new NPZInputStream("test.npz"); 56 | 57 | // we can check of an archive contains a file 58 | if (input.Contains("color.npy")) 59 | { 60 | // and peek at its header 61 | header = input.Peek("color.npy"); 62 | } 63 | 64 | color = input.ReadUInt8("color.npy"); 65 | gray = input.ReadFloat32("gray.npy"); 66 | input.Close(); 67 | } 68 | } -------------------------------------------------------------------------------- /samples/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | matplotlib -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set( SOURCES 2 | dtype.cpp 3 | npy.cpp 4 | npz.cpp 5 | tensor.cpp 6 | zip.cpp 7 | miniz/miniz.cpp 8 | ) 9 | 10 | add_definitions( -DLIBNPY_VERSION=${LIBNPY_VERSION} ) 11 | 12 | add_library( npy STATIC ${SOURCES} ) 13 | add_library( npy::npy ALIAS npy ) 14 | 15 | if (LIBNPY_SANITIZE) 16 | target_compile_options(npy PUBLIC -g -fsanitize=${REGOCPP_SANITIZE} -fno-omit-frame-pointer) 17 | target_link_libraries(npy PUBLIC -fsanitize=${REGOCPP_SANITIZE}) 18 | endif() 19 | 20 | if ( CMAKE_COMPILER_IS_GNUCC ) 21 | target_compile_options(npy PRIVATE "-Wall" "-Wextra") 22 | elseif( MSVC ) 23 | target_compile_options(npy PRIVATE "/W4") 24 | endif() 25 | -------------------------------------------------------------------------------- /src/dtype.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "npy/core.h" 6 | 7 | namespace { 8 | std::array BIG_ENDIAN_DTYPES = { 9 | "|i1", "|u1", ">i2", ">u2", ">i4", ">u4", ">i8", ">u8", ">f4", ">f8"}; 10 | 11 | std::array LITTLE_ENDIAN_DTYPES = { 12 | "|i1", "|u1", "> DTYPE_MAP = { 15 | {"|u1", {npy::data_type_t::UINT8, npy::endian_t::NATIVE}}, 16 | {"|i1", {npy::data_type_t::INT8, npy::endian_t::NATIVE}}, 17 | {"u2", {npy::data_type_t::UINT16, npy::endian_t::BIG}}, 19 | {"i2", {npy::data_type_t::INT16, npy::endian_t::BIG}}, 21 | {"u4", {npy::data_type_t::UINT32, npy::endian_t::BIG}}, 23 | {"i4", {npy::data_type_t::INT32, npy::endian_t::BIG}}, 25 | {"u8", {npy::data_type_t::UINT64, npy::endian_t::BIG}}, 27 | {"i8", {npy::data_type_t::INT64, npy::endian_t::BIG}}, 29 | {"f4", {npy::data_type_t::FLOAT32, npy::endian_t::BIG}}, 31 | {"f8", {npy::data_type_t::FLOAT64, npy::endian_t::BIG}}, 33 | }; 34 | } // namespace 35 | 36 | namespace npy { 37 | const std::string &to_dtype(data_type_t dtype, endian_t endianness) { 38 | if (endianness == npy::endian_t::NATIVE) { 39 | endianness = native_endian(); 40 | } 41 | 42 | if (endianness == npy::endian_t::BIG) { 43 | return BIG_ENDIAN_DTYPES[static_cast(dtype)]; 44 | } 45 | 46 | return LITTLE_ENDIAN_DTYPES[static_cast(dtype)]; 47 | } 48 | 49 | const std::pair &from_dtype(const std::string &dtype) { 50 | return DTYPE_MAP[dtype]; 51 | } 52 | 53 | std::ostream &operator<<(std::ostream &os, const data_type_t &value) { 54 | os << static_cast(value); 55 | return os; 56 | } 57 | 58 | std::ostream &operator<<(std::ostream &os, const endian_t &value) { 59 | os << static_cast(value); 60 | return os; 61 | } 62 | 63 | } // namespace npy -------------------------------------------------------------------------------- /src/npy.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "npy/npy.h" 8 | 9 | namespace { 10 | const int BUFFER_SIZE = 64 * 1024; 11 | char BUFFER[BUFFER_SIZE]; 12 | 13 | void read(std::istream &input, char expected) { 14 | char actual; 15 | input.get(actual); 16 | assert(actual == expected); 17 | } 18 | 19 | void read(std::istream &input, const std::string &expected) { 20 | input.read(BUFFER, expected.length()); 21 | std::string actual(BUFFER, BUFFER + expected.length()); 22 | assert(actual == expected); 23 | } 24 | 25 | void skip_whitespace(std::istream &input) { 26 | char skip; 27 | while (std::isspace(input.peek())) { 28 | input.get(skip); 29 | } 30 | } 31 | 32 | std::string read_to(std::istream &input, char delim) { 33 | if (input.peek() == delim) { 34 | return ""; 35 | } 36 | 37 | input.get(BUFFER, BUFFER_SIZE, delim); 38 | auto length = input.gcount(); 39 | assert(length < BUFFER_SIZE); 40 | 41 | return std::string(BUFFER, BUFFER + length); 42 | } 43 | 44 | std::string read_string(std::istream &input) { 45 | read(input, '\''); 46 | std::string token = read_to(input, '\''); 47 | read(input, '\''); 48 | return token; 49 | } 50 | 51 | bool read_bool(std::istream &input) { 52 | if (input.peek() == 'T') { 53 | read(input, "True"); 54 | return true; 55 | } else if (input.peek() == 'F') { 56 | read(input, "False"); 57 | return false; 58 | } 59 | 60 | throw std::logic_error("Dictionary value is not a boolean"); 61 | } 62 | 63 | std::vector read_shape(std::istream &input) { 64 | read(input, '('); 65 | std::stringstream tuple(read_to(input, ')')); 66 | read(input, ')'); 67 | 68 | std::vector shape; 69 | size_t size; 70 | 71 | while (tuple >> size) { 72 | shape.push_back(size); 73 | if (tuple.peek() == ',') { 74 | read(tuple, ','); 75 | skip_whitespace(tuple); 76 | } 77 | } 78 | 79 | return shape; 80 | } 81 | } // namespace 82 | 83 | namespace npy { 84 | header_info::header_info(const std::string &dictionary) { 85 | std::istringstream input(dictionary); 86 | read(input, '{'); 87 | while (input.peek() != '}') { 88 | skip_whitespace(input); 89 | std::string key = read_string(input); 90 | skip_whitespace(input); 91 | read(input, ':'); 92 | skip_whitespace(input); 93 | if (key == "descr") { 94 | std::string dtype_code = read_string(input); 95 | if (dtype_code[1] == 'U') { 96 | this->dtype = npy::data_type_t::UNICODE_STRING; 97 | endianness = 98 | dtype_code[0] == '>' ? npy::endian_t::BIG : npy::endian_t::LITTLE; 99 | max_element_length = std::stoi(dtype_code.substr(2)); 100 | } else { 101 | std::tie(this->dtype, endianness) = from_dtype(dtype_code); 102 | max_element_length = 0; 103 | } 104 | 105 | } else if (key == "fortran_order") { 106 | fortran_order = read_bool(input); 107 | } else if (key == "shape") { 108 | shape = read_shape(input); 109 | } else { 110 | throw std::logic_error("Unsupported key: " + key); 111 | } 112 | 113 | read(input, ','); 114 | skip_whitespace(input); 115 | } 116 | 117 | read(input, '}'); 118 | } 119 | 120 | header_info::header_info(data_type_t dtype, npy::endian_t endianness, 121 | bool fortran_order, const std::vector &shape) { 122 | this->dtype = dtype; 123 | this->endianness = endianness; 124 | this->fortran_order = fortran_order; 125 | this->shape = shape; 126 | } 127 | 128 | header_info peek(const std::string &path) { 129 | std::ifstream input(path, std::ios::in | std::ios::binary); 130 | if (!input.is_open()) { 131 | throw std::invalid_argument("path"); 132 | } 133 | 134 | return peek(input); 135 | } 136 | 137 | } // namespace npy -------------------------------------------------------------------------------- /src/npz.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "npy/npz.h" 9 | #include "zip.h" 10 | 11 | namespace { 12 | const std::array LOCAL_HEADER_SIG = {0x50, 0x4B, 0x03, 0x04}; 13 | const std::array CD_HEADER_SIG = {0x50, 0x4B, 0x01, 0x02}; 14 | const std::array CD_END_SIG = {0x50, 0x4B, 0x05, 0x06}; 15 | 16 | const std::array EXTERNAL_ATTR = {0x00, 0x00, 0x80, 0x01}; 17 | const std::array TIME = {0x00, 0x00, 0x21, 0x00}; 18 | const int CD_END_SIZE = 22; 19 | const std::uint16_t STANDARD_VERSION = 20 | 20; // 2.0 File is encrypted using traditional PKWARE encryption 21 | const std::uint16_t ZIP64_VERSION = 45; // 4.5 File uses ZIP64 format extensions 22 | 23 | const std::uint16_t ZIP64_TAG = 1; 24 | const std::uint64_t ZIP64_LIMIT = 0x8FFFFFFF; 25 | const std::uint32_t ZIP64_PLACEHOLDER = 0xFFFFFFFF; 26 | 27 | void write(std::ostream &stream, std::uint16_t value) { 28 | stream.put(value & 0x00FF); 29 | stream.put(value >> 8); 30 | } 31 | 32 | void write(std::ostream &stream, std::uint32_t value) { 33 | for (int i = 0; i < 4; ++i) { 34 | stream.put(value & 0x000000FF); 35 | value >>= 8; 36 | } 37 | } 38 | 39 | void write32(std::ostream &stream, std::uint64_t value) { 40 | if (value > ZIP64_LIMIT) { 41 | write(stream, ZIP64_PLACEHOLDER); 42 | } else { 43 | write(stream, static_cast(value)); 44 | } 45 | } 46 | 47 | void write(std::ostream &stream, std::uint64_t value) { 48 | for (int i = 0; i < 8; ++i) { 49 | stream.put(value & 0x00000000000000FF); 50 | value >>= 8; 51 | } 52 | } 53 | 54 | std::uint16_t read16(std::istream &stream) { 55 | std::uint16_t low = static_cast(stream.get()); 56 | std::uint16_t high = static_cast(stream.get()); 57 | return low | (high << 8); 58 | } 59 | 60 | std::uint32_t read32(std::istream &stream) { 61 | std::uint32_t result = 0; 62 | int shift = 0; 63 | for (int i = 0; i < 4; ++i, shift += 8) { 64 | std::uint32_t part = stream.get(); 65 | result |= part << shift; 66 | } 67 | 68 | return result; 69 | } 70 | 71 | std::uint64_t read64(std::istream &stream) { 72 | std::uint64_t result = 0; 73 | int shift = 0; 74 | for (int i = 0; i < 8; ++i, shift += 8) { 75 | std::uint64_t part = stream.get(); 76 | result |= part << shift; 77 | } 78 | 79 | return result; 80 | } 81 | 82 | void assert_sig(std::istream &stream, 83 | const std::array &expected) { 84 | std::array actual; 85 | stream.read(reinterpret_cast(actual.data()), actual.size()); 86 | if (actual != expected) { 87 | throw std::logic_error("Invalid signature (Not a valid NPZ file)"); 88 | } 89 | } 90 | 91 | std::uint16_t determine_extra_length(const npy::file_entry &header, 92 | bool include_offset) { 93 | std::uint16_t length = 0; 94 | if (header.compressed_size > ZIP64_LIMIT) { 95 | length += 8; 96 | } 97 | 98 | if (header.uncompressed_size > ZIP64_LIMIT) { 99 | length += 8; 100 | } 101 | 102 | if (include_offset && header.offset > ZIP64_LIMIT) { 103 | length += 8; 104 | } 105 | 106 | return length; 107 | } 108 | 109 | void write_zip64_extra(std::ostream &stream, const npy::file_entry &header, 110 | bool include_offset) { 111 | std::vector extra; 112 | if (header.uncompressed_size > ZIP64_LIMIT) { 113 | extra.push_back(header.uncompressed_size); 114 | } 115 | 116 | if (header.compressed_size > ZIP64_LIMIT) { 117 | extra.push_back(header.compressed_size); 118 | } 119 | 120 | if (include_offset && header.offset > ZIP64_LIMIT) { 121 | extra.push_back(header.offset); 122 | } 123 | 124 | write(stream, ZIP64_TAG); 125 | write(stream, static_cast(extra.size() * 8)); 126 | for (auto val : extra) { 127 | write(stream, val); 128 | } 129 | } 130 | 131 | void read_zip64_extra(std::istream &stream, npy::file_entry &header, 132 | bool include_offset) { 133 | std::uint16_t tag = read16(stream); 134 | if (tag != ZIP64_TAG) { 135 | throw std::logic_error("Invalid tag (expected ZIP64)"); 136 | } 137 | 138 | std::uint16_t actual_size = read16(stream); 139 | std::uint16_t expected_size = 0; 140 | 141 | if (header.uncompressed_size == ZIP64_PLACEHOLDER) { 142 | header.uncompressed_size = read64(stream); 143 | expected_size += 8; 144 | } 145 | 146 | if (header.compressed_size == ZIP64_PLACEHOLDER) { 147 | header.compressed_size = read64(stream); 148 | expected_size += 8; 149 | } 150 | 151 | if (include_offset && header.offset == ZIP64_PLACEHOLDER) { 152 | header.offset = read64(stream); 153 | expected_size += 8; 154 | } 155 | 156 | if (actual_size < expected_size) { 157 | throw std::logic_error("ZIP64 extra info missing"); 158 | } 159 | 160 | if (actual_size > expected_size) { 161 | // this can be the result of force_zip64 being set in Python's zipfile 162 | stream.seekg(actual_size - expected_size, std::ios::cur); 163 | } 164 | } 165 | 166 | void write_shared_header(std::ostream &stream, const npy::file_entry &header) { 167 | std::uint16_t general_purpose_big_flag = 0; 168 | write(stream, general_purpose_big_flag); 169 | write(stream, header.compression_method); 170 | stream.write(reinterpret_cast(TIME.data()), TIME.size()); 171 | write(stream, header.crc32); 172 | write32(stream, header.compressed_size); 173 | write32(stream, header.uncompressed_size); 174 | write(stream, static_cast(header.filename.length())); 175 | } 176 | 177 | std::uint16_t read_shared_header(std::istream &stream, 178 | npy::file_entry &header) { 179 | read16(stream); // general purpose bit flag 180 | header.compression_method = read16(stream); 181 | read32(stream); // time 182 | header.crc32 = read32(stream); 183 | header.compressed_size = read32(stream); 184 | header.uncompressed_size = read32(stream); 185 | return read16(stream); 186 | } 187 | 188 | void write_local_header(std::ostream &stream, const npy::file_entry &header, 189 | bool zip64) { 190 | stream.write(reinterpret_cast(LOCAL_HEADER_SIG.data()), 191 | LOCAL_HEADER_SIG.size()); 192 | write(stream, zip64 ? ZIP64_VERSION : STANDARD_VERSION); 193 | write_shared_header(stream, header); 194 | std::uint16_t extra_field_length = determine_extra_length(header, false); 195 | write(stream, extra_field_length); 196 | stream.write(header.filename.data(), header.filename.length()); 197 | if (extra_field_length > 0) { 198 | write_zip64_extra(stream, header, false); 199 | } 200 | } 201 | 202 | npy::file_entry read_local_header(std::istream &stream) { 203 | assert_sig(stream, LOCAL_HEADER_SIG); 204 | std::uint16_t version = read16(stream); 205 | if (version > ZIP64_VERSION) { 206 | throw std::logic_error("Unsupported NPZ version"); 207 | } 208 | 209 | npy::file_entry entry; 210 | 211 | std::uint16_t filename_length = read_shared_header(stream, entry); 212 | std::uint16_t extra_field_length = read16(stream); 213 | std::vector buffer(filename_length); 214 | stream.read(buffer.data(), filename_length); 215 | entry.filename = std::string(buffer.begin(), buffer.end()); 216 | 217 | if (extra_field_length > 0) { 218 | read_zip64_extra(stream, entry, false); 219 | } 220 | 221 | return entry; 222 | } 223 | 224 | void write_central_directory_header(std::ostream &stream, 225 | const npy::file_entry &header) { 226 | std::uint16_t extra_field_length = determine_extra_length(header, true); 227 | stream.write(reinterpret_cast(CD_HEADER_SIG.data()), 228 | CD_HEADER_SIG.size()); 229 | write(stream, STANDARD_VERSION); 230 | write(stream, extra_field_length > 0 ? ZIP64_VERSION : STANDARD_VERSION); 231 | write_shared_header(stream, header); 232 | write(stream, extra_field_length); 233 | std::uint16_t file_comment_length = 0; 234 | write(stream, file_comment_length); 235 | std::uint16_t disk_number_start = 0; 236 | write(stream, disk_number_start); 237 | std::uint16_t internal_file_attributes = 0; 238 | write(stream, internal_file_attributes); 239 | stream.write(reinterpret_cast(EXTERNAL_ATTR.data()), 240 | EXTERNAL_ATTR.size()); 241 | write32(stream, header.offset); 242 | stream.write(header.filename.data(), header.filename.length()); 243 | if (extra_field_length > 0) { 244 | write_zip64_extra(stream, header, true); 245 | } 246 | } 247 | 248 | npy::file_entry read_central_directory_header(std::istream &stream) { 249 | assert_sig(stream, CD_HEADER_SIG); 250 | read16(stream); // version made by 251 | std::uint16_t version = read16(stream); 252 | if (version > ZIP64_VERSION) { 253 | throw std::logic_error("Unsupported NPZ version"); 254 | } 255 | 256 | npy::file_entry entry; 257 | std::uint16_t filename_length = read_shared_header(stream, entry); 258 | std::uint16_t extra_field_length = read16(stream); 259 | read16(stream); // file comment length 260 | read16(stream); // disk number start 261 | read16(stream); // internal file attributes 262 | read32(stream); // external file attributes 263 | entry.offset = read32(stream); 264 | 265 | std::vector buffer(filename_length); 266 | stream.read(buffer.data(), filename_length); 267 | entry.filename = std::string(buffer.begin(), buffer.end()); 268 | 269 | if (extra_field_length > 0) { 270 | read_zip64_extra(stream, entry, true); 271 | } 272 | 273 | return entry; 274 | } 275 | 276 | struct CentralDirectory { 277 | std::uint16_t num_entries; 278 | std::uint32_t size; 279 | std::uint32_t offset; 280 | }; 281 | 282 | void write_end_of_central_directory(std::ostream &stream, 283 | const CentralDirectory &dir) { 284 | stream.write(reinterpret_cast(CD_END_SIG.data()), 285 | CD_END_SIG.size()); 286 | uint16_t disk_number = 0; 287 | write(stream, disk_number); 288 | write(stream, disk_number); 289 | write(stream, dir.num_entries); 290 | write(stream, dir.num_entries); 291 | write(stream, dir.size); 292 | write(stream, dir.offset); 293 | std::uint16_t file_comment_length = 0; 294 | write(stream, file_comment_length); 295 | } 296 | 297 | CentralDirectory read_end_of_central_directory(std::istream &stream) { 298 | assert_sig(stream, CD_END_SIG); 299 | 300 | CentralDirectory result; 301 | read16(stream); // number of this disk 302 | read16(stream); // number of the disk with the start of the central directory 303 | result.num_entries = read16(stream); 304 | read16(stream); // num_entries_on_disk 305 | result.size = read32(stream); 306 | result.offset = read32(stream); 307 | return result; 308 | } 309 | 310 | } // namespace 311 | 312 | namespace npy { 313 | bool file_entry::check(const file_entry &other) const { 314 | return !(other.filename != this->filename || other.crc32 != this->crc32 || 315 | other.compression_method != this->compression_method || 316 | other.compressed_size != this->compressed_size || 317 | other.uncompressed_size != this->uncompressed_size); 318 | } 319 | 320 | onpzstream::onpzstream(const std::shared_ptr &output, 321 | compression_method_t compression, endian_t endianness) 322 | : m_closed(false), m_output(output), m_compression_method(compression), 323 | m_endianness(endianness) {} 324 | 325 | onpzstream::onpzstream(const std::string &path, compression_method_t method, 326 | endian_t endianness) 327 | : m_closed(false), m_output(std::make_shared( 328 | path, std::ios::out | std::ios::binary)), 329 | m_compression_method(method), m_endianness(endianness) {} 330 | 331 | onpzstream::~onpzstream() { 332 | if (!m_closed) { 333 | close(); 334 | } 335 | } 336 | 337 | void onpzstream::write_file(const std::string &filename, std::string &&bytes) { 338 | std::uint32_t uncompressed_size = static_cast(bytes.size()); 339 | std::uint32_t compressed_size = 0; 340 | std::string compressed_bytes; 341 | std::uint32_t checksum = npy_crc32(bytes); 342 | if (m_compression_method == compression_method_t::STORED) { 343 | compressed_bytes = bytes; 344 | compressed_size = uncompressed_size; 345 | } else if (m_compression_method == compression_method_t::DEFLATED) { 346 | compressed_bytes = npy_deflate(std::move(bytes)); 347 | compressed_size = static_cast(compressed_bytes.size()); 348 | } else { 349 | throw std::invalid_argument("m_compression_method"); 350 | } 351 | 352 | file_entry entry = {filename, 353 | checksum, 354 | compressed_size, 355 | uncompressed_size, 356 | static_cast(m_compression_method), 357 | static_cast(m_output->tellp())}; 358 | 359 | bool zip64 = uncompressed_size > ZIP64_LIMIT || compressed_size > ZIP64_LIMIT; 360 | write_local_header(*m_output, entry, zip64); 361 | m_output->write(compressed_bytes.data(), compressed_size); 362 | m_entries.push_back(std::move(entry)); 363 | } 364 | 365 | bool onpzstream::is_open() const { 366 | std::shared_ptr output = 367 | std::dynamic_pointer_cast(m_output); 368 | if (output) { 369 | return output->is_open(); 370 | } 371 | 372 | return true; 373 | } 374 | 375 | void onpzstream::close() { 376 | if (!m_closed) { 377 | CentralDirectory dir; 378 | dir.offset = static_cast(m_output->tellp()); 379 | for (auto &header : m_entries) { 380 | write_central_directory_header(*m_output, header); 381 | } 382 | 383 | dir.size = static_cast(m_output->tellp()) - dir.offset; 384 | dir.num_entries = static_cast(m_entries.size()); 385 | write_end_of_central_directory(*m_output, dir); 386 | 387 | std::shared_ptr output = 388 | std::dynamic_pointer_cast(m_output); 389 | if (output) { 390 | output->close(); 391 | } 392 | 393 | m_closed = true; 394 | } 395 | } 396 | 397 | inpzstream::inpzstream(const std::shared_ptr &stream) 398 | : m_input(stream) { 399 | read_entries(); 400 | } 401 | 402 | inpzstream::inpzstream(const std::string &path) { 403 | m_input = 404 | std::make_shared(path, std::ios::in | std::ios::binary); 405 | read_entries(); 406 | } 407 | 408 | void inpzstream::read_entries() { 409 | m_input->seekg(-CD_END_SIZE, std::ios::end); 410 | CentralDirectory dir = read_end_of_central_directory(*m_input); 411 | 412 | m_input->seekg(dir.offset, std::ios::beg); 413 | 414 | for (size_t i = 0; i < dir.num_entries; ++i) { 415 | file_entry entry = read_central_directory_header(*m_input); 416 | m_entries[entry.filename] = entry; 417 | m_keys.push_back(entry.filename); 418 | } 419 | 420 | std::sort(m_keys.begin(), m_keys.end()); 421 | } 422 | 423 | const std::vector &inpzstream::keys() const { return m_keys; } 424 | 425 | std::string inpzstream::read_file(const std::string &temp_filename) { 426 | std::string filename = temp_filename; 427 | if (m_entries.count(filename) == 0) { 428 | filename += ".npy"; 429 | if (m_entries.count(filename) == 0) { 430 | throw std::invalid_argument("filename"); 431 | } 432 | } 433 | 434 | const file_entry &entry = m_entries[filename]; 435 | m_input->seekg(entry.offset, std::ios::beg); 436 | 437 | file_entry local = read_local_header(*m_input); 438 | if (!entry.check(local)) { 439 | throw std::logic_error("Central directory and local headers disagree"); 440 | } 441 | 442 | std::string uncompressed_bytes; 443 | uncompressed_bytes.resize(entry.compressed_size); 444 | m_input->read(reinterpret_cast(uncompressed_bytes.data()), 445 | uncompressed_bytes.size()); 446 | compression_method_t cmethod = 447 | static_cast(entry.compression_method); 448 | if (cmethod == compression_method_t::DEFLATED) { 449 | uncompressed_bytes = npy_inflate(std::move(uncompressed_bytes)); 450 | } 451 | 452 | std::uint32_t actual_crc32 = npy_crc32(uncompressed_bytes); 453 | if (actual_crc32 != entry.crc32) { 454 | throw std::logic_error("CRC mismatch"); 455 | } 456 | 457 | return uncompressed_bytes; 458 | } 459 | 460 | bool inpzstream::is_open() const { 461 | std::shared_ptr input = 462 | std::dynamic_pointer_cast(m_input); 463 | if (input) { 464 | return input->is_open(); 465 | } 466 | 467 | return true; 468 | } 469 | 470 | void inpzstream::close() { 471 | std::shared_ptr input = 472 | std::dynamic_pointer_cast(m_input); 473 | if (input) { 474 | input->close(); 475 | } 476 | } 477 | 478 | bool inpzstream::contains(const std::string &filename) { 479 | return m_entries.count(filename); 480 | } 481 | 482 | header_info inpzstream::peek(const std::string &filename) { 483 | imemstream stream(read_file(filename)); 484 | return npy::peek(stream); 485 | } 486 | } // namespace npy -------------------------------------------------------------------------------- /src/tensor.cpp: -------------------------------------------------------------------------------- 1 | #include "npy/tensor.h" 2 | 3 | namespace npy { 4 | template <> data_type_t tensor::get_dtype() { 5 | return data_type_t::INT8; 6 | }; 7 | 8 | template <> data_type_t tensor::get_dtype() { 9 | return data_type_t::UINT8; 10 | }; 11 | 12 | template <> data_type_t tensor::get_dtype() { 13 | return data_type_t::INT16; 14 | }; 15 | 16 | template <> data_type_t tensor::get_dtype() { 17 | return data_type_t::UINT16; 18 | }; 19 | 20 | template <> data_type_t tensor::get_dtype() { 21 | return data_type_t::INT32; 22 | }; 23 | 24 | template <> data_type_t tensor::get_dtype() { 25 | return data_type_t::UINT32; 26 | }; 27 | 28 | template <> data_type_t tensor::get_dtype() { 29 | return data_type_t::INT64; 30 | }; 31 | 32 | template <> data_type_t tensor::get_dtype() { 33 | return data_type_t::UINT64; 34 | }; 35 | 36 | template <> data_type_t tensor::get_dtype() { 37 | return data_type_t::FLOAT32; 38 | }; 39 | 40 | template <> data_type_t tensor::get_dtype() { 41 | return data_type_t::FLOAT64; 42 | }; 43 | 44 | template <> data_type_t tensor::get_dtype() { 45 | return data_type_t::UNICODE_STRING; 46 | } 47 | 48 | } // namespace npy -------------------------------------------------------------------------------- /src/zip.cpp: -------------------------------------------------------------------------------- 1 | #include "miniz/miniz.h" 2 | #include 3 | #include 4 | 5 | #include "npy/core.h" 6 | #include "zip.h" 7 | 8 | namespace { 9 | const size_t CHUNK = 1024 * 1024; 10 | const int WINDOW_BITS = -15; 11 | const int MEM_LEVEL = 8; 12 | } // namespace 13 | 14 | namespace npy { 15 | 16 | std::uint32_t npy_crc32(const std::string &bytes) { 17 | uLong crc = ::crc32(0L, Z_NULL, 0); 18 | const Bytef *buf = reinterpret_cast(bytes.data()); 19 | uInt len = static_cast(bytes.size()); 20 | return ::crc32(crc, buf, len); 21 | } 22 | 23 | std::string npy_deflate(std::string &&bytes) { 24 | int ret, flush; 25 | unsigned have; 26 | z_stream strm; 27 | std::vector in(CHUNK); 28 | std::vector out(CHUNK); 29 | 30 | /* allocate deflate state */ 31 | strm.zalloc = Z_NULL; 32 | strm.zfree = Z_NULL; 33 | strm.opaque = Z_NULL; 34 | ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, WINDOW_BITS, 35 | MEM_LEVEL, Z_DEFAULT_STRATEGY); 36 | if (ret != Z_OK) { 37 | throw std::logic_error("Unable to initialize deflate algorithm"); 38 | } 39 | 40 | std::string str(bytes.begin(), bytes.end()); 41 | imemstream input(std::move(str)); 42 | omemstream output; 43 | 44 | /* compress until end of file */ 45 | do { 46 | input.read(reinterpret_cast(in.data()), CHUNK); 47 | strm.avail_in = static_cast(input.gcount()); 48 | if (input.eof()) { 49 | flush = Z_FINISH; 50 | } else { 51 | if (input.fail() || input.bad()) { 52 | (void)deflateEnd(&strm); 53 | throw std::logic_error("Error reading from input stream"); 54 | } 55 | 56 | flush = Z_NO_FLUSH; 57 | } 58 | 59 | strm.next_in = in.data(); 60 | 61 | /* run deflate() on input until output buffer not full, finish 62 | compression if all of source has been read in */ 63 | do { 64 | strm.avail_out = CHUNK; 65 | strm.next_out = out.data(); 66 | ret = deflate(&strm, flush); 67 | assert(ret != Z_STREAM_ERROR); /* state not clobbered */ 68 | have = CHUNK - strm.avail_out; 69 | output.write(reinterpret_cast(out.data()), have); 70 | if (output.fail() || output.bad()) { 71 | (void)deflateEnd(&strm); 72 | throw std::logic_error("Error writing to output stream"); 73 | } 74 | } while (strm.avail_out == 0); 75 | assert(strm.avail_in == 0); /* all input will be used */ 76 | /* done when last data in file processed */ 77 | } while (flush != Z_FINISH); 78 | assert(ret == Z_STREAM_END); /* stream will be complete */ 79 | 80 | /* clean up and return */ 81 | (void)deflateEnd(&strm); 82 | 83 | return output.str(); 84 | } 85 | 86 | std::string npy_inflate(std::string &&bytes) { 87 | int ret; 88 | unsigned have; 89 | z_stream strm; 90 | std::vector in(CHUNK); 91 | std::vector out(CHUNK); 92 | /* allocate inflate state */ 93 | strm.zalloc = Z_NULL; 94 | strm.zfree = Z_NULL; 95 | strm.opaque = Z_NULL; 96 | strm.avail_in = 0; 97 | strm.next_in = Z_NULL; 98 | ret = inflateInit2(&strm, WINDOW_BITS); 99 | if (ret != Z_OK) { 100 | throw std::logic_error("Unable to initialize inflate algorithm"); 101 | } 102 | 103 | std::string str(bytes.begin(), bytes.end()); 104 | imemstream input(std::move(str)); 105 | omemstream output; 106 | 107 | /* decompress until deflate stream ends or end of file */ 108 | do { 109 | input.read(reinterpret_cast(in.data()), CHUNK); 110 | strm.avail_in = static_cast(input.gcount()); 111 | if ((input.fail() && !input.eof()) || input.bad()) { 112 | (void)inflateEnd(&strm); 113 | throw std::logic_error("Error reading from input stream"); 114 | } 115 | 116 | if (strm.avail_in == 0) { 117 | break; 118 | } 119 | 120 | strm.next_in = in.data(); 121 | 122 | /* run inflate() on input until output buffer not full */ 123 | do { 124 | strm.avail_out = CHUNK; 125 | strm.next_out = out.data(); 126 | ret = inflate(&strm, Z_NO_FLUSH); 127 | assert(ret != Z_STREAM_ERROR); /* state not clobbered */ 128 | switch (ret) { 129 | case Z_NEED_DICT: 130 | ret = Z_DATA_ERROR; /* and fall through */ 131 | case Z_DATA_ERROR: 132 | case Z_MEM_ERROR: 133 | (void)inflateEnd(&strm); 134 | throw std::logic_error("Error inflating stream"); 135 | } 136 | 137 | have = CHUNK - strm.avail_out; 138 | output.write(reinterpret_cast(out.data()), have); 139 | if (output.fail() || output.bad()) { 140 | (void)inflateEnd(&strm); 141 | throw std::logic_error("Error writing to output stream"); 142 | } 143 | } while (strm.avail_out == 0); 144 | 145 | /* done when inflate() says it's done */ 146 | } while (ret != Z_STREAM_END); 147 | 148 | /* clean up and return */ 149 | (void)inflateEnd(&strm); 150 | 151 | if (ret == Z_STREAM_END) { 152 | return output.str(); 153 | } 154 | 155 | throw std::logic_error("Error inflating stream"); 156 | } 157 | 158 | } // namespace npy -------------------------------------------------------------------------------- /src/zip.h: -------------------------------------------------------------------------------- 1 | // ---------------------------------------------------------------------------- 2 | // 3 | // zip.h -- simple wrapper around miniz 4 | // 5 | // Copyright (C) 2021 Matthew Johnson 6 | // 7 | // For conditions of distribution and use, see copyright notice in LICENSE 8 | // 9 | // ---------------------------------------------------------------------------- 10 | 11 | #ifndef _ZIP_H_ 12 | #define _ZIP_H_ 13 | 14 | #include 15 | #include 16 | 17 | namespace npy { 18 | /** Deflate the bytes and return the compressed result. 19 | * \param bytes the raw bytes 20 | * \return the compressed bytes 21 | */ 22 | std::string npy_deflate(std::string &&bytes); 23 | 24 | /** Inflate the bytes and return the decompressed result. 25 | * \param bytes the compressed bytes 26 | * \return the raw bytes 27 | */ 28 | std::string npy_inflate(std::string &&bytes); 29 | 30 | /** Perform a fast CRC32 checksum of a set of bytes. 31 | * \param bytes the bytes to check 32 | * \return the CRC32 checksum 33 | */ 34 | std::uint32_t npy_crc32(const std::string &bytes); 35 | } // namespace npy 36 | 37 | #endif -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set( CPP_TEST_SOURCES 2 | libnpy_tests.cpp 3 | libnpy_tests.h 4 | ) 5 | 6 | set( TESTS 7 | crc32 8 | exceptions 9 | memstream 10 | npy_peek 11 | npy_read 12 | npy_write 13 | npz_peek 14 | npz_read 15 | npz_write 16 | tensor 17 | ) 18 | 19 | foreach( test ${TESTS} ) 20 | list( APPEND CPP_TEST_SOURCES "${test}.cpp" ) 21 | endforeach() 22 | 23 | set( TEST_DRIVER libnpy_tests ) 24 | add_executable( ${TEST_DRIVER} ${CPP_TEST_SOURCES} ) 25 | target_link_libraries( ${TEST_DRIVER} npy::npy ) 26 | target_include_directories( ${TEST_DRIVER} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../src) 27 | 28 | foreach( test ${TESTS} ) 29 | add_test( NAME ${test} 30 | COMMAND ${TEST_DRIVER} ${test} 31 | WORKING_DIRECTORY ${CMAKE_HOME_DIRECTORY} ) 32 | endforeach() 33 | 34 | 35 | if( INCLUDE_CSHARP ) 36 | add_subdirectory( CSharpTests ) 37 | endif() 38 | -------------------------------------------------------------------------------- /test/CSharpTests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set( CSHARP_TEST_SOURCES 2 | test_exceptions.cs 3 | test_npy_peek.cs 4 | test_npy_read.cs 5 | test_npy_write.cs 6 | test_npz_peek.cs 7 | test_npz_read.cs 8 | test_npz_write.cs 9 | ) 10 | 11 | if( WIN32 ) 12 | set( TEST_FOLDER ${CMAKE_CURRENT_BINARY_DIR}/$ ) 13 | else() 14 | set( TEST_FOLDER ${CMAKE_CURRENT_BINARY_DIR} ) 15 | endif() 16 | 17 | foreach( file ${CSHARP_TEST_SOURCES} ) 18 | get_filename_component( test_name "${file}" NAME_WE ) 19 | set( test_name "CSharpTests_${test_name}" ) 20 | 21 | add_executable( ${test_name} ${file} Test.cs ) 22 | target_link_libraries( ${test_name} NumpyIO ${SWIG_MODULE_NumpyIONative_REAL_NAME} ) 23 | 24 | # copy the NumpyIO dynamic libraries to the test build folder 25 | add_custom_command( TARGET ${test_name} POST_BUILD COMMAND ${CMAKE_COMMAND} ARGS -E copy_if_different ${LIBNPY_CSHARP_DIR}/NumpyIO.dll ${TEST_FOLDER}/NumpyIO.dll ) 26 | add_custom_command( TARGET ${test_name} POST_BUILD COMMAND ${CMAKE_COMMAND} ARGS -E copy_if_different ${LIBNPY_CSHARP_DIR}/${NUMPYIO_NATIVE} ${TEST_FOLDER}/${NUMPYIO_NATIVE} ) 27 | 28 | if( NOT WIN32 ) 29 | add_test( NAME ${test_name} 30 | COMMAND ${CSHARP_INTERPRETER} ${CMAKE_CURRENT_BINARY_DIR}/${test_name}.exe 31 | WORKING_DIRECTORY ${CMAKE_HOME_DIRECTORY} ) 32 | else() 33 | add_test( NAME ${test_name} 34 | COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${test_name}.exe 35 | WORKING_DIRECTORY ${CMAKE_HOME_DIRECTORY} ) 36 | endif() 37 | 38 | endforeach() -------------------------------------------------------------------------------- /test/CSharpTests/test.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Runtime.InteropServices; 5 | using System.Linq; 6 | using NumpyIO; 7 | 8 | namespace Testing 9 | { 10 | static class Test 11 | { 12 | public const int EXIT_SUCCESS = 0; 13 | public const int EXIT_FAILURE = 1; 14 | 15 | public static void AssertEqual(T expected, T actual, ref int result, string tag) 16 | { 17 | if (!actual.Equals(expected)) 18 | { 19 | Console.WriteLine("{0} is incorrect: {1} != {2}", tag, actual, expected); 20 | result = EXIT_FAILURE; 21 | } 22 | } 23 | 24 | public static void AssertEqual(B expected, B actual, ref int result, string tag) where B : IList 25 | { 26 | AssertEqual(expected.Count, actual.Count, ref result, tag + " Count"); 27 | if (result == EXIT_SUCCESS) 28 | { 29 | for (int i = 0; i < actual.Count; ++i) 30 | { 31 | AssertEqual(expected[i], actual[i], ref result, tag + "[" + i + "]"); 32 | if (result == EXIT_FAILURE) 33 | { 34 | break; 35 | } 36 | } 37 | } 38 | } 39 | 40 | public static void AssertEqual(Tensor expected, Tensor actual, ref int result, string tag) where B : IList 41 | { 42 | AssertEqual(expected.DataType, actual.DataType, ref result, tag + " DataType"); 43 | AssertEqual(expected.FortranOrder, actual.FortranOrder, ref result, tag + " FortranOrder"); 44 | AssertEqual>(expected.Shape.ToList(), actual.Shape.ToList(), ref result, tag + " Shape"); 45 | AssertEqual>(expected.Values, actual.Values, ref result, tag); 46 | } 47 | 48 | public static void AssertEqual(HeaderInfo expected, HeaderInfo actual, ref int result, string tag) 49 | { 50 | AssertEqual(expected.DataType, actual.DataType, ref result, tag + " DataType"); 51 | AssertEqual(expected.Endianness, actual.Endianness, ref result, tag + " Endianness"); 52 | AssertEqual(expected.FortranOrder, actual.FortranOrder, ref result, tag + " FortranOrder"); 53 | AssertEqual>(expected.Shape.ToList(), actual.Shape.ToList(), ref result, tag + " Shape"); 54 | } 55 | 56 | public static void AssertThrows(Action action, ref int result, string tag) where E:Exception 57 | { 58 | try 59 | { 60 | action(); 61 | result = EXIT_FAILURE; 62 | Console.WriteLine("{0} did not throw an exception", tag); 63 | } 64 | catch (E) 65 | { 66 | } 67 | catch(Exception e) 68 | { 69 | result = EXIT_FAILURE; 70 | Console.WriteLine("{0} threw an unexpected exception: {1}", tag, e); 71 | } 72 | } 73 | 74 | public static T Tensor(Shape shape) 75 | where B : IList 76 | where T : Tensor 77 | { 78 | T tensor = (T)Activator.CreateInstance(typeof(T), new object[] { shape }); 79 | for (int i = 0; i < tensor.Size; ++i) 80 | { 81 | tensor.Values[i] = (D)Convert.ChangeType(i, typeof(D)); 82 | } 83 | 84 | return tensor; 85 | } 86 | 87 | public static string AssetPath(string filename) 88 | { 89 | return Path.Combine("assets", "test", filename); 90 | } 91 | } 92 | } -------------------------------------------------------------------------------- /test/CSharpTests/test_exceptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using NumpyIO; 4 | 5 | namespace Testing 6 | { 7 | class TestExceptions 8 | { 9 | static UInt8Tensor TENSOR = new UInt8Tensor(new Shape(new uint[] { 5, 2, 5 })); 10 | static string TEMP_NPZ = "temp.npz"; 11 | 12 | static void PeekInvalidPath() 13 | { 14 | NumpyIO.NumpyIO.Peek(Path.Combine("does_not_exist", "bad.npy")); 15 | } 16 | 17 | static void SaveInvalidPath() 18 | { 19 | TENSOR.Save(Path.Combine("does_not_exist", "bad.npy")); 20 | } 21 | 22 | static void LoadInvalidPath() 23 | { 24 | var tensor = new UInt8Tensor(Path.Combine("does_not_exist", "bad.npy")); 25 | } 26 | 27 | static void NPZInputStreamInvalidPath() 28 | { 29 | var stream = new NPZInputStream(Path.Combine("does_not_exist", "bad.npz")); 30 | } 31 | 32 | static void NPZOutputStreamCompression() 33 | { 34 | CompressionMethod method = (CompressionMethod)99; 35 | using(var stream = new NPZOutputStream(TEMP_NPZ, method)) 36 | { 37 | stream.Write("error.npy", TENSOR); 38 | } 39 | } 40 | 41 | static void NPZInputStreamReadInvalidFilename() 42 | { 43 | using(var stream = new NPZInputStream(Test.AssetPath("test.npz"))) 44 | { 45 | var tensor = stream.ReadUInt8("not_there.npy"); 46 | } 47 | } 48 | 49 | static void NPZInputStreamPeekInvalidFilename() 50 | { 51 | using(var stream = new NPZInputStream(Test.AssetPath("test.npz"))) 52 | { 53 | var header = stream.Peek("not_there.npy"); 54 | } 55 | } 56 | 57 | static void TensorCopyFrom() 58 | { 59 | UInt8Buffer buffer = new UInt8Buffer(new byte[10]); 60 | TENSOR.CopyFrom(buffer); 61 | } 62 | 63 | static void TensorIndexSize() 64 | { 65 | byte value = TENSOR[0, 0]; 66 | } 67 | 68 | static void TensorIndexRange() 69 | { 70 | byte value = TENSOR[2, 3, 3]; 71 | } 72 | 73 | static void NPZOutputStreamClosed() 74 | { 75 | using(var stream = new NPZOutputStream(TEMP_NPZ)) 76 | { 77 | stream.Close(); 78 | stream.Write("error.npy", TENSOR); 79 | } 80 | } 81 | 82 | static void NPZInputStreamInvalidFile() 83 | { 84 | var stream = new NPZInputStream(Test.AssetPath("uint8.npy")); 85 | } 86 | 87 | public static int Main(string[] args) 88 | { 89 | int result = Test.EXIT_SUCCESS; 90 | 91 | Test.AssertThrows(PeekInvalidPath, ref result, "PeekInvalidPath"); 92 | Test.AssertThrows(SaveInvalidPath, ref result, "SaveInvalidPath"); 93 | Test.AssertThrows(LoadInvalidPath, ref result, "LoadInvalidPath"); 94 | Test.AssertThrows(NPZInputStreamInvalidPath, ref result, "NPZInputStreamInvalidPath"); 95 | Test.AssertThrows(NPZInputStreamReadInvalidFilename, ref result, "NPZInputStreamReadInvalidFilename"); 96 | Test.AssertThrows(NPZInputStreamPeekInvalidFilename, ref result, "NPZInputStreamPeekInvalidFilename"); 97 | Test.AssertThrows(NPZOutputStreamCompression, ref result, "NPZOutputStreamCompression"); 98 | Test.AssertThrows(TensorCopyFrom, ref result, "TensorCopyFrom"); 99 | Test.AssertThrows(TensorIndexSize, ref result, "TensorIndexSize"); 100 | 101 | Test.AssertThrows(TensorIndexRange, ref result, "TensorIndexRange"); 102 | 103 | Test.AssertThrows(NPZInputStreamInvalidFile, ref result, "NPZInputStreamInvalidFile"); 104 | Test.AssertThrows(NPZOutputStreamClosed, ref result, "NPZOutputStreamClosed"); 105 | 106 | File.Delete(TEMP_NPZ); 107 | 108 | return result; 109 | } 110 | } 111 | } -------------------------------------------------------------------------------- /test/CSharpTests/test_npy_peek.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using NumpyIO; 5 | 6 | namespace Testing 7 | { 8 | class TestNPYPeek 9 | { 10 | static void TestPeek(ref int result, string tag, DataType dtype) 11 | { 12 | TestPeek(ref result, tag, dtype, Endian.LITTLE, false); 13 | } 14 | 15 | static void TestPeek(ref int result, string tag, DataType dtype, Endian endianness) 16 | { 17 | TestPeek(ref result, tag, dtype, endianness, false); 18 | } 19 | 20 | static void TestPeek(ref int result, string tag, DataType dtype, bool fortranOrder) 21 | { 22 | TestPeek(ref result, tag, dtype, Endian.LITTLE, fortranOrder); 23 | } 24 | static void TestPeek(ref int result, string tag, DataType dtype, Endian endianness, bool fortranOrder) 25 | { 26 | Shape shape = new Shape(new uint[]{5, 2, 5}); 27 | HeaderInfo expected = new HeaderInfo(dtype, endianness, fortranOrder, shape); 28 | HeaderInfo actual = NumpyIO.NumpyIO.Peek(Test.AssetPath(tag + ".npy")); 29 | Test.AssertEqual(expected, actual, ref result, "c#_npy_peek_" + tag); 30 | } 31 | public static int Main() 32 | { 33 | int result = Test.EXIT_SUCCESS; 34 | 35 | TestPeek(ref result, "uint8", DataType.UINT8, Endian.NATIVE); 36 | TestPeek(ref result, "uint8_fortran", DataType.UINT8, Endian.NATIVE,true); 37 | TestPeek(ref result, "int8", DataType.INT8, Endian.NATIVE); 38 | TestPeek(ref result, "uint16", DataType.UINT16); 39 | TestPeek(ref result, "int16", DataType.INT16); 40 | TestPeek(ref result, "uint32", DataType.UINT32); 41 | TestPeek(ref result, "int32", DataType.INT32); 42 | TestPeek(ref result, "int32_big", DataType.INT32, Endian.BIG); 43 | TestPeek(ref result, "uint64", DataType.UINT64); 44 | TestPeek(ref result, "int64", DataType.INT64); 45 | TestPeek(ref result, "float32", DataType.FLOAT32); 46 | TestPeek(ref result, "float64", DataType.FLOAT64); 47 | 48 | return result; 49 | } 50 | } 51 | } -------------------------------------------------------------------------------- /test/CSharpTests/test_npy_read.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using NumpyIO; 5 | 6 | namespace Testing 7 | { 8 | class TestNPYRead 9 | { 10 | static void TestRead(ref int result, string tag) 11 | where B : IList 12 | where T : Tensor 13 | { 14 | Shape shape = new Shape(new uint[] { 5, 2, 5 }); 15 | T expected = Test.Tensor(shape); 16 | string path = Test.AssetPath(tag + ".npy"); 17 | T actual = (T)Activator.CreateInstance(typeof(T), new object[] { path }); 18 | Test.AssertEqual(expected, actual, ref result, "c#_npy_read_" + tag); 19 | } 20 | public static int Main() 21 | { 22 | int result = Test.EXIT_SUCCESS; 23 | 24 | TestRead(ref result, "uint8"); 25 | TestRead(ref result, "int8"); 26 | TestRead(ref result, "uint16"); 27 | TestRead(ref result, "int16"); 28 | TestRead(ref result, "uint32"); 29 | TestRead(ref result, "int32"); 30 | TestRead(ref result, "uint64"); 31 | TestRead(ref result, "int64"); 32 | 33 | return result; 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /test/CSharpTests/test_npy_write.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using NumpyIO; 5 | 6 | namespace Testing 7 | { 8 | class TestNPYWrite 9 | { 10 | static void TestWrite(ref int result, string tag) 11 | where B : IList 12 | where T : Tensor 13 | { 14 | Shape shape = new Shape(new uint[] { 5, 2, 5 }); 15 | T tensor = Test.Tensor(shape); 16 | string path = Path.GetRandomFileName(); 17 | tensor.Save(path); 18 | byte[] actual = File.ReadAllBytes(path); 19 | byte[] expected = File.ReadAllBytes(Test.AssetPath(tag + ".npy")); 20 | Test.AssertEqual(expected, actual, ref result, "c#_npy_write_" + tag); 21 | File.Delete(path); 22 | } 23 | public static int Main() 24 | { 25 | int result = Test.EXIT_SUCCESS; 26 | 27 | TestWrite(ref result, "uint8"); 28 | TestWrite(ref result, "int8"); 29 | TestWrite(ref result, "uint16"); 30 | TestWrite(ref result, "int16"); 31 | TestWrite(ref result, "uint32"); 32 | TestWrite(ref result, "int32"); 33 | TestWrite(ref result, "uint64"); 34 | TestWrite(ref result, "int64"); 35 | 36 | return result; 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /test/CSharpTests/test_npz_peek.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using NumpyIO; 5 | 6 | namespace Testing 7 | { 8 | class TestNPZPeek 9 | { 10 | static void TestPeek(bool compressed, ref int result) 11 | { 12 | HeaderInfo expectedColor = new HeaderInfo(DataType.UINT8, Endian.NATIVE, false, new Shape(new uint[] { 5, 5, 3 })); 13 | HeaderInfo expectedDepth = new HeaderInfo(DataType.FLOAT32, Endian.LITTLE, false, new Shape(new uint[] { 5, 5 })); 14 | 15 | string filename = compressed ? "test_compressed.npz" : "test.npz"; 16 | NPZInputStream stream = new NPZInputStream(Test.AssetPath(filename)); 17 | 18 | HeaderInfo actualColor = stream.Peek("color.npy"); 19 | HeaderInfo actualDepth = stream.Peek("depth.npy"); 20 | 21 | string tag = "c#_npz_read"; 22 | if (compressed) 23 | { 24 | tag += "_compressed"; 25 | } 26 | 27 | StringList keys = stream.Keys(); 28 | Test.AssertEqual(keys[0], "color.npy", ref result, tag + " keys"); 29 | Test.AssertEqual(keys[1], "depth.npy", ref result, tag + " keys"); 30 | Test.AssertEqual(keys[2], "unicode.npy", ref result, tag + " keys"); 31 | 32 | Test.AssertEqual(false, stream.Contains("not_there.npy"), ref result, tag + " contains not_there"); 33 | Test.AssertEqual(true, stream.Contains("color.npy"), ref result, tag + " contains color"); 34 | Test.AssertEqual(true, stream.Contains("depth.npy"), ref result, tag + " contains depth"); 35 | Test.AssertEqual(expectedColor, actualColor, ref result, tag + " color"); 36 | Test.AssertEqual(expectedDepth, actualDepth, ref result, tag + " depth"); 37 | } 38 | public static int Main() 39 | { 40 | int result = Test.EXIT_SUCCESS; 41 | 42 | TestPeek(false, ref result); 43 | TestPeek(true, ref result); 44 | 45 | return result; 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /test/CSharpTests/test_npz_read.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using NumpyIO; 5 | 6 | namespace Testing 7 | { 8 | class TestNPZRead 9 | { 10 | static void TestRead(bool compressed, ref int result) 11 | { 12 | UInt8Tensor expectedColor = Test.Tensor(new Shape(new uint[] { 5, 5, 3 })); 13 | Float32Tensor expectedDepth = Test.Tensor(new Shape(new uint[] { 5, 5 })); 14 | UnicodeStringTensor expectedUnicode = Test.Tensor(new Shape(new uint[]{5, 2, 5})); 15 | 16 | string filename = compressed ? "test_compressed.npz" : "test.npz"; 17 | NPZInputStream stream = new NPZInputStream(Test.AssetPath(filename)); 18 | 19 | UInt8Tensor actualColor = stream.ReadUInt8("color"); 20 | Float32Tensor actualDepth = stream.ReadFloat32("depth.npy"); 21 | UnicodeStringTensor actualUnicode = stream.ReadUnicodeString("unicode.npy"); 22 | 23 | string tag = "c#_npz_read"; 24 | if (compressed) 25 | { 26 | tag += "_compressed"; 27 | } 28 | 29 | Test.AssertEqual(expectedColor, actualColor, ref result, tag + " color"); 30 | Test.AssertEqual(expectedDepth, actualDepth, ref result, tag + " depth"); 31 | Test.AssertEqual(expectedUnicode, actualUnicode, ref result, tag + " unicode"); 32 | } 33 | public static int Main() 34 | { 35 | int result = Test.EXIT_SUCCESS; 36 | 37 | TestRead(false, ref result); 38 | TestRead(true, ref result); 39 | 40 | return result; 41 | } 42 | } 43 | } -------------------------------------------------------------------------------- /test/CSharpTests/test_npz_write.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using NumpyIO; 5 | 6 | namespace Testing 7 | { 8 | class TestNPZWrite 9 | { 10 | static void TestWrite(bool compressed, ref int result) 11 | { 12 | string filename = compressed ? "test_compressed.npz" : "test.npz"; 13 | byte[] expected = File.ReadAllBytes(Test.AssetPath(filename)); 14 | 15 | UInt8Tensor color = Test.Tensor(new Shape(new uint[] { 5, 5, 3 })); 16 | Float32Tensor depth = Test.Tensor(new Shape(new uint[] { 5, 5 })); 17 | UnicodeStringTensor unicode = Test.Tensor(new Shape(new uint[]{5, 2, 5})); 18 | string path = Path.GetRandomFileName(); 19 | NPZOutputStream stream = new NPZOutputStream(path, compressed ? CompressionMethod.DEFLATED : CompressionMethod.STORED); 20 | stream.Write("color.npy", color); 21 | stream.Write("depth", depth); 22 | stream.Write("unicode.npy", unicode); 23 | stream.Close(); 24 | 25 | byte[] actual = File.ReadAllBytes(path); 26 | 27 | string tag = "c#_npz_write"; 28 | if (compressed) 29 | { 30 | tag += "_compressed"; 31 | } 32 | Test.AssertEqual(expected, actual, ref result, tag); 33 | 34 | File.Delete(path); 35 | } 36 | 37 | public static int Main() 38 | { 39 | int result = Test.EXIT_SUCCESS; 40 | 41 | TestWrite(false, ref result); 42 | TestWrite(true, ref result); 43 | 44 | return result; 45 | } 46 | } 47 | } -------------------------------------------------------------------------------- /test/crc32.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "libnpy_tests.h" 4 | #include "zip.h" 5 | 6 | const static int BUF_SIZE = 4096; 7 | 8 | int test_crc32() { 9 | int result = EXIT_SUCCESS; 10 | std::ifstream stream(test::asset_path("float32.npy"), 11 | std::ios_base::in | std::ios_base::binary); 12 | char buffer[BUF_SIZE]; 13 | std::string bytes; 14 | while (stream.good()) { 15 | stream.read(buffer, BUF_SIZE); 16 | std::copy(buffer, buffer + stream.gcount(), std::back_inserter(bytes)); 17 | } 18 | 19 | int actual = npy::npy_crc32(bytes); 20 | int expected = 928602993; 21 | test::assert_equal(expected, actual, result, "crc32"); 22 | return result; 23 | } -------------------------------------------------------------------------------- /test/exceptions.cpp: -------------------------------------------------------------------------------- 1 | #include "libnpy_tests.h" 2 | #include "npy/npy.h" 3 | #include "npy/npz.h" 4 | #include "npy/tensor.h" 5 | 6 | namespace { 7 | npy::tensor TENSOR(std::vector({5, 2, 5})); 8 | 9 | void save_invalid_path() { 10 | npy::save(test::path_join({"does_not_exist", "bad.npy"}), TENSOR); 11 | } 12 | 13 | void load_invalid_path() { 14 | npy::load( 15 | test::path_join({"does_not_exist", "bad.npy"})); 16 | } 17 | 18 | void peek_invalid_path() { 19 | npy::peek(test::path_join({"does_not_exist", "bad.npy"})); 20 | } 21 | 22 | void inpzstream_invalid_path() { 23 | npy::inpzstream(test::path_join({"does_not_exist", "bad.npz"})); 24 | } 25 | 26 | void inpzstream_read_invalid_filename() { 27 | npy::inpzstream stream(test::path_join({"assets", "test", "test.npz"})); 28 | npy::tensor tensor = stream.read("not_there.npy"); 29 | } 30 | 31 | void inpzstream_peek_invalid_filename() { 32 | npy::inpzstream stream(test::path_join({"assets", "test", "test.npz"})); 33 | npy::header_info header = stream.peek("not_there.npy"); 34 | } 35 | 36 | void onpzstream_compression() { 37 | npy::compression_method_t compression_method = 38 | static_cast(99); 39 | npy::onpzstream stream("test.npz", compression_method); 40 | stream.write("test.npy", TENSOR); 41 | } 42 | 43 | void tensor_copy_from_0() { 44 | std::vector buffer; 45 | TENSOR.copy_from(buffer.data(), buffer.size()); 46 | } 47 | 48 | void tensor_copy_from_1() { 49 | std::vector buffer; 50 | TENSOR.copy_from(buffer); 51 | } 52 | 53 | void tensor_move_from() { 54 | std::vector buffer; 55 | TENSOR.copy_from(std::move(buffer)); 56 | } 57 | 58 | void tensor_index_size() { std::uint8_t value = TENSOR(0, 0); } 59 | 60 | void tensor_index_range() { std::uint8_t value = TENSOR(2, 3, 3); } 61 | 62 | void load_wrong_dtype() { 63 | npy::tensor tensor = npy::load( 64 | test::path_join({"assets", "test", "uint8.npy"})); 65 | } 66 | 67 | void onpzstream_closed() { 68 | npy::onpzstream stream("test.npz"); 69 | stream.close(); 70 | stream.write("error.npy", TENSOR); 71 | } 72 | 73 | void inpzstream_invalid_file() { 74 | npy::inpzstream stream(test::path_join({"assets", "test", "uint8.npy"})); 75 | } 76 | 77 | } // namespace 78 | 79 | int test_exceptions() { 80 | int result = EXIT_SUCCESS; 81 | 82 | test::assert_throws(peek_invalid_path, result, 83 | "peek_invalid_path"); 84 | test::assert_throws(save_invalid_path, result, 85 | "save_invalid_path"); 86 | test::assert_throws(load_invalid_path, result, 87 | "load_invalid_path"); 88 | test::assert_throws(inpzstream_invalid_path, result, 89 | "inpzstream_invalid_path"); 90 | test::assert_throws( 91 | inpzstream_read_invalid_filename, result, 92 | "inpzstream_read_invalid_filename"); 93 | test::assert_throws( 94 | inpzstream_peek_invalid_filename, result, 95 | "inpzstream_peek_invalid_filename"); 96 | test::assert_throws(onpzstream_compression, result, 97 | "onpzstream_compression"); 98 | test::assert_throws(tensor_copy_from_0, result, 99 | "tensor_copy_from_0"); 100 | test::assert_throws(tensor_copy_from_1, result, 101 | "tensor_copy_from_1"); 102 | test::assert_throws(tensor_move_from, result, 103 | "tensor_move_from"); 104 | test::assert_throws(tensor_index_size, result, 105 | "tensor_index"); 106 | 107 | test::assert_throws(tensor_index_range, result, 108 | "tensor_index_range"); 109 | 110 | test::assert_throws(load_wrong_dtype, result, 111 | "load_wrong_dtype"); 112 | test::assert_throws(onpzstream_closed, result, 113 | "onpzstream_closed"); 114 | test::assert_throws(inpzstream_invalid_file, result, 115 | "inpzstream_invalid_file"); 116 | 117 | std::remove("test.npz"); 118 | 119 | return result; 120 | } -------------------------------------------------------------------------------- /test/generate_large_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import numpy as np 4 | 5 | if __name__ == "__main__": 6 | test_int = np.arange(1000000).astype(np.int32) 7 | test_int = test_int.reshape(200, 5, 1000) 8 | test_float = np.arange(1000000).astype(np.float32) 9 | test_float = test_float.reshape(1000, 5, 20, 10) 10 | 11 | path = os.path.join(os.path.dirname(__file__), "..", 12 | "assets", "test", "test_large.npz") 13 | np.savez(path, test_int=test_int, test_float=test_float) 14 | 15 | path = os.path.join(os.path.dirname(__file__), "..", 16 | "assets", "test", "test_large_compressed.npz") 17 | np.savez_compressed(path, test_int=test_int, test_float=test_float) 18 | -------------------------------------------------------------------------------- /test/libnpy_tests.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "libnpy_tests.h" 9 | 10 | namespace { 11 | inline char sep() { 12 | #if defined(_WIN32) || defined(WIN32) 13 | return '\\'; 14 | #else 15 | return '/'; 16 | #endif 17 | } 18 | 19 | } // namespace 20 | 21 | namespace test { 22 | 23 | std::string path_join(const std::vector &parts) { 24 | std::stringstream result; 25 | result << parts.front(); 26 | 27 | for (auto it = parts.begin() + 1; it < parts.end(); ++it) { 28 | result << sep() << *it; 29 | } 30 | 31 | return result.str(); 32 | } 33 | 34 | std::string read_file(const std::string &path) { 35 | std::ifstream file(path, std::ios::in | std::ios::binary); 36 | if (!file.is_open()) { 37 | throw std::invalid_argument("path"); 38 | } 39 | 40 | std::ostringstream stream; 41 | stream << file.rdbuf(); 42 | return stream.str(); 43 | } 44 | 45 | std::string asset_path(const std::string &filename) { 46 | return path_join({"assets", "test", filename}); 47 | } 48 | 49 | std::string read_asset(const std::string &filename) { 50 | return read_file(asset_path(filename)); 51 | } 52 | 53 | } // namespace test 54 | 55 | typedef std::function TestFunction; 56 | 57 | int main(int argc, char **argv) { 58 | std::map tests; 59 | 60 | tests["crc32"] = test_crc32; 61 | tests["exceptions"] = test_exceptions; 62 | tests["memstream"] = test_memstream; 63 | tests["npy_peek"] = test_npy_peek; 64 | tests["npy_read"] = test_npy_read; 65 | tests["npy_write"] = test_npy_write; 66 | tests["npz_peek"] = test_npz_peek; 67 | tests["npz_read"] = test_npz_read; 68 | tests["npz_write"] = test_npz_write; 69 | tests["tensor"] = test_tensor; 70 | 71 | if (argc == 2) { 72 | std::string test(argv[1]); 73 | if (tests.count(test)) { 74 | return tests[test](); 75 | } else { 76 | std::cout << "Invalid test: " << test << std::endl; 77 | return EXIT_FAILURE; 78 | } 79 | } else { 80 | int result = EXIT_SUCCESS; 81 | for (auto &test : tests) { 82 | std::cout << "Running " << test.first << "..." << std::endl; 83 | if (test.second()) { 84 | result = EXIT_FAILURE; 85 | std::cout << test.first << " failed." << std::endl; 86 | } else { 87 | std::cout << test.first << " succeeded." << std::endl; 88 | } 89 | } 90 | 91 | return result; 92 | } 93 | } -------------------------------------------------------------------------------- /test/libnpy_tests.h: -------------------------------------------------------------------------------- 1 | #ifndef _LIBNPY_TESTS_H_ 2 | #define _LIBNPY_TESTS_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "npy/core.h" 10 | #include "npy/npy.h" 11 | #include "npy/tensor.h" 12 | 13 | int test_crc32(); 14 | int test_exceptions(); 15 | int test_memstream(); 16 | int test_npy_peek(); 17 | int test_npy_read(); 18 | int test_npy_write(); 19 | int test_npz_peek(); 20 | int test_npz_read(); 21 | int test_npz_write(); 22 | int test_tensor(); 23 | 24 | namespace test { 25 | template 26 | void assert_equal(const T &expected, const T &actual, int &result, 27 | const std::string &tag) { 28 | if (expected != actual) { 29 | result = EXIT_FAILURE; 30 | std::cout << tag << " is incorrect: " << actual << " != " << expected 31 | << std::endl; 32 | } 33 | } 34 | 35 | template 36 | void assert_equal(const std::vector &expected, const std::vector &actual, 37 | int &result, const std::string &tag) { 38 | assert_equal(expected.size(), actual.size(), result, tag + " size"); 39 | if (result == EXIT_SUCCESS) { 40 | for (std::size_t i = 0; i < expected.size(); ++i) { 41 | assert_equal(expected[i], actual[i], result, 42 | tag + "[" + std::to_string(i) + "]"); 43 | if (result == EXIT_FAILURE) { 44 | break; 45 | } 46 | } 47 | } 48 | } 49 | 50 | template 51 | void assert_equal(const npy::tensor &expected, const npy::tensor &actual, 52 | int &result, const std::string &tag) { 53 | assert_equal(to_dtype(expected.dtype()), to_dtype(actual.dtype()), result, 54 | tag + " dtype"); 55 | assert_equal(expected.fortran_order(), actual.fortran_order(), result, 56 | tag + " fortran_order"); 57 | assert_equal(expected.shape(), actual.shape(), result, tag + " shape"); 58 | assert_equal(expected.values(), actual.values(), result, tag); 59 | } 60 | 61 | template <> 62 | inline void assert_equal(const std::string &expected, 63 | const std::string &actual, int &result, 64 | const std::string &tag) { 65 | assert_equal(expected.length(), actual.length(), result, tag + " length"); 66 | if (result == EXIT_SUCCESS) { 67 | for (std::size_t i = 0; i < expected.size(); ++i) { 68 | int expected_val = static_cast(expected[i]); 69 | int actual_val = static_cast(actual[i]); 70 | assert_equal(expected_val, actual_val, result, 71 | tag + "[" + std::to_string(i) + "]"); 72 | if (result == EXIT_FAILURE) { 73 | break; 74 | } 75 | } 76 | } 77 | } 78 | 79 | template <> 80 | inline void assert_equal(const std::wstring &expected, 81 | const std::wstring &actual, int &result, 82 | const std::string &tag) { 83 | assert_equal(expected.length(), actual.length(), result, tag + " length"); 84 | if (result == EXIT_SUCCESS) { 85 | for (std::size_t i = 0; i < expected.size(); ++i) { 86 | int expected_val = static_cast(expected[i]); 87 | int actual_val = static_cast(actual[i]); 88 | assert_equal(expected_val, actual_val, result, 89 | tag + "[" + std::to_string(i) + "]"); 90 | if (result == EXIT_FAILURE) { 91 | break; 92 | } 93 | } 94 | } 95 | } 96 | 97 | template <> 98 | inline void assert_equal(const npy::header_info &expected, 99 | const npy::header_info &actual, 100 | int &result, 101 | const std::string &tag) { 102 | assert_equal(expected.dtype, actual.dtype, result, tag + " dtype"); 103 | assert_equal(expected.endianness, actual.endianness, result, 104 | tag + " endianness"); 105 | assert_equal(expected.fortran_order, actual.fortran_order, result, 106 | tag + " fortran_order"); 107 | assert_equal(expected.shape, actual.shape, result, tag + " shape"); 108 | } 109 | 110 | template <> 111 | inline void assert_equal>( 112 | const npy::tensor &expected, 113 | const npy::tensor &actual, int &result, 114 | const std::string &tag) { 115 | assert_equal(to_dtype(expected.dtype()), to_dtype(actual.dtype()), result, 116 | tag + " dtype"); 117 | assert_equal(expected.fortran_order(), actual.fortran_order(), result, 118 | tag + " fortran_order"); 119 | assert_equal(expected.shape(), actual.shape(), result, tag + " shape"); 120 | 121 | auto expected_it = expected.begin(); 122 | auto actual_it = actual.begin(); 123 | for (std::size_t i = 0; i < expected.size(); 124 | ++i, ++expected_it, ++actual_it) { 125 | if (*expected_it != *actual_it) { 126 | result = EXIT_FAILURE; 127 | std::wcout << std::wstring(tag.begin(), tag.end()) 128 | << " is incorrect: " << *actual_it << " != " << *expected_it 129 | << std::endl; 130 | break; 131 | } 132 | } 133 | } 134 | 135 | template 136 | void assert_throws(void (*function)(), int &result, const std::string &tag) { 137 | try { 138 | function(); 139 | result = EXIT_FAILURE; 140 | std::cout << tag << " did not throw an exception" << std::endl; 141 | } catch (EXCEPTION &) { 142 | } catch (std::exception &e) { 143 | result = EXIT_FAILURE; 144 | std::cout << tag << " threw unexpected exception: " << e.what() 145 | << std::endl; 146 | } 147 | } 148 | 149 | template 150 | npy::tensor test_tensor(const std::vector &shape) { 151 | npy::tensor tensor(shape); 152 | std::vector values(tensor.size()); 153 | auto curr = values.begin(); 154 | for (int i = 0; curr < values.end(); ++i, ++curr) { 155 | *curr = static_cast(i); 156 | } 157 | tensor.copy_from(values); 158 | 159 | return tensor; 160 | }; 161 | 162 | template <> 163 | inline npy::tensor test_tensor(const std::vector &shape) { 164 | npy::tensor tensor(shape); 165 | int i = 0; 166 | for (auto &word : tensor) { 167 | word = std::to_wstring(i); 168 | i += 1; 169 | } 170 | 171 | return tensor; 172 | } 173 | 174 | template npy::tensor test_fortran_tensor() { 175 | std::vector values = {0, 10, 20, 30, 40, 5, 15, 25, 35, 45, 1, 11, 21, 176 | 31, 41, 6, 16, 26, 36, 46, 2, 12, 22, 32, 42, 7, 177 | 17, 27, 37, 47, 3, 13, 23, 33, 43, 8, 18, 28, 38, 178 | 48, 4, 14, 24, 34, 44, 9, 19, 29, 39, 49}; 179 | npy::tensor tensor({5, 2, 5}, true); 180 | auto dst = tensor.data(); 181 | auto src = values.begin(); 182 | for (; dst < tensor.data() + tensor.size(); ++src, ++dst) { 183 | *dst = static_cast(*src); 184 | } 185 | 186 | return tensor; 187 | } 188 | 189 | template <> inline npy::tensor test_fortran_tensor() { 190 | std::vector values = {0, 10, 20, 30, 40, 5, 15, 25, 35, 45, 1, 11, 21, 191 | 31, 41, 6, 16, 26, 36, 46, 2, 12, 22, 32, 42, 7, 192 | 17, 27, 37, 47, 3, 13, 23, 33, 43, 8, 18, 28, 38, 193 | 48, 4, 14, 24, 34, 44, 9, 19, 29, 39, 49}; 194 | npy::tensor tensor({5, 2, 5}, true); 195 | auto dst = tensor.data(); 196 | auto src = values.begin(); 197 | for (; dst < tensor.data() + tensor.size(); ++src, ++dst) { 198 | *dst = std::to_wstring(*src); 199 | } 200 | 201 | return tensor; 202 | } 203 | 204 | template 205 | std::string npy_stream(npy::endian_t endianness = npy::endian_t::NATIVE) { 206 | std::ostringstream actual_stream; 207 | npy::tensor tensor = test_tensor({5, 2, 5}); 208 | npy::save(actual_stream, tensor, endianness); 209 | return actual_stream.str(); 210 | } 211 | 212 | template 213 | std::string 214 | npy_scalar_stream(npy::endian_t endianness = npy::endian_t::NATIVE) { 215 | std::ostringstream actual_stream; 216 | npy::tensor tensor = test_tensor({}); 217 | *tensor.data() = static_cast(42); 218 | npy::save(actual_stream, tensor, endianness); 219 | return actual_stream.str(); 220 | } 221 | 222 | template 223 | std::string npy_array_stream(npy::endian_t endianness = npy::endian_t::NATIVE) { 224 | std::ostringstream actual_stream; 225 | npy::tensor tensor = test_tensor({25}); 226 | npy::save(actual_stream, tensor, endianness); 227 | return actual_stream.str(); 228 | } 229 | 230 | template 231 | std::string 232 | npy_fortran_stream(npy::endian_t endianness = npy::endian_t::NATIVE) { 233 | std::ostringstream actual_stream; 234 | npy::tensor tensor = test_fortran_tensor(); 235 | npy::save(actual_stream, tensor, endianness); 236 | return actual_stream.str(); 237 | } 238 | 239 | std::string read_file(const std::string &path); 240 | std::string read_asset(const std::string &filename); 241 | std::string asset_path(const std::string &filename); 242 | std::string path_join(const std::vector &parts); 243 | } // namespace test 244 | 245 | #endif -------------------------------------------------------------------------------- /test/memstream.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "libnpy_tests.h" 4 | 5 | namespace { 6 | const size_t SIZE = 50; 7 | 8 | void test_read(int &result) { 9 | std::vector values(SIZE); 10 | std::iota(values.begin(), values.end(), 0); 11 | std::string expected(values.begin(), values.end()); 12 | 13 | npy::imemstream stream(expected); 14 | stream.read(values.data(), SIZE); 15 | std::string actual(values.begin(), values.end()); 16 | 17 | test::assert_equal(expected, actual, result, "memstream_test_copy_read"); 18 | 19 | stream = npy::imemstream(std::move(expected)); 20 | std::fill(actual.begin(), actual.end(), 0); 21 | stream.read(actual.data(), SIZE); 22 | 23 | expected = std::move(stream.str()); 24 | 25 | test::assert_equal(expected, actual, result, "memstream_test_move_read"); 26 | } 27 | 28 | void test_write(int &result) { 29 | std::vector values(SIZE); 30 | std::iota(values.begin(), values.end(), 0); 31 | std::string expected(values.begin(), values.end()); 32 | 33 | npy::omemstream stream; 34 | stream.write(expected.data(), SIZE); 35 | 36 | std::string actual = stream.str(); 37 | test::assert_equal(expected, actual, result, "memstream_test_copy_write"); 38 | 39 | std::fill(actual.begin(), actual.end(), 0); 40 | stream = npy::omemstream(std::move(actual)); 41 | stream.write(expected.data(), SIZE); 42 | actual = std::move(stream.str()); 43 | 44 | test::assert_equal(expected, actual, result, "memstream_test_move_write"); 45 | } 46 | } // namespace 47 | 48 | int test_memstream() { 49 | int result = EXIT_SUCCESS; 50 | 51 | test_read(result); 52 | 53 | return result; 54 | } -------------------------------------------------------------------------------- /test/npy_peek.cpp: -------------------------------------------------------------------------------- 1 | #include "libnpy_tests.h" 2 | #include "npy/npy.h" 3 | 4 | namespace { 5 | void test_peek(int &result, const std::string &tag, npy::data_type_t data_type, 6 | npy::endian_t endianness = npy::endian_t::LITTLE, 7 | bool fortran_order = false) { 8 | npy::header_info expected = {data_type, endianness, fortran_order, {5, 2, 5}}; 9 | npy::header_info actual = npy::peek(test::asset_path(tag + ".npy")); 10 | test::assert_equal(expected, actual, result, tag); 11 | } 12 | } // namespace 13 | 14 | int test_npy_peek() { 15 | int result = EXIT_SUCCESS; 16 | 17 | test_peek(result, "uint8", npy::data_type_t::UINT8, npy::endian_t::NATIVE); 18 | test_peek(result, "uint8_fortran", npy::data_type_t::UINT8, 19 | npy::endian_t::NATIVE, true); 20 | test_peek(result, "int8", npy::data_type_t::INT8, npy::endian_t::NATIVE); 21 | test_peek(result, "uint16", npy::data_type_t::UINT16); 22 | test_peek(result, "int16", npy::data_type_t::INT16); 23 | test_peek(result, "uint32", npy::data_type_t::UINT32); 24 | test_peek(result, "int32", npy::data_type_t::INT32); 25 | test_peek(result, "int32_big", npy::data_type_t::INT32, npy::endian_t::BIG); 26 | test_peek(result, "uint64", npy::data_type_t::UINT64); 27 | test_peek(result, "int64", npy::data_type_t::INT64); 28 | test_peek(result, "float32", npy::data_type_t::FLOAT32); 29 | test_peek(result, "float64", npy::data_type_t::FLOAT64); 30 | 31 | return result; 32 | } -------------------------------------------------------------------------------- /test/npy_read.cpp: -------------------------------------------------------------------------------- 1 | #include "npy_read.h" 2 | #include "libnpy_tests.h" 3 | 4 | int test_npy_read() { 5 | int result = EXIT_SUCCESS; 6 | 7 | test_read(result, "uint8"); 8 | test_read(result, "uint8_fortran", true); 9 | test_read(result, "int8"); 10 | test_read(result, "uint16"); 11 | test_read(result, "int16"); 12 | test_read(result, "uint32"); 13 | test_read(result, "int32"); 14 | test_read(result, "int32_big"); 15 | test_read_scalar(result, "int32_scalar"); 16 | test_read_array(result, "int32_array"); 17 | test_read(result, "uint64"); 18 | test_read(result, "int64"); 19 | test_read(result, "float32"); 20 | test_read(result, "float64"); 21 | test_read(result, "unicode"); 22 | 23 | return result; 24 | } -------------------------------------------------------------------------------- /test/npy_read.h: -------------------------------------------------------------------------------- 1 | #ifndef _NPY_READ_H_ 2 | #define _NPY_READ_H_ 3 | 4 | #include "libnpy_tests.h" 5 | #include "npy/npy.h" 6 | #include "npy/tensor.h" 7 | 8 | template 9 | void test_read(int &result, const std::string &name, 10 | bool fortran_order = false) { 11 | npy::tensor expected = test::test_tensor({5, 2, 5}); 12 | if (fortran_order) { 13 | expected = test::test_fortran_tensor(); 14 | } 15 | 16 | npy::tensor actual = 17 | npy::load(test::asset_path(name + ".npy")); 18 | test::assert_equal(expected, actual, result, "npy_read_" + name); 19 | } 20 | 21 | template 22 | void test_read_scalar(int &result, const std::string &name) { 23 | npy::tensor expected = test::test_tensor({}); 24 | *expected.data() = static_cast(42); 25 | npy::tensor actual = 26 | npy::load(test::asset_path(name + ".npy")); 27 | test::assert_equal(expected, actual, result, "npy_read_" + name); 28 | } 29 | 30 | template 31 | void test_read_array(int &result, const std::string &name) { 32 | npy::tensor expected = test::test_tensor({25}); 33 | npy::tensor actual = 34 | npy::load(test::asset_path(name + ".npy")); 35 | test::assert_equal(expected, actual, result, "npy_read_" + name); 36 | } 37 | 38 | #endif -------------------------------------------------------------------------------- /test/npy_write.cpp: -------------------------------------------------------------------------------- 1 | #include "libnpy_tests.h" 2 | #include "npy/npy.h" 3 | #include "npy/tensor.h" 4 | 5 | int test_npy_write() { 6 | int result = EXIT_SUCCESS; 7 | 8 | std::string expected, actual; 9 | 10 | npy::endian_t endianness = npy::native_endian(); 11 | std::string dtype = 12 | npy::to_dtype(npy::data_type_t::FLOAT32, npy::endian_t::BIG); 13 | 14 | expected = test::read_asset("uint8.npy"); 15 | actual = test::npy_stream(); 16 | test::assert_equal(expected, actual, result, "npy_write_uint8"); 17 | 18 | expected = test::read_asset("uint8_fortran.npy"); 19 | actual = test::npy_fortran_stream(); 20 | test::assert_equal(expected, actual, result, "npy_write_uint8_fortran"); 21 | 22 | expected = test::read_asset("int8.npy"); 23 | actual = test::npy_stream(); 24 | test::assert_equal(expected, actual, result, "npy_write_int8"); 25 | 26 | expected = test::read_asset("uint16.npy"); 27 | actual = test::npy_stream(npy::endian_t::LITTLE); 28 | test::assert_equal(expected, actual, result, "npy_write_uint16"); 29 | 30 | expected = test::read_asset("int16.npy"); 31 | actual = test::npy_stream(npy::endian_t::LITTLE); 32 | test::assert_equal(expected, actual, result, "npy_write_int16"); 33 | 34 | expected = test::read_asset("uint32.npy"); 35 | actual = test::npy_stream(npy::endian_t::LITTLE); 36 | test::assert_equal(expected, actual, result, "npy_write_uint32"); 37 | 38 | expected = test::read_asset("int32.npy"); 39 | actual = test::npy_stream(npy::endian_t::LITTLE); 40 | test::assert_equal(expected, actual, result, "npy_write_int32"); 41 | 42 | expected = test::read_asset("int32_big.npy"); 43 | actual = test::npy_stream(npy::endian_t::BIG); 44 | test::assert_equal(expected, actual, result, "npy_write_int32_big"); 45 | 46 | expected = test::read_asset("int32_scalar.npy"); 47 | actual = test::npy_scalar_stream(npy::endian_t::LITTLE); 48 | test::assert_equal(expected, actual, result, "npy_write_int32_scalar"); 49 | 50 | expected = test::read_asset("int32_array.npy"); 51 | actual = test::npy_array_stream(npy::endian_t::LITTLE); 52 | test::assert_equal(expected, actual, result, "npy_write_int32_array"); 53 | 54 | expected = test::read_asset("uint64.npy"); 55 | actual = test::npy_stream(npy::endian_t::LITTLE); 56 | test::assert_equal(expected, actual, result, "npy_write_uint64"); 57 | 58 | expected = test::read_asset("int64.npy"); 59 | actual = test::npy_stream(npy::endian_t::LITTLE); 60 | test::assert_equal(expected, actual, result, "npy_write_int64"); 61 | 62 | expected = test::read_asset("float32.npy"); 63 | actual = test::npy_stream(npy::endian_t::LITTLE); 64 | test::assert_equal(expected, actual, result, "npy_write_float32"); 65 | 66 | expected = test::read_asset("float64.npy"); 67 | actual = test::npy_stream(npy::endian_t::LITTLE); 68 | test::assert_equal(expected, actual, result, "npy_write_float64"); 69 | 70 | expected = test::read_asset("unicode.npy"); 71 | actual = test::npy_stream(npy::endian_t::LITTLE); 72 | test::assert_equal(expected, actual, result, "npy_write_unicode"); 73 | 74 | return result; 75 | }; 76 | -------------------------------------------------------------------------------- /test/npz_peek.cpp: -------------------------------------------------------------------------------- 1 | #include "libnpy_tests.h" 2 | #include "npy/npz.h" 3 | 4 | namespace { 5 | void _test(int &result, const std::string &filename, bool compressed) { 6 | npy::header_info expected_color(npy::data_type_t::UINT8, 7 | npy::endian_t::NATIVE, false, {5, 5, 3}); 8 | npy::header_info expected_depth(npy::data_type_t::FLOAT32, 9 | npy::endian_t::LITTLE, false, {5, 5}); 10 | 11 | npy::inpzstream stream(test::asset_path(filename)); 12 | const auto &keys = stream.keys(); 13 | 14 | test::assert_equal(keys[0], std::string("color.npy"), result, 15 | "npz_keys_incorrect"); 16 | test::assert_equal(keys[1], std::string("depth.npy"), result, 17 | "npz_keys_incorrect"); 18 | test::assert_equal(keys[2], std::string("unicode.npy"), result, 19 | "npz_keys_incorrect"); 20 | test::assert_equal(false, stream.contains("not_there.npy"), result, 21 | "npz_contains_missing"); 22 | test::assert_equal(true, stream.contains("color.npy"), result, 23 | "npz_contains_color"); 24 | test::assert_equal(true, stream.contains("depth.npy"), result, 25 | "npz_contains_depth"); 26 | 27 | npy::header_info actual_color = stream.peek("color.npy"); 28 | npy::header_info actual_depth = stream.peek("depth.npy"); 29 | 30 | std::string suffix = compressed ? "_compressed" : ""; 31 | test::assert_equal(expected_color, actual_color, result, 32 | "npz_peek_color" + suffix); 33 | test::assert_equal(expected_depth, actual_depth, result, 34 | "npz_peek_depth" + suffix); 35 | } 36 | } // namespace 37 | 38 | int test_npz_peek() { 39 | int result = EXIT_SUCCESS; 40 | 41 | _test(result, "test.npz", false); 42 | _test(result, "test_compressed.npz", true); 43 | 44 | return result; 45 | } -------------------------------------------------------------------------------- /test/npz_read.cpp: -------------------------------------------------------------------------------- 1 | #include "libnpy_tests.h" 2 | #include "npy/core.h" 3 | #include "npy/npz.h" 4 | 5 | namespace { 6 | void _test(int &result, const std::string &filename, bool compressed) { 7 | auto expected_color = test::test_tensor({5, 5, 3}); 8 | auto expected_depth = test::test_tensor({5, 5}); 9 | auto expected_unicode = test::test_tensor({5, 2, 5}); 10 | 11 | npy::inpzstream stream(test::asset_path(filename)); 12 | auto actual_color = stream.read("color.npy"); 13 | auto actual_depth = stream.read("depth"); 14 | auto actual_unicode = stream.read("unicode"); 15 | 16 | std::string suffix = compressed ? "_compressed" : ""; 17 | test::assert_equal(expected_color, actual_color, result, 18 | "npz_read_color" + suffix); 19 | test::assert_equal(expected_depth, actual_depth, result, 20 | "npz_read_depth" + suffix); 21 | test::assert_equal(expected_unicode, actual_unicode, result, 22 | "npz_read_unicode" + suffix); 23 | } 24 | 25 | void _test_large(int &result, const std::string &filename, bool compressed) { 26 | auto expected_int = test::test_tensor({200, 5, 1000}); 27 | auto expected_float = test::test_tensor({1000, 5, 20, 10}); 28 | 29 | npy::inpzstream stream(test::asset_path(filename)); 30 | auto actual_int = stream.read("test_int"); 31 | auto actual_float = stream.read("test_float"); 32 | 33 | std::string suffix = compressed ? "_compressed" : ""; 34 | test::assert_equal(expected_int, actual_int, result, 35 | "npz_read_large_int" + suffix); 36 | test::assert_equal(expected_float, actual_float, result, 37 | "npz_read_large_float" + suffix); 38 | } 39 | 40 | void _test_memory(int &result, const std::string &filename) { 41 | std::ifstream input(test::asset_path(filename), 42 | std::ios::in | std::ios::binary); 43 | std::string contents((std::istreambuf_iterator(input)), 44 | std::istreambuf_iterator()); 45 | auto memory = std::make_shared(contents); 46 | 47 | auto expected_color = test::test_tensor({5, 5, 3}); 48 | auto expected_depth = test::test_tensor({5, 5}); 49 | auto expected_unicode = test::test_tensor({5, 2, 5}); 50 | 51 | npy::inpzstream stream(memory); 52 | auto actual_color = stream.read("color.npy"); 53 | auto actual_depth = stream.read("depth"); 54 | auto actual_unicode = stream.read("unicode"); 55 | 56 | test::assert_equal(expected_color, actual_color, result, 57 | "npz_read_color_memory"); 58 | test::assert_equal(expected_depth, actual_depth, result, 59 | "npz_read_depth_memory"); 60 | test::assert_equal(expected_unicode, actual_unicode, result, 61 | "npz_read_unicode_memory"); 62 | } 63 | } // namespace 64 | 65 | int test_npz_read() { 66 | int result = EXIT_SUCCESS; 67 | 68 | _test(result, "test.npz", false); 69 | _test(result, "test_compressed.npz", true); 70 | _test_large(result, "test_large.npz", false); 71 | _test_large(result, "test_large_compressed.npz", true); 72 | _test_memory(result, "test.npz"); 73 | 74 | return result; 75 | } -------------------------------------------------------------------------------- /test/npz_write.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "libnpy_tests.h" 7 | #include "npy/core.h" 8 | #include "npy/npz.h" 9 | 10 | namespace { 11 | const char *TEMP_NPZ = "temp.npz"; 12 | } 13 | 14 | namespace { 15 | void _test(int &result, npy::compression_method_t compression_method) { 16 | std::string asset_name = "test.npz"; 17 | std::string suffix = ""; 18 | if (compression_method == npy::compression_method_t::DEFLATED) { 19 | asset_name = "test_compressed.npz"; 20 | suffix = "_compressed"; 21 | } 22 | 23 | std::string expected = test::read_asset(asset_name); 24 | 25 | { 26 | npy::onpzstream npz(TEMP_NPZ, compression_method, npy::endian_t::LITTLE); 27 | npz.write("color", test::test_tensor({5, 5, 3})); 28 | npz.write("depth.npy", test::test_tensor({5, 5})); 29 | npz.write("unicode.npy", test::test_tensor({5, 2, 5})); 30 | } 31 | 32 | std::string actual = test::read_file(TEMP_NPZ); 33 | test::assert_equal(expected, actual, result, "npz_write" + suffix); 34 | 35 | std::remove(TEMP_NPZ); 36 | } 37 | 38 | void _test_memory(int &result) { 39 | std::string asset_name = "test.npz"; 40 | 41 | std::string expected = test::read_asset(asset_name); 42 | auto memory = std::make_shared(); 43 | 44 | { 45 | npy::onpzstream npz(memory, npy::compression_method_t::STORED, 46 | npy::endian_t::LITTLE); 47 | npz.write("color", test::test_tensor({5, 5, 3})); 48 | npz.write("depth.npy", test::test_tensor({5, 5})); 49 | npz.write("unicode.npy", test::test_tensor({5, 2, 5})); 50 | } 51 | 52 | std::string actual = memory->str(); 53 | test::assert_equal(expected, actual, result, "npz_write_memory"); 54 | } 55 | } // namespace 56 | 57 | int test_npz_write() { 58 | int result = EXIT_SUCCESS; 59 | 60 | _test(result, npy::compression_method_t::STORED); 61 | _test(result, npy::compression_method_t::DEFLATED); 62 | _test_memory(result); 63 | 64 | return result; 65 | } -------------------------------------------------------------------------------- /test/tensor.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "libnpy_tests.h" 5 | #include "npy/tensor.h" 6 | 7 | namespace { 8 | const char *TEMP_NPY = "temp.npy"; 9 | } 10 | 11 | int test_tensor() { 12 | int result = EXIT_SUCCESS; 13 | 14 | npy::tensor fortran({3, 4, 5}, true); 15 | std::uint8_t value = 0; 16 | for (auto i = 0; i < 3; ++i) { 17 | for (auto j = 0; j < 4; ++j) { 18 | for (auto k = 0; k < 5; ++k, ++value) { 19 | fortran(i, j, k) = value; 20 | fortran({static_cast(i), static_cast(j), 21 | static_cast(k)}) = value; 22 | } 23 | } 24 | } 25 | 26 | fortran.save(TEMP_NPY); 27 | 28 | npy::tensor from_file(TEMP_NPY); 29 | npy::tensor standard(from_file.shape(), false); 30 | for (int i = 0; i < 3; ++i) { 31 | for (int j = 0; j < 4; ++j) { 32 | for (int k = 0; k < 5; ++k) { 33 | standard(i, j, k) = fortran(i, j, k); 34 | } 35 | } 36 | } 37 | 38 | for (std::uint8_t i = 0; i < 60; ++i) { 39 | test::assert_equal(i, standard.values()[i], result, "tensor read/write"); 40 | } 41 | 42 | std::remove(TEMP_NPY); 43 | 44 | return result; 45 | }; 46 | --------------------------------------------------------------------------------