├── .clang-format ├── .github └── workflows │ └── cmake.yml ├── .gitignore ├── .gitmodules ├── ANNOUNCE.md ├── CATERVA_METALAYER.rst ├── CMakeLists.txt ├── CONTRIBUTING.rst ├── COPYING ├── CPPLINT.cfg ├── DEVELOPING-GUIDE.rst ├── FUNDING.yml ├── LICENSE ├── LICENSES └── BLOSC.txt ├── README.md ├── RELEASE_NOTES.rst ├── RELEASING.rst ├── ROADMAP.rst ├── bench ├── CMakeLists.txt ├── bench_get_slice.c └── bench_zfp_getitem.c ├── caterva ├── caterva.c ├── caterva.h ├── caterva_utils.c └── caterva_utils.h ├── cmake └── FindBLOSC2.cmake ├── code_of_conduct.md ├── doc ├── doxygen │ └── Doxyfile ├── requirements.txt └── sphinx │ ├── Makefile │ ├── _static │ ├── caterva-icon.png │ ├── caterva-logo.png │ └── css │ │ └── custom.css │ ├── conf.py │ ├── development │ ├── code_of_conduct.md │ ├── contributing.rst │ ├── index.rst │ └── roadmap.rst │ ├── getting_started │ ├── index.rst │ ├── installation.rst │ ├── overview.png │ ├── overview.rst │ └── tutorials │ │ ├── basics.rst │ │ └── index.rst │ ├── index.rst │ ├── logo-caterva.png │ ├── reference │ ├── array.rst │ ├── context.rst │ ├── index.rst │ ├── metalayers.rst │ └── wrappers.rst │ └── release_notes │ └── index.rst ├── examples ├── CMakeLists.txt ├── example_empty_shape.c ├── example_frame_generator.c ├── example_oindex.c ├── example_plainbuffer.c ├── example_plugins_codecs.c ├── example_plugins_filters.c ├── example_print_meta.c └── example_serialize.c └── tests ├── CMakeLists.txt ├── cutest.h ├── test_append.c ├── test_common.h ├── test_copy.c ├── test_delete.c ├── test_full.c ├── test_get_slice.c ├── test_get_slice_buffer.c ├── test_insert.c ├── test_metalayers.c ├── test_persistency.c ├── test_resize.c ├── test_roundtrip.c ├── test_save.c ├── test_serialize.c ├── test_set_slice_buffer.c ├── test_squeeze.c ├── test_squeeze_index.c └── test_zeros.c /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | # We'll use defaults from the LLVM style, but with 4 columns indentation. 3 | BasedOnStyle: Google 4 | IndentWidth: 4 5 | ColumnLimit: 100 6 | SpaceAfterCStyleCast: true 7 | --- 8 | -------------------------------------------------------------------------------- /.github/workflows/cmake.yml: -------------------------------------------------------------------------------- 1 | name: CI CMake 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | branches: 8 | - master 9 | jobs: 10 | ci-cmake: 11 | name: ${{ matrix.name }} 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | include: 17 | - name: Ubuntu GCC 18 | os: ubuntu-latest 19 | compiler: gcc 20 | 21 | - name: Ubuntu Clang 22 | os: ubuntu-latest 23 | compiler: clang 24 | 25 | - name: Windows MSVC Win32 26 | os: windows-latest 27 | compiler: cl 28 | cmake-args: -A Win32 29 | 30 | - name: Windows MSVC Win64 31 | os: windows-latest 32 | compiler: cl 33 | cmake-args: -A x64 34 | 35 | - name: Windows GCC Ninja 36 | os: windows-latest 37 | compiler: gcc 38 | cmake-args: -G Ninja 39 | 40 | - name: Windows GCC MinGW 41 | os: windows-latest 42 | compiler: gcc 43 | cmake-args: -G "MinGW Makefiles" 44 | 45 | - name: macOS Clang 46 | os: macOS-latest 47 | compiler: clang 48 | 49 | - name: macOS GCC 50 | os: macOS-latest 51 | compiler: gcc 52 | 53 | steps: 54 | - uses: actions/checkout@v1 55 | with: 56 | submodules: 'recursive' 57 | 58 | - name: Install packages (Ubuntu) 59 | if: runner.os == 'Linux' && matrix.packages 60 | run: | 61 | sudo apt-get update 62 | sudo apt-get install -y ${{ matrix.packages }} 63 | 64 | - name: Install packages (Windows) 65 | if: runner.os == 'Windows' 66 | run: | 67 | choco install ninja ${{ matrix.packages }} 68 | 69 | - name: Install packages (macOS) 70 | if: runner.os == 'macOS' 71 | run: | 72 | brew install ninja ${{ matrix.packages }} 73 | 74 | - name: Generate project files 75 | run: | 76 | mkdir ${{ matrix.build-dir || '.not-used' }} 77 | cd ${{ matrix.build-dir || '.' }} 78 | cmake ${{ matrix.build-src-dir || '.' }} ${{ matrix.cmake-args }} -DCMAKE_BUILD_TYPE=${{ matrix.build-config || 'Release' }} 79 | env: 80 | CC: ${{ matrix.compiler }} 81 | CFLAGS: ${{ matrix.cflags }} 82 | LDFLAGS: ${{ matrix.ldflags }} 83 | CI: true 84 | 85 | - name: Compile source code 86 | run: | 87 | cd ${{ matrix.build-dir || '.' }} 88 | cmake --build . --config ${{ matrix.build-config || 'Release' }} 89 | 90 | - name: Run test cases 91 | run: | 92 | cd ${{ matrix.build-dir || '.' }} 93 | ctest -C Release -L caterva --output-on-failure --max-width 120 94 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Clion IDE 2 | .idea/ 3 | # CMake 4 | cmake-build-*/ 5 | build*/ 6 | # macOS 7 | .DS_Store 8 | # Caterva 9 | doc/doxygen/xml/ 10 | doc/sphinx/_build/ 11 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "contribs/c-blosc2"] 2 | path = contribs/c-blosc2 3 | url = https://github.com/Blosc/c-blosc2.git 4 | -------------------------------------------------------------------------------- /ANNOUNCE.md: -------------------------------------------------------------------------------- 1 | # Announcing Caterva 0.5.0: a simple multidimensional container for compressed data 2 | 3 | 4 | ## What is new? 5 | 6 | A redesign of the caterva guts and a code refactorization have been performed 7 | in order to simplify the code. This includes API renaming. 8 | 9 | Also a set_slice function for arrays backed by Blosc has been implemented. 10 | This allows users to update the values in the array whenever and wherever 11 | they want. 12 | 13 | Finally, some constructors (empty, zeros, full) have been introduced using the 14 | special-values features introduced in Blosc. 15 | 16 | For more info, please see the release notes in: 17 | 18 | https://github.com/Blosc/Caterva/releases 19 | 20 | 21 | ## What is it? 22 | 23 | Caterva is an open source C library and a format that allows to store large 24 | multidimensional, chunked, compressed datasets. 25 | 26 | Data can be stored either in-memory or on-disk, but the API to handle both 27 | versions is the same. Compression is handled transparently for the user by 28 | adopting the Blosc2 library. 29 | 30 | ## Download sources 31 | 32 | The github repository is over here: 33 | 34 | https://github.com/Blosc/Caterva 35 | 36 | Caterva is distributed using the BSD license, see 37 | [LICENSE](https://github.com/Blosc/Caterva/blob/master/LICENSE) for details. 38 | 39 | ## Mailing list 40 | 41 | There is an official Blosc mailing list where discussions about Caterva are 42 | welcome: 43 | 44 | blosc@googlegroups.com 45 | 46 | http://groups.google.es/group/blosc 47 | 48 | 49 | Enjoy Data! 50 | - The Blosc Development Team 51 | -------------------------------------------------------------------------------- /CATERVA_METALAYER.rst: -------------------------------------------------------------------------------- 1 | Caterva metalayer 2 | +++++++++++++++++ 3 | 4 | Caterva is created by specifying a metalayer on top of a Blosc2 container for storing 5 | multidimensional information. Specifically, the Caterva metalayer is named 'caterva' 6 | and follows this format:: 7 | 8 | |-0-|-1-|-2-|-3-|~~~~~~~~~~~~~~~~|---|~~~~~~~~~~~~~~~~|---|~~~~~~~~~~~~~~~~| 9 | | 9X| v | nd| 9X| shape | 9X| chunkshape | 9X| blockshape | 10 | |---|---|---|---|~~~~~~~~~~~~~~~~|---|~~~~~~~~~~~~~~~~|---|~~~~~~~~~~~~~~~~| 11 | ^ ^ ^ ^ ^ ^ 12 | | | | | | | 13 | | | | | | +--[msgpack] fixarray with X=nd elements 14 | | | | | +--[msgpack] fixarray with X=nd elements 15 | | | | +--[msgpack] fixarray with X=nd elements 16 | | | +--[msgpack] positive fixnum for the number of dimensions (up to 127) 17 | | +--[msgpack] positive fixnum for the metalayer format version (up to 127) 18 | +---[msgpack] fixarray with X=5 elements 19 | 20 | The `shape` section is meant to store the actual shape info:: 21 | 22 | |---|--8 bytes---|---|--8 bytes---|~~~~~|---|--8 bytes---| 23 | | d3| first_dim | d3| second_dim | ... | d3| nth_dim | 24 | |---|------------|---|------------|~~~~~|---|------------| 25 | ^ ^ ^ 26 | | | | 27 | | | +--[msgpack] int64 28 | | +--[msgpack] int64 29 | +--[msgpack] int64 30 | 31 | 32 | Next, the `chunkshape` section is meant to store the actual chunk shape info:: 33 | 34 | |---|--4 bytes---|---|--4 bytes---|~~~~~|---|--4 bytes---| 35 | | d2| first_dim | d2| second_dim | ... | d2| nth_dim | 36 | |---|------------|---|------------|~~~~~|---|------------| 37 | ^ ^ ^ 38 | | | | 39 | | | +--[msgpack] int32 40 | | +--[msgpack] int32 41 | +--[msgpack] int32 42 | 43 | 44 | Finally, the `blockshape` section is meant to store the actual block shape info:: 45 | 46 | |---|--4 bytes---|---|--4 bytes---|~~~~~|---|--4 bytes---| 47 | | d2| first_dim | d2| second_dim | ... | d2| nth_dim | 48 | |---|------------|---|------------|~~~~~|---|------------| 49 | ^ ^ ^ 50 | | | | 51 | | | +--[msgpack] int32 52 | | +--[msgpack] int32 53 | +--[msgpack] int32 54 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 2 | # Copyright (C) 2019-present Blosc Development team 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # You may select, at your option, one of the above-listed licenses. 9 | 10 | cmake_minimum_required(VERSION 3.12) 11 | 12 | 13 | cmake_policy(SET CMP0048 NEW) 14 | cmake_policy(SET CMP0077 NEW) 15 | 16 | project(caterva VERSION 0.3.3 LANGUAGES C) 17 | 18 | set(CMAKE_POSITION_INDEPENDENT_CODE ON) 19 | 20 | option(CATERVA_SHARED_LIB "Create shared library" ON) 21 | option(CATERVA_STATIC_LIB "Create static library" ON) 22 | option(CATERVA_BUILD_TESTS "Build tests" ON) 23 | option(CATERVA_BUILD_EXAMPLES "Build examples" ON) 24 | option(CATERVA_BUILD_BENCH "Build benchmarks" ON) 25 | option(CATERVA_PREFER_BLOSC2_EXTERNAL "Prefer external c-blosc2 shared library" OFF) 26 | option(CATERVA_ENABLE_ASAN "Enable ASAN" OFF) 27 | option(CATERVA_ENABLE_COVERAGE "Enable COVERAGE" OFF) 28 | 29 | if(MSVC) 30 | # warning level 4 and all warnings as errors 31 | # add_compile_options(/W4 /WX) 32 | add_compile_options(/W4) 33 | add_compile_options(/wd4232) # TODO: fix this (warning C4232: nonstandard extension used) 34 | add_compile_options(/wd4127) # TODO: fix this (warning C4127: conditional expression is constant) 35 | else() 36 | # lots of warnings and all warnings as errors 37 | # add_compile_options(-Wall -Wextra -pedantic -Werror) 38 | add_compile_options(-Wall -Wextra) 39 | endif() 40 | 41 | if(CATERVA_ENABLE_ASAN) 42 | message(STATUS "Enable sanitizers") 43 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Os -fno-omit-frame-pointer -fsanitize=address") 44 | endif() 45 | 46 | if(CATERVA_ENABLE_COVERAGE) 47 | message(STATUS "Coverage is enabled") 48 | endif() 49 | 50 | set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") 51 | 52 | if(CATERVA_PREFER_BLOSC2_EXTERNAL) 53 | message(STATUS "Looking for external c-blosc2 installation") 54 | find_package(BLOSC2) 55 | endif() 56 | 57 | set(CATERVA_ROOT ${CMAKE_CURRENT_LIST_DIR}) 58 | set(CATERVA_SRC ${CMAKE_CURRENT_LIST_DIR}/caterva) 59 | 60 | if(CATERVA_SHARED_LIB) 61 | set(CATERVA_LIB caterva_shared) 62 | elseif(CATERVA_STATIC_LIB) 63 | set(CATERVA_LIB caterva_static) 64 | else() 65 | message(FATAL_ERROR "No library is created") 66 | endif() 67 | 68 | 69 | set(BUILD_STATIC ON CACHE BOOL "Build a static version of the blosc library.") 70 | set(BUILD_SHARED OFF CACHE BOOL "Build a shared library version of the blosc library.") 71 | set(BUILD_TESTS OFF CACHE BOOL "Build test programs form the blosc compression library") 72 | set(BUILD_BENCHMARKS OFF CACHE BOOL "Build benchmark programs form the blosc compression library") 73 | set(BUILD_EXAMPLES OFF CACHE BOOL "Build benchmark programs form the blosc compression library") 74 | 75 | if(BLOSC2_FOUND) 76 | # See cmake/FindBLOSC2.cmake 77 | set(LIBS ${LIBS} ${BLOSC2_LIBRARIES}) 78 | include_directories(${BLOSC2_INCLUDE_DIRS}) 79 | else() 80 | if(CATERVA_PREFER_BLOSC2_EXTERNAL) 81 | message(WARNING "External c-blosc2 not found") 82 | endif() 83 | message(STATUS "Using c-blosc2 internal sources") 84 | add_subdirectory(contribs/c-blosc2) 85 | include_directories(contribs/c-blosc2/include) 86 | set(LIBS ${LIBS} blosc2_static) 87 | endif() 88 | 89 | 90 | include_directories(${CATERVA_SRC}) 91 | 92 | include(CTest) 93 | 94 | file(GLOB SRC_FILES ${CATERVA_SRC}/*.c) 95 | 96 | if(CATERVA_SHARED_LIB) 97 | message(STATUS "Building caterva shared lib") 98 | add_library(caterva_shared SHARED ${SRC_FILES}) 99 | if(CATERVA_ENABLE_COVERAGE) 100 | target_compile_options(caterva_shared PRIVATE -fprofile-arcs -ftest-coverage) 101 | target_link_libraries(caterva_shared ${LIBS} -fprofile-arcs) 102 | else() 103 | target_compile_options(caterva_shared PRIVATE) 104 | target_link_libraries(caterva_shared ${LIBS}) 105 | endif() 106 | set_target_properties(caterva_shared PROPERTIES OUTPUT_NAME caterva) 107 | install(TARGETS caterva_shared DESTINATION lib) 108 | endif() 109 | 110 | if(CATERVA_STATIC_LIB) 111 | message(STATUS "Building caterva static lib") 112 | add_library(caterva_static STATIC ${SRC_FILES}) 113 | if(CATERVA_ENABLE_COVERAGE) 114 | target_compile_options(caterva_static PRIVATE -fprofile-arcs -ftest-coverage) 115 | target_link_libraries(caterva_static ${LIBS} -fprofile-arcs) 116 | else() 117 | target_compile_options(caterva_static PRIVATE) 118 | target_link_libraries(caterva_static ${LIBS}) 119 | endif() 120 | set_target_properties(caterva_static PROPERTIES OUTPUT_NAME caterva) 121 | if(MSVC) 122 | set_target_properties(caterva_static PROPERTIES PREFIX lib) 123 | endif() 124 | install(TARGETS caterva_static DESTINATION lib) 125 | endif() 126 | 127 | install(FILES ${CATERVA_SRC}/caterva.h DESTINATION include) 128 | 129 | if(CATERVA_BUILD_EXAMPLES) 130 | message(STATUS "Adding Caterva examples") 131 | add_subdirectory(examples) 132 | endif() 133 | 134 | if(CATERVA_BUILD_BENCH) 135 | message(STATUS "Adding Caterva benchmarks") 136 | add_subdirectory(bench) 137 | endif() 138 | 139 | if(CATERVA_BUILD_TESTS) 140 | message(STATUS "Adding Caterva tests") 141 | enable_testing() 142 | add_subdirectory(tests) 143 | endif() 144 | 145 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | Contributing to Caterva 2 | ======================= 3 | 4 | Caterva is a community maintained project. We want to make contributing to 5 | this project as easy and transparent as possible. 6 | 7 | 8 | Asking for help 9 | --------------- 10 | 11 | If you have a question about how to use Caterva, please post your question on 12 | StackOverflow using the `“caterva” tag `_. 13 | 14 | 15 | 16 | Bug reports 17 | ----------- 18 | 19 | We use `GitHub issues `_ to track 20 | public bugs. Please ensure your description is clear and has sufficient 21 | instructions to be able to reproduce the issue. The ideal report should 22 | contain the following: 23 | 24 | 1. Summarize the problem: Include details about your goal, describe expected 25 | and actual results and include any error messages. 26 | 27 | 2. Describe what you’ve tried: Show what you’ve tried, tell us what you 28 | found and why it didn’t meet your needs. 29 | 30 | 3. Minimum reproducible example: Share the minimum amount of code needed to 31 | reproduce your issue. You can format the code nicely using markdown:: 32 | 33 | ```C 34 | #import 35 | 36 | int main() { 37 | ... 38 | } 39 | ``` 40 | 41 | 42 | 4. Determine the environment: Indicates the Caterva version and the operating 43 | system the code is running on. 44 | 45 | Contributing to code 46 | -------------------- 47 | 48 | We actively welcome your code contributions. By contributing to Caterva, you 49 | agree that your contributions will be licensed under the ` `_ file of 50 | the project. 51 | 52 | Fork the repo 53 | +++++++++++++ 54 | 55 | Make a fork of the Caterva repository and clone it:: 56 | 57 | git clone https://github.com//caterva 58 | 59 | 60 | Create your branch 61 | ++++++++++++++++++++ 62 | 63 | Before you do any new work or submit a pull request, please open an `issue on 64 | GitHub `_ to report the bug or 65 | propose the feature you’d like to add. 66 | 67 | Then create a new, separate branch for each piece of work you want to do. 68 | 69 | 70 | Update docstrings 71 | +++++++++++++++++ 72 | 73 | If you've changed APIs, update the involved docstrings using the `doxygen 74 | format `_. 75 | 76 | 77 | Run the test suite 78 | ++++++++++++++++++ 79 | 80 | If you have added code that needs to be tested, add the necessary tests and 81 | verify that all tests pass successfully. 82 | -------------------------------------------------------------------------------- /CPPLINT.cfg: -------------------------------------------------------------------------------- 1 | linelength=100 2 | filter=-readability/casting 3 | filter=-build/include_subdir 4 | 5 | filter=-whitespace/line_length 6 | 7 | filter=-build/include 8 | exclude_files=.*\.c 9 | -------------------------------------------------------------------------------- /DEVELOPING-GUIDE.rst: -------------------------------------------------------------------------------- 1 | Development guide 2 | ================= 3 | 4 | Our Development Process 5 | ----------------------- 6 | 7 | New versions are being developed in the "master" branch, 8 | or in their own feature branch. 9 | When they are deemed ready for a release, they are merged back into "master" 10 | again. 11 | 12 | So all contributions must stage first through "master" 13 | or their own feature branch. 14 | 15 | Pull Requests 16 | ------------- 17 | 18 | We actively welcome your pull requests. 19 | 20 | 1. Fork the repo and create your branch from `master`. 21 | 2. If you've added code that should be tested, add tests. 22 | 3. If you've changed APIs, update the documentation. 23 | 4. Ensure the test suite passes. 24 | 5. Make sure your code does not issue new compiler warnings. 25 | 26 | Issues 27 | ------ 28 | 29 | We use GitHub issues to track public bugs. Please ensure your description is 30 | clear and has sufficient instructions to be able to reproduce the issue. 31 | 32 | Coding Style 33 | ------------ 34 | * 4 spaces for indentation rather than tabs. 35 | 36 | License 37 | ------- 38 | 39 | By contributing to Caterva, you agree that your contributions will be licensed 40 | under the ``_ file of the project. 41 | -------------------------------------------------------------------------------- /FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [numfocus] 2 | custom: ['https://numfocus.org/donate-to-blosc'] 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD License 2 | 3 | For Caterva - A multidimensional data container on top of Blosc2. 4 | 5 | Copyright (C) 2018 Francesc Alted 6 | Copyright (C) 2018 Aleix Alcacer 7 | Copyright (C) 2019-present Blosc Development team 8 | 9 | Redistribution and use in source and binary forms, with or without modification, 10 | are permitted provided that the following conditions are met: 11 | 12 | * Redistributions of source code must retain the above copyright notice, this 13 | list of conditions and the following disclaimer. 14 | 15 | * Redistributions in binary form must reproduce the above copyright notice, 16 | this list of conditions and the following disclaimer in the documentation 17 | and/or other materials provided with the distribution. 18 | 19 | * Neither the names of the Blosc Development team nor the names of its 20 | contributors may be used to endorse or promote products derived from this 21 | software without specific prior written permission. 22 | 23 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 24 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 25 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 26 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 27 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 28 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 29 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 30 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /LICENSES/BLOSC.txt: -------------------------------------------------------------------------------- 1 | BSD License 2 | 3 | For Blosc - A blocking, shuffling and lossless compression library 4 | 5 | Copyright (C) 2009-2018 Francesc Alted 6 | Copyright (C) 2019-present Blosc Development team 7 | 8 | Redistribution and use in source and binary forms, with or without modification, 9 | are permitted provided that the following conditions are met: 10 | 11 | * Redistributions of source code must retain the above copyright notice, this 12 | list of conditions and the following disclaimer. 13 | 14 | * Redistributions in binary form must reproduce the above copyright notice, 15 | this list of conditions and the following disclaimer in the documentation 16 | and/or other materials provided with the distribution. 17 | 18 | * Neither the name Francesc Alted nor the names of its contributors may be used 19 | to endorse or promote products derived from this software without specific 20 | prior written permission. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 23 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 24 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 26 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 27 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 29 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 31 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Caterva 2 | 3 | **Important:** All the features of Caterva have been included in the [C-Blosc2 NDim object](https://www.blosc.org/c-blosc2/c-blosc2.html). As a result, this project is now obsolete. 4 | 5 | [![CI CMake](https://github.com/Blosc/caterva/actions/workflows/cmake.yml/badge.svg)](https://github.com/Blosc/caterva/actions/workflows/cmake.yml) 6 | [![Documentation Status](https://readthedocs.org/projects/caterva/badge/?version=latest)](https://caterva.readthedocs.io/en/latest/?badge=latest) 7 | [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg)](code_of_conduct.md) 8 | ## What it is 9 | 10 | Caterva is a C library for handling multi-dimensional, compressed datasets in 11 | an easy and convenient manner. It implements a thin metalayer on top of 12 | [C-Blosc2](https://github.com/Blosc/c-blosc2) for specifying not only the 13 | dimensionality of a dataset, but also the dimensionality of the chunks 14 | inside the dataset. In addition, Caterva adds machinery for retrieving 15 | arbitrary multi-dimensional slices (aka hyper-slices) out of the 16 | multi-dimensional containers in the most efficient way. Hence, Caterva brings 17 | the convenience of multi-dimensional and compressed containers to your 18 | application very easily. 19 | 20 | For more info, check out the 21 | [Caterva documentation](https://caterva.readthedocs.io). 22 | 23 | Here are slides of a gentle 24 | [introductory talk to Caterva](https://blosc.github.io/caterva-scipy21). 25 | 26 | ## Credits 27 | * Aleix Alcacer: original code creator 28 | * Oscar Griñón: contributed the code for the second partitioning 29 | * Marta Iborra: contributed the resizing operations for arrays 30 | * Francesc Alted: direction, ideas and support code 31 | 32 | This library has received the support from [a donation from Huawei to the Blosc project](https://www.blosc.org/posts/blosc-donation/). 33 | Also, [ironArray SL](https://ironarray.io) has generously donated to the Blosc project, and Caterva in particular. 34 | We are always grateful for donations that help the development of libraries under the umbrella 35 | of Blosc. Thanks! 36 | -------------------------------------------------------------------------------- /RELEASE_NOTES.rst: -------------------------------------------------------------------------------- 1 | Release notes 2 | ============= 3 | 4 | Changes from 0.5.0 to 0.5.1 5 | --------------------------- 6 | 7 | XXX version-specific blurb XXX 8 | 9 | Changes from 0.4.0 to 0.5.0 10 | --------------------------- 11 | 12 | * Redesign the caterva guts and perform a code refactorization in order 13 | to simplify the code. This includes API renaming. A performance improvement 14 | is obtained as can be seen in https://github.com/Blosc/caterva/pull/69#issuecomment-843835622 15 | 16 | * Implement a set_slice for arrays backed by Blosc. This allows users to 17 | update the values in the array whenever and wherever they want. 18 | 19 | * Implement constructors (empty, zeros, full) using the special-values 20 | features introduced in Blosc. 21 | 22 | * Use the `pydata_sphinx_theme` in the documentation. 23 | 24 | 25 | Changes from 0.3.3 to 0.4.0 26 | --------------------------- 27 | 28 | * API renaming. The function names simulate a *namespace* and some 29 | variable names have been changed to those used by the general community 30 | (*part -> chunk*). 31 | 32 | * Add a new level of multi-dimensionality. As a result, unlike other 33 | libraries, Caterva supports two levels of multi-dimensional chunking (chunks 34 | and blocks). 35 | 36 | * Improve library compilation to allow users to avoid building tests and 37 | examples. 38 | 39 | * Simplify the test's suite for a proper integration in Windows. 40 | 41 | * Update documentation to improve the library description and to add the 42 | *Release notes* and the *Roadmap* in a new section. 43 | 44 | Changes from 0.3.0 to 0.3.3 45 | --------------------------- 46 | 47 | * Fixing that 0.3.1 and 0.3.2 tags were not made in master :-/ 48 | 49 | Changes from 0.2.2 to 0.3.0 50 | --------------------------- 51 | 52 | * Big code and API refactorization. As result, the API is more consistent and 53 | hopefully more intuitive to use. For more info on the new API, see 54 | https://caterva.readthedocs.io. 55 | 56 | Changes from 0.2.1 to 0.2.2 57 | --------------------------- 58 | 59 | * Add a `caterva_from_sframe()` function. 60 | 61 | Changes from 0.2.0 to 0.2.1 62 | --------------------------- 63 | 64 | * Both static and dynamic libraries are created by default now. If you want to 65 | disable the creation of one of them, just set the cmake options 66 | `SHARED_LIB=OFF` or `STATIC_LIB=OFF`. 67 | 68 | * Add a `copy` parameter to `caterva_from_file()`. 69 | -------------------------------------------------------------------------------- /RELEASING.rst: -------------------------------------------------------------------------------- 1 | Release procedure 2 | ================= 3 | 4 | Preliminaries 5 | ------------- 6 | 7 | - Make sure that you are in `master` branch:: 8 | 9 | git checkout master 10 | 11 | - Make sure that ``RELEASE_NOTES.rst`` and ``ANNOUNCE.md`` are up to 12 | date with the latest news in the release. 13 | 14 | - Check that *VERSION* symbols in caterva/caterva.h contains the correct info. 15 | 16 | - Commit the changes with:: 17 | 18 | git commit -a -m "Getting ready for release X.Y.Z" 19 | 20 | 21 | Testing 22 | ------- 23 | 24 | Code 25 | ++++ 26 | 27 | Create a new build/ directory, change into it and issue:: 28 | 29 | cmake .. 30 | cmake --build . 31 | ctest 32 | 33 | Documentation 34 | +++++++++++++ 35 | 36 | Make sure the documentation is rendering well:: 37 | 38 | cd docs/sphinx 39 | make html 40 | open _build/html/index.html 41 | 42 | Tagging 43 | ------- 44 | 45 | - Create a tag ``X.Y.Z`` from ``master``. Use the next message:: 46 | 47 | git tag -a vX.Y.Z -m "Tagging version X.Y.Z" 48 | 49 | - Push the tag to the github repo:: 50 | 51 | git push 52 | git push --tags 53 | 54 | - Add the release notes for this tag in the releases tab of github project at: 55 | https://github.com/Blosc/Caterva/releases 56 | 57 | 58 | Announcing 59 | ---------- 60 | 61 | - Send an announcement to the blosc list. Use the ``ANNOUNCE.md`` file as skeleton 62 | (possibly as the definitive version). 63 | 64 | - Tweet about this from the @Blosc2 account and rejoice! 65 | 66 | 67 | Post-release actions 68 | -------------------- 69 | 70 | - Edit *VERSION* symbols in caterva/caterva.h in master to increment the 71 | version to the next minor one (i.e. X.Y.Z --> X.Y.(Z+1)-dev). 72 | 73 | - Create new headers for adding new features in ``RELEASE_NOTES.md`` and 74 | add this place-holder instead: 75 | 76 | XXX version-specific blurb XXX 77 | 78 | - Commit the changes:: 79 | 80 | git commit -a -m "Post X.Y.Z release actions done" 81 | git push 82 | 83 | That's all folks! 84 | 85 | -------------------------------------------------------------------------------- /ROADMAP.rst: -------------------------------------------------------------------------------- 1 | Roadmap 2 | ======= 3 | 4 | Caterva is a multidimensional container for binary data. It is a pure C library, allowing for better interoperatibility between different languages (although Python stands out high in the list). 5 | 6 | This document lists the goals for a production release of Caterva. 7 | 8 | 9 | Existing features 10 | ----------------- 11 | 12 | * **Built on top of Blosc2:** besides transparent compression, this allows to store large compressed datasets either in-memory or on-disk. In addition, Caterva inherits all the improvements that are being introduced in Blosc2 (see https://github.com/Blosc/c-blosc2/blob/main/ROADMAP.rst). 13 | 14 | * **Two-level multidimensional chunking:** like other libraries, Caterva stores the data into multidimensional chunks for efficient slicing. But in addition, Caterva introduces a new level of chunking. Within each chunk, the data is re-chunked into smaller multidimensional sets called blocks, leading to more fine-grained, and hence, to `even more efficient slicing capabilities `_. 15 | 16 | * **Update values:** it allows to populate an array in any order. 17 | 18 | 19 | Actions to be done 20 | ------------------ 21 | 22 | * **Resize array dimensions:** this will allow to increase or decrease in size any dimension of the arrays. 23 | 24 | * **Improve slicing capabilities:** currently Caterva only supports basic slicing based on `start:stop` ranges; we would like to extend this to `start:stop:step` as well as selections based on an array of booleans (similar to NumPy). 25 | 26 | * **Add support for DLPack:** support for `DLPack `_ would be nice for being able to share data between different frameworks and devices. This should complement (or even replace in the long term) the existing plainbuffer support. See `this dicussion `_ for more insight on what advantages could the support for DLPack bring for Caterva. 27 | 28 | * **Support for multidimensional filters:** this will improve the in-memory spatial locally for data that is **n-dim closer** in the array; by n-dim closer we mean that the multidimensional norm (in an Euclidean space) between two different positions of elements is shorter. This may led to better compression opportunities when spatial locality (Euclidean space) is high. 29 | 30 | * **Support for multidimensional codecs:** this is the equivalent for multidim filters, but for codecs. Multidim codecs can leverage n-dim spatial locality in order to compress better/faster. Such codecs could be used in combination with others, uni-dim codecs (e.g. LZ4), so as to get better ratios. 31 | 32 | * **Provide wheels:** this will make the installation much easier for the user. 33 | 34 | 35 | Outreaching 36 | ----------- 37 | 38 | * **Improve the main Caterva README:** this should allow a better overview at first glance of all the features that Caterva offers right now. 39 | 40 | * **Attend to meetings and conferences:** it is very important to plan going to conferences for advertising Caterva and meeting people in-person. We need to decide which meetings to attend. As there are not that much conferences about C libraries, it is important to leverage the `python-caterva `_ wrapper so as to try to promote Caterva on Python conferences too. 41 | 42 | * Other outreaching activities would be to produce videos of the kind 'Caterva in 10 minutes', or producing compelling tutorials (preferably based on Jupyter notebook, and using services like `binder `_ that allows a low entry level for quick trials). 43 | -------------------------------------------------------------------------------- /bench/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 2 | # Copyright (C) 2019-present Blosc Development team 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # You may select, at your option, one of the above-listed licenses. 9 | 10 | file(GLOB SOURCES bench_*.c) 11 | 12 | foreach(source ${SOURCES}) 13 | get_filename_component(target_name ${source} NAME_WE) 14 | set(target caterva_${target_name}) 15 | add_executable(${target} ${target_name}.c) 16 | target_link_libraries(${target} caterva_static ${LIBS}) 17 | endforeach(source) 18 | -------------------------------------------------------------------------------- /bench/bench_get_slice.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #define DATA_TYPE int64_t 13 | 14 | # include 15 | 16 | int main() { 17 | blosc_timestamp_t t0, t1; 18 | 19 | int nslices = 10; 20 | 21 | int8_t ndim = 3; 22 | uint8_t itemsize = sizeof(DATA_TYPE); 23 | 24 | int64_t shape[] = {1250, 745, 400}; 25 | 26 | int32_t chunkshape[] = {50, 150, 100}; 27 | int32_t blockshape[] = {13, 21, 30}; 28 | 29 | int64_t nbytes = itemsize; 30 | for (int i = 0; i < ndim; ++i) { 31 | nbytes *= shape[i]; 32 | } 33 | 34 | DATA_TYPE *src = malloc(nbytes); 35 | for (int i = 0; i < nbytes / itemsize; ++i) { 36 | src[i] = i; 37 | } 38 | 39 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 40 | cfg.nthreads = 4; 41 | 42 | caterva_ctx_t *ctx; 43 | caterva_ctx_new(&cfg, &ctx); 44 | 45 | caterva_params_t params; 46 | params.itemsize = itemsize; 47 | params.ndim = ndim; 48 | for (int i = 0; i < ndim; ++i) { 49 | params.shape[i] = shape[i]; 50 | } 51 | 52 | caterva_storage_t storage = {0}; 53 | for (int i = 0; i < ndim; ++i) { 54 | storage.chunkshape[i] = chunkshape[i]; 55 | storage.blockshape[i] = blockshape[i]; 56 | } 57 | 58 | caterva_array_t *arr; 59 | blosc_set_timestamp(&t0); 60 | CATERVA_ERROR(caterva_from_buffer(ctx, src, nbytes, ¶ms, &storage, &arr)); 61 | blosc_set_timestamp(&t1); 62 | printf("from_buffer: %.4f s\n", blosc_elapsed_secs(t0, t1)); 63 | 64 | blosc_set_timestamp(&t0); 65 | 66 | for (int dim = 0; dim < ndim; ++dim) { 67 | int64_t slice_start[CATERVA_MAX_DIM], slice_stop[CATERVA_MAX_DIM], slice_shape[CATERVA_MAX_DIM]; 68 | int64_t buffersize = itemsize; 69 | for (int j = 0; j < ndim; ++j) { 70 | slice_start[j] = 0; 71 | slice_stop[j] = j == dim ? 1 : shape[j]; 72 | slice_shape[j] = slice_stop[j] - slice_start[j]; 73 | buffersize *= slice_shape[j]; 74 | } 75 | 76 | DATA_TYPE *buffer = malloc(buffersize); 77 | 78 | for (int slice = 0; slice < nslices; ++slice) { 79 | slice_start[dim] = rand() % shape[dim]; 80 | slice_stop[dim] = slice_start[dim] + 1; 81 | CATERVA_ERROR(caterva_get_slice_buffer(ctx, arr, slice_start, slice_stop, buffer, slice_shape, buffersize)); 82 | } 83 | free(buffer); 84 | } 85 | 86 | blosc_set_timestamp(&t1); 87 | printf("get_slice: %.4f s\n", blosc_elapsed_secs(t0, t1)); 88 | 89 | free(src); 90 | 91 | caterva_free(ctx, &arr); 92 | caterva_ctx_free(&ctx); 93 | 94 | return 0; 95 | } 96 | -------------------------------------------------------------------------------- /bench/bench_zfp_getitem.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | * 11 | * Benchmark to measure the retrieval time of a number of elements in random positions 12 | * in caterva arrays. To get the necessary arrays (air1.cat, precip1.cat, snow1.cat...) 13 | * you can use the following script, changing the dataset by your preference (see 14 | * https://docs.digitalearthafrica.org/fr/latest/sandbox/notebooks/Datasets/Climate_Data_ERA5_AWS.html): 15 | 16 | 8<---snip---- "fetch_data.py" 17 | #!/usr/bin/env python 18 | import os 19 | import sys 20 | import xarray as xr 21 | import numpy as np 22 | import s3fs 23 | import caterva as cat 24 | 25 | def open_zarr(year, month, datestart, dateend): 26 | fs = s3fs.S3FileSystem(anon=True) 27 | datestring = "era5-pds/zarr/{year}/{month:02d}/data/".format(year=year, month=month) 28 | s3map = s3fs.S3Map(datestring + "precipitation_amount_1hour_Accumulation.zarr/", s3=fs) 29 | precip_zarr = xr.open_dataset(s3map, engine="zarr") 30 | precip_zarr = precip_zarr.sel(time1=slice(np.datetime64(datestart), np.datetime64(dateend))) 31 | 32 | return precip_zarr.precipitation_amount_1hour_Accumulation 33 | 34 | print("Fetching data from S3 (era5-pds)...") 35 | precip_m0 = open_zarr(1987, 10, "1987-10-01", "1987-10-30 23:59") 36 | 37 | if os.path.exists("precip1.cat"): 38 | cat.remove(path) 39 | 40 | # ia.set_config_defaults(favor=ia.Favor.SPEED) 41 | m_shape = precip_m0.shape 42 | m_chunks = (128, 128, 256) 43 | m_blocks = (32, 32, 32) 44 | cat_precip0 = cat.empty(m_shape, itemsize=4, chunks=m_chunks, blocks=m_blocks, 45 | urlpath="precip1.cat", contiguous=True) 46 | print("Fetching and storing 1st month...") 47 | values = precip_m0.values 48 | cat_precip0[:] = values 49 | 8<---snip---- 50 | 51 | * To call this script, you can run the following commands: 52 | * $ pip install caterva 53 | * $ python fetch_data.py 54 | * 55 | */ 56 | 57 | # include 58 | # include "../contribs/c-blosc2/include/blosc2/codecs-registry.h" 59 | # include "../contribs/c-blosc2/plugins/codecs/zfp/blosc2-zfp.h" 60 | #include 61 | int comp(const char* urlpath) { 62 | blosc2_init(); 63 | 64 | blosc2_schunk *schunk = blosc2_schunk_open(urlpath); 65 | 66 | if (schunk->typesize != 4) { 67 | printf("Error: This test is only for floats.\n"); 68 | return -1; 69 | } 70 | 71 | blosc2_remove_urlpath("schunk_rate.cat"); 72 | blosc2_remove_urlpath("schunk.cat"); 73 | 74 | // Get multidimensional parameters and configure Caterva array 75 | int8_t ndim; 76 | int64_t shape[4]; 77 | int64_t *shape_aux = malloc(8 * sizeof(int64_t)); 78 | int32_t *chunkshape = malloc(8 * sizeof(int32_t)); 79 | int32_t *blockshape = malloc(8 * sizeof(int32_t)); 80 | uint8_t *smeta; 81 | int32_t smeta_len; 82 | if (blosc2_meta_get(schunk, "caterva", &smeta, &smeta_len) < 0) { 83 | printf("This benchmark only supports Caterva datasets"); 84 | free(chunkshape); 85 | free(blockshape); 86 | return -1; 87 | } 88 | caterva_deserialize_meta(smeta, smeta_len, &ndim, shape_aux, chunkshape, blockshape); 89 | free(smeta); 90 | 91 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 92 | cfg.nthreads = 6; 93 | caterva_ctx_t *ctx, *ctx_zfp; 94 | caterva_ctx_new(&cfg, &ctx); 95 | 96 | caterva_storage_t storage = {0}; 97 | for (int i = 0; i < ndim; ++i) { 98 | storage.chunkshape[i] = chunkshape[i]; 99 | storage.blockshape[i] = blockshape[i]; 100 | shape[i] = (int32_t) shape_aux[i]; 101 | } 102 | 103 | caterva_array_t *arr; 104 | caterva_from_schunk(ctx, schunk, &arr); 105 | int copied; 106 | printf("LZ4 comp ratio: %f \n",(float) arr->sc->nbytes / (float) arr->sc->cbytes); 107 | 108 | /* Use BLOSC_CODEC_ZFP_FIXED_RATE */ 109 | storage.urlpath = "schunk_rate.cat"; 110 | caterva_array_t *arr_rate; 111 | ctx_zfp = ctx; 112 | ctx_zfp->cfg->compcodec = BLOSC_CODEC_ZFP_FIXED_RATE; 113 | ctx_zfp->cfg->splitmode = BLOSC_NEVER_SPLIT; 114 | ctx_zfp->cfg->compmeta = (uint8_t) (100.0 * (float) arr->sc->cbytes / (float) arr->sc->nbytes); 115 | ctx_zfp->cfg->filters[BLOSC2_MAX_FILTERS - 1] = 0; 116 | ctx_zfp->cfg->filtersmeta[BLOSC2_MAX_FILTERS - 1] = 0; 117 | copied = caterva_copy(ctx_zfp, arr, &storage, &arr_rate); 118 | if (copied != 0) { 119 | printf("Error BLOSC_CODEC_ZFP_FIXED_RATE \n"); 120 | free(chunkshape); 121 | free(blockshape); 122 | caterva_free(ctx_zfp, &arr); 123 | return -1; 124 | } 125 | printf("ZFP_FIXED_RATE comp ratio: %f \n",(float) arr_rate->sc->nbytes / (float) arr_rate->sc->cbytes); 126 | 127 | int64_t nelems = arr_rate->nitems; 128 | int dsize_zfp, dsize_blosc; 129 | int64_t index; 130 | float item_zfp, item_blosc; 131 | blosc_timestamp_t t0, t1; 132 | double zfp_time, blosc_time; 133 | zfp_time = blosc_time = 0; 134 | int64_t index_ndim[ZFP_MAX_DIM]; 135 | int64_t index_chunk_ndim[ZFP_MAX_DIM]; 136 | int64_t ind_ndim[ZFP_MAX_DIM]; 137 | int64_t stride_chunk, ind_chunk; 138 | int64_t nchunk; 139 | bool needs_free_blosc, needs_free_zfp; 140 | uint8_t *chunk_blosc, *chunk_zfp; 141 | int32_t chunk_nbytes_zfp, chunk_cbytes_zfp, chunk_nbytes_lossy, chunk_cbytes_lossy; 142 | double ntests = 500.0; 143 | for (int i = 0; i < ntests; ++i) { 144 | srand(i); 145 | index = rand() % nelems; 146 | blosc2_unidim_to_multidim(ndim, shape, index, index_ndim); 147 | for (int j = 0; j < ndim; ++j) { 148 | index_chunk_ndim[j] = index_ndim[j] / chunkshape[j]; 149 | ind_ndim[j] = index_ndim[j] % chunkshape[j]; 150 | } 151 | stride_chunk = (shape[1] - 1) / chunkshape[1] + 1; 152 | nchunk = index_chunk_ndim[0] * stride_chunk + index_chunk_ndim[1]; 153 | ind_chunk = (int32_t)(ind_ndim[0] * chunkshape[1] + ind_ndim[1]); 154 | blosc2_schunk_get_lazychunk(arr->sc, nchunk, &chunk_blosc, &needs_free_blosc); 155 | blosc2_cbuffer_sizes(chunk_blosc, &chunk_nbytes_lossy, &chunk_cbytes_lossy, NULL); 156 | blosc_set_timestamp(&t0); 157 | dsize_blosc = blosc2_getitem_ctx(arr->sc->dctx, chunk_blosc, chunk_cbytes_lossy, 158 | ind_chunk, 1, &item_blosc, sizeof(item_blosc)); 159 | blosc_set_timestamp(&t1); 160 | blosc_time += blosc_elapsed_secs(t0, t1); 161 | blosc2_schunk_get_lazychunk(arr_rate->sc, nchunk, &chunk_zfp, &needs_free_zfp); 162 | blosc2_cbuffer_sizes(chunk_zfp, &chunk_nbytes_zfp, &chunk_cbytes_zfp, NULL); 163 | blosc_set_timestamp(&t0); 164 | dsize_zfp = blosc2_getitem_ctx(arr_rate->sc->dctx, chunk_zfp, chunk_cbytes_zfp, 165 | ind_chunk, 1, &item_zfp, sizeof(item_zfp)); 166 | blosc_set_timestamp(&t1); 167 | zfp_time += blosc_elapsed_secs(t0, t1); 168 | if (dsize_blosc != dsize_zfp) { 169 | printf("Different amount of items gotten"); 170 | return -1; 171 | } 172 | } 173 | printf("ZFP_FIXED_RATE time: %.5f microseconds\n", (zfp_time * 1000000.0 / ntests)); 174 | printf("Blosc2 time: %.5f microseconds\n", (blosc_time * 1000000.0 / ntests)); 175 | 176 | free(shape_aux); 177 | free(chunkshape); 178 | free(blockshape); 179 | caterva_free(ctx_zfp, &arr); 180 | caterva_free(ctx_zfp, &arr_rate); 181 | caterva_ctx_free(&ctx_zfp); 182 | if (needs_free_blosc) { 183 | free(chunk_blosc); 184 | } 185 | if (needs_free_zfp) { 186 | free(chunk_zfp); 187 | } 188 | blosc2_destroy(); 189 | 190 | return CATERVA_SUCCEED; 191 | } 192 | 193 | int solar1(void) { 194 | const char* urlpath = "../../bench/solar1.cat"; 195 | 196 | int result = comp(urlpath); 197 | return result; 198 | } 199 | 200 | int air1(void) { 201 | const char* urlpath = "../../bench/air1.cat"; 202 | 203 | int result = comp(urlpath); 204 | return result; 205 | } 206 | 207 | int snow1(void) { 208 | const char* urlpath = "../../bench/snow1.cat"; 209 | 210 | int result = comp(urlpath); 211 | return result; 212 | } 213 | 214 | int wind1(void) { 215 | const char* urlpath = "../../bench/wind1.cat"; 216 | 217 | int result = comp(urlpath); 218 | return result; 219 | } 220 | 221 | int precip1(void) { 222 | const char* urlpath = "../../bench/precip1.cat"; 223 | 224 | int result = comp(urlpath); 225 | return result; 226 | } 227 | 228 | int precip2(void) { 229 | const char* urlpath = "../../bench/precip2.cat"; 230 | 231 | int result = comp(urlpath); 232 | return result; 233 | } 234 | 235 | int precip3(void) { 236 | const char* urlpath = "../../bench/precip3.cat"; 237 | 238 | int result = comp(urlpath); 239 | return result; 240 | } 241 | 242 | int precip3m(void) { 243 | const char* urlpath = "../../bench/precip-3m.cat"; 244 | 245 | int result = comp(urlpath); 246 | return result; 247 | } 248 | 249 | 250 | int main() { 251 | 252 | printf("wind1 \n"); 253 | CATERVA_ERROR(wind1()); 254 | printf("air1 \n"); 255 | CATERVA_ERROR(air1()); 256 | printf("solar1 \n"); 257 | CATERVA_ERROR(solar1()); 258 | printf("snow1 \n"); 259 | CATERVA_ERROR(snow1()); 260 | printf("precip1 \n"); 261 | CATERVA_ERROR(precip1()); 262 | printf("precip2 \n"); 263 | CATERVA_ERROR(precip2()); 264 | printf("precip3 \n"); 265 | CATERVA_ERROR(precip3()); 266 | // printf("precip3m \n"); 267 | // CATERVA_ERROR(precip3m()); 268 | return CATERVA_SUCCEED; 269 | 270 | } 271 | -------------------------------------------------------------------------------- /caterva/caterva_utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018-present Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #ifndef CATERVA_CATERVA_UTILS_H_ 13 | #define CATERVA_CATERVA_UTILS_H_ 14 | 15 | #include 16 | #include <../contribs/c-blosc2/plugins/plugin_utils.h> 17 | 18 | #ifdef __cplusplus 19 | extern "C" { 20 | #endif 21 | 22 | 23 | int caterva_copy_buffer(int8_t ndim, 24 | uint8_t itemsize, 25 | void *src, const int64_t *src_pad_shape, 26 | int64_t *src_start, const int64_t *src_stop, 27 | void *dst, const int64_t *dst_pad_shape, 28 | int64_t *dst_start); 29 | 30 | int create_blosc_params(caterva_ctx_t *ctx, 31 | caterva_params_t *params, 32 | caterva_storage_t *storage, 33 | blosc2_cparams *cparams, 34 | blosc2_dparams *dparams, 35 | blosc2_storage *b_storage); 36 | 37 | int caterva_config_from_schunk(caterva_ctx_t *ctx, blosc2_schunk *sc, caterva_config_t *cfg); 38 | 39 | #ifdef __cplusplus 40 | } 41 | #endif 42 | 43 | #endif // CATERVA_CATERVA_UTILS_H_ 44 | -------------------------------------------------------------------------------- /cmake/FindBLOSC2.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find c-blosc2 2 | # Once done this will define 3 | # BLOSC2_FOUND - System has c-blosc2 4 | # BLOSC2_INCLUDE_DIRS - The c-blosc2 include directories 5 | # BLOSC2_LIBRARIES - The shared libraries required for using c-blosc2 6 | # BLOSC2_DEFINITIONS - 7 | # 8 | # Advanced variables 9 | # BLOSC2_INCLUDE_DIR - Used to set BLOSC2_INCLUDE_DIRS 10 | # BLOSC2_LIBRARY - Used to set BLOSC2_LIBRARIES 11 | # BLOSC2_LIBRARY_STATIC - Name of the static library ending in .a or .lib 12 | # 13 | # Based upon the LibXml2 example in the CMake wiki: 14 | # https://gitlab.kitware.com/cmake/community/-/wikis/doc/tutorials/How-To-Find-Libraries#writing-find-modules 15 | 16 | find_package(PkgConfig) 17 | pkg_check_modules(PC_BLOSC2 QUIET blosc) 18 | set(BLOSC2_DEFINITIONS ${PC_BLOSC2_CFLAGS_OTHER}) 19 | 20 | find_path(BLOSC2_INCLUDE_DIR blosc2.h 21 | HINTS ${PC_BLOSC2_INCLUDEDIR} ${PC_BLOSC2_INCLUDE_DIRS} ) 22 | 23 | find_library(BLOSC2_LIBRARY NAMES blosc2 24 | HINTS ${PC_BLOSC2_LIBDIR} ${PC_BLOSC2_LIBRARY_DIRS} ) 25 | find_library(BLOSC2_LIBRARY_STATIC NAMES libblosc2.a libblosc2.lib 26 | HINTS ${PC_BLOSC2_LIBDIR} ${PC_BLOSC2_LIBRARY_DIRS} ) 27 | 28 | if(BLOSC2_INCLUDE_DIR AND BLOSC2_LIBRARY) 29 | # Debugging messages 30 | message(VERBOSE "Found BLOSC2 include dir: ${BLOSC2_INCLUDE_DIR}") 31 | message(VERBOSE "Found BLOSC2 library: ${BLOSC2_LIBRARY}") 32 | endif() 33 | 34 | if(BLOSC2_LIBRARY_STATIC) 35 | set(BLOSC2_STATIC_FOUND TRUE) 36 | message(VERBOSE "Found BLOSC2 static library: ${BLOSC2_LIBRARY_STATIC}") 37 | endif() 38 | 39 | include(FindPackageHandleStandardArgs) 40 | find_package_handle_standard_args(BLOSC2 DEFAULT_MSG BLOSC2_INCLUDE_DIR BLOSC2_LIBRARY) 41 | 42 | mark_as_advanced(BLOSC2_INCLUDE_DIR BLOSC2_LIBRARY BLOSC2_LIBRARY_STATIC) 43 | 44 | set(BLOSC2_LIBRARIES ${BLOSC2_LIBRARY}) 45 | set(BLOSC2_INCLUDE_DIRS ${BLOSC2_INCLUDE_DIR}) 46 | -------------------------------------------------------------------------------- /code_of_conduct.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | The Blosc community has adopted a Code of Conduct that we expect project participants to adhere to. 4 | Please read the [full text](https://github.com/Blosc/community/blob/master/code_of_conduct.md) 5 | so that you can understand what actions will and will not be tolerated. 6 | -------------------------------------------------------------------------------- /doc/doxygen/Doxyfile: -------------------------------------------------------------------------------- 1 | # Doxyfile 1.8.16 2 | 3 | #--------------------------------------------------------------------------- 4 | # Project related configuration options 5 | #--------------------------------------------------------------------------- 6 | DOXYFILE_ENCODING = UTF-8 7 | PROJECT_NAME = "Caterva" 8 | PROJECT_NUMBER = 9 | PROJECT_BRIEF = 10 | PROJECT_LOGO = 11 | OUTPUT_DIRECTORY = 12 | CREATE_SUBDIRS = NO 13 | ALLOW_UNICODE_NAMES = NO 14 | OUTPUT_LANGUAGE = English 15 | OUTPUT_TEXT_DIRECTION = None 16 | BRIEF_MEMBER_DESC = YES 17 | REPEAT_BRIEF = YES 18 | ABBREVIATE_BRIEF = "The $name class" \ 19 | "The $name widget" \ 20 | "The $name file" \ 21 | is \ 22 | provides \ 23 | specifies \ 24 | contains \ 25 | represents \ 26 | a \ 27 | an \ 28 | the 29 | ALWAYS_DETAILED_SEC = NO 30 | INLINE_INHERITED_MEMB = NO 31 | FULL_PATH_NAMES = YES 32 | STRIP_FROM_PATH = 33 | STRIP_FROM_INC_PATH = 34 | SHORT_NAMES = NO 35 | JAVADOC_AUTOBRIEF = YES 36 | QT_AUTOBRIEF = NO 37 | MULTILINE_CPP_IS_BRIEF = NO 38 | INHERIT_DOCS = YES 39 | SEPARATE_MEMBER_PAGES = NO 40 | TAB_SIZE = 4 41 | ALIASES = 42 | TCL_SUBST = 43 | OPTIMIZE_OUTPUT_FOR_C = NO 44 | OPTIMIZE_OUTPUT_JAVA = NO 45 | OPTIMIZE_FOR_FORTRAN = NO 46 | OPTIMIZE_OUTPUT_VHDL = NO 47 | OPTIMIZE_OUTPUT_SLICE = NO 48 | EXTENSION_MAPPING = 49 | MARKDOWN_SUPPORT = YES 50 | TOC_INCLUDE_HEADINGS = 0 51 | AUTOLINK_SUPPORT = YES 52 | BUILTIN_STL_SUPPORT = NO 53 | CPP_CLI_SUPPORT = NO 54 | SIP_SUPPORT = NO 55 | IDL_PROPERTY_SUPPORT = YES 56 | DISTRIBUTE_GROUP_DOC = NO 57 | GROUP_NESTED_COMPOUNDS = NO 58 | SUBGROUPING = YES 59 | INLINE_GROUPED_CLASSES = NO 60 | INLINE_SIMPLE_STRUCTS = NO 61 | TYPEDEF_HIDES_STRUCT = NO 62 | LOOKUP_CACHE_SIZE = 0 63 | #--------------------------------------------------------------------------- 64 | # Build related configuration options 65 | #--------------------------------------------------------------------------- 66 | EXTRACT_ALL = NO 67 | EXTRACT_PRIVATE = NO 68 | EXTRACT_PACKAGE = NO 69 | EXTRACT_STATIC = NO 70 | EXTRACT_LOCAL_CLASSES = YES 71 | EXTRACT_LOCAL_METHODS = NO 72 | EXTRACT_ANON_NSPACES = NO 73 | HIDE_UNDOC_MEMBERS = NO 74 | HIDE_UNDOC_CLASSES = NO 75 | HIDE_FRIEND_COMPOUNDS = NO 76 | HIDE_IN_BODY_DOCS = NO 77 | INTERNAL_DOCS = NO 78 | CASE_SENSE_NAMES = NO 79 | HIDE_SCOPE_NAMES = NO 80 | HIDE_COMPOUND_REFERENCE= NO 81 | SHOW_INCLUDE_FILES = YES 82 | SHOW_GROUPED_MEMB_INC = NO 83 | FORCE_LOCAL_INCLUDES = NO 84 | INLINE_INFO = YES 85 | SORT_MEMBER_DOCS = YES 86 | SORT_BRIEF_DOCS = NO 87 | SORT_MEMBERS_CTORS_1ST = NO 88 | SORT_GROUP_NAMES = NO 89 | SORT_BY_SCOPE_NAME = NO 90 | STRICT_PROTO_MATCHING = NO 91 | GENERATE_TODOLIST = YES 92 | GENERATE_TESTLIST = YES 93 | GENERATE_BUGLIST = YES 94 | GENERATE_DEPRECATEDLIST= YES 95 | ENABLED_SECTIONS = 96 | MAX_INITIALIZER_LINES = 1 97 | SHOW_USED_FILES = YES 98 | SHOW_FILES = YES 99 | SHOW_NAMESPACES = YES 100 | FILE_VERSION_FILTER = 101 | LAYOUT_FILE = 102 | CITE_BIB_FILES = 103 | #--------------------------------------------------------------------------- 104 | # Configuration options related to warning and progress messages 105 | #--------------------------------------------------------------------------- 106 | QUIET = NO 107 | WARNINGS = YES 108 | WARN_IF_UNDOCUMENTED = YES 109 | WARN_IF_DOC_ERROR = YES 110 | WARN_NO_PARAMDOC = NO 111 | WARN_AS_ERROR = NO 112 | WARN_FORMAT = "$file:$line: $text" 113 | WARN_LOGFILE = 114 | #--------------------------------------------------------------------------- 115 | # Configuration options related to the input files 116 | #--------------------------------------------------------------------------- 117 | INPUT = ../../caterva/ 118 | INPUT_ENCODING = UTF-8 119 | FILE_PATTERNS = *.cc \ 120 | *.cxx \ 121 | *.cpp \ 122 | *.c++ \ 123 | *.java \ 124 | *.ii \ 125 | *.ixx \ 126 | *.ipp \ 127 | *.i++ \ 128 | *.inl \ 129 | *.idl \ 130 | *.ddl \ 131 | *.odl \ 132 | *.h \ 133 | *.hh \ 134 | *.hxx \ 135 | *.hpp \ 136 | *.h++ \ 137 | *.cs \ 138 | *.d \ 139 | *.php \ 140 | *.php4 \ 141 | *.php5 \ 142 | *.phtml \ 143 | *.inc \ 144 | *.m \ 145 | *.markdown \ 146 | *.md \ 147 | *.mm \ 148 | *.dox \ 149 | *.py \ 150 | *.pyw \ 151 | *.f90 \ 152 | *.f95 \ 153 | *.f03 \ 154 | *.f08 \ 155 | *.f \ 156 | *.for \ 157 | *.tcl \ 158 | *.vhd \ 159 | *.vhdl \ 160 | *.ucf \ 161 | *.qsf \ 162 | *.ice 163 | RECURSIVE = NO 164 | EXCLUDE = 165 | EXCLUDE_SYMLINKS = NO 166 | EXCLUDE_PATTERNS = 167 | EXCLUDE_SYMBOLS = 168 | EXAMPLE_PATH = 169 | EXAMPLE_PATTERNS = * 170 | EXAMPLE_RECURSIVE = NO 171 | IMAGE_PATH = 172 | INPUT_FILTER = 173 | FILTER_PATTERNS = 174 | FILTER_SOURCE_FILES = NO 175 | FILTER_SOURCE_PATTERNS = 176 | USE_MDFILE_AS_MAINPAGE = 177 | #--------------------------------------------------------------------------- 178 | # Configuration options related to source browsing 179 | #--------------------------------------------------------------------------- 180 | SOURCE_BROWSER = NO 181 | INLINE_SOURCES = NO 182 | STRIP_CODE_COMMENTS = YES 183 | REFERENCED_BY_RELATION = NO 184 | REFERENCES_RELATION = NO 185 | REFERENCES_LINK_SOURCE = YES 186 | SOURCE_TOOLTIPS = YES 187 | USE_HTAGS = NO 188 | VERBATIM_HEADERS = YES 189 | #--------------------------------------------------------------------------- 190 | # Configuration options related to the alphabetical class index 191 | #--------------------------------------------------------------------------- 192 | ALPHABETICAL_INDEX = YES 193 | COLS_IN_ALPHA_INDEX = 5 194 | IGNORE_PREFIX = 195 | #--------------------------------------------------------------------------- 196 | # Configuration options related to the HTML output 197 | #--------------------------------------------------------------------------- 198 | GENERATE_HTML = NO 199 | HTML_OUTPUT = html 200 | HTML_FILE_EXTENSION = .html 201 | HTML_HEADER = 202 | HTML_FOOTER = 203 | HTML_STYLESHEET = 204 | HTML_EXTRA_STYLESHEET = 205 | HTML_EXTRA_FILES = 206 | HTML_COLORSTYLE_HUE = 220 207 | HTML_COLORSTYLE_SAT = 100 208 | HTML_COLORSTYLE_GAMMA = 80 209 | HTML_TIMESTAMP = NO 210 | HTML_DYNAMIC_MENUS = YES 211 | HTML_DYNAMIC_SECTIONS = NO 212 | HTML_INDEX_NUM_ENTRIES = 100 213 | GENERATE_DOCSET = NO 214 | DOCSET_FEEDNAME = "Doxygen generated docs" 215 | DOCSET_BUNDLE_ID = org.doxygen.Project 216 | DOCSET_PUBLISHER_ID = org.doxygen.Publisher 217 | DOCSET_PUBLISHER_NAME = Publisher 218 | GENERATE_HTMLHELP = NO 219 | CHM_FILE = 220 | HHC_LOCATION = 221 | GENERATE_CHI = NO 222 | CHM_INDEX_ENCODING = 223 | BINARY_TOC = NO 224 | TOC_EXPAND = NO 225 | GENERATE_QHP = NO 226 | QCH_FILE = 227 | QHP_NAMESPACE = org.doxygen.Project 228 | QHP_VIRTUAL_FOLDER = doc 229 | QHP_CUST_FILTER_NAME = 230 | QHP_CUST_FILTER_ATTRS = 231 | QHP_SECT_FILTER_ATTRS = 232 | QHG_LOCATION = 233 | GENERATE_ECLIPSEHELP = NO 234 | ECLIPSE_DOC_ID = org.doxygen.Project 235 | DISABLE_INDEX = NO 236 | GENERATE_TREEVIEW = NO 237 | ENUM_VALUES_PER_LINE = 4 238 | TREEVIEW_WIDTH = 250 239 | EXT_LINKS_IN_WINDOW = NO 240 | FORMULA_FONTSIZE = 10 241 | FORMULA_TRANSPARENT = YES 242 | USE_MATHJAX = YES 243 | MATHJAX_FORMAT = HTML-CSS 244 | MATHJAX_RELPATH = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/ 245 | MATHJAX_EXTENSIONS = 246 | MATHJAX_CODEFILE = 247 | SEARCHENGINE = YES 248 | SERVER_BASED_SEARCH = NO 249 | EXTERNAL_SEARCH = NO 250 | SEARCHENGINE_URL = 251 | SEARCHDATA_FILE = searchdata.xml 252 | EXTERNAL_SEARCH_ID = 253 | EXTRA_SEARCH_MAPPINGS = 254 | #--------------------------------------------------------------------------- 255 | # Configuration options related to the LaTeX output 256 | #--------------------------------------------------------------------------- 257 | GENERATE_LATEX = NO 258 | LATEX_OUTPUT = latex 259 | LATEX_CMD_NAME = 260 | MAKEINDEX_CMD_NAME = makeindex 261 | LATEX_MAKEINDEX_CMD = \makeindex 262 | COMPACT_LATEX = NO 263 | PAPER_TYPE = a4 264 | EXTRA_PACKAGES = 265 | LATEX_HEADER = 266 | LATEX_FOOTER = 267 | LATEX_EXTRA_STYLESHEET = 268 | LATEX_EXTRA_FILES = 269 | PDF_HYPERLINKS = YES 270 | USE_PDFLATEX = YES 271 | LATEX_BATCHMODE = NO 272 | LATEX_HIDE_INDICES = NO 273 | LATEX_SOURCE_CODE = NO 274 | LATEX_BIB_STYLE = plain 275 | LATEX_TIMESTAMP = NO 276 | LATEX_EMOJI_DIRECTORY = 277 | #--------------------------------------------------------------------------- 278 | # Configuration options related to the RTF output 279 | #--------------------------------------------------------------------------- 280 | GENERATE_RTF = NO 281 | RTF_OUTPUT = rtf 282 | COMPACT_RTF = NO 283 | RTF_HYPERLINKS = NO 284 | RTF_STYLESHEET_FILE = 285 | RTF_EXTENSIONS_FILE = 286 | RTF_SOURCE_CODE = NO 287 | #--------------------------------------------------------------------------- 288 | # Configuration options related to the man page output 289 | #--------------------------------------------------------------------------- 290 | GENERATE_MAN = NO 291 | MAN_OUTPUT = man 292 | MAN_EXTENSION = .3 293 | MAN_SUBDIR = 294 | MAN_LINKS = NO 295 | #--------------------------------------------------------------------------- 296 | # Configuration options related to the XML output 297 | #--------------------------------------------------------------------------- 298 | GENERATE_XML = YES 299 | XML_OUTPUT = xml 300 | XML_PROGRAMLISTING = YES 301 | XML_NS_MEMB_FILE_SCOPE = NO 302 | #--------------------------------------------------------------------------- 303 | # Configuration options related to the DOCBOOK output 304 | #--------------------------------------------------------------------------- 305 | GENERATE_DOCBOOK = NO 306 | DOCBOOK_OUTPUT = docbook 307 | DOCBOOK_PROGRAMLISTING = NO 308 | #--------------------------------------------------------------------------- 309 | # Configuration options for the AutoGen Definitions output 310 | #--------------------------------------------------------------------------- 311 | GENERATE_AUTOGEN_DEF = NO 312 | #--------------------------------------------------------------------------- 313 | # Configuration options related to the Perl module output 314 | #--------------------------------------------------------------------------- 315 | GENERATE_PERLMOD = NO 316 | PERLMOD_LATEX = NO 317 | PERLMOD_PRETTY = YES 318 | PERLMOD_MAKEVAR_PREFIX = 319 | #--------------------------------------------------------------------------- 320 | # Configuration options related to the preprocessor 321 | #--------------------------------------------------------------------------- 322 | ENABLE_PREPROCESSING = YES 323 | MACRO_EXPANSION = NO 324 | EXPAND_ONLY_PREDEF = NO 325 | SEARCH_INCLUDES = YES 326 | INCLUDE_PATH = 327 | INCLUDE_FILE_PATTERNS = 328 | PREDEFINED = 329 | EXPAND_AS_DEFINED = 330 | SKIP_FUNCTION_MACROS = YES 331 | #--------------------------------------------------------------------------- 332 | # Configuration options related to external references 333 | #--------------------------------------------------------------------------- 334 | TAGFILES = 335 | GENERATE_TAGFILE = 336 | ALLEXTERNALS = NO 337 | EXTERNAL_GROUPS = YES 338 | EXTERNAL_PAGES = YES 339 | PERL_PATH = /usr/bin/perl 340 | #--------------------------------------------------------------------------- 341 | # Configuration options related to the dot tool 342 | #--------------------------------------------------------------------------- 343 | CLASS_DIAGRAMS = YES 344 | MSCGEN_PATH = 345 | DIA_PATH = 346 | HIDE_UNDOC_RELATIONS = YES 347 | HAVE_DOT = NO 348 | DOT_NUM_THREADS = 0 349 | DOT_FONTNAME = Helvetica 350 | DOT_FONTSIZE = 10 351 | DOT_FONTPATH = 352 | CLASS_GRAPH = YES 353 | COLLABORATION_GRAPH = YES 354 | GROUP_GRAPHS = YES 355 | UML_LOOK = NO 356 | UML_LIMIT_NUM_FIELDS = 10 357 | TEMPLATE_RELATIONS = NO 358 | INCLUDE_GRAPH = YES 359 | INCLUDED_BY_GRAPH = YES 360 | CALL_GRAPH = NO 361 | CALLER_GRAPH = NO 362 | GRAPHICAL_HIERARCHY = YES 363 | DIRECTORY_GRAPH = YES 364 | DOT_IMAGE_FORMAT = png 365 | INTERACTIVE_SVG = NO 366 | DOT_PATH = 367 | DOTFILE_DIRS = 368 | MSCFILE_DIRS = 369 | DIAFILE_DIRS = 370 | PLANTUML_JAR_PATH = 371 | PLANTUML_CFG_FILE = 372 | PLANTUML_INCLUDE_PATH = 373 | DOT_GRAPH_MAX_NODES = 50 374 | MAX_DOT_GRAPH_DEPTH = 0 375 | DOT_TRANSPARENT = NO 376 | DOT_MULTI_TARGETS = NO 377 | GENERATE_LEGEND = YES 378 | DOT_CLEANUP = YES 379 | -------------------------------------------------------------------------------- /doc/requirements.txt: -------------------------------------------------------------------------------- 1 | breathe==4.30.0 2 | sphinx==4.0.2 3 | pydata-sphinx-theme 4 | myst-parser 5 | sphinx-inline-tabs 6 | sphinx-panels 7 | -------------------------------------------------------------------------------- /doc/sphinx/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = . 8 | BUILDDIR = _build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /doc/sphinx/_static/caterva-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FrancescAlted/caterva/4a2415e9d1c0075890418f01c74dbc29c5e774f3/doc/sphinx/_static/caterva-icon.png -------------------------------------------------------------------------------- /doc/sphinx/_static/caterva-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FrancescAlted/caterva/4a2415e9d1c0075890418f01c74dbc29c5e774f3/doc/sphinx/_static/caterva-logo.png -------------------------------------------------------------------------------- /doc/sphinx/_static/css/custom.css: -------------------------------------------------------------------------------- 1 | :root { 2 | /***************************************************************************** 3 | * Theme config 4 | **/ 5 | --pst-header-height: 60px; 6 | 7 | /***************************************************************************** 8 | * Font size 9 | **/ 10 | --pst-font-size-base: 15px; /* base font size - applied at body / html level */ 11 | 12 | /* heading font sizes */ 13 | --pst-font-size-h1: 36px; 14 | --pst-font-size-h2: 32px; 15 | --pst-font-size-h3: 26px; 16 | --pst-font-size-h4: 21px; 17 | --pst-font-size-h5: 18px; 18 | --pst-font-size-h6: 16px; 19 | 20 | /* smaller then heading font sizes*/ 21 | --pst-font-size-milli: 12px; 22 | 23 | --pst-sidebar-font-size: .9em; 24 | --pst-sidebar-caption-font-size: .9em; 25 | 26 | /***************************************************************************** 27 | * Font family 28 | **/ 29 | /* These are adapted from https://systemfontstack.com/ */ 30 | --pst-font-family-base-system: -apple-system, BlinkMacSystemFont, Segoe UI, "Helvetica Neue", 31 | Arial, sans-serif, Apple Color Emoji, Segoe UI Emoji, Segoe UI Symbol; 32 | --pst-font-family-monospace-system: "SFMono-Regular", Menlo, Consolas, Monaco, 33 | Liberation Mono, Lucida Console, monospace; 34 | 35 | --pst-font-family-base: var(--pst-font-family-base-system); 36 | --pst-font-family-heading: var(--pst-font-family-base); 37 | --pst-font-family-monospace: var(--pst-font-family-monospace-system); 38 | 39 | /***************************************************************************** 40 | * Color 41 | * 42 | * Colors are defined in rgb string way, "red, green, blue" 43 | **/ 44 | --pst-color-primary: 15, 109, 126; 45 | --pst-color-primary-dark: 2, 89, 100; 46 | --pst-color-secondary: 212, 168, 12; 47 | --pst-color-success: 40, 167, 69; 48 | --pst-color-info: 0, 123, 255; /*23, 162, 184;*/ 49 | --pst-color-warning: 255, 193, 7; 50 | --pst-color-danger: 220, 53, 69; 51 | --pst-color-text-base: 51, 51, 51; 52 | 53 | --pst-color-h1: var(--pst-color-primary-dark); 54 | --pst-color-h2: var(--pst-color-primary-dark); 55 | --pst-color-h3: var(--pst-color-text-base); 56 | --pst-color-h4: var(--pst-color-text-base); 57 | --pst-color-h5: var(--pst-color-text-base); 58 | --pst-color-h6: var(--pst-color-text-base); 59 | --pst-color-paragraph: var(--pst-color-text-base); 60 | --pst-color-link: var(--pst-color-primary); 61 | --pst-color-link-hover: var(--pst-color-secondary); 62 | --pst-color-headerlink: var(--pst-color-primary); 63 | --pst-color-headerlink-hover: 255, 255, 255; 64 | --pst-color-preformatted-text: 34, 34, 34; 65 | --pst-color-preformatted-background: 250, 250, 250; 66 | --pst-color-inline-code: var(--pst-color-primary); 67 | 68 | --pst-color-active-navigation: var(--pst-color-secondary); 69 | --pst-color-navbar-link: 77, 77, 77; 70 | --pst-color-navbar-link-hover: var(--pst-color-active-navigation); 71 | --pst-color-navbar-link-active: var(--pst-color-active-navigation); 72 | --pst-color-sidebar-link: 77, 77, 77; 73 | --pst-color-sidebar-link-hover: var(--pst-color-active-navigation); 74 | --pst-color-sidebar-link-active: var(--pst-color-active-navigation); 75 | --pst-color-sidebar-expander-background-hover: 244, 244, 244; 76 | --pst-color-sidebar-caption: 77, 77, 77; 77 | --pst-color-toc-link: 119, 117, 122; 78 | --pst-color-toc-link-hover: var(--pst-color-active-navigation); 79 | --pst-color-toc-link-active: var(--pst-color-active-navigation); 80 | 81 | /***************************************************************************** 82 | * Icon 83 | **/ 84 | 85 | /* font awesome icons*/ 86 | --pst-icon-check-circle: '\f058'; 87 | --pst-icon-info-circle: '\f05a'; 88 | --pst-icon-exclamation-triangle: '\f071'; 89 | --pst-icon-exclamation-circle: '\f06a'; 90 | --pst-icon-times-circle: '\f057'; 91 | --pst-icon-lightbulb: '\f0eb'; 92 | 93 | /***************************************************************************** 94 | * Admonitions 95 | **/ 96 | 97 | --pst-color-admonition-default: var(--pst-color-info); 98 | --pst-color-admonition-note: var(--pst-color-info); 99 | --pst-color-admonition-attention: var(--pst-color-warning); 100 | --pst-color-admonition-caution: var(--pst-color-warning); 101 | --pst-color-admonition-warning: var(--pst-color-warning); 102 | --pst-color-admonition-danger: var(--pst-color-danger); 103 | --pst-color-admonition-error: var(--pst-color-danger); 104 | --pst-color-admonition-hint: var(--pst-color-success); 105 | --pst-color-admonition-tip: var(--pst-color-success); 106 | --pst-color-admonition-important: var(--pst-color-success); 107 | 108 | --pst-icon-admonition-default: var(--pst-icon-info-circle); 109 | --pst-icon-admonition-note: var(--pst-icon-info-circle); 110 | --pst-icon-admonition-attention: var(--pst-icon-exclamation-circle); 111 | --pst-icon-admonition-caution: var(--pst-icon-exclamation-triangle); 112 | --pst-icon-admonition-warning: var(--pst-icon-exclamation-triangle); 113 | --pst-icon-admonition-danger: var(--pst-icon-exclamation-triangle); 114 | --pst-icon-admonition-error: var(--pst-icon-times-circle); 115 | --pst-icon-admonition-hint: var(--pst-icon-lightbulb); 116 | --pst-icon-admonition-tip: var(--pst-icon-lightbulb); 117 | --pst-icon-admonition-important: var(--pst-icon-exclamation-circle); 118 | 119 | --tabs--label-text: rgb(var(--pst-color-primary)); 120 | --tabs--label-text--active: rgb(var(--pst-color-secondary)); 121 | --tabs--label-text--hover: rgb(var(--pst-color-secondary)); 122 | --tabs--label-background--active: transparent; 123 | --tabs--label-background--hover: transparent; 124 | --tabs--border: transparent; 125 | } 126 | 127 | /* Main index page overview cards */ 128 | 129 | .intro-card { 130 | background: #fff; 131 | border-radius: 0; 132 | padding: 30px 10px 20px 10px; 133 | margin: 10px 0px; 134 | } 135 | 136 | .intro-card p.card-text { 137 | margin: 0px; 138 | } 139 | 140 | .intro-card .card-img-top { 141 | margin: 10px; 142 | height: 52px; 143 | } 144 | 145 | .intro-card .card-header { 146 | border: none; 147 | background-color: white; 148 | font-size: var(--pst-font-size-h5); 149 | font-weight: bold; 150 | padding: 2.5rem 0rem 0.5rem 0rem; 151 | } 152 | 153 | .intro-card .card-header .card-text { 154 | color: rgb(var(--pst-color-primary)); 155 | } 156 | 157 | .intro-card .card-footer { 158 | border: none; 159 | background-color:white; 160 | } 161 | 162 | .intro-card .card-footer p.card-text{ 163 | max-width: 220px; 164 | margin-left: auto; 165 | margin-right: auto; 166 | } 167 | -------------------------------------------------------------------------------- /doc/sphinx/conf.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import datetime 3 | import re 4 | 5 | 6 | subprocess.call('cd ../doxygen && doxygen Doxyfile && cd ../sphinx', shell=True) 7 | 8 | project = 'Caterva' 9 | author = 'The Blosc Developers' 10 | now = datetime.datetime.now() 11 | 12 | copyright = f'2019-{now.year}, {author}' 13 | 14 | extensions = ["sphinx.ext.mathjax", 15 | "breathe", 16 | "myst_parser", 17 | "sphinx_inline_tabs", 18 | "sphinx_panels"] 19 | 20 | source_suffix = ['.rst', '.md'] 21 | 22 | master_doc = 'index' 23 | language = None 24 | 25 | exclude_patterns = ['_build', '**.ipynb_checkpoints', 'Thumbs.db', '.DS_Store'] 26 | 27 | panels_add_bootstrap_css = False 28 | pygments_style = "sphinx" 29 | 30 | html_static_path = ["_static"] 31 | html_theme = "pydata_sphinx_theme" 32 | html_logo = "_static/caterva-logo.png" 33 | html_favicon = "_static/caterva-icon.png" 34 | html_show_sourcelink = False 35 | 36 | html_theme_options = { 37 | } 38 | 39 | html_css_files = [ 40 | 'css/custom.css', 41 | "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css", 42 | ] 43 | 44 | breathe_projects = {"caterva": "../doxygen/xml/"} 45 | breathe_default_project = "caterva" 46 | -------------------------------------------------------------------------------- /doc/sphinx/development/code_of_conduct.md: -------------------------------------------------------------------------------- 1 | ```{include} ../../../code_of_conduct.md 2 | ``` 3 | -------------------------------------------------------------------------------- /doc/sphinx/development/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../../CONTRIBUTING.rst 2 | 3 | -------------------------------------------------------------------------------- /doc/sphinx/development/index.rst: -------------------------------------------------------------------------------- 1 | Development 2 | =========== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | contributing 8 | roadmap 9 | code_of_conduct 10 | -------------------------------------------------------------------------------- /doc/sphinx/development/roadmap.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../../ROADMAP.rst 2 | -------------------------------------------------------------------------------- /doc/sphinx/getting_started/index.rst: -------------------------------------------------------------------------------- 1 | Getting Started 2 | =============== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | overview 8 | installation 9 | Tutorials -------------------------------------------------------------------------------- /doc/sphinx/getting_started/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | Caterva can be built, tested and installed using CMake. The following procedure 5 | describes a typical CMake build. 6 | 7 | 1. Download the source code from Github: 8 | 9 | .. tab:: Unix 10 | 11 | .. code-block:: 12 | 13 | git clone --recurse-submodules git@github.com:Blosc/caterva.git 14 | 15 | .. tab:: Windows 16 | 17 | .. code-block:: 18 | 19 | git clone --recurse-submodules git@github.com:Blosc/caterva.git 20 | 21 | 22 | 23 | 2. Create the build directory inside the sources and move into it: 24 | 25 | .. tab:: Unix 26 | 27 | .. code-block:: 28 | 29 | cd caterva 30 | mkdir build 31 | cd build 32 | 33 | .. tab:: Windows 34 | 35 | .. code-block:: 36 | 37 | cd caterva 38 | mkdir build 39 | cd build 40 | 41 | 42 | 3. Now run CMake configuration: 43 | 44 | .. tab:: Unix 45 | 46 | .. code-block:: 47 | 48 | cmake -DCMAKE_BUILD_TYPE='Debug/Release' .. 49 | 50 | .. tab:: Windows 51 | 52 | .. code-block:: 53 | 54 | cmake .. 55 | 56 | 57 | 4. Build and test Caterva: 58 | 59 | .. tab:: Unix 60 | 61 | .. code-block:: 62 | 63 | cmake --build . 64 | ctest 65 | 66 | .. tab:: Windows 67 | 68 | .. code-block:: 69 | 70 | cmake --build . --config 'Debug/Release' 71 | ctest --build-config 'Debug/Release' 72 | 73 | 74 | 5. If desired, install Caterva: 75 | 76 | .. tab:: Unix 77 | 78 | .. code-block:: 79 | 80 | cmake --build . --target install 81 | 82 | 83 | .. tab:: Windows 84 | 85 | .. code-block:: 86 | 87 | cmake --build . --target install --config 'Debug/Release' 88 | 89 | 90 | That's all folks! 91 | -------------------------------------------------------------------------------- /doc/sphinx/getting_started/overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FrancescAlted/caterva/4a2415e9d1c0075890418f01c74dbc29c5e774f3/doc/sphinx/getting_started/overview.png -------------------------------------------------------------------------------- /doc/sphinx/getting_started/overview.rst: -------------------------------------------------------------------------------- 1 | What is caterva? 2 | ================ 3 | 4 | Caterva is a container for multidimensional data that is specially designed to 5 | read, in an incredibly efficient way, datasets slices. To achieve this, a new 6 | chunking-based data layout has been created. 7 | 8 | .. image:: overview.png 9 | :width: 90% 10 | :align: center 11 | 12 | Like other libraries like Zarr, HDF5 or TileDB, Caterva stores the data into 13 | multidimensional chunks (yellow cubes). These chunks can then be read 14 | individually, improving performance when reading slices of the dataset. But 15 | also, Caterva introduces a new level of chunking. Within each chunk, the data is 16 | re-partitioned into smaller multidimensional sets called blocks (green cubes). 17 | In this way, Caterva can read blocks individually (and also in parallel) instead 18 | of chunks. 19 | 20 | These partition levels allow to access data efficiently with a larger set of 21 | data access patterns. This is due to obtain the desired slice, instead of 22 | reading the data using the chunks, data is obtained using the blocks. 23 | 24 | Blosc 25 | ----- 26 | 27 | In Caterva the compression is handled transparently for the user by leveraging 28 | the Blosc2 library. Blosc is an extremely fast compressor specially designed 29 | for binary data. It uses the blocking technique to reduce activity on the memory 30 | bus as much as possible. It also leverages SIMD (SSE2, AVX2 for Intel, NEON for 31 | ARM, Altivec for Power) and multi-threading capabilities present in multi-core 32 | processors so as to accelerate the compression/decompression process to a 33 | maximum. 34 | 35 | Being able to store in an in-memory data container does not mean that data 36 | cannot be persisted. It is critical to find a way to store and retrieve data 37 | efficiently. Also, it is important to adopt open formats for reducing the 38 | maintenance burden and facilitate its adoption more quickly. Blosc2 brings such 39 | an efficient and open format for `persistency `__. 41 | 42 | An additional feature that introduces Blosc2 is the concept of metalayers. They 43 | are small metadata for informing about the kind of data that is stored on a 44 | Blosc2 container. They are handy for defining layers with different specs: data 45 | types, geo-spatial... 46 | 47 | .. include:: ../../../CATERVA_METALAYER.rst 48 | -------------------------------------------------------------------------------- /doc/sphinx/getting_started/tutorials/basics.rst: -------------------------------------------------------------------------------- 1 | Caterva basics 2 | ============== 3 | 4 | .. literalinclude:: ../../../../examples/tutorial_basics.c 5 | :language: c -------------------------------------------------------------------------------- /doc/sphinx/getting_started/tutorials/index.rst: -------------------------------------------------------------------------------- 1 | Caterva Tutorials 2 | ================= 3 | 4 | Coming soon! 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | .. 10 | basics 11 | -------------------------------------------------------------------------------- /doc/sphinx/index.rst: -------------------------------------------------------------------------------- 1 | Caterva Documentation 2 | ===================== 3 | 4 | Caterva is an open source C library specially designed to deal with large 5 | multidimensional, chunked, compressed datasets. 6 | 7 | .. panels:: 8 | :card: + intro-card text-center 9 | :column: col-lg-6 col-md-12 col-sm-12 col-xs-12 d-flex 10 | 11 | --- 12 | 13 | Getting Started 14 | ^^^^^^^^^^^^^^^ 15 | 16 | New to *caterva*? Check out the getting started guides. They contain an 17 | introduction to *caterva'* main concepts and an installation tutorial. 18 | 19 | +++ 20 | 21 | .. link-button:: getting_started/index 22 | :type: ref 23 | :text: To the getting started guides 24 | :classes: btn-light 25 | 26 | --- 27 | 28 | API Reference 29 | ^^^^^^^^^^^^^ 30 | 31 | The reference guide contains a detailed description of the caterva API. 32 | The reference describes how the functions work and which parameters can 33 | be used. 34 | 35 | +++ 36 | 37 | .. link-button:: reference/index 38 | :type: ref 39 | :text: To the reference guide 40 | :classes: btn-light 41 | 42 | 43 | --- 44 | 45 | Development 46 | ^^^^^^^^^^^ 47 | 48 | Saw a typo in the documentation? Want to improve 49 | existing functionalities? The contributing guidelines will guide 50 | you through the process of improving caterva. 51 | 52 | +++ 53 | 54 | .. link-button:: development/index 55 | :type: ref 56 | :text: To the development guide 57 | :classes: btn-light 58 | 59 | --- 60 | 61 | Release Notes 62 | ^^^^^^^^^^^^^ 63 | 64 | Want to see what's new in the latest release? Check out the release notes to find out! 65 | 66 | +++ 67 | 68 | .. link-button:: release_notes/index 69 | :type: ref 70 | :text: To the release notes 71 | :classes: btn-light 72 | 73 | 74 | .. toctree:: 75 | :maxdepth: 1 76 | :hidden: 77 | 78 | Getting Started 79 | API Reference 80 | Development 81 | Release Notes 82 | -------------------------------------------------------------------------------- /doc/sphinx/logo-caterva.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FrancescAlted/caterva/4a2415e9d1c0075890418f01c74dbc29c5e774f3/doc/sphinx/logo-caterva.png -------------------------------------------------------------------------------- /doc/sphinx/reference/array.rst: -------------------------------------------------------------------------------- 1 | Array 2 | ===== 3 | A Caterva array is a multidimensional dataset (compressed or not) easy to handle using the Caterva functions. 4 | Furthermore, Caterva only stores item size instead of the data type and every item of a Caterva array has the same size. 5 | On the other hand, Caterva functions let users to perform different operations with these arrays like copying, slicing, setting them or converting them into buffers or files and vice versa. 6 | 7 | .. doxygenstruct:: caterva_array_t 8 | 9 | 10 | Parameters 11 | ---------- 12 | 13 | 14 | General parameters 15 | ++++++++++++++++++ 16 | 17 | .. doxygenstruct:: caterva_params_t 18 | :members: 19 | 20 | 21 | Storage parameters 22 | ++++++++++++++++++ 23 | .. doxygenstruct:: caterva_storage_t 24 | :members: 25 | 26 | .. doxygenstruct:: caterva_metalayer_t 27 | :members: 28 | 29 | 30 | Creation 31 | -------- 32 | 33 | Fast constructors 34 | +++++++++++++++++ 35 | 36 | .. doxygenfunction:: caterva_empty 37 | 38 | .. doxygenfunction:: caterva_zeros 39 | 40 | .. doxygenfunction:: caterva_full 41 | 42 | 43 | From/To buffer 44 | ++++++++++++++ 45 | .. doxygenfunction:: caterva_from_buffer 46 | 47 | .. doxygenfunction:: caterva_to_buffer 48 | 49 | 50 | From/To file 51 | ++++++++++++ 52 | 53 | .. doxygenfunction:: caterva_open 54 | .. doxygenfunction:: caterva_save 55 | 56 | 57 | From Blosc object 58 | +++++++++++++++++ 59 | 60 | .. doxygenfunction:: caterva_from_schunk 61 | 62 | 63 | From/To cframe buffer 64 | +++++++++++++++++++++ 65 | 66 | .. doxygenfunction:: caterva_from_cframe 67 | 68 | .. doxygenfunction:: caterva_to_cframe 69 | 70 | Copying 71 | +++++++ 72 | 73 | .. doxygenfunction:: caterva_copy 74 | 75 | 76 | Slicing 77 | ------- 78 | 79 | .. doxygenfunction:: caterva_get_slice_buffer 80 | 81 | .. doxygenfunction:: caterva_set_slice_buffer 82 | 83 | .. doxygenfunction:: caterva_get_slice 84 | 85 | .. doxygenfunction:: caterva_squeeze 86 | 87 | .. doxygenfunction:: caterva_squeeze_index 88 | 89 | 90 | Destruction 91 | ----------- 92 | 93 | .. doxygenfunction:: caterva_free 94 | 95 | .. doxygenfunction:: caterva_remove 96 | -------------------------------------------------------------------------------- /doc/sphinx/reference/context.rst: -------------------------------------------------------------------------------- 1 | Caterva has an internal structure managed by parameters that record the different properties of each dataset. 2 | The most fundamental parts of this structure are the Caterva context and the configuration parameters it depends on. 3 | 4 | Context 5 | ======= 6 | .. doxygenstruct:: caterva_ctx_t 7 | 8 | 9 | Configuration parameters 10 | ++++++++++++++++++++++++ 11 | 12 | .. doxygenstruct:: caterva_config_t 13 | :members: 14 | 15 | .. doxygenvariable:: CATERVA_CONFIG_DEFAULTS 16 | 17 | 18 | Creation 19 | ++++++++ 20 | .. doxygenfunction:: caterva_ctx_new 21 | 22 | 23 | Destruction 24 | +++++++++++ 25 | 26 | .. doxygenfunction:: caterva_ctx_free 27 | -------------------------------------------------------------------------------- /doc/sphinx/reference/index.rst: -------------------------------------------------------------------------------- 1 | Caterva Reference 2 | ================= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | context 8 | array 9 | metalayers 10 | wrappers 11 | -------------------------------------------------------------------------------- /doc/sphinx/reference/metalayers.rst: -------------------------------------------------------------------------------- 1 | Metalayers 2 | ========== 3 | Metalayers are metadata for informing about the properties of data that is stored on a container. Caterva implements its own metalayer on top of C-Blosc2 for storing multidimensional information. 4 | In general, you can use metalayers for adapting Blosc2 containers (and in particular, Caterva arrays) to your own needs. 5 | 6 | Fixed-length metalayers 7 | ----------------------- 8 | These small metalayers are stored in the header of the frame. This allows adding chunks to the frame without the need to rewrite the whole meta information and data coming after it. 9 | However, fixed-length metalayers cannot be resized and once the first chunk of data is added to the super-chunk, no more fixed-length metalayers can be added either. 10 | 11 | .. doxygenfunction:: caterva_meta_exists 12 | 13 | .. doxygenfunction:: caterva_meta_get 14 | 15 | .. doxygenfunction:: caterva_meta_update 16 | 17 | 18 | Variable-length metalayers 19 | -------------------------- 20 | Unlike fixed-length metalayers, these can be resized and are stored in the trailer section of the frame. 21 | This provides users a lot of flexibility to define their own metadata. 22 | 23 | .. doxygenfunction:: caterva_vlmeta_add 24 | 25 | .. doxygenfunction:: caterva_vlmeta_exists 26 | 27 | .. doxygenfunction:: caterva_vlmeta_get 28 | 29 | .. doxygenfunction:: caterva_vlmeta_update 30 | -------------------------------------------------------------------------------- /doc/sphinx/reference/wrappers.rst: -------------------------------------------------------------------------------- 1 | High-level APIs 2 | =============== 3 | 4 | Here you can find a list with the wrappers available for Caterva: 5 | 6 | - `Python Caterva `_: a Pythonic wrapper of Caterva. 7 | -------------------------------------------------------------------------------- /doc/sphinx/release_notes/index.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../../RELEASE_NOTES.rst 2 | -------------------------------------------------------------------------------- /examples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 2 | # Copyright (C) 2019-present Blosc Development team 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # You may select, at your option, one of the above-listed licenses. 9 | 10 | file(GLOB SOURCES example_*.c) 11 | 12 | foreach(source ${SOURCES}) 13 | get_filename_component(target_name ${source} NAME_WE) 14 | set(target caterva_${target_name}) 15 | add_executable(${target} ${target_name}.c) 16 | target_link_libraries(${target} caterva_static ${LIBS}) 17 | endforeach(source) 18 | -------------------------------------------------------------------------------- /examples/example_empty_shape.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | # include 13 | 14 | int main() { 15 | 16 | int8_t ndim = 2; 17 | int64_t shape[] = {10, 10}; 18 | int32_t chunkshape[] = {4, 4}; 19 | int32_t blockshape[] = {2, 2}; 20 | int8_t itemsize = 8; 21 | 22 | int64_t slice_start[] = {2, 5}; 23 | int64_t slice_stop[] = {2, 6}; 24 | int32_t slice_chunkshape[] = {0, 1}; 25 | int32_t slice_blockshape[] = {0, 1}; 26 | 27 | int64_t nelem = 1; 28 | for (int i = 0; i < ndim; ++i) { 29 | nelem *= shape[i]; 30 | } 31 | int64_t size = nelem * itemsize; 32 | int8_t *data = malloc(size); 33 | 34 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 35 | 36 | caterva_ctx_t *ctx; 37 | caterva_ctx_new(&cfg, &ctx); 38 | 39 | caterva_params_t params = {0}; 40 | params.ndim = ndim; 41 | params.itemsize = itemsize; 42 | for (int i = 0; i < ndim; ++i) { 43 | params.shape[i] = shape[i]; 44 | } 45 | 46 | caterva_storage_t storage = {0}; 47 | for (int i = 0; i < ndim; ++i) { 48 | storage.chunkshape[i] = chunkshape[i]; 49 | storage.blockshape[i] = blockshape[i]; 50 | } 51 | 52 | caterva_array_t *arr; 53 | CATERVA_ERROR(caterva_from_buffer(ctx, data, size, ¶ms, &storage, &arr)); 54 | 55 | 56 | caterva_storage_t slice_storage = {0}; 57 | slice_storage.urlpath = "example_hola.b2frame"; 58 | for (int i = 0; i < ndim; ++i) { 59 | slice_storage.chunkshape[i] = slice_chunkshape[i]; 60 | slice_storage.blockshape[i] = slice_blockshape[i]; 61 | } 62 | 63 | caterva_array_t *slice; 64 | CATERVA_ERROR(caterva_get_slice(ctx, arr, slice_start, slice_stop, &slice_storage, 65 | &slice)); 66 | 67 | 68 | uint8_t *buffer; 69 | uint64_t buffer_size = 1; 70 | for (int i = 0; i < slice->ndim; ++i) { 71 | buffer_size *= slice->shape[i]; 72 | } 73 | buffer_size *= slice->itemsize; 74 | buffer = malloc(buffer_size); 75 | 76 | CATERVA_ERROR(caterva_to_buffer(ctx, slice, buffer, buffer_size)); 77 | 78 | // printf("Elapsed seconds: %.5f\n", blosc_elapsed_secs(t0, t1)); 79 | 80 | return 0; 81 | } 82 | -------------------------------------------------------------------------------- /examples/example_frame_generator.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2019-present Blosc Development team 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | # include 12 | # include 13 | 14 | 15 | int frame_generator(int8_t *data, int8_t ndim, const int64_t shape[8], const int32_t chunkshape[8], 16 | const int32_t blockshape[8], int8_t itemsize, int64_t size, char *urlpath) { 17 | 18 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 19 | caterva_ctx_t *ctx; 20 | caterva_ctx_new(&cfg, &ctx); 21 | 22 | caterva_params_t params = {0}; 23 | params.ndim = ndim; 24 | params.itemsize = itemsize; 25 | for (int i = 0; i < ndim; ++i) { 26 | params.shape[i] = shape[i]; 27 | } 28 | 29 | caterva_storage_t storage = {0}; 30 | storage.urlpath = urlpath; 31 | storage.contiguous = true; 32 | for (int i = 0; i < ndim; ++i) { 33 | storage.chunkshape[i] = chunkshape[i]; 34 | storage.blockshape[i] = blockshape[i]; 35 | } 36 | 37 | caterva_array_t *arr; 38 | CATERVA_ERROR(caterva_from_buffer(ctx, data, size, ¶ms, &storage, &arr)); 39 | caterva_print_meta(arr); 40 | 41 | return 0; 42 | } 43 | 44 | int all_eq() { 45 | int8_t ndim = 3; 46 | int64_t shape[] = {100, 50, 100}; 47 | int32_t chunkshape[] = {40, 20, 60}; 48 | int32_t blockshape[] = {20, 10, 30}; 49 | int8_t itemsize = 8; 50 | int64_t nelem = 1; 51 | for (int i = 0; i < ndim; ++i) { 52 | nelem *= shape[i]; 53 | } 54 | int64_t size = nelem * itemsize; 55 | 56 | int8_t *data = malloc(size); 57 | for (int i= 0; i < nelem; i++) { 58 | data[i] = (int8_t) 22; 59 | } 60 | char *urlpath = "all_eq.caterva"; 61 | CATERVA_ERROR(frame_generator(data, ndim, shape, chunkshape, blockshape, itemsize, size, urlpath)); 62 | 63 | return 0; 64 | } 65 | 66 | int cyclic() { 67 | int8_t ndim = 3; 68 | int64_t shape[] = {100, 50, 100}; 69 | int32_t chunkshape[] = {40, 20, 60}; 70 | int32_t blockshape[] = {20, 10, 30}; 71 | int8_t itemsize = 8; 72 | int64_t nelem = 1; 73 | for (int i = 0; i < ndim; ++i) { 74 | nelem *= shape[i]; 75 | } 76 | int64_t size = nelem * itemsize; 77 | 78 | int8_t *data = malloc(size); 79 | for (int i= 0; i < nelem; i++) { 80 | data[i] = (int8_t) i; 81 | } 82 | char *urlpath = "cyclic.caterva"; 83 | CATERVA_ERROR(frame_generator(data, ndim, shape, chunkshape, blockshape, itemsize, size, urlpath)); 84 | 85 | return 0; 86 | } 87 | 88 | int many_matches() { 89 | int8_t ndim = 3; 90 | int64_t shape[] = {80, 120, 111}; 91 | int32_t chunkshape[] = {40, 30, 50}; 92 | int32_t blockshape[] = {11, 14, 24}; 93 | int8_t itemsize = 8; 94 | int64_t nelem = 1; 95 | for (int i = 0; i < ndim; ++i) { 96 | nelem *= shape[i]; 97 | } 98 | int64_t size = nelem * itemsize; 99 | 100 | int8_t *data = malloc(size); 101 | for (int i = 0; i < nelem; i += 2) { 102 | data[i] = (int8_t) i; 103 | data[i + 1] = (int8_t) 2; 104 | } 105 | char *urlpath = "many_matches.caterva"; 106 | CATERVA_ERROR(frame_generator(data, ndim, shape, chunkshape, blockshape, itemsize, size, urlpath)); 107 | 108 | return 0; 109 | } 110 | 111 | int float_cyclic() { 112 | int8_t ndim = 3; 113 | int64_t shape[] = {40, 60, 20}; 114 | int32_t chunkshape[] = {20, 30, 16}; 115 | int32_t blockshape[] = {11, 14, 7}; 116 | int8_t itemsize = sizeof(float); 117 | int64_t nelem = 1; 118 | for (int i = 0; i < ndim; ++i) { 119 | nelem *= shape[i]; 120 | } 121 | int64_t size = nelem * itemsize; 122 | 123 | float *data = malloc(size); 124 | for (int i = 0; i < nelem; i += 2) { 125 | float j = (float) i; 126 | data[i] = (j + j / 10 + j / 100); 127 | data[i + 1] = (2 + j / 10 + j / 1000); 128 | } 129 | char *urlpath = "example_float_cyclic.caterva"; 130 | CATERVA_ERROR(frame_generator((int8_t *)data, ndim, shape, chunkshape, blockshape, itemsize, size, urlpath)); 131 | 132 | return 0; 133 | } 134 | 135 | int double_same_cells() { 136 | int8_t ndim = 2; 137 | int64_t shape[] = {40, 60}; 138 | int32_t chunkshape[] = {20, 30}; 139 | int32_t blockshape[] = {16, 16}; 140 | int8_t itemsize = sizeof(double); 141 | int64_t nelem = 1; 142 | for (int i = 0; i < ndim; ++i) { 143 | nelem *= shape[i]; 144 | } 145 | int64_t size = nelem * itemsize; 146 | 147 | double *data = malloc(size); 148 | for (int i = 0; i < nelem; i += 4) { 149 | data[i] = 1.5; 150 | data[i + 1] = 14.7; 151 | data[i + 2] = 23.6; 152 | data[i + 3] = 3.2; 153 | } 154 | char *urlpath = "example_double_same_cells.caterva"; 155 | CATERVA_ERROR(frame_generator((int8_t *)data, ndim, shape, chunkshape, blockshape, itemsize, size, urlpath)); 156 | 157 | return 0; 158 | } 159 | 160 | int big_float_frame() { 161 | int ndim = 3; 162 | int64_t shape[] = {200, 310, 214}; 163 | int32_t chunkshape[] = {110, 120, 76}; 164 | int32_t blockshape[] = {57, 52, 35}; 165 | int8_t itemsize = sizeof(float); 166 | int64_t nelem = 1; 167 | for (int i = 0; i < ndim; ++i) { 168 | nelem *= shape[i]; 169 | } 170 | int64_t size = nelem * itemsize; 171 | 172 | float *data = malloc(size); 173 | for (int i = 0; i < nelem; i += 4) { 174 | float j = (float) i; 175 | data[i] = (float) 2.73; 176 | data[i + 1] = (2 + j / 10 + j / 1000); 177 | data[i + 2] = (7 + j / 10 - j / 100); 178 | data[i + 3] = (11 + j / 100 - j / 1000); 179 | } 180 | char *urlpath = "example_big_float_frame.caterva"; 181 | CATERVA_ERROR(frame_generator((int8_t *)data, ndim, shape, chunkshape, blockshape, itemsize, size, urlpath)); 182 | 183 | return 0; 184 | } 185 | 186 | int day_month_temp() { 187 | int ndim = 2; 188 | int64_t shape[] = {400, 3}; 189 | int32_t chunkshape[] = {110, 3}; 190 | int32_t blockshape[] = {57, 3}; 191 | int8_t itemsize = sizeof(float); 192 | int64_t nelem = 1; 193 | for (int i = 0; i < ndim; ++i) { 194 | nelem *= shape[i]; 195 | } 196 | int64_t size = nelem * itemsize; 197 | 198 | float temp_min = -20; 199 | float temp_max = 40; 200 | srand(time(NULL)); 201 | float *data = malloc(size); 202 | for (int i = 0; i < nelem / 3; i++) { 203 | data[i] = (float) (rand() % 31); 204 | data[i + 1] = (float) (rand() % 12); 205 | data[i + 2] = ((float) (rand() % 10000) / 10000 * (temp_max - temp_min) + temp_min); 206 | i += 3; 207 | } 208 | char *urlpath = "example_day_month_temp.caterva"; 209 | CATERVA_ERROR(frame_generator((int8_t *)data, ndim, shape, chunkshape, blockshape, itemsize, size, urlpath)); 210 | 211 | return 0; 212 | } 213 | 214 | int item_prices() { 215 | int ndim = 3; 216 | int64_t shape[] = {12, 25, 250}; 217 | int32_t chunkshape[] = {6, 10, 50}; 218 | int32_t blockshape[] = {3, 5, 10}; 219 | int8_t itemsize = sizeof(float); 220 | int64_t nelem = 1; 221 | for (int i = 0; i < ndim; ++i) { 222 | nelem *= shape[i]; 223 | } 224 | int64_t size = nelem * itemsize; 225 | 226 | float price_min = (float) 1; // if I choose 0.99 results are less aproppiate 227 | float price_max = (float) 251; 228 | float *data = malloc(size); 229 | int index = 0; 230 | for (int month = 1; month <= shape[0]; month ++) { // month (1 to 12) 231 | for (int store = 1; store <= shape[1]; store ++) { // store ID (less to more expensive) 232 | for (int item = 1; item <= shape[2]; item++) { // item ID 233 | srand(item); 234 | data[index] = ((float) store + (float) (3 - (month % 3)) * 235 | ((float) (rand() % 1000) / 1000 * (price_max - price_min) + price_min)); 236 | index++; 237 | } 238 | } 239 | } 240 | char *urlpath = "example_item_prices.caterva"; 241 | CATERVA_ERROR(frame_generator((int8_t *)data, ndim, shape, chunkshape, blockshape, itemsize, size, urlpath)); 242 | 243 | return 0; 244 | } 245 | 246 | 247 | 248 | int main() { 249 | int err; 250 | err = all_eq(); 251 | if (err != CATERVA_SUCCEED) { 252 | printf("\n All_eq error: %d", err); 253 | } 254 | err = cyclic(); 255 | if (err != CATERVA_SUCCEED) { 256 | printf("\n Cyclic error: %d", err); 257 | } 258 | err = many_matches(); 259 | if (err != CATERVA_SUCCEED) { 260 | printf("\n Many_matches error: %d", err); 261 | } 262 | err = float_cyclic(); 263 | if (err != CATERVA_SUCCEED) { 264 | printf("\n Float_cyclic error: %d", err); 265 | } 266 | err = double_same_cells(); 267 | if (err != CATERVA_SUCCEED) { 268 | printf("\n Double_same_cells error: %d", err); 269 | } 270 | err = big_float_frame(); 271 | if (err != CATERVA_SUCCEED) { 272 | printf("\n Double_same_cells error: %d", err); 273 | } 274 | err = day_month_temp(); 275 | if (err != CATERVA_SUCCEED) { 276 | printf("\n Day_month_temp error: %d", err); 277 | } 278 | err = item_prices(); 279 | if (err != CATERVA_SUCCEED) { 280 | printf("\n Item_prices error: %d", err); 281 | } 282 | 283 | return err; 284 | } 285 | -------------------------------------------------------------------------------- /examples/example_oindex.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #include 13 | 14 | int main() { 15 | 16 | int8_t ndim = 2; 17 | int64_t shape[] = {10, 10}; 18 | int32_t chunkshape[] = {4, 4}; 19 | int32_t blockshape[] = {2, 2}; 20 | int8_t itemsize = 8; 21 | 22 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 23 | 24 | caterva_ctx_t *ctx; 25 | caterva_ctx_new(&cfg, &ctx); 26 | 27 | caterva_params_t params = {0}; 28 | params.ndim = ndim; 29 | params.itemsize = itemsize; 30 | for (int i = 0; i < ndim; ++i) { 31 | params.shape[i] = shape[i]; 32 | } 33 | 34 | caterva_storage_t storage = {0}; 35 | for (int i = 0; i < ndim; ++i) { 36 | storage.chunkshape[i] = chunkshape[i]; 37 | storage.blockshape[i] = blockshape[i]; 38 | } 39 | int64_t dataitems = 1; 40 | for (int i = 0; i < ndim; ++i) { 41 | dataitems *= shape[i]; 42 | } 43 | int64_t datasize = dataitems * itemsize; 44 | double *data = malloc(datasize); 45 | for (int i = 0; i < dataitems; ++i) { 46 | data[i] = (double) i; 47 | } 48 | caterva_array_t *arr; 49 | CATERVA_ERROR(caterva_from_buffer(ctx, data, datasize, ¶ms, &storage, &arr)); 50 | free(data); 51 | 52 | int64_t sel0[] = {3, 1, 2}; 53 | int64_t sel1[] = {2, 5}; 54 | int64_t sel2[] = {3, 3, 3, 9,3, 1, 0}; 55 | int64_t *selection[] = {sel0, sel1, sel2}; 56 | int64_t selection_size[] = {sizeof(sel0)/sizeof(int64_t), sizeof(sel1)/(sizeof(int64_t)), sizeof(sel2)/(sizeof(int64_t))}; 57 | int64_t *buffershape = selection_size; 58 | int64_t nitems = 1; 59 | for (int i = 0; i < ndim; ++i) { 60 | nitems *= buffershape[i]; 61 | } 62 | int64_t buffersize = nitems * arr->itemsize; 63 | double *buffer = calloc(nitems, arr->itemsize); 64 | CATERVA_ERROR(caterva_set_orthogonal_selection(ctx, arr, selection, selection_size, buffer, buffershape, buffersize)); 65 | CATERVA_ERROR(caterva_get_orthogonal_selection(ctx, arr, selection, selection_size, buffer, buffershape, buffersize)); 66 | 67 | printf("Results: \n"); 68 | for (int i = 0; i < nitems; ++i) { 69 | if (i % buffershape[1] == 0) { 70 | printf("\n"); 71 | } 72 | printf(" %f ", buffer[i]); 73 | } 74 | printf("\n"); 75 | free(buffer); 76 | CATERVA_ERROR(caterva_free(ctx, &arr)); 77 | CATERVA_ERROR(caterva_ctx_free(&ctx)); 78 | 79 | return 0; 80 | } 81 | -------------------------------------------------------------------------------- /examples/example_plainbuffer.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #include 13 | 14 | int main() { 15 | 16 | int8_t ndim = 2; 17 | int64_t shape[] = {10, 10}; 18 | int32_t chunkshape[] = {4, 4}; 19 | int32_t blockshape[] = {2, 2}; 20 | int8_t itemsize = 8; 21 | 22 | int64_t slice_start[] = {2, 5}; 23 | int64_t slice_stop[] = {3, 6}; 24 | int32_t slice_chunkshape[] = {1, 1}; 25 | int32_t slice_blockshape[] = {1, 1}; 26 | 27 | int64_t nelem = 1; 28 | for (int i = 0; i < ndim; ++i) { 29 | nelem *= shape[i]; 30 | } 31 | int64_t size = nelem * itemsize; 32 | int8_t *data = malloc(size); 33 | 34 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 35 | 36 | caterva_ctx_t *ctx; 37 | caterva_ctx_new(&cfg, &ctx); 38 | 39 | caterva_params_t params = {0}; 40 | params.ndim = ndim; 41 | params.itemsize = itemsize; 42 | for (int i = 0; i < ndim; ++i) { 43 | params.shape[i] = shape[i]; 44 | } 45 | 46 | caterva_storage_t storage = {0}; 47 | for (int i = 0; i < ndim; ++i) { 48 | storage.chunkshape[i] = chunkshape[i]; 49 | storage.blockshape[i] = blockshape[i]; 50 | } 51 | 52 | caterva_array_t *arr; 53 | CATERVA_ERROR(caterva_from_buffer(ctx, data, size, ¶ms, &storage, &arr)); 54 | 55 | 56 | caterva_storage_t slice_storage = {0}; 57 | for (int i = 0; i < ndim; ++i) { 58 | slice_storage.chunkshape[i] = slice_chunkshape[i]; 59 | slice_storage.blockshape[i] = slice_blockshape[i]; 60 | } 61 | 62 | caterva_array_t *slice; 63 | CATERVA_ERROR(caterva_get_slice(ctx, arr, slice_start, slice_stop, &slice_storage, 64 | &slice)); 65 | 66 | CATERVA_ERROR(caterva_squeeze(ctx, slice)); 67 | 68 | uint8_t *buffer; 69 | uint64_t buffer_size = 1; 70 | for (int i = 0; i < slice->ndim; ++i) { 71 | buffer_size *= slice->shape[i]; 72 | } 73 | buffer_size *= slice->itemsize; 74 | buffer = malloc(buffer_size); 75 | 76 | CATERVA_ERROR(caterva_to_buffer(ctx, slice, buffer, buffer_size)); 77 | 78 | // printf("Elapsed seconds: %.5f\n", blosc_elapsed_secs(t0, t1)); 79 | 80 | return 0; 81 | } 82 | -------------------------------------------------------------------------------- /examples/example_plugins_codecs.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | * 11 | * Example program demonstrating use of the Blosc plugins from C code. 12 | * 13 | * To compile this program: 14 | * $ gcc example_plugins_codecs.c -o example_plugins_codecs -lblosc2 15 | * 16 | * To run: 17 | * $ ./example_plugins_codecs 18 | * 19 | * from_buffer: 0.0668 s 20 | * to_buffer: 0.0068 s 21 | * Process finished with exit code 0 22 | */ 23 | 24 | 25 | 26 | #include 27 | #include 28 | #include 29 | #include "../contribs/c-blosc2/plugins/codecs/codecs-registry.c" 30 | #include 31 | 32 | int main() { 33 | blosc_timestamp_t t0, t1; 34 | 35 | blosc2_init(); 36 | int8_t ndim = 2; 37 | uint8_t itemsize = sizeof(int64_t); 38 | 39 | int64_t shape[] = {745, 400}; 40 | int32_t chunkshape[] = {150, 100}; 41 | int32_t blockshape[] = {21, 30}; 42 | 43 | int64_t nbytes = itemsize; 44 | for (int i = 0; i < ndim; ++i) { 45 | nbytes *= shape[i]; 46 | } 47 | 48 | int64_t *src = malloc((size_t) nbytes); 49 | for (int i = 0; i < nbytes / itemsize; ++i) { 50 | src[i] = (int64_t) i; 51 | } 52 | 53 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 54 | cfg.nthreads = 1; 55 | /* 56 | * Use the NDLZ codec through its plugin. 57 | * NDLZ metainformation: - it calls the 4x4 version if meta == 4 58 | - it calls the 8x8 version if meta == 8 59 | */ 60 | cfg.compcodec = BLOSC_CODEC_NDLZ; 61 | cfg.splitmode = BLOSC_ALWAYS_SPLIT; 62 | cfg.compmeta = 4; 63 | cfg.complevel = 5; 64 | // We could use a filter plugin by setting cfg.filters[]. 65 | 66 | caterva_ctx_t *ctx; 67 | caterva_ctx_new(&cfg, &ctx); 68 | 69 | caterva_params_t params = {0}; 70 | params.itemsize = itemsize; 71 | params.ndim = ndim; 72 | for (int i = 0; i < ndim; ++i) { 73 | params.shape[i] = shape[i]; 74 | } 75 | 76 | caterva_storage_t storage = {0}; 77 | for (int i = 0; i < ndim; ++i) { 78 | storage.chunkshape[i] = chunkshape[i]; 79 | storage.blockshape[i] = blockshape[i]; 80 | } 81 | 82 | caterva_array_t *arr; 83 | blosc_set_timestamp(&t0); 84 | CATERVA_ERROR(caterva_from_buffer(ctx, src, nbytes, ¶ms, &storage, &arr)); 85 | blosc_set_timestamp(&t1); 86 | printf("from_buffer: %.4f s\n", blosc_elapsed_secs(t0, t1)); 87 | 88 | int64_t *buffer = malloc(nbytes); 89 | int64_t buffer_size = nbytes; 90 | blosc_set_timestamp(&t0); 91 | CATERVA_ERROR(caterva_to_buffer(ctx, arr, buffer, buffer_size)); 92 | blosc_set_timestamp(&t1); 93 | printf("to_buffer: %.4f s\n", blosc_elapsed_secs(t0, t1)); 94 | 95 | blosc2_destroy(); 96 | 97 | for (int i = 0; i < buffer_size / itemsize; i++) { 98 | if (src[i] != buffer[i]) { 99 | printf("\n Decompressed data differs from original!\n"); 100 | printf("i: %d, data %" PRId64 ", dest %" PRId64 "", i, src[i], buffer[i]); 101 | return -1; 102 | } 103 | } 104 | 105 | free(src); 106 | free(buffer); 107 | 108 | caterva_free(ctx, &arr); 109 | caterva_ctx_free(&ctx); 110 | 111 | return 0; 112 | } 113 | -------------------------------------------------------------------------------- /examples/example_plugins_filters.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | * 11 | * Example program demonstrating use of the Blosc plugins from C code. 12 | * 13 | * To compile this program: 14 | * $ gcc example_plugins_filters.c -o example_plugins_filters -lblosc2 15 | * 16 | * To run: 17 | * $ ./example_plugins_filters 18 | * 19 | * from_buffer: 0.0668 s 20 | * to_buffer: 0.0068 s 21 | * Process finished with exit code 0 22 | */ 23 | 24 | 25 | 26 | #include 27 | #include 28 | #include 29 | #include "../contribs/c-blosc2/plugins/filters/filters-registry.c" 30 | #include 31 | 32 | int main() { 33 | blosc_timestamp_t t0, t1; 34 | 35 | blosc2_init(); 36 | int8_t ndim = 3; 37 | uint8_t itemsize = sizeof(int64_t); 38 | 39 | int64_t shape[] = {745, 400, 350}; 40 | int32_t chunkshape[] = {150, 100, 150}; 41 | int32_t blockshape[] = {21, 30, 27}; 42 | 43 | int64_t nbytes = itemsize; 44 | for (int i = 0; i < ndim; ++i) { 45 | nbytes *= shape[i]; 46 | } 47 | 48 | int64_t *src = malloc((size_t) nbytes); 49 | for (int i = 0; i < nbytes / itemsize; ++i) { 50 | src[i] = (int64_t) i; 51 | } 52 | 53 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 54 | cfg.nthreads = 1; 55 | /* 56 | * Use the NDCELL filter through its plugin. 57 | * NDCELL metainformation: user must specify the parameter meta as the cellshape, so 58 | * if in a 3-dim dataset user specifies meta = 4, then cellshape will be 4x4x4. 59 | */ 60 | cfg.filters[4] = BLOSC_FILTER_NDCELL; 61 | cfg.filtersmeta[4] = 4; 62 | // We could use a codec plugin by setting cfg.compcodec. 63 | 64 | caterva_ctx_t *ctx; 65 | caterva_ctx_new(&cfg, &ctx); 66 | 67 | caterva_params_t params = {0}; 68 | params.itemsize = itemsize; 69 | params.ndim = ndim; 70 | for (int i = 0; i < ndim; ++i) { 71 | params.shape[i] = shape[i]; 72 | } 73 | 74 | caterva_storage_t storage = {0}; 75 | for (int i = 0; i < ndim; ++i) { 76 | storage.chunkshape[i] = chunkshape[i]; 77 | storage.blockshape[i] = blockshape[i]; 78 | } 79 | 80 | caterva_array_t *arr; 81 | blosc_set_timestamp(&t0); 82 | CATERVA_ERROR(caterva_from_buffer(ctx, src, nbytes, ¶ms, &storage, &arr)); 83 | blosc_set_timestamp(&t1); 84 | printf("from_buffer: %.4f s\n", blosc_elapsed_secs(t0, t1)); 85 | 86 | int64_t *buffer = malloc(nbytes); 87 | int64_t buffer_size = nbytes; 88 | blosc_set_timestamp(&t0); 89 | CATERVA_ERROR(caterva_to_buffer(ctx, arr, buffer, buffer_size)); 90 | blosc_set_timestamp(&t1); 91 | printf("to_buffer: %.4f s\n", blosc_elapsed_secs(t0, t1)); 92 | 93 | blosc2_destroy(); 94 | 95 | for (int i = 0; i < buffer_size / itemsize; i++) { 96 | if (src[i] != buffer[i]) { 97 | printf("\n Decompressed data differs from original!\n"); 98 | printf("i: %d, data %" PRId64 ", dest %" PRId64 "", i, src[i], buffer[i]); 99 | return -1; 100 | } 101 | } 102 | 103 | free(src); 104 | free(buffer); 105 | 106 | caterva_free(ctx, &arr); 107 | caterva_ctx_free(&ctx); 108 | 109 | return 0; 110 | } 111 | -------------------------------------------------------------------------------- /examples/example_print_meta.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2019-present Blosc Development team 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | * 10 | * Example program demonstrating how to print metainfo from a caterva frame. 11 | * You can build frames with example_frame_generator.c 12 | * 13 | * Usage: 14 | * $ ./example_print_meta 15 | * 16 | * Example of output: 17 | * $ ./caterva_example_print_meta example_big_float_frame.caterva 18 | * Caterva metalayer parameters: 19 | * Ndim: 3 20 | * Shape: 200, 310, 214 21 | * Chunkshape: 110, 120, 76 22 | * Blockshape: 57, 52, 35 23 | * 24 | */ 25 | 26 | # include 27 | 28 | int print_meta(char *urlpath) { 29 | 30 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 31 | caterva_ctx_t *ctx; 32 | caterva_ctx_new(&cfg, &ctx); 33 | caterva_array_t *arr; 34 | CATERVA_ERROR(caterva_open(ctx, urlpath, &arr)); 35 | caterva_print_meta(arr); 36 | caterva_free(ctx, &arr); 37 | caterva_ctx_free(&ctx); 38 | 39 | return 0; 40 | } 41 | 42 | int main(int argc, char *argv[]) { 43 | if (argc != 2) { 44 | printf("Usage: %s urlpath", argv[0]); 45 | exit(-1); 46 | } 47 | 48 | char* urlpath = argv[1]; 49 | print_meta(urlpath); 50 | 51 | return 0; 52 | } 53 | -------------------------------------------------------------------------------- /examples/example_serialize.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | # include 13 | 14 | int main() { 15 | 16 | int8_t ndim = 2; 17 | int64_t shape[] = {10, 10}; 18 | int32_t chunkshape[] = {4, 4}; 19 | int32_t blockshape[] = {2, 2}; 20 | int8_t itemsize = 8; 21 | 22 | int64_t slice_start[] = {2, 5}; 23 | int64_t slice_stop[] = {2, 6}; 24 | int32_t slice_chunkshape[] = {0, 1}; 25 | int32_t slice_blockshape[] = {0, 1}; 26 | 27 | int64_t nelem = 1; 28 | for (int i = 0; i < ndim; ++i) { 29 | nelem *= shape[i]; 30 | } 31 | int64_t size = nelem * itemsize; 32 | double *data = malloc(size); 33 | 34 | for (int i = 0; i < nelem; ++i) { 35 | data[i] = i; 36 | } 37 | 38 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 39 | 40 | caterva_ctx_t *ctx; 41 | caterva_ctx_new(&cfg, &ctx); 42 | 43 | caterva_params_t params = {0}; 44 | params.ndim = ndim; 45 | params.itemsize = itemsize; 46 | for (int i = 0; i < ndim; ++i) { 47 | params.shape[i] = shape[i]; 48 | } 49 | 50 | caterva_storage_t storage = {0}; 51 | for (int i = 0; i < ndim; ++i) { 52 | storage.chunkshape[i] = chunkshape[i]; 53 | storage.blockshape[i] = blockshape[i]; 54 | } 55 | storage.contiguous = false; 56 | 57 | caterva_array_t *arr; 58 | CATERVA_ERROR(caterva_from_buffer(ctx, data, size, ¶ms, &storage, &arr)); 59 | 60 | uint8_t *cframe; 61 | int64_t cframe_len; 62 | bool needs_free; 63 | CATERVA_ERROR(caterva_to_cframe(ctx, arr, &cframe, &cframe_len, &needs_free)); 64 | 65 | caterva_array_t *dest; 66 | CATERVA_ERROR(caterva_from_cframe(ctx, cframe, cframe_len, true, &dest)); 67 | 68 | /* Fill dest array with caterva_array_t data */ 69 | uint8_t *data_dest = malloc(size); 70 | CATERVA_ERROR(caterva_to_buffer(ctx, dest, data_dest, size)); 71 | 72 | for (int i = 0; i < nelem; ++i) { 73 | if (data[i] != data_dest[i] && data[i] != i) { 74 | return -1; 75 | } 76 | } 77 | 78 | /* Free mallocs */ 79 | free(data); 80 | free(data_dest); 81 | 82 | caterva_free(ctx, &arr); 83 | caterva_free(ctx, &dest); 84 | 85 | return 0; 86 | } 87 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 2 | # Copyright (C) 2019-present Blosc Development team 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # You may select, at your option, one of the above-listed licenses. 9 | 10 | file(GLOB SOURCES test_*.c) 11 | 12 | foreach(source ${SOURCES}) 13 | get_filename_component(target_name ${source} NAME_WE) 14 | set(target caterva_${target_name}) 15 | add_executable(${target} ${target_name}.c) 16 | target_link_libraries(${target} caterva_static ${LIBS}) 17 | add_test(NAME ${target} COMMAND ${target}) 18 | set_tests_properties(${target} PROPERTIES LABELS "caterva") 19 | endforeach(source) 20 | -------------------------------------------------------------------------------- /tests/cutest.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2021 Aleix Alcacer 3 | Copyright (C) 2021 The Blosc Developers 4 | http://blosc.org 5 | License: BSD (see LICENSE.txt) 6 | 7 | See LICENSE.txt for details about copyright and rights to use. 8 | */ 9 | 10 | #ifndef CUTEST_CUTEST_H 11 | #define CUTEST_CUTEST_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | 18 | #define RED "\033[31m" /* Red */ 19 | #define GREEN "\033[32m" /* Green */ 20 | #define RESET "\033[0m" 21 | 22 | 23 | #define CUNIT_OK 0 24 | #define CUNIT_FAIL 1 25 | 26 | 27 | #define CUTEST_DATA(...) __VA_ARGS__ 28 | 29 | 30 | #define CUTEST_PARAMETRIZE(name, type, ...) \ 31 | do { \ 32 | type cutest_##name[] = {__VA_ARGS__}; \ 33 | _cutest_parametrize(#name, cutest_##name, \ 34 | sizeof(cutest_##name) / sizeof(type), sizeof(type)); \ 35 | } while(0) 36 | 37 | #define CUTEST_PARAMETRIZE2(name, type, params_len, params) \ 38 | do { \ 39 | (type) *cutest_##name = params; \ 40 | _cutest_parametrize(#name, cutest_##name, params_len, sizeof(type)); \ 41 | } while(0) 42 | 43 | #define CUTEST_GET_PARAMETER(name, type) \ 44 | type name = * (type *) _cutest_get_parameter(#name) 45 | 46 | #define CUTEST_TEST_DATA(sname) \ 47 | struct sname##_data 48 | 49 | #define CUTEST_TEST_SETUP(sname) \ 50 | void sname##_setup(struct sname##_data *data) 51 | 52 | #define CUTEST_TEST_TEARDOWN(sname) \ 53 | void sname##_teardown(struct sname##_data *data) 54 | 55 | #define CUTEST_TEST_TEST(sname) \ 56 | static struct sname##_data test_##sname##_data; \ 57 | CUTEST_TEST_SETUP(sname); \ 58 | CUTEST_TEST_TEARDOWN(sname); \ 59 | int sname##_test(struct sname##_data* data); \ 60 | int sname##_test(struct sname##_data* data) \ 61 | 62 | 63 | #define CUTEST_TEST_RUN(sname) \ 64 | _cutest_setup(); \ 65 | sname##_setup(&test_##sname##_data); \ 66 | int rc = _cutest_run((int (*)(void *)) sname##_test, \ 67 | (void *) &test_##sname##_data, \ 68 | #sname); \ 69 | sname##_teardown(&test_##sname##_data); \ 70 | _cutest_teardown(); \ 71 | return rc; 72 | 73 | 74 | #define CUTEST_ASSERT(msg, cond) \ 75 | do { \ 76 | if (!(cond)) { \ 77 | sprintf(_cutest_error_msg, "Error: %s %s:%d", msg, __FILE__, __LINE__); \ 78 | return CUNIT_FAIL; \ 79 | } \ 80 | } while(0) 81 | 82 | 83 | #define CUTEST_PARAMS_MAX 16 84 | #define MAXLEN_TESTNAME 1024 85 | 86 | 87 | typedef struct { 88 | char *name; 89 | uint8_t *params; 90 | int32_t params_len; 91 | int32_t param_size; 92 | } cutest_param_t; 93 | 94 | static cutest_param_t cutest_params[CUTEST_PARAMS_MAX] = {0}; 95 | static int8_t cutest_params_ind[CUTEST_PARAMS_MAX] = {0}; 96 | 97 | 98 | void _cutest_parametrize(char *name, void *params, int32_t params_len, int32_t param_size) { 99 | int i = 0; 100 | while (cutest_params[i].name != NULL) { 101 | i++; 102 | } 103 | uint8_t *new_params = malloc(param_size * params_len); 104 | char *new_name = strdup(name); 105 | memcpy(new_params, params, param_size * params_len); 106 | cutest_params[i].name = new_name; 107 | cutest_params[i].params = new_params; 108 | cutest_params[i].param_size = param_size; 109 | cutest_params[i].params_len = params_len; 110 | } 111 | 112 | uint8_t *_cutest_get_parameter(char *name) { 113 | int i = 0; 114 | while (strcmp(cutest_params[i].name, name) != 0) { 115 | i++; 116 | } 117 | return cutest_params[i].params + cutest_params_ind[i] * cutest_params[i].param_size; 118 | } 119 | 120 | 121 | void _cutest_setup() { 122 | for (int i = 0; i < CUTEST_PARAMS_MAX; ++i) { 123 | cutest_params[i].name = NULL; 124 | } 125 | } 126 | 127 | 128 | void _cutest_teardown() { 129 | int i = 0; 130 | while (cutest_params[i].name != NULL) { 131 | free(cutest_params[i].params); 132 | free(cutest_params[i].name); 133 | i++; 134 | } 135 | } 136 | 137 | 138 | char _cutest_error_msg[1024]; 139 | 140 | 141 | int _cutest_run(int (*test)(void *), void *test_data, char *name) { 142 | int cutest_ok = 0; 143 | int cutest_failed = 0; 144 | int cutest_total = 0; 145 | 146 | int nparams = 0; 147 | while (cutest_params[nparams].name != NULL) { 148 | nparams++; 149 | } 150 | 151 | int niters = 1; 152 | for (int i = 0; i < nparams; ++i) { 153 | niters *= cutest_params[i].params_len; 154 | } 155 | 156 | int32_t params_strides[CUTEST_PARAMS_MAX] = {0}; 157 | params_strides[0] = 1; 158 | for (int i = 1; i < nparams; ++i) { 159 | params_strides[i] = params_strides[i - 1] * cutest_params[i - 1].params_len; 160 | } 161 | 162 | char test_name[MAXLEN_TESTNAME + 1]; 163 | char aux[MAXLEN_TESTNAME + 1]; 164 | uint8_t count = 0; 165 | int num = niters; 166 | do { 167 | count++; 168 | num /= 10; 169 | } while (num != 0); 170 | for (int niter = 0; niter < niters; ++niter) { 171 | sprintf(test_name, "[%0*d/%d] %s(", count, niter + 1, niters, name); 172 | for (int i = 0; i < nparams; ++i) { 173 | cutest_params_ind[i] = (int8_t) (niter / params_strides[i] % 174 | cutest_params[i].params_len); 175 | strcpy(aux, test_name); 176 | snprintf(test_name, MAXLEN_TESTNAME, "%s%s[%d], ", aux, cutest_params[i].name, 177 | cutest_params_ind[i]); 178 | } 179 | test_name[strlen(test_name) - 1] = 0; 180 | strncpy(&test_name[strlen(test_name) - 1], ")", 1); 181 | if (nparams == 0) { 182 | test_name[strlen(test_name) - 1] = 0; 183 | } 184 | printf("%s ", test_name); 185 | 186 | cutest_total++; 187 | 188 | int rc = test(test_data); 189 | if (rc == CUNIT_OK) { 190 | cutest_ok++; 191 | fprintf(stdout, GREEN "[ OK ]\n" RESET); 192 | } else { 193 | cutest_failed++; 194 | fprintf(stdout, RED "[FAILED]\n" RESET); 195 | } 196 | if (_cutest_error_msg[0] != 0) { 197 | fprintf(stdout, RED " %s\n" RESET, _cutest_error_msg); 198 | _cutest_error_msg[0] = 0; 199 | } 200 | } 201 | 202 | printf("\nTEST RESULTS: %d tests (%d ok, %d failed)\n", 203 | cutest_total, cutest_ok, cutest_failed); 204 | 205 | return cutest_failed; 206 | } 207 | 208 | 209 | #endif //CUTEST_CUTEST_H 210 | -------------------------------------------------------------------------------- /tests/test_append.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | 13 | #include "test_common.h" 14 | 15 | typedef struct { 16 | int8_t ndim; 17 | int64_t shape[CATERVA_MAX_DIM]; 18 | int32_t chunkshape[CATERVA_MAX_DIM]; 19 | int32_t blockshape[CATERVA_MAX_DIM]; 20 | int64_t buffershape[CATERVA_MAX_DIM]; 21 | int8_t axis; 22 | } test_shapes_t; 23 | 24 | 25 | CUTEST_TEST_DATA(append) { 26 | caterva_ctx_t *ctx; 27 | }; 28 | 29 | 30 | CUTEST_TEST_SETUP(append) { 31 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 32 | cfg.nthreads = 2; 33 | cfg.compcodec = BLOSC_BLOSCLZ; 34 | caterva_ctx_new(&cfg, &data->ctx); 35 | 36 | // Add parametrizations 37 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA( 38 | 1, 39 | 2, 40 | 4, 41 | 8, 42 | )); 43 | 44 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 45 | {false, false}, 46 | {true, false}, 47 | {true, true}, 48 | {false, true}, 49 | )); 50 | 51 | 52 | CUTEST_PARAMETRIZE(shapes, test_shapes_t, CUTEST_DATA( 53 | {1, {5}, {3}, {2}, {10}, 0}, 54 | {2, {18, 6}, {6, 6}, {3, 3}, {18, 12}, 1}, 55 | {3, {12, 10, 14}, {3, 5, 9}, {3, 4, 4}, {12, 10, 18}, 2}, 56 | {4, {10, 10, 5, 5}, {5, 7, 3, 3}, {2, 2, 1, 1}, {10, 10, 5, 30}, 3}, 57 | 58 | )); 59 | } 60 | 61 | CUTEST_TEST_TEST(append) { 62 | CUTEST_GET_PARAMETER(backend, _test_backend); 63 | CUTEST_GET_PARAMETER(shapes, test_shapes_t); 64 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 65 | 66 | char *urlpath = "test_append_shape.b2frame"; 67 | caterva_remove(data->ctx, urlpath); 68 | 69 | caterva_params_t params; 70 | params.itemsize = itemsize; 71 | params.ndim = shapes.ndim; 72 | for (int i = 0; i < params.ndim; ++i) { 73 | params.shape[i] = shapes.shape[i]; 74 | } 75 | 76 | caterva_storage_t storage = {0}; 77 | if (backend.persistent) { 78 | storage.urlpath = urlpath; 79 | } 80 | storage.contiguous = backend.contiguous; 81 | for (int i = 0; i < params.ndim; ++i) { 82 | storage.chunkshape[i] = shapes.chunkshape[i]; 83 | storage.blockshape[i] = shapes.blockshape[i]; 84 | } 85 | 86 | int64_t buffersize = itemsize; 87 | for (int i = 0; i < params.ndim; ++i) { 88 | buffersize *= shapes.buffershape[i]; 89 | } 90 | 91 | /* Create caterva_array_t with original data */ 92 | caterva_array_t *src; 93 | uint8_t *value = malloc(itemsize); 94 | int8_t fill_value = 1; 95 | switch (itemsize) { 96 | case 8: 97 | ((int64_t *) value)[0] = (int64_t) fill_value; 98 | break; 99 | case 4: 100 | ((int32_t *) value)[0] = (int32_t) fill_value; 101 | break; 102 | case 2: 103 | ((int16_t *) value)[0] = (int16_t) fill_value; 104 | break; 105 | case 1: 106 | ((int8_t *) value)[0] = fill_value; 107 | break; 108 | default: 109 | break; 110 | } 111 | CATERVA_ERROR(caterva_full(data->ctx, ¶ms, &storage, value, &src)); 112 | 113 | uint8_t *buffer = data->ctx->cfg->alloc(buffersize); 114 | fill_buf(buffer, itemsize, buffersize / itemsize); 115 | CATERVA_ERROR(caterva_append(data->ctx, src, buffer, buffersize, shapes.axis)); 116 | 117 | int64_t start[CATERVA_MAX_DIM] = {0}; 118 | start[shapes.axis] = shapes.shape[shapes.axis]; 119 | int64_t stop[CATERVA_MAX_DIM]; 120 | for (int i = 0; i < shapes.ndim; ++i) { 121 | stop[i] = shapes.shape[i]; 122 | } 123 | stop[shapes.axis] = shapes.shape[shapes.axis] + shapes.buffershape[shapes.axis]; 124 | 125 | /* Fill buffer with a slice from the new chunks */ 126 | uint8_t *res_buffer = data->ctx->cfg->alloc(buffersize); 127 | CATERVA_ERROR(caterva_get_slice_buffer(data->ctx, src, start, stop, res_buffer, 128 | shapes.buffershape, buffersize)); 129 | 130 | for (uint64_t i = 0; i < (uint64_t) buffersize / itemsize; ++i) { 131 | switch (itemsize) { 132 | case 8: 133 | CUTEST_ASSERT("Elements are not equal!", 134 | ((uint64_t *) buffer)[i] == ((uint64_t *) res_buffer)[i]); 135 | break; 136 | case 4: 137 | CUTEST_ASSERT("Elements are not equal!", 138 | ((uint32_t *) buffer)[i] == ((uint32_t *) res_buffer)[i]); 139 | break; 140 | case 2: 141 | CUTEST_ASSERT("Elements are not equal!", 142 | ((uint16_t *) buffer)[i] == ((uint16_t *) res_buffer)[i]); 143 | break; 144 | case 1: 145 | CUTEST_ASSERT("Elements are not equal!", 146 | ((uint8_t *) buffer)[i] == ((uint8_t *) res_buffer)[i]); 147 | break; 148 | default: 149 | CATERVA_TEST_ASSERT(CATERVA_ERR_INVALID_ARGUMENT); 150 | } 151 | } 152 | /* Free mallocs */ 153 | free(value); 154 | data->ctx->cfg->free(buffer); 155 | data->ctx->cfg->free(res_buffer); 156 | 157 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 158 | caterva_remove(data->ctx, urlpath); 159 | 160 | return 0; 161 | } 162 | 163 | CUTEST_TEST_TEARDOWN(append) { 164 | caterva_ctx_free(&data->ctx); 165 | } 166 | 167 | int main() { 168 | CUTEST_TEST_RUN(append); 169 | } 170 | -------------------------------------------------------------------------------- /tests/test_common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #ifndef CATERVA_TEST_COMMON_H 13 | #define CATERVA_TEST_COMMON_H 14 | 15 | #include 16 | #include "cutest.h" 17 | 18 | 19 | #define CATERVA_TEST_ASSERT(rc) CUTEST_ASSERT(print_error(rc), (rc) == CATERVA_SUCCEED); 20 | 21 | #ifdef __GNUC__ 22 | #define CATERVA_TEST_UNUSED __attribute__((unused)) 23 | #else 24 | #define CATERVA_TEST_UNUSED 25 | #endif 26 | 27 | static bool fill_buf(void *buf, uint8_t itemsize, size_t buf_size) CATERVA_TEST_UNUSED; 28 | static bool fill_buf(void *buf, uint8_t itemsize, size_t buf_size) { 29 | switch (itemsize) { 30 | case 8: 31 | for (size_t i = 0; i < buf_size; ++i) { 32 | ((uint64_t *) buf)[i] = (uint64_t) i + 1; 33 | } 34 | break; 35 | case 4: 36 | for (size_t i = 0; i < buf_size; ++i) { 37 | ((uint32_t *) buf)[i] = (uint32_t) i + 1; 38 | } 39 | break; 40 | case 2: 41 | for (size_t i = 0; i < buf_size; ++i) { 42 | ((uint16_t *) buf)[i] = (uint16_t ) i + 1; 43 | } 44 | break; 45 | case 1: 46 | for (size_t i = 0; i < buf_size; ++i) { 47 | ((uint8_t *) buf)[i] = (uint8_t ) i + 1; 48 | } 49 | break; 50 | default: 51 | return false; 52 | } 53 | return true; 54 | } 55 | 56 | 57 | /* Tests data */ 58 | 59 | typedef struct { 60 | int8_t ndim; 61 | int64_t shape[CATERVA_MAX_DIM]; 62 | int32_t chunkshape[CATERVA_MAX_DIM]; 63 | int32_t blockshape[CATERVA_MAX_DIM]; 64 | } _test_shapes; 65 | 66 | 67 | typedef struct { 68 | bool contiguous; 69 | bool persistent; 70 | } _test_backend; 71 | 72 | 73 | void caterva_default_parameters() { 74 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA(1, 2, 4, 8)); 75 | CUTEST_PARAMETRIZE(shapes, _test_shapes, CUTEST_DATA( 76 | {2, {100, 100}, {20, 20}, {10, 10}}, 77 | {3, {100, 55, 123}, {31, 5, 22}, {4, 4, 4}}, 78 | {3, {100, 0, 12}, {31, 0, 12}, {10, 0, 12}}, 79 | {4, {50, 160, 31, 12}, {25, 20, 20, 10}, {5, 5, 5, 10}}, 80 | {5, {1, 1, 1024, 1, 1}, {1, 1, 500, 1, 1}, {1, 1, 200, 1, 1}}, 81 | {6, {5, 1, 200, 3, 1, 2}, {5, 1, 50, 2, 1, 2}, {2, 1, 20, 2, 1, 2}}, 82 | )); 83 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 84 | {false, false}, 85 | {true, false}, 86 | {false, true}, 87 | {true, true}, 88 | )); 89 | } 90 | 91 | 92 | #define CATERVA_TEST_ASSERT_BUFFER(buffer1, buffer2, buffersize) \ 93 | for (int i = 0; i < (buffersize); ++i) { \ 94 | CUTEST_ASSERT("elements are not equals!", (buffer1)[i] == (buffer2)[i]); \ 95 | } 96 | 97 | 98 | #endif //CATERVA_TEST_COMMON_H 99 | -------------------------------------------------------------------------------- /tests/test_copy.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #include "test_common.h" 13 | 14 | typedef struct { 15 | int8_t ndim; 16 | int64_t shape[CATERVA_MAX_DIM]; 17 | int32_t chunkshape[CATERVA_MAX_DIM]; 18 | int32_t blockshape[CATERVA_MAX_DIM]; 19 | int32_t chunkshape2[CATERVA_MAX_DIM]; 20 | int32_t blockshape2[CATERVA_MAX_DIM]; 21 | } test_shapes_t; 22 | 23 | 24 | CUTEST_TEST_DATA(copy) { 25 | caterva_ctx_t *ctx; 26 | }; 27 | 28 | 29 | CUTEST_TEST_SETUP(copy) { 30 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 31 | cfg.nthreads = 2; 32 | cfg.compcodec = BLOSC_BLOSCLZ; 33 | caterva_ctx_new(&cfg, &data->ctx); 34 | 35 | // Add parametrizations 36 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA( 37 | 2, 38 | 4, 39 | )); 40 | CUTEST_PARAMETRIZE(shapes, test_shapes_t, CUTEST_DATA( 41 | {2, {100, 100}, {20, 20}, {10, 10}, 42 | {20, 20}, {10, 10}}, 43 | {3, {100, 55, 123}, {31, 5, 22}, {4, 4, 4}, 44 | {50, 15, 20}, {10, 4, 4}}, 45 | {3, {100, 0, 12}, {31, 0, 12}, {10, 0, 12}, 46 | {50, 0, 12}, {25, 0, 6}}, 47 | // The following cases are skipped to reduce the execution time of the test 48 | // {4, {25, 60, 31, 12}, {12, 20, 20, 10}, {5, 5, 5, 10}, 49 | // {25, 20, 20, 10}, {5, 5, 5, 10}}, 50 | // {5, {1, 1, 1024, 1, 1}, {1, 1, 500, 1, 1}, {1, 1, 200, 1, 1}, 51 | // {1, 1, 300, 1, 1}, {1, 1, 50, 1, 1}}, 52 | // {6, {5, 1, 100, 3, 1, 2}, {4, 1, 50, 2, 1, 2}, {2, 1, 20, 2, 1, 2}, 53 | // {4, 1, 50, 2, 1, 1}, {2, 1, 20, 2, 1, 1}}, 54 | )); 55 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 56 | {false, false}, 57 | {true, false}, 58 | {false, true}, 59 | {true, true}, 60 | )); 61 | 62 | CUTEST_PARAMETRIZE(backend2, _test_backend, CUTEST_DATA( 63 | {false, false}, 64 | {false, false}, 65 | {true, false}, 66 | {false, true}, 67 | {true, true}, 68 | )); 69 | } 70 | 71 | CUTEST_TEST_TEST(copy) { 72 | CUTEST_GET_PARAMETER(backend, _test_backend); 73 | CUTEST_GET_PARAMETER(shapes, test_shapes_t); 74 | CUTEST_GET_PARAMETER(backend2, _test_backend); 75 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 76 | 77 | char *urlpath = "test_copy.b2frame"; 78 | char *urlpath2 = "test_copy2.b2frame"; 79 | 80 | caterva_remove(data->ctx, urlpath); 81 | caterva_remove(data->ctx, urlpath2); 82 | 83 | caterva_params_t params; 84 | params.itemsize = itemsize; 85 | params.ndim = shapes.ndim; 86 | for (int i = 0; i < shapes.ndim; ++i) { 87 | params.shape[i] = shapes.shape[i]; 88 | } 89 | 90 | double datatoserialize = 8.34; 91 | 92 | caterva_storage_t storage = {0}; 93 | if (backend.persistent) { 94 | storage.urlpath = urlpath; 95 | } else { 96 | storage.urlpath = NULL; 97 | } 98 | storage.contiguous = backend.contiguous; 99 | for (int i = 0; i < params.ndim; ++i) { 100 | storage.chunkshape[i] = shapes.chunkshape[i]; 101 | storage.blockshape[i] = shapes.blockshape[i]; 102 | } 103 | storage.nmetalayers = 1; 104 | storage.metalayers[0].name = "random"; 105 | storage.metalayers[0].sdata = (uint8_t *) &datatoserialize; 106 | storage.metalayers[0].size = 8; 107 | 108 | 109 | /* Create original data */ 110 | size_t buffersize = itemsize; 111 | for (int i = 0; i < params.ndim; ++i) { 112 | buffersize *= (size_t) params.shape[i]; 113 | } 114 | uint8_t *buffer = malloc(buffersize); 115 | CUTEST_ASSERT("Buffer filled incorrectly", fill_buf(buffer, itemsize, (buffersize / itemsize))); 116 | 117 | /* Create caterva_array_t with original data */ 118 | caterva_array_t *src; 119 | CATERVA_TEST_ASSERT(caterva_from_buffer(data->ctx, buffer, buffersize, ¶ms, &storage, 120 | &src)); 121 | 122 | /* Assert the metalayers creation */ 123 | bool exists; 124 | CATERVA_TEST_ASSERT(caterva_meta_exists(data->ctx, src, "random", &exists)); 125 | if (!exists) { 126 | CATERVA_TEST_ASSERT(CATERVA_ERR_BLOSC_FAILED); 127 | } 128 | caterva_metalayer_t meta; 129 | CATERVA_TEST_ASSERT(caterva_meta_get(data->ctx, src, "random", &meta)); 130 | double serializeddata = *((double *) meta.sdata); 131 | if (serializeddata != datatoserialize) { 132 | CATERVA_TEST_ASSERT(CATERVA_ERR_BLOSC_FAILED); 133 | } 134 | 135 | CATERVA_TEST_ASSERT(caterva_vlmeta_add(data->ctx, src, &meta)); 136 | free(meta.sdata); 137 | free(meta.name); 138 | 139 | 140 | /* Create storage for dest container */ 141 | caterva_storage_t storage2 = {0}; 142 | 143 | if (backend2.persistent) { 144 | storage2.urlpath = urlpath2; 145 | } else { 146 | storage2.urlpath = NULL; 147 | } 148 | storage2.contiguous = backend2.contiguous; 149 | for (int i = 0; i < shapes.ndim; ++i) { 150 | storage2.chunkshape[i] = shapes.chunkshape2[i]; 151 | storage2.blockshape[i] = shapes.blockshape2[i]; 152 | } 153 | 154 | 155 | caterva_array_t *dest; 156 | CATERVA_TEST_ASSERT(caterva_copy(data->ctx, src, &storage2, &dest)); 157 | 158 | /* Assert the metalayers creation */ 159 | CATERVA_TEST_ASSERT(caterva_meta_get(data->ctx, dest, "random", &meta)); 160 | serializeddata = *((double *) meta.sdata); 161 | if (serializeddata != datatoserialize) { 162 | CATERVA_TEST_ASSERT(CATERVA_ERR_BLOSC_FAILED); 163 | } 164 | free(meta.sdata); 165 | free(meta.name); 166 | 167 | caterva_metalayer_t vlmeta; 168 | CATERVA_TEST_ASSERT(caterva_vlmeta_get(data->ctx, dest, "random", &vlmeta)); 169 | serializeddata = *((double *) vlmeta.sdata); 170 | if (serializeddata != datatoserialize) { 171 | CATERVA_TEST_ASSERT(CATERVA_ERR_BLOSC_FAILED); 172 | } 173 | free(vlmeta.sdata); 174 | free(vlmeta.name); 175 | 176 | 177 | uint8_t *buffer_dest = malloc(buffersize); 178 | CATERVA_TEST_ASSERT(caterva_to_buffer(data->ctx, dest, buffer_dest, buffersize)); 179 | 180 | /* Testing */ 181 | CATERVA_TEST_ASSERT_BUFFER(buffer, buffer_dest, (int) buffersize); 182 | 183 | /* Free mallocs */ 184 | free(buffer); 185 | free(buffer_dest); 186 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 187 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &dest)); 188 | 189 | caterva_remove(data->ctx, urlpath); 190 | caterva_remove(data->ctx, urlpath2); 191 | 192 | return 0; 193 | } 194 | 195 | CUTEST_TEST_TEARDOWN(copy) { 196 | caterva_ctx_free(&data->ctx); 197 | } 198 | 199 | int main() { 200 | CUTEST_TEST_RUN(copy); 201 | } 202 | -------------------------------------------------------------------------------- /tests/test_delete.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | 13 | #include "test_common.h" 14 | 15 | typedef struct { 16 | int8_t ndim; 17 | int64_t shape[CATERVA_MAX_DIM]; 18 | int32_t chunkshape[CATERVA_MAX_DIM]; 19 | int32_t blockshape[CATERVA_MAX_DIM]; 20 | int8_t axis; 21 | int64_t start; 22 | int64_t delete_len; 23 | } test_shapes_t; 24 | 25 | 26 | CUTEST_TEST_DATA(delete) { 27 | caterva_ctx_t *ctx; 28 | }; 29 | 30 | 31 | CUTEST_TEST_SETUP(delete) { 32 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 33 | cfg.nthreads = 2; 34 | cfg.compcodec = BLOSC_LZ4; 35 | caterva_ctx_new(&cfg, &data->ctx); 36 | 37 | // Add parametrizations 38 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA( 39 | 1, 40 | 2, 41 | 4, 42 | 8, 43 | )); 44 | 45 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 46 | {false, false}, 47 | {true, false}, 48 | {true, true}, 49 | {false, true}, 50 | )); 51 | 52 | 53 | CUTEST_PARAMETRIZE(shapes, test_shapes_t, CUTEST_DATA( 54 | {1, {10}, {3}, {2}, 0, 5, 5}, // delete the end 55 | {2, {18, 12}, {6, 6}, {3, 3}, 1, 0, 6}, // delete at the beginning 56 | {3, {12, 10, 27}, {3, 5, 9}, {3, 4, 4}, 2, 9, 9}, // delete in the middle 57 | {4, {10, 10, 5, 30}, {5, 7, 3, 3}, {2, 2, 1, 1}, 3, 12, 9}, // delete in the middle 58 | 59 | )); 60 | } 61 | 62 | CUTEST_TEST_TEST(delete) { 63 | CUTEST_GET_PARAMETER(backend, _test_backend); 64 | CUTEST_GET_PARAMETER(shapes, test_shapes_t); 65 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 66 | 67 | char *urlpath = "test_delete.b2frame"; 68 | caterva_remove(data->ctx, urlpath); 69 | 70 | caterva_params_t params; 71 | params.itemsize = itemsize; 72 | params.ndim = shapes.ndim; 73 | for (int i = 0; i < params.ndim; ++i) { 74 | params.shape[i] = shapes.shape[i]; 75 | } 76 | 77 | caterva_storage_t storage = {0}; 78 | if (backend.persistent) { 79 | storage.urlpath = urlpath; 80 | } 81 | storage.contiguous = backend.contiguous; 82 | for (int i = 0; i < params.ndim; ++i) { 83 | storage.chunkshape[i] = shapes.chunkshape[i]; 84 | storage.blockshape[i] = shapes.blockshape[i]; 85 | } 86 | 87 | /* Create caterva_array_t with original data */ 88 | caterva_array_t *src; 89 | uint8_t *value = malloc(itemsize); 90 | int8_t fill_value = 1; 91 | switch (itemsize) { 92 | case 8: 93 | ((int64_t *) value)[0] = (int64_t) fill_value; 94 | break; 95 | case 4: 96 | ((int32_t *) value)[0] = (int32_t) fill_value; 97 | break; 98 | case 2: 99 | ((int16_t *) value)[0] = (int16_t) fill_value; 100 | break; 101 | case 1: 102 | ((int8_t *) value)[0] = fill_value; 103 | break; 104 | default: 105 | break; 106 | } 107 | CATERVA_ERROR(caterva_full(data->ctx, ¶ms, &storage, value, &src)); 108 | 109 | int64_t bufferlen = 1; 110 | int64_t stop[CATERVA_MAX_DIM]; 111 | int64_t buffer_shape[CATERVA_MAX_DIM]; 112 | for (int i = 0; i < params.ndim; ++i) { 113 | if (i != shapes.axis) { 114 | bufferlen *= shapes.shape[i]; 115 | stop[i] = shapes.shape[i]; 116 | buffer_shape[i] = shapes.shape[i]; 117 | } 118 | else { 119 | bufferlen *= shapes.delete_len; 120 | stop[i] = shapes.start + shapes.delete_len; 121 | buffer_shape[i] = shapes.delete_len; 122 | } 123 | } 124 | // Set future deleted values to 0 125 | int64_t start[CATERVA_MAX_DIM] = {0}; 126 | start[shapes.axis] = shapes.start; 127 | uint8_t *buffer = calloc((size_t) bufferlen, (size_t) itemsize); 128 | CATERVA_ERROR(caterva_set_slice_buffer(data->ctx, buffer, buffer_shape, bufferlen * itemsize, start, stop, src)); 129 | 130 | CATERVA_ERROR(caterva_delete(data->ctx, src, shapes.axis, shapes.start, shapes.delete_len)); 131 | 132 | int64_t newshape[CATERVA_MAX_DIM] = {0}; 133 | for (int i = 0; i < shapes.ndim; ++i) { 134 | newshape[i] = shapes.shape[i]; 135 | } 136 | newshape[shapes.axis] -= shapes.delete_len; 137 | 138 | // Create aux array to compare values 139 | caterva_array_t *aux; 140 | caterva_params_t aux_params; 141 | aux_params.itemsize = itemsize; 142 | aux_params.ndim = shapes.ndim; 143 | for (int i = 0; i < aux_params.ndim; ++i) { 144 | aux_params.shape[i] = newshape[i]; 145 | } 146 | caterva_storage_t aux_storage = {0}; 147 | aux_storage.contiguous = backend.contiguous; 148 | for (int i = 0; i < params.ndim; ++i) { 149 | aux_storage.chunkshape[i] = shapes.chunkshape[i]; 150 | aux_storage.blockshape[i] = shapes.blockshape[i]; 151 | } 152 | CATERVA_ERROR(caterva_full(data->ctx, &aux_params, &aux_storage, value, &aux)); 153 | 154 | 155 | /* Fill buffer with whole array data */ 156 | uint8_t *src_buffer = data->ctx->cfg->alloc((size_t) (src->nitems * itemsize)); 157 | CATERVA_TEST_ASSERT(caterva_to_buffer(data->ctx, src, src_buffer, src->nitems * itemsize)); 158 | 159 | for (uint64_t i = 0; i < (uint64_t) src->nitems; ++i) { 160 | switch (itemsize) { 161 | case 8: 162 | CUTEST_ASSERT("Elements are not equal!", 163 | ((uint64_t *) src_buffer)[i] == (uint64_t) fill_value); 164 | break; 165 | case 4: 166 | CUTEST_ASSERT("Elements are not equal!", 167 | ((uint32_t *) src_buffer)[i] == (uint32_t) fill_value); 168 | break; 169 | case 2: 170 | CUTEST_ASSERT("Elements are not equal!", 171 | ((uint16_t *) src_buffer)[i] == (uint16_t) fill_value); 172 | break; 173 | case 1: 174 | CUTEST_ASSERT("Elements are not equal!", 175 | ((uint8_t *) src_buffer)[i] == (uint8_t) fill_value); 176 | break; 177 | default: 178 | CATERVA_TEST_ASSERT(CATERVA_ERR_INVALID_ARGUMENT); 179 | } 180 | } 181 | /* Free mallocs */ 182 | free(value); 183 | free(buffer); 184 | data->ctx->cfg->free(src_buffer); 185 | 186 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 187 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &aux)); 188 | 189 | caterva_remove(data->ctx, urlpath); 190 | 191 | return 0; 192 | } 193 | 194 | CUTEST_TEST_TEARDOWN(delete) { 195 | caterva_ctx_free(&data->ctx); 196 | } 197 | 198 | int main() { 199 | CUTEST_TEST_RUN(delete); 200 | } 201 | -------------------------------------------------------------------------------- /tests/test_full.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #include "test_common.h" 13 | 14 | 15 | CUTEST_TEST_DATA(full) { 16 | caterva_ctx_t *ctx; 17 | }; 18 | 19 | 20 | CUTEST_TEST_SETUP(full) { 21 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 22 | cfg.nthreads = 2; 23 | cfg.compcodec = BLOSC_BLOSCLZ; 24 | caterva_ctx_new(&cfg, &data->ctx); 25 | 26 | // Add parametrizations 27 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA( 28 | 1, 2, 4, 8 29 | )); 30 | CUTEST_PARAMETRIZE(shapes, _test_shapes, CUTEST_DATA( 31 | {0, {0}, {0}, {0}}, // 0-dim 32 | {1, {5}, {3}, {2}}, // 1-idim 33 | {2, {20, 0}, {7, 0}, {3, 0}}, // 0-shape 34 | {2, {20, 10}, {7, 5}, {3, 5}}, // 0-shape 35 | {2, {14, 10}, {8, 5}, {2, 2}}, // general, 36 | {3, {12, 10, 14}, {3, 5, 9}, {3, 4, 4}}, // general 37 | {3, {10, 21, 30, 55}, {8, 7, 15, 3}, {5, 5, 10, 1}}, // general, 38 | )); 39 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 40 | {false, false}, 41 | {true, false}, 42 | {true, true}, 43 | {false, true}, 44 | )); 45 | CUTEST_PARAMETRIZE(fill_value, int8_t, CUTEST_DATA( 46 | 3, 113, 33, -5 47 | )); 48 | } 49 | 50 | 51 | CUTEST_TEST_TEST(full) { 52 | CUTEST_GET_PARAMETER(backend, _test_backend); 53 | CUTEST_GET_PARAMETER(shapes, _test_shapes); 54 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 55 | CUTEST_GET_PARAMETER(fill_value, int8_t); 56 | 57 | 58 | char *urlpath = "test_full.b2frame"; 59 | caterva_remove(data->ctx, urlpath); 60 | 61 | caterva_params_t params; 62 | params.itemsize = itemsize; 63 | params.ndim = shapes.ndim; 64 | for (int i = 0; i < shapes.ndim; ++i) { 65 | params.shape[i] = shapes.shape[i]; 66 | } 67 | 68 | caterva_storage_t storage = {0}; 69 | if (backend.persistent) { 70 | storage.urlpath = urlpath; 71 | } 72 | storage.contiguous = backend.contiguous; 73 | for (int i = 0; i < shapes.ndim; ++i) { 74 | storage.chunkshape[i] = shapes.chunkshape[i]; 75 | storage.blockshape[i] = shapes.blockshape[i]; 76 | } 77 | 78 | /* Create original data */ 79 | int64_t buffersize = itemsize; 80 | for (int i = 0; i < shapes.ndim; ++i) { 81 | buffersize *= shapes.shape[i]; 82 | } 83 | 84 | /* Create caterva_array_t with original data */ 85 | caterva_array_t *src; 86 | uint8_t *value = malloc(itemsize); 87 | switch (itemsize) { 88 | case 8: 89 | ((int64_t *) value)[0] = (int64_t) fill_value; 90 | break; 91 | case 4: 92 | ((int32_t *) value)[0] = (int32_t) fill_value; 93 | break; 94 | case 2: 95 | ((int16_t *) value)[0] = (int16_t) fill_value; 96 | break; 97 | case 1: 98 | ((int8_t *) value)[0] = fill_value; 99 | break; 100 | default: 101 | break; 102 | } 103 | 104 | CATERVA_TEST_ASSERT(caterva_full(data->ctx, ¶ms, &storage, value, &src)); 105 | 106 | /* Fill dest array with caterva_array_t data */ 107 | uint8_t *buffer_dest = malloc( buffersize); 108 | CATERVA_TEST_ASSERT(caterva_to_buffer(data->ctx, src, buffer_dest, buffersize)); 109 | 110 | /* Testing */ 111 | for (int i = 0; i < buffersize / itemsize; ++i) { 112 | bool is_true = false; 113 | switch (itemsize) { 114 | case 8: 115 | is_true = ((int64_t *) buffer_dest)[i] == fill_value; 116 | break; 117 | case 4: 118 | is_true = ((int32_t *) buffer_dest)[i] == fill_value; 119 | break; 120 | case 2: 121 | is_true = ((int16_t *) buffer_dest)[i] == fill_value; 122 | break; 123 | case 1: 124 | is_true = ((int8_t *) buffer_dest)[i] == fill_value; 125 | break; 126 | default: 127 | break; 128 | } 129 | CUTEST_ASSERT("Elements are not equals", is_true); 130 | } 131 | 132 | /* Free mallocs */ 133 | free(buffer_dest); 134 | free(value); 135 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 136 | caterva_remove(data->ctx, urlpath); 137 | 138 | return CATERVA_SUCCEED; 139 | } 140 | 141 | 142 | CUTEST_TEST_TEARDOWN(full) { 143 | caterva_ctx_free(&data->ctx); 144 | } 145 | 146 | int main() { 147 | CUTEST_TEST_RUN(full); 148 | } 149 | -------------------------------------------------------------------------------- /tests/test_get_slice.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #include "test_common.h" 13 | 14 | uint64_t result0[1024] = {0}; 15 | uint64_t result1[1024] = {2, 3, 4, 5, 6, 7, 8}; 16 | uint64_t result2[1024] = {53, 54, 55, 56, 57, 58, 59, 63, 64, 65, 66, 67, 68, 69, 73, 74, 75, 76, 17 | 77, 78, 79, 83, 84, 85, 86, 87, 88, 89}; 18 | uint64_t result3[1024] = {303, 304, 305, 306, 307, 308, 309, 313, 314, 315, 316, 317, 318, 319, 19 | 323, 324, 325, 326, 327, 328, 329, 333, 334, 335, 336, 337, 338, 339, 20 | 343, 344, 345, 346, 347, 348, 349, 353, 354, 355, 356, 357, 358, 359, 21 | 363, 364, 365, 366, 367, 368, 369, 403, 404, 405, 406, 407, 408, 409, 22 | 413, 414, 415, 416, 417, 418, 419, 423, 424, 425, 426, 427, 428, 429, 23 | 433, 434, 435, 436, 437, 438, 439, 443, 444, 445, 446, 447, 448, 449, 24 | 453, 454, 455, 456, 457, 458, 459, 463, 464, 465, 466, 467, 468, 469, 25 | 503, 504, 505, 506, 507, 508, 509, 513, 514, 515, 516, 517, 518, 519, 26 | 523, 524, 525, 526, 527, 528, 529, 533, 534, 535, 536, 537, 538, 539, 27 | 543, 544, 545, 546, 547, 548, 549, 553, 554, 555, 556, 557, 558, 559, 28 | 563, 564, 565, 566, 567, 568, 569}; 29 | uint64_t result4[1024] = {0}; 30 | uint64_t result5[1024] = {0}; 31 | 32 | typedef struct { 33 | int8_t ndim; 34 | int64_t shape[CATERVA_MAX_DIM]; 35 | int32_t chunkshape[CATERVA_MAX_DIM]; 36 | int32_t blockshape[CATERVA_MAX_DIM]; 37 | int32_t chunkshape2[CATERVA_MAX_DIM]; 38 | int32_t blockshape2[CATERVA_MAX_DIM]; 39 | int64_t start[CATERVA_MAX_DIM]; 40 | int64_t stop[CATERVA_MAX_DIM]; 41 | uint64_t *result; 42 | } test_shapes_t; 43 | 44 | 45 | CUTEST_TEST_DATA(get_slice) { 46 | caterva_ctx_t *ctx; 47 | }; 48 | 49 | 50 | CUTEST_TEST_SETUP(get_slice) { 51 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 52 | cfg.nthreads = 2; 53 | cfg.compcodec = BLOSC_BLOSCLZ; 54 | caterva_ctx_new(&cfg, &data->ctx); 55 | 56 | // Add parametrizations 57 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA(8)); 58 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 59 | {false, false}, 60 | {true, false}, 61 | {true, true}, 62 | {false, true}, 63 | )); 64 | CUTEST_PARAMETRIZE(backend2, _test_backend, CUTEST_DATA( 65 | {false, false}, 66 | {true, false}, 67 | {true, true}, 68 | {false, true}, 69 | )); 70 | 71 | CUTEST_PARAMETRIZE(shapes, test_shapes_t, CUTEST_DATA( 72 | {0, {0}, {0}, {0}, {0}, {0}, {0}, {0}, result0}, // 0-dim 73 | {1, {10}, {7}, {2}, {6}, {2}, {2}, {9}, result1}, // 1-idim 74 | {2, {14, 10}, {8, 5}, {2, 2}, {4, 4}, {2, 3}, {5, 3}, {9, 10}, result2}, // general, 75 | {3, {10, 10, 10}, {3, 5, 9}, {3, 4, 4}, {3, 7, 7}, {2, 5, 5}, {3, 0, 3}, {6, 7, 10}, result3}, // general 76 | {2, {20, 0}, {7, 0}, {3, 0}, {5, 0}, {2, 0}, {2, 0}, {8, 0}, result4}, // 0-shape 77 | {2, {20, 10}, {7, 5}, {3, 5}, {5, 5}, {2, 2}, {2, 0}, {18, 0}, result5}, // 0-shape 78 | )); 79 | } 80 | 81 | CUTEST_TEST_TEST(get_slice) { 82 | CUTEST_GET_PARAMETER(backend, _test_backend); 83 | CUTEST_GET_PARAMETER(shapes, test_shapes_t); 84 | CUTEST_GET_PARAMETER(backend2, _test_backend); 85 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 86 | 87 | char* urlpath = "test_get_slice.b2frame"; 88 | char* urlpath2 = "test_get_slice2.b2frame"; 89 | 90 | caterva_remove(data->ctx, urlpath); 91 | caterva_remove(data->ctx, urlpath2); 92 | 93 | caterva_params_t params; 94 | params.itemsize = itemsize; 95 | params.ndim = shapes.ndim; 96 | for (int i = 0; i < params.ndim; ++i) { 97 | params.shape[i] = shapes.shape[i]; 98 | } 99 | 100 | caterva_storage_t storage = {0}; 101 | if (backend.persistent) { 102 | storage.urlpath = urlpath; 103 | } 104 | storage.contiguous = backend.contiguous; 105 | for (int i = 0; i < params.ndim; ++i) { 106 | storage.chunkshape[i] = shapes.chunkshape[i]; 107 | storage.blockshape[i] = shapes.blockshape[i]; 108 | } 109 | 110 | /* Create original data */ 111 | size_t buffersize = itemsize; 112 | for (int i = 0; i < params.ndim; ++i) { 113 | buffersize *= (size_t) shapes.shape[i]; 114 | } 115 | uint8_t *buffer = data->ctx->cfg->alloc(buffersize); 116 | CUTEST_ASSERT("Buffer filled incorrectly", fill_buf(buffer, itemsize, buffersize / itemsize)); 117 | 118 | /* Create caterva_array_t with original data */ 119 | caterva_array_t *src; 120 | CATERVA_TEST_ASSERT(caterva_from_buffer(data->ctx, buffer, buffersize, ¶ms, &storage, 121 | &src)); 122 | 123 | /* Add vlmeta */ 124 | 125 | caterva_metalayer_t vlmeta; 126 | vlmeta.name = "test_get_slice"; 127 | double sdata = 2.3; 128 | vlmeta.sdata = (uint8_t *) &sdata; 129 | vlmeta.size = sizeof(double); 130 | 131 | CATERVA_TEST_ASSERT(caterva_vlmeta_add(data->ctx, src, &vlmeta)); 132 | 133 | /* Create storage for dest container */ 134 | 135 | caterva_storage_t storage2 = {0}; 136 | if (backend2.persistent) { 137 | storage2.urlpath = urlpath2; 138 | } 139 | storage2.contiguous = backend2.contiguous; 140 | for (int i = 0; i < params.ndim; ++i) { 141 | storage2.chunkshape[i] = shapes.chunkshape2[i]; 142 | storage2.blockshape[i] = shapes.blockshape2[i]; 143 | } 144 | 145 | caterva_array_t *dest; 146 | CATERVA_TEST_ASSERT(caterva_get_slice(data->ctx, src, shapes.start, shapes.stop, 147 | &storage2, &dest)); 148 | 149 | /* Check metalayers */ 150 | 151 | bool exists; 152 | CATERVA_TEST_ASSERT(caterva_meta_exists(data->ctx, dest, "caterva", &exists)); 153 | CUTEST_ASSERT("metalayer not exists", exists == true); 154 | CATERVA_TEST_ASSERT(caterva_vlmeta_exists(data->ctx, dest, vlmeta.name, &exists)); 155 | CUTEST_ASSERT("vlmetalayer not exists", exists == false); 156 | 157 | int64_t destbuffersize = itemsize; 158 | for (int i = 0; i < src->ndim; ++i) { 159 | destbuffersize *= (shapes.stop[i] - shapes.start[i]); 160 | } 161 | 162 | uint64_t *buffer_dest = data->ctx->cfg->alloc((size_t) destbuffersize); 163 | CATERVA_TEST_ASSERT(caterva_to_buffer(data->ctx, dest, buffer_dest, destbuffersize)); 164 | 165 | for (int i = 0; i < destbuffersize / itemsize; ++i) { 166 | uint64_t a = shapes.result[i] + 1; 167 | uint64_t b = buffer_dest[i]; 168 | CUTEST_ASSERT("Elements are not equals!", a == b); 169 | } 170 | 171 | /* Free mallocs */ 172 | data->ctx->cfg->free(buffer); 173 | data->ctx->cfg->free(buffer_dest); 174 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 175 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &dest)); 176 | caterva_remove(data->ctx, urlpath); 177 | caterva_remove(data->ctx, urlpath2); 178 | 179 | return 0; 180 | } 181 | 182 | CUTEST_TEST_TEARDOWN(get_slice) { 183 | caterva_ctx_free(&data->ctx); 184 | } 185 | 186 | int main() { 187 | CUTEST_TEST_RUN(get_slice); 188 | } 189 | -------------------------------------------------------------------------------- /tests/test_get_slice_buffer.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | 13 | #include "test_common.h" 14 | 15 | 16 | uint64_t result0[1024] = {0}; 17 | uint64_t result1[1024] = {2, 3, 4, 5, 6, 7, 8}; 18 | uint64_t result2[1024] = {53, 54, 55, 56, 57, 58, 59, 63, 64, 65, 66, 67, 68, 69, 73, 74, 75, 76, 19 | 77, 78, 79, 83, 84, 85, 86, 87, 88, 89}; 20 | uint64_t result3[1024] = {303, 304, 305, 306, 307, 308, 309, 313, 314, 315, 316, 317, 318, 319, 21 | 323, 324, 325, 326, 327, 328, 329, 333, 334, 335, 336, 337, 338, 339, 22 | 343, 344, 345, 346, 347, 348, 349, 353, 354, 355, 356, 357, 358, 359, 23 | 363, 364, 365, 366, 367, 368, 369, 403, 404, 405, 406, 407, 408, 409, 24 | 413, 414, 415, 416, 417, 418, 419, 423, 424, 425, 426, 427, 428, 429, 25 | 433, 434, 435, 436, 437, 438, 439, 443, 444, 445, 446, 447, 448, 449, 26 | 453, 454, 455, 456, 457, 458, 459, 463, 464, 465, 466, 467, 468, 469, 27 | 503, 504, 505, 506, 507, 508, 509, 513, 514, 515, 516, 517, 518, 519, 28 | 523, 524, 525, 526, 527, 528, 529, 533, 534, 535, 536, 537, 538, 539, 29 | 543, 544, 545, 546, 547, 548, 549, 553, 554, 555, 556, 557, 558, 559, 30 | 563, 564, 565, 566, 567, 568, 569}; 31 | uint64_t result4[1024] = {0}; 32 | uint64_t result5[1024] = {0}; 33 | 34 | typedef struct { 35 | int8_t ndim; 36 | int64_t shape[CATERVA_MAX_DIM]; 37 | int32_t chunkshape[CATERVA_MAX_DIM]; 38 | int32_t blockshape[CATERVA_MAX_DIM]; 39 | int32_t chunkshape2[CATERVA_MAX_DIM]; 40 | int32_t blockshape2[CATERVA_MAX_DIM]; 41 | int64_t start[CATERVA_MAX_DIM]; 42 | int64_t stop[CATERVA_MAX_DIM]; 43 | uint64_t *result; 44 | } test_shapes_t; 45 | 46 | 47 | CUTEST_TEST_DATA(get_slice_buffer) { 48 | caterva_ctx_t *ctx; 49 | }; 50 | 51 | 52 | CUTEST_TEST_SETUP(get_slice_buffer) { 53 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 54 | cfg.nthreads = 1; 55 | cfg.compcodec = BLOSC_BLOSCLZ; 56 | caterva_ctx_new(&cfg, &data->ctx); 57 | 58 | // Add parametrizations 59 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA(8)); 60 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 61 | {false, false}, 62 | {true, false}, 63 | {true, true}, 64 | {false, true}, 65 | )); 66 | 67 | CUTEST_PARAMETRIZE(shapes, test_shapes_t, CUTEST_DATA( 68 | {0, {0}, {0}, {0}, {0}, {0}, {0}, {0}, result0}, // 0-dim 69 | {1, {10}, {7}, {2}, {6}, {2}, {2}, {9}, result1}, // 1-idim 70 | {2, {14, 10}, {8, 5}, {2, 2}, {4, 4}, {2, 3}, {5, 3}, {9, 10}, result2}, // general, 71 | {3, {10, 10, 10}, {3, 5, 9}, {3, 4, 4}, {3, 7, 7}, {2, 5, 5}, {3, 0, 3}, {6, 7, 10}, result3}, // general 72 | {2, {20, 0}, {7, 0}, {3, 0}, {5, 0}, {2, 0}, {2, 0}, {8, 0}, result4}, // 0-shape 73 | {2, {20, 10}, {7, 5}, {3, 5}, {5, 5}, {2, 2}, {2, 0}, {18, 0}, result5}, // 0-shape 74 | )); 75 | } 76 | 77 | CUTEST_TEST_TEST(get_slice_buffer) { 78 | CUTEST_GET_PARAMETER(backend, _test_backend); 79 | CUTEST_GET_PARAMETER(shapes, test_shapes_t); 80 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 81 | 82 | char *urlpath = "test_get_slice_buffer.b2frame"; 83 | caterva_remove(data->ctx, urlpath); 84 | 85 | caterva_params_t params; 86 | params.itemsize = itemsize; 87 | params.ndim = shapes.ndim; 88 | for (int i = 0; i < params.ndim; ++i) { 89 | params.shape[i] = shapes.shape[i]; 90 | } 91 | 92 | caterva_storage_t storage = {0}; 93 | if (backend.persistent) { 94 | storage.urlpath = urlpath; 95 | } 96 | storage.contiguous = backend.contiguous; 97 | for (int i = 0; i < params.ndim; ++i) { 98 | storage.chunkshape[i] = shapes.chunkshape[i]; 99 | storage.blockshape[i] = shapes.blockshape[i]; 100 | } 101 | 102 | /* Create original data */ 103 | size_t buffersize = itemsize; 104 | for (int i = 0; i < params.ndim; ++i) { 105 | buffersize *= (size_t) shapes.shape[i]; 106 | } 107 | uint8_t *buffer = data->ctx->cfg->alloc(buffersize); 108 | 109 | CUTEST_ASSERT("Buffer filled incorrectly", fill_buf(buffer, itemsize, buffersize / itemsize)); 110 | 111 | /* Create caterva_array_t with original data */ 112 | caterva_array_t *src; 113 | CATERVA_TEST_ASSERT(caterva_from_buffer(data->ctx, buffer, buffersize, ¶ms, &storage, 114 | &src)); 115 | 116 | /* Create dest buffer */ 117 | int64_t destshape[CATERVA_MAX_DIM] = {0}; 118 | int64_t destbuffersize = itemsize; 119 | for (int i = 0; i < params.ndim; ++i) { 120 | destshape[i] = shapes.stop[i] - shapes.start[i]; 121 | destbuffersize *= destshape[i]; 122 | } 123 | 124 | uint64_t *destbuffer = data->ctx->cfg->alloc((size_t) destbuffersize); 125 | 126 | /* Fill dest buffer with a slice*/ 127 | CATERVA_TEST_ASSERT(caterva_get_slice_buffer(data->ctx, src, shapes.start, shapes.stop, 128 | destbuffer, 129 | destshape, destbuffersize)); 130 | 131 | 132 | for (int i = 0; i < destbuffersize / itemsize; ++i) { 133 | uint64_t a = destbuffer[i]; 134 | uint64_t b = shapes.result[i] + 1; 135 | CUTEST_ASSERT("Elements are not equals!", a == b); 136 | } 137 | 138 | /* Free mallocs */ 139 | data->ctx->cfg->free(buffer); 140 | data->ctx->cfg->free(destbuffer); 141 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 142 | caterva_remove(data->ctx, urlpath); 143 | 144 | return 0; 145 | } 146 | 147 | CUTEST_TEST_TEARDOWN(get_slice_buffer) { 148 | caterva_ctx_free(&data->ctx); 149 | } 150 | 151 | int main() { 152 | CUTEST_TEST_RUN(get_slice_buffer); 153 | } 154 | -------------------------------------------------------------------------------- /tests/test_insert.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | 13 | #include "test_common.h" 14 | 15 | typedef struct { 16 | int8_t ndim; 17 | int64_t shape[CATERVA_MAX_DIM]; 18 | int32_t chunkshape[CATERVA_MAX_DIM]; 19 | int32_t blockshape[CATERVA_MAX_DIM]; 20 | int64_t buffershape[CATERVA_MAX_DIM]; 21 | int8_t axis; 22 | int64_t start; 23 | } test_shapes_t; 24 | 25 | 26 | CUTEST_TEST_DATA(insert) { 27 | caterva_ctx_t *ctx; 28 | }; 29 | 30 | 31 | CUTEST_TEST_SETUP(insert) { 32 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 33 | cfg.nthreads = 2; 34 | cfg.compcodec = BLOSC_BLOSCLZ; 35 | caterva_ctx_new(&cfg, &data->ctx); 36 | 37 | // Add parametrizations 38 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA( 39 | 1, 40 | 2, 41 | 4, 42 | 8, 43 | )); 44 | 45 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 46 | {false, false}, 47 | {true, false}, 48 | {true, true}, 49 | {false, true}, 50 | )); 51 | 52 | 53 | CUTEST_PARAMETRIZE(shapes, test_shapes_t, CUTEST_DATA( 54 | {1, {5}, {3}, {2}, {10}, 0, 5}, // insert at the end (old padding modified) 55 | {2, {18, 6}, {6, 6}, {3, 3}, {18, 12}, 1, 0}, // insert at the beginning 56 | {3, {12, 10, 14}, {3, 5, 9}, {3, 4, 4}, {12, 10, 18}, 2, 9}, // insert in the middle 57 | {4, {10, 10, 5, 5}, {5, 7, 3, 3}, {2, 2, 1, 1}, {10, 10, 5, 30}, 3, 3}, // insert in the middle 58 | 59 | )); 60 | } 61 | 62 | CUTEST_TEST_TEST(insert) { 63 | CUTEST_GET_PARAMETER(backend, _test_backend); 64 | CUTEST_GET_PARAMETER(shapes, test_shapes_t); 65 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 66 | 67 | char *urlpath = "test_insert_shape.b2frame"; 68 | caterva_remove(data->ctx, urlpath); 69 | 70 | caterva_params_t params; 71 | params.itemsize = itemsize; 72 | params.ndim = shapes.ndim; 73 | for (int i = 0; i < params.ndim; ++i) { 74 | params.shape[i] = shapes.shape[i]; 75 | } 76 | 77 | caterva_storage_t storage = {0}; 78 | if (backend.persistent) { 79 | storage.urlpath = urlpath; 80 | } 81 | storage.contiguous = backend.contiguous; 82 | for (int i = 0; i < params.ndim; ++i) { 83 | storage.chunkshape[i] = shapes.chunkshape[i]; 84 | storage.blockshape[i] = shapes.blockshape[i]; 85 | } 86 | 87 | int64_t buffersize = itemsize; 88 | for (int i = 0; i < params.ndim; ++i) { 89 | buffersize *= shapes.buffershape[i]; 90 | } 91 | 92 | /* Create caterva_array_t filled with 1 */ 93 | caterva_array_t *src; 94 | uint8_t *value = malloc(itemsize); 95 | int8_t fill_value = 1; 96 | switch (itemsize) { 97 | case 8: 98 | ((int64_t *) value)[0] = (int64_t) fill_value; 99 | break; 100 | case 4: 101 | ((int32_t *) value)[0] = (int32_t) fill_value; 102 | break; 103 | case 2: 104 | ((int16_t *) value)[0] = (int16_t) fill_value; 105 | break; 106 | case 1: 107 | ((int8_t *) value)[0] = fill_value; 108 | break; 109 | default: 110 | break; 111 | } 112 | CATERVA_ERROR(caterva_full(data->ctx, ¶ms, &storage, value, &src)); 113 | 114 | uint8_t *buffer = data->ctx->cfg->alloc(buffersize); 115 | fill_buf(buffer, itemsize, buffersize / itemsize); 116 | CATERVA_ERROR(caterva_insert(data->ctx, src, buffer, buffersize, shapes.axis, shapes.start)); 117 | 118 | int64_t start[CATERVA_MAX_DIM] = {0}; 119 | start[shapes.axis] = shapes.start; 120 | int64_t stop[CATERVA_MAX_DIM]; 121 | for (int i = 0; i < shapes.ndim; ++i) { 122 | stop[i] = shapes.shape[i]; 123 | } 124 | stop[shapes.axis] = shapes.start + shapes.buffershape[shapes.axis]; 125 | 126 | /* Fill buffer with a slice from the new chunks */ 127 | uint8_t *res_buffer = data->ctx->cfg->alloc(buffersize); 128 | CATERVA_ERROR(caterva_get_slice_buffer(data->ctx, src, start, stop, res_buffer, 129 | shapes.buffershape, buffersize)); 130 | 131 | for (uint64_t i = 0; i < (uint64_t) buffersize / itemsize; ++i) { 132 | switch (itemsize) { 133 | case 8: 134 | CUTEST_ASSERT("Elements are not equal!", 135 | ((uint64_t *) buffer)[i] == ((uint64_t *) res_buffer)[i]); 136 | break; 137 | case 4: 138 | CUTEST_ASSERT("Elements are not equal!", 139 | ((uint32_t *) buffer)[i] == ((uint32_t *) res_buffer)[i]); 140 | break; 141 | case 2: 142 | CUTEST_ASSERT("Elements are not equal!", 143 | ((uint16_t *) buffer)[i] == ((uint16_t *) res_buffer)[i]); 144 | break; 145 | case 1: 146 | CUTEST_ASSERT("Elements are not equal!", 147 | ((uint8_t *) buffer)[i] == ((uint8_t *) res_buffer)[i]); 148 | break; 149 | default: 150 | CATERVA_TEST_ASSERT(CATERVA_ERR_INVALID_ARGUMENT); 151 | } 152 | } 153 | /* Free mallocs */ 154 | free(value); 155 | data->ctx->cfg->free(buffer); 156 | data->ctx->cfg->free(res_buffer); 157 | 158 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 159 | caterva_remove(data->ctx, urlpath); 160 | 161 | return 0; 162 | } 163 | 164 | CUTEST_TEST_TEARDOWN(insert) { 165 | caterva_ctx_free(&data->ctx); 166 | } 167 | 168 | int main() { 169 | CUTEST_TEST_RUN(insert); 170 | } 171 | -------------------------------------------------------------------------------- /tests/test_metalayers.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #include "test_common.h" 13 | 14 | 15 | CUTEST_TEST_DATA(metalayers) { 16 | caterva_ctx_t *ctx; 17 | }; 18 | 19 | 20 | CUTEST_TEST_SETUP(metalayers) { 21 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 22 | cfg.nthreads = 2; 23 | cfg.compcodec = BLOSC_BLOSCLZ; 24 | caterva_ctx_new(&cfg, &data->ctx); 25 | 26 | // Add parametrizations 27 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA(1, 2, 4, 8)); 28 | CUTEST_PARAMETRIZE(shapes, _test_shapes, CUTEST_DATA( 29 | {0, {0}, {0}, {0}}, // 0-dim 30 | {1, {10}, {7}, {2}}, // 1-idim 31 | {2, {100, 100}, {20, 20}, {10, 10}}, 32 | )); 33 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 34 | {true, true}, 35 | {false, true}, 36 | )); 37 | } 38 | 39 | 40 | CUTEST_TEST_TEST(metalayers) { 41 | CUTEST_GET_PARAMETER(shapes, _test_shapes); 42 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 43 | CUTEST_GET_PARAMETER(backend, _test_backend); 44 | 45 | char *urlpath = "test_metalayers.caterva"; 46 | caterva_remove(data->ctx, urlpath); 47 | caterva_params_t params; 48 | params.itemsize = itemsize; 49 | params.ndim = shapes.ndim; 50 | for (int i = 0; i < params.ndim; ++i) { 51 | params.shape[i] = shapes.shape[i]; 52 | } 53 | 54 | caterva_storage_t storage = {0}; 55 | if (backend.persistent) { 56 | storage.urlpath = urlpath; 57 | } 58 | storage.contiguous = backend.contiguous; 59 | for (int i = 0; i < params.ndim; ++i) { 60 | storage.chunkshape[i] = shapes.chunkshape[i]; 61 | storage.blockshape[i] = shapes.blockshape[i]; 62 | } 63 | storage.nmetalayers = 1; 64 | caterva_metalayer_t *meta0 = &storage.metalayers[0]; 65 | meta0->name = "test_meta"; 66 | meta0->size = 3; 67 | double sdata0 = 5.789; 68 | meta0->sdata = (uint8_t *) &sdata0; 69 | meta0->size = sizeof(sdata0); 70 | 71 | /* Create original data */ 72 | size_t buffersize = itemsize; 73 | for (int i = 0; i < params.ndim; ++i) { 74 | buffersize *= (size_t) params.shape[i]; 75 | } 76 | 77 | uint8_t *buffer = malloc(buffersize); 78 | CUTEST_ASSERT("Buffer filled incorrectly", fill_buf(buffer, itemsize, buffersize / itemsize)); 79 | 80 | /* Create caterva_array_t with original data */ 81 | caterva_array_t *src; 82 | CATERVA_TEST_ASSERT(caterva_from_buffer(data->ctx, buffer, buffersize, ¶ms, &storage, 83 | &src)); 84 | 85 | caterva_metalayer_t vlmeta1; 86 | 87 | uint64_t sdata1 = 56; 88 | vlmeta1.name = "vlmeta1"; 89 | vlmeta1.sdata = (uint8_t *) &sdata1; 90 | vlmeta1.size = sizeof(sdata1); 91 | 92 | CATERVA_TEST_ASSERT(caterva_vlmeta_add(data->ctx, src, &vlmeta1)); 93 | 94 | bool exists = false; 95 | CATERVA_TEST_ASSERT(caterva_vlmeta_exists(data->ctx, src, "vlmeta2", &exists)); 96 | CUTEST_ASSERT("", exists == false); 97 | CATERVA_TEST_ASSERT(caterva_vlmeta_exists(data->ctx, src, vlmeta1.name, &exists)); 98 | CUTEST_ASSERT("", exists == true); 99 | 100 | caterva_metalayer_t vlmeta2; 101 | CATERVA_TEST_ASSERT(caterva_vlmeta_get(data->ctx, src, vlmeta1.name, &vlmeta2)); 102 | CUTEST_ASSERT("Contents are not equals", 103 | *((uint64_t *) vlmeta1.sdata) == *((uint64_t *) vlmeta2.sdata)); 104 | CUTEST_ASSERT("Sizes are not equals", vlmeta1.size == vlmeta2.size); 105 | free(vlmeta2.name); 106 | free(vlmeta2.sdata); 107 | 108 | float sdata11 = 4.5f; 109 | vlmeta1.sdata = (uint8_t *) &sdata11; 110 | vlmeta1.size = sizeof(sdata11); 111 | 112 | CATERVA_TEST_ASSERT(caterva_vlmeta_update(data->ctx, src, &vlmeta1)); 113 | 114 | caterva_metalayer_t vlmeta3; 115 | CATERVA_TEST_ASSERT(caterva_vlmeta_get(data->ctx, src, vlmeta1.name, &vlmeta3)); 116 | CUTEST_ASSERT("Contents are not equals", *((float *) vlmeta1.sdata) == *((float *) vlmeta3.sdata)); 117 | CUTEST_ASSERT("Sizes are not equals", vlmeta1.size == vlmeta3.size); 118 | free(vlmeta3.name); 119 | free(vlmeta3.sdata); 120 | 121 | vlmeta2.name = "vlmeta2"; 122 | vlmeta2.sdata = (uint8_t *) &sdata1; 123 | vlmeta2.size = sizeof(sdata1); 124 | CATERVA_TEST_ASSERT(caterva_vlmeta_add(data->ctx, src, &vlmeta2)); 125 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 126 | 127 | caterva_array_t *src2; 128 | caterva_open(data->ctx, urlpath, &src2); 129 | 130 | CATERVA_TEST_ASSERT(caterva_vlmeta_get(data->ctx, src2, vlmeta2.name, &vlmeta3)); 131 | CUTEST_ASSERT("Contents are not equals", *((uint64_t *) vlmeta2.sdata) == *((uint64_t *) vlmeta3.sdata)); 132 | CUTEST_ASSERT("Sizes are not equals", vlmeta2.size == vlmeta3.size); 133 | free(vlmeta3.name); 134 | free(vlmeta3.sdata); 135 | 136 | sdata0 = 1e-10; 137 | caterva_metalayer_t meta1; 138 | meta1.name = meta0->name; 139 | meta1.sdata = (uint8_t *) &sdata0; 140 | meta1.size = sizeof(sdata0); 141 | 142 | CATERVA_TEST_ASSERT(caterva_meta_exists(data->ctx, src2, meta0->name, &exists)); 143 | CATERVA_TEST_ASSERT(caterva_meta_update(data->ctx, src2, &meta1)); 144 | 145 | caterva_metalayer_t meta2; 146 | CATERVA_TEST_ASSERT(caterva_meta_get(data->ctx, src2, meta1.name, &meta2)); 147 | 148 | CUTEST_ASSERT("Contents are not equals", *((double *) meta2.sdata) == *((double *) meta1.sdata)); 149 | CUTEST_ASSERT("Sizes are not equals", meta2.size == meta1.size); 150 | free(meta2.name); 151 | free(meta2.sdata); 152 | 153 | /* Free mallocs */ 154 | free(buffer); 155 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src2)); 156 | caterva_remove(data->ctx, urlpath); 157 | return 0; 158 | } 159 | 160 | 161 | CUTEST_TEST_TEARDOWN(metalayers) { 162 | caterva_ctx_free(&data->ctx); 163 | } 164 | 165 | int main() { 166 | CUTEST_TEST_RUN(metalayers); 167 | } 168 | -------------------------------------------------------------------------------- /tests/test_persistency.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #include "test_common.h" 13 | #ifdef __GNUC__ 14 | #include 15 | #define FILE_EXISTS(urlpath) access(urlpath, F_OK) 16 | #else 17 | #include 18 | #define FILE_EXISTS(urlpath) _access(urlpath, 0) 19 | #endif 20 | 21 | 22 | typedef struct { 23 | int8_t ndim; 24 | int64_t shape[CATERVA_MAX_DIM]; 25 | int32_t chunkshape[CATERVA_MAX_DIM]; 26 | int32_t blockshape[CATERVA_MAX_DIM]; 27 | } test_shapes_t; 28 | 29 | 30 | CUTEST_TEST_DATA(persistency) { 31 | caterva_ctx_t *ctx; 32 | }; 33 | 34 | 35 | CUTEST_TEST_SETUP(persistency) { 36 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 37 | cfg.nthreads = 2; 38 | cfg.compcodec = BLOSC_BLOSCLZ; 39 | caterva_ctx_new(&cfg, &data->ctx); 40 | 41 | // Add parametrizations 42 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA(1, 2, 4, 8)); 43 | CUTEST_PARAMETRIZE(shapes, test_shapes_t, CUTEST_DATA( 44 | {0, {0}, {0}, {0}}, // 0-dim 45 | {1, {10}, {7}, {2}}, // 1-idim 46 | {2, {100, 100}, {20, 20}, {10, 10}}, 47 | {3, {100, 55, 123}, {31, 5, 22}, {4, 4, 4}}, 48 | {3, {100, 0, 12}, {31, 0, 12}, {10, 0, 12}}, 49 | {4, {50, 160, 31, 12}, {25, 20, 20, 10}, {5, 5, 5, 10}}, 50 | {5, {1, 1, 1024, 1, 1}, {1, 1, 500, 1, 1}, {1, 1, 200, 1, 1}}, 51 | {6, {5, 1, 200, 3, 1, 2}, {5, 1, 50, 2, 1, 2}, {2, 1, 20, 2, 1, 2}} 52 | )); 53 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 54 | {true, true}, 55 | {false, true}, 56 | )); 57 | } 58 | 59 | CUTEST_TEST_TEST(persistency) { 60 | CUTEST_GET_PARAMETER(backend, _test_backend); 61 | CUTEST_GET_PARAMETER(shapes, test_shapes_t); 62 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 63 | 64 | char* urlpath = "test_persistency.b2frame"; 65 | 66 | caterva_remove(data->ctx, urlpath); 67 | 68 | caterva_params_t params; 69 | params.itemsize = itemsize; 70 | params.ndim = shapes.ndim; 71 | for (int i = 0; i < params.ndim; ++i) { 72 | params.shape[i] = shapes.shape[i]; 73 | } 74 | 75 | caterva_storage_t storage = {0}; 76 | if (backend.persistent) { 77 | storage.urlpath = urlpath; 78 | } 79 | storage.contiguous = backend.contiguous; 80 | for (int i = 0; i < params.ndim; ++i) { 81 | storage.chunkshape[i] = shapes.chunkshape[i]; 82 | storage.blockshape[i] = shapes.blockshape[i]; 83 | } 84 | 85 | 86 | /* Create original data */ 87 | int64_t buffersize = itemsize; 88 | for (int i = 0; i < params.ndim; ++i) { 89 | buffersize *= shapes.shape[i]; 90 | } 91 | uint8_t *buffer = malloc(buffersize); 92 | CUTEST_ASSERT("Buffer filled incorrectly", fill_buf(buffer, itemsize, buffersize / itemsize)); 93 | 94 | /* Create caterva_array_t with original data */ 95 | caterva_array_t *src; 96 | CATERVA_TEST_ASSERT(caterva_from_buffer(data->ctx, buffer, buffersize, ¶ms, &storage, 97 | &src)); 98 | 99 | caterva_array_t *dest; 100 | CATERVA_TEST_ASSERT(caterva_open(data->ctx, urlpath, &dest)); 101 | 102 | /* Fill dest array with caterva_array_t data */ 103 | uint8_t *buffer_dest = malloc(buffersize); 104 | CATERVA_TEST_ASSERT(caterva_to_buffer(data->ctx, dest, buffer_dest, buffersize)); 105 | 106 | /* Testing */ 107 | if (dest->nitems != 0) { 108 | for (int i = 0; i < buffersize / itemsize; ++i) { 109 | // printf("%d - %d\n", buffer[i], buffer_dest[i]); 110 | CUTEST_ASSERT("Elements are not equals!", buffer[i] == buffer_dest[i]); 111 | } 112 | } 113 | 114 | /* Free mallocs */ 115 | free(buffer); 116 | free(buffer_dest); 117 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 118 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &dest)); 119 | 120 | caterva_remove(data->ctx, urlpath); 121 | 122 | return 0; 123 | } 124 | 125 | 126 | CUTEST_TEST_TEARDOWN(persistency) { 127 | caterva_ctx_free(&data->ctx); 128 | } 129 | 130 | int main() { 131 | CUTEST_TEST_RUN(persistency); 132 | } 133 | -------------------------------------------------------------------------------- /tests/test_resize.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | 13 | #include "test_common.h" 14 | 15 | typedef struct { 16 | int8_t ndim; 17 | int64_t shape[CATERVA_MAX_DIM]; 18 | int32_t chunkshape[CATERVA_MAX_DIM]; 19 | int32_t blockshape[CATERVA_MAX_DIM]; 20 | int64_t newshape[CATERVA_MAX_DIM]; 21 | bool given_pos; 22 | int64_t start_resize[CATERVA_MAX_DIM]; 23 | } test_shapes_t; 24 | 25 | 26 | CUTEST_TEST_DATA(resize_shape) { 27 | caterva_ctx_t *ctx; 28 | }; 29 | 30 | 31 | CUTEST_TEST_SETUP(resize_shape) { 32 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 33 | cfg.nthreads = 2; 34 | cfg.compcodec = BLOSC_ZSTD; 35 | caterva_ctx_new(&cfg, &data->ctx); 36 | 37 | // Add parametrizations 38 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA( 39 | 1, 40 | 2, 41 | 4, 42 | 8, 43 | )); 44 | 45 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 46 | {false, false}, 47 | {true, false}, 48 | {true, true}, 49 | {false, true}, 50 | )); 51 | 52 | 53 | CUTEST_PARAMETRIZE(shapes, test_shapes_t, CUTEST_DATA( 54 | {1, {5}, {3}, {2}, {10}, false, {5}}, // extend only 55 | {2, {20, 5}, {7, 5}, {3, 3}, {27, 10}, true, {14, 5}}, // extend only - start 56 | {2, {20, 10}, {7, 5}, {3, 5}, {10, 10}, false, {10, 10}}, // shrink only 57 | {2, {30, 20}, {8, 5}, {2, 2}, {22, 10}, true, {8, 5}}, // shrink only - start 58 | {3, {12, 10, 14}, {3, 5, 9}, {3, 4, 4}, {10, 15, 14}, false, {10, 10, 14}}, // shrink and extend 59 | {3, {10, 21, 30}, {8, 7, 15}, {5, 5, 10}, {10, 13, 10}, false, {10, 13, 10}}, // shrink only 60 | {3, {10, 23, 30}, {8, 7, 15}, {5, 5, 10}, {10, 16, 45}, true, {0, 0, 0}}, // shrink and extend - start 61 | {2, {75, 50}, {25, 13}, {8, 8}, {50, 76}, true, {50, 13}}, // shrink and extend - start 62 | {2, {50, 50}, {25, 13}, {8, 8}, {49, 51}, false, {49, 50}}, // shrink and extend 63 | {2, {143, 41}, {18, 13}, {7, 7}, {50, 50}, false, {50, 41}}, // shrink and extend 64 | {4, {10, 10, 5, 5}, {5, 7, 3, 3}, {2, 2, 1, 1}, {11, 20, 2, 2}, false, {10, 10, 2, 2}}, // shrink and extend 65 | 66 | )); 67 | } 68 | 69 | CUTEST_TEST_TEST(resize_shape) { 70 | CUTEST_GET_PARAMETER(backend, _test_backend); 71 | CUTEST_GET_PARAMETER(shapes, test_shapes_t); 72 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 73 | 74 | char *urlpath = "test_resize_shape.b2frame"; 75 | caterva_remove(data->ctx, urlpath); 76 | 77 | caterva_params_t params; 78 | params.itemsize = itemsize; 79 | params.ndim = shapes.ndim; 80 | for (int i = 0; i < params.ndim; ++i) { 81 | params.shape[i] = shapes.shape[i]; 82 | } 83 | 84 | caterva_storage_t storage = {0}; 85 | if (backend.persistent) { 86 | storage.urlpath = urlpath; 87 | } 88 | storage.contiguous = backend.contiguous; 89 | for (int i = 0; i < params.ndim; ++i) { 90 | storage.chunkshape[i] = shapes.chunkshape[i]; 91 | storage.blockshape[i] = shapes.blockshape[i]; 92 | } 93 | 94 | int64_t buffersize = itemsize; 95 | bool only_shrink = true; 96 | for (int i = 0; i < params.ndim; ++i) { 97 | if (shapes.newshape[i] > shapes.shape[i]) { 98 | only_shrink = false; 99 | } 100 | buffersize *= shapes.newshape[i]; 101 | } 102 | 103 | /* Create caterva_array_t with original data */ 104 | caterva_array_t *src; 105 | uint8_t *value = malloc(itemsize); 106 | int8_t fill_value = 1; 107 | switch (itemsize) { 108 | case 8: 109 | ((int64_t *) value)[0] = (int64_t) fill_value; 110 | break; 111 | case 4: 112 | ((int32_t *) value)[0] = (int32_t) fill_value; 113 | break; 114 | case 2: 115 | ((int16_t *) value)[0] = (int16_t) fill_value; 116 | break; 117 | case 1: 118 | ((int8_t *) value)[0] = fill_value; 119 | break; 120 | default: 121 | break; 122 | } 123 | CATERVA_ERROR(caterva_full(data->ctx, ¶ms, &storage, value, &src)); 124 | 125 | if (shapes.given_pos) { 126 | CATERVA_ERROR(caterva_resize(data->ctx, src, shapes.newshape, shapes.start_resize)); 127 | } 128 | else { 129 | CATERVA_ERROR(caterva_resize(data->ctx, src, shapes.newshape, NULL)); 130 | } 131 | 132 | // Create aux array to compare values 133 | caterva_array_t *aux; 134 | caterva_params_t aux_params; 135 | aux_params.itemsize = itemsize; 136 | aux_params.ndim = shapes.ndim; 137 | for (int i = 0; i < aux_params.ndim; ++i) { 138 | aux_params.shape[i] = shapes.newshape[i]; 139 | } 140 | caterva_storage_t aux_storage = {0}; 141 | aux_storage.contiguous = backend.contiguous; 142 | for (int i = 0; i < params.ndim; ++i) { 143 | aux_storage.chunkshape[i] = shapes.chunkshape[i]; 144 | aux_storage.blockshape[i] = shapes.blockshape[i]; 145 | } 146 | CATERVA_ERROR(caterva_full(data->ctx, &aux_params, &aux_storage, value, &aux)); 147 | if (!only_shrink) { 148 | for (int i = 0; i < shapes.ndim; ++i) { 149 | if (shapes.newshape[i] <= shapes.shape[i]) { 150 | continue; 151 | } 152 | int64_t slice_start[CATERVA_MAX_DIM] = {0}; 153 | int64_t slice_stop[CATERVA_MAX_DIM]; 154 | int64_t slice_shape[CATERVA_MAX_DIM] = {0}; 155 | int64_t buffer_len = 1; 156 | for (int j = 0; j < shapes.ndim; ++j) { 157 | if (j != i) { 158 | slice_shape[j] = shapes.newshape[j]; 159 | buffer_len *= slice_shape[j]; 160 | slice_stop[j] = shapes.newshape[j]; 161 | } 162 | } 163 | slice_start[i] = shapes.start_resize[i]; 164 | slice_shape[i] = shapes.newshape[i] - shapes.shape[i]; 165 | if (slice_start[i] % shapes.chunkshape[i] != 0) { 166 | // Old padding was filled with ones 167 | slice_shape[i] -= shapes.chunkshape[i] - slice_start[i] % shapes.chunkshape[i]; 168 | slice_start[i] += shapes.chunkshape[i] - slice_start[i] % shapes.chunkshape[i]; 169 | } 170 | if (slice_start[i] > shapes.newshape[i]) { 171 | continue; 172 | } 173 | slice_stop[i] = slice_start[i] + slice_shape[i]; 174 | buffer_len *= slice_shape[i]; 175 | uint8_t *buffer = calloc((size_t) buffer_len, (size_t) itemsize); 176 | CATERVA_ERROR(caterva_set_slice_buffer(data->ctx, buffer, slice_shape, buffer_len * itemsize, 177 | slice_start, slice_stop, aux)); 178 | free(buffer); 179 | } 180 | } 181 | 182 | /* Fill buffers with whole arrays */ 183 | uint8_t *src_buffer = data->ctx->cfg->alloc((size_t) buffersize); 184 | uint8_t *aux_buffer = data->ctx->cfg->alloc((size_t) buffersize); 185 | CATERVA_TEST_ASSERT(caterva_to_buffer(data->ctx, src, src_buffer, buffersize)); 186 | CATERVA_TEST_ASSERT(caterva_to_buffer(data->ctx, aux, aux_buffer, buffersize)); 187 | for (uint64_t i = 0; i < (uint64_t) buffersize / itemsize; ++i) { 188 | switch (itemsize) { 189 | case 8: 190 | CUTEST_ASSERT("Elements are not equal!", 191 | ((uint64_t *) src_buffer)[i] == ((uint64_t *) aux_buffer)[i]); 192 | break; 193 | case 4: 194 | CUTEST_ASSERT("Elements are not equal!", 195 | ((uint32_t *) src_buffer)[i] == ((uint32_t *) aux_buffer)[i]); 196 | break; 197 | case 2: 198 | CUTEST_ASSERT("Elements are not equal!", 199 | ((uint16_t *) src_buffer)[i] == ((uint16_t *) aux_buffer)[i]); 200 | break; 201 | case 1: 202 | CUTEST_ASSERT("Elements are not equal!", 203 | ((uint8_t *) src_buffer)[i] == ((uint8_t *) aux_buffer)[i]); 204 | break; 205 | default: 206 | CATERVA_TEST_ASSERT(CATERVA_ERR_INVALID_ARGUMENT); 207 | } 208 | } 209 | /* Free mallocs */ 210 | free(value); 211 | data->ctx->cfg->free(src_buffer); 212 | data->ctx->cfg->free(aux_buffer); 213 | 214 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 215 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &aux)); 216 | caterva_remove(data->ctx, urlpath); 217 | 218 | return 0; 219 | } 220 | 221 | CUTEST_TEST_TEARDOWN(resize_shape) { 222 | caterva_ctx_free(&data->ctx); 223 | } 224 | 225 | int main() { 226 | CUTEST_TEST_RUN(resize_shape); 227 | } 228 | -------------------------------------------------------------------------------- /tests/test_roundtrip.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #include "test_common.h" 13 | 14 | 15 | CUTEST_TEST_DATA(roundtrip) { 16 | caterva_ctx_t *ctx; 17 | }; 18 | 19 | 20 | CUTEST_TEST_SETUP(roundtrip) { 21 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 22 | cfg.nthreads = 2; 23 | cfg.compcodec = BLOSC_BLOSCLZ; 24 | caterva_ctx_new(&cfg, &data->ctx); 25 | 26 | // Add parametrizations 27 | caterva_default_parameters(); 28 | } 29 | 30 | 31 | CUTEST_TEST_TEST(roundtrip) { 32 | CUTEST_GET_PARAMETER(backend, _test_backend); 33 | CUTEST_GET_PARAMETER(shapes, _test_shapes); 34 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 35 | 36 | char *urlpath = "test_roundtrip.b2frame"; 37 | caterva_remove(data->ctx, urlpath); 38 | 39 | caterva_params_t params; 40 | params.itemsize = itemsize; 41 | params.ndim = shapes.ndim; 42 | for (int i = 0; i < shapes.ndim; ++i) { 43 | params.shape[i] = shapes.shape[i]; 44 | } 45 | 46 | caterva_storage_t storage = {0}; 47 | if (backend.persistent) { 48 | storage.urlpath = urlpath; 49 | } 50 | storage.contiguous = backend.contiguous; 51 | for (int i = 0; i < shapes.ndim; ++i) { 52 | storage.chunkshape[i] = shapes.chunkshape[i]; 53 | storage.blockshape[i] = shapes.blockshape[i]; 54 | } 55 | 56 | /* Create original data */ 57 | size_t buffersize = (size_t) itemsize; 58 | for (int i = 0; i < shapes.ndim; ++i) { 59 | buffersize *= (size_t) shapes.shape[i]; 60 | } 61 | uint8_t *buffer = malloc(buffersize); 62 | CUTEST_ASSERT("Buffer filled incorrectly", fill_buf(buffer, itemsize, buffersize / itemsize)); 63 | 64 | /* Create caterva_array_t with original data */ 65 | caterva_array_t *src; 66 | CATERVA_TEST_ASSERT(caterva_from_buffer(data->ctx, buffer, buffersize, ¶ms, 67 | &storage, 68 | &src)); 69 | 70 | /* Fill dest array with caterva_array_t data */ 71 | uint8_t *buffer_dest = malloc( buffersize); 72 | CATERVA_TEST_ASSERT(caterva_to_buffer(data->ctx, src, buffer_dest, buffersize)); 73 | 74 | /* Testing */ 75 | CATERVA_TEST_ASSERT_BUFFER(buffer, buffer_dest, (int) buffersize); 76 | 77 | /* Free mallocs */ 78 | free(buffer); 79 | free(buffer_dest); 80 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 81 | caterva_remove(data->ctx, urlpath); 82 | 83 | return CATERVA_SUCCEED; 84 | } 85 | 86 | 87 | CUTEST_TEST_TEARDOWN(roundtrip) { 88 | caterva_ctx_free(&data->ctx); 89 | } 90 | 91 | int main() { 92 | CUTEST_TEST_RUN(roundtrip); 93 | } 94 | -------------------------------------------------------------------------------- /tests/test_save.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #include "test_common.h" 13 | #ifdef __GNUC__ 14 | #include 15 | #define FILE_EXISTS(urlpath) access(urlpath, F_OK) 16 | #else 17 | #include 18 | #define FILE_EXISTS(urlpath) _access(urlpath, 0) 19 | #endif 20 | 21 | 22 | typedef struct { 23 | int8_t ndim; 24 | int64_t shape[CATERVA_MAX_DIM]; 25 | int32_t chunkshape[CATERVA_MAX_DIM]; 26 | int32_t blockshape[CATERVA_MAX_DIM]; 27 | } test_shapes_t; 28 | 29 | 30 | CUTEST_TEST_DATA(save) { 31 | caterva_ctx_t *ctx; 32 | }; 33 | 34 | 35 | CUTEST_TEST_SETUP(save) { 36 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 37 | cfg.nthreads = 2; 38 | cfg.compcodec = BLOSC_BLOSCLZ; 39 | caterva_ctx_new(&cfg, &data->ctx); 40 | 41 | // Add parametrizations 42 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA(1, 2, 4, 8)); 43 | CUTEST_PARAMETRIZE(shapes, test_shapes_t, CUTEST_DATA( 44 | // {0, {0}, {0}, {0}}, // 0-dim 45 | {1, {10}, {7}, {2}}, // 1-idim 46 | {2, {100, 100}, {20, 20}, {10, 10}}, 47 | {3, {100, 55, 123}, {31, 5, 22}, {4, 4, 4}}, 48 | {3, {100, 0, 12}, {31, 0, 12}, {10, 0, 12}}, 49 | )); 50 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 51 | {true, false}, 52 | {false, false}, 53 | )); 54 | } 55 | 56 | CUTEST_TEST_TEST(save) { 57 | CUTEST_GET_PARAMETER(backend, _test_backend); 58 | CUTEST_GET_PARAMETER(shapes, test_shapes_t); 59 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 60 | 61 | char* urlpath = "test_save.b2frame"; 62 | 63 | caterva_remove(data->ctx, urlpath); 64 | 65 | caterva_params_t params; 66 | params.itemsize = itemsize; 67 | params.ndim = shapes.ndim; 68 | for (int i = 0; i < params.ndim; ++i) { 69 | params.shape[i] = shapes.shape[i]; 70 | } 71 | 72 | caterva_storage_t storage = {0}; 73 | storage.urlpath = NULL; 74 | storage.contiguous = backend.contiguous; 75 | for (int i = 0; i < params.ndim; ++i) { 76 | storage.chunkshape[i] = shapes.chunkshape[i]; 77 | storage.blockshape[i] = shapes.blockshape[i]; 78 | } 79 | 80 | /* Create original data */ 81 | int64_t buffersize = itemsize; 82 | for (int i = 0; i < params.ndim; ++i) { 83 | buffersize *= shapes.shape[i]; 84 | } 85 | uint8_t *buffer = malloc(buffersize); 86 | CUTEST_ASSERT("Buffer filled incorrectly", fill_buf(buffer, itemsize, buffersize / itemsize)); 87 | 88 | /* Create caterva_array_t with original data */ 89 | caterva_array_t *src; 90 | CATERVA_TEST_ASSERT(caterva_from_buffer(data->ctx, buffer, buffersize, ¶ms, &storage, 91 | &src)); 92 | 93 | CATERVA_TEST_ASSERT(caterva_save(data->ctx, src, urlpath)); 94 | caterva_array_t *dest; 95 | CATERVA_TEST_ASSERT(caterva_open(data->ctx, urlpath, &dest)); 96 | 97 | /* Fill dest array with caterva_array_t data */ 98 | uint8_t *buffer_dest = malloc(buffersize); 99 | CATERVA_TEST_ASSERT(caterva_to_buffer(data->ctx, dest, buffer_dest, buffersize)); 100 | 101 | /* Testing */ 102 | if (dest->nitems != 0) { 103 | for (int i = 0; i < buffersize / itemsize; ++i) { 104 | // printf("%d - %d\n", buffer[i], buffer_dest[i]); 105 | CUTEST_ASSERT("Elements are not equals!", buffer[i] == buffer_dest[i]); 106 | } 107 | } 108 | 109 | /* Free mallocs */ 110 | free(buffer); 111 | free(buffer_dest); 112 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 113 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &dest)); 114 | 115 | caterva_remove(data->ctx, urlpath); 116 | 117 | return 0; 118 | } 119 | 120 | 121 | CUTEST_TEST_TEARDOWN(save) { 122 | caterva_ctx_free(&data->ctx); 123 | } 124 | 125 | int main() { 126 | CUTEST_TEST_RUN(save); 127 | } 128 | -------------------------------------------------------------------------------- /tests/test_serialize.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #include "test_common.h" 13 | 14 | 15 | CUTEST_TEST_DATA(serialize) { 16 | caterva_ctx_t *ctx; 17 | }; 18 | 19 | 20 | CUTEST_TEST_SETUP(serialize) { 21 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 22 | cfg.nthreads = 2; 23 | cfg.compcodec = BLOSC_BLOSCLZ; 24 | caterva_ctx_new(&cfg, &data->ctx); 25 | 26 | // Add parametrizations 27 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA(1, 2, 4, 8)); 28 | CUTEST_PARAMETRIZE(shapes, _test_shapes, CUTEST_DATA( 29 | {0, {0}, {0}, {0}}, // 0-dim 30 | {1, {10}, {7}, {2}}, // 1-idim 31 | {2, {100, 100}, {20, 20}, {10, 10}}, 32 | {3, {100, 55, 123}, {31, 5, 22}, {4, 4, 4}}, 33 | {3, {100, 0, 12}, {31, 0, 12}, {10, 0, 12}}, 34 | {4, {50, 160, 31, 12}, {25, 20, 20, 10}, {5, 5, 5, 10}}, 35 | {5, {1, 1, 1024, 1, 1}, {1, 1, 500, 1, 1}, {1, 1, 200, 1, 1}}, 36 | {6, {5, 1, 200, 3, 1, 2}, {5, 1, 50, 2, 1, 2}, {2, 1, 20, 2, 1, 2}} 37 | )); 38 | CUTEST_PARAMETRIZE(contiguous, bool, CUTEST_DATA(true, false)); 39 | } 40 | 41 | 42 | CUTEST_TEST_TEST(serialize) { 43 | CUTEST_GET_PARAMETER(shapes, _test_shapes); 44 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 45 | CUTEST_GET_PARAMETER(contiguous, bool); 46 | 47 | caterva_params_t params; 48 | params.itemsize = itemsize; 49 | params.ndim = shapes.ndim; 50 | for (int i = 0; i < params.ndim; ++i) { 51 | params.shape[i] = shapes.shape[i]; 52 | } 53 | 54 | caterva_storage_t storage = {0}; 55 | storage.urlpath = NULL; 56 | storage.contiguous = contiguous; 57 | for (int i = 0; i < params.ndim; ++i) { 58 | storage.chunkshape[i] = shapes.chunkshape[i]; 59 | storage.blockshape[i] = shapes.blockshape[i]; 60 | } 61 | 62 | /* Create original data */ 63 | size_t buffersize = itemsize; 64 | for (int i = 0; i < params.ndim; ++i) { 65 | buffersize *= (size_t) params.shape[i]; 66 | } 67 | 68 | 69 | uint8_t *buffer = malloc(buffersize); 70 | CUTEST_ASSERT("Buffer filled incorrectly", fill_buf(buffer, itemsize, buffersize / itemsize)); 71 | 72 | /* Create caterva_array_t with original data */ 73 | caterva_array_t *src; 74 | CATERVA_TEST_ASSERT(caterva_from_buffer(data->ctx, buffer, buffersize, ¶ms, &storage, 75 | &src)); 76 | 77 | uint8_t *cframe; 78 | int64_t cframe_len; 79 | bool needs_free; 80 | CATERVA_TEST_ASSERT(caterva_to_cframe(data->ctx, src, &cframe, &cframe_len, &needs_free)); 81 | 82 | caterva_array_t *dest; 83 | CATERVA_TEST_ASSERT(caterva_from_cframe(data->ctx, cframe, cframe_len, true, &dest)); 84 | 85 | /* Fill dest array with caterva_array_t data */ 86 | uint8_t *buffer_dest = malloc(buffersize); 87 | CATERVA_TEST_ASSERT(caterva_to_buffer(data->ctx, dest, buffer_dest, buffersize)); 88 | 89 | /* Testing */ 90 | CATERVA_TEST_ASSERT_BUFFER(buffer, buffer_dest, (int) buffersize); 91 | 92 | /* Free mallocs */ 93 | if (needs_free) { 94 | free(cframe); 95 | } 96 | 97 | free(buffer); 98 | free(buffer_dest); 99 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 100 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &dest)); 101 | 102 | return 0; 103 | } 104 | 105 | 106 | CUTEST_TEST_TEARDOWN(serialize) { 107 | caterva_ctx_free(&data->ctx); 108 | } 109 | 110 | int main() { 111 | CUTEST_TEST_RUN(serialize); 112 | } 113 | -------------------------------------------------------------------------------- /tests/test_set_slice_buffer.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | 13 | #include "test_common.h" 14 | 15 | typedef struct { 16 | int8_t ndim; 17 | int64_t shape[CATERVA_MAX_DIM]; 18 | int32_t chunkshape[CATERVA_MAX_DIM]; 19 | int32_t blockshape[CATERVA_MAX_DIM]; 20 | int64_t start[CATERVA_MAX_DIM]; 21 | int64_t stop[CATERVA_MAX_DIM]; 22 | } test_shapes_t; 23 | 24 | 25 | CUTEST_TEST_DATA(set_slice_buffer) { 26 | caterva_ctx_t *ctx; 27 | }; 28 | 29 | 30 | CUTEST_TEST_SETUP(set_slice_buffer) { 31 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 32 | cfg.nthreads = 2; 33 | cfg.compcodec = BLOSC_ZSTD; 34 | caterva_ctx_new(&cfg, &data->ctx); 35 | 36 | // Add parametrizations 37 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA( 38 | 1, 39 | 2, 40 | 4, 41 | 8, 42 | )); 43 | 44 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 45 | {false, false}, 46 | {true, false}, 47 | {true, true}, 48 | {false, true}, 49 | )); 50 | 51 | 52 | CUTEST_PARAMETRIZE(shapes, test_shapes_t, CUTEST_DATA( 53 | {0, {0}, {0}, {0}, {0}, {0}}, // 0-dim 54 | {1, {5}, {3}, {2}, {2}, {5}}, // 1-idim 55 | {2, {20, 0}, {7, 0}, {3, 0}, {2, 0}, {8, 0}}, // 0-shape 56 | {2, {20, 10}, {7, 5}, {3, 5}, {2, 0}, {18, 0}}, // 0-shape 57 | {2, {14, 10}, {8, 5}, {2, 2}, {5, 3}, {9, 10}}, // general, 58 | {3, {12, 10, 14}, {3, 5, 9}, {3, 4, 4}, {3, 0, 3}, {6, 7, 10}}, // general 59 | {3, {10, 21, 30, 55}, {8, 7, 15, 3}, {5, 5, 10, 1}, {5, 4, 3, 3}, {10, 8, 8, 34}}, // general, 60 | {2, {50, 50}, {25, 13}, {8, 8}, {0, 0}, {10, 10}}, // general, 61 | {2, {143, 41}, {18, 13}, {7, 7}, {4, 2}, {6, 5}}, // general, 62 | {2, {10, 10}, {5, 7}, {2, 2}, {0, 0}, {5, 5}}, 63 | 64 | )); 65 | } 66 | 67 | CUTEST_TEST_TEST(set_slice_buffer) { 68 | CUTEST_GET_PARAMETER(backend, _test_backend); 69 | CUTEST_GET_PARAMETER(shapes, test_shapes_t); 70 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 71 | 72 | char *urlpath = "test_set_slice_buffer.b2frame"; 73 | caterva_remove(data->ctx, urlpath); 74 | 75 | caterva_params_t params; 76 | params.itemsize = itemsize; 77 | params.ndim = shapes.ndim; 78 | for (int i = 0; i < params.ndim; ++i) { 79 | params.shape[i] = shapes.shape[i]; 80 | } 81 | 82 | caterva_storage_t storage = {0}; 83 | if (backend.persistent) { 84 | storage.urlpath = urlpath; 85 | } 86 | storage.contiguous = backend.contiguous; 87 | for (int i = 0; i < params.ndim; ++i) { 88 | storage.chunkshape[i] = shapes.chunkshape[i]; 89 | storage.blockshape[i] = shapes.blockshape[i]; 90 | } 91 | 92 | /* Create dest buffer */ 93 | int64_t shape[CATERVA_MAX_DIM] = {0}; 94 | int64_t buffersize = itemsize; 95 | for (int i = 0; i < params.ndim; ++i) { 96 | shape[i] = shapes.stop[i] - shapes.start[i]; 97 | buffersize *= shape[i]; 98 | } 99 | 100 | uint8_t *buffer = data->ctx->cfg->alloc(buffersize); 101 | CUTEST_ASSERT("Buffer filled incorrectly", fill_buf(buffer, itemsize, buffersize / itemsize)); 102 | 103 | /* Create caterva_array_t with original data */ 104 | caterva_array_t *src; 105 | CATERVA_ERROR(caterva_zeros(data->ctx, ¶ms, &storage, &src)); 106 | 107 | 108 | CATERVA_ERROR(caterva_set_slice_buffer(data->ctx, buffer, shape, buffersize, 109 | shapes.start, shapes.stop, src)); 110 | 111 | 112 | uint8_t *destbuffer = data->ctx->cfg->alloc((size_t) buffersize); 113 | 114 | /* Fill dest buffer with a slice*/ 115 | CATERVA_TEST_ASSERT(caterva_get_slice_buffer(data->ctx, src, shapes.start, shapes.stop, 116 | destbuffer, 117 | shape, buffersize)); 118 | 119 | for (uint64_t i = 0; i < (uint64_t) buffersize / itemsize; ++i) { 120 | uint64_t k = i + 1; 121 | switch (itemsize) { 122 | case 8: 123 | CUTEST_ASSERT("Elements are not equals!", 124 | (uint64_t) k == ((uint64_t *) destbuffer)[i]); 125 | break; 126 | case 4: 127 | CUTEST_ASSERT("Elements are not equals!", 128 | (uint32_t) k == ((uint32_t *) destbuffer)[i]); 129 | break; 130 | case 2: 131 | CUTEST_ASSERT("Elements are not equals!", 132 | (uint16_t) k == ((uint16_t *) destbuffer)[i]); 133 | break; 134 | case 1: 135 | CUTEST_ASSERT("Elements are not equals!", 136 | (uint8_t) k == ((uint8_t *) destbuffer)[i]); 137 | break; 138 | default: 139 | CATERVA_TEST_ASSERT(CATERVA_ERR_INVALID_ARGUMENT); 140 | } 141 | } 142 | 143 | /* Free mallocs */ 144 | data->ctx->cfg->free(buffer); 145 | data->ctx->cfg->free(destbuffer); 146 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 147 | caterva_remove(data->ctx, urlpath); 148 | 149 | return 0; 150 | } 151 | 152 | CUTEST_TEST_TEARDOWN(set_slice_buffer) { 153 | caterva_ctx_free(&data->ctx); 154 | } 155 | 156 | int main() { 157 | CUTEST_TEST_RUN(set_slice_buffer); 158 | } 159 | -------------------------------------------------------------------------------- /tests/test_squeeze.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #include "test_common.h" 13 | 14 | typedef struct { 15 | int8_t ndim; 16 | int64_t shape[CATERVA_MAX_DIM]; 17 | int32_t chunkshape[CATERVA_MAX_DIM]; 18 | int32_t blockshape[CATERVA_MAX_DIM]; 19 | int32_t chunkshape2[CATERVA_MAX_DIM]; 20 | int32_t blockshape2[CATERVA_MAX_DIM]; 21 | int64_t start[CATERVA_MAX_DIM]; 22 | int64_t stop[CATERVA_MAX_DIM]; 23 | } test_shapes_t; 24 | 25 | 26 | CUTEST_TEST_DATA(squeeze) { 27 | caterva_ctx_t *ctx; 28 | }; 29 | 30 | 31 | CUTEST_TEST_SETUP(squeeze) { 32 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 33 | cfg.nthreads = 2; 34 | cfg.compcodec = BLOSC_BLOSCLZ; 35 | caterva_ctx_new(&cfg, &data->ctx); 36 | 37 | // Add parametrizations 38 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA( 39 | 1, 40 | 2, 41 | 4, 42 | 8 43 | )); 44 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 45 | {false, false}, 46 | {true, false}, 47 | {true, true}, 48 | {false, true}, 49 | )); 50 | CUTEST_PARAMETRIZE(backend2, _test_backend, CUTEST_DATA( 51 | {false, false}, 52 | {true, false}, 53 | {true, true}, 54 | {false, true}, 55 | )); 56 | 57 | 58 | CUTEST_PARAMETRIZE(shapes, test_shapes_t, CUTEST_DATA( 59 | {0, {0}, {0}, {0}, {0}, {0}, {0}, {0}}, // 0-dim 60 | {1, {10}, {7}, {2}, {1}, {1}, {2}, {3}}, // 1-idim 61 | {2, {14, 10}, {8, 5}, {2, 2}, {4, 1}, {2, 1}, {5, 3}, {9, 4}}, // general, 62 | {3, {10, 10, 10}, {3, 5, 9}, {3, 4, 4}, {1, 7, 1}, {1, 5, 1}, {3, 0, 9}, {4, 7, 10}}, 63 | {2, {20, 0}, {7, 0}, {3, 0}, {1, 0}, {1, 0}, {1, 0}, {2, 0}}, // 0-shape 64 | {2, {20, 10}, {7, 5}, {3, 5}, {1, 0}, {1, 0}, {17, 0}, {18, 0}}, // 0-shape 65 | )); 66 | } 67 | 68 | 69 | CUTEST_TEST_TEST(squeeze) { 70 | CUTEST_GET_PARAMETER(backend, _test_backend); 71 | CUTEST_GET_PARAMETER(shapes, test_shapes_t); 72 | CUTEST_GET_PARAMETER(backend2, _test_backend); 73 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 74 | 75 | char *urlpath = "test_squeeze.b2frame"; 76 | char *urlpath2 = "test_squeeze2.b2frame"; 77 | 78 | caterva_remove(data->ctx, urlpath); 79 | caterva_remove(data->ctx, urlpath2); 80 | 81 | caterva_params_t params; 82 | params.itemsize = itemsize; 83 | params.ndim = shapes.ndim; 84 | for (int i = 0; i < params.ndim; ++i) { 85 | params.shape[i] = shapes.shape[i]; 86 | } 87 | 88 | caterva_storage_t storage = {0}; 89 | if (backend.persistent) { 90 | storage.urlpath = urlpath; 91 | } 92 | storage.contiguous = backend.contiguous; 93 | for (int i = 0; i < params.ndim; ++i) { 94 | storage.chunkshape[i] = shapes.chunkshape[i]; 95 | storage.blockshape[i] = shapes.blockshape[i]; 96 | } 97 | 98 | /* Create original data */ 99 | size_t buffersize = itemsize; 100 | for (int i = 0; i < params.ndim; ++i) { 101 | buffersize *= (size_t) params.shape[i]; 102 | } 103 | uint8_t *buffer = malloc(buffersize); 104 | CUTEST_ASSERT("Buffer filled incorrectly", fill_buf(buffer, itemsize, buffersize / itemsize)); 105 | 106 | /* Create caterva_array_t with original data */ 107 | caterva_array_t *src; 108 | CATERVA_TEST_ASSERT(caterva_from_buffer(data->ctx, buffer, buffersize, ¶ms, &storage, 109 | &src)); 110 | 111 | 112 | /* Create storage for dest container */ 113 | 114 | caterva_storage_t storage2 = {0}; 115 | if (backend2.persistent) { 116 | storage2.urlpath = urlpath2; 117 | } 118 | storage2.contiguous = backend2.contiguous; 119 | for (int i = 0; i < params.ndim; ++i) { 120 | storage2.chunkshape[i] = shapes.chunkshape2[i]; 121 | storage2.blockshape[i] = shapes.blockshape2[i]; 122 | } 123 | 124 | caterva_array_t *dest; 125 | CATERVA_TEST_ASSERT(caterva_get_slice(data->ctx, src, shapes.start, shapes.stop, 126 | &storage2, &dest)); 127 | 128 | CATERVA_TEST_ASSERT(caterva_squeeze(data->ctx, dest)); 129 | 130 | if (params.ndim != 0) { 131 | CUTEST_ASSERT("dims are equal", src->ndim != dest->ndim); 132 | } 133 | 134 | free(buffer); 135 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 136 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &dest)); 137 | caterva_remove(data->ctx, urlpath); 138 | caterva_remove(data->ctx, urlpath2); 139 | 140 | return CATERVA_SUCCEED; 141 | } 142 | 143 | CUTEST_TEST_TEARDOWN(squeeze) { 144 | caterva_ctx_free(&data->ctx); 145 | } 146 | 147 | int main() { 148 | CUTEST_TEST_RUN(squeeze); 149 | } 150 | -------------------------------------------------------------------------------- /tests/test_squeeze_index.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #include "test_common.h" 13 | 14 | typedef struct { 15 | int8_t ndim; 16 | int64_t shape[CATERVA_MAX_DIM]; 17 | int32_t chunkshape[CATERVA_MAX_DIM]; 18 | int32_t blockshape[CATERVA_MAX_DIM]; 19 | int32_t chunkshape2[CATERVA_MAX_DIM]; 20 | int32_t blockshape2[CATERVA_MAX_DIM]; 21 | int64_t start[CATERVA_MAX_DIM]; 22 | int64_t stop[CATERVA_MAX_DIM]; 23 | bool squeeze_indexes[CATERVA_MAX_DIM]; 24 | } test_squeeze_index_shapes_t; 25 | 26 | 27 | CUTEST_TEST_DATA(squeeze_index) { 28 | caterva_ctx_t *ctx; 29 | }; 30 | 31 | 32 | CUTEST_TEST_SETUP(squeeze_index) { 33 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 34 | cfg.nthreads = 2; 35 | cfg.compcodec = BLOSC_BLOSCLZ; 36 | caterva_ctx_new(&cfg, &data->ctx); 37 | 38 | // Add parametrizations 39 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA( 40 | 1, 41 | 2, 42 | 4, 43 | 8 44 | )); 45 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 46 | {false, false}, 47 | {true, false}, 48 | {true, true}, 49 | {false, true}, 50 | )); 51 | CUTEST_PARAMETRIZE(backend2, _test_backend, CUTEST_DATA( 52 | {false, false}, 53 | {true, false}, 54 | {true, true}, 55 | {false, true}, 56 | )); 57 | 58 | 59 | CUTEST_PARAMETRIZE(shapes, test_squeeze_index_shapes_t, CUTEST_DATA( 60 | {0, {0}, {0}, {0}, {0}, {0}, 61 | {0}, {0}, {0}}, // 0-dim 62 | {1, {10}, {7}, {2}, {1}, {1}, 63 | {2}, {3}, {0}}, // 1-idim 64 | {2, {14, 10}, {8, 5}, {2, 2}, {4, 1}, {2, 1}, 65 | {5, 3}, {9, 4}, {0, 1}}, // general, 66 | {3, {10, 10, 10}, {3, 5, 9}, {3, 4, 4}, {1, 7, 1}, {1, 5, 1}, 67 | {3, 0, 9}, {4, 7, 10}, {1, 0, 0}}, 68 | {2, {20, 0}, {7, 0}, {3, 0}, {1, 0}, {1, 0}, 69 | {1, 0}, {2, 0}, {1, 0}}, // 0-shape 70 | {2, {20, 10}, {7, 5}, {3, 5}, {1, 0}, {1, 0}, 71 | {17, 0}, {18, 0}, {1, 0}}, // 0-shape 72 | {4, {10, 10, 10, 10}, {7, 5, 1, 4}, {2, 2, 1, 2}, {1, 1, 5, 1}, {1, 1, 2, 1}, 73 | {4, 4, 0, 4}, {5, 5, 10, 5}, {1, 0, 0, 1}} // general 74 | )); 75 | } 76 | 77 | 78 | CUTEST_TEST_TEST(squeeze_index) { 79 | CUTEST_GET_PARAMETER(backend, _test_backend); 80 | CUTEST_GET_PARAMETER(shapes, test_squeeze_index_shapes_t); 81 | CUTEST_GET_PARAMETER(backend2, _test_backend); 82 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 83 | 84 | char *urlpath = "test_squeeze_index.b2frame"; 85 | char *urlpath2 = "test_squezze_index2.b2frame"; 86 | 87 | caterva_remove(data->ctx, urlpath); 88 | caterva_remove(data->ctx, urlpath2); 89 | 90 | caterva_params_t params; 91 | params.itemsize = itemsize; 92 | params.ndim = shapes.ndim; 93 | for (int i = 0; i < params.ndim; ++i) { 94 | params.shape[i] = shapes.shape[i]; 95 | } 96 | 97 | caterva_storage_t storage = {0}; 98 | if (backend.persistent) { 99 | storage.urlpath = urlpath; 100 | } 101 | storage.contiguous = backend.persistent; 102 | for (int i = 0; i < params.ndim; ++i) { 103 | storage.chunkshape[i] = shapes.chunkshape[i]; 104 | storage.blockshape[i] = shapes.blockshape[i]; 105 | } 106 | 107 | /* Create original data */ 108 | size_t buffersize = itemsize; 109 | for (int i = 0; i < params.ndim; ++i) { 110 | buffersize *= (size_t) params.shape[i]; 111 | } 112 | uint8_t *buffer = malloc(buffersize); 113 | CUTEST_ASSERT("Buffer filled incorrectly", fill_buf(buffer, itemsize, buffersize / itemsize)); 114 | 115 | /* Create caterva_array_t with original data */ 116 | caterva_array_t *src; 117 | CATERVA_TEST_ASSERT(caterva_from_buffer(data->ctx, buffer, buffersize, ¶ms, &storage, 118 | &src)); 119 | 120 | 121 | /* Create storage for dest container */ 122 | 123 | caterva_storage_t storage2 = {0}; 124 | if (backend2.persistent) { 125 | storage2.urlpath = urlpath2; 126 | } 127 | storage2.contiguous = backend2.contiguous; 128 | for (int i = 0; i < params.ndim; ++i) { 129 | storage2.chunkshape[i] = shapes.chunkshape2[i]; 130 | storage2.blockshape[i] = shapes.blockshape2[i]; 131 | } 132 | 133 | caterva_array_t *dest; 134 | CATERVA_TEST_ASSERT(caterva_get_slice(data->ctx, src, shapes.start, shapes.stop, 135 | &storage2, &dest)); 136 | 137 | CATERVA_TEST_ASSERT(caterva_squeeze_index(data->ctx, dest, shapes.squeeze_indexes)); 138 | 139 | int8_t nsq = 0; 140 | for (int i = 0; i < params.ndim; ++i) { 141 | if (shapes.squeeze_indexes[i] == true) { 142 | nsq++; 143 | } 144 | } 145 | CUTEST_ASSERT("dims are not correct", src->ndim == dest->ndim + nsq); 146 | 147 | free(buffer); 148 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 149 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &dest)); 150 | 151 | caterva_remove(data->ctx, urlpath); 152 | caterva_remove(data->ctx, urlpath2); 153 | 154 | return 0; 155 | } 156 | 157 | 158 | CUTEST_TEST_TEARDOWN(squeeze_index) { 159 | caterva_ctx_free(&data->ctx); 160 | } 161 | 162 | int main() { 163 | CUTEST_TEST_RUN(squeeze_index); 164 | } 165 | -------------------------------------------------------------------------------- /tests/test_zeros.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Francesc Alted, Aleix Alcacer. 3 | * Copyright (C) 2019-present Blosc Development team 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #include "test_common.h" 13 | 14 | 15 | CUTEST_TEST_DATA(zeros) { 16 | caterva_ctx_t *ctx; 17 | }; 18 | 19 | 20 | CUTEST_TEST_SETUP(zeros) { 21 | caterva_config_t cfg = CATERVA_CONFIG_DEFAULTS; 22 | cfg.nthreads = 2; 23 | cfg.compcodec = BLOSC_BLOSCLZ; 24 | caterva_ctx_new(&cfg, &data->ctx); 25 | 26 | // Add parametrizations 27 | CUTEST_PARAMETRIZE(itemsize, uint8_t, CUTEST_DATA( 28 | 1, 2, 4, 7 29 | )); 30 | CUTEST_PARAMETRIZE(shapes, _test_shapes, CUTEST_DATA( 31 | {0, {0}, {0}, {0}}, // 0-dim 32 | {1, {5}, {3}, {2}}, // 1-idim 33 | {2, {20, 0}, {7, 0}, {3, 0}}, // 0-shape 34 | {2, {20, 10}, {7, 5}, {3, 5}}, // 0-shape 35 | {2, {14, 10}, {8, 5}, {2, 2}}, // general, 36 | {3, {12, 10, 14}, {3, 5, 9}, {3, 4, 4}}, // general 37 | {3, {10, 21, 30, 55}, {8, 7, 15, 3}, {5, 5, 10, 1}}, // general, 38 | )); 39 | CUTEST_PARAMETRIZE(backend, _test_backend, CUTEST_DATA( 40 | {false, false}, 41 | {true, false}, 42 | {true, true}, 43 | {false, true}, 44 | )); 45 | } 46 | 47 | 48 | CUTEST_TEST_TEST(zeros) { 49 | CUTEST_GET_PARAMETER(backend, _test_backend); 50 | CUTEST_GET_PARAMETER(shapes, _test_shapes); 51 | CUTEST_GET_PARAMETER(itemsize, uint8_t); 52 | 53 | char *urlpath = "test_zeros.b2frame"; 54 | caterva_remove(data->ctx, urlpath); 55 | 56 | caterva_params_t params; 57 | params.itemsize = itemsize; 58 | params.ndim = shapes.ndim; 59 | for (int i = 0; i < shapes.ndim; ++i) { 60 | params.shape[i] = shapes.shape[i]; 61 | } 62 | 63 | caterva_storage_t storage = {0}; 64 | if (backend.persistent) { 65 | storage.urlpath = urlpath; 66 | } 67 | storage.contiguous = backend.contiguous; 68 | for (int i = 0; i < shapes.ndim; ++i) { 69 | storage.chunkshape[i] = shapes.chunkshape[i]; 70 | storage.blockshape[i] = shapes.blockshape[i]; 71 | } 72 | 73 | /* Create original data */ 74 | int64_t buffersize = itemsize; 75 | for (int i = 0; i < shapes.ndim; ++i) { 76 | buffersize *= shapes.shape[i]; 77 | } 78 | 79 | /* Create caterva_array_t with original data */ 80 | caterva_array_t *src; 81 | CATERVA_TEST_ASSERT(caterva_zeros(data->ctx, ¶ms, &storage, &src)); 82 | 83 | /* Fill dest array with caterva_array_t data */ 84 | uint8_t *buffer_dest = malloc( buffersize); 85 | CATERVA_TEST_ASSERT(caterva_to_buffer(data->ctx, src, buffer_dest, buffersize)); 86 | 87 | /* Testing */ 88 | for (int i = 0; i < buffersize; ++i) { 89 | CUTEST_ASSERT("Elements are not equals", buffer_dest[i] == 0); 90 | } 91 | 92 | /* Free mallocs */ 93 | free(buffer_dest); 94 | CATERVA_TEST_ASSERT(caterva_free(data->ctx, &src)); 95 | caterva_remove(data->ctx, urlpath); 96 | 97 | return CATERVA_SUCCEED; 98 | } 99 | 100 | 101 | CUTEST_TEST_TEARDOWN(zeros) { 102 | caterva_ctx_free(&data->ctx); 103 | } 104 | 105 | int main() { 106 | CUTEST_TEST_RUN(zeros); 107 | } 108 | --------------------------------------------------------------------------------