├── .appveyor.yml ├── .conda ├── libxnd │ ├── bld.bat │ ├── build.sh │ └── meta.yaml └── xnd │ ├── bld.bat │ ├── build.sh │ └── meta.yaml ├── .gitignore ├── .travis.yml ├── AUTHORS.txt ├── INSTALL.txt ├── LICENSE.txt ├── MANIFEST.in ├── Makefile.in ├── README.rst ├── config.guess ├── config.h.in ├── config.sub ├── configure ├── configure.ac ├── doc ├── Makefile ├── _static │ └── copybutton.js ├── conf.py ├── index.rst ├── libxnd │ ├── data-structures.rst │ ├── functions.rst │ └── index.rst ├── releases │ └── index.rst └── xnd │ ├── align-pack.rst │ ├── buffer-protocol.rst │ ├── index.rst │ ├── quickstart.rst │ └── xnd.rst ├── install-sh ├── libxnd ├── .objs │ └── README.txt ├── Makefile.in ├── Makefile.vc ├── bitmaps.c ├── bounds.c ├── contrib.h ├── contrib │ └── bfloat16.h ├── copy.c ├── cuda │ ├── cuda_memory.cu │ └── cuda_memory.h ├── equal.c ├── inline.h ├── overflow.h ├── shape.c ├── split.c ├── tests │ ├── Makefile.in │ ├── Makefile.vc │ ├── README.txt │ ├── runtest.c │ ├── test.h │ └── test_fixed.c ├── xnd.c └── xnd.h ├── python ├── test_xnd.py ├── xnd │ ├── __init__.py │ ├── _version.py │ ├── _xnd.c │ ├── contrib │ │ └── pretty.py │ ├── docstrings.h │ ├── pyxnd.h │ └── util.h ├── xnd_randvalue.py └── xnd_support.py ├── setup.py └── vcbuild ├── INSTALL.txt ├── runtest32.bat ├── runtest64.bat ├── vcbuild32.bat ├── vcbuild64.bat ├── vcclean.bat └── vcdistclean.bat /.appveyor.yml: -------------------------------------------------------------------------------- 1 | skip_tags: true 2 | 3 | matrix: 4 | fast_finish: true 5 | 6 | os: Visual Studio 2015 7 | 8 | environment: 9 | matrix: 10 | - PLATFORM: "x64" 11 | VCBUILD: "vcbuild64.bat" 12 | RUNTEST: "runtest64.bat" 13 | PYTHON: "C:\\Python36-x64" 14 | - PLATFORM: "x86" 15 | VCBUILD: "vcbuild32.bat" 16 | RUNTEST: "runtest32.bat" 17 | PYTHON: "C:\\Python36" 18 | 19 | before_build: 20 | "\"C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat\" %PLATFORM%" 21 | 22 | build_script: 23 | - git clone https://github.com/plures/ndtypes.git 24 | - cd ndtypes 25 | - "%PYTHON%\\python.exe setup.py install --local=..\\python" 26 | - cd ..\vcbuild 27 | - "%VCBUILD%" 28 | - "%RUNTEST%" 29 | - cd .. 30 | - "%PYTHON%\\python.exe setup.py module" 31 | - "%PYTHON%\\python.exe setup.py test" 32 | - "%PYTHON%\\python.exe setup.py distclean" 33 | 34 | 35 | -------------------------------------------------------------------------------- /.conda/libxnd/bld.bat: -------------------------------------------------------------------------------- 1 | set "LIBNDTYPESDIR=%PREFIX%\Library\bin" 2 | set "LIBNDTYPESINCLUDE=%PREFIX%\Library\include" 3 | cd "%RECIPE_DIR%\..\..\vcbuild" || exit 1 4 | call vcbuild64.bat || exit 1 5 | call runtest64.bat || exit 1 6 | copy /y dist64\lib* "%PREFIX%\Library\bin\" 7 | copy /y dist64\xnd.h "%PREFIX%\Library\include\" 8 | -------------------------------------------------------------------------------- /.conda/libxnd/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | cd "$RECIPE_DIR/../../" || exit 1 4 | ./configure --prefix="$PREFIX" --with-includes="$PREFIX/include" --with-libs="$PREFIX/lib" --without-docs || exit 1 5 | make check || exit 1 6 | make install 7 | -------------------------------------------------------------------------------- /.conda/libxnd/meta.yaml: -------------------------------------------------------------------------------- 1 | package: 2 | name: libxnd 3 | version: 0.2.0dev3 4 | 5 | build: 6 | number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }} 7 | string: py{{ environ.get('CONDA_PY', 'NA') }}_{{ environ.get('GIT_BUILD_STR', 'NA') }} 8 | 9 | test: 10 | skip: True # tests are in build.sh 11 | 12 | requirements: 13 | build: 14 | - libndtypes ==0.2.0dev3 15 | run: 16 | - libndtypes ==0.2.0dev3 17 | 18 | about: 19 | home: https://github.com/plures/ 20 | license: BSD 3-clause 21 | 22 | source: 23 | git_url: ../../ 24 | -------------------------------------------------------------------------------- /.conda/xnd/bld.bat: -------------------------------------------------------------------------------- 1 | cd "%RECIPE_DIR%\..\..\" || exit 1 2 | "%PYTHON%" setup.py conda_install || exit 1 3 | if not exist "%SP_DIR%\xnd\contrib" mkdir "%SP_DIR%\xnd\contrib" 4 | copy /y python\xnd\contrib\*.py "%SP_DIR%\xnd\contrib" 5 | if not exist "%RECIPE_DIR%\test" mkdir "%RECIPE_DIR%\test" 6 | copy /y python\*.py "%RECIPE_DIR%\test" 7 | -------------------------------------------------------------------------------- /.conda/xnd/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | cd $RECIPE_DIR/../../ || exit 1 4 | $PYTHON setup.py conda_install || exit 1 5 | mkdir -p $SP_DIR/xnd/contrib && cp python/xnd/contrib/*.py $SP_DIR/xnd/contrib 6 | mkdir -p $RECIPE_DIR/test && cp python/*.py $RECIPE_DIR/test 7 | -------------------------------------------------------------------------------- /.conda/xnd/meta.yaml: -------------------------------------------------------------------------------- 1 | package: 2 | name: xnd 3 | version: 0.2.0dev3 4 | 5 | build: 6 | number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }} 7 | string: py{{ environ.get('CONDA_PY', 'NA') }}_{{ environ.get('GIT_BUILD_STR', 'NA') }} 8 | 9 | test: 10 | files: 11 | - test 12 | commands: 13 | - python test/test_xnd.py --long 14 | requires: 15 | - libndtypes ==0.2.0dev3 16 | - libxnd ==0.2.0dev3 17 | - ndtypes ==0.2.0dev3 18 | - nomkl # [not win] 19 | - numpy 20 | - python ==3.7 21 | 22 | requirements: 23 | build: 24 | - libndtypes ==0.2.0dev3 25 | - libxnd ==0.2.0dev3 26 | - ndtypes ==0.2.0dev3 27 | - python ==3.7 28 | run: 29 | - libndtypes ==0.2.0dev3 30 | - libxnd ==0.2.0dev3 31 | - ndtypes ==0.2.0dev3 32 | - python ==3.7 33 | 34 | about: 35 | home: https://github.com/plures/ 36 | license: BSD 3-clause 37 | 38 | source: 39 | git_url: ../../ 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore ./configure generated files 2 | Makefile 3 | config.h 4 | config.log 5 | config.status 6 | 7 | # Ignore make generated C Files 8 | *.a 9 | *.o 10 | *.so 11 | *.so.* 12 | libxnd/tests/runtest 13 | libxnd/tests/runtest_shared 14 | 15 | # Ignore python generated build files 16 | build/ 17 | doc/build/ 18 | python/xnd/xnd.h 19 | *.egg-info 20 | __pycache__ 21 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | 3 | dist: xenial 4 | 5 | language: python 6 | python: 7 | - 3.6 8 | 9 | matrix: 10 | include: 11 | - compiler: gcc 12 | before_install: 13 | - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test 14 | - sudo apt-get update -qq 15 | install: 16 | - sudo apt-get install -qq gcc-6 17 | - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-6 90 18 | before_script: 19 | - python3 -m pip install numpy sphinx 20 | - git clone https://github.com/plures/ndtypes.git 21 | - cd ndtypes 22 | - python3 setup.py install --local=../python 23 | - cd .. 24 | script: 25 | - ./configure 26 | - make check 27 | - python3 setup.py module 28 | - python3 setup.py test 29 | - python3 setup.py doctest 30 | 31 | - compiler: gcc 32 | before_install: 33 | - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test 34 | - sudo apt-get update -qq 35 | install: 36 | - sudo apt-get install -qq gcc-7 37 | - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90 38 | before_script: 39 | - sudo apt-get install -y valgrind 40 | - git clone https://github.com/plures/ndtypes.git 41 | - cd ndtypes 42 | - python3 setup.py install --local=../python 43 | - cd .. 44 | script: 45 | - ./configure 46 | - make memcheck 47 | - python3 setup.py module 48 | - python3 setup.py test 49 | 50 | - compiler: clang 51 | before_script: 52 | - git clone https://github.com/plures/ndtypes.git 53 | - export CC=clang 54 | - cd ndtypes 55 | - python3 setup.py install --local=../python 56 | - cd .. 57 | script: 58 | - ./configure 59 | - make check 60 | - python3 setup.py module 61 | - python3 setup.py test 62 | 63 | - language: objective-c 64 | os: osx 65 | compiler: clang 66 | before_install: 67 | - brew install python3 68 | before_script: 69 | - git clone https://github.com/plures/ndtypes.git 70 | - export CC=clang 71 | - cd ndtypes 72 | - python3 setup.py install --local=../python 73 | - cd .. 74 | script: 75 | - ./configure 76 | - make check 77 | - python3 setup.py module 78 | - python3 setup.py test 79 | 80 | - compiler: gcc 81 | before_install: 82 | - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test 83 | - sudo apt-get update -qq 84 | install: 85 | - sudo apt-get install -qq gcc-8 86 | - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 90 87 | before_script: 88 | - git clone https://github.com/plures/ndtypes.git 89 | - cd ndtypes 90 | - python3 setup.py install --local=../python 91 | - cd .. 92 | script: 93 | - python3 setup.py build 94 | - python3 setup.py test 95 | -------------------------------------------------------------------------------- /AUTHORS.txt: -------------------------------------------------------------------------------- 1 | 2 | Version 0.2.0 3 | ------------- 4 | 5 | Stefan Krah , sponsored by Anaconda Inc. and Quansight LLC. 6 | -------------------------------------------------------------------------------- /INSTALL.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | Requirements (libndtypes) 4 | ========================= 5 | 6 | Unless libndtypes has already been installed in the system directories, 7 | it is recommended to clone libndtypes into the xnd directory. 8 | 9 | 10 | Get ndtypes 11 | ----------- 12 | git clone https://github.com/plures/ndtypes.git 13 | 14 | 15 | For building libxnd only 16 | ------------------------ 17 | cd ndtypes 18 | ./configure 19 | make 20 | cd .. 21 | 22 | 23 | For building the xnd Python module 24 | ---------------------------------- 25 | # This also builds libndtypes and copies the ndtypes Python package into 26 | # the python directory next to the xnd package. 27 | cd ndtypes 28 | python3 setup.py install --local=../python 29 | cd .. 30 | 31 | 32 | Unix: libxnd build instructions 33 | =============================== 34 | 35 | # Build 36 | ./configure 37 | make 38 | 39 | # Test 40 | make check 41 | 42 | # Install 43 | make install 44 | 45 | # Clean 46 | make clean 47 | 48 | # Distclean 49 | make distclean 50 | 51 | 52 | Windows: libxnd build instructions 53 | ================================== 54 | 55 | See vcbuild/INSTALL.txt. 56 | 57 | 58 | Unix/Windows: Python module build instructions 59 | ============================================== 60 | 61 | To avoid shared library mismatches, the Python module builds with an rpath 62 | and ships the library inside the package. 63 | 64 | Unless you are a distributor with tight control over the system library 65 | versions, it is not recommended to install the library for the Python 66 | module. 67 | 68 | 69 | Build all 70 | --------- 71 | 72 | # Build libxnd and the module (libxnd is copied into the package) 73 | python3 setup.py build 74 | 75 | # Test 76 | python3 setup.py test 77 | 78 | # Doctest (optional, relies on Sphinx) 79 | python3 setup.py doctest 80 | 81 | # Install 82 | python3 setup.py install 83 | 84 | # Clean libxnd and the module 85 | python3 setup.py distclean 86 | 87 | 88 | Build the module only (for developing) 89 | -------------------------------------- 90 | 91 | First, build libxnd as above. This also copies the shared library into 92 | the package. Then, to avoid rebuilding the library repeatedly, use: 93 | 94 | # Build the module 95 | python3 setup.py module 96 | 97 | # Clean the module 98 | python3 setup.py clean 99 | 100 | 101 | Alternative install (for developing) 102 | ------------------------------------ 103 | 104 | # Install the package into a local directory. This is mainly useful for 105 | # developing gufuncs: 106 | python3 setup.py install --local="$PWD/../python" 107 | 108 | # Windows: 109 | python.exe setup.py install --local="%cd%\..\python" 110 | 111 | 112 | Alternative install with conda (for developing) 113 | ----------------------------------------------- 114 | 115 | # Install the xnd package into a local directory. 116 | 117 | # Create and activate a new conda environment: 118 | conda create --name xnd python=3.7 119 | conda activate xnd 120 | 121 | # Use conda to install ndtypes (if you also want to work on ndtypes, please 122 | # check the project repository https://github.com/plures/ndtypes): 123 | conda install -c xnd/label/dev ndtypes 124 | 125 | # Use conda to install numpy (optional, some tests require the library): 126 | conda install numpy 127 | 128 | # Use conda to build libxnd and xnd: 129 | conda build .conda/libxnd -c xnd/label/dev 130 | conda build .conda/xnd -c xnd/label/dev 131 | 132 | # Use conda to install the local version of the library and the 133 | # Python module: 134 | conda install --use-local xnd 135 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017-2018, plures 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | global-include *.ac *.guess *.in *.sub *.vc *.sh *.bat *.yaml 2 | global-include *.c *.h *.py *.txt *.rst *.js 3 | global-include configure install-sh doc/Makefile 4 | -------------------------------------------------------------------------------- /Makefile.in: -------------------------------------------------------------------------------- 1 | 2 | # ============================================================================== 3 | # Unix Makefile for libxnd 4 | # ============================================================================== 5 | 6 | 7 | PACKAGE_TARNAME = @PACKAGE_TARNAME@ 8 | LIBSTATIC = @LIBSTATIC@ 9 | LIBNAME = @LIBNAME@ 10 | LIBSONAME = @LIBSONAME@ 11 | LIBSHARED = @LIBSHARED@ 12 | 13 | LIBRARY_PATH = ..:@CONFIGURE_LIBS_TEST@ 14 | INSTALL = @INSTALL@ 15 | 16 | prefix = @prefix@ 17 | exec_prefix = @exec_prefix@ 18 | includedir = @includedir@ 19 | libdir = @libdir@ 20 | datarootdir = @datarootdir@ 21 | docdir = @docdir@ 22 | 23 | 24 | default: FORCE 25 | cd libxnd && $(MAKE) 26 | $(INSTALL) -m 644 libxnd/xnd.h python/xnd 27 | $(INSTALL) -m 644 libxnd/$(LIBSTATIC) python/xnd 28 | $(INSTALL) -m 755 libxnd/$(LIBSHARED) python/xnd 29 | cd python/xnd && ln -sf $(LIBSHARED) $(LIBSONAME) && ln -sf $(LIBSHARED) $(LIBNAME) 30 | 31 | runtest: default 32 | cd libxnd/tests && $(MAKE) 33 | 34 | check: runtest 35 | cd libxnd/tests && ./runtest 36 | @printf "\n\n" 37 | cd libxnd/tests && @LIBRARY_PATH@=$(LIBRARY_PATH) ./runtest_shared 38 | 39 | memcheck: runtest 40 | cd libxnd/tests && valgrind --leak-check=full --show-leak-kinds=all ./runtest 41 | @printf "\n\n" 42 | cd libxnd/tests && @LIBRARY_PATH@=$(LIBRARY_PATH) valgrind --leak-check=full --show-leak-kinds=all ./runtest_shared 43 | 44 | install: install_libs @NDT_INSTALL_DOCS@ 45 | 46 | install_libs: FORCE 47 | $(INSTALL) -d -m 755 $(DESTDIR)$(includedir) 48 | $(INSTALL) -m 644 libxnd/xnd.h $(DESTDIR)$(includedir) 49 | $(INSTALL) -d -m 755 $(DESTDIR)$(libdir) 50 | $(INSTALL) -m 644 libxnd/$(LIBSTATIC) $(DESTDIR)$(libdir) 51 | $(INSTALL) -m 755 libxnd/$(LIBSHARED) $(DESTDIR)$(libdir) 52 | cd $(DESTDIR)$(libdir) && ln -sf $(LIBSHARED) $(LIBSONAME) && ln -sf $(LIBSHARED) $(LIBNAME) 53 | 54 | install_docs: FORCE 55 | $(INSTALL) -d -m 755 $(DESTDIR)$(docdir) 56 | cp -R doc/* $(DESTDIR)$(docdir) 57 | 58 | clean: FORCE 59 | cd libxnd && if [ -f Makefile ]; then $(MAKE) clean; else exit 0; fi 60 | cd libxnd/tests && if [ -f Makefile ]; then $(MAKE) clean; else exit 0; fi 61 | rm -rf build 62 | cd python/xnd && rm -f *.so $(LIBSTATIC) $(LIBSHARED) $(LIBSONAME) $(LIBNAME) 63 | cd python/xnd && rm -rf __pycache__ 64 | cd python/xnd/contrib && rm -rf __pycache__ 65 | 66 | distclean: FORCE 67 | cd libxnd && if [ -f Makefile ]; then $(MAKE) distclean; else exit 0; fi 68 | cd libxnd/tests && if [ -f Makefile ]; then $(MAKE) distclean; else exit 0; fi 69 | rm -f config.h config.log config.status Makefile 70 | rm -rf build dist MANIFEST ndtypes record.txt 71 | cd python && rm -rf ndtypes *.egg-info __pycache__ ndtypes.egg-info 72 | cd python/xnd && rm -f *.so $(LIBSTATIC) $(LIBSHARED) $(LIBSONAME) $(LIBNAME) xnd.h 73 | cd python/xnd && rm -rf __pycache__ 74 | cd python/xnd/contrib && rm -rf __pycache__ 75 | cd .conda/xnd && rm -rf test 76 | 77 | FORCE: 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | 2 | xnd 3 | === 4 | 5 | libxnd 6 | ------ 7 | 8 | libxnd is a lightweight library for initializing and managing typed memory 9 | blocks. It relies on the libndtypes library for typing and traversing 10 | memory. 11 | 12 | http://xnd.readthedocs.io/en/latest/ 13 | 14 | 15 | xnd (Python module) 16 | ------------------- 17 | 18 | The xnd Python module implements a container type that maps most Python 19 | values relevant for scientific computing directly to typed memory. 20 | 21 | Whenever possible, a single, pointer-free memory block is used. 22 | 23 | xnd supports ragged arrays, categorical types, indexing, slicing, aligned 24 | memory blocks and type inference. 25 | 26 | 27 | Operations like indexing and slicing return zero-copy typed views on the 28 | data. 29 | 30 | 31 | http://xnd.readthedocs.io/en/latest/ 32 | 33 | 34 | Authors 35 | ------- 36 | 37 | libxnd/xnd was created by `Stefan Krah `_. 38 | The funding that made this project possible came from `Anaconda Inc. `_ 39 | and currently from `Quansight LLC `_. 40 | 41 | 42 | .. image:: https://badges.gitter.im/Plures/xnd.svg 43 | :alt: Join the chat at https://gitter.im/Plures/xnd 44 | :target: https://gitter.im/Plures/xnd?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge 45 | -------------------------------------------------------------------------------- /config.h.in: -------------------------------------------------------------------------------- 1 | /* config.h.in. Generated from configure.ac by autoheader. */ 2 | 3 | /* Define to 1 if you have the nvcc cuda compiler. */ 4 | #undef HAVE_CUDA 5 | 6 | /* Define to the address where bug reports for this package should be sent. */ 7 | #undef PACKAGE_BUGREPORT 8 | 9 | /* Define to the full name of this package. */ 10 | #undef PACKAGE_NAME 11 | 12 | /* Define to the full name and version of this package. */ 13 | #undef PACKAGE_STRING 14 | 15 | /* Define to the one symbol short name of this package. */ 16 | #undef PACKAGE_TARNAME 17 | 18 | /* Define to the home page for this package. */ 19 | #undef PACKAGE_URL 20 | 21 | /* Define to the version of this package. */ 22 | #undef PACKAGE_VERSION 23 | 24 | /* Define to 1 if you have the ANSI C header files. */ 25 | #undef STDC_HEADERS 26 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | 2 | AC_PREREQ([2.67]) 3 | AC_INIT(xnd, 0.2.0dev3, skrah@bytereef.org, xnd, https://github.com/plures/) 4 | AC_CONFIG_HEADER(config.h) 5 | AC_CONFIG_FILES([Makefile libxnd/Makefile libxnd/tests/Makefile]) 6 | 7 | # System and machine type (only used for Darwin): 8 | AC_MSG_CHECKING(system as reported by uname -s) 9 | ac_sys_system=`uname -s` 10 | AC_MSG_RESULT($ac_sys_system) 11 | 12 | LIBSTATIC=libxnd.a 13 | case $ac_sys_system in 14 | darwin*|Darwin*) 15 | LIBRARY_PATH="DYLD_LIBRARY_PATH" 16 | LIBNAME="libxnd.dylib" 17 | LIBSONAME="libxnd.0.dylib" 18 | LIBSHARED="libxnd.0.2.0dev3.dylib" 19 | CONFIGURE_LDFLAGS="-dynamiclib -install_name @rpath/$LIBSONAME -undefined dynamic_lookup -compatibility_version 0.2 -current_version 0.2.0" 20 | ;; 21 | *) 22 | LIBRARY_PATH="LD_LIBRARY_PATH" 23 | LIBNAME="libxnd.so" 24 | LIBSONAME="libxnd.so.0" 25 | LIBSHARED="libxnd.so.0.2.0dev3" 26 | CONFIGURE_LDFLAGS="-shared -Wl,-soname,$LIBSONAME" 27 | ;; 28 | esac 29 | 30 | AC_SUBST(LIBSTATIC) 31 | AC_SUBST(LIBRARY_PATH) 32 | AC_SUBST(LIBNAME) 33 | AC_SUBST(LIBSONAME) 34 | AC_SUBST(LIBSHARED) 35 | 36 | 37 | # Apparently purely informational for this particular build: 38 | AC_CANONICAL_HOST 39 | AC_SUBST(build) 40 | AC_SUBST(host) 41 | 42 | # Language and compiler: 43 | AC_LANG_C 44 | saved_cflags="$CFLAGS" 45 | saved_cxxflags="$CXXFLAGS" 46 | AC_PROG_CC 47 | AC_PROG_CXX 48 | CFLAGS="$saved_cflags" 49 | CXXFLAGS="$saved_cxxflags" 50 | 51 | # ar and ranlib: 52 | AC_CHECK_TOOL(AR, ar, ar) 53 | AC_PROG_RANLIB 54 | AC_SUBST(RANLIB) 55 | 56 | 57 | # Checks for header files: 58 | AC_HEADER_STDC 59 | 60 | # Install program: 61 | AC_PROG_INSTALL 62 | AC_SUBST(INSTALL) 63 | 64 | # Cuda compiler: 65 | AC_MSG_CHECKING(for nvcc) 66 | saved_cc="$CC" 67 | saved_cflags="$CFLAGS" 68 | saved_cxxflags="$CXXFLAGS" 69 | CC=nvcc 70 | AC_COMPILE_IFELSE([AC_LANG_SOURCE([[ 71 | void 72 | f(void) 73 | { 74 | return; 75 | } 76 | ]])], 77 | [have_nvcc=yes], 78 | [have_nvcc=no], 79 | [have_nvcc=undefined]) 80 | CC="$saved_cc" 81 | CFLAGS="$saved_cflags" 82 | CXXFLAGS="$saved_cxxflags" 83 | AC_MSG_RESULT($have_nvcc) 84 | 85 | CUDA_CXX= 86 | CONFIGURE_CUDA_CXXFLAGS= 87 | if test "$have_nvcc" = yes; then 88 | CUDA_CXX="nvcc" 89 | CONFIGURE_CUDA_CXXFLAGS="-std=c++11" 90 | AC_DEFINE(HAVE_CUDA, 1, [Define to 1 if you have the nvcc cuda compiler.]) 91 | fi 92 | AC_SUBST(CUDA_CXX) 93 | AC_SUBST(CONFIGURE_CUDA_CXXFLAGS) 94 | 95 | # Add an explicit include directory. 96 | AC_MSG_CHECKING(for --with-includes) 97 | AC_ARG_WITH(includes, 98 | AS_HELP_STRING([--with-includes="/usr/local/include"], 99 | [absolute path to an include directory]), 100 | [], 101 | [with_includes="none"]) 102 | 103 | AC_MSG_RESULT($with_includes) 104 | 105 | if test "$with_includes" = "none"; then 106 | CONFIGURE_INCLUDES=../ndtypes/libndtypes 107 | CONFIGURE_INCLUDES_TEST=../../ndtypes/libndtypes 108 | else 109 | CONFIGURE_INCLUDES=$with_includes 110 | CONFIGURE_INCLUDES_TEST=$with_includes 111 | fi 112 | 113 | AC_SUBST(CONFIGURE_INCLUDES) 114 | AC_SUBST(CONFIGURE_INCLUDES_TEST) 115 | 116 | # Add an explicit library path. 117 | AC_MSG_CHECKING(for --with-libs) 118 | AC_ARG_WITH(libs, 119 | AS_HELP_STRING([--with-libs="/usr/local/lib"], 120 | [absolute path to a library directory]), 121 | [], 122 | [with_libs="none"]) 123 | 124 | AC_MSG_RESULT($with_libs) 125 | 126 | if test "$with_libs" = "none"; then 127 | CONFIGURE_LIBS=../ndtypes/libndtypes 128 | CONFIGURE_LIBS_TEST=../../ndtypes/libndtypes 129 | else 130 | CONFIGURE_LIBS=$with_libs 131 | CONFIGURE_LIBS_TEST=$with_libs 132 | fi 133 | 134 | AC_SUBST(CONFIGURE_LIBS) 135 | AC_SUBST(CONFIGURE_LIBS_TEST) 136 | 137 | # Without documentation: 138 | AC_MSG_CHECKING([for --with-docs]) 139 | AC_ARG_WITH([docs], 140 | AS_HELP_STRING([--with-docs], [install documentation - enabled by default]),, 141 | with_docs=yes) 142 | AC_MSG_RESULT([$with_docs]) 143 | if test "$with_docs" = yes; then 144 | NDT_INSTALL_DOCS="install_docs" 145 | else 146 | NDT_INSTALL_DOCS="" 147 | fi 148 | 149 | AC_SUBST(NDT_INSTALL_DOCS) 150 | 151 | # Compiler dependent settings: 152 | XND_WARN= 153 | XND_OPT="-O2 -g" 154 | case $CC in 155 | *gcc*) 156 | XND_WARN="-Wall -Wextra -std=c11 -pedantic" 157 | XND_OPT="-O2 -g" 158 | ;; 159 | *icc*) 160 | AR=xiar 161 | XND_WARN="-Wall" 162 | XND_OPT="-O2 -g" 163 | ;; 164 | *clang*) 165 | XND_WARN="-Wall -Wextra -std=c11 -pedantic" 166 | XND_OPT="-O2 -g" 167 | ;; 168 | esac 169 | 170 | 171 | 172 | # Substitute variables and generate output: 173 | if test -z "$LD"; then 174 | LD="$CXX" 175 | fi 176 | AC_SUBST(LD) 177 | AC_SUBST(AR) 178 | AC_SUBST(XND_WARN) 179 | AC_SUBST(XND_OPT) 180 | 181 | 182 | if test -z "$CFLAGS"; then 183 | CONFIGURE_CFLAGS="$XND_INCLUDE $XND_WARN $XND_CONFIG $XND_OPT" 184 | else 185 | CONFIGURE_CFLAGS="$XND_INCLUDE $XND_WARN $XND_CONFIG $XND_OPT $CFLAGS" 186 | fi 187 | 188 | if test -z "$CXXFLAGS"; then 189 | CONFIGURE_CXXFLAGS="$XND_INCLUDE -Wall -Wextra -O2 -g" 190 | else 191 | CONFIGURE_CXXFLAGS="$XND_INCLUDE -Wall -Wextra -O2 -g $CXXFLAGS" 192 | fi 193 | 194 | if test -z "$LDFLAGS"; then 195 | CONFIGURE_LDFLAGS="$XND_LINK $CONFIGURE_LDFLAGS" 196 | else 197 | CONFIGURE_LDFLAGS="$XND_LINK $CONFIGURE_LDFLAGS $LDFLAGS" 198 | fi 199 | 200 | AC_SUBST(CONFIGURE_CFLAGS) 201 | AC_SUBST(CONFIGURE_CXXFLAGS) 202 | AC_SUBST(CONFIGURE_LDFLAGS) 203 | 204 | AC_OUTPUT 205 | 206 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation. 2 | SPHINXBUILD = sphinx-build 3 | BUILDDIR = build 4 | 5 | default: html 6 | 7 | html: 8 | sphinx-build -b html -d build/doctrees . build/html 9 | 10 | doctest: 11 | sphinx-build -b doctest -d build/doctrees . build/html 12 | 13 | clean: 14 | rm -rf $(BUILDDIR)/* 15 | -------------------------------------------------------------------------------- /doc/_static/copybutton.js: -------------------------------------------------------------------------------- 1 | // Copyright 2014 PSF. Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 2 | // File originates from the cpython source found in Doc/tools/sphinxext/static/copybutton.js 3 | 4 | $(document).ready(function() { 5 | /* Add a [>>>] button on the top-right corner of code samples to hide 6 | * the >>> and ... prompts and the output and thus make the code 7 | * copyable. */ 8 | var div = $('.highlight-py .highlight,' + 9 | '.highlight-default .highlight,' + 10 | '.highlight-py3 .highlight') 11 | var pre = div.find('pre'); 12 | 13 | // get the styles from the current theme 14 | pre.parent().parent().css('position', 'relative'); 15 | var hide_text = 'Hide the prompts and output'; 16 | var show_text = 'Show the prompts and output'; 17 | var border_width = pre.css('border-top-width'); 18 | var border_style = pre.css('border-top-style'); 19 | var border_color = pre.css('border-top-color'); 20 | var button_styles = { 21 | 'cursor':'pointer', 'position': 'absolute', 'top': '0', 'right': '0', 22 | 'border-color': border_color, 'border-style': border_style, 23 | 'border-width': border_width, 'color': border_color, 'text-size': '75%', 24 | 'font-family': 'monospace', 'padding-left': '0.2em', 'padding-right': '0.2em', 25 | 'border-radius': '0 3px 0 0' 26 | } 27 | 28 | // create and add the button to all the code blocks that contain >>> 29 | div.each(function(index) { 30 | var jthis = $(this); 31 | if (jthis.find('.gp').length > 0) { 32 | var button = $('>>>'); 33 | button.css(button_styles) 34 | button.attr('title', hide_text); 35 | button.data('hidden', 'false'); 36 | jthis.prepend(button); 37 | } 38 | // tracebacks (.gt) contain bare text elements that need to be 39 | // wrapped in a span to work with .nextUntil() (see later) 40 | jthis.find('pre:has(.gt)').contents().filter(function() { 41 | return ((this.nodeType == 3) && (this.data.trim().length > 0)); 42 | }).wrap(''); 43 | }); 44 | 45 | // define the behavior of the button when it's clicked 46 | $('.copybutton').click(function(e){ 47 | e.preventDefault(); 48 | var button = $(this); 49 | if (button.data('hidden') === 'false') { 50 | // hide the code output 51 | button.parent().find('.go, .gp, .gt').hide(); 52 | button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden'); 53 | button.css('text-decoration', 'line-through'); 54 | button.attr('title', show_text); 55 | button.data('hidden', 'true'); 56 | } else { 57 | // show the code output 58 | button.parent().find('.go, .gp, .gt').show(); 59 | button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible'); 60 | button.css('text-decoration', 'none'); 61 | button.attr('title', hide_text); 62 | button.data('hidden', 'false'); 63 | } 64 | }); 65 | }); 66 | 67 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | import sys, os, docutils 2 | 3 | 4 | extensions = ['sphinx.ext.intersphinx', 'sphinx.ext.doctest'] 5 | 6 | source_suffix = '.rst' 7 | master_doc = 'index' 8 | project = 'xnd' 9 | copyright = '2017-2018, Plures Project' 10 | version = 'v0.2.0dev3' 11 | release = 'v0.2.0dev3' 12 | exclude_patterns = ['doc', 'build'] 13 | pygments_style = 'sphinx' 14 | html_static_path = ['_static'] 15 | 16 | primary_domain = 'py' 17 | add_function_parentheses = False 18 | 19 | 20 | def setup(app): 21 | app.add_crossref_type('topic', 'topic', 'single: %s', 22 | docutils.nodes.strong) 23 | app.add_javascript("copybutton.js") 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :robots: index, follow 3 | :description: xnd documentation 4 | :keywords: libxnd, xnd, C, Python, array computing 5 | 6 | .. sectionauthor:: Stefan Krah 7 | 8 | 9 | xnd 10 | === 11 | 12 | xnd is a package for typed memory blocks. 13 | 14 | 15 | Libxnd 16 | ------ 17 | 18 | C library. 19 | 20 | .. toctree:: 21 | :maxdepth: 1 22 | 23 | libxnd/index.rst 24 | 25 | 26 | Xnd 27 | --- 28 | 29 | Python module. 30 | 31 | 32 | .. toctree:: 33 | :maxdepth: 1 34 | 35 | xnd/index.rst 36 | 37 | 38 | Releases 39 | -------- 40 | 41 | .. toctree:: 42 | :maxdepth: 1 43 | 44 | releases/index.rst 45 | -------------------------------------------------------------------------------- /doc/libxnd/data-structures.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :robots: index,follow 3 | :description: libxnd documentation 4 | 5 | .. sectionauthor:: Stefan Krah 6 | 7 | 8 | Data structures 9 | =============== 10 | 11 | libxnd is a lightweight container library that leaves most of the work to 12 | libndtypes. The underlying idea is to have a small struct that contains 13 | bitmaps, a linear index, a type and a data pointer. 14 | 15 | The type contains all low level access information. 16 | 17 | Since the struct is small, it can easily be passed around by value when 18 | it serves as a view on the data. 19 | 20 | The owner of the data is a master buffer with some additional bookkeeping 21 | fields. 22 | 23 | 24 | Bitmaps 25 | ------- 26 | 27 | .. topic:: bitmaps 28 | 29 | .. code-block:: c 30 | 31 | typedef struct xnd_bitmap xnd_bitmap_t; 32 | 33 | struct xnd_bitmap { 34 | uint8_t *data; /* bitmap */ 35 | int64_t size; /* number of subtree bitmaps in the "next" array */ 36 | xnd_bitmap_t *next; /* array of bitmaps for subtrees */ 37 | }; 38 | 39 | libxnd supports data with optional values. Any type can have a bitmap for its 40 | data. Bitmaps are addressed using the linear index in the :c:macro:`xnd_t` 41 | struct. 42 | 43 | For a scalar, the bitmap contains just one addressable bit. 44 | 45 | For fixed and variable arrays, the bitmap contains one bit for each item. 46 | 47 | Container types like tuples and records have new bitmaps for each of their 48 | fields if any of the field subtrees contains optional data. 49 | 50 | These field bitmaps are in the *next* array. 51 | 52 | 53 | View 54 | ---- 55 | 56 | .. topic:: xnd_view 57 | 58 | .. code-block:: c 59 | 60 | 61 | /* Typed memory block, usually a view. */ 62 | typedef struct xnd { 63 | xnd_bitmap_t bitmap; /* bitmap tree */ 64 | int64_t index; /* linear index for var dims */ 65 | const ndt_t *type; /* type of the data */ 66 | char *ptr; /* data */ 67 | } xnd_t; 68 | 69 | This is the xnd view, with a typed data pointer, the current linear index 70 | and a bitmap. 71 | 72 | When passing the view around, the linear index needs to be maintained because 73 | it is required to determine if a value is missing (NA). 74 | 75 | The convention is to apply the linear index (adjust the *ptr* and set it 76 | to *0*) only when a non-optional element at *ndim == 0* is actually 77 | accessed. 78 | 79 | This happens for example when getting the value of an element or when 80 | descending into a record. 81 | 82 | 83 | Flags 84 | ----- 85 | 86 | .. topic:: flags 87 | 88 | .. code-block:: c 89 | 90 | #define XND_OWN_TYPE 0x00000001U /* type pointer */ 91 | #define XND_OWN_DATA 0x00000002U /* data pointer */ 92 | #define XND_OWN_STRINGS 0x00000004U /* embedded string pointers */ 93 | #define XND_OWN_BYTES 0x00000008U /* embedded bytes pointers */ 94 | #define XND_OWN_POINTERS 0x00000010U /* embedded pointers */ 95 | 96 | #define XND_OWN_ALL (XND_OWN_TYPE | \ 97 | XND_OWN_DATA | \ 98 | XND_OWN_STRINGS | \ 99 | XND_OWN_BYTES | \ 100 | XND_OWN_POINTERS) 101 | 102 | #define XND_OWN_EMBEDDED (XND_OWN_DATA | \ 103 | XND_OWN_STRINGS | \ 104 | XND_OWN_BYTES | \ 105 | XND_OWN_POINTERS) 106 | 107 | 108 | The ownership flags for the xnd master buffer (see below). Like libndtypes, 109 | libxnd itself has no notion of how many exported views a master buffer has. 110 | 111 | This is deliberately done in order to prevent two different memory management 112 | schemes from getting in each other's way. 113 | 114 | However, for deallocating a master buffer the flags must be set correctly. 115 | 116 | :c:macro:`XND_OWN_TYPE` is set if the master buffer owns the :c:macro:`ndt_t`. 117 | 118 | :c:macro:`XND_OWN_DATA` is set if the master buffer owns the data pointer. 119 | 120 | 121 | The *string*, *bytes* and *ref* types have pointers that are embedded in the 122 | data. Usually, these are owned and deallocated by libxnd. 123 | 124 | For strings, the Python bindings use the convention that :c:macro:`NULL` strings 125 | are interpreted as the empty string. Once a string pointer is initialized it 126 | belongs to the master buffer. 127 | 128 | 129 | Macros 130 | ------ 131 | 132 | .. topic:: macros 133 | 134 | .. code-block:: c 135 | 136 | /* Convenience macros to extract embedded values. */ 137 | #define XND_POINTER_DATA(ptr) (*((char **)ptr)) 138 | #define XND_BYTES_SIZE(ptr) (((ndt_bytes_t *)ptr)->size) 139 | #define XND_BYTES_DATA(ptr) (((ndt_bytes_t *)ptr)->data) 140 | 141 | These macros should be used to extract embedded *ref*, *string* and *bytes* 142 | data. 143 | 144 | 145 | 146 | Master buffer 147 | ------------- 148 | 149 | .. topic:: xnd_master 150 | 151 | .. code-block:: c 152 | 153 | /* Master memory block. */ 154 | typedef struct xnd_master { 155 | uint32_t flags; /* ownership flags */ 156 | xnd_t master; /* typed memory */ 157 | } xnd_master_t; 158 | 159 | This is the master buffer. *flags* are explained above, the *master* buffer 160 | should be considered constant. 161 | 162 | For traversing memory, copy a new view buffer by value. 163 | 164 | 165 | Slice and index keys 166 | -------------------- 167 | 168 | .. topic:: xnd_key 169 | 170 | .. code-block:: c 171 | 172 | enum xnd_key { Index, FieldName, Slice }; 173 | typedef struct { 174 | enum xnd_key tag; 175 | union { 176 | int64_t Index; 177 | const char *FieldName; 178 | ndt_slice_t Slice; 179 | }; 180 | } xnd_index_t; 181 | 182 | Slicing and indexing uses the same model as Python. Indices are usually 183 | integers, but record fields may also be indexed with field names. 184 | 185 | *ndt_slice_t* has *start*, *stop*, *step* fields that must be filled in with 186 | normalized values following the same protocol as :c:func:`PySlice_Unpack`. 187 | -------------------------------------------------------------------------------- /doc/libxnd/functions.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :robots: index,follow 3 | :description: libndtypes documentation 4 | 5 | .. sectionauthor:: Stefan Krah 6 | 7 | 8 | Functions 9 | ========= 10 | 11 | Create typed memory blocks 12 | -------------------------- 13 | 14 | The main use case for libxnd is to create and manage typed memory blocks. 15 | These blocks are fully initialized to *0*. References to additional memory 16 | blocks are allocated and initialized recursively. 17 | 18 | *bytes* and *string* types are initialized to :c:macro:`NULL`, since their 19 | actual length is not known yet. 20 | 21 | 22 | .. topic:: xnd_empty_from_string 23 | 24 | .. code-block:: c 25 | 26 | xnd_master_t *xnd_empty_from_string(const char *s, uint32_t flags, ndt_context_t *ctx); 27 | 28 | Return a new master buffer according to the type string in *s*. *flags* 29 | must include :c:macro:`XND_OWN_TYPE`. 30 | 31 | 32 | .. topic:: xnd_empty_from_type 33 | 34 | .. code-block:: c 35 | 36 | xnd_master_t *xnd_empty_from_type(const ndt_t *t, uint32_t flags, ndt_context_t *ctx); 37 | 38 | 39 | Return a new master buffer according to *type*. *flags* must not include 40 | :c:macro:`XND_OWN_TYPE`, i.e. the type is externally managed. 41 | 42 | This is the case in the Python bindings, where the ndtypes module creates 43 | and manages types. 44 | 45 | 46 | Delete typed memory blocks 47 | -------------------------- 48 | 49 | .. topic:: xnd_del 50 | 51 | .. code-block:: c 52 | 53 | void xnd_del(xnd_master_t *x); 54 | 55 | Delete the master buffer according to its flags. *x* may be :c:macro:`NULL`. 56 | *x->master.ptr* and *x->master.type* may be :c:macro:`NULL`. 57 | 58 | The latter situation should only arise when breaking up reference cycles. 59 | This is used in the Python module. 60 | 61 | 62 | Bitmaps 63 | ------- 64 | 65 | .. topic:: xnd_bitmap_next 66 | 67 | .. code-block:: c 68 | 69 | xnd_bitmap_t xnd_bitmap_next(const xnd_t *x, int64_t i, ndt_context_t *ctx); 70 | 71 | Get the next bitmap for the *Tuple*, *Record*, *Ref* and *Constr* types. 72 | 73 | This is a convenience function that checks if the types have optional 74 | subtrees. 75 | 76 | If yes, return the bitmap at index *i*. If not, it return an empty bitmap 77 | that must not be accessed. 78 | 79 | 80 | .. topic:: xnd_set_valid 81 | 82 | .. code-block:: c 83 | 84 | void xnd_set_valid(xnd_t *x); 85 | 86 | Set the validity bit at *x->index*. *x* must have an optional type. 87 | 88 | 89 | .. topic:: xnd_set_na 90 | 91 | .. code-block:: c 92 | 93 | void xnd_set_na(xnd_t *x); 94 | 95 | Clear the validity bit at *x->index*. *x* must have an optional type. 96 | 97 | 98 | .. topic:: xnd_is_valid 99 | 100 | .. code-block:: c 101 | 102 | int xnd_is_valid(const xnd_t *x); 103 | 104 | Check if the element at *x->index* is valid. If *x* does not have an optional 105 | type, return *1*. Otherwise, return the validity bit (zero or nonzero). 106 | 107 | 108 | .. topic:: xnd_is_na 109 | 110 | .. code-block:: c 111 | 112 | int xnd_is_na(const xnd_t *x); 113 | 114 | Check if the element at *x->index* is valid. If *x* does not have an optional 115 | type, return *0*. Otherwise, return the negation of the validity bit. 116 | 117 | 118 | .. topic:: xnd_subtree 119 | 120 | .. code-block:: c 121 | 122 | xnd_t xnd_subtree(const xnd_t *x, const xnd_index_t indices[], int len, 123 | ndt_context_t *ctx); 124 | 125 | Apply zero or more indices to the input *x* and return a typed view. Valid 126 | indices are integers or strings for record fields. 127 | 128 | This function is more general than pure array indexing, hence the name. For 129 | example, it is possible to index into nested records that in turn contain 130 | arrays. 131 | 132 | 133 | .. topic:: xnd_multikey 134 | 135 | .. code-block:: c 136 | 137 | xnd_t xnd_multikey(const xnd_t *x, const xnd_index_t indices[], int len, 138 | ndt_context_t *ctx); 139 | 140 | Apply zero or more keys to the input *x* and return a typed view. Valid 141 | keys are integers or slices. 142 | 143 | This function differs from :c:func:`xnd_subtree` in that it allows 144 | mixed indexing and slicing for fixed dimensions. Records and tuples 145 | cannot be sliced. 146 | 147 | Variable dimensions can be sliced, but do not support mixed indexing 148 | and slicing. 149 | -------------------------------------------------------------------------------- /doc/libxnd/index.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :robots: index, follow 3 | :description: libxnd documentation 4 | :keywords: libxnd, C, array computing 5 | 6 | .. sectionauthor:: Stefan Krah 7 | 8 | 9 | libxnd 10 | ------ 11 | 12 | libxnd implements support for typed memory blocks using the libndtypes 13 | type library. 14 | 15 | Types include ndarrays, ragged arrays (compatible with the Arrow list type), 16 | optional data (bitmaps are compatible with Arrow), tuples, records (structs), 17 | strings, bytes and categorical values. 18 | 19 | 20 | .. toctree:: 21 | 22 | data-structures.rst 23 | functions.rst 24 | 25 | 26 | -------------------------------------------------------------------------------- /doc/releases/index.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :robots: index, follow 3 | :description: libndtypes documentation 4 | :keywords: libndtypes, C, array computing 5 | 6 | .. sectionauthor:: Stefan Krah 7 | 8 | 9 | ======== 10 | Releases 11 | ======== 12 | 13 | 14 | v0.2.0b2 (February 5th 2018) 15 | ============================ 16 | 17 | Second release (beta2). This release addresses several build and packaging issues: 18 | 19 | - Avoid copying libraries into the Python package if system libraries are used. 20 | 21 | - The build and install partially relied on the dev setup (ndtypes checked out 22 | in the xnd directory). This dependency has been removed. 23 | 24 | - The conda build now supports separate library and Python module installs. 25 | 26 | - Configure now has a **–without-docs** option for skipping the doc install. 27 | 28 | 29 | v0.2.0b1 (January 20th 2018) 30 | ============================ 31 | 32 | First release (beta1). 33 | 34 | 35 | -------------------------------------------------------------------------------- /doc/xnd/align-pack.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :robots: index,follow 3 | :description: xnd container 4 | :keywords: xnd, alignment, packing 5 | 6 | .. sectionauthor:: Stefan Krah 7 | 8 | 9 | Alignment and packing 10 | ===================== 11 | 12 | The xnd memory allocators support explicit alignment. Alignment is specified 13 | in the types. 14 | 15 | 16 | Tuples and records 17 | ------------------ 18 | 19 | Tuples and records have the *align* and *pack* keywords that have the same 20 | purpose as gcc's *aligned* and *packed* struct attributes. 21 | 22 | 23 | Field alignment 24 | ~~~~~~~~~~~~~~~ 25 | 26 | The *align* keyword can be used to specify an alignment that is greater 27 | than the natural alignment of a field: 28 | 29 | .. doctest:: 30 | 31 | >>> from xnd import * 32 | >>> s = "(uint8, uint64 |align=32|, uint64)" 33 | >>> x = xnd.empty(s) 34 | >>> x.align 35 | 32 36 | >>> x.type.datasize 37 | 64 38 | 39 | 40 | 41 | Field packing 42 | ~~~~~~~~~~~~~ 43 | 44 | The *pack* keyword can be used to specify an alignment that is smaller 45 | than the natural alignment of a field: 46 | 47 | .. doctest:: 48 | 49 | >>> s = "(uint8, uint64 |pack=2|, uint64)" 50 | >>> x = xnd.empty(s) 51 | >>> x.align 52 | 8 53 | >>> x.type.datasize 54 | 24 55 | 56 | 57 | 58 | Struct packing 59 | ~~~~~~~~~~~~~~ 60 | 61 | The *pack* and *align* keywords can be applied to the entire struct: 62 | 63 | .. doctest:: 64 | 65 | >>> s = "(uint8, uint64, uint64, pack=1)" 66 | >>> x = xnd.empty(s) 67 | >>> x.align 68 | 1 69 | >>> x.type.datasize 70 | 17 71 | 72 | 73 | Individual field and struct directives are mutually exclusive: 74 | 75 | .. doctest:: 76 | 77 | >>> s = "2 * (uint8 |align=16|, uint64, pack=1)" 78 | >>> x = xnd.empty(s) 79 | Traceback (most recent call last): 80 | File "", line 1, in 81 | ValueError: cannot have 'pack' tuple attribute and field attributes 82 | 83 | 84 | Array alignment 85 | ~~~~~~~~~~~~~~~ 86 | 87 | An array has the same alignment as its elements: 88 | 89 | .. doctest:: 90 | 91 | >>> s = "2 * (uint8, uint64, pack=1)" 92 | >>> x = xnd.empty(s) 93 | >>> x.align 94 | 1 95 | >>> x.type.datasize 96 | 18 97 | -------------------------------------------------------------------------------- /doc/xnd/buffer-protocol.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :robots: index,follow 3 | :description: xnd container 4 | :keywords: xnd, buffer protocol 5 | 6 | .. sectionauthor:: Stefan Krah 7 | 8 | 9 | Buffer protocol 10 | =============== 11 | 12 | xnd supports importing PEP-3118 buffers. 13 | 14 | 15 | From NumPy 16 | ---------- 17 | 18 | Import a simple ndarray: 19 | 20 | .. doctest:: 21 | 22 | >>> import numpy as np 23 | >>> from xnd import * 24 | >>> x = np.array([[[0,1,2], [3,4,5]], [[6,7,8], [9,10,11]]]) 25 | >>> y = xnd.from_buffer(x) 26 | >>> y.type 27 | ndt("2 * 2 * 3 * int64") 28 | >>> y.value 29 | [[[0, 1, 2], [3, 4, 5]], [[6, 7, 8], [9, 10, 11]]] 30 | 31 | 32 | Import an ndarray with a struct dtype: 33 | 34 | .. doctest:: 35 | 36 | >>> x = np.array([(1000, 400.25, 'abc'), (-23, -1e10, 'cba')], 37 | ... dtype=[('x', 'f4'), ('z', 'S3')]) 38 | >>> y = xnd.from_buffer(x) 39 | >>> y.type 40 | ndt("2 * {x : int32, y : >float32, z : fixed_bytes(size=3)}") 41 | >>> y.value 42 | [{'x': 1000, 'y': 400.25, 'z': b'abc'}, {'x': -23, 'y': -10000000000.0, 'z': b'cba'}] 43 | -------------------------------------------------------------------------------- /doc/xnd/index.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :robots: index, follow 3 | :description: xnd documentation 4 | :keywords: memory blocks, unboxed values, array computing, Python 5 | 6 | .. sectionauthor:: Stefan Krah 7 | 8 | 9 | xnd 10 | --- 11 | 12 | The xnd module implements a container type that maps most Python values 13 | relevant for scientific computing directly to typed memory. 14 | 15 | Whenever possible, a single, pointer-free memory block is used. 16 | 17 | xnd supports ragged arrays, categorical types, indexing, slicing, aligned memory blocks and type inference. 18 | 19 | Operations like indexing and slicing return zero-copy typed views on the data. 20 | 21 | Importing PEP-3118 buffers is supported. 22 | 23 | 24 | 25 | .. toctree:: 26 | :maxdepth: 1 27 | 28 | xnd.rst 29 | array.rst 30 | align-pack.rst 31 | buffer-protocol.rst 32 | quickstart.rst 33 | -------------------------------------------------------------------------------- /doc/xnd/quickstart.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :robots: index,follow 3 | :description: xnd quickstart 4 | :keywords: xnd, install 5 | 6 | .. sectionauthor:: Stefan Krah 7 | 8 | 9 | Quick Start 10 | =========== 11 | 12 | Prerequisites 13 | ~~~~~~~~~~~~~ 14 | 15 | Python2 is not supported. If not already present, install the Python3 16 | development packages: 17 | 18 | .. code-block:: sh 19 | 20 | # Debian, Ubuntu: 21 | sudo apt-get install gcc make 22 | sudo apt-get install python3-dev 23 | 24 | # Fedora, RedHat: 25 | sudo yum install gcc make 26 | sudo yum install python3-devel 27 | 28 | # openSUSE: 29 | sudo zypper install gcc make 30 | sudo zypper install python3-devel 31 | 32 | # BSD: 33 | # You know what to do. 34 | 35 | # Mac OS X: 36 | # Install Xcode and Python 3 headers. 37 | 38 | 39 | Install 40 | ~~~~~~~ 41 | 42 | If `pip `_ is present on the system, installation 43 | should be as easy as: 44 | 45 | .. code-block:: sh 46 | 47 | pip install xnd 48 | 49 | 50 | Otherwise: 51 | 52 | .. code-block:: sh 53 | 54 | tar xvzf xnd.2.0b1.tar.gz 55 | cd xnd.2.0b1 56 | python3 setup.py install 57 | 58 | 59 | Windows 60 | ~~~~~~~ 61 | 62 | Refer to the instructions in the *vcbuild* directory in the source distribution. 63 | -------------------------------------------------------------------------------- /doc/xnd/xnd.rst: -------------------------------------------------------------------------------- 1 | .. meta:: 2 | :robots: index,follow 3 | :description: xnd container 4 | :keywords: xnd, types, examples 5 | 6 | .. sectionauthor:: Stefan Krah 7 | 8 | 9 | Types 10 | ===== 11 | 12 | The xnd object is a container that maps a wide range of Python values directly 13 | to memory. xnd unpacks complex types of arbitrary nesting depth to a single 14 | memory block. 15 | 16 | Pointers only occur in explicit pointer types like *Ref* (reference), *Bytes* 17 | and *String*, but not in the general case. 18 | 19 | 20 | Type inference 21 | -------------- 22 | 23 | If no explicit type is given, xnd supports type inference by assuming 24 | types for the most common Python values. 25 | 26 | 27 | Fixed arrays 28 | ~~~~~~~~~~~~ 29 | 30 | .. doctest:: 31 | 32 | >>> from xnd import * 33 | >>> x = xnd([[0, 1, 2], [3, 4, 5]]) 34 | >>> x 35 | xnd([[0, 1, 2], [3, 4, 5]], type='2 * 3 * int64') 36 | 37 | 38 | As expected, lists are mapped to ndarrays and integers to int64. Indexing and 39 | slicing works the usual way. For performance reasons these operations return 40 | zero-copy views whenever possible: 41 | 42 | .. doctest:: 43 | 44 | >>> x[0][1] # Indexing returns views, even for scalars. 45 | xnd(1, type='int64') 46 | >>> 47 | >>> y = x[:, ::-1] # Containers are returned as views. 48 | >>> y 49 | xnd([[2, 1, 0], [5, 4, 3]], type='2 * 3 * int64') 50 | 51 | 52 | Subarrays are views and properly typed: 53 | 54 | .. doctest:: 55 | 56 | >>> x[1] 57 | xnd([3, 4, 5], type='3 * int64') 58 | 59 | 60 | The representation of large values is abbreviated: 61 | 62 | .. doctest:: 63 | 64 | >>> x = xnd(10 * [200 * [1]]) 65 | >>> x 66 | xnd([[1, 1, 1, 1, 1, 1, 1, 1, 1, ...], 67 | [1, 1, 1, 1, 1, 1, 1, 1, 1, ...], 68 | [1, 1, 1, 1, 1, 1, 1, 1, 1, ...], 69 | [1, 1, 1, 1, 1, 1, 1, 1, 1, ...], 70 | [1, 1, 1, 1, 1, 1, 1, 1, 1, ...], 71 | [1, 1, 1, 1, 1, 1, 1, 1, 1, ...], 72 | [1, 1, 1, 1, 1, 1, 1, 1, 1, ...], 73 | [1, 1, 1, 1, 1, 1, 1, 1, 1, ...], 74 | [1, 1, 1, 1, 1, 1, 1, 1, 1, ...], 75 | ...], 76 | type='10 * 200 * int64') 77 | 78 | 79 | Values can be accessed in full using the *value* property: 80 | 81 | .. doctest:: 82 | 83 | >>> x = xnd(11 * [1]) 84 | >>> x 85 | xnd([1, 1, 1, 1, 1, 1, 1, 1, 1, ...], type='11 * int64') 86 | >>> x.value 87 | [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] 88 | 89 | 90 | Types can be accessed using the *type* property: 91 | 92 | .. doctest:: 93 | 94 | >>> x = xnd(11 * [1]) 95 | >>> x.type 96 | ndt("11 * int64") 97 | 98 | 99 | Ragged arrays 100 | ~~~~~~~~~~~~~ 101 | 102 | Ragged arrays are compatible with the Arrow list representation. The data 103 | is pointer-free, addressing the elements works by having one offset array 104 | per dimension. 105 | 106 | .. doctest:: 107 | 108 | >>> xnd([[0.1j], [3+2j, 4+5j, 10j]]) 109 | xnd([[0.1j], [(3+2j), (4+5j), 10j]], type='var * var * complex128') 110 | 111 | 112 | Indexing and slicing works as usual, returning properly typed views or 113 | values in the case of scalars: 114 | 115 | .. doctest:: 116 | 117 | >>> x = xnd([[0.1j], [3+2j, 4+5j, 10j]]) 118 | >>> x[1, 2] 119 | xnd(10j, type='complex128') 120 | 121 | >>> x[1] 122 | xnd([(3+2j), (4+5j), 10j], type='var * complex128') 123 | 124 | 125 | Eliminating dimensions through mixed slicing and indexing is not supported 126 | because it would require copying and adjusting potentially huge offset arrays: 127 | 128 | .. doctest:: 129 | 130 | >>> y = x[:, 1] 131 | Traceback (most recent call last): 132 | File "", line 1, in 133 | IndexError: mixed indexing and slicing is not supported for var dimensions 134 | 135 | 136 | Records (structs) 137 | ~~~~~~~~~~~~~~~~~ 138 | 139 | From Python 3.6 on, dicts retain their order, so they can be used directly 140 | for initializing C structs. 141 | 142 | .. doctest:: 143 | 144 | >>> xnd({'a': 'foo', 'b': 10.2}) 145 | xnd({'a': 'foo', 'b': 10.2}, type='{a : string, b : float64}') 146 | 147 | 148 | Tuples 149 | ~~~~~~ 150 | 151 | Python tuples are directly translated to the libndtypes tuple type: 152 | 153 | .. doctest:: 154 | 155 | >>> xnd(('foo', b'bar', [None, 10.0, 20.0])) 156 | xnd(('foo', b'bar', [None, 10.0, 20.0]), type='(string, bytes, 3 * ?float64)') 157 | 158 | 159 | Nested arrays in structs 160 | ~~~~~~~~~~~~~~~~~~~~~~~~ 161 | 162 | xnd seamlessly supports nested values of arbitrary depth: 163 | 164 | .. doctest:: 165 | 166 | >>> lst = [{'name': 'John', 'internet_points': [1, 2, 3]}, 167 | ... {'name': 'Jane', 'internet_points': [4, 5, 6]}] 168 | >>> xnd(lst) 169 | xnd([{'name': 'John', 'internet_points': [1, 2, 3]}, {'name': 'Jane', 'internet_points': [4, 5, 6]}], 170 | type='2 * {name : string, internet_points : 3 * int64}') 171 | 172 | 173 | Optional data (missing values) 174 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 175 | 176 | Optional data is currently specified using *None*. It is under debate if 177 | a separate *NA* singleton object would be more suitable. 178 | 179 | .. doctest:: 180 | 181 | >>> lst = [0, 1, None, 2, 3, None, 5, 10] 182 | >>> xnd(lst) 183 | xnd([0, 1, None, 2, 3, None, 5, 10], type='8 * ?int64') 184 | 185 | 186 | Categorical data 187 | ~~~~~~~~~~~~~~~~ 188 | 189 | Type inference would be ambiguous, so it cannot work directly. xnd supports 190 | the *levels* argument that is internally translated to the type. 191 | 192 | .. doctest:: 193 | 194 | >>> levels = ['January', 'August', 'December', None] 195 | >>> x = xnd(['January', 'January', None, 'December', 'August', 'December', 'December'], levels=levels) 196 | >>> x.value 197 | ['January', 'January', None, 'December', 'August', 'December', 'December'] 198 | >>> x.type 199 | ndt("7 * categorical('January', 'August', 'December', NA)") 200 | 201 | 202 | The above is equivalent to specifying the type directly: 203 | 204 | .. doctest:: 205 | 206 | >>> from ndtypes import * 207 | >>> t = ndt("7 * categorical('January', 'August', 'December', NA)") 208 | >>> x = xnd(['January', 'January', None, 'December', 'August', 'December', 'December'], type=t) 209 | >>> x.value 210 | ['January', 'January', None, 'December', 'August', 'December', 'December'] 211 | >>> x.type 212 | ndt("7 * categorical('January', 'August', 'December', NA)") 213 | 214 | 215 | Explicit types 216 | -------------- 217 | 218 | While type inference is well-defined, it necessarily makes assumptions about 219 | the programmer's intent. 220 | 221 | There are two cases where types should be given: 222 | 223 | 224 | Different types are intended 225 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 226 | 227 | .. doctest:: 228 | 229 | >>> xnd([[0,1,2], [3,4,5]], type="2 * 3 * uint8") 230 | xnd([[0, 1, 2], [3, 4, 5]], type='2 * 3 * uint8') 231 | 232 | Here, type inference would deduce :c:macro:`int64`, so :c:macro:`uint8` needs 233 | to be passed explicitly. 234 | 235 | 236 | All supported types 237 | ------------------- 238 | 239 | Fixed arrays 240 | ~~~~~~~~~~~~ 241 | 242 | Fixed arrays are similar to NumPy's ndarray. One difference is that internally 243 | xnd uses steps instead of strides. One step is the amount of indices required 244 | to move the linear index from one dimension element to the next. 245 | 246 | This facilitates optional data, whose bitmaps need to be addressed by the 247 | linear index. The equation *stride = step * itemsize* always holds. 248 | 249 | 250 | .. doctest:: 251 | 252 | >>> xnd([[[1,2], [None, 3]], [[4, None], [5, 6]]]) 253 | xnd([[[1, 2], [None, 3]], [[4, None], [5, 6]]], type='2 * 2 * 2 * ?int64') 254 | 255 | This is a fixed array with optional data. 256 | 257 | 258 | .. doctest:: 259 | 260 | >>> xnd([(1,2.0,3j), (4,5.0,6j)]) 261 | xnd([(1, 2.0, 3j), (4, 5.0, 6j)], type='2 * (int64, float64, complex128)') 262 | 263 | An array with tuple elements. 264 | 265 | 266 | Fortran order 267 | ~~~~~~~~~~~~~ 268 | 269 | Fortran order is specified by prefixing the dimensions with an exclamation mark: 270 | 271 | .. doctest:: 272 | 273 | >>> lst = [[1, 2, 3], [4, 5, 6]] 274 | >>> x = xnd(lst, type='!2 * 3 * uint16') 275 | >>> 276 | >>> x.type.shape 277 | (2, 3) 278 | >>> x.type.strides 279 | (2, 4) 280 | 281 | 282 | Alternatively, steps can be passed as arguments to the fixed dimension type: 283 | 284 | .. doctest:: 285 | 286 | >>> from ndtypes import * 287 | >>> lst = [[1, 2, 3], [4, 5, 6]] 288 | >>> t = ndt("fixed(shape=2, step=1) * fixed(shape=3, step=2) * uint16") 289 | >>> x = xnd(lst, type=t) 290 | >>> x.type.shape 291 | (2, 3) 292 | >>> x.type.strides 293 | (2, 4) 294 | 295 | 296 | Ragged arrays 297 | ~~~~~~~~~~~~~ 298 | 299 | Ragged arrays with explicit types are easiest to construct using the *dtype* 300 | argument to the xnd constructor. 301 | 302 | .. doctest:: 303 | 304 | >>> lst = [[0], [1, 2], [3, 4, 5]] 305 | >>> xnd(lst, dtype="int32") 306 | xnd([[0], [1, 2], [3, 4, 5]], type='var * var * int32') 307 | 308 | 309 | Alternatively, offsets can be passed as arguments to the var dimension type: 310 | 311 | .. doctest:: 312 | 313 | >>> from ndtypes import ndt 314 | >>> t = ndt("var(offsets=[0,3]) * var(offsets=[0,1,3,6]) * int32") 315 | >>> xnd(lst, type=t) 316 | xnd([[0], [1, 2], [3, 4, 5]], type='var * var * int32') 317 | 318 | 319 | Tuples 320 | ~~~~~~ 321 | 322 | In memory, tuples are the same as C structs. 323 | 324 | .. doctest:: 325 | 326 | >>> xnd(("foo", 1.0)) 327 | xnd(('foo', 1.0), type='(string, float64)') 328 | 329 | 330 | Indexing works the same as for arrays: 331 | 332 | .. doctest:: 333 | 334 | >>> x = xnd(("foo", 1.0)) 335 | >>> x[0] 336 | xnd('foo', type='string') 337 | 338 | 339 | Nested tuples are more general than ragged arrays. They can a) hold different 340 | data types and b) the trees they represent may be unbalanced. 341 | 342 | They do not allow slicing though and are probably less efficient. 343 | 344 | This is an example of an unbalanced tree that cannot be represented as a 345 | ragged array: 346 | 347 | .. doctest:: 348 | 349 | >>> unbalanced_tree = (((1.0, 2.0), (3.0)), 4.0, ((5.0, 6.0, 7.0), ())) 350 | >>> x = xnd(unbalanced_tree) 351 | >>> x.value 352 | (((1.0, 2.0), 3.0), 4.0, ((5.0, 6.0, 7.0), ())) 353 | >>> x.type 354 | ndt("(((float64, float64), float64), float64, ((float64, float64, float64), ()))") 355 | >>> 356 | >>> x[0] 357 | xnd(((1.0, 2.0), 3.0), type='((float64, float64), float64)') 358 | >>> x[0][0] 359 | xnd((1.0, 2.0), type='(float64, float64)') 360 | 361 | 362 | Note that the data in the above tree example is packed into a single contiguous 363 | memory block. 364 | 365 | 366 | Records 367 | ~~~~~~~ 368 | 369 | In memory, records are C structs. The field names are only stored in the type. 370 | 371 | The following examples use Python-3.6, which keeps the dict initialization 372 | order. 373 | 374 | .. doctest:: 375 | 376 | >>> x = xnd({'a': b'123', 'b': {'x': 1.2, 'y': 100+3j}}) 377 | >>> x.value 378 | {'a': b'123', 'b': {'x': 1.2, 'y': (100+3j)}} 379 | >>> x.type 380 | ndt("{a : bytes, b : {x : float64, y : complex128}}") 381 | 382 | 383 | Indexing works the same as for arrays. Additionally, fields can be indexed 384 | by name: 385 | 386 | .. doctest:: 387 | 388 | >>> x[0] 389 | xnd(b'123', type='bytes') 390 | >>> x['a'] 391 | xnd(b'123', type='bytes') 392 | >>> x['b'] 393 | xnd({'x': 1.2, 'y': (100+3j)}, type='{x : float64, y : complex128}') 394 | 395 | 396 | The nesting depth is arbitrary. In the following example, the data -- except 397 | for strings, which are pointers -- is packed into a single contiguous memory 398 | block: 399 | 400 | .. code-block:: py 401 | 402 | >>> from pprint import pprint 403 | >>> item = { 404 | ... "id": 1001, 405 | ... "name": "cyclotron", 406 | ... "price": 5998321.99, 407 | ... "tags": ["connoisseur", "luxury"], 408 | ... "stock": { 409 | ... "warehouse": 722, 410 | ... "retail": 20 411 | ... } 412 | ... } 413 | >>> x = xnd(item) 414 | >>> 415 | >>> pprint(x.value) 416 | {'id': 1001, 417 | 'name': 'cyclotron', 418 | 'price': 5998321.99, 419 | 'stock': {'retail': 20, 'warehouse': 722}, 420 | 'tags': ['connoisseur', 'luxury']} 421 | >>> 422 | >>> x.type.pprint() 423 | { 424 | id : int64, 425 | name : string, 426 | price : float64, 427 | tags : 2 * string, 428 | stock : { 429 | warehouse : int64, 430 | retail : int64 431 | } 432 | } 433 | 434 | 435 | Strings can be embedded into the array by specifying the fixed string type. 436 | In this case, the memory block is pointer-free. 437 | 438 | .. code-block:: py 439 | 440 | >>> from ndtypes import ndt 441 | >>> 442 | >>> t = """ 443 | ... { id : int64, 444 | ... name : fixed_string(30), 445 | ... price : float64, 446 | ... tags : 2 * fixed_string(30), 447 | ... stock : {warehouse : int64, retail : int64} 448 | ... } 449 | ... """ 450 | >>> 451 | >>> x = xnd(item, type=t) 452 | >>> x.type.pprint() 453 | { 454 | id : int64, 455 | name : fixed_string(30), 456 | price : float64, 457 | tags : 2 * fixed_string(30), 458 | stock : { 459 | warehouse : int64, 460 | retail : int64 461 | } 462 | } 463 | 464 | 465 | Record of arrays 466 | ~~~~~~~~~~~~~~~~ 467 | 468 | Often it is more memory efficient to store an array of records as a record of 469 | arrays. This example with columnar data is from the Arrow homepage: 470 | 471 | .. doctest:: 472 | 473 | >>> data = {'session_id': [1331247700, 1331247702, 1331247709, 1331247799], 474 | ... 'timestamp': [1515529735.4895875, 1515529746.2128427, 1515529756.4485607, 1515529766.2181058], 475 | ... 'source_ip': ['8.8.8.100', '100.2.0.11', '99.101.22.222', '12.100.111.200']} 476 | >>> x = xnd(data) 477 | >>> x.type 478 | ndt("{session_id : 4 * int64, timestamp : 4 * float64, source_ip : 4 * string}") 479 | 480 | 481 | 482 | References 483 | ~~~~~~~~~~ 484 | 485 | References are transparent pointers to new memory blocks (meaning a new 486 | data pointer, not a whole new xnd buffer). 487 | 488 | For example, this is an array of pointer to array: 489 | 490 | .. doctest:: 491 | 492 | >>> t = ndt("3 * ref(4 * uint64)") 493 | >>> lst = [[0,1,2,3], [4,5,6,7], [8,9,10,11]] 494 | >>> xnd(lst, type=t) 495 | xnd([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], type='3 * ref(4 * uint64)') 496 | 497 | The user sees no difference to a regular 3 by 4 array, but internally 498 | the outer dimension consists of three pointers to the inner arrays. 499 | 500 | For memory blocks generated by xnd itself the feature is not so useful -- 501 | after all, it is usually better to have a single memory block than one 502 | with additional pointers. 503 | 504 | 505 | However, suppose that in the above columnar data example another application 506 | represents the arrays inside the record with pointers. Using the *ref* type, 507 | data structures borrowed from such an application can be properly typed: 508 | 509 | .. doctest:: 510 | 511 | >>> t = ndt("{session_id : &4 * int64, timestamp : &4 * float64, source_ip : &4 * string}") 512 | >>> x = xnd(data, type=t) 513 | >>> x.type 514 | ndt("{session_id : ref(4 * int64), timestamp : ref(4 * float64), source_ip : ref(4 * string)}") 515 | 516 | The ampersand is the shorthand for "ref". 517 | 518 | 519 | 520 | Constructors 521 | ~~~~~~~~~~~~ 522 | 523 | Constructors are xnd's way of creating distinct named types. The constructor 524 | argument is a regular type. 525 | 526 | Constructors open up a new dtype, so named arrays can be the dtype of 527 | other arrays. Type inference currently isn't aware of constructors, 528 | so types have to be provided. 529 | 530 | .. doctest:: 531 | 532 | >>> t = ndt("3 * SomeMatrix(2 * 2 * float32)") 533 | >>> lst = [[[1,2], [3,4]], [[5,6], [7,8]], [[9,10], [11,12]]] 534 | >>> x = xnd(lst, type=t) 535 | >>> x 536 | xnd([[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]], [[9.0, 10.0], [11.0, 12.0]]], 537 | type='3 * SomeMatrix(2 * 2 * float32)') 538 | >>> x[0] 539 | xnd([[1.0, 2.0], [3.0, 4.0]], type='SomeMatrix(2 * 2 * float32)') 540 | 541 | 542 | Categorical 543 | ~~~~~~~~~~~ 544 | 545 | Categorical types contain values. The data stored in xnd buffers are indices 546 | (:c:macro:`int64`) into the type's categories. 547 | 548 | .. doctest:: 549 | 550 | >>> t = ndt("categorical('a', 'b', 'c', NA)") 551 | >>> data = ['a', 'a', 'b', 'a', 'a', 'a', 'foo', 'c'] 552 | >>> x = xnd(data, dtype=t) 553 | >>> x.value 554 | ['a', 'a', 'b', 'a', 'a', 'a', None, 'c'] 555 | 556 | 557 | Fixed String 558 | ~~~~~~~~~~~~ 559 | 560 | Fixed strings are embedded into arrays. Supported encodings are 'ascii', 561 | 'utf8', 'utf16' and 'utf32'. The string size argument denotes the number 562 | of code points rather than bytes. 563 | 564 | .. doctest:: 565 | 566 | >>> t = ndt("10 * fixed_string(3, 'utf32')") 567 | >>> x = xnd.empty(t) 568 | >>> x.value 569 | ['', '', '', '', '', '', '', '', '', ''] 570 | >>> x[3] = "\U000003B1\U000003B2\U000003B3" 571 | >>> x.value 572 | ['', '', '', 'αβγ', '', '', '', '', '', ''] 573 | 574 | 575 | Fixed Bytes 576 | ~~~~~~~~~~~ 577 | 578 | Fixed bytes are embedded into arrays. 579 | 580 | .. doctest:: 581 | 582 | >>> t = ndt("3 * fixed_bytes(size=3)") 583 | >>> x = xnd.empty(t) 584 | >>> x[2] = b'123' 585 | >>> x.value 586 | [b'\x00\x00\x00', b'\x00\x00\x00', b'123'] 587 | >>> x.align 588 | 1 589 | 590 | Alignment can be requested with the requirement that size is a multiple of 591 | alignment: 592 | 593 | .. doctest:: 594 | 595 | >>> t = ndt("3 * fixed_bytes(size=32, align=16)") 596 | >>> x = xnd.empty(t) 597 | >>> x.align 598 | 16 599 | 600 | 601 | String 602 | ~~~~~~ 603 | 604 | Strings are pointers to :c:macro:`NUL`-terminated UTF-8 strings. 605 | 606 | .. doctest:: 607 | 608 | >>> x = xnd.empty("10 * string") 609 | >>> x.value 610 | ['', '', '', '', '', '', '', '', '', ''] 611 | >>> x[0] = "abc" 612 | >>> x.value 613 | ['abc', '', '', '', '', '', '', '', '', ''] 614 | 615 | 616 | 617 | Bytes 618 | ~~~~~ 619 | 620 | Internally, bytes are structs with a size field and a pointer to the data. 621 | 622 | .. doctest:: 623 | 624 | >>> xnd([b'123', b'45678']) 625 | xnd([b'123', b'45678'], type='2 * bytes') 626 | 627 | 628 | The bytes constructor takes an optional *align* argument that specifies the 629 | alignment of the allocated data: 630 | 631 | .. doctest:: 632 | 633 | >>> x = xnd([b'abc', b'123'], type="2 * bytes(align=64)") 634 | >>> x.value 635 | [b'abc', b'123'] 636 | >>> x.align 637 | 8 638 | 639 | Note that *x.align* is the alignment of the array. The embedded pointers 640 | to the bytes data are aligned at *64*. 641 | 642 | 643 | Primitive types 644 | ~~~~~~~~~~~~~~~ 645 | 646 | As a short example, here is a tuple that contains all primitive types: 647 | 648 | .. doctest:: 649 | 650 | >>> s = """ 651 | ... (bool, 652 | ... int8, int16, int32, int64, 653 | ... uint8, uint16, uint32, uint64, 654 | ... bfloat16, float16, float32, float64, 655 | ... bcomplex32, complex32, complex64, complex128) 656 | ... """ 657 | >>> x = xnd.empty(s) 658 | >>> x.value 659 | (False, 0, 0, 0, 0, 0, 0, 0, 0, 0.0, 0.0, 0.0, 0.0, 0j, 0j, 0j, 0j) 660 | -------------------------------------------------------------------------------- /install-sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # install - install a program, script, or datafile 3 | 4 | scriptversion=2011-01-19.21; # UTC 5 | 6 | # This originates from X11R5 (mit/util/scripts/install.sh), which was 7 | # later released in X11R6 (xc/config/util/install.sh) with the 8 | # following copyright and license. 9 | # 10 | # Copyright (C) 1994 X Consortium 11 | # 12 | # Permission is hereby granted, free of charge, to any person obtaining a copy 13 | # of this software and associated documentation files (the "Software"), to 14 | # deal in the Software without restriction, including without limitation the 15 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 16 | # sell copies of the Software, and to permit persons to whom the Software is 17 | # furnished to do so, subject to the following conditions: 18 | # 19 | # The above copyright notice and this permission notice shall be included in 20 | # all copies or substantial portions of the Software. 21 | # 22 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 | # X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN 26 | # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC- 27 | # TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 28 | # 29 | # Except as contained in this notice, the name of the X Consortium shall not 30 | # be used in advertising or otherwise to promote the sale, use or other deal- 31 | # ings in this Software without prior written authorization from the X Consor- 32 | # tium. 33 | # 34 | # 35 | # FSF changes to this file are in the public domain. 36 | # 37 | # Calling this script install-sh is preferred over install.sh, to prevent 38 | # `make' implicit rules from creating a file called install from it 39 | # when there is no Makefile. 40 | # 41 | # This script is compatible with the BSD install script, but was written 42 | # from scratch. 43 | 44 | nl=' 45 | ' 46 | IFS=" "" $nl" 47 | 48 | # set DOITPROG to echo to test this script 49 | 50 | # Don't use :- since 4.3BSD and earlier shells don't like it. 51 | doit=${DOITPROG-} 52 | if test -z "$doit"; then 53 | doit_exec=exec 54 | else 55 | doit_exec=$doit 56 | fi 57 | 58 | # Put in absolute file names if you don't have them in your path; 59 | # or use environment vars. 60 | 61 | chgrpprog=${CHGRPPROG-chgrp} 62 | chmodprog=${CHMODPROG-chmod} 63 | chownprog=${CHOWNPROG-chown} 64 | cmpprog=${CMPPROG-cmp} 65 | cpprog=${CPPROG-cp} 66 | mkdirprog=${MKDIRPROG-mkdir} 67 | mvprog=${MVPROG-mv} 68 | rmprog=${RMPROG-rm} 69 | stripprog=${STRIPPROG-strip} 70 | 71 | posix_glob='?' 72 | initialize_posix_glob=' 73 | test "$posix_glob" != "?" || { 74 | if (set -f) 2>/dev/null; then 75 | posix_glob= 76 | else 77 | posix_glob=: 78 | fi 79 | } 80 | ' 81 | 82 | posix_mkdir= 83 | 84 | # Desired mode of installed file. 85 | mode=0755 86 | 87 | chgrpcmd= 88 | chmodcmd=$chmodprog 89 | chowncmd= 90 | mvcmd=$mvprog 91 | rmcmd="$rmprog -f" 92 | stripcmd= 93 | 94 | src= 95 | dst= 96 | dir_arg= 97 | dst_arg= 98 | 99 | copy_on_change=false 100 | no_target_directory= 101 | 102 | usage="\ 103 | Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE 104 | or: $0 [OPTION]... SRCFILES... DIRECTORY 105 | or: $0 [OPTION]... -t DIRECTORY SRCFILES... 106 | or: $0 [OPTION]... -d DIRECTORIES... 107 | 108 | In the 1st form, copy SRCFILE to DSTFILE. 109 | In the 2nd and 3rd, copy all SRCFILES to DIRECTORY. 110 | In the 4th, create DIRECTORIES. 111 | 112 | Options: 113 | --help display this help and exit. 114 | --version display version info and exit. 115 | 116 | -c (ignored) 117 | -C install only if different (preserve the last data modification time) 118 | -d create directories instead of installing files. 119 | -g GROUP $chgrpprog installed files to GROUP. 120 | -m MODE $chmodprog installed files to MODE. 121 | -o USER $chownprog installed files to USER. 122 | -s $stripprog installed files. 123 | -t DIRECTORY install into DIRECTORY. 124 | -T report an error if DSTFILE is a directory. 125 | 126 | Environment variables override the default commands: 127 | CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG 128 | RMPROG STRIPPROG 129 | " 130 | 131 | while test $# -ne 0; do 132 | case $1 in 133 | -c) ;; 134 | 135 | -C) copy_on_change=true;; 136 | 137 | -d) dir_arg=true;; 138 | 139 | -g) chgrpcmd="$chgrpprog $2" 140 | shift;; 141 | 142 | --help) echo "$usage"; exit $?;; 143 | 144 | -m) mode=$2 145 | case $mode in 146 | *' '* | *' '* | *' 147 | '* | *'*'* | *'?'* | *'['*) 148 | echo "$0: invalid mode: $mode" >&2 149 | exit 1;; 150 | esac 151 | shift;; 152 | 153 | -o) chowncmd="$chownprog $2" 154 | shift;; 155 | 156 | -s) stripcmd=$stripprog;; 157 | 158 | -t) dst_arg=$2 159 | # Protect names problematic for `test' and other utilities. 160 | case $dst_arg in 161 | -* | [=\(\)!]) dst_arg=./$dst_arg;; 162 | esac 163 | shift;; 164 | 165 | -T) no_target_directory=true;; 166 | 167 | --version) echo "$0 $scriptversion"; exit $?;; 168 | 169 | --) shift 170 | break;; 171 | 172 | -*) echo "$0: invalid option: $1" >&2 173 | exit 1;; 174 | 175 | *) break;; 176 | esac 177 | shift 178 | done 179 | 180 | if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then 181 | # When -d is used, all remaining arguments are directories to create. 182 | # When -t is used, the destination is already specified. 183 | # Otherwise, the last argument is the destination. Remove it from $@. 184 | for arg 185 | do 186 | if test -n "$dst_arg"; then 187 | # $@ is not empty: it contains at least $arg. 188 | set fnord "$@" "$dst_arg" 189 | shift # fnord 190 | fi 191 | shift # arg 192 | dst_arg=$arg 193 | # Protect names problematic for `test' and other utilities. 194 | case $dst_arg in 195 | -* | [=\(\)!]) dst_arg=./$dst_arg;; 196 | esac 197 | done 198 | fi 199 | 200 | if test $# -eq 0; then 201 | if test -z "$dir_arg"; then 202 | echo "$0: no input file specified." >&2 203 | exit 1 204 | fi 205 | # It's OK to call `install-sh -d' without argument. 206 | # This can happen when creating conditional directories. 207 | exit 0 208 | fi 209 | 210 | if test -z "$dir_arg"; then 211 | do_exit='(exit $ret); exit $ret' 212 | trap "ret=129; $do_exit" 1 213 | trap "ret=130; $do_exit" 2 214 | trap "ret=141; $do_exit" 13 215 | trap "ret=143; $do_exit" 15 216 | 217 | # Set umask so as not to create temps with too-generous modes. 218 | # However, 'strip' requires both read and write access to temps. 219 | case $mode in 220 | # Optimize common cases. 221 | *644) cp_umask=133;; 222 | *755) cp_umask=22;; 223 | 224 | *[0-7]) 225 | if test -z "$stripcmd"; then 226 | u_plus_rw= 227 | else 228 | u_plus_rw='% 200' 229 | fi 230 | cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;; 231 | *) 232 | if test -z "$stripcmd"; then 233 | u_plus_rw= 234 | else 235 | u_plus_rw=,u+rw 236 | fi 237 | cp_umask=$mode$u_plus_rw;; 238 | esac 239 | fi 240 | 241 | for src 242 | do 243 | # Protect names problematic for `test' and other utilities. 244 | case $src in 245 | -* | [=\(\)!]) src=./$src;; 246 | esac 247 | 248 | if test -n "$dir_arg"; then 249 | dst=$src 250 | dstdir=$dst 251 | test -d "$dstdir" 252 | dstdir_status=$? 253 | else 254 | 255 | # Waiting for this to be detected by the "$cpprog $src $dsttmp" command 256 | # might cause directories to be created, which would be especially bad 257 | # if $src (and thus $dsttmp) contains '*'. 258 | if test ! -f "$src" && test ! -d "$src"; then 259 | echo "$0: $src does not exist." >&2 260 | exit 1 261 | fi 262 | 263 | if test -z "$dst_arg"; then 264 | echo "$0: no destination specified." >&2 265 | exit 1 266 | fi 267 | dst=$dst_arg 268 | 269 | # If destination is a directory, append the input filename; won't work 270 | # if double slashes aren't ignored. 271 | if test -d "$dst"; then 272 | if test -n "$no_target_directory"; then 273 | echo "$0: $dst_arg: Is a directory" >&2 274 | exit 1 275 | fi 276 | dstdir=$dst 277 | dst=$dstdir/`basename "$src"` 278 | dstdir_status=0 279 | else 280 | # Prefer dirname, but fall back on a substitute if dirname fails. 281 | dstdir=` 282 | (dirname "$dst") 2>/dev/null || 283 | expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ 284 | X"$dst" : 'X\(//\)[^/]' \| \ 285 | X"$dst" : 'X\(//\)$' \| \ 286 | X"$dst" : 'X\(/\)' \| . 2>/dev/null || 287 | echo X"$dst" | 288 | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ 289 | s//\1/ 290 | q 291 | } 292 | /^X\(\/\/\)[^/].*/{ 293 | s//\1/ 294 | q 295 | } 296 | /^X\(\/\/\)$/{ 297 | s//\1/ 298 | q 299 | } 300 | /^X\(\/\).*/{ 301 | s//\1/ 302 | q 303 | } 304 | s/.*/./; q' 305 | ` 306 | 307 | test -d "$dstdir" 308 | dstdir_status=$? 309 | fi 310 | fi 311 | 312 | obsolete_mkdir_used=false 313 | 314 | if test $dstdir_status != 0; then 315 | case $posix_mkdir in 316 | '') 317 | # Create intermediate dirs using mode 755 as modified by the umask. 318 | # This is like FreeBSD 'install' as of 1997-10-28. 319 | umask=`umask` 320 | case $stripcmd.$umask in 321 | # Optimize common cases. 322 | *[2367][2367]) mkdir_umask=$umask;; 323 | .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;; 324 | 325 | *[0-7]) 326 | mkdir_umask=`expr $umask + 22 \ 327 | - $umask % 100 % 40 + $umask % 20 \ 328 | - $umask % 10 % 4 + $umask % 2 329 | `;; 330 | *) mkdir_umask=$umask,go-w;; 331 | esac 332 | 333 | # With -d, create the new directory with the user-specified mode. 334 | # Otherwise, rely on $mkdir_umask. 335 | if test -n "$dir_arg"; then 336 | mkdir_mode=-m$mode 337 | else 338 | mkdir_mode= 339 | fi 340 | 341 | posix_mkdir=false 342 | case $umask in 343 | *[123567][0-7][0-7]) 344 | # POSIX mkdir -p sets u+wx bits regardless of umask, which 345 | # is incompatible with FreeBSD 'install' when (umask & 300) != 0. 346 | ;; 347 | *) 348 | tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$ 349 | trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0 350 | 351 | if (umask $mkdir_umask && 352 | exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1 353 | then 354 | if test -z "$dir_arg" || { 355 | # Check for POSIX incompatibilities with -m. 356 | # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or 357 | # other-writeable bit of parent directory when it shouldn't. 358 | # FreeBSD 6.1 mkdir -m -p sets mode of existing directory. 359 | ls_ld_tmpdir=`ls -ld "$tmpdir"` 360 | case $ls_ld_tmpdir in 361 | d????-?r-*) different_mode=700;; 362 | d????-?--*) different_mode=755;; 363 | *) false;; 364 | esac && 365 | $mkdirprog -m$different_mode -p -- "$tmpdir" && { 366 | ls_ld_tmpdir_1=`ls -ld "$tmpdir"` 367 | test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1" 368 | } 369 | } 370 | then posix_mkdir=: 371 | fi 372 | rmdir "$tmpdir/d" "$tmpdir" 373 | else 374 | # Remove any dirs left behind by ancient mkdir implementations. 375 | rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null 376 | fi 377 | trap '' 0;; 378 | esac;; 379 | esac 380 | 381 | if 382 | $posix_mkdir && ( 383 | umask $mkdir_umask && 384 | $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir" 385 | ) 386 | then : 387 | else 388 | 389 | # The umask is ridiculous, or mkdir does not conform to POSIX, 390 | # or it failed possibly due to a race condition. Create the 391 | # directory the slow way, step by step, checking for races as we go. 392 | 393 | case $dstdir in 394 | /*) prefix='/';; 395 | [-=\(\)!]*) prefix='./';; 396 | *) prefix='';; 397 | esac 398 | 399 | eval "$initialize_posix_glob" 400 | 401 | oIFS=$IFS 402 | IFS=/ 403 | $posix_glob set -f 404 | set fnord $dstdir 405 | shift 406 | $posix_glob set +f 407 | IFS=$oIFS 408 | 409 | prefixes= 410 | 411 | for d 412 | do 413 | test X"$d" = X && continue 414 | 415 | prefix=$prefix$d 416 | if test -d "$prefix"; then 417 | prefixes= 418 | else 419 | if $posix_mkdir; then 420 | (umask=$mkdir_umask && 421 | $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break 422 | # Don't fail if two instances are running concurrently. 423 | test -d "$prefix" || exit 1 424 | else 425 | case $prefix in 426 | *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;; 427 | *) qprefix=$prefix;; 428 | esac 429 | prefixes="$prefixes '$qprefix'" 430 | fi 431 | fi 432 | prefix=$prefix/ 433 | done 434 | 435 | if test -n "$prefixes"; then 436 | # Don't fail if two instances are running concurrently. 437 | (umask $mkdir_umask && 438 | eval "\$doit_exec \$mkdirprog $prefixes") || 439 | test -d "$dstdir" || exit 1 440 | obsolete_mkdir_used=true 441 | fi 442 | fi 443 | fi 444 | 445 | if test -n "$dir_arg"; then 446 | { test -z "$chowncmd" || $doit $chowncmd "$dst"; } && 447 | { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } && 448 | { test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false || 449 | test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1 450 | else 451 | 452 | # Make a couple of temp file names in the proper directory. 453 | dsttmp=$dstdir/_inst.$$_ 454 | rmtmp=$dstdir/_rm.$$_ 455 | 456 | # Trap to clean up those temp files at exit. 457 | trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0 458 | 459 | # Copy the file name to the temp name. 460 | (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") && 461 | 462 | # and set any options; do chmod last to preserve setuid bits. 463 | # 464 | # If any of these fail, we abort the whole thing. If we want to 465 | # ignore errors from any of these, just make sure not to ignore 466 | # errors from the above "$doit $cpprog $src $dsttmp" command. 467 | # 468 | { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } && 469 | { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } && 470 | { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } && 471 | { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } && 472 | 473 | # If -C, don't bother to copy if it wouldn't change the file. 474 | if $copy_on_change && 475 | old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` && 476 | new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` && 477 | 478 | eval "$initialize_posix_glob" && 479 | $posix_glob set -f && 480 | set X $old && old=:$2:$4:$5:$6 && 481 | set X $new && new=:$2:$4:$5:$6 && 482 | $posix_glob set +f && 483 | 484 | test "$old" = "$new" && 485 | $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1 486 | then 487 | rm -f "$dsttmp" 488 | else 489 | # Rename the file to the real destination. 490 | $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null || 491 | 492 | # The rename failed, perhaps because mv can't rename something else 493 | # to itself, or perhaps because mv is so ancient that it does not 494 | # support -f. 495 | { 496 | # Now remove or move aside any old file at destination location. 497 | # We try this two ways since rm can't unlink itself on some 498 | # systems and the destination file might be busy for other 499 | # reasons. In this case, the final cleanup might fail but the new 500 | # file should still install successfully. 501 | { 502 | test ! -f "$dst" || 503 | $doit $rmcmd -f "$dst" 2>/dev/null || 504 | { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null && 505 | { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; } 506 | } || 507 | { echo "$0: cannot unlink or rename $dst" >&2 508 | (exit 1); exit 1 509 | } 510 | } && 511 | 512 | # Now rename the file to the real destination. 513 | $doit $mvcmd "$dsttmp" "$dst" 514 | } 515 | fi || exit 1 516 | 517 | trap '' 0 518 | fi 519 | done 520 | 521 | # Local variables: 522 | # eval: (add-hook 'write-file-hooks 'time-stamp) 523 | # time-stamp-start: "scriptversion=" 524 | # time-stamp-format: "%:y-%02m-%02d.%02H" 525 | # time-stamp-time-zone: "UTC" 526 | # time-stamp-end: "; # UTC" 527 | # End: 528 | -------------------------------------------------------------------------------- /libxnd/.objs/README.txt: -------------------------------------------------------------------------------- 1 | 2 | Directory for shared object files. 3 | 4 | 5 | -------------------------------------------------------------------------------- /libxnd/Makefile.in: -------------------------------------------------------------------------------- 1 | 2 | # ============================================================================== 3 | # Unix Makefile for libxnd 4 | # ============================================================================== 5 | 6 | 7 | LIBSTATIC = @LIBSTATIC@ 8 | LIBNAME = @LIBNAME@ 9 | LIBSONAME = @LIBSONAME@ 10 | LIBSHARED = @LIBSHARED@ 11 | 12 | CC = @CC@ 13 | LD = @LD@ 14 | AR = @AR@ 15 | RANLIB = @RANLIB@ 16 | CUDA_CXX = @CUDA_CXX@ 17 | 18 | XND_INCLUDES = @CONFIGURE_INCLUDES@ 19 | 20 | CONFIGURE_CFLAGS = @CONFIGURE_CFLAGS@ 21 | XND_CFLAGS = $(strip -I.. -I$(XND_INCLUDES) $(CONFIGURE_CFLAGS) $(CFLAGS)) 22 | XND_CFLAGS_SHARED = $(XND_CFLAGS) -fPIC 23 | 24 | CONFIGURE_CXXFLAGS = @CONFIGURE_CXXFLAGS@ 25 | XND_CXXFLAGS = $(strip -I.. -I$(XND_INCLUDES) $(CONFIGURE_CXXFLAGS) $(CXXFLAGS)) 26 | XND_CXXFLAGS_SHARED = $(XND_CXXFLAGS) -fPIC 27 | 28 | CONFIGURE_LDFLAGS = @CONFIGURE_LDFLAGS@ 29 | XND_LDFLAGS = $(strip $(CONFIGURE_LDFLAGS) $(LDFLAGS)) 30 | 31 | CONFIGURE_CUDA_CXXFLAGS = @CONFIGURE_CUDA_CXXFLAGS@ 32 | GM_CUDA_CXXFLAGS = $(strip $(CONFIGURE_CUDA_CXXFLAGS) $(CUDA_CXXFLAGS)) 33 | 34 | 35 | default: $(LIBSTATIC) $(LIBSHARED) 36 | 37 | 38 | OBJS = bitmaps.o bounds.o copy.o equal.o shape.o split.o xnd.o 39 | 40 | SHARED_OBJS = .objs/bitmaps.o .objs/bounds.o .objs/copy.o .objs/equal.o .objs/shape.o .objs/split.o .objs/xnd.o 41 | 42 | ifdef CUDA_CXX 43 | OBJS += cuda_memory.o 44 | SHARED_OBJS += .objs/cuda_memory.o 45 | endif 46 | 47 | 48 | $(LIBSTATIC): Makefile $(OBJS) 49 | $(AR) rc $(LIBSTATIC) $(OBJS) 50 | $(RANLIB) $(LIBSTATIC) 51 | 52 | $(LIBSHARED): Makefile $(SHARED_OBJS) 53 | $(LD) $(XND_LDFLAGS) -o $(LIBSHARED) $(SHARED_OBJS) 54 | ln -sf $(LIBSHARED) $(LIBNAME) 55 | ln -sf $(LIBSHARED) $(LIBSONAME) 56 | 57 | 58 | bitmaps.o:\ 59 | Makefile bitmaps.c xnd.h 60 | $(CC) $(XND_CFLAGS) -c bitmaps.c 61 | 62 | .objs/bitmaps.o:\ 63 | Makefile bitmaps.c xnd.h 64 | $(CC) $(XND_CFLAGS_SHARED) -c bitmaps.c -o .objs/bitmaps.o 65 | 66 | bounds.o:\ 67 | Makefile bounds.c xnd.h 68 | $(CC) $(XND_CFLAGS) -c bounds.c 69 | 70 | .objs/bounds.o:\ 71 | Makefile bounds.c xnd.h 72 | $(CC) $(XND_CFLAGS_SHARED) -c bounds.c -o .objs/bounds.o 73 | 74 | copy.o:\ 75 | Makefile copy.c xnd.h 76 | $(CC) $(XND_CFLAGS) -c copy.c 77 | 78 | .objs/copy.o:\ 79 | Makefile copy.c xnd.h 80 | $(CC) $(XND_CFLAGS_SHARED) -c copy.c -o .objs/copy.o 81 | 82 | equal.o:\ 83 | Makefile equal.c xnd.h 84 | $(CC) $(XND_CFLAGS) -c equal.c 85 | 86 | .objs/equal.o:\ 87 | Makefile equal.c xnd.h 88 | $(CC) $(XND_CFLAGS_SHARED) -c equal.c -o .objs/equal.o 89 | 90 | shape.o:\ 91 | Makefile shape.c overflow.h xnd.h 92 | $(CC) $(XND_CFLAGS) -c shape.c 93 | 94 | .objs/shape.o:\ 95 | Makefile shape.c overflow.h xnd.h 96 | $(CC) $(XND_CFLAGS_SHARED) -c shape.c -o .objs/shape.o 97 | 98 | split.o:\ 99 | Makefile split.c overflow.h xnd.h 100 | $(CC) $(XND_CFLAGS) -c split.c 101 | 102 | .objs/split.o:\ 103 | Makefile split.c overflow.h xnd.h 104 | $(CC) $(XND_CFLAGS_SHARED) -c split.c -o .objs/split.o 105 | 106 | xnd.o:\ 107 | Makefile xnd.c xnd.h 108 | $(CC) $(XND_CFLAGS) -c xnd.c 109 | 110 | .objs/xnd.o:\ 111 | Makefile xnd.c xnd.h 112 | $(CC) $(XND_CFLAGS_SHARED) -c xnd.c -o .objs/xnd.o 113 | 114 | 115 | # Cuda 116 | cuda_memory.o:\ 117 | Makefile cuda/cuda_memory.cu cuda/cuda_memory.h 118 | $(CUDA_CXX) --compiler-options "$(XND_CXXFLAGS)" $(GM_CUDA_CXXFLAGS) -c cuda/cuda_memory.cu 119 | 120 | .objs/cuda_memory.o:\ 121 | Makefile cuda/cuda_memory.cu cuda/cuda_memory.h 122 | $(CUDA_CXX) --compiler-options "$(XND_CXXFLAGS_SHARED)" $(GM_CUDA_CXXFLAGS) -c cuda/cuda_memory.cu -o .objs/cuda_memory.o 123 | 124 | 125 | # Coverage 126 | coverage:\ 127 | Makefile clean runtest 128 | ./tests/runtest 129 | for file in *.c; do gcov -l "$$file" > /dev/null 2>&1; done 130 | 131 | FORCE: 132 | 133 | clean: FORCE 134 | rm -f *.o *.so *.gch *.gcda *.gcno *.gcov *.dyn *.dpi *.lock 135 | rm -f $(LIBSTATIC) $(LIBSHARED) $(LIBSONAME) $(LIBNAME) 136 | cd .objs && rm -f *.o *.so *.gch *.gcda *.gcno *.gcov *.dyn *.dpi *.lock 137 | 138 | distclean: clean 139 | rm -f Makefile 140 | 141 | 142 | 143 | -------------------------------------------------------------------------------- /libxnd/Makefile.vc: -------------------------------------------------------------------------------- 1 | 2 | # ====================================================================== 3 | # Visual C (nmake) Makefile for libxnd 4 | # ====================================================================== 5 | 6 | LIBSTATIC = libxnd-0.2.0dev3.lib 7 | LIBIMPORT = libxnd-0.2.0dev3.dll.lib 8 | LIBSHARED = libxnd-0.2.0dev3.dll 9 | LIBNDTYPESIMPORT = libndtypes-0.2.0dev3.dll.lib 10 | 11 | !ifndef LIBNDTYPESINCLUDE 12 | LIBNDTYPESINCLUDE = ..\ndtypes\libndtypes 13 | !endif 14 | 15 | !ifndef LIBNDTYPESDIR 16 | LIBNDTYPESDIR = ..\ndtypes\libndtypes 17 | !endif 18 | 19 | OPT = /MT /Ox /GS /EHsc 20 | OPT_SHARED = /DXND_EXPORT /MD /Ox /GS /EHsc /Fo.objs^\ 21 | 22 | COMMON_CFLAGS = -I.. /nologo /W4 /wd4200 /wd4201 /wd4204 23 | CFLAGS = $(COMMON_CFLAGS) $(OPT) 24 | CFLAGS_SHARED = $(COMMON_CFLAGS) $(OPT_SHARED) 25 | 26 | 27 | default: $(LIBSTATIC) $(LIBSHARED) 28 | copy /y xnd.h ..\python\xnd 29 | copy /y $(LIBSTATIC) ..\python\xnd 30 | copy /y $(LIBIMPORT) ..\python\xnd 31 | copy /y $(LIBSHARED) ..\python\xnd 32 | 33 | 34 | OBJS = bitmaps.obj bounds.obj copy.obj equal.obj shape.obj split.obj xnd.obj 35 | 36 | SHARED_OBJS = .objs\bitmaps.obj .objs\bounds.obj .objs\copy.obj .objs\equal.obj .objs\shape.obj .objs\split.obj .objs\xnd.obj 37 | 38 | 39 | $(LIBSTATIC):\ 40 | Makefile $(OBJS) 41 | -@if exist $@ del $(LIBSTATIC) 42 | lib /nologo /out:$(LIBSTATIC) $(OBJS) 43 | 44 | $(LIBSHARED):\ 45 | Makefile $(SHARED_OBJS) 46 | -@if exist $@ del $(LIBSHARED) 47 | link /nologo /DLL /MANIFEST /out:$(LIBSHARED) /implib:$(LIBIMPORT) $(SHARED_OBJS) "/LIBPATH:$(LIBNDTYPESDIR)" $(LIBNDTYPESIMPORT) 48 | mt /nologo -manifest $(LIBSHARED).manifest -outputresource:$(LIBSHARED);2 49 | 50 | bitmaps.obj:\ 51 | Makefile bitmaps.c xnd.h 52 | $(CC) "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS) -c bitmaps.c 53 | 54 | .objs\bitmaps.obj:\ 55 | Makefile bitmaps.c xnd.h 56 | $(CC) "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS_SHARED) -c bitmaps.c 57 | 58 | bounds.obj:\ 59 | Makefile bounds.c xnd.h 60 | $(CC) "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS) -c bounds.c 61 | 62 | .objs\bounds.obj:\ 63 | Makefile bounds.c xnd.h 64 | $(CC) "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS_SHARED) -c bounds.c 65 | 66 | copy.obj:\ 67 | Makefile copy.c xnd.h 68 | $(CC) "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS) -c copy.c 69 | 70 | .objs\copy.obj:\ 71 | Makefile copy.c xnd.h 72 | $(CC) "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS_SHARED) -c copy.c 73 | 74 | equal.obj:\ 75 | Makefile equal.c xnd.h 76 | $(CC) "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS) -c equal.c 77 | 78 | .objs\equal.obj:\ 79 | Makefile equal.c xnd.h 80 | $(CC) "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS_SHARED) -c equal.c 81 | 82 | shape.obj:\ 83 | Makefile shape.c overflow.h xnd.h 84 | $(CC) "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS) -c shape.c 85 | 86 | .objs\shape.obj:\ 87 | Makefile shape.c overflow.h xnd.h 88 | $(CC) "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS_SHARED) -c shape.c 89 | 90 | split.obj:\ 91 | Makefile split.c overflow.h xnd.h 92 | $(CC) "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS) -c split.c 93 | 94 | .objs\split.obj:\ 95 | Makefile split.c overflow.h xnd.h 96 | $(CC) "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS_SHARED) -c split.c 97 | 98 | xnd.obj:\ 99 | Makefile xnd.c xnd.h 100 | $(CC) "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS) -c xnd.c 101 | 102 | .objs\xnd.obj:\ 103 | Makefile xnd.c xnd.h 104 | $(CC) "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS_SHARED) -c xnd.c 105 | 106 | check:\ 107 | Makefile default 108 | cd tests && copy /y Makefile.vc Makefile && nmake /nologo 109 | .\tests\runtest.exe 110 | .\tests\runtest_shared.exe 111 | 112 | 113 | FORCE: 114 | 115 | clean: FORCE 116 | del /q /f *.exe *.obj *.lib *.dll *.exp *.manifest 2>NUL 117 | cd .objs && del /q /f *.obj 2>NUL 118 | if exist "..\build\" rd /q /s "..\build\" 119 | if exist "..\dist\" rd /q /s "..\dist\" 120 | if exist "..\MANIFEST" del "..\MANIFEST" 121 | if exist "..\record.txt" del "..\record.txt" 122 | cd ..\python\xnd && del *.lib *.dll *.pyd xnd.h 2>NUL 123 | 124 | distclean: clean 125 | del Makefile 2>NUL 126 | 127 | 128 | 129 | -------------------------------------------------------------------------------- /libxnd/bitmaps.c: -------------------------------------------------------------------------------- 1 | /* 2 | * BSD 3-Clause License 3 | * 4 | * Copyright (c) 2017-2018, plures 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * 1. Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above copyright notice, 14 | * this list of conditions and the following disclaimer in the documentation 15 | * and/or other materials provided with the distribution. 16 | * 17 | * 3. Neither the name of the copyright holder nor the names of its 18 | * contributors may be used to endorse or promote products derived from 19 | * this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include "ndtypes.h" 39 | #include "xnd.h" 40 | 41 | 42 | const xnd_bitmap_t xnd_bitmap_empty = { .data = NULL, .size = 0, .next = NULL}; 43 | 44 | 45 | static int64_t 46 | bitmap_size(int64_t nelem) 47 | { 48 | return (nelem + 7) / 8; 49 | } 50 | 51 | static uint8_t * 52 | bits_new(int64_t n, ndt_context_t *ctx) 53 | { 54 | uint8_t *bits; 55 | 56 | bits = ndt_calloc(bitmap_size(n), 1); 57 | if (bits == NULL) { 58 | return ndt_memory_error(ctx); 59 | } 60 | 61 | return bits; 62 | } 63 | 64 | static xnd_bitmap_t * 65 | bitmap_array_new(int64_t n, ndt_context_t *ctx) 66 | { 67 | xnd_bitmap_t *b; 68 | 69 | b = ndt_calloc(n, sizeof *b); 70 | if (b == NULL) { 71 | return ndt_memory_error(ctx); 72 | } 73 | 74 | return b; 75 | } 76 | 77 | static int 78 | bitmap_init(xnd_bitmap_t *b, const ndt_t *t, int64_t nitems, ndt_context_t *ctx) 79 | { 80 | xnd_bitmap_t *next; 81 | int64_t shape, i, k; 82 | int64_t n; 83 | 84 | assert(ndt_is_concrete(t)); 85 | assert(b->data == NULL); 86 | assert(b->size == 0); 87 | assert(b->next == NULL); 88 | 89 | if (ndt_is_optional(t)) { 90 | if (t->ndim > 0) { 91 | ndt_err_format(ctx, NDT_NotImplementedError, 92 | "optional dimensions are not implemented"); 93 | return -1; 94 | } 95 | 96 | b->data = bits_new(nitems, ctx); 97 | if (b->data == NULL) { 98 | return -1; 99 | } 100 | } 101 | 102 | if (!ndt_subtree_is_optional(t)) { 103 | return 0; 104 | } 105 | 106 | switch (t->tag) { 107 | case FixedDim: { 108 | shape = t->FixedDim.shape; 109 | return bitmap_init(b, t->FixedDim.type, nitems * shape, ctx); 110 | } 111 | 112 | case VarDim: { 113 | assert(nitems == 1); 114 | n = nitems; 115 | 116 | if (t->ndim == 1) { 117 | int32_t noffsets = t->Concrete.VarDim.offsets->n; 118 | n = t->Concrete.VarDim.offsets->v[noffsets-1]; 119 | } 120 | 121 | return bitmap_init(b, t->VarDim.type, n, ctx); 122 | } 123 | 124 | case Tuple: { 125 | shape = t->Tuple.shape; 126 | 127 | n = nitems * shape; 128 | b->next = bitmap_array_new(n, ctx); 129 | if (b->next == NULL) { 130 | xnd_bitmap_clear(b); 131 | return -1; 132 | } 133 | b->size = n; 134 | 135 | for (i = 0; i < nitems; i++) { 136 | for (k = 0; k < shape; k++) { 137 | next = b->next + i*shape + k; 138 | if (bitmap_init(next, t->Tuple.types[k], 1, ctx) < 0) { 139 | xnd_bitmap_clear(b); 140 | return -1; 141 | } 142 | } 143 | } 144 | 145 | return 0; 146 | } 147 | 148 | case Record: { 149 | shape = t->Record.shape; 150 | 151 | n = nitems * shape; 152 | b->next = bitmap_array_new(n, ctx); 153 | if (b->next == NULL) { 154 | xnd_bitmap_clear(b); 155 | return -1; 156 | } 157 | b->size = n; 158 | 159 | for (i = 0; i < nitems; i++) { 160 | for (k = 0; k < shape; k++) { 161 | next = b->next + i*shape + k; 162 | if (bitmap_init(next, t->Record.types[k], 1, ctx) < 0) { 163 | xnd_bitmap_clear(b); 164 | return -1; 165 | } 166 | } 167 | } 168 | 169 | return 0; 170 | } 171 | 172 | case Union: { 173 | shape = t->Union.ntags; 174 | 175 | n = nitems * shape; 176 | b->next = bitmap_array_new(n, ctx); 177 | if (b->next == NULL) { 178 | xnd_bitmap_clear(b); 179 | return -1; 180 | } 181 | b->size = n; 182 | 183 | for (i = 0; i < nitems; i++) { 184 | for (k = 0; k < shape; k++) { 185 | next = b->next + i*shape + k; 186 | if (bitmap_init(next, t->Union.types[k], 1, ctx) < 0) { 187 | xnd_bitmap_clear(b); 188 | return -1; 189 | } 190 | } 191 | } 192 | 193 | return 0; 194 | } 195 | 196 | case Ref: { 197 | b->next = bitmap_array_new(nitems, ctx); 198 | if (b->next == NULL) { 199 | xnd_bitmap_clear(b); 200 | return -1; 201 | } 202 | b->size = nitems; 203 | 204 | for (i = 0; i < nitems; i++) { 205 | next = b->next + i; 206 | if (bitmap_init(next, t->Ref.type, 1, ctx) < 0) { 207 | xnd_bitmap_clear(b); 208 | return -1; 209 | } 210 | } 211 | 212 | return 0; 213 | } 214 | 215 | case Constr: { 216 | b->next = bitmap_array_new(nitems, ctx); 217 | if (b->next == NULL) { 218 | xnd_bitmap_clear(b); 219 | return -1; 220 | } 221 | b->size = nitems; 222 | 223 | for (i = 0; i < nitems; i++) { 224 | next = b->next + i; 225 | if (bitmap_init(next, t->Constr.type, 1, ctx) < 0) { 226 | xnd_bitmap_clear(b); 227 | return -1; 228 | } 229 | } 230 | 231 | return 0; 232 | } 233 | 234 | case Nominal: { 235 | b->next = bitmap_array_new(nitems, ctx); 236 | if (b->next == NULL) { 237 | xnd_bitmap_clear(b); 238 | return -1; 239 | } 240 | b->size = nitems; 241 | 242 | for (i = 0; i < nitems; i++) { 243 | next = b->next + i; 244 | if (bitmap_init(next, t->Nominal.type, 1, ctx) < 0) { 245 | xnd_bitmap_clear(b); 246 | return -1; 247 | } 248 | } 249 | 250 | return 0; 251 | } 252 | 253 | case Array: { 254 | ndt_err_format(ctx, NDT_NotImplementedError, 255 | "the element type of flexible arrays cannot be optional"); 256 | return -1; 257 | } 258 | 259 | default: 260 | return 0; 261 | } 262 | } 263 | 264 | int 265 | xnd_bitmap_init(xnd_bitmap_t *b, const ndt_t *t, ndt_context_t *ctx) 266 | { 267 | return bitmap_init(b, t, 1, ctx); 268 | } 269 | 270 | void 271 | xnd_bitmap_clear(xnd_bitmap_t *b) 272 | { 273 | int64_t i; 274 | 275 | ndt_free(b->data); 276 | b->data = NULL; 277 | 278 | if (b->next) { 279 | for (i = 0; i < b->size; i++) { 280 | xnd_bitmap_clear(b->next + i); 281 | } 282 | 283 | ndt_free(b->next); 284 | b->next = NULL; 285 | } 286 | } 287 | 288 | xnd_bitmap_t 289 | xnd_bitmap_next(const xnd_t *x, int64_t i, ndt_context_t *ctx) 290 | { 291 | const ndt_t *t = x->type; 292 | xnd_bitmap_t next = {.data=NULL, .size=0, .next=NULL}; 293 | int64_t shape; 294 | 295 | if (!ndt_subtree_is_optional(t)) { 296 | return next; 297 | } 298 | 299 | if (x->bitmap.next == NULL) { 300 | ndt_err_format(ctx, NDT_RuntimeError, "missing bitmap"); 301 | return next; 302 | } 303 | 304 | switch (t->tag) { 305 | case Tuple: 306 | shape = t->Tuple.shape; 307 | break; 308 | case Record: 309 | shape = t->Record.shape; 310 | break; 311 | case Union: 312 | shape = t->Union.ntags; 313 | break; 314 | case Ref: case Constr: case Nominal: 315 | shape = 1; 316 | break; 317 | default: 318 | ndt_err_format(ctx, NDT_RuntimeError, "type has no subtree bitmaps"); 319 | return next; 320 | } 321 | 322 | if (i < 0 || i >= shape) { 323 | ndt_err_format(ctx, NDT_ValueError, "invalid index"); 324 | return next; 325 | } 326 | 327 | return x->bitmap.next[x->index * shape + i]; 328 | } 329 | 330 | void 331 | xnd_set_valid(xnd_t *x) 332 | { 333 | const ndt_t *t = x->type; 334 | int64_t n = x->index; 335 | 336 | assert(ndt_is_optional(t)); 337 | assert(0 <= n); 338 | 339 | x->bitmap.data[n / 8] |= ((uint8_t)1 << (n % 8)); 340 | } 341 | 342 | void 343 | xnd_set_na(xnd_t *x) 344 | { 345 | const ndt_t *t = x->type; 346 | int64_t n = x->index; 347 | 348 | assert(ndt_is_optional(t)); 349 | assert(0 <= n); 350 | 351 | x->bitmap.data[n / 8] &= ~((uint8_t)1 << (n % 8)); 352 | } 353 | 354 | static int 355 | _xnd_is_valid(const xnd_t *x) 356 | { 357 | const ndt_t *t = x->type; 358 | int64_t n = x->index; 359 | 360 | assert(ndt_is_optional(t)); 361 | assert(0 <= n); 362 | 363 | return x->bitmap.data[n / 8] & ((uint8_t)1 << (n % 8)); 364 | } 365 | 366 | int 367 | xnd_is_valid(const xnd_t *x) 368 | { 369 | if (!ndt_is_optional(x->type)) { 370 | return 1; 371 | } 372 | 373 | return _xnd_is_valid(x); 374 | } 375 | 376 | int 377 | xnd_is_na(const xnd_t *x) 378 | { 379 | if (!ndt_is_optional(x->type)) { 380 | return 0; 381 | } 382 | 383 | return !_xnd_is_valid(x); 384 | } 385 | -------------------------------------------------------------------------------- /libxnd/bounds.c: -------------------------------------------------------------------------------- 1 | /* 2 | * BSD 3-Clause License 3 | * 4 | * Copyright (c) 2017-2018, plures 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * 1. Redistributions of source code must retain the above copyright notice, 11 | * tbufsizes list of conditions and the following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above copyright notice, 14 | * tbufsizes list of conditions and the following disclaimer in the documentation 15 | * and/or other materials provided with the distribution. 16 | * 17 | * 3. Neither the name of the copyright holder nor the names of its 18 | * contributors may be used to endorse or promote products derived from 19 | * tbufsizes software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | 34 | #include 35 | #include 36 | #include "ndtypes.h" 37 | #include "xnd.h" 38 | #include "overflow.h" 39 | 40 | #ifndef _MSC_VER 41 | #include "config.h" 42 | #endif 43 | 44 | 45 | typedef struct { 46 | int64_t index; 47 | const ndt_t *type; 48 | int64_t ptr; 49 | } xnd_bounds_t; 50 | 51 | 52 | static inline xnd_bounds_t 53 | _fixed_dim_next(const xnd_bounds_t *x, const int64_t i, bool *overflow) 54 | { 55 | const ndt_t *t = x->type; 56 | const ndt_t *u = t->FixedDim.type; 57 | const int64_t step = MULi64(i, t->Concrete.FixedDim.step, overflow); 58 | xnd_bounds_t next; 59 | 60 | next.index = ADDi64(x->index, step, overflow); 61 | next.type = u; 62 | next.ptr = x->ptr; 63 | 64 | if (u->ndim == 0) { 65 | int64_t tmp = MULi64(next.index, next.type->datasize, overflow); 66 | next.ptr = ADDi64(x->ptr, tmp, overflow); 67 | } 68 | 69 | return next; 70 | } 71 | 72 | static inline xnd_bounds_t 73 | _var_dim_next(const xnd_bounds_t *x, const int64_t start, const int64_t step, 74 | const int64_t i, bool *overflow) 75 | { 76 | const ndt_t *t = x->type; 77 | const ndt_t *u = t->VarDim.type; 78 | xnd_bounds_t next; 79 | int64_t tmp; 80 | 81 | tmp = MULi64(i, step, overflow); 82 | next.index = ADDi64(start, tmp, overflow); 83 | next.type = u; 84 | next.ptr = x->ptr; 85 | 86 | if (u->ndim == 0) { 87 | tmp = MULi64(next.index, next.type->datasize, overflow); 88 | next.ptr = ADDi64(x->ptr, tmp, overflow); 89 | } 90 | 91 | return next; 92 | } 93 | 94 | static inline xnd_bounds_t 95 | _tuple_next(const xnd_bounds_t *x, const int64_t i, bool *overflow) 96 | { 97 | const ndt_t *t = x->type; 98 | xnd_bounds_t next; 99 | 100 | next.index = 0; 101 | next.type = t->Tuple.types[i]; 102 | next.ptr = ADDi64(x->ptr, t->Concrete.Tuple.offset[i], overflow); 103 | 104 | return next; 105 | } 106 | 107 | static inline xnd_bounds_t 108 | _record_next(const xnd_bounds_t *x, const int64_t i, bool *overflow) 109 | { 110 | const ndt_t *t = x->type; 111 | xnd_bounds_t next; 112 | 113 | next.index = 0; 114 | next.type = t->Record.types[i]; 115 | next.ptr = ADDi64(x->ptr, t->Concrete.Record.offset[i], overflow); 116 | 117 | return next; 118 | } 119 | 120 | static inline xnd_bounds_t 121 | _constr_next(const xnd_bounds_t *x) 122 | { 123 | const ndt_t *t = x->type; 124 | xnd_bounds_t next; 125 | 126 | next.index = 0; 127 | next.type = t->Constr.type; 128 | next.ptr = x->ptr; 129 | 130 | return next; 131 | } 132 | 133 | static inline xnd_bounds_t 134 | _nominal_next(const xnd_bounds_t *x) 135 | { 136 | const ndt_t *t = x->type; 137 | xnd_bounds_t next; 138 | 139 | next.index = 0; 140 | next.type = t->Nominal.type; 141 | next.ptr = x->ptr; 142 | 143 | return next; 144 | } 145 | 146 | 147 | static int 148 | _xnd_bounds_check(const xnd_bounds_t * const x, const int64_t bufsize, ndt_context_t *ctx) 149 | { 150 | const ndt_t * const t = x->type; 151 | bool overflow = false; 152 | 153 | if (ndt_is_abstract(t)) { 154 | ndt_err_format(ctx, NDT_ValueError, 155 | "bounds checking requires a concrete type"); 156 | return -1; 157 | } 158 | 159 | if (ndt_subtree_is_optional(t)) { 160 | ndt_err_format(ctx, NDT_NotImplementedError, 161 | "bounds checking not implemented for optional types"); 162 | return -1; 163 | } 164 | 165 | switch (t->tag) { 166 | case FixedDim: { 167 | if (t->FixedDim.shape > 0) { 168 | xnd_bounds_t next = _fixed_dim_next(x, 0, &overflow); 169 | if (_xnd_bounds_check(&next, bufsize, ctx) < 0) { 170 | return -1; 171 | } 172 | } 173 | 174 | if (t->FixedDim.shape > 1) { 175 | xnd_bounds_t next = _fixed_dim_next(x, t->FixedDim.shape-1, &overflow); 176 | if (_xnd_bounds_check(&next, bufsize, ctx) < 0) { 177 | return -1; 178 | } 179 | } 180 | 181 | if (overflow) { 182 | goto overflow_error; 183 | } 184 | 185 | return 0; 186 | } 187 | 188 | case VarDim: { 189 | int64_t start, step, shape; 190 | 191 | shape = ndt_var_indices(&start, &step, t, x->index, ctx); 192 | if (shape < 0) { 193 | return -1; 194 | } 195 | 196 | if (shape > 0) { 197 | xnd_bounds_t next = _var_dim_next(x, start, step, 0, &overflow); 198 | if (_xnd_bounds_check(&next, bufsize, ctx) < 0) { 199 | return -1; 200 | } 201 | } 202 | 203 | if (shape > 1) { 204 | xnd_bounds_t next = _var_dim_next(x, start, step, shape-1, &overflow); 205 | if (_xnd_bounds_check(&next, bufsize, ctx) < 0) { 206 | return -1; 207 | } 208 | } 209 | 210 | if (overflow) { 211 | goto overflow_error; 212 | } 213 | 214 | return 0; 215 | } 216 | 217 | case Tuple: { 218 | if (t->Tuple.shape > 0) { 219 | xnd_bounds_t next = _tuple_next(x, 0, &overflow); 220 | if (_xnd_bounds_check(&next, bufsize, ctx) < 0) { 221 | return -1; 222 | } 223 | } 224 | 225 | if (t->Tuple.shape > 1) { 226 | xnd_bounds_t next = _tuple_next(x, t->Tuple.shape-1, &overflow); 227 | if (_xnd_bounds_check(&next, bufsize, ctx) < 0) { 228 | return -1; 229 | } 230 | } 231 | 232 | if (overflow) { 233 | goto overflow_error; 234 | } 235 | 236 | return 0; 237 | } 238 | 239 | case Record: { 240 | if (t->Record.shape > 0) { 241 | xnd_bounds_t next = _record_next(x, 0, &overflow); 242 | if (_xnd_bounds_check(&next, bufsize, ctx) < 0) { 243 | return -1; 244 | } 245 | } 246 | 247 | if (t->Record.shape > 1) { 248 | xnd_bounds_t next = _record_next(x, t->Record.shape-1, &overflow); 249 | if (_xnd_bounds_check(&next, bufsize, ctx) < 0) { 250 | return -1; 251 | } 252 | } 253 | 254 | if (overflow) { 255 | goto overflow_error; 256 | } 257 | 258 | return 0; 259 | } 260 | 261 | case Union: { 262 | ndt_err_format(ctx, NDT_NotImplementedError, 263 | "bounds checking union types is not implemented"); 264 | return -1; 265 | } 266 | 267 | case Ref: { 268 | ndt_err_format(ctx, NDT_NotImplementedError, 269 | "bounds checking ref types is not implemented"); 270 | return -1; 271 | } 272 | 273 | case Constr: { 274 | xnd_bounds_t next = _constr_next(x); 275 | if (_xnd_bounds_check(&next, bufsize, ctx) < 0) { 276 | return -1; 277 | } 278 | 279 | return 0; 280 | } 281 | 282 | case Nominal: { 283 | xnd_bounds_t next = _nominal_next(x); 284 | if (_xnd_bounds_check(&next, bufsize, ctx) < 0) { 285 | return -1; 286 | } 287 | 288 | return 0; 289 | } 290 | 291 | case VarDimElem: { 292 | ndt_err_format(ctx, NDT_NotImplementedError, 293 | "cannot bounds check var elem dimension"); 294 | return -1; 295 | } 296 | 297 | case Char: { 298 | ndt_err_format(ctx, NDT_NotImplementedError, 299 | "char not implemented"); 300 | return -1; 301 | } 302 | 303 | case String: case Bytes: { 304 | ndt_err_format(ctx, NDT_NotImplementedError, 305 | "serialization for string and bytes is not implemented"); 306 | return -1; 307 | } 308 | 309 | case Array: { 310 | ndt_err_format(ctx, NDT_NotImplementedError, 311 | "serialization for flexible arrays is not implemented"); 312 | return -1; 313 | } 314 | 315 | case Categorical: 316 | case Bool: 317 | case Int8: case Int16: case Int32: case Int64: 318 | case Uint8: case Uint16: case Uint32: case Uint64: 319 | case BFloat16: case Float16: case Float32: case Float64: 320 | case BComplex32: case Complex32: case Complex64: case Complex128: 321 | case FixedString: case FixedBytes: { 322 | const int64_t min = x->ptr; 323 | const int64_t max = ADDi64(min, t->datasize, &overflow); 324 | 325 | if (overflow) { 326 | goto overflow_error; 327 | } 328 | 329 | if (min < 0 || max > bufsize) { 330 | ndt_err_format(ctx, NDT_ValueError, "bounds check failed"); 331 | return -1; 332 | } 333 | 334 | return 0; 335 | } 336 | 337 | /* NOT REACHED: intercepted by ndt_is_abstract(). */ 338 | case Module: case Function: 339 | case AnyKind: case SymbolicDim: case EllipsisDim: case Typevar: 340 | case ScalarKind: case SignedKind: case UnsignedKind: case FloatKind: 341 | case ComplexKind: case FixedStringKind: case FixedBytesKind: 342 | ndt_err_format(ctx, NDT_RuntimeError, "unexpected abstract type"); 343 | return -1; 344 | } 345 | 346 | /* NOT REACHED: tags should be exhaustive */ 347 | ndt_err_format(ctx, NDT_RuntimeError, "invalid type tag"); 348 | return -1; 349 | 350 | overflow_error: 351 | ndt_err_format(ctx, NDT_ValueError, "overflow in bounds check"); 352 | return -1; 353 | } 354 | 355 | int 356 | xnd_bounds_check(const ndt_t *t, const int64_t linear_index, const int64_t bufsize, 357 | ndt_context_t *ctx) 358 | { 359 | xnd_bounds_t x; 360 | 361 | x.index = linear_index; 362 | x.type = t; 363 | x.ptr = 0; 364 | 365 | return _xnd_bounds_check(&x, bufsize, ctx); 366 | } 367 | -------------------------------------------------------------------------------- /libxnd/contrib.h: -------------------------------------------------------------------------------- 1 | #ifndef CONTRIB_H 2 | #define CONTRIB_H 3 | 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | 12 | /* PSF copyright: Written by Jim Hugunin and Chris Chase. */ 13 | static inline int64_t 14 | xnd_slice_adjust_indices(int64_t length, 15 | int64_t *start, int64_t *stop, int64_t step) 16 | { 17 | /* this is harder to get right than you might think */ 18 | 19 | assert(step != 0); 20 | assert(step >= -INT64_MAX); 21 | 22 | if (*start < 0) { 23 | *start += length; 24 | if (*start < 0) { 25 | *start = (step < 0) ? -1 : 0; 26 | } 27 | } 28 | else if (*start >= length) { 29 | *start = (step < 0) ? length - 1 : length; 30 | } 31 | 32 | if (*stop < 0) { 33 | *stop += length; 34 | if (*stop < 0) { 35 | *stop = (step < 0) ? -1 : 0; 36 | } 37 | } 38 | else if (*stop >= length) { 39 | *stop = (step < 0) ? length - 1 : length; 40 | } 41 | 42 | if (step < 0) { 43 | if (*stop < *start) { 44 | return (*start - *stop - 1) / (-step) + 1; 45 | } 46 | } 47 | else { 48 | if (*start < *stop) { 49 | return (*stop - *start - 1) / step + 1; 50 | } 51 | } 52 | 53 | return 0; 54 | } 55 | 56 | /* PSF copyright: Original by Eli Stevens and Mark Dickinson. */ 57 | static inline int 58 | xnd_float_pack2(double x, unsigned char *p, int le, ndt_context_t *ctx) 59 | { 60 | unsigned char sign; 61 | int e; 62 | double f; 63 | unsigned short bits; 64 | int incr = 1; 65 | 66 | if (x == 0.0) { 67 | sign = (copysign(1.0, x) == -1.0); 68 | e = 0; 69 | bits = 0; 70 | } 71 | else if (isinf(x)) { 72 | sign = (x < 0.0); 73 | e = 0x1f; 74 | bits = 0; 75 | } 76 | else if (isnan(x)) { 77 | /* There are 2046 distinct half-precision NaNs (1022 signaling and 78 | 1024 quiet), but there are only two quiet NaNs that don't arise by 79 | quieting a signaling NaN; we get those by setting the topmost bit 80 | of the fraction field and clearing all other fraction bits. We 81 | choose the one with the appropriate sign. */ 82 | sign = (copysign(1.0, x) == -1.0); 83 | e = 0x1f; 84 | bits = 512; 85 | } 86 | else { 87 | sign = (x < 0.0); 88 | if (sign) { 89 | x = -x; 90 | } 91 | 92 | f = frexp(x, &e); 93 | if (f < 0.5 || f >= 1.0) { 94 | ndt_err_format(ctx, NDT_RuntimeError, 95 | "frexp() result out of range"); 96 | return -1; 97 | } 98 | 99 | /* Normalize f to be in the range [1.0, 2.0) */ 100 | f *= 2.0; 101 | e--; 102 | 103 | if (e >= 16) { 104 | goto overflow; 105 | } 106 | else if (e < -25) { 107 | /* |x| < 2**-25. Underflow to zero. */ 108 | f = 0.0; 109 | e = 0; 110 | } 111 | else if (e < -14) { 112 | /* |x| < 2**-14. Gradual underflow */ 113 | f = ldexp(f, 14 + e); 114 | e = 0; 115 | } 116 | else /* if (!(e == 0 && f == 0.0)) */ { 117 | e += 15; 118 | f -= 1.0; /* Get rid of leading 1 */ 119 | } 120 | 121 | f *= 1024.0; /* 2**10 */ 122 | /* Round to even */ 123 | bits = (unsigned short)f; /* Note the truncation */ 124 | assert(bits < 1024); 125 | assert(e < 31); 126 | if ((f - bits > 0.5) || ((f - bits == 0.5) && (bits % 2 == 1))) { 127 | ++bits; 128 | if (bits == 1024) { 129 | /* The carry propagated out of a string of 10 1 bits. */ 130 | bits = 0; 131 | ++e; 132 | if (e == 31) 133 | goto overflow; 134 | } 135 | } 136 | } 137 | 138 | bits |= (e << 10) | (sign << 15); 139 | 140 | /* Write out result. */ 141 | if (le) { 142 | p += 1; 143 | incr = -1; 144 | } 145 | 146 | /* First byte */ 147 | *p = (unsigned char)((bits >> 8) & 0xFF); 148 | p += incr; 149 | 150 | /* Second byte */ 151 | *p = (unsigned char)(bits & 0xFF); 152 | 153 | return 0; 154 | 155 | overflow: 156 | ndt_err_format(ctx, NDT_ValueError, 157 | "float too large to pack with float16 type"); 158 | return -1; 159 | } 160 | 161 | /* PSF copyright: Original by Tim Peters. */ 162 | static inline int 163 | xnd_float_pack4(double x, unsigned char *p, int le, ndt_context_t *ctx) 164 | { 165 | float y = (float)x; 166 | int i, incr = 1; 167 | 168 | if (isinf(y) && !isinf(x)) { 169 | goto overflow; 170 | } 171 | 172 | unsigned char s[sizeof(float)]; 173 | memcpy(s, &y, sizeof(float)); 174 | 175 | if ((xnd_float_is_little_endian() && !le) || 176 | (xnd_float_is_big_endian() && le)) { 177 | p += 3; 178 | incr = -1; 179 | } 180 | 181 | for (i = 0; i < 4; i++) { 182 | *p = s[i]; 183 | p += incr; 184 | } 185 | 186 | return 0; 187 | 188 | overflow: 189 | ndt_err_format(ctx, NDT_ValueError, 190 | "float too large to pack with float32 type"); 191 | return -1; 192 | } 193 | 194 | /* PSF copyright: Original by Tim Peters. */ 195 | static inline void 196 | xnd_float_pack8(double x, unsigned char *p, int le) 197 | { 198 | const unsigned char *s = (unsigned char*)&x; 199 | int i, incr = 1; 200 | 201 | if ((xnd_double_is_little_endian() && !le) || 202 | (xnd_double_is_big_endian() && le)) { 203 | p += 7; 204 | incr = -1; 205 | } 206 | 207 | for (i = 0; i < 8; i++) { 208 | *p = *s++; 209 | p += incr; 210 | } 211 | } 212 | 213 | /* PSF copyright: Original by Eli Stevens and Mark Dickinson. */ 214 | static inline double 215 | xnd_float_unpack2(const unsigned char *p, int le) 216 | { 217 | unsigned char sign; 218 | int e; 219 | unsigned int f; 220 | double x; 221 | int incr = 1; 222 | 223 | if (le) { 224 | p += 1; 225 | incr = -1; 226 | } 227 | 228 | /* First byte */ 229 | sign = (*p >> 7) & 1; 230 | e = (*p & 0x7C) >> 2; 231 | f = (*p & 0x03) << 8; 232 | p += incr; 233 | 234 | /* Second byte */ 235 | f |= *p; 236 | 237 | if (e == 0x1f) { 238 | if (f == 0) { 239 | /* Infinity */ 240 | return sign ? -INFINITY : INFINITY; 241 | } 242 | else { 243 | /* NaN */ 244 | return sign ? -NAN : NAN; 245 | } 246 | } 247 | 248 | x = (double)f / 1024.0; 249 | 250 | if (e == 0) { 251 | e = -14; 252 | } 253 | else { 254 | x += 1.0; 255 | e -= 15; 256 | } 257 | x = ldexp(x, e); 258 | 259 | if (sign) 260 | x = -x; 261 | 262 | return x; 263 | } 264 | 265 | /* PSF copyright: Original by Tim Peters. */ 266 | static inline double 267 | xnd_float_unpack4(const unsigned char *p, int le) 268 | { 269 | float x; 270 | 271 | if ((xnd_float_is_little_endian() && !le) || 272 | (xnd_float_is_big_endian() && le)) { 273 | char buf[4]; 274 | char *d = &buf[3]; 275 | int i; 276 | 277 | for (i = 0; i < 4; i++) { 278 | *d-- = *p++; 279 | } 280 | memcpy(&x, buf, 4); 281 | } 282 | else { 283 | memcpy(&x, p, 4); 284 | } 285 | 286 | return x; 287 | } 288 | 289 | /* PSF copyright: Original by Tim Peters. */ 290 | static inline double 291 | xnd_float_unpack8(const unsigned char *p, int le) 292 | { 293 | double x; 294 | 295 | if ((xnd_double_is_little_endian() && !le) || 296 | (xnd_double_is_big_endian() && le)) { 297 | char buf[8]; 298 | char *d = &buf[7]; 299 | int i; 300 | 301 | for (i = 0; i < 8; i++) { 302 | *d-- = *p++; 303 | } 304 | memcpy(&x, buf, 8); 305 | } 306 | else { 307 | memcpy(&x, p, 8); 308 | } 309 | 310 | return x; 311 | } 312 | 313 | /* NumPy copyright: Original by David Cournapeau. */ 314 | static inline int 315 | xnd_nocopy_reshape(int64_t *newdims, int64_t *newstrides, int newnd, 316 | const int64_t *srcdims, const int64_t *srcstrides, const int srcnd, 317 | int is_f_order) 318 | { 319 | int oldnd; 320 | int64_t olddims[NDT_MAX_DIM]; 321 | int64_t oldstrides[NDT_MAX_DIM]; 322 | int64_t last_stride; 323 | int oi, oj, ok, ni, nj, nk; 324 | 325 | oldnd = 0; 326 | /* 327 | * Remove axes with dimension 1 from the old array. They have no effect 328 | * but would need special cases since their strides do not matter. 329 | */ 330 | for (oi = 0; oi < srcnd; oi++) { 331 | if (srcdims[oi] != 1) { 332 | olddims[oldnd] = srcdims[oi]; 333 | oldstrides[oldnd] = srcstrides[oi]; 334 | oldnd++; 335 | } 336 | } 337 | 338 | /* oi to oj and ni to nj give the axis ranges currently worked with */ 339 | oi = 0; 340 | oj = 1; 341 | ni = 0; 342 | nj = 1; 343 | while (ni < newnd && oi < oldnd) { 344 | int64_t np = newdims[ni]; 345 | int64_t op = olddims[oi]; 346 | 347 | while (np != op) { 348 | if (np < op) { 349 | /* Misses trailing 1s, these are handled later */ 350 | np *= newdims[nj++]; 351 | } else { 352 | op *= olddims[oj++]; 353 | } 354 | } 355 | 356 | /* Check whether the original axes can be combined */ 357 | for (ok = oi; ok < oj - 1; ok++) { 358 | if (is_f_order) { 359 | if (oldstrides[ok+1] != olddims[ok]*oldstrides[ok]) { 360 | /* not contiguous enough */ 361 | return 0; 362 | } 363 | } 364 | else { 365 | /* C order */ 366 | if (oldstrides[ok] != olddims[ok+1]*oldstrides[ok+1]) { 367 | /* not contiguous enough */ 368 | return 0; 369 | } 370 | } 371 | } 372 | 373 | /* Calculate new strides for all axes currently worked with */ 374 | if (is_f_order) { 375 | newstrides[ni] = oldstrides[oi]; 376 | for (nk = ni + 1; nk < nj; nk++) { 377 | newstrides[nk] = newstrides[nk - 1]*newdims[nk - 1]; 378 | } 379 | } 380 | else { 381 | /* C order */ 382 | newstrides[nj - 1] = oldstrides[oj - 1]; 383 | for (nk = nj - 1; nk > ni; nk--) { 384 | newstrides[nk - 1] = newstrides[nk]*newdims[nk]; 385 | } 386 | } 387 | ni = nj++; 388 | oi = oj++; 389 | } 390 | 391 | /* 392 | * Set strides corresponding to trailing 1s of the new shape. 393 | */ 394 | if (ni >= 1) { 395 | last_stride = newstrides[ni - 1]; 396 | } 397 | else { 398 | last_stride = 1; 399 | } 400 | if (is_f_order) { 401 | last_stride *= newdims[ni - 1]; 402 | } 403 | for (nk = ni; nk < newnd; nk++) { 404 | newstrides[nk] = last_stride; 405 | } 406 | 407 | return 1; 408 | } 409 | 410 | 411 | #endif /* CONTRIB_H */ 412 | -------------------------------------------------------------------------------- /libxnd/contrib/bfloat16.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | /* Modified and adapted for gumath. */ 17 | 18 | #ifndef BFLOAT16_H 19 | #define BFLOAT16_H 20 | 21 | 22 | #include 23 | #include 24 | 25 | 26 | union FP32 { 27 | unsigned int u; 28 | float f; 29 | }; 30 | 31 | // Converts a float point to bfloat16, with round-nearest-to-even as rounding 32 | // method. 33 | // TODO: There is a slightly faster implementation (8% faster on CPU) 34 | // than this (documented in cl/175987786), that is exponentially harder to 35 | // understand and document. Switch to the faster version when converting to 36 | // BF16 becomes compute-bound. 37 | static inline uint16_t 38 | xnd_round_to_bfloat16(float v) 39 | { 40 | uint32_t input; 41 | union FP32 f; 42 | f.f = v; 43 | input = f.u; 44 | uint16_t output; 45 | 46 | if (isnan(v)) { 47 | // If the value is a NaN, squash it to a qNaN with msb of fraction set, 48 | // this makes sure after truncation we don't end up with an inf. 49 | // 50 | // qNaN magic: All exponent bits set + most significant bit of fraction 51 | // set. 52 | output = 0x7fc0; 53 | } else { 54 | // Fast rounding algorithm that rounds a half value to nearest even. This 55 | // reduces expected error when we convert a large number of floats. Here 56 | // is how it works: 57 | // 58 | // Definitions: 59 | // To convert a float 32 to bfloat16, a float 32 can be viewed as 32 bits 60 | // with the following tags: 61 | // 62 | // Sign | Exp (8 bits) | Frac (23 bits) 63 | // S EEEEEEEE FFFFFFLRTTTTTTTTTTTTTTT 64 | // 65 | // S: Sign bit. 66 | // E: Exponent bits. 67 | // F: First 6 bits of fraction. 68 | // L: Least significant bit of resulting bfloat16 if we truncate away the 69 | // rest of the float32. This is also the 7th bit of fraction 70 | // R: Rounding bit, 8th bit of fraction. 71 | // T: Sticky bits, rest of fraction, 15 bits. 72 | // 73 | // To round half to nearest even, there are 3 cases where we want to round 74 | // down (simply truncate the result of the bits away, which consists of 75 | // rounding bit and sticky bits) and two cases where we want to round up 76 | // (truncate then add one to the result). 77 | // 78 | // The fast converting algorithm simply adds lsb (L) to 0x7fff (15 bits of 79 | // 1s) as the rounding bias, adds the rounding bias to the input, then 80 | // truncates the last 16 bits away. 81 | // 82 | // To understand how it works, we can analyze this algorithm case by case: 83 | // 84 | // 1. L = 0, R = 0: 85 | // Expect: round down, this is less than half value. 86 | // 87 | // Algorithm: 88 | // - Rounding bias: 0x7fff + 0 = 0x7fff 89 | // - Adding rounding bias to input may create any carry, depending on 90 | // whether there is any value set to 1 in T bits. 91 | // - R may be set to 1 if there is a carry. 92 | // - L remains 0. 93 | // - Note that this case also handles Inf and -Inf, where all fraction 94 | // bits, including L, R and Ts are all 0. The output remains Inf after 95 | // this algorithm. 96 | // 97 | // 2. L = 1, R = 0: 98 | // Expect: round down, this is less than half value. 99 | // 100 | // Algorithm: 101 | // - Rounding bias: 0x7fff + 1 = 0x8000 102 | // - Adding rounding bias to input doesn't change sticky bits but 103 | // adds 1 to rounding bit. 104 | // - L remains 1. 105 | // 106 | // 3. L = 0, R = 1, all of T are 0: 107 | // Expect: round down, this is exactly at half, the result is already 108 | // even (L=0). 109 | // 110 | // Algorithm: 111 | // - Rounding bias: 0x7fff + 0 = 0x7fff 112 | // - Adding rounding bias to input sets all sticky bits to 1, but 113 | // doesn't create a carry. 114 | // - R remains 1. 115 | // - L remains 0. 116 | // 117 | // 4. L = 1, R = 1: 118 | // Expect: round up, this is exactly at half, the result needs to be 119 | // round to the next even number. 120 | // 121 | // Algorithm: 122 | // - Rounding bias: 0x7fff + 1 = 0x8000 123 | // - Adding rounding bias to input doesn't change sticky bits, but 124 | // creates a carry from rounding bit. 125 | // - The carry sets L to 0, creates another carry bit and propagate 126 | // forward to F bits. 127 | // - If all the F bits are 1, a carry then propagates to the exponent 128 | // bits, which then creates the minimum value with the next exponent 129 | // value. Note that we won't have the case where exponents are all 1, 130 | // since that's either a NaN (handled in the other if condition) or inf 131 | // (handled in case 1). 132 | // 133 | // 5. L = 0, R = 1, any of T is 1: 134 | // Expect: round up, this is greater than half. 135 | // 136 | // Algorithm: 137 | // - Rounding bias: 0x7fff + 0 = 0x7fff 138 | // - Adding rounding bias to input creates a carry from sticky bits, 139 | // sets rounding bit to 0, then create another carry. 140 | // - The second carry sets L to 1. 141 | // 142 | // Examples: 143 | // 144 | // Exact half value that is already even: 145 | // Input: 146 | // Sign | Exp (8 bit) | Frac (first 7 bit) | Frac (last 16 bit) 147 | // S E E E E E E E E F F F F F F L RTTTTTTTTTTTTTTT 148 | // 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1000000000000000 149 | // 150 | // This falls into case 3. We truncate the rest of 16 bits and no 151 | // carry is created into F and L: 152 | // 153 | // Output: 154 | // Sign | Exp (8 bit) | Frac (first 7 bit) 155 | // S E E E E E E E E F F F F F F L 156 | // 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 157 | // 158 | // Exact half value, round to next even number: 159 | // Input: 160 | // Sign | Exp (8 bit) | Frac (first 7 bit) | Frac (last 16 bit) 161 | // S E E E E E E E E F F F F F F L RTTTTTTTTTTTTTTT 162 | // 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1000000000000000 163 | // 164 | // This falls into case 4. We create a carry from R and T, 165 | // which then propagates into L and F: 166 | // 167 | // Output: 168 | // Sign | Exp (8 bit) | Frac (first 7 bit) 169 | // S E E E E E E E E F F F F F F L 170 | // 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 171 | // 172 | // 173 | // Max denormal value round to min normal value: 174 | // Input: 175 | // Sign | Exp (8 bit) | Frac (first 7 bit) | Frac (last 16 bit) 176 | // S E E E E E E E E F F F F F F L RTTTTTTTTTTTTTTT 177 | // 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1111111111111111 178 | // 179 | // This falls into case 4. We create a carry from R and T, 180 | // propagate into L and F, which then propagates into exponent 181 | // bits: 182 | // 183 | // Output: 184 | // Sign | Exp (8 bit) | Frac (first 7 bit) 185 | // S E E E E E E E E F F F F F F L 186 | // 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 187 | // 188 | // Max normal value round to Inf: 189 | // Input: 190 | // Sign | Exp (8 bit) | Frac (first 7 bit) | Frac (last 16 bit) 191 | // S E E E E E E E E F F F F F F L RTTTTTTTTTTTTTTT 192 | // 0 1 1 1 1 1 1 1 0 1 1 1 1 1 1 1 1111111111111111 193 | // 194 | // This falls into case 4. We create a carry from R and T, 195 | // propagate into L and F, which then propagates into exponent 196 | // bits: 197 | // 198 | // Sign | Exp (8 bit) | Frac (first 7 bit) 199 | // S E E E E E E E E F F F F F F L 200 | // 0 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 201 | // 202 | // 203 | // Least significant bit of resulting bfloat. 204 | uint32_t lsb = (input >> 16) & 1; 205 | uint32_t rounding_bias = 0x7fff + lsb; 206 | input += rounding_bias; 207 | output = (uint16_t)(input >> 16); 208 | } 209 | return output; 210 | } 211 | 212 | 213 | #endif // BFLOAT16_H 214 | -------------------------------------------------------------------------------- /libxnd/cuda/cuda_memory.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * BSD 3-Clause License 3 | * 4 | * Copyright (c) 2017-2018, plures 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * 1. Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above copyright notice, 14 | * this list of conditions and the following disclaimer in the documentation 15 | * and/or other materials provided with the distribution. 16 | * 17 | * 3. Neither the name of the copyright holder nor the names of its 18 | * contributors may be used to endorse or promote products derived from 19 | * this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | 34 | #include 35 | #include "ndtypes.h" 36 | 37 | 38 | /*****************************************************************************/ 39 | /* Expose some cuda memory functions */ 40 | /*****************************************************************************/ 41 | 42 | extern "C" void * 43 | xnd_cuda_calloc_managed(uint16_t align, int64_t size, ndt_context_t *ctx) 44 | { 45 | char *ptr; 46 | cudaError_t err; 47 | 48 | size = size == 0 ? 1 : size; 49 | 50 | if (size < 0 || (uint64_t)size > SIZE_MAX) { 51 | ndt_err_format(ctx, NDT_ValueError, 52 | "cudaMallocManaged: invalid size"); 53 | return NULL; 54 | } 55 | 56 | err = cudaMallocManaged(&ptr, (size_t)size); 57 | if (err != cudaSuccess) { 58 | ndt_err_format(ctx, NDT_MemoryError, 59 | "cudaMallocManaged: allocation failed"); 60 | return NULL; 61 | } 62 | 63 | if (((uintptr_t)ptr) % align != 0) { 64 | ndt_err_format(ctx, NDT_ValueError, 65 | "cudaMallocManaged: alignment requirement too large"); 66 | cudaFree(ptr); 67 | return NULL; 68 | } 69 | 70 | memset(ptr, '\0', (size_t)size); 71 | 72 | return ptr; 73 | } 74 | 75 | extern "C" void 76 | xnd_cuda_free(void *ptr) 77 | { 78 | cudaError_t err; 79 | 80 | err = cudaFree(ptr); 81 | if (err != cudaSuccess) { 82 | fprintf(stderr, "cudaFree: unexpected failure\n"); 83 | } 84 | } 85 | 86 | extern "C" int 87 | xnd_cuda_mem_prefetch_async(const void *ptr, int64_t count, int dev, 88 | ndt_context_t *ctx) 89 | { 90 | cudaError_t err; 91 | 92 | if (count < 0 || (uint64_t)count > SIZE_MAX) { 93 | ndt_err_format(ctx, NDT_ValueError, 94 | "cudaMemPrefetchAsync: invalid count"); 95 | return -1; 96 | } 97 | 98 | err = cudaMemPrefetchAsync(ptr, (size_t)count, dev); 99 | if (err != cudaSuccess) { 100 | ndt_err_format(ctx, NDT_MemoryError, 101 | "cudaMemPrefetchAsync: prefetching failed"); 102 | return -1; 103 | } 104 | 105 | return 0; 106 | } 107 | 108 | extern "C" int 109 | xnd_cuda_device_synchronize(ndt_context_t *ctx) 110 | { 111 | cudaError_t err; 112 | 113 | err = cudaDeviceSynchronize(); 114 | if (err != cudaSuccess) { 115 | ndt_err_format(ctx, NDT_RuntimeError, 116 | "cuda device synchronization failed"); 117 | return -1; 118 | } 119 | 120 | return 0; 121 | } 122 | -------------------------------------------------------------------------------- /libxnd/cuda/cuda_memory.h: -------------------------------------------------------------------------------- 1 | /* 2 | * BSD 3-Clause License 3 | * 4 | * Copyright (c) 2017-2018, plures 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * 1. Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above copyright notice, 14 | * this list of conditions and the following disclaimer in the documentation 15 | * and/or other materials provided with the distribution. 16 | * 17 | * 3. Neither the name of the copyright holder nor the names of its 18 | * contributors may be used to endorse or promote products derived from 19 | * this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | 34 | #ifndef CUDA_MEMORY_H 35 | #define CUDA_MEMORY_H 36 | 37 | #include 38 | #include "ndtypes.h" 39 | 40 | 41 | /*****************************************************************************/ 42 | /* Expose some cuda memory functions */ 43 | /*****************************************************************************/ 44 | 45 | #ifdef __cpplusplus 46 | extern "C" void *xnd_cuda_calloc_managed(uint16_t align, int64_t size, ndt_context_t *ctx); 47 | extern "C" void xnd_cuda_free(void *ptr); 48 | extern "C" int xnd_cuda_mem_prefetch_async(const void *ptr, int64_t count, int dev, ndt_context_t *ctx); 49 | extern "C" int xnd_cuda_device_synchronize(ndt_context_t *ctx); 50 | #else 51 | void *xnd_cuda_calloc_managed(uint16_t align, int64_t size, ndt_context_t *ctx); 52 | void xnd_cuda_free(void *ptr); 53 | int xnd_cuda_mem_prefetch_async(const void *ptr, int64_t count, int dev, ndt_context_t *ctx); 54 | int xnd_cuda_device_synchronize(ndt_context_t *ctx); 55 | #endif 56 | 57 | 58 | #endif /* CUDA_MEMORY_H */ 59 | -------------------------------------------------------------------------------- /libxnd/inline.h: -------------------------------------------------------------------------------- 1 | /* 2 | * BSD 3-Clause License 3 | * 4 | * Copyright (c) 2017-2018, plures 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * 1. Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above copyright notice, 14 | * this list of conditions and the following disclaimer in the documentation 15 | * and/or other materials provided with the distribution. 16 | * 17 | * 3. Neither the name of the copyright holder nor the names of its 18 | * contributors may be used to endorse or promote products derived from 19 | * this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | 34 | #ifndef INLINE_H 35 | #define INLINE_H 36 | 37 | 38 | #include 39 | #include 40 | #include 41 | #include "ndtypes.h" 42 | #include "xnd.h" 43 | 44 | 45 | /*****************************************************************************/ 46 | /* Internal inline functions */ 47 | /*****************************************************************************/ 48 | 49 | /* Counterparts of the functions in xnd.h. These versions ignore the bitmaps. */ 50 | 51 | static inline xnd_t 52 | _fixed_dim_next(const xnd_t *x, const int64_t i) 53 | { 54 | const ndt_t *t = x->type; 55 | const ndt_t *u = t->FixedDim.type; 56 | const int64_t step = i * t->Concrete.FixedDim.step; 57 | xnd_t next; 58 | 59 | next.bitmap = xnd_bitmap_empty; 60 | next.index = x->index + step; 61 | next.type = u; 62 | next.ptr = u->ndim==0 ? x->ptr + next.index * next.type->datasize : x->ptr; 63 | 64 | return next; 65 | } 66 | 67 | static inline xnd_t 68 | _var_dim_next(const xnd_t *x, const int64_t start, const int64_t step, 69 | const int64_t i) 70 | { 71 | const ndt_t *t = x->type; 72 | const ndt_t *u = t->VarDim.type; 73 | xnd_t next; 74 | 75 | next.bitmap = xnd_bitmap_empty; 76 | next.index = start + i * step; 77 | next.type = u; 78 | next.ptr = u->ndim==0 ? x->ptr + next.index * next.type->datasize : x->ptr; 79 | 80 | return next; 81 | } 82 | 83 | static inline xnd_t 84 | _array_next(const xnd_t *x, const int64_t i) 85 | { 86 | const ndt_t *t = x->type; 87 | const ndt_t *u = t->Array.type; 88 | xnd_t next; 89 | 90 | assert(t->tag == Array); 91 | 92 | next.bitmap = xnd_bitmap_empty; 93 | next.index = 0; 94 | next.type = u; 95 | next.ptr = XND_ARRAY_DATA(x->ptr) + i * next.type->datasize; 96 | 97 | return next; 98 | } 99 | 100 | static inline xnd_t 101 | _tuple_next(const xnd_t *x, const int64_t i) 102 | { 103 | const ndt_t *t = x->type; 104 | xnd_t next; 105 | 106 | next.bitmap = xnd_bitmap_empty; 107 | next.index = 0; 108 | next.type = t->Tuple.types[i]; 109 | next.ptr = x->ptr + t->Concrete.Tuple.offset[i]; 110 | 111 | return next; 112 | } 113 | 114 | static inline xnd_t 115 | _record_next(const xnd_t *x, const int64_t i) 116 | { 117 | const ndt_t *t = x->type; 118 | xnd_t next; 119 | 120 | next.bitmap = xnd_bitmap_empty; 121 | next.index = 0; 122 | next.type = t->Record.types[i]; 123 | next.ptr = x->ptr + t->Concrete.Record.offset[i]; 124 | 125 | return next; 126 | } 127 | 128 | static inline xnd_t 129 | _union_next(const xnd_t *x) 130 | { 131 | uint8_t i = XND_UNION_TAG(x->ptr); 132 | const ndt_t *t = x->type; 133 | xnd_t next; 134 | 135 | next.bitmap = xnd_bitmap_empty; 136 | next.index = 0; 137 | next.type = t->Union.types[i]; 138 | next.ptr = x->ptr+1; 139 | 140 | return next; 141 | } 142 | 143 | static inline xnd_t 144 | _ref_next(const xnd_t *x) 145 | { 146 | const ndt_t *t = x->type; 147 | xnd_t next; 148 | 149 | next.bitmap = xnd_bitmap_empty; 150 | next.index = 0; 151 | next.type = t->Ref.type; 152 | next.ptr = XND_POINTER_DATA(x->ptr); 153 | 154 | return next; 155 | } 156 | 157 | static inline xnd_t 158 | _constr_next(const xnd_t *x) 159 | { 160 | const ndt_t *t = x->type; 161 | xnd_t next; 162 | 163 | next.bitmap = xnd_bitmap_empty; 164 | next.index = 0; 165 | next.type = t->Constr.type; 166 | next.ptr = x->ptr; 167 | 168 | return next; 169 | } 170 | 171 | static inline xnd_t 172 | _nominal_next(const xnd_t *x) 173 | { 174 | const ndt_t *t = x->type; 175 | xnd_t next; 176 | 177 | next.bitmap = xnd_bitmap_empty; 178 | next.index = 0; 179 | next.type = t->Nominal.type; 180 | next.ptr = x->ptr; 181 | 182 | return next; 183 | } 184 | 185 | 186 | #endif /* INLINE_H */ 187 | -------------------------------------------------------------------------------- /libxnd/overflow.h: -------------------------------------------------------------------------------- 1 | /* 2 | * BSD 3-Clause License 3 | * 4 | * Copyright (c) 2017-2018, plures 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * 1. Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above copyright notice, 14 | * this list of conditions and the following disclaimer in the documentation 15 | * and/or other materials provided with the distribution. 16 | * 17 | * 3. Neither the name of the copyright holder nor the names of its 18 | * contributors may be used to endorse or promote products derived from 19 | * this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | 34 | #include 35 | 36 | 37 | /*****************************************************************************/ 38 | /* Arithmetic with overflow checking */ 39 | /*****************************************************************************/ 40 | 41 | #if defined(__GNUC__) && __GNUC__ >= 5 && !defined(__INTEL_COMPILER) 42 | static inline int64_t 43 | ADDi64(int64_t a, int64_t b, bool *overflow) 44 | { 45 | int64_t c; 46 | *overflow |= __builtin_add_overflow(a, b, &c); 47 | return c; 48 | } 49 | 50 | static inline int64_t 51 | SUBi64(int64_t a, int64_t b, bool *overflow) 52 | { 53 | int64_t c; 54 | *overflow |= __builtin_sub_overflow(a, b, &c); 55 | return c; 56 | } 57 | 58 | static inline int64_t 59 | MULi64(int64_t a, int64_t b, bool *overflow) 60 | { 61 | int64_t c; 62 | *overflow |= __builtin_mul_overflow(a, b, &c); 63 | return c; 64 | } 65 | 66 | static inline size_t 67 | MULi64_size(int64_t a, int64_t b, bool *overflow) 68 | { 69 | int64_t c; 70 | *overflow |= __builtin_mul_overflow(a, b, &c); 71 | #if SIZE_MAX < INT64_MAX 72 | *overflow |= (c > INT32_MAX); 73 | #endif 74 | return (size_t)c; 75 | } 76 | 77 | static inline int64_t 78 | ABSi64(int64_t a, bool *overflow) 79 | { 80 | if (a == INT64_MIN) { 81 | *overflow = 1; 82 | return INT64_MIN; 83 | } 84 | return a >= 0 ? a : -a; 85 | } 86 | 87 | static inline uint16_t 88 | ADDu16(uint16_t a, uint16_t b, bool *overflow) 89 | { 90 | uint16_t c; 91 | *overflow |= __builtin_add_overflow(a, b, &c); 92 | return c; 93 | } 94 | #else 95 | static inline int64_t 96 | ADDi64(int64_t a, int64_t b, bool *overflow) 97 | { 98 | int64_t c = (uint64_t)a + (uint64_t)b; 99 | *overflow |= ((a < 0 && b < 0 && c >= 0) || (a >= 0 && b >= 0 && c < 0)); 100 | return c; 101 | } 102 | 103 | static inline int64_t 104 | SUBi64(int64_t a, int64_t b, bool *overflow) 105 | { 106 | int64_t c = (uint64_t)a - (uint64_t)b; 107 | *overflow |= ((a < 0 && b >= 0 && c >= 0) || (a >= 0 && b < 0 && c < 0)); 108 | return c; 109 | } 110 | 111 | static inline int64_t 112 | MULi64(int64_t a, int64_t b, bool *overflow) 113 | { 114 | int64_t c = (uint64_t)a * (uint64_t)b; 115 | *overflow |= ((b < 0 && a == INT64_MIN) || (b != 0 && a != c / b)); 116 | return c; 117 | } 118 | 119 | static inline size_t 120 | MULi64_size(int64_t a, int64_t b, bool *overflow) 121 | { 122 | int64_t c = (uint64_t)a * (uint64_t)b; 123 | *overflow |= ((b < 0 && a == INT64_MIN) || (b != 0 && a != c / b)); 124 | #if SIZE_MAX < INT64_MAX 125 | *overflow |= (c > INT32_MAX); 126 | #endif 127 | return (size_t)c; 128 | } 129 | 130 | static inline int64_t 131 | ABSi64(int64_t a, bool *overflow) 132 | { 133 | if (a == INT64_MIN) { 134 | *overflow = 1; 135 | return INT64_MIN; 136 | } 137 | return a >= 0 ? a : -a; 138 | } 139 | 140 | static inline uint16_t 141 | ADDu16(uint16_t a, uint16_t b, bool *overflow) 142 | { 143 | uint16_t c = a + b; 144 | *overflow |= (c < a); 145 | return c; 146 | } 147 | #endif /* OVERFLOW_H */ 148 | -------------------------------------------------------------------------------- /libxnd/shape.c: -------------------------------------------------------------------------------- 1 | /* 2 | * BSD 3-Clause License 3 | * 4 | * Copyright (c) 2017-2018, plures 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * 1. Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above copyright notice, 14 | * this list of conditions and the following disclaimer in the documentation 15 | * and/or other materials provided with the distribution. 16 | * 17 | * 3. Neither the name of the copyright holder nor the names of its 18 | * contributors may be used to endorse or promote products derived from 19 | * this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include "ndtypes.h" 39 | #include "xnd.h" 40 | #include "contrib.h" 41 | #include "overflow.h" 42 | 43 | 44 | static bool 45 | shape_equal(const ndt_ndarray_t *dest, const ndt_ndarray_t *src) 46 | { 47 | if (dest->ndim != src->ndim) { 48 | return false; 49 | } 50 | 51 | for (int i = 0; i < src->ndim; i++) { 52 | if (dest->shape[i] != src->shape[i]) { 53 | return false; 54 | } 55 | } 56 | 57 | return true; 58 | } 59 | 60 | static int64_t 61 | prod(const int64_t shape[], int N) 62 | { 63 | bool overflow = false; 64 | int64_t p = 1; 65 | 66 | for (int64_t i = 0; i < N; i++) { 67 | p = MULi64(p, shape[i], &overflow); 68 | if (overflow) { 69 | return -1; 70 | } 71 | } 72 | 73 | return p; 74 | } 75 | 76 | static inline bool 77 | zero_in_shape(const ndt_ndarray_t *x) 78 | { 79 | for (int i = 0; i < x->ndim; i++) { 80 | if (x->shape[i] == 0) { 81 | return true; 82 | } 83 | } 84 | 85 | return false; 86 | } 87 | 88 | static void 89 | init_contiguous_c_strides(ndt_ndarray_t *dest, const ndt_ndarray_t *src) 90 | { 91 | int64_t q; 92 | int64_t i; 93 | 94 | if (src->ndim == 0 && dest->ndim == 0) { 95 | return; 96 | } 97 | 98 | q = 1; 99 | for (i = dest->ndim-1; i >= 0; i--) { 100 | dest->steps[i] = q; 101 | q *= dest->shape[i]; 102 | } 103 | } 104 | 105 | static void 106 | init_contiguous_f_strides(ndt_ndarray_t *dest, const ndt_ndarray_t *src) 107 | { 108 | int64_t q; 109 | int64_t i; 110 | 111 | if (src->ndim == 0 && dest->ndim == 0) { 112 | return; 113 | } 114 | 115 | q = 1; 116 | for (i = 0; i < dest->ndim; i++) { 117 | dest->steps[i] = q; 118 | q *= dest->shape[i]; 119 | } 120 | } 121 | 122 | xnd_t 123 | xnd_reshape(const xnd_t *x, int64_t shape[], int ndim, char order, 124 | ndt_context_t *ctx) 125 | { 126 | const ndt_t *t = x->type; 127 | ndt_ndarray_t src, dest; 128 | int64_t p, q; 129 | int ret; 130 | int use_fortran = 0; 131 | 132 | if (order == 'F') { 133 | use_fortran = 1; 134 | } 135 | else if (order == 'A') { 136 | use_fortran = ndt_is_f_contiguous(t); 137 | } 138 | else if (order != 'C') { 139 | ndt_err_format(ctx, NDT_ValueError, "'order' must be 'C', 'F' or 'A'"); 140 | return xnd_error; 141 | } 142 | 143 | if (ndt_as_ndarray(&src, t, ctx) < 0) { 144 | return xnd_error; 145 | } 146 | 147 | dest.ndim = ndim; 148 | dest.itemsize = src.itemsize; 149 | for (int i = 0; i < ndim; i++) { 150 | dest.shape[i] = shape[i]; 151 | dest.steps[i] = 0; 152 | dest.strides[i] = 0; 153 | } 154 | 155 | p = prod(src.shape, src.ndim); 156 | q = prod(dest.shape, dest.ndim); 157 | if (p < 0 || q < 0) { 158 | ndt_err_format(ctx, NDT_ValueError, 159 | "reshaped array has too many elements"); 160 | return xnd_error; 161 | } 162 | if (p != q) { 163 | ndt_err_format(ctx, NDT_ValueError, 164 | "shapes do not have the same number of elements"); 165 | return xnd_error; 166 | } 167 | 168 | if (shape_equal(&dest, &src)) { 169 | dest = src; 170 | } 171 | else if (zero_in_shape(&dest)) { 172 | ; 173 | } 174 | else if (!use_fortran && ndt_is_c_contiguous(t)) { 175 | init_contiguous_c_strides(&dest, &src); 176 | } 177 | else if (use_fortran && ndt_is_f_contiguous(t)) { 178 | init_contiguous_f_strides(&dest, &src); 179 | } 180 | else { 181 | ret = xnd_nocopy_reshape(dest.shape, dest.steps, dest.ndim, 182 | src.shape, src.steps, src.ndim, use_fortran); 183 | if (!ret) { 184 | ndt_err_format(ctx, NDT_ValueError, "inplace reshape not possible"); 185 | return xnd_error; 186 | } 187 | } 188 | 189 | xnd_t res = *x; 190 | 191 | const ndt_t *u = ndt_copy(ndt_dtype(t), ctx); 192 | if (u == NULL) { 193 | return xnd_error; 194 | } 195 | 196 | for (int i = dest.ndim-1; i >= 0; i--) { 197 | const ndt_t *v = ndt_fixed_dim(u, dest.shape[i], dest.steps[i], ctx); 198 | ndt_decref(u); 199 | if (v == NULL) { 200 | return xnd_error; 201 | } 202 | u = v; 203 | } 204 | 205 | res.type = u; 206 | return res; 207 | } 208 | -------------------------------------------------------------------------------- /libxnd/split.c: -------------------------------------------------------------------------------- 1 | /* 2 | * BSD 3-Clause License 3 | * 4 | * Copyright (c) 2017-2018, plures 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * 1. Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above copyright notice, 14 | * this list of conditions and the following disclaimer in the documentation 15 | * and/or other materials provided with the distribution. 16 | * 17 | * 3. Neither the name of the copyright holder nor the names of its 18 | * contributors may be used to endorse or promote products derived from 19 | * this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include "ndtypes.h" 39 | #include "xnd.h" 40 | #include "overflow.h" 41 | 42 | 43 | static const xnd_index_t init_slice = 44 | { .tag = Slice, 45 | .Slice = {.start = -1, .stop = -1, .step = -1}}; 46 | 47 | 48 | static int64_t column( 49 | int64_t nrows, int64_t ncols, 50 | xnd_index_t *indices, int *nindices, 51 | int64_t row, int64_t col, int64_t r, int64_t q, 52 | int64_t m, const int64_t *ms, int len); 53 | 54 | 55 | static void 56 | free_slices(xnd_t *lst, int64_t len) 57 | { 58 | for (int64_t i = 0; i < len; i++) { 59 | ndt_decref(lst[i].type); 60 | } 61 | 62 | ndt_free(lst); 63 | } 64 | 65 | static inline int64_t 66 | start(int64_t i, int64_t r, int64_t q) 67 | { 68 | return i < r ? i*(q+1) : r+i*q; 69 | } 70 | 71 | static inline int64_t 72 | stop(int64_t i, int64_t r, int64_t q) 73 | { 74 | return i < r ? (i+1)*(q+1) : r+(i+1)*q; 75 | } 76 | 77 | static inline int64_t 78 | step(int64_t i, int64_t r, int64_t q) 79 | { 80 | return i < r ? q+1 : q; 81 | } 82 | 83 | static inline xnd_index_t 84 | single_step_slice(int64_t i) 85 | { 86 | xnd_index_t x; 87 | 88 | x.tag = Slice; 89 | x.Slice.start = i; 90 | x.Slice.stop = i+1; 91 | x.Slice.step = 1; 92 | 93 | return x; 94 | } 95 | 96 | static inline xnd_index_t 97 | slice(int64_t i, int64_t r, int64_t q) 98 | { 99 | xnd_index_t x; 100 | 101 | x.tag = Slice; 102 | x.Slice.start = start(i, r, q); 103 | x.Slice.stop = stop(i, r, q); 104 | x.Slice.step = 1; 105 | 106 | return x; 107 | } 108 | 109 | static int64_t 110 | prepend(int64_t nrows, int64_t ncols, 111 | xnd_index_t *indices, int *nindices, 112 | int64_t row, int64_t col, xnd_index_t s, int64_t n) 113 | { 114 | for (int64_t i = 0; i < n; i++) { 115 | assert(row+i < nrows && col < ncols); 116 | indices[(row+i)*ncols + col] = s; 117 | nindices[row+i]++; 118 | } 119 | 120 | return n; 121 | } 122 | 123 | static int64_t 124 | last_column(int64_t nrows, int64_t ncols, 125 | xnd_index_t *indices, int *nindices, 126 | int64_t row, int64_t col, int64_t r, int64_t q, int64_t n) 127 | { 128 | for (int64_t i = 0; i < n; i++) { 129 | assert(row+i < nrows && col < ncols); 130 | indices[(row+i)*ncols + col] = slice(i, r, q); 131 | nindices[row+i]++; 132 | } 133 | 134 | return n; 135 | } 136 | 137 | static int64_t 138 | schedule(int64_t nrows, int64_t ncols, 139 | xnd_index_t *indices, int *nindices, 140 | int64_t row, int64_t col, int64_t n, const int64_t *shape, int len) 141 | { 142 | int64_t m; 143 | int64_t q; 144 | int64_t r; 145 | 146 | if (len == 0) { 147 | return 1; 148 | } 149 | 150 | m = shape[0]; 151 | if (n <= m) { 152 | q = m / n; 153 | r = m % n; 154 | return last_column(nrows, ncols, indices, nindices, row, col, r, q, n); 155 | } 156 | else { 157 | q = n / m; 158 | r = n % m; 159 | return column(nrows, ncols, indices, nindices, row, col, r, q, m, shape+1, len-1); 160 | } 161 | } 162 | 163 | static int64_t 164 | column(int64_t nrows, int64_t ncols, 165 | xnd_index_t *indices, int *nindices, 166 | int64_t row, int64_t col, int64_t r, int64_t q, int64_t m, 167 | const int64_t *ms, int len) 168 | { 169 | int64_t column_len = 0; 170 | int64_t n, subtree_len, block_len; 171 | xnd_index_t s; 172 | 173 | for (int64_t i = 0; i < m; i++) { 174 | n = step(i, r, q); 175 | s = single_step_slice(i); 176 | subtree_len = schedule(nrows, ncols, indices, nindices, row, col+1, n, ms, len); 177 | block_len = prepend(nrows, ncols, indices, nindices, row, col, s, subtree_len); 178 | row += block_len; 179 | column_len += block_len; 180 | } 181 | 182 | return column_len; 183 | } 184 | 185 | static int 186 | get_shape(int64_t *shape, const ndt_t *t, int max_outer, ndt_context_t *ctx) 187 | { 188 | int i; 189 | 190 | if (!ndt_is_ndarray(t)) { 191 | ndt_err_format(ctx, NDT_ValueError, 192 | "split function called on non-ndarray"); 193 | return -1; 194 | } 195 | 196 | for (i = 0; i < max_outer && t->ndim > 0; i++, t=t->FixedDim.type) { 197 | shape[i] = t->FixedDim.shape; 198 | if (shape[i] <= 0) { 199 | ndt_err_format(ctx, NDT_ValueError, 200 | "split function called on invalid shape or shape with zeros"); 201 | return -1; 202 | } 203 | } 204 | for (; t->ndim > 0; t=t->FixedDim.type) { 205 | if (t->FixedDim.shape <= 0) { 206 | ndt_err_format(ctx, NDT_ValueError, 207 | "split function called on invalid shape or shape with zeros"); 208 | return -1; 209 | } 210 | } 211 | 212 | return i; 213 | } 214 | 215 | xnd_t * 216 | xnd_split(const xnd_t *x, int64_t *nparts, int max_outer, ndt_context_t *ctx) 217 | { 218 | bool overflow = false; 219 | int64_t shape[NDT_MAX_DIM]; 220 | xnd_index_t *indices; 221 | int *nindices; 222 | xnd_t *result; 223 | int64_t nrows, nmemb; 224 | int ncols; 225 | 226 | if (*nparts < 1) { 227 | ndt_err_format(ctx, NDT_ValueError, "'n' parameter must be >= 1"); 228 | return NULL; 229 | } 230 | nrows = *nparts; 231 | 232 | ncols = get_shape(shape, x->type, max_outer, ctx); 233 | if (ncols < 0) { 234 | return NULL; 235 | } 236 | 237 | nmemb = MULi64(nrows, ncols, &overflow); 238 | if (overflow) { 239 | ndt_err_format(ctx, NDT_ValueError, "'n' parameter is too large"); 240 | return NULL; 241 | } 242 | 243 | indices = ndt_alloc(nmemb, sizeof *indices); 244 | if (indices == NULL) { 245 | return ndt_memory_error(ctx); 246 | } 247 | for (int64_t i = 0; i < nrows; i++) { 248 | for (int64_t k = 0; k < ncols; k++) { 249 | indices[i*ncols + k] = init_slice; 250 | } 251 | } 252 | 253 | nindices = ndt_alloc(nrows, sizeof *nindices); 254 | if (nindices == NULL) { 255 | ndt_free(indices); 256 | return ndt_memory_error(ctx); 257 | } 258 | for (int64_t i = 0; i < nrows; i++) { 259 | nindices[i] = 0; 260 | } 261 | 262 | nrows = schedule(nrows, ncols, indices, nindices, 0, 0, nrows, shape, ncols); 263 | 264 | result = ndt_alloc(nrows, sizeof *result); 265 | if (result == NULL) { 266 | ndt_free(nindices); 267 | ndt_free(indices); 268 | return ndt_memory_error(ctx); 269 | } 270 | 271 | for (int64_t i = 0; i < nrows; i++) { 272 | result[i] = xnd_subscript(x, indices+(i*ncols), nindices[i], ctx); 273 | if (ndt_err_occurred(ctx)) { 274 | ndt_free(nindices); 275 | ndt_free(indices); 276 | free_slices(result, i); 277 | return NULL; 278 | } 279 | } 280 | 281 | ndt_free(nindices); 282 | ndt_free(indices); 283 | *nparts = nrows; 284 | 285 | return result; 286 | } 287 | -------------------------------------------------------------------------------- /libxnd/tests/Makefile.in: -------------------------------------------------------------------------------- 1 | 2 | SRCDIR = .. 3 | 4 | CC = @CC@ 5 | LIBSTATIC = @LIBSTATIC@ 6 | LIBSHARED = @LIBSHARED@ 7 | 8 | INCLUDES = @CONFIGURE_INCLUDES_TEST@ 9 | LIBS = @CONFIGURE_LIBS_TEST@ 10 | 11 | CONFIGURE_CFLAGS = @CONFIGURE_CFLAGS@ 12 | XND_CFLAGS = $(strip $(CONFIGURE_CFLAGS) $(CFLAGS)) 13 | 14 | 15 | default: runtest runtest_shared 16 | 17 | 18 | runtest:\ 19 | Makefile runtest.c test_fixed.c test.h $(SRCDIR)/xnd.h $(SRCDIR)/$(LIBSTATIC) 20 | $(CC) -I$(SRCDIR) -I$(INCLUDES) $(XND_CFLAGS) \ 21 | -o runtest runtest.c test_fixed.c $(SRCDIR)/libxnd.a \ 22 | $(LIBS)/libndtypes.a 23 | 24 | runtest_shared:\ 25 | Makefile runtest.c test_fixed.c test.h $(SRCDIR)/xnd.h $(SRCDIR)/$(LIBSHARED) 26 | $(CC) -I$(SRCDIR) -I$(INCLUDES) -L$(SRCDIR) -L$(LIBS) \ 27 | $(XND_CFLAGS) -o runtest_shared runtest.c test_fixed.c -lxnd -lndtypes 28 | 29 | 30 | FORCE: 31 | 32 | clean: FORCE 33 | rm -f *.o *.gch *.gcda *.gcno *.gcov *.dyn *.dpi *.lock 34 | rm -f runtest runtest_shared 35 | 36 | distclean: clean 37 | rm -rf Makefile 38 | 39 | 40 | -------------------------------------------------------------------------------- /libxnd/tests/Makefile.vc: -------------------------------------------------------------------------------- 1 | 2 | SRCDIR = .. 3 | 4 | LIBSTATIC = libxnd-0.2.0dev3.lib 5 | LIBSHARED = libxnd-0.2.0dev3.dll.lib 6 | LIBNDTYPESSTATIC = libndtypes-0.2.0dev3.lib 7 | LIBNDTYPESIMPORT = libndtypes-0.2.0dev3.dll.lib 8 | 9 | !ifndef LIBNDTYPESINCLUDE 10 | LIBNDTYPESINCLUDE = ..\..\ndtypes\libndtypes 11 | !endif 12 | 13 | !ifndef LIBNDTYPESDIR 14 | LIBNDTYPESDIR = ..\..\ndtypes\libndtypes 15 | !endif 16 | 17 | 18 | CC = cl.exe 19 | CFLAGS = /nologo /MT /Ox /GS /EHsc 20 | CFLAGS_SHARED = /nologo /DXND_IMPORT /MD /Ox /GS /EHsc 21 | 22 | default: runtest runtest_shared 23 | 24 | 25 | runtest:\ 26 | Makefile runtest.c test_fixed.c test.h $(SRCDIR)\xnd.h $(SRCDIR)\$(LIBSTATIC) 27 | $(CC) "-I$(SRCDIR)" "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS) /Feruntest runtest.c \ 28 | test_fixed.c $(SRCDIR)\$(LIBSTATIC) /link "/LIBPATH:$(LIBNDTYPESDIR)" $(LIBNDTYPESSTATIC) 29 | 30 | runtest_shared:\ 31 | Makefile runtest.c test_fixed.c test.h $(SRCDIR)\xnd.h $(SRCDIR)\$(LIBSHARED) 32 | $(CC) "-I$(SRCDIR)" "-I$(LIBNDTYPESINCLUDE)" $(CFLAGS_SHARED) /Feruntest_shared \ 33 | runtest.c test_fixed.c $(SRCDIR)\$(LIBSHARED) /link "/LIBPATH:$(LIBNDTYPESDIR)" $(LIBNDTYPESIMPORT) 34 | 35 | 36 | FORCE: 37 | 38 | clean: FORCE 39 | del /q /f *.exe *.obj *.lib *.dll *.exp *.manifest 2>NUL 40 | 41 | distclean: clean 42 | del /q /f Makefile 2>NUL 43 | 44 | 45 | -------------------------------------------------------------------------------- /libxnd/tests/README.txt: -------------------------------------------------------------------------------- 1 | 2 | The library is tested extensively through the unit tests of the Python bindings. 3 | -------------------------------------------------------------------------------- /libxnd/tests/runtest.c: -------------------------------------------------------------------------------- 1 | /* 2 | * BSD 3-Clause License 3 | * 4 | * Copyright (c) 2017-2018, plures 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * 1. Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above copyright notice, 14 | * this list of conditions and the following disclaimer in the documentation 15 | * and/or other materials provided with the distribution. 16 | * 17 | * 3. Neither the name of the copyright holder nor the names of its 18 | * contributors may be used to endorse or promote products derived from 19 | * this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include "ndtypes.h" 39 | #include "test.h" 40 | 41 | 42 | static int 43 | init_tests(void) 44 | { 45 | ndt_context_t *ctx; 46 | 47 | ctx = ndt_context_new(); 48 | if (ctx == NULL) { 49 | fprintf(stderr, "error: out of memory"); 50 | return -1; 51 | } 52 | 53 | if (ndt_init(ctx) < 0) { 54 | ndt_err_fprint(stderr, ctx); 55 | ndt_context_del(ctx); 56 | return -1; 57 | } 58 | 59 | if (xnd_init_float(ctx) < 0) { 60 | ndt_err_fprint(stderr, ctx); 61 | ndt_context_del(ctx); 62 | return -1; 63 | } 64 | 65 | ndt_context_del(ctx); 66 | return 0; 67 | } 68 | 69 | static int (*tests[])(void) = { 70 | test_fixed, 71 | NULL 72 | }; 73 | 74 | int 75 | main(void) 76 | { 77 | int (**f)(void); 78 | int success = 0; 79 | int fail = 0; 80 | 81 | if (init_tests() < 0) { 82 | return 1; 83 | } 84 | 85 | for (f = tests; *f != NULL; f++) { 86 | if ((*f)() < 0) 87 | fail++; 88 | else 89 | success++; 90 | } 91 | 92 | if (fail) { 93 | fprintf(stderr, "\nFAIL (failures=%d)\n", fail); 94 | } 95 | else { 96 | fprintf(stderr, "\n%d tests OK.\n", success); 97 | } 98 | 99 | ndt_finalize(); 100 | return fail ? 1 : 0; 101 | } 102 | -------------------------------------------------------------------------------- /libxnd/tests/test.h: -------------------------------------------------------------------------------- 1 | /* 2 | * BSD 3-Clause License 3 | * 4 | * Copyright (c) 2017-2018, plures 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * 1. Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above copyright notice, 14 | * this list of conditions and the following disclaimer in the documentation 15 | * and/or other materials provided with the distribution. 16 | * 17 | * 3. Neither the name of the copyright holder nor the names of its 18 | * contributors may be used to endorse or promote products derived from 19 | * this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | 34 | #ifndef TEST_H 35 | #define TEST_H 36 | 37 | 38 | #include "ndtypes.h" 39 | #include "xnd.h" 40 | 41 | 42 | #define ARRAY_SIZE(a) ((int)(sizeof(a)/sizeof(a[0]))) 43 | 44 | 45 | int test_fixed(void); 46 | 47 | 48 | #endif /* TEST_H */ 49 | -------------------------------------------------------------------------------- /libxnd/tests/test_fixed.c: -------------------------------------------------------------------------------- 1 | /* 2 | * BSD 3-Clause License 3 | * 4 | * Copyright (c) 2017-2018, plures 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * 1. Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above copyright notice, 14 | * this list of conditions and the following disclaimer in the documentation 15 | * and/or other materials provided with the distribution. 16 | * 17 | * 3. Neither the name of the copyright holder nor the names of its 18 | * contributors may be used to endorse or promote products derived from 19 | * this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | 34 | #include 35 | #include 36 | #include 37 | #include "ndtypes.h" 38 | #include "test.h" 39 | 40 | 41 | int 42 | test_fixed(void) 43 | { 44 | ndt_context_t *ctx; 45 | xnd_master_t *x; 46 | xnd_t view; 47 | uint16_t *ptr; 48 | int ret = 0; 49 | int i, j, k, l; 50 | int64_t indices[3]; 51 | 52 | /* a1 = [[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[8, 9], [10, 11]]] */ 53 | const char *s = "3 * 2 * 2 * uint16"; 54 | uint16_t data[12] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; 55 | 56 | 57 | ctx = ndt_context_new(); 58 | if (ctx == NULL) { 59 | fprintf(stderr, "out of memory\n"); 60 | return 1; 61 | } 62 | 63 | 64 | /***** Type with fixed dimensions *****/ 65 | x = xnd_empty_from_string(s, XND_OWN_ALL, ctx); 66 | if (x == NULL) { 67 | goto error; 68 | } 69 | 70 | ptr = (uint16_t *)x->master.ptr; 71 | for (i = 0; i < ARRAY_SIZE(data); i++) { 72 | ptr[i] = data[i]; 73 | } 74 | 75 | for (i = 0; i < 3; i++) { 76 | for (j = 0; j < 2; j++) { 77 | for (k = 0; k < 2; k++) { 78 | indices[0] = i; indices[1] = j; indices[2] = k; 79 | l = i * 4 + j * 2 + k; 80 | view = xnd_subtree_index(&x->master, indices, 3, ctx); 81 | if (view.ptr == NULL) { 82 | goto error; 83 | } 84 | assert(view.type->tag == Uint16); 85 | if (*(uint16_t *)(view.ptr) != data[l]) { 86 | ndt_err_format(ctx, NDT_RuntimeError, "unexpected value"); 87 | goto error; 88 | } 89 | } 90 | } 91 | } 92 | 93 | 94 | fprintf(stderr, "test_fixed (1 test case)\n"); 95 | 96 | 97 | out: 98 | xnd_del(x); 99 | ndt_context_del(ctx); 100 | return ret; 101 | 102 | error: 103 | ret = -1; 104 | ndt_err_fprint(stderr, ctx); 105 | goto out; 106 | } 107 | 108 | 109 | -------------------------------------------------------------------------------- /python/xnd/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.2.0dev3' 2 | -------------------------------------------------------------------------------- /python/xnd/docstrings.h: -------------------------------------------------------------------------------- 1 | /* 2 | * BSD 3-Clause License 3 | * 4 | * Copyright (c) 2017-2018, plures 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * 1. Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above copyright notice, 14 | * this list of conditions and the following disclaimer in the documentation 15 | * and/or other materials provided with the distribution. 16 | * 17 | * 3. Neither the name of the copyright holder nor the names of its 18 | * contributors may be used to endorse or promote products derived from 19 | * this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | 34 | #include "pymacro.h" 35 | 36 | 37 | /******************************************************************************/ 38 | /* Module */ 39 | /******************************************************************************/ 40 | 41 | 42 | PyDoc_STRVAR(doc_module, "xnd module"); 43 | 44 | 45 | 46 | /******************************************************************************/ 47 | /* xnd properties */ 48 | /******************************************************************************/ 49 | 50 | PyDoc_STRVAR(doc_value, 51 | "\n\ 52 | Convert the in-memory value to a Python value.\n\ 53 | \n\ 54 | >>> x = xnd([1, 2, 3, 4, 5])\n\ 55 | >>> x.value\n\ 56 | [1, 2, 3, 4, 5]\n\ 57 | \n"); 58 | 59 | PyDoc_STRVAR(doc_type, 60 | "\n\ 61 | Return the type of the in-memory value.\n\ 62 | \n\ 63 | >>> x = xnd([[1, 2, 3], [4, 5]])\n\ 64 | >>> x.type\n\ 65 | ndt(\"var * var * int64\")\n\ 66 | \n"); 67 | 68 | PyDoc_STRVAR(doc_ndim, 69 | "\n\ 70 | Return the number of dimensions of the in-memory value.\n\ 71 | \n\ 72 | >>> x = xnd([[(1, \"a\"), (2, \"b\")], [(3, \"x\"), (4, \"y\")]])\n\ 73 | >>> x.ndim\n\ 74 | 2\n\ 75 | \n"); 76 | 77 | PyDoc_STRVAR(doc_align, 78 | "\n\ 79 | Return the alignment of the memory block.\n\ 80 | \n\ 81 | >>> t = \"2 * {a: uint8, b: float64, align=512}\"\n\ 82 | >>> x = xnd([{'a': 1, 'b': 2.0}, {'a': 2, 'b': 3.0}], type=t)\n\ 83 | >>> x.align\n\ 84 | 512\n\ 85 | \n"); 86 | 87 | 88 | /******************************************************************************/ 89 | /* xnd methods */ 90 | /******************************************************************************/ 91 | 92 | PyDoc_STRVAR(doc_short_value, 93 | "short_value($self, /, maxshape=None)\n--\n\n\ 94 | Convert the in-memory value to an abbreviated Python value. Container types\n\ 95 | are converted to maxshape-1 actual values followed by an Ellipsis. If maxshape\n\ 96 | is None, the representation is not abbreviated.\n\ 97 | \n\ 98 | >>> x = xnd([1, 2, 3, 4, 5])\n\ 99 | >>> x.short_value(maxshape=3)\n\ 100 | [1, 2, ...]\n\ 101 | \n"); 102 | 103 | PyDoc_STRVAR(doc_empty, 104 | "empty($type, type, /)\n--\n\n\ 105 | Class method that constructs a new xnd container according to the type\n\ 106 | argument. All values are initialized to zero.\n\ 107 | \n\ 108 | >>> xnd.empty(\"10 * 2 * (int64, string, complex128)\")\n\ 109 | xnd([[(0, '', 0j), (0, '', 0j)], [(0, '', 0j), (0, '', 0j)],\n\ 110 | [(0, '', 0j), (0, '', 0j)], [(0, '', 0j), (0, '', 0j)],\n\ 111 | [(0, '', 0j), (0, '', 0j)], [(0, '', 0j), (0, '', 0j)],\n\ 112 | [(0, '', 0j), (0, '', 0j)], [(0, '', 0j), (0, '', 0j)],\n\ 113 | [(0, '', 0j), (0, '', 0j)], ...], type=\"10 * 2 * (int64, string, complex128)\")\n\ 114 | \n"); 115 | 116 | PyDoc_STRVAR(doc_from_buffer, 117 | "from_buffer($type, obj, /)\n--\n\n\ 118 | Class method that constructs a new xnd container from an object that supports\n\ 119 | the buffer protocol.\n\ 120 | \n\ 121 | >>> xnd.from_buffer(b\"123\")\n\ 122 | xnd([49, 50, 51], type=\"3 * uint8\")\n\ 123 | \n\ 124 | >>> import numpy as np\n\ 125 | >>> a = np.array([[1, 2, 3], [4, 5, 6]])\n\ 126 | >>> xnd.from_buffer(a)\n\ 127 | xnd([[1, 2, 3], [4, 5, 6]], type=\"2 * 3 * int64\")\n\ 128 | \n\ 129 | \n"); 130 | -------------------------------------------------------------------------------- /python/xnd/pyxnd.h: -------------------------------------------------------------------------------- 1 | /* 2 | * BSD 3-Clause License 3 | * 4 | * Copyright (c) 2017-2018, plures 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * 1. Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above copyright notice, 14 | * this list of conditions and the following disclaimer in the documentation 15 | * and/or other materials provided with the distribution. 16 | * 17 | * 3. Neither the name of the copyright holder nor the names of its 18 | * contributors may be used to endorse or promote products derived from 19 | * this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | 34 | #ifndef PYXND_H 35 | #define PYXND_H 36 | #ifdef __cplusplus 37 | extern "C" { 38 | #endif 39 | 40 | 41 | #include 42 | #include "ndtypes.h" 43 | #include "xnd.h" 44 | 45 | 46 | /****************************************************************************/ 47 | /* MemoryBlock Object */ 48 | /****************************************************************************/ 49 | 50 | /* This object owns the memory that is shared by several xnd objects. It is 51 | never exposed to the Python level. 52 | 53 | The memory block is created by the primary xnd object on initialization. 54 | Sub-views, slices etc. share the memory block. 55 | 56 | PEP-3118 imports are supported. At a later stage the mblock object will 57 | potentially need to communicate with Arrow or other formats in order 58 | to acquire and manage external memory blocks. 59 | */ 60 | 61 | /* Exposed here for the benefit of Numba. The API should not be regarded 62 | stable across versions. */ 63 | 64 | typedef struct { 65 | PyObject_HEAD 66 | PyObject *type; /* type owner */ 67 | xnd_master_t *xnd; /* memblock owner */ 68 | Py_buffer *view; /* PEP-3118 imports */ 69 | } MemoryBlockObject; 70 | 71 | 72 | /****************************************************************************/ 73 | /* xnd object */ 74 | /****************************************************************************/ 75 | 76 | /* Exposed here for the benefit of Numba. The API should not be regarded 77 | stable across versions. */ 78 | 79 | typedef struct { 80 | PyObject_HEAD 81 | MemoryBlockObject *mblock; /* owner of the primary type and memory block */ 82 | PyObject *type; /* owner of the current type */ 83 | xnd_t xnd; /* typed view, does not own anything */ 84 | } XndObject; 85 | 86 | #define TYPE_OWNER(v) (((XndObject *)v)->type) 87 | #define XND(v) (&(((XndObject *)v)->xnd)) 88 | #define XND_INDEX(v) (((XndObject *)v)->xnd.index) 89 | #define XND_TYPE(v) (((XndObject *)v)->xnd.type) 90 | #define XND_PTR(v) (((XndObject *)v)->xnd.ptr) 91 | 92 | 93 | /****************************************************************************/ 94 | /* Capsule API */ 95 | /****************************************************************************/ 96 | 97 | #define Xnd_CheckExact_INDEX 0 98 | #define Xnd_CheckExact_RETURN int 99 | #define Xnd_CheckExact_ARGS (const PyObject *) 100 | 101 | #define Xnd_Check_INDEX 1 102 | #define Xnd_Check_RETURN int 103 | #define Xnd_Check_ARGS (const PyObject *) 104 | 105 | #define CONST_XND_INDEX 2 106 | #define CONST_XND_RETURN const xnd_t * 107 | #define CONST_XND_ARGS (const PyObject *) 108 | 109 | #define Xnd_EmptyFromType_INDEX 3 110 | #define Xnd_EmptyFromType_RETURN PyObject * 111 | #define Xnd_EmptyFromType_ARGS (PyTypeObject *, const ndt_t *t, uint32_t flags) 112 | 113 | #define Xnd_ViewMoveNdt_INDEX 4 114 | #define Xnd_ViewMoveNdt_RETURN PyObject * 115 | #define Xnd_ViewMoveNdt_ARGS (const PyObject *, ndt_t *t) 116 | 117 | #define Xnd_FromXnd_INDEX 5 118 | #define Xnd_FromXnd_RETURN PyObject * 119 | #define Xnd_FromXnd_ARGS (PyTypeObject *, xnd_t *x) 120 | 121 | #define Xnd_Subscript_INDEX 6 122 | #define Xnd_Subscript_RETURN PyObject * 123 | #define Xnd_Subscript_ARGS (const PyObject *self, const PyObject *key) 124 | 125 | #define Xnd_FromXndMoveType_INDEX 7 126 | #define Xnd_FromXndMoveType_RETURN PyObject * 127 | #define Xnd_FromXndMoveType_ARGS (const PyObject *xnd, xnd_t *x) 128 | 129 | #define Xnd_FromXndView_INDEX 8 130 | #define Xnd_FromXndView_RETURN PyObject * 131 | #define Xnd_FromXndView_ARGS (xnd_view_t *x) 132 | 133 | #define Xnd_GetType_INDEX 9 134 | #define Xnd_GetType_RETURN PyTypeObject * 135 | #define Xnd_GetType_ARGS (void) 136 | 137 | #define XND_MAX_API 10 138 | 139 | 140 | #ifdef XND_MODULE 141 | static Xnd_CheckExact_RETURN Xnd_CheckExact Xnd_CheckExact_ARGS; 142 | static Xnd_Check_RETURN Xnd_Check Xnd_Check_ARGS; 143 | static CONST_XND_RETURN CONST_XND CONST_XND_ARGS; 144 | static Xnd_EmptyFromType_RETURN Xnd_EmptyFromType Xnd_EmptyFromType_ARGS; 145 | static Xnd_ViewMoveNdt_RETURN Xnd_ViewMoveNdt Xnd_ViewMoveNdt_ARGS; 146 | static Xnd_FromXnd_RETURN Xnd_FromXnd Xnd_FromXnd_ARGS; 147 | static Xnd_Subscript_RETURN Xnd_Subscript Xnd_Subscript_ARGS; 148 | static Xnd_FromXndMoveType_RETURN Xnd_FromXndMoveType Xnd_FromXndMoveType_ARGS; 149 | static Xnd_FromXndView_RETURN Xnd_FromXndView Xnd_FromXndView_ARGS; 150 | static Xnd_GetType_RETURN Xnd_GetType Xnd_GetType_ARGS; 151 | #else 152 | static void **_xnd_api; 153 | 154 | #define Xnd_CheckExact \ 155 | (*(Xnd_CheckExact_RETURN (*)Xnd_CheckExact_ARGS) _xnd_api[Xnd_CheckExact_INDEX]) 156 | 157 | #define Xnd_Check \ 158 | (*(Xnd_Check_RETURN (*)Xnd_Check_ARGS) _xnd_api[Xnd_Check_INDEX]) 159 | 160 | #define CONST_XND \ 161 | (*(CONST_XND_RETURN (*)CONST_XND_ARGS) _xnd_api[CONST_XND_INDEX]) 162 | 163 | #define Xnd_EmptyFromType \ 164 | (*(Xnd_EmptyFromType_RETURN (*)Xnd_EmptyFromType_ARGS) _xnd_api[Xnd_EmptyFromType_INDEX]) 165 | 166 | #define Xnd_ViewMoveNdt \ 167 | (*(Xnd_ViewMoveNdt_RETURN (*)Xnd_ViewMoveNdt_ARGS) _xnd_api[Xnd_ViewMoveNdt_INDEX]) 168 | 169 | #define Xnd_FromXnd \ 170 | (*(Xnd_FromXnd_RETURN (*)Xnd_FromXnd_ARGS) _xnd_api[Xnd_FromXnd_INDEX]) 171 | 172 | #define Xnd_Subscript \ 173 | (*(Xnd_Subscript_RETURN (*)Xnd_Subscript_ARGS) _xnd_api[Xnd_Subscript_INDEX]) 174 | 175 | #define Xnd_FromXndMoveType \ 176 | (*(Xnd_FromXndMoveType_RETURN (*)Xnd_FromXndMoveType_ARGS) _xnd_api[Xnd_FromXndMoveType_INDEX]) 177 | 178 | #define Xnd_FromXndView \ 179 | (*(Xnd_FromXndView_RETURN (*)Xnd_FromXndView_ARGS) _xnd_api[Xnd_FromXndView_INDEX]) 180 | 181 | #define Xnd_GetType \ 182 | (*(Xnd_GetType_RETURN (*)Xnd_GetType_ARGS) _xnd_api[Xnd_GetType_INDEX]) 183 | 184 | static int 185 | import_xnd(void) 186 | { 187 | _xnd_api = (void **)PyCapsule_Import("xnd._xnd._API", 0); 188 | if (_xnd_api == NULL) { 189 | return -1; 190 | } 191 | 192 | return 0; 193 | } 194 | #endif 195 | 196 | #ifdef __cplusplus 197 | } 198 | #endif 199 | 200 | #endif /* PYXND_H */ 201 | -------------------------------------------------------------------------------- /python/xnd/util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * BSD 3-Clause License 3 | * 4 | * Copyright (c) 2017-2018, plures 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * 1. Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above copyright notice, 14 | * this list of conditions and the following disclaimer in the documentation 15 | * and/or other materials provided with the distribution. 16 | * 17 | * 3. Neither the name of the copyright holder nor the names of its 18 | * contributors may be used to endorse or promote products derived from 19 | * this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | 34 | #ifndef UTIL_H 35 | #define UTIL_H 36 | 37 | 38 | #include 39 | #if PY_VERSION_HEX < 0x030B0000 40 | /* 41 | * Python 3.11 moved longintrepr.h and it is already included in Python.h 42 | * https://github.com/python/cpython/pull/28968 43 | */ 44 | #include 45 | #endif 46 | #include 47 | #include 48 | #include 49 | #include "ndtypes.h" 50 | 51 | 52 | static inline Py_ssize_t 53 | safe_downcast(int64_t size) 54 | { 55 | #if SIZE_MAX < INT64_MAX 56 | if (size > INT32_MAX) { 57 | PyErr_SetString(PyExc_ValueError, 58 | "sizes should never exceed INT32_MAX on 32-bit platforms"); 59 | return -1; 60 | } 61 | #endif 62 | return (Py_ssize_t)size; 63 | } 64 | 65 | static inline bool 66 | check_invariants(const ndt_t *t) 67 | { 68 | #if SIZE_MAX < INT64_MAX 69 | return safe_downcast(t->datasize) >= 0; 70 | #else 71 | (void)t; 72 | return 1; 73 | #endif 74 | } 75 | 76 | static inline PyObject * 77 | list_new(int64_t size) 78 | { 79 | #if SIZE_MAX < INT64_MAX 80 | Py_ssize_t n = safe_downcast(size); 81 | return n < 0 ? NULL : PyList_New(n); 82 | #else 83 | return PyList_New(size); 84 | #endif 85 | } 86 | 87 | static PyObject * 88 | tuple_new(int64_t size) 89 | { 90 | #if SIZE_MAX < INT64_MAX 91 | Py_ssize_t n = safe_downcast(size); 92 | return n < 0 ? NULL : PyTuple_New(n); 93 | #else 94 | return PyTuple_New(size); 95 | #endif 96 | } 97 | 98 | static inline PyObject * 99 | unicode_from_kind_and_data(int kind, const void *buffer, int64_t size) 100 | { 101 | #if SIZE_MAX < INT64_MAX 102 | Py_ssize_t n = safe_downcast(size); 103 | return n < 0 ? NULL : PyUnicode_FromKindAndData(kind, buffer, n); 104 | #else 105 | return PyUnicode_FromKindAndData(kind, buffer, size); 106 | #endif 107 | } 108 | 109 | static inline PyObject * 110 | bytes_from_string_and_size(const char *str, int64_t size) 111 | { 112 | #if SIZE_MAX < INT64_MAX 113 | Py_ssize_t n = safe_downcast(size); 114 | return n < 0 ? NULL : PyBytes_FromStringAndSize(str, n); 115 | #else 116 | return PyBytes_FromStringAndSize(str, size); 117 | #endif 118 | } 119 | 120 | static inline int 121 | py_slice_get_indices_ex(PyObject *key, int64_t length, 122 | int64_t *start, int64_t *stop, int64_t *step, 123 | int64_t *slicelength) 124 | { 125 | Py_ssize_t n, _start, _stop, _step, _slicelength; 126 | int ret; 127 | 128 | n = safe_downcast(length); 129 | if (n < 0) { 130 | return -1; 131 | } 132 | 133 | ret = PySlice_GetIndicesEx(key, n, &_start, &_stop, &_step, &_slicelength); 134 | *start = _start; 135 | *stop = _stop; 136 | *step = _step; 137 | *slicelength = _slicelength; 138 | 139 | return ret; 140 | } 141 | 142 | /* PSF copyright: Written by Jim Hugunin and Chris Chase. */ 143 | int 144 | pyslice_unpack(PyObject *_r, 145 | Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step) 146 | { 147 | PySliceObject *r = (PySliceObject*)_r; 148 | /* this is harder to get right than you might think */ 149 | 150 | assert(PY_SSIZE_T_MIN + 1 <= -PY_SSIZE_T_MAX); 151 | 152 | if (r->step == Py_None) { 153 | *step = 1; 154 | } 155 | else { 156 | if (!_PyEval_SliceIndex(r->step, step)) return -1; 157 | if (*step == 0) { 158 | PyErr_SetString(PyExc_ValueError, 159 | "slice step cannot be zero"); 160 | return -1; 161 | } 162 | /* Here *step might be -PY_SSIZE_T_MAX-1; in this case we replace it 163 | * with -PY_SSIZE_T_MAX. This doesn't affect the semantics, and it 164 | * guards against later undefined behaviour resulting from code that 165 | * does "step = -step" as part of a slice reversal. 166 | */ 167 | if (*step < -PY_SSIZE_T_MAX) 168 | *step = -PY_SSIZE_T_MAX; 169 | } 170 | 171 | if (r->start == Py_None) { 172 | *start = *step < 0 ? PY_SSIZE_T_MAX : 0; 173 | } 174 | else { 175 | if (!_PyEval_SliceIndex(r->start, start)) return -1; 176 | } 177 | 178 | if (r->stop == Py_None) { 179 | *stop = *step < 0 ? PY_SSIZE_T_MIN : PY_SSIZE_T_MAX; 180 | } 181 | else { 182 | if (!_PyEval_SliceIndex(r->stop, stop)) return -1; 183 | } 184 | 185 | return 0; 186 | } 187 | 188 | /* longobject.c */ 189 | static inline int 190 | long_compare(PyLongObject *a, PyLongObject *b) 191 | { 192 | Py_ssize_t sign; 193 | 194 | if (Py_SIZE(a) != Py_SIZE(b)) { 195 | sign = Py_SIZE(a) - Py_SIZE(b); 196 | } 197 | else { 198 | Py_ssize_t i = Py_ABS(Py_SIZE(a)); 199 | while (--i >= 0 && a->ob_digit[i] == b->ob_digit[i]) 200 | ; 201 | if (i < 0) 202 | sign = 0; 203 | else { 204 | sign = (sdigit)a->ob_digit[i] - (sdigit)b->ob_digit[i]; 205 | if (Py_SIZE(a) < 0) 206 | sign = -sign; 207 | } 208 | } 209 | return sign < 0 ? -1 : sign > 0 ? 1 : 0; 210 | } 211 | 212 | 213 | #endif /* UTIL_H */ 214 | -------------------------------------------------------------------------------- /python/xnd_support.py: -------------------------------------------------------------------------------- 1 | # 2 | # BSD 3-Clause License 3 | # 4 | # Copyright (c) 2017-2018, plures 5 | # All rights reserved. 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted provided that the following conditions are met: 9 | # 10 | # 1. Redistributions of source code must retain the above copyright notice, 11 | # this list of conditions and the following disclaimer. 12 | # 13 | # 2. Redistributions in binary form must reproduce the above copyright notice, 14 | # this list of conditions and the following disclaimer in the documentation 15 | # and/or other materials provided with the distribution. 16 | # 17 | # 3. Neither the name of the copyright holder nor the names of its 18 | # contributors may be used to endorse or promote products derived from 19 | # this software without specific prior written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | # 32 | 33 | from collections import OrderedDict 34 | import unittest 35 | import sys 36 | 37 | 38 | # Support for test_xnd.py. 39 | 40 | 41 | # OrderedDict literals hack. 42 | class Record(OrderedDict): 43 | @staticmethod 44 | def _kv(s): 45 | if not isinstance(s, slice): 46 | raise TypeError("expect key-value pair") 47 | if s.step is not None: 48 | raise ValueError("expect key-value pair") 49 | return s.start, s.stop 50 | def __getitem__(self, items): 51 | if not isinstance(items, tuple): 52 | items = (items,) 53 | return OrderedDict(list(map(self._kv, items))) 54 | 55 | R = Record() 56 | 57 | 58 | # Broken classes. 59 | class BoolMemoryError(object): 60 | def __bool__(self): 61 | raise MemoryError 62 | 63 | class Index(object): 64 | def __index__(self): 65 | return 10 66 | 67 | class IndexMemoryError(object): 68 | def __index__(self): 69 | raise MemoryError 70 | 71 | class IndexTypeError(object): 72 | def __index__(self): 73 | return "" 74 | 75 | def assertEqualWithEx(self, func, x, y): 76 | if x.value is y is None: 77 | return 78 | 79 | xerr = None 80 | try: 81 | xres = func(x) 82 | except Exception as e: 83 | xerr = e.__class__ 84 | 85 | yerr = None 86 | try: 87 | yres = func(y) 88 | except Exception as e: 89 | yerr = e.__class__ 90 | 91 | if xerr is None is yerr: 92 | self.assertEqual(xres, yres, msg="x: %s y: %s" % (x.type, y)) 93 | elif xerr is TypeError and yerr is None and isinstance(y, tuple) and len(y) == 2: 94 | # Tagged unions like ('Int', 0) have len() in the Python representation 95 | # but are scalars in xnd. 96 | pass 97 | else: 98 | self.assertEqual(xerr, yerr, msg="x: %s y: %s" % (x.type, y)) 99 | 100 | def skip_if(condition, reason): 101 | if condition: 102 | raise unittest.SkipTest(reason) 103 | 104 | HAVE_64_BIT = sys.maxsize == 2**63-1 105 | 106 | HAVE_PYTHON_36 = sys.version_info >= (3, 6, 0) 107 | 108 | requires_py36 = unittest.skipUnless( 109 | sys.version_info > (3, 6), 110 | "test requires Python 3.6 or greater") 111 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # 2 | # BSD 3-Clause License 3 | # 4 | # Copyright (c) 2017-2018, plures 5 | # All rights reserved. 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted provided that the following conditions are met: 9 | # 10 | # 1. Redistributions of source code must retain the above copyright notice, 11 | # this list of conditions and the following disclaimer. 12 | # 13 | # 2. Redistributions in binary form must reproduce the above copyright notice, 14 | # this list of conditions and the following disclaimer in the documentation 15 | # and/or other materials provided with the distribution. 16 | # 17 | # 3. Neither the name of the copyright holder nor the names of its 18 | # contributors may be used to endorse or promote products derived from 19 | # this software without specific prior written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | # 32 | 33 | import sys, os 34 | 35 | if "bdist_wheel" in sys.argv: 36 | from setuptools import setup, Extension 37 | else: 38 | from distutils.core import setup, Extension 39 | 40 | from distutils.sysconfig import get_python_lib 41 | from glob import glob 42 | import platform 43 | import subprocess 44 | import shutil 45 | import argparse 46 | import warnings 47 | 48 | try: 49 | import sphinx.cmd 50 | SPHINX_BUILD = "sphinx.cmd.build" 51 | except: 52 | SPHINX_BUILD = "sphinx" 53 | 54 | 55 | DESCRIPTION = """\ 56 | General container that maps a wide range of Python values directly to memory.\ 57 | """ 58 | 59 | LONG_DESCRIPTION = """ 60 | Overview 61 | -------- 62 | 63 | The ``xnd`` module implements a container type that maps most Python values 64 | relevant for scientific computing directly to typed memory. 65 | 66 | Whenever possible, a single, pointer-free memory block is used. 67 | 68 | ``xnd`` supports ragged arrays, categorical types, indexing, slicing, aligned 69 | memory blocks and type inference. 70 | 71 | Operations like indexing and slicing return zero-copy typed views on the data. 72 | 73 | Importing PEP-3118 buffers is supported. 74 | 75 | Links 76 | ----- 77 | 78 | * https://github.com/plures/ 79 | * http://ndtypes.readthedocs.io/en/latest/ 80 | * http://xnd.readthedocs.io/en/latest/ 81 | 82 | """ 83 | 84 | warnings.simplefilter("ignore", UserWarning) 85 | 86 | 87 | # Pre-parse and remove the '-j' argument from sys.argv. 88 | parser = argparse.ArgumentParser() 89 | parser.add_argument('-j', default=None) 90 | values, rest = parser.parse_known_args() 91 | PARALLEL = values.j 92 | sys.argv = sys.argv[:1] + rest 93 | 94 | 95 | if sys.platform == "darwin": 96 | LIBNAME = "libxnd.dylib" 97 | LIBSONAME = "libxnd.0.dylib" 98 | LIBSHARED = "libxnd.0.2.0dev3.dylib" 99 | else: 100 | LIBNAME = "libxnd.so" 101 | LIBSONAME = "libxnd.so.0" 102 | LIBSHARED = "libxnd.so.0.2.0dev3" 103 | LIBNDTYPES = "libndtypes.so.0.2.0dev3" 104 | 105 | if "install" in sys.argv or "bdist_wheel" in sys.argv: 106 | CONFIGURE_INCLUDES = "%s/ndtypes" % get_python_lib() 107 | CONFIGURE_LIBS = CONFIGURE_INCLUDES 108 | INCLUDES = LIBS = [CONFIGURE_INCLUDES] 109 | LIBXNDDIR = "%s/xnd" % get_python_lib() 110 | INSTALL_LIBS = True 111 | elif "conda_install" in sys.argv: 112 | site = "%s/ndtypes" % get_python_lib() 113 | sys_includes = os.path.join(os.environ['PREFIX'], "include") 114 | libdir = "Library/bin" if sys.platform == "win32" else "lib" 115 | sys_libs = os.path.join(os.environ['PREFIX'], libdir) 116 | CONFIGURE_INCLUDES = INCLUDES = [sys_includes, site] 117 | LIBS = [sys_libs, site] 118 | LIBXNDDIR = "%s/xnd" % get_python_lib() 119 | INSTALL_LIBS = False 120 | else: 121 | CONFIGURE_INCLUDES = "../python/ndtypes" 122 | CONFIGURE_LIBS = CONFIGURE_INCLUDES 123 | INCLUDES = LIBS = [CONFIGURE_INCLUDES] 124 | LIBXNDDIR = "../python/xnd" 125 | INSTALL_LIBS = False 126 | 127 | PY_MAJOR = sys.version_info[0] 128 | PY_MINOR = sys.version_info[1] 129 | ARCH = platform.architecture()[0] 130 | BUILD_ALL = \ 131 | "build" in sys.argv or "install" in sys.argv or "bdist_wheel" in sys.argv 132 | 133 | 134 | if PY_MAJOR < 3: 135 | raise NotImplementedError( 136 | "python2 support is not implemented") 137 | 138 | 139 | def get_module_path(): 140 | pathlist = glob("build/lib.*/") 141 | if pathlist: 142 | return pathlist[0] 143 | raise RuntimeError("cannot find xnd module in build directory") 144 | 145 | def copy_ext(): 146 | if sys.platform == "win32": 147 | pathlist = glob("build/lib.*/xnd/_xnd.*.pyd") 148 | else: 149 | pathlist = glob("build/lib.*/xnd/_xnd.*.so") 150 | if pathlist: 151 | shutil.copy2(pathlist[0], "python/xnd") 152 | 153 | def make_symlinks(): 154 | os.chdir(LIBXNDDIR) 155 | os.chmod(LIBSHARED, 0o755) 156 | os.system("ln -sf %s %s" % (LIBSHARED, LIBSONAME)) 157 | os.system("ln -sf %s %s" % (LIBSHARED, LIBNAME)) 158 | 159 | 160 | if len(sys.argv) == 3 and sys.argv[1] == "install" and \ 161 | sys.argv[2].startswith("--local"): 162 | localdir = sys.argv[2].split("=")[1] 163 | sys.argv = sys.argv[:2] + [ 164 | "--install-base=" + localdir, 165 | "--install-purelib=" + localdir, 166 | "--install-platlib=" + localdir, 167 | "--install-scripts=" + localdir, 168 | "--install-data=" + localdir, 169 | "--install-headers=" + localdir] 170 | 171 | LIBNDTYPESDIR = "%s/ndtypes" % localdir 172 | CONFIGURE_INCLUDES = "%s/ndtypes" % localdir 173 | CONFIGURE_LIBS = CONFIGURE_INCLUDES 174 | INCLUDES = LIBS = [CONFIGURE_INCLUDES] 175 | LIBXNDDIR = "%s/xnd" % localdir 176 | INSTALL_LIBS = True 177 | 178 | if sys.platform == "darwin": # homebrew bug 179 | sys.argv.append("--prefix=") 180 | 181 | elif len(sys.argv) == 2: 182 | if sys.argv[1] == 'module': 183 | sys.argv[1] = 'build' 184 | if sys.argv[1] == 'module_install' or sys.argv[1] == 'conda_install': 185 | sys.argv[1] = 'install' 186 | if sys.argv[1] == 'test': 187 | module_path = get_module_path() 188 | python_path = os.getenv('PYTHONPATH') 189 | path = module_path + ':' + python_path if python_path else module_path 190 | env = os.environ.copy() 191 | env['PYTHONPATH'] = path 192 | ret = subprocess.call([sys.executable, "python/test_xnd.py", "--long"], env=env) 193 | sys.exit(ret) 194 | elif sys.argv[1] == 'doctest': 195 | os.chdir("doc") 196 | module_path = '../python' 197 | python_path = os.getenv('PYTHONPATH') 198 | path = module_path + ':' + python_path if python_path else module_path 199 | env = os.environ.copy() 200 | env['PYTHONPATH'] = path 201 | cmd = [sys.executable, "-m", SPHINX_BUILD, 202 | "-b", "doctest", "-d", "build/doctrees", ".", "build/html"] 203 | ret = subprocess.call(cmd, env=env) 204 | sys.exit(ret) 205 | elif sys.argv[1] == 'clean': 206 | shutil.rmtree("build", ignore_errors=True) 207 | os.chdir("python/xnd") 208 | shutil.rmtree("__pycache__", ignore_errors=True) 209 | for f in glob("_xnd*.so"): 210 | os.remove(f) 211 | sys.exit(0) 212 | elif sys.argv[1] == 'distclean': 213 | if sys.platform == "win32": 214 | os.chdir("vcbuild") 215 | os.system("vcdistclean.bat") 216 | else: 217 | os.system("make distclean") 218 | sys.exit(0) 219 | else: 220 | pass 221 | 222 | def configure(configure_includes, configure_libs): 223 | os.chmod("./configure", 0x1ed) # pip removes execute permissions. 224 | make = "make -j%d" % int(PARALLEL) if PARALLEL else "make" 225 | os.system("./configure --with-includes='%s' --with-libs='%s' && %s" % 226 | (configure_includes, configure_libs, make)) 227 | 228 | def get_config_vars(): 229 | f = open("config.h") 230 | config_vars = {} 231 | for line in f: 232 | if line.startswith("#define"): 233 | l = line.split() 234 | try: 235 | config_vars[l[1]] = int(l[2]) 236 | except ValueError: 237 | pass 238 | elif line.startswith("/* #undef"): 239 | l = line.split() 240 | config_vars[l[2]] = 0 241 | f.close() 242 | return config_vars 243 | 244 | def xnd_ext(): 245 | include_dirs = ["libxnd", "ndtypes/python/ndtypes"] + INCLUDES 246 | library_dirs = ["libxnd", "ndtypes/libndtypes"] + LIBS 247 | depends = ["libxnd/xnd.h", "python/xnd/util.h", "python/xnd/pyxnd.h"] 248 | sources = ["python/xnd/_xnd.c"] 249 | 250 | if sys.platform == "win32": 251 | libraries = ["libndtypes-0.2.0dev3.dll", "libxnd-0.2.0dev3.dll"] 252 | extra_compile_args = ["/DXND_IMPORT"] 253 | extra_link_args = [] 254 | runtime_library_dirs = [] 255 | 256 | if BUILD_ALL: 257 | from distutils.msvc9compiler import MSVCCompiler 258 | MSVCCompiler().initialize() 259 | os.chdir("vcbuild") 260 | os.environ['LIBNDTYPESINCLUDE'] = os.path.normpath(CONFIGURE_INCLUDES) 261 | os.environ['LIBNDTYPESDIR'] = os.path.normpath(CONFIGURE_LIBS) 262 | if ARCH == "64bit": 263 | os.system("vcbuild64.bat") 264 | else: 265 | os.system("vcbuild32.bat") 266 | os.chdir("..") 267 | 268 | else: 269 | if BUILD_ALL: 270 | configure(CONFIGURE_INCLUDES, CONFIGURE_LIBS) 271 | 272 | config_vars = get_config_vars() 273 | 274 | extra_compile_args = ["-Wextra", "-Wno-missing-field-initializers", "-std=c11"] 275 | if sys.platform == "darwin": 276 | libraries = ["ndtypes", "xnd"] 277 | extra_link_args = ["-Wl,-rpath,@loader_path"] 278 | runtime_library_dirs = [] 279 | else: 280 | libraries = [":%s" % LIBNDTYPES, ":%s" % LIBSHARED] 281 | extra_link_args = [] 282 | runtime_library_dirs = ["$ORIGIN"] 283 | 284 | if config_vars["HAVE_CUDA"]: 285 | libraries += ["cudart"] 286 | 287 | for d in [ 288 | "/usr/cuda/lib", 289 | "/usr/cuda/lib64", 290 | "/usr/local/cuda/lib/", 291 | "/usr/local/cuda/lib64"]: 292 | if os.path.isdir(d): 293 | library_dirs.append(d) 294 | 295 | return Extension ( 296 | "xnd._xnd", 297 | include_dirs = include_dirs, 298 | library_dirs = library_dirs, 299 | depends = depends, 300 | sources = sources, 301 | libraries = libraries, 302 | extra_compile_args = extra_compile_args, 303 | extra_link_args = extra_link_args, 304 | runtime_library_dirs = runtime_library_dirs 305 | ) 306 | 307 | setup ( 308 | name = "xnd", 309 | version = "0.2.0dev3", 310 | description = DESCRIPTION, 311 | long_description = LONG_DESCRIPTION, 312 | url = "https://github.com/plures/xnd", 313 | author = 'Stefan Krah', 314 | author_email = 'skrah@bytereef.org', 315 | license = "BSD License", 316 | keywords = ["xnd", "array computing", "container", "memory blocks"], 317 | platforms = ["Many"], 318 | classifiers = [ 319 | "Development Status :: 4 - Beta", 320 | "Intended Audience :: Developers", 321 | "Intended Audience :: Education", 322 | "Intended Audience :: End Users/Desktop", 323 | "Intended Audience :: Financial and Insurance Industry", 324 | "Intended Audience :: Science/Research", 325 | "License :: OSI Approved :: BSD License", 326 | "Programming Language :: C", 327 | "Programming Language :: Python :: 3", 328 | "Programming Language :: Python :: 3.6", 329 | "Programming Language :: Python :: 3.7", 330 | "Operating System :: OS Independent", 331 | "Topic :: Scientific/Engineering :: Mathematics", 332 | "Topic :: Software Development" 333 | ], 334 | install_requires = ["ndtypes == v0.2.0dev3"], 335 | package_dir = {"": "python"}, 336 | packages = ["xnd"], 337 | package_data = {"xnd": ["libxnd*", "xnd.h", "pyxnd.h", "contrib/*"] 338 | if INSTALL_LIBS else ["pyxnd.h", "contrib/*"]}, 339 | ext_modules = [xnd_ext()], 340 | ) 341 | 342 | copy_ext() 343 | 344 | if INSTALL_LIBS and sys.platform != "win32" and not "bdist_wheel" in sys.argv: 345 | make_symlinks() 346 | -------------------------------------------------------------------------------- /vcbuild/INSTALL.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | libxnd build instructions for Visual Studio 4 | =========================================== 5 | 6 | 7 | Requirements 8 | ------------ 9 | 10 | - Visual Studio 2015 or later. 11 | 12 | 13 | 64-bit build 14 | ------------ 15 | 16 | # Set the build environment 17 | vcvarsall.bat x64 18 | 19 | # Build: If successful, the static library, the dynamic library, the 20 | # common header file and two executables for running the unit tests 21 | # should be in the dist64 directory. 22 | vcbuild64.bat 23 | 24 | # Test 25 | runtest64.bat 26 | 27 | 28 | 32-bit build 29 | ------------ 30 | 31 | # Set the build environment 32 | vcvarsall.bat x86 33 | 34 | # Build: If successful, the static library, the dynamic library, the 35 | # common header file and two executables for running the unit tests 36 | # should be in the dist32 directory. 37 | vcbuild32.bat 38 | 39 | # Test 40 | runtest32.bat 41 | 42 | 43 | -------------------------------------------------------------------------------- /vcbuild/runtest32.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | echo. 3 |