├── docs ├── _static │ ├── placeholder.txt │ └── custom.css ├── _templates │ └── placeholder.txt ├── changes.rst ├── requirements.txt ├── index.rst ├── conf.py ├── api.rst ├── make.bat └── Makefile ├── COPYRIGHT.txt ├── src └── BTrees │ ├── tests │ ├── __init__.py │ ├── test_compile_flags.py │ ├── testPersistency.py │ ├── test_dynamic_btrees.py │ ├── test_utils.py │ ├── test_fsBTree.py │ ├── test_Length.py │ ├── test_btreesubclass.py │ ├── test__datatypes.py │ ├── test_OOBTree.py │ ├── test_check.py │ └── _test_builder.py │ ├── SetOpTemplate.h │ ├── objectvaluemacros.h │ ├── floatvaluemacros.h │ ├── _IOBTree.c │ ├── _OIBTree.c │ ├── _OOBTree.c │ ├── _IIBTree.c │ ├── _IFBTree.c │ ├── _UOBTree.c │ ├── _OUBTree.c │ ├── _UIBTree.c │ ├── _IUBTree.c │ ├── _LOBTree.c │ ├── _OLBTree.c │ ├── _UFBTree.c │ ├── _LLBTree.c │ ├── _LFBTree.c │ ├── _OQBTree.c │ ├── _QOBTree.c │ ├── _UUBTree.c │ ├── _LQBTree.c │ ├── _QLBTree.c │ ├── _QFBTree.c │ ├── _QQBTree.c │ ├── objectkeymacros.h │ ├── utils.py │ ├── Length.py │ ├── _compat.py │ ├── __init__.py │ ├── intkeymacros.h │ ├── intvaluemacros.h │ ├── _fsBTree.c │ ├── _module_builder.py │ ├── MergeTemplate.c │ ├── _datatypes.py │ └── sorters.c ├── .gitignore ├── .manylinux.sh ├── MANIFEST.in ├── setup.cfg ├── .readthedocs.yaml ├── .pre-commit-config.yaml ├── CONTRIBUTING.md ├── .github └── workflows │ └── pre-commit.yml ├── .editorconfig ├── README.rst ├── include └── persistent │ └── persistent │ ├── _compat.h │ ├── ring.h │ └── cPersistence.h ├── LICENSE.txt ├── tox.ini ├── pyproject.toml ├── .meta.toml ├── .manylinux-install.sh └── setup.py /docs/_static/placeholder.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/_templates/placeholder.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /COPYRIGHT.txt: -------------------------------------------------------------------------------- 1 | Zope Foundation and Contributors -------------------------------------------------------------------------------- /docs/changes.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGES.rst 2 | -------------------------------------------------------------------------------- /src/BTrees/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Make this a package. 2 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | Sphinx 2 | repoze.sphinx.autointerface 3 | furo 4 | -------------------------------------------------------------------------------- /src/BTrees/SetOpTemplate.h: -------------------------------------------------------------------------------- 1 | # ifndef SETOPTEMPLATE_H 2 | # define SETOPTEMPLATE_H 3 | 4 | #include "Python.h" 5 | 6 | static PyObject * 7 | union_m(PyObject *ignored, PyObject *args); 8 | 9 | static PyObject * 10 | intersection_m(PyObject *ignored, PyObject *args); 11 | 12 | static PyObject * 13 | difference_m(PyObject *ignored, PyObject *args); 14 | 15 | # endif 16 | -------------------------------------------------------------------------------- /docs/_static/custom.css: -------------------------------------------------------------------------------- 1 | /* 2 | sphinx_rtd_theme pulls from wyrm, which applies white-space: nowrap to tables. 3 | https://github.com/snide/wyrm/blob/fd41b56978f009e8c33cb26f384dd0dfaf430a7d/sass/wyrm_core/_table.sass#L144 4 | That makes it hard to have a table with more than three columns without 5 | pointless scrolling. 6 | */ 7 | .wrapped td, .wrapped th { 8 | white-space: normal !important; 9 | } 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated with zope.meta (https://zopemeta.readthedocs.io/) from: 2 | # https://github.com/zopefoundation/meta/tree/master/src/zope/meta/c-code 3 | *.dll 4 | *.egg-info/ 5 | *.profraw 6 | *.pyc 7 | *.pyo 8 | *.so 9 | .coverage 10 | .coverage.* 11 | .eggs/ 12 | .installed.cfg 13 | .mr.developer.cfg 14 | .tox/ 15 | .vscode/ 16 | __pycache__/ 17 | bin/ 18 | build/ 19 | coverage.xml 20 | develop-eggs/ 21 | develop/ 22 | dist/ 23 | docs/_build 24 | eggs/ 25 | etc/ 26 | lib/ 27 | lib64 28 | log/ 29 | parts/ 30 | pyvenv.cfg 31 | testing.log 32 | var/ 33 | -------------------------------------------------------------------------------- /.manylinux.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Generated with zope.meta (https://zopemeta.readthedocs.io/) from: 3 | # https://github.com/zopefoundation/meta/tree/master/src/zope/meta/c-code 4 | 5 | set -e -x 6 | 7 | # Mount the current directory as /io 8 | # Mount the pip cache directory as /cache 9 | # `pip cache` requires pip 20.1 10 | echo Setting up caching 11 | python --version 12 | python -mpip --version 13 | LCACHE="$(dirname `python -mpip cache dir`)" 14 | echo Sharing pip cache at $LCACHE $(ls -ld $LCACHE) 15 | 16 | docker run --rm -e GITHUB_ACTIONS -v "$(pwd)":/io -v "$LCACHE:/cache" $DOCKER_IMAGE $PRE_CMD /io/.manylinux-install.sh 17 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ====================== 2 | BTrees Documentation 3 | ====================== 4 | 5 | This package contains a set of persistent object containers built around 6 | a modified BTree data structure. The trees are optimized for use inside 7 | ZODB's "optimistic concurrency" paradigm, and include explicit resolution 8 | of conflicts detected by that mechanism. 9 | 10 | Contents: 11 | 12 | .. toctree:: 13 | :maxdepth: 2 14 | 15 | overview 16 | api 17 | development 18 | changes 19 | 20 | ==================== 21 | Indices and tables 22 | ==================== 23 | 24 | * :ref:`genindex` 25 | * :ref:`modindex` 26 | * :ref:`search` 27 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # Generated with zope.meta (https://zopemeta.readthedocs.io/) from: 2 | # https://github.com/zopefoundation/meta/tree/master/src/zope/meta/c-code 3 | include *.md 4 | include *.rst 5 | include *.txt 6 | include buildout.cfg 7 | include tox.ini 8 | include .pre-commit-config.yaml 9 | 10 | recursive-include docs *.py 11 | recursive-include docs *.rst 12 | recursive-include docs *.txt 13 | recursive-include docs Makefile 14 | 15 | recursive-include src *.py 16 | include *.yaml 17 | include *.sh 18 | recursive-include docs *.bat 19 | recursive-include docs *.css 20 | recursive-include include/persistent *.h 21 | recursive-include src *.c 22 | recursive-include src *.h 23 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | # Generated with zope.meta (https://zopemeta.readthedocs.io/) from: 2 | # https://github.com/zopefoundation/meta/tree/master/src/zope/meta/c-code 3 | 4 | [zest.releaser] 5 | create-wheel = no 6 | 7 | [flake8] 8 | doctests = 1 9 | per-file-ignores = 10 | src/BTrees/check.py: F401 11 | 12 | [check-manifest] 13 | ignore = 14 | .editorconfig 15 | .meta.toml 16 | docs/_build/html/_sources/* 17 | docs/_build/doctest/* 18 | docs/_build/html/_static/* 19 | docs/_build/html/_static/*/* 20 | 21 | [isort] 22 | force_single_line = True 23 | combine_as_imports = True 24 | sections = FUTURE,STDLIB,THIRDPARTY,ZOPE,FIRSTPARTY,LOCALFOLDER 25 | known_third_party = docutils, pkg_resources, pytz 26 | known_zope = 27 | known_first_party = 28 | default_section = ZOPE 29 | line_length = 79 30 | lines_after_imports = 2 31 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Generated with zope.meta (https://zopemeta.readthedocs.io/) from: 2 | # https://github.com/zopefoundation/meta/tree/master/src/zope/meta/c-code 3 | # Read the Docs configuration file 4 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 5 | 6 | # Required 7 | version: 2 8 | 9 | # Set the version of Python and other tools you might need 10 | build: 11 | os: ubuntu-22.04 12 | tools: 13 | python: "3.11" 14 | 15 | # Build documentation in the docs/ directory with Sphinx 16 | sphinx: 17 | configuration: docs/conf.py 18 | 19 | # We recommend specifying your dependencies to enable reproducible builds: 20 | # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 21 | python: 22 | install: 23 | - requirements: docs/requirements.txt 24 | - method: pip 25 | path: . 26 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # Generated with zope.meta (https://zopemeta.readthedocs.io/) from: 2 | # https://github.com/zopefoundation/meta/tree/master/src/zope/meta/c-code 3 | minimum_pre_commit_version: '3.6' 4 | repos: 5 | - repo: https://github.com/pycqa/isort 6 | rev: "7.0.0" 7 | hooks: 8 | - id: isort 9 | - repo: https://github.com/hhatto/autopep8 10 | rev: "v2.3.2" 11 | hooks: 12 | - id: autopep8 13 | args: [--in-place, --aggressive, --aggressive] 14 | - repo: https://github.com/asottile/pyupgrade 15 | rev: v3.21.0 16 | hooks: 17 | - id: pyupgrade 18 | args: [--py310-plus] 19 | - repo: https://github.com/isidentical/teyit 20 | rev: 0.4.3 21 | hooks: 22 | - id: teyit 23 | - repo: https://github.com/PyCQA/flake8 24 | rev: "7.3.0" 25 | hooks: 26 | - id: flake8 27 | additional_dependencies: 28 | - flake8-debugger == 4.1.2 29 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 5 | # Contributing to zopefoundation projects 6 | 7 | The projects under the zopefoundation GitHub organization are open source and 8 | welcome contributions in different forms: 9 | 10 | * bug reports 11 | * code improvements and bug fixes 12 | * documentation improvements 13 | * pull request reviews 14 | 15 | For any changes in the repository besides trivial typo fixes you are required 16 | to sign the contributor agreement. See 17 | https://www.zope.dev/developer/becoming-a-committer.html for details. 18 | 19 | Please visit our [Developer 20 | Guidelines](https://www.zope.dev/developer/guidelines.html) if you'd like to 21 | contribute code changes and our [guidelines for reporting 22 | bugs](https://www.zope.dev/developer/reporting-bugs.html) if you want to file a 23 | bug report. 24 | -------------------------------------------------------------------------------- /src/BTrees/objectvaluemacros.h: -------------------------------------------------------------------------------- 1 | #define VALUEMACROS_H "$Id$\n" 2 | 3 | /* Note that the second comparison is skipped if the first comparison returns: 4 | 5 | 1 -> There was no error and the answer is -1 6 | -1 -> There was an error, which the caller will detect with PyError_Occurred. 7 | */ 8 | #define COMPARE(lhs, rhs) \ 9 | (lhs == Py_None ? (rhs == Py_None ? 0 : -1) : (rhs == Py_None ? 1 : \ 10 | (PyObject_RichCompareBool((lhs), (rhs), Py_LT) != 0 ? -1 : \ 11 | (PyObject_RichCompareBool((lhs), (rhs), Py_EQ) > 0 ? 0 : 1)))) 12 | 13 | #define VALUE_TYPE PyObject * 14 | #define VALUE_TYPE_IS_PYOBJECT 15 | #define TEST_VALUE(VALUE, TARGET) (COMPARE((VALUE),(TARGET))) 16 | #define DECLARE_VALUE(NAME) VALUE_TYPE NAME 17 | #define INCREF_VALUE(k) Py_INCREF(k) 18 | #define DECREF_VALUE(k) Py_DECREF(k) 19 | #define COPY_VALUE(k,e) k=(e) 20 | #define COPY_VALUE_TO_OBJECT(O, K) O=(K); Py_INCREF(O) 21 | #define COPY_VALUE_FROM_ARG(TARGET, ARG, S) TARGET=(ARG) 22 | #define NORMALIZE_VALUE(V, MIN) Py_INCREF(V) 23 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yml: -------------------------------------------------------------------------------- 1 | # Generated with zope.meta (https://zopemeta.readthedocs.io/) from: 2 | # https://github.com/zopefoundation/meta/tree/master/src/zope/meta/c-code 3 | name: pre-commit 4 | 5 | on: 6 | pull_request: 7 | push: 8 | branches: 9 | - master 10 | # Allow to run this workflow manually from the Actions tab 11 | workflow_dispatch: 12 | 13 | env: 14 | FORCE_COLOR: 1 15 | 16 | jobs: 17 | pre-commit: 18 | permissions: 19 | contents: read 20 | pull-requests: write 21 | name: linting 22 | runs-on: ubuntu-latest 23 | steps: 24 | - uses: actions/checkout@v6 25 | - uses: actions/setup-python@v6 26 | with: 27 | python-version: '3.13' 28 | - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd #v3.0.1 29 | with: 30 | extra_args: --all-files --show-diff-on-failure 31 | env: 32 | PRE_COMMIT_COLOR: always 33 | - uses: pre-commit-ci/lite-action@5d6cc0eb514c891a40562a58a8e71576c5c7fb43 #v1.1.0 34 | if: always() 35 | with: 36 | msg: Apply pre-commit code formatting 37 | -------------------------------------------------------------------------------- /src/BTrees/floatvaluemacros.h: -------------------------------------------------------------------------------- 1 | 2 | #define VALUEMACROS_H "$Id$\n" 3 | 4 | #define VALUE_TYPE float 5 | #undef VALUE_TYPE_IS_PYOBJECT 6 | #define TEST_VALUE(K, T) (((K) < (T)) ? -1 : (((K) > (T)) ? 1: 0)) 7 | #define VALUE_SAME(VALUE, TARGET) ( (VALUE) == (TARGET) ) 8 | #define DECLARE_VALUE(NAME) VALUE_TYPE NAME 9 | #define VALUE_PARSE "f" 10 | #define DECREF_VALUE(k) 11 | #define INCREF_VALUE(k) 12 | #define COPY_VALUE(V, E) (V=(E)) 13 | #define COPY_VALUE_TO_OBJECT(O, K) O=PyFloat_FromDouble(K) 14 | 15 | #define COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS) \ 16 | if (PyFloat_Check(ARG)) TARGET = (float)PyFloat_AsDouble(ARG); \ 17 | else if (PyLong_Check(ARG)) TARGET = (float)PyLong_AsLong(ARG); \ 18 | else { \ 19 | PyErr_SetString(PyExc_TypeError, "expected float or int value"); \ 20 | (STATUS)=0; (TARGET)=0; } 21 | 22 | #define NORMALIZE_VALUE(V, MIN) ((MIN) > 0) ? ((V)/=(MIN)) : 0 23 | 24 | #define MERGE_DEFAULT 1.0f 25 | #define MERGE(O1, w1, O2, w2) ((O1)*(w1)+(O2)*(w2)) 26 | #define MERGE_WEIGHT(O, w) ((O)*(w)) 27 | -------------------------------------------------------------------------------- /src/BTrees/_IOBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id$\n" 16 | 17 | /* IOBTree - int key, object value BTree 18 | 19 | Implements a collection using int type keys 20 | and object type values 21 | */ 22 | 23 | #define PERSISTENT 24 | 25 | #define MOD_NAME_PREFIX "IO" 26 | 27 | 28 | 29 | 30 | #include "Python.h" 31 | #include "intkeymacros.h" 32 | #include "objectvaluemacros.h" 33 | 34 | #define INITMODULE PyInit__IOBTree 35 | #include "BTreeModuleTemplate.c" 36 | -------------------------------------------------------------------------------- /src/BTrees/_OIBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id$\n" 16 | 17 | /* OIBTree - object key, int value BTree 18 | 19 | Implements a collection using object type keys 20 | and int type values 21 | */ 22 | 23 | #define PERSISTENT 24 | 25 | #define MOD_NAME_PREFIX "OI" 26 | 27 | 28 | 29 | 30 | #include "Python.h" 31 | #include "objectkeymacros.h" 32 | #include "intvaluemacros.h" 33 | 34 | #define INITMODULE PyInit__OIBTree 35 | #include "BTreeModuleTemplate.c" 36 | -------------------------------------------------------------------------------- /src/BTrees/_OOBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id$\n" 16 | 17 | /* OOBTree - object key, object value BTree 18 | 19 | Implements a collection using object type keys 20 | and object type values 21 | */ 22 | 23 | #define PERSISTENT 24 | 25 | #define MOD_NAME_PREFIX "OO" 26 | 27 | 28 | 29 | 30 | #include "Python.h" 31 | #include "objectkeymacros.h" 32 | #include "objectvaluemacros.h" 33 | 34 | #define INITMODULE PyInit__OOBTree 35 | #include "BTreeModuleTemplate.c" 36 | -------------------------------------------------------------------------------- /src/BTrees/_IIBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id$\n" 16 | 17 | /* IIBTree - int key, int value BTree 18 | 19 | Implements a collection using int type keys 20 | and int type values 21 | */ 22 | 23 | /* Setup template macros */ 24 | 25 | #define PERSISTENT 26 | 27 | #define MOD_NAME_PREFIX "II" 28 | 29 | 30 | 31 | 32 | #include "Python.h" 33 | #include "intkeymacros.h" 34 | #include "intvaluemacros.h" 35 | 36 | #define INITMODULE PyInit__IIBTree 37 | #include "BTreeModuleTemplate.c" 38 | -------------------------------------------------------------------------------- /src/BTrees/_IFBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id$\n" 16 | 17 | /* IFBTree - int key, float value BTree 18 | 19 | Implements a collection using int type keys 20 | and float type values 21 | */ 22 | 23 | /* Setup template macros */ 24 | 25 | #define PERSISTENT 26 | 27 | #define MOD_NAME_PREFIX "IF" 28 | 29 | 30 | 31 | 32 | #include "Python.h" 33 | #include "intkeymacros.h" 34 | #include "floatvaluemacros.h" 35 | 36 | #define INITMODULE PyInit__IFBTree 37 | #include "BTreeModuleTemplate.c" 38 | -------------------------------------------------------------------------------- /src/BTrees/_UOBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id$\n" 16 | 17 | /* UOBTree - unsigned int key, object value BTree 18 | 19 | Implements a collection using unsigned int type keys 20 | and object type values 21 | */ 22 | 23 | #define PERSISTENT 24 | 25 | #define MOD_NAME_PREFIX "UO" 26 | 27 | 28 | 29 | #define ZODB_UNSIGNED_KEY_INTS 30 | #include "Python.h" 31 | #include "intkeymacros.h" 32 | #include "objectvaluemacros.h" 33 | 34 | #define INITMODULE PyInit__UOBTree 35 | #include "BTreeModuleTemplate.c" 36 | -------------------------------------------------------------------------------- /src/BTrees/_OUBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id$\n" 16 | 17 | /* OUBTree - object key, uint32_t value BTree 18 | 19 | Implements a collection using object type keys 20 | and int type values 21 | */ 22 | 23 | #define PERSISTENT 24 | 25 | #define MOD_NAME_PREFIX "OU" 26 | 27 | 28 | 29 | 30 | #define ZODB_UNSIGNED_VALUE_INTS 31 | 32 | #include "Python.h" 33 | #include "objectkeymacros.h" 34 | #include "intvaluemacros.h" 35 | 36 | #define INITMODULE PyInit__OUBTree 37 | #include "BTreeModuleTemplate.c" 38 | -------------------------------------------------------------------------------- /src/BTrees/_UIBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id$\n" 16 | 17 | /* IIBTree - unsigned int key, int value BTree 18 | 19 | Implements a collection using int type keys 20 | and int type values 21 | */ 22 | 23 | /* Setup template macros */ 24 | 25 | #define PERSISTENT 26 | 27 | #define MOD_NAME_PREFIX "UI" 28 | 29 | 30 | 31 | #define ZODB_UNSIGNED_KEY_INTS 32 | 33 | #include "Python.h" 34 | #include "intkeymacros.h" 35 | #include "intvaluemacros.h" 36 | 37 | #define INITMODULE PyInit__UIBTree 38 | #include "BTreeModuleTemplate.c" 39 | -------------------------------------------------------------------------------- /src/BTrees/_IUBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id$\n" 16 | 17 | /* IIBTree - int32_t key, uint32_t value BTree 18 | 19 | Implements a collection using int type keys 20 | and int type values 21 | */ 22 | 23 | /* Setup template macros */ 24 | 25 | #define PERSISTENT 26 | 27 | #define MOD_NAME_PREFIX "IU" 28 | 29 | 30 | 31 | 32 | #define ZODB_UNSIGNED_VALUE_INTS 33 | 34 | #include "Python.h" 35 | #include "intkeymacros.h" 36 | #include "intvaluemacros.h" 37 | 38 | #define INITMODULE PyInit__IUBTree 39 | #include "BTreeModuleTemplate.c" 40 | -------------------------------------------------------------------------------- /src/BTrees/_LOBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id: _IOBTree.c 25186 2004-06-02 15:07:33Z jim $\n" 16 | 17 | /* IOBTree - int key, object value BTree 18 | 19 | Implements a collection using int type keys 20 | and object type values 21 | */ 22 | 23 | #define PERSISTENT 24 | 25 | #define MOD_NAME_PREFIX "LO" 26 | 27 | 28 | 29 | 30 | #define ZODB_64BIT_INTS 31 | 32 | #include "Python.h" 33 | #include "intkeymacros.h" 34 | #include "objectvaluemacros.h" 35 | 36 | #define INITMODULE PyInit__LOBTree 37 | #include "BTreeModuleTemplate.c" 38 | -------------------------------------------------------------------------------- /src/BTrees/_OLBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id: _OIBTree.c 25186 2004-06-02 15:07:33Z jim $\n" 16 | 17 | /* OIBTree - object key, int value BTree 18 | 19 | Implements a collection using object type keys 20 | and int type values 21 | */ 22 | 23 | #define PERSISTENT 24 | 25 | #define MOD_NAME_PREFIX "OL" 26 | 27 | 28 | 29 | 30 | #define ZODB_64BIT_INTS 31 | 32 | #include "Python.h" 33 | #include "objectkeymacros.h" 34 | #include "intvaluemacros.h" 35 | 36 | #define INITMODULE PyInit__OLBTree 37 | #include "BTreeModuleTemplate.c" 38 | -------------------------------------------------------------------------------- /src/BTrees/_UFBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id$\n" 16 | 17 | /* IFBTree - unsigned int key, float value BTree 18 | 19 | Implements a collection using int type keys 20 | and float type values 21 | */ 22 | 23 | /* Setup template macros */ 24 | 25 | #define PERSISTENT 26 | 27 | #define MOD_NAME_PREFIX "UF" 28 | 29 | 30 | 31 | #define ZODB_UNSIGNED_KEY_INTS 32 | 33 | #include "Python.h" 34 | #include "intkeymacros.h" 35 | #include "floatvaluemacros.h" 36 | 37 | #define INITMODULE PyInit__UFBTree 38 | #include "BTreeModuleTemplate.c" 39 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # Generated with zope.meta (https://zopemeta.readthedocs.io/) from: 2 | # https://github.com/zopefoundation/meta/tree/master/src/zope/meta/c-code 3 | # 4 | # EditorConfig Configuration file, for more details see: 5 | # https://EditorConfig.org 6 | # EditorConfig is a convention description, that could be interpreted 7 | # by multiple editors to enforce common coding conventions for specific 8 | # file types 9 | 10 | # top-most EditorConfig file: 11 | # Will ignore other EditorConfig files in Home directory or upper tree level. 12 | root = true 13 | 14 | 15 | [*] 16 | # Unix-style newlines with a newline ending every file 17 | end_of_line = lf 18 | insert_final_newline = true 19 | trim_trailing_whitespace = true 20 | # Set default charset 21 | charset = utf-8 22 | # Indent style default 23 | indent_style = space 24 | # Max Line Length - a hard line wrap, should be disabled 25 | max_line_length = off 26 | 27 | [*.{py,cfg,ini}] 28 | # 4 space indentation 29 | indent_size = 4 30 | 31 | [*.{yml,zpt,pt,dtml,zcml}] 32 | # 2 space indentation 33 | indent_size = 2 34 | 35 | [{Makefile,.gitmodules}] 36 | # Tab indentation (no size specified, but view as 4 spaces) 37 | indent_style = tab 38 | indent_size = unset 39 | tab_width = unset 40 | -------------------------------------------------------------------------------- /src/BTrees/_LLBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id: _IIBTree.c 25186 2004-06-02 15:07:33Z jim $\n" 16 | 17 | /* IIBTree - int key, int value BTree 18 | 19 | Implements a collection using int type keys 20 | and int type values 21 | */ 22 | 23 | /* Setup template macros */ 24 | 25 | #define PERSISTENT 26 | 27 | #define MOD_NAME_PREFIX "LL" 28 | 29 | 30 | 31 | 32 | #define ZODB_64BIT_INTS 33 | 34 | #include "Python.h" 35 | #include "intkeymacros.h" 36 | #include "intvaluemacros.h" 37 | 38 | #define INITMODULE PyInit__LLBTree 39 | #include "BTreeModuleTemplate.c" 40 | -------------------------------------------------------------------------------- /src/BTrees/_LFBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id: _IFBTree.c 67074 2006-04-17 19:13:39Z fdrake $\n" 16 | 17 | /* IFBTree - int key, float value BTree 18 | 19 | Implements a collection using int type keys 20 | and float type values 21 | */ 22 | 23 | /* Setup template macros */ 24 | 25 | #define PERSISTENT 26 | 27 | #define MOD_NAME_PREFIX "LF" 28 | 29 | 30 | 31 | 32 | #define ZODB_64BIT_INTS 33 | 34 | #include "Python.h" 35 | #include "intkeymacros.h" 36 | #include "floatvaluemacros.h" 37 | 38 | #define INITMODULE PyInit__LFBTree 39 | #include "BTreeModuleTemplate.c" 40 | -------------------------------------------------------------------------------- /src/BTrees/_OQBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id: _OIBTree.c 25186 2004-06-02 15:07:33Z jim $\n" 16 | 17 | /* OQBTree - object key, uint64_t value BTree 18 | 19 | Implements a collection using object type keys 20 | and int type values 21 | */ 22 | 23 | #define PERSISTENT 24 | 25 | #define MOD_NAME_PREFIX "OQ" 26 | 27 | 28 | 29 | 30 | #define ZODB_64BIT_INTS 31 | #define ZODB_UNSIGNED_VALUE_INTS 32 | 33 | #include "Python.h" 34 | #include "objectkeymacros.h" 35 | #include "intvaluemacros.h" 36 | 37 | #define INITMODULE PyInit__OQBTree 38 | #include "BTreeModuleTemplate.c" 39 | -------------------------------------------------------------------------------- /src/BTrees/_QOBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id: _IOBTree.c 25186 2004-06-02 15:07:33Z jim $\n" 16 | 17 | /* QOBTree - uint64_t key, object value BTree 18 | 19 | Implements a collection using int type keys 20 | and object type values 21 | */ 22 | 23 | #define PERSISTENT 24 | 25 | #define MOD_NAME_PREFIX "QO" 26 | 27 | 28 | 29 | 30 | #define ZODB_64BIT_INTS 31 | #define ZODB_UNSIGNED_KEY_INTS 32 | 33 | #include "Python.h" 34 | #include "intkeymacros.h" 35 | #include "objectvaluemacros.h" 36 | 37 | #define INITMODULE PyInit__QOBTree 38 | #include "BTreeModuleTemplate.c" 39 | -------------------------------------------------------------------------------- /src/BTrees/_UUBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id$\n" 16 | 17 | /* IIBTree - unsigned int key, unsigned int value BTree 18 | 19 | Implements a collection using int type keys 20 | and int type values 21 | */ 22 | 23 | /* Setup template macros */ 24 | 25 | #define PERSISTENT 26 | 27 | #define MOD_NAME_PREFIX "UU" 28 | 29 | 30 | 31 | #define ZODB_UNSIGNED_KEY_INTS 32 | #define ZODB_UNSIGNED_VALUE_INTS 33 | 34 | #include "Python.h" 35 | #include "intkeymacros.h" 36 | #include "intvaluemacros.h" 37 | 38 | #define INITMODULE PyInit__UUBTree 39 | #include "BTreeModuleTemplate.c" 40 | -------------------------------------------------------------------------------- /src/BTrees/_LQBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id: _IIBTree.c 25186 2004-06-02 15:07:33Z jim $\n" 16 | 17 | /* IIBTree - int64_t key, uint64_t value BTree 18 | 19 | Implements a collection using int type keys 20 | and int type values 21 | */ 22 | 23 | /* Setup template macros */ 24 | 25 | #define PERSISTENT 26 | 27 | #define MOD_NAME_PREFIX "LQ" 28 | 29 | 30 | 31 | 32 | #define ZODB_64BIT_INTS 33 | #define ZODB_UNSIGNED_VALUE_INTS 34 | 35 | #include "Python.h" 36 | #include "intkeymacros.h" 37 | #include "intvaluemacros.h" 38 | 39 | #define INITMODULE PyInit__LQBTree 40 | #include "BTreeModuleTemplate.c" 41 | -------------------------------------------------------------------------------- /src/BTrees/_QLBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id: _IIBTree.c 25186 2004-06-02 15:07:33Z jim $\n" 16 | 17 | /* QLBTree - uint64_t key, int64_t value BTree 18 | 19 | Implements a collection using int type keys 20 | and int type values 21 | */ 22 | 23 | /* Setup template macros */ 24 | 25 | #define PERSISTENT 26 | 27 | #define MOD_NAME_PREFIX "QL" 28 | 29 | 30 | 31 | 32 | #define ZODB_64BIT_INTS 33 | #define ZODB_UNSIGNED_KEY_INTS 34 | 35 | #include "Python.h" 36 | #include "intkeymacros.h" 37 | #include "intvaluemacros.h" 38 | 39 | #define INITMODULE PyInit__QLBTree 40 | #include "BTreeModuleTemplate.c" 41 | -------------------------------------------------------------------------------- /src/BTrees/_QFBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id: _IFBTree.c 67074 2006-04-17 19:13:39Z fdrake $\n" 16 | 17 | /* QFBTree - uint64_t key, float value BTree 18 | 19 | Implements a collection using int type keys 20 | and float type values 21 | */ 22 | 23 | /* Setup template macros */ 24 | 25 | #define PERSISTENT 26 | 27 | #define MOD_NAME_PREFIX "QF" 28 | 29 | 30 | 31 | 32 | #define ZODB_64BIT_INTS 33 | #define ZODB_UNSIGNED_KEY_INTS 34 | 35 | #include "Python.h" 36 | #include "intkeymacros.h" 37 | #include "floatvaluemacros.h" 38 | 39 | #define INITMODULE PyInit__QFBTree 40 | #include "BTreeModuleTemplate.c" 41 | -------------------------------------------------------------------------------- /src/BTrees/_QQBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id: _IIBTree.c 25186 2004-06-02 15:07:33Z jim $\n" 16 | 17 | /* QQBTree - uint64_t key, uint64_t value BTree 18 | 19 | Implements a collection using int type keys 20 | and int type values 21 | */ 22 | 23 | /* Setup template macros */ 24 | 25 | #define PERSISTENT 26 | 27 | #define MOD_NAME_PREFIX "QQ" 28 | 29 | 30 | 31 | 32 | #define ZODB_64BIT_INTS 33 | #define ZODB_UNSIGNED_KEY_INTS 34 | #define ZODB_UNSIGNED_VALUE_INTS 35 | 36 | #include "Python.h" 37 | #include "intkeymacros.h" 38 | #include "intvaluemacros.h" 39 | 40 | #define INITMODULE PyInit__QQBTree 41 | #include "BTreeModuleTemplate.c" 42 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============================================= 2 | ``BTrees``: scalable persistent components 3 | ============================================= 4 | 5 | .. image:: https://github.com/zopefoundation/BTrees/actions/workflows/tests.yml/badge.svg 6 | :target: https://github.com/zopefoundation/BTrees/actions/workflows/tests.yml 7 | 8 | .. image:: https://coveralls.io/repos/github/zopefoundation/BTrees/badge.svg?branch=master 9 | :target: https://coveralls.io/github/zopefoundation/BTrees?branch=master 10 | 11 | .. image:: https://readthedocs.org/projects/btrees/badge/?version=latest 12 | :target: https://btrees.readthedocs.io/en/latest/ 13 | :alt: Documentation Status 14 | 15 | .. image:: https://img.shields.io/pypi/v/BTrees.svg 16 | :target: https://pypi.org/project/BTrees/ 17 | :alt: Current version on PyPI 18 | 19 | .. image:: https://img.shields.io/pypi/pyversions/BTrees.svg 20 | :target: https://pypi.org/project/BTrees/ 21 | :alt: Supported Python versions 22 | 23 | 24 | This package contains a set of persistent object containers built around 25 | a modified BTree data structure. The trees are optimized for use inside 26 | ZODB's "optimistic concurrency" paradigm, and include explicit resolution 27 | of conflicts detected by that mechanism. 28 | 29 | Please see `the Sphinx documentation `_ for 30 | further information. 31 | -------------------------------------------------------------------------------- /src/BTrees/tests/test_compile_flags.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2022 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | import struct 15 | import unittest 16 | 17 | from BTrees import OOBTree # noqa: try to load a C module for side effects 18 | 19 | 20 | class TestFloatingPoint(unittest.TestCase): 21 | 22 | def test_no_fast_math_optimization(self): 23 | # Building with -Ofast enables -ffast-math, which sets certain FPU 24 | # flags that can cause breakage elsewhere. A library such as BTrees 25 | # has no business changing global FPU flags for the entire process. 26 | zero_bits = struct.unpack("!Q", struct.pack("!d", 0.0))[0] 27 | next_up = zero_bits + 1 28 | smallest_subnormal = struct.unpack("!d", struct.pack("!Q", next_up))[0] 29 | self.assertNotEqual(smallest_subnormal, 0.0) 30 | -------------------------------------------------------------------------------- /include/persistent/persistent/_compat.h: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | 3 | Copyright (c) 2012 Zope Foundation and Contributors. 4 | All Rights Reserved. 5 | 6 | This software is subject to the provisions of the Zope Public License, 7 | Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | FOR A PARTICULAR PURPOSE 12 | 13 | ****************************************************************************/ 14 | 15 | #ifndef PERSISTENT__COMPAT_H 16 | #define PERSISTENT__COMPAT_H 17 | 18 | #include "Python.h" 19 | 20 | #define INTERN PyUnicode_InternFromString 21 | #define INTERN_INPLACE PyUnicode_InternInPlace 22 | #define NATIVE_CHECK_EXACT PyUnicode_CheckExact 23 | #define NATIVE_FROM_STRING_AND_SIZE PyUnicode_FromStringAndSize 24 | 25 | #define Py_TPFLAGS_HAVE_RICHCOMPARE 0 26 | 27 | #define INT_FROM_LONG(x) PyLong_FromLong(x) 28 | #define INT_CHECK(x) PyLong_Check(x) 29 | #define INT_AS_LONG(x) PyLong_AsLong(x) 30 | #define CAPI_CAPSULE_NAME "persistent.cPersistence.CAPI" 31 | 32 | #else 33 | #define INTERN PyString_InternFromString 34 | #define INTERN_INPLACE PyString_InternInPlace 35 | #define NATIVE_CHECK_EXACT PyString_CheckExact 36 | #define NATIVE_FROM_STRING_AND_SIZE PyString_FromStringAndSize 37 | 38 | #define INT_FROM_LONG(x) PyInt_FromLong(x) 39 | #define INT_CHECK(x) PyInt_Check(x) 40 | #define INT_AS_LONG(x) PyInt_AS_LONG(x) 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /src/BTrees/objectkeymacros.h: -------------------------------------------------------------------------------- 1 | #define KEYMACROS_H "$Id$\n" 2 | #define KEY_TYPE PyObject * 3 | #define KEY_TYPE_IS_PYOBJECT 4 | 5 | #include "Python.h" 6 | 7 | /* Note that the second comparison is skipped if the first comparison returns: 8 | 9 | 1 -> There was no error and the answer is -1 10 | -1 -> There was an error, which the caller will detect with PyError_Occurred. 11 | */ 12 | #define COMPARE(lhs, rhs) \ 13 | (lhs == Py_None ? (rhs == Py_None ? 0 : -1) : (rhs == Py_None ? 1 : \ 14 | (PyObject_RichCompareBool((lhs), (rhs), Py_LT) != 0 ? -1 : \ 15 | (PyObject_RichCompareBool((lhs), (rhs), Py_EQ) > 0 ? 0 : 1)))) 16 | 17 | static PyObject *object_; /* initialized in BTreeModuleTemplate init */ 18 | 19 | static int 20 | check_argument_cmp(PyObject *arg) 21 | { 22 | /* printf("check cmp %p %p %p %p\n", */ 23 | /* arg->ob_type->tp_richcompare, */ 24 | /* ((PyTypeObject *)object_)->ob_type->tp_richcompare, */ 25 | /* arg->ob_type->tp_compare, */ 26 | /* ((PyTypeObject *)object_)->ob_type->tp_compare); */ 27 | if (arg == Py_None) { 28 | return 1; 29 | } 30 | 31 | 32 | if (Py_TYPE(arg)->tp_richcompare == Py_TYPE(object_)->tp_richcompare) 33 | { 34 | PyErr_Format(PyExc_TypeError, 35 | "Object of class %s has default comparison", 36 | Py_TYPE(arg)->tp_name); 37 | return 0; 38 | } 39 | return 1; 40 | } 41 | 42 | #define TEST_KEY_SET_OR(V, KEY, TARGET) \ 43 | if ( ( (V) = COMPARE((KEY),(TARGET)) ), PyErr_Occurred() ) 44 | #define INCREF_KEY(k) Py_INCREF(k) 45 | #define DECREF_KEY(KEY) Py_DECREF(KEY) 46 | #define COPY_KEY(KEY, E) KEY=(E) 47 | #define COPY_KEY_TO_OBJECT(O, K) O=(K); Py_INCREF(O) 48 | #define COPY_KEY_FROM_ARG(TARGET, ARG, S) \ 49 | TARGET=(ARG); \ 50 | (S) = 1; 51 | #define KEY_CHECK_ON_SET check_argument_cmp 52 | -------------------------------------------------------------------------------- /src/BTrees/tests/testPersistency.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2020 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | 15 | from unittest import TestCase 16 | 17 | from ..OOBTree import OOBTree 18 | from .common import ZODBAccess 19 | from .common import _skip_wo_ZODB 20 | 21 | 22 | BUCKET_SIZE = OOBTree.max_leaf_size 23 | 24 | 25 | class TestPersistency(ZODBAccess, TestCase): 26 | @_skip_wo_ZODB 27 | def test_empty_bucket_persistency(self): 28 | from transaction import commit 29 | root = self._getRoot() 30 | try: 31 | # tree with 3 buckets (internal implementation details) 32 | tree = OOBTree( 33 | {i: i for i in range(3 * BUCKET_SIZE // 2 + 2)}) 34 | root["tree"] = tree 35 | commit() 36 | # almost clear the second bucket keeping the last element 37 | for i in range(BUCKET_SIZE // 2, BUCKET_SIZE - 1): 38 | del tree[i] 39 | commit() 40 | del tree[BUCKET_SIZE - 1] # remove the last element 41 | commit() 42 | tree._check() 43 | tree._p_deactivate() 44 | tree._check() # fails in case of bad persistency 45 | finally: 46 | self._closeRoot(root) 47 | -------------------------------------------------------------------------------- /src/BTrees/tests/test_dynamic_btrees.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2020 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | # Dynamically creates test modules and suites for expected BTree families 15 | # that do not have their own test file on disk. 16 | import importlib 17 | import sys 18 | import types 19 | import unittest 20 | 21 | from BTrees import _FAMILIES 22 | 23 | from ._test_builder import update_module 24 | 25 | 26 | # If there is no .py file on disk, create the module in memory. 27 | # This is helpful during early development. However, it 28 | # doesn't work with zope-testrunner's ``-m`` filter. 29 | _suite = unittest.TestSuite() 30 | for family in _FAMILIES: 31 | mod_qname = "BTrees.tests.test_" + family + 'BTree' 32 | try: 33 | importlib.import_module(mod_qname) 34 | except ModuleNotFoundError: 35 | btree = importlib.import_module("BTrees." + family + 'BTree') 36 | mod = types.ModuleType(mod_qname) 37 | update_module(vars(mod), btree) 38 | sys.modules[mod_qname] = mod 39 | globals()[mod_qname.split('.', 1)[1]] = mod 40 | _suite.addTest(unittest.defaultTestLoader.loadTestsFromModule(mod)) 41 | 42 | 43 | def test_suite(): 44 | # zope.testrunner protocol 45 | return _suite 46 | 47 | 48 | def load_tests(loader, standard_tests, pattern): 49 | # Pure unittest protocol. 50 | return test_suite() 51 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | 6 | import datetime 7 | 8 | 9 | # -- Project information ----------------------------------------------------- 10 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 11 | year = datetime.datetime.now().year 12 | 13 | project = 'BTrees' 14 | copyright = f'2012-{year}, Zope Foundation and contributors' 15 | author = 'Zope Foundation and contributors' 16 | 17 | # -- General configuration --------------------------------------------------- 18 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 19 | 20 | extensions = [ 21 | 'sphinx.ext.autodoc', 22 | 'sphinx.ext.doctest', 23 | 'sphinx.ext.intersphinx', 24 | 'sphinx.ext.todo', 25 | 'sphinx.ext.viewcode', 26 | 'repoze.sphinx.autointerface', 27 | ] 28 | 29 | autodoc_default_options = { 30 | 'members': None, 31 | 'show-inheritance': None, 32 | 'special-members': None, 33 | 'undoc-members': None, 34 | } 35 | autodoc_member_order = 'groupwise' 36 | autoclass_content = 'both' 37 | 38 | templates_path = ['_templates'] 39 | exclude_patterns = [] 40 | 41 | 42 | # -- Options for HTML output ------------------------------------------------- 43 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 44 | 45 | html_theme = 'furo' 46 | html_static_path = ['_static'] 47 | html_css_files = [ 48 | 'custom.css' 49 | ] 50 | 51 | 52 | # Example configuration for intersphinx: refer to the Python standard library. 53 | intersphinx_mapping = { 54 | 'python': ('https://docs.python.org/3/', None), 55 | 'persistent': ("https://persistent.readthedocs.io/en/latest/", None), 56 | 'zodb': ("https://zodb.org/en/latest/", None), 57 | 'zopeinterface': ("https://zopeinterface.readthedocs.io/en/latest/", None), 58 | } 59 | -------------------------------------------------------------------------------- /src/BTrees/utils.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001-2012 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | # Copied from ZODB/utils.py 15 | 16 | from binascii import hexlify 17 | 18 | 19 | def non_negative(int_val): 20 | if int_val < 0: 21 | # Coerce to non-negative. 22 | int_val &= 0x7FFFFFFFFFFFFFFF 23 | return int_val 24 | 25 | 26 | def positive_id(obj): # pragma: no cover 27 | """Return id(obj) as a non-negative integer.""" 28 | return non_negative(id(obj)) 29 | 30 | 31 | def oid_repr(oid): 32 | if isinstance(oid, bytes) and len(oid) == 8: 33 | # Convert to hex and strip leading zeroes. 34 | as_hex = hexlify(oid).lstrip(b'0') 35 | # Ensure two characters per input byte. 36 | chunks = [b'0x'] 37 | if len(as_hex) & 1: 38 | chunks.append(b'0') 39 | elif as_hex == b'': 40 | as_hex = b'00' 41 | chunks.append(as_hex) 42 | return b''.join(chunks) 43 | else: 44 | return repr(oid) 45 | 46 | 47 | class Lazy: 48 | """ 49 | A simple version of ``Lazy`` from ``zope.cachedescriptors`` 50 | """ 51 | 52 | __slots__ = ('func',) 53 | 54 | def __init__(self, func): 55 | self.func = func 56 | 57 | def __get__(self, inst, class_): 58 | if inst is None: 59 | return self 60 | 61 | func = self.func 62 | value = func(inst) 63 | inst.__dict__[func.__name__] = value 64 | return value 65 | -------------------------------------------------------------------------------- /src/BTrees/Length.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | 15 | import persistent 16 | 17 | 18 | class Length(persistent.Persistent): 19 | """BTree lengths are often too expensive to compute. 20 | 21 | Objects that use BTrees need to keep track of lengths themselves. 22 | This class provides an object for doing this. 23 | 24 | As a bonus, the object support application-level conflict 25 | resolution. 26 | 27 | It is tempting to to assign length objects to __len__ attributes 28 | to provide instance-specific __len__ methods. However, this no 29 | longer works as expected, because new-style classes cache 30 | class-defined slot methods (like __len__) in C type slots. Thus, 31 | instance-defined slot fillers are ignored. 32 | """ 33 | # class-level default required to keep copy.deepcopy happy -- see 34 | # https://bugs.launchpad.net/zodb/+bug/516653 35 | value = 0 36 | 37 | def __init__(self, v=0): 38 | self.value = v 39 | 40 | def __getstate__(self): 41 | return self.value 42 | 43 | def __setstate__(self, v): 44 | self.value = v 45 | 46 | def set(self, v): 47 | "Set the length value to v." 48 | self.value = v 49 | 50 | def _p_resolveConflict(self, old, s1, s2): 51 | return s1 + s2 - old 52 | 53 | def change(self, delta): 54 | "Add delta to the length value." 55 | self.value += delta 56 | 57 | def __call__(self, *args): 58 | "Return the current length value." 59 | return self.value 60 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Zope Public License (ZPL) Version 2.1 2 | 3 | A copyright notice accompanies this license document that identifies the 4 | copyright holders. 5 | 6 | This license has been certified as open source. It has also been designated as 7 | GPL compatible by the Free Software Foundation (FSF). 8 | 9 | Redistribution and use in source and binary forms, with or without 10 | modification, are permitted provided that the following conditions are met: 11 | 12 | 1. Redistributions in source code must retain the accompanying copyright 13 | notice, this list of conditions, and the following disclaimer. 14 | 15 | 2. Redistributions in binary form must reproduce the accompanying copyright 16 | notice, this list of conditions, and the following disclaimer in the 17 | documentation and/or other materials provided with the distribution. 18 | 19 | 3. Names of the copyright holders must not be used to endorse or promote 20 | products derived from this software without prior written permission from the 21 | copyright holders. 22 | 23 | 4. The right to distribute this software or to use it for any purpose does not 24 | give you the right to use Servicemarks (sm) or Trademarks (tm) of the 25 | copyright 26 | holders. Use of them is covered by separate agreement with the copyright 27 | holders. 28 | 29 | 5. If any files are modified, you must cause the modified files to carry 30 | prominent notices stating that you changed the files and the date of any 31 | change. 32 | 33 | Disclaimer 34 | 35 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESSED 36 | OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 37 | OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 38 | EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT, 39 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 41 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 42 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 43 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 44 | EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 | -------------------------------------------------------------------------------- /src/BTrees/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001-2012 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | import unittest 15 | 16 | 17 | class Test_non_negative(unittest.TestCase): 18 | 19 | def _callFUT(self, int_val): 20 | from BTrees.utils import non_negative 21 | return non_negative(int_val) 22 | 23 | def test_w_big_negative(self): 24 | self.assertEqual(self._callFUT(-(2**63 - 1)), 1) 25 | 26 | def test_w_negative(self): 27 | self.assertEqual(self._callFUT(-1), 2**63 - 1) 28 | 29 | def test_w_zero(self): 30 | self.assertEqual(self._callFUT(0), 0) 31 | 32 | def test_w_positive(self): 33 | self.assertEqual(self._callFUT(1), 1) 34 | 35 | 36 | class Test_oid_repr(unittest.TestCase): 37 | 38 | def _callFUT(self, oid): 39 | from BTrees.utils import oid_repr 40 | return oid_repr(oid) 41 | 42 | def test_w_non_strings(self): 43 | self.assertEqual(self._callFUT(None), repr(None)) 44 | self.assertEqual(self._callFUT(()), repr(())) 45 | self.assertEqual(self._callFUT([]), repr([])) 46 | self.assertEqual(self._callFUT({}), repr({})) 47 | self.assertEqual(self._callFUT(0), repr(0)) 48 | 49 | def test_w_short_strings(self): 50 | for length in range(8): 51 | faux = 'x' * length 52 | self.assertEqual(self._callFUT(faux), repr(faux)) 53 | 54 | def test_w_long_strings(self): 55 | for length in range(9, 1024): 56 | faux = 'x' * length 57 | self.assertEqual(self._callFUT(faux), repr(faux)) 58 | 59 | def test_w_zero(self): 60 | self.assertEqual(self._callFUT(b'\0\0\0\0\0\0\0\0'), b'0x00') 61 | 62 | def test_w_one(self): 63 | self.assertEqual(self._callFUT(b'\0\0\0\0\0\0\0\1'), b'0x01') 64 | 65 | def test_w_even_length(self): 66 | self.assertEqual(self._callFUT(b'\0\0\0\0\0\0\xAB\xC4'), b'0xabc4') 67 | 68 | def test_w_odd_length(self): 69 | self.assertEqual(self._callFUT(b'\0\0\0\0\0\0\x0D\xEF'), b'0x0def') 70 | -------------------------------------------------------------------------------- /src/BTrees/tests/test_fsBTree.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2010 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | import unittest 15 | 16 | from BTrees import fsBTree 17 | 18 | from ._test_builder import update_module 19 | 20 | 21 | class fsBucketTests(unittest.TestCase): 22 | 23 | def _getTargetClass(self): 24 | return fsBTree.fsBucket 25 | 26 | def _makeOne(self, *args, **kw): 27 | return self._getTargetClass()(*args, **kw) 28 | 29 | def _makeBytesItems(self): 30 | from .._compat import _ascii 31 | return [(_ascii(c * 2), _ascii(c * 6)) for c in 'abcdef'] 32 | 33 | def test_toString(self): 34 | bucket = self._makeOne(self._makeBytesItems()) 35 | self.assertEqual(bucket.toString(), 36 | b'aabbccddeeffaaaaaabbbbbbccccccddddddeeeeeeffffff') 37 | 38 | def test_fromString(self): 39 | before = self._makeOne(self._makeBytesItems()) 40 | after = before.fromString(before.toString()) 41 | self.assertEqual(before.__getstate__(), after.__getstate__()) 42 | 43 | def test_fromString_empty(self): 44 | before = self._makeOne(self._makeBytesItems()) 45 | after = before.fromString(b'') 46 | self.assertEqual(after.__getstate__(), ((),)) 47 | 48 | def test_fromString_invalid_length(self): 49 | bucket = self._makeOne(self._makeBytesItems()) 50 | self.assertRaises(ValueError, bucket.fromString, b'xxx') 51 | 52 | 53 | class fsBucketPyTests(fsBucketTests): 54 | 55 | def _getTargetClass(self): 56 | return fsBTree.fsBucketPy 57 | 58 | 59 | class fsTreeTests(unittest.TestCase): 60 | 61 | def _check_sizes(self, cls): 62 | self.assertEqual(cls.max_leaf_size, 500) 63 | self.assertEqual(cls.max_internal_size, 500) 64 | 65 | def test_BTree_sizes(self): 66 | self._check_sizes(fsBTree.BTree) 67 | self._check_sizes(fsBTree.BTreePy) 68 | 69 | def test_TreeSet_sizes(self): 70 | self._check_sizes(fsBTree.TreeSet) 71 | self._check_sizes(fsBTree.TreeSetPy) 72 | 73 | 74 | update_module(globals(), fsBTree) 75 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Generated with zope.meta (https://zopemeta.readthedocs.io/) from: 2 | # https://github.com/zopefoundation/meta/tree/master/src/zope/meta/c-code 3 | [tox] 4 | minversion = 4.0 5 | envlist = 6 | release-check 7 | lint 8 | py310,py310-pure 9 | py311,py311-pure 10 | py312,py312-pure 11 | py313,py313-pure 12 | py314,py314-pure 13 | pypy3 14 | docs 15 | coverage 16 | w_zodb 17 | w_zodb-pure 18 | 19 | [testenv] 20 | deps = 21 | setuptools >= 78.1.1,< 81 22 | setenv = 23 | pure: PURE_PYTHON=1 24 | !pure-!pypy3: PURE_PYTHON=0 25 | PYTHONFAULTHANDLER=1 26 | PYTHONDEVMODE=1 27 | ZOPE_INTERFACE_STRICT_IRO=1 28 | ZOPE_INTERFACE_LOG_CHANGED_IRO=1 29 | commands = 30 | zope-testrunner --test-path=src {posargs:-vc} 31 | sphinx-build -b doctest -d {envdir}/.cache/doctrees docs {envdir}/.cache/doctest 32 | extras = 33 | test 34 | docs 35 | 36 | [testenv:w_zodb] 37 | basepython = python3.13 38 | deps = ZODB 39 | 40 | [testenv:w_zodb-pure] 41 | basepython = python3.13 42 | deps = ZODB 43 | 44 | [testenv:setuptools-latest] 45 | basepython = python3 46 | deps = 47 | git+https://github.com/pypa/setuptools.git\#egg=setuptools 48 | 49 | [testenv:coverage] 50 | basepython = python3 51 | allowlist_externals = 52 | mkdir 53 | deps = 54 | coverage[toml] 55 | setenv = 56 | PURE_PYTHON=1 57 | commands = 58 | mkdir -p {toxinidir}/parts/htmlcov 59 | coverage run -m zope.testrunner --test-path=src {posargs:-vc} 60 | coverage html 61 | coverage report 62 | 63 | [testenv:release-check] 64 | description = ensure that the distribution is ready to release 65 | basepython = python3 66 | skip_install = true 67 | deps = 68 | setuptools 69 | wheel 70 | persistent 71 | twine 72 | build 73 | check-manifest 74 | check-python-versions >= 0.20.0 75 | wheel 76 | commands_pre = 77 | commands = 78 | check-manifest 79 | check-python-versions --only pyproject.toml,setup.py,tox.ini,.github/workflows/tests.yml 80 | python -m build --sdist --no-isolation 81 | twine check dist/* 82 | 83 | [testenv:lint] 84 | description = This env runs all linters configured in .pre-commit-config.yaml 85 | basepython = python3 86 | skip_install = true 87 | deps = 88 | pre-commit 89 | commands_pre = 90 | commands = 91 | pre-commit run --all-files --show-diff-on-failure 92 | 93 | [testenv:docs] 94 | basepython = python3 95 | skip_install = false 96 | commands_pre = 97 | commands = 98 | sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html 99 | sphinx-build -b doctest -d docs/_build/doctrees docs docs/_build/doctest 100 | -------------------------------------------------------------------------------- /src/BTrees/tests/test_Length.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2008 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | import unittest 15 | 16 | 17 | _marker = object() 18 | 19 | 20 | class TestLength(unittest.TestCase): 21 | 22 | def _getTargetClass(self): 23 | from BTrees.Length import Length 24 | return Length 25 | 26 | def _makeOne(self, value=_marker): 27 | if value is _marker: 28 | return self._getTargetClass()() 29 | return self._getTargetClass()(value) 30 | 31 | def test_ctor_default(self): 32 | length = self._makeOne() 33 | self.assertEqual(length.value, 0) 34 | 35 | def test_ctor_explict(self): 36 | length = self._makeOne(42) 37 | self.assertEqual(length.value, 42) 38 | 39 | def test___getstate___(self): 40 | length = self._makeOne(42) 41 | self.assertEqual(length.__getstate__(), 42) 42 | 43 | def test___setstate__(self): 44 | length = self._makeOne() 45 | length.__setstate__(42) 46 | self.assertEqual(length.value, 42) 47 | 48 | def test_set(self): 49 | length = self._makeOne() 50 | length.set(42) 51 | self.assertEqual(length.value, 42) 52 | 53 | def test__p_resolveConflict(self): 54 | length = self._makeOne() 55 | self.assertEqual(length._p_resolveConflict(5, 7, 9), 11) 56 | 57 | def test_change_w_positive_delta(self): 58 | length = self._makeOne() 59 | length.change(3) 60 | self.assertEqual(length.value, 3) 61 | 62 | def test_change_w_negative_delta(self): 63 | length = self._makeOne() 64 | length.change(-3) 65 | self.assertEqual(length.value, -3) 66 | 67 | def test___call___no_args(self): 68 | length = self._makeOne(42) 69 | self.assertEqual(length(), 42) 70 | 71 | def test___call___w_args(self): 72 | length = self._makeOne(42) 73 | self.assertEqual(length(0, None, (), [], {}), 42) 74 | 75 | def test_lp_516653(self): 76 | # Test for https://bugs.launchpad.net/zodb/+bug/516653 77 | import copy 78 | length = self._makeOne() 79 | other = copy.copy(length) 80 | self.assertEqual(other(), 0) 81 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # Generated with zope.meta (https://zopemeta.readthedocs.io/) from: 2 | # https://github.com/zopefoundation/meta/tree/master/src/zope/meta/c-code 3 | [build-system] 4 | requires = [ 5 | "setuptools", 6 | "wheel", 7 | "persistent", 8 | ] 9 | build-backend = "setuptools.build_meta" 10 | 11 | [project] 12 | name = "BTrees" 13 | version = "6.4.dev0" 14 | description = "Scalable persistent object containers" 15 | dynamic = ["readme"] 16 | requires-python = ">=3.10" 17 | license = "ZPL-2.1" 18 | authors = [ 19 | { name = "Zope Foundation and contributors", email = "zope-dev@zope.dev" }, 20 | ] 21 | maintainers = [ 22 | { name = "Plone Foundation and contributors", email = "zope-dev@zope.dev" }, 23 | ] 24 | classifiers = [ 25 | "Development Status :: 6 - Mature", 26 | "Framework :: ZODB", 27 | "Intended Audience :: Developers", 28 | "Operating System :: OS Independent", 29 | "Programming Language :: Python", 30 | "Programming Language :: Python :: 3", 31 | "Programming Language :: Python :: 3 :: Only", 32 | "Programming Language :: Python :: 3.10", 33 | "Programming Language :: Python :: 3.11", 34 | "Programming Language :: Python :: 3.12", 35 | "Programming Language :: Python :: 3.13", 36 | "Programming Language :: Python :: 3.14", 37 | "Programming Language :: Python :: Implementation :: CPython", 38 | "Programming Language :: Python :: Implementation :: PyPy", 39 | "Topic :: Database", 40 | "Topic :: Software Development :: Libraries :: Python Modules", 41 | ] 42 | dependencies = [ 43 | "persistent", 44 | "zope.interface", 45 | ] 46 | 47 | [project.optional-dependencies] 48 | docs = [ 49 | "Sphinx", 50 | "repoze.sphinx.autointerface", 51 | "furo", 52 | ] 53 | test = [ 54 | "persistent", 55 | "transaction", 56 | "zope.testrunner >= 6.4", 57 | ] 58 | ZODB = ["ZODB"] 59 | 60 | [project.urls] 61 | Documentation = "https://btrees.readthedocs.io" 62 | Issues = "https://github.com/zopefoundation/BTrees/issues" 63 | Source = "https://github.com/zopefoundation/BTrees" 64 | Changelog = "https://github.com/zopefoundation/BTrees/blob/master/CHANGES.rst" 65 | 66 | [tool.coverage.run] 67 | branch = true 68 | source = ["BTrees"] 69 | relative_files = true 70 | 71 | [tool.coverage.report] 72 | fail_under = 93 73 | precision = 2 74 | ignore_errors = true 75 | show_missing = true 76 | exclude_lines = [ 77 | "pragma: no cover", 78 | "pragma: nocover", 79 | "except ImportError:", 80 | "raise NotImplementedError", 81 | "if __name__ == '__main__':", 82 | "self.fail", 83 | "raise AssertionError", 84 | "raise unittest.Skip", 85 | ] 86 | 87 | [tool.coverage.html] 88 | directory = "parts/htmlcov" 89 | 90 | [tool.coverage.paths] 91 | source = [ 92 | "src/", 93 | ".tox/*/lib/python*/site-packages/", 94 | ".tox/pypy*/site-packages/", 95 | ] 96 | 97 | [tool.setuptools.dynamic] 98 | readme = {file = ["README.rst", "CHANGES.rst"]} 99 | -------------------------------------------------------------------------------- /include/persistent/persistent/ring.h: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | 3 | Copyright (c) 2003 Zope Foundation and Contributors. 4 | All Rights Reserved. 5 | 6 | This software is subject to the provisions of the Zope Public License, 7 | Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | FOR A PARTICULAR PURPOSE 12 | 13 | ****************************************************************************/ 14 | 15 | /* Support routines for the doubly-linked list of cached objects. 16 | 17 | The cache stores a headed, doubly-linked, circular list of persistent 18 | objects, with space for the pointers allocated in the objects themselves. 19 | The cache stores the distinguished head of the list, which is not a valid 20 | persistent object. The other list members are non-ghost persistent 21 | objects, linked in LRU (least-recently used) order. 22 | 23 | The r_next pointers traverse the ring starting with the least recently used 24 | object. The r_prev pointers traverse the ring starting with the most 25 | recently used object. 26 | 27 | Obscure: While each object is pointed at twice by list pointers (once by 28 | its predecessor's r_next, again by its successor's r_prev), the refcount 29 | on the object is bumped only by 1. This leads to some possibly surprising 30 | sequences of incref and decref code. Note that since the refcount is 31 | bumped at least once, the list does hold a strong reference to each 32 | object in it. 33 | */ 34 | 35 | typedef struct CPersistentRing_struct 36 | { 37 | struct CPersistentRing_struct *r_prev; 38 | struct CPersistentRing_struct *r_next; 39 | } CPersistentRing; 40 | 41 | 42 | /* The list operations here take constant time independent of the 43 | * number of objects in the list: 44 | */ 45 | 46 | /* Add elt as the most recently used object. elt must not already be 47 | * in the list, although this isn't checked. 48 | */ 49 | void ring_add(CPersistentRing *ring, CPersistentRing *elt); 50 | 51 | /* Remove elt from the list. elt must already be in the list, although 52 | * this isn't checked. 53 | */ 54 | void ring_del(CPersistentRing *elt); 55 | 56 | /* elt must already be in the list, although this isn't checked. It's 57 | * unlinked from its current position, and relinked into the list as the 58 | * most recently used object (which is arguably the tail of the list 59 | * instead of the head -- but the name of this function could be argued 60 | * either way). This is equivalent to 61 | * 62 | * ring_del(elt); 63 | * ring_add(ring, elt); 64 | * 65 | * but may be a little quicker. 66 | */ 67 | void ring_move_to_head(CPersistentRing *ring, CPersistentRing *elt); 68 | -------------------------------------------------------------------------------- /src/BTrees/tests/test_btreesubclass.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | import unittest 15 | 16 | from BTrees.OOBTree import OOBTree 17 | from BTrees.OOBTree import OOBucket 18 | 19 | 20 | class B(OOBucket): 21 | pass 22 | 23 | 24 | class T(OOBTree): 25 | _bucket_type = B 26 | max_leaf_size = 2 27 | max_internal_size = 3 28 | 29 | 30 | class S(T): 31 | pass 32 | 33 | 34 | class SubclassTest(unittest.TestCase): 35 | 36 | def testSubclass(self): 37 | # test that a subclass that defines _bucket_type gets buckets 38 | # of that type 39 | t = T() 40 | t[0] = 0 41 | self.assertIs(t._firstbucket.__class__, B) 42 | 43 | def testCustomNodeSizes(self, TreeKind=S, BucketKind=B): 44 | # We override btree and bucket split sizes in BTree subclasses. 45 | t = TreeKind() 46 | for i in range(8): 47 | t[i] = i 48 | 49 | state = t.__getstate__()[0] 50 | self.assertEqual(len(state), 5) 51 | sub = state[0] 52 | # __class__ is a property in the Python implementation, and 53 | # if the C extension is available it returns the C version. 54 | self.assertIsInstance(sub, TreeKind) 55 | sub = sub.__getstate__()[0] 56 | self.assertEqual(len(sub), 5) 57 | sub = sub[0] 58 | self.assertIsInstance(sub, BucketKind) 59 | self.assertEqual(len(sub), 1) 60 | 61 | def _checkReplaceNodeSizes(self, TreeKind, BucketKind): 62 | # We can also change the node sizes globally. 63 | orig_leaf = TreeKind.max_leaf_size 64 | orig_internal = TreeKind.max_internal_size 65 | TreeKind.max_leaf_size = T.max_leaf_size 66 | TreeKind.max_internal_size = T.max_internal_size 67 | try: 68 | self.testCustomNodeSizes(TreeKind, BucketKind) 69 | finally: 70 | TreeKind.max_leaf_size = orig_leaf 71 | TreeKind.max_internal_size = orig_internal 72 | 73 | def testReplaceNodeSizesNative(self): 74 | self._checkReplaceNodeSizes(OOBTree, OOBucket) 75 | 76 | def testReplaceNodeSizesPython(self): 77 | from BTrees.OOBTree import OOBTreePy 78 | from BTrees.OOBTree import OOBucketPy 79 | self._checkReplaceNodeSizes(OOBTreePy, OOBucketPy) 80 | -------------------------------------------------------------------------------- /.meta.toml: -------------------------------------------------------------------------------- 1 | # Generated with zope.meta (https://zopemeta.readthedocs.io/) from: 2 | # https://github.com/zopefoundation/meta/tree/master/src/zope/meta/c-code 3 | [meta] 4 | template = "c-code" 5 | commit-id = "f62d8bab" 6 | 7 | [python] 8 | with-windows = true 9 | with-pypy = true 10 | with-future-python = false 11 | with-docs = true 12 | with-sphinx-doctests = true 13 | with-macos = false 14 | 15 | [tox] 16 | additional-envlist = [ 17 | "w_zodb", 18 | "w_zodb-pure", 19 | ] 20 | testenv-setenv = [ 21 | "PYTHONFAULTHANDLER=1", 22 | "PYTHONDEVMODE=1", 23 | "ZOPE_INTERFACE_STRICT_IRO=1", 24 | "ZOPE_INTERFACE_LOG_CHANGED_IRO=1", 25 | ] 26 | testenv-additional = [ 27 | "", 28 | "[testenv:w_zodb]", 29 | "basepython = python3.13", 30 | "deps = ZODB", 31 | "", 32 | "[testenv:w_zodb-pure]", 33 | "basepython = python3.13", 34 | "deps = ZODB", 35 | ] 36 | 37 | [coverage] 38 | fail-under = 93 39 | 40 | [flake8] 41 | additional-config = [ 42 | "per-file-ignores =", 43 | " src/BTrees/check.py: F401", 44 | ] 45 | 46 | [manifest] 47 | additional-rules = [ 48 | "include *.yaml", 49 | "include *.sh", 50 | "recursive-include docs *.bat", 51 | "recursive-include docs *.css", 52 | "recursive-include include/persistent *.h", 53 | "recursive-include src *.c", 54 | "recursive-include src *.h", 55 | ] 56 | 57 | [check-manifest] 58 | additional-ignores = [ 59 | "docs/_build/html/_static/*", 60 | "docs/_build/html/_static/*/*", 61 | ] 62 | 63 | [c-code] 64 | manylinux-install-setup = [ 65 | "export CFLAGS=\"-pipe\"", 66 | "if [ `uname -m` == 'aarch64' ]; then", 67 | " # Compiling with -O3 on the arm emulator takes hours. The default settings have -O3,", 68 | " # and adding -Os doesn't help much; -O1 seems too.", 69 | " echo \"Compiling with -O1\"", 70 | " export CFLAGS=\"$CFLAGS -O1\"", 71 | "else", 72 | " echo \"Compiling with -O3\"", 73 | " export CFLAGS=\"-O3 $CFLAGS\"", 74 | "fi", 75 | "", 76 | "export PURE_PYTHON=0", 77 | ] 78 | manylinux-aarch64-tests = [ 79 | "# Running the test suite takes forever in", 80 | "# emulation; an early run (using tox, which is also slow)", 81 | "# took over an hour to build and then run the tests sequentially", 82 | "# for the Python versions. We still want to run tests, though!", 83 | "# We don't want to distribute wheels for a platform that's", 84 | "# completely untested. Consequently, we limit it to running", 85 | "# in just one interpreter, the newest one on the list (which in principle", 86 | "# should be the fastest), and we don't install the ZODB extra.", 87 | "if [[ \"${PYBIN}\" == *\"cp311\"* ]]; then", 88 | " cd /io/", 89 | " \"${PYBIN}/pip\" install -e .[test]", 90 | " \"${PYBIN}/python\" -c 'import BTrees.OOBTree; print(BTrees.OOBTree.BTree, BTrees.OOBTree.BTreePy)'", 91 | " \"${PYBIN}/python\" -m unittest discover -s src", 92 | " cd ..", 93 | "fi", 94 | ] 95 | -------------------------------------------------------------------------------- /src/BTrees/tests/test__datatypes.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright 2012 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | 15 | import unittest 16 | 17 | from BTrees import _datatypes 18 | 19 | 20 | to_ob = _datatypes.Any() 21 | to_int = _datatypes.I() 22 | to_float = _datatypes.F() 23 | to_long = _datatypes.L() 24 | to_2_bytes = _datatypes.f() 25 | to_6_bytes = _datatypes.s() 26 | 27 | 28 | class TestDatatypes(unittest.TestCase): 29 | def test_to_ob(self): 30 | for thing in "abc", 0, 1.3, (), frozenset((1, 2)), object(): 31 | self.assertIs(to_ob(thing), thing) 32 | 33 | def test_to_int_w_int(self): 34 | self.assertEqual(to_int(3), 3) 35 | 36 | def test_to_int_w_overflow(self): 37 | self.assertRaises(TypeError, to_int, 2**64) 38 | 39 | def test_to_int_w_invalid(self): 40 | self.assertRaises(TypeError, to_int, ()) 41 | 42 | def test_to_float_w_float(self): 43 | self.assertEqual(to_float(3.14159), 3.14159) 44 | 45 | def test_to_float_w_int(self): 46 | self.assertEqual(to_float(3), 3.0) 47 | 48 | def test_to_float_w_invalid(self): 49 | self.assertRaises(TypeError, to_float, ()) 50 | 51 | def test_to_long_w_int(self): 52 | self.assertEqual(to_long(3), 3) 53 | 54 | def test_to_long_w_overflow(self): 55 | self.assertRaises(TypeError, to_long, 2**64) 56 | 57 | def test_to_long_w_invalid(self): 58 | self.assertRaises(TypeError, to_long, ()) 59 | 60 | def test_to_2_bytes_w_ok(self): 61 | self.assertEqual(to_2_bytes(b'ab'), b'ab') 62 | 63 | def test_to_2_bytes_w_invalid_length(self): 64 | self.assertRaises(TypeError, to_2_bytes, b'a') 65 | self.assertRaises(TypeError, to_2_bytes, b'abcd') 66 | 67 | def test_to_6_bytes_w_ok(self): 68 | self.assertEqual(to_6_bytes(b'abcdef'), b'abcdef') 69 | 70 | def test_to_6_bytes_w_invalid_length(self): 71 | self.assertRaises(TypeError, to_6_bytes, b'a') 72 | self.assertRaises(TypeError, to_6_bytes, b'abcd') 73 | 74 | def test_coerce_to_6_bytes(self): 75 | # correct input is passed through 76 | self.assertEqual(to_6_bytes.coerce(b'abcdef'), b'abcdef') 77 | 78 | # small positive integers are converted 79 | self.assertEqual(to_6_bytes.coerce(1), b'\x00\x00\x00\x00\x00\x01') 80 | 81 | # negative values are disallowed 82 | self.assertRaises(TypeError, to_6_bytes.coerce, -1) 83 | 84 | # values outside the bigger than 64-bits are disallowed 85 | self.assertRaises(TypeError, to_6_bytes.coerce, 2 ** 64 + 1) 86 | -------------------------------------------------------------------------------- /.manylinux-install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Generated with zope.meta (https://zopemeta.readthedocs.io/) from: 3 | # https://github.com/zopefoundation/meta/tree/master/src/zope/meta/c-code 4 | 5 | set -e -x 6 | 7 | # Running inside docker 8 | # Set a cache directory for pip. This was 9 | # mounted to be the same as it is outside docker so it 10 | # can be persisted. 11 | export XDG_CACHE_HOME="/cache" 12 | # XXX: This works for macOS, where everything bind-mounted 13 | # is seen as owned by root in the container. But when the host is Linux 14 | # the actual UIDs come through to the container, triggering 15 | # pip to disable the cache when it detects that the owner doesn't match. 16 | # The below is an attempt to fix that, taken from bcrypt. It seems to work on 17 | # Github Actions. 18 | if [ -n "$GITHUB_ACTIONS" ]; then 19 | echo Adjusting pip cache permissions 20 | mkdir -p $XDG_CACHE_HOME/pip 21 | chown -R $(whoami) $XDG_CACHE_HOME 22 | fi 23 | ls -ld /cache 24 | ls -ld /cache/pip 25 | 26 | export CFLAGS="-pipe" 27 | if [ `uname -m` == 'aarch64' ]; then 28 | # Compiling with -O3 on the arm emulator takes hours. The default settings have -O3, 29 | # and adding -Os doesn't help much; -O1 seems too. 30 | echo "Compiling with -O1" 31 | export CFLAGS="$CFLAGS -O1" 32 | else 33 | echo "Compiling with -O3" 34 | export CFLAGS="-O3 $CFLAGS" 35 | fi 36 | 37 | export PURE_PYTHON=0 38 | # We need some libraries because we build wheels from scratch: 39 | yum -y install libffi-devel 40 | 41 | tox_env_map() { 42 | case $1 in 43 | *"cp310"*) echo 'py310';; 44 | *"cp311"*) echo 'py311';; 45 | *"cp312"*) echo 'py312';; 46 | *"cp313"*) echo 'py313';; 47 | *"cp314"*) echo 'py314';; 48 | *) echo 'py';; 49 | esac 50 | } 51 | 52 | # Compile wheels 53 | for PYBIN in /opt/python/*/bin; do 54 | if \ 55 | [[ "${PYBIN}" == *"cp310/"* ]] || \ 56 | [[ "${PYBIN}" == *"cp311/"* ]] || \ 57 | [[ "${PYBIN}" == *"cp312/"* ]] || \ 58 | [[ "${PYBIN}" == *"cp313/"* ]] || \ 59 | [[ "${PYBIN}" == *"cp314/"* ]] ; then 60 | "${PYBIN}/pip" install -e /io/ 61 | "${PYBIN}/pip" wheel /io/ -w wheelhouse/ 62 | if [ `uname -m` == 'aarch64' ]; then 63 | # Running the test suite takes forever in 64 | # emulation; an early run (using tox, which is also slow) 65 | # took over an hour to build and then run the tests sequentially 66 | # for the Python versions. We still want to run tests, though! 67 | # We don't want to distribute wheels for a platform that's 68 | # completely untested. Consequently, we limit it to running 69 | # in just one interpreter, the newest one on the list (which in principle 70 | # should be the fastest), and we don't install the ZODB extra. 71 | if [[ "${PYBIN}" == *"cp311"* ]]; then 72 | cd /io/ 73 | "${PYBIN}/pip" install -e .[test] 74 | "${PYBIN}/python" -c 'import BTrees.OOBTree; print(BTrees.OOBTree.BTree, BTrees.OOBTree.BTreePy)' 75 | "${PYBIN}/python" -m unittest discover -s src 76 | cd .. 77 | fi 78 | fi 79 | rm -rf /io/build /io/*.egg-info 80 | fi 81 | done 82 | 83 | # Show what wheels we have 84 | echo "Fixing up the following wheels:" 85 | ls -l wheelhouse/btrees*.whl 86 | # Bundle external shared libraries into the wheels 87 | for whl in wheelhouse/btrees*.whl; do 88 | auditwheel repair "$whl" -w /io/wheelhouse/ 89 | done 90 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | API Reference 3 | =============== 4 | 5 | 6 | Protocol APIs 7 | ============= 8 | 9 | .. module:: BTrees.Interfaces 10 | 11 | .. versionchanged:: 4.8.0 12 | Previously, ``ISized`` was defined here, but now it is 13 | imported from :mod:`zope.interface.common.collections`. The 14 | definition is the same. 15 | 16 | Similarly, ``IReadSequence``, previously defined here, 17 | has been replaced with 18 | :mod:`zope.interface.common.sequence.IMinimalSequence `. 19 | 20 | .. caution:: 21 | 22 | Before version 4.8.0, most of these interfaces served as 23 | documentation only, and were *not* implemented by the classes of 24 | this package. For example, :class:`BTrees.OOBTree.BTree` did *not* 25 | implement `IBTree`. (The exceptions were the :class:`IBTreeModule` 26 | and :class:`IBTreeFamily` families of interfaces and 27 | implementations.) 28 | 29 | Beginning with version 4.8.0, objects implement the expected 30 | interface; the ``BTree`` classes implement ``IBTree``, the set 31 | classes implement the appropriate set interface and so on. 32 | 33 | 34 | .. autointerface:: ICollection 35 | .. autointerface:: IKeyed 36 | .. autointerface:: ISetMutable 37 | .. autointerface:: IKeySequence 38 | .. autointerface:: IMinimalDictionary 39 | .. autointerface:: IDictionaryIsh 40 | .. autointerface:: IMerge 41 | .. autointerface:: IIMerge 42 | .. autointerface:: IMergeIntegerKey 43 | 44 | BTree Family APIs 45 | ----------------- 46 | .. autointerface:: ISet 47 | .. autointerface:: ITreeSet 48 | .. autointerface:: IBTree 49 | .. autointerface:: IBTreeFamily 50 | 51 | There are two families defined: 52 | 53 | .. autodata:: BTrees.family32 54 | .. autodata:: BTrees.family64 55 | 56 | Module APIs 57 | ----------- 58 | .. autointerface:: IBTreeModule 59 | .. autointerface:: IObjectObjectBTreeModule 60 | .. autointerface:: IIntegerObjectBTreeModule 61 | .. autointerface:: IObjectIntegerBTreeModule 62 | .. autointerface:: IIntegerIntegerBTreeModule 63 | .. autointerface:: IIntegerFloatBTreeModule 64 | 65 | 66 | Utilities 67 | ========= 68 | 69 | .. automodule:: BTrees.Length 70 | 71 | .. automodule:: BTrees.check 72 | 73 | 74 | BTree Data Structure Variants 75 | ============================= 76 | 77 | Integer Keys 78 | ------------ 79 | 80 | Float Values 81 | ~~~~~~~~~~~~ 82 | .. automodule:: BTrees.IFBTree 83 | 84 | Integer Values 85 | ~~~~~~~~~~~~~~ 86 | .. automodule:: BTrees.IIBTree 87 | 88 | Object Values 89 | ~~~~~~~~~~~~~ 90 | .. automodule:: BTrees.IOBTree 91 | 92 | Unsigned Integer Values 93 | ~~~~~~~~~~~~~~~~~~~~~~~ 94 | .. automodule:: BTrees.IUBTree 95 | 96 | Long Integer Keys 97 | ----------------- 98 | 99 | Float Values 100 | ~~~~~~~~~~~~ 101 | .. automodule:: BTrees.LFBTree 102 | 103 | Long Integer Values 104 | ~~~~~~~~~~~~~~~~~~~ 105 | .. automodule:: BTrees.LLBTree 106 | 107 | 108 | Object Values 109 | ~~~~~~~~~~~~~ 110 | .. automodule:: BTrees.LOBTree 111 | 112 | Quad Unsigned Integer Values 113 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 114 | .. automodule:: BTrees.LQBTree 115 | 116 | 117 | Object Keys 118 | ----------- 119 | 120 | Integer Values 121 | ~~~~~~~~~~~~~~ 122 | .. automodule:: BTrees.OIBTree 123 | 124 | Long Integer Values 125 | ~~~~~~~~~~~~~~~~~~~ 126 | .. automodule:: BTrees.OLBTree 127 | 128 | Object Values 129 | ~~~~~~~~~~~~~ 130 | .. automodule:: BTrees.OOBTree 131 | 132 | Quad Unsigned Integer Values 133 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 134 | .. automodule:: BTrees.OQBTree 135 | 136 | Unsigned Integer Values 137 | ~~~~~~~~~~~~~~~~~~~~~~~ 138 | .. automodule:: BTrees.OUBTree 139 | 140 | 141 | Quad Unsigned Integer Keys 142 | -------------------------- 143 | 144 | Float Values 145 | ~~~~~~~~~~~~ 146 | .. automodule:: BTrees.QFBTree 147 | 148 | Long Integer Values 149 | ~~~~~~~~~~~~~~~~~~~ 150 | .. automodule:: BTrees.QLBTree 151 | 152 | Object Values 153 | ~~~~~~~~~~~~~ 154 | .. automodule:: BTrees.QOBTree 155 | 156 | Quad Unsigned Integer Values 157 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 158 | .. automodule:: BTrees.QQBTree 159 | 160 | Unsigned Integer Keys 161 | --------------------- 162 | 163 | Float Values 164 | ~~~~~~~~~~~~ 165 | .. automodule:: BTrees.UFBTree 166 | 167 | Integer Values 168 | ~~~~~~~~~~~~~~ 169 | .. automodule:: BTrees.UIBTree 170 | 171 | Object Values 172 | ~~~~~~~~~~~~~ 173 | .. automodule:: BTrees.UOBTree 174 | 175 | Unsigned Integer Values 176 | ~~~~~~~~~~~~~~~~~~~~~~~ 177 | .. automodule:: BTrees.UUBTree 178 | -------------------------------------------------------------------------------- /src/BTrees/_compat.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001-2012 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | import os 15 | import sys 16 | 17 | 18 | PYPY = hasattr(sys, 'pypy_version_info') 19 | 20 | 21 | def compare(x, y): 22 | if x is None: 23 | if y is None: 24 | return 0 25 | else: 26 | return -1 27 | elif y is None: 28 | return 1 29 | else: 30 | return (x > y) - (y > x) 31 | 32 | 33 | def _ascii(x): 34 | return bytes(x, 'ascii') 35 | 36 | 37 | def _c_optimizations_required(): 38 | """Return a true value if the C optimizations are required. 39 | 40 | Uses the ``PURE_PYTHON`` variable as documented in `import_c_extension`. 41 | """ 42 | pure_env = os.environ.get('PURE_PYTHON') 43 | require_c = pure_env == "0" 44 | return require_c 45 | 46 | 47 | def _c_optimizations_available(module_name): 48 | """ 49 | Return the C optimization module, if available, otherwise 50 | a false value. 51 | 52 | If the optimizations are required but not available, this 53 | raises the ImportError. 54 | 55 | This does not say whether they should be used or not. 56 | """ 57 | import importlib 58 | catch = () if _c_optimizations_required() else (ImportError,) 59 | try: 60 | return importlib.import_module('BTrees._' + module_name) 61 | except catch: # pragma: no cover 62 | return False 63 | 64 | 65 | def _c_optimizations_ignored(): 66 | """ 67 | The opposite of `_c_optimizations_required`. 68 | """ 69 | pure_env = os.environ.get('PURE_PYTHON') 70 | return pure_env != "0" if pure_env is not None else PYPY 71 | 72 | 73 | def _should_attempt_c_optimizations(): 74 | """ 75 | Return a true value if we should attempt to use the C optimizations. 76 | 77 | This takes into account whether we're on PyPy and the value of the 78 | ``PURE_PYTHON`` environment variable, as defined in `import_c_extension`. 79 | """ 80 | if PYPY: 81 | return False 82 | 83 | if _c_optimizations_required(): 84 | return True 85 | return not _c_optimizations_ignored() 86 | 87 | 88 | def import_c_extension(mod_globals): 89 | """ 90 | Call this function with the globals of a module that implements 91 | Python versions of a BTree family to find the C optimizations. 92 | 93 | If the ``PURE_PYTHON`` environment variable is set to any value 94 | other than ``"0"``, or we're on PyPy, ignore the C implementation. 95 | If the C implementation cannot be imported, return the Python 96 | version. If ``PURE_PYTHON`` is set to ``"0"``, *require* the C 97 | implementation (let the ImportError propagate); the exception again 98 | is PyPy, where we never use the C extension (although it builds here, the 99 | ``persistent`` library doesn't provide native extensions for PyPy). 100 | 101 | """ 102 | c_module = None 103 | module_name = mod_globals['__name__'] 104 | assert module_name.startswith('BTrees.') 105 | module_name = module_name.split('.')[1] 106 | if _should_attempt_c_optimizations(): 107 | c_module = _c_optimizations_available(module_name) 108 | 109 | if c_module: 110 | new_values = dict(c_module.__dict__) 111 | new_values.pop("__name__", None) 112 | new_values.pop('__file__', None) 113 | new_values.pop('__doc__', None) 114 | mod_globals.update(new_values) 115 | else: 116 | # No C extension, make the Py versions available without that 117 | # extension. The list comprehension both filters and prevents 118 | # concurrent modification errors. 119 | for py in [k for k in mod_globals if k.endswith('Py')]: 120 | mod_globals[py[:-2]] = mod_globals[py] 121 | 122 | # Assign the global aliases 123 | prefix = module_name[:2] 124 | for name in ('Bucket', 'Set', 'BTree', 'TreeSet'): 125 | mod_globals[name] = mod_globals[prefix + name] 126 | 127 | # Cleanup 128 | mod_globals.pop('import_c_extension', None) 129 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2012 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | 15 | import os 16 | import sys 17 | from distutils.errors import CCompilerError 18 | from distutils.errors import DistutilsExecError 19 | from distutils.errors import DistutilsPlatformError 20 | 21 | from setuptools import Extension 22 | from setuptools import setup 23 | from setuptools.command.build_ext import build_ext 24 | 25 | 26 | class optional_build_ext(build_ext): 27 | """This class subclasses build_ext and allows 28 | the building of C extensions to fail. 29 | """ 30 | 31 | def run(self): 32 | try: 33 | build_ext.run(self) 34 | except DistutilsPlatformError as e: 35 | self._unavailable(e) 36 | 37 | def build_extension(self, ext): 38 | try: 39 | build_ext.build_extension(self, ext) 40 | except (CCompilerError, DistutilsExecError, OSError) as e: 41 | self._unavailable(e) 42 | 43 | def _unavailable(self, e): 44 | print('*' * 80) 45 | print("""WARNING: 46 | An optional code optimization (C extension) could not be compiled. 47 | Optimizations for this package will not be available!""") 48 | print() 49 | print(e) 50 | print('*' * 80) 51 | if 'bdist_wheel' in sys.argv and not os.environ.get("PURE_PYTHON"): 52 | # pip uses bdist_wheel by default, and hides the error output. 53 | # Let this error percolate up so the user can see it. 54 | # pip will then go ahead and run 'setup.py install' directly. 55 | raise 56 | 57 | 58 | # Set up dependencies for the BTrees package 59 | base_btrees_depends = [ 60 | "src/BTrees/BTreeItemsTemplate.c", 61 | "src/BTrees/BTreeModuleTemplate.c", 62 | "src/BTrees/BTreeTemplate.c", 63 | "src/BTrees/BucketTemplate.c", 64 | "src/BTrees/MergeTemplate.c", 65 | "src/BTrees/SetOpTemplate.c", 66 | "src/BTrees/SetTemplate.c", 67 | "src/BTrees/TreeSetTemplate.c", 68 | "src/BTrees/sorters.c", 69 | ] 70 | 71 | FLAVORS = { 72 | "O": "object", 73 | "F": "float", 74 | "I": "int", # Signed 32-bit 75 | "L": "int", # Signed 64-bit 76 | "U": "int", # Unsigned 32-bit 77 | "Q": "int" # Unsigned 64-bit (from the printf "q" modifier for quad_t) 78 | } 79 | # XXX should 'fs' be in ZODB instead? 80 | FAMILIES = ( 81 | # Signed 32-bit keys 82 | "IO", # object value 83 | "II", # self value 84 | "IF", # float value 85 | "IU", # opposite sign value 86 | # Unsigned 32-bit keys 87 | "UO", # object value 88 | "UU", # self value 89 | "UF", # float value 90 | "UI", # opposite sign value 91 | # Signed 64-bit keys 92 | "LO", # object value 93 | "LL", # self value 94 | "LF", # float value 95 | "LQ", # opposite sign value 96 | # Unsigned 64-bit keys 97 | "QO", # object value 98 | "QQ", # self value 99 | "QF", # float value 100 | "QL", # opposite sign value 101 | # Object keys 102 | "OO", # object 103 | "OI", # 32-bit signed 104 | "OU", # 32-bit unsigned 105 | "OL", # 64-bit signed 106 | "OQ", # 64-bit unsigned 107 | "fs", 108 | ) 109 | 110 | KEY_H = "src/BTrees/%skeymacros.h" 111 | VALUE_H = "src/BTrees/%svaluemacros.h" 112 | 113 | 114 | def BTreeExtension(family): 115 | key = family[0] 116 | value = family[1] 117 | name = "BTrees._%sBTree" % family 118 | sources = ["src/BTrees/_%sBTree.c" % family] 119 | kwargs = {"include_dirs": [os.path.join('include', 'persistent')]} 120 | if family != "fs": 121 | kwargs["depends"] = (base_btrees_depends + [KEY_H % FLAVORS[key], 122 | VALUE_H % FLAVORS[value]]) 123 | else: 124 | kwargs["depends"] = base_btrees_depends 125 | if key != "O": 126 | kwargs["define_macros"] = [('EXCLUDE_INTSET_SUPPORT', None)] 127 | return Extension(name, sources, **kwargs) 128 | 129 | 130 | ext_modules = [BTreeExtension(family) for family in FAMILIES] 131 | 132 | 133 | setup(ext_modules=ext_modules, 134 | cmdclass={'build_ext': optional_build_ext}) 135 | -------------------------------------------------------------------------------- /src/BTrees/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################# 2 | # 3 | # Copyright (c) 2007 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################# 14 | 15 | import sys 16 | 17 | import zope.interface 18 | 19 | import BTrees.Interfaces 20 | 21 | from ._module_builder import create_module 22 | 23 | 24 | __all__ = [ 25 | 'family32', 26 | 'family64', 27 | ] 28 | 29 | _FAMILIES = ( 30 | # Signed 32-bit keys 31 | "IO", # object value 32 | "II", # self value 33 | "IF", # float value 34 | "IU", # opposite sign value 35 | # Unsigned 32-bit keys 36 | "UO", # object value 37 | "UU", # self value 38 | "UF", # float value 39 | "UI", # opposite sign value 40 | # Signed 64-bit keys 41 | "LO", # object value 42 | "LL", # self value 43 | "LF", # float value 44 | "LQ", # opposite sign value 45 | # Unsigned 64-bit keys 46 | "QO", # object value 47 | "QQ", # self value 48 | "QF", # float value 49 | "QL", # opposite sign value 50 | # Object keys 51 | "OO", # object 52 | "OI", # 32-bit signed 53 | "OU", # 32-bit unsigned 54 | "OL", # 64-bit signed 55 | "OQ", # 64-bit unsigned 56 | # Special purpose 57 | 'fs', # 2-byte -> 6-byte 58 | ) 59 | 60 | # XXX: Do this without completely ruining static analysis. 61 | for family in _FAMILIES: 62 | mod = create_module(family) 63 | name = vars(mod)['__name__'] 64 | sys.modules[name] = mod 65 | globals()[name.split('.', 1)[1]] = mod 66 | __all__.append(name) 67 | 68 | 69 | @zope.interface.implementer(BTrees.Interfaces.IBTreeFamily) 70 | class _Family: 71 | from BTrees import OOBTree as OO 72 | _BITSIZE = 0 73 | minint = maxint = maxuint = None 74 | 75 | def __init__(self): 76 | self.maxint = int(2 ** (self._BITSIZE - 1) - 1) 77 | self.minint = int(-self.maxint - 1) 78 | self.maxuint = int(2 ** self._BITSIZE - 1) 79 | 80 | def __str__(self): 81 | return ( 82 | "BTree family using {} bits. " 83 | "Supports signed integer values from {:,} to {:,} " 84 | "and maximum unsigned integer value {:,}." 85 | ).format(self._BITSIZE, self.minint, self.maxint, self.maxuint) 86 | 87 | def __repr__(self): 88 | return "<%s>" % ( 89 | self 90 | ) 91 | 92 | 93 | class _Family32(_Family): 94 | _BITSIZE = 32 95 | from BTrees import IFBTree as IF 96 | from BTrees import IIBTree as II 97 | from BTrees import IOBTree as IO 98 | from BTrees import IUBTree as IU 99 | from BTrees import OIBTree as OI 100 | from BTrees import OUBTree as OU 101 | from BTrees import UFBTree as UF 102 | from BTrees import UIBTree as UI 103 | from BTrees import UOBTree as UO 104 | from BTrees import UUBTree as UU 105 | 106 | def __reduce__(self): 107 | return _family32, () 108 | 109 | 110 | class _Family64(_Family): 111 | _BITSIZE = 64 112 | from BTrees import LFBTree as IF 113 | from BTrees import LLBTree as II 114 | from BTrees import LOBTree as IO 115 | from BTrees import LQBTree as IU 116 | from BTrees import OLBTree as OI 117 | from BTrees import OQBTree as OU 118 | from BTrees import QFBTree as UF 119 | from BTrees import QLBTree as UI 120 | from BTrees import QOBTree as UO 121 | from BTrees import QQBTree as UU 122 | 123 | def __reduce__(self): 124 | return _family64, () 125 | 126 | 127 | def _family32(): 128 | return family32 129 | 130 | 131 | _family32.__safe_for_unpickling__ = True # noqa E305 132 | 133 | 134 | def _family64(): 135 | return family64 136 | 137 | 138 | _family64.__safe_for_unpickling__ = True # noqa E305 139 | 140 | #: 32-bit BTree family. 141 | family32 = _Family32() 142 | 143 | #: 64-bit BTree family. 144 | family64 = _Family64() 145 | 146 | for _family in family32, family64: 147 | for _mod_name in ( 148 | "OI", "OU", 149 | 'IO', "II", "IF", "IU", 150 | "UO", "UU", "UF", "UI", 151 | ): 152 | getattr(_family, _mod_name).family = _family 153 | 154 | # The IMergeBTreeModule interface specifies the ``family`` attribute, 155 | # and fsBTree implements IIntegerObjectBTreeModule, which extends that 156 | # interface. But for fsBTrees, no family makes particular sense, so we 157 | # arbitrarily pick one. 158 | globals()['fsBTree'].family = family64 159 | -------------------------------------------------------------------------------- /src/BTrees/intkeymacros.h: -------------------------------------------------------------------------------- 1 | 2 | #define KEYMACROS_H "$Id$\n" 3 | 4 | #if !defined(ZODB_UNSIGNED_KEY_INTS) 5 | /* signed keys */ 6 | 7 | #if defined(ZODB_64BIT_INTS) 8 | /* PY_LONG_LONG as key */ 9 | 10 | #define NEED_LONG_LONG_SUPPORT 11 | 12 | #define NEED_LONG_LONG_KEYS 13 | #define KEY_TYPE PY_LONG_LONG 14 | 15 | #define NEED_LONG_LONG_CHECK 16 | #define KEY_CHECK longlong_check 17 | 18 | #define NEED_LONG_LONG_AS_OBJECT 19 | #define COPY_KEY_TO_OBJECT(O, K) O=longlong_as_object(K) 20 | 21 | #define NEED_LONG_LONG_CONVERT 22 | #define COPY_KEY_FROM_ARG(TARGET, ARG, STATUS) \ 23 | if (!longlong_convert((ARG), &TARGET)) \ 24 | { \ 25 | (STATUS)=0; (TARGET)=0; \ 26 | } 27 | 28 | #else /* !defined(ZODB_64BIT_INTS) */ 29 | /* C int as key */ 30 | 31 | #define KEY_TYPE int 32 | #define KEY_CHECK PyLong_Check 33 | #define COPY_KEY_TO_OBJECT(O, K) O=PyLong_FromLong(K) 34 | #define COPY_KEY_FROM_ARG(TARGET, ARG, STATUS) \ 35 | if (PyLong_Check(ARG)) { \ 36 | long vcopy = PyLong_AsLong(ARG); \ 37 | if (PyErr_Occurred()) { \ 38 | if (PyErr_ExceptionMatches(PyExc_OverflowError)) { \ 39 | PyErr_Clear(); \ 40 | PyErr_SetString(PyExc_TypeError, "integer out of range"); \ 41 | } \ 42 | (STATUS)=0; (TARGET)=0; \ 43 | } \ 44 | else if ((int)vcopy != vcopy) { \ 45 | PyErr_SetString(PyExc_TypeError, "integer out of range"); \ 46 | (STATUS)=0; (TARGET)=0; \ 47 | } \ 48 | else TARGET = vcopy; \ 49 | } else { \ 50 | PyErr_SetString(PyExc_TypeError, "expected integer key"); \ 51 | (STATUS)=0; (TARGET)=0; } 52 | 53 | #endif /* !defined(ZODB_64BIT_INTS) */ 54 | 55 | #else 56 | /* Unsigned keys */ 57 | 58 | #if defined(ZODB_64BIT_INTS) 59 | /* PY_LONG_LONG as key */ 60 | 61 | #define NEED_LONG_LONG_SUPPORT 62 | #define NEED_LONG_LONG_KEYS 63 | #define KEY_TYPE unsigned PY_LONG_LONG 64 | 65 | #define NEED_ULONG_LONG_CHECK 66 | #define KEY_CHECK ulonglong_check 67 | 68 | #define NEED_ULONG_LONG_AS_OBJECT 69 | #define COPY_KEY_TO_OBJECT(O, K) O=ulonglong_as_object(K) 70 | 71 | #define NEED_ULONG_LONG_CONVERT 72 | #define COPY_KEY_FROM_ARG(TARGET, ARG, STATUS) \ 73 | if (!ulonglong_convert((ARG), &TARGET)) \ 74 | { \ 75 | (STATUS)=0; (TARGET)=0; \ 76 | } 77 | 78 | #else /* !defined(ZODB_64BIT_INTS) */ 79 | /* C int as key */ 80 | 81 | #define KEY_TYPE unsigned int 82 | #define KEY_CHECK PyLong_Check 83 | #define COPY_KEY_TO_OBJECT(O, K) O=PyLong_FromUnsignedLongLong(K) 84 | 85 | #define COPY_KEY_FROM_ARG(TARGET, ARG, STATUS) \ 86 | if (PyLong_Check(ARG)) { \ 87 | long vcopy = PyLong_AsLong(ARG); \ 88 | if (PyErr_Occurred()) { \ 89 | if (PyErr_ExceptionMatches(PyExc_OverflowError)) { \ 90 | PyErr_Clear(); \ 91 | PyErr_SetString( \ 92 | PyExc_TypeError, "integer out of range"); \ 93 | } \ 94 | (STATUS)=0; (TARGET)=0; \ 95 | } \ 96 | else if (vcopy < 0) { \ 97 | PyErr_SetString( \ 98 | PyExc_TypeError, \ 99 | "can't convert negative value to unsigned int"); \ 100 | (STATUS)=0; (TARGET)=0; \ 101 | } \ 102 | else if ((unsigned int)vcopy != vcopy) { \ 103 | PyErr_SetString(PyExc_TypeError, "integer out of range"); \ 104 | (STATUS)=0; (TARGET)=0; \ 105 | } \ 106 | else TARGET = vcopy; \ 107 | } else { \ 108 | PyErr_SetString(PyExc_TypeError, "expected integer key"); \ 109 | (STATUS)=0; (TARGET)=0; } 110 | 111 | #endif /* !defined(ZODB_64BIT_INTS) */ 112 | 113 | #endif /* ZODB_SIGNED_KEY_INTS */ 114 | 115 | #undef KEY_TYPE_IS_PYOBJECT 116 | 117 | #define TEST_KEY_SET_OR(V, K, T) if ( ( (V) = (((K) < (T)) ? -1 : (((K) > (T)) ? 1: 0)) ) , 0 ) 118 | #define DECREF_KEY(KEY) 119 | #define INCREF_KEY(k) 120 | #define COPY_KEY(KEY, E) (KEY=(E)) 121 | #define MULTI_INT_UNION 1 122 | -------------------------------------------------------------------------------- /src/BTrees/intvaluemacros.h: -------------------------------------------------------------------------------- 1 | 2 | #define VALUEMACROS_H "$Id$\n" 3 | 4 | /* 5 | VALUE_PARSE is used exclusively in SetOpTemplate.c to accept the weight 6 | values for merging. The PyArg_ParseTuple function it uses has no trivial way 7 | to express "unsigned with check", so in the unsigned case, passing negative 8 | values as weights will produce weird output no matter what VALUE_PARSE we 9 | use (because it will immediately get cast to an unsigned). 10 | */ 11 | 12 | #ifndef ZODB_UNSIGNED_VALUE_INTS 13 | /*signed values */ 14 | #ifdef ZODB_64BIT_INTS 15 | #define NEED_LONG_LONG_SUPPORT 16 | #define VALUE_TYPE PY_LONG_LONG 17 | #define VALUE_PARSE "L" 18 | 19 | #define NEED_LONG_LONG_AS_OBJECT 20 | #define COPY_VALUE_TO_OBJECT(O, K) O=longlong_as_object(K) 21 | 22 | #define NEED_LONG_LONG_CONVERT 23 | #define COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS) \ 24 | if (!longlong_convert((ARG), &TARGET)) \ 25 | { \ 26 | (STATUS)=0; (TARGET)=0; \ 27 | } 28 | #else 29 | #define VALUE_TYPE int 30 | #define VALUE_PARSE "i" 31 | #define COPY_VALUE_TO_OBJECT(O, K) O=PyLong_FromLong(K) 32 | 33 | #define COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS) \ 34 | if (PyLong_Check(ARG)) { \ 35 | long vcopy = PyLong_AsLong(ARG); \ 36 | if (PyErr_Occurred()) { \ 37 | if (PyErr_ExceptionMatches(PyExc_OverflowError)) { \ 38 | PyErr_Clear(); \ 39 | PyErr_SetString(PyExc_TypeError, "integer out of range"); \ 40 | } \ 41 | (STATUS)=0; (TARGET)=0; \ 42 | } \ 43 | else if ((int)vcopy != vcopy) { \ 44 | PyErr_SetString(PyExc_TypeError, "integer out of range"); \ 45 | (STATUS)=0; (TARGET)=0; \ 46 | } \ 47 | else TARGET = vcopy; \ 48 | } else { \ 49 | PyErr_SetString(PyExc_TypeError, "expected integer key"); \ 50 | (STATUS)=0; (TARGET)=0; } 51 | 52 | #endif 53 | #else 54 | /* unsigned values */ 55 | #ifdef ZODB_64BIT_INTS 56 | /* unsigned, 64-bit values */ 57 | #define NEED_LONG_LONG_SUPPORT 58 | #define VALUE_TYPE unsigned PY_LONG_LONG 59 | #define VALUE_PARSE "K" 60 | 61 | #define NEED_ULONG_LONG_AS_OBJECT 62 | #define COPY_VALUE_TO_OBJECT(O, K) O=ulonglong_as_object(K) 63 | 64 | #define NEED_ULONG_LONG_CONVERT 65 | #define COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS) \ 66 | if (!ulonglong_convert((ARG), &TARGET)) \ 67 | { \ 68 | (STATUS)=0; (TARGET)=0; \ 69 | } 70 | #else 71 | /* unsigned, 32-bit values */ 72 | #define VALUE_TYPE unsigned int 73 | #define VALUE_PARSE "I" 74 | #define COPY_VALUE_TO_OBJECT(O, K) O=PyLong_FromUnsignedLongLong(K) 75 | 76 | #define COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS) \ 77 | if (PyLong_Check(ARG)) { \ 78 | long vcopy = PyLong_AsLong(ARG); \ 79 | if (PyErr_Occurred()) { \ 80 | if (PyErr_ExceptionMatches(PyExc_OverflowError)) { \ 81 | PyErr_Clear(); \ 82 | PyErr_SetString( \ 83 | PyExc_TypeError, "integer out of range"); \ 84 | } \ 85 | (STATUS)=0; (TARGET)=0; \ 86 | } \ 87 | else if (vcopy < 0) { \ 88 | PyErr_SetString( \ 89 | PyExc_TypeError, \ 90 | "can't convert negative value to unsigned int"); \ 91 | (STATUS)=0; (TARGET)=0; \ 92 | } \ 93 | else if ((unsigned int)vcopy != vcopy) { \ 94 | PyErr_SetString(PyExc_TypeError, "integer out of range"); \ 95 | (STATUS)=0; (TARGET)=0; \ 96 | } \ 97 | else TARGET = vcopy; \ 98 | } else { \ 99 | PyErr_SetString(PyExc_TypeError, "expected integer key"); \ 100 | (STATUS)=0; (TARGET)=0; } 101 | 102 | #endif 103 | #endif 104 | 105 | #undef VALUE_TYPE_IS_PYOBJECT 106 | #define TEST_VALUE(K, T) (((K) < (T)) ? -1 : (((K) > (T)) ? 1: 0)) 107 | #define VALUE_SAME(VALUE, TARGET) ( (VALUE) == (TARGET) ) 108 | #define DECLARE_VALUE(NAME) VALUE_TYPE NAME 109 | #define DECREF_VALUE(k) 110 | #define INCREF_VALUE(k) 111 | #define COPY_VALUE(V, E) (V=(E)) 112 | 113 | #define NORMALIZE_VALUE(V, MIN) ((MIN) > 0) ? ((V)/=(MIN)) : 0 114 | 115 | #define MERGE_DEFAULT 1 116 | #define MERGE(O1, w1, O2, w2) ((O1)*(w1)+(O2)*(w2)) 117 | #define MERGE_WEIGHT(O, w) ((O)*(w)) 118 | -------------------------------------------------------------------------------- /src/BTrees/_fsBTree.c: -------------------------------------------------------------------------------- 1 | /*############################################################################ 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################*/ 14 | 15 | #define MASTER_ID "$Id$\n" 16 | 17 | /* fsBTree - FileStorage index BTree 18 | 19 | This BTree implements a mapping from 2-character strings 20 | to six-character strings. This allows us to efficiently store 21 | a FileStorage index as a nested mapping of 6-character oid prefix 22 | to mapping of 2-character oid suffix to 6-character (byte) file 23 | positions. 24 | */ 25 | 26 | typedef unsigned char char2[2]; 27 | typedef unsigned char char6[6]; 28 | 29 | /* Setup template macros */ 30 | 31 | #define PERSISTENT 32 | 33 | #define MOD_NAME_PREFIX "fs" 34 | 35 | 36 | 37 | 38 | #include "Python.h" 39 | /*#include "intkeymacros.h"*/ 40 | 41 | #define KEYMACROS_H "$Id$\n" 42 | #define KEY_TYPE char2 43 | #undef KEY_TYPE_IS_PYOBJECT 44 | #define KEY_CHECK(K) (PyBytes_Check(K) && PyBytes_GET_SIZE(K)==2) 45 | #define TEST_KEY_SET_OR(V, K, T) if ( ( (V) = ((*(K) < *(T) || (*(K) == *(T) && (K)[1] < (T)[1])) ? -1 : ((*(K) == *(T) && (K)[1] == (T)[1]) ? 0 : 1)) ), 0 ) 46 | #define DECREF_KEY(KEY) 47 | #define INCREF_KEY(k) 48 | #define COPY_KEY(KEY, E) (*(KEY)=*(E), (KEY)[1]=(E)[1]) 49 | #define COPY_KEY_TO_OBJECT(O, K) O=PyBytes_FromStringAndSize((const char*)K,2) 50 | #define COPY_KEY_FROM_ARG(TARGET, ARG, STATUS) \ 51 | if (KEY_CHECK(ARG)) memcpy(TARGET, PyBytes_AS_STRING(ARG), 2); else { \ 52 | PyErr_SetString(PyExc_TypeError, "expected two-character string key"); \ 53 | (STATUS)=0; } 54 | 55 | /*#include "intvaluemacros.h"*/ 56 | #define VALUEMACROS_H "$Id$\n" 57 | #define VALUE_TYPE char6 58 | #undef VALUE_TYPE_IS_PYOBJECT 59 | #define TEST_VALUE(K, T) memcmp(K,T,6) 60 | #define DECREF_VALUE(k) 61 | #define INCREF_VALUE(k) 62 | #define COPY_VALUE(V, E) (memcpy(V, E, 6)) 63 | #define COPY_VALUE_TO_OBJECT(O, K) O=PyBytes_FromStringAndSize((const char*)K,6) 64 | #define COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS) \ 65 | if ((PyBytes_Check(ARG) && PyBytes_GET_SIZE(ARG)==6)) \ 66 | memcpy(TARGET, PyBytes_AS_STRING(ARG), 6); else { \ 67 | PyErr_SetString(PyExc_TypeError, "expected six-character string key"); \ 68 | (STATUS)=0; } 69 | 70 | #define NORMALIZE_VALUE(V, MIN) 71 | 72 | #include "Python.h" 73 | 74 | static PyObject *bucket_toBytes(PyObject *self); 75 | 76 | static PyObject *bucket_fromBytes(PyObject *self, PyObject *state); 77 | 78 | #define EXTRA_BUCKET_METHODS \ 79 | {"toBytes", (PyCFunction) bucket_toBytes, METH_NOARGS, \ 80 | "toBytes() -- Return the state as a bytes array"}, \ 81 | {"fromBytes", (PyCFunction) bucket_fromBytes, METH_O, \ 82 | "fromSBytes(s) -- Set the state of the object from a bytes array"}, \ 83 | {"toString", (PyCFunction) bucket_toBytes, METH_NOARGS, \ 84 | "toString() -- Deprecated alias for 'toBytes'"}, \ 85 | {"fromString", (PyCFunction) bucket_fromBytes, METH_O, \ 86 | "fromString(s) -- Deprecated alias for 'fromBytes'"}, \ 87 | 88 | #define INITMODULE PyInit__fsBTree 89 | #include "BTreeModuleTemplate.c" 90 | 91 | static PyObject * 92 | bucket_toBytes(PyObject *oself) 93 | { 94 | Bucket *self = (Bucket *)oself; 95 | PyObject *items = NULL; 96 | int len; 97 | 98 | PER_USE_OR_RETURN(self, NULL); 99 | 100 | len = self->len; 101 | 102 | items = PyBytes_FromStringAndSize(NULL, len*8); 103 | if (items == NULL) 104 | goto err; 105 | memcpy(PyBytes_AS_STRING(items), self->keys, len*2); 106 | memcpy(PyBytes_AS_STRING(items)+len*2, self->values, len*6); 107 | 108 | PER_UNUSE(self); 109 | return items; 110 | 111 | err: 112 | PER_UNUSE(self); 113 | Py_XDECREF(items); 114 | return NULL; 115 | } 116 | 117 | static PyObject * 118 | bucket_fromBytes(PyObject *oself, PyObject *state) 119 | { 120 | Bucket *self = (Bucket *)oself; 121 | int len; 122 | KEY_TYPE *keys; 123 | VALUE_TYPE *values; 124 | 125 | len = PyBytes_Size(state); 126 | if (len < 0) 127 | return NULL; 128 | 129 | if (len%8) 130 | { 131 | PyErr_SetString(PyExc_ValueError, "state string of wrong size"); 132 | return NULL; 133 | } 134 | len /= 8; 135 | 136 | if (self->next) { 137 | Py_DECREF(self->next); 138 | self->next = NULL; 139 | } 140 | 141 | if (len > self->size) { 142 | keys = BTree_Realloc(self->keys, sizeof(KEY_TYPE)*len); 143 | if (keys == NULL) 144 | return NULL; 145 | values = BTree_Realloc(self->values, sizeof(VALUE_TYPE)*len); 146 | if (values == NULL) 147 | return NULL; 148 | self->keys = keys; 149 | self->values = values; 150 | self->size = len; 151 | } 152 | 153 | memcpy(self->keys, PyBytes_AS_STRING(state), len*2); 154 | memcpy(self->values, PyBytes_AS_STRING(state)+len*2, len*6); 155 | 156 | self->len = len; 157 | 158 | Py_INCREF(self); 159 | return (PyObject *)self; 160 | } 161 | -------------------------------------------------------------------------------- /include/persistent/persistent/cPersistence.h: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | 3 | Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | All Rights Reserved. 5 | 6 | This software is subject to the provisions of the Zope Public License, 7 | Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | FOR A PARTICULAR PURPOSE 12 | 13 | ****************************************************************************/ 14 | 15 | #ifndef CPERSISTENCE_H 16 | #define CPERSISTENCE_H 17 | 18 | #include "_compat.h" 19 | #include "bytesobject.h" 20 | 21 | #include "ring.h" 22 | 23 | #define CACHE_HEAD \ 24 | PyObject_HEAD \ 25 | CPersistentRing ring_home; \ 26 | int non_ghost_count; \ 27 | Py_ssize_t total_estimated_size; 28 | 29 | struct ccobject_head_struct; 30 | 31 | typedef struct ccobject_head_struct PerCache; 32 | 33 | /* How big is a persistent object? 34 | 35 | 12 PyGC_Head is two pointers and an int 36 | 8 PyObject_HEAD is an int and a pointer 37 | 38 | 12 jar, oid, cache pointers 39 | 8 ring struct 40 | 8 serialno 41 | 4 state + extra 42 | 4 size info 43 | 44 | (56) so far 45 | 46 | 4 dict ptr 47 | 4 weaklist ptr 48 | ------------------------- 49 | 68 only need 62, but obmalloc rounds up to multiple of eight 50 | 51 | Even a ghost requires 64 bytes. It's possible to make a persistent 52 | instance with slots and no dict, which changes the storage needed. 53 | 54 | */ 55 | 56 | #define cPersistent_HEAD \ 57 | PyObject_HEAD \ 58 | PyObject *jar; \ 59 | PyObject *oid; \ 60 | PerCache *cache; \ 61 | CPersistentRing ring; \ 62 | char serial[8]; \ 63 | signed state:8; \ 64 | unsigned estimated_size:24; 65 | 66 | /* We recently added estimated_size. We originally added it as a new 67 | unsigned long field after a signed char state field and a 68 | 3-character reserved field. This didn't work because there 69 | are packages in the wild that have their own copies of cPersistence.h 70 | that didn't see the update. 71 | 72 | To get around this, we used the reserved space by making 73 | estimated_size a 24-bit bit field in the space occupied by the old 74 | 3-character reserved field. To fit in 24 bits, we made the units 75 | of estimated_size 64-character blocks. This allows is to handle up 76 | to a GB. We should never see that, but to be paranoid, we also 77 | truncate sizes greater than 1GB. We also set the minimum size to 78 | 64 bytes. 79 | 80 | We use the _estimated_size_in_24_bits and _estimated_size_in_bytes 81 | macros both to avoid repetition and to make intent a little clearer. 82 | */ 83 | #define _estimated_size_in_24_bits(I) ((I) > 1073741696 ? 16777215 : (I)/64+1) 84 | #define _estimated_size_in_bytes(I) ((I)*64) 85 | 86 | #define cPersistent_GHOST_STATE -1 87 | #define cPersistent_UPTODATE_STATE 0 88 | #define cPersistent_CHANGED_STATE 1 89 | #define cPersistent_STICKY_STATE 2 90 | 91 | typedef struct { 92 | cPersistent_HEAD 93 | } cPersistentObject; 94 | 95 | typedef void (*percachedelfunc)(PerCache *, PyObject *); 96 | 97 | typedef struct { 98 | PyTypeObject *pertype; 99 | getattrofunc getattro; 100 | setattrofunc setattro; 101 | int (*changed)(cPersistentObject*); 102 | void (*accessed)(cPersistentObject*); 103 | void (*ghostify)(cPersistentObject*); 104 | int (*setstate)(PyObject*); 105 | percachedelfunc percachedel; 106 | int (*readCurrent)(cPersistentObject*); 107 | } cPersistenceCAPIstruct; 108 | 109 | #define cPersistenceType cPersistenceCAPI->pertype 110 | 111 | #ifndef DONT_USE_CPERSISTENCECAPI 112 | static cPersistenceCAPIstruct *cPersistenceCAPI; 113 | #endif 114 | 115 | #define cPersistanceModuleName "cPersistence" 116 | 117 | #define PER_TypeCheck(O) PyObject_TypeCheck((O), cPersistenceCAPI->pertype) 118 | 119 | #define PER_USE_OR_RETURN(O,R) {if((O)->state==cPersistent_GHOST_STATE && cPersistenceCAPI->setstate((PyObject*)(O)) < 0) return (R); else if ((O)->state==cPersistent_UPTODATE_STATE) (O)->state=cPersistent_STICKY_STATE;} 120 | 121 | #define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O))) 122 | 123 | #define PER_READCURRENT(O, E) \ 124 | if (cPersistenceCAPI->readCurrent((cPersistentObject*)(O)) < 0) { E; } 125 | 126 | #define PER_GHOSTIFY(O) (cPersistenceCAPI->ghostify((cPersistentObject*)(O))) 127 | 128 | /* If the object is sticky, make it non-sticky, so that it can be ghostified. 129 | The value is not meaningful 130 | */ 131 | #define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE)) 132 | 133 | #define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE)) 134 | 135 | /* 136 | Make a persistent object usable from C by: 137 | 138 | - Making sure it is not a ghost 139 | 140 | - Making it sticky. 141 | 142 | IMPORTANT: If you call this and don't call PER_ALLOW_DEACTIVATION, 143 | your object will not be ghostified. 144 | 145 | PER_USE returns a 1 on success and 0 failure, where failure means 146 | error. 147 | */ 148 | #define PER_USE(O) \ 149 | (((O)->state != cPersistent_GHOST_STATE \ 150 | || (cPersistenceCAPI->setstate((PyObject*)(O)) >= 0)) \ 151 | ? (((O)->state==cPersistent_UPTODATE_STATE) \ 152 | ? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0) 153 | 154 | #define PER_ACCESSED(O) (cPersistenceCAPI->accessed((cPersistentObject*)(O))) 155 | 156 | #endif 157 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. linkcheck to check all external links for integrity 37 | echo. doctest to run all doctests embedded in the documentation if enabled 38 | goto end 39 | ) 40 | 41 | if "%1" == "clean" ( 42 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 43 | del /q /s %BUILDDIR%\* 44 | goto end 45 | ) 46 | 47 | if "%1" == "html" ( 48 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 49 | if errorlevel 1 exit /b 1 50 | echo. 51 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 52 | goto end 53 | ) 54 | 55 | if "%1" == "dirhtml" ( 56 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 57 | if errorlevel 1 exit /b 1 58 | echo. 59 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 60 | goto end 61 | ) 62 | 63 | if "%1" == "singlehtml" ( 64 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 68 | goto end 69 | ) 70 | 71 | if "%1" == "pickle" ( 72 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished; now you can process the pickle files. 76 | goto end 77 | ) 78 | 79 | if "%1" == "json" ( 80 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished; now you can process the JSON files. 84 | goto end 85 | ) 86 | 87 | if "%1" == "htmlhelp" ( 88 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can run HTML Help Workshop with the ^ 92 | .hhp project file in %BUILDDIR%/htmlhelp. 93 | goto end 94 | ) 95 | 96 | if "%1" == "qthelp" ( 97 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 98 | if errorlevel 1 exit /b 1 99 | echo. 100 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 101 | .qhcp project file in %BUILDDIR%/qthelp, like this: 102 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\BTrees.qhcp 103 | echo.To view the help file: 104 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\BTrees.ghc 105 | goto end 106 | ) 107 | 108 | if "%1" == "devhelp" ( 109 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 110 | if errorlevel 1 exit /b 1 111 | echo. 112 | echo.Build finished. 113 | goto end 114 | ) 115 | 116 | if "%1" == "epub" ( 117 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 118 | if errorlevel 1 exit /b 1 119 | echo. 120 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 121 | goto end 122 | ) 123 | 124 | if "%1" == "latex" ( 125 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 129 | goto end 130 | ) 131 | 132 | if "%1" == "text" ( 133 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The text files are in %BUILDDIR%/text. 137 | goto end 138 | ) 139 | 140 | if "%1" == "man" ( 141 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 145 | goto end 146 | ) 147 | 148 | if "%1" == "texinfo" ( 149 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 150 | if errorlevel 1 exit /b 1 151 | echo. 152 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 153 | goto end 154 | ) 155 | 156 | if "%1" == "gettext" ( 157 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 158 | if errorlevel 1 exit /b 1 159 | echo. 160 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 161 | goto end 162 | ) 163 | 164 | if "%1" == "changes" ( 165 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 166 | if errorlevel 1 exit /b 1 167 | echo. 168 | echo.The overview file is in %BUILDDIR%/changes. 169 | goto end 170 | ) 171 | 172 | if "%1" == "linkcheck" ( 173 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 174 | if errorlevel 1 exit /b 1 175 | echo. 176 | echo.Link check complete; look for any errors in the above output ^ 177 | or in %BUILDDIR%/linkcheck/output.txt. 178 | goto end 179 | ) 180 | 181 | if "%1" == "doctest" ( 182 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 183 | if errorlevel 1 exit /b 1 184 | echo. 185 | echo.Testing of doctests in the sources finished, look at the ^ 186 | results in %BUILDDIR%/doctest/output.txt. 187 | goto end 188 | ) 189 | 190 | :end 191 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/BTrees.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/BTrees.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/BTrees" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/BTrees" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /src/BTrees/tests/test_OOBTree.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001-2012 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | from BTrees import OOBTree 15 | 16 | from ._test_builder import update_module 17 | from .common import BTreeTests 18 | 19 | 20 | class OOBTreeTest(BTreeTests): 21 | 22 | def test_byValue(self): 23 | ITEMS = [(y, x) for x, y in enumerate('abcdefghijklmnopqrstuvwxyz')] 24 | tree = self._makeOne(ITEMS) 25 | self.assertEqual(list(tree.byValue(22)), 26 | [(y, x) for x, y in reversed(ITEMS[22:])]) 27 | 28 | def testRejectDefaultComparisonOnSet(self): 29 | # Check that passing in keys w default comparison fails. Only 30 | # applies to new-style class instances if we're using the C 31 | # extensions; old-style instances are too hard to introspect 32 | # in C. 33 | 34 | # This is white box because we know that the check is being 35 | # used in a function that's used in lots of places. 36 | # Otherwise, there are many permutations that would have to be 37 | # checked. 38 | t = self._makeOne() 39 | 40 | class OldStyle: 41 | pass 42 | 43 | if self._getTargetClass() is OOBTree.OOBTreePy: 44 | with self.assertRaises(TypeError): 45 | t[OldStyle()] = 1 46 | 47 | class C: 48 | pass 49 | 50 | with self.assertRaises(TypeError) as raising: 51 | t[C()] = 1 52 | 53 | self.assertEqual( 54 | raising.exception.args[0], 55 | "Object of class C has default comparison") 56 | 57 | class With___lt__: 58 | def __lt__(*args): 59 | return 1 60 | 61 | c = With___lt__() 62 | t[c] = 1 63 | t.clear() 64 | 65 | class With___lt__Old: 66 | def __lt__(*args): 67 | return 1 68 | 69 | c = With___lt__Old() 70 | t[c] = 1 71 | 72 | t.clear() 73 | 74 | def testAcceptDefaultComparisonOnGet(self): 75 | # Issue #42 76 | t = self._makeOne() 77 | 78 | class C: 79 | pass 80 | 81 | self.assertEqual(t.get(C(), 42), 42) 82 | self.assertRaises(KeyError, t.__getitem__, C()) 83 | self.assertNotIn(C(), t) 84 | 85 | def testNewStyleClassWithCustomMetaClassAllowed(self): 86 | class Meta(type): 87 | def __lt__(cls, other): 88 | return 1 89 | 90 | cls = Meta('Class', (object,), {}) 91 | m = self._makeOne() 92 | m[cls] = self.getTwoValues()[0] 93 | 94 | def test_None_is_smallest(self): 95 | t = self._makeOne() 96 | for i in range(999): # Make sure we multiple buckets 97 | t[i] = i * i 98 | t[None] = -1 99 | for i in range(-99, 0): # Make sure we multiple buckets 100 | t[i] = i * i 101 | self.assertEqual(list(t), [None] + list(range(-99, 999))) 102 | self.assertEqual(list(t.values()), 103 | [-1] + [i * i for i in range(-99, 999)]) 104 | self.assertEqual(t[2], 4) 105 | self.assertEqual(t[-2], 4) 106 | self.assertEqual(t[None], -1) 107 | t[None] = -2 108 | self.assertEqual(t[None], -2) 109 | t2 = t.__class__(t) 110 | del t[None] 111 | self.assertEqual(list(t), list(range(-99, 999))) 112 | 113 | if 'Py' in self.__class__.__name__: 114 | return 115 | from BTrees.OOBTree import difference 116 | from BTrees.OOBTree import intersection 117 | from BTrees.OOBTree import union 118 | self.assertEqual(list(difference(t2, t).items()), [(None, -2)]) 119 | self.assertEqual(list(union(t, t2)), list(t2)) 120 | self.assertEqual(list(intersection(t, t2)), list(t)) 121 | 122 | def testDeleteNoneKey(self): 123 | # Check that a None key can be deleted in Python 2. 124 | # This doesn't work on Python 3 because None is unorderable, 125 | # so the tree can't be searched. But None also can't be inserted, 126 | # and we don't support migrating Python 2 databases to Python 3. 127 | t = self._makeOne() 128 | bucket_state = ((None, 42),) 129 | tree_state = ((bucket_state,),) 130 | t.__setstate__(tree_state) 131 | 132 | self.assertEqual(t[None], 42) 133 | del t[None] 134 | 135 | def testUnpickleNoneKey(self): 136 | # All versions (py2 and py3, C and Python) can unpickle 137 | # data that looks like this: {None: 42}, even though None 138 | # is unorderable.. 139 | # This pickle was captured in BTree/ZODB3 3.10.7 140 | import pickle 141 | 142 | data = ( 143 | b'ccopy_reg\n__newobj__\np0\n(' 144 | b'cBTrees.OOBTree\nOOBTree\np1\ntp2\nRp3\n(' 145 | b'(((NI42\ntp4\ntp5\ntp6\ntp7\nb.' 146 | ) 147 | 148 | t = pickle.loads(data) 149 | keys = list(t) 150 | self.assertEqual([None], keys) 151 | 152 | def testIdentityTrumpsBrokenComparison(self): 153 | # Identical keys always match, even if their comparison is 154 | # broken. See https://github.com/zopefoundation/BTrees/issues/50 155 | from functools import total_ordering 156 | 157 | @total_ordering 158 | class Bad: 159 | def __eq__(self, other): 160 | return False 161 | 162 | __lt__ = __cmp__ = __eq__ 163 | 164 | t = self._makeOne() 165 | bad_key = Bad() 166 | t[bad_key] = 42 167 | 168 | self.assertIn(bad_key, t) 169 | self.assertEqual(list(t), [bad_key]) 170 | 171 | del t[bad_key] 172 | self.assertNotIn(bad_key, t) 173 | self.assertEqual(list(t), []) 174 | 175 | 176 | update_module(globals(), OOBTree, btree_tests_base=OOBTreeTest) 177 | -------------------------------------------------------------------------------- /src/BTrees/_module_builder.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """ 15 | Support functions to eliminate the boilerplate involved in defining 16 | BTree modules. 17 | """ 18 | import sys 19 | 20 | from zope.interface import classImplements 21 | from zope.interface import directlyProvides 22 | 23 | 24 | def _create_classes( 25 | module_name, key_datatype, value_datatype, 26 | ): 27 | from ._base import MERGE # Won't always want this. 28 | from ._base import Bucket 29 | from ._base import Set 30 | from ._base import Tree 31 | from ._base import TreeSet 32 | from ._base import _TreeItems as TreeItems 33 | from ._base import _TreeIterator 34 | 35 | classes = {} 36 | 37 | prefix = key_datatype.prefix_code + value_datatype.prefix_code 38 | 39 | classes['TreeItems'] = classes['TreeItemsPy'] = TreeItems 40 | for base in ( 41 | Bucket, 42 | Set, 43 | (Tree, 'BTree'), 44 | TreeSet, 45 | (_TreeIterator, 'TreeIterator'), 46 | ): 47 | if isinstance(base, tuple): 48 | base, base_name = base 49 | else: 50 | base_name = base.__name__ 51 | 52 | # XXX: Consider defining these with their natural names 53 | # now and only aliasing them to 'Py' instead of the 54 | # opposite. That should make pickling easier. 55 | name = prefix + base_name + 'Py' 56 | cls = type(name, (base,), dict( 57 | _to_key=key_datatype, 58 | _to_value=value_datatype, 59 | MERGE=MERGE, 60 | MERGE_WEIGHT=value_datatype.apply_weight, 61 | MERGE_DEFAULT=value_datatype.multiplication_identity, 62 | # max_leaf_size and max_internal_size are set 63 | # for BTree and TreeSet later, when we do the same thing 64 | # for C. 65 | )) 66 | cls.__module__ = module_name 67 | key_datatype.add_extra_methods(base_name, cls) 68 | 69 | classes[cls.__name__] = cls 70 | # Importing the C extension does this for the non-py 71 | # classes. 72 | # TODO: Unify that. 73 | classes[base_name + 'Py'] = cls 74 | 75 | for cls in classes.values(): 76 | cls._mapping_type = classes['BucketPy'] 77 | cls._set_type = classes['SetPy'] 78 | 79 | if 'Set' in cls.__name__: 80 | cls._bucket_type = classes['SetPy'] 81 | else: 82 | cls._bucket_type = classes['BucketPy'] 83 | 84 | return classes 85 | 86 | 87 | def _create_set_operations(module_name, key_type, value_type, set_type): 88 | from ._base import difference 89 | from ._base import intersection 90 | from ._base import multiunion 91 | from ._base import set_operation 92 | from ._base import union 93 | from ._base import weightedIntersection 94 | from ._base import weightedUnion 95 | 96 | ops = { 97 | op.__name__ + 'Py': set_operation(op, set_type) 98 | for op in ( 99 | difference, intersection, 100 | union, 101 | ) + ( 102 | (weightedIntersection, weightedUnion,) 103 | if value_type.supports_value_union() 104 | else () 105 | ) + ( 106 | (multiunion,) 107 | if key_type.supports_value_union() 108 | else () 109 | ) 110 | } 111 | 112 | for key, op in ops.items(): 113 | op.__module__ = module_name 114 | op.__name__ = key 115 | 116 | # TODO: Pickling. These things should be looked up by name. 117 | return ops 118 | 119 | 120 | def _create_globals(module_name, key_datatype, value_datatype): 121 | classes = _create_classes(module_name, key_datatype, value_datatype) 122 | set_type = classes['SetPy'] 123 | set_ops = _create_set_operations( 124 | module_name, key_datatype, value_datatype, set_type, 125 | ) 126 | 127 | classes.update(set_ops) 128 | return classes 129 | 130 | 131 | def populate_module(mod_globals, 132 | key_datatype, value_datatype, 133 | interface, module=None): 134 | import collections.abc 135 | 136 | from . import Interfaces as interfaces 137 | from ._base import _fix_pickle 138 | from ._compat import import_c_extension 139 | 140 | module_name = mod_globals['__name__'] 141 | # Define the Python implementations 142 | mod_globals.update( 143 | _create_globals(module_name, key_datatype, value_datatype) 144 | ) 145 | # Import the C versions, if possible. Whether or not this is possible, 146 | # this currently makes the non-`Py' suffixed names available. This should 147 | # change if we start defining the Python classes with their natural name, 148 | # only aliased to the 'Py` suffix (which simplifies pickling) 149 | import_c_extension(mod_globals) 150 | 151 | # Next, define __all__ after all the name aliasing is done. 152 | # XXX: Maybe derive this from the values we create. 153 | mod_all = ( 154 | 'Bucket', 'Set', 'BTree', 'TreeSet', 155 | 'union', 'intersection', 'difference', 156 | 'weightedUnion', 'weightedIntersection', 'multiunion', 157 | ) 158 | prefix = key_datatype.prefix_code + value_datatype.prefix_code 159 | 160 | mod_all += tuple(prefix + c for c in ('Bucket', 'Set', 'BTree', 'TreeSet')) 161 | 162 | mod_globals['__all__'] = tuple(c for c in mod_all if c in mod_globals) 163 | 164 | mod_globals['using64bits'] = ( 165 | key_datatype.using64bits or value_datatype.using64bits 166 | ) 167 | 168 | # XXX: We can probably do better than fix_pickle now; 169 | # we can know if we're going to be renaming classes 170 | # ahead of time. See above. 171 | _fix_pickle(mod_globals, module_name) 172 | 173 | # Apply interface definitions. 174 | directlyProvides(module or sys.modules[module_name], interface) 175 | for cls_name, iface in { 176 | 'BTree': interfaces.IBTree, 177 | 'Bucket': interfaces.IMinimalDictionary, 178 | 'Set': interfaces.ISet, 179 | 'TreeSet': interfaces.ITreeSet, 180 | 'TreeItems': interfaces.IMinimalSequence, 181 | }.items(): 182 | classImplements(mod_globals[cls_name], iface) 183 | classImplements(mod_globals[cls_name + 'Py'], iface) 184 | 185 | for cls_name, abc in { 186 | 'BTree': collections.abc.MutableMapping, 187 | 'Bucket': collections.abc.MutableMapping, 188 | 'Set': collections.abc.MutableSet, 189 | 'TreeSet': collections.abc.MutableSet, 190 | }.items(): 191 | abc.register(mod_globals[cls_name]) 192 | # Because of some quirks in the implementation of 193 | # ABCMeta.__instancecheck__, and the shenanigans we currently do to 194 | # make Python classes pickle without the 'Py' suffix, it's not actually 195 | # necessary to register the Python version of the class. Specifically, 196 | # ABCMeta asks for the object's ``__class__`` instead of using 197 | # ``type()``, and our objects have a ``@property`` for ``__class__`` 198 | # that returns the C version. 199 | # 200 | # That's too many coincidences to rely on though. 201 | abc.register(mod_globals[cls_name + 'Py']) 202 | 203 | # Set node sizes. 204 | for cls_name in ('BTree', 'TreeSet'): 205 | 206 | for suffix in ('', 'Py'): 207 | cls = mod_globals[cls_name + suffix] 208 | cls.max_leaf_size = key_datatype.bucket_size_for_value( 209 | value_datatype 210 | ) 211 | cls.max_internal_size = key_datatype.tree_size 212 | 213 | 214 | def create_module(prefix): 215 | import types 216 | 217 | from . import Interfaces 218 | from . import _datatypes as datatypes 219 | 220 | mod = types.ModuleType('BTrees.' + prefix + 'BTree') 221 | 222 | key_type = getattr(datatypes, prefix[0])() 223 | val_type = getattr(datatypes, prefix[1])().as_value_type() 224 | 225 | iface_name = 'I' + key_type.long_name + val_type.long_name + 'BTreeModule' 226 | 227 | iface = getattr(Interfaces, iface_name) 228 | 229 | populate_module(vars(mod), key_type, val_type, iface, mod) 230 | return mod 231 | -------------------------------------------------------------------------------- /src/BTrees/tests/test_check.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2003 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | import unittest 15 | 16 | 17 | def _assertRaises(self, e_type, checked, *args, **kw): 18 | try: 19 | checked(*args, **kw) 20 | except e_type as e: 21 | return e 22 | self.fail("Didn't raise: %s" % e_type.__name__) 23 | 24 | 25 | class Test_classify(unittest.TestCase): 26 | 27 | def _callFUT(self, obj): 28 | from BTrees.check import classify 29 | return classify(obj) 30 | 31 | def test_classify_w_unknown(self): 32 | class NotClassified: 33 | pass 34 | self.assertRaises(KeyError, self._callFUT, NotClassified()) 35 | 36 | def test_classify_w_bucket(self): 37 | from BTrees.check import TYPE_BUCKET 38 | from BTrees.OOBTree import OOBucketPy 39 | kind, is_mapping = self._callFUT(OOBucketPy()) 40 | self.assertEqual(kind, TYPE_BUCKET) 41 | self.assertTrue(is_mapping) 42 | 43 | def test_classify_w_set(self): 44 | from BTrees.check import TYPE_BUCKET 45 | from BTrees.OOBTree import OOSetPy 46 | kind, is_mapping = self._callFUT(OOSetPy()) 47 | self.assertEqual(kind, TYPE_BUCKET) 48 | self.assertFalse(is_mapping) 49 | 50 | def test_classify_w_tree(self): 51 | from BTrees.check import TYPE_BTREE 52 | from BTrees.OOBTree import OOBTreePy 53 | kind, is_mapping = self._callFUT(OOBTreePy()) 54 | self.assertEqual(kind, TYPE_BTREE) 55 | self.assertTrue(is_mapping) 56 | 57 | def test_classify_w_treeset(self): 58 | from BTrees.check import TYPE_BTREE 59 | from BTrees.OOBTree import OOTreeSetPy 60 | kind, is_mapping = self._callFUT(OOTreeSetPy()) 61 | self.assertEqual(kind, TYPE_BTREE) 62 | self.assertFalse(is_mapping) 63 | 64 | 65 | class Test_crack_btree(unittest.TestCase): 66 | 67 | def _callFUT(self, obj, is_mapping): 68 | from BTrees.check import crack_btree 69 | return crack_btree(obj, is_mapping) 70 | 71 | def test_w_empty_tree(self): 72 | from BTrees.check import BTREE_EMPTY 73 | 74 | class Empty: 75 | def __getstate__(self): 76 | return None 77 | 78 | kind, keys, kids = self._callFUT(Empty(), True) 79 | self.assertEqual(kind, BTREE_EMPTY) 80 | self.assertEqual(keys, []) 81 | self.assertEqual(kids, []) 82 | 83 | def test_w_degenerate_tree(self): 84 | from BTrees.check import BTREE_ONE 85 | 86 | class Degenerate: 87 | def __getstate__(self): 88 | return ((('a', 1, 'b', 2),),) 89 | 90 | kind, keys, kids = self._callFUT(Degenerate(), True) 91 | self.assertEqual(kind, BTREE_ONE) 92 | self.assertEqual(keys, ('a', 1, 'b', 2)) 93 | self.assertEqual(kids, None) 94 | 95 | def test_w_normal_tree(self): 96 | from BTrees.check import BTREE_NORMAL 97 | first_bucket = [object()] * 8 98 | second_bucket = [object()] * 8 99 | 100 | class Normal: 101 | def __getstate__(self): 102 | return ((first_bucket, 'b', second_bucket), first_bucket) 103 | 104 | kind, keys, kids = self._callFUT(Normal(), True) 105 | self.assertEqual(kind, BTREE_NORMAL) 106 | self.assertEqual(keys, ['b']) 107 | self.assertEqual(kids, [first_bucket, second_bucket]) 108 | 109 | 110 | class Test_crack_bucket(unittest.TestCase): 111 | 112 | def _callFUT(self, obj, is_mapping): 113 | from BTrees.check import crack_bucket 114 | return crack_bucket(obj, is_mapping) 115 | 116 | def test_w_empty_set(self): 117 | 118 | class EmptySet: 119 | def __getstate__(self): 120 | return ([],) 121 | 122 | keys, values = self._callFUT(EmptySet(), False) 123 | self.assertEqual(keys, []) 124 | self.assertEqual(values, []) 125 | 126 | def test_w_non_empty_set(self): 127 | 128 | class NonEmptySet: 129 | def __getstate__(self): 130 | return (['a', 'b', 'c'],) 131 | 132 | keys, values = self._callFUT(NonEmptySet(), False) 133 | self.assertEqual(keys, ['a', 'b', 'c']) 134 | self.assertEqual(values, []) 135 | 136 | def test_w_empty_mapping(self): 137 | 138 | class EmptyMapping: 139 | def __getstate__(self): 140 | return ([], object()) 141 | 142 | keys, values = self._callFUT(EmptyMapping(), True) 143 | self.assertEqual(keys, []) 144 | self.assertEqual(values, []) 145 | 146 | def test_w_non_empty_mapping(self): 147 | 148 | class NonEmptyMapping: 149 | def __getstate__(self): 150 | return (['a', 1, 'b', 2, 'c', 3], object()) 151 | 152 | keys, values = self._callFUT(NonEmptyMapping(), True) 153 | self.assertEqual(keys, ['a', 'b', 'c']) 154 | self.assertEqual(values, [1, 2, 3]) 155 | 156 | 157 | class Test_type_and_adr(unittest.TestCase): 158 | 159 | def _callFUT(self, obj): 160 | from BTrees.check import type_and_adr 161 | return type_and_adr(obj) 162 | 163 | def test_type_and_adr_w_oid(self): 164 | from BTrees.utils import oid_repr 165 | 166 | class WithOid: 167 | _p_oid = b'DEADBEEF' 168 | 169 | t_and_a = self._callFUT(WithOid()) 170 | self.assertTrue(t_and_a.startswith('WithOid (0x')) 171 | self.assertTrue(t_and_a.endswith('oid=%s)' % oid_repr(b'DEADBEEF'))) 172 | 173 | def test_type_and_adr_wo_oid(self): 174 | class WithoutOid: 175 | pass 176 | t_and_a = self._callFUT(WithoutOid()) 177 | self.assertTrue(t_and_a.startswith('WithoutOid (0x')) 178 | self.assertTrue(t_and_a.endswith('oid=None)')) 179 | 180 | 181 | class WalkerTests(unittest.TestCase): 182 | 183 | def _getTargetClass(self): 184 | from BTrees.check import Walker 185 | return Walker 186 | 187 | def _makeOne(self, obj): 188 | return self._getTargetClass()(obj) 189 | 190 | def test_visit_btree_abstract(self): 191 | walker = self._makeOne(object()) 192 | obj = object() 193 | path = '/' 194 | parent = object() 195 | is_mapping = True 196 | keys = [] 197 | kids = [] 198 | lo = 0 199 | hi = None 200 | self.assertRaises(NotImplementedError, walker.visit_btree, 201 | obj, path, parent, is_mapping, keys, kids, lo, hi) 202 | 203 | def test_visit_bucket_abstract(self): 204 | walker = self._makeOne(object()) 205 | obj = object() 206 | path = '/' 207 | parent = object() 208 | is_mapping = True 209 | keys = [] 210 | kids = [] 211 | lo = 0 212 | hi = None 213 | self.assertRaises(NotImplementedError, walker.visit_bucket, 214 | obj, path, parent, is_mapping, keys, kids, lo, hi) 215 | 216 | def test_walk_w_empty_bucket(self): 217 | from BTrees.OOBTree import OOBucket 218 | obj = OOBucket() 219 | walker = self._makeOne(obj) 220 | self.assertRaises(NotImplementedError, walker.walk) 221 | 222 | def test_walk_w_empty_btree(self): 223 | from BTrees.OOBTree import OOBTree 224 | obj = OOBTree() 225 | walker = self._makeOne(obj) 226 | self.assertRaises(NotImplementedError, walker.walk) 227 | 228 | def test_walk_w_degenerate_btree(self): 229 | from BTrees.OOBTree import OOBTree 230 | obj = OOBTree() 231 | obj['a'] = 1 232 | walker = self._makeOne(obj) 233 | self.assertRaises(NotImplementedError, walker.walk) 234 | 235 | def test_walk_w_normal_btree(self): 236 | from BTrees.IIBTree import IIBTree 237 | obj = IIBTree() 238 | for i in range(1000): 239 | obj[i] = i 240 | walker = self._makeOne(obj) 241 | self.assertRaises(NotImplementedError, walker.walk) 242 | 243 | 244 | class CheckerTests(unittest.TestCase): 245 | 246 | assertRaises = _assertRaises 247 | 248 | def _getTargetClass(self): 249 | from BTrees.check import Checker 250 | return Checker 251 | 252 | def _makeOne(self, obj): 253 | return self._getTargetClass()(obj) 254 | 255 | def test_walk_w_empty_bucket(self): 256 | from BTrees.OOBTree import OOBucket 257 | obj = OOBucket() 258 | checker = self._makeOne(obj) 259 | checker.check() # noraise 260 | 261 | def test_walk_w_empty_btree(self): 262 | obj = _makeTree(False) 263 | checker = self._makeOne(obj) 264 | checker.check() # noraise 265 | 266 | def test_walk_w_degenerate_btree(self): 267 | obj = _makeTree(False) 268 | obj['a'] = 1 269 | checker = self._makeOne(obj) 270 | checker.check() # noraise 271 | 272 | def test_walk_w_normal_btree(self): 273 | obj = _makeTree(False) 274 | checker = self._makeOne(obj) 275 | checker.check() # noraise 276 | 277 | def test_walk_w_key_too_large(self): 278 | obj = _makeTree(True) 279 | state = obj.__getstate__() 280 | # Damage an invariant by dropping the BTree key to 14. 281 | new_state = (state[0][0], 14, state[0][2]), state[1] 282 | obj.__setstate__(new_state) 283 | checker = self._makeOne(obj) 284 | e = self.assertRaises(AssertionError, checker.check) 285 | self.assertIn(">= upper bound", str(e)) 286 | 287 | def test_walk_w_key_too_small(self): 288 | obj = _makeTree(True) 289 | state = obj.__getstate__() 290 | # Damage an invariant by bumping the BTree key to 16. 291 | new_state = (state[0][0], 16, state[0][2]), state[1] 292 | obj.__setstate__(new_state) 293 | checker = self._makeOne(obj) 294 | e = self.assertRaises(AssertionError, checker.check) 295 | self.assertIn("< lower bound", str(e)) 296 | 297 | def test_walk_w_keys_swapped(self): 298 | obj = _makeTree(True) 299 | state = obj.__getstate__() 300 | # Damage an invariant by bumping the BTree key to 16. 301 | (b0, num, b1), firstbucket = state 302 | self.assertEqual(b0[4], 8) 303 | self.assertEqual(b0[5], 10) 304 | b0state = b0.__getstate__() 305 | self.assertEqual(len(b0state), 2) 306 | # b0state looks like 307 | # ((k0, v0, k1, v1, ...), nextbucket) 308 | pairs, nextbucket = b0state 309 | self.assertEqual(pairs[8], 4) 310 | self.assertEqual(pairs[9], 8) 311 | self.assertEqual(pairs[10], 5) 312 | self.assertEqual(pairs[11], 10) 313 | newpairs = pairs[:8] + (5, 10, 4, 8) + pairs[12:] 314 | b0.__setstate__((newpairs, nextbucket)) 315 | checker = self._makeOne(obj) 316 | e = self.assertRaises(AssertionError, checker.check) 317 | self.assertIn("key 5 at index 4 >= key 4 at index 5", str(e)) 318 | 319 | 320 | class Test_check(unittest.TestCase): 321 | 322 | def _callFUT(self, tree): 323 | from BTrees.check import check 324 | return check(tree) 325 | 326 | def _makeOne(self): 327 | from BTrees.OOBTree import OOBTree 328 | tree = OOBTree() 329 | for i in range(31): 330 | tree[i] = 2 * i 331 | return tree 332 | 333 | def test_normal(self): 334 | from BTrees.OOBTree import OOBTree 335 | tree = OOBTree() 336 | for i in range(31): 337 | tree[i] = 2 * i 338 | state = tree.__getstate__() 339 | self.assertEqual(len(state), 2) 340 | self.assertEqual(len(state[0]), 3) 341 | self.assertEqual(state[0][1], 15) 342 | self._callFUT(tree) # noraise 343 | 344 | 345 | def _makeTree(fill): 346 | from BTrees.OOBTree import OOBTree 347 | from BTrees.OOBTree import OOBTreePy 348 | tree = OOBTree() 349 | if fill: 350 | for i in range(OOBTreePy.max_leaf_size + 1): 351 | tree[i] = 2 * i 352 | return tree 353 | -------------------------------------------------------------------------------- /src/BTrees/tests/_test_builder.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | 15 | import unittest 16 | 17 | from .common import BTreeTests 18 | from .common import ExtendedSetTests 19 | from .common import I_SetsBase 20 | from .common import InternalKeysMappingTest 21 | from .common import MappingBase 22 | from .common import MappingConflictTestBase 23 | from .common import ModuleTest 24 | from .common import MultiUnion 25 | from .common import NormalSetTests 26 | from .common import SetConflictTestBase 27 | from .common import SetResult 28 | from .common import TestLongIntKeys 29 | from .common import TestLongIntValues 30 | from .common import Weighted 31 | from .common import itemsToSet 32 | from .common import makeMapBuilder 33 | from .common import makeSetBuilder 34 | 35 | 36 | class _FilteredModuleProxy: 37 | """ 38 | Accesses either ```` or ``Py`` from a module. 39 | 40 | This conveniently lets us avoid lots of 'getattr' calls. 41 | 42 | Accessing ``def_`` returns a callable that 43 | returns ````. This is suitable for use as class attributes. 44 | """ 45 | # Lets us easily access by name a particular attribute 46 | # in either the Python or C implementation, based on the 47 | # suffix 48 | 49 | def __init__(self, btree_module, suffix): 50 | self.btree_module = btree_module 51 | self.suffix = suffix 52 | 53 | def __getattr__(self, name): 54 | attr_name = name[4:] if name.startswith('def_') else name 55 | attr_name += self.suffix 56 | attr = getattr(self.btree_module, attr_name) 57 | if name.startswith('def_'): 58 | return staticmethod(lambda: attr) 59 | return attr 60 | 61 | 62 | def _flattened(*args): 63 | def f(tuple_or_klass): 64 | if isinstance(tuple_or_klass, tuple): 65 | for x in tuple_or_klass: 66 | yield from f(x) 67 | else: 68 | yield tuple_or_klass 69 | 70 | return tuple(f(args)) 71 | 72 | 73 | class ClassBuilder: 74 | 75 | # Use TestAuto as a prefix to avoid clashing with manual tests 76 | TESTCASE_PREFIX = 'TestAuto' 77 | 78 | def __init__(self, btree_module, btree_tests_base=BTreeTests): 79 | self.btree_module = btree_module 80 | # These will be instances of _datatypes.DataType 81 | self.key_type = btree_module.BTreePy._to_key 82 | self.value_type = btree_module.BTreePy._to_value 83 | 84 | class _BoundsMixin: 85 | # For test purposes, we can only support negative keys if they are 86 | # ordered like integers. Our int -> 2 byte conversion for fsBTree 87 | # doesn't do this. 88 | # 89 | # -1 is \xff\xff which is the largest possible key. 90 | SUPPORTS_NEGATIVE_KEYS = ( 91 | self.key_type.get_lower_bound() != 0 92 | and self.key_type.coerce(-1) < self.key_type.coerce(0) 93 | ) 94 | SUPPORTS_NEGATIVE_VALUES = self.value_type.get_lower_bound() != 0 95 | if SUPPORTS_NEGATIVE_KEYS: 96 | KEY_RANDRANGE_ARGS = (-2000, 2001) 97 | else: 98 | KEY_RANDRANGE_ARGS = (0, 4002) 99 | 100 | coerce_to_key = self.key_type.coerce 101 | coerce_to_value = self.value_type.coerce 102 | KEYS = tuple(self.key_type.coerce(x) for x in range(2001)) 103 | VALUES = tuple(self.value_type.coerce(x) for x in range(2001)) 104 | 105 | self.bounds_mixin = _BoundsMixin 106 | 107 | self.btree_tests_base = btree_tests_base 108 | 109 | self.prefix = btree_module.__name__.split('.', )[-1][:2] 110 | self.test_module = 'BTrees.tests.test_' + self.prefix + 'BTree' 111 | 112 | self.test_classes = {} 113 | # Keep track of tested classes so that we don't 114 | # double test in PURE_PYTHON mode (e.g., BTreePy is BTree) 115 | self.tested_classes = set() 116 | 117 | def _store_class(self, test_cls): 118 | assert test_cls.__name__ not in self.test_classes 119 | assert isinstance(test_cls, type) 120 | assert issubclass(test_cls, unittest.TestCase) 121 | self.test_classes[test_cls.__name__] = test_cls 122 | 123 | def _fixup_and_store_class(self, btree_module, fut, test_cls): 124 | base = [x for x in test_cls.__bases__ 125 | if x.__module__ != __name__ and x.__module__ != 'unittest'][0] 126 | 127 | test_name = self._name_for_test(btree_module, fut, base) 128 | test_cls.__name__ = test_name 129 | test_cls.__module__ = self.test_module 130 | test_cls.__qualname__ = self.test_module + '.' + test_name 131 | self._store_class(test_cls) 132 | 133 | def _name_for_test(self, btree_module, fut, test_base): 134 | fut = getattr(fut, '__name__', fut) 135 | fut = str(fut) 136 | if isinstance(test_base, tuple): 137 | test_base = test_base[0] 138 | test_name = ( 139 | self.TESTCASE_PREFIX 140 | + (self.prefix if not fut.startswith(self.prefix) else '') 141 | + fut 142 | + test_base.__name__ 143 | + btree_module.suffix 144 | ) 145 | return test_name 146 | 147 | def _needs_test(self, fut, test_base): 148 | key = (fut, test_base) 149 | if key in self.tested_classes: 150 | return False 151 | self.tested_classes.add(key) 152 | return True 153 | 154 | def _create_set_op_test(self, btree_module, base): 155 | tree = btree_module.BTree 156 | if not self._needs_test(tree, base): 157 | return 158 | 159 | class Test(self.bounds_mixin, base, unittest.TestCase): 160 | # There are two set operation tests, 161 | # Weighted and MultiUnion. 162 | 163 | # These attributes are used in both 164 | mkbucket = btree_module.Bucket 165 | # Weighted uses union as a factory, self.union()(...). 166 | # MultiUnion calls it directly. 167 | __union = btree_module.def_union 168 | 169 | def union(self, *args): 170 | if args: 171 | return self.__union()(*args) 172 | return self.__union() 173 | 174 | intersection = btree_module.def_intersection 175 | # These are specific to Weighted; modules that 176 | # don't have weighted values can'd do them. 177 | if base is Weighted: 178 | weightedUnion = btree_module.def_weightedUnion 179 | weightedIntersection = btree_module.def_weightedIntersection 180 | 181 | # These are specific to MultiUnion, and may not exist 182 | # in key types that don't support unions (``'O'``) 183 | multiunion = getattr(btree_module, 'multiunion', None) 184 | mkset = btree_module.Set 185 | mktreeset = btree_module.TreeSet 186 | mkbtree = tree 187 | 188 | def builders(self): 189 | return ( 190 | btree_module.Bucket, 191 | btree_module.BTree, 192 | itemsToSet(btree_module.Set), 193 | itemsToSet(btree_module.TreeSet) 194 | ) 195 | 196 | self._fixup_and_store_class(btree_module, '', Test) 197 | 198 | def _create_set_result_test(self, btree_module): 199 | tree = btree_module.BTree 200 | base = SetResult 201 | if not self._needs_test(tree, base): 202 | return 203 | 204 | class Test(self.bounds_mixin, base, unittest.TestCase): 205 | union = btree_module.union 206 | intersection = btree_module.intersection 207 | difference = btree_module.difference 208 | 209 | def builders(self): 210 | return ( 211 | makeSetBuilder(self, btree_module.Set), 212 | makeSetBuilder(self, btree_module.TreeSet), 213 | makeMapBuilder(self, btree_module.BTree), 214 | makeMapBuilder(self, btree_module.Bucket) 215 | ) 216 | 217 | self._fixup_and_store_class(btree_module, '', Test) 218 | 219 | def _create_module_test(self): 220 | from BTrees import Interfaces as interfaces 221 | mod = self.btree_module 222 | iface_name = ( 223 | f'I{self.key_type.long_name}{self.value_type.long_name}' 224 | f'BTreeModule' 225 | ) 226 | iface = getattr(interfaces, iface_name) 227 | 228 | class Test(ModuleTest, unittest.TestCase): 229 | prefix = self.prefix 230 | key_type = self.key_type 231 | value_type = self.value_type 232 | 233 | def _getModule(self): 234 | return mod 235 | 236 | def _getInterface(self): 237 | return iface 238 | 239 | self._fixup_and_store_class( 240 | _FilteredModuleProxy(self.btree_module, ''), '', Test 241 | ) 242 | 243 | def _create_type_tests(self, btree_module, type_name, test_bases): 244 | from BTrees import Interfaces as interfaces 245 | tree = getattr(btree_module, type_name) 246 | iface = { 247 | 'BTree': interfaces.IBTree, 248 | 'Bucket': interfaces.IMinimalDictionary, 249 | 'Set': interfaces.ISet, 250 | 'TreeSet': interfaces.ITreeSet 251 | }[type_name] 252 | 253 | for test_base in test_bases: 254 | if not self._needs_test(tree, test_base): 255 | continue 256 | 257 | test_name = self._name_for_test(btree_module, tree, test_base) 258 | bases = _flattened(self.bounds_mixin, test_base, unittest.TestCase) 259 | test_cls = type(test_name, bases, { 260 | '__module__': self.test_module, 261 | '_getTargetClass': lambda _, t=tree: t, 262 | '_getTargetInterface': lambda _, i=iface: i, 263 | 'getTwoKeys': self.key_type.getTwoExamples, 264 | 'getTwoValues': self.value_type.getTwoExamples, 265 | 'key_type': self.key_type, 266 | 'value_type': self.value_type, 267 | }) 268 | self._store_class(test_cls) 269 | 270 | def create_classes(self): 271 | self._create_module_test() 272 | 273 | btree_tests_base = (self.btree_tests_base,) 274 | if self.key_type.using64bits: 275 | btree_tests_base += (TestLongIntKeys,) 276 | if self.value_type.using64bits: 277 | btree_tests_base += (TestLongIntValues,) 278 | 279 | set_ops = () 280 | if self.key_type.supports_value_union(): 281 | set_ops += (MultiUnion,) 282 | if self.value_type.supports_value_union(): 283 | set_ops += (Weighted,) 284 | 285 | for suffix in ('', 'Py'): 286 | btree_module = _FilteredModuleProxy(self.btree_module, suffix) 287 | 288 | for type_name, test_bases in ( 289 | ('BTree', (InternalKeysMappingTest, 290 | MappingConflictTestBase, 291 | btree_tests_base)), 292 | ('Bucket', (MappingBase, 293 | MappingConflictTestBase,)), 294 | ('Set', (ExtendedSetTests, 295 | I_SetsBase, 296 | SetConflictTestBase,)), 297 | ('TreeSet', (I_SetsBase, 298 | NormalSetTests, 299 | SetConflictTestBase,)) 300 | ): 301 | self._create_type_tests(btree_module, type_name, test_bases) 302 | 303 | for test_base in set_ops: 304 | self._create_set_op_test(btree_module, test_base) 305 | 306 | self._create_set_result_test(btree_module) 307 | 308 | 309 | def update_module(test_module_globals, btree_module, *args, **kwargs): 310 | builder = ClassBuilder(btree_module, *args, **kwargs) 311 | builder.create_classes() 312 | test_module_globals.update(builder.test_classes) 313 | -------------------------------------------------------------------------------- /src/BTrees/MergeTemplate.c: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | 3 | Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | All Rights Reserved. 5 | 6 | This software is subject to the provisions of the Zope Public License, 7 | Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | FOR A PARTICULAR PURPOSE 12 | 13 | ****************************************************************************/ 14 | 15 | #define MERGETEMPLATE_C "$Id$\n" 16 | 17 | /**************************************************************************** 18 | Set operations 19 | ****************************************************************************/ 20 | 21 | static int 22 | merge_output(Bucket *r, SetIteration *i, int mapping) 23 | { 24 | if (r->len >= r->size && Bucket_grow(r, -1, !mapping) < 0) 25 | return -1; 26 | COPY_KEY(r->keys[r->len], i->key); 27 | INCREF_KEY(r->keys[r->len]); 28 | if (mapping) { 29 | COPY_VALUE(r->values[r->len], i->value); 30 | INCREF_VALUE(r->values[r->len]); 31 | } 32 | r->len++; 33 | return 0; 34 | } 35 | 36 | /* The "reason" argument is a little integer giving "a reason" for the 37 | * error. In the Zope3 codebase, these are mapped to explanatory strings 38 | * via zodb/btrees/interfaces.py. 39 | */ 40 | static PyObject * 41 | merge_error(int p1, int p2, int p3, int reason) 42 | { 43 | PyObject *r; 44 | 45 | UNLESS (r=Py_BuildValue("iiii", p1, p2, p3, reason)) r=Py_None; 46 | if (ConflictError == NULL) { 47 | ConflictError = PyExc_ValueError; 48 | Py_INCREF(ConflictError); 49 | } 50 | PyErr_SetObject(ConflictError, r); 51 | if (r != Py_None) 52 | { 53 | Py_DECREF(r); 54 | } 55 | 56 | return NULL; 57 | } 58 | 59 | /* It's hard to explain "the rules" for bucket_merge, in large part because 60 | * any automatic conflict-resolution scheme is going to be incorrect for 61 | * some endcases of *some* app. The scheme here is pretty conservative, 62 | * and should be OK for most apps. It's easier to explain what the code 63 | * allows than what it forbids: 64 | * 65 | * Leaving things alone: it's OK if both s2 and s3 leave a piece of s1 66 | * alone (don't delete the key, and don't change the value). 67 | * 68 | * Key deletion: a transaction (s2 or s3) can delete a key (from s1), but 69 | * only if the other transaction (of s2 and s3) doesn't delete the same key. 70 | * However, it's not OK for s2 and s3 to, between them, end up deleting all 71 | * the keys. This is a higher-level constraint, due to that the caller of 72 | * bucket_merge() doesn't have enough info to unlink the resulting empty 73 | * bucket from its BTree correctly. It's also not OK if s2 or s3 are empty, 74 | * because the transaction that emptied the bucket unlinked the bucket from 75 | * the tree, and nothing we do here can get it linked back in again. 76 | * 77 | * Key insertion: s2 or s3 can add a new key, provided the other transaction 78 | * doesn't insert the same key. It's not OK even if they insert the same 79 | * pair. 80 | * 81 | * Mapping value modification: s2 or s3 can modify the value associated 82 | * with a key in s1, provided the other transaction doesn't make a 83 | * modification of the same key to a different value. It's OK if s2 and s3 84 | * both give the same new value to the key while it's hard to be precise about 85 | * why, this doesn't seem consistent with that it's *not* OK for both to add 86 | * a new key mapping to the same value). 87 | */ 88 | static PyObject * 89 | bucket_merge(Bucket *s1, Bucket *s2, Bucket *s3) 90 | { 91 | Bucket *r=0; 92 | PyObject *s; 93 | SetIteration i1 = {0,0,0}, i2 = {0,0,0}, i3 = {0,0,0}; 94 | int cmp12, cmp13, cmp23, mapping, set; 95 | 96 | /* If either "after" bucket is empty, punt. */ 97 | if (s2->len == 0 || s3->len == 0) 98 | { 99 | merge_error(-1, -1, -1, 12); 100 | goto err; 101 | } 102 | 103 | if (initSetIteration(&i1, OBJECT(s1), 1) < 0) 104 | goto err; 105 | if (initSetIteration(&i2, OBJECT(s2), 1) < 0) 106 | goto err; 107 | if (initSetIteration(&i3, OBJECT(s3), 1) < 0) 108 | goto err; 109 | 110 | mapping = i1.usesValue | i2.usesValue | i3.usesValue; 111 | set = !mapping; 112 | 113 | if (mapping) 114 | r = (Bucket *)PyObject_CallObject((PyObject *)&BucketType, NULL); 115 | else 116 | r = (Bucket *)PyObject_CallObject((PyObject *)&SetType, NULL); 117 | if (r == NULL) 118 | goto err; 119 | 120 | if (i1.next(&i1) < 0) 121 | goto err; 122 | if (i2.next(&i2) < 0) 123 | goto err; 124 | if (i3.next(&i3) < 0) 125 | goto err; 126 | 127 | /* Consult zodb/btrees/interfaces.py for the meaning of the last 128 | * argument passed to merge_error(). 129 | */ 130 | /* TODO: This isn't passing on errors raised by value comparisons. */ 131 | while (i1.position >= 0 && i2.position >= 0 && i3.position >= 0) 132 | { 133 | TEST_KEY_SET_OR(cmp12, i1.key, i2.key) goto err; 134 | TEST_KEY_SET_OR(cmp13, i1.key, i3.key) goto err; 135 | if (cmp12==0) 136 | { 137 | if (cmp13==0) 138 | { 139 | if (set || (TEST_VALUE(i1.value, i2.value) == 0)) 140 | { /* change in i3 value or all same */ 141 | if (merge_output(r, &i3, mapping) < 0) goto err; 142 | } 143 | else if (set || (TEST_VALUE(i1.value, i3.value) == 0)) 144 | { /* change in i2 value */ 145 | if (merge_output(r, &i2, mapping) < 0) goto err; 146 | } 147 | else 148 | { /* conflicting value changes in i2 and i3 */ 149 | merge_error(i1.position, i2.position, i3.position, 1); 150 | goto err; 151 | } 152 | if (i1.next(&i1) < 0) goto err; 153 | if (i2.next(&i2) < 0) goto err; 154 | if (i3.next(&i3) < 0) goto err; 155 | } 156 | else if (cmp13 > 0) 157 | { /* insert i3 */ 158 | if (merge_output(r, &i3, mapping) < 0) goto err; 159 | if (i3.next(&i3) < 0) goto err; 160 | } 161 | else if (set || (TEST_VALUE(i1.value, i2.value) == 0)) 162 | { /* deleted in i3 */ 163 | if (i3.position == 1) 164 | { 165 | /* Deleted the first item. This will modify the 166 | parent node, so we don't know if merging will be 167 | safe 168 | */ 169 | merge_error(i1.position, i2.position, i3.position, 13); 170 | goto err; 171 | } 172 | if (i1.next(&i1) < 0) goto err; 173 | if (i2.next(&i2) < 0) goto err; 174 | } 175 | else 176 | { /* conflicting del in i3 and change in i2 */ 177 | merge_error(i1.position, i2.position, i3.position, 2); 178 | goto err; 179 | } 180 | } 181 | else if (cmp13 == 0) 182 | { 183 | if (cmp12 > 0) 184 | { /* insert i2 */ 185 | if (merge_output(r, &i2, mapping) < 0) goto err; 186 | if (i2.next(&i2) < 0) goto err; 187 | } 188 | else if (set || (TEST_VALUE(i1.value, i3.value) == 0)) 189 | { /* deleted in i2 */ 190 | if (i2.position == 1) 191 | { 192 | /* Deleted the first item. This will modify the 193 | parent node, so we don't know if merging will be 194 | safe 195 | */ 196 | merge_error(i1.position, i2.position, i3.position, 13); 197 | goto err; 198 | } 199 | if (i1.next(&i1) < 0) goto err; 200 | if (i3.next(&i3) < 0) goto err; 201 | } 202 | else 203 | { /* conflicting del in i2 and change in i3 */ 204 | merge_error(i1.position, i2.position, i3.position, 3); 205 | goto err; 206 | } 207 | } 208 | else 209 | { /* Both keys changed */ 210 | TEST_KEY_SET_OR(cmp23, i2.key, i3.key) goto err; 211 | if (cmp23==0) 212 | { /* dueling inserts or deletes */ 213 | merge_error(i1.position, i2.position, i3.position, 4); 214 | goto err; 215 | } 216 | if (cmp12 > 0) 217 | { /* insert i2 */ 218 | if (cmp23 > 0) 219 | { /* insert i3 first */ 220 | if (merge_output(r, &i3, mapping) < 0) goto err; 221 | if (i3.next(&i3) < 0) goto err; 222 | } 223 | else 224 | { /* insert i2 first */ 225 | if (merge_output(r, &i2, mapping) < 0) goto err; 226 | if (i2.next(&i2) < 0) goto err; 227 | } 228 | } 229 | else if (cmp13 > 0) 230 | { /* Insert i3 */ 231 | if (merge_output(r, &i3, mapping) < 0) goto err; 232 | if (i3.next(&i3) < 0) goto err; 233 | } 234 | else 235 | { /* 1<2 and 1<3: both deleted 1.key */ 236 | merge_error(i1.position, i2.position, i3.position, 5); 237 | goto err; 238 | } 239 | } 240 | } 241 | 242 | while (i2.position >= 0 && i3.position >= 0) 243 | { /* New inserts */ 244 | TEST_KEY_SET_OR(cmp23, i2.key, i3.key) goto err; 245 | if (cmp23==0) 246 | { /* dueling inserts */ 247 | merge_error(i1.position, i2.position, i3.position, 6); 248 | goto err; 249 | } 250 | if (cmp23 > 0) 251 | { /* insert i3 */ 252 | if (merge_output(r, &i3, mapping) < 0) goto err; 253 | if (i3.next(&i3) < 0) goto err; 254 | } 255 | else 256 | { /* insert i2 */ 257 | if (merge_output(r, &i2, mapping) < 0) goto err; 258 | if (i2.next(&i2) < 0) goto err; 259 | } 260 | } 261 | 262 | while (i1.position >= 0 && i2.position >= 0) 263 | { /* remainder of i1 deleted in i3 */ 264 | TEST_KEY_SET_OR(cmp12, i1.key, i2.key) goto err; 265 | if (cmp12 > 0) 266 | { /* insert i2 */ 267 | if (merge_output(r, &i2, mapping) < 0) goto err; 268 | if (i2.next(&i2) < 0) goto err; 269 | } 270 | else if (cmp12==0 && (set || (TEST_VALUE(i1.value, i2.value) == 0))) 271 | { /* delete i3 */ 272 | if (i1.next(&i1) < 0) goto err; 273 | if (i2.next(&i2) < 0) goto err; 274 | } 275 | else 276 | { /* Dueling deletes or delete and change */ 277 | merge_error(i1.position, i2.position, i3.position, 7); 278 | goto err; 279 | } 280 | } 281 | 282 | while (i1.position >= 0 && i3.position >= 0) 283 | { /* remainder of i1 deleted in i2 */ 284 | TEST_KEY_SET_OR(cmp13, i1.key, i3.key) goto err; 285 | if (cmp13 > 0) 286 | { /* insert i3 */ 287 | if (merge_output(r, &i3, mapping) < 0) goto err; 288 | if (i3.next(&i3) < 0) goto err; 289 | } 290 | else if (cmp13==0 && (set || (TEST_VALUE(i1.value, i3.value) == 0))) 291 | { /* delete i2 */ 292 | if (i1.next(&i1) < 0) goto err; 293 | if (i3.next(&i3) < 0) goto err; 294 | } 295 | else 296 | { /* Dueling deletes or delete and change */ 297 | merge_error(i1.position, i2.position, i3.position, 8); 298 | goto err; 299 | } 300 | } 301 | 302 | if (i1.position >= 0) 303 | { /* Dueling deletes */ 304 | merge_error(i1.position, i2.position, i3.position, 9); 305 | goto err; 306 | } 307 | 308 | while (i2.position >= 0) 309 | { /* Inserting i2 at end */ 310 | if (merge_output(r, &i2, mapping) < 0) goto err; 311 | if (i2.next(&i2) < 0) goto err; 312 | } 313 | 314 | while (i3.position >= 0) 315 | { /* Inserting i3 at end */ 316 | if (merge_output(r, &i3, mapping) < 0) goto err; 317 | if (i3.next(&i3) < 0) goto err; 318 | } 319 | 320 | /* If the output bucket is empty, conflict resolution doesn't have 321 | * enough info to unlink it from its containing BTree correctly. 322 | */ 323 | if (r->len == 0) 324 | { 325 | merge_error(-1, -1, -1, 10); 326 | goto err; 327 | } 328 | 329 | finiSetIteration(&i1); 330 | finiSetIteration(&i2); 331 | finiSetIteration(&i3); 332 | 333 | if (s1->next) 334 | { 335 | Py_INCREF(s1->next); 336 | r->next = s1->next; 337 | } 338 | s = bucket_getstate(r); 339 | Py_DECREF(r); 340 | 341 | return s; 342 | 343 | err: 344 | finiSetIteration(&i1); 345 | finiSetIteration(&i2); 346 | finiSetIteration(&i3); 347 | Py_XDECREF(r); 348 | return NULL; 349 | } 350 | -------------------------------------------------------------------------------- /src/BTrees/_datatypes.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """ 15 | Descriptions of the datatypes supported by this package. 16 | """ 17 | 18 | import abc 19 | import operator 20 | import struct 21 | 22 | from .utils import Lazy 23 | 24 | 25 | class DataType: 26 | """ 27 | Describes a data type used as a value. 28 | 29 | Subclasses will be defined for each particular 30 | supported type. 31 | """ 32 | 33 | # The name for this datatype as used in interface names. 34 | long_name = None 35 | 36 | # The prefix code for this data type. Usually a single letter. 37 | prefix_code = None 38 | 39 | # The multiplication identity for this data type. Used in 40 | # combining (merging) data types. Leave undefined if this is 41 | # not a valid operation. 42 | multiplication_identity = None 43 | 44 | # Does the data take up 64-bits? Currently only relevant for the 45 | # integer key types. 46 | using64bits = False 47 | 48 | def __init__(self): 49 | if not self.prefix_code: 50 | self.prefix_code = type(self).__name__ 51 | 52 | def __call__(self, item): 53 | """ 54 | Verify *item* is in the correct format (or "close" enough) 55 | and return the item or its suitable conversion. 56 | 57 | If this cannot be done, raise a :exc:`TypeError`. 58 | 59 | The definition of "close" varies according to the datatypes. 60 | For example, integer datatypes will accept anything that can 61 | be converted into an integer using normal python coercion 62 | rules (calling ``__index__``) and where the integer fits into 63 | the required native type size (e.g., 4 bytes). 64 | """ 65 | raise NotImplementedError 66 | 67 | def coerce(self, item): 68 | """ 69 | Coerce *item* into something that can be used with 70 | ``__call__`` and return it. 71 | 72 | The coercion rules will vary by datatype. This exists only 73 | for test cases. The default is to perform the same validation 74 | as ``__call__``. 75 | """ 76 | return self(item) 77 | 78 | def apply_weight(self, item, weight): 79 | """ 80 | Apply a *weight* multiplier to *item*. 81 | 82 | Used when merging data structures. The *item* will be a 83 | value. 84 | """ 85 | return item 86 | 87 | def as_value_type(self): 88 | # Because ``O'`` is used for both key and value, 89 | # we can override this to get the less restrictive value type. 90 | return self 91 | 92 | def supports_value_union(self): 93 | raise NotImplementedError 94 | 95 | def getTwoExamples(self): 96 | """ 97 | Provide two distinct (non equal) examples acceptable to `__call__`. 98 | 99 | This is for testing. 100 | """ 101 | return "object1", "object2" 102 | 103 | def get_lower_bound(self): 104 | """ 105 | If there is a lower bound (inclusive) on the data type, return 106 | it. Otherwise, return ``None``. 107 | 108 | For integer types, this will only depend on whether it 109 | supports signed or unsigned values, and the answer will be 0 110 | or a negative number. For object types, ``None`` is always 111 | defined to sort as the lowest bound. 112 | 113 | This can be relevant for both key and value types. 114 | """ 115 | return None 116 | 117 | def get_upper_bound(self): 118 | """ 119 | If there is an upper bound (inclusive) on the data type, 120 | return it. Otherwise, return ``None``. 121 | 122 | Remarks are as for `get_lower_bound`. 123 | """ 124 | return None 125 | 126 | def add_extra_methods(self, base_name, cls): 127 | """ 128 | Hook method called on the key datatype to add zero or more 129 | desired arbitrary additional, non-standard, methods to the 130 | *cls* being constructed. 131 | 132 | *base_name* will be a string identifying the particular family 133 | of class being constructed, such as 'Bucket' or 'BTree'. 134 | """ 135 | 136 | 137 | class KeyDataType(DataType): 138 | """ 139 | Describes a data type that has additional restrictions allowing it 140 | to be used as a key. 141 | """ 142 | 143 | # When used as the key, this number determines the 144 | # max_internal_size. 145 | tree_size = 500 146 | 147 | default_bucket_size = 120 148 | 149 | def __call__(self, item): 150 | raise NotImplementedError 151 | 152 | def bucket_size_for_value(self, value_type): 153 | """ 154 | What should the bucket (``max_leaf_size``) be when 155 | this data type is used with the given *value_type*? 156 | """ 157 | if isinstance(value_type, Any): 158 | return self.default_bucket_size // 2 159 | return self.default_bucket_size 160 | 161 | 162 | class Any(DataType): 163 | """ 164 | Arbitrary Python objects. 165 | """ 166 | prefix_code = 'O' 167 | long_name = 'Object' 168 | 169 | def __call__(self, item): 170 | return item 171 | 172 | def supports_value_union(self): 173 | return False 174 | 175 | 176 | class _HasDefaultComparison(abc.ABC): 177 | """ 178 | An `abc.ABC _` for 179 | checking whether an item has default comparison. 180 | 181 | All we have to do is override ``__subclasshook__`` to implement an 182 | algorithm determining whether a class has default comparison. 183 | Python and the ABC machinery will take care of translating 184 | ``isinstance(thing, _HasDefaultComparison)`` into something like 185 | ``_HasDefaultComparison.__subclasshook__(type(thing))``. The ABC 186 | handles caching the answer (based on exact classes, no MRO), and 187 | getting the type from ``thing`` (including mostly dealing with 188 | old-style) classes on Python 2. 189 | """ 190 | 191 | # Comparisons only use special methods defined on the 192 | # type, not instance variables. 193 | 194 | # On CPython 3, classes inherit __lt__ with ``__objclass__`` of ``object``. 195 | # On PyPy3, they do. 196 | # 197 | # Test these conditions at runtime and define the method variant 198 | # appropriately. 199 | # 200 | # Remember the method is checking if the object has default comparison 201 | assert '__lt__' not in DataType.__dict__ 202 | if getattr(DataType.__lt__, '__objclass__', None) is object: 203 | # CPython 3 204 | @classmethod 205 | def __subclasshook__(cls, C, _NoneType=type(None)): 206 | if C is _NoneType: 207 | return False 208 | defining_class = getattr(C.__lt__, '__objclass__', None) 209 | if defining_class is None: 210 | # Implemented in Python 211 | return False 212 | return C.__lt__.__objclass__ is object 213 | else: 214 | # PyPy3 215 | @classmethod 216 | def __subclasshook__( 217 | cls, C, _object_lt=object.__lt__, _NoneType=type(None) 218 | ): 219 | if C is _NoneType: 220 | return False 221 | return C.__lt__ is _object_lt 222 | 223 | 224 | class O(KeyDataType): # noqa E742 225 | """ 226 | Arbitrary (sortable) Python objects. 227 | """ 228 | long_name = 'Object' 229 | tree_size = 250 230 | default_bucket_size = 60 231 | 232 | def as_value_type(self): 233 | return Any() 234 | 235 | def supports_value_union(self): 236 | return False 237 | 238 | def __call__(self, item): 239 | if isinstance(item, _HasDefaultComparison): 240 | raise TypeError( 241 | "Object of class {} has default comparison".format( 242 | type(item).__name__ 243 | ) 244 | ) 245 | return item 246 | 247 | 248 | class _AbstractNativeDataType(KeyDataType): 249 | """ 250 | Uses `struct.Struct` to verify that the data can fit into a native 251 | type. 252 | """ 253 | 254 | _struct_format = None 255 | _as_python_type = NotImplementedError 256 | _required_python_type = object 257 | _error_description = None 258 | _as_packable = operator.index 259 | 260 | @Lazy 261 | def _check_native(self): 262 | return struct.Struct(self._struct_format).pack 263 | 264 | def __call__(self, item): 265 | try: 266 | self._check_native(self._as_packable(item)) 267 | except (struct.error, TypeError, ValueError): 268 | # PyPy can raise ValueError converting a negative number to a 269 | # unsigned value. 270 | if isinstance(item, int): 271 | raise TypeError("Value out of range", item) 272 | raise TypeError(self._error_description) 273 | 274 | return self._as_python_type(item) 275 | 276 | def apply_weight(self, item, weight): 277 | return item * weight 278 | 279 | def supports_value_union(self): 280 | return True 281 | 282 | 283 | class _AbstractIntDataType(_AbstractNativeDataType): 284 | _as_python_type = int 285 | _required_python_type = int 286 | multiplication_identity = 1 287 | long_name = "Integer" 288 | 289 | def getTwoExamples(self): 290 | return 1, 2 291 | 292 | def get_lower_bound(self): 293 | exp = 64 if self.using64bits else 32 294 | exp -= 1 295 | return int(-(2 ** exp)) 296 | 297 | def get_upper_bound(self): 298 | exp = 64 if self.using64bits else 32 299 | exp -= 1 300 | return int(2 ** exp - 1) 301 | 302 | 303 | class _AbstractUIntDataType(_AbstractIntDataType): 304 | long_name = 'Unsigned' 305 | 306 | def get_lower_bound(self): 307 | return 0 308 | 309 | def get_upper_bound(self): 310 | exp = 64 if self.using64bits else 32 311 | return int(2 ** exp - 1) 312 | 313 | 314 | class I(_AbstractIntDataType): # noqa E742 315 | _struct_format = 'i' 316 | _error_description = "32-bit integer expected" 317 | 318 | 319 | class U(_AbstractUIntDataType): 320 | _struct_format = 'I' 321 | _error_description = 'non-negative 32-bit integer expected' 322 | 323 | 324 | class F(_AbstractNativeDataType): 325 | _struct_format = 'f' 326 | _as_python_type = float 327 | _error_description = 'float expected' 328 | multiplication_identity = 1.0 329 | long_name = 'Float' 330 | 331 | def _as_packable(self, k): # identity 332 | return k 333 | 334 | def getTwoExamples(self): 335 | return 0.5, 1.5 336 | 337 | 338 | class L(_AbstractIntDataType): 339 | _struct_format = 'q' 340 | _error_description = '64-bit integer expected' 341 | using64bits = True 342 | 343 | 344 | class Q(_AbstractUIntDataType): 345 | _struct_format = 'Q' 346 | _error_description = 'non-negative 64-bit integer expected' 347 | using64bits = True 348 | 349 | 350 | class _AbstractBytes(KeyDataType): 351 | """ 352 | An exact-length byte string type. 353 | 354 | This must be subclassed to provide the actual byte length. 355 | """ 356 | tree_size = 500 357 | default_bucket_size = 500 358 | _length = None 359 | 360 | def __call__(self, item): 361 | if not isinstance(item, bytes) or len(item) != self._length: 362 | raise TypeError( 363 | f"{self._length}-byte array expected, not {item!r}" 364 | ) 365 | return item 366 | 367 | def supports_value_union(self): 368 | # We don't implement 'multiunion' for fsBTree. 369 | return False 370 | 371 | 372 | class f(_AbstractBytes): 373 | """ 374 | The key type for an ``fs`` tree. 375 | 376 | This is a two-byte prefix of an overall 8-byte value 377 | like a ZODB object ID or transaction ID. 378 | """ 379 | 380 | # Our keys are treated like integers; the module 381 | # implements IIntegerObjectBTreeModule 382 | long_name = 'Integer' 383 | prefix_code = 'f' 384 | _length = 2 385 | 386 | # Check it can be converted to a two-byte 387 | # value. Note that even though we allow negative values 388 | # that can break test assumptions: -1 < 0 < 1, but the byte 389 | # values for those are \xff\xff > \x00\x00 < \x00\x01. 390 | _as_2_bytes = struct.Struct('>h').pack 391 | 392 | def coerce(self, item): 393 | try: 394 | return self(item) 395 | except TypeError: 396 | try: 397 | return self._as_2_bytes(operator.index(item)) 398 | except struct.error as e: 399 | raise TypeError(e) 400 | 401 | @staticmethod 402 | def _make_Bucket_toString(): 403 | def toString(self): 404 | return b''.join(self._keys) + b''.join(self._values) 405 | return toString 406 | 407 | @staticmethod 408 | def _make_Bucket_fromString(): 409 | def fromString(self, v): 410 | length = len(v) 411 | if length % 8 != 0: 412 | raise ValueError() 413 | count = length // 8 414 | keys, values = v[:count * 2], v[count * 2:] 415 | self.clear() 416 | while keys and values: 417 | key, keys = keys[:2], keys[2:] 418 | value, values = values[:6], values[6:] 419 | self._keys.append(key) 420 | self._values.append(value) 421 | return self 422 | return fromString 423 | 424 | def add_extra_methods(self, base_name, cls): 425 | if base_name == 'Bucket': 426 | cls.toString = self._make_Bucket_toString() 427 | cls.fromString = self._make_Bucket_fromString() 428 | 429 | 430 | class s(_AbstractBytes): 431 | """ 432 | The value type for an ``fs`` tree. 433 | 434 | This is a 6-byte suffix of an overall 8-byte value 435 | like a ZODB object ID or transaction ID. 436 | """ 437 | 438 | # Our values are treated like objects; the 439 | # module implements IIntegerObjectBTreeModule 440 | long_name = 'Object' 441 | prefix_code = 's' 442 | _length = 6 443 | 444 | def get_lower_bound(self): 445 | # Negative values have the high bit set, which is incompatible 446 | # with our transformation. 447 | return 0 448 | 449 | # To coerce an integer, as used in tests, first convert to 8 bytes 450 | # in big-endian order, then ensure the first two 451 | # are 0 and cut them off. 452 | _as_8_bytes = struct.Struct('>q').pack 453 | 454 | def coerce(self, item): 455 | try: 456 | return self(item) 457 | except TypeError: 458 | try: 459 | as_bytes = self._as_8_bytes(operator.index(item)) 460 | except struct.error as e: 461 | raise TypeError(e) 462 | 463 | if as_bytes[:2] != b'\x00\x00': 464 | raise TypeError( 465 | "Cannot convert {!r} to 6 bytes ({!r})".format( 466 | item, as_bytes 467 | ) 468 | ) 469 | return as_bytes[2:] 470 | -------------------------------------------------------------------------------- /src/BTrees/sorters.c: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | 3 | Copyright (c) 2002 Zope Foundation and Contributors. 4 | All Rights Reserved. 5 | 6 | This software is subject to the provisions of the Zope Public License, 7 | Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | FOR A PARTICULAR PURPOSE 12 | 13 | ****************************************************************************/ 14 | 15 | /* Revision information: $Id$ */ 16 | 17 | /* The only routine here intended to be used outside the file is 18 | size_t sort_int_nodups(int *p, size_t n) 19 | 20 | Sort the array of n ints pointed at by p, in place, and also remove 21 | duplicates. Return the number of unique elements remaining, which occupy 22 | a contiguous and monotonically increasing slice of the array starting at p. 23 | 24 | Example: If the input array is [3, 1, 2, 3, 1, 5, 2], sort_int_nodups 25 | returns 4, and the first 4 elements of the array are changed to 26 | [1, 2, 3, 5]. The content of the remaining array positions is not defined. 27 | 28 | Notes: 29 | 30 | + This is specific to n-byte signed ints, with endianness natural to the 31 | platform. `n` is determined based on ZODB_64BIT_INTS. 32 | 33 | + 4*n bytes of available heap memory are required for best speed 34 | (8*n when ZODB_64BIT_INTS is defined). 35 | */ 36 | 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | 43 | /* The type of array elements to be sorted. Most of the routines don't 44 | care about the type, and will work fine for any scalar C type (provided 45 | they're recompiled with element_type appropriately redefined). However, 46 | the radix sort has to know everything about the type's internal 47 | representation. 48 | */ 49 | typedef KEY_TYPE element_type; 50 | 51 | /* The radixsort is faster than the quicksort for large arrays, but radixsort 52 | has high fixed overhead, making it a poor choice for small arrays. The 53 | crossover point isn't critical, and is sensitive to things like compiler 54 | and machine cache structure, so don't worry much about this. 55 | */ 56 | #define QUICKSORT_BEATS_RADIXSORT 800U 57 | 58 | /* In turn, the quicksort backs off to an insertion sort for very small 59 | slices. MAX_INSERTION is the largest slice quicksort leaves entirely to 60 | insertion. Because this version of quicksort uses a median-of-3 rule for 61 | selecting a pivot, MAX_INSERTION must be at least 2 (so that quicksort 62 | has at least 3 values to look at in a slice). Again, the exact value here 63 | isn't critical. 64 | */ 65 | #define MAX_INSERTION 25U 66 | 67 | #if MAX_INSERTION < 2U 68 | # error "MAX_INSERTION must be >= 2" 69 | #endif 70 | 71 | /* LSB-first radix sort of the n elements in 'in'. 72 | 'work' is work storage at least as large as 'in'. Depending on how many 73 | swaps are done internally, the final result may come back in 'in' or 'work'; 74 | and that pointer is returned. 75 | 76 | radixsort_int is specific to signed n-byte ints, with natural machine 77 | endianness. `n` is determined based on ZODB_64BIT_INTS. 78 | */ 79 | static element_type* 80 | radixsort_int(element_type *in, element_type *work, size_t n) 81 | { 82 | /* count[i][j] is the number of input elements that have byte value j 83 | in byte position i, where byte position 0 is the LSB. Note that 84 | holding i fixed, the sum of count[i][j] over all j in range(256) 85 | is n. 86 | */ 87 | #ifdef ZODB_64BIT_INTS 88 | size_t count[8][256]; 89 | #else 90 | size_t count[4][256]; 91 | #endif 92 | size_t i; 93 | int offset, offsetinc; 94 | 95 | /* Which byte position are we working on now? 0=LSB, 1, 2, ... */ 96 | size_t bytenum; 97 | 98 | #ifdef ZODB_64BIT_INTS 99 | assert(sizeof(element_type) == 8); 100 | #else 101 | assert(sizeof(element_type) == 4); 102 | #endif 103 | assert(in); 104 | assert(work); 105 | 106 | /* Compute all of count in one pass. */ 107 | memset(count, 0, sizeof(count)); 108 | for (i = 0; i < n; ++i) { 109 | element_type const x = in[i]; 110 | ++count[0][(x ) & 0xff]; 111 | ++count[1][(x >> 8) & 0xff]; 112 | ++count[2][(x >> 16) & 0xff]; 113 | ++count[3][(x >> 24) & 0xff]; 114 | #ifdef ZODB_64BIT_INTS 115 | ++count[4][(x >> 32) & 0xff]; 116 | ++count[5][(x >> 40) & 0xff]; 117 | ++count[6][(x >> 48) & 0xff]; 118 | ++count[7][(x >> 56) & 0xff]; 119 | #endif 120 | } 121 | 122 | /* For p an element_type* cast to char*, offset is how much farther we 123 | have to go to get to the LSB of the element; this is 0 for little- 124 | endian boxes and sizeof(element_type)-1 for big-endian. 125 | offsetinc is 1 or -1, respectively, telling us which direction to go 126 | from p+offset to get to the element's more-significant bytes. 127 | */ 128 | { 129 | element_type one = 1; 130 | if (*(char*)&one) { 131 | /* Little endian. */ 132 | offset = 0; 133 | offsetinc = 1; 134 | } 135 | else { 136 | /* Big endian. */ 137 | offset = sizeof(element_type) - 1; 138 | offsetinc = -1; 139 | } 140 | } 141 | 142 | /* The radix sort. */ 143 | for (bytenum = 0; 144 | bytenum < sizeof(element_type); 145 | ++bytenum, offset += offsetinc) { 146 | 147 | /* Do a stable distribution sort on byte position bytenum, 148 | from in to work. index[i] tells us the work index at which 149 | to store the next in element with byte value i. pinbyte 150 | points to the correct byte in the input array. 151 | */ 152 | size_t index[256]; 153 | unsigned char* pinbyte; 154 | size_t total = 0; 155 | size_t *pcount = count[bytenum]; 156 | 157 | /* Compute the correct output starting index for each possible 158 | byte value. 159 | */ 160 | if (bytenum < sizeof(element_type) - 1) { 161 | for (i = 0; i < 256; ++i) { 162 | const size_t icount = pcount[i]; 163 | index[i] = total; 164 | total += icount; 165 | if (icount == n) 166 | break; 167 | } 168 | if (i < 256) { 169 | /* All bytes in the current position have value 170 | i, so there's nothing to do on this pass. 171 | */ 172 | continue; 173 | } 174 | } 175 | else { 176 | /* The MSB of signed ints needs to be distributed 177 | differently than the other bytes, in order 178 | 0x80, 0x81, ... 0xff, 0x00, 0x01, ... 0x7f 179 | */ 180 | for (i = 128; i < 256; ++i) { 181 | const size_t icount = pcount[i]; 182 | index[i] = total; 183 | total += icount; 184 | if (icount == n) 185 | break; 186 | } 187 | if (i < 256) 188 | continue; 189 | for (i = 0; i < 128; ++i) { 190 | const size_t icount = pcount[i]; 191 | index[i] = total; 192 | total += icount; 193 | if (icount == n) 194 | break; 195 | } 196 | if (i < 128) 197 | continue; 198 | } 199 | assert(total == n); 200 | 201 | /* Distribute the elements according to byte value. Note that 202 | this is where most of the time is spent. 203 | Note: The loop is unrolled 4x by hand, for speed. This 204 | may be a pessimization someday, but was a significant win 205 | on my MSVC 6.0 timing tests. 206 | */ 207 | pinbyte = (unsigned char *)in + offset; 208 | i = 0; 209 | /* Reduce number of elements to copy to a multiple of 4. */ 210 | while ((n - i) & 0x3) { 211 | unsigned char byte = *pinbyte; 212 | work[index[byte]++] = in[i]; 213 | ++i; 214 | pinbyte += sizeof(element_type); 215 | } 216 | for (; i < n; i += 4, pinbyte += 4 * sizeof(element_type)) { 217 | unsigned char byte1 = *(pinbyte ); 218 | unsigned char byte2 = *(pinbyte + sizeof(element_type)); 219 | unsigned char byte3 = *(pinbyte + 2 * sizeof(element_type)); 220 | unsigned char byte4 = *(pinbyte + 3 * sizeof(element_type)); 221 | 222 | element_type in1 = in[i ]; 223 | element_type in2 = in[i+1]; 224 | element_type in3 = in[i+2]; 225 | element_type in4 = in[i+3]; 226 | 227 | work[index[byte1]++] = in1; 228 | work[index[byte2]++] = in2; 229 | work[index[byte3]++] = in3; 230 | work[index[byte4]++] = in4; 231 | } 232 | /* Swap in and work (just a pointer swap). */ 233 | { 234 | element_type *temp = in; 235 | in = work; 236 | work = temp; 237 | } 238 | } 239 | 240 | return in; 241 | } 242 | 243 | /* Remove duplicates from sorted array in, storing exactly one of each distinct 244 | element value into sorted array out. It's OK (and expected!) for in == out, 245 | but otherwise the n elements beginning at in must not overlap with the n 246 | beginning at out. 247 | Return the number of elements in out. 248 | */ 249 | static size_t 250 | uniq(element_type *out, element_type *in, size_t n) 251 | { 252 | size_t i; 253 | element_type lastelt; 254 | element_type *pout; 255 | 256 | assert(out); 257 | assert(in); 258 | if (n == 0) 259 | return 0; 260 | 261 | /* i <- first index in 'in' that contains a duplicate. 262 | in[0], in[1], ... in[i-1] are unique, but in[i-1] == in[i]. 263 | Set i to n if everything is unique. 264 | */ 265 | for (i = 1; i < n; ++i) { 266 | if (in[i-1] == in[i]) 267 | break; 268 | } 269 | 270 | /* in[:i] is unique; copy to out[:i] if needed. */ 271 | assert(i > 0); 272 | if (in != out) 273 | memcpy(out, in, i * sizeof(element_type)); 274 | 275 | pout = out + i; 276 | lastelt = in[i-1]; /* safe even when i == n */ 277 | for (++i; i < n; ++i) { 278 | element_type elt = in[i]; 279 | if (elt != lastelt) 280 | *pout++ = lastelt = elt; 281 | } 282 | return pout - out; 283 | } 284 | 285 | #if 0 286 | /* insertionsort is no longer referenced directly, but I'd like to keep 287 | * the code here just in case. 288 | */ 289 | 290 | /* Straight insertion sort of the n elements starting at 'in'. */ 291 | static void 292 | insertionsort(element_type *in, size_t n) 293 | { 294 | element_type *p, *q; 295 | element_type minimum; /* smallest seen so far */ 296 | element_type *plimit = in + n; 297 | 298 | assert(in); 299 | if (n < 2) 300 | return; 301 | 302 | minimum = *in; 303 | for (p = in+1; p < plimit; ++p) { 304 | /* *in <= *(in+1) <= ... <= *(p-1). Slide *p into place. */ 305 | element_type thiselt = *p; 306 | if (thiselt < minimum) { 307 | /* This is a new minimum. This saves p-in compares 308 | when it happens, but should happen so rarely that 309 | it's not worth checking for its own sake: the 310 | point is that the far more popular 'else' branch can 311 | exploit that thiselt is *not* the smallest so far. 312 | */ 313 | memmove(in+1, in, (p - in) * sizeof(*in)); 314 | *in = minimum = thiselt; 315 | } 316 | else { 317 | /* thiselt >= minimum, so the loop will find a q 318 | with *q <= thiselt. This saves testing q >= in 319 | on each trip. It's such a simple loop that saving 320 | a per-trip test is a major speed win. 321 | */ 322 | for (q = p-1; *q > thiselt; --q) 323 | *(q+1) = *q; 324 | *(q+1) = thiselt; 325 | } 326 | } 327 | } 328 | #endif 329 | 330 | /* The maximum number of elements in the pending-work stack quicksort 331 | maintains. The maximum stack depth is approximately log2(n), so 332 | arrays of size up to approximately MAX_INSERTION * 2**STACKSIZE can be 333 | sorted. The memory burden for the stack is small, so better safe than 334 | sorry. 335 | */ 336 | #define STACKSIZE 60 337 | 338 | /* A _stacknode remembers a contiguous slice of an array that needs to sorted. 339 | lo must be <= hi, and, unlike Python array slices, this includes both ends. 340 | */ 341 | struct _stacknode { 342 | element_type *lo; 343 | element_type *hi; 344 | }; 345 | 346 | static void 347 | quicksort(element_type *plo, size_t n) 348 | { 349 | element_type *phi; 350 | 351 | /* Swap two array elements. */ 352 | element_type _temp; 353 | #define SWAP(P, Q) (_temp = *(P), *(P) = *(Q), *(Q) = _temp) 354 | 355 | /* Stack of pending array slices to be sorted. */ 356 | struct _stacknode stack[STACKSIZE]; 357 | struct _stacknode *stackfree = stack; /* available stack slot */ 358 | 359 | /* Push an array slice on the pending-work stack. */ 360 | #define PUSH(PLO, PHI) \ 361 | do { \ 362 | assert(stackfree - stack < STACKSIZE); \ 363 | assert((PLO) <= (PHI)); \ 364 | stackfree->lo = (PLO); \ 365 | stackfree->hi = (PHI); \ 366 | ++stackfree; \ 367 | } while(0) 368 | 369 | assert(plo); 370 | phi = plo + n - 1; 371 | 372 | for (;;) { 373 | element_type pivot; 374 | element_type *pi, *pj; 375 | 376 | assert(plo <= phi); 377 | n = phi - plo + 1; 378 | if (n <= MAX_INSERTION) { 379 | /* Do a small insertion sort. Contra Knuth, we do 380 | this now instead of waiting until the end, because 381 | this little slice is likely still in cache now. 382 | */ 383 | element_type *p, *q; 384 | element_type minimum = *plo; 385 | 386 | for (p = plo+1; p <= phi; ++p) { 387 | /* *plo <= *(plo+1) <= ... <= *(p-1). 388 | Slide *p into place. */ 389 | element_type thiselt = *p; 390 | if (thiselt < minimum) { 391 | /* New minimum. */ 392 | memmove(plo+1, 393 | plo, 394 | (p - plo) * sizeof(*p)); 395 | *plo = minimum = thiselt; 396 | } 397 | else { 398 | /* thiselt >= minimum, so the loop will 399 | find a q with *q <= thiselt. 400 | */ 401 | for (q = p-1; *q > thiselt; --q) 402 | *(q+1) = *q; 403 | *(q+1) = thiselt; 404 | } 405 | } 406 | 407 | /* Pop another slice off the stack. */ 408 | if (stack == stackfree) 409 | break; /* no more slices -- we're done */ 410 | --stackfree; 411 | plo = stackfree->lo; 412 | phi = stackfree->hi; 413 | continue; 414 | } 415 | 416 | /* Parition the slice. 417 | For pivot, take the median of the leftmost, rightmost, and 418 | middle elements. First sort those three; then the median 419 | is the middle one. For technical reasons, the middle 420 | element is swapped to plo+1 first (see Knuth Vol 3 Ed 2 421 | section 5.2.2 exercise 55 -- reverse-sorted arrays can 422 | take quadratic time otherwise!). 423 | */ 424 | { 425 | element_type *plop1 = plo + 1; 426 | element_type *pmid = plo + (n >> 1); 427 | 428 | assert(plo < pmid && pmid < phi); 429 | SWAP(plop1, pmid); 430 | 431 | /* Sort plo, plop1, phi. */ 432 | /* Smaller of rightmost two -> middle. */ 433 | if (*plop1 > *phi) 434 | SWAP(plop1, phi); 435 | /* Smallest of all -> left; if plo is already the 436 | smallest, the sort is complete. 437 | */ 438 | if (*plo > *plop1) { 439 | SWAP(plo, plop1); 440 | /* Largest of all -> right. */ 441 | if (*plop1 > *phi) 442 | SWAP(plop1, phi); 443 | } 444 | pivot = *plop1; 445 | pi = plop1; 446 | } 447 | assert(*plo <= pivot); 448 | assert(*pi == pivot); 449 | assert(*phi >= pivot); 450 | pj = phi; 451 | 452 | /* Partition wrt pivot. This is the time-critical part, and 453 | nearly every decision in the routine aims at making this 454 | loop as fast as possible -- even small points like 455 | arranging that all loop tests can be done correctly at the 456 | bottoms of loops instead of the tops, and that pointers can 457 | be derefenced directly as-is (without fiddly +1 or -1). 458 | The aim is to make the C here so simple that a compiler 459 | has a good shot at doing as well as hand-crafted assembler. 460 | */ 461 | for (;;) { 462 | /* Invariants: 463 | 1. pi < pj. 464 | 2. All elements at plo, plo+1 .. pi are <= pivot. 465 | 3. All elements at pj, pj+1 .. phi are >= pivot. 466 | 4. There is an element >= pivot to the right of pi. 467 | 5. There is an element <= pivot to the left of pj. 468 | 469 | Note that #4 and #5 save us from needing to check 470 | that the pointers stay in bounds. 471 | */ 472 | assert(pi < pj); 473 | 474 | do { ++pi; } while (*pi < pivot); 475 | assert(pi <= pj); 476 | 477 | do { --pj; } while (*pj > pivot); 478 | assert(pj >= pi - 1); 479 | 480 | if (pi < pj) 481 | SWAP(pi, pj); 482 | else 483 | break; 484 | } 485 | assert(plo+1 < pi && pi <= phi); 486 | assert(plo < pj && pj < phi); 487 | assert(*pi >= pivot); 488 | assert( (pi == pj && *pj == pivot) || 489 | (pj + 1 == pi && *pj <= pivot) ); 490 | 491 | /* Swap pivot into its final position, pj. */ 492 | assert(plo[1] == pivot); 493 | plo[1] = *pj; 494 | *pj = pivot; 495 | 496 | /* Subfiles are from plo to pj-1 inclusive, and pj+1 to phi 497 | inclusive. Push the larger one, and loop back to do the 498 | smaller one directly. 499 | */ 500 | if (pj - plo >= phi - pj) { 501 | PUSH(plo, pj-1); 502 | plo = pj+1; 503 | } 504 | else { 505 | PUSH(pj+1, phi); 506 | phi = pj-1; 507 | } 508 | } 509 | 510 | #undef PUSH 511 | #undef SWAP 512 | } 513 | 514 | /* Sort p and remove duplicates, as fast as we can. */ 515 | static size_t 516 | sort_int_nodups(KEY_TYPE *p, size_t n) 517 | { 518 | size_t nunique; 519 | element_type *work; 520 | 521 | assert(sizeof(KEY_TYPE) == sizeof(element_type)); 522 | assert(p); 523 | 524 | /* Use quicksort if the array is small, OR if malloc can't find 525 | enough temp memory for radixsort. 526 | */ 527 | work = NULL; 528 | if (n > QUICKSORT_BEATS_RADIXSORT) 529 | work = (element_type *)malloc(n * sizeof(element_type)); 530 | 531 | if (work) { 532 | element_type *out = radixsort_int(p, work, n); 533 | nunique = uniq(p, out, n); 534 | free(work); 535 | } 536 | else { 537 | quicksort(p, n); 538 | nunique = uniq(p, p, n); 539 | } 540 | 541 | return nunique; 542 | } 543 | --------------------------------------------------------------------------------