├── .editorconfig ├── .github ├── ISSUE_TEMPLATE.md └── workflows │ ├── deploy.yml │ └── main.yml ├── .gitignore ├── .gitmodules ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── deploy.sh ├── docs ├── Makefile ├── _static │ ├── performance_test1.ods │ ├── performance_test1.png │ ├── performance_test2.png │ ├── zlmdb_dbformat.ods │ └── zlmdb_dbformat.pdf ├── conf.py ├── contents.rst ├── gen.py ├── index.rst ├── performance.rst ├── reference.rst └── spelling_wordlist.txt ├── flatbuffers ├── __init__.py ├── _version.py ├── builder.py ├── compat.py ├── encode.py ├── flexbuffers.py ├── number_types.py ├── packer.py ├── reflection │ ├── AdvancedFeatures.py │ ├── BaseType.py │ ├── Enum.py │ ├── EnumVal.py │ ├── Field.py │ ├── KeyValue.py │ ├── Object.py │ ├── RPCCall.py │ ├── Schema.py │ ├── SchemaFile.py │ ├── Service.py │ ├── Type.py │ └── __init__.py ├── table.py └── util.py ├── mypy.ini ├── pytest.ini ├── readthedocs.yml ├── requirements-dev.txt ├── requirements-rtd.txt ├── requirements-test.txt ├── setup.cfg ├── setup.py ├── tests ├── Makefile ├── _gen │ └── crossbarfx │ │ ├── Date.py │ │ ├── Rating.py │ │ ├── Tag.py │ │ ├── User.py │ │ └── __init__.py ├── test_cbor.py ├── test_fbs_reflection.py ├── test_flatbuffers.py ├── test_new.py ├── test_ops.py ├── test_pickle.py ├── test_zlmdb.py ├── user.py ├── user_typed.py └── zdb │ ├── README.md │ ├── _schema_fbs.py │ ├── test_zdb_df.py │ ├── test_zdb_dyn.py │ ├── test_zdb_etcd.py │ ├── test_zdb_fbs.py │ └── zdb.yml ├── tox.ini ├── yapf.ini └── zlmdb ├── __init__.py ├── _database.py ├── _errors.py ├── _meta.py ├── _pmap.py ├── _schema.py ├── _transaction.py ├── _types.py ├── _version.py ├── cli.py ├── flatbuffers ├── __init__.py ├── reflection.fbs └── reflection │ ├── AdvancedFeatures.py │ ├── BaseType.py │ ├── Enum.py │ ├── EnumVal.py │ ├── Field.py │ ├── KeyValue.py │ ├── Object.py │ ├── RPCCall.py │ ├── Schema.py │ ├── SchemaFile.py │ ├── Service.py │ ├── Type.py │ └── __init__.py └── tests ├── MNodeLog.py ├── _schema_fbs.py ├── _schema_mnode_log.py ├── _schema_py2.py ├── _schema_py3.py ├── _test_flatbuffers.py ├── _test_serialization.py ├── test_basic.py ├── test_etcd.py ├── test_lmdb.py ├── test_pmap_indexes.py ├── test_pmap_types.py ├── test_pmaps.py └── test_select.py /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | [LICENSE] 18 | insert_final_newline = false 19 | 20 | [Makefile] 21 | indent_style = tab 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | * ZLMDB version: 2 | * Python version: 3 | * Operating System: 4 | 5 | ### Description 6 | 7 | Describe what you were trying to get done. 8 | Tell us what happened, what went wrong, and what you expected to happen. 9 | 10 | ### What I Did 11 | 12 | ``` 13 | Paste the command(s) you ran and the output. 14 | If there was a crash, please include the traceback here. 15 | ``` 16 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: deploy 2 | 3 | on: 4 | # Trigger this workflow when the "main" workflow has completed successfully 5 | # https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows#workflow_run 6 | workflow_run: 7 | workflows: 8 | - main 9 | branches: 10 | - master 11 | types: 12 | - completed 13 | 14 | jobs: 15 | 16 | deploy: 17 | if: github.ref == 'refs/heads/master' 18 | 19 | # https://github.blog/changelog/2020-12-15-github-actions-environments-environment-protection-rules-and-environment-secrets-beta/ 20 | # https://docs.github.com/en/free-pro-team@latest/actions/reference/environments 21 | environment: deploy_aws 22 | 23 | env: 24 | AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} 25 | AWS_S3_BUCKET_NAME: ${{ secrets.AWS_S3_BUCKET_NAME }} 26 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 27 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 28 | WAMP_PRIVATE_KEY: ${{ secrets.WAMP_PRIVATE_KEY }} 29 | 30 | runs-on: ubuntu-latest 31 | steps: 32 | - uses: actions/checkout@v2 33 | 34 | # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable 35 | # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#adding-a-system-path 36 | - name: Set environment 37 | run: | 38 | echo "${HOME}/.local/bin" >> $GITHUB_PATH 39 | echo BUILD_DATE=`date -u +"%Y-%m-%d"` >> $GITHUB_ENV 40 | echo ZLMDB_VCS_REF=`git rev-parse --short ${GITHUB_SHA}` >> $GITHUB_ENV 41 | echo ZLMDB_BUILD_ID=$(date --utc +%Y%m%d)-$(git rev-parse --short ${GITHUB_SHA}) >> $GITHUB_ENV 42 | echo ZLMDB_VERSION=$(grep -E '^(__version__)' ./zlmdb/_version.py | cut -d ' ' -f3 | sed -e 's|[u"'\'']||g') >> $GITHUB_ENV 43 | 44 | # - name: Set environment - 2 45 | # run: | 46 | # echo ZLMDB_VCS_REF=`git --git-dir="./.git" rev-list -n 1 v${ZLMDB_VERSION} --abbrev-commit` >> $GITHUB_ENV 47 | 48 | - name: Install OS package dependencies 49 | run: | 50 | sudo apt update 51 | sudo apt install build-essential libssl-dev libffi-dev libunwind-dev \ 52 | libreadline-dev zlib1g-dev libbz2-dev libsqlite3-dev libncurses5-dev \ 53 | libsnappy-dev 54 | 55 | - name: Set up Python 3.x 56 | uses: actions/setup-python@v2 57 | with: 58 | python-version: '3.x' 59 | architecture: 'x64' 60 | 61 | - name: Install Python package dependencies 62 | run: | 63 | python -m pip install --upgrade pip 64 | pip install -r requirements-dev.txt 65 | 66 | - name: Deploy (build, package and upload) 67 | run: | 68 | ./deploy.sh 69 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: main 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | 11 | jobs: 12 | check: 13 | runs-on: ubuntu-22.04 14 | steps: 15 | - uses: actions/checkout@v3 16 | 17 | - name: Install OS package dependencies 18 | run: | 19 | sudo apt update 20 | sudo apt install libenchant-2-dev libbz2-dev libsnappy-dev libunwind-dev 21 | 22 | - name: Set up Python 3.11 23 | uses: actions/setup-python@v3 24 | with: 25 | python-version: '3.11' 26 | architecture: 'x64' 27 | 28 | - name: Install Python package dependencies 29 | run: | 30 | python -m pip install --upgrade pip 31 | pip install -r requirements-dev.txt 32 | 33 | - name: Run Flake8 34 | run: tox -c tox.ini -e flake8 35 | 36 | - name: Run Yapf 37 | run: tox -c tox.ini -e yapf 38 | 39 | - name: Run MyPy 40 | run: tox -c tox.ini -e mypy 41 | 42 | test: 43 | env: 44 | CB_FULLTESTS: 1 45 | 46 | runs-on: ${{ matrix.os }} 47 | strategy: 48 | matrix: 49 | os: [ubuntu-22.04] 50 | python-version: ['3.9', '3.11', 'pypy-3.9'] 51 | 52 | # https://github.blog/changelog/2020-04-15-github-actions-new-workflow-features/ 53 | # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstepscontinue-on-error 54 | continue-on-error: false 55 | 56 | steps: 57 | # Checkout sources 58 | - uses: actions/checkout@v3 59 | 60 | # Install OS packages, as we install Python packages from source: 61 | # libenchant-dev: needed for pyenchant, needed for sphinx-spellcheck 62 | # libbz2-dev, libsnappy-dev: needed for compression 63 | # libunwind-dev: needed for vmprof 64 | - name: Install OS package dependencies 65 | run: | 66 | sudo apt update 67 | sudo apt install build-essential libssl-dev libffi-dev libunwind-dev \ 68 | libreadline-dev zlib1g-dev libbz2-dev libsqlite3-dev libncurses5-dev \ 69 | libsnappy-dev 70 | 71 | # Use this Python 72 | # https://github.com/actions/setup-python/blob/main/README.md 73 | - name: Set up Python ${{ matrix.python-version }} 74 | uses: actions/setup-python@v3 75 | with: 76 | python-version: ${{ matrix.python-version }} 77 | 78 | - name: Install Python package dependencies 79 | run: | 80 | python -m pip install -U pip 81 | pip install -U -r requirements-dev.txt 82 | 83 | - name: Install zLMDB 84 | run: | 85 | pip install . 86 | 87 | - name: Run unit tests (PyTest) 88 | run: | 89 | tox -c tox.ini 90 | 91 | docs: 92 | runs-on: ubuntu-22.04 93 | steps: 94 | - uses: actions/checkout@v3 95 | 96 | - name: Install OS package dependencies 97 | run: | 98 | sudo apt update 99 | sudo apt install libenchant-2-dev libbz2-dev libsnappy-dev libunwind-dev 100 | 101 | - name: Set up Python 3.11 102 | uses: actions/setup-python@v3 103 | with: 104 | python-version: '3.11' 105 | architecture: 'x64' 106 | 107 | - name: Install Python package dependencies 108 | run: | 109 | python -m pip install --upgrade pip 110 | pip install -r requirements-dev.txt 111 | 112 | - name: Run Sphinx 113 | run: tox -c tox.ini -e sphinx 114 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | 58 | # Flask stuff: 59 | instance/ 60 | .webassets-cache 61 | 62 | # Scrapy stuff: 63 | .scrapy 64 | 65 | # Sphinx documentation 66 | docs/_build/ 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | # Jupyter Notebook 72 | .ipynb_checkpoints 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # celery beat schedule file 78 | celerybeat-schedule 79 | 80 | # SageMath parsed files 81 | *.sage.py 82 | 83 | # dotenv 84 | .env 85 | 86 | # virtualenv 87 | .venv 88 | venv/ 89 | ENV/ 90 | 91 | # Spyder project settings 92 | .spyderproject 93 | .spyproject 94 | 95 | # Rope project settings 96 | .ropeproject 97 | 98 | # mkdocs documentation 99 | /site 100 | 101 | # mypy 102 | .mypy_cache/ 103 | .idea 104 | .test* 105 | testdb 106 | __pycache__/* 107 | *.pyc 108 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "flatbuffers"] 2 | path = deps/flatbuffers 3 | url = https://github.com/google/flatbuffers.git 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c), Crossbar.io Technologies GmbH 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CONTRIBUTING.rst 2 | include HISTORY.rst 3 | include LICENSE 4 | include README.rst 5 | 6 | recursive-include tests * 7 | recursive-exclude * __pycache__ 8 | recursive-exclude * *.py[co] 9 | 10 | recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif 11 | recursive-include zlmdb/flatbuffers *.fbs *.bfbs 12 | 13 | include requirements.txt 14 | include requirements-dev.txt 15 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean clean-docs clean-test clean-pyc clean-build docs help 2 | .DEFAULT_GOAL := help 3 | 4 | define BROWSER_PYSCRIPT 5 | import os, webbrowser, sys 6 | 7 | try: 8 | from urllib import pathname2url 9 | except: 10 | from urllib.request import pathname2url 11 | 12 | webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) 13 | endef 14 | export BROWSER_PYSCRIPT 15 | 16 | define PRINT_HELP_PYSCRIPT 17 | import re, sys 18 | 19 | for line in sys.stdin: 20 | match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) 21 | if match: 22 | target, help = match.groups() 23 | print("%-20s %s" % (target, help)) 24 | endef 25 | export PRINT_HELP_PYSCRIPT 26 | 27 | BROWSER := python -c "$$BROWSER_PYSCRIPT" 28 | 29 | help: 30 | @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) 31 | 32 | clean: clean_build clean_pyc clean_test clean_docs ## remove all build, test, coverage, Python artifacts and docs 33 | 34 | clean_docs: 35 | rm -fr docs/_build 36 | 37 | clean_build: ## remove build artifacts 38 | rm -fr build/ 39 | rm -fr dist/ 40 | rm -fr .eggs/ 41 | find . -name '*.egg-info' -exec rm -fr {} + 42 | find . -name '*.egg' -exec rm -f {} + 43 | 44 | clean_pyc: ## remove Python file artifacts 45 | find . -name '*.pyc' -exec rm -f {} + 46 | find . -name '*.pyo' -exec rm -f {} + 47 | find . -name '*~' -exec rm -f {} + 48 | find . -name '__pycache__' -exec rm -fr {} + 49 | 50 | clean_test: ## remove test and coverage artifacts 51 | rm -fr .tox/ 52 | rm -f .coverage 53 | rm -f .coverage.* 54 | rm -fr htmlcov/ 55 | rm -fr .pytest_cache 56 | -rm -rf .test* 57 | -rm -rf .mypy_cache 58 | 59 | lint: ## check style with flake8 60 | flake8 zlmdb tests 61 | 62 | test_single: 63 | clear && pytest -v -s zlmdb/tests/test_basic.py 64 | 65 | test_pmaps: 66 | clear && pytest -v -s zlmdb/tests/test_pmaps.py 67 | 68 | test_indexes: 69 | clear && pytest -v -s zlmdb/tests/test_pmap_indexes.py 70 | 71 | test_select: 72 | clear && pytest -v -s zlmdb/tests/test_select.py 73 | 74 | # 75 | # test ZLMDB high level API 76 | # 77 | test_zdb: test-zdb-etcd test-zdb-df test-zdb-dyn 78 | 79 | test_zdb_etcd: 80 | python tests/zdb/test_zdb_etcd.py 81 | 82 | test_zdb_df: 83 | python tests/zdb/test_zdb_df.py 84 | 85 | test_zdb_dyn: 86 | python tests/zdb/test_zdb_dyn.py 87 | 88 | test_zdb_fbs: 89 | python tests/zdb/test_zdb_fbs.py 90 | 91 | 92 | test_quick: 93 | pytest 94 | 95 | test: 96 | tox -e py36,flake8,coverage,mypy,yapf,sphinx 97 | 98 | test_all: 99 | tox 100 | 101 | coverage: ## check code coverage quickly with the default Python 102 | #coverage run --source zlmdb -m pytest 103 | coverage run --source zlmdb --omit="zlmdb/flatbuffer/reflection/*,zlmdb/flatbuffer/demo/*,zlmdb/tests/*" -m pytest -v -s zlmdb 104 | coverage report -m 105 | coverage html 106 | $(BROWSER) htmlcov/index.html 107 | 108 | docs: ## generate Sphinx HTML documentation, including API docs 109 | sphinx-build -b html ./docs ./docs/_build 110 | $(BROWSER) docs/_build/index.html 111 | 112 | dist: clean ## builds source and wheel package 113 | python setup.py sdist bdist_wheel 114 | ls -la dist 115 | unzip -l dist/zlmdb-*-py2.py3-none-any.whl 116 | 117 | # publish to PyPI 118 | publish: dist 119 | twine upload dist/* 120 | 121 | install: 122 | -pip uninstall -y pytest_asyncio # remove the broken shit 123 | -pip uninstall -y pytest_cov # remove the broken shit 124 | pip install -e . 125 | pip install -r requirements-dev.txt 126 | 127 | yapf: 128 | yapf --version 129 | yapf -rd --style=yapf.ini --exclude="zlmdb/flatbuffers/*" --exclude="zlmdb/tests/MNodeLog.py" zlmdb 130 | 131 | # auto-format code - WARNING: this my change files, in-place! 132 | autoformat: 133 | yapf -ri --style=yapf.ini --exclude="zlmdb/flatbuffers/*" zlmdb 134 | 135 | 136 | FLATC=/usr/local/bin/flatc 137 | 138 | # git submodule update --init --recursive 139 | # git submodule update --remote --merge 140 | # git submodule foreach git pull 141 | update_flatbuffers: 142 | rm -rf ./flatbuffers 143 | cp -R deps/flatbuffers/python/flatbuffers . 144 | 145 | generate_flatbuffers_reflection: 146 | $(FLATC) --python -o zlmdb/flatbuffers/ deps/flatbuffers/reflection/reflection.fbs 147 | 148 | fix_copyright: 149 | find . -type f -exec sed -i 's/Copyright (c) Crossbar.io Technologies GmbH/Copyright (c) typedef int GmbH/g' {} \; 150 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Introduction to zLMDB 2 | ===================== 3 | 4 | .. image:: https://img.shields.io/pypi/v/zlmdb.svg 5 | :target: https://pypi.python.org/pypi/zlmdb 6 | :alt: PyPI 7 | 8 | .. image:: https://github.com/crossbario/zlmdb/workflows/main/badge.svg 9 | :target: https://github.com/crossbario/zlmdb/actions?query=workflow%3Amain 10 | :alt: Build 11 | 12 | .. image:: https://readthedocs.org/projects/zlmdb/badge/?version=latest 13 | :target: https://zlmdb.readthedocs.io/en/latest/?badge=latest 14 | :alt: Documentation 15 | 16 | .. image:: https://github.com/crossbario/zlmdb/workflows/deploy/badge.svg 17 | :target: https://github.com/crossbario/zlmdb/actions?query=workflow%3Adeploy 18 | :alt: Deploy 19 | 20 | Object-relational in-memory database layer based on LMDB: 21 | 22 | * High-performance (see below) 23 | * Supports multiple serializers (JSON, CBOR, Pickle, Flatbuffers) 24 | * Supports export/import from/to Apache Arrow 25 | * Support native Numpy arrays and Pandas data frames 26 | * Automatic indexes 27 | * Free software (MIT license) 28 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +o verbose -o errexit 4 | 5 | # AWS_DEFAULT_REGION : must be set in CI build context! 6 | # AWS_S3_BUCKET_NAME : must be set in CI build context! 7 | # AWS_ACCESS_KEY_ID : must be set in CI build context! 8 | # AWS_SECRET_ACCESS_KEY : must be set in CI build context! 9 | # WAMP_PRIVATE_KEY : must be set in CI build context! 10 | 11 | echo 'AWS env vars (should be 4):' 12 | env | grep AWS_ | wc -l 13 | 14 | echo 'WAMP_PRIVATE_KEY env var (should be 1):' 15 | env | grep WAMP_PRIVATE_KEY | wc -l 16 | 17 | # set up awscli package 18 | echo 'installing aws tools ..' 19 | pip install awscli wheel 20 | which aws 21 | aws --version 22 | 23 | # build python source dist and wheels 24 | echo 'building package ..' 25 | python setup.py sdist bdist_wheel --universal 26 | ls -la ./dist 27 | 28 | # upload to S3: https://s3.eu-central-1.amazonaws.com/crossbarbuilder/wheels/ 29 | echo 'uploading package ..' 30 | # aws s3 cp --recursive ./dist s3://${AWS_S3_BUCKET_NAME}/wheels 31 | aws s3 rm s3://${AWS_S3_BUCKET_NAME}/wheels/zlmdb-${ZLMDB_VERSION}-py2.py3-none-any.whl 32 | aws s3 rm s3://${AWS_S3_BUCKET_NAME}/wheels/zlmdb-latest-py2.py3-none-any.whl 33 | 34 | aws s3 cp --acl public-read ./dist/zlmdb-${ZLMDB_VERSION}-py2.py3-none-any.whl s3://${AWS_S3_BUCKET_NAME}/wheels/zlmdb-${ZLMDB_VERSION}-py2.py3-none-any.whl 35 | aws s3 cp --acl public-read ./dist/zlmdb-${ZLMDB_VERSION}-py2.py3-none-any.whl s3://${AWS_S3_BUCKET_NAME}/wheels/zlmdb-latest-py2.py3-none-any.whl 36 | 37 | #aws s3api copy-object --acl public-read \ 38 | # --copy-source wheels/zlmdb-${ZLMDB_VERSION}-py2.py3-none-any.whl --bucket ${AWS_S3_BUCKET_NAME} \ 39 | # --key wheels/zlmdb-latest-py2.py3-none-any.whl 40 | 41 | aws s3 ls ${AWS_S3_BUCKET_NAME}/wheels/zlmdb- 42 | 43 | # tell crossbar-builder about this new wheel push 44 | # get 'wamp' command, always with latest autobahn master 45 | pip install -q -I https://github.com/crossbario/autobahn-python/archive/master.zip#egg=autobahn[twisted,serialization,encryption] 46 | 47 | # use 'wamp' to notify crossbar-builder 48 | wamp --max-failures 3 \ 49 | --authid wheel_pusher \ 50 | --url ws://office2dmz.crossbario.com:8008/ \ 51 | --realm webhook call builder.wheel_pushed \ 52 | --keyword name zlmdb \ 53 | --keyword publish true 54 | 55 | echo '' 56 | echo 'package uploaded to:' 57 | echo '' 58 | echo ' https://crossbarbuilder.s3.eu-central-1.amazonaws.com/wheels/zlmdb-'${ZLMDB_VERSION}'-py2.py3-none-any.whl' 59 | echo ' https://crossbarbuilder.s3.eu-central-1.amazonaws.com/wheels/zlmdb-latest-py2.py3-none-any.whl' 60 | echo '' 61 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = python -msphinx 7 | SPHINXPROJ = zlmdb 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/_static/performance_test1.ods: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crossbario/zlmdb/16270e41d5fadd2615cd735723cb629700fa12e8/docs/_static/performance_test1.ods -------------------------------------------------------------------------------- /docs/_static/performance_test1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crossbario/zlmdb/16270e41d5fadd2615cd735723cb629700fa12e8/docs/_static/performance_test1.png -------------------------------------------------------------------------------- /docs/_static/performance_test2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crossbario/zlmdb/16270e41d5fadd2615cd735723cb629700fa12e8/docs/_static/performance_test2.png -------------------------------------------------------------------------------- /docs/_static/zlmdb_dbformat.ods: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crossbario/zlmdb/16270e41d5fadd2615cd735723cb629700fa12e8/docs/_static/zlmdb_dbformat.ods -------------------------------------------------------------------------------- /docs/_static/zlmdb_dbformat.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crossbario/zlmdb/16270e41d5fadd2615cd735723cb629700fa12e8/docs/_static/zlmdb_dbformat.pdf -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | 17 | import sys 18 | import os 19 | import time 20 | 21 | try: 22 | import sphinx_rtd_theme 23 | except ImportError: 24 | sphinx_rtd_theme = None 25 | 26 | try: 27 | from sphinxcontrib import spelling 28 | except ImportError: 29 | spelling = None 30 | 31 | sys.path.insert(0, os.path.abspath('..')) 32 | 33 | # monkey-patch txaio so that we can "use" both twisted *and* asyncio, 34 | # at least at import time -- this is so the autodoc stuff can 35 | # successfully import autobahn.twisted.* as well as autobahn.asyncio.* 36 | # (usually, you can only import one or the other in a single Python 37 | # interpreter) 38 | import txaio 39 | 40 | def use_tx(): 41 | "monkey-patched for doc-building" 42 | from txaio import tx 43 | txaio._use_framework(tx) 44 | 45 | def use_aio(): 46 | "monkey-patched for doc-building" 47 | from txaio import aio 48 | txaio._use_framework(aio) 49 | 50 | txaio.use_twisted = use_tx 51 | txaio.use_asyncio = use_aio 52 | 53 | # -- Project information ----------------------------------------------------- 54 | 55 | project = 'zLMDB' 56 | author = 'Crossbar.io Project' 57 | language = 'en' 58 | 59 | this_year = '{0}'.format(time.strftime('%Y')) 60 | if this_year != '2018': 61 | copyright = '2018-{0}, Crossbar.io Technologies GmbH'.format(this_year) 62 | else: 63 | copyright = '2018, Crossbar.io Technologies GmbH' 64 | 65 | base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) 66 | with open(os.path.join(base_dir, "zlmdb", "_version.py")) as f: 67 | # defines __version__ 68 | exec(f.read()) 69 | 70 | version = release = __version__ # noqa 71 | 72 | # -- General configuration --------------------------------------------------- 73 | 74 | # https://sphinx-autoapi.readthedocs.io/ 75 | # https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html 76 | autoclass_content = 'both' 77 | autodoc_member_order = 'bysource' 78 | 79 | autoapi_keep_files = True 80 | autoapi_type = 'python' 81 | autoapi_dirs = [base_dir, os.path.join(base_dir, 'zlmdb')] 82 | 83 | # 'members', 84 | # 'undoc-members', 85 | # 'private-members', 86 | # 'show-inheritance', 87 | # 'show-inheritance-diagram', 88 | # 'show-module-summary', 89 | # 'special-members', 90 | # 'imported-members', 91 | autoapi_options = ['members', 'show-inheritance'] 92 | 93 | # Check if we are building on readthedocs 94 | RTD_BUILD = os.environ.get('READTHEDOCS', None) == 'True' 95 | 96 | # Add any Sphinx extension module names here, as strings. They can be 97 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 98 | # ones. 99 | extensions = [ 100 | 'sphinx.ext.viewcode', 101 | 'sphinx.ext.intersphinx', 102 | 'sphinx.ext.ifconfig', 103 | 'sphinx.ext.todo', 104 | 'sphinx.ext.doctest', 105 | 'sphinx.ext.inheritance_diagram', 106 | 107 | # https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html 108 | 'sphinx.ext.autodoc', 109 | 'sphinx.ext.autodoc.typehints', 110 | 111 | 'sphinxcontrib.spelling', 112 | 113 | # https://sphinx-autoapi.readthedocs.io 114 | # 'autoapi.extension', 115 | 116 | # Usage: .. thumbnail:: picture.png 117 | # Installation: pip install sphinxcontrib-images 118 | # Source: https://github.com/sphinx-contrib/images 119 | # 'sphinxcontrib.images', 120 | ] 121 | 122 | # The language for content autogenerated by Sphinx. Refer to documentation 123 | # for a list of supported languages. 124 | language = 'en' 125 | 126 | # extensions not available on RTD 127 | if spelling is not None: 128 | extensions.append('sphinxcontrib.spelling') 129 | 130 | spelling_lang = 'en_US' 131 | spelling_show_suggestions = False 132 | spelling_word_list_filename = 'spelling_wordlist.txt' 133 | 134 | # Add any paths that contain templates here, relative to this directory. 135 | templates_path = ['_templates'] 136 | 137 | # The suffix of source filenames. 138 | source_suffix = '.rst' 139 | 140 | # The encoding of source files. 141 | #source_encoding = 'utf-8-sig' 142 | 143 | # The master toctree document. 144 | master_doc = 'contents' 145 | 146 | # List of patterns, relative to source directory, that match files and 147 | # directories to ignore when looking for source files. 148 | # This pattern also affects html_static_path and html_extra_path. 149 | exclude_patterns = [] 150 | 151 | 152 | # -- Options for HTML output ------------------------------------------------- 153 | 154 | # The theme to use for HTML and HTML Help pages. See the documentation for 155 | # a list of builtin themes. 156 | # 157 | if sphinx_rtd_theme: 158 | html_theme = "sphinx_rtd_theme" 159 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 160 | else: 161 | html_theme = "default" 162 | 163 | pygments_style = 'sphinx' 164 | 165 | # Add any paths that contain custom static files (such as style sheets) here, 166 | # relative to this directory. They are copied after the builtin static files, 167 | # so a file named "default.css" will overwrite the builtin "default.css". 168 | html_static_path = ['_static'] 169 | 170 | intersphinx_mapping = { 171 | 'py3': ('https://docs.python.org/3', None), 172 | 'python': ('https://docs.python.org/3', None), 173 | 'rtd': ('https://docs.readthedocs.io/en/latest/', None), 174 | 'txaio': ('https://txaio.readthedocs.io/en/latest/', None), 175 | 'autobahn': ('https://autobahn.readthedocs.io/en/latest/', None), 176 | 'zlmdb': ('https://zlmdb.readthedocs.io/en/latest/', None), 177 | 'numpy': ('http://docs.scipy.org/doc/numpy', None), 178 | 'scipy': ('http://docs.scipy.org/doc/scipy/reference', None), 179 | 'matplotlib': ('http://matplotlib.org/stable', None), 180 | } 181 | -------------------------------------------------------------------------------- /docs/contents.rst: -------------------------------------------------------------------------------- 1 | :tocdepth: 0 2 | :orphan: 3 | 4 | .. _site_contents: 5 | 6 | Site Contents 7 | ============= 8 | 9 | .. toctree:: 10 | :maxdepth: 3 11 | 12 | index 13 | performance 14 | reference 15 | 16 | -------------------------------------------------------------------------------- /docs/gen.py: -------------------------------------------------------------------------------- 1 | 2 | def print_tables(tables, prefix='zlmdb', inherit=False): 3 | tables = sorted(tables) 4 | 5 | for table in tables: 6 | print('* :class:`{}.{}`'.format(prefix, table)) 7 | 8 | print('') 9 | print('------') 10 | print('') 11 | 12 | tmpl = """.. autoclass:: {}.{} 13 | :members: 14 | """ 15 | if inherit: 16 | tmpl += " :show-inheritance:" 17 | 18 | for table in tables: 19 | print(tmpl.format(prefix, table)) 20 | 21 | 22 | from zlmdb import __all__ as a1 23 | print_tables([x for x in a1 if x.startswith('Map')], inherit=True) 24 | 25 | from zlmdb._types import __dict__ as a2 26 | print_tables([x for x in a2 if x.endswith('KeysMixin')], prefix='zlmdb._types') 27 | print_tables([x for x in a2 if x.endswith('ValuesMixin')], prefix='zlmdb._types') 28 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ../README.rst -------------------------------------------------------------------------------- /docs/spelling_wordlist.txt: -------------------------------------------------------------------------------- 1 | Twisted 2 | asyncio 3 | Trollius 4 | inlineCallbacks 5 | Deferreds 6 | inline 7 | excludeMe 8 | wxPython 9 | Tox 10 | CPython 11 | stdlib 12 | timestamp 13 | Lua 14 | rawsocket 15 | serializer 16 | subprotocol 17 | subprotocols 18 | Hybi 19 | args 20 | kwargs 21 | unserialized 22 | unserialize 23 | serializable 24 | Testee 25 | Jython 26 | wallclock 27 | walltime 28 | mixin 29 | Mixin 30 | hostname 31 | serializers 32 | app 33 | apps 34 | util 35 | wamp 36 | WAMP 37 | Ctor 38 | Iff 39 | iff 40 | ping 41 | pong 42 | openHandshakeTimeout 43 | closeHandshakeTimeout 44 | wasClean 45 | logOctets 46 | logFrames 47 | reasonUtf 48 | sendMessage 49 | websocket 50 | validator 51 | Bjoern 52 | Hoehrmann 53 | codepoint 54 | currentIndex 55 | totalIndex 56 | websocket 57 | xormasker 58 | localhost 59 | polyfill 60 | useragent 61 | sendPing 62 | sendPong 63 | sendClose 64 | Nagle 65 | endsOnCodePoint 66 | Changelog 67 | changelog 68 | docstrings 69 | websockify 70 | ws 71 | wss 72 | slowsquare 73 | plugin 74 | pubsub 75 | Peticolas 76 | isSecure 77 | permessage 78 | Nagle 79 | endsOnCodePoint 80 | blog 81 | backport 82 | backports 83 | twistd 84 | frontend 85 | backend 86 | frontends 87 | backends 88 | setProtocolOptions 89 | serverConnectionDropTimeout 90 | reasonRaw 91 | serverStatus 92 | onConnect 93 | namespace 94 | unsubscribe 95 | bzip 96 | http 97 | uri 98 | longpoll 99 | choosereactor 100 | flashpolicy 101 | autoimport 102 | longpoll 103 | chopsize 104 | lifecycle 105 | Lifecycle 106 | Callees 107 | callees 108 | Testsuite 109 | testsuite 110 | Subpackages 111 | subpath 112 | subpaths 113 | pongs 114 | pings 115 | params 116 | unescaped 117 | utf 118 | acknowledgement 119 | unregistration 120 | unregister 121 | unregistered 122 | unsubscription 123 | unsubscribe 124 | unsubscribed 125 | deserialization 126 | deserialize 127 | -------------------------------------------------------------------------------- /flatbuffers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .builder import Builder 16 | from .table import Table 17 | from .compat import range_func as compat_range 18 | from ._version import __version__ 19 | from . import util 20 | -------------------------------------------------------------------------------- /flatbuffers/_version.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Placeholder, to be updated during the release process 16 | # by the setup.py 17 | __version__ = u"latest" 18 | -------------------------------------------------------------------------------- /flatbuffers/compat.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ A tiny version of `six` to help with backwards compability. Also includes 16 | compatibility helpers for numpy. """ 17 | 18 | import sys 19 | 20 | PY2 = sys.version_info[0] == 2 21 | PY26 = sys.version_info[0:2] == (2, 6) 22 | PY27 = sys.version_info[0:2] == (2, 7) 23 | PY275 = sys.version_info[0:3] >= (2, 7, 5) 24 | PY3 = sys.version_info[0] == 3 25 | PY34 = sys.version_info[0:2] >= (3, 4) 26 | 27 | if PY3: 28 | import importlib.machinery 29 | string_types = (str,) 30 | binary_types = (bytes,bytearray) 31 | range_func = range 32 | memoryview_type = memoryview 33 | struct_bool_decl = "?" 34 | else: 35 | import imp 36 | string_types = (unicode,) 37 | if PY26 or PY27: 38 | binary_types = (str,bytearray) 39 | else: 40 | binary_types = (str,) 41 | range_func = xrange 42 | if PY26 or (PY27 and not PY275): 43 | memoryview_type = buffer 44 | struct_bool_decl = "=9.0.1 2 | bumpversion>=0.5.3 3 | wheel>=0.30.0 4 | watchdog>=0.8.3 5 | flake8>=3.5.0 6 | tox>=2.9.1 7 | tox-gh-actions>=2.2.0 8 | codecov>=2.0.15 9 | sphinx>=1.7.1 10 | sphinxcontrib-images 11 | sphinxcontrib-spelling 12 | sphinx-autoapi 13 | sphinx_rtd_theme 14 | twine>=1.10.0 15 | pytest>=3.4.2 16 | pytest-runner>=2.11.1 17 | humanize>=0.5.1 18 | backports.tempfile>=1.0 19 | # https://github.com/google/yapf/issues/712 20 | yapf==0.29.0 21 | pylint>=1.9.2 22 | pyyaml>=4.2b4 23 | types-PyYAML>=6.0.1 24 | mypy>=0.610; python_version >= '3.4' and platform_python_implementation != 'PyPy' 25 | twisted>=18.7.0 26 | -------------------------------------------------------------------------------- /requirements-rtd.txt: -------------------------------------------------------------------------------- 1 | # requirements for building the docs on RTD 2 | txaio 3 | twisted 4 | sphinxcontrib-images 5 | sphinxcontrib-spelling 6 | sphinx-autoapi 7 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | twisted 2 | txaioetcd 3 | numpy 4 | pandas 5 | pyarrow 6 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.1.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | search = version='{current_version}' 8 | replace = version='{new_version}' 9 | 10 | [bumpversion:file:zlmdb/__init__.py] 11 | search = __version__ = '{current_version}' 12 | replace = __version__ = '{new_version}' 13 | 14 | [bdist_wheel] 15 | universal = 1 16 | 17 | [flake8] 18 | exclude = docs 19 | 20 | [aliases] 21 | # Define setup.py command aliases here 22 | test = pytest 23 | 24 | [tool:pytest] 25 | collect_ignore = ['setup.py'] 26 | 27 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | ############################################################################### 5 | # 6 | # The MIT License (MIT) 7 | # 8 | # Copyright (c) typedef int GmbH 9 | # 10 | # Permission is hereby granted, free of charge, to any person obtaining a copy 11 | # of this software and associated documentation files (the "Software"), to deal 12 | # in the Software without restriction, including without limitation the rights 13 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | # copies of the Software, and to permit persons to whom the Software is 15 | # furnished to do so, subject to the following conditions: 16 | # 17 | # The above copyright notice and this permission notice shall be included in 18 | # all copies or substantial portions of the Software. 19 | # 20 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 23 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 25 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 26 | # THE SOFTWARE. 27 | # 28 | ############################################################################### 29 | 30 | """The setup script.""" 31 | 32 | import os 33 | from setuptools import setup 34 | 35 | with open('zlmdb/_version.py') as f: 36 | exec(f.read()) # defines __version__ 37 | 38 | with open('README.rst') as readme_file: 39 | readme = readme_file.read() 40 | 41 | # enforce use of CFFI for LMDB 42 | os.environ['LMDB_FORCE_CFFI'] = '1' 43 | 44 | # enforce use of bundled libsodium with PyNaCl 45 | os.environ['SODIUM_INSTALL'] = 'bundled' 46 | 47 | requirements = [ 48 | 'cffi>=1.15.1', 49 | 'cbor2>=5.4.6', 50 | 'click>=8.1.3', 51 | 'flatbuffers>=23.1.4', 52 | 'lmdb>=1.4.0', 53 | 'pynacl>=1.5.0', 54 | 'pyyaml>=6.0', 55 | 'txaio>=23.1.1', 56 | 'numpy>=1.24.1', 57 | ] 58 | 59 | extras_require = { 60 | 'dev': [] 61 | } 62 | 63 | with open('requirements-dev.txt') as f: 64 | for line in f.read().splitlines(): 65 | extras_require['dev'].append(line.strip()) 66 | 67 | test_requirements = ['pytest', 'pytest-runner'] 68 | 69 | packages = [ 70 | 'flatbuffers', 71 | 'zlmdb', 72 | 'zlmdb.flatbuffers', 73 | 'zlmdb.flatbuffers.reflection', 74 | ] 75 | 76 | setup( 77 | author="typedef int GmbH", 78 | classifiers=[ 79 | 'Development Status :: 5 - Production/Stable', 80 | 'Intended Audience :: Developers', 81 | 'License :: OSI Approved :: MIT License', 82 | 'Natural Language :: English', 83 | 'Programming Language :: Python :: 3', 84 | 'Programming Language :: Python :: 3.7', 85 | 'Programming Language :: Python :: 3.8', 86 | 'Programming Language :: Python :: 3.9', 87 | 'Programming Language :: Python :: 3.10', 88 | 'Programming Language :: Python :: 3.11', 89 | 'Programming Language :: Python :: Implementation :: CPython', 90 | 'Programming Language :: Python :: Implementation :: PyPy', 91 | ], 92 | description="Object-relational zero-copy in-memory database layer for LMDB.", 93 | entry_points={ 94 | 'console_scripts': [ 95 | 'zlmdb=zlmdb.cli:main', 96 | ], 97 | }, 98 | python_requires='>=3.7', 99 | install_requires=requirements, 100 | extras_require=extras_require, 101 | license="MIT license", 102 | long_description=readme, 103 | include_package_data=True, 104 | keywords='zlmdb', 105 | name='zlmdb', 106 | packages=packages, 107 | test_suite='tests', 108 | tests_require=test_requirements, 109 | url='https://github.com/crossbario/zlmdb', 110 | version=__version__, 111 | zip_safe=True, 112 | ) 113 | -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- 1 | generate: 2 | ~/scm/3rdparty/flatbuffers/flatc --python -o _gen user.fbs 3 | -------------------------------------------------------------------------------- /tests/_gen/crossbarfx/Date.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: crossbarfx 4 | 5 | import flatbuffers 6 | 7 | class Date(object): 8 | __slots__ = ['_tab'] 9 | 10 | # Date 11 | def Init(self, buf, pos): 12 | self._tab = flatbuffers.table.Table(buf, pos) 13 | 14 | # Date 15 | def Year(self): return self._tab.Get(flatbuffers.number_types.Uint16Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(0)) 16 | # Date 17 | def Month(self): return self._tab.Get(flatbuffers.number_types.Uint8Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(2)) 18 | # Date 19 | def Day(self): return self._tab.Get(flatbuffers.number_types.Uint8Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(3)) 20 | 21 | def CreateDate(builder, year, month, day): 22 | builder.Prep(2, 4) 23 | builder.PrependUint8(day) 24 | builder.PrependUint8(month) 25 | builder.PrependUint16(year) 26 | return builder.Offset() 27 | -------------------------------------------------------------------------------- /tests/_gen/crossbarfx/Rating.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: crossbarfx 4 | 5 | import flatbuffers 6 | 7 | class Rating(object): 8 | __slots__ = ['_tab'] 9 | 10 | @classmethod 11 | def GetRootAsRating(cls, buf, offset): 12 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 13 | x = Rating() 14 | x.Init(buf, n + offset) 15 | return x 16 | 17 | # Rating 18 | def Init(self, buf, pos): 19 | self._tab = flatbuffers.table.Table(buf, pos) 20 | 21 | # Rating 22 | def Name(self): 23 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 24 | if o != 0: 25 | return self._tab.String(o + self._tab.Pos) 26 | return None 27 | 28 | # Rating 29 | def Rating(self): 30 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 31 | if o != 0: 32 | return self._tab.Get(flatbuffers.number_types.Float32Flags, o + self._tab.Pos) 33 | return 0.0 34 | 35 | def RatingStart(builder): builder.StartObject(2) 36 | def RatingAddName(builder, name): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(name), 0) 37 | def RatingAddRating(builder, rating): builder.PrependFloat32Slot(1, rating, 0.0) 38 | def RatingEnd(builder): return builder.EndObject() 39 | -------------------------------------------------------------------------------- /tests/_gen/crossbarfx/Tag.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: crossbarfx 4 | 5 | class Tag(object): 6 | GEEK = 0 7 | VIP = 1 8 | 9 | -------------------------------------------------------------------------------- /tests/_gen/crossbarfx/User.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: crossbarfx 4 | 5 | import flatbuffers 6 | 7 | class User(object): 8 | __slots__ = ['_tab'] 9 | 10 | @classmethod 11 | def GetRootAsUser(cls, buf, offset): 12 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 13 | x = User() 14 | x.Init(buf, n + offset) 15 | return x 16 | 17 | # User 18 | def Init(self, buf, pos): 19 | self._tab = flatbuffers.table.Table(buf, pos) 20 | 21 | # User 22 | def Name(self): 23 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 24 | if o != 0: 25 | return self._tab.String(o + self._tab.Pos) 26 | return None 27 | 28 | # User 29 | def Authid(self): 30 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 31 | if o != 0: 32 | return self._tab.String(o + self._tab.Pos) 33 | return None 34 | 35 | # User 36 | def Email(self): 37 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) 38 | if o != 0: 39 | return self._tab.String(o + self._tab.Pos) 40 | return None 41 | 42 | # User 43 | def Birthday(self): 44 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) 45 | if o != 0: 46 | x = o + self._tab.Pos 47 | from .Date import Date 48 | obj = Date() 49 | obj.Init(self._tab.Bytes, x) 50 | return obj 51 | return None 52 | 53 | # User 54 | def IsFriendly(self): 55 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) 56 | if o != 0: 57 | return bool(self._tab.Get(flatbuffers.number_types.BoolFlags, o + self._tab.Pos)) 58 | return False 59 | 60 | # User 61 | def Tags(self, j): 62 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) 63 | if o != 0: 64 | a = self._tab.Vector(o) 65 | return self._tab.Get(flatbuffers.number_types.Int8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1)) 66 | return 0 67 | 68 | # User 69 | def TagsAsNumpy(self): 70 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) 71 | if o != 0: 72 | return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int8Flags, o) 73 | return 0 74 | 75 | # User 76 | def TagsLength(self): 77 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) 78 | if o != 0: 79 | return self._tab.VectorLen(o) 80 | return 0 81 | 82 | # User 83 | def Ratings(self, j): 84 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) 85 | if o != 0: 86 | x = self._tab.Vector(o) 87 | x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 88 | x = self._tab.Indirect(x) 89 | from .Rating import Rating 90 | obj = Rating() 91 | obj.Init(self._tab.Bytes, x) 92 | return obj 93 | return None 94 | 95 | # User 96 | def RatingsLength(self): 97 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) 98 | if o != 0: 99 | return self._tab.VectorLen(o) 100 | return 0 101 | 102 | def UserStart(builder): builder.StartObject(7) 103 | def UserAddName(builder, name): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(name), 0) 104 | def UserAddAuthid(builder, authid): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(authid), 0) 105 | def UserAddEmail(builder, email): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(email), 0) 106 | def UserAddBirthday(builder, birthday): builder.PrependStructSlot(3, flatbuffers.number_types.UOffsetTFlags.py_type(birthday), 0) 107 | def UserAddIsFriendly(builder, isFriendly): builder.PrependBoolSlot(4, isFriendly, 0) 108 | def UserAddTags(builder, tags): builder.PrependUOffsetTRelativeSlot(5, flatbuffers.number_types.UOffsetTFlags.py_type(tags), 0) 109 | def UserStartTagsVector(builder, numElems): return builder.StartVector(1, numElems, 1) 110 | def UserAddRatings(builder, ratings): builder.PrependUOffsetTRelativeSlot(6, flatbuffers.number_types.UOffsetTFlags.py_type(ratings), 0) 111 | def UserStartRatingsVector(builder, numElems): return builder.StartVector(4, numElems, 4) 112 | def UserEnd(builder): return builder.EndObject() 113 | -------------------------------------------------------------------------------- /tests/_gen/crossbarfx/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crossbario/zlmdb/16270e41d5fadd2615cd735723cb629700fa12e8/tests/_gen/crossbarfx/__init__.py -------------------------------------------------------------------------------- /tests/test_cbor.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import pickle 3 | import json 4 | import cbor2 5 | from typing import Optional, List, Dict 6 | 7 | RESULT = { 8 | 'objects': 0, 9 | 'bytes': 0 10 | } 11 | 12 | 13 | class Tag(object): 14 | GEEK = 1 15 | VIP = 2 16 | 17 | # requires python 3.6+: 18 | # 19 | # class User(object): 20 | # oid: int 21 | # name: str 22 | # authid: str 23 | # email: str 24 | # birthday: datetime.date 25 | # is_friendly: bool 26 | # tags: Optional[List[str]] 27 | # ratings: Dict[str, float] = {} 28 | # friends: List[int] = [] 29 | # referred_by: int = None 30 | 31 | 32 | class User(object): 33 | def __init__(self): 34 | self.oid = None 35 | self.name = None 36 | self.authid = None 37 | self.email = None 38 | self.birthday = None 39 | self.is_friendly = None 40 | self.tags = None 41 | self.ratings = {} 42 | self.friends = [] 43 | self.referred_by = None 44 | 45 | def marshal(self): 46 | obj = { 47 | 'oid': self.oid, 48 | 'name': self.name, 49 | 'authid': self.authid, 50 | 'email': self.email, 51 | 'birthday': self.birthday, 52 | 'is_friendly': self.is_friendly, 53 | 'tags': self.tags, 54 | 'ratings': self.ratings, 55 | 'friends': self.friends, 56 | 'referred_by': self.referred_by, 57 | } 58 | return obj 59 | 60 | @staticmethod 61 | def parse(obj): 62 | user = User() 63 | user.oid = obj.get('oid', None) 64 | user.name = obj.get('name', None) 65 | user.authid = obj.get('authid', None) 66 | user.email = obj.get('email', None) 67 | user.birthday = obj.get('birthday', None) 68 | user.is_friendly = obj.get('is_friendly', None) 69 | user.tags = obj.get('tags', None) 70 | user.ratings = obj.get('ratings', {}) 71 | user.friends = obj.get('friends', []) 72 | user.referred_by = obj.get('referred_by', None) 73 | return user 74 | 75 | 76 | def test(): 77 | global RESULT 78 | 79 | user = User() 80 | user.oid = 23 81 | user.name = 'Homer Simpson' 82 | user.authid = 'homer' 83 | user.email = 'homer.simpson@example.com' 84 | user.birthday = { 85 | 'year': 1950, 86 | 'month': 12, 87 | 'day': 24 88 | } 89 | user.is_friendly = True 90 | user.tags = [Tag.GEEK, Tag.VIP] 91 | 92 | #data = json.dumps(user.marshal(), ensure_ascii=False) 93 | data = cbor2.dumps(user.marshal()) 94 | 95 | RESULT['objects'] += 1 96 | RESULT['bytes'] += len(data) 97 | 98 | 99 | import timeit 100 | 101 | N = 1000 102 | M = 100000 103 | 104 | for i in range(N): 105 | secs = timeit.timeit(test, number=M) 106 | ops = round(float(M) / secs, 1) 107 | print('{} objects/sec'.format(ops)) 108 | print(RESULT) 109 | -------------------------------------------------------------------------------- /tests/test_fbs_reflection.py: -------------------------------------------------------------------------------- 1 | import txaio 2 | txaio.use_twisted() 3 | 4 | from autobahn.xbr._schema import FbsSchema 5 | 6 | for filename in [ 7 | '/tmp/test/bfbs/climate.bfbs', 8 | '/tmp/test/bfbs/network.bfbs', 9 | '/tmp/test/bfbs/location.bfbs']: 10 | schema = FbsSchema.load(filename) 11 | print(schema) 12 | -------------------------------------------------------------------------------- /tests/test_flatbuffers.py: -------------------------------------------------------------------------------- 1 | import flatbuffers 2 | from _gen.crossbarfx import User, Date, Tag, Rating 3 | 4 | RESULT = { 5 | 'objects': 0, 6 | 'bytes': 0 7 | } 8 | 9 | 10 | def test(): 11 | global RESULT 12 | 13 | builder = flatbuffers.Builder(0) 14 | 15 | name = builder.CreateString('Homer Simpson') 16 | authid = builder.CreateString('homer') 17 | email = builder.CreateString('homer.simpson@example.com') 18 | 19 | User.UserStartTagsVector(builder, 2) 20 | builder.PrependUOffsetTRelative(Tag.Tag.GEEK) 21 | builder.PrependUOffsetTRelative(Tag.Tag.VIP) 22 | tags = builder.EndVector(2) 23 | 24 | ratings = None 25 | if False: 26 | _ratings = { 27 | 'dawn-of-the-dead': 6.9, 28 | 'day-of-the-dead': 7.5, 29 | 'land-of-the-dead': 8.9 30 | } 31 | 32 | _ratings_strings = { 33 | } 34 | for name in _ratings.keys(): 35 | _name = builder.CreateString(name) 36 | _ratings_strings[_name] = name 37 | 38 | User.UserStartRatingsVector(builder, len(_ratings)) 39 | l = [] 40 | for _name, _rating in _ratings.items(): 41 | Rating.RatingStart(builder) 42 | Rating.RatingAddName(builder, _ratings_strings[_name]) 43 | Rating.RatingAddRating(builder, _rating) 44 | rating = Rating.RatingEnd(builder) 45 | l.append(rating) 46 | ratings = builder.EndVector(len(_ratings)) 47 | 48 | User.UserStart(builder) 49 | 50 | User.UserAddName(builder, name) 51 | User.UserAddAuthid(builder, authid) 52 | User.UserAddEmail(builder, email) 53 | User.UserAddBirthday(builder, Date.CreateDate(builder, 1950, 12, 24)) 54 | User.UserAddIsFriendly(builder, True) 55 | User.UserAddTags(builder, tags) 56 | 57 | if ratings: 58 | User.UserAddRatings(builder, ratings) 59 | 60 | user = User.UserEnd(builder) 61 | 62 | builder.Finish(user) 63 | 64 | buf = builder.Output() 65 | 66 | #data = bytes(buf) 67 | 68 | RESULT['objects'] += 1 69 | RESULT['bytes'] += len(buf) 70 | # RESULT['bytes'] += len(data) 71 | 72 | 73 | import timeit 74 | 75 | N = 1000 76 | M = 100000 77 | 78 | for i in range(N): 79 | secs = timeit.timeit(test, number=M) 80 | ops = round(float(M) / secs, 1) 81 | print('{} objects/sec'.format(ops)) 82 | print(RESULT) 83 | -------------------------------------------------------------------------------- /tests/test_new.py: -------------------------------------------------------------------------------- 1 | import random 2 | import zlmdb 3 | 4 | 5 | class Foo(object): 6 | 7 | oid: int 8 | value: float 9 | msg: str 10 | 11 | def __init__(self, oid=None, value=None, msg=None): 12 | self.oid = oid 13 | self.value = value 14 | self.msg = msg 15 | 16 | @staticmethod 17 | def unmarshal(obj): 18 | return Foo(obj['oid'], obj['value'], obj['msg']) 19 | 20 | def marshal(self): 21 | return {'oid': self.oid, 'value': self.value, 'msg': self.msg} 22 | 23 | 24 | class MySchema(zlmdb.Schema): 25 | 26 | tab1: zlmdb.MapOidPickle = zlmdb.MapOidPickle(1) 27 | # tab1: zlmdb.MapOidJson = zlmdb.MapOidJson(1, marshal=Foo.marshal, unmarshal=Foo.unmarshal) 28 | # tab1: zlmdb.MapOidCbor = zlmdb.MapOidCbor(1, marshal=Foo.marshal, unmarshal=Foo.unmarshal) 29 | 30 | 31 | schema = MySchema() 32 | 33 | with schema.open('.testdb') as db: 34 | 35 | with db.begin(write=True) as txn: 36 | 37 | o1 = Foo(23, random.random(), 'Hello, world!') 38 | 39 | schema.tab1[txn, o1.oid] = o1 40 | print('object saved:', o1.oid, o1.value, o1.msg) 41 | 42 | o2 = schema.tab1[txn, o1.oid] 43 | assert o2 44 | print('object loaded:', o2.oid, o2.value, o2.msg) 45 | 46 | print('transaction committed') 47 | 48 | print('database closed') 49 | -------------------------------------------------------------------------------- /tests/test_ops.py: -------------------------------------------------------------------------------- 1 | def test_insert1(env): 2 | users = [] 3 | 4 | user1 = User() 5 | user1.oid = 1 6 | user1.name = 'Homer Simpson' 7 | user1.authid = 'homer' 8 | user1.email = 'homer.simpson@example.com' 9 | user1.birthday = datetime.date(1950, 12, 24) 10 | user1.is_friendly = True 11 | user1.tags = ['relaxed', 'beerfan'] 12 | users.append(user1) 13 | 14 | user2 = User() 15 | user2.oid = 2 16 | user2.name = 'Crocodile Dundee' 17 | user2.authid = 'crocoboss' 18 | user2.email = 'croco@example.com' 19 | user2.birthday = datetime.date(1960, 2, 4) 20 | user2.is_friendly = False 21 | user2.tags = ['red', 'yellow'] 22 | user2.referred_by = user1.oid 23 | users.append(user2) 24 | 25 | user3 = User() 26 | user3.oid = 3 27 | user3.name = 'Foobar Space' 28 | user3.authid = 'foobar' 29 | user3.email = 'foobar@example.com' 30 | user3.birthday = datetime.date(1970, 5, 7) 31 | user3.is_friendly = True 32 | user3.tags = ['relaxed', 'beerfan'] 33 | user3.referred_by = user1.oid 34 | users.append(user3) 35 | 36 | with Transaction(env, write=True) as txn: 37 | for user in users: 38 | _user = txn.users[user.oid] 39 | if not _user: 40 | txn.users[user.oid] = user 41 | #txn.users_by_authid[user.authid] = user.oid 42 | print('user stored', user) 43 | else: 44 | print('user loaded', _user) 45 | 46 | def test_insert2(env): 47 | with Transaction(env, write=True) as txn: 48 | for i in range(100): 49 | user = User() 50 | user.oid = i + 10 51 | user.name = 'Test {}'.format(i) 52 | user.authid = 'test-{}'.format(i) 53 | user.email = '{}@example.com'.format(user.authid) 54 | for j in range(10): 55 | user.ratings['test-rating-{}'.format(j)] = random.random() 56 | 57 | _user = txn.users[user.oid] 58 | if not _user: 59 | txn.users[user.oid] = user 60 | #txn.users_by_authid[user.authid] = user.oid 61 | print('user stored', user, user.oid, user.authid) 62 | else: 63 | print('user loaded', _user, _user.oid, _user.authid) 64 | 65 | 66 | def test_insert3(env): 67 | oid = 4 68 | 69 | with Transaction(env, write=True) as txn: 70 | user = txn.users[oid] 71 | if not user: 72 | user = User() 73 | user.oid = oid 74 | user.name = 'Foobar Space' 75 | user.authid = 'foobar' 76 | user.email = 'foobar@example.com' 77 | user.birthday = datetime.date(1970, 5, 7) 78 | user.is_friendly = True 79 | user.tags = ['relaxed', 'beerfan'] 80 | user.referred_by = 1 81 | 82 | txn.users[oid] = user 83 | print('user stored', user) 84 | else: 85 | print('user loaded', user) 86 | 87 | 88 | def test_by_auth(env): 89 | with Transaction(env) as txn: 90 | for i in range(100): 91 | authid = 'test-{}'.format(i) 92 | oid = txn.idx_users_by_authid[authid] 93 | if oid: 94 | user = txn.users[oid] 95 | print('success: user "{}" loaded by authid "{}"'.format(oid, authid)) 96 | else: 97 | print('failure: user not found for authid "{}"'.format(authid)) 98 | 99 | 100 | def test_by_email(env): 101 | with Transaction(env) as txn: 102 | for i in range(100): 103 | email = 'test-{}@example.com'.format(i) 104 | oid = txn.idx_users_by_email[email] 105 | if oid: 106 | user = txn.users[oid] 107 | print('success: user "{}" loaded by email "{}"'.format(oid, email)) 108 | else: 109 | print('failure: user not found for email "{}"'.format(email)) 110 | 111 | 112 | def test_truncate_index(env): 113 | with Transaction(env, write=True) as txn: 114 | rows = txn.users_by_authid.truncate() 115 | print('users_by_authid truncated: {} rows'.format(rows)) 116 | -------------------------------------------------------------------------------- /tests/test_pickle.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import pickle 3 | from typing import Optional, List, Dict 4 | 5 | RESULT = { 6 | 'objects': 0, 7 | 'bytes': 0 8 | } 9 | 10 | 11 | class Tag(object): 12 | GEEK = 1 13 | VIP = 2 14 | 15 | # requires python 3.6+: 16 | # 17 | # class User(object): 18 | # oid: int 19 | # name: str 20 | # authid: str 21 | # email: str 22 | # birthday: datetime.date 23 | # is_friendly: bool 24 | # tags: Optional[List[str]] 25 | # ratings: Dict[str, float] = {} 26 | # friends: List[int] = [] 27 | # referred_by: int = None 28 | 29 | 30 | class User(object): 31 | def __init__(self): 32 | self.oid = None 33 | self.name = None 34 | self.authid = None 35 | self.email = None 36 | self.birthday = None 37 | self.is_friendly = None 38 | self.tags = None 39 | self.friends = None 40 | self.referred_by = None 41 | 42 | 43 | def test(): 44 | global RESULT 45 | 46 | user = User() 47 | user.oid = 23 48 | user.name = 'Homer Simpson' 49 | user.authid = 'homer' 50 | user.email = 'homer.simpson@example.com' 51 | 52 | user.birthday = datetime.date(1950, 12, 24) 53 | user.is_friendly = True 54 | user.tags = [Tag.GEEK, Tag.VIP] 55 | 56 | data = pickle.dumps(user, protocol=4) 57 | 58 | RESULT['objects'] += 1 59 | RESULT['bytes'] += len(data) 60 | 61 | 62 | import timeit 63 | 64 | N = 1000 65 | M = 100000 66 | 67 | for i in range(N): 68 | secs = timeit.timeit(test, number=M) 69 | ops = round(float(M) / secs, 1) 70 | print('{} objects/sec'.format(ops)) 71 | print(RESULT) 72 | -------------------------------------------------------------------------------- /tests/test_zlmdb.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Tests for `zlmdb` package.""" 5 | 6 | import pytest 7 | 8 | from click.testing import CliRunner 9 | 10 | from zlmdb import _transaction 11 | from zlmdb import cli 12 | 13 | 14 | @pytest.fixture 15 | def response(): 16 | """Sample pytest fixture. 17 | 18 | See more at: http://doc.pytest.org/en/latest/fixture.html 19 | """ 20 | # import requests 21 | # return requests.get('https://github.com/audreyr/cookiecutter-pypackage') 22 | 23 | 24 | def test_content(response): 25 | """Sample pytest test function with the pytest fixture as an argument.""" 26 | # from bs4 import BeautifulSoup 27 | # assert 'GitHub' in BeautifulSoup(response.content).title.string 28 | 29 | 30 | def test_command_line_interface(): 31 | """Test the CLI.""" 32 | runner = CliRunner() 33 | result = runner.invoke(cli.main) 34 | assert result.exit_code == 0 35 | assert 'zlmdb.cli.main' in result.output 36 | help_result = runner.invoke(cli.main, ['--help']) 37 | assert help_result.exit_code == 0 38 | assert '--help Show this message and exit.' in help_result.output 39 | -------------------------------------------------------------------------------- /tests/user.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | import datetime 3 | 4 | 5 | class User(object): 6 | 7 | def __init__(self): 8 | self.oid = None 9 | self.name = None 10 | self.authid = None 11 | self.uuid = None 12 | self.email = None 13 | self.birthday = None 14 | self.is_friendly = None 15 | self.tags = None 16 | self.ratings = {} 17 | self.friends = [] 18 | self.referred_by = None 19 | 20 | def marshal(self): 21 | obj = { 22 | 'oid': self.oid, 23 | 'name': self.name, 24 | 'authid': self.authid, 25 | 'uuid': self.uuid.hex if self.uuid else None, 26 | 'email': self.email, 27 | 'birthday': { 28 | 'year': self.birthday.year if self.birthday else None, 29 | 'month': self.birthday.month if self.birthday else None, 30 | 'day': self.birthday.day if self.birthday else None, 31 | }, 32 | 'is_friendly': self.is_friendly, 33 | 'tags': self.tags, 34 | 'ratings': self.ratings, 35 | 'friends': self.friends, 36 | 'referred_by': self.referred_by, 37 | } 38 | return obj 39 | 40 | @staticmethod 41 | def parse(obj): 42 | user = User() 43 | user.oid = obj.get('oid', None) 44 | user.name = obj.get('name', None) 45 | user.authid = obj.get('authid', None) 46 | if 'uuid' in obj: 47 | user.uuid = uuid.UUID(hex=obj['uuid']) 48 | user.email = obj.get('email', None) 49 | if 'birthday' in obj: 50 | b = obj['birthday'] 51 | user.birthday = datetime.date(b.year, b.month, b.day) 52 | user.is_friendly = obj.get('is_friendly', None) 53 | user.tags = obj.get('tags', None) 54 | user.ratings = obj.get('ratings', {}) 55 | user.friends = obj.get('friends', []) 56 | user.referred_by = obj.get('referred_by', None) 57 | return user 58 | -------------------------------------------------------------------------------- /tests/user_typed.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | import datetime 3 | from typing import Optional, List, Dict 4 | 5 | 6 | class User(object): 7 | oid: int 8 | name: str 9 | authid: str 10 | uuid: uuid.UUID 11 | email: str 12 | birthday: datetime.date 13 | is_friendly: bool 14 | tags: Optional[List[str]] 15 | ratings: Dict[str, float] = {} 16 | friends: List[int] = [] 17 | referred_by: int = None 18 | 19 | def marshal(self): 20 | obj = { 21 | 'oid': self.oid, 22 | 'name': self.name, 23 | 'authid': self.authid, 24 | 'uuid': self.uuid.hex if self.uuid else None, 25 | 'email': self.email, 26 | 'birthday': { 27 | 'year': self.birthday.year if self.birthday else None, 28 | 'month': self.birthday.month if self.birthday else None, 29 | 'day': self.birthday.day if self.birthday else None, 30 | }, 31 | 'is_friendly': self.is_friendly, 32 | 'tags': self.tags, 33 | 'ratings': self.ratings, 34 | 'friends': self.friends, 35 | 'referred_by': self.referred_by, 36 | } 37 | return obj 38 | 39 | @staticmethod 40 | def parse(obj): 41 | user = User() 42 | user.oid = obj.get('oid', None) 43 | user.name = obj.get('name', None) 44 | user.authid = obj.get('authid', None) 45 | if 'uuid' in obj: 46 | user.uuid = uuid.UUID(hex=obj['uuid']) 47 | user.email = obj.get('email', None) 48 | if 'birthday' in obj: 49 | b = obj['birthday'] 50 | user.birthday = datetime.date(b.year, b.month, b.day) 51 | user.is_friendly = obj.get('is_friendly', None) 52 | user.tags = obj.get('tags', None) 53 | user.ratings = obj.get('ratings', {}) 54 | user.friends = obj.get('friends', []) 55 | user.referred_by = obj.get('referred_by', None) 56 | return user 57 | -------------------------------------------------------------------------------- /tests/zdb/README.md: -------------------------------------------------------------------------------- 1 | # ZLMDB high level API tests 2 | 3 | * [test_zdb_df.py](test_zdb_df.py): test pandas dataframe integration 4 | * [test_zdb_etcd.py](test_zdb_etcd.py): test etcd data integration 5 | * [test_zdb_dyn.py](test_zdb_dyn.py): test self describing database format 6 | 7 | ## Notes 8 | 9 | ```console 10 | signify -S -s key.sec -m message.txt -x msg.sig 11 | ``` 12 | -------------------------------------------------------------------------------- /tests/zdb/_schema_fbs.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) typedef int GmbH 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in 15 | # all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | # THE SOFTWARE. 24 | # 25 | ############################################################################### 26 | 27 | import random 28 | import uuid 29 | import datetime 30 | 31 | from zlmdb.flatbuffers.demo import User as _user 32 | from zlmdb.flatbuffers.demo import Date as _date 33 | 34 | 35 | class User(object): 36 | def __init__(self, from_fbs=None): 37 | self._from_fbs = from_fbs 38 | 39 | self._name = None 40 | self._authid = None 41 | self._uuid = None 42 | self._email = None 43 | self._birthday = None 44 | self._is_friendly = None 45 | self._tags = None 46 | self._ratings = None 47 | self._ratings_cached = None 48 | self._friends = None 49 | self._friends_cached = None 50 | self._referred_by = None 51 | 52 | @property 53 | def name(self): 54 | return self._name or self._from_fbs.Name() 55 | 56 | @name.setter 57 | def name(self, value): 58 | self._name = value 59 | 60 | @property 61 | def authid(self): 62 | return self._authid or self._from_fbs.Authid() 63 | 64 | @authid.setter 65 | def authid(self, value): 66 | self._authid = value 67 | 68 | @property 69 | def uuid(self): 70 | return self._uuid or self._from_fbs.Uuid() 71 | 72 | @uuid.setter 73 | def uuid(self, value): 74 | self._uuid = value 75 | 76 | @property 77 | def email(self): 78 | return self._email or self._from_fbs.Email() 79 | 80 | @email.setter 81 | def email(self, value): 82 | self._email = value 83 | 84 | @property 85 | def birthday(self): 86 | return self._birthday or self._from_fbs.Birthday() 87 | 88 | @birthday.setter 89 | def birthday(self, value): 90 | self._birthday = value 91 | 92 | @property 93 | def is_friendly(self): 94 | return self._is_friendly or self._from_fbs.IsFriendly() 95 | 96 | @is_friendly.setter 97 | def is_friendly(self, value): 98 | self._is_friendly = value 99 | 100 | @property 101 | def ratings(self): 102 | if self._ratings is not None: 103 | return self._ratings 104 | if self._ratings_cached is None: 105 | self._ratings_cached = {} 106 | if self._from_fbs: 107 | for i in range(self._from_fbs.RatingsLength()): 108 | rat = self._from_fbs.Ratings(i) 109 | self._ratings_cached[rat.Name()] = rat.Rating() 110 | return self._ratings_cached 111 | 112 | @ratings.setter 113 | def ratings(self, value): 114 | self._ratings = value 115 | 116 | @property 117 | def friends(self): 118 | if self._friends is not None: 119 | return self._friends 120 | if self._friends_cached is None: 121 | self._friends_cached = [] 122 | if self._from_fbs: 123 | for i in range(self._from_fbs.FriendsLength()): 124 | friend_oid = self._from_fbs.Friends(i) 125 | self._friends_cached.append(friend_oid) 126 | return self._friends_cached 127 | 128 | @friends.setter 129 | def friends(self, value): 130 | self._friends = value 131 | 132 | @property 133 | def referred_by(self): 134 | return self._referred_by or self._from_fbs.ReferredBy() 135 | 136 | @referred_by.setter 137 | def referred_by(self, value): 138 | self._referred_by = value 139 | 140 | def build(self, builder): 141 | if self._name is not None: 142 | name = builder.CreateString(self._name) 143 | else: 144 | name = builder.CreateString(self._from_fbs.Name()) 145 | 146 | if self._authid is not None: 147 | authid = builder.CreateString(self._authid) 148 | else: 149 | authid = builder.CreateString(self._from_fbs.Authid()) 150 | 151 | if self._email is not None: 152 | email = builder.CreateString(self._email) 153 | else: 154 | email = builder.CreateString(self._from_fbs.Email()) 155 | 156 | _user.UserStart(builder) 157 | _user.UserAddName(builder, name) 158 | _user.UserAddAuthid(builder, authid) 159 | _user.UserAddEmail(builder, email) 160 | 161 | if self._birthday is not None: 162 | _user.UserAddBirthday( 163 | builder, _date.CreateDate(builder, self._birthday.year, self._birthday.month, self._birthday.day)) 164 | else: 165 | bd = self._from_fbs.Birthday() 166 | _user.UserAddBirthday(builder, _date.CreateDate(builder, bd.Year(), bd.Month(), bd.Day())) 167 | 168 | # FIXME: tags 169 | # FIXME: ratings 170 | # FIXME: friends 171 | 172 | if self._is_friendly is not None: 173 | _user.UserAddIsFriendly(builder, self._is_friendly) 174 | else: 175 | _user.UserAddIsFriendly(builder, self._from_fbs.IsFriendly()) 176 | 177 | if self._referred_by is not None: 178 | _user.UserAddReferredBy(builder, self._referred_by) 179 | else: 180 | _user.UserAddReferredBy(builder, self._from_fbs.ReferredBy()) 181 | 182 | return _user.UserEnd(builder) 183 | 184 | @staticmethod 185 | def cast(buf): 186 | return User(_user.User.GetRootAsUser(buf, 0)) 187 | 188 | @staticmethod 189 | def create_test_user(oid=None): 190 | user = User() 191 | if oid is not None: 192 | user.oid = oid 193 | else: 194 | user.oid = random.randint(0, 9007199254740992) 195 | user.name = 'Test {}'.format(user.oid) 196 | user.authid = 'test-{}'.format(user.oid) 197 | user.uuid = uuid.uuid4() 198 | user.email = '{}@example.com'.format(user.authid) 199 | user.birthday = datetime.date(1950, 12, 24) 200 | user.is_friendly = True 201 | user.tags = ['geek', 'sudoko', 'yellow'] 202 | for j in range(10): 203 | user.ratings['test-rating-{}'.format(j)] = random.random() 204 | user.friends = [random.randint(0, 9007199254740992) for _ in range(10)] 205 | user.referred_by = random.randint(0, 9007199254740992) 206 | return user 207 | -------------------------------------------------------------------------------- /tests/zdb/test_zdb_df.py: -------------------------------------------------------------------------------- 1 | 2 | from twisted.internet.task import react 3 | from twisted.internet.defer import inlineCallbacks 4 | 5 | import txaio 6 | from autobahn.twisted import util 7 | 8 | import numpy as np 9 | import pandas as pd 10 | import pyarrow as pa 11 | 12 | import zlmdb 13 | from zlmdb._pmap import _StringKeysMixin, PersistentMap 14 | 15 | 16 | try: 17 | from tempfile import TemporaryDirectory 18 | except ImportError: 19 | from backports.tempfile import TemporaryDirectory 20 | 21 | 22 | class _DataFrameValuesMixin(object): 23 | 24 | def __init__(self, marshal=None, unmarshal=None): 25 | self._marshal = marshal or self._zlmdb_marshal 26 | self._unmarshal = unmarshal or self._zlmdb_unmarshal 27 | 28 | def _serialize_value(self, value): 29 | return pa.serialize(value).to_buffer() 30 | 31 | def _deserialize_value(self, data): 32 | return pa.deserialize(data) 33 | 34 | 35 | class MapStringDataFrame(_StringKeysMixin, _DataFrameValuesMixin, PersistentMap): 36 | 37 | def __init__(self, slot=None, compress=None): 38 | PersistentMap.__init__(self, slot=slot, compress=compress) 39 | 40 | 41 | class MySchema(zlmdb.Schema): 42 | 43 | samples: MapStringDataFrame 44 | 45 | def __init__(self): 46 | self.samples = MapStringDataFrame(1) 47 | 48 | 49 | @inlineCallbacks 50 | def main(reactor): 51 | 52 | with TemporaryDirectory() as dbpath: 53 | print('Using temporary directory {} for database'.format(dbpath)) 54 | 55 | schema = MySchema() 56 | db = zlmdb.Database(dbpath) 57 | 58 | # WRITE some native pandas data frames to zlmdb 59 | with db.begin(write=True) as txn: 60 | for i in range(10): 61 | if i % 2: 62 | key = 'key{}'.format(i) 63 | value = pd.DataFrame(np.random.randn(8, 4), 64 | columns=['A','B','C','D']) 65 | schema.samples[txn, key] = value 66 | 67 | # READ back native pandas data frames from zlmdb 68 | with db.begin() as txn: 69 | for i in range(10): 70 | key = 'key{}'.format(i) 71 | value = schema.samples[txn, key] 72 | print('key={} : value=\n{}'.format(key, value)) 73 | 74 | yield util.sleep(1) 75 | 76 | 77 | if __name__ == '__main__': 78 | txaio.start_logging(level='info') 79 | react(main) 80 | -------------------------------------------------------------------------------- /tests/zdb/test_zdb_dyn.py: -------------------------------------------------------------------------------- 1 | from twisted.internet.task import react 2 | from twisted.internet.defer import inlineCallbacks 3 | 4 | from autobahn.twisted import util 5 | 6 | import txaio 7 | 8 | import yaml 9 | 10 | from pprint import pprint, pformat 11 | 12 | import zlmdb 13 | from zlmdb._pmap import MapStringJson, MapStringCbor, MapUuidJson, MapUuidCbor 14 | 15 | import random 16 | import uuid 17 | import datetime 18 | from typing import Optional, List, Dict 19 | 20 | 21 | class Tag(object): 22 | GEEK = 1 23 | VIP = 2 24 | 25 | 26 | class User(object): 27 | oid: int 28 | name: str 29 | authid: str 30 | uuid: uuid.UUID 31 | email: str 32 | birthday: datetime.date 33 | is_friendly: bool 34 | tags: Optional[List[str]] 35 | ratings: Dict[str, float] = {} 36 | friends: List[int] = [] 37 | referred_by: int = None 38 | 39 | def __eq__(self, other): 40 | if not isinstance(other, self.__class__): 41 | return False 42 | if other.oid != self.oid: 43 | return False 44 | if other.name != self.name: 45 | return False 46 | if other.authid != self.authid: 47 | return False 48 | if other.uuid != self.uuid: 49 | return False 50 | if other.email != self.email: 51 | return False 52 | if other.birthday != self.birthday: 53 | return False 54 | if other.is_friendly != self.is_friendly: 55 | return False 56 | if (self.tags and not other.tags) or (not self.tags and other.tags): 57 | return False 58 | return True 59 | 60 | def __ne__(self, other): 61 | return not self.__eq__(other) 62 | 63 | def __str__(self): 64 | return '\n{}\n'.format(pformat(self.marshal())) 65 | 66 | def marshal(self): 67 | obj = { 68 | 'oid': self.oid, 69 | 'name': self.name, 70 | 'authid': self.authid, 71 | 'uuid': self.uuid.hex if self.uuid else None, 72 | 'email': self.email, 73 | 'birthday': { 74 | 'year': self.birthday.year if self.birthday else None, 75 | 'month': self.birthday.month if self.birthday else None, 76 | 'day': self.birthday.day if self.birthday else None, 77 | }, 78 | 'is_friendly': self.is_friendly, 79 | 'tags': self.tags, 80 | 'ratings': self.ratings, 81 | 'friends': self.friends, 82 | 'referred_by': self.referred_by, 83 | } 84 | return obj 85 | 86 | @staticmethod 87 | def parse(obj): 88 | user = User() 89 | user.oid = obj.get('oid', None) 90 | user.name = obj.get('name', None) 91 | user.authid = obj.get('authid', None) 92 | if 'uuid' in obj: 93 | user.uuid = uuid.UUID(hex=obj['uuid']) 94 | user.email = obj.get('email', None) 95 | if 'birthday' in obj: 96 | b = obj['birthday'] 97 | user.birthday = datetime.date(b.get('year', None), b.get('month', None), b.get('day', None)) 98 | user.is_friendly = obj.get('is_friendly', None) 99 | user.tags = obj.get('tags', None) 100 | user.ratings = obj.get('ratings', {}) 101 | user.friends = obj.get('friends', []) 102 | user.referred_by = obj.get('referred_by', None) 103 | return user 104 | 105 | @staticmethod 106 | def create_test_user(oid=None): 107 | user = User() 108 | if oid is not None: 109 | user.oid = oid 110 | else: 111 | user.oid = random.randint(0, 9007199254740992) 112 | user.name = 'Test {}'.format(user.oid) 113 | user.authid = 'test-{}'.format(user.oid) 114 | user.uuid = uuid.uuid4() 115 | user.email = '{}@example.com'.format(user.authid) 116 | user.birthday = datetime.date(1950, 12, 24) 117 | user.is_friendly = True 118 | user.tags = ['geek', 'sudoko', 'yellow'] 119 | for j in range(10): 120 | user.ratings['test-rating-{}'.format(j)] = random.random() 121 | user.friends = [random.randint(0, 9007199254740992) for _ in range(10)] 122 | user.referred_by = random.randint(0, 9007199254740992) 123 | return user 124 | 125 | 126 | 127 | KV_TYPE_TO_CLASS = { 128 | 'string-json': (MapStringJson, lambda x: x, lambda x: x), 129 | 'string-json-user': (MapStringJson, User.marshal, User.parse), 130 | 'string-cbor-user': (MapStringCbor, User.marshal, User.parse), 131 | 'uuid-json-user': (MapUuidJson, User.marshal, User.parse), 132 | 'uuid-cbor-user': (MapUuidCbor, User.marshal, User.parse), 133 | } 134 | DBPATH = '/tmp/zlmdb1' 135 | DBSCHEMA = 'tests/zdb/zdb.yml' 136 | 137 | @inlineCallbacks 138 | def main(reactor): 139 | 140 | schema = zlmdb._database.Schema.parse(DBSCHEMA, KV_TYPE_TO_CLASS) 141 | 142 | print('Using database directory {} and schema {}:\n{}'.format(DBPATH, DBSCHEMA, schema)) 143 | 144 | with zlmdb.Database(DBPATH, schema) as db: 145 | with db.begin(write=True) as txn: 146 | users = schema['users'] 147 | users2 = schema['users2'] 148 | 149 | print('users', users) 150 | print('users2', users2) 151 | 152 | key = 'user1' 153 | for table in [users, users2]: 154 | user = table[txn, key] 155 | if user: 156 | print('user object already exists in {} for key {}: {}'.format(table, key, user)) 157 | else: 158 | print('user does not exist in {}, storing new object ..'.format(table)) 159 | user = User.create_test_user() 160 | table[txn, key] = user 161 | print('user object created for key {}: {}'.format(key, user)) 162 | 163 | yield util.sleep(1) 164 | 165 | 166 | if __name__ == '__main__': 167 | txaio.start_logging(level='info') 168 | react(main) 169 | -------------------------------------------------------------------------------- /tests/zdb/test_zdb_etcd.py: -------------------------------------------------------------------------------- 1 | from twisted.internet.task import react 2 | from twisted.internet.defer import inlineCallbacks 3 | 4 | from autobahn.twisted import util 5 | 6 | import txaio 7 | from txaioetcd import Client, KeySet 8 | 9 | import yaml 10 | 11 | from pprint import pprint, pformat 12 | import random 13 | 14 | import zlmdb 15 | from zlmdb._pmap import MapStringString 16 | 17 | 18 | try: 19 | from tempfile import TemporaryDirectory 20 | except ImportError: 21 | from backports.tempfile import TemporaryDirectory 22 | 23 | 24 | 25 | class MySchema(zlmdb.Schema): 26 | 27 | samples: MapStringString 28 | 29 | def __init__(self): 30 | self.samples = MapStringString(1) 31 | 32 | 33 | @inlineCallbacks 34 | def main(reactor): 35 | if True: 36 | with TemporaryDirectory() as dbpath: 37 | print('Using temporary directory {} for database'.format(dbpath)) 38 | 39 | schema = MySchema() 40 | 41 | with zlmdb.Database(dbpath) as db: 42 | # write records into zlmdb 43 | with db.begin(write=True) as txn: 44 | for i in range(10): 45 | key = 'key{}'.format(i) 46 | value = 'value{}'.format(random.randint(0, 1000)) 47 | schema.samples[txn, key] = value 48 | 49 | # read records from zlmdb 50 | with db.begin() as txn: 51 | for i in range(10): 52 | key = 'key{}'.format(i) 53 | value = schema.samples[txn, key] 54 | print('key={} : value={}'.format(key, value)) 55 | 56 | if True: 57 | # etcd database 58 | etcd = Client(reactor) 59 | status = yield etcd.status() 60 | print(status) 61 | 62 | # zlmdb database 63 | schema = MySchema() 64 | dbpath = '/tmp/.test-zlmdb' 65 | 66 | with zlmdb.Database(dbpath) as db: 67 | print('zlmdb open on {}'.format(dbpath)) 68 | 69 | # check current record count 70 | with db.begin() as txn: 71 | cnt = schema.samples.count(txn) 72 | print('currently {} rows in table'.format(cnt)) 73 | 74 | # watch changes in etcd and write to local zlmdb 75 | def on_change(kv): 76 | key = kv.key.decode() 77 | value = kv.value.decode() 78 | with db.begin(write=True) as txn: 79 | schema.samples[txn, key] = value 80 | print('on_change received from etcd and written to zlmdb: key={} value={}'.format(key, value)) 81 | 82 | # start watching for etcd changes .. 83 | ks = [KeySet('k'.encode(), prefix=True)] 84 | d = etcd.watch(ks, on_change) 85 | 86 | print('watching for 1s ..') 87 | yield txaio.sleep(1) 88 | 89 | # loop every 1s and write a key-value in etcd directly 90 | for i in range(5): 91 | print('watching for 1s ..') 92 | yield txaio.sleep(1) 93 | 94 | key = 'key{}'.format(i).encode() 95 | value = 'value{}'.format(random.randint(0, 1000)).encode() 96 | 97 | etcd.set(key, value) 98 | 99 | # cancel our watch 100 | d.cancel() 101 | 102 | yield util.sleep(1) 103 | 104 | # check current record count 105 | with db.begin() as txn: 106 | cnt = schema.samples.count(txn) 107 | print('currently {} rows in table'.format(cnt)) 108 | 109 | yield util.sleep(1) 110 | 111 | 112 | if __name__ == '__main__': 113 | txaio.start_logging(level='info') 114 | react(main) 115 | -------------------------------------------------------------------------------- /tests/zdb/test_zdb_fbs.py: -------------------------------------------------------------------------------- 1 | from twisted.internet.task import react 2 | from twisted.internet.defer import inlineCallbacks 3 | import txaio 4 | txaio.use_twisted() 5 | from autobahn.twisted import util 6 | 7 | import os 8 | import sys 9 | import random 10 | 11 | try: 12 | from tempfile import TemporaryDirectory 13 | except ImportError: 14 | from backports.tempfile import TemporaryDirectory 15 | 16 | import zlmdb 17 | 18 | sys.path.append(os.path.dirname(os.path.abspath(__file__))) 19 | 20 | from _schema_fbs import User as UserFbs # noqa 21 | 22 | 23 | class UsersSchema(zlmdb.Schema): 24 | def __init__(self): 25 | self.tab_oid_fbs = zlmdb.MapOidFlatBuffers(1, build=UserFbs.build, cast=UserFbs.cast) 26 | 27 | 28 | @inlineCallbacks 29 | def main2(reactor): 30 | dbpath = '/tmp/zlmdb1' 31 | 32 | print('Using database directory {}'.format(dbpath)) 33 | 34 | schema = UsersSchema() 35 | 36 | with zlmdb.Database(dbpath) as db: 37 | 38 | N = 1000 39 | with db.begin() as txn: 40 | cnt_begin = schema.tab_oid_fbs.count(txn) 41 | 42 | stats = zlmdb.TransactionStats() 43 | 44 | with db.begin(write=True, stats=stats) as txn: 45 | for i in range(N): 46 | user = UserFbs.create_test_user() 47 | schema.tab_oid_fbs[txn, user.oid] = user 48 | 49 | assert stats.puts == N 50 | assert stats.dels == 0 51 | stats.reset() 52 | 53 | with db.begin() as txn: 54 | cnt_end = schema.tab_oid_fbs.count(txn) 55 | 56 | cnt = cnt_end - cnt_begin 57 | assert cnt == N 58 | 59 | print('{} records written, {} records total'.format(cnt, cnt_end)) 60 | 61 | yield util.sleep(1) 62 | 63 | 64 | if __name__ == '__main__': 65 | txaio.start_logging(level='info') 66 | react(main2) 67 | -------------------------------------------------------------------------------- /tests/zdb/zdb.yml: -------------------------------------------------------------------------------- 1 | slots: 2 | - index: 100 3 | name: users 4 | key: string 5 | value: json 6 | schema: user 7 | description: Arbitrary user data (serialized in JSON format). 8 | 9 | - index: 101 10 | name: mrealms 11 | key: uuid 12 | value: cbor 13 | schema: user 14 | description: Management realms. 15 | 16 | - index: 102 17 | name: users2 18 | key: string 19 | schema: user 20 | value: cbor 21 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skip_missing_interpreters = true 3 | envlist = 4 | py37 5 | py38 6 | py39 7 | py310 8 | py311 9 | pypy37 10 | pypy38 11 | pypy39 12 | flake8 13 | mypy 14 | yapf 15 | sphinx 16 | 17 | 18 | # MAP: GitHub Actions Python Name => Tox Env Name (for Python) 19 | # 20 | # when called without a specific environment ("-e"), detect the 21 | # python version / get from GH action, and map to tox env 22 | # 23 | # https://github.com/ymyzk/tox-gh-actions 24 | # 25 | [gh-actions] 26 | python = 27 | 3.7: py37 28 | 3.8: py38 29 | 3.9: py39 30 | 3.10: py310 31 | 3.11: py311 32 | pypy-3.7: pypy37 33 | pypy-3.8: pypy38 34 | pypy-3.9: pypy39 35 | 36 | 37 | [testenv] 38 | allowlist_externals = * 39 | setenv = 40 | PYTHONPATH = {toxinidir} 41 | # LMDB_FORCE_CFFI = "1" 42 | # SODIUM_INSTALL = "bundled" 43 | deps = 44 | -r{toxinidir}/requirements-dev.txt 45 | commands = 46 | {py37,py38,py39,py310,py311,pypy37,pypy38,pypy39}: pytest -v -s --basetemp={envtmpdir} zlmdb 47 | 48 | 49 | [testenv:flake8] 50 | skip_install = True 51 | deps = 52 | flake8 53 | commands = 54 | python -V 55 | flake8 --max-line-length=119 --exclude=zlmdb/tests/user_typed.py --exclude=zlmdb/flatbuffers,zlmdb/tests/MNodeLog.py zlmdb 56 | 57 | 58 | [testenv:yapf] 59 | description = 60 | Run yapf style checks. 61 | skip_install = True 62 | deps = 63 | # https://github.com/google/yapf/issues/712 64 | yapf==0.29.0 65 | commands = 66 | python -V 67 | yapf --version 68 | yapf -rd --style=yapf.ini --exclude="zlmdb/flatbuffers/*" --exclude="zlmdb/tests/MNodeLog.py" zlmdb 69 | 70 | 71 | [testenv:mypy] 72 | description = 73 | Run mypy type checks. 74 | skip_install = True 75 | deps = 76 | mypy 77 | commands= 78 | python -V 79 | mypy --version 80 | mypy --install-types --non-interactive --ignore-missing-imports --config-file {toxinidir}/mypy.ini zlmdb 81 | 82 | 83 | [testenv:pylint] 84 | description = 85 | Run pylint checks. 86 | skip_install = False 87 | deps = 88 | pylint 89 | commands= 90 | python -V 91 | pylint --version 92 | pylint --errors-only --ignore-patterns="zlmdb/flatbuffers/*","zlmdb/test/*" zlmdb 93 | 94 | 95 | [testenv:sphinx] 96 | description = 97 | Generate docs using Sphinx. 98 | skip_install = False 99 | deps = 100 | sphinx>=1.7.1 101 | sphinxcontrib-images 102 | sphinxcontrib-spelling 103 | sphinx-autoapi 104 | sphinx_rtd_theme 105 | commands = 106 | python -V 107 | sphinx-build --version 108 | 109 | # first test with all warnings fatal 110 | sphinx-build -nWT -b dummy ./docs ./docs/_build 111 | 112 | # generate HTML output 113 | sphinx-build -b html ./docs ./docs/_build 114 | -------------------------------------------------------------------------------- /yapf.ini: -------------------------------------------------------------------------------- 1 | [style] 2 | based_on_style = pep8 3 | column_limit = 119 4 | 5 | #ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=True 6 | #ALLOW_MULTILINE_LAMBDAS=False 7 | #ALLOW_MULTILINE_DICTIONARY_KEYS=False 8 | #ALLOW_SPLIT_BEFORE_DEFAULT_OR_NAMED_ASSIGNS=True 9 | #ALLOW_SPLIT_BEFORE_DICT_VALUE=True 10 | #ARITHMETIC_PRECEDENCE_INDICATION=False 11 | #BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=False 12 | #BLANK_LINE_BEFORE_CLASS_DOCSTRING=False 13 | #BLANK_LINE_BEFORE_MODULE_DOCSTRING=False 14 | #BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION=2 15 | #COALESCE_BRACKETS=False 16 | ##COLUMN_LIMIT=79 17 | #CONTINUATION_ALIGN_STYLE='SPACE' 18 | #CONTINUATION_INDENT_WIDTH=4 19 | #DEDENT_CLOSING_BRACKETS=False 20 | #DISABLE_ENDING_COMMA_HEURISTIC=False 21 | #EACH_DICT_ENTRY_ON_SEPARATE_LINE=True 22 | #I18N_COMMENT='' 23 | #I18N_FUNCTION_CALL='' 24 | #INDENT_DICTIONARY_VALUE=False 25 | #INDENT_WIDTH=4 26 | #INDENT_BLANK_LINES=False 27 | #JOIN_MULTIPLE_LINES=True 28 | #NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS=set() 29 | #SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET=True 30 | #SPACES_AROUND_POWER_OPERATOR=False 31 | #SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN=False 32 | #SPACES_BEFORE_COMMENT=2 33 | #SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED=False 34 | #SPLIT_ALL_COMMA_SEPARATED_VALUES=False 35 | #SPLIT_BEFORE_ARITHMETIC_OPERATOR=False 36 | #SPLIT_BEFORE_BITWISE_OPERATOR=True 37 | #SPLIT_BEFORE_CLOSING_BRACKET=True 38 | #SPLIT_BEFORE_DICT_SET_GENERATOR=True 39 | #SPLIT_BEFORE_DOT=False 40 | #SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN=False 41 | #SPLIT_BEFORE_FIRST_ARGUMENT=False 42 | #SPLIT_BEFORE_LOGICAL_OPERATOR=True 43 | #SPLIT_BEFORE_NAMED_ASSIGNS=True 44 | #SPLIT_COMPLEX_COMPREHENSION=False 45 | #SPLIT_PENALTY_AFTER_OPENING_BRACKET=300 46 | #SPLIT_PENALTY_AFTER_UNARY_OPERATOR=10000 47 | #SPLIT_PENALTY_ARITHMETIC_OPERATOR=300 48 | #SPLIT_PENALTY_BEFORE_IF_EXPR=0 49 | #SPLIT_PENALTY_BITWISE_OPERATOR=300 50 | #SPLIT_PENALTY_COMPREHENSION=80 51 | #SPLIT_PENALTY_EXCESS_CHARACTER=7000 52 | #SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT=30 53 | #SPLIT_PENALTY_IMPORT_NAMES=0 54 | #SPLIT_PENALTY_LOGICAL_OPERATOR=300 55 | #USE_TABS=False 56 | -------------------------------------------------------------------------------- /zlmdb/_errors.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) typedef int GmbH 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in 15 | # all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | # THE SOFTWARE. 24 | # 25 | ############################################################################### 26 | 27 | 28 | class NullValueConstraint(RuntimeError): 29 | """ 30 | Null value in indexed column violates not-null constraint. 31 | """ 32 | -------------------------------------------------------------------------------- /zlmdb/_meta.py: -------------------------------------------------------------------------------- 1 | ############################################################################# 2 | # 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) typedef int GmbH 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in 15 | # all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | # THE SOFTWARE. 24 | # 25 | ############################################################################### 26 | 27 | MAGIC = b'ZLMDB-S1' 28 | -------------------------------------------------------------------------------- /zlmdb/_schema.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) typedef int GmbH 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in 15 | # all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | # THE SOFTWARE. 24 | # 25 | ############################################################################### 26 | 27 | from zlmdb._pmap import PersistentMap 28 | 29 | 30 | class Slot(object): 31 | """ 32 | LMDB database slot. A slot is defined just by the convention of using 33 | the first 2 bytes of keys in a LMDB database as the "slot index". 34 | 35 | The 2 bytes are interpreted as an uint16 in big endian byte order. 36 | """ 37 | def __init__(self, slot, name, pmap): 38 | """ 39 | 40 | :param slot: 41 | :param name: 42 | :param pmap: 43 | """ 44 | self.slot = slot 45 | self.name = name 46 | self.pmap = pmap 47 | 48 | 49 | class Schema(object): 50 | """ 51 | ZLMDB database schema definition. 52 | """ 53 | 54 | SLOT_DATA_EMPTY = 0 55 | """ 56 | Database slot is empty (unused, not necessarily zero'ed, but uninitialized). 57 | """ 58 | 59 | SLOT_DATA_METADATA = 1 60 | """ 61 | FIXME. 62 | """ 63 | 64 | SLOT_DATA_TYPE = 2 65 | """ 66 | FIXME. 67 | """ 68 | 69 | SLOT_DATA_SEQUENCE = 3 70 | """ 71 | FIXME. 72 | """ 73 | 74 | SLOT_DATA_TABLE = 4 75 | """ 76 | Database slot contains a persistent map, for example a map of type OID to Pickle. 77 | """ 78 | 79 | SLOT_DATA_INDEX = 5 80 | """ 81 | FIXME. 82 | """ 83 | 84 | SLOT_DATA_REPLICATION = 6 85 | """ 86 | FIXME. 87 | """ 88 | 89 | SLOT_DATA_MATERIALIZATION = 7 90 | """ 91 | FIXME. 92 | """ 93 | def __init__(self): 94 | self._index_to_slot = {} 95 | self._name_to_slot = {} 96 | 97 | def slot(self, slot_index, marshal=None, unmarshal=None, build=None, cast=None, compress=False): 98 | """ 99 | Decorator for use on classes derived from zlmdb.PersistentMap. The decorator define slots 100 | in a LMDB database schema based on persistent maps, and slot configuration. 101 | 102 | :param slot_index: 103 | :param marshal: 104 | :param unmarshal: 105 | :param build: 106 | :param cast: 107 | :param compress: 108 | :return: 109 | """ 110 | def decorate(o): 111 | assert isinstance(o, PersistentMap) 112 | name = o.__class__.__name__ 113 | assert slot_index not in self._index_to_slot 114 | assert name not in self._name_to_slot 115 | o._zlmdb_slot = slot_index 116 | o._zlmdb_marshal = marshal 117 | o._zlmdb_unmarshal = unmarshal 118 | o._zlmdb_build = build 119 | o._zlmdb_cast = cast 120 | o._zlmdb_compress = compress 121 | _slot = Slot(slot_index, name, o) 122 | self._index_to_slot[slot_index] = _slot 123 | self._name_to_slot[name] = _slot 124 | return o 125 | 126 | return decorate 127 | -------------------------------------------------------------------------------- /zlmdb/_transaction.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) typedef int GmbH 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in 15 | # all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | # THE SOFTWARE. 24 | # 25 | ############################################################################### 26 | """Transactions""" 27 | 28 | import struct 29 | import lmdb 30 | from typing import Optional 31 | 32 | from txaio import time_ns as walltime 33 | 34 | 35 | class TransactionStats(object): 36 | """ 37 | Value class for holding transaction statistics. 38 | """ 39 | def __init__(self): 40 | self.puts = 0 41 | self.dels = 0 42 | self._started = walltime() 43 | 44 | @property 45 | def started(self): 46 | """ 47 | 48 | :return: start time in ns since epoch 49 | """ 50 | return self._started 51 | 52 | @property 53 | def duration(self): 54 | """ 55 | 56 | :return: duration in ns 57 | """ 58 | if self._started: 59 | return walltime() - self._started 60 | else: 61 | return 0 62 | 63 | def reset(self): 64 | """ 65 | 66 | :return: 67 | """ 68 | self.puts = 0 69 | self.dels = 0 70 | self._started = walltime() 71 | 72 | 73 | class Transaction(object): 74 | """ 75 | Transactions in zLMDB are always run under an instance of this class. 76 | """ 77 | 78 | PUT = 1 79 | DEL = 2 80 | 81 | def __init__(self, db, write=False, buffers=False, stats=None): 82 | """ 83 | 84 | :param db: 85 | :type db: zlmdb.Database 86 | 87 | :param write: 88 | :type write: bool 89 | 90 | :param stats: 91 | :type stats: TransactionStats 92 | """ 93 | self._db = db 94 | self._write = write 95 | self._buffers = buffers 96 | self._stats = stats 97 | self._txn: Optional[lmdb.Transaction] = None 98 | self._log = None 99 | 100 | def __enter__(self): 101 | assert (self._txn is None) 102 | 103 | self._txn = lmdb.Transaction(self._db._env, write=self._write, buffers=self._buffers) 104 | return self 105 | 106 | def __exit__(self, exc_type, exc_value, traceback): 107 | assert (self._txn is not None) 108 | 109 | # https://docs.python.org/3/reference/datamodel.html#object.__exit__ 110 | # If the context was exited without an exception, all three arguments will be None. 111 | if exc_type is None: 112 | if self._log: 113 | cnt = 0 114 | for op, key in self._log: 115 | _key = struct.pack('>H', 0) 116 | _data = struct.pack('>H', op) + key 117 | self._txn.put(_key, _data) 118 | cnt += 1 119 | self._txn.commit() 120 | else: 121 | self._txn.abort() 122 | 123 | self._txn = None 124 | 125 | def id(self): 126 | """ 127 | 128 | :return: 129 | """ 130 | assert (self._txn is not None) 131 | 132 | return self._txn.id() 133 | 134 | def get(self, key): 135 | """ 136 | 137 | :param key: 138 | :return: 139 | """ 140 | assert (self._txn is not None) 141 | 142 | return self._txn.get(key) 143 | 144 | def put(self, key, data, overwrite=True): 145 | """ 146 | 147 | :param key: 148 | :param data: 149 | :param overwrite: 150 | :return: 151 | """ 152 | assert (self._txn is not None) 153 | 154 | # store the record, returning True if it was written, or False to indicate the key 155 | # was already present and overwrite=False. 156 | was_written = self._txn.put(key, data, overwrite=overwrite) 157 | if was_written: 158 | if self._stats: 159 | self._stats.puts += 1 160 | if self._log: 161 | self._log.append((Transaction.PUT, key)) 162 | return was_written 163 | 164 | def delete(self, key): 165 | """ 166 | 167 | :param key: 168 | :return: 169 | """ 170 | assert (self._txn is not None) 171 | 172 | was_deleted = self._txn.delete(key) 173 | if was_deleted: 174 | if self._stats: 175 | self._stats.dels += 1 176 | if self._log: 177 | self._log.append((Transaction.DEL, key)) 178 | return was_deleted 179 | -------------------------------------------------------------------------------- /zlmdb/_version.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) typedef int GmbH 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in 15 | # all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | # THE SOFTWARE. 24 | # 25 | ############################################################################### 26 | 27 | __version__ = '23.1.1' 28 | -------------------------------------------------------------------------------- /zlmdb/cli.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) typedef int GmbH 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in 15 | # all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | # THE SOFTWARE. 24 | # 25 | ############################################################################### 26 | """Console script for zlmdb.""" 27 | 28 | import sys 29 | import click 30 | 31 | 32 | @click.command() 33 | def main(args=None): 34 | """Console script for zlmdb.""" 35 | click.echo("Replace this message by putting your code into " "zlmdb.cli.main") 36 | click.echo("See click documentation at http://click.pocoo.org/") 37 | return 0 38 | 39 | 40 | if __name__ == "__main__": 41 | sys.exit(main()) # pragma: no cover 42 | -------------------------------------------------------------------------------- /zlmdb/flatbuffers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) FlatBuffers Contributors, Apache License 2.0 2 | 3 | # Copied from (master branch, 05/29/2022): 4 | # * https://github.com/google/flatbuffers/blob/master/reflection/reflection.fbs 5 | # * https://github.com/google/flatbuffers/tree/master/python/flatbuffers/reflection 6 | -------------------------------------------------------------------------------- /zlmdb/flatbuffers/reflection.fbs: -------------------------------------------------------------------------------- 1 | // This schema defines objects that represent a parsed schema, like 2 | // the binary version of a .fbs file. 3 | // This could be used to operate on unknown FlatBuffers at runtime. 4 | // It can even ... represent itself (!) 5 | 6 | namespace reflection; 7 | 8 | // These must correspond to the enum in idl.h. 9 | enum BaseType : byte { 10 | None, 11 | UType, 12 | Bool, 13 | Byte, 14 | UByte, 15 | Short, 16 | UShort, 17 | Int, 18 | UInt, 19 | Long, 20 | ULong, 21 | Float, 22 | Double, 23 | String, 24 | Vector, 25 | Obj, // Used for tables & structs. 26 | Union, 27 | Array, 28 | 29 | // Add any new type above this value. 30 | MaxBaseType 31 | } 32 | 33 | table Type { 34 | base_type:BaseType; 35 | element:BaseType = None; // Only if base_type == Vector 36 | // or base_type == Array. 37 | index:int = -1; // If base_type == Object, index into "objects" below. 38 | // If base_type == Union, UnionType, or integral derived 39 | // from an enum, index into "enums" below. 40 | // If base_type == Vector && element == Union or UnionType. 41 | fixed_length:uint16 = 0; // Only if base_type == Array. 42 | /// The size (octets) of the `base_type` field. 43 | base_size:uint = 4; // 4 Is a common size due to offsets being that size. 44 | /// The size (octets) of the `element` field, if present. 45 | element_size:uint = 0; 46 | } 47 | 48 | table KeyValue { 49 | key:string (required, key); 50 | value:string; 51 | } 52 | 53 | table EnumVal { 54 | name:string (required); 55 | value:long (key); 56 | object:Object (deprecated); 57 | union_type:Type; 58 | documentation:[string]; 59 | } 60 | 61 | table Enum { 62 | name:string (required, key); 63 | values:[EnumVal] (required); // In order of their values. 64 | is_union:bool = false; 65 | underlying_type:Type (required); 66 | attributes:[KeyValue]; 67 | documentation:[string]; 68 | /// File that this Enum is declared in. 69 | declaration_file: string; 70 | } 71 | 72 | table Field { 73 | name:string (required, key); 74 | type:Type (required); 75 | id:ushort; 76 | offset:ushort; // Offset into the vtable for tables, or into the struct. 77 | default_integer:long = 0; 78 | default_real:double = 0.0; 79 | deprecated:bool = false; 80 | required:bool = false; 81 | key:bool = false; 82 | attributes:[KeyValue]; 83 | documentation:[string]; 84 | optional:bool = false; 85 | /// Number of padding octets to always add after this field. Structs only. 86 | padding:uint16 = 0; 87 | } 88 | 89 | table Object { // Used for both tables and structs. 90 | name:string (required, key); 91 | fields:[Field] (required); // Sorted. 92 | is_struct:bool = false; 93 | minalign:int; 94 | bytesize:int; // For structs. 95 | attributes:[KeyValue]; 96 | documentation:[string]; 97 | /// File that this Object is declared in. 98 | declaration_file: string; 99 | } 100 | 101 | table RPCCall { 102 | name:string (required, key); 103 | request:Object (required); // must be a table (not a struct) 104 | response:Object (required); // must be a table (not a struct) 105 | attributes:[KeyValue]; 106 | documentation:[string]; 107 | } 108 | 109 | table Service { 110 | name:string (required, key); 111 | calls:[RPCCall]; 112 | attributes:[KeyValue]; 113 | documentation:[string]; 114 | /// File that this Service is declared in. 115 | declaration_file: string; 116 | } 117 | 118 | /// New schema language features that are not supported by old code generators. 119 | enum AdvancedFeatures : ulong (bit_flags) { 120 | AdvancedArrayFeatures, 121 | AdvancedUnionFeatures, 122 | OptionalScalars, 123 | DefaultVectorsAndStrings, 124 | } 125 | 126 | /// File specific information. 127 | /// Symbols declared within a file may be recovered by iterating over all 128 | /// symbols and examining the `declaration_file` field. 129 | table SchemaFile { 130 | /// Filename, relative to project root. 131 | filename:string (required, key); 132 | /// Names of included files, relative to project root. 133 | included_filenames:[string]; 134 | } 135 | 136 | table Schema { 137 | objects:[Object] (required); // Sorted. 138 | enums:[Enum] (required); // Sorted. 139 | file_ident:string; 140 | file_ext:string; 141 | root_table:Object; 142 | services:[Service]; // Sorted. 143 | advanced_features:AdvancedFeatures; 144 | /// All the files used in this compilation. Files are relative to where 145 | /// flatc was invoked. 146 | fbs_files:[SchemaFile]; // Sorted. 147 | } 148 | 149 | root_type Schema; 150 | 151 | file_identifier "BFBS"; 152 | file_extension "bfbs"; 153 | -------------------------------------------------------------------------------- /zlmdb/flatbuffers/reflection/AdvancedFeatures.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: reflection 4 | 5 | # New schema language features that are not supported by old code generators. 6 | class AdvancedFeatures(object): 7 | AdvancedArrayFeatures = 1 8 | AdvancedUnionFeatures = 2 9 | OptionalScalars = 4 10 | DefaultVectorsAndStrings = 8 11 | -------------------------------------------------------------------------------- /zlmdb/flatbuffers/reflection/BaseType.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: reflection 4 | 5 | class BaseType(object): 6 | None_ = 0 7 | UType = 1 8 | Bool = 2 9 | Byte = 3 10 | UByte = 4 11 | Short = 5 12 | UShort = 6 13 | Int = 7 14 | UInt = 8 15 | Long = 9 16 | ULong = 10 17 | Float = 11 18 | Double = 12 19 | String = 13 20 | Vector = 14 21 | Obj = 15 22 | Union = 16 23 | Array = 17 24 | MaxBaseType = 18 25 | -------------------------------------------------------------------------------- /zlmdb/flatbuffers/reflection/Enum.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: reflection 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | class Enum(object): 10 | __slots__ = ['_tab'] 11 | 12 | @classmethod 13 | def GetRootAs(cls, buf, offset=0): 14 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 15 | x = Enum() 16 | x.Init(buf, n + offset) 17 | return x 18 | 19 | @classmethod 20 | def GetRootAsEnum(cls, buf, offset=0): 21 | """This method is deprecated. Please switch to GetRootAs.""" 22 | return cls.GetRootAs(buf, offset) 23 | @classmethod 24 | def EnumBufferHasIdentifier(cls, buf, offset, size_prefixed=False): 25 | return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x42\x46\x42\x53", size_prefixed=size_prefixed) 26 | 27 | # Enum 28 | def Init(self, buf, pos): 29 | self._tab = flatbuffers.table.Table(buf, pos) 30 | 31 | # Enum 32 | def Name(self): 33 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 34 | if o != 0: 35 | return self._tab.String(o + self._tab.Pos) 36 | return None 37 | 38 | # Enum 39 | def Values(self, j): 40 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 41 | if o != 0: 42 | x = self._tab.Vector(o) 43 | x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 44 | x = self._tab.Indirect(x) 45 | from zlmdb.flatbuffers.reflection.EnumVal import EnumVal 46 | obj = EnumVal() 47 | obj.Init(self._tab.Bytes, x) 48 | return obj 49 | return None 50 | 51 | # Enum 52 | def ValuesLength(self): 53 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 54 | if o != 0: 55 | return self._tab.VectorLen(o) 56 | return 0 57 | 58 | # Enum 59 | def ValuesIsNone(self): 60 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 61 | return o == 0 62 | 63 | # Enum 64 | def IsUnion(self): 65 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) 66 | if o != 0: 67 | return bool(self._tab.Get(flatbuffers.number_types.BoolFlags, o + self._tab.Pos)) 68 | return False 69 | 70 | # Enum 71 | def UnderlyingType(self): 72 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) 73 | if o != 0: 74 | x = self._tab.Indirect(o + self._tab.Pos) 75 | from zlmdb.flatbuffers.reflection.Type import Type 76 | obj = Type() 77 | obj.Init(self._tab.Bytes, x) 78 | return obj 79 | return None 80 | 81 | # Enum 82 | def Attributes(self, j): 83 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) 84 | if o != 0: 85 | x = self._tab.Vector(o) 86 | x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 87 | x = self._tab.Indirect(x) 88 | from zlmdb.flatbuffers.reflection.KeyValue import KeyValue 89 | obj = KeyValue() 90 | obj.Init(self._tab.Bytes, x) 91 | return obj 92 | return None 93 | 94 | # Enum 95 | def AttributesLength(self): 96 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) 97 | if o != 0: 98 | return self._tab.VectorLen(o) 99 | return 0 100 | 101 | # Enum 102 | def AttributesIsNone(self): 103 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) 104 | return o == 0 105 | 106 | # Enum 107 | def Documentation(self, j): 108 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) 109 | if o != 0: 110 | a = self._tab.Vector(o) 111 | return self._tab.String(a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) 112 | return "" 113 | 114 | # Enum 115 | def DocumentationLength(self): 116 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) 117 | if o != 0: 118 | return self._tab.VectorLen(o) 119 | return 0 120 | 121 | # Enum 122 | def DocumentationIsNone(self): 123 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) 124 | return o == 0 125 | 126 | # File that this Enum is declared in. 127 | # Enum 128 | def DeclarationFile(self): 129 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) 130 | if o != 0: 131 | return self._tab.String(o + self._tab.Pos) 132 | return None 133 | 134 | def EnumStart(builder): builder.StartObject(7) 135 | def Start(builder): 136 | return EnumStart(builder) 137 | def EnumAddName(builder, name): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(name), 0) 138 | def AddName(builder, name): 139 | return EnumAddName(builder, name) 140 | def EnumAddValues(builder, values): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(values), 0) 141 | def AddValues(builder, values): 142 | return EnumAddValues(builder, values) 143 | def EnumStartValuesVector(builder, numElems): return builder.StartVector(4, numElems, 4) 144 | def StartValuesVector(builder, numElems): 145 | return EnumStartValuesVector(builder, numElems) 146 | def EnumAddIsUnion(builder, isUnion): builder.PrependBoolSlot(2, isUnion, 0) 147 | def AddIsUnion(builder, isUnion): 148 | return EnumAddIsUnion(builder, isUnion) 149 | def EnumAddUnderlyingType(builder, underlyingType): builder.PrependUOffsetTRelativeSlot(3, flatbuffers.number_types.UOffsetTFlags.py_type(underlyingType), 0) 150 | def AddUnderlyingType(builder, underlyingType): 151 | return EnumAddUnderlyingType(builder, underlyingType) 152 | def EnumAddAttributes(builder, attributes): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(attributes), 0) 153 | def AddAttributes(builder, attributes): 154 | return EnumAddAttributes(builder, attributes) 155 | def EnumStartAttributesVector(builder, numElems): return builder.StartVector(4, numElems, 4) 156 | def StartAttributesVector(builder, numElems): 157 | return EnumStartAttributesVector(builder, numElems) 158 | def EnumAddDocumentation(builder, documentation): builder.PrependUOffsetTRelativeSlot(5, flatbuffers.number_types.UOffsetTFlags.py_type(documentation), 0) 159 | def AddDocumentation(builder, documentation): 160 | return EnumAddDocumentation(builder, documentation) 161 | def EnumStartDocumentationVector(builder, numElems): return builder.StartVector(4, numElems, 4) 162 | def StartDocumentationVector(builder, numElems): 163 | return EnumStartDocumentationVector(builder, numElems) 164 | def EnumAddDeclarationFile(builder, declarationFile): builder.PrependUOffsetTRelativeSlot(6, flatbuffers.number_types.UOffsetTFlags.py_type(declarationFile), 0) 165 | def AddDeclarationFile(builder, declarationFile): 166 | return EnumAddDeclarationFile(builder, declarationFile) 167 | def EnumEnd(builder): return builder.EndObject() 168 | def End(builder): 169 | return EnumEnd(builder) 170 | -------------------------------------------------------------------------------- /zlmdb/flatbuffers/reflection/EnumVal.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: reflection 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | class EnumVal(object): 10 | __slots__ = ['_tab'] 11 | 12 | @classmethod 13 | def GetRootAs(cls, buf, offset=0): 14 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 15 | x = EnumVal() 16 | x.Init(buf, n + offset) 17 | return x 18 | 19 | @classmethod 20 | def GetRootAsEnumVal(cls, buf, offset=0): 21 | """This method is deprecated. Please switch to GetRootAs.""" 22 | return cls.GetRootAs(buf, offset) 23 | @classmethod 24 | def EnumValBufferHasIdentifier(cls, buf, offset, size_prefixed=False): 25 | return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x42\x46\x42\x53", size_prefixed=size_prefixed) 26 | 27 | # EnumVal 28 | def Init(self, buf, pos): 29 | self._tab = flatbuffers.table.Table(buf, pos) 30 | 31 | # EnumVal 32 | def Name(self): 33 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 34 | if o != 0: 35 | return self._tab.String(o + self._tab.Pos) 36 | return None 37 | 38 | # EnumVal 39 | def Value(self): 40 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 41 | if o != 0: 42 | return self._tab.Get(flatbuffers.number_types.Int64Flags, o + self._tab.Pos) 43 | return 0 44 | 45 | # EnumVal 46 | def UnionType(self): 47 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) 48 | if o != 0: 49 | x = self._tab.Indirect(o + self._tab.Pos) 50 | from zlmdb.flatbuffers.reflection.Type import Type 51 | obj = Type() 52 | obj.Init(self._tab.Bytes, x) 53 | return obj 54 | return None 55 | 56 | # EnumVal 57 | def Documentation(self, j): 58 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) 59 | if o != 0: 60 | a = self._tab.Vector(o) 61 | return self._tab.String(a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) 62 | return "" 63 | 64 | # EnumVal 65 | def DocumentationLength(self): 66 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) 67 | if o != 0: 68 | return self._tab.VectorLen(o) 69 | return 0 70 | 71 | # EnumVal 72 | def DocumentationIsNone(self): 73 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) 74 | return o == 0 75 | 76 | def EnumValStart(builder): builder.StartObject(5) 77 | def Start(builder): 78 | return EnumValStart(builder) 79 | def EnumValAddName(builder, name): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(name), 0) 80 | def AddName(builder, name): 81 | return EnumValAddName(builder, name) 82 | def EnumValAddValue(builder, value): builder.PrependInt64Slot(1, value, 0) 83 | def AddValue(builder, value): 84 | return EnumValAddValue(builder, value) 85 | def EnumValAddUnionType(builder, unionType): builder.PrependUOffsetTRelativeSlot(3, flatbuffers.number_types.UOffsetTFlags.py_type(unionType), 0) 86 | def AddUnionType(builder, unionType): 87 | return EnumValAddUnionType(builder, unionType) 88 | def EnumValAddDocumentation(builder, documentation): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(documentation), 0) 89 | def AddDocumentation(builder, documentation): 90 | return EnumValAddDocumentation(builder, documentation) 91 | def EnumValStartDocumentationVector(builder, numElems): return builder.StartVector(4, numElems, 4) 92 | def StartDocumentationVector(builder, numElems): 93 | return EnumValStartDocumentationVector(builder, numElems) 94 | def EnumValEnd(builder): return builder.EndObject() 95 | def End(builder): 96 | return EnumValEnd(builder) 97 | -------------------------------------------------------------------------------- /zlmdb/flatbuffers/reflection/KeyValue.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: reflection 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | class KeyValue(object): 10 | __slots__ = ['_tab'] 11 | 12 | @classmethod 13 | def GetRootAs(cls, buf, offset=0): 14 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 15 | x = KeyValue() 16 | x.Init(buf, n + offset) 17 | return x 18 | 19 | @classmethod 20 | def GetRootAsKeyValue(cls, buf, offset=0): 21 | """This method is deprecated. Please switch to GetRootAs.""" 22 | return cls.GetRootAs(buf, offset) 23 | @classmethod 24 | def KeyValueBufferHasIdentifier(cls, buf, offset, size_prefixed=False): 25 | return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x42\x46\x42\x53", size_prefixed=size_prefixed) 26 | 27 | # KeyValue 28 | def Init(self, buf, pos): 29 | self._tab = flatbuffers.table.Table(buf, pos) 30 | 31 | # KeyValue 32 | def Key(self): 33 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 34 | if o != 0: 35 | return self._tab.String(o + self._tab.Pos) 36 | return None 37 | 38 | # KeyValue 39 | def Value(self): 40 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 41 | if o != 0: 42 | return self._tab.String(o + self._tab.Pos) 43 | return None 44 | 45 | def KeyValueStart(builder): builder.StartObject(2) 46 | def Start(builder): 47 | return KeyValueStart(builder) 48 | def KeyValueAddKey(builder, key): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(key), 0) 49 | def AddKey(builder, key): 50 | return KeyValueAddKey(builder, key) 51 | def KeyValueAddValue(builder, value): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(value), 0) 52 | def AddValue(builder, value): 53 | return KeyValueAddValue(builder, value) 54 | def KeyValueEnd(builder): return builder.EndObject() 55 | def End(builder): 56 | return KeyValueEnd(builder) -------------------------------------------------------------------------------- /zlmdb/flatbuffers/reflection/RPCCall.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: reflection 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | class RPCCall(object): 10 | __slots__ = ['_tab'] 11 | 12 | @classmethod 13 | def GetRootAs(cls, buf, offset=0): 14 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 15 | x = RPCCall() 16 | x.Init(buf, n + offset) 17 | return x 18 | 19 | @classmethod 20 | def GetRootAsRPCCall(cls, buf, offset=0): 21 | """This method is deprecated. Please switch to GetRootAs.""" 22 | return cls.GetRootAs(buf, offset) 23 | @classmethod 24 | def RPCCallBufferHasIdentifier(cls, buf, offset, size_prefixed=False): 25 | return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x42\x46\x42\x53", size_prefixed=size_prefixed) 26 | 27 | # RPCCall 28 | def Init(self, buf, pos): 29 | self._tab = flatbuffers.table.Table(buf, pos) 30 | 31 | # RPCCall 32 | def Name(self): 33 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 34 | if o != 0: 35 | return self._tab.String(o + self._tab.Pos) 36 | return None 37 | 38 | # RPCCall 39 | def Request(self): 40 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 41 | if o != 0: 42 | x = self._tab.Indirect(o + self._tab.Pos) 43 | from zlmdb.flatbuffers.reflection.Object import Object 44 | obj = Object() 45 | obj.Init(self._tab.Bytes, x) 46 | return obj 47 | return None 48 | 49 | # RPCCall 50 | def Response(self): 51 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) 52 | if o != 0: 53 | x = self._tab.Indirect(o + self._tab.Pos) 54 | from zlmdb.flatbuffers.reflection.Object import Object 55 | obj = Object() 56 | obj.Init(self._tab.Bytes, x) 57 | return obj 58 | return None 59 | 60 | # RPCCall 61 | def Attributes(self, j): 62 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) 63 | if o != 0: 64 | x = self._tab.Vector(o) 65 | x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 66 | x = self._tab.Indirect(x) 67 | from zlmdb.flatbuffers.reflection.KeyValue import KeyValue 68 | obj = KeyValue() 69 | obj.Init(self._tab.Bytes, x) 70 | return obj 71 | return None 72 | 73 | # RPCCall 74 | def AttributesLength(self): 75 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) 76 | if o != 0: 77 | return self._tab.VectorLen(o) 78 | return 0 79 | 80 | # RPCCall 81 | def AttributesIsNone(self): 82 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) 83 | return o == 0 84 | 85 | # RPCCall 86 | def Documentation(self, j): 87 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) 88 | if o != 0: 89 | a = self._tab.Vector(o) 90 | return self._tab.String(a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) 91 | return "" 92 | 93 | # RPCCall 94 | def DocumentationLength(self): 95 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) 96 | if o != 0: 97 | return self._tab.VectorLen(o) 98 | return 0 99 | 100 | # RPCCall 101 | def DocumentationIsNone(self): 102 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) 103 | return o == 0 104 | 105 | def RPCCallStart(builder): builder.StartObject(5) 106 | def Start(builder): 107 | return RPCCallStart(builder) 108 | def RPCCallAddName(builder, name): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(name), 0) 109 | def AddName(builder, name): 110 | return RPCCallAddName(builder, name) 111 | def RPCCallAddRequest(builder, request): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(request), 0) 112 | def AddRequest(builder, request): 113 | return RPCCallAddRequest(builder, request) 114 | def RPCCallAddResponse(builder, response): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(response), 0) 115 | def AddResponse(builder, response): 116 | return RPCCallAddResponse(builder, response) 117 | def RPCCallAddAttributes(builder, attributes): builder.PrependUOffsetTRelativeSlot(3, flatbuffers.number_types.UOffsetTFlags.py_type(attributes), 0) 118 | def AddAttributes(builder, attributes): 119 | return RPCCallAddAttributes(builder, attributes) 120 | def RPCCallStartAttributesVector(builder, numElems): return builder.StartVector(4, numElems, 4) 121 | def StartAttributesVector(builder, numElems): 122 | return RPCCallStartAttributesVector(builder, numElems) 123 | def RPCCallAddDocumentation(builder, documentation): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(documentation), 0) 124 | def AddDocumentation(builder, documentation): 125 | return RPCCallAddDocumentation(builder, documentation) 126 | def RPCCallStartDocumentationVector(builder, numElems): return builder.StartVector(4, numElems, 4) 127 | def StartDocumentationVector(builder, numElems): 128 | return RPCCallStartDocumentationVector(builder, numElems) 129 | def RPCCallEnd(builder): return builder.EndObject() 130 | def End(builder): 131 | return RPCCallEnd(builder) 132 | -------------------------------------------------------------------------------- /zlmdb/flatbuffers/reflection/SchemaFile.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: reflection 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | # File specific information. 10 | # Symbols declared within a file may be recovered by iterating over all 11 | # symbols and examining the `declaration_file` field. 12 | class SchemaFile(object): 13 | __slots__ = ['_tab'] 14 | 15 | @classmethod 16 | def GetRootAs(cls, buf, offset=0): 17 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 18 | x = SchemaFile() 19 | x.Init(buf, n + offset) 20 | return x 21 | 22 | @classmethod 23 | def GetRootAsSchemaFile(cls, buf, offset=0): 24 | """This method is deprecated. Please switch to GetRootAs.""" 25 | return cls.GetRootAs(buf, offset) 26 | @classmethod 27 | def SchemaFileBufferHasIdentifier(cls, buf, offset, size_prefixed=False): 28 | return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x42\x46\x42\x53", size_prefixed=size_prefixed) 29 | 30 | # SchemaFile 31 | def Init(self, buf, pos): 32 | self._tab = flatbuffers.table.Table(buf, pos) 33 | 34 | # Filename, relative to project root. 35 | # SchemaFile 36 | def Filename(self): 37 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 38 | if o != 0: 39 | return self._tab.String(o + self._tab.Pos) 40 | return None 41 | 42 | # Names of included files, relative to project root. 43 | # SchemaFile 44 | def IncludedFilenames(self, j): 45 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 46 | if o != 0: 47 | a = self._tab.Vector(o) 48 | return self._tab.String(a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) 49 | return "" 50 | 51 | # SchemaFile 52 | def IncludedFilenamesLength(self): 53 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 54 | if o != 0: 55 | return self._tab.VectorLen(o) 56 | return 0 57 | 58 | # SchemaFile 59 | def IncludedFilenamesIsNone(self): 60 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 61 | return o == 0 62 | 63 | def SchemaFileStart(builder): builder.StartObject(2) 64 | def Start(builder): 65 | return SchemaFileStart(builder) 66 | def SchemaFileAddFilename(builder, filename): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(filename), 0) 67 | def AddFilename(builder, filename): 68 | return SchemaFileAddFilename(builder, filename) 69 | def SchemaFileAddIncludedFilenames(builder, includedFilenames): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(includedFilenames), 0) 70 | def AddIncludedFilenames(builder, includedFilenames): 71 | return SchemaFileAddIncludedFilenames(builder, includedFilenames) 72 | def SchemaFileStartIncludedFilenamesVector(builder, numElems): return builder.StartVector(4, numElems, 4) 73 | def StartIncludedFilenamesVector(builder, numElems): 74 | return SchemaFileStartIncludedFilenamesVector(builder, numElems) 75 | def SchemaFileEnd(builder): return builder.EndObject() 76 | def End(builder): 77 | return SchemaFileEnd(builder) -------------------------------------------------------------------------------- /zlmdb/flatbuffers/reflection/Service.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: reflection 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | class Service(object): 10 | __slots__ = ['_tab'] 11 | 12 | @classmethod 13 | def GetRootAs(cls, buf, offset=0): 14 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 15 | x = Service() 16 | x.Init(buf, n + offset) 17 | return x 18 | 19 | @classmethod 20 | def GetRootAsService(cls, buf, offset=0): 21 | """This method is deprecated. Please switch to GetRootAs.""" 22 | return cls.GetRootAs(buf, offset) 23 | @classmethod 24 | def ServiceBufferHasIdentifier(cls, buf, offset, size_prefixed=False): 25 | return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x42\x46\x42\x53", size_prefixed=size_prefixed) 26 | 27 | # Service 28 | def Init(self, buf, pos): 29 | self._tab = flatbuffers.table.Table(buf, pos) 30 | 31 | # Service 32 | def Name(self): 33 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 34 | if o != 0: 35 | return self._tab.String(o + self._tab.Pos) 36 | return None 37 | 38 | # Service 39 | def Calls(self, j): 40 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 41 | if o != 0: 42 | x = self._tab.Vector(o) 43 | x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 44 | x = self._tab.Indirect(x) 45 | from zlmdb.flatbuffers.reflection.RPCCall import RPCCall 46 | obj = RPCCall() 47 | obj.Init(self._tab.Bytes, x) 48 | return obj 49 | return None 50 | 51 | # Service 52 | def CallsLength(self): 53 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 54 | if o != 0: 55 | return self._tab.VectorLen(o) 56 | return 0 57 | 58 | # Service 59 | def CallsIsNone(self): 60 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 61 | return o == 0 62 | 63 | # Service 64 | def Attributes(self, j): 65 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) 66 | if o != 0: 67 | x = self._tab.Vector(o) 68 | x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 69 | x = self._tab.Indirect(x) 70 | from zlmdb.flatbuffers.reflection.KeyValue import KeyValue 71 | obj = KeyValue() 72 | obj.Init(self._tab.Bytes, x) 73 | return obj 74 | return None 75 | 76 | # Service 77 | def AttributesLength(self): 78 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) 79 | if o != 0: 80 | return self._tab.VectorLen(o) 81 | return 0 82 | 83 | # Service 84 | def AttributesIsNone(self): 85 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) 86 | return o == 0 87 | 88 | # Service 89 | def Documentation(self, j): 90 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) 91 | if o != 0: 92 | a = self._tab.Vector(o) 93 | return self._tab.String(a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) 94 | return "" 95 | 96 | # Service 97 | def DocumentationLength(self): 98 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) 99 | if o != 0: 100 | return self._tab.VectorLen(o) 101 | return 0 102 | 103 | # Service 104 | def DocumentationIsNone(self): 105 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) 106 | return o == 0 107 | 108 | # File that this Service is declared in. 109 | # Service 110 | def DeclarationFile(self): 111 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) 112 | if o != 0: 113 | return self._tab.String(o + self._tab.Pos) 114 | return None 115 | 116 | def ServiceStart(builder): builder.StartObject(5) 117 | def Start(builder): 118 | return ServiceStart(builder) 119 | def ServiceAddName(builder, name): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(name), 0) 120 | def AddName(builder, name): 121 | return ServiceAddName(builder, name) 122 | def ServiceAddCalls(builder, calls): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(calls), 0) 123 | def AddCalls(builder, calls): 124 | return ServiceAddCalls(builder, calls) 125 | def ServiceStartCallsVector(builder, numElems): return builder.StartVector(4, numElems, 4) 126 | def StartCallsVector(builder, numElems): 127 | return ServiceStartCallsVector(builder, numElems) 128 | def ServiceAddAttributes(builder, attributes): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(attributes), 0) 129 | def AddAttributes(builder, attributes): 130 | return ServiceAddAttributes(builder, attributes) 131 | def ServiceStartAttributesVector(builder, numElems): return builder.StartVector(4, numElems, 4) 132 | def StartAttributesVector(builder, numElems): 133 | return ServiceStartAttributesVector(builder, numElems) 134 | def ServiceAddDocumentation(builder, documentation): builder.PrependUOffsetTRelativeSlot(3, flatbuffers.number_types.UOffsetTFlags.py_type(documentation), 0) 135 | def AddDocumentation(builder, documentation): 136 | return ServiceAddDocumentation(builder, documentation) 137 | def ServiceStartDocumentationVector(builder, numElems): return builder.StartVector(4, numElems, 4) 138 | def StartDocumentationVector(builder, numElems): 139 | return ServiceStartDocumentationVector(builder, numElems) 140 | def ServiceAddDeclarationFile(builder, declarationFile): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(declarationFile), 0) 141 | def AddDeclarationFile(builder, declarationFile): 142 | return ServiceAddDeclarationFile(builder, declarationFile) 143 | def ServiceEnd(builder): return builder.EndObject() 144 | def End(builder): 145 | return ServiceEnd(builder) 146 | -------------------------------------------------------------------------------- /zlmdb/flatbuffers/reflection/Type.py: -------------------------------------------------------------------------------- 1 | # automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | # namespace: reflection 4 | 5 | import flatbuffers 6 | from flatbuffers.compat import import_numpy 7 | np = import_numpy() 8 | 9 | class Type(object): 10 | __slots__ = ['_tab'] 11 | 12 | @classmethod 13 | def GetRootAs(cls, buf, offset=0): 14 | n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) 15 | x = Type() 16 | x.Init(buf, n + offset) 17 | return x 18 | 19 | @classmethod 20 | def GetRootAsType(cls, buf, offset=0): 21 | """This method is deprecated. Please switch to GetRootAs.""" 22 | return cls.GetRootAs(buf, offset) 23 | @classmethod 24 | def TypeBufferHasIdentifier(cls, buf, offset, size_prefixed=False): 25 | return flatbuffers.util.BufferHasIdentifier(buf, offset, b"\x42\x46\x42\x53", size_prefixed=size_prefixed) 26 | 27 | # Type 28 | def Init(self, buf, pos): 29 | self._tab = flatbuffers.table.Table(buf, pos) 30 | 31 | # Type 32 | def BaseType(self): 33 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) 34 | if o != 0: 35 | return self._tab.Get(flatbuffers.number_types.Int8Flags, o + self._tab.Pos) 36 | return 0 37 | 38 | # Type 39 | def Element(self): 40 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) 41 | if o != 0: 42 | return self._tab.Get(flatbuffers.number_types.Int8Flags, o + self._tab.Pos) 43 | return 0 44 | 45 | # Type 46 | def Index(self): 47 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) 48 | if o != 0: 49 | return self._tab.Get(flatbuffers.number_types.Int32Flags, o + self._tab.Pos) 50 | return -1 51 | 52 | # Type 53 | def FixedLength(self): 54 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) 55 | if o != 0: 56 | return self._tab.Get(flatbuffers.number_types.Uint16Flags, o + self._tab.Pos) 57 | return 0 58 | 59 | # The size (octets) of the `base_type` field. 60 | # Type 61 | def BaseSize(self): 62 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) 63 | if o != 0: 64 | return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos) 65 | return 4 66 | 67 | # The size (octets) of the `element` field, if present. 68 | # Type 69 | def ElementSize(self): 70 | o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) 71 | if o != 0: 72 | return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos) 73 | return 0 74 | 75 | def TypeStart(builder): builder.StartObject(6) 76 | def Start(builder): 77 | return TypeStart(builder) 78 | def TypeAddBaseType(builder, baseType): builder.PrependInt8Slot(0, baseType, 0) 79 | def AddBaseType(builder, baseType): 80 | return TypeAddBaseType(builder, baseType) 81 | def TypeAddElement(builder, element): builder.PrependInt8Slot(1, element, 0) 82 | def AddElement(builder, element): 83 | return TypeAddElement(builder, element) 84 | def TypeAddIndex(builder, index): builder.PrependInt32Slot(2, index, -1) 85 | def AddIndex(builder, index): 86 | return TypeAddIndex(builder, index) 87 | def TypeAddFixedLength(builder, fixedLength): builder.PrependUint16Slot(3, fixedLength, 0) 88 | def AddFixedLength(builder, fixedLength): 89 | return TypeAddFixedLength(builder, fixedLength) 90 | def TypeAddBaseSize(builder, baseSize): builder.PrependUint32Slot(4, baseSize, 4) 91 | def AddBaseSize(builder, baseSize): 92 | return TypeAddBaseSize(builder, baseSize) 93 | def TypeAddElementSize(builder, elementSize): builder.PrependUint32Slot(5, elementSize, 0) 94 | def AddElementSize(builder, elementSize): 95 | return TypeAddElementSize(builder, elementSize) 96 | def TypeEnd(builder): return builder.EndObject() 97 | def End(builder): 98 | return TypeEnd(builder) -------------------------------------------------------------------------------- /zlmdb/flatbuffers/reflection/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crossbario/zlmdb/16270e41d5fadd2615cd735723cb629700fa12e8/zlmdb/flatbuffers/reflection/__init__.py -------------------------------------------------------------------------------- /zlmdb/tests/_test_flatbuffers.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) typedef int GmbH 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in 15 | # all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | # THE SOFTWARE. 24 | # 25 | ############################################################################### 26 | 27 | import os 28 | import sys 29 | import random 30 | 31 | import txaio 32 | txaio.use_twisted() 33 | 34 | try: 35 | from tempfile import TemporaryDirectory 36 | except ImportError: 37 | from backports.tempfile import TemporaryDirectory # type:ignore 38 | 39 | import zlmdb # noqa 40 | 41 | sys.path.append(os.path.dirname(os.path.abspath(__file__))) 42 | 43 | from _schema_fbs import User # noqa 44 | 45 | 46 | class UsersSchema(zlmdb.Schema): 47 | def __init__(self): 48 | self.tab_oid_fbs = zlmdb.MapOidFlatBuffers(1, build=User.build, cast=User.cast) 49 | 50 | 51 | def test_pmap_flatbuffers_values(): 52 | with TemporaryDirectory() as dbpath: 53 | print('Using temporary directory {} for database'.format(dbpath)) 54 | 55 | schema = UsersSchema() 56 | 57 | with zlmdb.Database(dbpath) as db: 58 | 59 | N = 100 60 | stats = zlmdb.TransactionStats() 61 | 62 | with db.begin(write=True, stats=stats) as txn: 63 | for i in range(N): 64 | user = User.create_test_user() 65 | schema.tab_oid_fbs[txn, user.oid] = user 66 | 67 | assert stats.puts == N 68 | assert stats.dels == 0 69 | stats.reset() 70 | 71 | with db.begin() as txn: 72 | cnt = schema.tab_oid_fbs.count(txn) 73 | 74 | assert cnt == N 75 | 76 | 77 | def test_pmap_flatbuffers_count(): 78 | with TemporaryDirectory() as dbpath: 79 | print('Using temporary directory {} for database'.format(dbpath)) 80 | 81 | schema = UsersSchema() 82 | 83 | # max DB size is 100 MB 84 | with zlmdb.Database(dbpath, maxsize=100 * (2**20)) as db: 85 | 86 | oids = set() 87 | oid_to_referred_by = {} 88 | 89 | stats = zlmdb.TransactionStats() 90 | 91 | # number of transactions 92 | M = 5 93 | 94 | # number of insert rows per transaction 95 | N = 10000 96 | for j in range(M): 97 | with db.begin(write=True, stats=stats) as txn: 98 | for i in range(N): 99 | user = User.create_test_user() 100 | schema.tab_oid_fbs[txn, user.oid] = user 101 | oids.add(user.oid) 102 | oid_to_referred_by[user.oid] = user.referred_by 103 | 104 | assert stats.puts == N 105 | assert stats.dels == 0 106 | duration_ns = stats.duration 107 | duration_ms = int(duration_ns / 1000000.) 108 | rows_per_sec = int(round(float(stats.puts + stats.dels) * 1000. / float(duration_ms))) 109 | print('Transaction ended: puts={} / dels={} rows in {} ms, {} rows/sec'.format( 110 | stats.puts, stats.dels, duration_ms, rows_per_sec)) 111 | 112 | stats.reset() 113 | 114 | # count all rows 115 | with db.begin() as txn: 116 | cnt = schema.tab_oid_fbs.count(txn) 117 | 118 | assert cnt == N * M 119 | 120 | # retrieve 121 | with db.begin() as txn: 122 | for j in range(5): 123 | started = zlmdb.walltime() 124 | M = 100 125 | for i in range(M): 126 | for oid in random.sample(oids, N): 127 | user = schema.tab_oid_fbs[txn, oid] 128 | assert user 129 | assert user.referred_by == oid_to_referred_by.get(oid, None) 130 | duration_ns = zlmdb.walltime() - started 131 | duration_ms = int(duration_ns / 1000000.) 132 | rows_per_sec = int(round(float(M * N) * 1000. / float(duration_ms))) 133 | print('Transaction ended: {} rows read in {} ms, {} rows/sec'.format( 134 | M * N, duration_ms, rows_per_sec)) 135 | -------------------------------------------------------------------------------- /zlmdb/tests/_test_serialization.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) typedef int GmbH 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in 15 | # all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | # THE SOFTWARE. 24 | # 25 | ############################################################################### 26 | 27 | import os 28 | import sys 29 | import timeit 30 | import uuid 31 | import platform 32 | 33 | import humanize 34 | 35 | import txaio 36 | txaio.use_twisted() 37 | 38 | from zlmdb import _types # noqa 39 | from _schema_fbs import User as UserFbs # noqa 40 | 41 | sys.path.append(os.path.dirname(os.path.abspath(__file__))) 42 | 43 | if sys.version_info >= (3, 6): 44 | from _schema_py3 import User 45 | else: 46 | from _schema_py2 import User 47 | 48 | _TEST = {'oid': 0, 'uuid': None, 'bytes': 0, 'serializer': None} 49 | 50 | 51 | def _serializer_run_fbs(): 52 | serialize = _TEST['serializer']._serialize_value 53 | user = UserFbs.create_test_user() 54 | data = serialize(user) 55 | _TEST['bytes'] += len(data) 56 | 57 | 58 | def _serializer_run(): 59 | serialize = _TEST['serializer']._serialize_value 60 | user = User.create_test_user() 61 | data = serialize(user) 62 | _TEST['bytes'] += len(data) 63 | 64 | 65 | def _serialization_speed(serializer, testfun): 66 | N = 10 67 | M = 10000 68 | 69 | samples = [] 70 | 71 | print('running on:') 72 | print(sys.version) 73 | print(platform.uname()) 74 | 75 | _TEST['oid'] = 0 76 | _TEST['uuid'] = uuid.uuid4() 77 | _TEST['bytes'] = 0 78 | _TEST['bytes'] = 0 79 | _TEST['serializer'] = serializer 80 | 81 | for i in range(N): 82 | secs = timeit.timeit(testfun, number=M) 83 | ops = round(float(M) / secs, 1) 84 | samples.append(ops) 85 | print('{} objects/sec {}'.format(ops, humanize.naturalsize(_TEST['bytes']))) 86 | 87 | ops_max = max(samples) 88 | bytes_per_obj = float(_TEST['bytes']) / float(N * M) 89 | print('{} objects/sec max, {} bytes total, {} bytes/obj'.format(ops_max, humanize.naturalsize(_TEST['bytes']), 90 | humanize.naturalsize(bytes_per_obj))) 91 | 92 | return ops_max, _TEST['bytes'] 93 | 94 | 95 | def test_json_serialization_speed(): 96 | ser = _types._JsonValuesMixin(marshal=User.marshal, unmarshal=User.parse) 97 | ops_max, total = _serialization_speed(ser, _serializer_run) 98 | # cpy36: 19564.6 objects/sec max, 135456153 bytes total 99 | assert ops_max > 1000 100 | assert total > 1000000 101 | 102 | 103 | def test_cbor_serialization_speed(): 104 | ser = _types._CborValuesMixin(marshal=User.marshal, unmarshal=User.parse) 105 | ops_max, total = _serialization_speed(ser, _serializer_run) 106 | # cpy36: 7787.4 objects/sec max, 97815364 bytes total 107 | assert ops_max > 1000 108 | assert total > 1000000 109 | 110 | 111 | def test_pickle_serialization_speed(): 112 | ser = _types._PickleValuesMixin() 113 | ops_max, total = _serialization_speed(ser, _serializer_run) 114 | # cpy36: 33586.0 objects/sec max, 137738869 bytes total 115 | assert ops_max > 1000 116 | assert total > 1000000 117 | 118 | 119 | def test_flatbuffer_serialization_speed(): 120 | ser = _types._FlatBuffersValuesMixin(build=UserFbs.build, cast=UserFbs.cast) 121 | ops_max, total = _serialization_speed(ser, _serializer_run_fbs) 122 | assert ops_max > 1000 123 | assert total > 1000000 124 | 125 | 126 | if __name__ == '__main__': 127 | from typing import List 128 | 129 | sers: List[object] = [] 130 | sers.append(_types._JsonValuesMixin(marshal=User.marshal, unmarshal=User.parse)) 131 | sers.append(_types._CborValuesMixin(marshal=User.marshal, unmarshal=User.parse)) 132 | sers.append(_types._PickleValuesMixin()) 133 | sers.append(_types._FlatBuffersValuesMixin(build=UserFbs.build, cast=UserFbs.cast)) 134 | for ser in sers: 135 | print(_serialization_speed(ser, _serializer_run)) 136 | -------------------------------------------------------------------------------- /zlmdb/tests/test_lmdb.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) typedef int GmbH 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in 15 | # all copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | # THE SOFTWARE. 24 | # 25 | ############################################################################### 26 | 27 | import random 28 | import os 29 | import lmdb 30 | import struct 31 | import platform 32 | 33 | import pytest 34 | 35 | try: 36 | from tempfile import TemporaryDirectory 37 | except ImportError: 38 | from backports.tempfile import TemporaryDirectory # type:ignore 39 | 40 | 41 | def test_lmdb_create(): 42 | """ 43 | Test creation of LMDB database. 44 | """ 45 | with TemporaryDirectory() as dbpath: 46 | 47 | env = lmdb.open(dbpath) 48 | 49 | with env.begin() as txn: 50 | assert txn.id() == 0 51 | 52 | 53 | def test_lmdb_insert_empty_key_raises(): 54 | """ 55 | Test that LMDB raises on inserting record with empty (bytes) key. 56 | """ 57 | with TemporaryDirectory() as dbpath: 58 | 59 | env = lmdb.open(dbpath) 60 | 61 | with env.begin(write=True) as txn: 62 | key = b'' 63 | value = random.randint(0, 2**32 - 1) 64 | data = struct.pack('= (3, 6): 43 | from _schema_py3 import User, Schema1 44 | else: 45 | from _schema_py2 import User, Schema1 46 | 47 | 48 | def test_pmap_value_types(): 49 | with TemporaryDirectory() as dbpath: 50 | print('Using temporary directory {} for database'.format(dbpath)) 51 | 52 | schema = Schema1() 53 | 54 | n = 100 55 | stats = zlmdb.TransactionStats() 56 | 57 | tabs = [ 58 | (schema.tab_oid_json, schema.tab_str_json, schema.tab_uuid_json), 59 | (schema.tab_oid_cbor, schema.tab_str_cbor, schema.tab_uuid_cbor), 60 | (schema.tab_oid_pickle, schema.tab_str_pickle, schema.tab_uuid_pickle), 61 | ] 62 | 63 | with zlmdb.Database(dbpath) as db: 64 | for tab_oid, tab_str, tab_uuid in tabs: 65 | with db.begin(write=True, stats=stats) as txn: 66 | for i in range(n): 67 | user = User.create_test_user(i) 68 | tab_oid[txn, user.oid] = user 69 | tab_str[txn, user.authid] = user 70 | tab_uuid[txn, user.uuid] = user 71 | 72 | print('transaction committed') 73 | assert stats.puts == n * 3 74 | assert stats.dels == 0 75 | 76 | stats.reset() 77 | 78 | with db.begin() as txn: 79 | cnt = tab_oid.count(txn) 80 | assert cnt == n 81 | 82 | cnt = tab_str.count(txn) 83 | assert cnt == n 84 | 85 | cnt = tab_uuid.count(txn) 86 | assert cnt == n 87 | 88 | print('database closed') 89 | --------------------------------------------------------------------------------