├── .editorconfig ├── .github └── workflows │ ├── publish.yml │ └── tests.yml ├── .gitignore ├── .gitmodules ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── examples ├── benchmark.py └── reader.py ├── lsm ├── __init__.py ├── _lsm.pyi └── py.typed ├── lsm_tool.py ├── pylava.ini ├── scripts └── make-wheels.sh ├── setup.py ├── src └── _lsm.c ├── tests ├── __init__.py ├── test_arguments.py ├── test_common_cases.py ├── test_dealloc.py ├── test_read_only.py └── test_simple.py └── tox.ini /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | 9 | [*.{py,yml}] 10 | indent_style = space 11 | 12 | [*.py] 13 | indent_size = 4 14 | 15 | [docs/**.py] 16 | max_line_length = 80 17 | 18 | [*.rst] 19 | indent_size = 3 20 | 21 | [Makefile] 22 | indent_style = tab 23 | 24 | [*.yml] 25 | indent_size = 2 26 | 27 | [*.c] 28 | max_line_length = 120 29 | indent_style = tab 30 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: publish 5 | 6 | on: 7 | release: 8 | types: 9 | - created 10 | 11 | jobs: 12 | sdist: 13 | 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | 18 | - uses: actions/checkout@v2 19 | with: 20 | submodules: recursive 21 | 22 | - name: Setup python3.9 23 | uses: actions/setup-python@v2 24 | with: 25 | python-version: "3.12" 26 | 27 | - name: Install requires 28 | run: python -m pip install setuptools twine wheel 29 | 30 | - name: Build source package 31 | run: python setup.py sdist 32 | 33 | - name: Publishing to pypi 34 | run: twine upload --skip-existing --disable-progress-bar dist/*.tar.gz 35 | env: 36 | TWINE_USERNAME: __token__ 37 | TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} 38 | 39 | wheel: 40 | 41 | runs-on: ${{ matrix.os }} 42 | 43 | strategy: 44 | fail-fast: false 45 | 46 | matrix: 47 | include: 48 | # MacOS 49 | - python: '3.9' 50 | os: macos-latest 51 | - python: '3.10' 52 | os: macos-latest 53 | - python: '3.11' 54 | os: macos-latest 55 | - python: '3.12' 56 | os: macos-latest 57 | - python: '3.13' 58 | os: macos-latest 59 | 60 | # Windows 61 | - python: '3.9' 62 | os: windows-latest 63 | - python: '3.10' 64 | os: windows-latest 65 | - python: '3.11' 66 | os: windows-latest 67 | - python: '3.12' 68 | os: windows-latest 69 | - python: '3.13' 70 | os: windows-latest 71 | 72 | steps: 73 | - uses: actions/checkout@v2 74 | with: 75 | submodules: recursive 76 | 77 | - name: Setup python${{ matrix.python }} 78 | uses: actions/setup-python@v2 79 | with: 80 | python-version: "${{ matrix.python }}" 81 | 82 | - name: Install requires 83 | run: python -m pip install twine wheel setuptools 84 | 85 | - name: Build wheel for python "${{ matrix.python }}" 86 | run: python setup.py bdist_wheel 87 | 88 | - name: Publishing to pypi 89 | run: twine upload --skip-existing --disable-progress-bar dist/*.whl 90 | env: 91 | TWINE_USERNAME: __token__ 92 | TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} 93 | 94 | 95 | linux-wheels: 96 | 97 | runs-on: ubuntu-latest 98 | 99 | steps: 100 | - uses: actions/checkout@v2 101 | with: 102 | submodules: recursive 103 | 104 | - name: Building manylinux2014_x86_64 wheels 105 | uses: docker://quay.io/pypa/manylinux_2_34_x86_64 106 | with: 107 | args: /bin/bash scripts/make-wheels.sh 108 | 109 | - name: Setup python${{ matrix.python }} 110 | uses: actions/setup-python@v2 111 | with: 112 | python-version: "3.9" 113 | 114 | - name: Install requires 115 | run: python -m pip install twine 116 | 117 | - name: Publishing to pypi 118 | run: twine upload --skip-existing --disable-progress-bar dist/*.whl 119 | env: 120 | TWINE_USERNAME: __token__ 121 | TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} 122 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | 5 | name: tests 6 | 7 | on: 8 | push: 9 | branches: [ master ] 10 | pull_request: 11 | branches: [ master ] 12 | 13 | jobs: 14 | tests: 15 | 16 | runs-on: ${{ matrix.os }} 17 | 18 | strategy: 19 | fail-fast: false 20 | 21 | matrix: 22 | include: 23 | - python: '3.9' 24 | toxenv: py39 25 | os: ubuntu-latest 26 | - python: '3.10' 27 | toxenv: py310 28 | os: ubuntu-latest 29 | - python: '3.11' 30 | toxenv: py311 31 | os: ubuntu-latest 32 | - python: '3.12' 33 | toxenv: py312 34 | os: ubuntu-latest 35 | - python: '3.13' 36 | toxenv: py313 37 | os: ubuntu-latest 38 | 39 | - python: '3.9' 40 | toxenv: py39 41 | os: macos-latest 42 | - python: '3.10' 43 | toxenv: py310 44 | os: macos-latest 45 | - python: '3.11' 46 | toxenv: py311 47 | os: macos-latest 48 | - python: '3.12' 49 | toxenv: py312 50 | os: macos-latest 51 | - python: '3.13' 52 | toxenv: py313 53 | os: macos-latest 54 | 55 | - python: '3.9' 56 | toxenv: py39 57 | os: windows-latest 58 | - python: '3.10' 59 | toxenv: py310 60 | os: windows-latest 61 | - python: '3.11' 62 | toxenv: py311 63 | os: windows-latest 64 | - python: '3.12' 65 | toxenv: py312 66 | os: windows-latest 67 | - python: '3.13' 68 | toxenv: py313 69 | os: windows-latest 70 | steps: 71 | - uses: actions/checkout@v2 72 | with: 73 | submodules: recursive 74 | 75 | - name: Setup python${{ matrix.python }} 76 | uses: actions/setup-python@v2 77 | with: 78 | python-version: "${{ matrix.python }}" 79 | 80 | - name: Install tox 81 | run: python -m pip install tox 82 | 83 | - name: Run tox 84 | run: tox 85 | env: 86 | FORCE_COLOR: yes 87 | TOXENV: ${{ matrix.toxenv }} 88 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | *.bin 131 | /.venv*/ 132 | /.env*/ 133 | *.ldb 134 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "src/lz4"] 2 | path = src/lz4 3 | url = https://github.com/lz4/lz4.git 4 | [submodule "src/zstd"] 5 | path = src/zstd 6 | url = https://github.com/facebook/zstd.git 7 | [submodule "src/sqlite"] 8 | path = src/sqlite 9 | url = https://github.com/sqlite/sqlite.git 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | 2 | # Add requirement licenses 3 | include src/lz4/LICENSE 4 | include src/zstd/LICENSE 5 | include src/sqlite/LICENSE.md 6 | 7 | recursive-include src/lz4/lib *.c *.h 8 | recursive-include src/zstd/lib *.c *.h 9 | recursive-include src/sqlite/ext/lsm1 *.c *.h 10 | 11 | # Add library interface 12 | include lsm/lsm.pyi 13 | include README.md 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | build: sdist 2 | 3 | sdist: 4 | python3 setup.py sdist 5 | 6 | linux-wheels: 7 | docker run --rm -v $(PWD):/mnt -w /mnt quay.io/pypa/manylinux_2_34_x86_64 bash /mnt/scripts/make-wheels.sh 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | lsm 2 | === 3 | 4 | Fast Python bindings for [SQLite's LSM key/value store](http://www.sqlite.org/src4/doc/trunk/www/lsmusr.wiki>). 5 | The LSM storage engine was initially written as part of the experimental 6 | SQLite4 rewrite (now abandoned). More recently, the LSM source code was moved 7 | into the SQLite3 [source tree](https://www.sqlite.org/cgi/src/dir?ci=e148cdad35520e66&name=ext/lsm1) 8 | and has seen some improvements and fixes. This project uses the LSM code from 9 | the SQLite3 source tree. 10 | 11 | Features: 12 | 13 | * Embedded zero-conf database. 14 | * Keys support in-order traversal using cursors. 15 | * Transactional (including nested transactions). 16 | * Single writer/multiple reader MVCC based transactional concurrency model. 17 | * On-disk database stored in a single file. 18 | * Data is durable in the face of application or power failure. 19 | * Thread-safe. 20 | * Releases GIL for read and write operations 21 | (each connection has own mutex) 22 | * Page compression (lz4 or zstd) 23 | * Zero dependency static library 24 | * Python 3.x. 25 | 26 | Limitations: 27 | 28 | The source for Python lsm is 29 | [hosted on GitHub](https://github.com/mosquito/python-lsm). 30 | 31 | If you encounter any bugs in the library, please 32 | [open an issue](https://github.com/mosquito/python-lsm/issues/new), 33 | including a description of the bug and any related traceback. 34 | 35 | ## Quick-start 36 | 37 | Below is a sample interactive console session designed to show some of the 38 | basic features and functionality of the ``lsm`` Python library. 39 | 40 | To begin, instantiate a `LSM` object, specifying a path to a database file. 41 | 42 | 43 | ```python 44 | from lsm import LSM 45 | db = LSM('test.ldb') 46 | assert db.open() 47 | ``` 48 | 49 | More pythonic variant is using context manager: 50 | 51 | 52 | ```python 53 | from lsm import LSM 54 | with LSM("test.ldb") as db: 55 | assert db.info() 56 | ``` 57 | 58 | Not opened database will raise a RuntimeError: 59 | 60 | 61 | ```python 62 | import pytest 63 | from lsm import LSM 64 | 65 | db = LSM('test.ldb') 66 | 67 | with pytest.raises(RuntimeError): 68 | db.info() 69 | ``` 70 | 71 | ### Binary/string mode 72 | 73 | You should select mode for opening the database with ``binary: bool = True`` 74 | argument. 75 | 76 | For example when you want to store strings just pass ``binary=False``: 77 | 78 | 79 | ```python 80 | from lsm import LSM 81 | with LSM("test_0.ldb", binary=False) as db: 82 | # must be str for keys and values 83 | db['foo'] = 'bar' 84 | assert db['foo'] == "bar" 85 | ``` 86 | 87 | Otherwise, you must pass keys and values ad ``bytes`` (default behaviour): 88 | 89 | 90 | ```python 91 | from lsm import LSM 92 | 93 | with LSM("test.ldb") as db: 94 | db[b'foo'] = b'bar' 95 | assert db[b'foo'] == b'bar' 96 | ``` 97 | 98 | ### Key/Value Features 99 | 100 | ``lsm`` is a key/value store, and has a dictionary-like API: 101 | 102 | 103 | ```python 104 | from lsm import LSM 105 | with LSM("test.ldb", binary=False) as db: 106 | db['foo'] = 'bar' 107 | assert db['foo'] == 'bar' 108 | ``` 109 | 110 | Database apply changes as soon as possible: 111 | 112 | 113 | ```python 114 | import pytest 115 | from lsm import LSM 116 | 117 | with LSM("test.ldb", binary=False) as db: 118 | for i in range(4): 119 | db[f'k{i}'] = str(i) 120 | 121 | assert 'k3' in db 122 | assert 'k4' not in db 123 | del db['k3'] 124 | 125 | with pytest.raises(KeyError): 126 | print(db['k3']) 127 | ``` 128 | 129 | By default, when you attempt to look up a key, ``lsm`` will search for an 130 | exact match. You can also search for the closest key, if the specific key you 131 | are searching for does not exist: 132 | 133 | 134 | ```python 135 | import pytest 136 | from lsm import LSM, SEEK_LE, SEEK_GE, SEEK_LEFAST 137 | 138 | 139 | with LSM("test.ldb", binary=False) as db: 140 | for i in range(4): 141 | db[f'k{i}'] = str(i) 142 | 143 | # Here we will match "k1". 144 | assert db['k1xx', SEEK_LE] == '1' 145 | 146 | # Here we will match "k1" but do not fetch a value 147 | # In this case the value will always be ``True`` or there will 148 | # be an exception if the key is not found 149 | assert db['k1xx', SEEK_LEFAST] is True 150 | 151 | with pytest.raises(KeyError): 152 | print(db['000', SEEK_LEFAST]) 153 | 154 | # Here we will match "k2". 155 | assert db['k1xx', SEEK_GE] == "2" 156 | ``` 157 | 158 | `LSM` supports other common dictionary methods such as: 159 | 160 | * `keys()` 161 | * `values()` 162 | * `items()` 163 | * `update()` 164 | 165 | 166 | ### Slices and Iteration 167 | 168 | The database can be iterated through directly, or sliced. When you are slicing 169 | the database the start and end keys need not exist -- ``lsm`` will find the 170 | closest key (details can be found in the [LSM.fetch_range()](https://lsm-db.readthedocs.io/en/latest/api.html#lsm.LSM.fetch_range) 171 | documentation). 172 | 173 | 176 | ```python 177 | from lsm import LSM 178 | 179 | with LSM("test_slices.ldb", binary=False) as db: 180 | 181 | # clean database 182 | for key in db.keys(): 183 | del db[key] 184 | 185 | db['foo'] = 'bar' 186 | 187 | for i in range(3): 188 | db[f'k{i}'] = str(i) 189 | 190 | # Can easily iterate over the database items 191 | assert ( 192 | sorted(item for item in db.items()) == [ 193 | ('foo', 'bar'), ('k0', '0'), ('k1', '1'), ('k2', '2') 194 | ] 195 | ) 196 | 197 | # However, you will not read the entire database into memory, as special 198 | # iterator objects are used. 199 | assert str(db['k0':'k99']).startswith(" 213 | ```python 214 | with LSM("test_slices.ldb", binary=False, readonly=True) as db: 215 | assert list(db['k0':]) == [('k0', '0'), ('k1', '1'), ('k2', '2')] 216 | assert list(db[:'k1']) == [('foo', 'bar'), ('k0', '0'), ('k1', '1')] 217 | assert list(db[:'aaa']) == [] 218 | ``` 219 | 220 | To retrieve keys in reverse order or stepping over more than one item, 221 | simply use a third slice argument as usual. 222 | Negative step value means reverse order, but first and second arguments 223 | must be ordinarily ordered. 224 | 225 | 229 | ```python 230 | with LSM("test_slices.ldb", binary=False, readonly=True) as db: 231 | assert list(db['k0':'k99':2]) == [('k0', '0'), ('k2', '2')] 232 | assert list(db['k0'::-1]) == [('k2', '2'), ('k1', '1'), ('k0', '0')] 233 | assert list(db['k0'::-2]) == [('k2', '2'), ('k0', '0')] 234 | assert list(db['k0'::3]) == [('k0', '0')] 235 | ``` 236 | 237 | You can also **delete** slices of keys, but note that delete **will not** 238 | include the keys themselves: 239 | 240 | 244 | ```python 245 | with LSM("test_slices.ldb", binary=False) as db: 246 | del db['k0':'k99'] 247 | 248 | # Note that 'k0' still exists. 249 | assert list(db.items()) == [('foo', 'bar'), ('k0', '0')] 250 | ``` 251 | 252 | ### Cursors 253 | 254 | While slicing may cover most use-cases, for finer-grained control you can use 255 | cursors for traversing records. 256 | 257 | 261 | ```python 262 | from lsm import LSM, SEEK_GE, SEEK_LE 263 | 264 | with LSM("test_cursors.ldb", binary=False) as db: 265 | del db["a":"z"] 266 | 267 | db["spam"] = "spam" 268 | 269 | with db.cursor() as cursor: 270 | cursor.seek('spam') 271 | key, value = cursor.retrieve() 272 | assert key == 'spam' 273 | assert value == 'spam' 274 | ``` 275 | 276 | Seeking over cursors: 277 | 278 | 282 | ```python 283 | with LSM("test_cursors.ldb", binary=False) as db: 284 | db.update({'k0': '0', 'k1': '1', 'k2': '2', 'k3': '3', 'foo': 'bar'}) 285 | 286 | with db.cursor() as cursor: 287 | 288 | cursor.first() 289 | key, value = cursor.retrieve() 290 | assert key == "foo" 291 | assert value == "bar" 292 | 293 | cursor.last() 294 | key, value = cursor.retrieve() 295 | assert key == "spam" 296 | assert value == "spam" 297 | 298 | cursor.previous() 299 | key, value = cursor.retrieve() 300 | assert key == "k3" 301 | assert value == "3" 302 | 303 | ``` 304 | 305 | Finding the first match that is greater than or equal to `'k0'` and move 306 | forward until the key is less than `'k99'` 307 | 308 | 312 | ```python 313 | with LSM("test_cursors.ldb", binary=False) as db: 314 | with db.cursor() as cursor: 315 | cursor.seek("k0", SEEK_GE) 316 | results = [] 317 | 318 | while cursor.compare("k99") > 0: 319 | key, value = cursor.retrieve() 320 | results.append((key, value)) 321 | cursor.next() 322 | 323 | assert results == [('k0', '0'), ('k1', '1'), ('k2', '2'), ('k3', '3')] 324 | 325 | ``` 326 | 327 | Finding the last match that is lower than or equal to `'k99'` and move 328 | backward until the key is less than `'k0'` 329 | 330 | 334 | ```python 335 | with LSM("test_cursors.ldb", binary=False) as db: 336 | with db.cursor() as cursor: 337 | cursor.seek("k99", SEEK_LE) 338 | results = [] 339 | 340 | while cursor.compare("k0") >= 0: 341 | key, value = cursor.retrieve() 342 | results.append((key, value)) 343 | cursor.previous() 344 | 345 | assert results == [('k3', '3'), ('k2', '2'), ('k1', '1'), ('k0', '0')] 346 | ``` 347 | 348 | It is very important to close a cursor when you are through using it. For this 349 | reason, it is recommended you use the `LSM.cursor()` context-manager, which 350 | ensures the cursor is closed properly. 351 | 352 | ### Transactions 353 | 354 | ``lsm`` supports nested transactions. The simplest way to use transactions 355 | is with the `LSM.transaction()` method, which returns a context-manager: 356 | 357 | 358 | ```python 359 | from lsm import LSM 360 | 361 | with LSM("test_tx.ldb", binary=False) as db: 362 | del db["a":"z"] 363 | for i in range(10): 364 | db[f"k{i}"] = f"{i}" 365 | 366 | 367 | with LSM("test_tx.ldb", binary=False) as db: 368 | with db.transaction() as tx1: 369 | db['k1'] = '1-mod' 370 | 371 | with db.transaction() as tx2: 372 | db['k2'] = '2-mod' 373 | tx2.rollback() 374 | 375 | assert db['k1'] == '1-mod' 376 | assert db['k2'] == '2' 377 | ``` 378 | 379 | You can commit or roll-back transactions part-way through a wrapped block: 380 | 381 | 382 | ```python 383 | from lsm import LSM 384 | 385 | with LSM("test_tx_2.ldb", binary=False) as db: 386 | del db["a":"z"] 387 | for i in range(10): 388 | db[f"k{i}"] = f"{i}" 389 | 390 | with LSM("test_tx_2.ldb", binary=False) as db: 391 | with db.transaction() as txn: 392 | db['k1'] = 'outer txn' 393 | 394 | # The write operation is preserved. 395 | txn.commit() 396 | 397 | db['k1'] = 'outer txn-2' 398 | 399 | with db.transaction() as txn2: 400 | # This is committed after the block ends. 401 | db['k1'] = 'inner-txn' 402 | 403 | assert db['k1'] == "inner-txn" 404 | 405 | # Rolls back both the changes from txn2 and the preceding write. 406 | txn.rollback() 407 | 408 | assert db['k1'] == 'outer txn', db['k1'] 409 | ``` 410 | 411 | 412 | If you like, you can also explicitly call `LSM.begin()`, `LSM.commit()`, and 413 | `LSM.rollback()`. 414 | 415 | 416 | ```python 417 | from lsm import LSM 418 | 419 | # fill db 420 | with LSM("test_db_tx.ldb", binary=False) as db: 421 | del db["k":"z"] 422 | for i in range(10): 423 | db[f"k{i}"] = f"{i}" 424 | 425 | 426 | with LSM("test_db_tx.ldb", binary=False) as db: 427 | # start transaction 428 | db.begin() 429 | db['k1'] = '1-mod' 430 | 431 | # nested transaction 432 | db.begin() 433 | db['k2'] = '2-mod' 434 | # rolling back nested transaction 435 | db.rollback() 436 | 437 | # comitting top-level transaction 438 | db.commit() 439 | 440 | assert db['k1'] == '1-mod' 441 | assert db['k2'] == '2' 442 | ``` 443 | 444 | ### Thanks to 445 | 446 | * [@coleifer](https://github.com/coleifer) - this project was inspired by 447 | [coleifer/python-lsm-db](https://github.com/coleifer/python-lsm-db). 448 | -------------------------------------------------------------------------------- /examples/benchmark.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import struct 5 | import tempfile 6 | from argparse import Action, ArgumentParser 7 | from glob import glob 8 | from multiprocessing import cpu_count 9 | from multiprocessing.pool import ThreadPool 10 | from pathlib import Path 11 | from random import shuffle 12 | from threading import RLock, local 13 | from typing import Union 14 | 15 | from mimesis import Address, Business, Datetime, Person 16 | from rich.logging import RichHandler 17 | from rich.progress import ( 18 | BarColumn, Progress, ProgressColumn, Task, TaskID, TextColumn, 19 | TimeRemainingColumn, 20 | ) 21 | from rich.text import Text 22 | 23 | import lsm 24 | 25 | 26 | class SpeedColumn(ProgressColumn): 27 | """Renders human readable transfer speed.""" 28 | 29 | def render(self, task: Task) -> Text: 30 | """Show data transfer speed.""" 31 | speed = task.finished_speed or task.speed 32 | if speed is None: 33 | return Text("?", style="progress.data.speed") 34 | return Text(f"{int(speed)}it/s", style="progress.data.speed") 35 | 36 | 37 | progress = Progress( 38 | TextColumn("[bold red]{task.description} [bold blue]{task.completed}/{task.total}", justify="right"), 39 | BarColumn(bar_width=None), 40 | "[progress.percentage]{task.percentage:>3.1f}%", 41 | "[bold blue] ETA:", 42 | TimeRemainingColumn(), 43 | SpeedColumn(), 44 | auto_refresh=True, 45 | ) 46 | 47 | progress.start() 48 | log = logging.getLogger("rich") 49 | 50 | 51 | class AppendConstAction(Action): 52 | def __init__( 53 | self, option_strings, dest, const=None, default=None, 54 | type=None, choices=None, required=False, 55 | help=None, metavar=None, 56 | ): 57 | assert const, "const= required" 58 | super().__init__( 59 | option_strings, dest, const=const, nargs=0, 60 | default=default, type=type, choices=choices, 61 | required=required, help=help, metavar=metavar, 62 | ) 63 | 64 | def __call__(self, parser, namespace, value, option_string=None): 65 | if getattr(namespace, self.dest) is None: 66 | setattr(namespace, self.dest, list()) 67 | 68 | lst = getattr(namespace, self.dest) 69 | lst.append(self.const) 70 | 71 | 72 | parser = ArgumentParser() 73 | parser.add_argument("-n", "--count", default=100000, type=int) 74 | parser.add_argument("--pool-size", type=int, default=cpu_count()) 75 | parser.add_argument( 76 | "--clear", 77 | help="Keep existent database before writing", 78 | action="store_true", 79 | ) 80 | parser.add_argument( 81 | "--path", 82 | default=os.path.join(tempfile.gettempdir(), "lsm-compressed"), 83 | ) 84 | 85 | parser.add_argument("--run-sequentially", action="store_true") 86 | 87 | group = parser.add_argument_group("cases") 88 | group.add_argument( 89 | "--case-all", 90 | dest="cases", 91 | const="all", 92 | action=AppendConstAction, 93 | ) 94 | group.add_argument( 95 | "--case-lz4", 96 | dest="cases", 97 | const="lz4", 98 | action=AppendConstAction, 99 | ) 100 | group.add_argument( 101 | "--case-zstd", 102 | dest="cases", 103 | const="zstd", 104 | action=AppendConstAction, 105 | ) 106 | group.add_argument( 107 | "--case-raw", 108 | dest="cases", 109 | const="raw", 110 | action=AppendConstAction, 111 | ) 112 | 113 | group = parser.add_argument_group("benchmarks") 114 | group.add_argument( 115 | "--bench-all", 116 | dest="benchmarks", 117 | const="all", 118 | action=AppendConstAction, 119 | ) 120 | group.add_argument( 121 | "--bench-insert", 122 | dest="benchmarks", 123 | const="insert", 124 | action=AppendConstAction, 125 | ) 126 | group.add_argument( 127 | "--bench-select-seq", 128 | dest="benchmarks", 129 | const="select-seq", 130 | action=AppendConstAction, 131 | ) 132 | group.add_argument( 133 | "--bench-select-rnd", 134 | dest="benchmarks", 135 | const="select-rnd", 136 | action=AppendConstAction, 137 | ) 138 | 139 | group.add_argument( 140 | "--bench-copy-seq", 141 | dest="benchmarks", 142 | const="copy-seq", 143 | action=AppendConstAction, 144 | ) 145 | 146 | 147 | person_generator = Person("en") 148 | address_generator = Address("en") 149 | business_generator = Business("en") 150 | datetime_generator = Datetime("en") 151 | 152 | 153 | class Cases: 154 | @classmethod 155 | def lz4(cls, path): 156 | return [ 157 | path + ".lsm.lz4", 158 | dict(multiple_processes=False, compress="lz4"), 159 | ] 160 | 161 | @classmethod 162 | def zstd(cls, path): 163 | return [ 164 | path + ".lsm.zst", 165 | dict(multiple_processes=False, compress="zstd"), 166 | ] 167 | 168 | @classmethod 169 | def raw(cls, path): 170 | return [ 171 | path + ".lsm", 172 | dict(multiple_processes=False), 173 | ] 174 | 175 | 176 | def get_key(idx) -> Union[bytes, str]: 177 | return struct.pack("I", idx) 178 | 179 | 180 | def get_value(idx) -> Union[bytes, str]: 181 | return json.dumps({ 182 | "id": idx, 183 | "person": { 184 | "full_name": person_generator.full_name(), 185 | "email": person_generator.email( 186 | domains=[ 187 | "gmail.com", 188 | "hotmail.com", 189 | "yandex.ru", 190 | "mail.ru", 191 | ], 192 | ), 193 | "phone": person_generator.telephone(mask="+7(9##)-###-####"), 194 | "avatar": person_generator.avatar(), 195 | "language": person_generator.language(), 196 | }, 197 | "gps": { 198 | "lat": address_generator.latitude(), 199 | "lon": address_generator.longitude(), 200 | }, 201 | "address": { 202 | "city": address_generator.city(), 203 | "country": address_generator.country_code(), 204 | "address": address_generator.address(), 205 | "zip": address_generator.zip_code(), 206 | "region": address_generator.region(), 207 | }, 208 | "business": { 209 | "company": business_generator.company(), 210 | "type": business_generator.company_type(), 211 | "copyright": business_generator.copyright(), 212 | "currency": business_generator.currency_symbol(), 213 | }, 214 | "registration": { 215 | "join_date": datetime_generator.datetime().isoformat(), 216 | }, 217 | }).encode() 218 | 219 | 220 | DATA_HEADER = struct.Struct("!I") 221 | 222 | 223 | def gen_data(path, n, task_id: TaskID): 224 | with open(path, "a+b") as fp: 225 | fp.seek(0) 226 | head = fp.read(DATA_HEADER.size) 227 | 228 | if len(head) == DATA_HEADER.size and DATA_HEADER.unpack(head)[0] == n: 229 | log.info("Using previously generated file. Skipping") 230 | return 231 | 232 | fp.truncate(0) 233 | fp.flush() 234 | 235 | fp.write(b"\x00" * DATA_HEADER.size) 236 | 237 | progress.start_task(task_id) 238 | track = progress.track( 239 | range(n), task_id=task_id, total=n, 240 | ) 241 | 242 | for i in track: 243 | value = get_value(i) 244 | fp.write(DATA_HEADER.pack(len(value))) 245 | fp.write(value) 246 | 247 | fp.seek(0) 248 | os.pwrite(fp.fileno(), DATA_HEADER.pack(n), 0) 249 | fp.flush() 250 | progress.stop_task(task_id) 251 | 252 | 253 | def fill_db(path, *, pool_size, data_file, **kwargs): 254 | with ThreadPool(pool_size) as pool, \ 255 | lsm.LSM(path, **kwargs) as db, \ 256 | open(data_file, "rb") as fp: 257 | 258 | n = DATA_HEADER.unpack(fp.read(DATA_HEADER.size))[0] 259 | read_lock = RLock() 260 | 261 | task_id = progress.add_task( 262 | description=( 263 | f"Fill DB " 264 | f"[bold green]compress={kwargs.get('compress', 'none')}" 265 | ), total=n, 266 | ) 267 | 268 | count = 0 269 | 270 | def insert(i): 271 | nonlocal count 272 | 273 | with read_lock: 274 | line = fp.read( 275 | DATA_HEADER.unpack(fp.read(DATA_HEADER.size))[0], 276 | ) 277 | 278 | db[get_key(i)] = line 279 | count += 1 280 | progress.update(task_id, completed=count) 281 | 282 | for _ in pool.imap_unordered(insert, range(n)): 283 | pass 284 | 285 | db.work(complete=True) 286 | 287 | 288 | def select_thread_pool( 289 | path, *, keys_iter, keys_total, pool_size, **kwargs 290 | ): 291 | tls = local() 292 | db_pool = set() 293 | 294 | log.info("Opening: %s with %r", path, kwargs) 295 | with ThreadPool(pool_size) as pool: 296 | task_id = progress.add_task( 297 | description=( 298 | f"[bold blue]Select all keys sequentially " 299 | f"[bold green]compress={kwargs.get('compress', 'none')}" 300 | ), total=keys_total, 301 | ) 302 | 303 | count = 0 304 | 305 | def select(k): 306 | nonlocal count 307 | if not hasattr(tls, "db"): 308 | tls.db = lsm.LSM(path, readonly=True, **kwargs) 309 | tls.db.open() 310 | db_pool.add(tls.db) 311 | 312 | _ = tls.db[get_key(k)] 313 | count += 1 314 | progress.update(task_id=task_id, completed=count) 315 | return 0 316 | 317 | for _ in pool.imap_unordered(select, keys_iter): 318 | pass 319 | 320 | for conn in db_pool: 321 | conn.close() 322 | 323 | 324 | def copy_seq(path, **kwargs): 325 | with tempfile.TemporaryDirectory() as dest: 326 | dest = lsm.LSM(os.path.join(dest, "lsm-copy"), **kwargs) 327 | src = lsm.LSM(path, readonly=True, **kwargs) 328 | 329 | with src, dest: 330 | total_keys = len(src.keys()) 331 | 332 | task_id = progress.add_task( 333 | description=f"Copy [bold green]{kwargs.get('compress', 'none')}", 334 | ) 335 | track = progress.track( 336 | src.items(), task_id=task_id, total=total_keys, 337 | ) 338 | 339 | for key, value in track: 340 | dest[key] = value 341 | 342 | 343 | def run_parallel(func, cases): 344 | with ThreadPool(len(cases)) as pool: 345 | for _ in pool.imap_unordered(func, cases): 346 | pass 347 | 348 | 349 | def run_sequentially(func, cases): 350 | for case in cases: 351 | func(case) 352 | 353 | 354 | def main(): 355 | arguments = parser.parse_args() 356 | 357 | run_insert = False 358 | run_select_seq = False 359 | run_select_rnd = False 360 | run_copy_seq = False 361 | 362 | if not arguments.benchmarks: 363 | run_insert = True 364 | run_select_seq = True 365 | run_select_rnd = True 366 | run_copy_seq = True 367 | else: 368 | if "insert" in arguments.benchmarks: 369 | run_insert = True 370 | if "select-seq" in arguments.benchmarks: 371 | run_select_seq = True 372 | if "select-rnd" in arguments.benchmarks: 373 | run_select_rnd = True 374 | if "copy-seq" in arguments.benchmarks: 375 | run_copy_seq = True 376 | 377 | if "all" in arguments.benchmarks: 378 | run_insert = True 379 | run_select_seq = True 380 | run_select_rnd = True 381 | run_copy_seq = True 382 | 383 | if not arguments.cases or "all" in arguments.cases: 384 | cases = [ 385 | Cases.zstd(arguments.path), 386 | Cases.lz4(arguments.path), 387 | Cases.raw(arguments.path), 388 | ] 389 | else: 390 | cases = [] 391 | if "zstd" in arguments.cases: 392 | cases.append(Cases.zstd(arguments.path)) 393 | if "lz4" in arguments.cases: 394 | cases.append(Cases.lz4(arguments.path)) 395 | if "raw" in arguments.cases: 396 | cases.append(Cases.raw(arguments.path)) 397 | 398 | if arguments.run_sequentially: 399 | run = run_sequentially 400 | else: 401 | run = run_parallel 402 | 403 | if arguments.clear: 404 | for file_name in glob(arguments.path + ".*"): 405 | log.info("Removing: %r", file_name) 406 | os.remove(file_name) 407 | 408 | data_path = Path(arguments.path).parent / "data.json" 409 | 410 | def fill_job(item): 411 | path, kwargs = item 412 | return fill_db( 413 | path, 414 | pool_size=arguments.pool_size, 415 | data_file=data_path, 416 | **kwargs 417 | ) 418 | 419 | def select_job(item): 420 | path, kwargs = item 421 | return select_thread_pool( 422 | path, 423 | pool_size=arguments.pool_size, 424 | keys_iter=range(arguments.count), 425 | keys_total=arguments.count, 426 | **kwargs 427 | ) 428 | 429 | def select_random_job(item): 430 | 431 | path, kwargs = item 432 | keys = list(range(arguments.count)) 433 | shuffle(keys) 434 | 435 | return select_thread_pool( 436 | path, 437 | pool_size=arguments.pool_size, 438 | keys_iter=iter(keys), 439 | keys_total=arguments.count, 440 | **kwargs 441 | ) 442 | 443 | def copy_seq_job(item): 444 | path, kwargs = item 445 | return copy_seq(path, **kwargs) 446 | 447 | with progress: 448 | if run_insert: 449 | gen_data( 450 | data_path, arguments.count, progress.add_task( 451 | description="Generate data", 452 | ), 453 | ) 454 | 455 | run(fill_job, cases) 456 | 457 | if run_select_seq: 458 | log.info("Select all keys sequentially") 459 | run(select_job, cases) 460 | 461 | if run_select_rnd: 462 | log.info("Select all keys random") 463 | run(select_random_job, cases) 464 | 465 | if run_copy_seq: 466 | log.info("Copy database") 467 | run(copy_seq_job, cases) 468 | 469 | 470 | if __name__ == "__main__": 471 | logging.basicConfig( 472 | level="NOTSET", 473 | format="%(message)s", 474 | datefmt="[%X]", 475 | ) 476 | logging.getLogger().handlers.clear() 477 | logging.getLogger().handlers.append( 478 | RichHandler( 479 | rich_tracebacks=True, 480 | tracebacks_show_locals=True 481 | ) 482 | ) 483 | 484 | main() 485 | -------------------------------------------------------------------------------- /examples/reader.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import hashlib 3 | import logging 4 | import sys 5 | from pathlib import Path 6 | 7 | import aiomisc 8 | from aiomisc.service import MemoryTracer 9 | from lsm import LSM 10 | 11 | 12 | @aiomisc.threaded 13 | def seq_reader(fname: Path): 14 | hasher = hashlib.md5() 15 | 16 | logging.info("Start reading %s", fname) 17 | with LSM(fname, binary=True, readonly=True) as db: 18 | for key, value in db.items(): 19 | hasher.update(key) 20 | hasher.update(value) 21 | 22 | logging.info("DIGEST: %s", hasher.hexdigest()) 23 | logging.info("Reading done for %s", fname) 24 | 25 | 26 | async def main(): 27 | await asyncio.gather( 28 | *[seq_reader(Path(sys.argv[1])) for _ in range(2)] 29 | ) 30 | 31 | 32 | aiomisc.run( 33 | main(), 34 | MemoryTracer(interval=2, ), 35 | pool_size=8, 36 | log_format="rich_tb", 37 | log_level="info", 38 | ) 39 | -------------------------------------------------------------------------------- /lsm/__init__.py: -------------------------------------------------------------------------------- 1 | from lsm._lsm import * 2 | -------------------------------------------------------------------------------- /lsm/_lsm.pyi: -------------------------------------------------------------------------------- 1 | from typing import ( 2 | Any, Callable, Dict, Union, Optional, Tuple, 3 | KeysView, ValuesView, ItemsView, Mapping 4 | ) 5 | 6 | SAFETY_OFF: int 7 | SAFETY_NORMAL: int 8 | SAFETY_FULL: int 9 | 10 | STATE_INITIALIZED: int 11 | STATE_OPENED: int 12 | STATE_CLOSED: int 13 | 14 | SEEK_EQ: int 15 | SEEK_LE: int 16 | SEEK_GE: int 17 | SEEK_LEFAST: int 18 | 19 | 20 | class Transaction: 21 | level: int 22 | def __enter__(self) -> "Transaction": ... 23 | def __exit__(self, exc_type, exc_val, exc_tb) -> None: ... 24 | def commit(self) -> bool: ... 25 | def rollback(self) -> bool: ... 26 | 27 | 28 | class Cursor: 29 | state: int 30 | seek_mode: int 31 | 32 | def close(self) -> None:... 33 | def first(self) -> None: ... 34 | def last(self) -> None: ... 35 | def seek( 36 | self, key: Union[bytes, str], seek_mode: int = SEEK_EQ 37 | ) -> bool: ... 38 | def retrieve(self) -> Optional[ 39 | Tuple[Union[bytes, str], Union[bytes, str]] 40 | ]: ... 41 | def next(self) -> bool: ... 42 | def previous(self) -> bool: ... 43 | def compare(self, key: Union[bytes, str]) -> int: ... 44 | def __enter__(self) -> "Cursor": ... 45 | def __exit__(self, exc_type, exc_val, exc_tb) -> None: ... 46 | 47 | 48 | class LSM: 49 | path: str 50 | compressed: bool 51 | state: int 52 | page_size: int 53 | block_size: int 54 | safety: int 55 | autowork: int 56 | autocheckpoint: int 57 | mmap: bool 58 | use_log: bool 59 | automerge: int 60 | max_freelist: int 61 | multiple_processes: bool 62 | readonly: bool 63 | compress: str 64 | compress_level: int 65 | tx_level: int 66 | 67 | def __init__( 68 | self, path: Any, *, 69 | autoflush: int = 1024, 70 | page_size: int = 4096, 71 | safety: int = SAFETY_NORMAL, 72 | block_size: int = 1024, 73 | automerge: int = 4, 74 | max_freelist: int = 24, 75 | autocheckpoint: int = 2048, 76 | autowork: bool = True, 77 | mmap: bool = True, 78 | use_log: bool = True, 79 | multiple_processes: bool = True, 80 | readonly: bool = False, 81 | binary: bool = True, 82 | logger: Callable[[str, int], Any] = None, 83 | compress: str = None, 84 | compress_level: int = None, 85 | ): ... 86 | 87 | def open(self) -> bool: ... 88 | def close(self) -> bool: ... 89 | def info(self) -> Dict[str, int]: ... 90 | def work( 91 | self, *, nmerge: int = 4, nkb: int = 1024, complete: bool = True 92 | ) -> int: ... 93 | def flush(self) -> bool: ... 94 | def __enter__(self) -> "LSM": ... 95 | def __exit__(self, exc_type, exc_val, exc_tb) -> None: ... 96 | def checkpoint(self) -> int: ... 97 | def cursor(self, seek_mode: int = SEEK_GE) -> Cursor: ... 98 | def insert( 99 | self, key: Union[bytes, str], value: Union[bytes, str] 100 | ) -> None: ... 101 | def delete(self, key: Union[bytes, str]) -> None: ... 102 | def delete_range( 103 | self, start: Union[bytes, str], end: Union[bytes, str] 104 | ) -> None: ... 105 | def begin(self) -> bool: ... 106 | def commit(self) -> bool: ... 107 | def rollback(self) -> bool: ... 108 | def __getitem__( 109 | self, item: Union[bytes, str, slice] 110 | ) -> Union[ 111 | Union[bytes, str, bool], 112 | ItemsView[Union[bytes, str], Union[bytes, str]] 113 | ]: ... 114 | def __delitem__(self, key: Union[bytes, str]): ... 115 | def __setitem__(self, key: Union[bytes, str], value: Union[bytes, str]): ... 116 | def keys(self) -> KeysView[Union[bytes, str]]: ... 117 | def values(self) -> ValuesView[Union[bytes, str]]: ... 118 | def items(self) -> ItemsView[Union[bytes, str], Union[bytes, str]]: ... 119 | def update( 120 | self, value: Mapping[Union[bytes, str], Union[bytes, str]] 121 | ) -> None: ... 122 | def transaction(self) -> Transaction: ... 123 | def tx(self) -> Transaction: ... 124 | def __len__(self) -> int: ... 125 | -------------------------------------------------------------------------------- /lsm/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mosquito/python-lsm/c97ebb31897494ed0c977d3b18b6a500f3f92c5c/lsm/py.typed -------------------------------------------------------------------------------- /lsm_tool.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | from argparse import ArgumentParser 4 | from pathlib import Path 5 | 6 | from lsm import LSM, SAFETY_OFF 7 | 8 | 9 | def main(): 10 | parser = ArgumentParser( 11 | description=( 12 | "This tool is useful for various manipulations on LSM databases. " 13 | "It can apply one of page compression algorithms " 14 | "(--compress=lz4, --compress=zstd) or copies the database without " 15 | "compression (--compress=none). You can also merge one or more " 16 | "databases into one file if the destination file exists, or " 17 | "simply copy the database element by element. You can also use " 18 | "it to check the integrity of a database." 19 | ) 20 | ) 21 | parser.add_argument( 22 | "source", type=Path, help="Source LSM database file path" 23 | ) 24 | parser.add_argument( 25 | "dest", type=Path, help="Destination LSM database file path" 26 | ) 27 | parser.add_argument( 28 | "-c", "--compress", choices=["none", "zstd", "lz4"], 29 | help="Page compression algorithm" 30 | ) 31 | parser.add_argument( 32 | "-L", "--compress-level", choices=list(range(1, 10)), type=int, 33 | default=6, help="Page compression level" 34 | ) 35 | parser.add_argument( 36 | "-R", "--replace", action="store_true", 37 | help="Replace existent database file, " 38 | "otherwise merge keys and values from the source" 39 | ) 40 | parser.add_argument( 41 | "--dest-page-size", choices=list(str(1024 << i) for i in range(7)), 42 | help="Destination file page_size" 43 | ) 44 | 45 | arguments = parser.parse_args() 46 | 47 | if arguments.replace and arguments.dest.exists(): 48 | sys.stderr.write( 49 | f"Removing {arguments.dest} because replace flag passed\n" 50 | ) 51 | arguments.dest.unlink() 52 | 53 | dest = LSM( 54 | path=arguments.dest, 55 | compress=arguments.compress, 56 | compress_level=arguments.compress_level, 57 | multiple_processes=False, 58 | automerge=8, 59 | use_log=False, 60 | safety=SAFETY_OFF 61 | ) 62 | 63 | prompt = f"\rCopying {arguments.source} -> {arguments.dest}: " 64 | 65 | def log_progress(idx): 66 | sys.stderr.write(prompt) 67 | sys.stderr.write(f"{idx:>10}") 68 | sys.stderr.flush() 69 | 70 | with LSM(arguments.source, readonly=True) as src, dest: 71 | idx = 0 72 | log_progress(idx) 73 | started_at = time.monotonic() 74 | 75 | for idx, (key, value) in enumerate(src.items()): 76 | dest[key] = value 77 | if idx % 1024 == 0: 78 | log_progress(idx) 79 | 80 | log_progress(idx) 81 | finished_at = time.monotonic() 82 | 83 | sys.stdout.write("\n") 84 | sys.stdout.write( 85 | f"Copied {idx} items in {finished_at - started_at} seconds\n" 86 | ) 87 | 88 | 89 | if __name__ == '__main__': 90 | main() 91 | -------------------------------------------------------------------------------- /pylava.ini: -------------------------------------------------------------------------------- 1 | [pylava] 2 | ignore=C901,E252,W0401 3 | skip = .env*,.tox*,*build*,*/src/* 4 | 5 | [pylava:pycodestyle] 6 | max_line_length = 80 7 | -------------------------------------------------------------------------------- /scripts/make-wheels.sh: -------------------------------------------------------------------------------- 1 | set -ex 2 | 3 | mkdir -p dist 4 | 5 | function build_wheel() { 6 | /opt/python/$1/bin/pip install setuptools wheel 7 | /opt/python/$1/bin/pip wheel . -f . -w dist 8 | } 9 | 10 | build_wheel cp39-cp39 11 | build_wheel cp310-cp310 12 | build_wheel cp311-cp311 13 | build_wheel cp312-cp312 14 | build_wheel cp313-cp313 15 | 16 | cd dist 17 | for f in ./*linux_*; 18 | do if [ -f $f ]; then auditwheel repair $f -w . ; rm $f; fi; 19 | done 20 | cd - 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import platform 3 | 4 | from setuptools import Extension, setup 5 | 6 | 7 | module_name = "lsm" 8 | 9 | 10 | define_macros = [] 11 | compiller_args = [] 12 | libraries = [] 13 | 14 | 15 | if platform.system() in ("Darwin", "Linux"): 16 | define_macros.append(('LSM_MUTEX_PTHREADS', None)) 17 | compiller_args += ( 18 | "-g3", 19 | "-std=c99", 20 | "-O0", 21 | "-fPIC", 22 | "-Wall", 23 | "-ftrapv", 24 | "-fwrapv", 25 | ) 26 | libraries.append("pthread") 27 | 28 | 29 | if platform.system() in ("Windows",): 30 | define_macros.append(('LSM_MUTEX_WIN32', None)) 31 | 32 | 33 | sources = { 34 | "sqlite/ext/lsm1": [ 35 | "lsm_main.c", 36 | "lsm_win32.c", 37 | "lsm_file.c", 38 | "lsm_tree.c", 39 | "lsm_log.c", 40 | "lsm_ckpt.c", 41 | "lsm_mutex.c", 42 | "lsm_mem.c", 43 | "lsm_str.c", 44 | "lsm_unix.c", 45 | "lsm_varint.c", 46 | "lsm_shared.c", 47 | "lsm_sorted.c", 48 | ], 49 | "lz4/lib": ["lz4.c",], 50 | "zstd/lib": [ 51 | "compress/zstd_compress.c", 52 | "compress/zstd_compress_literals.c", 53 | "compress/zstd_compress_sequences.c", 54 | "compress/zstd_compress_superblock.c", 55 | "compress/zstdmt_compress.c", 56 | "compress/zstd_fast.c", 57 | "compress/zstd_double_fast.c", 58 | "compress/zstd_lazy.c", 59 | "compress/zstd_opt.c", 60 | "compress/zstd_ldm.c", 61 | "compress/fse_compress.c", 62 | "compress/huf_compress.c", 63 | "compress/hist.c", 64 | "common/fse_decompress.c", 65 | "decompress/zstd_decompress.c", 66 | "decompress/zstd_decompress_block.c", 67 | "decompress/zstd_ddict.c", 68 | "decompress/huf_decompress.c", 69 | "common/entropy_common.c", 70 | "common/zstd_common.c", 71 | "common/xxhash.c", 72 | "common/error_private.c", 73 | "common/pool.c", 74 | "common/threading.c", 75 | ], 76 | "": ["_lsm.c"] 77 | } 78 | 79 | 80 | def library_sources(): 81 | result = [] 82 | for parent_dir, files in sources.items(): 83 | result += [os.path.join("src", parent_dir, f) for f in files] 84 | return result 85 | 86 | 87 | setup( 88 | name=module_name, 89 | version="0.5.10", 90 | ext_modules=[ 91 | Extension( 92 | "lsm._lsm", 93 | library_sources(), 94 | include_dirs=[ 95 | "src/sqlite/ext/lsm1", 96 | "src/zstd/lib", 97 | "src/lz4/lib", 98 | ], 99 | undef_macros=["NDEBUG"], 100 | define_macros=define_macros, 101 | libraries=libraries, 102 | extra_compile_args=compiller_args, 103 | ), 104 | ], 105 | include_package_data=True, 106 | py_modules=['lsm_tool'], 107 | description="Python bindings for SQLite's LSM key/value engine", 108 | long_description=open("README.md").read(), 109 | long_description_content_type='text/markdown', 110 | license="Apache Software License", 111 | author="Dmitry Orlov", 112 | author_email="me@mosquito.su", 113 | url="https://github.com/mosquito/python-lsm/", 114 | project_urls={ 115 | "Documentation": "https://github.com/mosquito/python-lsm/", 116 | "Source": "https://github.com/mosquito/python-lsm/", 117 | "Tracker": "https://github.com/mosquito/python-lsm/issues", 118 | "Say Thanks!": "https://saythanks.io/to/mosquito", 119 | }, 120 | packages=["lsm"], 121 | package_data={"lsm": ["py.typed", "_lsm.pyi"]}, 122 | classifiers=[ 123 | "Intended Audience :: Developers", 124 | "License :: OSI Approved :: Apache Software License", 125 | "Natural Language :: English", 126 | "Operating System :: MacOS", 127 | "Operating System :: Microsoft", 128 | "Operating System :: POSIX", 129 | "Programming Language :: Python :: 3", 130 | "Programming Language :: Python :: 3.9", 131 | "Programming Language :: Python :: 3.10", 132 | "Programming Language :: Python :: 3.11", 133 | "Programming Language :: Python :: 3.12", 134 | "Programming Language :: Python :: 3.13", 135 | "Programming Language :: Python :: Implementation :: CPython", 136 | "Programming Language :: Python", 137 | "Topic :: Database :: Database Engines/Servers", 138 | "Topic :: Database", 139 | "Topic :: Software Development :: Libraries", 140 | "Topic :: Software Development", 141 | "Typing :: Typed", 142 | ], 143 | entry_points={ 144 | "console_scripts": [ 145 | "lsm-tool = lsm_tool:main" 146 | ] 147 | }, 148 | python_requires=">=3.9,<4", 149 | extras_require={ 150 | "develop": [ 151 | "pytest", 152 | "pytest-subtests", 153 | "markdown-pytest>=0.3.0", 154 | ], 155 | }, 156 | ) 157 | -------------------------------------------------------------------------------- /src/_lsm.c: -------------------------------------------------------------------------------- 1 | #define PY_SSIZE_T_CLEAN 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "lz4/lib/lz4.h" 9 | #include "zstd/lib/zstd.h" 10 | #include "sqlite/ext/lsm1/lsm.h" 11 | 12 | #define IS_64_BIT (sizeof(void*)==8) 13 | #define LSM_MAX_AUTOFLUSH 1048576 14 | #define PYLSM_DEFAULT_COMPRESS_LEVEL -65535 15 | 16 | #define LZ4_COMP_LEVEL_DEFAULT 16 17 | #define LZ4_COMP_LEVEL_MAX 16 18 | 19 | typedef struct { 20 | PyObject_HEAD 21 | char *path; 22 | lsm_db *lsm; 23 | int state; 24 | int compressed; 25 | unsigned int compressor_id; 26 | int autoflush; 27 | int page_size; 28 | int block_size; 29 | int safety; 30 | int autowork; 31 | int mmap; 32 | int use_log; 33 | int automerge; 34 | int max_freelist; 35 | int multiple_processes; 36 | int autocheckpoint; 37 | int readonly; 38 | int tx_level; 39 | int compress_level; 40 | char binary; 41 | PyObject *logger; 42 | lsm_compress lsm_compress; 43 | lsm_env *lsm_env; 44 | lsm_mutex *lsm_mutex; 45 | PyObject* weakrefs; 46 | } LSM; 47 | 48 | 49 | typedef struct { 50 | PyObject_HEAD 51 | uint8_t state; 52 | lsm_cursor* cursor; 53 | LSM* db; 54 | int seek_mode; 55 | PyObject* weakrefs; 56 | } LSMCursor; 57 | 58 | 59 | typedef struct { 60 | PyObject_HEAD 61 | LSM *db; 62 | uint8_t state; 63 | lsm_cursor *cursor; 64 | PyObject* weakrefs; 65 | } LSMIterView; 66 | 67 | 68 | typedef struct { 69 | PyObject_HEAD 70 | LSM *db; 71 | lsm_cursor *cursor; 72 | 73 | PyObject *start; 74 | char* pStart; 75 | Py_ssize_t nStart; 76 | 77 | PyObject *stop; 78 | char* pStop; 79 | Py_ssize_t nStop; 80 | 81 | int state; 82 | 83 | long step; 84 | char direction; 85 | 86 | Py_ssize_t counter; 87 | PyObject* weakrefs; 88 | } LSMSliceView; 89 | 90 | 91 | typedef struct { 92 | PyObject_HEAD 93 | LSM *db; 94 | int tx_level; 95 | int state; 96 | PyObject* weakrefs; 97 | } LSMTransaction; 98 | 99 | 100 | static PyTypeObject LSMType; 101 | static PyTypeObject LSMCursorType; 102 | static PyTypeObject LSMKeysType; 103 | static PyTypeObject LSMValuesType; 104 | static PyTypeObject LSMItemsType; 105 | static PyTypeObject LSMSliceType; 106 | static PyTypeObject LSMTransactionType; 107 | 108 | 109 | static PyObject* LSMCursor_new(PyTypeObject*, LSM*, int); 110 | static PyObject* LSMTransaction_new(PyTypeObject *type, LSM*); 111 | 112 | 113 | enum { 114 | PY_LSM_INITIALIZED = 0, 115 | PY_LSM_OPENED = 1, 116 | PY_LSM_CLOSED = 2, 117 | PY_LSM_ITERATING = 3 118 | }; 119 | 120 | enum { 121 | PY_LSM_SLICE_FORWARD = 0, 122 | PY_LSM_SLICE_BACKWARD = 1 123 | }; 124 | 125 | enum { 126 | PY_LSM_COMPRESSOR_EMPTY = LSM_COMPRESSION_EMPTY, 127 | PY_LSM_COMPRESSOR_NONE = LSM_COMPRESSION_NONE, 128 | PY_LSM_COMPRESSOR_LZ4 = 1024, 129 | PY_LSM_COMPRESSOR_ZSTD = 2048, 130 | }; 131 | 132 | 133 | static int pylsm_error(int rc) { 134 | switch (rc) { 135 | case LSM_OK: 136 | break; 137 | case LSM_ERROR: 138 | PyErr_SetString(PyExc_RuntimeError, "Error occurred"); 139 | break; 140 | case LSM_BUSY: 141 | PyErr_SetString(PyExc_RuntimeError, "Busy"); 142 | break; 143 | case LSM_NOMEM: 144 | PyErr_SetString(PyExc_MemoryError, "LSM memory error"); 145 | break; 146 | case LSM_READONLY: 147 | PyErr_SetString(PyExc_PermissionError, "Read only"); 148 | break; 149 | case LSM_IOERR: 150 | PyErr_SetString(PyExc_OSError, "IO error"); 151 | break; 152 | case LSM_CORRUPT: 153 | PyErr_SetString(PyExc_RuntimeError, "Corrupted"); 154 | break; 155 | case LSM_FULL: 156 | PyErr_SetString(PyExc_RuntimeError, "Full"); 157 | break; 158 | case LSM_CANTOPEN: 159 | PyErr_SetString(PyExc_FileNotFoundError, "Can not open"); 160 | break; 161 | case LSM_PROTOCOL: 162 | PyErr_SetString(PyExc_FileNotFoundError, "Protocol error"); 163 | break; 164 | case LSM_MISUSE: 165 | PyErr_SetString(PyExc_RuntimeError, "Misuse"); 166 | break; 167 | case LSM_MISMATCH: 168 | PyErr_SetString(PyExc_RuntimeError, "Mismatch"); 169 | break; 170 | case LSM_IOERR_NOENT: 171 | PyErr_SetString(PyExc_SystemError, "NOENT"); 172 | break; 173 | default: 174 | PyErr_Format(PyExc_RuntimeError, "Unhandled error: %d", rc); 175 | break; 176 | } 177 | 178 | return rc; 179 | } 180 | 181 | 182 | static int LSM_MutexLock(LSM* self) { 183 | self->lsm_env->xMutexEnter(self->lsm_mutex); 184 | return LSM_OK; 185 | } 186 | 187 | 188 | static int LSM_MutexLeave(LSM* self) { 189 | self->lsm_env->xMutexLeave(self->lsm_mutex); 190 | return LSM_OK; 191 | } 192 | 193 | 194 | static int pylsm_lz4_xBound(LSM* self, int nIn) { 195 | int rc = LZ4_compressBound(nIn); 196 | assert(rc > 0); 197 | return rc; 198 | } 199 | 200 | 201 | static int pylsm_lz4_xCompress(LSM* self, char *pOut, int *pnOut, const char *pIn, int nIn) { 202 | int acceleration = (2 << (15 - self->compress_level)) + 1; 203 | int rc = LZ4_compress_fast((const char*)pIn, pOut, nIn, *pnOut, acceleration); 204 | assert(rc > 0); 205 | *pnOut = rc; 206 | return LSM_OK; 207 | } 208 | 209 | 210 | static int pylsm_lz4_xUncompress(LSM* self, char *pOut, int *pnOut, const char *pIn, int nIn) { 211 | int rc = LZ4_decompress_safe((const char*)pIn, (char*)pOut, nIn, *pnOut); 212 | assert(rc > 0); 213 | *pnOut = rc; 214 | return LSM_OK; 215 | } 216 | 217 | 218 | static size_t pylsm_zstd_xBound(LSM* self, int nIn) { 219 | return ZSTD_compressBound(nIn); 220 | } 221 | 222 | 223 | static size_t pylsm_zstd_xCompress(LSM* self, char *pOut, Py_ssize_t *pnOut, const char *pIn, int nIn) { 224 | size_t rc = ZSTD_compress(pOut, *pnOut, pIn, nIn, self->compress_level); 225 | 226 | assert(!ZSTD_isError(rc)); 227 | 228 | *pnOut = rc; 229 | return LSM_OK; 230 | } 231 | 232 | 233 | static int pylsm_zstd_xUncompress(LSM* self, char *pOut, Py_ssize_t *pnOut, const char *pIn, int nIn) { 234 | Py_ssize_t rc = ZSTD_decompress((char*)pOut, *pnOut, (const char*)pIn, nIn); 235 | assert(!ZSTD_isError(rc)); 236 | *pnOut = rc; 237 | return 0; 238 | } 239 | 240 | 241 | static uint32_t is_power_of_two(uint32_t n) { 242 | if (n==0) return 0; 243 | return (ceil(log2(n)) == floor(log2(n))); 244 | } 245 | 246 | 247 | static void pylsm_logger(LSM* self, int rc, const char * message) { 248 | if (self->logger == NULL) return; 249 | 250 | PyGILState_STATE state = PyGILState_Ensure(); 251 | PyObject_CallFunction(self->logger, "sI", message, rc); 252 | if (PyErr_Occurred()) PyErr_Print(); 253 | PyGILState_Release(state); 254 | } 255 | 256 | 257 | static int pylsm_seek_mode_check(int seek_mode) { 258 | switch (seek_mode) { 259 | case LSM_SEEK_EQ: 260 | return 0; 261 | case LSM_SEEK_LE: 262 | return 0; 263 | case LSM_SEEK_GE: 264 | return 0; 265 | case LSM_SEEK_LEFAST: 266 | return 0; 267 | default: 268 | PyErr_Format( 269 | PyExc_ValueError, 270 | "\"seek_mode\" should be one of SEEK_LEFAST (%d), SEEK_LE (%d), SEEK_EQ(%d) or SEEK_GE (%d) not %d", 271 | LSM_SEEK_LEFAST, LSM_SEEK_LE, LSM_SEEK_EQ, LSM_SEEK_GE, seek_mode 272 | ); 273 | return -1; 274 | } 275 | } 276 | 277 | 278 | static Py_ssize_t pylsm_csr_length(lsm_cursor* cursor, Py_ssize_t *result) { 279 | Py_ssize_t counter = 0; 280 | int rc = 0; 281 | 282 | if ((rc = lsm_csr_first(cursor))) return rc; 283 | 284 | while (lsm_csr_valid(cursor)) { 285 | counter++; 286 | if ((rc = lsm_csr_next(cursor))) break; 287 | } 288 | 289 | *result = counter; 290 | return rc; 291 | } 292 | 293 | 294 | static Py_ssize_t pylsm_length(lsm_db* lsm, Py_ssize_t *result) { 295 | Py_ssize_t rc = 0; 296 | lsm_cursor *cursor; 297 | 298 | if ((rc = lsm_csr_open(lsm, &cursor))) return rc; 299 | rc = pylsm_csr_length(cursor, result); 300 | lsm_csr_close(cursor); 301 | return rc; 302 | } 303 | 304 | 305 | static int pylsm_getitem( 306 | lsm_db* lsm, 307 | const char * pKey, 308 | int nKey, 309 | char** ppVal, 310 | int* pnVal, 311 | int seek_mode 312 | ) { 313 | int rc; 314 | lsm_cursor *cursor; 315 | char* pValue = NULL; 316 | int nValue = 0; 317 | char* result = NULL; 318 | 319 | if ((rc = lsm_csr_open(lsm, &cursor))) return rc; 320 | if ((rc = lsm_csr_seek(cursor, pKey, nKey, seek_mode))) { 321 | lsm_csr_close(cursor); 322 | return rc; 323 | } 324 | if (!lsm_csr_valid(cursor)) { 325 | lsm_csr_close(cursor); 326 | return -1; 327 | } 328 | 329 | if (seek_mode == LSM_SEEK_LEFAST) { 330 | *pnVal = 0; 331 | lsm_csr_close(cursor); 332 | return rc; 333 | } 334 | 335 | if ((rc = lsm_csr_value(cursor, (const void **)&pValue, &nValue))) { 336 | lsm_csr_close(cursor); 337 | return rc; 338 | } 339 | 340 | result = calloc(nValue, sizeof(char)); 341 | memcpy(result, pValue, nValue); 342 | lsm_csr_close(cursor); 343 | 344 | *ppVal = result; 345 | *pnVal = nValue; 346 | return 0; 347 | } 348 | 349 | 350 | static int pylsm_delitem( 351 | lsm_db* lsm, 352 | const char * pKey, 353 | int nKey 354 | ) { 355 | int rc = 0; 356 | lsm_cursor *cursor; 357 | 358 | if ((rc = lsm_csr_open(lsm, &cursor))) return rc; 359 | if ((rc = lsm_csr_seek(cursor, pKey, nKey, LSM_SEEK_EQ))) { 360 | lsm_csr_close(cursor); 361 | return rc; 362 | } 363 | if (!lsm_csr_valid(cursor)) { 364 | lsm_csr_close(cursor); 365 | return -1; 366 | } 367 | lsm_csr_close(cursor); 368 | if ((rc = lsm_delete(lsm, pKey, nKey))) return rc; 369 | return 0; 370 | } 371 | 372 | 373 | static int pylsm_contains(lsm_db* lsm, const char* pKey, int nKey) { 374 | int rc; 375 | lsm_cursor *cursor; 376 | 377 | if ((rc = lsm_csr_open(lsm, &cursor))) return rc; 378 | if ((rc = lsm_csr_seek(cursor, pKey, nKey, LSM_SEEK_EQ))) { 379 | lsm_csr_close(cursor); 380 | return rc; 381 | } 382 | 383 | if (!lsm_csr_valid(cursor)) { rc = -1; } else { rc = 0; } 384 | lsm_csr_close(cursor); 385 | return rc; 386 | } 387 | 388 | 389 | static int pylsm_ensure_opened(LSM* self) { 390 | if (self == NULL) { 391 | PyErr_SetString(PyExc_MemoryError, "Instance deallocated"); 392 | return -1; 393 | } 394 | if (self->state == PY_LSM_OPENED) return 0; 395 | 396 | PyErr_SetString(PyExc_RuntimeError, "Database has not opened"); 397 | return -1; 398 | } 399 | 400 | static int pylsm_ensure_writable(LSM* self) { 401 | if (pylsm_ensure_opened(self)) return -1; 402 | if (self->readonly) return pylsm_error(LSM_READONLY); 403 | return 0; 404 | } 405 | 406 | static int pylsm_ensure_csr_opened(LSMCursor* self) { 407 | if (pylsm_ensure_opened(self->db)) return 0; 408 | 409 | switch (self->state) { 410 | case PY_LSM_OPENED: 411 | case PY_LSM_ITERATING: 412 | if (!lsm_csr_valid(self->cursor)) { 413 | PyErr_SetString(PyExc_RuntimeError, "Invalid cursor"); 414 | return -1; 415 | } 416 | return 0; 417 | default: 418 | PyErr_SetString(PyExc_RuntimeError, "Cursor closed"); 419 | return -1; 420 | } 421 | } 422 | 423 | 424 | int pylsm_slice_first(LSMSliceView* self) { 425 | int rc; 426 | int cmp_res; 427 | 428 | if (self->pStop != NULL) { 429 | if ((rc = lsm_csr_cmp(self->cursor, self->pStop, (int) self->nStop, &cmp_res))) return rc; 430 | if (self->direction == PY_LSM_SLICE_FORWARD && cmp_res > 0) return -1; 431 | if (self->direction == PY_LSM_SLICE_BACKWARD && cmp_res < 0) return -1; 432 | } 433 | 434 | if (!lsm_csr_valid(self->cursor)) return -1; 435 | 436 | return 0; 437 | } 438 | 439 | 440 | int pylsm_slice_next(LSMSliceView* self) { 441 | int rc; 442 | int cmp_res = -65535; 443 | 444 | while (lsm_csr_valid(self->cursor)) { 445 | switch (self->direction) { 446 | case PY_LSM_SLICE_FORWARD: 447 | if ((rc = lsm_csr_next(self->cursor))) return rc; 448 | break; 449 | case PY_LSM_SLICE_BACKWARD: 450 | if ((rc = lsm_csr_prev(self->cursor))) return rc; 451 | break; 452 | } 453 | 454 | if (!lsm_csr_valid(self->cursor)) break; 455 | 456 | if (self->pStop != NULL) { 457 | if ((rc = lsm_csr_cmp(self->cursor, self->pStop, (int) self->nStop, &cmp_res))) return rc; 458 | if (self->direction == PY_LSM_SLICE_FORWARD && cmp_res > 0) break; 459 | if (self->direction == PY_LSM_SLICE_BACKWARD && cmp_res < 0) break; 460 | } 461 | 462 | self->counter++; 463 | if ((self->counter % self->step) == 0) return 0; 464 | } 465 | 466 | return -1; 467 | } 468 | 469 | 470 | static inline int pylsm_seek_mode_direction(int direction) { 471 | return (direction == PY_LSM_SLICE_FORWARD) ? LSM_SEEK_GE : LSM_SEEK_LE; 472 | } 473 | 474 | 475 | static int pylsm_slice_view_iter(LSMSliceView *self) { 476 | int rc; 477 | 478 | if ((rc = lsm_csr_open(self->db->lsm, &self->cursor))) return rc; 479 | 480 | int seek_mode = pylsm_seek_mode_direction(self->direction); 481 | 482 | if (self->pStart != NULL) { 483 | if ((rc = lsm_csr_seek(self->cursor, self->pStart, (int) self->nStart, seek_mode))) return rc; 484 | } else { 485 | switch (self->direction) { 486 | case PY_LSM_SLICE_FORWARD: 487 | if ((rc = lsm_csr_first(self->cursor))) return rc; 488 | break; 489 | case PY_LSM_SLICE_BACKWARD: 490 | if ((rc = lsm_csr_last(self->cursor))) return rc; 491 | break; 492 | } 493 | } 494 | 495 | return LSM_OK; 496 | } 497 | 498 | 499 | static int str_or_bytes_check(char binary, PyObject* pObj, const char** ppBuff, Py_ssize_t* nBuf) { 500 | const char * buff = NULL; 501 | Py_ssize_t buff_len = 0; 502 | 503 | if (binary) { 504 | if (PyBytes_Check(pObj)) { 505 | buff_len = PyBytes_GET_SIZE(pObj); 506 | buff = PyBytes_AS_STRING(pObj); 507 | } else { 508 | PyErr_Format(PyExc_ValueError, "bytes expected not %R", PyObject_Type(pObj)); 509 | return -1; 510 | } 511 | } else { 512 | if (PyUnicode_Check(pObj)) { 513 | buff = PyUnicode_AsUTF8AndSize(pObj, &buff_len); 514 | if (buff == NULL) return -1; 515 | } else { 516 | PyErr_Format(PyExc_ValueError, "str expected not %R", PyObject_Type(pObj)); 517 | return -1; 518 | } 519 | } 520 | 521 | *ppBuff = buff; 522 | *nBuf = buff_len; 523 | 524 | return 0; 525 | } 526 | 527 | 528 | static PyObject* pylsm_cursor_key_fetch(lsm_cursor* cursor, uint8_t binary) { 529 | char *pKey = NULL; 530 | int nKey = 0; 531 | char *pValue = NULL; 532 | int nValue = 0; 533 | 534 | if (pylsm_error(lsm_csr_key(cursor, (const void**) &pKey, &nKey))) return NULL; 535 | if (pylsm_error(lsm_csr_value(cursor, (const void**) &pValue, &nValue))) return NULL; 536 | 537 | if (binary) { 538 | return PyBytes_FromStringAndSize(pKey, nKey); 539 | } else { 540 | return PyUnicode_FromStringAndSize(pKey, nKey); 541 | } 542 | } 543 | 544 | 545 | static PyObject* pylsm_cursor_value_fetch(lsm_cursor* cursor, uint8_t binary) { 546 | char *pKey = NULL; 547 | int nKey = 0; 548 | char *pValue = NULL; 549 | int nValue = 0; 550 | 551 | if (pylsm_error(lsm_csr_key(cursor, (const void**) &pKey, &nKey))) return NULL; 552 | if (pylsm_error(lsm_csr_value(cursor, (const void**) &pValue, &nValue))) return NULL; 553 | 554 | if (binary) { 555 | return PyBytes_FromStringAndSize(pValue, nValue); 556 | } else { 557 | return PyUnicode_FromStringAndSize(pValue, nValue); 558 | } 559 | } 560 | 561 | 562 | static PyObject* pylsm_cursor_items_fetch(lsm_cursor* cursor, uint8_t binary) { 563 | char *pKey = NULL; 564 | int nKey = 0; 565 | char *pValue = NULL; 566 | int nValue = 0; 567 | 568 | lsm_csr_key(cursor, (const void**) &pKey, &nKey); 569 | lsm_csr_value(cursor, (const void**) &pValue, &nValue); 570 | 571 | PyObject* pyKey; 572 | PyObject* pyValue; 573 | 574 | if (binary) { 575 | pyKey = PyBytes_FromStringAndSize(pKey, nKey); 576 | pyValue = PyBytes_FromStringAndSize(pValue, nValue); 577 | } else { 578 | pyKey = PyUnicode_FromStringAndSize(pKey, nKey); 579 | pyValue = PyUnicode_FromStringAndSize(pValue, nValue); 580 | } 581 | 582 | PyObject* result = PyTuple_Pack(2, pyKey, pyValue); 583 | Py_DECREF(pyKey); 584 | Py_DECREF(pyValue); 585 | return result; 586 | } 587 | 588 | 589 | static PyObject* LSMIterView_new(PyTypeObject *type) { 590 | LSMIterView *self; 591 | self = (LSMIterView *) type->tp_alloc(type, 0); 592 | return (PyObject *) self; 593 | } 594 | 595 | 596 | static void LSMIterView_dealloc(LSMIterView *self) { 597 | if (self->db == NULL) return; 598 | 599 | if (self->cursor != NULL) { 600 | Py_BEGIN_ALLOW_THREADS 601 | LSM_MutexLock(self->db); 602 | lsm_csr_close(self->cursor); 603 | LSM_MutexLeave(self->db); 604 | Py_END_ALLOW_THREADS 605 | } 606 | 607 | if (self->state == PY_LSM_OPENED) { 608 | self->state = PY_LSM_CLOSED; 609 | } 610 | 611 | Py_DECREF(self->db); 612 | 613 | self->cursor = NULL; 614 | self->db = NULL; 615 | 616 | if (self->weakrefs != NULL) PyObject_ClearWeakRefs((PyObject *) self); 617 | } 618 | 619 | 620 | static int LSMIterView_init(LSMIterView *self, LSM* lsm) { 621 | if (pylsm_ensure_opened(lsm)) return -1; 622 | 623 | self->db = lsm; 624 | Py_INCREF(self->db); 625 | 626 | self->state = PY_LSM_INITIALIZED; 627 | return 0; 628 | } 629 | 630 | 631 | static Py_ssize_t LSMIterView_len(LSMIterView* self) { 632 | if (pylsm_ensure_opened(self->db)) return -1; 633 | 634 | Py_ssize_t result = 0; 635 | Py_ssize_t rc = 0; 636 | 637 | Py_BEGIN_ALLOW_THREADS 638 | LSM_MutexLock(self->db); 639 | rc = pylsm_length(self->db->lsm, &result); 640 | LSM_MutexLeave(self->db); 641 | Py_END_ALLOW_THREADS 642 | 643 | if (pylsm_error(rc)) return -1; 644 | return result; 645 | } 646 | 647 | static LSMIterView* LSMIterView_iter(LSMIterView* self) { 648 | if (pylsm_ensure_opened(self->db)) return NULL; 649 | 650 | if (self->state != PY_LSM_INITIALIZED) { 651 | Py_INCREF(self); 652 | return self; 653 | } 654 | 655 | if (self->state == PY_LSM_OPENED) { 656 | PyErr_SetString(PyExc_RuntimeError, "Can not modify started iterator"); 657 | return NULL; 658 | } 659 | 660 | self->state = PY_LSM_OPENED; 661 | 662 | LSM_MutexLock(self->db); 663 | if (pylsm_error(lsm_csr_open(self->db->lsm, &self->cursor))) { 664 | LSM_MutexLeave(self->db); 665 | return NULL; 666 | } 667 | 668 | if (pylsm_error(lsm_csr_first(self->cursor))) { 669 | LSM_MutexLeave(self->db); 670 | return NULL; 671 | } 672 | 673 | LSM_MutexLeave(self->db); 674 | 675 | Py_INCREF(self); 676 | return self; 677 | } 678 | 679 | 680 | static PyObject* LSMKeysView_next(LSMIterView *self) { 681 | if (pylsm_ensure_opened(self->db)) return NULL; 682 | if (self->state != PY_LSM_OPENED) { 683 | PyErr_SetString(PyExc_RuntimeError, "Must call __iter__ before __next__"); 684 | return NULL; 685 | } 686 | 687 | if (!lsm_csr_valid(self->cursor)) { 688 | if (self->state != PY_LSM_CLOSED) { 689 | self->state = PY_LSM_CLOSED; 690 | } 691 | 692 | PyErr_SetNone(PyExc_StopIteration); 693 | return NULL; 694 | } 695 | 696 | LSM_MutexLock(self->db); 697 | PyObject* result = pylsm_cursor_key_fetch(self->cursor, self->db->binary); 698 | 699 | if (result == NULL) { 700 | LSM_MutexLeave(self->db); 701 | return NULL; 702 | } 703 | 704 | if (pylsm_error(lsm_csr_next(self->cursor))) { 705 | LSM_MutexLeave(self->db); 706 | return NULL; 707 | }; 708 | 709 | LSM_MutexLeave(self->db); 710 | return result; 711 | } 712 | 713 | 714 | static PyObject* LSMValuesView_next(LSMIterView *self) { 715 | if (pylsm_ensure_opened(self->db)) return NULL; 716 | 717 | if (!lsm_csr_valid(self->cursor)) { 718 | if (self->state != PY_LSM_CLOSED) { 719 | self->state = PY_LSM_CLOSED; 720 | } 721 | PyErr_SetNone(PyExc_StopIteration); 722 | return NULL; 723 | } 724 | 725 | LSM_MutexLock(self->db); 726 | PyObject* result = pylsm_cursor_value_fetch(self->cursor, self->db->binary); 727 | if (result == NULL) { 728 | LSM_MutexLeave(self->db); 729 | return NULL; 730 | } 731 | 732 | if (pylsm_error(lsm_csr_next(self->cursor))) { 733 | LSM_MutexLeave(self->db); 734 | return NULL; 735 | }; 736 | 737 | LSM_MutexLeave(self->db); 738 | 739 | return result; 740 | } 741 | 742 | 743 | static PyObject* LSMItemsView_next(LSMIterView *self) { 744 | if (pylsm_ensure_opened(self->db)) return NULL; 745 | 746 | if (!lsm_csr_valid(self->cursor)) { 747 | if (self->state != PY_LSM_CLOSED) { 748 | self->state = PY_LSM_CLOSED; 749 | } 750 | PyErr_SetNone(PyExc_StopIteration); 751 | return NULL; 752 | } 753 | 754 | LSM_MutexLock(self->db); 755 | PyObject* result = pylsm_cursor_items_fetch( 756 | self->cursor, 757 | self->db->binary 758 | ); 759 | if (result == NULL) { 760 | LSM_MutexLeave(self->db); 761 | return NULL; 762 | } 763 | 764 | if (pylsm_error(lsm_csr_next(self->cursor))) { 765 | LSM_MutexLeave(self->db); 766 | return NULL; 767 | }; 768 | 769 | LSM_MutexLeave(self->db); 770 | return result; 771 | } 772 | 773 | 774 | static int LSM_contains(LSM *self, PyObject *key); 775 | 776 | static int LSMKeysView_contains(LSMIterView* self, PyObject* key) { 777 | return LSM_contains(self->db, key); 778 | } 779 | 780 | static PySequenceMethods LSMKeysView_sequence = { 781 | .sq_length = (lenfunc) LSMIterView_len, 782 | .sq_contains = (objobjproc) LSMKeysView_contains 783 | }; 784 | 785 | 786 | static int LSMIterView_contains(LSMIterView* self, PyObject* key) { 787 | PyErr_SetNone(PyExc_NotImplementedError); 788 | return 0; 789 | } 790 | 791 | 792 | static PySequenceMethods LSMIterView_sequence = { 793 | .sq_length = (lenfunc) LSMIterView_len, 794 | .sq_contains = (objobjproc) LSMIterView_contains 795 | }; 796 | 797 | static PyTypeObject LSMKeysType = { 798 | PyVarObject_HEAD_INIT(NULL, 0) 799 | .tp_name = "lsm_keys", 800 | .tp_basicsize = sizeof(LSMIterView), 801 | .tp_itemsize = 0, 802 | .tp_flags = Py_TPFLAGS_DEFAULT, 803 | .tp_dealloc = (destructor) LSMIterView_dealloc, 804 | .tp_iter = (getiterfunc) LSMIterView_iter, 805 | .tp_iternext = (iternextfunc) LSMKeysView_next, 806 | .tp_as_sequence = &LSMKeysView_sequence, 807 | .tp_weaklistoffset = offsetof(LSMIterView, weakrefs) 808 | }; 809 | 810 | 811 | static PyTypeObject LSMItemsType = { 812 | PyVarObject_HEAD_INIT(NULL, 0) 813 | .tp_name = "lsm_items", 814 | .tp_basicsize = sizeof(LSMIterView), 815 | .tp_itemsize = 0, 816 | .tp_flags = Py_TPFLAGS_DEFAULT, 817 | .tp_dealloc = (destructor) LSMIterView_dealloc, 818 | .tp_iter = (getiterfunc) LSMIterView_iter, 819 | .tp_iternext = (iternextfunc) LSMItemsView_next, 820 | .tp_as_sequence = &LSMIterView_sequence, 821 | .tp_weaklistoffset = offsetof(LSMIterView, weakrefs) 822 | }; 823 | 824 | 825 | static PyTypeObject LSMValuesType = { 826 | PyVarObject_HEAD_INIT(NULL, 0) 827 | .tp_name = "lsm_values", 828 | .tp_basicsize = sizeof(LSMIterView), 829 | .tp_itemsize = 0, 830 | .tp_flags = Py_TPFLAGS_DEFAULT, 831 | .tp_dealloc = (destructor) LSMIterView_dealloc, 832 | .tp_iter = (getiterfunc) LSMIterView_iter, 833 | .tp_iternext = (iternextfunc) LSMValuesView_next, 834 | .tp_as_sequence = &LSMIterView_sequence, 835 | .tp_weaklistoffset = offsetof(LSMIterView, weakrefs) 836 | }; 837 | 838 | 839 | static PyObject* LSMSliceView_new(PyTypeObject *type) { 840 | LSMSliceView *self; 841 | self = (LSMSliceView *) type->tp_alloc(type, 0); 842 | return (PyObject *) self; 843 | } 844 | 845 | 846 | static void LSMSliceView_dealloc(LSMSliceView *self) { 847 | if (self->db == NULL) return; 848 | 849 | if (self->cursor != NULL) { 850 | LSM_MutexLock(self->db); 851 | lsm_csr_close(self->cursor); 852 | LSM_MutexLeave(self->db); 853 | } 854 | 855 | if (self->start != NULL) Py_DECREF(self->start); 856 | if (self->stop != NULL) Py_DECREF(self->stop); 857 | 858 | Py_DECREF(self->db); 859 | 860 | self->cursor = NULL; 861 | self->db = NULL; 862 | self->pStart = NULL; 863 | self->pStop = NULL; 864 | self->stop = NULL; 865 | 866 | if (self->weakrefs != NULL) PyObject_ClearWeakRefs((PyObject *) self); 867 | } 868 | 869 | 870 | static int LSMSliceView_init( 871 | LSMSliceView *self, 872 | LSM* lsm, 873 | PyObject* start, 874 | PyObject* stop, 875 | PyObject* step 876 | ) { 877 | assert(lsm != NULL); 878 | if (pylsm_ensure_opened(lsm)) return -1; 879 | 880 | if (step == Py_None) { 881 | self->step = 1; 882 | } else { 883 | if (!PyLong_Check(step)) { 884 | PyErr_Format( 885 | PyExc_ValueError, 886 | "step must be int not %R", 887 | PyObject_Type(step) 888 | ); 889 | return -1; 890 | } 891 | self->step = PyLong_AsLong(step); 892 | } 893 | 894 | self->direction = (self->step > 0) ? PY_LSM_SLICE_FORWARD : PY_LSM_SLICE_BACKWARD; 895 | 896 | self->db = lsm; 897 | 898 | switch (self->direction) { 899 | case PY_LSM_SLICE_FORWARD: 900 | self->stop = stop; 901 | self->start = start; 902 | break; 903 | case PY_LSM_SLICE_BACKWARD: 904 | self->stop = start; 905 | self->start = stop; 906 | break; 907 | } 908 | 909 | self->pStop = NULL; 910 | self->nStop = 0; 911 | self->counter = 0; 912 | 913 | if (self->stop != Py_None) { 914 | if (str_or_bytes_check(self->db->binary, self->stop, (const char **) &self->pStop, &self->nStop)) return -1; 915 | Py_INCREF(self->stop); 916 | } 917 | 918 | if (self->start != Py_None) { 919 | if (str_or_bytes_check(self->db->binary, self->start, (const char **) &self->pStart, &self->nStart)) return -1; 920 | Py_INCREF(self->start); 921 | } 922 | 923 | self->state = PY_LSM_INITIALIZED; 924 | Py_INCREF(self->db); 925 | return 0; 926 | } 927 | 928 | 929 | static LSMSliceView* LSMSliceView_iter(LSMSliceView* self) { 930 | if (pylsm_ensure_opened(self->db)) return NULL; 931 | 932 | 933 | if (self->state != PY_LSM_INITIALIZED) { 934 | Py_INCREF(self); 935 | return self; 936 | } 937 | 938 | if (self->state == PY_LSM_OPENED) { 939 | PyErr_SetString(PyExc_RuntimeError, "Can not modify started iterator"); 940 | return NULL; 941 | } 942 | 943 | self->state = PY_LSM_OPENED; 944 | 945 | int err; 946 | Py_BEGIN_ALLOW_THREADS 947 | LSM_MutexLock(self->db); 948 | err = pylsm_slice_view_iter(self); 949 | LSM_MutexLeave(self->db); 950 | Py_END_ALLOW_THREADS 951 | 952 | if (pylsm_error(err)) return NULL; 953 | 954 | Py_INCREF(self); 955 | return self; 956 | } 957 | 958 | 959 | static PyObject* LSMSliceView_next(LSMSliceView *self) { 960 | if (pylsm_ensure_opened(self->db)) return NULL; 961 | 962 | switch (self->state) { 963 | case PY_LSM_OPENED: 964 | break; 965 | case PY_LSM_ITERATING: 966 | break; 967 | case PY_LSM_CLOSED: 968 | PyErr_SetNone(PyExc_StopIteration); 969 | return NULL; 970 | default: 971 | PyErr_SetString(PyExc_RuntimeError, "Must call __iter__ before __next__"); 972 | return NULL; 973 | } 974 | 975 | if (!lsm_csr_valid(self->cursor)) { 976 | if (self->state != PY_LSM_CLOSED) { 977 | self->state = PY_LSM_CLOSED; 978 | } 979 | PyErr_SetNone(PyExc_StopIteration); 980 | return NULL; 981 | } 982 | 983 | int rc; 984 | 985 | Py_BEGIN_ALLOW_THREADS 986 | LSM_MutexLock(self->db); 987 | 988 | if (self->state == PY_LSM_OPENED) { 989 | self->state = PY_LSM_ITERATING; 990 | rc = pylsm_slice_first(self); 991 | } else { 992 | rc = pylsm_slice_next(self); 993 | } 994 | 995 | LSM_MutexLeave(self->db); 996 | Py_END_ALLOW_THREADS 997 | 998 | if (rc == -1) { 999 | self->state = PY_LSM_CLOSED; 1000 | PyErr_SetNone(PyExc_StopIteration); 1001 | return NULL; 1002 | } 1003 | 1004 | if (pylsm_error(rc)) return NULL; 1005 | 1006 | if (!lsm_csr_valid(self->cursor)) { 1007 | self->state = PY_LSM_CLOSED; 1008 | PyErr_SetNone(PyExc_StopIteration); 1009 | return NULL; 1010 | } 1011 | 1012 | return pylsm_cursor_items_fetch(self->cursor, self->db->binary); 1013 | } 1014 | 1015 | 1016 | static PyTypeObject LSMSliceType = { 1017 | PyVarObject_HEAD_INIT(NULL, 0) 1018 | .tp_name = "lsm_slice", 1019 | .tp_basicsize = sizeof(LSMSliceView), 1020 | .tp_itemsize = 0, 1021 | .tp_flags = Py_TPFLAGS_DEFAULT, 1022 | .tp_dealloc = (destructor) LSMSliceView_dealloc, 1023 | .tp_iter = (getiterfunc) LSMSliceView_iter, 1024 | .tp_iternext = (iternextfunc) LSMSliceView_next, 1025 | .tp_weaklistoffset = offsetof(LSMSliceView, weakrefs) 1026 | }; 1027 | 1028 | 1029 | static PyObject* LSM_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { 1030 | LSM *self; 1031 | self = (LSM *) type->tp_alloc(type, 0); 1032 | return (PyObject *) self; 1033 | } 1034 | 1035 | 1036 | static int _LSM_close(LSM* self) { 1037 | int result; 1038 | 1039 | Py_BEGIN_ALLOW_THREADS; 1040 | LSM_MutexLock(self); 1041 | result = lsm_close(self->lsm); 1042 | LSM_MutexLeave(self); 1043 | Py_END_ALLOW_THREADS; 1044 | 1045 | if (result == LSM_OK) { 1046 | self->state = PY_LSM_CLOSED; 1047 | self->lsm = NULL; 1048 | self->lsm_env = NULL; 1049 | self->lsm_mutex = NULL; 1050 | } 1051 | 1052 | return result; 1053 | } 1054 | 1055 | 1056 | static void LSM_dealloc(LSM *self) { 1057 | if (self->state != PY_LSM_CLOSED && self->lsm != NULL) pylsm_error(_LSM_close(self)); 1058 | if (self->lsm_mutex != NULL) self->lsm_env->xMutexDel(self->lsm_mutex); 1059 | if (self->logger != NULL) Py_DECREF(self->logger); 1060 | if (self->path != NULL) PyMem_Free(self->path); 1061 | if (self->weakrefs != NULL) PyObject_ClearWeakRefs((PyObject *) self); 1062 | } 1063 | 1064 | 1065 | static int LSM_init(LSM *self, PyObject *args, PyObject *kwds) { 1066 | self->autocheckpoint = 2048; 1067 | self->autoflush = 1024; 1068 | self->automerge = 4; 1069 | self->autowork = 1; 1070 | self->mmap = 0; 1071 | self->block_size = 1024; 1072 | self->max_freelist = 24; 1073 | self->multiple_processes = 1; 1074 | self->page_size = 4 * 1024; 1075 | self->readonly = 0; 1076 | self->safety = LSM_SAFETY_NORMAL; 1077 | self->use_log = 1; 1078 | self->tx_level = 0; 1079 | self->compressed = 0; 1080 | self->logger = NULL; 1081 | self->compress_level = PYLSM_DEFAULT_COMPRESS_LEVEL; 1082 | self->path = NULL; 1083 | self->binary = 1; 1084 | memset(&self->lsm_compress, 0, sizeof(lsm_compress)); 1085 | 1086 | static char* kwlist[] = { 1087 | "path", 1088 | "autoflush", 1089 | "page_size", 1090 | "safety", 1091 | "block_size", 1092 | "automerge", 1093 | "max_freelist", 1094 | "autocheckpoint", 1095 | "autowork", 1096 | "mmap", 1097 | "use_log", 1098 | "multiple_processes", 1099 | "readonly", 1100 | "binary", 1101 | "logger", 1102 | "compress", 1103 | "compress_level", 1104 | NULL 1105 | }; 1106 | 1107 | PyObject* compress = Py_None; 1108 | int compressor_id = LSM_COMPRESSION_NONE; 1109 | 1110 | PyObject* pyPath; 1111 | const char *path; 1112 | Py_ssize_t path_len; 1113 | 1114 | if (!PyArg_ParseTupleAndKeywords( 1115 | args, kwds, "O|iiIIIIIppppppOOi", kwlist, 1116 | &pyPath, 1117 | &self->autoflush, 1118 | &self->page_size, 1119 | &self->safety, 1120 | &self->block_size, 1121 | &self->automerge, 1122 | &self->max_freelist, 1123 | &self->autocheckpoint, 1124 | &self->autowork, 1125 | &self->mmap, 1126 | &self->use_log, 1127 | &self->multiple_processes, 1128 | &self->readonly, 1129 | &self->binary, 1130 | &self->logger, 1131 | &compress, 1132 | &self->compress_level 1133 | )) return -1; 1134 | 1135 | if (!PyUnicode_Check(pyPath)) pyPath = PyObject_Str(pyPath); 1136 | 1137 | path = PyUnicode_AsUTF8AndSize(pyPath, &path_len); 1138 | if (path == NULL) return -1; 1139 | 1140 | self->path = PyMem_Calloc(sizeof(char), path_len + 1); 1141 | memcpy(self->path, path, path_len); 1142 | 1143 | self->state = PY_LSM_INITIALIZED; 1144 | 1145 | if (self->autoflush > LSM_MAX_AUTOFLUSH) { 1146 | PyErr_Format( 1147 | PyExc_ValueError, 1148 | "The maximum allowable value for autoflush parameter " 1149 | "is 1048576 (1GB). Not %d", self->autoflush 1150 | ); 1151 | return -1; 1152 | } 1153 | 1154 | if (self->autoflush < 0) { 1155 | PyErr_Format( 1156 | PyExc_ValueError, 1157 | "The minimum allowable value for autoflush parameter " 1158 | "is 0. Not %d", self->autoflush 1159 | ); 1160 | return -1; 1161 | } 1162 | 1163 | if (self->autocheckpoint <= 0) { 1164 | PyErr_SetString( 1165 | PyExc_ValueError, 1166 | "autocheckpoint is not able to be zero or lower" 1167 | ); 1168 | return -1; 1169 | } 1170 | 1171 | if (!( 1172 | is_power_of_two(self->block_size) && 1173 | self->block_size >= 64 && 1174 | self->block_size < 65537 1175 | )) { 1176 | PyErr_Format( 1177 | PyExc_ValueError, 1178 | "block_size parameter must be power of two between " 1179 | "64 and 65535. Not %d", 1180 | self->block_size 1181 | ); 1182 | return -1; 1183 | } 1184 | 1185 | switch (self->safety) { 1186 | case LSM_SAFETY_OFF: 1187 | break; 1188 | case LSM_SAFETY_NORMAL: 1189 | break; 1190 | case LSM_SAFETY_FULL: 1191 | break; 1192 | default: 1193 | PyErr_Format( 1194 | PyExc_ValueError, 1195 | "safety parameter must be SAFETY_OFF SAFETY_NORMAL " 1196 | "or SAFETY_FULL. Not %d", self->safety 1197 | ); 1198 | return -1; 1199 | } 1200 | 1201 | if (compress == Py_None) { 1202 | compressor_id = PY_LSM_COMPRESSOR_EMPTY; 1203 | } else if (!PyUnicode_Check(compress)) { 1204 | PyErr_Format(PyExc_ValueError, "str expected not %R", PyObject_Type(compress)); 1205 | return -1; 1206 | } else if (PyUnicode_CompareWithASCIIString(compress, "none") == 0) { 1207 | compressor_id = PY_LSM_COMPRESSOR_NONE; 1208 | } else if (PyUnicode_CompareWithASCIIString(compress, "lz4") == 0) { 1209 | compressor_id = PY_LSM_COMPRESSOR_LZ4; 1210 | 1211 | if (self->compress_level == PYLSM_DEFAULT_COMPRESS_LEVEL) { 1212 | self->compress_level = LZ4_COMP_LEVEL_DEFAULT; 1213 | } 1214 | 1215 | if (self->compress_level > LZ4_COMP_LEVEL_MAX || self->compress_level < 1) { 1216 | PyErr_Format( 1217 | PyExc_ValueError, 1218 | "compress_level for lz4 must be between 1 and %d", 1219 | LZ4_COMP_LEVEL_MAX 1220 | ); 1221 | return -1; 1222 | } 1223 | } else if (PyUnicode_CompareWithASCIIString(compress, "zstd") == 0) { 1224 | compressor_id = PY_LSM_COMPRESSOR_ZSTD; 1225 | if (self->compress_level == PYLSM_DEFAULT_COMPRESS_LEVEL) { 1226 | self->compress_level = ZSTD_CLEVEL_DEFAULT; 1227 | } 1228 | 1229 | if (self->compress_level > ZSTD_maxCLevel() || self->compress_level < 1) { 1230 | PyErr_Format( 1231 | PyExc_ValueError, 1232 | "compress_level for zstd must be between 1 and %d", ZSTD_maxCLevel() 1233 | ); 1234 | return -1; 1235 | } 1236 | 1237 | } else { 1238 | PyErr_Format( 1239 | PyExc_ValueError, 1240 | "compressor argument must be one of \"none\" (or None) \"lz4\" or \"zstd\", but not %R", 1241 | compress 1242 | ); 1243 | return -1; 1244 | } 1245 | 1246 | if (compressor_id > PY_LSM_COMPRESSOR_NONE) self->compressed = 1; 1247 | 1248 | if (self->logger != NULL && !PyCallable_Check(self->logger)) { 1249 | PyErr_Format(PyExc_ValueError, "object %R is not callable", self->logger); 1250 | return -1; 1251 | } 1252 | 1253 | if (self->logger != NULL) Py_INCREF(self->logger); 1254 | if (pylsm_error(lsm_new(NULL, &self->lsm))) return -1; 1255 | 1256 | self->lsm_env = lsm_get_env(self->lsm); 1257 | 1258 | if (pylsm_error(self->lsm_env->xMutexNew(self->lsm_env, &self->lsm_mutex))) return -1; 1259 | 1260 | if (self->logger != NULL) { 1261 | lsm_config_log(self->lsm, (void (*)(void *, int, const char *)) pylsm_logger, self); 1262 | } else { 1263 | lsm_config_log(self->lsm, NULL, NULL); 1264 | } 1265 | 1266 | if (self->lsm == NULL) { 1267 | PyErr_SetString(PyExc_MemoryError, "Can not allocate memory"); 1268 | return -1; 1269 | } 1270 | 1271 | // Only before lsm_open 1272 | if (self->compressed) { 1273 | self->lsm_compress.pCtx = self; 1274 | self->lsm_compress.iId = compressor_id; 1275 | 1276 | switch (compressor_id) { 1277 | case PY_LSM_COMPRESSOR_LZ4: 1278 | self->lsm_compress.xCompress = (int (*)(void *, char *, int *, const char *, int)) pylsm_lz4_xCompress; 1279 | self->lsm_compress.xUncompress = (int (*)(void *, char *, int *, const char *, int)) pylsm_lz4_xUncompress; 1280 | self->lsm_compress.xBound = (int (*)(void *, int)) pylsm_lz4_xBound; 1281 | self->lsm_compress.xFree = NULL; 1282 | break; 1283 | case PY_LSM_COMPRESSOR_ZSTD: 1284 | self->lsm_compress.xCompress = (int (*)(void *, char *, int *, const char *, int)) pylsm_zstd_xCompress; 1285 | self->lsm_compress.xUncompress = (int (*)(void *, char *, int *, const char *, int)) pylsm_zstd_xUncompress; 1286 | self->lsm_compress.xBound = (int (*)(void *, int)) pylsm_zstd_xBound; 1287 | self->lsm_compress.xFree = NULL; 1288 | break; 1289 | } 1290 | 1291 | if (pylsm_error(lsm_config(self->lsm, LSM_CONFIG_SET_COMPRESSION, &self->lsm_compress))) return -1; 1292 | } 1293 | 1294 | if (pylsm_error(lsm_config(self->lsm, LSM_CONFIG_BLOCK_SIZE, &self->block_size))) return -1; 1295 | if (pylsm_error(lsm_config(self->lsm, LSM_CONFIG_MULTIPLE_PROCESSES, &self->multiple_processes))) return -1; 1296 | if (pylsm_error(lsm_config(self->lsm, LSM_CONFIG_PAGE_SIZE, &self->page_size))) return -1; 1297 | if (pylsm_error(lsm_config(self->lsm, LSM_CONFIG_READONLY, &self->readonly))) return -1; 1298 | 1299 | // Not only before lsm_open 1300 | if (pylsm_error(lsm_config(self->lsm, LSM_CONFIG_AUTOCHECKPOINT, &self->autocheckpoint))) return -1; 1301 | if (pylsm_error(lsm_config(self->lsm, LSM_CONFIG_AUTOFLUSH, &self->autoflush))) return -1; 1302 | if (pylsm_error(lsm_config(self->lsm, LSM_CONFIG_AUTOMERGE, &self->automerge))) return -1; 1303 | if (pylsm_error(lsm_config(self->lsm, LSM_CONFIG_AUTOWORK, &self->autowork))) return -1; 1304 | if (pylsm_error(lsm_config(self->lsm, LSM_CONFIG_MAX_FREELIST, &self->max_freelist))) return -1; 1305 | if (pylsm_error(lsm_config(self->lsm, LSM_CONFIG_MMAP, &self->mmap))) return -1; 1306 | if (pylsm_error(lsm_config(self->lsm, LSM_CONFIG_SAFETY, &self->safety))) return -1; 1307 | if (pylsm_error(lsm_config(self->lsm, LSM_CONFIG_USE_LOG, &self->use_log))) return -1; 1308 | 1309 | if (PyErr_Occurred()) return -1; 1310 | 1311 | return 0; 1312 | } 1313 | 1314 | 1315 | static PyObject* LSM_open(LSM *self) { 1316 | if (self->state == PY_LSM_OPENED) { 1317 | PyErr_SetString(PyExc_RuntimeError, "Database already opened"); 1318 | return NULL; 1319 | } 1320 | 1321 | if (self->state == PY_LSM_CLOSED) { 1322 | PyErr_SetString(PyExc_RuntimeError, "Database closed"); 1323 | return NULL; 1324 | } 1325 | 1326 | int result; 1327 | result = lsm_open(self->lsm, self->path); 1328 | 1329 | if (pylsm_error(result)) return NULL; 1330 | 1331 | if (self->readonly == 0) { 1332 | Py_BEGIN_ALLOW_THREADS 1333 | result = lsm_flush(self->lsm); 1334 | Py_END_ALLOW_THREADS 1335 | 1336 | if (pylsm_error(result)) return NULL; 1337 | 1338 | Py_BEGIN_ALLOW_THREADS 1339 | result = lsm_work(self->lsm, self->automerge, self->page_size, NULL); 1340 | Py_END_ALLOW_THREADS 1341 | 1342 | if (pylsm_error(result)) return NULL; 1343 | } 1344 | 1345 | self->state = PY_LSM_OPENED; 1346 | Py_RETURN_TRUE; 1347 | } 1348 | 1349 | static PyObject* LSM_close(LSM *self) { 1350 | if (self->state == PY_LSM_CLOSED) { 1351 | PyErr_SetString(PyExc_RuntimeError, "Database already closed"); 1352 | return NULL; 1353 | } 1354 | 1355 | if (pylsm_error(_LSM_close(self))) return NULL; 1356 | Py_RETURN_TRUE; 1357 | } 1358 | 1359 | 1360 | static PyObject* LSM_info(LSM *self) { 1361 | if (pylsm_ensure_opened(self)) return NULL; 1362 | 1363 | int nwrite_result = 0, 1364 | nread_result = 0, 1365 | checkpoint_size_result = 0; 1366 | 1367 | int nwrite = 0, 1368 | nread = 0, 1369 | checkpoint_size = 0, 1370 | tree_size_old = 0, 1371 | tree_size_current = 0, 1372 | tree_size_result = 0; 1373 | 1374 | Py_BEGIN_ALLOW_THREADS 1375 | LSM_MutexLock(self); 1376 | 1377 | nread_result = lsm_info( 1378 | self->lsm, LSM_INFO_NREAD, &nread 1379 | ); 1380 | 1381 | if (!self->readonly) nwrite_result = lsm_info( 1382 | self->lsm, LSM_INFO_NWRITE, &nwrite 1383 | ); 1384 | 1385 | if (!self->readonly) checkpoint_size_result = lsm_info( 1386 | self->lsm, LSM_INFO_CHECKPOINT_SIZE, &checkpoint_size 1387 | ); 1388 | 1389 | if (!self->readonly) tree_size_result = lsm_info( 1390 | self->lsm, LSM_INFO_TREE_SIZE, &tree_size_old, &tree_size_current 1391 | ); 1392 | 1393 | LSM_MutexLeave(self); 1394 | Py_END_ALLOW_THREADS 1395 | 1396 | if (pylsm_error(nread_result)) return NULL; 1397 | if (self->readonly) return Py_BuildValue("{si}", "nread", nread); 1398 | 1399 | if (pylsm_error(nwrite_result)) return NULL; 1400 | if (pylsm_error(checkpoint_size_result)) return NULL; 1401 | if (pylsm_error(tree_size_result)) return NULL; 1402 | 1403 | return Py_BuildValue( 1404 | "{sisisis{sisi}}", 1405 | "nwrite", nwrite, 1406 | "nread", nread, 1407 | "checkpoint_size_result", checkpoint_size, 1408 | "tree_size", "old", tree_size_old, "current", tree_size_current 1409 | ); 1410 | } 1411 | 1412 | 1413 | static PyObject* LSM_ctx_enter(LSM *self) { 1414 | if (self->state == PY_LSM_OPENED) return (PyObject*) self; 1415 | 1416 | Py_INCREF(self); 1417 | 1418 | LSM_open(self); 1419 | if (PyErr_Occurred()) return NULL; 1420 | 1421 | return (PyObject*) self; 1422 | } 1423 | 1424 | 1425 | static PyObject* LSM_commit_inner(LSM *self, int tx_level); 1426 | static PyObject* LSM_rollback_inner(LSM *self, int tx_level); 1427 | 1428 | 1429 | static PyObject* LSM_ctx_exit(LSM *self, PyObject* args) { 1430 | if (self->state == PY_LSM_CLOSED) { Py_RETURN_NONE; }; 1431 | 1432 | PyObject *exc_type, *exc_value, *exc_tb; 1433 | if (!PyArg_ParseTuple(args, "OOO", &exc_type, &exc_value, &exc_tb)) return NULL; 1434 | if (exc_type == Py_None) { 1435 | if (self->tx_level > 0) LSM_commit_inner(self, 0); 1436 | } else { 1437 | if (self->tx_level > 0) LSM_rollback_inner(self, 0); 1438 | } 1439 | 1440 | if (pylsm_error(_LSM_close(self))) return NULL; 1441 | Py_RETURN_NONE; 1442 | } 1443 | 1444 | 1445 | static PyObject* LSM_work(LSM *self, PyObject *args, PyObject *kwds) { 1446 | if (pylsm_ensure_writable(self)) return NULL; 1447 | 1448 | static char *kwlist[] = {"nmerge", "nkb", "complete", NULL}; 1449 | 1450 | char complete = 1; 1451 | int nmerge = self->automerge; 1452 | int nkb = self->page_size; 1453 | 1454 | if (!PyArg_ParseTupleAndKeywords( 1455 | args, kwds, "|IIp", kwlist, &nmerge, &nkb, &complete 1456 | )) return NULL; 1457 | 1458 | int result; 1459 | int total_written = 0; 1460 | int written = 0; 1461 | 1462 | Py_BEGIN_ALLOW_THREADS 1463 | LSM_MutexLock(self); 1464 | 1465 | do { 1466 | result = lsm_work(self->lsm, nmerge, nkb, &written); 1467 | total_written += written; 1468 | if (nmerge < self->automerge) nmerge++; 1469 | } while (complete && written > 0); 1470 | 1471 | LSM_MutexLeave(self); 1472 | Py_END_ALLOW_THREADS 1473 | 1474 | if (pylsm_error(result)) return NULL; 1475 | return Py_BuildValue("i", total_written); 1476 | } 1477 | 1478 | 1479 | static PyObject* LSM_flush(LSM *self) { 1480 | if (pylsm_ensure_writable(self)) return NULL; 1481 | 1482 | int rc; 1483 | 1484 | Py_BEGIN_ALLOW_THREADS 1485 | LSM_MutexLock(self); 1486 | rc = lsm_flush(self->lsm); 1487 | LSM_MutexLeave(self); 1488 | Py_END_ALLOW_THREADS 1489 | 1490 | if (pylsm_error(rc)) return NULL; 1491 | Py_RETURN_TRUE; 1492 | } 1493 | 1494 | static PyObject* LSM_checkpoint(LSM *self) { 1495 | if (pylsm_ensure_writable(self)) return NULL; 1496 | 1497 | int result; 1498 | int bytes_written = 0; 1499 | 1500 | Py_BEGIN_ALLOW_THREADS 1501 | LSM_MutexLock(self); 1502 | result = lsm_checkpoint(self->lsm, &bytes_written); 1503 | LSM_MutexLeave(self); 1504 | Py_END_ALLOW_THREADS 1505 | 1506 | if (pylsm_error(result)) return NULL; 1507 | return Py_BuildValue("i", bytes_written); 1508 | } 1509 | 1510 | static PyObject* LSM_cursor(LSM *self, PyObject *args, PyObject *kwds) { 1511 | if (pylsm_ensure_opened(self)) return NULL; 1512 | 1513 | int seek_mode = LSM_SEEK_GE; 1514 | static char *kwlist[] = {"seek_mode", NULL}; 1515 | 1516 | if (!PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &seek_mode)) return NULL; 1517 | if (pylsm_seek_mode_check(seek_mode)) return NULL; 1518 | 1519 | LSMCursor* cursor = (LSMCursor*) LSMCursor_new(&LSMCursorType, self, seek_mode); 1520 | if (cursor == NULL) return NULL; 1521 | 1522 | return (PyObject*) cursor; 1523 | } 1524 | 1525 | 1526 | static PyObject* LSM_insert(LSM *self, PyObject *args, PyObject *kwds) { 1527 | if (pylsm_ensure_writable(self)) return NULL; 1528 | 1529 | static char *kwlist[] = {"key", "value", NULL}; 1530 | 1531 | const char* pKey = NULL; 1532 | Py_ssize_t nKey = 0; 1533 | 1534 | const char* pVal = NULL; 1535 | Py_ssize_t nVal = 0; 1536 | 1537 | if (self->binary) { 1538 | if (!PyArg_ParseTupleAndKeywords(args, kwds, "y#y#", kwlist, &pKey, &nKey, &pVal, &nVal)) return NULL; 1539 | } else { 1540 | if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#s#", kwlist, &pKey, &nKey, &pVal, &nVal)) return NULL; 1541 | } 1542 | 1543 | if (nKey >= INT_MAX) { 1544 | PyErr_SetString(PyExc_OverflowError, "length of key is too large"); 1545 | return NULL; 1546 | } 1547 | if (nVal >= INT_MAX) { 1548 | PyErr_SetString(PyExc_OverflowError, "length of value is too large"); 1549 | return NULL; 1550 | } 1551 | 1552 | int result; 1553 | 1554 | Py_BEGIN_ALLOW_THREADS 1555 | LSM_MutexLock(self); 1556 | result = lsm_insert(self->lsm, pKey, (int) nKey, pVal, (int) nVal); 1557 | LSM_MutexLeave(self); 1558 | Py_END_ALLOW_THREADS 1559 | 1560 | if (pylsm_error(result)) return NULL; 1561 | Py_RETURN_NONE; 1562 | } 1563 | 1564 | 1565 | static PyObject* LSM_delete(LSM *self, PyObject *args, PyObject *kwds) { 1566 | if (pylsm_ensure_writable(self)) return NULL; 1567 | 1568 | static char *kwlist[] = {"key", NULL}; 1569 | 1570 | const char* pKey = NULL; 1571 | Py_ssize_t nKey = 0; 1572 | 1573 | if (self->binary) { 1574 | if (!PyArg_ParseTupleAndKeywords(args, kwds, "y#", kwlist, &pKey, &nKey)) return NULL; 1575 | } else { 1576 | if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#", kwlist, &pKey, &nKey)) return NULL; 1577 | } 1578 | 1579 | if (nKey >= INT_MAX) { 1580 | PyErr_SetString(PyExc_OverflowError, "length of key is too large"); 1581 | return NULL; 1582 | } 1583 | 1584 | int result; 1585 | Py_BEGIN_ALLOW_THREADS 1586 | LSM_MutexLock(self); 1587 | result = lsm_delete(self->lsm, pKey, (int) nKey); 1588 | LSM_MutexLeave(self); 1589 | Py_END_ALLOW_THREADS 1590 | 1591 | if (pylsm_error(result)) return NULL; 1592 | Py_RETURN_NONE; 1593 | } 1594 | 1595 | 1596 | static PyObject* LSM_delete_range(LSM *self, PyObject *args, PyObject *kwds) { 1597 | if (pylsm_ensure_writable(self)) return NULL; 1598 | 1599 | static char *kwlist[] = {"start", "end", NULL}; 1600 | 1601 | const char* pStart = NULL; 1602 | Py_ssize_t nStart = 0; 1603 | 1604 | const char* pEnd = NULL; 1605 | Py_ssize_t nEnd = 0; 1606 | 1607 | if (self->binary) { 1608 | if (!PyArg_ParseTupleAndKeywords(args, kwds, "y#y#", kwlist, &pStart, &nStart, &pEnd, &nEnd)) return NULL; 1609 | } else { 1610 | if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#s#", kwlist, &pStart, &nStart, &pEnd, &nEnd)) return NULL; 1611 | } 1612 | 1613 | if (nStart >= INT_MAX) { 1614 | PyErr_SetString(PyExc_OverflowError, "length of start is too large"); 1615 | return NULL; 1616 | } 1617 | if (nEnd >= INT_MAX) { 1618 | PyErr_SetString(PyExc_OverflowError, "length of end is too large"); 1619 | return NULL; 1620 | } 1621 | 1622 | int result; 1623 | Py_BEGIN_ALLOW_THREADS 1624 | LSM_MutexLock(self); 1625 | result = lsm_delete_range(self->lsm, pStart, (int) nStart, pEnd, (int) nEnd); 1626 | LSM_MutexLeave(self); 1627 | Py_END_ALLOW_THREADS 1628 | 1629 | if (pylsm_error(result)) return NULL; 1630 | Py_RETURN_NONE; 1631 | } 1632 | 1633 | 1634 | static PyObject* LSM_begin(LSM *self) { 1635 | if (pylsm_ensure_writable(self)) return NULL; 1636 | if (self->tx_level < 0) self->tx_level = 0; 1637 | 1638 | int level = self->tx_level + 1; 1639 | int result; 1640 | 1641 | Py_BEGIN_ALLOW_THREADS 1642 | result = lsm_begin(self->lsm, level); 1643 | Py_END_ALLOW_THREADS 1644 | 1645 | if (pylsm_error(result)) return NULL; 1646 | 1647 | self->tx_level = level; 1648 | 1649 | Py_RETURN_TRUE; 1650 | } 1651 | 1652 | 1653 | static PyObject* LSM_commit_inner(LSM *self, int tx_level) { 1654 | if (pylsm_ensure_writable(self)) return NULL; 1655 | if (tx_level < 0) tx_level = 0; 1656 | 1657 | int result; 1658 | Py_BEGIN_ALLOW_THREADS 1659 | LSM_MutexLock(self); 1660 | result = lsm_commit(self->lsm, tx_level); 1661 | LSM_MutexLeave(self); 1662 | Py_END_ALLOW_THREADS 1663 | 1664 | if (pylsm_error(result)) return NULL; 1665 | Py_RETURN_TRUE; 1666 | } 1667 | 1668 | 1669 | static PyObject* LSM_rollback_inner(LSM *self, int tx_level) { 1670 | if (pylsm_ensure_writable(self)) return NULL; 1671 | if (tx_level < 0) tx_level = 0; 1672 | 1673 | int result; 1674 | Py_BEGIN_ALLOW_THREADS 1675 | LSM_MutexLock(self); 1676 | result = lsm_rollback(self->lsm, tx_level); 1677 | LSM_MutexLeave(self); 1678 | Py_END_ALLOW_THREADS 1679 | 1680 | if (pylsm_error(result)) return NULL; 1681 | Py_RETURN_TRUE; 1682 | } 1683 | 1684 | 1685 | 1686 | static PyObject* LSM_commit(LSM *self) { 1687 | if (self->tx_level < 0) self->tx_level = 0; 1688 | return LSM_commit_inner(self, self->tx_level); 1689 | } 1690 | 1691 | 1692 | static PyObject* LSM_rollback(LSM *self) { 1693 | if (self->tx_level < 0) self->tx_level = 0; 1694 | return LSM_rollback_inner(self, self->tx_level); 1695 | } 1696 | 1697 | 1698 | static PyObject* LSM_getitem(LSM *self, PyObject *arg) { 1699 | if (pylsm_ensure_opened(self)) return NULL; 1700 | 1701 | PyObject* key = arg; 1702 | const char* pKey = NULL; 1703 | Py_ssize_t nKey = 0; 1704 | Py_ssize_t tuple_size; 1705 | int seek_mode = LSM_SEEK_EQ; 1706 | 1707 | if (PySlice_Check(arg)) { 1708 | PySliceObject* slice = (PySliceObject*) arg; 1709 | 1710 | LSMSliceView* view = (LSMSliceView*) LSMSliceView_new(&LSMSliceType); 1711 | if (LSMSliceView_init(view, self, slice->start, slice->stop, slice->step)) return NULL; 1712 | return (PyObject*) view; 1713 | } 1714 | 1715 | if (PyTuple_Check(arg)) { 1716 | tuple_size = PyTuple_GET_SIZE(arg); 1717 | if (tuple_size != 2) { 1718 | PyErr_Format( 1719 | PyExc_ValueError, 1720 | "tuple argument must be pair of key and seek_mode passed tuple has size %d", 1721 | tuple_size 1722 | ); 1723 | return NULL; 1724 | } 1725 | 1726 | key = PyTuple_GetItem(arg, 0); 1727 | PyObject* seek_mode_obj = PyTuple_GetItem(arg, 1); 1728 | 1729 | if (!PyLong_Check(seek_mode_obj)) { 1730 | PyErr_Format( 1731 | PyExc_ValueError, 1732 | "second tuple argument must be int not %R", 1733 | PyObject_Type(seek_mode_obj) 1734 | ); 1735 | return NULL; 1736 | } 1737 | 1738 | seek_mode = PyLong_AsLong(seek_mode_obj); 1739 | } 1740 | 1741 | if (pylsm_seek_mode_check(seek_mode)) return NULL; 1742 | 1743 | if (str_or_bytes_check(self->binary, key, &pKey, &nKey)) return NULL; 1744 | if (nKey >= INT_MAX) { 1745 | PyErr_SetString(PyExc_OverflowError, "length of key is too large"); 1746 | return NULL; 1747 | } 1748 | 1749 | int result; 1750 | char *pValue = NULL; 1751 | int nValue = 0; 1752 | 1753 | Py_BEGIN_ALLOW_THREADS 1754 | LSM_MutexLock(self); 1755 | 1756 | result = pylsm_getitem( 1757 | self->lsm, 1758 | pKey, 1759 | (int) nKey, 1760 | &pValue, 1761 | &nValue, 1762 | seek_mode 1763 | ); 1764 | 1765 | LSM_MutexLeave(self); 1766 | Py_END_ALLOW_THREADS 1767 | 1768 | if (result == -1) { 1769 | PyErr_Format( 1770 | PyExc_KeyError, 1771 | "Key %R was not found", 1772 | key 1773 | ); 1774 | if (pValue != NULL) free(pValue); 1775 | return NULL; 1776 | } 1777 | if (pValue == NULL) Py_RETURN_TRUE; 1778 | 1779 | if (pylsm_error(result)) { 1780 | if (pValue != NULL) free(pValue); 1781 | return NULL; 1782 | } 1783 | 1784 | PyObject* py_value = Py_BuildValue(self->binary ? "y#" : "s#", pValue, nValue); 1785 | 1786 | if (pValue != NULL) free(pValue); 1787 | 1788 | return py_value; 1789 | } 1790 | 1791 | 1792 | static int LSM_set_del_item(LSM* self, PyObject* key, PyObject* value) { 1793 | if (pylsm_ensure_writable(self)) return -1; 1794 | 1795 | int rc; 1796 | const char* pKey = NULL; 1797 | Py_ssize_t nKey = 0; 1798 | 1799 | const char* pVal = NULL; 1800 | Py_ssize_t nVal = 0; 1801 | 1802 | // Delete slice 1803 | if (PySlice_Check(key)) { 1804 | 1805 | if (value != NULL) { 1806 | PyErr_SetString(PyExc_NotImplementedError, "setting range doesn't supported yet"); 1807 | return -1; 1808 | } 1809 | 1810 | PySliceObject* slice = (PySliceObject*) key; 1811 | 1812 | if (slice->step != Py_None) { 1813 | PyErr_SetString(PyExc_ValueError, "Stepping not allowed in delete_range operation"); 1814 | return -1; 1815 | } 1816 | 1817 | if (slice->start == Py_None || slice->stop == Py_None) { 1818 | PyErr_SetString(PyExc_ValueError, "You must provide range start and range stop values"); 1819 | return -1; 1820 | } 1821 | 1822 | char *pStop = NULL; 1823 | char *pStart = NULL; 1824 | Py_ssize_t nStart = 0; 1825 | Py_ssize_t nStop = 0; 1826 | 1827 | if (str_or_bytes_check(self->binary, slice->start, (const char **) &pStart, &nStart)) return -1; 1828 | if (str_or_bytes_check(self->binary, slice->stop, (const char **) &pStop, &nStop)) return -1; 1829 | 1830 | if (nStart >= INT_MAX) { 1831 | PyErr_SetString(PyExc_OverflowError, "length of start is too large"); 1832 | return -1; 1833 | } 1834 | if (nStop >= INT_MAX) { 1835 | PyErr_SetString(PyExc_OverflowError, "length of stop is too large"); 1836 | return -1; 1837 | } 1838 | 1839 | Py_INCREF(slice->start); 1840 | Py_INCREF(slice->stop); 1841 | 1842 | int rc; 1843 | 1844 | Py_BEGIN_ALLOW_THREADS 1845 | LSM_MutexLock(self); 1846 | rc = lsm_delete_range( 1847 | self->lsm, 1848 | pStart, (int) nStart, 1849 | pStop, (int) nStop 1850 | ); 1851 | LSM_MutexLeave(self); 1852 | Py_END_ALLOW_THREADS 1853 | 1854 | Py_DECREF(slice->start); 1855 | Py_DECREF(slice->stop); 1856 | 1857 | if (pylsm_error(rc)) return -1; 1858 | 1859 | return 0; 1860 | } 1861 | 1862 | if (str_or_bytes_check(self->binary, key, &pKey, &nKey)) return -1; 1863 | if (value != NULL) { if (str_or_bytes_check(self->binary, value, &pVal, &nVal)) return -1; } 1864 | 1865 | if (nKey >= INT_MAX) { 1866 | PyErr_SetString(PyExc_OverflowError, "length of key is too large"); 1867 | return -1; 1868 | } 1869 | if (nVal >= INT_MAX) { 1870 | PyErr_SetString(PyExc_OverflowError, "length of value is too large"); 1871 | return -1; 1872 | } 1873 | 1874 | Py_BEGIN_ALLOW_THREADS 1875 | LSM_MutexLock(self); 1876 | if (pVal == NULL) { 1877 | rc = pylsm_delitem(self->lsm, pKey, (int) nKey); 1878 | } else { 1879 | rc = lsm_insert(self->lsm, pKey, (int) nKey, pVal, (int) nVal); 1880 | } 1881 | LSM_MutexLeave(self); 1882 | Py_END_ALLOW_THREADS 1883 | 1884 | if (rc == -1) { 1885 | PyErr_Format( 1886 | PyExc_KeyError, 1887 | "Key %R was not found", 1888 | key 1889 | ); 1890 | return -1; 1891 | } 1892 | 1893 | if (pylsm_error(rc)) return -1; 1894 | 1895 | return 0; 1896 | } 1897 | 1898 | 1899 | static int LSM_contains(LSM *self, PyObject *key) { 1900 | if (pylsm_ensure_opened(self)) return 0; 1901 | 1902 | const char* pKey = NULL; 1903 | Py_ssize_t nKey = 0; 1904 | 1905 | if (str_or_bytes_check(self->binary, key, (const char**) &pKey, &nKey)) return 0; 1906 | 1907 | if (nKey >= INT_MAX) { 1908 | PyErr_SetString(PyExc_OverflowError, "length of key is too large"); 1909 | return -1; 1910 | } 1911 | 1912 | int rc; 1913 | 1914 | Py_BEGIN_ALLOW_THREADS 1915 | LSM_MutexLock(self); 1916 | rc = pylsm_contains(self->lsm, pKey, (int) nKey); 1917 | LSM_MutexLeave(self); 1918 | Py_END_ALLOW_THREADS 1919 | 1920 | if (rc == -1) return 0; 1921 | if (rc == 0) return 1; 1922 | 1923 | pylsm_error(rc); 1924 | return -1; 1925 | } 1926 | 1927 | 1928 | static PyObject* LSM_compress_get(LSM* self) { 1929 | switch (self->lsm_compress.iId) { 1930 | case PY_LSM_COMPRESSOR_NONE: 1931 | Py_RETURN_NONE; 1932 | case PY_LSM_COMPRESSOR_LZ4: 1933 | return Py_BuildValue("s", "lz4"); 1934 | case PY_LSM_COMPRESSOR_ZSTD: 1935 | return Py_BuildValue("s", "zstd"); 1936 | } 1937 | 1938 | PyErr_SetString(PyExc_RuntimeError, "invalid compressor"); 1939 | return NULL; 1940 | } 1941 | 1942 | 1943 | static PyObject* LSM_repr(LSM *self) { 1944 | char * path = self->path; 1945 | if (path == NULL) path = ""; 1946 | return PyUnicode_FromFormat( 1947 | "<%s at \"%s\" as %p>", Py_TYPE(self)->tp_name, path, self 1948 | ); 1949 | } 1950 | 1951 | 1952 | static Py_ssize_t LSM_length(LSM *self) { 1953 | Py_ssize_t result = 0; 1954 | Py_ssize_t rc = 0; 1955 | 1956 | Py_BEGIN_ALLOW_THREADS 1957 | LSM_MutexLock(self); 1958 | 1959 | rc = pylsm_length(self->lsm, &result); 1960 | LSM_MutexLeave(self); 1961 | Py_END_ALLOW_THREADS 1962 | 1963 | if (pylsm_error(rc)) return -1; 1964 | return result; 1965 | } 1966 | 1967 | 1968 | static LSMIterView* LSM_keys(LSM* self) { 1969 | if (pylsm_ensure_opened(self)) return NULL; 1970 | 1971 | LSMIterView* view = (LSMIterView*) LSMIterView_new(&LSMKeysType); 1972 | if (LSMIterView_init(view, self)) return NULL; 1973 | return view; 1974 | } 1975 | 1976 | static LSMIterView* LSM_values(LSM* self) { 1977 | if (pylsm_ensure_opened(self)) return NULL; 1978 | 1979 | LSMIterView* view = (LSMIterView*) LSMIterView_new(&LSMValuesType); 1980 | if (LSMIterView_init(view, self)) return NULL; 1981 | return view; 1982 | } 1983 | 1984 | static LSMIterView* LSM_items(LSM* self) { 1985 | if (pylsm_ensure_opened(self)) return NULL; 1986 | 1987 | LSMIterView* view = (LSMIterView*) LSMIterView_new(&LSMItemsType); 1988 | if (LSMIterView_init(view, self)) return NULL; 1989 | return view; 1990 | } 1991 | 1992 | static LSMIterView* LSM_iter(LSM* self) { 1993 | if (pylsm_ensure_opened(self)) return NULL; 1994 | 1995 | LSMIterView* view = (LSMIterView*) LSMIterView_new(&LSMKeysType); 1996 | if (LSMIterView_init(view, self)) return NULL; 1997 | view = LSMIterView_iter(view); 1998 | Py_DECREF(view); 1999 | return view; 2000 | } 2001 | 2002 | static PyObject* LSM_update(LSM* self, PyObject *args) { 2003 | if (pylsm_ensure_writable(self)) return NULL; 2004 | 2005 | PyObject * value = NULL; 2006 | 2007 | if (!PyArg_ParseTuple(args, "O", &value)) return NULL; 2008 | if (!PyMapping_Check(value)) { 2009 | PyErr_Format( 2010 | PyExc_ValueError, 2011 | "Mapping expected not %R", 2012 | PyObject_Type(value) 2013 | ); 2014 | return NULL; 2015 | } 2016 | 2017 | PyObject* items = PyMapping_Items(value); 2018 | 2019 | if (!PyList_Check(items)) { 2020 | PyErr_Format( 2021 | PyExc_ValueError, 2022 | "Iterable expected not %R", 2023 | PyObject_Type(items) 2024 | ); 2025 | return NULL; 2026 | } 2027 | 2028 | Py_ssize_t mapping_size = PyMapping_Length(value); 2029 | 2030 | PyObject **keys_objects = PyMem_Calloc(mapping_size, sizeof(PyObject*)); 2031 | PyObject **values_objects = PyMem_Calloc(mapping_size, sizeof(PyObject*)); 2032 | char **keys = PyMem_Calloc(mapping_size, sizeof(char*)); 2033 | char **values = PyMem_Calloc(mapping_size, sizeof(char*)); 2034 | Py_ssize_t *key_sizes = PyMem_Calloc(mapping_size, sizeof(Py_ssize_t*)); 2035 | Py_ssize_t *value_sizes = PyMem_Calloc(mapping_size, sizeof(Py_ssize_t*)); 2036 | 2037 | PyObject *item; 2038 | int count = 0; 2039 | PyObject *iterator = PyObject_GetIter(items); 2040 | 2041 | PyObject* obj; 2042 | 2043 | unsigned short is_ok = 1; 2044 | 2045 | while ((item = PyIter_Next(iterator))) { 2046 | if (PyTuple_Size(item) != 2) { 2047 | Py_DECREF(item); 2048 | PyErr_Format( 2049 | PyExc_ValueError, 2050 | "Mapping items must be tuple with pair not %R", 2051 | item 2052 | ); 2053 | is_ok = 0; 2054 | break; 2055 | } 2056 | 2057 | obj = PyTuple_GET_ITEM(item, 0); 2058 | if (str_or_bytes_check(self->binary, obj, (const char**) &keys[count], &key_sizes[count])) { 2059 | Py_DECREF(item); 2060 | is_ok = 0; 2061 | break; 2062 | } 2063 | 2064 | if (key_sizes[count] >= INT_MAX) { 2065 | PyErr_SetString(PyExc_OverflowError, "length of key is too large"); 2066 | return NULL; 2067 | } 2068 | 2069 | keys_objects[count] = obj; 2070 | Py_INCREF(obj); 2071 | 2072 | obj = PyTuple_GET_ITEM(item, 1); 2073 | if (str_or_bytes_check(self->binary, obj, (const char**) &values[count], &value_sizes[count])) { 2074 | Py_DECREF(item); 2075 | is_ok = 0; 2076 | break; 2077 | } 2078 | 2079 | if (value_sizes[count] >= INT_MAX) { 2080 | PyErr_SetString(PyExc_OverflowError, "length of value is too large"); 2081 | return NULL; 2082 | } 2083 | 2084 | values_objects[count] = obj; 2085 | Py_INCREF(obj); 2086 | 2087 | Py_DECREF(item); 2088 | count++; 2089 | } 2090 | 2091 | int rc; 2092 | 2093 | if (is_ok) { 2094 | Py_BEGIN_ALLOW_THREADS 2095 | LSM_MutexLock(self); 2096 | for (int i=0; i < mapping_size; i++) { 2097 | if ((rc = lsm_insert(self->lsm, keys[i], (int) key_sizes[i], values[i], (int) value_sizes[i]))) break; 2098 | } 2099 | LSM_MutexLeave(self); 2100 | Py_END_ALLOW_THREADS 2101 | 2102 | if (pylsm_error(rc)) is_ok = 0; 2103 | } 2104 | 2105 | for (int i = 0; i < mapping_size && keys_objects[i] != NULL; i++) Py_DECREF(keys_objects[i]); 2106 | for (int i = 0; i < mapping_size && values_objects[i] != NULL; i++) Py_DECREF(values_objects[i]); 2107 | 2108 | PyMem_Free(key_sizes); 2109 | PyMem_Free(value_sizes); 2110 | PyMem_Free(keys); 2111 | PyMem_Free(values); 2112 | PyMem_Free(keys_objects); 2113 | PyMem_Free(values_objects); 2114 | 2115 | Py_CLEAR(items); 2116 | Py_CLEAR(iterator); 2117 | 2118 | if (is_ok) { 2119 | Py_RETURN_NONE; 2120 | } else { 2121 | return NULL; 2122 | } 2123 | } 2124 | 2125 | 2126 | static LSMTransaction* LSM_transaction(LSM* self) { 2127 | LSM_begin(self); 2128 | if (PyErr_Occurred()) return NULL; 2129 | 2130 | LSMTransaction* tx = (LSMTransaction*) LSMTransaction_new(&LSMTransactionType, self); 2131 | if (PyErr_Occurred()) return NULL; 2132 | 2133 | return tx; 2134 | } 2135 | 2136 | 2137 | static PyMemberDef LSM_members[] = { 2138 | { 2139 | "path", 2140 | T_STRING, 2141 | offsetof(LSM, path), 2142 | READONLY, 2143 | "path" 2144 | }, 2145 | { 2146 | "compressed", 2147 | T_BOOL, 2148 | offsetof(LSM, compressed), 2149 | READONLY, 2150 | "compressed" 2151 | }, 2152 | { 2153 | "state", 2154 | T_INT, 2155 | offsetof(LSM, state), 2156 | READONLY, 2157 | "state" 2158 | }, 2159 | { 2160 | "page_size", 2161 | T_INT, 2162 | offsetof(LSM, page_size), 2163 | READONLY, 2164 | "page_size" 2165 | }, 2166 | { 2167 | "block_size", 2168 | T_INT, 2169 | offsetof(LSM, block_size), 2170 | READONLY, 2171 | "block_size" 2172 | }, 2173 | { 2174 | "safety", 2175 | T_INT, 2176 | offsetof(LSM, safety), 2177 | READONLY, 2178 | "safety" 2179 | }, 2180 | { 2181 | "autowork", 2182 | T_BOOL, 2183 | offsetof(LSM, autowork), 2184 | READONLY, 2185 | "autowork" 2186 | }, 2187 | { 2188 | "autocheckpoint", 2189 | T_INT, 2190 | offsetof(LSM, autocheckpoint), 2191 | READONLY, 2192 | "autocheckpoint" 2193 | }, 2194 | { 2195 | "mmap", 2196 | T_BOOL, 2197 | offsetof(LSM, mmap), 2198 | READONLY, 2199 | "mmap" 2200 | }, 2201 | { 2202 | "use_log", 2203 | T_BOOL, 2204 | offsetof(LSM, use_log), 2205 | READONLY, 2206 | "use_log" 2207 | }, 2208 | { 2209 | "automerge", 2210 | T_INT, 2211 | offsetof(LSM, automerge), 2212 | READONLY, 2213 | "automerge" 2214 | }, 2215 | { 2216 | "max_freelist", 2217 | T_INT, 2218 | offsetof(LSM, max_freelist), 2219 | READONLY, 2220 | "max_freelist" 2221 | }, 2222 | { 2223 | "multiple_processes", 2224 | T_BOOL, 2225 | offsetof(LSM, multiple_processes), 2226 | READONLY, 2227 | "multiple_processes" 2228 | }, 2229 | { 2230 | "readonly", 2231 | T_BOOL, 2232 | offsetof(LSM, readonly), 2233 | READONLY, 2234 | "readonly" 2235 | }, 2236 | { 2237 | "binary", 2238 | T_BOOL, 2239 | offsetof(LSM, binary), 2240 | READONLY, 2241 | "binary" 2242 | }, 2243 | { 2244 | "compress_level", 2245 | T_INT, 2246 | offsetof(LSM, compress_level), 2247 | READONLY, 2248 | "compress_level" 2249 | }, 2250 | { 2251 | "tx_level", 2252 | T_INT, 2253 | offsetof(LSM, tx_level), 2254 | READONLY, 2255 | "Transaction nesting level" 2256 | }, 2257 | {NULL} /* Sentinel */ 2258 | }; 2259 | 2260 | 2261 | static PyMethodDef LSM_methods[] = { 2262 | { 2263 | "__enter__", 2264 | (PyCFunction) LSM_ctx_enter, METH_NOARGS, 2265 | "Enter context" 2266 | }, 2267 | { 2268 | "__exit__", 2269 | (PyCFunction) LSM_ctx_exit, METH_VARARGS | METH_KEYWORDS, 2270 | "Exit context" 2271 | }, 2272 | { 2273 | "open", 2274 | (PyCFunction) LSM_open, METH_NOARGS, 2275 | "Open database" 2276 | }, 2277 | { 2278 | "close", 2279 | (PyCFunction) LSM_close, METH_NOARGS, 2280 | "Close database" 2281 | }, 2282 | { 2283 | "info", 2284 | (PyCFunction) LSM_info, METH_NOARGS, 2285 | "Database info" 2286 | }, 2287 | { 2288 | "work", 2289 | (PyCFunction) LSM_work, METH_VARARGS | METH_KEYWORDS, 2290 | "Explicit Database work" 2291 | }, 2292 | { 2293 | "flush", 2294 | (PyCFunction) LSM_flush, METH_NOARGS, 2295 | "Explicit Database flush" 2296 | }, 2297 | { 2298 | "checkpoint", 2299 | (PyCFunction) LSM_checkpoint, METH_NOARGS, 2300 | "Explicit Database checkpointing" 2301 | }, 2302 | { 2303 | "cursor", 2304 | (PyCFunction) LSM_cursor, METH_VARARGS | METH_KEYWORDS, 2305 | "Create a cursor" 2306 | }, 2307 | { 2308 | "insert", 2309 | (PyCFunction) LSM_insert, METH_VARARGS | METH_KEYWORDS, 2310 | "Insert key and value" 2311 | }, 2312 | { 2313 | "delete", 2314 | (PyCFunction) LSM_delete, METH_VARARGS | METH_KEYWORDS, 2315 | "Delete value by key" 2316 | }, 2317 | { 2318 | "delete_range", 2319 | (PyCFunction) LSM_delete_range, METH_VARARGS | METH_KEYWORDS, 2320 | "Delete values by range" 2321 | }, 2322 | { 2323 | "begin", 2324 | (PyCFunction) LSM_begin, METH_NOARGS, 2325 | "Start transaction" 2326 | }, 2327 | { 2328 | "commit", 2329 | (PyCFunction) LSM_commit, METH_NOARGS, 2330 | "Commit transaction" 2331 | }, 2332 | { 2333 | "rollback", 2334 | (PyCFunction) LSM_rollback, METH_NOARGS, 2335 | "Rollback transaction" 2336 | }, 2337 | { 2338 | "transaction", 2339 | (PyCFunction) LSM_transaction, METH_NOARGS, 2340 | "Return transaction instance" 2341 | }, 2342 | { 2343 | "tx", 2344 | (PyCFunction) LSM_transaction, METH_NOARGS, 2345 | "Alias of transaction method" 2346 | }, 2347 | { 2348 | "keys", 2349 | (PyCFunction) LSM_keys, METH_NOARGS, 2350 | "Returns lsm_keys instance" 2351 | }, 2352 | { 2353 | "values", 2354 | (PyCFunction) LSM_values, METH_NOARGS, 2355 | "Returns lsm_keys instance" 2356 | }, 2357 | { 2358 | "items", 2359 | (PyCFunction) LSM_items, METH_NOARGS, 2360 | "Returns lsm_keys instance" 2361 | }, 2362 | { 2363 | "update", 2364 | (PyCFunction) LSM_update, METH_VARARGS, 2365 | "dict-like update method" 2366 | 2367 | }, 2368 | {NULL} /* Sentinel */ 2369 | }; 2370 | 2371 | 2372 | static PyGetSetDef LSMTypeGetSet[] = { 2373 | { 2374 | .name = "compress", 2375 | .get = (PyObject *(*)(PyObject *, void *)) LSM_compress_get, 2376 | .doc = "Compression algorithm" 2377 | }, 2378 | {NULL} /* Sentinel */ 2379 | }; 2380 | 2381 | static PyMappingMethods LSMTypeMapping = { 2382 | .mp_subscript = (binaryfunc) LSM_getitem, 2383 | .mp_ass_subscript = (objobjargproc) LSM_set_del_item, 2384 | .mp_length = (lenfunc) LSM_length 2385 | }; 2386 | 2387 | 2388 | static PySequenceMethods LSMTypeSequence = { 2389 | .sq_contains = (objobjproc) LSM_contains 2390 | }; 2391 | 2392 | 2393 | static PyTypeObject LSMType = { 2394 | PyVarObject_HEAD_INIT(NULL, 0) 2395 | .tp_name = "LSM", 2396 | .tp_doc = "", 2397 | .tp_basicsize = sizeof(LSM), 2398 | .tp_itemsize = 0, 2399 | .tp_flags = Py_TPFLAGS_DEFAULT, 2400 | .tp_new = LSM_new, 2401 | .tp_init = (initproc) LSM_init, 2402 | .tp_dealloc = (destructor) LSM_dealloc, 2403 | .tp_members = LSM_members, 2404 | .tp_methods = LSM_methods, 2405 | .tp_repr = (reprfunc) LSM_repr, 2406 | .tp_as_mapping = &LSMTypeMapping, 2407 | .tp_as_sequence = &LSMTypeSequence, 2408 | .tp_getset = (struct PyGetSetDef *) &LSMTypeGetSet, 2409 | .tp_iter = (getiterfunc) LSM_iter, 2410 | .tp_weaklistoffset = offsetof(LSM, weakrefs) 2411 | }; 2412 | 2413 | 2414 | static PyObject* LSMCursor_new(PyTypeObject *type, LSM *db, int seek_mode) { 2415 | if (pylsm_ensure_opened(db)) return NULL; 2416 | 2417 | LSMCursor *self; 2418 | 2419 | self = (LSMCursor *) type->tp_alloc(type, 0); 2420 | self->state = PY_LSM_INITIALIZED; 2421 | self->db = db; 2422 | 2423 | self->seek_mode = seek_mode; 2424 | 2425 | int rc; 2426 | 2427 | LSM_MutexLock(db); 2428 | rc = lsm_csr_open(self->db->lsm, &self->cursor); 2429 | LSM_MutexLeave(db); 2430 | 2431 | if(pylsm_error(rc)) return NULL; 2432 | 2433 | Py_BEGIN_ALLOW_THREADS 2434 | LSM_MutexLock(self->db); 2435 | rc = lsm_csr_first(self->cursor); 2436 | LSM_MutexLeave(self->db); 2437 | Py_END_ALLOW_THREADS 2438 | 2439 | if (pylsm_error(rc)) return NULL; 2440 | 2441 | self->state = PY_LSM_OPENED; 2442 | 2443 | Py_INCREF(self->db); 2444 | 2445 | return (PyObject *) self; 2446 | } 2447 | 2448 | 2449 | static void LSMCursor_dealloc(LSMCursor *self) { 2450 | if (self->state != PY_LSM_CLOSED && self->cursor != NULL) { 2451 | lsm_csr_close(self->cursor); 2452 | self->cursor = NULL; 2453 | self->state = PY_LSM_CLOSED; 2454 | } 2455 | 2456 | if (self->db != NULL) { 2457 | Py_DECREF(self->db); 2458 | self->db = NULL; 2459 | } 2460 | 2461 | if (self->weakrefs != NULL) PyObject_ClearWeakRefs((PyObject *) self); 2462 | } 2463 | 2464 | 2465 | static PyObject* LSMCursor_first(LSMCursor *self) { 2466 | if (self->state == PY_LSM_ITERATING) { 2467 | PyErr_SetString(PyExc_RuntimeError, "can not change cursor during iteration"); 2468 | return NULL; 2469 | } 2470 | 2471 | if (pylsm_ensure_csr_opened(self)) return NULL; 2472 | int result; 2473 | 2474 | Py_BEGIN_ALLOW_THREADS 2475 | LSM_MutexLock(self->db); 2476 | result = lsm_csr_first(self->cursor); 2477 | LSM_MutexLeave(self->db); 2478 | Py_END_ALLOW_THREADS 2479 | 2480 | if (pylsm_error(result)) return NULL; 2481 | self->state = PY_LSM_OPENED; 2482 | 2483 | if (!lsm_csr_valid(self->cursor)) Py_RETURN_FALSE; 2484 | 2485 | Py_RETURN_TRUE; 2486 | } 2487 | 2488 | 2489 | static PyObject* LSMCursor_last(LSMCursor *self) { 2490 | if (self->state == PY_LSM_ITERATING) { 2491 | PyErr_SetString(PyExc_RuntimeError, "can not change cursor during iteration"); 2492 | return NULL; 2493 | } 2494 | if (pylsm_ensure_csr_opened(self)) return NULL; 2495 | int result; 2496 | 2497 | Py_BEGIN_ALLOW_THREADS 2498 | LSM_MutexLock(self->db); 2499 | result = lsm_csr_last(self->cursor); 2500 | LSM_MutexLeave(self->db); 2501 | Py_END_ALLOW_THREADS 2502 | 2503 | if (pylsm_error(result)) return NULL; 2504 | self->state = PY_LSM_OPENED; 2505 | 2506 | if (!lsm_csr_valid(self->cursor)) Py_RETURN_FALSE; 2507 | 2508 | Py_RETURN_TRUE; 2509 | } 2510 | 2511 | 2512 | static PyObject* LSMCursor_close(LSMCursor *self) { 2513 | switch (self->state == PY_LSM_CLOSED) { 2514 | PyErr_SetString(PyExc_RuntimeError, "Cursor closed"); 2515 | return NULL; 2516 | } 2517 | 2518 | int result; 2519 | result = lsm_csr_close(self->cursor); 2520 | 2521 | if (pylsm_error(result)) return NULL; 2522 | 2523 | if (self->db != NULL) Py_DECREF(self->db); 2524 | self->db = NULL; 2525 | 2526 | self->cursor = NULL; 2527 | self->state = PY_LSM_CLOSED; 2528 | Py_RETURN_NONE; 2529 | } 2530 | 2531 | static PyObject* LSMCursor_seek(LSMCursor *self, PyObject* args, PyObject* kwds) { 2532 | if (self->state == PY_LSM_ITERATING) { 2533 | PyErr_SetString(PyExc_RuntimeError, "can not change cursor during iteration"); 2534 | return NULL; 2535 | } 2536 | 2537 | if (pylsm_ensure_csr_opened(self)) return NULL; 2538 | static char *kwlist[] = {"key", "seek_mode", NULL}; 2539 | 2540 | self->seek_mode = LSM_SEEK_EQ; 2541 | 2542 | PyObject* key = NULL; 2543 | const char* pKey = NULL; 2544 | Py_ssize_t nKey = 0; 2545 | 2546 | if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|i", kwlist, &key, &self->seek_mode)) return NULL; 2547 | if (pylsm_seek_mode_check(self->seek_mode)) return NULL; 2548 | 2549 | int rc; 2550 | 2551 | if (str_or_bytes_check(self->db->binary, key, &pKey, &nKey)) return NULL; 2552 | 2553 | if (nKey >= INT_MAX) { 2554 | PyErr_SetString(PyExc_OverflowError, "length of key is too large"); 2555 | return NULL; 2556 | } 2557 | Py_BEGIN_ALLOW_THREADS 2558 | LSM_MutexLock(self->db); 2559 | rc = lsm_csr_seek(self->cursor, pKey, (int) nKey, self->seek_mode); 2560 | LSM_MutexLeave(self->db); 2561 | Py_END_ALLOW_THREADS 2562 | 2563 | if (pylsm_error(rc)) return NULL; 2564 | if (lsm_csr_valid(self->cursor)) { Py_RETURN_TRUE; } else { Py_RETURN_FALSE; } 2565 | } 2566 | 2567 | 2568 | static PyObject* LSMCursor_compare(LSMCursor *self, PyObject* args, PyObject* kwds) { 2569 | if (pylsm_ensure_csr_opened(self)) return NULL; 2570 | 2571 | if (!lsm_csr_valid(self->cursor)) { 2572 | PyErr_SetString(PyExc_RuntimeError, "Invalid cursor"); 2573 | return NULL; 2574 | }; 2575 | 2576 | static char *kwlist[] = {"key", NULL}; 2577 | 2578 | PyObject * key = NULL; 2579 | const char* pKey = NULL; 2580 | Py_ssize_t nKey = 0; 2581 | 2582 | if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &key)) return NULL; 2583 | if (str_or_bytes_check(self->db->binary, key, &pKey, &nKey)) return NULL; 2584 | 2585 | int cmp_result = 0; 2586 | int result; 2587 | 2588 | if (nKey >= INT_MAX) { 2589 | PyErr_SetString(PyExc_OverflowError, "length of key is too large"); 2590 | return NULL; 2591 | } 2592 | 2593 | LSM_MutexLock(self->db); 2594 | result = lsm_csr_cmp(self->cursor, pKey, (int) nKey, &cmp_result); 2595 | LSM_MutexLeave(self->db); 2596 | 2597 | if (self->seek_mode == LSM_SEEK_GE) cmp_result = -cmp_result; 2598 | 2599 | if (pylsm_error(result)) return NULL; 2600 | return Py_BuildValue("i", cmp_result); 2601 | } 2602 | 2603 | static PyObject* LSMCursor_retrieve(LSMCursor *self) { 2604 | if (self->state == PY_LSM_ITERATING) { 2605 | PyErr_SetString(PyExc_RuntimeError, "can not change cursor during iteration"); 2606 | return NULL; 2607 | } 2608 | if (pylsm_ensure_csr_opened(self)) return NULL; 2609 | if(!lsm_csr_valid(self->cursor)) { Py_RETURN_NONE; } 2610 | 2611 | LSM_MutexLock(self->db); 2612 | PyObject* result = pylsm_cursor_items_fetch(self->cursor, self->db->binary); 2613 | LSM_MutexLeave(self->db); 2614 | return result; 2615 | } 2616 | 2617 | 2618 | static PyObject* LSMCursor_key(LSMCursor *self) { 2619 | if (self->state == PY_LSM_ITERATING) { 2620 | PyErr_SetString(PyExc_RuntimeError, "can not change cursor during iteration"); 2621 | return NULL; 2622 | } 2623 | if (pylsm_ensure_csr_opened(self)) return NULL; 2624 | if(!lsm_csr_valid(self->cursor)) { Py_RETURN_NONE; } 2625 | 2626 | LSM_MutexLock(self->db); 2627 | PyObject* result = pylsm_cursor_key_fetch(self->cursor, self->db->binary); 2628 | LSM_MutexLeave(self->db); 2629 | 2630 | return result; 2631 | } 2632 | 2633 | 2634 | static PyObject* LSMCursor_value(LSMCursor *self) { 2635 | if (self->state == PY_LSM_ITERATING) { 2636 | PyErr_SetString(PyExc_RuntimeError, "can not change cursor during iteration"); 2637 | return NULL; 2638 | } 2639 | if (pylsm_ensure_csr_opened(self)) return NULL; 2640 | if(!lsm_csr_valid(self->cursor)) { Py_RETURN_NONE; } 2641 | 2642 | LSM_MutexLock(self->db); 2643 | PyObject* result = pylsm_cursor_value_fetch(self->cursor, self->db->binary); 2644 | LSM_MutexLeave(self->db); 2645 | 2646 | return result; 2647 | } 2648 | 2649 | 2650 | static PyObject* LSMCursor_next(LSMCursor *self) { 2651 | if (self->state == PY_LSM_ITERATING) { 2652 | PyErr_SetString(PyExc_RuntimeError, "can not change cursor during iteration"); 2653 | return NULL; 2654 | } 2655 | if (pylsm_ensure_csr_opened(self)) return NULL; 2656 | if (self->seek_mode == LSM_SEEK_EQ) Py_RETURN_FALSE; 2657 | if (!lsm_csr_valid(self->cursor)) Py_RETURN_FALSE; 2658 | 2659 | int err; 2660 | Py_BEGIN_ALLOW_THREADS 2661 | LSM_MutexLock(self->db); 2662 | err = lsm_csr_next(self->cursor); 2663 | LSM_MutexLeave(self->db); 2664 | Py_END_ALLOW_THREADS 2665 | 2666 | if (pylsm_error(err)) return NULL; 2667 | 2668 | if (!lsm_csr_valid(self->cursor)) Py_RETURN_FALSE; 2669 | Py_RETURN_TRUE; 2670 | } 2671 | 2672 | 2673 | static PyObject* LSMCursor_previous(LSMCursor *self) { 2674 | if (self->state == PY_LSM_ITERATING) { 2675 | PyErr_SetString(PyExc_RuntimeError, "can not change cursor during iteration"); 2676 | return NULL; 2677 | } 2678 | if (pylsm_ensure_csr_opened(self)) return NULL; 2679 | if (self->seek_mode == LSM_SEEK_EQ) { 2680 | PyErr_SetString(PyExc_RuntimeError, "can not seek in SEEK_EQ mode"); 2681 | return NULL; 2682 | }; 2683 | 2684 | if (!lsm_csr_valid(self->cursor)) Py_RETURN_FALSE; 2685 | 2686 | int err; 2687 | Py_BEGIN_ALLOW_THREADS 2688 | LSM_MutexLock(self->db); 2689 | err = lsm_csr_prev(self->cursor); 2690 | LSM_MutexLeave(self->db); 2691 | Py_END_ALLOW_THREADS 2692 | 2693 | if (pylsm_error(err)) return NULL; 2694 | if (!lsm_csr_valid(self->cursor)) Py_RETURN_FALSE; 2695 | Py_RETURN_TRUE; 2696 | } 2697 | 2698 | 2699 | static PyObject* LSMCursor_ctx_enter(LSMCursor *self) { 2700 | if (self->state == PY_LSM_ITERATING) { 2701 | PyErr_SetString(PyExc_RuntimeError, "can not change cursor during iteration"); 2702 | return NULL; 2703 | } 2704 | if (pylsm_ensure_csr_opened(self)) return NULL; 2705 | return (PyObject*) self; 2706 | } 2707 | 2708 | 2709 | static PyObject* LSMCursor_ctx_exit(LSMCursor *self) { 2710 | if (self->state == PY_LSM_CLOSED) { Py_RETURN_NONE; }; 2711 | 2712 | LSMCursor_close(self); 2713 | if (PyErr_Occurred()) return NULL; 2714 | 2715 | Py_RETURN_NONE; 2716 | } 2717 | 2718 | 2719 | static PyObject* LSMCursor_repr(LSMCursor *self) { 2720 | return PyUnicode_FromFormat( 2721 | "<%s as %p>", 2722 | Py_TYPE(self)->tp_name, self 2723 | ); 2724 | } 2725 | 2726 | 2727 | static PyMemberDef LSMCursor_members[] = { 2728 | { 2729 | "state", 2730 | T_INT, 2731 | offsetof(LSMCursor, state), 2732 | READONLY, 2733 | "state" 2734 | }, 2735 | { 2736 | "seek_mode", 2737 | T_INT, 2738 | offsetof(LSMCursor, seek_mode), 2739 | READONLY, 2740 | "seek_mode" 2741 | }, 2742 | {NULL} /* Sentinel */ 2743 | }; 2744 | 2745 | 2746 | static PyMethodDef LSMCursor_methods[] = { 2747 | { 2748 | "__enter__", 2749 | (PyCFunction) LSMCursor_ctx_enter, METH_NOARGS, 2750 | "Enter context" 2751 | }, 2752 | { 2753 | "__exit__", 2754 | (PyCFunction) LSMCursor_ctx_exit, METH_VARARGS | METH_KEYWORDS, 2755 | "Exit context" 2756 | }, 2757 | { 2758 | "close", 2759 | (PyCFunction) LSMCursor_close, METH_NOARGS, 2760 | "Close database" 2761 | }, 2762 | { 2763 | "first", 2764 | (PyCFunction) LSMCursor_first, METH_NOARGS, 2765 | "Move cursor to first item" 2766 | }, 2767 | { 2768 | "last", 2769 | (PyCFunction) LSMCursor_last, METH_NOARGS, 2770 | "Move cursor to last item" 2771 | }, 2772 | { 2773 | "seek", 2774 | (PyCFunction) LSMCursor_seek, METH_VARARGS | METH_KEYWORDS, 2775 | "Seek to key" 2776 | }, 2777 | { 2778 | "retrieve", 2779 | (PyCFunction) LSMCursor_retrieve, METH_NOARGS, 2780 | "Retrieve key and value" 2781 | }, 2782 | { 2783 | "key", 2784 | (PyCFunction) LSMCursor_key, METH_NOARGS, 2785 | "Retrieve key" 2786 | }, 2787 | { 2788 | "value", 2789 | (PyCFunction) LSMCursor_value, METH_NOARGS, 2790 | "Retrieve value" 2791 | }, 2792 | { 2793 | "next", 2794 | (PyCFunction) LSMCursor_next, METH_NOARGS, 2795 | "Seek next" 2796 | }, 2797 | { 2798 | "previous", 2799 | (PyCFunction) LSMCursor_previous, METH_NOARGS, 2800 | "Seek previous" 2801 | }, 2802 | { 2803 | "compare", 2804 | (PyCFunction) LSMCursor_compare, METH_VARARGS | METH_KEYWORDS, 2805 | "Compare current position against key" 2806 | }, 2807 | 2808 | {NULL} /* Sentinel */ 2809 | }; 2810 | 2811 | static PyTypeObject LSMCursorType = { 2812 | PyVarObject_HEAD_INIT(NULL, 0) 2813 | .tp_name = "Cursor", 2814 | .tp_doc = "", 2815 | .tp_basicsize = sizeof(LSMCursor), 2816 | .tp_itemsize = 0, 2817 | .tp_flags = Py_TPFLAGS_DEFAULT, 2818 | .tp_dealloc = (destructor) LSMCursor_dealloc, 2819 | .tp_members = LSMCursor_members, 2820 | .tp_methods = LSMCursor_methods, 2821 | .tp_repr = (reprfunc) LSMCursor_repr, 2822 | .tp_weaklistoffset = offsetof(LSMCursor, weakrefs) 2823 | }; 2824 | 2825 | 2826 | static PyObject* LSMTransaction_new(PyTypeObject *type, LSM* db) { 2827 | LSMTransaction *self; 2828 | 2829 | self = (LSMTransaction *) type->tp_alloc(type, 0); 2830 | self->state = PY_LSM_INITIALIZED; 2831 | self->db = db; 2832 | self->tx_level = self->db->tx_level; 2833 | 2834 | Py_INCREF(self->db); 2835 | 2836 | return (PyObject *) self; 2837 | } 2838 | 2839 | 2840 | static void LSMTransaction_dealloc(LSMTransaction *self) { 2841 | if (self->weakrefs != NULL) PyObject_ClearWeakRefs((PyObject *) self); 2842 | if (self->db == NULL) return; 2843 | 2844 | Py_DECREF(self->db); 2845 | if (self->state != PY_LSM_CLOSED && self->db->state != PY_LSM_CLOSED) { 2846 | LSM_rollback_inner(self->db, self->tx_level); 2847 | } 2848 | } 2849 | 2850 | 2851 | static PyObject* LSMTransaction_ctx_enter(LSMTransaction *self) { 2852 | if (pylsm_ensure_writable(self->db)) return NULL; 2853 | return (PyObject*) self; 2854 | } 2855 | 2856 | 2857 | static PyObject* LSMTransaction_commit(LSMTransaction *self); 2858 | static PyObject* LSMTransaction_rollback(LSMTransaction *self); 2859 | 2860 | 2861 | static PyObject* LSMTransaction_ctx_exit( 2862 | LSMTransaction *self, PyObject *args 2863 | ) { 2864 | if (self->state == PY_LSM_CLOSED) Py_RETURN_NONE; 2865 | 2866 | PyObject *exc_type, *exc_value, *exc_tb; 2867 | if (!PyArg_ParseTuple(args, "OOO", &exc_type, &exc_value, &exc_tb)) return NULL; 2868 | 2869 | self->state = PY_LSM_CLOSED; 2870 | 2871 | if (exc_type == Py_None) { 2872 | LSM_commit_inner(self->db, self->tx_level - 1); 2873 | } else { 2874 | LSM_rollback_inner(self->db, self->tx_level); 2875 | } 2876 | 2877 | if (PyErr_Occurred()) return NULL; 2878 | 2879 | Py_RETURN_NONE; 2880 | } 2881 | 2882 | 2883 | static PyObject* LSMTransaction_commit(LSMTransaction *self) { 2884 | PyObject * result = LSM_commit_inner(self->db, self->tx_level -1); 2885 | if (PyErr_Occurred()) return NULL; 2886 | if (pylsm_error(lsm_begin(self->db->lsm, self->tx_level))) return NULL; 2887 | return result; 2888 | } 2889 | 2890 | 2891 | static PyObject* LSMTransaction_rollback(LSMTransaction *self) { 2892 | return LSM_rollback_inner(self->db, self->tx_level); 2893 | } 2894 | 2895 | 2896 | 2897 | static PyMemberDef LSMTransaction_members[] = { 2898 | { 2899 | "level", 2900 | T_INT, 2901 | offsetof(LSMTransaction, tx_level), 2902 | READONLY, 2903 | "Transaction level" 2904 | }, 2905 | {NULL} /* Sentinel */ 2906 | }; 2907 | 2908 | 2909 | static PyMethodDef LSMTransaction_methods[] = { 2910 | { 2911 | "__enter__", 2912 | (PyCFunction) LSMTransaction_ctx_enter, METH_NOARGS, 2913 | "Enter context" 2914 | }, 2915 | { 2916 | "__exit__", 2917 | (PyCFunction) LSMTransaction_ctx_exit, METH_VARARGS, 2918 | "Exit context" 2919 | }, 2920 | { 2921 | "commit", 2922 | (PyCFunction) LSMTransaction_commit, METH_NOARGS, 2923 | "Commit transaction" 2924 | }, 2925 | { 2926 | "rollback", 2927 | (PyCFunction) LSMTransaction_rollback, METH_NOARGS, 2928 | "Rollback transaction" 2929 | }, 2930 | 2931 | {NULL} /* Sentinel */ 2932 | }; 2933 | 2934 | static PyTypeObject LSMTransactionType = { 2935 | PyVarObject_HEAD_INIT(NULL, 0) 2936 | .tp_name = "Transaction", 2937 | .tp_doc = "", 2938 | .tp_basicsize = sizeof(LSMTransaction), 2939 | .tp_itemsize = 0, 2940 | .tp_flags = Py_TPFLAGS_DEFAULT, 2941 | .tp_dealloc = (destructor) LSMTransaction_dealloc, 2942 | .tp_methods = LSMTransaction_methods, 2943 | .tp_members = LSMTransaction_members, 2944 | .tp_weaklistoffset = offsetof(LSMTransaction, weakrefs) 2945 | }; 2946 | 2947 | 2948 | static PyModuleDef lsm_module = { 2949 | PyModuleDef_HEAD_INIT, 2950 | .m_name = "_lsm", 2951 | .m_doc = "LSM DB python binding", 2952 | .m_size = -1, 2953 | }; 2954 | 2955 | 2956 | PyMODINIT_FUNC PyInit__lsm(void) { 2957 | PyObject *m; 2958 | 2959 | m = PyModule_Create(&lsm_module); 2960 | 2961 | if (m == NULL) return NULL; 2962 | 2963 | if (PyType_Ready(&LSMType) < 0) return NULL; 2964 | Py_INCREF(&LSMType); 2965 | 2966 | if (PyModule_AddObject(m, "LSM", (PyObject *) &LSMType) < 0) { 2967 | Py_XDECREF(&LSMType); 2968 | Py_XDECREF(m); 2969 | return NULL; 2970 | } 2971 | 2972 | if (PyType_Ready(&LSMCursorType) < 0) return NULL; 2973 | Py_INCREF(&LSMCursorType); 2974 | 2975 | if (PyModule_AddObject(m, "Cursor", (PyObject *) &LSMCursorType) < 0) { 2976 | Py_XDECREF(&LSMCursorType); 2977 | Py_XDECREF(m); 2978 | return NULL; 2979 | } 2980 | 2981 | if (PyType_Ready(&LSMTransactionType) < 0) return NULL; 2982 | Py_INCREF(&LSMTransactionType); 2983 | 2984 | if (PyModule_AddObject(m, "Transaction", (PyObject *) &LSMTransactionType) < 0) { 2985 | Py_XDECREF(&LSMTransactionType); 2986 | Py_XDECREF(m); 2987 | return NULL; 2988 | } 2989 | 2990 | if (PyType_Ready(&LSMItemsType) < 0) return NULL; 2991 | Py_INCREF(&LSMItemsType); 2992 | 2993 | if (PyType_Ready(&LSMValuesType) < 0) return NULL; 2994 | Py_INCREF(&LSMValuesType); 2995 | 2996 | if (PyType_Ready(&LSMKeysType) < 0) return NULL; 2997 | Py_INCREF(&LSMKeysType); 2998 | 2999 | if (PyType_Ready(&LSMSliceType) < 0) return NULL; 3000 | Py_INCREF(&LSMSliceType); 3001 | 3002 | PyModule_AddIntConstant(m, "SAFETY_OFF", LSM_SAFETY_OFF); 3003 | PyModule_AddIntConstant(m, "SAFETY_NORMAL", LSM_SAFETY_NORMAL); 3004 | PyModule_AddIntConstant(m, "SAFETY_FULL", LSM_SAFETY_FULL); 3005 | 3006 | PyModule_AddIntConstant(m, "STATE_INITIALIZED", PY_LSM_INITIALIZED); 3007 | PyModule_AddIntConstant(m, "STATE_OPENED", PY_LSM_OPENED); 3008 | PyModule_AddIntConstant(m, "STATE_CLOSED", PY_LSM_CLOSED); 3009 | 3010 | PyModule_AddIntConstant(m, "SEEK_EQ", LSM_SEEK_EQ); 3011 | PyModule_AddIntConstant(m, "SEEK_LE", LSM_SEEK_LE); 3012 | PyModule_AddIntConstant(m, "SEEK_GE", LSM_SEEK_GE); 3013 | PyModule_AddIntConstant(m, "SEEK_LEFAST", LSM_SEEK_LEFAST); 3014 | 3015 | return m; 3016 | } 3017 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | comp_algo = ["none", "lz4", "zstd"] 2 | -------------------------------------------------------------------------------- /tests/test_arguments.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from lsm import LSM 3 | 4 | 5 | def test_argument_checks(subtests, tmp_path): 6 | with subtests.test("blank context manager"): 7 | with LSM(str(tmp_path / "test-filled.lsm"), binary=False) as db: 8 | for i in range(1000): 9 | db[str(i)] = str(i) 10 | 11 | with LSM(str(tmp_path / "test-filled.lsm")): 12 | pass 13 | 14 | with subtests.test("autoflush=1048577"), pytest.raises(ValueError): 15 | LSM(str(tmp_path / "test.lsm"), autoflush=1048577) 16 | 17 | with subtests.test("autoflush=-1"), pytest.raises(ValueError): 18 | LSM(str(tmp_path / "test.lsm"), autoflush=-1) 19 | 20 | with subtests.test("autocheckpoint=0"), pytest.raises(ValueError): 21 | LSM(str(tmp_path / "test.lsm"), autocheckpoint=0) 22 | 23 | with subtests.test("autocheckpoint=-1"), pytest.raises(ValueError): 24 | LSM(str(tmp_path / "test.lsm"), autocheckpoint=-1) 25 | 26 | with subtests.test("block_size=65"), pytest.raises(ValueError): 27 | LSM(str(tmp_path / "test.lsm"), block_size=65) 28 | 29 | with subtests.test("safety=32"), pytest.raises(ValueError): 30 | LSM(str(tmp_path / "test.lsm"), safety=32) 31 | 32 | with subtests.test("compress='zip'"), pytest.raises(ValueError): 33 | LSM(str(tmp_path / "test.lsm"), compress='zip') 34 | 35 | with subtests.test("logger=1"), pytest.raises(ValueError): 36 | LSM(str(tmp_path / "test.lsm"), logger=1) 37 | 38 | 39 | -------------------------------------------------------------------------------- /tests/test_common_cases.py: -------------------------------------------------------------------------------- 1 | import struct 2 | 3 | import pytest 4 | from lsm import LSM, SEEK_LE, SEEK_GE, SEEK_EQ, SEEK_LEFAST 5 | 6 | from tests import comp_algo 7 | 8 | 9 | @pytest.fixture(params=comp_algo) 10 | def db(request, tmp_path): 11 | with LSM( 12 | tmp_path / ("db.lsm" + request.param), 13 | compress=request.param, 14 | binary=False 15 | ) as db: 16 | yield db 17 | 18 | 19 | @pytest.fixture(params=comp_algo) 20 | def db_binary(request, tmp_path): 21 | with LSM( 22 | tmp_path / ("db.lsm" + request.param), 23 | compress=request.param, 24 | binary=True 25 | ) as db: 26 | yield db 27 | 28 | 29 | @pytest.mark.parametrize("n", range(1, 10)) 30 | def test_ranges(n, subtests, db_binary: LSM): 31 | def make_key(*args): 32 | result = struct.pack("!" + "b" * len(args), *args) 33 | return result 34 | 35 | for i in range(n): 36 | for j in range(n): 37 | key = make_key(i, j, 0) 38 | db_binary[key] = b"\x00" 39 | 40 | with subtests.test("one key"): 41 | s = list(db_binary[make_key(0):make_key(1)]) 42 | assert len(s) == n, s 43 | 44 | 45 | def test_insert_select(subtests, db): 46 | with subtests.test("one key"): 47 | db["foo"] = "bar" 48 | assert db['foo'] == 'bar' 49 | assert 'foo' in db 50 | assert len(db) == 1 51 | assert list(db) == ['foo'] 52 | assert list(db.keys()) == ['foo'] 53 | assert list(db.values()) == ['bar'] 54 | assert list(db.items()) == [('foo', 'bar')] 55 | 56 | del db['foo'] 57 | 58 | assert 'foo' not in db 59 | assert len(db) == 0 60 | assert list(db) == [] 61 | assert list(db.keys()) == [] 62 | assert list(db.values()) == [] 63 | assert list(db.items()) == [] 64 | 65 | with subtests.test("100 keys"): 66 | for i in range(100): 67 | db['k{}'.format(i)] = str(i) 68 | 69 | assert len(db) == 100 70 | assert set(db) == set('k{}'.format(i) for i in range(100)) 71 | assert set(db.keys()) == set('k{}'.format(i) for i in range(100)) 72 | assert set(db.values()) == set(str(i) for i in range(100)) 73 | assert set(db.items()) == set( 74 | ('k{}'.format(i), str(i)) for i in range(100) 75 | ) 76 | 77 | with subtests.test("slice select ['k90':'k99']"): 78 | assert list(db['k90':'k99']) == list( 79 | ('k{}'.format(i), str(i)) for i in range(90, 100) 80 | ) 81 | 82 | with subtests.test("slice select ['k90':'k99':-1]"): 83 | assert list(db['k90':'k99':-1]) == list( 84 | ('k{}'.format(i), str(i)) for i in range(99, 89, -1) 85 | ) 86 | 87 | with subtests.test("select ['k90xx', SEEK_LE]"): 88 | assert db['k90xx', SEEK_LE] == '90' 89 | 90 | with subtests.test("select ['k90xx', SEEK_LEFAST]"): 91 | assert db['k90xx', SEEK_LEFAST] 92 | 93 | with subtests.test("select ['k90xx', SEEK_GE]"): 94 | assert db['k90xx', SEEK_GE] == '91' 95 | 96 | with subtests.test("select ['k90xx', SEEK_GE]"): 97 | with pytest.raises(KeyError): 98 | _ = db['k90xx', SEEK_EQ] 99 | 100 | with subtests.test("delete range ['k':'z']"): 101 | del db['k':'l'] 102 | assert len(db) == 0 103 | 104 | with subtests.test("update"): 105 | db.update({"k{}".format(i): str(i) for i in range(100)}) 106 | assert len(db) == 100 107 | assert db['k19'] == '19' 108 | 109 | 110 | @pytest.mark.parametrize("comp", comp_algo) 111 | def test_info(comp, tmp_path): 112 | with LSM(tmp_path / ("test.lsm." + comp), compress=comp, 113 | binary=False) as db: 114 | for i in map(str, range(10000)): 115 | db[i] = i 116 | 117 | info = db.info() 118 | assert 'checkpoint_size_result' in info 119 | assert 'nread' in info 120 | assert 'nwrite' in info 121 | assert 'tree_size' in info 122 | 123 | with LSM(tmp_path / ("test.lsm." + comp), binary=False, 124 | compress=comp, readonly=True) as db: 125 | info = db.info() 126 | assert 'checkpoint_size_result' not in info 127 | assert 'nread' in info 128 | assert 'nwrite' not in info 129 | assert 'tree_size' not in info 130 | 131 | 132 | @pytest.mark.parametrize("comp", comp_algo) 133 | def test_len(comp, tmp_path): 134 | with LSM(tmp_path / ("test.lsm." + comp), compress=comp, 135 | binary=False) as db: 136 | for i in map(str, range(10000)): 137 | db[i] = i 138 | 139 | assert len(db) == 10000 140 | assert len(db.keys()) == 10000 141 | assert len(db.values()) == 10000 142 | assert len(db.items()) == 10000 143 | -------------------------------------------------------------------------------- /tests/test_dealloc.py: -------------------------------------------------------------------------------- 1 | from concurrent.futures import Future 2 | from typing import Any 3 | from weakref import finalize 4 | 5 | import lsm 6 | import pytest 7 | 8 | 9 | # noinspection PyMethodMayBeStatic,SpellCheckingInspection 10 | from tests import comp_algo 11 | 12 | 13 | class DeallocCases: 14 | TIMEOUT = 1 15 | 16 | def test_weakref_finalize(self, instance_maker): 17 | future = Future() 18 | finalize(instance_maker(), future.set_result, True) 19 | future.result(timeout=1) 20 | 21 | 22 | class TestDeallocClass(DeallocCases): 23 | """ 24 | Just for test base class 25 | """ 26 | 27 | class Instance: 28 | pass 29 | 30 | @pytest.fixture 31 | def instance_maker(self) -> Any: 32 | def maker(): 33 | return self.Instance() 34 | return maker 35 | 36 | 37 | class TestLSMDealloc(DeallocCases): 38 | @pytest.fixture(params=comp_algo) 39 | def instance_maker(self, request, tmp_path) -> Any: 40 | def maker(): 41 | return lsm.LSM( 42 | tmp_path / ("db.lsm." + request.param), 43 | compress=request.param 44 | ) 45 | return maker 46 | 47 | 48 | class TestLSMKeysDeallocCtx(DeallocCases): 49 | @pytest.fixture(params=comp_algo) 50 | def instance_maker(self, request, tmp_path) -> Any: 51 | def maker(): 52 | db = lsm.LSM( 53 | tmp_path / ("db.lsm." + request.param), 54 | compress=request.param 55 | ) 56 | db.open() 57 | return db.keys() 58 | return maker 59 | 60 | 61 | class TestLSMValuesDeallocCtx(DeallocCases): 62 | @pytest.fixture(params=comp_algo) 63 | def instance_maker(self, request, tmp_path) -> Any: 64 | def maker(): 65 | db = lsm.LSM( 66 | tmp_path / ("db.lsm." + request.param), 67 | compress=request.param 68 | ) 69 | db.open() 70 | return db.values() 71 | return maker 72 | 73 | 74 | class TestLSMItemsDeallocCtx(DeallocCases): 75 | @pytest.fixture(params=comp_algo) 76 | def instance_maker(self, request, tmp_path) -> Any: 77 | def maker(): 78 | db = lsm.LSM( 79 | tmp_path / ("db.lsm." + request.param), 80 | compress=request.param 81 | ) 82 | db.open() 83 | return db.items() 84 | return maker 85 | 86 | 87 | class TestLSMSliceDeallocCtx(DeallocCases): 88 | @pytest.fixture(params=comp_algo) 89 | def instance_maker(self, request, tmp_path) -> Any: 90 | def maker(): 91 | db = lsm.LSM( 92 | tmp_path / ("db.lsm." + request.param), 93 | compress=request.param 94 | ) 95 | db.open() 96 | return db[::-1] 97 | return maker 98 | 99 | 100 | class TestLSMCursorDeallocCtx(DeallocCases): 101 | @pytest.fixture(params=comp_algo) 102 | def instance_maker(self, request, tmp_path) -> Any: 103 | with lsm.LSM( 104 | tmp_path / ("db.lsm." + request.param), 105 | compress=request.param 106 | ) as db: 107 | yield db.cursor 108 | 109 | 110 | class TestLSMTransactionDeallocCtx(DeallocCases): 111 | @pytest.fixture(params=comp_algo) 112 | def instance_maker(self, request, tmp_path) -> Any: 113 | with lsm.LSM( 114 | tmp_path / ("db.lsm." + request.param), 115 | compress=request.param 116 | ) as db: 117 | yield db.transaction 118 | 119 | 120 | @pytest.fixture(params=[1, 10, 25, 50, 100, 256, 1024, 2048]) 121 | def filler(request): 122 | def fill_db(db: lsm.LSM): 123 | for i in range(request.param): 124 | db[str(i).encode()] = str(i).encode() 125 | return fill_db 126 | 127 | 128 | class TestFilledLSMDealloc(DeallocCases): 129 | @pytest.fixture(params=comp_algo) 130 | def instance_maker(self, request, tmp_path, filler) -> Any: 131 | def maker(): 132 | db = lsm.LSM( 133 | tmp_path / ("db.lsm." + request.param), 134 | compress=request.param 135 | ) 136 | db.open() 137 | filler(db) 138 | return db 139 | return maker 140 | 141 | 142 | class TestFilledLSMKeysDeallocCtx(DeallocCases): 143 | @pytest.fixture(params=comp_algo) 144 | def instance_maker(self, request, tmp_path, filler) -> Any: 145 | def maker(): 146 | db = lsm.LSM( 147 | tmp_path / ("db.lsm." + request.param), 148 | compress=request.param 149 | ) 150 | db.open() 151 | filler(db) 152 | return db.keys() 153 | return maker 154 | 155 | 156 | class TestFilledLSMValuesDeallocCtx(DeallocCases): 157 | @pytest.fixture(params=comp_algo) 158 | def instance_maker(self, request, tmp_path, filler) -> Any: 159 | def maker(): 160 | db = lsm.LSM( 161 | tmp_path / ("db.lsm." + request.param), 162 | compress=request.param 163 | ) 164 | db.open() 165 | filler(db) 166 | return db.values() 167 | return maker 168 | 169 | 170 | class TestFilledLSMItemsDeallocCtx(DeallocCases): 171 | @pytest.fixture(params=comp_algo) 172 | def instance_maker(self, request, tmp_path, filler) -> Any: 173 | def maker(): 174 | db = lsm.LSM( 175 | tmp_path / ("db.lsm." + request.param), 176 | compress=request.param 177 | ) 178 | db.open() 179 | filler(db) 180 | return db.items() 181 | return maker 182 | 183 | 184 | class TestFilledAndCheckLSMDeallocCtx(DeallocCases): 185 | @pytest.fixture(params=comp_algo) 186 | def instance_maker(self, request, tmp_path, filler) -> Any: 187 | def maker(): 188 | db = lsm.LSM( 189 | tmp_path / ("db.lsm." + request.param), 190 | compress=request.param 191 | ) 192 | db.open() 193 | filler(db) 194 | 195 | for key, value in db.items(): 196 | assert key == value, (key, value) 197 | 198 | return db.items() 199 | return maker 200 | 201 | 202 | class TestFilledIterLSMDealloc(DeallocCases): 203 | @pytest.fixture(params=comp_algo) 204 | def instance_maker(self, request, tmp_path, filler) -> Any: 205 | def maker(): 206 | db = lsm.LSM( 207 | tmp_path / ("db.lsm." + request.param), 208 | compress=request.param 209 | ) 210 | db.open() 211 | filler(db) 212 | return iter(db) 213 | return maker 214 | 215 | 216 | class TestFilledIterLSMKeysDeallocCtx(DeallocCases): 217 | @pytest.fixture(params=comp_algo) 218 | def instance_maker(self, request, tmp_path, filler) -> Any: 219 | def maker(): 220 | db = lsm.LSM( 221 | tmp_path / ("db.lsm." + request.param), 222 | compress=request.param 223 | ) 224 | db.open() 225 | filler(db) 226 | return iter(db.keys()) 227 | return maker 228 | 229 | 230 | class TestFilledSliceLSMKeysDeallocCtx(DeallocCases): 231 | @pytest.fixture(params=comp_algo) 232 | def instance_maker(self, request, tmp_path, filler) -> Any: 233 | def maker(): 234 | db = lsm.LSM( 235 | tmp_path / ("db.lsm." + request.param), 236 | compress=request.param 237 | ) 238 | db.open() 239 | filler(db) 240 | return iter(db.keys()) 241 | return maker 242 | 243 | 244 | class TestFilledIterLSMValuesDeallocCtx(DeallocCases): 245 | @pytest.fixture(params=comp_algo) 246 | def instance_maker(self, request, tmp_path, filler) -> Any: 247 | def maker(): 248 | db = lsm.LSM( 249 | tmp_path / ("db.lsm." + request.param), 250 | compress=request.param 251 | ) 252 | db.open() 253 | filler(db) 254 | return iter(db[::-1]) 255 | return maker 256 | 257 | 258 | class TestFilledIterLSMItemsDeallocCtx(DeallocCases): 259 | @pytest.fixture(params=comp_algo) 260 | def instance_maker(self, request, tmp_path, filler) -> Any: 261 | def maker(): 262 | db = lsm.LSM( 263 | tmp_path / ("db.lsm." + request.param), 264 | compress=request.param 265 | ) 266 | db.open() 267 | filler(db) 268 | return iter(db.items()) 269 | return maker 270 | 271 | 272 | class TestFilledIterAndCheckLSMDeallocCtx(DeallocCases): 273 | @pytest.fixture(params=comp_algo) 274 | def instance_maker(self, request, tmp_path, filler) -> Any: 275 | def maker(): 276 | db = lsm.LSM( 277 | tmp_path / ("db.lsm." + request.param), 278 | compress=request.param 279 | ) 280 | db.open() 281 | filler(db) 282 | 283 | for key, value in db.items(): 284 | assert key == value, (key, value) 285 | 286 | return iter(db.items()) 287 | return maker 288 | 289 | 290 | class TestFilledIterIterLSMDealloc(DeallocCases): 291 | @pytest.fixture(params=comp_algo) 292 | def instance_maker(self, request, tmp_path, filler) -> Any: 293 | def maker(): 294 | db = lsm.LSM( 295 | tmp_path / ("db.lsm." + request.param), 296 | compress=request.param 297 | ) 298 | db.open() 299 | filler(db) 300 | return iter(iter(db)) 301 | return maker 302 | 303 | 304 | class TestFilledIterIterLSMKeysDeallocCtx(DeallocCases): 305 | @pytest.fixture(params=comp_algo) 306 | def instance_maker(self, request, tmp_path, filler) -> Any: 307 | def maker(): 308 | db = lsm.LSM( 309 | tmp_path / ("db.lsm." + request.param), 310 | compress=request.param 311 | ) 312 | db.open() 313 | filler(db) 314 | return iter(iter(db.keys())) 315 | return maker 316 | 317 | 318 | class TestFilledIterSliceLSMKeysDeallocCtx(DeallocCases): 319 | @pytest.fixture(params=comp_algo) 320 | def instance_maker(self, request, tmp_path, filler) -> Any: 321 | def maker(): 322 | db = lsm.LSM( 323 | tmp_path / ("db.lsm." + request.param), 324 | compress=request.param 325 | ) 326 | db.open() 327 | filler(db) 328 | return iter(iter(db.keys())) 329 | return maker 330 | 331 | 332 | class TestFilledIterIterLSMValuesDeallocCtx(DeallocCases): 333 | @pytest.fixture(params=comp_algo) 334 | def instance_maker(self, request, tmp_path, filler) -> Any: 335 | def maker(): 336 | db = lsm.LSM( 337 | tmp_path / ("db.lsm." + request.param), 338 | compress=request.param 339 | ) 340 | db.open() 341 | filler(db) 342 | return iter(iter(db[::-1])) 343 | return maker 344 | 345 | 346 | class TestFilledIterIterLSMItemsDeallocCtx(DeallocCases): 347 | @pytest.fixture(params=comp_algo) 348 | def instance_maker(self, request, tmp_path, filler) -> Any: 349 | def maker(): 350 | db = lsm.LSM( 351 | tmp_path / ("db.lsm." + request.param), 352 | compress=request.param 353 | ) 354 | db.open() 355 | filler(db) 356 | return iter(iter(db.items())) 357 | return maker 358 | 359 | 360 | class TestFilledIterIterAndCheckLSMDeallocCtx(DeallocCases): 361 | @pytest.fixture(params=comp_algo) 362 | def instance_maker(self, request, tmp_path, filler) -> Any: 363 | def maker(): 364 | db = lsm.LSM( 365 | tmp_path / ("db.lsm." + request.param), 366 | compress=request.param 367 | ) 368 | db.open() 369 | filler(db) 370 | 371 | for key, value in db.items(): 372 | assert key == value, (key, value) 373 | 374 | return iter(iter(db.items())) 375 | return maker 376 | -------------------------------------------------------------------------------- /tests/test_read_only.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | from pathlib import Path 3 | 4 | import pytest 5 | from lsm import LSM 6 | 7 | from tests import comp_algo 8 | 9 | 10 | @pytest.fixture(params=comp_algo, ids=comp_algo) 11 | def db(request, tmp_path: Path): 12 | db_path = tmp_path / ("readonly.lsm" + request.param) 13 | with LSM(db_path, binary=False, multiple_processes=False) as db: 14 | db.update({"k{}".format(i): str(i) for i in range(100000)}) 15 | 16 | with LSM( 17 | db_path, readonly=True, 18 | binary=False, multiple_processes=False 19 | ) as db: 20 | yield db 21 | 22 | 23 | @contextmanager 24 | def ensure_readonly(): 25 | with pytest.raises(PermissionError) as e: 26 | yield 27 | 28 | assert e.value.args[0] == "Read only" 29 | 30 | 31 | def test_readonly_update(db: LSM): 32 | with ensure_readonly(): 33 | db.update({"foo": "bar"}) 34 | 35 | 36 | def test_readonly_setitem(db: LSM): 37 | with ensure_readonly(): 38 | db["foo"] = "bar" 39 | 40 | 41 | def test_readonly_insert(db: LSM): 42 | with ensure_readonly(): 43 | db.insert("foo", "bar") 44 | 45 | 46 | def test_readonly_flush(db: LSM): 47 | with ensure_readonly(): 48 | db.flush() 49 | 50 | 51 | def test_readonly_work(db: LSM): 52 | with ensure_readonly(): 53 | db.work() 54 | 55 | 56 | def test_readonly_checkpoint(db: LSM): 57 | with ensure_readonly(): 58 | db.checkpoint() 59 | 60 | 61 | def test_readonly_delete(db: LSM): 62 | with ensure_readonly(): 63 | db.delete("foo") 64 | 65 | 66 | def test_readonly_delitem(db: LSM): 67 | with ensure_readonly(): 68 | del db["foo"] 69 | 70 | 71 | def test_readonly_delete_range(db: LSM): 72 | with ensure_readonly(): 73 | db.delete_range("foo", "bar") 74 | 75 | 76 | def test_readonly_delitem_slice(db: LSM): 77 | with ensure_readonly(): 78 | del db["foo":"bar"] 79 | 80 | 81 | def test_readonly_begin(db: LSM): 82 | with ensure_readonly(): 83 | db.begin() 84 | 85 | 86 | def test_readonly_commit(db: LSM): 87 | with ensure_readonly(): 88 | db.commit() 89 | 90 | 91 | def test_readonly_rollback(db: LSM): 92 | with ensure_readonly(): 93 | db.rollback() 94 | 95 | 96 | def test_readonly_tx(db: LSM): 97 | with ensure_readonly(): 98 | with db.tx(): 99 | raise RuntimeError("Impossible case") 100 | 101 | 102 | def test_readonly_transaction(db: LSM): 103 | with ensure_readonly(): 104 | with db.transaction(): 105 | raise RuntimeError("Impossible case") 106 | -------------------------------------------------------------------------------- /tests/test_simple.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import lsm 3 | 4 | 5 | def test_multiple_open(subtests, tmp_path): 6 | kwargs = { 7 | 'autocheckpoint': 8 * 1024, # 8 MB 8 | 'autoflush': 8 * 1024, # 8 MB 9 | 'multiple_processes': False, 10 | 'safety': lsm.SAFETY_OFF, # do not fsync manually 11 | 'use_log': False, 12 | 'binary': False, 13 | } 14 | 15 | count = 2048 16 | 17 | for prefix in ("k", "z", "a", "f", "1"): 18 | with lsm.LSM(str(tmp_path / "test.lsm"), **kwargs) as db: 19 | for i in range(count): 20 | db['{}{}'.format(prefix, i)] = str(i) 21 | 22 | with lsm.LSM(str(tmp_path / "test.lsm"), binary=False, readonly=True) as db: 23 | for prefix in ("k", "z", "a", "f", "1"): 24 | with subtests.test(msg="prefix {}".format(i)): 25 | for i in range(count): 26 | assert db['{}{}'.format(prefix, i)] == str(i) 27 | 28 | for key, value in db.items(): 29 | assert key[1:] == value, (key, value) 30 | 31 | 32 | def test_db_binary(subtests, tmp_path): 33 | with lsm.LSM(str(tmp_path / "test.lsm"), binary=True) as db: 34 | with subtests.test(msg="str to binary mode"): 35 | with pytest.raises(ValueError): 36 | _ = db['foo'] 37 | 38 | with pytest.raises(ValueError): 39 | db['foo'] = 'bar' 40 | 41 | with subtests.test(msg="test KeyError"): 42 | with pytest.raises(KeyError): 43 | _ = db[b'foo'] 44 | 45 | with pytest.raises(KeyError): 46 | del db[b'foo'] 47 | 48 | with subtests.test(msg="test mapping-like set"): 49 | assert b'foo' not in db 50 | db[b'foo'] = b'bar' 51 | assert b'foo' in db 52 | 53 | with subtests.test(msg="test mapping-like get"): 54 | assert db[b'foo'] == b'bar' 55 | 56 | with subtests.test(msg="test KeyError"): 57 | del db[b'foo'] 58 | with pytest.raises(KeyError): 59 | _ = db[b'foo'] 60 | 61 | 62 | def test_db_strings(subtests, tmp_path): 63 | with lsm.LSM(str(tmp_path / "test.lsm"), binary=False) as db: 64 | with subtests.test(msg="bytes to string mode"): 65 | with pytest.raises(ValueError): 66 | _ = db[b'foo'] 67 | 68 | with pytest.raises(ValueError): 69 | db[b'foo'] = b'bar' 70 | 71 | with subtests.test(msg="test KeyError"): 72 | with pytest.raises(KeyError): 73 | _ = db['foo'] 74 | 75 | with pytest.raises(KeyError): 76 | del db['foo'] 77 | 78 | with subtests.test(msg="test mapping-like set"): 79 | assert 'foo' not in db 80 | db['foo'] = 'bar' 81 | assert 'foo' in db 82 | 83 | with subtests.test(msg="test mapping-like get"): 84 | assert db['foo'] == 'bar' 85 | 86 | with subtests.test(msg="test KeyError"): 87 | del db['foo'] 88 | with pytest.raises(KeyError): 89 | _ = db['foo'] 90 | 91 | 92 | def test_db_cursors(subtests, tmp_path): 93 | with lsm.LSM(str(tmp_path / "test.lsm"), binary=False) as db: 94 | for i in range(10): 95 | db[f"key_{i}"] = str(i) 96 | 97 | with subtests.test(msg="basic"): 98 | with db.cursor() as cursor: 99 | cursor.first() 100 | 101 | key1, value1 = cursor.retrieve() 102 | key2, value2 = cursor.retrieve() 103 | 104 | assert cursor.compare(key1) >= 0 105 | assert cursor.compare(key1[1:]) < 0 106 | 107 | assert key1 == key2 108 | assert value1 == value2 109 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = lint,py3{9,10,11,12,13} 3 | 4 | [testenv] 5 | passenv = FORCE_COLOR 6 | usedevelop = true 7 | 8 | extras = 9 | develop 10 | 11 | commands= 12 | py.test -sv tests README.md 13 | --------------------------------------------------------------------------------