├── .dockerignore ├── .github └── workflows │ ├── mac │ ├── build-bzip2.sh │ ├── build-lz4.sh │ ├── build-rocksdb.sh │ ├── build-snappy.sh │ ├── build-zlib.sh │ └── build-zstd.sh │ └── python-package.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── README.md ├── deps └── rocksdb_sched.patch ├── docker ├── audit-wheels.sh ├── build-bzip2.sh ├── build-deps.sh ├── build-lz4.sh ├── build-rocksdb.sh ├── build-snappy.sh ├── build-wheels.sh ├── build-zlib.sh └── build-zstd.sh ├── pyproject.toml ├── requirements.dev.txt ├── setup.py ├── src └── aimrocks │ ├── __init__.py │ ├── backup.pxd │ ├── cache.pxd │ ├── comparator.pxd │ ├── db.pxd │ ├── env.pxd │ ├── errors.py │ ├── filter_policy.pxd │ ├── include │ └── rdb_include │ │ ├── comparator_wrapper.hpp │ │ ├── filter_policy_wrapper.hpp │ │ ├── memtable_factories.hpp │ │ ├── merge_operator_wrapper.hpp │ │ ├── slice_transform_wrapper.hpp │ │ ├── utils.hpp │ │ └── write_batch_iter_helper.hpp │ ├── interfaces.py │ ├── iterator.pxd │ ├── lib_rocksdb.pxd │ ├── lib_rocksdb.pyi │ ├── lib_rocksdb.pyx │ ├── lib_utils.py │ ├── logger.pxd │ ├── memtablerep.pxd │ ├── merge_operator.pxd │ ├── merge_operators.py │ ├── options.pxd │ ├── slice_.pxd │ ├── slice_transform.pxd │ ├── snapshot.pxd │ ├── status.pxd │ ├── std_memory.pxd │ ├── table_factory.pxd │ └── universal_compaction.pxd └── tests ├── __init__.py ├── test_db.py ├── test_memtable.py └── test_options.py /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .github 3 | Dockerfile 4 | -------------------------------------------------------------------------------- /.github/workflows/mac/build-bzip2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | cd $AIM_DEP_DIR 5 | curl https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz -o bzip2-1.0.8.tar.gz 6 | tar zxvf bzip2-1.0.8.tar.gz 7 | cd bzip2-1.0.8/ 8 | make CFLAGS="-fPIC" CXXFLAGS="-fPIC" 9 | make install PREFIX=.. 10 | cd ../ 11 | rm -rf bzip2-1.0.8 bzip2-1.0.8.tar.gz 12 | -------------------------------------------------------------------------------- /.github/workflows/mac/build-lz4.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | cd $AIM_DEP_DIR 5 | curl -L https://github.com/lz4/lz4/archive/v1.9.3.tar.gz -o lz4-1.9.3.tar.gz 6 | tar zxvf lz4-1.9.3.tar.gz 7 | cd lz4-1.9.3 8 | make CFLAGS="-fPIC" CXXFLAGS="-fPIC" 9 | make PREFIX=$PWD/.. install 10 | cd .. 11 | rm -rf lz4-1.9.3 lz4-1.9.3.tar.gz 12 | -------------------------------------------------------------------------------- /.github/workflows/mac/build-rocksdb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | cd $AIM_DEP_DIR 5 | rm lib/*.dylib 6 | curl -L https://github.com/facebook/rocksdb/archive/6.29.fb.tar.gz -o rocksdb-6.29.fb.tar.gz 7 | tar zxvf rocksdb-6.29.fb.tar.gz 8 | cd rocksdb-6.29.fb 9 | LIBRARY_PATH=$AIM_DEP_DIR/lib PORTABLE=1 make shared_lib PLATFORM_SHARED_VERSIONED=false EXTRA_CXXFLAGS="-fPIC -I$AIM_DEP_DIR/include" USE_RTTI=0 DEBUG_LEVEL=0 -j12 10 | strip -S librocksdb.dylib 11 | install_name_tool -id @rpath/librocksdb.dylib librocksdb.dylib 12 | cp librocksdb.dylib $AIM_DEP_DIR/lib 13 | LIBRARY_PATH=$AIM_DEP_DIR/lib PREFIX="$AIM_DEP_DIR" PORTABLE=1 make install-headers PLATFORM_SHARED_VERSIONED=false EXTRA_CXXFLAGS="-fPIC -I$AIM_DEP_DIR/include" USE_RTTI=0 DEBUG_LEVEL=0 -j12 14 | cd .. 15 | rm -rf rocksdb-6.29.fb rocksdb-6.29.fb.tar.gz 16 | -------------------------------------------------------------------------------- /.github/workflows/mac/build-snappy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | cd $AIM_DEP_DIR 5 | curl -L https://github.com/google/snappy/archive/1.1.8.tar.gz -o snappy-1.1.8.tar.gz 6 | tar zxvf snappy-1.1.8.tar.gz 7 | cd snappy-1.1.8 8 | mkdir build 9 | cd build 10 | $CMAKE -DCMAKE_INSTALL_PREFIX=../.. CFLAGS="-fPIC" CXXFLAGS="-fPIC" -DCMAKE_POSITION_INDEPENDENT_CODE=ON .. 11 | make 12 | make install 13 | cd ../.. 14 | rm -rf snappy-1.1.8 snappy-1.1.8.tar.gz 15 | -------------------------------------------------------------------------------- /.github/workflows/mac/build-zlib.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | cd $AIM_DEP_DIR 5 | curl -L https://www.zlib.net/fossils/zlib-1.2.11.tar.gz -o zlib-1.2.11.tar.gz 6 | tar zxvf zlib-1.2.11.tar.gz 7 | cd zlib-1.2.11/ 8 | ./configure 9 | make CFLAGS="-fPIC" CXXFLAGS="-fPIC" 10 | make install prefix=.. 11 | cd ../ 12 | rm -rf zlib-1.2.11 zlib-1.2.11.tar.gz 13 | -------------------------------------------------------------------------------- /.github/workflows/mac/build-zstd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | cd $AIM_DEP_DIR 5 | curl -L https://github.com/facebook/zstd/archive/v1.1.3.tar.gz -o zstd-1.1.3.tar.gz 6 | tar zxvf zstd-1.1.3.tar.gz 7 | cd zstd-1.1.3 8 | make CFLAGS="-fPIC" CXXFLAGS="-fPIC" 9 | make install PREFIX=$PWD/.. 10 | cd .. 11 | rm -rf zstd-1.1.3 zstd-1.1.3.tar.gz 12 | -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | name: aimrocks packaging pipeline 2 | 3 | on: workflow_dispatch 4 | 5 | jobs: 6 | uploading-wheels: 7 | needs: 8 | - linux-dist-aarch64 9 | - linux-dist-x86_64 10 | - macos-dist 11 | runs-on: ubuntu-latest 12 | name: Uploading wheels 13 | steps: 14 | - name: Setup python 15 | uses: actions/setup-python@v2 16 | with: 17 | python-version: '3.9' 18 | architecture: x64 19 | 20 | - name: Install dev dependencies 21 | run: | 22 | python -m pip install twine 23 | 24 | - name: Download all the wheels 25 | uses: actions/download-artifact@v3 26 | 27 | - name: Publish wheels 28 | env: 29 | PYPI_PASSWORD: ${{ secrets.pypi_password }} 30 | run: | 31 | python -m twine upload -u __token__ -p "${PYPI_PASSWORD}" artifact/*.whl 32 | 33 | linux-dist-aarch64: 34 | runs-on: m1 35 | strategy: 36 | fail-fast: false 37 | matrix: 38 | manylinux-version: [ 'manylinux_2_28_aarch64', 'manylinux2014_aarch64', 'manylinux_2_24_aarch64' ] 39 | name: ${{ matrix.manylinux-version }} using Docker on M1 Mac 40 | steps: 41 | - name: Checkout sources 42 | uses: actions/checkout@v2 43 | 44 | - name: Pull Docker Images 45 | run: | 46 | docker pull quay.io/pypa/${{ matrix.manylinux-version }} 47 | 48 | - name: Building dependencies for ${{ matrix.manylinux-version }} 49 | run: | 50 | docker build \ 51 | --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version }} \ 52 | --target deps . 53 | 54 | - name: Building rocksdb for ${{ matrix.manylinux-version }} 55 | run: | 56 | docker build \ 57 | --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version }} \ 58 | --target rocksdb . 59 | 60 | - name: Building wheels for ${{ matrix.manylinux-version }} 61 | run: | 62 | docker build \ 63 | --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version }} \ 64 | --target wheels . \ 65 | -t aimhubio/aimrocks:${{ matrix.manylinux-version }} 66 | 67 | - name: Auditing wheels for ${{ matrix.manylinux-version }} 68 | run: | 69 | mkdir -p manylinux_dist/ && \ 70 | docker run --rm \ 71 | --mount type=bind,source=$PWD/manylinux_dist,target=/opt/aimrocks/manylinux_dist \ 72 | aimhubio/aimrocks:${{ matrix.manylinux-version }} \ 73 | bash -e /opt/aimrocks/docker/audit-wheels.sh 74 | 75 | - uses: actions/upload-artifact@v3 76 | with: 77 | path: manylinux_dist/*.whl 78 | 79 | linux-dist-x86_64: 80 | runs-on: ubuntu-latest 81 | strategy: 82 | fail-fast: false 83 | matrix: 84 | manylinux-version: [ 'manylinux1_x86_64', 'manylinux2010_x86_64', 'manylinux2014_x86_64', 'manylinux_2_24_x86_64' ] 85 | name: ${{ matrix.manylinux-version }} 86 | steps: 87 | - name: Install Docker & images 88 | run: | 89 | apt update && apt install -y docker.io 90 | sudo systemctl enable --now docker 91 | 92 | - name: Checkout sources 93 | uses: actions/checkout@v2 94 | 95 | - name: Setup python 96 | uses: actions/setup-python@v2 97 | with: 98 | python-version: '3.9' 99 | architecture: x64 100 | 101 | - name: Pull Docker Images 102 | run: | 103 | docker pull quay.io/pypa/${{ matrix.manylinux-version }} 104 | 105 | - uses: satackey/action-docker-layer-caching@v0.0.11 106 | # Ignore the failure of a step and avoid terminating the job. 107 | continue-on-error: true 108 | with: 109 | key: aimrocks-cython-manylinux-build-${{ matrix.manylinux-version }}-{hash} 110 | restore-keys: | 111 | aimrocks-cython-manylinux-build-${{ matrix.manylinux-version }}- 112 | 113 | - name: Building dependencies for ${{ matrix.manylinux-version }} 114 | run: | 115 | docker build \ 116 | --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version }} \ 117 | --target deps . 118 | 119 | - name: Building rocksdb for ${{ matrix.manylinux-version }} 120 | run: | 121 | docker build \ 122 | --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version }} \ 123 | --target rocksdb . 124 | 125 | - name: Building wheels for ${{ matrix.manylinux-version }} 126 | run: | 127 | docker build \ 128 | --build-arg FROM=quay.io/pypa/${{ matrix.manylinux-version }} \ 129 | --target wheels . \ 130 | -t aimhubio/aimrocks:${{ matrix.manylinux-version }} 131 | 132 | - name: Auditing wheels for ${{ matrix.manylinux-version }} 133 | run: | 134 | mkdir -p manylinux_dist/ && \ 135 | docker run --rm \ 136 | --mount type=bind,source=$PWD/manylinux_dist,target=/opt/aimrocks/manylinux_dist \ 137 | aimhubio/aimrocks:${{ matrix.manylinux-version }} \ 138 | bash -e /opt/aimrocks/docker/audit-wheels.sh 139 | 140 | - uses: actions/upload-artifact@v3 141 | with: 142 | path: manylinux_dist/*.whl 143 | 144 | macos-deps: 145 | runs-on: m1 146 | strategy: 147 | fail-fast: true 148 | matrix: 149 | arch: ['arm64', 'x86_64'] 150 | name: Preparing dependencies for ${{ matrix.arch }} Mac build 151 | env: 152 | MACOSX_DEPLOYMENT_TARGET: 10.14 153 | AIM_DEP_DIR: /tmp/run/${{ github.run_number }}/${{ matrix.arch }} 154 | CMAKE: /opt/homebrew/bin/cmake 155 | steps: 156 | - name: Preparing Build Dir for Dependencies 157 | run: | 158 | mkdir -p $AIM_DEP_DIR 159 | rm -rf $AIM_DEP_DIR/* 160 | 161 | - name: Checkout sources 162 | uses: actions/checkout@v2 163 | 164 | - name: Building ZLib 165 | run: | 166 | pwd 167 | ls -lhatr .github/workflows/mac 168 | arch -${{matrix.arch}} ./.github/workflows/mac/build-zlib.sh 169 | 170 | - name: Building BZip2 171 | run: | 172 | arch -${{matrix.arch}} ./.github/workflows/mac/build-bzip2.sh 173 | 174 | - name: Building LZ4 175 | run: | 176 | arch -${{matrix.arch}} ./.github/workflows/mac/build-lz4.sh 177 | 178 | - name: Building Snappy 179 | run: | 180 | arch -${{matrix.arch}} ./.github/workflows/mac/build-snappy.sh 181 | 182 | - name: Building ZSTD 183 | run: | 184 | arch -${{matrix.arch}} ./.github/workflows/mac/build-zstd.sh 185 | 186 | - name: Building RocksDB 187 | run: | 188 | arch -${{matrix.arch}} ./.github/workflows/mac/build-rocksdb.sh 189 | 190 | 191 | macos-dist: 192 | runs-on: m1 193 | needs: macos-deps 194 | strategy: 195 | fail-fast: true 196 | matrix: 197 | python-version: [ '3.6', '3.7', '3.8', '3.9', '3.10' , '3.11', '3.12'] 198 | arch: ['arm64', 'x86_64'] 199 | exclude: 200 | - arch: 'arm64' 201 | python-version: 3.6 202 | - arch: 'arm64' 203 | python-version: 3.7 204 | # requirement for custom runners: having cmake and python installed 205 | name: Python ${{ matrix.python-version }} for ${{ matrix.arch }} build 206 | env: 207 | MACOSX_DEPLOYMENT_TARGET: 10.14 208 | AIM_DEP_DIR: /tmp/run/${{ github.run_number }}/${{ matrix.arch }} 209 | PYTHON: /opt/conda/${{ matrix.arch }}/envs/py${{ matrix.python-version }}/bin/python 210 | steps: 211 | - name: Build and test wheels 212 | run: | 213 | arch -${{matrix.arch}} $PYTHON -m build 214 | 215 | - uses: actions/upload-artifact@v3 216 | with: 217 | path: dist/*.whl 218 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | venv 2 | build 3 | dist 4 | .idea 5 | .pytest_cache 6 | *.egg-info 7 | *.eggs 8 | src/aimrocks/lib_rocksdb.cpp 9 | *.so 10 | *.dylib 11 | lib*.so.* 12 | __pycache__ 13 | src/aimrocks/include/rocksdb 14 | .DS_Store 15 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG FROM=ubuntu 2 | 3 | FROM ${FROM} AS deps 4 | 5 | RUN mkdir -p /opt/aimrocks_deps/lib /opt/aimrocks_deps/include 6 | ENV AIM_DEP_DIR=/opt/aimrocks_deps 7 | RUN /opt/python/cp37-cp37m/bin/python -m pip install 'cmake<3.23' && \ 8 | ln -s /opt/python/cp37-cp37m/bin/cmake /usr/bin/cmake 9 | WORKDIR /opt/aimrocks_deps/ 10 | 11 | 12 | COPY docker/build-zlib.sh ./ 13 | RUN ./build-zlib.sh 14 | 15 | COPY docker/build-bzip2.sh ./ 16 | RUN ./build-bzip2.sh 17 | 18 | COPY docker/build-zstd.sh ./ 19 | RUN ./build-zstd.sh 20 | 21 | COPY docker/build-lz4.sh ./ 22 | RUN ./build-lz4.sh 23 | 24 | COPY docker/build-snappy.sh ./ 25 | RUN ./build-snappy.sh 26 | 27 | 28 | 29 | FROM deps AS rocksdb 30 | 31 | COPY docker/build-rocksdb.sh ./ 32 | COPY deps/rocksdb_sched.patch /opt/aimrocks_deps/ 33 | RUN ./build-rocksdb.sh 34 | 35 | 36 | FROM rocksdb AS wheels 37 | 38 | COPY . /opt/aimrocks 39 | COPY docker/build-wheels.sh ./ 40 | RUN ./build-wheels.sh 41 | 42 | 43 | FROM wheels AS audit 44 | 45 | COPY docker/audit-wheels.sh ./ 46 | RUN ./audit-wheels.sh 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | recursive-include src *.hpp *.pxd *.pyx *.h *.hxx *.pyi *.so* *.dylib* 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # aimrocks: python wrapper for rocksdb 2 | 3 | `aimrocks` is a python package written in Cython, similar to [python-rocksdb](https://python-rocksdb.readthedocs.io/en/latest/). 4 | 5 | It uses statically linked libraries for rocksdb (version 6.29.5) and compression libraries it depends on, 6 | so `aimrocks` can be used out of the box (without requiring additional installation of any of those). 7 | 8 | ### Example usage 9 | 10 | ```python 11 | import aimrocks 12 | 13 | db_options = dict( 14 | create_if_missing=True, 15 | paranoid_checks=False, 16 | ) 17 | 18 | db_path = '/tmp/example_db' 19 | rocks_db = aimrocks.DB(db_path, aimrocks.Options(**db_options), read_only=False) 20 | 21 | batch = aimrocks.WriteBatch() 22 | batch.put(b'key_1', b'value_1') 23 | batch.put(b'key_1', b'value_1') 24 | ... 25 | 26 | rocks_db.write(batch) 27 | 28 | ``` 29 | -------------------------------------------------------------------------------- /deps/rocksdb_sched.patch: -------------------------------------------------------------------------------- 1 | diff --git a/port/port_posix.cc b/port/port_posix.cc 2 | index 8615f11d6..4c5bbc590 100644 3 | --- a/port/port_posix.cc 4 | +++ b/port/port_posix.cc 5 | @@ -17,6 +17,25 @@ 6 | #endif 7 | #include 8 | #include 9 | + 10 | +/* Normally defined in */ 11 | +#ifndef SCHED_NORMAL 12 | +#define SCHED_NORMAL 0 13 | +#endif 14 | +#ifndef SCHED_FIFO 15 | +#define SCHED_FIFO 1 16 | +#endif 17 | +#ifndef SCHED_RR 18 | +#define SCHED_RR 2 19 | +#endif 20 | +#ifndef SCHED_BATCH 21 | +#define SCHED_BATCH 3 22 | +#endif 23 | +/* SCHED_ISO not yet implemented */ 24 | +#ifndef SCHED_IDLE 25 | +#define SCHED_IDLE 5 26 | +#endif 27 | + 28 | #include 29 | #include 30 | #include 31 | -------------------------------------------------------------------------------- /docker/audit-wheels.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | export AIM_DEP_DIR=/opt/aimrocks_deps 5 | 6 | # These libraries are manually handled 7 | LIBS_BUNDLED=`ls ${AIM_DEP_DIR}/lib/ \ 8 | | grep .so \ 9 | | sed -r 's/^(lib.*?\.so)\.*?$/\1/g' \ 10 | | uniq \ 11 | | paste -s -d','` 12 | export LD_LIBRARY_PATH=$AIM_DEP_DIR/lib:$LD_LIBRARY_PATH 13 | 14 | INTERNAL_PYTHON=/opt/python/cp38-cp38/bin/python 15 | 16 | cd /opt/aimrocks 17 | 18 | ls -lhatr dist 19 | 20 | $INTERNAL_PYTHON -m pip install --upgrade pip 21 | $INTERNAL_PYTHON -m pip install git+https://github.com/aimhubio/auditwheel.git@include-exclude-new 22 | 23 | for whl in dist/*.whl 24 | do 25 | $INTERNAL_PYTHON -m auditwheel repair ${whl} \ 26 | --exclude $LIBS_BUNDLED \ 27 | --wheel-dir manylinux_dist 28 | done 29 | rm -rf dist 30 | -------------------------------------------------------------------------------- /docker/build-bzip2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl -L https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz -o bzip2-1.0.8.tar.gz 3 | tar zxvf bzip2-1.0.8.tar.gz 4 | cd bzip2-1.0.8/ 5 | make CFLAGS="-fPIC" CXXFLAGS="-fPIC" 6 | make install PREFIX=.. 7 | cd .. 8 | rm -rf bzip2-1.0.8/ bzip2-1.0.8.tar.gz 9 | -------------------------------------------------------------------------------- /docker/build-deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Preparing local dependency directories 4 | mkdir -p /opt/aimrocks_deps 5 | cd /opt/aimrocks_deps/ 6 | export AIM_DEP_DIR=/opt/aimrocks_deps 7 | 8 | mkdir -p lib 9 | mkdir -p include 10 | 11 | # Installing CMake 12 | /opt/python/cp37-cp37m/bin/python -m pip install cmake 13 | ln -s /opt/python/cp37-cp37m/bin/cmake /usr/bin/cmake 14 | PATH=/opt/python/cp37-cp37m/bin:$PATH 15 | 16 | # Building third party dependencies 17 | /opt/aimrocks/docker/build-zlib.sh 18 | /opt/aimrocks/docker/build-bzip2.sh 19 | /opt/aimrocks/docker/build-zstd.sh 20 | /opt/aimrocks/docker/build-lz4.sh 21 | /opt/aimrocks/docker/build-snappy.sh 22 | /opt/aimrocks/docker/build-rocksdb.sh 23 | -------------------------------------------------------------------------------- /docker/build-lz4.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl -L https://github.com/lz4/lz4/archive/v1.9.3.tar.gz -o lz4-1.9.3.tar.gz 3 | tar zxvf lz4-1.9.3.tar.gz 4 | cd lz4-1.9.3 5 | make CFLAGS="-fPIC" CXXFLAGS="-fPIC" 6 | make PREFIX=.. install 7 | cd .. 8 | rm -rf lz4-1.9.3 lz4-1.9.3.tar.gz 9 | -------------------------------------------------------------------------------- /docker/build-rocksdb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # check OS version 3 | if [[ -f /etc/redhat-release ]] 4 | then 5 | if [[ $(< /etc/redhat-release) == "CentOS release 5"* ]] 6 | then 7 | # CentOS 5 8 | export platform=centos_5 9 | fi 10 | fi 11 | 12 | rm -rf lib/lib*.so* 13 | 14 | curl -L https://github.com/facebook/rocksdb/archive/6.29.fb.tar.gz -o rocksdb-6.29.fb.tar.gz 15 | tar zxvf rocksdb-6.29.fb.tar.gz 16 | cd rocksdb-6.29.fb 17 | if [[ $platform == centos_5 ]] 18 | then 19 | cp /opt/aimrocks_deps/rocksdb_sched.patch . 20 | patch port/port_posix.cc rocksdb_sched.patch 21 | fi 22 | LIBRARY_PATH=$PWD/../lib PORTABLE=1 make shared_lib PLATFORM_SHARED_VERSIONED=false EXTRA_CXXFLAGS="-fPIC -I../include" EXTRA_CFLAGS="-fPIC" USE_RTTI=0 DEBUG_LEVEL=0 -j4 23 | strip --strip-debug librocksdb.so 24 | cp librocksdb.so ../lib/ 25 | PORTABLE=1 make PREFIX="$PWD/.." DEBUG_LEVEL=0 install-headers 26 | cd .. 27 | rm -rf rocksdb-6.29.fb rocksdb-6.29.fb.tar.gz 28 | -------------------------------------------------------------------------------- /docker/build-snappy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl -L https://github.com/google/snappy/archive/1.1.8.tar.gz -o snappy-1.1.8.tar.gz 3 | tar zxvf snappy-1.1.8.tar.gz 4 | cd snappy-1.1.8 5 | mkdir build 6 | cd build 7 | cmake -DCMAKE_INSTALL_PREFIX="../.." CFLAGS="-fPIC" CXXFLAGS="fPIC" -DCMAKE_POSITION_INDEPENDENT_CODE=ON .. 8 | make 9 | make install 10 | cd ../.. 11 | rm -rf snappy-1.1.8 snappy-1.1.8.tar.gz 12 | -------------------------------------------------------------------------------- /docker/build-wheels.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | export AIM_DEP_DIR=/opt/aimrocks_deps 5 | 6 | # check OS version 7 | if [[ -f /etc/redhat-release ]] 8 | then 9 | if [[ $(< /etc/redhat-release) == "CentOS release 5"* ]] 10 | then 11 | # CentOS 5 12 | export platform=centos_5 13 | fi 14 | fi 15 | 16 | 17 | cd /opt/aimrocks 18 | 19 | echo "build python wheels" 20 | python_versions=("cp36-cp36m" "cp37-cp37m" "cp38-cp38" "cp39-cp39" "cp310-cp310" "cp311-cp311" "cp312-cp312") 21 | 22 | for python_version in "${python_versions[@]}" 23 | do 24 | python_exe=/opt/python/${python_version}/bin/python 25 | if [ -f "$python_exe" ] 26 | then 27 | $python_exe -m build 28 | rm -rf build 29 | fi 30 | done 31 | -------------------------------------------------------------------------------- /docker/build-zlib.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl -L https://www.zlib.net/fossils/zlib-1.2.11.tar.gz -o zlib-1.2.11.tar.gz 3 | tar zxvf zlib-1.2.11.tar.gz 4 | cd zlib-1.2.11/ 5 | ls -lhatr 6 | ./configure 7 | make CFLAGS="-fPIC" CXXFLAGS="-fPIC" 8 | make install prefix=.. 9 | cd .. 10 | rm -rf zlib-1.2.11/ zlib-1.2.11.tar.gz 11 | -------------------------------------------------------------------------------- /docker/build-zstd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl -L https://github.com/facebook/zstd/archive/v1.1.3.tar.gz -o zstd-1.1.3.tar.gz 3 | tar zxvf zstd-1.1.3.tar.gz 4 | cd zstd-1.1.3 5 | make CFLAGS="-fPIC" CXXFLAGS="-fPIC" 6 | make install PREFIX=$PWD/.. 7 | cd .. 8 | rm -rf zstd-1.1.3 zstd-1.1.3.tar.gz 9 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "cython >= 3.0.0a9"] 3 | -------------------------------------------------------------------------------- /requirements.dev.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | wheel 3 | twine 4 | flake8 5 | Cython==3.0.0a9 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import platform 4 | from glob import glob 5 | from setuptools import setup 6 | from setuptools import find_packages 7 | from setuptools import Extension 8 | from distutils.dir_util import copy_tree 9 | from distutils.file_util import copy_file 10 | 11 | 12 | try: 13 | from Cython.Build import cythonize 14 | except ImportError: 15 | print("Warning: Cython is not installed", file=sys.stderr) 16 | # Falling back to simpler build 17 | cythonize = lambda x: x 18 | 19 | 20 | aimrocks_extra_compile_args = [ 21 | '-std=c++11', 22 | '-O3', 23 | '-Wall', 24 | '-Wextra', 25 | '-Wconversion', 26 | '-fno-strict-aliasing', 27 | '-fno-rtti', 28 | '-fPIC', 29 | ] 30 | 31 | aimrocks_extra_link_args = [] 32 | 33 | if platform.system() == 'Darwin': 34 | aimrocks_extra_compile_args += ['-mmacosx-version-min=10.7', '-stdlib=libc++'] 35 | aimrocks_extra_link_args += ["-Wl,-rpath,@loader_path"] 36 | else: 37 | aimrocks_extra_link_args += ["-Wl,-rpath,$ORIGIN"] 38 | 39 | third_party_install_dir = os.environ.get('AIM_DEP_DIR', '/usr/local') 40 | 41 | # By default aimrocks is only linked to rocksdb. 42 | third_party_deps = ['rocksdb'] 43 | 44 | # `AIMROCKS_LINK_LIBS` can be used to link against additional libraries. 45 | if os.environ.get("AIMROCKS_LINK_LIBS") is not None: 46 | third_party_deps += os.environ.get("AIMROCKS_LINK_LIBS", "").split(",") 47 | 48 | third_party_lib_dir = os.path.join(third_party_install_dir, 'lib') 49 | 50 | # We define a local include directory to store all the required public headers. 51 | # The third party headers are copied into this directory to enable binding with 52 | # the precompiled aimrocks binaries without third-party dependencies. 53 | local_include_dir = os.path.abspath( 54 | os.path.join(os.path.dirname(__file__), 'src/aimrocks/include') 55 | ) 56 | local_lib_dir = os.path.abspath( 57 | os.path.join(os.path.dirname(__file__), 'src/aimrocks') 58 | ) 59 | 60 | if os.environ.get("AIMROCKS_EMBED_ROCKSDB", "1") == "1": 61 | 62 | third_party_libs = glob(os.path.join(third_party_lib_dir, 'librocksdb.*')) 63 | 64 | print('third party libs detected:', third_party_libs) 65 | for source in third_party_libs: 66 | print('copying third party lib', source, local_lib_dir) 67 | copy_file(source, local_lib_dir) 68 | 69 | third_party_headers = [os.path.join(third_party_install_dir, 'include/rocksdb')] 70 | for source in third_party_headers: 71 | basename = os.path.basename(source) 72 | destination = os.path.join(local_include_dir, basename) 73 | copy_tree(source, destination, 74 | preserve_symlinks=False, update=True) 75 | 76 | # `include_dirs` is used to specify the include directories for the extension. 77 | # It contains the aimrocks local headers and the rocksdb ones. 78 | include_dirs = [os.path.join(third_party_install_dir, 'include/'), local_include_dir] 79 | 80 | exts = [ 81 | Extension( 82 | 'aimrocks.lib_rocksdb', 83 | ['src/aimrocks/lib_rocksdb.pyx'], 84 | extra_compile_args=aimrocks_extra_compile_args, 85 | extra_link_args=aimrocks_extra_link_args, 86 | language='c++', 87 | include_dirs=include_dirs, 88 | library_dirs=[third_party_lib_dir], 89 | libraries=third_party_deps, 90 | ) 91 | ] 92 | 93 | setup( 94 | name="aimrocks", 95 | version='0.5.2', 96 | description='RocksDB wrapper implemented in Cython.', 97 | setup_requires=['setuptools>=25', 'Cython>=3.0.0a9'], 98 | packages=find_packages('./src'), 99 | package_dir={'': 'src'}, 100 | package_data={'aimrocks': ['src/*']}, 101 | ext_modules=cythonize(exts), 102 | include_package_data=True, 103 | zip_safe=False, 104 | classifiers=[ 105 | 'License :: OSI Approved :: Apache Software License', 106 | 'Programming Language :: Python', 107 | 'Programming Language :: Python :: 3', 108 | 'Programming Language :: Python :: 3.7', 109 | 'Programming Language :: Python :: 3.8', 110 | 'Programming Language :: Python :: 3.9', 111 | 'Programming Language :: Python :: 3.10', 112 | 'Programming Language :: Python :: 3.11', 113 | 'Programming Language :: Python :: 3.12', 114 | ], 115 | ) 116 | -------------------------------------------------------------------------------- /src/aimrocks/__init__.py: -------------------------------------------------------------------------------- 1 | from .lib_rocksdb import * 2 | -------------------------------------------------------------------------------- /src/aimrocks/backup.pxd: -------------------------------------------------------------------------------- 1 | from libcpp cimport bool as cpp_bool 2 | from libcpp.string cimport string 3 | from libcpp.vector cimport vector 4 | from libc.stdint cimport uint32_t 5 | from libc.stdint cimport int64_t 6 | from libc.stdint cimport uint64_t 7 | 8 | from aimrocks.status cimport Status 9 | from aimrocks.db cimport DB 10 | from aimrocks.env cimport Env 11 | 12 | cdef extern from "rocksdb/utilities/backup_engine.h" namespace "rocksdb": 13 | ctypedef uint32_t BackupID 14 | 15 | cdef cppclass BackupEngineOptions: 16 | BackupEngineOptions(const string& backup_dir) 17 | 18 | cdef struct BackupInfo: 19 | BackupID backup_id 20 | int64_t timestamp 21 | uint64_t size 22 | 23 | cdef cppclass BackupEngine: 24 | Status CreateNewBackup(DB*, cpp_bool) nogil except+ 25 | Status PurgeOldBackups(uint32_t) nogil except+ 26 | Status DeleteBackup(BackupID) nogil except+ 27 | void StopBackup() nogil except+ 28 | void GetBackupInfo(vector[BackupInfo]*) nogil except+ 29 | Status RestoreDBFromBackup(BackupID, string&, string&) nogil except+ 30 | Status RestoreDBFromLatestBackup(string&, string&) nogil except+ 31 | 32 | cdef Status BackupEngine_Open "rocksdb::BackupEngine::Open"( 33 | Env*, 34 | BackupEngineOptions&, 35 | BackupEngine**) 36 | -------------------------------------------------------------------------------- /src/aimrocks/cache.pxd: -------------------------------------------------------------------------------- 1 | from aimrocks.std_memory cimport shared_ptr 2 | 3 | cdef extern from "rocksdb/cache.h" namespace "rocksdb": 4 | cdef cppclass Cache: 5 | pass 6 | 7 | cdef extern shared_ptr[Cache] NewLRUCache(size_t) 8 | cdef extern shared_ptr[Cache] NewLRUCache(size_t, int) 9 | -------------------------------------------------------------------------------- /src/aimrocks/comparator.pxd: -------------------------------------------------------------------------------- 1 | from libcpp.string cimport string 2 | from aimrocks.slice_ cimport Slice 3 | from aimrocks.logger cimport Logger 4 | from aimrocks.std_memory cimport shared_ptr 5 | 6 | cdef extern from "rocksdb/comparator.h" namespace "rocksdb": 7 | cdef cppclass Comparator: 8 | const char* Name() 9 | int Compare(const Slice&, const Slice&) const 10 | 11 | cdef extern const Comparator* BytewiseComparator() nogil except + 12 | 13 | ctypedef int (*compare_func)( 14 | void*, 15 | Logger*, 16 | string&, 17 | const Slice&, 18 | const Slice&) 19 | 20 | cdef extern from "rdb_include/comparator_wrapper.hpp" namespace "py_rocks": 21 | cdef cppclass ComparatorWrapper: 22 | ComparatorWrapper(string, void*, compare_func) nogil except + 23 | void set_info_log(shared_ptr[Logger]) nogil except+ 24 | -------------------------------------------------------------------------------- /src/aimrocks/db.pxd: -------------------------------------------------------------------------------- 1 | cimport aimrocks.options as options 2 | from libc.stdint cimport uint64_t, uint32_t 3 | from aimrocks.status cimport Status 4 | from libcpp cimport bool as cpp_bool 5 | from libcpp.string cimport string 6 | from libcpp.vector cimport vector 7 | from aimrocks.slice_ cimport Slice 8 | from aimrocks.snapshot cimport Snapshot 9 | from aimrocks.iterator cimport Iterator 10 | 11 | cdef extern from "rocksdb/write_batch.h" namespace "rocksdb": 12 | cdef cppclass WriteBatch: 13 | WriteBatch() nogil except+ 14 | WriteBatch(string) nogil except+ 15 | void Put(const Slice&, const Slice&) nogil except+ 16 | void Put(ColumnFamilyHandle*, const Slice&, const Slice&) nogil except+ 17 | void Merge(const Slice&, const Slice&) nogil except+ 18 | void Merge(ColumnFamilyHandle*, const Slice&, const Slice&) nogil except+ 19 | void Delete(const Slice&) nogil except+ 20 | void Delete(ColumnFamilyHandle*, const Slice&) nogil except+ 21 | void DeleteRange(const Slice&, const Slice&) nogil except+ 22 | void DeleteRange(ColumnFamilyHandle*, const Slice&, const Slice&) nogil except+ 23 | void PutLogData(const Slice&) nogil except+ 24 | void Clear() nogil except+ 25 | const string& Data() nogil except+ 26 | int Count() nogil except+ 27 | 28 | cdef extern from "rdb_include/write_batch_iter_helper.hpp" namespace "py_rocks": 29 | cdef enum BatchItemOp "RecordItemsHandler::Optype": 30 | BatchItemOpPut "py_rocks::RecordItemsHandler::Optype::PutRecord" 31 | BatchItemOpMerge "py_rocks::RecordItemsHandler::Optype::MergeRecord" 32 | BatchItemOpDelte "py_rocks::RecordItemsHandler::Optype::DeleteRecord" 33 | 34 | cdef cppclass BatchItem "py_rocks::RecordItemsHandler::BatchItem": 35 | BatchItemOp op 36 | uint32_t column_family_id 37 | Slice key 38 | Slice value 39 | 40 | Status get_batch_items(WriteBatch* batch, vector[BatchItem]* items) 41 | 42 | 43 | cdef extern from "rocksdb/db.h" namespace "rocksdb": 44 | ctypedef uint64_t SequenceNumber 45 | string kDefaultColumnFamilyName 46 | 47 | cdef struct LiveFileMetaData: 48 | string name 49 | int level 50 | uint64_t size 51 | string smallestkey 52 | string largestkey 53 | SequenceNumber smallest_seqno 54 | SequenceNumber largest_seqno 55 | 56 | cdef cppclass Range: 57 | Range(const Slice&, const Slice&) 58 | 59 | cdef cppclass DB: 60 | Status Put( 61 | const options.WriteOptions&, 62 | ColumnFamilyHandle*, 63 | const Slice&, 64 | const Slice&) nogil except+ 65 | 66 | Status Delete( 67 | const options.WriteOptions&, 68 | ColumnFamilyHandle*, 69 | const Slice&) nogil except+ 70 | 71 | Status DeleteRange( 72 | const options.WriteOptions&, 73 | ColumnFamilyHandle*, 74 | const Slice&, 75 | const Slice&) nogil except+ 76 | 77 | Status Merge( 78 | const options.WriteOptions&, 79 | ColumnFamilyHandle*, 80 | const Slice&, 81 | const Slice&) nogil except+ 82 | 83 | Status Write( 84 | const options.WriteOptions&, 85 | WriteBatch*) nogil except+ 86 | 87 | Status Get( 88 | const options.ReadOptions&, 89 | ColumnFamilyHandle*, 90 | const Slice&, 91 | string*) nogil except+ 92 | 93 | vector[Status] MultiGet( 94 | const options.ReadOptions&, 95 | const vector[ColumnFamilyHandle*]&, 96 | const vector[Slice]&, 97 | vector[string]*) nogil except+ 98 | 99 | cpp_bool KeyMayExist( 100 | const options.ReadOptions&, 101 | ColumnFamilyHandle*, 102 | Slice&, 103 | string*, 104 | cpp_bool*) nogil except+ 105 | 106 | cpp_bool KeyMayExist( 107 | const options.ReadOptions&, 108 | ColumnFamilyHandle*, 109 | Slice&, 110 | string*) nogil except+ 111 | 112 | Iterator* NewIterator( 113 | const options.ReadOptions&, 114 | ColumnFamilyHandle*) nogil except+ 115 | 116 | void NewIterators( 117 | const options.ReadOptions&, 118 | vector[ColumnFamilyHandle*]&, 119 | vector[Iterator*]*) nogil except+ 120 | 121 | const Snapshot* GetSnapshot() nogil except+ 122 | 123 | void ReleaseSnapshot(const Snapshot*) nogil except+ 124 | 125 | cpp_bool GetProperty( 126 | ColumnFamilyHandle*, 127 | const Slice&, 128 | string*) nogil except+ 129 | 130 | void GetApproximateSizes( 131 | ColumnFamilyHandle*, 132 | const Range* 133 | int, 134 | uint64_t*) nogil except+ 135 | 136 | Status CompactRange( 137 | const options.CompactRangeOptions&, 138 | ColumnFamilyHandle*, 139 | const Slice*, 140 | const Slice*) nogil except+ 141 | 142 | Status CreateColumnFamily( 143 | const options.ColumnFamilyOptions&, 144 | const string&, 145 | ColumnFamilyHandle**) nogil except+ 146 | 147 | Status DropColumnFamily( 148 | ColumnFamilyHandle*) nogil except+ 149 | 150 | int NumberLevels(ColumnFamilyHandle*) nogil except+ 151 | int MaxMemCompactionLevel(ColumnFamilyHandle*) nogil except+ 152 | int Level0StopWriteTrigger(ColumnFamilyHandle*) nogil except+ 153 | const string& GetName() nogil except+ 154 | const options.Options& GetOptions(ColumnFamilyHandle*) nogil except+ 155 | Status Flush(const options.FlushOptions&, ColumnFamilyHandle*) nogil except+ 156 | Status FlushWAL(cpp_bool sync) nogil except+ 157 | Status DisableFileDeletions() nogil except+ 158 | Status EnableFileDeletions() nogil except+ 159 | 160 | Status TryCatchUpWithPrimary() nogil except+ 161 | 162 | # TODO: Status GetSortedWalFiles(VectorLogPtr& files) 163 | # TODO: SequenceNumber GetLatestSequenceNumber() 164 | # TODO: Status GetUpdatesSince( 165 | # SequenceNumber seq_number, 166 | # unique_ptr[TransactionLogIterator]*) 167 | 168 | Status DeleteFile(string) nogil except+ 169 | void GetLiveFilesMetaData(vector[LiveFileMetaData]*) nogil except+ 170 | ColumnFamilyHandle* DefaultColumnFamily() 171 | 172 | 173 | cdef Status DB_Open "rocksdb::DB::Open"( 174 | const options.Options&, 175 | const string&, 176 | DB**) nogil except+ 177 | 178 | cdef Status DB_Open_ColumnFamilies "rocksdb::DB::Open"( 179 | const options.Options&, 180 | const string&, 181 | const vector[ColumnFamilyDescriptor]&, 182 | vector[ColumnFamilyHandle*]*, 183 | DB**) nogil except+ 184 | 185 | cdef Status DB_OpenForReadOnly "rocksdb::DB::OpenForReadOnly"( 186 | const options.Options&, 187 | const string&, 188 | DB**, 189 | cpp_bool) nogil except+ 190 | 191 | cdef Status DB_OpenForReadOnly_ColumnFamilies "rocksdb::DB::OpenForReadOnly"( 192 | const options.Options&, 193 | const string&, 194 | const vector[ColumnFamilyDescriptor]&, 195 | vector[ColumnFamilyHandle*]*, 196 | DB**, 197 | cpp_bool) nogil except+ 198 | 199 | cdef Status DB_OpenAsSecondary "rocksdb::DB::OpenAsSecondary"( 200 | const options.Options&, 201 | const string&, 202 | const string&, 203 | DB**) nogil except+ 204 | 205 | cdef Status DB_OpenAsSecondary_ColumnFamilies "rocksdb::DB::OpenAsSecondary"( 206 | const options.Options&, 207 | const string&, 208 | const string&, 209 | const vector[ColumnFamilyDescriptor]&, 210 | vector[ColumnFamilyHandle*]*, 211 | DB**) nogil except+ 212 | 213 | cdef Status RepairDB(const string& dbname, const options.Options&) 214 | 215 | cdef Status ListColumnFamilies "rocksdb::DB::ListColumnFamilies" ( 216 | const options.Options&, 217 | const string&, 218 | vector[string]*) nogil except+ 219 | 220 | cdef cppclass ColumnFamilyHandle: 221 | const string& GetName() nogil except+ 222 | int GetID() nogil except+ 223 | 224 | cdef cppclass ColumnFamilyDescriptor: 225 | ColumnFamilyDescriptor() nogil except+ 226 | ColumnFamilyDescriptor( 227 | const string&, 228 | const options.ColumnFamilyOptions&) nogil except+ 229 | string name 230 | options.ColumnFamilyOptions options 231 | -------------------------------------------------------------------------------- /src/aimrocks/env.pxd: -------------------------------------------------------------------------------- 1 | cdef extern from "rocksdb/env.h" namespace "rocksdb": 2 | cdef cppclass Env: 3 | Env() 4 | 5 | cdef Env* Env_Default "rocksdb::Env::Default"() 6 | -------------------------------------------------------------------------------- /src/aimrocks/errors.py: -------------------------------------------------------------------------------- 1 | class NotFound(Exception): 2 | pass 3 | 4 | class Corruption(Exception): 5 | pass 6 | 7 | class NotSupported(Exception): 8 | pass 9 | 10 | class InvalidArgument(Exception): 11 | pass 12 | 13 | class RocksIOError(Exception): 14 | pass 15 | 16 | class MergeInProgress(Exception): 17 | pass 18 | 19 | class Incomplete(Exception): 20 | pass 21 | -------------------------------------------------------------------------------- /src/aimrocks/filter_policy.pxd: -------------------------------------------------------------------------------- 1 | from libcpp cimport bool as cpp_bool 2 | from libcpp.string cimport string 3 | from libc.string cimport const_char 4 | from aimrocks.slice_ cimport Slice 5 | from aimrocks.std_memory cimport shared_ptr 6 | from aimrocks.logger cimport Logger 7 | 8 | cdef extern from "rocksdb/filter_policy.h" namespace "rocksdb": 9 | cdef cppclass FilterPolicy: 10 | void CreateFilter(const Slice*, int, string*) nogil except+ 11 | cpp_bool KeyMayMatch(const Slice&, const Slice&) nogil except+ 12 | const_char* Name() nogil except+ 13 | 14 | cdef extern const FilterPolicy* NewBloomFilterPolicy(int) nogil except+ 15 | 16 | ctypedef void (*create_filter_func)( 17 | void*, 18 | Logger*, 19 | string&, 20 | const Slice*, 21 | int, 22 | string*) 23 | 24 | ctypedef cpp_bool (*key_may_match_func)( 25 | void*, 26 | Logger*, 27 | string&, 28 | const Slice&, 29 | const Slice&) 30 | 31 | cdef extern from "rdb_include/filter_policy_wrapper.hpp" namespace "py_rocks": 32 | cdef cppclass FilterPolicyWrapper: 33 | FilterPolicyWrapper( 34 | string, 35 | void*, 36 | create_filter_func, 37 | key_may_match_func) nogil except+ 38 | 39 | void set_info_log(shared_ptr[Logger]) nogil except+ 40 | -------------------------------------------------------------------------------- /src/aimrocks/include/rdb_include/comparator_wrapper.hpp: -------------------------------------------------------------------------------- 1 | #include "rocksdb/comparator.h" 2 | #include "rocksdb/env.h" 3 | #include 4 | 5 | using std::string; 6 | using rocksdb::Comparator; 7 | using rocksdb::Slice; 8 | using rocksdb::Logger; 9 | 10 | namespace py_rocks { 11 | class ComparatorWrapper: public Comparator { 12 | public: 13 | typedef int (*compare_func)( 14 | void*, 15 | Logger*, 16 | string&, 17 | const Slice&, 18 | const Slice&); 19 | 20 | ComparatorWrapper( 21 | string name, 22 | void* compare_context, 23 | compare_func compare_callback): 24 | name(name), 25 | compare_context(compare_context), 26 | compare_callback(compare_callback) 27 | {} 28 | 29 | virtual int Compare(const Slice& a, const Slice& b) const { 30 | string error_msg; 31 | int val; 32 | 33 | val = this->compare_callback( 34 | this->compare_context, 35 | this->info_log.get(), 36 | error_msg, 37 | a, 38 | b); 39 | 40 | if (error_msg.size()) { 41 | throw std::runtime_error(error_msg.c_str()); 42 | } 43 | return val; 44 | } 45 | 46 | virtual const char* Name() const { 47 | return this->name.c_str(); 48 | } 49 | 50 | virtual void FindShortestSeparator(string*, const Slice&) const {} 51 | virtual void FindShortSuccessor(string*) const {} 52 | 53 | void set_info_log(std::shared_ptr info_log) { 54 | this->info_log = info_log; 55 | } 56 | 57 | private: 58 | string name; 59 | void* compare_context; 60 | compare_func compare_callback; 61 | std::shared_ptr info_log; 62 | }; 63 | } 64 | -------------------------------------------------------------------------------- /src/aimrocks/include/rdb_include/filter_policy_wrapper.hpp: -------------------------------------------------------------------------------- 1 | #include "rocksdb/filter_policy.h" 2 | #include "rocksdb/env.h" 3 | #include 4 | 5 | using std::string; 6 | using rocksdb::FilterPolicy; 7 | using rocksdb::Slice; 8 | using rocksdb::Logger; 9 | 10 | namespace py_rocks { 11 | class FilterPolicyWrapper: public FilterPolicy { 12 | public: 13 | typedef void (*create_filter_func)( 14 | void* ctx, 15 | Logger*, 16 | string&, 17 | const Slice* keys, 18 | int n, 19 | string* dst); 20 | 21 | typedef bool (*key_may_match_func)( 22 | void* ctx, 23 | Logger*, 24 | string&, 25 | const Slice& key, 26 | const Slice& filter); 27 | 28 | FilterPolicyWrapper( 29 | string name, 30 | void* ctx, 31 | create_filter_func create_filter_callback, 32 | key_may_match_func key_may_match_callback): 33 | name(name), 34 | ctx(ctx), 35 | create_filter_callback(create_filter_callback), 36 | key_may_match_callback(key_may_match_callback) 37 | {} 38 | 39 | virtual void 40 | CreateFilter(const Slice* keys, int n, std::string* dst) const { 41 | string error_msg; 42 | 43 | this->create_filter_callback( 44 | this->ctx, 45 | this->info_log.get(), 46 | error_msg, 47 | keys, 48 | n, 49 | dst); 50 | 51 | if (error_msg.size()) { 52 | throw std::runtime_error(error_msg.c_str()); 53 | } 54 | } 55 | 56 | virtual bool 57 | KeyMayMatch(const Slice& key, const Slice& filter) const { 58 | string error_msg; 59 | bool val; 60 | 61 | val = this->key_may_match_callback( 62 | this->ctx, 63 | this->info_log.get(), 64 | error_msg, 65 | key, 66 | filter); 67 | 68 | if (error_msg.size()) { 69 | throw std::runtime_error(error_msg.c_str()); 70 | } 71 | return val; 72 | } 73 | 74 | virtual const char* Name() const { 75 | return this->name.c_str(); 76 | } 77 | 78 | void set_info_log(std::shared_ptr info_log) { 79 | this->info_log = info_log; 80 | } 81 | 82 | private: 83 | string name; 84 | void* ctx; 85 | create_filter_func create_filter_callback; 86 | key_may_match_func key_may_match_callback; 87 | std::shared_ptr info_log; 88 | }; 89 | } 90 | -------------------------------------------------------------------------------- /src/aimrocks/include/rdb_include/memtable_factories.hpp: -------------------------------------------------------------------------------- 1 | #include "rocksdb/memtablerep.h" 2 | 3 | using rocksdb::MemTableRepFactory; 4 | using rocksdb::VectorRepFactory; 5 | using rocksdb::SkipListFactory; 6 | 7 | namespace py_rocks { 8 | MemTableRepFactory* NewVectorRepFactory(size_t count = 0) { 9 | return new VectorRepFactory(count); 10 | } 11 | 12 | MemTableRepFactory* NewSkipListFactory() { 13 | return new SkipListFactory(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/aimrocks/include/rdb_include/merge_operator_wrapper.hpp: -------------------------------------------------------------------------------- 1 | #include "rocksdb/merge_operator.h" 2 | 3 | using std::string; 4 | using std::deque; 5 | using rocksdb::Slice; 6 | using rocksdb::Logger; 7 | using rocksdb::MergeOperator; 8 | using rocksdb::AssociativeMergeOperator; 9 | 10 | namespace py_rocks { 11 | class AssociativeMergeOperatorWrapper: public AssociativeMergeOperator { 12 | public: 13 | typedef bool (*merge_func)( 14 | void*, 15 | const Slice& key, 16 | const Slice* existing_value, 17 | const Slice& value, 18 | std::string* new_value, 19 | Logger* logger); 20 | 21 | 22 | AssociativeMergeOperatorWrapper( 23 | string name, 24 | void* merge_context, 25 | merge_func merge_callback): 26 | name(name), 27 | merge_context(merge_context), 28 | merge_callback(merge_callback) 29 | {} 30 | 31 | virtual bool Merge( 32 | const Slice& key, 33 | const Slice* existing_value, 34 | const Slice& value, 35 | std::string* new_value, 36 | Logger* logger) const 37 | { 38 | return this->merge_callback( 39 | this->merge_context, 40 | key, 41 | existing_value, 42 | value, 43 | new_value, 44 | logger); 45 | } 46 | 47 | virtual const char* Name() const { 48 | return this->name.c_str(); 49 | } 50 | 51 | private: 52 | string name; 53 | void* merge_context; 54 | merge_func merge_callback; 55 | }; 56 | 57 | class MergeOperatorWrapper: public MergeOperator { 58 | public: 59 | typedef bool (*full_merge_func)( 60 | void* ctx, 61 | const Slice& key, 62 | const Slice* existing_value, 63 | const deque& operand_list, 64 | string* new_value, 65 | Logger* logger); 66 | 67 | typedef bool (*partial_merge_func)( 68 | void* ctx, 69 | const Slice& key, 70 | const Slice& left_op, 71 | const Slice& right_op, 72 | string* new_value, 73 | Logger* logger); 74 | 75 | MergeOperatorWrapper( 76 | string name, 77 | void* full_merge_context, 78 | void* partial_merge_context, 79 | full_merge_func full_merge_callback, 80 | partial_merge_func partial_merge_callback): 81 | name(name), 82 | full_merge_context(full_merge_context), 83 | partial_merge_context(partial_merge_context), 84 | full_merge_callback(full_merge_callback), 85 | partial_merge_callback(partial_merge_callback) 86 | {} 87 | 88 | virtual bool FullMerge( 89 | const Slice& key, 90 | const Slice* existing_value, 91 | const deque& operand_list, 92 | string* new_value, 93 | Logger* logger) const 94 | { 95 | return this->full_merge_callback( 96 | this->full_merge_context, 97 | key, 98 | existing_value, 99 | operand_list, 100 | new_value, 101 | logger); 102 | } 103 | 104 | virtual bool PartialMerge ( 105 | const Slice& key, 106 | const Slice& left_operand, 107 | const Slice& right_operand, 108 | string* new_value, 109 | Logger* logger) const 110 | { 111 | return this->partial_merge_callback( 112 | this->partial_merge_context, 113 | key, 114 | left_operand, 115 | right_operand, 116 | new_value, 117 | logger); 118 | } 119 | 120 | virtual const char* Name() const { 121 | return this->name.c_str(); 122 | } 123 | 124 | private: 125 | string name; 126 | void* full_merge_context; 127 | void* partial_merge_context; 128 | full_merge_func full_merge_callback; 129 | partial_merge_func partial_merge_callback; 130 | 131 | }; 132 | } 133 | -------------------------------------------------------------------------------- /src/aimrocks/include/rdb_include/slice_transform_wrapper.hpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "rocksdb/slice_transform.h" 3 | #include "rocksdb/env.h" 4 | #include 5 | 6 | using std::string; 7 | using rocksdb::SliceTransform; 8 | using rocksdb::Slice; 9 | using rocksdb::Logger; 10 | 11 | namespace py_rocks { 12 | class SliceTransformWrapper: public SliceTransform { 13 | public: 14 | typedef Slice (*transform_func)( 15 | void*, 16 | Logger*, 17 | string&, 18 | const Slice&); 19 | 20 | typedef bool (*in_domain_func)( 21 | void*, 22 | Logger*, 23 | string&, 24 | const Slice&); 25 | 26 | typedef bool (*in_range_func)( 27 | void*, 28 | Logger*, 29 | string&, 30 | const Slice&); 31 | 32 | SliceTransformWrapper( 33 | string name, 34 | void* ctx, 35 | transform_func transfrom_callback, 36 | in_domain_func in_domain_callback, 37 | in_range_func in_range_callback): 38 | name(name), 39 | ctx(ctx), 40 | transfrom_callback(transfrom_callback), 41 | in_domain_callback(in_domain_callback), 42 | in_range_callback(in_range_callback) 43 | {} 44 | 45 | virtual const char* Name() const { 46 | return this->name.c_str(); 47 | } 48 | 49 | virtual Slice Transform(const Slice& src) const { 50 | string error_msg; 51 | Slice val; 52 | 53 | val = this->transfrom_callback( 54 | this->ctx, 55 | this->info_log.get(), 56 | error_msg, 57 | src); 58 | 59 | if (error_msg.size()) { 60 | throw std::runtime_error(error_msg.c_str()); 61 | } 62 | return val; 63 | } 64 | 65 | virtual bool InDomain(const Slice& src) const { 66 | string error_msg; 67 | bool val; 68 | 69 | val = this->in_domain_callback( 70 | this->ctx, 71 | this->info_log.get(), 72 | error_msg, 73 | src); 74 | 75 | if (error_msg.size()) { 76 | throw std::runtime_error(error_msg.c_str()); 77 | } 78 | return val; 79 | } 80 | 81 | virtual bool InRange(const Slice& dst) const { 82 | string error_msg; 83 | bool val; 84 | 85 | val = this->in_range_callback( 86 | this->ctx, 87 | this->info_log.get(), 88 | error_msg, 89 | dst); 90 | 91 | if (error_msg.size()) { 92 | throw std::runtime_error(error_msg.c_str()); 93 | } 94 | return val; 95 | } 96 | 97 | void set_info_log(std::shared_ptr info_log) { 98 | this->info_log = info_log; 99 | } 100 | 101 | private: 102 | string name; 103 | void* ctx; 104 | transform_func transfrom_callback; 105 | in_domain_func in_domain_callback; 106 | in_range_func in_range_callback; 107 | std::shared_ptr info_log; 108 | }; 109 | } 110 | -------------------------------------------------------------------------------- /src/aimrocks/include/rdb_include/utils.hpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | namespace py_rocks { 4 | template 5 | const T* vector_data(std::vector& v) { 6 | return v.data(); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/aimrocks/include/rdb_include/write_batch_iter_helper.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include "rocksdb/write_batch.h" 5 | 6 | namespace py_rocks { 7 | 8 | class RecordItemsHandler: public rocksdb::WriteBatch::Handler { 9 | public: 10 | enum Optype {PutRecord, MergeRecord, DeleteRecord}; 11 | 12 | class BatchItem { 13 | public: 14 | BatchItem( 15 | const Optype& op, 16 | uint32_t column_family_id, 17 | const rocksdb::Slice& key, 18 | const rocksdb::Slice& value): 19 | op(op), 20 | column_family_id(column_family_id), 21 | key(key), 22 | value(value) 23 | {} 24 | 25 | const Optype op; 26 | uint32_t column_family_id; 27 | const rocksdb::Slice key; 28 | const rocksdb::Slice value; 29 | }; 30 | 31 | typedef std::vector BatchItems; 32 | 33 | public: 34 | /* Items is filled during iteration. */ 35 | RecordItemsHandler(BatchItems* items): items(items) {} 36 | 37 | virtual rocksdb::Status PutCF( 38 | uint32_t column_family_id, const Slice& key, const Slice& value) { 39 | this->items->emplace_back(PutRecord, column_family_id, key, value); 40 | return rocksdb::Status::OK(); 41 | } 42 | 43 | virtual rocksdb::Status MergeCF( 44 | uint32_t column_family_id, const Slice& key, const Slice& value) { 45 | this->items->emplace_back(MergeRecord, column_family_id, key, value); 46 | return rocksdb::Status::OK(); 47 | } 48 | 49 | virtual rocksdb::Status DeleteCF( 50 | uint32_t column_family_id, const Slice& key) { 51 | this->items->emplace_back(DeleteRecord, column_family_id, key, rocksdb::Slice()); 52 | return rocksdb::Status::OK(); 53 | } 54 | 55 | private: 56 | BatchItems* items; 57 | }; 58 | 59 | rocksdb::Status 60 | get_batch_items(const rocksdb::WriteBatch* batch, RecordItemsHandler::BatchItems* items) { 61 | RecordItemsHandler handler(items); 62 | return batch->Iterate(&handler); 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/aimrocks/interfaces.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta 2 | from abc import abstractmethod 3 | 4 | 5 | class Comparator: 6 | __metaclass__ = ABCMeta 7 | 8 | @abstractmethod 9 | def compare(self, a, b): 10 | pass 11 | 12 | @abstractmethod 13 | def name(self): 14 | pass 15 | 16 | 17 | class AssociativeMergeOperator: 18 | __metaclass__ = ABCMeta 19 | 20 | @abstractmethod 21 | def merge(self, key, existing_value, value): 22 | pass 23 | 24 | @abstractmethod 25 | def name(self): 26 | pass 27 | 28 | 29 | class MergeOperator: 30 | __metaclass__ = ABCMeta 31 | 32 | @abstractmethod 33 | def full_merge(self, key, existing_value, operand_list): 34 | pass 35 | 36 | @abstractmethod 37 | def partial_merge(self, key, left_operand, right_operand): 38 | pass 39 | 40 | @abstractmethod 41 | def name(self): 42 | pass 43 | 44 | 45 | class FilterPolicy: 46 | __metaclass__ = ABCMeta 47 | 48 | @abstractmethod 49 | def name(self): 50 | pass 51 | 52 | @abstractmethod 53 | def create_filter(self, keys): 54 | pass 55 | 56 | @abstractmethod 57 | def key_may_match(self, key, filter_): 58 | pass 59 | 60 | class SliceTransform: 61 | __metaclass__ = ABCMeta 62 | 63 | @abstractmethod 64 | def name(self): 65 | pass 66 | 67 | @abstractmethod 68 | def transform(self, src): 69 | pass 70 | 71 | @abstractmethod 72 | def in_domain(self, src): 73 | pass 74 | 75 | @abstractmethod 76 | def in_range(self, dst): 77 | pass 78 | -------------------------------------------------------------------------------- /src/aimrocks/iterator.pxd: -------------------------------------------------------------------------------- 1 | from libcpp cimport bool as cpp_bool 2 | from aimrocks.slice_ cimport Slice 3 | from aimrocks.status cimport Status 4 | 5 | cdef extern from "rocksdb/iterator.h" namespace "rocksdb": 6 | cdef cppclass Iterator: 7 | cpp_bool Valid() nogil except+ 8 | void SeekToFirst() nogil except+ 9 | void SeekToLast() nogil except+ 10 | void Seek(const Slice&) nogil except+ 11 | void Next() nogil except+ 12 | void Prev() nogil except+ 13 | void SeekForPrev(const Slice&) nogil except+ 14 | Slice key() nogil except+ 15 | Slice value() nogil except+ 16 | Status status() nogil except+ 17 | -------------------------------------------------------------------------------- /src/aimrocks/lib_rocksdb.pxd: -------------------------------------------------------------------------------- 1 | # distutils: language = c++ 2 | 3 | import cython 4 | from libcpp.string cimport string 5 | from libcpp.deque cimport deque 6 | from libcpp.vector cimport vector 7 | from cpython cimport bool as py_bool 8 | from libcpp cimport bool as cpp_bool 9 | from libc.stdint cimport uint32_t 10 | from cython.operator cimport dereference as deref 11 | from cpython.bytes cimport PyBytes_AsString 12 | from cpython.bytes cimport PyBytes_Size 13 | from cpython.bytes cimport PyBytes_FromString 14 | from cpython.bytes cimport PyBytes_FromStringAndSize 15 | from cpython.unicode cimport PyUnicode_Decode 16 | 17 | from aimrocks.std_memory cimport shared_ptr 18 | cimport aimrocks.options as options 19 | cimport aimrocks.merge_operator as merge_operator 20 | cimport aimrocks.filter_policy as filter_policy 21 | cimport aimrocks.comparator as comparator 22 | cimport aimrocks.slice_transform as slice_transform 23 | cimport aimrocks.cache as cache 24 | cimport aimrocks.logger as logger 25 | cimport aimrocks.snapshot as snapshot 26 | cimport aimrocks.db as db 27 | cimport aimrocks.iterator as iterator 28 | cimport aimrocks.backup as backup 29 | cimport aimrocks.env as env 30 | cimport aimrocks.table_factory as table_factory 31 | cimport aimrocks.memtablerep as memtablerep 32 | cimport aimrocks.universal_compaction as universal_compaction 33 | 34 | # Enums are the only exception for direct imports 35 | # Their name als already unique enough 36 | from aimrocks.universal_compaction cimport kCompactionStopStyleSimilarSize 37 | from aimrocks.universal_compaction cimport kCompactionStopStyleTotalSize 38 | 39 | from aimrocks.options cimport FlushOptions 40 | from aimrocks.options cimport kCompactionStyleLevel 41 | from aimrocks.options cimport kCompactionStyleUniversal 42 | from aimrocks.options cimport kCompactionStyleFIFO 43 | from aimrocks.options cimport kCompactionStyleNone 44 | 45 | from aimrocks.slice_ cimport Slice 46 | from aimrocks.status cimport Status 47 | 48 | ctypedef const filter_policy.FilterPolicy ConstFilterPolicy 49 | 50 | cdef extern from "rdb_include/utils.hpp" namespace "py_rocks": 51 | cdef const Slice* vector_data(vector[Slice]&) 52 | 53 | cdef extern from "Python.h": 54 | void PyEval_InitThreads() 55 | 56 | cdef check_status(const Status& st) 57 | 58 | cdef string bytes_to_string(path) except * 59 | 60 | cdef string_to_bytes(string ob) 61 | 62 | cdef Slice bytes_to_slice(ob) except * 63 | 64 | cdef slice_to_bytes(Slice sl) 65 | 66 | cdef string path_to_string(object path) except * 67 | 68 | cdef object string_to_path(string path) 69 | 70 | cdef class PyComparator(object): 71 | cdef object get_ob(self) 72 | 73 | cdef const comparator.Comparator* get_comparator(self) 74 | 75 | cdef set_info_log(self, shared_ptr[logger.Logger] info_log) 76 | 77 | cdef class PyGenericComparator(PyComparator): 78 | cdef comparator.ComparatorWrapper* comparator_ptr 79 | cdef object ob 80 | # def __cinit__(self, object ob) 81 | # def __dealloc__(self) 82 | cdef object get_ob(self) 83 | cdef const comparator.Comparator* get_comparator(self) 84 | cdef set_info_log(self, shared_ptr[logger.Logger] info_log) 85 | 86 | cdef class PyBytewiseComparator(PyComparator): 87 | cdef const comparator.Comparator* comparator_ptr 88 | # def __cinit__(self) 89 | cpdef name(self) 90 | cpdef compare(self, a, b) 91 | cdef object get_ob(self) 92 | cdef const comparator.Comparator* get_comparator(self) 93 | 94 | 95 | cdef int compare_callback( 96 | void* ctx, 97 | logger.Logger* log, 98 | string& error_msg, 99 | const Slice& a, 100 | const Slice& b) with gil 101 | 102 | # BytewiseComparator = PyBytewiseComparator 103 | ######################################### 104 | 105 | 106 | 107 | ## Here comes the stuff for the filter policy 108 | cdef class PyFilterPolicy(object): 109 | cdef object get_ob(self) 110 | cdef shared_ptr[ConstFilterPolicy] get_policy(self) 111 | cdef set_info_log(self, shared_ptr[logger.Logger] info_log) 112 | 113 | cdef class PyGenericFilterPolicy(PyFilterPolicy): 114 | cdef shared_ptr[filter_policy.FilterPolicyWrapper] policy 115 | cdef object ob 116 | # def __cinit__(self, object ob) 117 | cdef object get_ob(self) 118 | cdef shared_ptr[ConstFilterPolicy] get_policy(self) 119 | cdef set_info_log(self, shared_ptr[logger.Logger] info_log) 120 | 121 | 122 | cdef void create_filter_callback( 123 | void* ctx, 124 | logger.Logger* log, 125 | string& error_msg, 126 | const Slice* keys, 127 | int n, 128 | string* dst) with gil 129 | 130 | cdef cpp_bool key_may_match_callback( 131 | void* ctx, 132 | logger.Logger* log, 133 | string& error_msg, 134 | const Slice& key, 135 | const Slice& filt) with gil 136 | 137 | cdef class PyBloomFilterPolicy(PyFilterPolicy): 138 | cdef shared_ptr[ConstFilterPolicy] policy 139 | # def __cinit__(self, int bits_per_key) 140 | cpdef name(self) 141 | cpdef create_filter(self, keys) 142 | cpdef key_may_match(self, key, filter_) 143 | cdef object get_ob(self) 144 | cdef shared_ptr[ConstFilterPolicy] get_policy(self) 145 | 146 | # BloomFilterPolicy = PyBloomFilterPolicy 147 | # ############################################# 148 | 149 | 150 | 151 | ## Here comes the stuff for the merge operator 152 | cdef class PyMergeOperator(object): 153 | cdef shared_ptr[merge_operator.MergeOperator] merge_op 154 | cdef object ob 155 | # def __cinit__(self, object ob) 156 | cdef object get_ob(self) 157 | cdef shared_ptr[merge_operator.MergeOperator] get_operator(self) 158 | 159 | cdef cpp_bool merge_callback( 160 | void* ctx, 161 | const Slice& key, 162 | const Slice* existing_value, 163 | const Slice& value, 164 | string* new_value, 165 | logger.Logger* log) with gil 166 | 167 | cdef cpp_bool full_merge_callback( 168 | void* ctx, 169 | const Slice& key, 170 | const Slice* existing_value, 171 | const deque[string]& op_list, 172 | string* new_value, 173 | logger.Logger* log) with gil 174 | 175 | cdef cpp_bool partial_merge_callback( 176 | void* ctx, 177 | const Slice& key, 178 | const Slice& left_op, 179 | const Slice& right_op, 180 | string* new_value, 181 | logger.Logger* log) with gil 182 | 183 | cdef class PyCache(object): 184 | cdef shared_ptr[cache.Cache] get_cache(self) 185 | 186 | cdef class PyLRUCache(PyCache): 187 | cdef shared_ptr[cache.Cache] cache_ob 188 | # def __cinit__(self, capacity, shard_bits = *) 189 | cdef shared_ptr[cache.Cache] get_cache(self) 190 | 191 | LRUCache = PyLRUCache 192 | ############################### 193 | 194 | ### Here comes the stuff for SliceTransform 195 | cdef class PySliceTransform(object): 196 | cdef shared_ptr[slice_transform.SliceTransform] transfomer 197 | cdef object ob 198 | # def __cinit__(self, object ob) 199 | cdef object get_ob(self) 200 | cdef shared_ptr[slice_transform.SliceTransform] get_transformer(self) 201 | cdef set_info_log(self, shared_ptr[logger.Logger] info_log) 202 | 203 | 204 | cdef Slice slice_transform_callback( 205 | void* ctx, 206 | logger.Logger* log, 207 | string& error_msg, 208 | const Slice& src) with gil 209 | 210 | cdef cpp_bool slice_in_domain_callback( 211 | void* ctx, 212 | logger.Logger* log, 213 | string& error_msg, 214 | const Slice& src) with gil 215 | 216 | cdef cpp_bool slice_in_range_callback( 217 | void* ctx, 218 | logger.Logger* log, 219 | string& error_msg, 220 | const Slice& src) with gil 221 | 222 | ## Here are the TableFactories 223 | cdef class PyTableFactory(object): 224 | cdef shared_ptr[table_factory.TableFactory] factory 225 | cdef shared_ptr[table_factory.TableFactory] get_table_factory(self) 226 | cdef set_info_log(self, shared_ptr[logger.Logger] info_log) 227 | 228 | cdef class BlockBasedTableFactory(PyTableFactory): 229 | cdef PyFilterPolicy py_filter_policy 230 | # def __init__(self, 231 | # index_type='binary_search', 232 | # py_bool hash_index_allow_collision=True, 233 | # checksum='crc32', 234 | # PyCache block_cache = *, 235 | # PyCache block_cache_compressed = *, 236 | # filter_policy = *, 237 | # no_block_cache=False, 238 | # block_size = *, 239 | # block_size_deviation = *, 240 | # block_restart_interval = *, 241 | # whole_key_filtering = *) 242 | cdef set_info_log(self, shared_ptr[logger.Logger] info_log) 243 | 244 | cdef class PlainTableFactory(PyTableFactory): 245 | pass 246 | # def __init__( 247 | # self, 248 | # user_key_len=0, 249 | # bloom_bits_per_key=10, 250 | # hash_table_ratio=0.75, 251 | # index_sparseness=10, 252 | # huge_page_tlb_size=0, 253 | # encoding_type='plain', 254 | # py_bool full_scan_mode=False) 255 | ############################################# 256 | 257 | ### Here are the MemtableFactories 258 | cdef class PyMemtableFactory(object): 259 | cdef shared_ptr[memtablerep.MemTableRepFactory] factory 260 | cdef shared_ptr[memtablerep.MemTableRepFactory] get_memtable_factory(self) 261 | 262 | cdef class SkipListMemtableFactory(PyMemtableFactory): 263 | pass 264 | # def __init__(self) 265 | 266 | cdef class VectorMemtableFactory(PyMemtableFactory): 267 | pass 268 | # def __init__(self, count=0) 269 | 270 | cdef class HashSkipListMemtableFactory(PyMemtableFactory): 271 | pass 272 | # def __init__( 273 | # self, 274 | # bucket_count=1000000, 275 | # skiplist_height=4, 276 | # skiplist_branching_factor=4) 277 | 278 | cdef class HashLinkListMemtableFactory(PyMemtableFactory): 279 | pass 280 | # def __init__(self, bucket_count=50000) 281 | ################################## 282 | 283 | 284 | cdef class CompressionType(object): 285 | pass 286 | # no_compression = u'no_compression' 287 | # snappy_compression = u'snappy_compression' 288 | # zlib_compression = u'zlib_compression' 289 | # bzip2_compression = u'bzip2_compression' 290 | # lz4_compression = u'lz4_compression' 291 | # lz4hc_compression = u'lz4hc_compression' 292 | # xpress_compression = u'xpress_compression' 293 | # zstd_compression = u'zstd_compression' 294 | # zstdnotfinal_compression = u'zstdnotfinal_compression' 295 | # disable_compression = u'disable_compression' 296 | 297 | cdef class CompactionPri(object): 298 | pass 299 | # by_compensated_size = u'by_compensated_size' 300 | # oldest_largest_seq_first = u'oldest_largest_seq_first' 301 | # oldest_smallest_seq_first = u'oldest_smallest_seq_first' 302 | # min_overlapping_ratio = u'min_overlapping_ratio' 303 | 304 | cdef class _ColumnFamilyHandle: 305 | """ This is an internal class that we will weakref for safety """ 306 | cdef db.ColumnFamilyHandle* handle 307 | cdef object __weakref__ 308 | cdef object weak_handle 309 | # def __cinit__(self) 310 | # def __dealloc__(self) 311 | @staticmethod 312 | cdef from_handle_ptr(db.ColumnFamilyHandle* handle) 313 | # @property 314 | # def name(self) 315 | # @property 316 | # def id(self) 317 | # @property 318 | # def weakref(self) 319 | 320 | cdef class ColumnFamilyHandle: 321 | """ This represents a ColumnFamilyHandle """ 322 | cdef object _ref 323 | cdef readonly bytes name 324 | cdef readonly int id 325 | # def __cinit__(self, weakhandle) 326 | # def __init__(self, *) 327 | @staticmethod 328 | cdef object from_wrapper(_ColumnFamilyHandle real_handle) 329 | # @property 330 | # cpdef is_valid(self) 331 | # def __repr__(self) 332 | cdef db.ColumnFamilyHandle* get_handle(self) except NULL 333 | # def __eq__(self, other) 334 | # def __lt__(self, other) 335 | # Since @total_ordering isn't a thing for cython 336 | # def __ne__(self, other) 337 | # def __gt__(self, other) 338 | # def __le__(self, other) 339 | # def __ge__(self, other) 340 | # def __hash__(self) 341 | 342 | cdef class ColumnFamilyOptions(object): 343 | cdef options.ColumnFamilyOptions* copts 344 | cdef PyComparator py_comparator 345 | cdef PyMergeOperator py_merge_operator 346 | cdef PySliceTransform py_prefix_extractor 347 | cdef PyTableFactory py_table_factory 348 | cdef PyMemtableFactory py_memtable_factory 349 | 350 | # Used to protect sharing of Options with many DB-objects 351 | cdef cpp_bool in_use 352 | # def __cinit__(self) 353 | # def __dealloc__(self) 354 | # def __init__(self, **kwargs) 355 | # property write_buffer_size: 356 | # def __get__(self) 357 | # def __set__(self, value) 358 | # property max_write_buffer_number: 359 | # def __get__(self) 360 | # def __set__(self, value) 361 | # property min_write_buffer_number_to_merge: 362 | # def __get__(self) 363 | # def __set__(self, value) 364 | # property compression_opts: 365 | # def __get__(self) 366 | # def __set__(self, dict value) 367 | # property compaction_pri: 368 | # def __get__(self) 369 | # def __set__(self, value) 370 | # property compression: 371 | # def __get__(self) 372 | # def __set__(self, value) 373 | # property max_compaction_bytes: 374 | # def __get__(self) 375 | # def __set__(self, value) 376 | # property num_levels: 377 | # def __get__(self) 378 | # def __set__(self, value) 379 | # property level0_file_num_compaction_trigger: 380 | # def __get__(self) 381 | # def __set__(self, value) 382 | # property level0_slowdown_writes_trigger: 383 | # def __get__(self) 384 | # def __set__(self, value) 385 | # property level0_stop_writes_trigger: 386 | # def __get__(self) 387 | # def __set__(self, value) 388 | # property max_mem_compaction_level: 389 | # def __get__(self) 390 | # def __set__(self, value) 391 | # property target_file_size_base: 392 | # def __get__(self) 393 | # def __set__(self, value) 394 | # property target_file_size_multiplier: 395 | # def __get__(self) 396 | # def __set__(self, value) 397 | # property max_bytes_for_level_base: 398 | # def __get__(self) 399 | # def __set__(self, value) 400 | # property max_bytes_for_level_multiplier: 401 | # def __get__(self) 402 | # def __set__(self, value) 403 | # property max_bytes_for_level_multiplier_additional: 404 | # def __get__(self) 405 | # def __set__(self, value) 406 | # property arena_block_size: 407 | # def __get__(self) 408 | # def __set__(self, value) 409 | # property disable_auto_compactions: 410 | # def __get__(self) 411 | # def __set__(self, value) 412 | # # FIXME: remove to util/options_helper.h 413 | # # property allow_os_buffer: 414 | # # def __get__(self): 415 | # # return self.copts.allow_os_buffer 416 | # # def __set__(self, value): 417 | # # self.copts.allow_os_buffer = value 418 | # property compaction_style: 419 | # def __get__(self) 420 | # def __set__(self, str value) 421 | # property compaction_options_universal: 422 | # def __get__(self) 423 | # def __set__(self, dict value) 424 | # # Deprecate 425 | # # property filter_deletes: 426 | # # def __get__(self): 427 | # # return self.copts.filter_deletes 428 | # # def __set__(self, value): 429 | # # self.copts.filter_deletes = value 430 | # property max_sequential_skip_in_iterations: 431 | # def __get__(self) 432 | # def __set__(self, value) 433 | # property inplace_update_support: 434 | # def __get__(self) 435 | # def __set__(self, value) 436 | # property table_factory: 437 | # def __get__(self) 438 | # def __set__(self, PyTableFactory value) 439 | # property memtable_factory: 440 | # def __get__(self) 441 | # def __set__(self, PyMemtableFactory value) 442 | # property inplace_update_num_locks: 443 | # def __get__(self) 444 | # def __set__(self, value) 445 | # property comparator: 446 | # def __get__(self) 447 | # def __set__(self, value) 448 | # property merge_operator: 449 | # def __get__(self) 450 | # def __set__(self, value) 451 | # property prefix_extractor: 452 | # def __get__(self) 453 | # def __set__(self, value) 454 | 455 | cdef class Options(ColumnFamilyOptions): 456 | cdef options.Options* opts 457 | cdef PyCache py_row_cache 458 | # def __cinit__(self) 459 | # def __dealloc__(self) 460 | # def __init__(self, **kwargs) 461 | # property create_if_missing: 462 | # def __get__(self) 463 | # def __set__(self, value) 464 | # property error_if_exists: 465 | # def __get__(self) 466 | # def __set__(self, value) 467 | # property paranoid_checks: 468 | # def __get__(self) 469 | # def __set__(self, value) 470 | # property max_open_files: 471 | # def __get__(self) 472 | # def __set__(self, value) 473 | # property use_fsync: 474 | # def __get__(self) 475 | # def __set__(self, value) 476 | # property db_log_dir: 477 | # def __get__(self) 478 | # def __set__(self, value) 479 | # property wal_dir: 480 | # def __get__(self) 481 | # def __set__(self, value) 482 | # property delete_obsolete_files_period_micros: 483 | # def __get__(self) 484 | # def __set__(self, value) 485 | # property max_background_compactions: 486 | # def __get__(self) 487 | # def __set__(self, value) 488 | # property max_background_flushes: 489 | # def __get__(self) 490 | # def __set__(self, value) 491 | # property max_log_file_size: 492 | # def __get__(self) 493 | # def __set__(self, value) 494 | # property db_write_buffer_size: 495 | # def __get__(self) 496 | # def __set__(self, value) 497 | # property log_file_time_to_roll: 498 | # def __get__(self) 499 | # def __set__(self, value) 500 | # property keep_log_file_num: 501 | # def __get__(self) 502 | # def __set__(self, value) 503 | # property max_manifest_file_size: 504 | # def __get__(self) 505 | # def __set__(self, value) 506 | # property table_cache_numshardbits: 507 | # def __get__(self) 508 | # def __set__(self, value) 509 | # property wal_ttl_seconds: 510 | # def __get__(self) 511 | # def __set__(self, value) 512 | # property wal_size_limit_mb: 513 | # def __get__(self) 514 | # def __set__(self, value) 515 | # property manifest_preallocation_size: 516 | # def __get__(self) 517 | # def __set__(self, value) 518 | # property enable_write_thread_adaptive_yield: 519 | # def __get__(self) 520 | # def __set__(self, value) 521 | # property allow_concurrent_memtable_write: 522 | # def __get__(self) 523 | # def __set__(self, value) 524 | # property allow_mmap_reads: 525 | # def __get__(self) 526 | # def __set__(self, value) 527 | # property allow_mmap_writes: 528 | # def __get__(self) 529 | # def __set__(self, value) 530 | # property is_fd_close_on_exec: 531 | # def __get__(self) 532 | # def __set__(self, value) 533 | # property stats_dump_period_sec: 534 | # def __get__(self) 535 | # def __set__(self, value) 536 | # property advise_random_on_open: 537 | # def __get__(self) 538 | # def __set__(self, value) 539 | # # TODO: need to remove -Wconversion to make this work 540 | # # property access_hint_on_compaction_start: 541 | # # def __get__(sel 542 | # # def __set__(self, AccessHint valu 543 | # property use_adaptive_mutex: 544 | # def __get__(self) 545 | # def __set__(self, value) 546 | # property bytes_per_sync: 547 | # def __get__(self) 548 | # def __set__(self, value) 549 | # property row_cache: 550 | # def __get__(self) 551 | # def __set__(self, value) 552 | # # cpp_bool skip_checking_sst_file_sizes_on_db_open 553 | # property skip_checking_sst_file_sizes_on_db_open: 554 | # def __get__(self) 555 | # def __set__(self, value) 556 | # # cpp_bool skip_stats_update_on_db_open 557 | # property skip_stats_update_on_db_open: 558 | # def __get__(self) 559 | # def __set__(self, value) 560 | 561 | # Forward declaration 562 | # cdef class Snapshot 563 | 564 | # cdef class KeysIterator 565 | # cdef class ValuesIterator 566 | # cdef class ItemsIterator 567 | # cdef class ReversedIterator 568 | 569 | # # Forward declaration 570 | # cdef class WriteBatchIterator 571 | 572 | cdef class WriteBatch(object): 573 | cdef db.WriteBatch* batch 574 | 575 | # def __cinit__(self, data = *) 576 | # def __dealloc__(self) 577 | cpdef void put(self, bytes key, bytes value, ColumnFamilyHandle column_family = *) 578 | cpdef void merge(self, bytes key, bytes value, ColumnFamilyHandle column_family = *) 579 | cpdef void delete_single(self, bytes key, ColumnFamilyHandle column_family = *) 580 | cpdef void delete_range(self, bytes begin_key, bytes end_key, ColumnFamilyHandle column_family = *) 581 | cpdef void clear(self) 582 | cpdef data(self) 583 | cpdef int count(self) 584 | # def __iter__(self) 585 | 586 | cdef class WriteBatchIterator(object): 587 | # Need a reference to the WriteBatch. 588 | # The BatchItems are only pointers to the memory in WriteBatch. 589 | cdef WriteBatch batch 590 | cdef vector[db.BatchItem] items 591 | cdef size_t pos 592 | # def __init__(self, WriteBatch batch) 593 | # def __iter__(self) 594 | # def __next__(self) 595 | 596 | cdef class IDB(object): 597 | cpdef void put(self, bytes key, bytes value, cpp_bool sync = *, cpp_bool disable_wal = *, ColumnFamilyHandle column_family = *) 598 | cpdef void delete_single(self, bytes key, cpp_bool sync = *, cpp_bool disable_wal = *, ColumnFamilyHandle column_family = *) 599 | cpdef void delete_range(self, bytes begin_key, bytes end_key, cpp_bool sync = *, cpp_bool disable_wal = *, ColumnFamilyHandle column_family = *) 600 | cpdef void flush(self) 601 | cpdef void flush_wal(self, cpp_bool sync = *) 602 | cpdef void merge(self, bytes key, bytes value, cpp_bool sync = *, cpp_bool disable_wal = *, ColumnFamilyHandle column_family = *) 603 | cpdef void write(self, WriteBatch batch, cpp_bool sync = *, cpp_bool disable_wal = *) 604 | cpdef get(self, bytes key, ColumnFamilyHandle column_family = *) 605 | cpdef multi_get(self, keys) 606 | cpdef key_may_exist(self, bytes key, cpp_bool fetch = *, ColumnFamilyHandle column_family = *) 607 | cpdef Iterator iterkeys(self, ColumnFamilyHandle column_family = *) 608 | cpdef Iterator itervalues(self, ColumnFamilyHandle column_family = *) 609 | cpdef Iterator iteritems(self, ColumnFamilyHandle column_family = *) 610 | 611 | @cython.no_gc_clear 612 | cdef class DB(IDB): 613 | cdef Options opts 614 | cdef db.DB* db 615 | cdef list cf_handles 616 | cdef list cf_options 617 | # def __cinit__(self, db_name, Options opts, dict column_families = *, read_only=False) 618 | # def __dealloc__(self) 619 | cpdef void close(self) 620 | # @property 621 | # cpdef column_families(self) 622 | cpdef get_column_family(self, bytes name) 623 | cpdef iterskeys(self, column_families) 624 | cpdef itersvalues(self, column_families) 625 | cpdef itersitems(self, column_families) 626 | cpdef snapshot(self) 627 | cpdef get_property(self, prop, ColumnFamilyHandle column_family = *) 628 | cpdef try_catch_up_with_primary(self) 629 | cpdef get_live_files_metadata(self) 630 | cpdef void compact_range( 631 | self, 632 | bytes begin = *, 633 | bytes end = *, 634 | cpp_bool change_level = *, 635 | int target_level = *, 636 | str bottommost_level_compaction = *, 637 | ColumnFamilyHandle column_family = * 638 | ) 639 | # @staticmethod 640 | # def __parse_read_opts( 641 | # verify_checksums = *, 642 | # fill_cache = *, 643 | # snapshot = *, 644 | # read_tier = *) 645 | cdef options.ReadOptions build_read_opts(self, dict py_opts) 646 | # property options: 647 | # def __get__(self) 648 | cpdef create_column_family(self, bytes name, ColumnFamilyOptions copts) 649 | cpdef void drop_column_family(self, ColumnFamilyHandle weak_handle) 650 | 651 | cpdef repair_db(db_name, Options opts) 652 | 653 | cpdef list_column_families(db_name, Options opts) 654 | 655 | @cython.no_gc_clear 656 | cdef class Snapshot(object): 657 | cdef const snapshot.Snapshot* ptr 658 | cdef DB db 659 | 660 | # def __cinit__(self, DB db) 661 | # def __dealloc__(self) 662 | 663 | 664 | cdef class Iterator: 665 | cdef object _current_value 666 | 667 | cpdef object next(self) 668 | cpdef object get(self) 669 | cpdef void skip(self) 670 | 671 | 672 | cdef class BaseIterator(Iterator): 673 | cdef iterator.Iterator* ptr 674 | cdef DB db 675 | cdef ColumnFamilyHandle handle 676 | 677 | # def __cinit__(self, DB db, ColumnFamilyHandle handle = None) 678 | # def __dealloc__(self) 679 | # def __iter__(self) 680 | # def __next__(self) 681 | # def __reversed__(self) 682 | cpdef void seek_to_first(self) 683 | cpdef void seek_to_last(self) 684 | cpdef void seek(self, bytes key) 685 | cpdef void seek_for_prev(self, bytes key) 686 | cdef object get_ob(self) 687 | 688 | cdef class KeysIterator(BaseIterator): 689 | cdef object get_ob(self) 690 | 691 | cdef class ValuesIterator(BaseIterator): 692 | cdef object get_ob(self) 693 | 694 | cdef class ItemsIterator(BaseIterator): 695 | cdef object get_ob(self) 696 | 697 | cdef class ReversedIterator(Iterator): 698 | cdef BaseIterator it 699 | 700 | # def __cinit__(self, BaseIterator it) 701 | cpdef void seek_to_first(self) 702 | cpdef void seek_to_last(self) 703 | cpdef void seek(self, bytes key) 704 | cpdef void seek_for_prev(self, bytes key) 705 | # def __iter__(self) 706 | # def __reversed__(self) 707 | # def __next__(self) 708 | 709 | cdef class BackupEngine(object): 710 | cdef backup.BackupEngine* engine 711 | 712 | # def __cinit__(self, backup_dir)s 713 | # def __dealloc__(self) 714 | cpdef create_backup(self, DB db, flush_before_backup = *) 715 | cpdef restore_backup(self, backup_id, db_dir, wal_dir) 716 | cpdef restore_latest_backup(self, db_dir, wal_dir) 717 | cpdef stop_backup(self) 718 | cpdef purge_old_backups(self, num_backups_to_keep) 719 | cpdef delete_backup(self, backup_id) 720 | cpdef get_backup_info(self) 721 | -------------------------------------------------------------------------------- /src/aimrocks/lib_rocksdb.pyi: -------------------------------------------------------------------------------- 1 | # import cython 2 | # from libcpp.string cimport string 3 | # from libcpp.deque cimport deque 4 | # from libcpp.vector cimport vector 5 | # from cpython cimport bool as py_bool 6 | # from libcpp cimport bool as cpp_bool 7 | # from libc.stdint cimport uint32_t 8 | # from cython.operator cimport dereference as deref 9 | # from cpython.bytes cimport PyBytes_AsString 10 | # from cpython.bytes cimport PyBytes_Size 11 | # from cpython.bytes cimport PyBytes_FromString 12 | # from cpython.bytes cimport PyBytes_FromStringAndSize 13 | # from cpython.unicode cimport PyUnicode_Decode 14 | 15 | # from std_memory cimport shared_ptr 16 | # cimport options 17 | # cimport merge_operator 18 | # cimport filter_policy 19 | # cimport comparator 20 | # cimport slice_transform 21 | # cimport cache 22 | # cimport logger 23 | # cimport snapshot 24 | # cimport db 25 | # cimport iterator 26 | # cimport backup 27 | # cimport env 28 | # cimport table_factory 29 | # cimport memtablerep 30 | # cimport universal_compaction 31 | 32 | # # Enums are the only exception for direct imports 33 | # # Their name als already unique enough 34 | # from universal_compaction cimport kCompactionStopStyleSimilarSize 35 | # from universal_compaction cimport kCompactionStopStyleTotalSize 36 | 37 | # from options cimport kCompactionStyleLevel 38 | # from options cimport kCompactionStyleUniversal 39 | # from options cimport kCompactionStyleFIFO 40 | # from options cimport kCompactionStyleNone 41 | 42 | # from slice_ cimport Slice 43 | # from status cimport Status 44 | 45 | # import sys 46 | # from interfaces import MergeOperator as IMergeOperator 47 | # from interfaces import AssociativeMergeOperator as IAssociativeMergeOperator 48 | # from interfaces import FilterPolicy as IFilterPolicy 49 | # from interfaces import Comparator as IComparator 50 | # from interfaces import SliceTransform as ISliceTransform 51 | # import traceback 52 | # import errors 53 | # import weakref 54 | 55 | # ctypedef const filter_policy.FilterPolicy ConstFilterPolicy 56 | 57 | # cdef extern from "cpp/utils.hpp" namespace "py_rocks": 58 | # cdef const Slice* vector_data(vector[Slice]&) 59 | 60 | # # Prepare python for threaded usage. 61 | # # Python callbacks (merge, comparator) 62 | # # could be executed in a rocksdb background thread (eg. compaction). 63 | # cdef extern from "Python.h": 64 | # void PyEval_InitThreads() 65 | # PyEval_InitThreads() 66 | 67 | # ## Here comes the stuff to wrap the status to exception 68 | # cdef check_status(const Status& st): 69 | # if st.ok(): 70 | # return 71 | 72 | # if st.IsNotFound(): 73 | # raise errors.NotFound(st.ToString()) 74 | 75 | # if st.IsCorruption(): 76 | # raise errors.Corruption(st.ToString()) 77 | 78 | # if st.IsNotSupported(): 79 | # raise errors.NotSupported(st.ToString()) 80 | 81 | # if st.IsInvalidArgument(): 82 | # raise errors.InvalidArgument(st.ToString()) 83 | 84 | # if st.IsIOError(): 85 | # raise errors.RocksIOError(st.ToString()) 86 | 87 | # if st.IsMergeInProgress(): 88 | # raise errors.MergeInProgress(st.ToString()) 89 | 90 | # if st.IsIncomplete(): 91 | # raise errors.Incomplete(st.ToString()) 92 | 93 | # raise Exception("Unknown error: %s" % st.ToString()) 94 | # ###################################################### 95 | 96 | 97 | # cdef string bytes_to_string(path) except *: 98 | # return string(PyBytes_AsString(path), PyBytes_Size(path)) 99 | 100 | # cdef string_to_bytes(string ob): 101 | # return PyBytes_FromStringAndSize(ob.c_str(), ob.size()) 102 | 103 | # cdef Slice bytes_to_slice(ob) except *: 104 | # return Slice(PyBytes_AsString(ob), PyBytes_Size(ob)) 105 | 106 | # cdef slice_to_bytes(Slice sl): 107 | # return PyBytes_FromStringAndSize(sl.data(), sl.size()) 108 | 109 | # ## only for filsystem paths 110 | # cdef string path_to_string(object path) except *: 111 | # if isinstance(path, bytes): 112 | # return bytes_to_string(path) 113 | # if isinstance(path, unicode): 114 | # path = path.encode(sys.getfilesystemencoding()) 115 | # return bytes_to_string(path) 116 | # else: 117 | # raise TypeError("Wrong type for path: %s" % path) 118 | 119 | # cdef object string_to_path(string path): 120 | # fs_encoding = sys.getfilesystemencoding().encode('ascii') 121 | # return PyUnicode_Decode(path.c_str(), path.size(), fs_encoding, "replace") 122 | 123 | # ## Here comes the stuff for the comparator 124 | # @cython.internal 125 | # cdef class PyComparator(object): 126 | # cdef object get_ob(self): 127 | # return None 128 | 129 | # cdef const comparator.Comparator* get_comparator(self): 130 | # return NULL 131 | 132 | # cdef set_info_log(self, shared_ptr[logger.Logger] info_log): 133 | # pass 134 | 135 | # @cython.internal 136 | # cdef class PyGenericComparator(PyComparator): 137 | # cdef comparator.ComparatorWrapper* comparator_ptr 138 | # cdef object ob 139 | 140 | # def __cinit__(self, object ob): 141 | # self.comparator_ptr = NULL 142 | # if not isinstance(ob, IComparator): 143 | # raise TypeError("%s is not of type %s" % (ob, IComparator)) 144 | 145 | # self.ob = ob 146 | # self.comparator_ptr = new comparator.ComparatorWrapper( 147 | # bytes_to_string(ob.name()), 148 | # ob, 149 | # compare_callback) 150 | 151 | # def __dealloc__(self): 152 | # if not self.comparator_ptr == NULL: 153 | # del self.comparator_ptr 154 | 155 | # cdef object get_ob(self): 156 | # return self.ob 157 | 158 | # cdef const comparator.Comparator* get_comparator(self): 159 | # return self.comparator_ptr 160 | 161 | # cdef set_info_log(self, shared_ptr[logger.Logger] info_log): 162 | # self.comparator_ptr.set_info_log(info_log) 163 | 164 | # @cython.internal 165 | # cdef class PyBytewiseComparator(PyComparator): 166 | # cdef const comparator.Comparator* comparator_ptr 167 | 168 | # def __cinit__(self): 169 | # self.comparator_ptr = comparator.BytewiseComparator() 170 | 171 | # def name(self): 172 | # return PyBytes_FromString(self.comparator_ptr.Name()) 173 | 174 | # def compare(self, a, b): 175 | # return self.comparator_ptr.Compare( 176 | # bytes_to_slice(a), 177 | # bytes_to_slice(b)) 178 | 179 | # cdef object get_ob(self): 180 | # return self 181 | 182 | # cdef const comparator.Comparator* get_comparator(self): 183 | # return self.comparator_ptr 184 | 185 | 186 | 187 | # cdef int compare_callback( 188 | # void* ctx, 189 | # logger.Logger* log, 190 | # string& error_msg, 191 | # const Slice& a, 192 | # const Slice& b) with gil: 193 | 194 | # try: 195 | # return (ctx).compare(slice_to_bytes(a), slice_to_bytes(b)) 196 | # except BaseException as error: 197 | # tb = traceback.format_exc() 198 | # logger.Log(log, "Error in compare callback: %s", tb) 199 | # error_msg.assign(str(error)) 200 | 201 | # BytewiseComparator = PyBytewiseComparator 202 | # ######################################### 203 | 204 | 205 | 206 | # ## Here comes the stuff for the filter policy 207 | # @cython.internal 208 | # cdef class PyFilterPolicy(object): 209 | # cdef object get_ob(self): 210 | # return None 211 | 212 | # cdef shared_ptr[ConstFilterPolicy] get_policy(self): 213 | # return shared_ptr[ConstFilterPolicy]() 214 | 215 | # cdef set_info_log(self, shared_ptr[logger.Logger] info_log): 216 | # pass 217 | 218 | # @cython.internal 219 | # cdef class PyGenericFilterPolicy(PyFilterPolicy): 220 | # cdef shared_ptr[filter_policy.FilterPolicyWrapper] policy 221 | # cdef object ob 222 | 223 | # def __cinit__(self, object ob): 224 | # if not isinstance(ob, IFilterPolicy): 225 | # raise TypeError("%s is not of type %s" % (ob, IFilterPolicy)) 226 | 227 | # self.ob = ob 228 | # self.policy.reset(new filter_policy.FilterPolicyWrapper( 229 | # bytes_to_string(ob.name()), 230 | # ob, 231 | # create_filter_callback, 232 | # key_may_match_callback)) 233 | 234 | # cdef object get_ob(self): 235 | # return self.ob 236 | 237 | # cdef shared_ptr[ConstFilterPolicy] get_policy(self): 238 | # return (self.policy) 239 | 240 | # cdef set_info_log(self, shared_ptr[logger.Logger] info_log): 241 | # self.policy.get().set_info_log(info_log) 242 | 243 | 244 | # cdef void create_filter_callback( 245 | # void* ctx, 246 | # logger.Logger* log, 247 | # string& error_msg, 248 | # const Slice* keys, 249 | # int n, 250 | # string* dst) with gil: 251 | 252 | # try: 253 | # ret = (ctx).create_filter( 254 | # [slice_to_bytes(keys[i]) for i in range(n)]) 255 | # dst.append(bytes_to_string(ret)) 256 | # except BaseException as error: 257 | # tb = traceback.format_exc() 258 | # logger.Log(log, "Error in create filter callback: %s", tb) 259 | # error_msg.assign(str(error)) 260 | 261 | # cdef cpp_bool key_may_match_callback( 262 | # void* ctx, 263 | # logger.Logger* log, 264 | # string& error_msg, 265 | # const Slice& key, 266 | # const Slice& filt) with gil: 267 | 268 | # try: 269 | # return (ctx).key_may_match( 270 | # slice_to_bytes(key), 271 | # slice_to_bytes(filt)) 272 | # except BaseException as error: 273 | # tb = traceback.format_exc() 274 | # logger.Log(log, "Error in key_mach_match callback: %s", tb) 275 | # error_msg.assign(str(error)) 276 | 277 | # @cython.internal 278 | # cdef class PyBloomFilterPolicy(PyFilterPolicy): 279 | # cdef shared_ptr[ConstFilterPolicy] policy 280 | 281 | # def __cinit__(self, int bits_per_key): 282 | # self.policy.reset(filter_policy.NewBloomFilterPolicy(bits_per_key)) 283 | 284 | # def name(self): 285 | # return PyBytes_FromString(self.policy.get().Name()) 286 | 287 | # def create_filter(self, keys): 288 | # cdef string dst 289 | # cdef vector[Slice] c_keys 290 | 291 | # for key in keys: 292 | # c_keys.push_back(bytes_to_slice(key)) 293 | 294 | # self.policy.get().CreateFilter( 295 | # vector_data(c_keys), 296 | # c_keys.size(), 297 | # cython.address(dst)) 298 | 299 | # return string_to_bytes(dst) 300 | 301 | # def key_may_match(self, key, filter_): 302 | # return self.policy.get().KeyMayMatch( 303 | # bytes_to_slice(key), 304 | # bytes_to_slice(filter_)) 305 | 306 | # cdef object get_ob(self): 307 | # return self 308 | 309 | # cdef shared_ptr[ConstFilterPolicy] get_policy(self): 310 | # return self.policy 311 | 312 | # BloomFilterPolicy = PyBloomFilterPolicy 313 | # ############################################# 314 | 315 | 316 | 317 | # ## Here comes the stuff for the merge operator 318 | # @cython.internal 319 | # cdef class PyMergeOperator(object): 320 | # cdef shared_ptr[merge_operator.MergeOperator] merge_op 321 | # cdef object ob 322 | 323 | # def __cinit__(self, object ob): 324 | # self.ob = ob 325 | # if isinstance(ob, IAssociativeMergeOperator): 326 | # self.merge_op.reset( 327 | # 328 | # new merge_operator.AssociativeMergeOperatorWrapper( 329 | # bytes_to_string(ob.name()), 330 | # (ob), 331 | # merge_callback)) 332 | 333 | # elif isinstance(ob, IMergeOperator): 334 | # self.merge_op.reset( 335 | # 336 | # new merge_operator.MergeOperatorWrapper( 337 | # bytes_to_string(ob.name()), 338 | # ob, 339 | # ob, 340 | # full_merge_callback, 341 | # partial_merge_callback)) 342 | # # elif isinstance(ob, str): 343 | # # if ob == "put": 344 | # # self.merge_op = merge_operator.MergeOperators.CreatePutOperator() 345 | # # elif ob == "put_v1": 346 | # # self.merge_op = merge_operator.MergeOperators.CreateDeprecatedPutOperator() 347 | # # elif ob == "uint64add": 348 | # # self.merge_op = merge_operator.MergeOperators.CreateUInt64AddOperator() 349 | # # elif ob == "stringappend": 350 | # # self.merge_op = merge_operator.MergeOperators.CreateStringAppendOperator() 351 | # # #TODO: necessary? 352 | # # # elif ob == "stringappendtest": 353 | # # # self.merge_op = merge_operator.MergeOperators.CreateStringAppendTESTOperator() 354 | # # elif ob == "max": 355 | # # self.merge_op = merge_operator.MergeOperators.CreateMaxOperator() 356 | # # else: 357 | # # msg = "{0} is not the default type".format(ob) 358 | # # raise TypeError(msg) 359 | # else: 360 | # msg = "%s is not of this types %s" 361 | # msg %= (ob, (IAssociativeMergeOperator, IMergeOperator)) 362 | # raise TypeError(msg) 363 | 364 | 365 | # cdef object get_ob(self): 366 | # return self.ob 367 | 368 | # cdef shared_ptr[merge_operator.MergeOperator] get_operator(self): 369 | # return self.merge_op 370 | 371 | # cdef cpp_bool merge_callback( 372 | # void* ctx, 373 | # const Slice& key, 374 | # const Slice* existing_value, 375 | # const Slice& value, 376 | # string* new_value, 377 | # logger.Logger* log) with gil: 378 | 379 | # if existing_value == NULL: 380 | # py_existing_value = None 381 | # else: 382 | # py_existing_value = slice_to_bytes(deref(existing_value)) 383 | 384 | # try: 385 | # ret = (ctx).merge( 386 | # slice_to_bytes(key), 387 | # py_existing_value, 388 | # slice_to_bytes(value)) 389 | 390 | # if ret[0]: 391 | # new_value.assign(bytes_to_string(ret[1])) 392 | # return True 393 | # return False 394 | 395 | # except: 396 | # tb = traceback.format_exc() 397 | # logger.Log(log, "Error in merge_callback: %s", tb) 398 | # return False 399 | 400 | # cdef cpp_bool full_merge_callback( 401 | # void* ctx, 402 | # const Slice& key, 403 | # const Slice* existing_value, 404 | # const deque[string]& op_list, 405 | # string* new_value, 406 | # logger.Logger* log) with gil: 407 | 408 | # if existing_value == NULL: 409 | # py_existing_value = None 410 | # else: 411 | # py_existing_value = slice_to_bytes(deref(existing_value)) 412 | 413 | # try: 414 | # ret = (ctx).full_merge( 415 | # slice_to_bytes(key), 416 | # py_existing_value, 417 | # [string_to_bytes(op_list[i]) for i in range(op_list.size())]) 418 | 419 | # if ret[0]: 420 | # new_value.assign(bytes_to_string(ret[1])) 421 | # return True 422 | # return False 423 | 424 | # except: 425 | # tb = traceback.format_exc() 426 | # logger.Log(log, "Error in full_merge_callback: %s", tb) 427 | # return False 428 | 429 | # cdef cpp_bool partial_merge_callback( 430 | # void* ctx, 431 | # const Slice& key, 432 | # const Slice& left_op, 433 | # const Slice& right_op, 434 | # string* new_value, 435 | # logger.Logger* log) with gil: 436 | 437 | # try: 438 | # ret = (ctx).partial_merge( 439 | # slice_to_bytes(key), 440 | # slice_to_bytes(left_op), 441 | # slice_to_bytes(right_op)) 442 | 443 | # if ret[0]: 444 | # new_value.assign(bytes_to_string(ret[1])) 445 | # return True 446 | # return False 447 | 448 | # except: 449 | # tb = traceback.format_exc() 450 | # logger.Log(log, "Error in partial_merge_callback: %s", tb) 451 | # return False 452 | # ############################################## 453 | 454 | # #### Here comes the Cache stuff 455 | # @cython.internal 456 | # cdef class PyCache(object): 457 | # cdef shared_ptr[cache.Cache] get_cache(self): 458 | # return shared_ptr[cache.Cache]() 459 | 460 | # @cython.internal 461 | # cdef class PyLRUCache(PyCache): 462 | # cdef shared_ptr[cache.Cache] cache_ob 463 | 464 | # def __cinit__(self, capacity, shard_bits=None): 465 | # if shard_bits is not None: 466 | # self.cache_ob = cache.NewLRUCache(capacity, shard_bits) 467 | # else: 468 | # self.cache_ob = cache.NewLRUCache(capacity) 469 | 470 | # cdef shared_ptr[cache.Cache] get_cache(self): 471 | # return self.cache_ob 472 | 473 | # LRUCache = PyLRUCache 474 | # ############################### 475 | 476 | # ### Here comes the stuff for SliceTransform 477 | # @cython.internal 478 | # cdef class PySliceTransform(object): 479 | # cdef shared_ptr[slice_transform.SliceTransform] transfomer 480 | # cdef object ob 481 | 482 | # def __cinit__(self, object ob): 483 | # if not isinstance(ob, ISliceTransform): 484 | # raise TypeError("%s is not of type %s" % (ob, ISliceTransform)) 485 | 486 | # self.ob = ob 487 | # self.transfomer.reset( 488 | # 489 | # new slice_transform.SliceTransformWrapper( 490 | # bytes_to_string(ob.name()), 491 | # ob, 492 | # slice_transform_callback, 493 | # slice_in_domain_callback, 494 | # slice_in_range_callback)) 495 | 496 | # cdef object get_ob(self): 497 | # return self.ob 498 | 499 | # cdef shared_ptr[slice_transform.SliceTransform] get_transformer(self): 500 | # return self.transfomer 501 | 502 | # cdef set_info_log(self, shared_ptr[logger.Logger] info_log): 503 | # cdef slice_transform.SliceTransformWrapper* ptr 504 | # ptr = self.transfomer.get() 505 | # ptr.set_info_log(info_log) 506 | 507 | 508 | # cdef Slice slice_transform_callback( 509 | # void* ctx, 510 | # logger.Logger* log, 511 | # string& error_msg, 512 | # const Slice& src) with gil: 513 | 514 | # cdef size_t offset 515 | # cdef size_t size 516 | 517 | # try: 518 | # ret = (ctx).transform(slice_to_bytes(src)) 519 | # offset = ret[0] 520 | # size = ret[1] 521 | # if (offset + size) > src.size(): 522 | # msg = "offset(%i) + size(%i) is bigger than slice(%i)" 523 | # raise Exception(msg % (offset, size, src.size())) 524 | 525 | # return Slice(src.data() + offset, size) 526 | # except BaseException as error: 527 | # tb = traceback.format_exc() 528 | # logger.Log(log, "Error in slice transfrom callback: %s", tb) 529 | # error_msg.assign(str(error)) 530 | 531 | # cdef cpp_bool slice_in_domain_callback( 532 | # void* ctx, 533 | # logger.Logger* log, 534 | # string& error_msg, 535 | # const Slice& src) with gil: 536 | 537 | # try: 538 | # return (ctx).in_domain(slice_to_bytes(src)) 539 | # except BaseException as error: 540 | # tb = traceback.format_exc() 541 | # logger.Log(log, "Error in slice transfrom callback: %s", tb) 542 | # error_msg.assign(str(error)) 543 | 544 | # cdef cpp_bool slice_in_range_callback( 545 | # void* ctx, 546 | # logger.Logger* log, 547 | # string& error_msg, 548 | # const Slice& src) with gil: 549 | 550 | # try: 551 | # return (ctx).in_range(slice_to_bytes(src)) 552 | # except BaseException as error: 553 | # tb = traceback.format_exc() 554 | # logger.Log(log, "Error in slice transfrom callback: %s", tb) 555 | # error_msg.assign(str(error)) 556 | # ########################################### 557 | 558 | # ## Here are the TableFactories 559 | # @cython.internal 560 | # cdef class PyTableFactory(object): 561 | # cdef shared_ptr[table_factory.TableFactory] factory 562 | 563 | # cdef shared_ptr[table_factory.TableFactory] get_table_factory(self): 564 | # return self.factory 565 | 566 | # cdef set_info_log(self, shared_ptr[logger.Logger] info_log): 567 | # pass 568 | 569 | # cdef class BlockBasedTableFactory(PyTableFactory): 570 | # cdef PyFilterPolicy py_filter_policy 571 | 572 | # def __init__(self, 573 | # index_type='binary_search', 574 | # py_bool hash_index_allow_collision=True, 575 | # checksum='crc32', 576 | # PyCache block_cache=None, 577 | # PyCache block_cache_compressed=None, 578 | # filter_policy=None, 579 | # no_block_cache=False, 580 | # block_size=None, 581 | # block_size_deviation=None, 582 | # block_restart_interval=None, 583 | # whole_key_filtering=None): 584 | 585 | # cdef table_factory.BlockBasedTableOptions table_options 586 | 587 | # if index_type == 'binary_search': 588 | # table_options.index_type = table_factory.kBinarySearch 589 | # elif index_type == 'hash_search': 590 | # table_options.index_type = table_factory.kHashSearch 591 | # else: 592 | # raise ValueError("Unknown index_type: %s" % index_type) 593 | 594 | # if hash_index_allow_collision: 595 | # table_options.hash_index_allow_collision = True 596 | # else: 597 | # table_options.hash_index_allow_collision = False 598 | 599 | # if checksum == 'crc32': 600 | # table_options.checksum = table_factory.kCRC32c 601 | # elif checksum == 'xxhash': 602 | # table_options.checksum = table_factory.kxxHash 603 | # else: 604 | # raise ValueError("Unknown checksum: %s" % checksum) 605 | 606 | # if no_block_cache: 607 | # table_options.no_block_cache = True 608 | # else: 609 | # table_options.no_block_cache = False 610 | 611 | # # If the following options are None use the rocksdb default. 612 | # if block_size is not None: 613 | # table_options.block_size = block_size 614 | 615 | # if block_size_deviation is not None: 616 | # table_options.block_size_deviation = block_size_deviation 617 | 618 | # if block_restart_interval is not None: 619 | # table_options.block_restart_interval = block_restart_interval 620 | 621 | # if whole_key_filtering is not None: 622 | # if whole_key_filtering: 623 | # table_options.whole_key_filtering = True 624 | # else: 625 | # table_options.whole_key_filtering = False 626 | 627 | # if block_cache is not None: 628 | # table_options.block_cache = block_cache.get_cache() 629 | 630 | # if block_cache_compressed is not None: 631 | # table_options.block_cache_compressed = block_cache_compressed.get_cache() 632 | 633 | # # Set the filter_policy 634 | # self.py_filter_policy = None 635 | # if filter_policy is not None: 636 | # if isinstance(filter_policy, PyFilterPolicy): 637 | # if (filter_policy).get_policy().get() == NULL: 638 | # raise Exception("Cannot set filter policy: %s" % filter_policy) 639 | # self.py_filter_policy = filter_policy 640 | # else: 641 | # self.py_filter_policy = PyGenericFilterPolicy(filter_policy) 642 | 643 | # table_options.filter_policy = self.py_filter_policy.get_policy() 644 | 645 | # self.factory.reset(table_factory.NewBlockBasedTableFactory(table_options)) 646 | 647 | # cdef set_info_log(self, shared_ptr[logger.Logger] info_log): 648 | # if self.py_filter_policy is not None: 649 | # self.py_filter_policy.set_info_log(info_log) 650 | 651 | # cdef class PlainTableFactory(PyTableFactory): 652 | # def __init__( 653 | # self, 654 | # user_key_len=0, 655 | # bloom_bits_per_key=10, 656 | # hash_table_ratio=0.75, 657 | # index_sparseness=10, 658 | # huge_page_tlb_size=0, 659 | # encoding_type='plain', 660 | # py_bool full_scan_mode=False): 661 | 662 | # cdef table_factory.PlainTableOptions table_options 663 | 664 | # table_options.user_key_len = user_key_len 665 | # table_options.bloom_bits_per_key = bloom_bits_per_key 666 | # table_options.hash_table_ratio = hash_table_ratio 667 | # table_options.index_sparseness = index_sparseness 668 | # table_options.huge_page_tlb_size = huge_page_tlb_size 669 | 670 | # if encoding_type == 'plain': 671 | # table_options.encoding_type = table_factory.kPlain 672 | # elif encoding_type == 'prefix': 673 | # table_options.encoding_type = table_factory.kPrefix 674 | # else: 675 | # raise ValueError("Unknown encoding_type: %s" % encoding_type) 676 | 677 | # table_options.full_scan_mode = full_scan_mode 678 | 679 | # self.factory.reset( table_factory.NewPlainTableFactory(table_options)) 680 | # ############################################# 681 | 682 | # ### Here are the MemtableFactories 683 | # @cython.internal 684 | # cdef class PyMemtableFactory(object): 685 | # cdef shared_ptr[memtablerep.MemTableRepFactory] factory 686 | 687 | # cdef shared_ptr[memtablerep.MemTableRepFactory] get_memtable_factory(self): 688 | # return self.factory 689 | 690 | # cdef class SkipListMemtableFactory(PyMemtableFactory): 691 | # def __init__(self): 692 | # self.factory.reset(memtablerep.NewSkipListFactory()) 693 | 694 | # cdef class VectorMemtableFactory(PyMemtableFactory): 695 | # def __init__(self, count=0): 696 | # self.factory.reset(memtablerep.NewVectorRepFactory(count)) 697 | 698 | # cdef class HashSkipListMemtableFactory(PyMemtableFactory): 699 | # def __init__( 700 | # self, 701 | # bucket_count=1000000, 702 | # skiplist_height=4, 703 | # skiplist_branching_factor=4): 704 | 705 | # self.factory.reset( 706 | # memtablerep.NewHashSkipListRepFactory( 707 | # bucket_count, 708 | # skiplist_height, 709 | # skiplist_branching_factor)) 710 | 711 | # cdef class HashLinkListMemtableFactory(PyMemtableFactory): 712 | # def __init__(self, bucket_count=50000): 713 | # self.factory.reset(memtablerep.NewHashLinkListRepFactory(bucket_count)) 714 | # ################################## 715 | 716 | 717 | # cdef class CompressionType(object): 718 | # no_compression = u'no_compression' 719 | # snappy_compression = u'snappy_compression' 720 | # zlib_compression = u'zlib_compression' 721 | # bzip2_compression = u'bzip2_compression' 722 | # lz4_compression = u'lz4_compression' 723 | # lz4hc_compression = u'lz4hc_compression' 724 | # xpress_compression = u'xpress_compression' 725 | # zstd_compression = u'zstd_compression' 726 | # zstdnotfinal_compression = u'zstdnotfinal_compression' 727 | # disable_compression = u'disable_compression' 728 | 729 | # cdef class CompactionPri(object): 730 | # by_compensated_size = u'by_compensated_size' 731 | # oldest_largest_seq_first = u'oldest_largest_seq_first' 732 | # oldest_smallest_seq_first = u'oldest_smallest_seq_first' 733 | # min_overlapping_ratio = u'min_overlapping_ratio' 734 | 735 | # @cython.internal 736 | # cdef class _ColumnFamilyHandle: 737 | # """ This is an internal class that we will weakref for safety """ 738 | # cdef db.ColumnFamilyHandle* handle 739 | # cdef object __weakref__ 740 | # cdef object weak_handle 741 | 742 | # def __cinit__(self): 743 | # self.handle = NULL 744 | 745 | # def __dealloc__(self): 746 | # if not self.handle == NULL: 747 | # del self.handle 748 | 749 | # @staticmethod 750 | # cdef from_handle_ptr(db.ColumnFamilyHandle* handle): 751 | # inst = <_ColumnFamilyHandle>_ColumnFamilyHandle.__new__(_ColumnFamilyHandle) 752 | # inst.handle = handle 753 | # return inst 754 | 755 | # @property 756 | # def name(self): 757 | # return self.handle.GetName() 758 | 759 | # @property 760 | # def id(self): 761 | # return self.handle.GetID() 762 | 763 | # @property 764 | # def weakref(self): 765 | # if self.weak_handle is None: 766 | # self.weak_handle = ColumnFamilyHandle.from_wrapper(self) 767 | # return self.weak_handle 768 | 769 | class ColumnFamilyHandle: 770 | 771 | @property 772 | def is_valid(self) -> bool: 773 | ... 774 | 775 | def __repr__(self) -> str: 776 | ... 777 | 778 | def __eq__(self, other) -> bool: 779 | ... 780 | 781 | def __lt__(self, other) -> bool: 782 | ... 783 | 784 | def __ne__(self, other): 785 | return not self == other 786 | 787 | def __gt__(self, other): 788 | return other < self 789 | 790 | def __le__(self, other): 791 | return not other < self 792 | 793 | def __ge__(self, other): 794 | return not self < other 795 | 796 | def __hash__(self) -> int: 797 | ... 798 | 799 | 800 | class ColumnFamilyOptions(object): 801 | 802 | def __init__(self, **kwargs): 803 | self.py_comparator = None 804 | self.py_merge_operator = None 805 | self.py_prefix_extractor = None 806 | self.py_table_factory = None 807 | self.py_memtable_factory = None 808 | self.write_buffer_size = None 809 | self.max_write_buffer_number = None 810 | self.min_write_buffer_number_to_merge = None 811 | self.compression_opts = None 812 | self.compaction_pri = None 813 | self.compression = None 814 | self.max_compaction_bytes = None 815 | self.num_levels = None 816 | self.level0_file_num_compaction_trigger = None 817 | self.level0_slowdown_writes_trigger = None 818 | self.level0_stop_writes_trigger = None 819 | self.max_mem_compaction_level = None 820 | self.target_file_size_base = None 821 | self.target_file_size_multiplier = None 822 | self.max_bytes_for_level_base = None 823 | self.max_bytes_for_level_multiplier = None 824 | self.max_bytes_for_level_multiplier_additional = None 825 | self.arena_block_size = None 826 | self.disable_auto_compactions = None 827 | self.compaction_style = None 828 | self.compaction_options_universal = None 829 | self.max_sequential_skip_in_iterations = None 830 | self.inplace_update_support = None 831 | self.table_factory = None 832 | self.memtable_factory = None 833 | self.inplace_update_num_locks = None 834 | self.comparator = None 835 | self.merge_operator = None 836 | self.prefix_extractors = None 837 | 838 | 839 | class Options(ColumnFamilyOptions): 840 | 841 | def __init__(self, **kwargs): 842 | super().__init__() 843 | self.create_if_missing: bool 844 | self.error_if_exists: bool 845 | self.paranoid_checks: bool 846 | self.max_open_files = None 847 | self.use_fsync: bool 848 | self.db_log_dir = None 849 | self.wal_dir = None 850 | self.delete_obsolete_files_period_micros: bool 851 | self.max_background_compactions = None 852 | self.max_background_flushes = None 853 | self.max_log_file_size = None 854 | self.log_file_time_to_roll = None 855 | self.keep_log_file_num = None 856 | self.max_manifest_file_size = None 857 | self.table_cache_numshardbits = None 858 | self.wal_ttl_seconds = None 859 | self.wal_size_limit_mb = None 860 | self.manifest_preallocation_size = None 861 | self.enable_write_thread_adaptive_yield = None 862 | self.allow_concurrent_memtable_write = None 863 | self.allow_mmap_reads = None 864 | self.allow_mmap_writes = None 865 | self.is_fd_close_on_exec = None 866 | self.stats_dump_period_sec = None 867 | self.advise_random_on_open = None 868 | self.use_adaptive_mutex = None 869 | self.bytes_per_sync = None 870 | self.row_cache = None 871 | 872 | 873 | # # Forward declaration 874 | # cdef class Snapshot 875 | 876 | # cdef class KeysIterator 877 | # cdef class ValuesIterator 878 | # cdef class ItemsIterator 879 | # cdef class ReversedIterator 880 | 881 | # # Forward declaration 882 | # cdef class WriteBatchIterator 883 | 884 | class WriteBatch(object): 885 | 886 | def __init__(self, data=None): 887 | ... 888 | 889 | def put(self, key: bytes, value: bytes): 890 | ... 891 | 892 | def merge(self, key: bytes, value: bytes): 893 | ... 894 | 895 | def delete(self, key: bytes): 896 | ... 897 | 898 | def delete_range(self, key: Tuple[bytes, bytes]): 899 | ... 900 | 901 | def clear(self): 902 | ... 903 | 904 | def data(self): 905 | ... 906 | 907 | def count(self): 908 | ... 909 | 910 | def __iter__(self): 911 | ... 912 | 913 | 914 | # @cython.internal 915 | # cdef class WriteBatchIterator(object): 916 | # # Need a reference to the WriteBatch. 917 | # # The BatchItems are only pointers to the memory in WriteBatch. 918 | # cdef WriteBatch batch 919 | # cdef vector[db.BatchItem] items 920 | # cdef size_t pos 921 | 922 | # def __init__(self, WriteBatch batch): 923 | # cdef Status st 924 | 925 | # self.batch = batch 926 | # self.pos = 0 927 | 928 | # st = db.get_batch_items(batch.batch, cython.address(self.items)) 929 | # check_status(st) 930 | 931 | # def __iter__(self): 932 | # return self 933 | 934 | # def __next__(self): 935 | # if self.pos == self.items.size(): 936 | # raise StopIteration() 937 | 938 | # cdef str op 939 | 940 | # if self.items[self.pos].op == db.BatchItemOpPut: 941 | # op = "Put" 942 | # elif self.items[self.pos].op == db.BatchItemOpMerge: 943 | # op = "Merge" 944 | # elif self.items[self.pos].op == db.BatchItemOpDelte: 945 | # op = "Delete" 946 | 947 | # if self.items[self.pos].column_family_id != 0: # Column Family is set 948 | # ret = ( 949 | # op, 950 | # ( 951 | # self.items[self.pos].column_family_id, 952 | # slice_to_bytes(self.items[self.pos].key) 953 | # ), 954 | # slice_to_bytes(self.items[self.pos].value) 955 | # ) 956 | # else: 957 | # ret = ( 958 | # op, 959 | # slice_to_bytes(self.items[self.pos].key), 960 | # slice_to_bytes(self.items[self.pos].value) 961 | # ) 962 | # self.pos += 1 963 | # return ret 964 | 965 | 966 | from typing import Any, Dict, Iterable, List, Tuple 967 | 968 | 969 | class DB(object): 970 | 971 | def __init__(self, 972 | db_name: str, 973 | opts: 'Options', 974 | column_families: dict = None, 975 | read_only: bool = False): 976 | ... 977 | self.opts = opts 978 | self.opts.in_use = True 979 | 980 | def __dealloc__(self): 981 | self.close() 982 | 983 | def close(self): 984 | ... 985 | 986 | @property 987 | def column_families(self) -> list: 988 | ... 989 | 990 | def get_column_family(self, name: bytes): 991 | ... 992 | 993 | def put( 994 | self, 995 | key: bytes, 996 | value: bytes, 997 | sync: bool = False, 998 | disable_wal: bool = False 999 | ): 1000 | ... 1001 | 1002 | def delete( 1003 | self, 1004 | key: bytes, 1005 | sync: bool = False, 1006 | disable_wal: bool = False 1007 | ): 1008 | ... 1009 | 1010 | def delete_range( 1011 | self, 1012 | key: Tuple[bytes, bytes], 1013 | sync: bool = False, 1014 | disable_wal: bool = False 1015 | ): 1016 | ... 1017 | 1018 | def merge( 1019 | self, 1020 | key: bytes, 1021 | value: bytes, 1022 | sync: bool = False, 1023 | disable_wal: bool = False 1024 | ): 1025 | ... 1026 | 1027 | def write( 1028 | self, 1029 | batch: 'WriteBatch', 1030 | sync: bool = False, 1031 | disable_wal: bool = False 1032 | ): 1033 | ... 1034 | 1035 | def get( 1036 | self, 1037 | key: bytes, 1038 | *args, 1039 | **kwargs 1040 | ) -> bytes: 1041 | ... 1042 | 1043 | def multi_get( 1044 | self, 1045 | keys: Iterable[bytes], 1046 | *args, 1047 | **kwargs 1048 | ) -> Dict[bytes, bytes]: 1049 | ... 1050 | 1051 | def key_may_exist( 1052 | self, 1053 | key: bytes, 1054 | fetch: bool = False, 1055 | *args, 1056 | **kwargs 1057 | ) -> Tuple[bool, Any]: 1058 | ... 1059 | 1060 | def iterkeys( 1061 | self, 1062 | column_family: 'ColumnFamilyHandle' = None, 1063 | *args, 1064 | **kwargs 1065 | ) -> 'KeysIterator': 1066 | ... 1067 | 1068 | def itervalues( 1069 | self, 1070 | column_family: 'ColumnFamilyHandle' = None, 1071 | *args, 1072 | **kwargs 1073 | ) -> 'ValuesIterator': 1074 | ... 1075 | 1076 | def iteritems( 1077 | self, 1078 | column_family: 'ColumnFamilyHandle' = None, 1079 | *args, 1080 | **kwargs 1081 | ) -> 'ItemsIterator': 1082 | ... 1083 | 1084 | # def iterskeys(self, column_families, *args, **kwargs): 1085 | # cdef vector[db.Iterator*] iters 1086 | # iters.resize(len(column_families)) 1087 | # cdef options.ReadOptions opts 1088 | # cdef db.Iterator* it_ptr 1089 | # cdef KeysIterator it 1090 | # cdef db.ColumnFamilyHandle* cf_handle 1091 | # cdef vector[db.ColumnFamilyHandle*] cf_handles 1092 | 1093 | # for column_family in column_families: 1094 | # cf_handle = (column_family).get_handle() 1095 | # cf_handles.push_back(cf_handle) 1096 | 1097 | # opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs)) 1098 | # with nogil: 1099 | # self.db.NewIterators(opts, cf_handles, &iters) 1100 | 1101 | # cf_iter = iter(column_families) 1102 | # cdef list ret = [] 1103 | # for it_ptr in iters: 1104 | # it = KeysIterator(self, next(cf_iter)) 1105 | # it.ptr = it_ptr 1106 | # ret.append(it) 1107 | # return ret 1108 | 1109 | # def itersvalues(self, column_families, *args, **kwargs): 1110 | # cdef vector[db.Iterator*] iters 1111 | # iters.resize(len(column_families)) 1112 | # cdef options.ReadOptions opts 1113 | # cdef db.Iterator* it_ptr 1114 | # cdef ValuesIterator it 1115 | # cdef db.ColumnFamilyHandle* cf_handle 1116 | # cdef vector[db.ColumnFamilyHandle*] cf_handles 1117 | 1118 | # for column_family in column_families: 1119 | # cf_handle = (column_family).get_handle() 1120 | # cf_handles.push_back(cf_handle) 1121 | 1122 | # opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs)) 1123 | # with nogil: 1124 | # self.db.NewIterators(opts, cf_handles, &iters) 1125 | 1126 | # cdef list ret = [] 1127 | # for it_ptr in iters: 1128 | # it = ValuesIterator(self) 1129 | # it.ptr = it_ptr 1130 | # ret.append(it) 1131 | # return ret 1132 | 1133 | # def iterskeys(self, column_families, *args, **kwargs): 1134 | # cdef vector[db.Iterator*] iters 1135 | # iters.resize(len(column_families)) 1136 | # cdef options.ReadOptions opts 1137 | # cdef db.Iterator* it_ptr 1138 | # cdef ItemsIterator it 1139 | # cdef db.ColumnFamilyHandle* cf_handle 1140 | # cdef vector[db.ColumnFamilyHandle*] cf_handles 1141 | 1142 | # for column_family in column_families: 1143 | # cf_handle = (column_family).get_handle() 1144 | # cf_handles.push_back(cf_handle) 1145 | 1146 | # opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs)) 1147 | # with nogil: 1148 | # self.db.NewIterators(opts, cf_handles, &iters) 1149 | 1150 | 1151 | # cf_iter = iter(column_families) 1152 | # cdef list ret = [] 1153 | # for it_ptr in iters: 1154 | # it = ItemsIterator(self, next(cf_iter)) 1155 | # it.ptr = it_ptr 1156 | # ret.append(it) 1157 | # return ret 1158 | 1159 | # def snapshot(self): 1160 | # return Snapshot(self) 1161 | 1162 | def get_property(self, prop: bytes, column_family: 'ColumnFamilyHandle' = None): 1163 | ... 1164 | 1165 | def get_live_files_metadata(self) -> List[Dict[str, Any]]: 1166 | ... 1167 | 1168 | # def compact_range(self, begin=None, end=None, ColumnFamilyHandle column_family=None, **py_options): 1169 | # cdef options.CompactRangeOptions c_options 1170 | 1171 | # c_options.change_level = py_options.get('change_level', False) 1172 | # c_options.target_level = py_options.get('target_level', -1) 1173 | 1174 | # blc = py_options.get('bottommost_level_compaction', 'if_compaction_filter') 1175 | # if blc == 'skip': 1176 | # c_options.bottommost_level_compaction = options.blc_skip 1177 | # elif blc == 'if_compaction_filter': 1178 | # c_options.bottommost_level_compaction = options.blc_is_filter 1179 | # elif blc == 'force': 1180 | # c_options.bottommost_level_compaction = options.blc_force 1181 | # else: 1182 | # raise ValueError("bottommost_level_compaction is not valid") 1183 | 1184 | # cdef Status st 1185 | # cdef Slice begin_val 1186 | # cdef Slice end_val 1187 | 1188 | # cdef Slice* begin_ptr 1189 | # cdef Slice* end_ptr 1190 | 1191 | # begin_ptr = NULL 1192 | # end_ptr = NULL 1193 | 1194 | # if begin is not None: 1195 | # begin_val = bytes_to_slice(begin) 1196 | # begin_ptr = cython.address(begin_val) 1197 | 1198 | # if end is not None: 1199 | # end_val = bytes_to_slice(end) 1200 | # end_ptr = cython.address(end_val) 1201 | 1202 | # cdef db.ColumnFamilyHandle* cf_handle = self.db.DefaultColumnFamily() 1203 | # if column_family: 1204 | # cf_handle = (column_family).get_handle() 1205 | 1206 | # st = self.db.CompactRange(c_options, cf_handle, begin_ptr, end_ptr) 1207 | # check_status(st) 1208 | 1209 | # @staticmethod 1210 | # def __parse_read_opts( 1211 | # verify_checksums=False, 1212 | # fill_cache=True, 1213 | # snapshot=None, 1214 | # read_tier="all"): 1215 | 1216 | # # TODO: Is this really effiencet ? 1217 | # return locals() 1218 | 1219 | # cdef options.ReadOptions build_read_opts(self, dict py_opts): 1220 | # cdef options.ReadOptions opts 1221 | # opts.verify_checksums = py_opts['verify_checksums'] 1222 | # opts.fill_cache = py_opts['fill_cache'] 1223 | # if py_opts['snapshot'] is not None: 1224 | # opts.snapshot = ((py_opts['snapshot'])).ptr 1225 | 1226 | # if py_opts['read_tier'] == "all": 1227 | # opts.read_tier = options.kReadAllTier 1228 | # elif py_opts['read_tier'] == 'cache': 1229 | # opts.read_tier = options.kBlockCacheTier 1230 | # else: 1231 | # raise ValueError("Invalid read_tier") 1232 | 1233 | # return opts 1234 | 1235 | # property options: 1236 | # def __get__(self): 1237 | # return self.opts 1238 | 1239 | # def create_column_family(self, bytes name, ColumnFamilyOptions copts): 1240 | # cdef db.ColumnFamilyHandle* cf_handle 1241 | # cdef Status st 1242 | # cdef string c_name = name 1243 | 1244 | # for handle in self.cf_handles: 1245 | # if handle.name == name: 1246 | # raise ValueError(f"{name} is already an existing column family") 1247 | 1248 | # if copts.in_use: 1249 | # raise Exception("ColumnFamilyOptions are in_use by another column family") 1250 | 1251 | # copts.in_use = True 1252 | # with nogil: 1253 | # st = self.db.CreateColumnFamily(deref(copts.copts), c_name, &cf_handle) 1254 | # check_status(st) 1255 | 1256 | # handle = _ColumnFamilyHandle.from_handle_ptr(cf_handle) 1257 | 1258 | # self.cf_handles.append(handle) 1259 | # self.cf_options.append(copts) 1260 | # return handle.weakref 1261 | 1262 | # def drop_column_family(self, ColumnFamilyHandle weak_handle not None): 1263 | # cdef db.ColumnFamilyHandle* cf_handle 1264 | # cdef ColumnFamilyOptions copts 1265 | # cdef Status st 1266 | 1267 | # cf_handle = weak_handle.get_handle() 1268 | 1269 | # with nogil: 1270 | # st = self.db.DropColumnFamily(cf_handle) 1271 | # check_status(st) 1272 | 1273 | # py_handle = weak_handle._ref() 1274 | # index = self.cf_handles.index(py_handle) 1275 | # copts = self.cf_options.pop(index) 1276 | # del self.cf_handles[index] 1277 | # del py_handle 1278 | # if copts: 1279 | # copts.in_use = False 1280 | 1281 | 1282 | # def repair_db(db_name, Options opts): 1283 | # cdef Status st 1284 | # cdef string db_path 1285 | 1286 | # db_path = path_to_string(db_name) 1287 | # st = db.RepairDB(db_path, deref(opts.opts)) 1288 | # check_status(st) 1289 | 1290 | 1291 | # def list_column_families(db_name, Options opts): 1292 | # cdef Status st 1293 | # cdef string db_path 1294 | # cdef vector[string] column_families 1295 | 1296 | # db_path = path_to_string(db_name) 1297 | # with nogil: 1298 | # st = db.ListColumnFamilies(deref(opts.opts), db_path, &column_families) 1299 | # check_status(st) 1300 | 1301 | # return column_families 1302 | 1303 | 1304 | # @cython.no_gc_clear 1305 | # @cython.internal 1306 | # cdef class Snapshot(object): 1307 | # cdef const snapshot.Snapshot* ptr 1308 | # cdef DB db 1309 | 1310 | # def __cinit__(self, DB db): 1311 | # self.db = db 1312 | # self.ptr = NULL 1313 | # with nogil: 1314 | # self.ptr = db.db.GetSnapshot() 1315 | 1316 | # def __dealloc__(self): 1317 | # if not self.ptr == NULL: 1318 | # with nogil: 1319 | # self.db.db.ReleaseSnapshot(self.ptr) 1320 | 1321 | 1322 | class BaseIterator(object): 1323 | def __init__(self, db: 'DB', handle: 'ColumnFamilyHandle' = None): 1324 | ... 1325 | 1326 | def __dealloc__(self): 1327 | ... 1328 | 1329 | def __iter__(self): 1330 | return self 1331 | 1332 | def __next__(self): 1333 | ... 1334 | 1335 | def get(self): 1336 | ... 1337 | 1338 | def __reversed__(self): 1339 | ... 1340 | 1341 | def seek_to_first(self): 1342 | ... 1343 | 1344 | def seek_to_last(self): 1345 | ... 1346 | 1347 | def seek(self, key: bytes): 1348 | ... 1349 | 1350 | def seek_for_prev(self, key): 1351 | ... 1352 | 1353 | def get_ob(self): 1354 | return None 1355 | 1356 | 1357 | class KeysIterator(BaseIterator): 1358 | def get(self) -> bytes: 1359 | ... 1360 | def __next__(self) -> bytes: 1361 | ... 1362 | 1363 | class ValuesIterator(BaseIterator): 1364 | def get(self) -> bytes: 1365 | ... 1366 | def __next__(self) -> bytes: 1367 | ... 1368 | 1369 | 1370 | class ItemsIterator(BaseIterator): 1371 | def get(self) -> Tuple[bytes, bytes]: 1372 | ... 1373 | def __next__(self) -> Tuple[bytes, bytes]: 1374 | ... 1375 | 1376 | # @cython.internal 1377 | # cdef class ReversedIterator(object): 1378 | # cdef BaseIterator it 1379 | 1380 | # def __cinit__(self, BaseIterator it): 1381 | # self.it = it 1382 | 1383 | # def seek_to_first(self): 1384 | # self.it.seek_to_first() 1385 | 1386 | # def seek_to_last(self): 1387 | # self.it.seek_to_last() 1388 | 1389 | # def seek(self, key): 1390 | # self.it.seek(key) 1391 | 1392 | # def seek_for_prev(self, key): 1393 | # self.it.seek_for_prev(key) 1394 | 1395 | # def get(self): 1396 | # return self.it.get() 1397 | 1398 | # def __iter__(self): 1399 | # return self 1400 | 1401 | # def __reversed__(self): 1402 | # return self.it 1403 | 1404 | # def __next__(self): 1405 | # if not self.it.ptr.Valid(): 1406 | # raise StopIteration() 1407 | 1408 | # cdef object ret = self.it.get_ob() 1409 | # with nogil: 1410 | # self.it.ptr.Prev() 1411 | # check_status(self.it.ptr.status()) 1412 | # return ret 1413 | 1414 | # cdef class BackupEngine(object): 1415 | # cdef backup.BackupEngine* engine 1416 | 1417 | # def __cinit__(self, backup_dir): 1418 | # cdef Status st 1419 | # cdef string c_backup_dir 1420 | # self.engine = NULL 1421 | 1422 | # c_backup_dir = path_to_string(backup_dir) 1423 | # st = backup.BackupEngine_Open( 1424 | # env.Env_Default(), 1425 | # backup.BackupEngineOptions(c_backup_dir), 1426 | # cython.address(self.engine)) 1427 | 1428 | # check_status(st) 1429 | 1430 | # def __dealloc__(self): 1431 | # if not self.engine == NULL: 1432 | # with nogil: 1433 | # del self.engine 1434 | 1435 | # def create_backup(self, DB db, flush_before_backup=False): 1436 | # cdef Status st 1437 | # cdef cpp_bool c_flush_before_backup 1438 | 1439 | # c_flush_before_backup = flush_before_backup 1440 | 1441 | # with nogil: 1442 | # st = self.engine.CreateNewBackup(db.db, c_flush_before_backup) 1443 | # check_status(st) 1444 | 1445 | # def restore_backup(self, backup_id, db_dir, wal_dir): 1446 | # cdef Status st 1447 | # cdef backup.BackupID c_backup_id 1448 | # cdef string c_db_dir 1449 | # cdef string c_wal_dir 1450 | 1451 | # c_backup_id = backup_id 1452 | # c_db_dir = path_to_string(db_dir) 1453 | # c_wal_dir = path_to_string(wal_dir) 1454 | 1455 | # with nogil: 1456 | # st = self.engine.RestoreDBFromBackup( 1457 | # c_backup_id, 1458 | # c_db_dir, 1459 | # c_wal_dir) 1460 | 1461 | # check_status(st) 1462 | 1463 | # def restore_latest_backup(self, db_dir, wal_dir): 1464 | # cdef Status st 1465 | # cdef string c_db_dir 1466 | # cdef string c_wal_dir 1467 | 1468 | # c_db_dir = path_to_string(db_dir) 1469 | # c_wal_dir = path_to_string(wal_dir) 1470 | 1471 | # with nogil: 1472 | # st = self.engine.RestoreDBFromLatestBackup(c_db_dir, c_wal_dir) 1473 | 1474 | # check_status(st) 1475 | 1476 | # def stop_backup(self): 1477 | # with nogil: 1478 | # self.engine.StopBackup() 1479 | 1480 | # def purge_old_backups(self, num_backups_to_keep): 1481 | # cdef Status st 1482 | # cdef uint32_t c_num_backups_to_keep 1483 | 1484 | # c_num_backups_to_keep = num_backups_to_keep 1485 | 1486 | # with nogil: 1487 | # st = self.engine.PurgeOldBackups(c_num_backups_to_keep) 1488 | # check_status(st) 1489 | 1490 | # def delete_backup(self, backup_id): 1491 | # cdef Status st 1492 | # cdef backup.BackupID c_backup_id 1493 | 1494 | # c_backup_id = backup_id 1495 | 1496 | # with nogil: 1497 | # st = self.engine.DeleteBackup(c_backup_id) 1498 | 1499 | # check_status(st) 1500 | 1501 | # def get_backup_info(self): 1502 | # cdef vector[backup.BackupInfo] backup_info 1503 | 1504 | # with nogil: 1505 | # self.engine.GetBackupInfo(cython.address(backup_info)) 1506 | 1507 | # ret = [] 1508 | # for ob in backup_info: 1509 | # t = {} 1510 | # t['backup_id'] = ob.backup_id 1511 | # t['timestamp'] = ob.timestamp 1512 | # t['size'] = ob.size 1513 | # ret.append(t) 1514 | 1515 | # return ret 1516 | -------------------------------------------------------------------------------- /src/aimrocks/lib_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import ctypes 3 | import platform 4 | from glob import glob 5 | 6 | 7 | def get_include_dir(): 8 | dirname = os.path.dirname(__file__) 9 | abs_dirname = os.path.abspath(dirname) 10 | path = os.path.join(abs_dirname, 'include') 11 | return path 12 | 13 | def get_lib_dir(): 14 | dirname = os.path.dirname(__file__) 15 | abs_dirname = os.path.abspath(dirname) 16 | path = abs_dirname 17 | return path 18 | 19 | def get_libs(): 20 | return [ 21 | 'rocksdb', 22 | ] 23 | 24 | def get_lib_filename(name, lib_dir): 25 | if platform.system() == 'Darwin': 26 | pattern = f'lib{name}*.dylib' 27 | else: 28 | pattern = f'lib{name}*.so*' 29 | 30 | files = glob(os.path.join(lib_dir, pattern)) 31 | assert len(files) == 1 32 | return os.path.basename(files[0]) 33 | 34 | def get_lib_filenames(): 35 | local_lib_dir = get_lib_dir() 36 | return [ 37 | get_lib_filename(name, local_lib_dir) 38 | for name in get_libs() 39 | ] 40 | 41 | def load_lib(filename): 42 | local_lib_dir = get_lib_dir() 43 | path = os.path.join(local_lib_dir, filename) 44 | print(f'Loading runtime library from {path}') 45 | ctypes.CDLL(path) 46 | 47 | def load_libs(): 48 | for filename in get_lib_filenames(): 49 | load_lib(filename) 50 | -------------------------------------------------------------------------------- /src/aimrocks/logger.pxd: -------------------------------------------------------------------------------- 1 | cdef extern from "rocksdb/env.h" namespace "rocksdb": 2 | cdef cppclass Logger: 3 | pass 4 | 5 | void Log(Logger*, const char*, ...) nogil except+ 6 | -------------------------------------------------------------------------------- /src/aimrocks/memtablerep.pxd: -------------------------------------------------------------------------------- 1 | from libc.stdint cimport int32_t 2 | 3 | cdef extern from "rocksdb/memtablerep.h" namespace "rocksdb": 4 | cdef cppclass MemTableRepFactory: 5 | MemTableRepFactory() 6 | 7 | cdef MemTableRepFactory* NewHashSkipListRepFactory(size_t, int32_t, int32_t) 8 | cdef MemTableRepFactory* NewHashLinkListRepFactory(size_t) 9 | 10 | cdef extern from "rdb_include/memtable_factories.hpp" namespace "py_rocks": 11 | cdef MemTableRepFactory* NewVectorRepFactory(size_t) 12 | cdef MemTableRepFactory* NewSkipListFactory() 13 | -------------------------------------------------------------------------------- /src/aimrocks/merge_operator.pxd: -------------------------------------------------------------------------------- 1 | from libcpp.string cimport string 2 | from libcpp cimport bool as cpp_bool 3 | from libcpp.deque cimport deque 4 | from aimrocks.slice_ cimport Slice 5 | from aimrocks.logger cimport Logger 6 | from aimrocks.std_memory cimport shared_ptr 7 | 8 | cdef extern from "rocksdb/merge_operator.h" namespace "rocksdb": 9 | cdef cppclass MergeOperator: 10 | pass 11 | 12 | ctypedef cpp_bool (*merge_func)( 13 | void*, 14 | const Slice&, 15 | const Slice*, 16 | const Slice&, 17 | string*, 18 | Logger*) 19 | 20 | ctypedef cpp_bool (*full_merge_func)( 21 | void* ctx, 22 | const Slice& key, 23 | const Slice* existing_value, 24 | const deque[string]& operand_list, 25 | string* new_value, 26 | Logger* logger) 27 | 28 | ctypedef cpp_bool (*partial_merge_func)( 29 | void* ctx, 30 | const Slice& key, 31 | const Slice& left_op, 32 | const Slice& right_op, 33 | string* new_value, 34 | Logger* logger) 35 | 36 | cdef extern from "rdb_include/merge_operator_wrapper.hpp" namespace "py_rocks": 37 | cdef cppclass AssociativeMergeOperatorWrapper: 38 | AssociativeMergeOperatorWrapper(string, void*, merge_func) nogil except+ 39 | 40 | cdef cppclass MergeOperatorWrapper: 41 | MergeOperatorWrapper( 42 | string, 43 | void*, 44 | void*, 45 | full_merge_func, 46 | partial_merge_func) nogil except+ 47 | -------------------------------------------------------------------------------- /src/aimrocks/merge_operators.py: -------------------------------------------------------------------------------- 1 | import struct as py_struct 2 | from aimrocks.interfaces import AssociativeMergeOperator 3 | 4 | class UintAddOperator(AssociativeMergeOperator): 5 | def merge(self, key, existing_value, value): 6 | if existing_value: 7 | s = py_struct.unpack('Q', existing_value)[0] + py_struct.unpack('Q', value)[0] 8 | return (True, py_struct.pack('Q', s)) 9 | return (True, value) 10 | 11 | def name(self): 12 | return b'uint64add' 13 | 14 | class StringAppendOperator(AssociativeMergeOperator): 15 | def merge(self, key, existing_value, value): 16 | if existing_value: 17 | s = existing_value + b',' + value 18 | return (True, s) 19 | return (True, value) 20 | 21 | def name(self): 22 | return b'StringAppendOperator' 23 | -------------------------------------------------------------------------------- /src/aimrocks/options.pxd: -------------------------------------------------------------------------------- 1 | from libcpp cimport bool as cpp_bool 2 | from libcpp.string cimport string 3 | from libcpp.vector cimport vector 4 | from libc.stdint cimport uint64_t 5 | from libc.stdint cimport uint32_t 6 | from aimrocks.std_memory cimport shared_ptr 7 | from aimrocks.comparator cimport Comparator 8 | from aimrocks.merge_operator cimport MergeOperator 9 | from aimrocks.logger cimport Logger 10 | from aimrocks.slice_ cimport Slice 11 | from aimrocks.snapshot cimport Snapshot 12 | from aimrocks.slice_transform cimport SliceTransform 13 | from aimrocks.table_factory cimport TableFactory 14 | from aimrocks.memtablerep cimport MemTableRepFactory 15 | from aimrocks.universal_compaction cimport CompactionOptionsUniversal 16 | from aimrocks.cache cimport Cache 17 | 18 | cdef extern from "rocksdb/options.h" namespace "rocksdb": 19 | cdef cppclass CompressionOptions: 20 | int window_bits; 21 | int level; 22 | int strategy; 23 | uint32_t max_dict_bytes 24 | CompressionOptions() except + 25 | CompressionOptions(int, int, int, int) except + 26 | 27 | ctypedef enum CompactionStyle: 28 | kCompactionStyleLevel 29 | kCompactionStyleUniversal 30 | kCompactionStyleFIFO 31 | kCompactionStyleNone 32 | 33 | ctypedef enum CompressionType: 34 | kNoCompression 35 | kSnappyCompression 36 | kZlibCompression 37 | kBZip2Compression 38 | kLZ4Compression 39 | kLZ4HCCompression 40 | kXpressCompression 41 | kZSTD 42 | kZSTDNotFinalCompression 43 | kDisableCompressionOption 44 | 45 | ctypedef enum ReadTier: 46 | kReadAllTier 47 | kBlockCacheTier 48 | 49 | ctypedef enum CompactionPri: 50 | kByCompensatedSize 51 | kOldestLargestSeqFirst 52 | kOldestSmallestSeqFirst 53 | kMinOverlappingRatio 54 | 55 | # This needs to be in _rocksdb.pxd so it will export into python 56 | #cpdef enum AccessHint "rocksdb::DBOptions::AccessHint": 57 | # NONE, 58 | # NORMAL, 59 | # SEQUENTIAL, 60 | # WILLNEED 61 | 62 | cdef cppclass DBOptions: 63 | cpp_bool create_if_missing 64 | cpp_bool create_missing_column_families 65 | cpp_bool error_if_exists 66 | cpp_bool paranoid_checks 67 | # TODO: env 68 | shared_ptr[Logger] info_log 69 | int max_open_files 70 | int max_file_opening_threads 71 | # TODO: statistics 72 | cpp_bool use_fsync 73 | string db_log_dir 74 | string wal_dir 75 | uint64_t delete_obsolete_files_period_micros 76 | int max_background_jobs 77 | int max_background_compactions 78 | uint32_t max_subcompactions 79 | int max_background_flushes 80 | size_t max_log_file_size 81 | size_t log_file_time_to_roll 82 | size_t keep_log_file_num 83 | size_t recycle_log_file_num 84 | uint64_t max_manifest_file_size 85 | int table_cache_numshardbits 86 | uint64_t WAL_ttl_seconds 87 | uint64_t WAL_size_limit_MB 88 | size_t manifest_preallocation_size 89 | cpp_bool allow_mmap_reads 90 | cpp_bool allow_mmap_writes 91 | cpp_bool use_direct_reads 92 | cpp_bool use_direct_io_for_flush_and_compaction 93 | cpp_bool allow_fallocate 94 | cpp_bool is_fd_close_on_exec 95 | unsigned int stats_dump_period_sec 96 | cpp_bool advise_random_on_open 97 | size_t db_write_buffer_size 98 | # AccessHint access_hint_on_compaction_start 99 | cpp_bool use_adaptive_mutex 100 | uint64_t bytes_per_sync 101 | cpp_bool allow_concurrent_memtable_write 102 | cpp_bool enable_write_thread_adaptive_yield 103 | shared_ptr[Cache] row_cache 104 | 105 | cpp_bool skip_checking_sst_file_sizes_on_db_open 106 | cpp_bool skip_stats_update_on_db_open 107 | 108 | cdef cppclass ColumnFamilyOptions: 109 | ColumnFamilyOptions() 110 | ColumnFamilyOptions(const Options& options) 111 | const Comparator* comparator 112 | shared_ptr[MergeOperator] merge_operator 113 | # TODO: compaction_filter 114 | # TODO: compaction_filter_factory 115 | size_t write_buffer_size 116 | int max_write_buffer_number 117 | int min_write_buffer_number_to_merge 118 | CompressionType compression 119 | CompactionPri compaction_pri 120 | # TODO: compression_per_level 121 | shared_ptr[SliceTransform] prefix_extractor 122 | int num_levels 123 | int level0_file_num_compaction_trigger 124 | int level0_slowdown_writes_trigger 125 | int level0_stop_writes_trigger 126 | int max_mem_compaction_level 127 | uint64_t target_file_size_base 128 | int target_file_size_multiplier 129 | uint64_t max_bytes_for_level_base 130 | double max_bytes_for_level_multiplier 131 | vector[int] max_bytes_for_level_multiplier_additional 132 | int expanded_compaction_factor 133 | int source_compaction_factor 134 | int max_grandparent_overlap_factor 135 | cpp_bool disableDataSync 136 | size_t arena_block_size 137 | # TODO: PrepareForBulkLoad() 138 | cpp_bool disable_auto_compactions 139 | cpp_bool allow_os_buffer 140 | cpp_bool verify_checksums_in_compaction 141 | CompactionStyle compaction_style 142 | CompactionOptionsUniversal compaction_options_universal 143 | cpp_bool filter_deletes 144 | uint64_t max_sequential_skip_in_iterations 145 | shared_ptr[MemTableRepFactory] memtable_factory 146 | shared_ptr[TableFactory] table_factory 147 | # TODO: table_properties_collectors 148 | cpp_bool inplace_update_support 149 | size_t inplace_update_num_locks 150 | # TODO: remove options source_compaction_factor, max_grandparent_overlap_bytes and expanded_compaction_factor from document 151 | uint64_t max_compaction_bytes 152 | CompressionOptions compression_opts 153 | 154 | cdef cppclass Options(DBOptions, ColumnFamilyOptions): 155 | pass 156 | 157 | cdef cppclass WriteOptions: 158 | cpp_bool sync 159 | cpp_bool disableWAL 160 | 161 | cdef cppclass ReadOptions: 162 | cpp_bool verify_checksums 163 | cpp_bool fill_cache 164 | const Snapshot* snapshot 165 | ReadTier read_tier 166 | 167 | cdef cppclass FlushOptions: 168 | cpp_bool wait 169 | 170 | ctypedef enum BottommostLevelCompaction: 171 | blc_skip "rocksdb::BottommostLevelCompaction::kSkip" 172 | blc_is_filter "rocksdb::BottommostLevelCompaction::kIfHaveCompactionFilter" 173 | blc_force "rocksdb::BottommostLevelCompaction::kForce" 174 | 175 | cdef cppclass CompactRangeOptions: 176 | cpp_bool change_level 177 | int target_level 178 | uint32_t target_path_id 179 | BottommostLevelCompaction bottommost_level_compaction 180 | -------------------------------------------------------------------------------- /src/aimrocks/slice_.pxd: -------------------------------------------------------------------------------- 1 | from libcpp.string cimport string 2 | from libcpp cimport bool as cpp_bool 3 | 4 | cdef extern from "rocksdb/slice.h" namespace "rocksdb": 5 | cdef cppclass Slice: 6 | Slice() nogil 7 | Slice(const char*, size_t) nogil 8 | Slice(const string&) nogil 9 | Slice(const char*) nogil 10 | 11 | const char* data() nogil 12 | size_t size() nogil 13 | cpp_bool empty() nogil 14 | char operator[](int) nogil 15 | void clear() nogil 16 | void remove_prefix(size_t) nogil 17 | string ToString() nogil 18 | string ToString(cpp_bool) nogil 19 | int compare(const Slice&) nogil 20 | cpp_bool starts_with(const Slice&) nogil 21 | -------------------------------------------------------------------------------- /src/aimrocks/slice_transform.pxd: -------------------------------------------------------------------------------- 1 | from aimrocks.slice_ cimport Slice 2 | from libcpp.string cimport string 3 | from libcpp cimport bool as cpp_bool 4 | from aimrocks.logger cimport Logger 5 | from aimrocks.std_memory cimport shared_ptr 6 | 7 | cdef extern from "rocksdb/slice_transform.h" namespace "rocksdb": 8 | cdef cppclass SliceTransform: 9 | pass 10 | 11 | ctypedef Slice (*transform_func)( 12 | void*, 13 | Logger*, 14 | string&, 15 | const Slice&) 16 | 17 | ctypedef cpp_bool (*in_domain_func)( 18 | void*, 19 | Logger*, 20 | string&, 21 | const Slice&) 22 | 23 | ctypedef cpp_bool (*in_range_func)( 24 | void*, 25 | Logger*, 26 | string&, 27 | const Slice&) 28 | 29 | cdef extern from "rdb_include/slice_transform_wrapper.hpp" namespace "py_rocks": 30 | cdef cppclass SliceTransformWrapper: 31 | SliceTransformWrapper( 32 | string name, 33 | void*, 34 | transform_func, 35 | in_domain_func, 36 | in_range_func) nogil except+ 37 | void set_info_log(shared_ptr[Logger]) nogil except+ 38 | -------------------------------------------------------------------------------- /src/aimrocks/snapshot.pxd: -------------------------------------------------------------------------------- 1 | cdef extern from "rocksdb/db.h" namespace "rocksdb": 2 | cdef cppclass Snapshot: 3 | pass 4 | -------------------------------------------------------------------------------- /src/aimrocks/status.pxd: -------------------------------------------------------------------------------- 1 | from libcpp cimport bool as cpp_bool 2 | from libcpp.string cimport string 3 | 4 | cdef extern from "rocksdb/status.h" namespace "rocksdb": 5 | cdef cppclass Status: 6 | Status() 7 | cpp_bool ok() nogil 8 | cpp_bool IsNotFound() nogil const 9 | cpp_bool IsCorruption() nogil const 10 | cpp_bool IsNotSupported() nogil const 11 | cpp_bool IsInvalidArgument() nogil const 12 | cpp_bool IsIOError() nogil const 13 | cpp_bool IsMergeInProgress() nogil const 14 | cpp_bool IsIncomplete() nogil const 15 | string ToString() nogil except+ 16 | -------------------------------------------------------------------------------- /src/aimrocks/std_memory.pxd: -------------------------------------------------------------------------------- 1 | cdef extern from "" namespace "std": 2 | cdef cppclass shared_ptr[T]: 3 | shared_ptr() nogil except+ 4 | shared_ptr(T*) nogil except+ 5 | void reset() nogil except+ 6 | void reset(T*) nogil except+ 7 | T* get() nogil except+ 8 | -------------------------------------------------------------------------------- /src/aimrocks/table_factory.pxd: -------------------------------------------------------------------------------- 1 | from libc.stdint cimport uint32_t 2 | from libcpp cimport bool as cpp_bool 3 | from aimrocks.std_memory cimport shared_ptr 4 | 5 | from aimrocks.cache cimport Cache 6 | from aimrocks.filter_policy cimport FilterPolicy 7 | 8 | cdef extern from "rocksdb/table.h" namespace "rocksdb": 9 | cdef cppclass TableFactory: 10 | TableFactory() 11 | 12 | ctypedef enum BlockBasedTableIndexType: 13 | kBinarySearch "rocksdb::BlockBasedTableOptions::IndexType::kBinarySearch" 14 | kHashSearch "rocksdb::BlockBasedTableOptions::IndexType::kHashSearch" 15 | 16 | ctypedef enum ChecksumType: 17 | kCRC32c 18 | kxxHash 19 | 20 | cdef cppclass BlockBasedTableOptions: 21 | BlockBasedTableOptions() 22 | BlockBasedTableIndexType index_type 23 | cpp_bool hash_index_allow_collision 24 | ChecksumType checksum 25 | cpp_bool no_block_cache 26 | size_t block_size 27 | int block_size_deviation 28 | int block_restart_interval 29 | cpp_bool whole_key_filtering 30 | shared_ptr[Cache] block_cache 31 | shared_ptr[Cache] block_cache_compressed 32 | shared_ptr[FilterPolicy] filter_policy 33 | 34 | cdef TableFactory* NewBlockBasedTableFactory(const BlockBasedTableOptions&) 35 | 36 | ctypedef enum EncodingType: 37 | kPlain 38 | kPrefix 39 | 40 | cdef cppclass PlainTableOptions: 41 | uint32_t user_key_len 42 | int bloom_bits_per_key 43 | double hash_table_ratio 44 | size_t index_sparseness 45 | size_t huge_page_tlb_size 46 | EncodingType encoding_type 47 | cpp_bool full_scan_mode 48 | cpp_bool store_index_in_file 49 | 50 | cdef TableFactory* NewPlainTableFactory(const PlainTableOptions&) 51 | -------------------------------------------------------------------------------- /src/aimrocks/universal_compaction.pxd: -------------------------------------------------------------------------------- 1 | cdef extern from "rocksdb/universal_compaction.h" namespace "rocksdb": 2 | 3 | ctypedef enum CompactionStopStyle: 4 | kCompactionStopStyleSimilarSize 5 | kCompactionStopStyleTotalSize 6 | 7 | cdef cppclass CompactionOptionsUniversal: 8 | CompactionOptionsUniversal() 9 | 10 | unsigned int size_ratio 11 | unsigned int min_merge_width 12 | unsigned int max_merge_width 13 | unsigned int max_size_amplification_percent 14 | int compression_size_percent 15 | CompactionStopStyle stop_style 16 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aimhubio/aimrocks/a477b4e9976454c31992056cf13033bb7bc534cc/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_db.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import shutil 4 | import gc 5 | import unittest 6 | import aimrocks 7 | from itertools import takewhile 8 | import struct 9 | import tempfile 10 | from aimrocks.merge_operators import UintAddOperator, StringAppendOperator 11 | 12 | def int_to_bytes(ob): 13 | return str(ob).encode('ascii') 14 | 15 | class TestHelper(unittest.TestCase): 16 | 17 | def setUp(self): 18 | self.db_loc = tempfile.mkdtemp() 19 | self.addCleanup(self._close_db) 20 | 21 | def _close_db(self): 22 | del self.db 23 | gc.collect() 24 | if os.path.exists(self.db_loc): 25 | shutil.rmtree(self.db_loc) 26 | 27 | 28 | class TestDB(TestHelper): 29 | def setUp(self): 30 | TestHelper.setUp(self) 31 | opts = aimrocks.Options(create_if_missing=True) 32 | self.db = aimrocks.DB(os.path.join(self.db_loc, "test"), opts) 33 | 34 | def test_options_used_twice(self): 35 | if sys.version_info[0] == 3: 36 | assertRaisesRegex = self.assertRaisesRegex 37 | else: 38 | assertRaisesRegex = self.assertRaisesRegexp 39 | expected = "Options object is already used by another DB" 40 | with assertRaisesRegex(Exception, expected): 41 | aimrocks.DB(os.path.join(self.db_loc, "test2"), self.db.options) 42 | 43 | def test_unicode_path(self): 44 | name = os.path.join(self.db_loc, b'M\xc3\xbcnchen'.decode('utf8')) 45 | aimrocks.DB(name, aimrocks.Options(create_if_missing=True)) 46 | self.addCleanup(shutil.rmtree, name) 47 | self.assertTrue(os.path.isdir(name)) 48 | 49 | def test_get_none(self): 50 | self.assertIsNone(self.db.get(b'xxx')) 51 | 52 | def test_put_get(self): 53 | self.db.put(b"a", b"b") 54 | self.assertEqual(b"b", self.db.get(b"a")) 55 | 56 | def test_multi_get(self): 57 | self.db.put(b"a", b"1") 58 | self.db.put(b"b", b"2") 59 | self.db.put(b"c", b"3") 60 | 61 | ret = self.db.multi_get([b'a', b'b', b'c']) 62 | ref = {b'a': b'1', b'c': b'3', b'b': b'2'} 63 | self.assertEqual(ref, ret) 64 | 65 | def test_delete(self): 66 | self.db.put(b"a", b"b") 67 | self.assertEqual(b"b", self.db.get(b"a")) 68 | self.db.delete(b"a") 69 | self.assertIsNone(self.db.get(b"a")) 70 | 71 | def test_write_batch(self): 72 | batch = aimrocks.WriteBatch() 73 | batch.put(b"key", b"v1") 74 | batch.delete(b"key") 75 | batch.put(b"key", b"v2") 76 | batch.put(b"key", b"v3") 77 | batch.put(b"a", b"b") 78 | 79 | self.db.write(batch) 80 | ref = {b'a': b'b', b'key': b'v3'} 81 | ret = self.db.multi_get([b'key', b'a']) 82 | self.assertEqual(ref, ret) 83 | 84 | def test_write_batch_iter(self): 85 | batch = aimrocks.WriteBatch() 86 | self.assertEqual([], list(batch)) 87 | 88 | batch.put(b"key1", b"v1") 89 | batch.put(b"key2", b"v2") 90 | batch.put(b"key3", b"v3") 91 | batch.delete(b'a') 92 | batch.delete(b'key1') 93 | batch.merge(b'xxx', b'value') 94 | 95 | it = iter(batch) 96 | del batch 97 | ref = [ 98 | ('Put', b'key1', b'v1'), 99 | ('Put', b'key2', b'v2'), 100 | ('Put', b'key3', b'v3'), 101 | ('Delete', b'a', b''), 102 | ('Delete', b'key1', b''), 103 | ('Merge', b'xxx', b'value') 104 | ] 105 | self.assertEqual(ref, list(it)) 106 | 107 | 108 | def test_key_may_exists(self): 109 | self.db.put(b"a", b'1') 110 | 111 | self.assertEqual((False, None), self.db.key_may_exist(b"x")) 112 | self.assertEqual((False, None), self.db.key_may_exist(b'x', True)) 113 | self.assertEqual((True, None), self.db.key_may_exist(b'a')) 114 | self.assertEqual((True, b'1'), self.db.key_may_exist(b'a', True)) 115 | 116 | def test_seek_for_prev(self): 117 | self.db.put(b'a1', b'a1_value') 118 | self.db.put(b'a3', b'a3_value') 119 | self.db.put(b'b1', b'b1_value') 120 | self.db.put(b'b2', b'b2_value') 121 | self.db.put(b'c2', b'c2_value') 122 | self.db.put(b'c4', b'c4_value') 123 | 124 | self.assertEqual(self.db.get(b'a1'), b'a1_value') 125 | 126 | it = self.db.iterkeys() 127 | 128 | it.seek(b'a1') 129 | self.assertEqual(it.get(), b'a1') 130 | it.seek(b'a3') 131 | self.assertEqual(it.get(), b'a3') 132 | it.seek_for_prev(b'c4') 133 | self.assertEqual(it.get(), b'c4') 134 | it.seek_for_prev(b'c3') 135 | self.assertEqual(it.get(), b'c2') 136 | 137 | it = self.db.itervalues() 138 | it.seek(b'a1') 139 | self.assertEqual(it.get(), b'a1_value') 140 | it.seek(b'a3') 141 | self.assertEqual(it.get(), b'a3_value') 142 | it.seek_for_prev(b'c4') 143 | self.assertEqual(it.get(), b'c4_value') 144 | it.seek_for_prev(b'c3') 145 | self.assertEqual(it.get(), b'c2_value') 146 | 147 | it = self.db.iteritems() 148 | it.seek(b'a1') 149 | self.assertEqual(it.get(), (b'a1', b'a1_value')) 150 | it.seek(b'a3') 151 | self.assertEqual(it.get(), (b'a3', b'a3_value')) 152 | it.seek_for_prev(b'c4') 153 | self.assertEqual(it.get(), (b'c4', b'c4_value')) 154 | it.seek_for_prev(b'c3') 155 | self.assertEqual(it.get(), (b'c2', b'c2_value')) 156 | 157 | reverse_it = reversed(it) 158 | it.seek_for_prev(b'c3') 159 | self.assertEqual(it.get(), (b'c2', b'c2_value')) 160 | 161 | 162 | def test_iter_keys(self): 163 | for x in range(300): 164 | self.db.put(int_to_bytes(x), int_to_bytes(x)) 165 | 166 | it = self.db.iterkeys() 167 | 168 | self.assertEqual([], list(it)) 169 | 170 | it.seek_to_last() 171 | self.assertEqual([b'99'], list(it)) 172 | 173 | ref = sorted([int_to_bytes(x) for x in range(300)]) 174 | it.seek_to_first() 175 | self.assertEqual(ref, list(it)) 176 | 177 | it.seek(b'90') 178 | ref = [ 179 | b'90', 180 | b'91', 181 | b'92', 182 | b'93', 183 | b'94', 184 | b'95', 185 | b'96', 186 | b'97', 187 | b'98', 188 | b'99' 189 | ] 190 | self.assertEqual(ref, list(it)) 191 | 192 | def test_iter_values(self): 193 | for x in range(300): 194 | self.db.put(int_to_bytes(x), int_to_bytes(x * 1000)) 195 | 196 | it = self.db.itervalues() 197 | 198 | self.assertEqual([], list(it)) 199 | 200 | it.seek_to_last() 201 | self.assertEqual([b'99000'], list(it)) 202 | 203 | ref = sorted([int_to_bytes(x) for x in range(300)]) 204 | ref = [int_to_bytes(int(x) * 1000) for x in ref] 205 | it.seek_to_first() 206 | self.assertEqual(ref, list(it)) 207 | 208 | it.seek(b'90') 209 | ref = [int_to_bytes(x * 1000) for x in range(90, 100)] 210 | self.assertEqual(ref, list(it)) 211 | 212 | def test_iter_items(self): 213 | for x in range(300): 214 | self.db.put(int_to_bytes(x), int_to_bytes(x * 1000)) 215 | 216 | it = self.db.iteritems() 217 | 218 | self.assertEqual([], list(it)) 219 | 220 | it.seek_to_last() 221 | self.assertEqual([(b'99', b'99000')], list(it)) 222 | 223 | ref = sorted([int_to_bytes(x) for x in range(300)]) 224 | ref = [(x, int_to_bytes(int(x) * 1000)) for x in ref] 225 | it.seek_to_first() 226 | self.assertEqual(ref, list(it)) 227 | 228 | it.seek(b'90') 229 | ref = [(int_to_bytes(x), int_to_bytes(x * 1000)) for x in range(90, 100)] 230 | self.assertEqual(ref, list(it)) 231 | 232 | def test_reverse_iter(self): 233 | for x in range(100): 234 | self.db.put(int_to_bytes(x), int_to_bytes(x * 1000)) 235 | 236 | it = self.db.iteritems() 237 | it.seek_to_last() 238 | 239 | ref = reversed(sorted([int_to_bytes(x) for x in range(100)])) 240 | ref = [(x, int_to_bytes(int(x) * 1000)) for x in ref] 241 | 242 | self.assertEqual(ref, list(reversed(it))) 243 | 244 | def test_snapshot(self): 245 | self.db.put(b"a", b"1") 246 | self.db.put(b"b", b"2") 247 | 248 | snapshot = self.db.snapshot() 249 | self.db.put(b"a", b"2") 250 | self.db.delete(b"b") 251 | 252 | it = self.db.iteritems() 253 | it.seek_to_first() 254 | self.assertEqual({b'a': b'2'}, dict(it)) 255 | 256 | # it = self.db.iteritems(snapshot=snapshot) 257 | # it.seek_to_first() 258 | # self.assertEqual({b'a': b'1', b'b': b'2'}, dict(it)) 259 | 260 | def test_get_property(self): 261 | for x in range(300): 262 | x = int_to_bytes(x) 263 | self.db.put(x, x) 264 | 265 | self.assertIsNotNone(self.db.get_property(b'rocksdb.stats')) 266 | self.assertIsNotNone(self.db.get_property(b'rocksdb.sstables')) 267 | self.assertIsNotNone(self.db.get_property(b'rocksdb.num-files-at-level0')) 268 | self.assertIsNone(self.db.get_property(b'does not exsits')) 269 | 270 | def test_compact_range(self): 271 | for x in range(10000): 272 | x = int_to_bytes(x) 273 | self.db.put(x, x) 274 | 275 | self.db.compact_range() 276 | 277 | 278 | class AssocCounter(aimrocks.interfaces.AssociativeMergeOperator): 279 | def merge(self, key, existing_value, value): 280 | if existing_value: 281 | return (True, int_to_bytes(int(existing_value) + int(value))) 282 | return (True, value) 283 | 284 | def name(self): 285 | return b'AssocCounter' 286 | 287 | 288 | class TestUint64Merge(TestHelper): 289 | def setUp(self): 290 | TestHelper.setUp(self) 291 | opts = aimrocks.Options() 292 | opts.create_if_missing = True 293 | opts.merge_operator = UintAddOperator() 294 | self.db = aimrocks.DB(os.path.join(self.db_loc, 'test'), opts) 295 | 296 | def test_merge(self): 297 | self.db.put(b'a', struct.pack('Q', 5566)) 298 | for x in range(1000): 299 | self.db.merge(b"a", struct.pack('Q', x)) 300 | self.assertEqual(5566 + sum(range(1000)), struct.unpack('Q', self.db.get(b'a'))[0]) 301 | 302 | 303 | # class TestPutMerge(TestHelper): 304 | # def setUp(self): 305 | # TestHelper.setUp(self) 306 | # opts = rocksdb.Options() 307 | # opts.create_if_missing = True 308 | # opts.merge_operator = "put" 309 | # self.db = rocksdb.DB(os.path.join(self.db_loc, 'test'), opts) 310 | 311 | # def test_merge(self): 312 | # self.db.put(b'a', b'ccc') 313 | # self.db.merge(b'a', b'ddd') 314 | # self.assertEqual(self.db.get(b'a'), 'ddd') 315 | 316 | # class TestPutV1Merge(TestHelper): 317 | # def setUp(self): 318 | # TestHelper.setUp(self) 319 | # opts = rocksdb.Options() 320 | # opts.create_if_missing = True 321 | # opts.merge_operator = "put_v1" 322 | # self.db = rocksdb.DB(os.path.join(self.db_loc, 'test'), opts) 323 | 324 | # def test_merge(self): 325 | # self.db.put(b'a', b'ccc') 326 | # self.db.merge(b'a', b'ddd') 327 | # self.assertEqual(self.db.get(b'a'), 'ddd') 328 | 329 | class TestStringAppendOperatorMerge(TestHelper): 330 | def setUp(self): 331 | TestHelper.setUp(self) 332 | opts = aimrocks.Options() 333 | opts.create_if_missing = True 334 | opts.merge_operator = StringAppendOperator() 335 | self.db = aimrocks.DB(os.path.join(self.db_loc, 'test'), opts) 336 | 337 | # NOTE(sileht): Raise "Corruption: Error: Could not perform merge." on PY3 338 | #@unittest.skipIf(sys.version_info[0] == 3, 339 | # "Unexpected behavior on PY3") 340 | def test_merge(self): 341 | self.db.put(b'a', b'ccc') 342 | self.db.merge(b'a', b'ddd') 343 | self.assertEqual(self.db.get(b'a'), b'ccc,ddd') 344 | 345 | # class TestStringMaxOperatorMerge(TestHelper): 346 | # def setUp(self): 347 | # TestHelper.setUp(self) 348 | # opts = rocksdb.Options() 349 | # opts.create_if_missing = True 350 | # opts.merge_operator = "max" 351 | # self.db = rocksdb.DB(os.path.join(self.db_loc, 'test'), opts) 352 | 353 | # def test_merge(self): 354 | # self.db.put(b'a', int_to_bytes(55)) 355 | # self.db.merge(b'a', int_to_bytes(56)) 356 | # self.assertEqual(int(self.db.get(b'a')), 56) 357 | 358 | 359 | class TestAssocMerge(TestHelper): 360 | def setUp(self): 361 | TestHelper.setUp(self) 362 | opts = aimrocks.Options() 363 | opts.create_if_missing = True 364 | opts.merge_operator = AssocCounter() 365 | self.db = aimrocks.DB(os.path.join(self.db_loc, 'test'), opts) 366 | 367 | def test_merge(self): 368 | for x in range(1000): 369 | self.db.merge(b"a", int_to_bytes(x)) 370 | self.assertEqual(sum(range(1000)), int(self.db.get(b'a'))) 371 | 372 | 373 | class FullCounter(aimrocks.interfaces.MergeOperator): 374 | def name(self): 375 | return b'fullcounter' 376 | 377 | def full_merge(self, key, existing_value, operand_list): 378 | ret = sum([int(x) for x in operand_list]) 379 | if existing_value: 380 | ret += int(existing_value) 381 | 382 | return (True, int_to_bytes(ret)) 383 | 384 | def partial_merge(self, key, left, right): 385 | return (True, int_to_bytes(int(left) + int(right))) 386 | 387 | 388 | class TestFullMerge(TestHelper): 389 | def setUp(self): 390 | TestHelper.setUp(self) 391 | opts = aimrocks.Options() 392 | opts.create_if_missing = True 393 | opts.merge_operator = FullCounter() 394 | self.db = aimrocks.DB(os.path.join(self.db_loc, 'test'), opts) 395 | 396 | def test_merge(self): 397 | for x in range(1000): 398 | self.db.merge(b"a", int_to_bytes(x)) 399 | self.assertEqual(sum(range(1000)), int(self.db.get(b'a'))) 400 | 401 | 402 | class SimpleComparator(aimrocks.interfaces.Comparator): 403 | def name(self): 404 | return b'mycompare' 405 | 406 | def compare(self, a, b): 407 | a = int(a) 408 | b = int(b) 409 | if a < b: 410 | return -1 411 | if a == b: 412 | return 0 413 | if a > b: 414 | return 1 415 | 416 | 417 | class TestComparator(TestHelper): 418 | def setUp(self): 419 | TestHelper.setUp(self) 420 | opts = aimrocks.Options() 421 | opts.create_if_missing = True 422 | opts.comparator = SimpleComparator() 423 | self.db = aimrocks.DB(os.path.join(self.db_loc, 'test'), opts) 424 | 425 | def test_compare(self): 426 | for x in range(1000): 427 | self.db.put(int_to_bytes(x), int_to_bytes(x)) 428 | 429 | self.assertEqual(b'300', self.db.get(b'300')) 430 | 431 | class StaticPrefix(aimrocks.interfaces.SliceTransform): 432 | def name(self): 433 | return b'static' 434 | 435 | def transform(self, src): 436 | return (0, 5) 437 | 438 | def in_domain(self, src): 439 | return len(src) >= 5 440 | 441 | def in_range(self, dst): 442 | return len(dst) == 5 443 | 444 | class TestPrefixExtractor(TestHelper): 445 | def setUp(self): 446 | TestHelper.setUp(self) 447 | opts = aimrocks.Options(create_if_missing=True) 448 | opts.prefix_extractor = StaticPrefix() 449 | self.db = aimrocks.DB(os.path.join(self.db_loc, 'test'), opts) 450 | 451 | def _fill_db(self): 452 | for x in range(3000): 453 | keyx = hex(x)[2:].zfill(5).encode('utf8') + b'.x' 454 | keyy = hex(x)[2:].zfill(5).encode('utf8') + b'.y' 455 | keyz = hex(x)[2:].zfill(5).encode('utf8') + b'.z' 456 | self.db.put(keyx, b'x') 457 | self.db.put(keyy, b'y') 458 | self.db.put(keyz, b'z') 459 | 460 | 461 | def test_prefix_iterkeys(self): 462 | self._fill_db() 463 | self.assertEqual(b'x', self.db.get(b'00001.x')) 464 | self.assertEqual(b'y', self.db.get(b'00001.y')) 465 | self.assertEqual(b'z', self.db.get(b'00001.z')) 466 | 467 | it = self.db.iterkeys() 468 | it.seek(b'00002') 469 | 470 | ref = [b'00002.x', b'00002.y', b'00002.z'] 471 | ret = takewhile(lambda key: key.startswith(b'00002'), it) 472 | self.assertEqual(ref, list(ret)) 473 | 474 | def test_prefix_iteritems(self): 475 | self._fill_db() 476 | 477 | it = self.db.iteritems() 478 | it.seek(b'00002') 479 | 480 | ref = {b'00002.z': b'z', b'00002.y': b'y', b'00002.x': b'x'} 481 | ret = takewhile(lambda item: item[0].startswith(b'00002'), it) 482 | self.assertEqual(ref, dict(ret)) 483 | 484 | class TestDBColumnFamilies(TestHelper): 485 | def setUp(self): 486 | TestHelper.setUp(self) 487 | opts = aimrocks.Options(create_if_missing=True) 488 | self.db = aimrocks.DB( 489 | os.path.join(self.db_loc, 'test'), 490 | opts, 491 | ) 492 | 493 | self.cf_a = self.db.create_column_family(b'A', aimrocks.ColumnFamilyOptions()) 494 | self.cf_b = self.db.create_column_family(b'B', aimrocks.ColumnFamilyOptions()) 495 | 496 | def test_column_families(self): 497 | families = self.db.column_families 498 | names = [handle.name for handle in families] 499 | self.assertEqual([b'default', b'A', b'B'], names) 500 | for name in names: 501 | self.assertIn(self.db.get_column_family(name), families) 502 | 503 | self.assertEqual( 504 | names, 505 | aimrocks.list_column_families( 506 | os.path.join(self.db_loc, 'test'), 507 | aimrocks.Options(), 508 | ) 509 | ) 510 | 511 | def test_get_none(self): 512 | self.assertIsNone(self.db.get(b'k')) 513 | self.assertIsNone(self.db.get(b'k', column_family=self.cf_a)) 514 | self.assertIsNone(self.db.get(b'k', column_family=self.cf_b)) 515 | 516 | def test_put_get(self): 517 | self.db.put(b'k', b"v", column_family=self.cf_a) 518 | self.assertEqual(b"v", self.db.get(b'k', column_family=self.cf_a)) 519 | self.assertIsNone(self.db.get(b"k")) 520 | self.assertIsNone(self.db.get(b"k", column_family=self.cf_b)) 521 | 522 | def test_multi_get(self): 523 | data = [ 524 | (b'a', b'1default'), 525 | (b'b', b'2default'), 526 | (b'c', b'3default'), 527 | ((self.cf_a, b'a'), b'1a'), 528 | ((self.cf_a, b'b'), b'2a'), 529 | ((self.cf_a, b'c'), b'3a'), 530 | ((self.cf_b, b'a'), b'1b'), 531 | ((self.cf_b, b'b'), b'2b'), 532 | ((self.cf_b, b'c'), b'3b'), 533 | ] 534 | for key, value in data: 535 | if not isinstance(key, tuple): 536 | key = (None, key) 537 | self.db.put(key[1], value, column_family=key[0]) 538 | 539 | multi_get_lookup = [value[0] for value in data] 540 | 541 | ret = self.db.multi_get(multi_get_lookup) 542 | ref = {value[0]: value[1] for value in data} 543 | self.assertEqual(ref, ret) 544 | 545 | def test_delete(self): 546 | self.db.put(b"a", b"b", column_family=self.cf_a) 547 | self.assertEqual(b"b", self.db.get(b"a", column_family=self.cf_a)) 548 | self.db.delete(b"a", column_family=self.cf_a) 549 | self.assertIsNone(self.db.get(b"a", column_family=self.cf_a)) 550 | 551 | def test_write_batch(self): 552 | cfa = self.db.get_column_family(b"A") 553 | batch = aimrocks.WriteBatch() 554 | batch.put(b"key", b"v1", column_family=cfa) 555 | batch.delete(b"key", column_family=self.cf_a) 556 | batch.put(b"key", b"v2", column_family=cfa) 557 | batch.put(b"key", b"v3", column_family=cfa) 558 | batch.put(b"a", b"1", column_family=cfa) 559 | batch.put(b"b", b"2", column_family=cfa) 560 | 561 | self.db.write(batch) 562 | query = [(cfa, b"key"), (cfa, b"a"), (cfa, b"b")] 563 | ret = self.db.multi_get(query) 564 | 565 | self.assertEqual(b"v3", ret[query[0]]) 566 | self.assertEqual(b"1", ret[query[1]]) 567 | self.assertEqual(b"2", ret[query[2]]) 568 | 569 | def test_key_may_exists(self): 570 | self.db.put(b"a", b'1', column_family=self.cf_a) 571 | 572 | self.assertEqual( 573 | (False, None), 574 | self.db.key_may_exist(b"x", column_family=self.cf_a) 575 | ) 576 | self.assertEqual( 577 | (False, None), 578 | self.db.key_may_exist(b'x', fetch=True, column_family=self.cf_a) 579 | ) 580 | self.assertEqual( 581 | (True, None), 582 | self.db.key_may_exist(b'a', column_family=self.cf_a) 583 | ) 584 | self.assertEqual( 585 | (True, b'1'), 586 | self.db.key_may_exist(b'a', fetch=True, column_family=self.cf_a) 587 | ) 588 | 589 | def test_iter_keys(self): 590 | for x in range(300): 591 | self.db.put(int_to_bytes(x), int_to_bytes(x), column_family=self.cf_a) 592 | 593 | it = self.db.iterkeys(self.cf_a) 594 | self.assertEqual([], list(it)) 595 | 596 | it.seek_to_last() 597 | self.assertEqual([b'99'], list(it)) 598 | 599 | ref = sorted([int_to_bytes(x) for x in range(300)]) 600 | it.seek_to_first() 601 | self.assertEqual(ref, list(it)) 602 | 603 | it.seek(b'90') 604 | ref = sorted([int_to_bytes(x) for x in range(90, 100)]) 605 | self.assertEqual(ref, list(it)) 606 | 607 | def test_iter_values(self): 608 | for x in range(300): 609 | self.db.put(int_to_bytes(x), int_to_bytes(x * 1000), column_family=self.cf_b) 610 | 611 | it = self.db.itervalues(self.cf_b) 612 | self.assertEqual([], list(it)) 613 | 614 | it.seek_to_last() 615 | self.assertEqual([b'99000'], list(it)) 616 | 617 | ref = sorted([int_to_bytes(x) for x in range(300)]) 618 | ref = [int_to_bytes(int(x) * 1000) for x in ref] 619 | it.seek_to_first() 620 | self.assertEqual(ref, list(it)) 621 | 622 | it.seek(b'90') 623 | ref = [int_to_bytes(x * 1000) for x in range(90, 100)] 624 | self.assertEqual(ref, list(it)) 625 | 626 | def test_iter_items(self): 627 | for x in range(300): 628 | self.db.put(int_to_bytes(x), int_to_bytes(x * 1000), column_family=self.cf_b) 629 | 630 | it = self.db.iteritems(self.cf_b) 631 | self.assertEqual([], list(it)) 632 | 633 | it.seek_to_last() 634 | self.assertEqual([(b'99', b'99000')], list(it)) 635 | 636 | ref = sorted([int_to_bytes(x) for x in range(300)]) 637 | ref = [(x, int_to_bytes(int(x) * 1000)) for x in ref] 638 | it.seek_to_first() 639 | self.assertEqual(ref, list(it)) 640 | 641 | it.seek(b'90') 642 | ref = [(int_to_bytes(x), int_to_bytes(x * 1000)) for x in range(90, 100)] 643 | self.assertEqual(ref, list(it)) 644 | 645 | def test_reverse_iter(self): 646 | for x in range(100): 647 | self.db.put(int_to_bytes(x), int_to_bytes(x * 1000), column_family=self.cf_a) 648 | 649 | it = self.db.iteritems(self.cf_a) 650 | it.seek_to_last() 651 | 652 | ref = reversed(sorted([ 653 | int_to_bytes(x) for x in range(100) 654 | ])) 655 | ref = [(x, int_to_bytes(int(x) * 1000)) for x in ref] 656 | 657 | self.assertEqual(ref, list(reversed(it))) 658 | 659 | def test_snapshot(self): 660 | cfa = self.db.get_column_family(b'A') 661 | self.db.put(b"a", b"1", column_family=cfa) 662 | self.db.put(b"b", b"2", column_family=cfa) 663 | 664 | snapshot = self.db.snapshot() 665 | self.db.put(b"a", b"2", column_family=cfa) 666 | self.db.delete(b"b", column_family=cfa) 667 | 668 | it = self.db.iteritems(cfa) 669 | it.seek_to_first() 670 | self.assertEqual({b'a': b'2'}, dict(it)) 671 | 672 | # it = self.db.iteritems(cfa, snapshot=snapshot) 673 | # it.seek_to_first() 674 | # self.assertEqual({(cfa, b'a'): b'1', (cfa, b'b'): b'2'}, dict(it)) 675 | 676 | def test_get_property(self): 677 | for x in range(300): 678 | x = int_to_bytes(x) 679 | self.db.put(x, x, column_family=self.cf_a) 680 | 681 | self.assertEqual(b"300", 682 | self.db.get_property(b'rocksdb.estimate-num-keys', 683 | self.cf_a)) 684 | self.assertIsNone(self.db.get_property(b'does not exsits', 685 | self.cf_a)) 686 | 687 | def test_compact_range(self): 688 | for x in range(10000): 689 | x = int_to_bytes(x) 690 | self.db.put(x, x, column_family=self.cf_b) 691 | 692 | self.db.compact_range(column_family=self.cf_b) 693 | -------------------------------------------------------------------------------- /tests/test_memtable.py: -------------------------------------------------------------------------------- 1 | # content of test_sample.py 2 | import aimrocks 3 | import pytest 4 | import shutil 5 | import os 6 | import tempfile 7 | 8 | def test_open_skiplist_memtable_factory(): 9 | opts = aimrocks.Options() 10 | opts.memtable_factory = aimrocks.SkipListMemtableFactory() 11 | opts.create_if_missing = True 12 | 13 | loc = tempfile.mkdtemp() 14 | try: 15 | test_db = aimrocks.DB(os.path.join(loc, "test"), opts) 16 | finally: 17 | shutil.rmtree(loc) 18 | 19 | 20 | def test_open_vector_memtable_factory(): 21 | opts = aimrocks.Options() 22 | opts.allow_concurrent_memtable_write = False 23 | opts.memtable_factory = aimrocks.VectorMemtableFactory() 24 | opts.create_if_missing = True 25 | loc = tempfile.mkdtemp() 26 | try: 27 | test_db = aimrocks.DB(os.path.join(loc, "test"), opts) 28 | finally: 29 | shutil.rmtree(loc) 30 | -------------------------------------------------------------------------------- /tests/test_options.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sys 3 | import aimrocks 4 | 5 | class TestFilterPolicy(aimrocks.interfaces.FilterPolicy): 6 | def create_filter(self, keys): 7 | return b'nix' 8 | 9 | def key_may_match(self, key, fil): 10 | return True 11 | 12 | def name(self): 13 | return b'testfilter' 14 | 15 | class TestMergeOperator(aimrocks.interfaces.MergeOperator): 16 | def full_merge(self, *args, **kwargs): 17 | return (False, None) 18 | 19 | def partial_merge(self, *args, **kwargs): 20 | return (False, None) 21 | 22 | def name(self): 23 | return b'testmergeop' 24 | 25 | class TestOptions(unittest.TestCase): 26 | # def test_default_merge_operator(self): 27 | # opts = rocksdb.Options() 28 | # self.assertEqual(True, opts.paranoid_checks) 29 | # opts.paranoid_checks = False 30 | # self.assertEqual(False, opts.paranoid_checks) 31 | 32 | # self.assertIsNone(opts.merge_operator) 33 | # opts.merge_operator = "uint64add" 34 | # self.assertIsNotNone(opts.merge_operator) 35 | # self.assertEqual(opts.merge_operator, "uint64add") 36 | # with self.assertRaises(TypeError): 37 | # opts.merge_operator = "not an operator" 38 | 39 | # FIXME: travis test should include the latest version of rocksdb 40 | # def test_compaction_pri(self): 41 | # opts = rocksdb.Options() 42 | # default compaction_pri 43 | # self.assertEqual(opts.compaction_pri, rocksdb.CompactionPri.by_compensated_size) 44 | # self.assertEqual(opts.compaction_pri, rocksdb.CompactionPri.min_overlapping_ratio) 45 | # opts.compaction_pri = rocksdb.CompactionPri.by_compensated_size 46 | # self.assertEqual(opts.compaction_pri, rocksdb.CompactionPri.by_compensated_size) 47 | # opts.compaction_pri = rocksdb.CompactionPri.oldest_largest_seq_first 48 | # self.assertEqual(opts.compaction_pri, rocksdb.CompactionPri.oldest_largest_seq_first) 49 | # opts.compaction_pri = rocksdb.CompactionPri.min_overlapping_ratio 50 | # self.assertEqual(opts.compaction_pri, rocksdb.CompactionPri.min_overlapping_ratio) 51 | 52 | def test_enable_write_thread_adaptive_yield(self): 53 | opts = aimrocks.Options() 54 | self.assertEqual(opts.enable_write_thread_adaptive_yield, True) 55 | opts.enable_write_thread_adaptive_yield = False 56 | self.assertEqual(opts.enable_write_thread_adaptive_yield, False) 57 | 58 | def test_allow_concurrent_memtable_write(self): 59 | opts = aimrocks.Options() 60 | self.assertEqual(opts.allow_concurrent_memtable_write, True) 61 | opts.allow_concurrent_memtable_write = False 62 | self.assertEqual(opts.allow_concurrent_memtable_write, False) 63 | 64 | def test_compression_opts(self): 65 | opts = aimrocks.Options() 66 | compression_opts = opts.compression_opts 67 | # default value 68 | self.assertEqual(isinstance(compression_opts, dict), True) 69 | self.assertEqual(compression_opts['window_bits'], -14) 70 | # This doesn't match rocksdb latest 71 | # self.assertEqual(compression_opts['level'], -1) 72 | self.assertEqual(compression_opts['strategy'], 0) 73 | self.assertEqual(compression_opts['max_dict_bytes'], 0) 74 | 75 | with self.assertRaises(TypeError): 76 | opts.compression_opts = list(1,2) 77 | 78 | opts.compression_opts = {'window_bits': 1, 'level': 2, 'strategy': 3, 'max_dict_bytes': 4} 79 | compression_opts = opts.compression_opts 80 | self.assertEqual(compression_opts['window_bits'], 1) 81 | self.assertEqual(compression_opts['level'], 2) 82 | self.assertEqual(compression_opts['strategy'], 3) 83 | self.assertEqual(compression_opts['max_dict_bytes'], 4) 84 | 85 | def test_simple(self): 86 | opts = aimrocks.Options() 87 | self.assertEqual(True, opts.paranoid_checks) 88 | opts.paranoid_checks = False 89 | self.assertEqual(False, opts.paranoid_checks) 90 | 91 | self.assertIsNone(opts.merge_operator) 92 | ob = TestMergeOperator() 93 | opts.merge_operator = ob 94 | self.assertEqual(opts.merge_operator, ob) 95 | 96 | self.assertIsInstance( 97 | opts.comparator, 98 | aimrocks.BytewiseComparator) 99 | 100 | self.assertIn(opts.compression, 101 | (aimrocks.CompressionType.no_compression, 102 | aimrocks.CompressionType.snappy_compression)) 103 | 104 | opts.compression = aimrocks.CompressionType.zstd_compression 105 | self.assertEqual(aimrocks.CompressionType.zstd_compression, opts.compression) 106 | 107 | def test_block_options(self): 108 | aimrocks.BlockBasedTableFactory( 109 | block_size=4096, 110 | filter_policy=TestFilterPolicy(), 111 | block_cache=aimrocks.LRUCache(100)) 112 | 113 | def test_unicode_path(self): 114 | name = b'/tmp/M\xc3\xbcnchen'.decode('utf8') 115 | opts = aimrocks.Options() 116 | opts.db_log_dir = name 117 | opts.wal_dir = name 118 | 119 | self.assertEqual(name, opts.db_log_dir) 120 | self.assertEqual(name, opts.wal_dir) 121 | 122 | def test_table_factory(self): 123 | opts = aimrocks.Options() 124 | self.assertIsNone(opts.table_factory) 125 | 126 | opts.table_factory = aimrocks.BlockBasedTableFactory() 127 | opts.table_factory = aimrocks.PlainTableFactory() 128 | 129 | def test_compaction_style(self): 130 | opts = aimrocks.Options() 131 | self.assertEqual('level', opts.compaction_style) 132 | 133 | opts.compaction_style = 'universal' 134 | self.assertEqual('universal', opts.compaction_style) 135 | 136 | opts.compaction_style = 'level' 137 | self.assertEqual('level', opts.compaction_style) 138 | 139 | if sys.version_info[0] == 3: 140 | assertRaisesRegex = self.assertRaisesRegex 141 | else: 142 | assertRaisesRegex = self.assertRaisesRegexp 143 | 144 | with assertRaisesRegex(Exception, 'Unknown compaction style'): 145 | opts.compaction_style = 'foo' 146 | 147 | def test_compaction_opts_universal(self): 148 | opts = aimrocks.Options() 149 | uopts = opts.compaction_options_universal 150 | self.assertEqual(-1, uopts['compression_size_percent']) 151 | self.assertEqual(200, uopts['max_size_amplification_percent']) 152 | self.assertEqual('total_size', uopts['stop_style']) 153 | self.assertEqual(1, uopts['size_ratio']) 154 | self.assertEqual(2, uopts['min_merge_width']) 155 | self.assertGreaterEqual(4294967295, uopts['max_merge_width']) 156 | 157 | new_opts = {'stop_style': 'similar_size', 'max_merge_width': 30} 158 | opts.compaction_options_universal = new_opts 159 | uopts = opts.compaction_options_universal 160 | 161 | self.assertEqual(-1, uopts['compression_size_percent']) 162 | self.assertEqual(200, uopts['max_size_amplification_percent']) 163 | self.assertEqual('similar_size', uopts['stop_style']) 164 | self.assertEqual(1, uopts['size_ratio']) 165 | self.assertEqual(2, uopts['min_merge_width']) 166 | self.assertEqual(30, uopts['max_merge_width']) 167 | 168 | def test_row_cache(self): 169 | opts = aimrocks.Options() 170 | self.assertIsNone(opts.row_cache) 171 | opts.row_cache = cache = aimrocks.LRUCache(2 * 1024 * 1024) 172 | self.assertEqual(cache, opts.row_cache) 173 | --------------------------------------------------------------------------------