├── .github
    ├── ISSUE_TEMPLATE.md
    └── workflows
    │   └── python-package.yml
├── .gitignore
├── ChangeLog
├── LICENSE
├── MANIFEST.in
├── README.md
├── docs
    ├── Makefile
    ├── conf.py
    ├── index.rst
    └── themes
    │   └── acid
    │       ├── conf.py.conf
    │       ├── layout.html
    │       ├── sourcelink.html
    │       ├── static
    │           └── acid.css
    │       └── theme.conf
├── examples
    ├── address-book.py
    ├── dirtybench-gdbm.py
    ├── dirtybench.py
    ├── keystore
    │   ├── README.md
    │   ├── __init__.py
    │   ├── interfaces.py
    │   ├── lmdb.py
    │   ├── main.py
    │   ├── static
    │   │   └── index.html
    │   ├── web.py
    │   └── webapi.py
    ├── nastybench.py
    ├── parabench.py
    └── words.gz
├── lib
    ├── lmdb.h
    ├── mdb.c
    ├── midl.c
    ├── midl.h
    ├── py-lmdb
    │   ├── env-copy-txn.patch
    │   └── preload.h
    ├── win32-stdint
    │   └── stdint.h
    └── win32
    │   ├── inttypes.h
    │   └── unistd.h
├── lmdb
    ├── __init__.py
    ├── __main__.py
    ├── cffi.py
    ├── cpython.c
    └── tool.py
├── misc
    ├── cursor-del-break.c
    ├── cursor-iter-bug.c
    ├── cursor_put_pyparse.diff
    ├── gdb.commands
    ├── its7733.c
    ├── readers_mrb_env.patch
    ├── run_in_vm.py
    └── test_monster_acid_trace.diff
├── setup.py
└── tests
    ├── crash_test.py
    ├── cursor_test.py
    ├── env_test.py
    ├── getmulti_test.py
    ├── iteration_test.py
    ├── package_test.py
    ├── testlib.py
    ├── tool_test.py
    └── txn_test.py


/.github/ISSUE_TEMPLATE.md:
--------------------------------------------------------------------------------
 1 | Edit this issue template as appropriate. If none of it seems relevant to your
 2 | issue, then you may delete the entire template, however, I may not respond to
 3 | issues that do not include important information that is mentioned in this
 4 | template!
 5 | 
 6 | 
 7 | ### Affected Operating Systems
 8 | 
 9 | * Linux
10 | * Windows
11 | * BSD
12 | * [Other]
13 | 
14 | ### Affected py-lmdb Version
15 | 
16 |     e.g. "print lmdb.__version__"
17 | 
18 | ### py-lmdb Installation Method
19 | 
20 |     e.g. sudo pip install lmdb
21 | 
22 | ### Using bundled or distribution-provided LMDB library?
23 | 
24 | Bundled
25 | 
26 | ### Distribution name and LMDB library version
27 | 
28 | Use "print lmdb.version()" from a Python prompt.
29 | 
30 | ### Machine "free -m" output
31 | 
32 | e.g.
33 | 
34 | ```
35 |              total       used       free     shared    buffers     cached
36 | Mem:         24154      23874        279         21        386       8175
37 | -/+ buffers/cache:      15313       8840
38 | Swap:            0          0          0
39 | ```
40 | 
41 | ### Other important machine info
42 | 
43 | Running under cgroups? Containers? Weird filesystems in use? Network
44 | filesystem? Patched kernel? ...
45 | 
46 | 
47 | ### Describe Your Problem
48 | 
49 | XXXXX
50 | 
51 | 
52 | ### Errors/exceptions Encountered
53 | 
54 | e.g.
55 | 
56 | ```
57 | Traceback (most recent call last):
58 |   File "<stdin>", line 1, in <module>
59 | MemoryError
60 | ```
61 | 
62 | 
63 | 
64 | ### Describe What You Expected To Happen
65 | 
66 | e.g.
67 | 
68 | I expected the transaction to commit successfully.
69 | 
70 | 
71 | 
72 | ### Describe What Happened Instead
73 | 
74 | e.g.
75 | 
76 | The Python process crashed.
77 | 


--------------------------------------------------------------------------------
/.github/workflows/python-package.yml:
--------------------------------------------------------------------------------
  1 | ---
  2 | name: Build, run, and test py-lmdb
  3 | 
  4 | 'on':
  5 |   push:
  6 |     branches: [master, release]
  7 |     tags:
  8 |       - 'py-lmdb_*'
  9 |   pull_request:
 10 |     branches: [master]
 11 | 
 12 | jobs:
 13 |   build:
 14 | 
 15 |     runs-on: ${{ matrix.os }}
 16 |     strategy:
 17 |       matrix:
 18 |         os: [ubuntu-20.04, macos-latest, windows-latest]
 19 |         python-version: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12', '3.13', pypy-3.10]
 20 |         impl: [cpython, cffi]
 21 |         purity: [pure, with-pylmdb-mods]
 22 | 
 23 |         exclude:
 24 |           # Pypy doesn't work with cpython
 25 |           - python-version: pypy-3.10
 26 |             impl: cpython
 27 |           # macos latest is now arm64 and 3.7 doesn't have arm64 build
 28 |           - python-version: 3.7
 29 |             os: macos-latest
 30 | 
 31 |     steps:
 32 |       - uses: actions/checkout@v2
 33 |       - name: Set up Python ${{ matrix.python-version }}
 34 |         uses: actions/setup-python@v5
 35 |         with:
 36 |           python-version: ${{ matrix.python-version }}
 37 |       - name: Set env vars Windows
 38 |         if: runner.os == 'Windows'
 39 |         run: >-
 40 |           if ( '${{ matrix.impl }}' -eq 'cpython' ) {
 41 |             echo "LMDB_FORCE_CPYTHON=1" |
 42 |               Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
 43 |           } else {
 44 |             echo "LMDB_FORCE_CFFI=1" |
 45 |               Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
 46 |           }
 47 |           if ( '${{ matrix.purity }}' -eq 'pure' ) {
 48 |             echo "LMDB_PURE=1" |
 49 |               Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
 50 |           } elseif ( '${{ matrix.purity }}' -eq 'system' ){
 51 |             echo "LMDB_FORCE_SYSTEM=1" |
 52 |               Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
 53 |           }
 54 |       - name: Set env vars *nix
 55 |         if: runner.os != 'Windows'
 56 |         run: |
 57 |           if [[ ${{ matrix.impl }} == cpython ]] ; then
 58 |             echo "LMDB_FORCE_CPYTHON=1" >> $GITHUB_ENV;
 59 |           else
 60 |             echo "LMDB_FORCE_CFFI=1" >> $GITHUB_ENV;
 61 |           fi
 62 |           if [[ ${{ matrix.purity }} == pure ]] ; then
 63 |             echo "LMDB_PURE=1" >> $GITHUB_ENV;
 64 |           elif [[ ${{ matrix.purity }} == system ]] ; then
 65 |             echo "LMDB_FORCE_SYSTEM=1" >> $GITHUB_ENV;
 66 |             sudo apt-get install liblmdb-dev;
 67 |           fi
 68 | 
 69 |       - name: Install dependencies
 70 |         run: |
 71 |           echo "Linux: Envs are cpython=$LMDB_FORCE_CPYTHON
 72 |             cffi=$LMDB_FORCE_CFFI pure=$LMDB_PURE system=$LMDB_FORCE_SYSTEM"
 73 |           echo "Windows: Envs are cpython=$Env:LMDB_FORCE_CPYTHON
 74 |             cffi=$Env:LMDB_FORCE_CFFI pure=$Env:LMDB_PURE system=$Env:LMDB_FORCE_SYSTEM"
 75 |           python -m pip install setuptools wheel
 76 |           # Install this separately since sometimes Github Actions can't find it
 77 |           python -m pip install cffi
 78 |           python -m pip install flake8 pytest patch-ng
 79 |           python setup.py develop sdist bdist_wheel
 80 |           ls dist
 81 |       # name: Lint with flake8
 82 |       # run: |
 83 |       #   # stop the build if there are Python syntax errors or undefined names
 84 |       #   flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
 85 |       #   # exit-zero treats all errors as warnings.
 86 |       #   flake8 . --count --exit-zero --max-complexity=10 --max-line-length=120 --statistics
 87 |       #
 88 |       - name: Test with pytest
 89 |         # Limit the test cycle a little
 90 |         if: >-
 91 |           matrix.python-version != '3.7' && matrix.python-version != '3.9' &&
 92 |           matrix.python-version != '3.11'
 93 |         run: |
 94 |           echo "Envs are cpython=$LMDB_FORCE_CPYTHON cffi=$LMDB_FORCE_CFFI
 95 |             pure=$LMDB_PURE system=$LMDB_FORCE_SYSTEM"
 96 |           pytest
 97 | 
 98 |       - name: Save wheel
 99 |         uses: actions/upload-artifact@v4
100 |         with:
101 |           name: "${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.impl }}-${{ matrix.purity }}.whl"
102 |           path: dist/lmdb-*.whl
103 | 
104 |       - name: Get version
105 |         if: >-
106 |           matrix.python-version == '3.10' && runner.os == 'Linux' &&
107 |           matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython'
108 |         run: |
109 |           python -c 'import lmdb; print(lmdb.__version__, end="")' > vers.txt
110 | 
111 |       - name: Save version
112 |         if: >-
113 |           matrix.python-version == '3.10' && runner.os == 'Linux' &&
114 |           matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython'
115 |         uses: actions/upload-artifact@v4
116 |         with:
117 |           path: vers.txt
118 |           name: vers.txt
119 | 
120 |       - name: Save source
121 |         # We only need a single target to upload the source
122 |         # (all targets have the same source)
123 |         if: >-
124 |           matrix.python-version == '3.10' && runner.os == 'Linux' &&
125 |           matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython'
126 | 
127 |         uses: actions/upload-artifact@v4
128 |         with:
129 |           path: dist/lmdb*.tar.gz
130 |           name: source
131 | 
132 |       - name: Build manylinux wheel
133 |         if: >-
134 |           matrix.python-version == '3.10' && runner.os == 'Linux' &&
135 |           matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython'
136 |         uses: RalfG/python-wheels-manylinux-build@v0.7.1
137 |         with:
138 |           python-versions: >-
139 |               cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313
140 |           build-requirements: 'patch-ng'
141 | 
142 |       - name: What do we have
143 |         run: ls dist
144 | 
145 |       - name: Save manylinux wheel
146 |         # We only need a single target to upload the source
147 |         # (all targets have the same source)
148 |         if: >-
149 |           matrix.python-version == '3.10' && runner.os == 'Linux' &&
150 |           matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython'
151 |         uses: actions/upload-artifact@v4
152 |         with:
153 |           path: dist/lmdb*manylinux*.whl
154 |           name: manylinux
155 | 
156 |   build_aarch64:
157 |     name: "build aarch64 ${{ matrix.PYTHON }} aarch64 ${{ matrix.impl }} ${{ matrix.purity }}"
158 |     runs-on: ubuntu-20.04
159 |     strategy:
160 |        matrix:
161 |         impl: [cpython, cffi]
162 |         purity: [pure, with-pylmdb-mods]
163 |         PYTHON: ["cp37-cp37m" , "cp38-cp38" , "cp39-cp39", "cp310-cp310", "cp311-cp311", 'cp312-cp312', 'cp313-cp313']
164 |     steps:
165 |       - uses: actions/checkout@v2
166 |       - run: |
167 |           docker run --rm --privileged hypriot/qemu-register
168 |       - uses: docker://quay.io/pypa/manylinux2014_aarch64
169 |         with:
170 |           args: |
171 |               bash -c "set -xe;
172 |               mkdir -p /github/home/.cache/pip;
173 |               chown -R $(whoami) /github/home/.cache;
174 |               python --version;
175 |               /opt/python/${{ matrix.PYTHON }}/bin/python -m venv .venv;
176 |               yum install -y libffi-devel;
177 |               .venv/bin/pip install -U pip setuptools wheel cffi six;
178 |               if \[ ${{ matrix.impl }} == cpython \] ; then
179 |                 echo LMDB_FORCE_CPYTHON=1
180 |               else
181 |                 echo LMDB_FORCE_CFFI=1
182 |               fi
183 |               if \[ ${{ matrix.purity }} == pure \] ; then
184 |                 echo LMDB_PURE=1
185 |               elif \[ ${{ matrix.purity }} == system \] ; then
186 |                 echo LMDB_FORCE_SYSTEM=1
187 |               fi
188 |               echo \"Linux: Envs are cpython=$LMDB_FORCE_CPYTHON
189 |                 cffi=$LMDB_FORCE_CFFI pure=$LMDB_PURE system=$LMDB_FORCE_SYSTEM\";
190 |               echo \"Windows: Envs are cpython=$Env:LMDB_FORCE_CPYTHON
191 |                 cffi=$Env:LMDB_FORCE_CFFI pure=$Env:LMDB_PURE system=$Env:LMDB_FORCE_SYSTEM\";
192 |               .venv/bin/pip install setuptools flake8 pytest patch-ng;
193 |               /opt/python/${{ matrix.PYTHON }}/bin/python -m pip install setuptools;
194 |               /opt/python/${{ matrix.PYTHON }}/bin/python setup.py develop bdist_wheel;
195 |               ls dist;
196 |               /opt/python/${{ matrix.PYTHON }}/bin/python -m pip install pytest;
197 |               echo \"Envs are cpython=$LMDB_FORCE_CPYTHON cffi=$LMDB_FORCE_CFFI pure=$LMDB_PURE system=$LMDB_FORCE_SYSTEM\";
198 |               if \[ ${{ matrix.PYTHON }} != cp36-cp36 \] && \[ ${{ matrix.PYTHON }} != cp38-cp38 \] && \[ ${{ matrix.PYTHON }} != cp311-cp311 \] ; then
199 |                 /opt/python/${{ matrix.PYTHON }}/bin/python -m pytest;
200 |               fi;
201 |               /opt/python/${{ matrix.PYTHON }}/bin/python -c \"import lmdb; print(lmdb.__version__, end='')\" > vers.txt;
202 |       - uses: actions/upload-artifact@v4
203 |         with:
204 |           name: "${{ runner.os }}-${{ matrix.PYTHON }}-${{ matrix.impl }}-${{ matrix.purity }}.whl"
205 |           path: dist/lmdb-*.whl
206 | 
207 |       - uses: actions/upload-artifact@v4
208 |         if: >-
209 |             matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' &&
210 |             matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython'
211 |         with:
212 |           path: vers.txt
213 |           name: vers.txt.aarch64
214 |       - uses: RalfG/python-wheels-manylinux-build@v0.7.1-manylinux2014_aarch64
215 |         if: >-
216 |             matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' &&
217 |             matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython'
218 |         with:
219 |           python-versions: >-
220 |               cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313
221 |           build-requirements: 'patch-ng'
222 |       - uses: actions/upload-artifact@v4
223 |         if: >-
224 |             matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' &&
225 |             matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython'
226 |         with:
227 |           path: dist/lmdb*manylinux*.whl
228 |           name: manylinux.aarch64
229 | 
230 |   publish:
231 |     permissions:
232 |       id-token: write # This is required for passwordless publishing to PyPI
233 |     needs: [build, build_aarch64]
234 |     # N.B. the host running twine to upload is distinct from the target image
235 |     runs-on: ubuntu-20.04
236 |     strategy:
237 |       # We publish a subset of the targets we test
238 |       matrix:
239 |         os: [macos-latest, windows-latest]
240 |         python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', pypy-3.10]
241 |         impl: [cpython, cffi]
242 |         purity: [with-pylmdb-mods]
243 | 
244 |         exclude:
245 |           - python-version: 'pypy-3.10'
246 |             impl: cpython
247 |           - python-version: '3.7'
248 |             os: macos-latest
249 |           - python-version: '3.7'
250 |             impl: cffi
251 |           - python-version: '3.8'
252 |             impl: cffi
253 |           - python-version: '3.9'
254 |             impl: cffi
255 |           - python-version: '3.10'
256 |             impl: cffi
257 |           - python-version: '3.11'
258 |             impl: cffi
259 |           - python-version: '3.12'
260 |             impl: cffi
261 |           - python-version: '3.13'
262 |             impl: cffi
263 | 
264 |         include:
265 |           # Ubuntu artifacts apply to all python versions
266 |           - os: ubuntu-20.04
267 |             python-version: '3.10'
268 |             impl: cpython
269 |             purity: with-pylmdb-mods
270 | 
271 |     steps:
272 |       - name: Download source
273 |         if: matrix.os == 'ubuntu-20.04'
274 |         uses: actions/download-artifact@v4
275 |         with:
276 |           name: source
277 |           path: dist
278 | 
279 |       - name: Download manylinux artifacts
280 |         if: matrix.os == 'ubuntu-20.04'
281 |         uses: actions/download-artifact@v4
282 |         with:
283 |           name: manylinux
284 |           path: dist
285 | 
286 |       - name: Download aarch64 manylinux artifacts
287 |         if: matrix.os == 'ubuntu-20.04'
288 |         uses: actions/download-artifact@v4
289 |         with:
290 |           name: manylinux.aarch64
291 |           path: dist
292 | 
293 |       - name: Download non-Linux wheel
294 |         if: matrix.os != 'ubuntu-20.04'
295 |         uses: actions/download-artifact@v4
296 |         with:
297 |           name: "${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.impl }}-${{ matrix.purity }}.whl"
298 |           path: dist
299 | 
300 |       - name: Inventory
301 |         run: >-
302 |           find .
303 | 
304 |       - name: Publish distribution 📦 to Test PyPI
305 |         if: ${{ !startsWith(github.ref, 'refs/tags') }}
306 |         uses: pypa/gh-action-pypi-publish@release/v1
307 | 
308 |         with:
309 |           # password: ${{ secrets.TEST_PYPI_API_TOKEN }}
310 |           repository-url: https://test.pypi.org/legacy/
311 |           skip-existing: true
312 | 
313 |       - name: Publish distribution 📦 to PyPI
314 |         if: startsWith(github.ref, 'refs/tags')
315 |         uses: pypa/gh-action-pypi-publish@release/v1
316 |         # with: password: ${{ secrets.PYPI_API_TOKEN }}
317 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | MANIFEST
 2 | __pycache__
 3 | build
 4 | dist
 5 | docs/_build
 6 | ll.sh
 7 | lmdb.egg-info
 8 | lmdb/_config.py
 9 | lo.sh
10 | old
11 | .tox/
12 | tests/test.py
13 | 


--------------------------------------------------------------------------------
/ChangeLog:
--------------------------------------------------------------------------------
  1 | 2025-01-05 1.6.2
  2 | * CI-only fix.
  3 | 
  4 | 2025-01-05 1.6.1
  5 | * CI-only fix.
  6 | 
  7 | 2025-01-05 1.6.0
  8 | * Support for Python 3.13.  Contributed by Miro Hrončok and Adam Williamson.
  9 | 
 10 | * CI:  Publish 3.13 binaries and Linux aarch64 wheels for multiple versions.
 11 | 
 12 | 2024-07-01 1.5.1
 13 | * CI-only fix.
 14 | 
 15 | 2024-06-30 1.5.0
 16 | * Add Python 3.12 binaries.
 17 | 
 18 | * Update bundled LMDB to 0.9.31.
 19 | 
 20 | * Remove Python 2.7 support.
 21 | 
 22 | 2022-04-04 v1.4.1
 23 | * Update CI to build manylinux binaries.
 24 | 
 25 | 2022-12-06 v1.4.0
 26 | * Add Python 3.11 support.
 27 | 
 28 | 2021-12-30 v1.3.0
 29 | * Add aarch64 architecture builds.  Contributed by odidev.
 30 | 
 31 | * Add Python 3.10 support.
 32 | 
 33 | * Fix crash relating to caching of transactions.  The 'max_spare_txns'
 34 |   parameter to Environment/open is currently ignored in cpython.
 35 | 
 36 | 2021-04-19 v1.2.1
 37 | * Resolve CI bug where non-Linux wheels were not being published to PyPI.
 38 | 
 39 | 2021-04-15 v1.2.0
 40 | * Update bundled LMDB to 0.9.29.
 41 | 
 42 | * Add non-bundled testing to CI.
 43 | 
 44 | * Remove wheel generation for 2.7 because the manylinux images no longer
 45 |   support it.
 46 | 
 47 | * Allow passing None as a value to transaction.del in CFFI implementation
 48 |   for parity with cpython implementation.
 49 | 
 50 | * Fix Cursor.put behavior on a dupsort DB with append=True.
 51 | 
 52 | * Add warning to docs about use of Environment.set_mapsize.  This is currently
 53 | an unresolved issue with upstream LMDB.
 54 | 
 55 | * CFFI implementation:  fix a seg fault when open_db returns map full.
 56 | 
 57 | * CFFI implementation:  fix a bug in open_db in a read-only environment.
 58 | 
 59 | 
 60 | 2021-02-05 v1.1.1
 61 | * Dowgrade underlying LMDB to 0.9.26.  0.9.27 has a minor defect that will
 62 |   need to get resolved.
 63 | 
 64 | 
 65 | 2021-02-04 v1.1.0
 66 | * Migrate CI pipeline from Travis and AppVeyor to Github Actions.  Now
 67 |   includes comprehensive testing across 4 dimensions (OS, Python version,
 68 |   cpython/CFFI, pure/with mods).  Also includes publishing to PyPI.
 69 | 
 70 | * Prevent invalid flag combinations when creating a database.
 71 | 
 72 | * Add a Cursor.getmulti method with optional buffer support.  Contributed by
 73 |   Will Thompson <willsthompson@gmail.com>.
 74 | 
 75 | * Upgrade underlying LMDB to 0.9.27.
 76 | 
 77 | 
 78 | 2020-08-28 v1.0.0
 79 | * Start of new semantic versioning scheme.  This would be a minor version
 80 |   bump from the 0.99 release if it were semantically versioned.
 81 | 
 82 | * Allow environment copy to take a passed-in transaction.  This is the
 83 |   first released feature that requires a (very small) patch to the
 84 |   underlying C library.  By default, the patch will be applied unless
 85 |   this module is built with LMDB_PURE environment variable set.
 86 | 
 87 | 
 88 | 2020-08-13 v0.99
 89 | * Fix lmdb.tool encoding issues.
 90 | 
 91 | * Fix -l lmdb invocation issue.
 92 | 
 93 | * Minor documentation improvements.
 94 | 
 95 | * Update LMDB to version 0.9.24.
 96 | 
 97 | * Update for Python 3.9 (current release candidate) support.
 98 | 
 99 | * Resolve a bug when using cursor.putmulti and append=True on dupsort DBs.
100 | 
101 | * Allow _Database.flags method to take no arguments since the one argument
102 |   wasn't being used.
103 | 
104 | 
105 | 2019-11-06 v0.98
106 | * Fix that a duplicate argument to a lmdb method would cause an assert.
107 | 
108 | * Solaris needs ```#include "python.h"``` as soon as possible.  Fix
109 |   contributed by Jesús Cea.
110 | 
111 | * Fix crash under debug cpython when mdb_cursor_open failed
112 | 
113 | 
114 | 2019-08-11 v0.97
115 | 
116 | * Fix a missed GIL unlock sequence.  Reported by ajschorr.
117 | 
118 | * Fix argv check in JEP (cpython under Java) environment.  Contributed by
119 |   de-code.
120 | 
121 | 
122 | 2019-07-14 v0.96
123 | 
124 | * First release under new maintainer, Nic Watson.
125 | 
126 | * Doc updates.
127 | 
128 | * More removal of code for now-unsupported Python versions.
129 | 
130 | * Only preload the value with the GIL unlocked when the value is actually
131 |   requested. This significantly improves read performance to retrieve keys
132 |   with large values when the value isn't retrieved. Reported by Dan Patton.
133 | 
134 | 
135 | 2019-06-08 v0.95
136 | 
137 | * The minimum supported version of Python is now 2.7.
138 | 
139 | * The library is no longer tested on Python 3.2.
140 | 
141 | * The address-book.py example was updated for Python 3. Contributed by Jamie
142 |   Bliss.
143 | 
144 | * Development-related files were removed from the distribution tarball.
145 | 
146 | * Handling of the Environment(create=True) flag was improved. Fix contributed
147 |   by Nir Soffer.
148 | 
149 | * Database names may be reused after they are dropped on CFFI, without
150 |   reopening the environment. Fix contributed by Gareth Bult.
151 | 
152 | 
153 | 2018-04-09 v0.94
154 | 
155 | * CPython argument parsing now matches the behaviour of CFFI, and most sane
156 |   Python APIs: a bool parameter is considered to be true if it is any truthy
157 |   value, not just if it is exactly True. Reported by Nic Watson.
158 | 
159 | * Removed Python 2.6 support due to urllib3 warnings and pytest dropping it.
160 | 
161 | * Updared LMDB to version 0.9.22.
162 | 
163 | * Fixed several 2.7/3 bugs in command line tool.
164 | 
165 | 
166 | 2017-07-16 v0.93
167 | 
168 | * py-lmdb is now built with AppVeyor CI, providing early feedback on Windows
169 |   build health. Egg and wheel artifacts are being generated, removing the need
170 |   for a dedicated Windows build machine, however there is no mechanism to
171 |   paublish these to PyPI yet.
172 | 
173 | * The "warm" tool command did not function on Python 3.x. Reported by Github
174 |   user dev351.
175 | 
176 | * Tests now pass on non-4kb page-sized machines, such as ppc64le. Reported by
177 |   Jonathan J. Helmus.
178 | 
179 | * Windows 3.6 eggs and wheels are now available on PyPI, and tests are run
180 |   against 3.6. Reported by Ofek Lev.
181 | 
182 | * Python 3.2 is no longer supported, due to yet more pointless breakage
183 |   introduced in pip/pkg_resources.
184 | 
185 | * py-lmdb currently does not support LMDB >=0.9.19 due to interface changes in
186 |   LMDB. Support will appear in a future release.
187 | 
188 | 
189 | 2016-10-17 v0.92
190 | 
191 | * Changes to support __all__ caused the CPython module to fail to import at
192 |   runtime on Python 3. This was hidden during testing as the CFFI module was
193 |   successfully imported.
194 | 
195 | 
196 | 2016-10-17 v0.91
197 | 
198 | * The docstring for NotFoundError was clarified to indicate that it is
199 |   not raised in normal circumstances.
200 | 
201 | * CFFI open_db() would always attempt to use a write transaction, even if the
202 |   environment was opened with readonly=True. Now both CPython and CFFI will
203 |   use a read-only transaction in this case. Reported by Github user
204 |   handloomweaver.
205 | 
206 | * The source distribution previously did not include a LICENSE file, and may
207 |   have included random cached junk from the source tree during build. Reported
208 |   by Thomas Petazzoni.
209 | 
210 | * Transaction.id() was broken on Python 2.5.
211 | 
212 | * Repair Travis CI build again.
213 | 
214 | * CFFI Cursor did not correctly return empty strings for key()/value()/item()
215 |   when iternext()/iterprev() had reached the start/end of the database.
216 |   Detected by tests contributed by Ong Teck Wu.
217 | 
218 | * The package can now be imported from within a CPython subinterpreter. Fix
219 |   contributed by Vitaly Repin.
220 | 
221 | * lmdb.tool --delete would not delete keys in some circumstances. Fix
222 |   contributed by Vitaly Repin.
223 | 
224 | * Calls to Cursor.set_range_dup() could lead to memory corruption due to
225 |   Cursor's idea of the key and value failing to be updated correctly. Reported
226 |   by Michael Lazarev.
227 | 
228 | * The lmdb.tool copy command now supports a --compact flag. Contributed by
229 |   Achal Dave.
230 | 
231 | * The lmdb.tool edit command selects the correct database when --delete is
232 |   specified. Contributed by ispequalnp.
233 | 
234 | * lmdb.tool correctly supports the -r flag to select a read-only environment.
235 |   Contributed by ispequalnp.
236 | 
237 | * The lmdb.tool --txn_size parameter was removed, as it was never implemented,
238 |   and its original function is no longer necessary with modern LMDB. Reported
239 |   by Achal Dave.
240 | 
241 | * The documentation template was updated to fix broken links. Contributed by
242 |   Adam Chainz.
243 | 
244 | * The Travis CI build configuration was heavily refactored by Alexander Zhukov.
245 |   Automated tests are running under Travis CI once more.
246 | 
247 | * The CPython extension module did not define __all__. It is now defined
248 |   contain the same names as on CFFI.
249 | 
250 | * Both implementations were updated to remove lmdb.open() from __all__,
251 |   ensuring "from lmdb import *" does not shadow the builtin open(). The
252 |   function can still be invoked using its fully qualified name, and the alias
253 |   "Environment" may be used when "from lmdb import *" is used. Reported by
254 |   Alexander Zhukov.
255 | 
256 | * The CPython extension exported BadRSlotError, instead of BadRslotError. The
257 |   exception's name was corrected to match CFFI.
258 | 
259 | * Environment.open_db() now supports integerdup=True, dupfixed=True, and
260 |   integerkey=True flags. Based on a patch by Jonathan Heyman.
261 | 
262 | 
263 | 2016-07-11 v0.90
264 | 
265 | * This release was deleted from PyPI due to an erroneous pull request
266 |   upgrading the bundled LMDB to mdb.master.
267 | 
268 | 
269 | 2016-02-12 v0.89
270 | 
271 | * LMDB 0.9.18 is bundled.
272 | 
273 | * CPython Iterator.next() was incorrectly defined as pointing at the
274 |   implementation for Cursor.next(), triggering a crash if the method was ever
275 |   invoked manually. Reported by Kimikazu Kato.
276 | 
277 | 
278 | 2016-01-24 v0.88
279 | 
280 | * LMDB 0.9.17 is bundled.
281 | 
282 | * Transaction.id() is exposed.
283 | 
284 | * Binary wheels are built for Python 3.5 Windows 32/64-bit.
285 | 
286 | 
287 | 2015-08-11 v0.87
288 | 
289 | * Environment.set_mapsize() was added to allow runtime adjustment of the
290 |   environment map size.
291 | 
292 | * Remove non-determinism from setup.py, to support Debian's reproducible
293 |   builds project. Patch by Chris Lamb.
294 | 
295 | * Documentation correctness and typo fixes. Patch by Gustav Larsson.
296 | 
297 | * examples/keystore: beginnings of example that integrates py-lmdb with an
298 |   asynchronous IO loop.
299 | 
300 | 
301 | 2015-06-07 v0.86
302 | 
303 | * LMDB_FORCE_SYSTEM builds were broken by the GIL/page fault change. This
304 |   release fixes the problem.
305 | 
306 | * Various cosmetic fixes to documentation.
307 | 
308 | 
309 | 2015-06-06 v0.85
310 | 
311 | * New exception class: lmdb.BadDbiError.
312 | 
313 | * Environment.copy() and Environment.copyfd() now support compact=True, to
314 |   trigger database compaction while copying.
315 | 
316 | * Various small documentation updates.
317 | 
318 | * CPython set_range_dup() and set_key_dup() both invoked MDB_GET_BOTH, however
319 |   set_range_dup() should have instead invoked MDB_GET_BOTH_RANGE. Fix by
320 |   Matthew Battifarano.
321 | 
322 | * lmdb.tool module was broken on Win32, since Win32 lacks signal.SIGWINCH. Fix
323 |   suggested by David Khess.
324 | 
325 | * LMDB 0.9.14 is bundled along with extra fixes from mdb.RE/0.9 (release
326 |   engineering) branch.
327 | 
328 | * CPython previously lacked a Cursor.close() method. Problem was noticed by
329 |   Jos Vos.
330 | 
331 | * Several memory leaks affecting the CFFI implementation when running on
332 |   CPython were fixed, apparent only when repeatedly opening and discarding a
333 |   large number of environments. Noticed by Jos Vos.
334 | 
335 | * The CPython extension previously did not support weakrefs on Environment
336 |   objects, and the implementation for Transaction objects was flawed. The
337 |   extension now correctly invalidates weakrefs during deallocation.
338 | 
339 | * Both variants now try to avoid taking page faults with the GIL held,
340 |   accomplished by touching one byte of every page in a value during reads.
341 |   This does not guarantee faults will never occur with the GIL held, but it
342 |   drastically reduces the possibility. The binding should now be suitable for
343 |   use in multi-threaded applications with databases containing >2KB values
344 |   where the entire database does not fit in RAM.
345 | 
346 | 
347 | 2014-09-22 v0.84
348 | 
349 | * LMDB 0.9.14 is bundled.
350 | 
351 | * CFFI Cursor.putmulti() could crash when append=False and a key already
352 |   existed.
353 | 
354 | 
355 | 2014-06-24 v0.83
356 | 
357 | * LMDB 0.9.13 is bundled along with extra fixes from upstream Git.
358 | 
359 | * Environment.__enter__() and __exit__() are implemented, allowing
360 |   Environments to behave like context managers.
361 | 
362 | * Cursor.close(), __enter__() and __exit__() are implemented, allowing Cursors
363 |   to be explicitly closed. In CFFI this mechanism *must* be used when many
364 |   cursors are used within a single transaction, otherwise a resource leak will
365 |   occur.
366 | 
367 | * Dependency tracking in CFFI is now much faster, especially on PyPy, however
368 |   at a cost: Cursor use must always be wrapped in a context manager, or
369 |   .close() must be manually invoked for discarded Cursors when the parent
370 |   transaction is long lived.
371 | 
372 | * Fixed crash in CFFI Cursor.putmulti().
373 | 
374 | 
375 | 2014-05-26 v0.82
376 | 
377 | * Both variants now implement max_spare_txns, reducing the cost of creating a
378 |   read-only transaction 4x for an uncontended database and by up to 20x for
379 |   very read-busy environments. By default only 1 read-only transaction is
380 |   cached, adjust max_spare_txns= parameter if your script operates multiple
381 |   simultaneous read transactions.
382 | 
383 | * Patch from Vladimir Vladimirov implementing MDB_NOLOCK.
384 | 
385 | * The max_spare_iters and max_spare_cursors parameters were removed, neither
386 |   ever had any effect.
387 | 
388 | * Cursor.putmulti() implemented based on a patch from Luke Kenneth Casson
389 |   Leighton. This function moves the loop required to batch populate a
390 |   database out of Python and into C.
391 | 
392 | * The bundled LMDB 0.9.11 has been updated with several fixes from upstream
393 |   Git.
394 | 
395 | * The cost of using keyword arguments in the CPython extension was
396 |   significantly reduced.
397 | 
398 | 
399 | 2014-04-26 v0.81
400 | 
401 | * On Python 2.x the extension module would silently interpret Unicode
402 |   instances as buffer objects, causing UCS-2/UCS-4 string data to end up in
403 |   the database. This was never intentional and now raises TypeError. Any
404 |   Unicode data passed to py-lmdb must explicitly be encoded with .encode()
405 |   first.
406 | 
407 | * open_db()'s name argument was renamed to key, and its semantics now match
408 |   get() and put(): in other words the key must be a bytestring, and passing
409 |   Unicode will raise TypeError.
410 | 
411 | * The extension module now builds under Python 3.4 on Windows.
412 | 
413 | 
414 | 2014-04-21 v0.80
415 | 
416 | * Both variants now build successfully as 32 bit / 64bit binaries on
417 |   Windows under Visual Studio 9.0, the compiler for Python 2.7. This enables
418 |   py-lmdb to be installed via pip on Windows without requiring a compiler to
419 |   be available. In future, .egg/.whl releases will be pre-built for all recent
420 |   Python versions on Windows.
421 | 
422 |   Known bugs: Environment.copy() and Environment.copyfd() currently produce a
423 |   database that cannot be reopened.
424 | 
425 | * The lmdb.enable_drop_gil() function was removed. Its purpose was
426 |   experimental at best, confusing at worst.
427 | 
428 | 
429 | 2014-03-17 v0.79
430 | 
431 | * CPython Cursor.delete() lacked dupdata argument, fixed.
432 | 
433 | * Fixed minor bug where CFFI _get_cursor() did not note its idea of
434 |   the current key and value were up to date.
435 | 
436 | * Cursor.replace() and Cursor.pop() updated for MDB_DUPSORT databases. For
437 |   pop(), the first data item is popped and returned. For replace(), the first
438 |   data item is returned, and all duplicates for the key are replaced.
439 | 
440 | * Implement remaining Cursor methods necessary for working with MDB_DUPSORT
441 |   databases: next_dup(), next_nodup(), prev_dup(), prev_nodup(), first_dup(),
442 |   last_dup(), set_key_dup(), set_range_dup(), iternext_dup(),
443 |   iternext_nodup(), iterprev_dup(), iterprev_nodup().
444 | 
445 | * The default for Transaction.put(dupdata=...) and Cursor.put(dupdata=...) has
446 |   changed from False to True. The previous default did not reflect LMDB's
447 |   normal mode of operation.
448 | 
449 | * LMDB 0.9.11 is bundled along with extra fixes from upstream Git.
450 | 
451 | 
452 | 2014-01-18 v0.78
453 | 
454 | * Patch from bra-fsn to fix LMDB_LIBDIR.
455 | 
456 | * Various inaccurate documentation improvements.
457 | 
458 | * Initial work towards Windows/Microsoft Visual C++ 9.0 build.
459 | 
460 | * LMDB 0.9.11 is now bundled.
461 | 
462 | * To work around install failures minimum CFFI version is now >=0.8.0.
463 | 
464 | * ticket #38: remove all buffer object hacks. This results in ~50% slowdown
465 |   for cursor enumeration, but results in far simpler object lifetimes. A
466 |   future version may introduce a better mechanism for achieving the same
467 |   performance without loss of sanity.
468 | 
469 | 
470 | 2013-11-30 v0.77
471 | 
472 | * Added Environment.max_key_size(), Environment.max_readers().
473 | 
474 | * CFFI now raises the correct Error subclass associated with an MDB_* return
475 |   code.
476 | 
477 | * Numerous CFFI vs. CPython behavioural inconsistencies have been fixed.
478 | 
479 | * An endless variety of Unicode related 2.x/3.x/CPython/CFFI fixes were made.
480 | 
481 | * LMDB 0.9.10 is now bundled, along with some extra fixes from Git.
482 | 
483 | * Added Environment(meminit=...) option.
484 | 
485 | 
486 | 2013-10-28 v0.76
487 | 
488 | * Added support for Environment(..., readahead=False).
489 | 
490 | * LMDB 0.9.9 is now bundled.
491 | 
492 | * Many Python 2.5 and 3.x fixes were made. Future changes are automatically
493 |   tested via Travis CI <https://travis-ci.org/dw/py-lmdb>.
494 | 
495 | * When multiple cursors exist, and one cursor performs a mutation,
496 |   remaining cursors may have returned corrupt results via key(), value(),
497 |   or item(). Mutations are now explicitly tracked and cause the cursor's
498 |   data to be refreshed in this case.
499 | 
500 | * setup.py was adjusted to ensure the distutils default of '-DNDEBUG' is never
501 |   defined while building LMDB. This caused many important checks in the engine
502 |   to be disabled.
503 | 
504 | * The old 'transactionless' API was removed. A future version may support the
505 |   same API, but the implementation will be different.
506 | 
507 | * Transaction.pop() and Cursor.pop() helpers added, to complement
508 |   Transaction.replace() and Cursor.replace().
509 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The OpenLDAP Public License
 2 |   Version 2.8, 17 August 2003
 3 | 
 4 | Redistribution and use of this software and associated documentation
 5 | ("Software"), with or without modification, are permitted provided
 6 | that the following conditions are met:
 7 | 
 8 | 1. Redistributions in source form must retain copyright statements
 9 |    and notices,
10 | 
11 | 2. Redistributions in binary form must reproduce applicable copyright
12 |    statements and notices, this list of conditions, and the following
13 |    disclaimer in the documentation and/or other materials provided
14 |    with the distribution, and
15 | 
16 | 3. Redistributions must contain a verbatim copy of this document.
17 | 
18 | The OpenLDAP Foundation may revise this license from time to time.
19 | Each revision is distinguished by a version number.  You may use
20 | this Software under terms of this license revision or under the
21 | terms of any subsequent revision of the license.
22 | 
23 | THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS
24 | CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
25 | INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
26 | AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT
27 | SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S)
28 | OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
29 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
30 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
31 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
32 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
34 | ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 | POSSIBILITY OF SUCH DAMAGE.
36 | 
37 | The names of the authors and copyright holders must not be used in
38 | advertising or otherwise to promote the sale, use or other dealing
39 | in this Software without specific, written prior permission.  Title
40 | to copyright in this Software shall at all times remain with copyright
41 | holders.
42 | 
43 | OpenLDAP is a registered trademark of the OpenLDAP Foundation.
44 | 
45 | Copyright 1999-2003 The OpenLDAP Foundation, Redwood City,
46 | California, USA.  All Rights Reserved.  Permission to copy and
47 | distribute verbatim copies of this document is granted.
48 | 


--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
 1 | graft docs
 2 | graft examples
 3 | graft lib
 4 | graft lmdb
 5 | graft tests
 6 | include ChangeLog
 7 | include LICENSE
 8 | prune docs/_build
 9 | prune lmdb/__pycache__
10 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | This is a universal Python binding for the LMDB ‘Lightning’ Database.
 2 | 
 3 | See [the documentation](https://lmdb.readthedocs.io) for more information.
 4 | 
 5 | ### CI State
 6 | [![master](https://github.com/jnwatson/py-lmdb/workflows/Build,%20run,%20and%20test%20py-lmdb/badge.svg)](https://github.com/jnwatson/py-lmdb/actions/workflows/python-package.yml)
 7 | 
 8 | # Python Version Support Statement
 9 | 
10 | This project has been around for a while.  Previously, it supported all the 
11 | way back to before Python 2.5.  Currently, py-lmdb supports Python >= 3.5 
12 | and pypy.
13 | 
14 | The last version of py-lmdb that supported Python 2.7 was 1.4.1.
15 | 


--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
  1 | # Makefile for Sphinx documentation
  2 | #
  3 | 
  4 | # You can set these variables from the command line.
  5 | SPHINXOPTS    =
  6 | SPHINXBUILD   = sphinx-build
  7 | PAPER         =
  8 | BUILDDIR      = _build
  9 | 
 10 | # Internal variables.
 11 | PAPEROPT_a4     = -D latex_paper_size=a4
 12 | PAPEROPT_letter = -D latex_paper_size=letter
 13 | ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 14 | # the i18n builder cannot share the environment and doctrees with the others
 15 | I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 16 | 
 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
 18 | 
 19 | help:
 20 | 	@echo "Please use \`make <target>' where <target> is one of"
 21 | 	@echo "  html       to make standalone HTML files"
 22 | 	@echo "  dirhtml    to make HTML files named index.html in directories"
 23 | 	@echo "  singlehtml to make a single large HTML file"
 24 | 	@echo "  pickle     to make pickle files"
 25 | 	@echo "  json       to make JSON files"
 26 | 	@echo "  htmlhelp   to make HTML files and a HTML help project"
 27 | 	@echo "  qthelp     to make HTML files and a qthelp project"
 28 | 	@echo "  devhelp    to make HTML files and a Devhelp project"
 29 | 	@echo "  epub       to make an epub"
 30 | 	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
 31 | 	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
 32 | 	@echo "  text       to make text files"
 33 | 	@echo "  man        to make manual pages"
 34 | 	@echo "  texinfo    to make Texinfo files"
 35 | 	@echo "  info       to make Texinfo files and run them through makeinfo"
 36 | 	@echo "  gettext    to make PO message catalogs"
 37 | 	@echo "  changes    to make an overview of all changed/added/deprecated items"
 38 | 	@echo "  linkcheck  to check all external links for integrity"
 39 | 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
 40 | 
 41 | clean:
 42 | 	-rm -rf $(BUILDDIR)/*
 43 | 
 44 | html:
 45 | 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 46 | 	@echo
 47 | 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 48 | 
 49 | dirhtml:
 50 | 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 51 | 	@echo
 52 | 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
 53 | 
 54 | singlehtml:
 55 | 	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
 56 | 	@echo
 57 | 	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
 58 | 
 59 | pickle:
 60 | 	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
 61 | 	@echo
 62 | 	@echo "Build finished; now you can process the pickle files."
 63 | 
 64 | json:
 65 | 	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
 66 | 	@echo
 67 | 	@echo "Build finished; now you can process the JSON files."
 68 | 
 69 | htmlhelp:
 70 | 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
 71 | 	@echo
 72 | 	@echo "Build finished; now you can run HTML Help Workshop with the" \
 73 | 	      ".hhp project file in $(BUILDDIR)/htmlhelp."
 74 | 
 75 | qthelp:
 76 | 	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
 77 | 	@echo
 78 | 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
 79 | 	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
 80 | 	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/lmdb.qhcp"
 81 | 	@echo "To view the help file:"
 82 | 	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/lmdb.qhc"
 83 | 
 84 | devhelp:
 85 | 	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
 86 | 	@echo
 87 | 	@echo "Build finished."
 88 | 	@echo "To view the help file:"
 89 | 	@echo "# mkdir -p $$HOME/.local/share/devhelp/lmdb"
 90 | 	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/lmdb"
 91 | 	@echo "# devhelp"
 92 | 
 93 | epub:
 94 | 	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
 95 | 	@echo
 96 | 	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
 97 | 
 98 | latex:
 99 | 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
100 | 	@echo
101 | 	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
102 | 	@echo "Run \`make' in that directory to run these through (pdf)latex" \
103 | 	      "(use \`make latexpdf' here to do that automatically)."
104 | 
105 | latexpdf:
106 | 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
107 | 	@echo "Running LaTeX files through pdflatex..."
108 | 	$(MAKE) -C $(BUILDDIR)/latex all-pdf
109 | 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
110 | 
111 | text:
112 | 	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
113 | 	@echo
114 | 	@echo "Build finished. The text files are in $(BUILDDIR)/text."
115 | 
116 | man:
117 | 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
118 | 	@echo
119 | 	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
120 | 
121 | texinfo:
122 | 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
123 | 	@echo
124 | 	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
125 | 	@echo "Run \`make' in that directory to run these through makeinfo" \
126 | 	      "(use \`make info' here to do that automatically)."
127 | 
128 | info:
129 | 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
130 | 	@echo "Running Texinfo files through makeinfo..."
131 | 	make -C $(BUILDDIR)/texinfo info
132 | 	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
133 | 
134 | gettext:
135 | 	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
136 | 	@echo
137 | 	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
138 | 
139 | changes:
140 | 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
141 | 	@echo
142 | 	@echo "The overview file is in $(BUILDDIR)/changes."
143 | 
144 | linkcheck:
145 | 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
146 | 	@echo
147 | 	@echo "Link check complete; look for any errors in the above output " \
148 | 	      "or in $(BUILDDIR)/linkcheck/output.txt."
149 | 
150 | doctest:
151 | 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
152 | 	@echo "Testing of doctests in the sources finished, look at the " \
153 | 	      "results in $(BUILDDIR)/doctest/output.txt."
154 | 


--------------------------------------------------------------------------------
/docs/conf.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | #
  3 | # lmdb documentation build configuration file, created by
  4 | # sphinx-quickstart on Tue Feb  5 00:39:26 2013.
  5 | #
  6 | # This file is execfile()d with the current directory set to its containing dir.
  7 | #
  8 | # Note that not all possible configuration values are present in this
  9 | # autogenerated file.
 10 | #
 11 | # All configuration values have a default; values that are commented out
 12 | # serve to show the default.
 13 | 
 14 | import sys, os
 15 | 
 16 | # If extensions (or modules to document with autodoc) are in another directory,
 17 | # add these directories to sys.path here. If the directory is relative to the
 18 | # documentation root, use os.path.abspath to make it absolute, like shown here.
 19 | sys.path.insert(0, os.path.abspath('..'))
 20 | 
 21 | # -- General configuration -----------------------------------------------------
 22 | 
 23 | # If your documentation needs a minimal Sphinx version, state it here.
 24 | #needs_sphinx = '1.0'
 25 | 
 26 | # Add any Sphinx extension module names here, as strings. They can be extensions
 27 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 28 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx']
 29 | 
 30 | intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None)}
 31 | 
 32 | # Add any paths that contain templates here, relative to this directory.
 33 | # templates_path = ['_templates']
 34 | 
 35 | # The suffix of source filenames.
 36 | source_suffix = '.rst'
 37 | 
 38 | # The encoding of source files.
 39 | #source_encoding = 'utf-8-sig'
 40 | 
 41 | # The master toctree document.
 42 | master_doc = 'index'
 43 | 
 44 | # General information about the project.
 45 | project = u'lmdb'
 46 | copyright = u'2013-2019 David Wilson, 2019-2022 Nic Watson'
 47 | 
 48 | # The version info for the project you're documenting, acts as replacement for
 49 | # |version| and |release|, also used in various other places throughout the
 50 | # built documents.
 51 | #
 52 | 
 53 | def grep_version():
 54 |     path = os.path.join(os.path.dirname(__file__), '../lmdb/__init__.py')
 55 |     with open(path) as fp:
 56 |         for line in fp:
 57 |             if line.startswith('__version__'):
 58 |                 return eval(line.split()[-1])
 59 | 
 60 | # The short X.Y version.
 61 | version = grep_version()
 62 | # The full version, including alpha/beta/rc tags.
 63 | release = version
 64 | 
 65 | # The language for content autogenerated by Sphinx. Refer to documentation
 66 | # for a list of supported languages.
 67 | #language = None
 68 | 
 69 | # There are two options for replacing |today|: either, you set today to some
 70 | # non-false value, then it is used:
 71 | #today = ''
 72 | # Else, today_fmt is used as the format for a strftime call.
 73 | #today_fmt = '%B %d, %Y'
 74 | 
 75 | # List of patterns, relative to source directory, that match files and
 76 | # directories to ignore when looking for source files.
 77 | exclude_patterns = ['_build']
 78 | 
 79 | # The reST default role (used for this markup: `text`) to use for all documents.
 80 | #default_role = None
 81 | 
 82 | # If true, '()' will be appended to :func: etc. cross-reference text.
 83 | #add_function_parentheses = True
 84 | 
 85 | # If true, the current module name will be prepended to all description
 86 | # unit titles (such as .. function::).
 87 | #add_module_names = True
 88 | 
 89 | # If true, sectionauthor and moduleauthor directives will be shown in the
 90 | # output. They are ignored by default.
 91 | #show_authors = False
 92 | 
 93 | # The name of the Pygments (syntax highlighting) style to use.
 94 | pygments_style = 'sphinx'
 95 | 
 96 | # A list of ignored prefixes for module index sorting.
 97 | #modindex_common_prefix = []
 98 | 
 99 | 
100 | # -- Options for HTML output ---------------------------------------------------
101 | 
102 | # The theme to use for HTML and HTML Help pages.  See the documentation for
103 | # a list of builtin themes.
104 | html_theme = 'acid'
105 | 
106 | # Theme options are theme-specific and customize the look and feel of a theme
107 | # further.  For a list of options available for each theme, see the
108 | # documentation.
109 | html_theme_options = {
110 |     'github_repo': 'https://github.com/jnwatson/py-lmdb/'
111 | }
112 | 
113 | # Add any paths that contain custom themes here, relative to this directory.
114 | html_theme_path = ['themes']
115 | 
116 | # The name for this set of Sphinx documents.  If None, it defaults to
117 | # "<project> v<release> documentation".
118 | #html_title = None
119 | 
120 | # A shorter title for the navigation bar.  Default is the same as html_title.
121 | #html_short_title = None
122 | 
123 | # The name of an image file (relative to this directory) to place at the top
124 | # of the sidebar.
125 | #html_logo = None
126 | 
127 | # The name of an image file (within the static path) to use as favicon of the
128 | # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
129 | # pixels large.
130 | #html_favicon = None
131 | 
132 | # Add any paths that contain custom static files (such as style sheets) here,
133 | # relative to this directory. They are copied after the builtin static files,
134 | # so a file named "default.css" will overwrite the builtin "default.css".
135 | html_static_path = ['_static']
136 | 
137 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
138 | # using the given strftime format.
139 | #html_last_updated_fmt = '%b %d, %Y'
140 | 
141 | # If true, SmartyPants will be used to convert quotes and dashes to
142 | # typographically correct entities.
143 | #html_use_smartypants = True
144 | 
145 | # Custom sidebar templates, maps document names to template names.
146 | html_sidebars = {}
147 | 
148 | # Additional templates that should be rendered to pages, maps page names to
149 | # template names.
150 | #html_additional_pages = {}
151 | 
152 | # If false, no module index is generated.
153 | #html_domain_indices = True
154 | 
155 | # If false, no index is generated.
156 | html_use_index = False
157 | 
158 | # If true, the index is split into individual pages for each letter.
159 | #html_split_index = False
160 | 
161 | # If true, links to the reST sources are added to the pages.
162 | html_show_sourcelink = False
163 | 
164 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
165 | html_show_sphinx = False
166 | 
167 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
168 | #html_show_copyright = True
169 | 
170 | # If true, an OpenSearch description file will be output, and all pages will
171 | # contain a <link> tag referring to it.  The value of this option must be the
172 | # base URL from which the finished HTML is served.
173 | #html_use_opensearch = ''
174 | 
175 | # This is the file name suffix for HTML files (e.g. ".xhtml").
176 | #html_file_suffix = None
177 | 
178 | # Output file base name for HTML help builder.
179 | htmlhelp_basename = 'lmdbdoc'
180 | 
181 | 
182 | # -- Options for LaTeX output --------------------------------------------------
183 | 
184 | latex_elements = {
185 | # The paper size ('letterpaper' or 'a4paper').
186 | #'papersize': 'letterpaper',
187 | 
188 | # The font size ('10pt', '11pt' or '12pt').
189 | #'pointsize': '10pt',
190 | 
191 | # Additional stuff for the LaTeX preamble.
192 | #'preamble': '',
193 | }
194 | 
195 | # Grouping the document tree into LaTeX files. List of tuples
196 | # (source start file, target name, title, author, documentclass [howto/manual]).
197 | latex_documents = [
198 |   ('index', 'lmdb.tex', u'lmdb Documentation',
199 |    u'David Wilson and Nic Watson', 'manual'),
200 | ]
201 | 
202 | # The name of an image file (relative to this directory) to place at the top of
203 | # the title page.
204 | #latex_logo = None
205 | 
206 | # For "manual" documents, if this is true, then toplevel headings are parts,
207 | # not chapters.
208 | #latex_use_parts = False
209 | 
210 | # If true, show page references after internal links.
211 | #latex_show_pagerefs = False
212 | 
213 | # If true, show URL addresses after external links.
214 | #latex_show_urls = False
215 | 
216 | # Documents to append as an appendix to all manuals.
217 | #latex_appendices = []
218 | 
219 | # If false, no module index is generated.
220 | #latex_domain_indices = True
221 | 
222 | 
223 | # -- Options for manual page output --------------------------------------------
224 | 
225 | # One entry per manual page. List of tuples
226 | # (source start file, name, description, authors, manual section).
227 | man_pages = [
228 |     ('index', 'lmdb', u'lmdb Documentation',
229 |      [u'David Wilson'], 1)
230 | ]
231 | 
232 | # If true, show URL addresses after external links.
233 | #man_show_urls = False
234 | 
235 | 
236 | # -- Options for Texinfo output ------------------------------------------------
237 | 
238 | # Grouping the document tree into Texinfo files. List of tuples
239 | # (source start file, target name, title, author,
240 | #  dir menu entry, description, category)
241 | texinfo_documents = [
242 |   ('index', 'lmdb', u'lmdb Documentation',
243 |    u'David Wilson', 'lmdb', 'One line description of project.',
244 |    'Miscellaneous'),
245 | ]
246 | 
247 | # Documents to append as an appendix to all manuals.
248 | #texinfo_appendices = []
249 | 
250 | # If false, no module index is generated.
251 | #texinfo_domain_indices = True
252 | 
253 | # How to display URL addresses: 'footnote', 'no', or 'inline'.
254 | #texinfo_show_urls = 'footnote'
255 | 
256 | 
257 | 
258 | import sys
259 | 
260 | class Mock(object):
261 |     def __init__(self, *args, **kwargs):
262 |         pass
263 | 
264 |     def __call__(self, *args, **kwargs):
265 |         return Mock()
266 | 
267 |     @classmethod
268 |     def __getattr__(cls, name):
269 |         if name in ('__file__', '__path__'):
270 |             return '/dev/null'
271 |         elif 0 and name[0] == name[0].upper():
272 |             mockType = type(name, (), {})
273 |             mockType.__module__ = __name__
274 |             return mockType
275 |         else:
276 |             return Mock()
277 | 
278 | MOCK_MODULES = ['cffi']
279 | for mod_name in MOCK_MODULES:
280 |     sys.modules[mod_name] = Mock()
281 | 


--------------------------------------------------------------------------------
/docs/themes/acid/conf.py.conf:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | 
 3 | import sys, os
 4 | extensions = []
 5 | templates_path = ['{{ template_dir }}', 'templates', '_templates', '.templates']
 6 | source_suffix = '{{ project.suffix }}'
 7 | master_doc = 'index'
 8 | project = u'{{ project.name }}'
 9 | copyright = u'{{ project.copyright }}'
10 | version = '{{ project.version }}'
11 | release = '{{ project.version }}'
12 | exclude_patterns = ['_build']
13 | pygments_style = 'sphinx'
14 | html_theme = '{{ project.theme }}'
15 | html_theme_path = ['.', '_theme', '.theme']
16 | htmlhelp_basename = '{{ project.slug }}'
17 | file_insertion_enabled = False
18 | latex_documents = [
19 |   ('index', '{{ project.slug }}.tex', u'{{ project.name }} Documentation',
20 |    u'{{ project.copyright }}', 'manual'),
21 | ]
22 | 


--------------------------------------------------------------------------------
/docs/themes/acid/layout.html:
--------------------------------------------------------------------------------
  1 | <!doctype html>
  2 | <meta http-equiv="Content-Type" content="text/html; charset={{ encoding }}">
  3 | 
  4 | {# Strip out the default script files we provide, but keep some, like mathjax. #}
  5 | {% set script_files = [] %}
  6 | {%- set url_root = pathto('', 1) %}
  7 | {%- if url_root == '#' %}{% set url_root = '' %}{% endif %}
  8 | {%- set titlesuffix = " &mdash; "|safe + docstitle|e %}
  9 | 
 10 | {%- macro sidebar() %}
 11 | <div class="sphinxsidebar">
 12 | <div class="sphinxsidebarwrapper" style="padding-top: 1em;">
 13 |     {%- if logo %}
 14 |       <p class="logo"><a href="{{ pathto(master_doc) }}">
 15 |         <img class="logo" src="{{ pathto('_static/' + logo, 1) }}" alt="">
 16 |       </a></p>
 17 |     {%- endif %}
 18 | 
 19 |     {% if theme_github_repo %}
 20 |     <p>
 21 |     <a href="{{ theme_github_repo }}">GitHub Repository</a>
 22 |     </p>
 23 |     {% endif %}
 24 | 
 25 |     <p>
 26 |     <a href="/en/release/">Docs for last release</a>
 27 |     </p>
 28 | 
 29 |     <p>
 30 |     <a href="/en/latest/">Docs for master branch</a>
 31 |     </p>
 32 | 
 33 |     {%- block sidebartoc %}
 34 |     {%- include "localtoc.html" %}
 35 |     {%- endblock %}
 36 |     {%- block sidebarrel %}
 37 |     {%- include "relations.html" %}
 38 |     {%- endblock %}
 39 | </div>
 40 | </div>
 41 | {%- endmacro %}
 42 | 
 43 | <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
 44 | <link rel="stylesheet" href="{{ pathto('_static/' + style, 1) }}">
 45 | <link rel="stylesheet" href="{{ pathto('_static/pygments.css', 1) }}">
 46 | {%- for cssfile in css_files %}
 47 |     <link rel="stylesheet" href="{{ pathto(cssfile, 1) }}">
 48 | {%- endfor %}
 49 | 
 50 | <script>
 51 | var DOCUMENTATION_OPTIONS = {
 52 |     URL_ROOT:    '{{ url_root }}',
 53 |     VERSION:     '{{ release|e }}',
 54 |     COLLAPSE_INDEX: false,
 55 |     FILE_SUFFIX: '{{ '' if no_search_suffix else file_suffix }}',
 56 |     HAS_SOURCE:  {{ has_source|lower }}
 57 | };
 58 | var doc_version = "{{ current_version }}";
 59 | var doc_slug = "{{ slug }}";
 60 | var page_name = "{{ pagename }}";
 61 | </script>
 62 | 
 63 | {%- for scriptfile in script_files %}
 64 | <script src="{{ pathto(scriptfile, 1) }}"></script>
 65 | {%- endfor %}
 66 | 
 67 | 
 68 | {%- block content %}
 69 |   {%- block sidebar1 %} {# possible location for sidebar #} {% endblock %}
 70 | 
 71 |     <div class="document">
 72 |   {%- block document %}
 73 |       <div class="documentwrapper">
 74 |         <div class="bodywrapper">
 75 |           <div class="body">
 76 |             {% block body %} {% endblock %}
 77 |           </div>
 78 |         </div>
 79 |       </div>
 80 |   {%- endblock %}
 81 | 
 82 |   {%- block sidebar2 %}{{ sidebar() }}{% endblock %}
 83 |       <div class="clearer"></div>
 84 |     </div>
 85 | {%- endblock %}
 86 | 
 87 | {% macro relbar() %}{% endmacro %}
 88 | {%- block relbar2 %}{{ relbar() }}{% endblock %}
 89 | 
 90 | 
 91 | <div class="footer">
 92 | {%- if show_copyright %}
 93 |     {%- if hasdoc('copyright') %}
 94 |     {% trans path=pathto('copyright'), copyright=copyright|e %}&copy; <a href="{{ path }}">Copyright</a> {{ copyright }}.{% endtrans %}
 95 |   {%- else %}
 96 |     {% trans copyright=copyright|e %}&copy; Copyright {{ copyright }}.{% endtrans %}
 97 |   {%- endif %}
 98 | {%- endif %}
 99 | {%- if last_updated %}
100 |   {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %}
101 | {%- endif %}
102 | {%- if show_sphinx %}
103 |   {% trans sphinx_version=sphinx_version|e %}Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> {{ sphinx_version }}.{% endtrans %}
104 | {%- endif %}
105 | </div>
106 | 
107 | {% if not using_theme %}
108 |     {# Keep this here, so that the RTD logo doesn't stomp on the bottom of the theme. #}
109 |     <br/>
110 |     <br/>
111 |     <br/>
112 | {% endif %}
113 | 
114 | 
115 | <script>
116 |   var _paq = _paq || [];
117 |   _paq.push(['trackPageView']);
118 |   _paq.push(['enableLinkTracking']);
119 |   (function() {
120 |     var u="https://k1.botanicus.net/tr/";
121 |     _paq.push(['enableHeartBeatTimer', 30]);
122 |     _paq.push(['setTrackerUrl', u+'ep']);
123 |     _paq.push(['setSiteId', 2]);
124 |     var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.type='text/javascript';
125 |     g.defer=true; g.async=true; g.src=u+'js'; s.parentNode.insertBefore(g,s);
126 |   })();
127 | </script>
128 | <noscript><p><img src="https://k1.botanicus.net/tr/ep?idsite=2" style="border:0" alt=""></p></noscript>
129 | 


--------------------------------------------------------------------------------
/docs/themes/acid/sourcelink.html:
--------------------------------------------------------------------------------
1 | {% if 0 %}{% endif %}
2 | 


--------------------------------------------------------------------------------
/docs/themes/acid/theme.conf:
--------------------------------------------------------------------------------
 1 | [theme]
 2 | inherit = basic
 3 | stylesheet = acid.css
 4 | pygments_style = sphinx
 5 | 
 6 | [options]
 7 | github_repo =
 8 | 
 9 | rightsidebar = false
10 | stickysidebar = false
11 | collapsiblesidebar = false
12 | externalrefs = false
13 | 
14 | footerbgcolor    = #11303d
15 | footertextcolor  = #ffffff
16 | sidebarbgcolor   = #1c4e63
17 | sidebarbtncolor  = #3c6e83
18 | sidebartextcolor = #ffffff
19 | sidebarlinkcolor = #98dbcc
20 | relbarbgcolor    = #133f52
21 | relbartextcolor  = #ffffff
22 | relbarlinkcolor  = #ffffff
23 | bgcolor          = #ffffff
24 | textcolor        = #000000
25 | headbgcolor      = #f2f2f2
26 | headtextcolor    = #20435c
27 | headlinkcolor    = #c60f0f
28 | linkcolor        = #355f7c
29 | visitedlinkcolor = #355f7c
30 | codebgcolor      = #eeffcc
31 | codetextcolor    = #333333
32 | 
33 | bodyfont = sans-serif
34 | headfont = 'Trebuchet MS', sans-serif
35 | 


--------------------------------------------------------------------------------
/examples/address-book.py:
--------------------------------------------------------------------------------
 1 | 
 2 | import lmdb
 3 | 
 4 | # Open (and create if necessary) our database environment. Must specify
 5 | # max_dbs=... since we're opening subdbs.
 6 | env = lmdb.open('/tmp/address-book.lmdb', max_dbs=10)
 7 | 
 8 | # Now create subdbs for home and business addresses.
 9 | home_db = env.open_db(b'home')
10 | business_db = env.open_db(b'business')
11 | 
12 | 
13 | # Add some telephone numbers to each DB:
14 | with env.begin(write=True) as txn:
15 |     txn.put(b'mum', b'012345678', db=home_db)
16 |     txn.put(b'dad', b'011232211', db=home_db)
17 |     txn.put(b'dentist', b'044415121', db=home_db)
18 |     txn.put(b'hospital', b'078126321', db=home_db)
19 | 
20 |     txn.put(b'vendor', b'0917465628', db=business_db)
21 |     txn.put(b'customer', b'0553211232', db=business_db)
22 |     txn.put(b'coworker', b'0147652935', db=business_db)
23 |     txn.put(b'boss', b'0123151232', db=business_db)
24 |     txn.put(b'manager', b'0644810485', db=business_db)
25 | 
26 | 
27 | # Iterate each DB to show the keys are sorted:
28 | with env.begin() as txn:
29 |     for name, db in ('home', home_db), ('business', business_db):
30 |         print('DB:', name)
31 |         for key, value in txn.cursor(db=db):
32 |             print('  ', key, value)
33 |         print()
34 | 
35 | 
36 | # Now let's update some phone numbers. We can specify the default subdb when
37 | # starting the transaction, rather than pass it in every time:
38 | with env.begin(write=True, db=home_db) as txn:
39 |     print('Updating number for dentist')
40 |     txn.put(b'dentist', b'099991231')
41 | 
42 |     print('Deleting number for hospital')
43 |     txn.delete(b'hospital')
44 |     print()
45 | 
46 |     print('Home DB is now:')
47 |     for key, value in txn.cursor():
48 |         print('  ', key, value)
49 |     print()
50 | 
51 | 
52 | # Now let's look up a number in the business DB
53 | with env.begin(db=business_db) as txn:
54 |     print('Boss telephone number:', txn.get(b'boss'))
55 |     print()
56 | 
57 | 
58 | # We got fired, time to delete all keys from the business DB.
59 | with env.begin(write=True) as txn:
60 |     print('Deleting all numbers from business DB:')
61 |     txn.drop(business_db, delete=False)
62 | 
63 |     print('Adding number for recruiter to business DB')
64 |     txn.put(b'recruiter', b'04123125324', db=business_db)
65 | 
66 |     print('Business DB is now:')
67 |     for key, value in txn.cursor(db=business_db):
68 |         print('  ', key, value)
69 |     print()
70 | 


--------------------------------------------------------------------------------
/examples/dirtybench-gdbm.py:
--------------------------------------------------------------------------------
  1 | 
  2 | from pprint import pprint
  3 | import os
  4 | import shutil
  5 | import tempfile
  6 | 
  7 | from time import time as now
  8 | import random
  9 | import gdbm
 10 | 
 11 | MAP_SIZE = 1048576 * 400
 12 | DB_PATH = '/ram/testdb-gdbm'
 13 | 
 14 | if os.path.exists('/ram'):
 15 |     DB_PATH = '/ram/testdb-gdbm'
 16 | else:
 17 |     DB_PATH = tempfile.mktemp(prefix='dirtybench-gdbm')
 18 | 
 19 | 
 20 | def x():
 21 |     big = '' # '*' * 400
 22 | 
 23 |     if os.path.exists(DB_PATH):
 24 |         os.unlink(DB_PATH)
 25 | 
 26 |     t0 = now()
 27 |     words = set(file('/usr/share/dict/words').readlines())
 28 |     words.update([w.upper() for w in words])
 29 |     words.update([w[::-1] for w in words])
 30 |     words.update([w[::-1].upper() for w in words])
 31 |     words.update(['-'.join(w) for w in words])
 32 |     #words.update(['+'.join(w) for w in words])
 33 |     #words.update(['/'.join(w) for w in words])
 34 |     words = list(words)
 35 |     alllen = sum(len(w) for w in words)
 36 |     avglen = alllen  / len(words)
 37 |     print 'permutate %d words avglen %d took %.2fsec' % (len(words), avglen, now()-t0)
 38 | 
 39 |     getword = iter(words).next
 40 | 
 41 |     env = gdbm.open(DB_PATH, 'c')
 42 | 
 43 |     run = True
 44 |     t0 = now()
 45 |     last = t0
 46 |     while run:
 47 |         try:
 48 |             for _ in xrange(50000):
 49 |                 word = getword()
 50 |                 env[word] = big or word
 51 |         except StopIteration:
 52 |             run = False
 53 | 
 54 |         t1 = now()
 55 |         if (t1 - last) > 2:
 56 |             print '%.2fs (%d/sec)' % (t1-t0, len(words)/(t1-t0))
 57 |             last = t1
 58 | 
 59 |     t1 = now()
 60 |     print 'done all %d in %.2fs (%d/sec)' % (len(words), t1-t0, len(words)/(t1-t0))
 61 |     last = t1
 62 | 
 63 |     print
 64 |     print
 65 | 
 66 |     t0 = now()
 67 |     lst = sum(env[k] and 1 for k in env.keys())
 68 |     t1 = now()
 69 |     print 'enum %d (key, value) pairs took %.2f sec' % ((lst), t1-t0)
 70 | 
 71 |     t0 = now()
 72 |     lst = sum(1 or env[k] for k in reversed(env.keys()))
 73 |     t1 = now()
 74 |     print 'reverse enum %d (key, value) pairs took %.2f sec' % ((lst), t1-t0)
 75 | 
 76 |     t0 = now()
 77 |     for word in words:
 78 |         env[word]
 79 |     t1 = now()
 80 |     print 'rand lookup all keys %.2f sec (%d/sec)' % (t1-t0, lst/(t1-t0))
 81 | 
 82 |     t0 = now()
 83 |     for word in words:
 84 |         hash(env[word])
 85 |     t1 = now()
 86 |     print 'per txn rand lookup+hash all keys %.2f sec (%d/sec)' % (t1-t0, lst/(t1-t0))
 87 | 
 88 |     t0 = now()
 89 |     for word in words:
 90 |         hash(env[word])
 91 |     t1 = now()
 92 |     print 'rand lookup+hash all keys %.2f sec (%d/sec)' % (t1-t0, lst/(t1-t0))
 93 | 
 94 |     t0 = now()
 95 |     for word in words:
 96 |         env[word]
 97 |     t1 = now()
 98 |     print 'rand lookup all buffers %.2f sec (%d/sec)' % (t1-t0, lst/(t1-t0))
 99 | 
100 |     t0 = now()
101 |     for word in words:
102 |         hash(env[word])
103 |     t1 = now()
104 |     print 'rand lookup+hash all buffers %.2f sec (%d/sec)' % (t1-t0, lst/(t1-t0))
105 | 
106 | 
107 |     #
108 |     # get+put
109 |     #
110 | 
111 |     getword = iter(sorted(words)).next
112 |     run = True
113 |     t0 = now()
114 |     last = t0
115 |     while run:
116 |         try:
117 |             for _ in xrange(50000):
118 |                 word = getword()
119 |                 old = env[word]
120 |                 env[word] = word
121 |         except StopIteration:
122 |             run = False
123 | 
124 |         t1 = now()
125 |         if (t1 - last) > 2:
126 |             print '%.2fs (%d/sec)' % (t1-t0, len(words)/(t1-t0))
127 |             last = t1
128 | 
129 |     t1 = now()
130 |     print 'get+put all %d in %.2fs (%d/sec)' % (len(words), t1-t0, len(words)/(t1-t0))
131 |     last = t1
132 | 
133 | 
134 |     #
135 |     # REPLACE
136 |     #
137 | 
138 |     getword = iter(sorted(words)).next
139 |     run = True
140 |     t0 = now()
141 |     last = t0
142 |     while run:
143 |         try:
144 |             for _ in xrange(50000):
145 |                 word = getword()
146 |                 old = env[word]
147 |         except StopIteration:
148 |             run = False
149 | 
150 |         t1 = now()
151 |         if (t1 - last) > 2:
152 |             print '%.2fs (%d/sec)' % (t1-t0, len(words)/(t1-t0))
153 |             last = t1
154 | 
155 |     t1 = now()
156 |     print 'replace all %d in %.2fs (%d/sec)' % (len(words), t1-t0, len(words)/(t1-t0))
157 |     last = t1
158 | 
159 | 
160 | 
161 | 
162 | x()
163 | 


--------------------------------------------------------------------------------
/examples/dirtybench.py:
--------------------------------------------------------------------------------
  1 | 
  2 | from pprint import pprint
  3 | import atexit
  4 | import gzip
  5 | import itertools
  6 | import os
  7 | import shutil
  8 | import sys
  9 | import tempfile
 10 | 
 11 | from time import time as now
 12 | import random
 13 | import lmdb
 14 | 
 15 | MAP_SIZE = 1048576 * 400
 16 | DB_PATH = '/ram/testdb'
 17 | USE_SPARSE_FILES = sys.platform != 'darwin'
 18 | 
 19 | if os.path.exists('/ram'):
 20 |     DB_PATH = '/ram/testdb'
 21 | else:
 22 |     DB_PATH = tempfile.mktemp(prefix='dirtybench')
 23 | 
 24 | 
 25 | env = None
 26 | @atexit.register
 27 | def cleanup():
 28 |     if env:
 29 |         env.close()
 30 |     if os.path.exists(DB_PATH):
 31 |         shutil.rmtree(DB_PATH)
 32 | 
 33 | 
 34 | def reopen_env(**kwargs):
 35 |     if env:
 36 |         env.close()
 37 |     if os.path.exists(DB_PATH):
 38 |         shutil.rmtree(DB_PATH)
 39 |     return lmdb.open(DB_PATH, map_size=MAP_SIZE, writemap=USE_SPARSE_FILES, **kwargs)
 40 | 
 41 | 
 42 | def case(title, **params):
 43 |     def wrapper(func):
 44 |         t0 = now()
 45 |         count = func()
 46 |         t1 = now()
 47 |         print('%40s:  %2.3fs   %8d/sec' % (title, t1-t0, count/(t1-t0)))
 48 |         return func
 49 |     return wrapper
 50 | 
 51 | 
 52 | def x():
 53 |     big = '' # '*' * 400
 54 | 
 55 |     t0 = now()
 56 |     words_path = os.path.join(os.path.dirname(__file__), 'words.gz')
 57 |     words = set(gzip.open(words_path).read().splitlines())
 58 |     words.update([w.upper() for w in words])
 59 |     words.update([w[::-1] for w in words])
 60 |     words.update([w[::-1].upper() for w in words])
 61 |     words.update(['-'.join(w) for w in words])
 62 |     #words.update(['+'.join(w) for w in words])
 63 |     #words.update(['/'.join(w) for w in words])
 64 |     words = list(words)
 65 |     alllen = sum(len(w) for w in words)
 66 |     avglen = alllen  / len(words)
 67 |     print 'permutate %d words avglen %d took %.2fsec' % (len(words), avglen, now()-t0)
 68 |     print 'DB_PATH:', DB_PATH
 69 | 
 70 |     words_sorted = sorted(words)
 71 |     items = [(w, big or w) for w in words]
 72 |     items_sorted = [(w, big or w) for w in words_sorted]
 73 | 
 74 |     global env
 75 |     env = reopen_env()
 76 | 
 77 |     @case('insert')
 78 |     def test():
 79 |         with env.begin(write=True) as txn:
 80 |             for word in words:
 81 |                 txn.put(word, big or word)
 82 |             return len(words)
 83 | 
 84 | 
 85 |     st = env.stat()
 86 |     print
 87 |     print 'stat:', st
 88 |     print 'k+v size %.2fkb avg %d, on-disk size: %.2fkb avg %d' %\
 89 |         ((2*alllen) / 1024., (2*alllen)/len(words),
 90 |          (st['psize'] * st['leaf_pages']) / 1024.,
 91 |          (st['psize'] * st['leaf_pages']) / len(words))
 92 |     print
 93 | 
 94 | 
 95 |     @case('enum (key, value) pairs')
 96 |     def test():
 97 |         with env.begin() as txn:
 98 |             return sum(1 for _ in txn.cursor())
 99 | 
100 | 
101 |     @case('reverse enum (key, value) pairs')
102 |     def test():
103 |         with env.begin() as txn:
104 |             return sum(1 for _ in txn.cursor().iterprev())
105 | 
106 | 
107 |     @case('enum (key, value) buffers')
108 |     def test():
109 |         with env.begin(buffers=True) as txn:
110 |             return sum(1 for _ in txn.cursor())
111 | 
112 | 
113 |     print
114 | 
115 | 
116 |     @case('rand lookup')
117 |     def test():
118 |         with env.begin() as txn:
119 |             for word in words:
120 |                 txn.get(word)
121 |         return len(words)
122 | 
123 | 
124 |     @case('per txn rand lookup')
125 |     def test():
126 |         for word in words:
127 |             with env.begin() as txn:
128 |                 txn.get(word)
129 |         return len(words)
130 | 
131 | 
132 |     @case('rand lookup+hash')
133 |     def test():
134 |         with env.begin() as txn:
135 |             for word in words:
136 |                 hash(txn.get(word))
137 |         return len(words)
138 | 
139 | 
140 |     @case('rand lookup buffers')
141 |     def test():
142 |         with env.begin(buffers=True) as txn:
143 |             for word in words:
144 |                 txn.get(word)
145 |         return len(words)
146 | 
147 | 
148 |     @case('rand lookup+hash buffers')
149 |     def test():
150 |         with env.begin(buffers=True) as txn:
151 |             for word in words:
152 |                 hash(txn.get(word))
153 |         return len(words)
154 | 
155 | 
156 |     @case('rand lookup buffers (cursor)')
157 |     def test():
158 |         with env.begin(buffers=True) as txn:
159 |             cursget = txn.cursor().get
160 |             for word in words:
161 |                 cursget(word)
162 |         return len(words)
163 | 
164 | 
165 |     print
166 | 
167 | 
168 |     @case('get+put')
169 |     def test():
170 |         with env.begin(write=True) as txn:
171 |             for word in words:
172 |                 txn.get(word)
173 |                 txn.put(word, word)
174 |             return len(words)
175 | 
176 | 
177 |     @case('replace')
178 |     def test():
179 |         with env.begin(write=True) as txn:
180 |             for word in words:
181 |                 txn.replace(word, word)
182 |         return len(words)
183 | 
184 | 
185 |     @case('get+put (cursor)')
186 |     def test():
187 |         with env.begin(write=True) as txn:
188 |             with txn.cursor() as cursor:
189 |                 for word in words:
190 |                     cursor.get(word)
191 |                     cursor.put(word, word)
192 |             return len(words)
193 | 
194 | 
195 |     @case('replace (cursor)')
196 |     def test():
197 |         with env.begin(write=True) as txn:
198 |             with txn.cursor() as cursor:
199 |                 for word in words:
200 |                     cursor.replace(word, word)
201 |         return len(words)
202 | 
203 | 
204 |     print
205 | 
206 | 
207 |     env = reopen_env()
208 |     @case('insert (rand)')
209 |     def test():
210 |         with env.begin(write=True) as txn:
211 |             for word in words:
212 |                 txn.put(word, big or word)
213 |         return len(words)
214 | 
215 | 
216 |     env = reopen_env()
217 |     @case('insert (seq)')
218 |     def test():
219 |         with env.begin(write=True) as txn:
220 |             for word in words_sorted:
221 |                 txn.put(word, big or word)
222 |         return len(words)
223 | 
224 | 
225 |     env = reopen_env()
226 |     @case('insert (rand), reuse cursor')
227 |     def test():
228 |         with env.begin(write=True) as txn:
229 |             curs = txn.cursor()
230 |             for word in words:
231 |                 curs.put(word, big or word)
232 |         return len(words)
233 |     env = reopen_env()
234 | 
235 | 
236 |     @case('insert (seq), reuse cursor')
237 |     def test():
238 |         with env.begin(write=True) as txn:
239 |             curs = txn.cursor()
240 |             for word in words_sorted:
241 |                 curs.put(word, big or word)
242 |         return len(words)
243 | 
244 | 
245 |     env = reopen_env()
246 |     @case('insert, putmulti')
247 |     def test():
248 |         with env.begin(write=True) as txn:
249 |             txn.cursor().putmulti(items)
250 |         return len(words)
251 | 
252 | 
253 |     env = reopen_env()
254 |     @case('insert, putmulti+generator')
255 |     def test():
256 |         with env.begin(write=True) as txn:
257 |             txn.cursor().putmulti((w, big or w) for w in words)
258 |         return len(words)
259 | 
260 | 
261 |     print
262 | 
263 | 
264 |     env = reopen_env()
265 |     @case('append')
266 |     def test():
267 |         with env.begin(write=True) as txn:
268 |             for word in words_sorted:
269 |                 txn.put(word, big or word, append=True)
270 |         return len(words)
271 | 
272 | 
273 |     env = reopen_env()
274 |     @case('append, reuse cursor')
275 |     def test():
276 |         with env.begin(write=True) as txn:
277 |             curs = txn.cursor()
278 |             for word in words_sorted:
279 |                 curs.put(word, big or word, append=True)
280 |         return len(words)
281 | 
282 | 
283 |     env = reopen_env()
284 |     @case('append+putmulti')
285 |     def test():
286 |         with env.begin(write=True) as txn:
287 |             txn.cursor().putmulti(items_sorted, append=True)
288 |         return len(words)
289 | 
290 | 
291 |     print
292 |     st = env.stat()
293 |     print 'stat:', st
294 |     print 'k+v size %.2fkb avg %d, on-disk size: %.2fkb avg %d' %\
295 |         ((2*alllen) / 1024., (2*alllen)/len(words),
296 |          (st['psize'] * st['leaf_pages']) / 1024.,
297 |          (st['psize'] * st['leaf_pages']) / len(words))
298 | 
299 | x()
300 | 


--------------------------------------------------------------------------------
/examples/keystore/README.md:
--------------------------------------------------------------------------------
 1 | 
 2 | ## Ultra minimal async IO example
 3 | 
 4 | This requires a recent version of Twisted to run. From the parent directory,
 5 | execute it as:
 6 | 
 7 |     python -m keystore.main
 8 | 
 9 | It wraps LMDB in a very basic REST API, demonstrating how to handle database IO
10 | using a thread pool without blocking the main thread's select loop.
11 | 
12 | The internals are structured relatively sanely, but the `keystore.lmdb`
13 | implementation itself is fairly horrid: a single GET will always cause a large
14 | amount of machinery to run. However, given a large and slow enough (non RAM,
15 | spinning rust) DB, this example will successfully keep the main loop responsive
16 | at all times even though multiple LMDB `mdb_get()` invocations are running
17 | concurrently.
18 | 
19 | Not shown here, but may be added sometime later to demonstrate the techniques:
20 | 
21 | * Combining multiple reads into a single transaction and context switch.
22 | * Combining writes into a single write transaction and context switch.
23 | 
24 | 
25 | ## Calling LMDB synchronously
26 | 
27 | This example never calls LMDB directly from the main loop, even though in some
28 | restricted circumstances that may be completely safe. Such a situation might
29 | look like:
30 | 
31 | * The database is guaranteed to always be in RAM.
32 | * Database writes are never contended.
33 | * Disk IO is very fast, or `sync=False` is used.
34 | 
35 | In almost every case, it is likely better to design an application that handles
36 | the possibility that calls into LMDB will trigger slow IO, if not now then at
37 | some point in 10 years when all the original developers have left your project.
38 | 


--------------------------------------------------------------------------------
/examples/keystore/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jnwatson/py-lmdb/4806170f185642d5a96b48ec10823a57d5e94080/examples/keystore/__init__.py


--------------------------------------------------------------------------------
/examples/keystore/interfaces.py:
--------------------------------------------------------------------------------
  1 | 
  2 | from __future__ import absolute_import
  3 | import zope.interface
  4 | 
  5 | 
  6 | class IKeyStoreSync(zope.interface.Interface):
  7 |     def get(key):
  8 |         """
  9 |         Fetch a key
 10 |         @return None or bytestring value.
 11 |         """
 12 | 
 13 |     def put(key, value):
 14 |         """
 15 |         Create or overwrite a key.
 16 |         @param key Bytestring key.
 17 |         @param value Bytestring alue.
 18 |         @return deferred producing None or error.
 19 |         """
 20 | 
 21 |     def delete(key):
 22 |         """
 23 |         Delete a key.
 24 |         @param key Bytestring key.
 25 |         @return deferred producing None or error.
 26 |         """
 27 | 
 28 |     def seek(key):
 29 |         """
 30 |         Seek to a key, or the next highest key.
 31 |         @param key Bytestring key.
 32 |         """
 33 | 
 34 |     def first():
 35 |         """
 36 |         Seek to the first key in the store.
 37 |         @return True if positioned on a key.
 38 |         """
 39 | 
 40 |     def last():
 41 |         """
 42 |         Seek to the last key in the store.
 43 |         @return True if positioned on a key.
 44 |         """
 45 | 
 46 |     def next():
 47 |         """
 48 |         Seek to the next key in the store.
 49 |         @return True if positioned on a key.
 50 |         """
 51 | 
 52 |     def prev():
 53 |         """
 54 |         Seek to the previous key in the store.
 55 |         @return True if positioned on a key.
 56 |         """
 57 | 
 58 | 
 59 | class IKeyStore(zope.interface.Interface):
 60 |     def get(key):
 61 |         """
 62 |         Fetch a key.
 63 |         @return deferred producing None or bytestring value.
 64 |         """
 65 | 
 66 |     def getKeys(key, count):
 67 |         """
 68 |         Fetch a list of keys.
 69 |         @param key
 70 |             Bytestring first key to return, or None for first/last key
 71 |             in space.
 72 |         @param count Number of keys including first key to return.
 73 |         """
 74 | 
 75 |     def getKeysReverse(key, count):
 76 |         """
 77 |         Fetch a list of keys, walking in reverse.
 78 |         @param key
 79 |             Bytestring first key to return, or None for first/last key
 80 |             in space.
 81 |         @param count Number of keys including first key to return.
 82 |         """
 83 | 
 84 |     def getItems(key, count):
 85 |         """
 86 |         Fetch a list of (key, value) tuples.
 87 |         @param key
 88 |             Bytestring first key to return, or None for first/last key
 89 |             in space.
 90 |         @param count Number of keys including first key to return.
 91 |         """
 92 | 
 93 |     def getItemsReverse(key, count):
 94 |         """
 95 |         Fetch a list of (key, value) tuples.
 96 |         @param key
 97 |             Bytestring first key to return, or None for first/last key
 98 |             in space.
 99 |         @param count Number of keys including first key to return.
100 |         """
101 | 
102 |     def put(key):
103 |         """
104 |         Create or overwrite a key.
105 | 
106 |         @param key Bytestring key.
107 |         @param value Bytestring alue.
108 |         @return deferred producing None or error.
109 |         """
110 | 
111 |     def delete(key):
112 |         """
113 |         Delete a key.
114 |         @param key Bytestring key.
115 |         @return deferred producing None or error.
116 |         """
117 | 
118 |     def putGroup(func):
119 |         """
120 |         Execute a function in the context of synchronous (IKeyStoreSync)
121 |         transaction, in a private thread.
122 | 
123 |         @param func Function accepting IKeyStoreSync parameter.
124 |         @returns deferred producing None or error.
125 |         """
126 | 


--------------------------------------------------------------------------------
/examples/keystore/lmdb.py:
--------------------------------------------------------------------------------
  1 | 
  2 | from __future__ import absolute_import
  3 | import operator
  4 | 
  5 | import twisted.internet.defer
  6 | import twisted.internet.threads
  7 | import zope.interface
  8 | 
  9 | import keystore.interfaces
 10 | 
 11 | 
 12 | class LmdbKeyStoreSync(object):
 13 |     zope.interface.implements(keystore.interfaces.IKeyStoreSync)
 14 |     cursor = None
 15 | 
 16 |     def __init__(self, env, write):
 17 |         self.txn = env.begin(write=write)
 18 | 
 19 |     def __enter__(self):
 20 |         return self
 21 | 
 22 |     def __exit__(self, e_type, e_val, e_tb):
 23 |         if e_type:
 24 |             self.txn.abort()
 25 |         else:
 26 |             self.txn.commit()
 27 |         self.txn = None
 28 | 
 29 |     def get(self, key):
 30 |         return self.txn.get(key)
 31 | 
 32 |     def put(self, key, value):
 33 |         self.txn.put(key, value)
 34 | 
 35 |     def delete(self, key):
 36 |         self.txn.delete(key)
 37 | 
 38 |     #
 39 |     # Cursor.
 40 |     #
 41 | 
 42 |     @property
 43 |     def key(self):
 44 |         return self.cursor.key()
 45 | 
 46 |     @property
 47 |     def value(self):
 48 |         return self.cursor.value()
 49 | 
 50 |     def seek(self, key):
 51 |         self.cursor.set_range(key)
 52 | 
 53 |     def next(self):
 54 |         return self.cursor.next()
 55 | 
 56 |     def prev(self):
 57 |         return self.cursor.prev()
 58 | 
 59 | 
 60 | def _reader_task(env, func):
 61 |     with LmdbKeyStoreSync(env, write=False) as sync:
 62 |         return func(sync)
 63 | 
 64 | 
 65 | def _writer_task(env, func):
 66 |     with LmdbKeyStoreSync(env, write=True) as sync:
 67 |         return func(sync)
 68 | 
 69 | 
 70 | class LmdbKeyStore(object):
 71 |     zope.interface.implements(keystore.interfaces.IKeyStore)
 72 | 
 73 |     def __init__(self, reactor, pool, env):
 74 |         self.reactor = reactor
 75 |         self.pool = pool
 76 |         self.env = env
 77 | 
 78 |     def _call_in_thread(self, func):
 79 |         return twisted.internet.threads.deferToThreadPool(
 80 |             self.reactor,
 81 |             self.pool,
 82 |             func)
 83 | 
 84 |     def get(self, key):
 85 |         twisted.python.log.msg('get(%r, %r)', key)
 86 |         get = lambda sync: sync.get(key)
 87 |         return self._call_in_thread(lambda: _reader_task(self.env, get))
 88 | 
 89 |     def _get_forward(self, sync, key, count, getter):
 90 |         positioned = sync.seek(key)
 91 |         out = []
 92 |         for x in xrange(count):
 93 |             if not positioned:
 94 |                 break
 95 |             out.append(getter(sync))
 96 |             positioned = sync.next()
 97 |         return out
 98 | 
 99 |     def _get_reverse(self, sync, key, count, getter):
100 |         out = []
101 |         positioned = sync.seek(key)
102 |         if not positioned:
103 |             positioned = sync.last()
104 |         for x in xrange(count):
105 |             if not positioned:
106 |                 break
107 |             out.append(sync.key)
108 |             positioned = sync.prev()
109 |         return out
110 | 
111 |     _key_getter = operator.attrgetter('key')
112 |     _item_getter = operator.attrgetter('key', 'value')
113 | 
114 |     def getKeys(self, key, count):
115 |         get = lambda sync: self._get_forward(sync, key, count,
116 |                                              self._key_getter)
117 |         return self._call_in_thread(lambda: _reader_task(self.env, get))
118 | 
119 |     def getKeysReverse(self, key, count):
120 |         get = lambda sync: self._get_reverse(sync, key, count,
121 |                                              self._key_getter)
122 |         return self._call_in_thread(lambda: _reader_task(self.env, get))
123 | 
124 |     def getItems(self, key, count):
125 |         get = lambda sync: self._get_forward(sync, key, count,
126 |                                              self._item_getter)
127 |         return self._call_in_thread(lambda: _reader_task(self.env, get))
128 | 
129 |     def getItemsReverse(self, key, count):
130 |         get = lambda sync: self._get_reverse(sync, key, count,
131 |                                              self._item_getter)
132 |         return self._call_in_thread(lambda: _reader_task(self.env, get))
133 | 
134 |     def put(self, key, value):
135 |         twisted.python.log.msg('put(%r, %r)', key, value)
136 |         put = lambda sync: sync.put(key, value)
137 |         return self._call_in_thread(lambda: _writer_task(self.env, put))
138 | 
139 |     def delete(self, key):
140 |         delete = lambda sync: sync.delete(key)
141 |         return self._call_in_thread(lambda: _writer_task(self.env, delete))
142 | 
143 |     def putGroup(self, func):
144 |         return self._call_in_thread(lambda: _writer_task(self.env, func))
145 | 


--------------------------------------------------------------------------------
/examples/keystore/main.py:
--------------------------------------------------------------------------------
 1 | 
 2 | from __future__ import absolute_import
 3 | import webbrowser
 4 | 
 5 | import twisted.internet.reactor
 6 | 
 7 | import lmdb
 8 | import keystore.lmdb
 9 | import keystore.webapi
10 | 
11 | 
12 | def main():
13 |     port = 9999
14 |     interface = '127.0.0.1'
15 |     url = 'http://%s:%d/' % (interface, port)
16 |     env = lmdb.open('/tmp/foo')
17 |     reactor = twisted.internet.reactor
18 |     pool = reactor.getThreadPool()
19 |     store = keystore.lmdb.LmdbKeyStore(reactor, pool, env)
20 |     site = keystore.webapi.create_site(store)
21 |     reactor.listenTCP(port, site, interface=interface)
22 |     reactor.callLater(0, webbrowser.open, url)
23 |     reactor.run()
24 | 
25 | if __name__ == '__main__':
26 |     main()
27 | 


--------------------------------------------------------------------------------
/examples/keystore/static/index.html:
--------------------------------------------------------------------------------
 1 | <!doctype html>
 2 | <meta charset="utf-8">
 3 | 
 4 | <script>
 5 |     function log(s) {
 6 |         var msg = document.createElement('div');
 7 |         msg.textContent = s;
 8 | 
 9 |         var logElem = document.getElementById('log');
10 |         logElem.insertBefore(msg, logElem.firstChild);
11 |     }
12 | 
13 |     function onRequestDone(request, path) {
14 |         var bits = [path, ': HTTP '];
15 |         bits.push(request.status);
16 |         bits.push(': ');
17 |         bits.push(request.responseText);
18 |         log(bits.join(''));
19 |     }
20 | 
21 |     function startRequest(method, path, body) {
22 |         var req = new XMLHttpRequest;
23 |         req.onreadystatechange = function() {
24 |             if(req.readyState === XMLHttpRequest.DONE) {
25 |                 onRequestDone(req, path);
26 |             }
27 |         }
28 |         req.open(method, path);
29 |         if(body !== undefined) {
30 |             req.send(body);
31 |         } else {
32 |             req.send();
33 |         }
34 |     }
35 | 
36 |     window.onload = function() {
37 |         document.getElementById('startRequest').onclick = function() {
38 |             var method = document.getElementById('method').value;
39 |             var path = '/db' + document.getElementById('path').value;
40 |             var body = document.getElementById('body').value;
41 |             startRequest(method, path, method=='PUT'?body:undefined);
42 |         }
43 |     }
44 | </script>
45 | 
46 | <style>
47 |     #log {
48 |         margin-top: 1em;
49 |         background: #efefef;
50 |     }
51 | 
52 |     #log div {
53 |         font-family: monospace;
54 |         white-space: pre-wrap;
55 |         border-bottom: 1px dotted silver;
56 |         padding-bottom: 0.2em;
57 |     }
58 | </style>
59 | 
60 | 
61 | <select id="method">
62 |     <option value="GET">GET</option>
63 |     <option value="PUT">PUT</option>
64 |     <option value="DELETE">DELETE</option>
65 | </select><br>
66 | Request path: <input type="text" id="path" value="/foo/"><br>
67 | Request body: <input type="text" id="body"><br>
68 | <button type="button" id="startRequest">Start Request</button>
69 | 
70 | <div id="log">
71 | </div>
72 | 


--------------------------------------------------------------------------------
/examples/keystore/web.py:
--------------------------------------------------------------------------------
 1 | 
 2 | from __future__ import absolute_import
 3 | import twisted.python.failure
 4 | import twisted.python.log
 5 | import twisted.web.http
 6 | import twisted.web.resource
 7 | import twisted.web.server
 8 | 
 9 | 
10 | class Response(object):
11 |     def __init__(self, body=None, headers=None, status=None):
12 |         self.body = body
13 |         self.headers = headers or {}
14 |         self.status = status
15 | 
16 | 
17 | class DeferrableResourceRender(object):
18 |     def __init__(self, resource_, request):
19 |         self.resource = resource_
20 |         self.request = request
21 |         self.closed = False
22 | 
23 |     def _notify_finish(self, resp):
24 |         self.closed = resp
25 | 
26 |     def _on_render_error(self, failure):
27 |         twisted.python.log.err(failure)
28 |         if not self.closed:
29 |             self.request.setResponseCode(twisted.web.http.INTERNAL_SERVER_ERROR)
30 |             self.request.finish()
31 | 
32 |     def _on_render_done(self, resp):
33 |         if not isinstance(resp, Response):
34 |             exc = TypeError("render() didn't return Response, got %r" % (resp,))
35 |             self._on_render_error(twisted.python.failure.Failure(exc))
36 |             return
37 | 
38 |         if resp.body is not None and type(resp.body) is not bytes:
39 |             exc = TypeError("render() returned %r, not bytes" %
40 |                             (type(resp.body),))
41 |             self._on_render_error(twisted.python.failure.Failure(exc))
42 |             return
43 | 
44 |         if self.closed:
45 |             return
46 | 
47 |         if resp.status is not None:
48 |             self.request.setResponseCode(resp.status)
49 |         for key, value in resp.headers.iteritems():
50 |             self.request.setHeader(key, value)
51 |         if resp.body is not None:
52 |             self.request.setHeader('Content-Length', len(resp.body))
53 |             self.request.write(resp.body)
54 |         self.request.finish()
55 | 
56 |     def start_render(self):
57 |         method = self.request.method
58 |         handler = getattr(self.resource, 'render_' + method, None)
59 |         if handler is None:
60 |             self._on_render_done(Response(status=405))
61 |             return
62 | 
63 |         self.request.notifyFinish().addBoth(self._notify_finish)
64 |         d = twisted.internet.defer.maybeDeferred(handler, self.request)
65 |         d.addCallbacks(self._on_render_done, self._on_render_error)
66 |         return twisted.web.server.NOT_DONE_YET
67 | 
68 | 
69 | class DeferrableResource(twisted.web.resource.Resource):
70 |     def render(self, request):
71 |         render = DeferrableResourceRender(self, request)
72 |         return render.start_render()
73 | 


--------------------------------------------------------------------------------
/examples/keystore/webapi.py:
--------------------------------------------------------------------------------
 1 | 
 2 | from __future__ import absolute_import
 3 | import pkg_resources
 4 | import keystore.web
 5 | import twisted.web.server
 6 | 
 7 | 
 8 | def read_resource(path):
 9 |     return pkg_resources.resource_string('keystore', path)
10 | 
11 | 
12 | class KeyResource(keystore.web.DeferrableResource):
13 |     isLeaf = True
14 | 
15 |     def __init__(self, store):
16 |         keystore.web.DeferrableResource.__init__(self)
17 |         self.store = store
18 | 
19 |     def _get_done(self, value):
20 |         if value is None:
21 |             return keystore.web.Response(status=404)
22 |         else:
23 |             return keystore.web.Response(body=value)
24 | 
25 |     def render_GET(self, request):
26 |         d = self.store.get(request.path)
27 |         d.addCallback(self._get_done)
28 |         return d
29 | 
30 |     def render_PUT(self, request):
31 |         value = request.content.read()
32 |         d = self.store.put(request.path, value)
33 |         d.addCallback(lambda _: keystore.web.Response(status=202))
34 |         return d
35 | 
36 |     def render_DELETE(self, request):
37 |         d = self.store.delete(request.path)
38 |         d.addCallback(lambda _: keystore.web.Response(status=204))
39 |         return d
40 | 
41 | 
42 | class NamespaceResource(keystore.web.DeferrableResource):
43 |     isLeaf = False
44 | 
45 |     def __init__(self, store):
46 |         keystore.web.DeferrableResource.__init__(self)
47 |         self.store = store
48 | 
49 |     def getChild(self, path, request):
50 |         return KeyResource(self.store)
51 | 
52 | 
53 | class StaticResource(twisted.web.resource.Resource):
54 |     def __init__(self, pkg_path):
55 |         twisted.web.resource.Resource.__init__(self)
56 |         self.data = read_resource(pkg_path)
57 | 
58 |     def render(self, request):
59 |         return self.data
60 | 
61 | 
62 | def create_site(store):
63 |     root = twisted.web.resource.Resource()
64 |     root.putChild('', StaticResource('static/index.html'))
65 |     root.putChild('db', NamespaceResource(store))
66 |     return twisted.web.server.Site(root)
67 | 


--------------------------------------------------------------------------------
/examples/nastybench.py:
--------------------------------------------------------------------------------
  1 | 
  2 | # Roughly approximates some of Symas microbenchmark.
  3 | 
  4 | from time import time
  5 | import random
  6 | import shutil
  7 | import os
  8 | import tempfile
  9 | 
 10 | import lmdb
 11 | 
 12 | 
 13 | val = ' ' * 100
 14 | MAX_KEYS = int(1e6)
 15 | 
 16 | t0 = time()
 17 | 
 18 | urandom = file('/dev/urandom', 'rb', 1048576).read
 19 | 
 20 | keys = set()
 21 | while len(keys) < MAX_KEYS:
 22 |     for _ in xrange(min(1000, MAX_KEYS - len(keys))):
 23 |         keys.add(urandom(16))
 24 | 
 25 | print 'make %d keys in %.2fsec' % (len(keys), time() - t0)
 26 | keys = list(keys)
 27 | 
 28 | if os.path.exists('/ram'):
 29 |     DB_PATH = '/ram/dbtest'
 30 | else:
 31 |     DB_PATH = tempfile.mktemp(prefix='nastybench')
 32 | 
 33 | if os.path.exists(DB_PATH):
 34 |     shutil.rmtree(DB_PATH)
 35 | 
 36 | env = lmdb.open(DB_PATH, map_size=1048576 * 1024,
 37 |     metasync=False, sync=False, map_async=True)
 38 | 
 39 | nextkey = iter(keys).next
 40 | run = True
 41 | while run:
 42 |     with env.begin(write=True) as txn:
 43 |         try:
 44 |             for _ in xrange(10000):
 45 |                 txn.put(nextkey(), val)
 46 |         except StopIteration:
 47 |             run = False
 48 | 
 49 | d = time() - t0
 50 | env.sync(True)
 51 | print 'insert %d keys in %.2fsec (%d/sec)' % (len(keys), d, len(keys) / d)
 52 | 
 53 | 
 54 | 
 55 | nextkey = iter(keys).next
 56 | t0 = time()
 57 | 
 58 | with env.begin() as txn:
 59 |     try:
 60 |         while 1:
 61 |             txn.get(nextkey())
 62 |     except StopIteration:
 63 |         pass
 64 | 
 65 | d = time() - t0
 66 | print 'random lookup %d keys in %.2fsec (%d/sec)' % (len(keys), d, len(keys)/d)
 67 | 
 68 | 
 69 | nextkey = iter(keys).next
 70 | t0 = time()
 71 | 
 72 | with env.begin(buffers=True) as txn:
 73 |     try:
 74 |         while 1:
 75 |             txn.get(nextkey())
 76 |     except StopIteration:
 77 |         pass
 78 | 
 79 | d = time() - t0
 80 | print 'random lookup %d buffers in %.2fsec (%d/sec)' % (len(keys), d, len(keys)/d)
 81 | 
 82 | 
 83 | nextkey = iter(keys).next
 84 | t0 = time()
 85 | 
 86 | with env.begin(buffers=True) as txn:
 87 |     try:
 88 |         while 1:
 89 |             hash(txn.get(nextkey()))
 90 |     except StopIteration:
 91 |         pass
 92 | 
 93 | d = time() - t0
 94 | print 'random lookup+hash %d buffers in %.2fsec (%d/sec)' % (len(keys), d, len(keys)/d)
 95 | 
 96 | 
 97 | 
 98 | nextkey = iter(keys).next
 99 | t0 = time()
100 | 
101 | with env.begin(buffers=True) as txn:
102 |     nextrec = txn.cursor().iternext().next
103 |     try:
104 |         while 1:
105 |             nextrec()
106 |     except StopIteration:
107 |         pass
108 | 
109 | d = time() - t0
110 | print 'seq read %d buffers in %.2fsec (%d/sec)' % (len(keys), d, len(keys)/d)
111 | 


--------------------------------------------------------------------------------
/examples/parabench.py:
--------------------------------------------------------------------------------
  1 | 
  2 | # Roughly approximates some of Symas microbenchmark.
  3 | 
  4 | import multiprocessing
  5 | import os
  6 | import random
  7 | import shutil
  8 | import sys
  9 | import tempfile
 10 | import time
 11 | 
 12 | try:
 13 |     import affinity
 14 | except:
 15 |     affinity = False
 16 | import lmdb
 17 | 
 18 | 
 19 | USE_SPARSE_FILES = sys.platform != 'darwin'
 20 | DB_PATH = '/ram/dbtest'
 21 | MAX_KEYS = int(4e6)
 22 | 
 23 | if os.path.exists('/ram'):
 24 |     DB_PATH = '/ram/dbtest'
 25 | else:
 26 |     DB_PATH = tempfile.mktemp(prefix='parabench')
 27 | 
 28 | 
 29 | def open_env():
 30 |     return lmdb.open(DB_PATH,
 31 |         map_size=1048576 * 1024,
 32 |         metasync=False,
 33 |         sync=False,
 34 |         map_async=True,
 35 |         writemap=USE_SPARSE_FILES)
 36 | 
 37 | 
 38 | def make_keys():
 39 |     t0 = time.time()
 40 |     urandom = file('/dev/urandom', 'rb', 1048576).read
 41 | 
 42 |     keys = set()
 43 |     while len(keys) < MAX_KEYS:
 44 |         for _ in xrange(min(1000, MAX_KEYS - len(keys))):
 45 |             keys.add(urandom(16))
 46 | 
 47 |     print 'make %d keys in %.2fsec' % (len(keys), time.time() - t0)
 48 |     keys = list(keys)
 49 | 
 50 |     nextkey = iter(keys).next
 51 |     run = True
 52 |     val = ' ' * 100
 53 |     env = open_env()
 54 |     while run:
 55 |         with env.begin(write=True) as txn:
 56 |             try:
 57 |                 for _ in xrange(10000):
 58 |                     txn.put(nextkey(), val)
 59 |             except StopIteration:
 60 |                 run = False
 61 | 
 62 |     d = time.time() - t0
 63 |     env.sync(True)
 64 |     env.close()
 65 |     print 'insert %d keys in %.2fsec (%d/sec)' % (len(keys), d, len(keys) / d)
 66 | 
 67 | 
 68 | if 'drop' in sys.argv and os.path.exists(DB_PATH):
 69 |     shutil.rmtree(DB_PATH)
 70 | 
 71 | if not os.path.exists(DB_PATH):
 72 |     make_keys()
 73 | 
 74 | 
 75 | env = open_env()
 76 | with env.begin() as txn:
 77 |     keys = list(txn.cursor().iternext(values=False))
 78 | env.close()
 79 | 
 80 | 
 81 | def run(idx):
 82 |     if affinity:
 83 |         affinity.set_process_affinity_mask(os.getpid(), 1 << idx)
 84 | 
 85 |     env = open_env()
 86 |     k = list(keys)
 87 |     random.shuffle(k)
 88 |     k = k[:1000]
 89 | 
 90 |     while 1:
 91 |         with env.begin() as txn:
 92 |             nextkey = iter(k).next
 93 |             try:
 94 |                 while 1:
 95 |                     hash(txn.get(nextkey()))
 96 |             except StopIteration:
 97 |                 pass
 98 |             arr[idx] += len(k)
 99 | 
100 | 
101 | 
102 | nproc = int(sys.argv[1])
103 | arr = multiprocessing.Array('L', xrange(nproc))
104 | for x in xrange(nproc):
105 |     arr[x] = 0
106 | procs = [multiprocessing.Process(target=run, args=(x,)) for x in xrange(nproc)]
107 | [p.start() for p in procs]
108 | 
109 | 
110 | t0 = time.time()
111 | while True:
112 |     time.sleep(2)
113 |     d = time.time() - t0
114 |     lk = sum(arr)
115 |     print 'lookup %d keys in %.2fsec (%d/sec)' % (lk, d, lk / d)
116 | 
117 | 


--------------------------------------------------------------------------------
/examples/words.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jnwatson/py-lmdb/4806170f185642d5a96b48ec10823a57d5e94080/examples/words.gz


--------------------------------------------------------------------------------
/lib/midl.c:
--------------------------------------------------------------------------------
  1 | /**	@file midl.c
  2 |  *	@brief ldap bdb back-end ID List functions */
  3 | /* $OpenLDAP$ */
  4 | /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  5 |  *
  6 |  * Copyright 2000-2021 The OpenLDAP Foundation.
  7 |  * Portions Copyright 2001-2021 Howard Chu, Symas Corp.
  8 |  * All rights reserved.
  9 |  *
 10 |  * Redistribution and use in source and binary forms, with or without
 11 |  * modification, are permitted only as authorized by the OpenLDAP
 12 |  * Public License.
 13 |  *
 14 |  * A copy of this license is available in the file LICENSE in the
 15 |  * top-level directory of the distribution or, alternatively, at
 16 |  * <http://www.OpenLDAP.org/license.html>.
 17 |  */
 18 | 
 19 | #include <limits.h>
 20 | #include <string.h>
 21 | #include <stdlib.h>
 22 | #include <errno.h>
 23 | #include <sys/types.h>
 24 | #include "midl.h"
 25 | 
 26 | /** @defgroup internal	LMDB Internals
 27 |  *	@{
 28 |  */
 29 | /** @defgroup idls	ID List Management
 30 |  *	@{
 31 |  */
 32 | #define CMP(x,y)	 ( (x) < (y) ? -1 : (x) > (y) )
 33 | 
 34 | unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id )
 35 | {
 36 | 	/*
 37 | 	 * binary search of id in ids
 38 | 	 * if found, returns position of id
 39 | 	 * if not found, returns first position greater than id
 40 | 	 */
 41 | 	unsigned base = 0;
 42 | 	unsigned cursor = 1;
 43 | 	int val = 0;
 44 | 	unsigned n = ids[0];
 45 | 
 46 | 	while( 0 < n ) {
 47 | 		unsigned pivot = n >> 1;
 48 | 		cursor = base + pivot + 1;
 49 | 		val = CMP( ids[cursor], id );
 50 | 
 51 | 		if( val < 0 ) {
 52 | 			n = pivot;
 53 | 
 54 | 		} else if ( val > 0 ) {
 55 | 			base = cursor;
 56 | 			n -= pivot + 1;
 57 | 
 58 | 		} else {
 59 | 			return cursor;
 60 | 		}
 61 | 	}
 62 | 
 63 | 	if( val > 0 ) {
 64 | 		++cursor;
 65 | 	}
 66 | 	return cursor;
 67 | }
 68 | 
 69 | #if 0	/* superseded by append/sort */
 70 | int mdb_midl_insert( MDB_IDL ids, MDB_ID id )
 71 | {
 72 | 	unsigned x, i;
 73 | 
 74 | 	x = mdb_midl_search( ids, id );
 75 | 	assert( x > 0 );
 76 | 
 77 | 	if( x < 1 ) {
 78 | 		/* internal error */
 79 | 		return -2;
 80 | 	}
 81 | 
 82 | 	if ( x <= ids[0] && ids[x] == id ) {
 83 | 		/* duplicate */
 84 | 		assert(0);
 85 | 		return -1;
 86 | 	}
 87 | 
 88 | 	if ( ++ids[0] >= MDB_IDL_DB_MAX ) {
 89 | 		/* no room */
 90 | 		--ids[0];
 91 | 		return -2;
 92 | 
 93 | 	} else {
 94 | 		/* insert id */
 95 | 		for (i=ids[0]; i>x; i--)
 96 | 			ids[i] = ids[i-1];
 97 | 		ids[x] = id;
 98 | 	}
 99 | 
100 | 	return 0;
101 | }
102 | #endif
103 | 
104 | MDB_IDL mdb_midl_alloc(int num)
105 | {
106 | 	MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID));
107 | 	if (ids) {
108 | 		*ids++ = num;
109 | 		*ids = 0;
110 | 	}
111 | 	return ids;
112 | }
113 | 
114 | void mdb_midl_free(MDB_IDL ids)
115 | {
116 | 	if (ids)
117 | 		free(ids-1);
118 | }
119 | 
120 | void mdb_midl_shrink( MDB_IDL *idp )
121 | {
122 | 	MDB_IDL ids = *idp;
123 | 	if (*(--ids) > MDB_IDL_UM_MAX &&
124 | 		(ids = realloc(ids, (MDB_IDL_UM_MAX+2) * sizeof(MDB_ID))))
125 | 	{
126 | 		*ids++ = MDB_IDL_UM_MAX;
127 | 		*idp = ids;
128 | 	}
129 | }
130 | 
131 | static int mdb_midl_grow( MDB_IDL *idp, int num )
132 | {
133 | 	MDB_IDL idn = *idp-1;
134 | 	/* grow it */
135 | 	idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID));
136 | 	if (!idn)
137 | 		return ENOMEM;
138 | 	*idn++ += num;
139 | 	*idp = idn;
140 | 	return 0;
141 | }
142 | 
143 | int mdb_midl_need( MDB_IDL *idp, unsigned num )
144 | {
145 | 	MDB_IDL ids = *idp;
146 | 	num += ids[0];
147 | 	if (num > ids[-1]) {
148 | 		num = (num + num/4 + (256 + 2)) & -256;
149 | 		if (!(ids = realloc(ids-1, num * sizeof(MDB_ID))))
150 | 			return ENOMEM;
151 | 		*ids++ = num - 2;
152 | 		*idp = ids;
153 | 	}
154 | 	return 0;
155 | }
156 | 
157 | int mdb_midl_append( MDB_IDL *idp, MDB_ID id )
158 | {
159 | 	MDB_IDL ids = *idp;
160 | 	/* Too big? */
161 | 	if (ids[0] >= ids[-1]) {
162 | 		if (mdb_midl_grow(idp, MDB_IDL_UM_MAX))
163 | 			return ENOMEM;
164 | 		ids = *idp;
165 | 	}
166 | 	ids[0]++;
167 | 	ids[ids[0]] = id;
168 | 	return 0;
169 | }
170 | 
171 | int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app )
172 | {
173 | 	MDB_IDL ids = *idp;
174 | 	/* Too big? */
175 | 	if (ids[0] + app[0] >= ids[-1]) {
176 | 		if (mdb_midl_grow(idp, app[0]))
177 | 			return ENOMEM;
178 | 		ids = *idp;
179 | 	}
180 | 	memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID));
181 | 	ids[0] += app[0];
182 | 	return 0;
183 | }
184 | 
185 | int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n )
186 | {
187 | 	MDB_ID *ids = *idp, len = ids[0];
188 | 	/* Too big? */
189 | 	if (len + n > ids[-1]) {
190 | 		if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX))
191 | 			return ENOMEM;
192 | 		ids = *idp;
193 | 	}
194 | 	ids[0] = len + n;
195 | 	ids += len;
196 | 	while (n)
197 | 		ids[n--] = id++;
198 | 	return 0;
199 | }
200 | 
201 | void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge )
202 | {
203 | 	MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k;
204 | 	idl[0] = (MDB_ID)-1;		/* delimiter for idl scan below */
205 | 	old_id = idl[j];
206 | 	while (i) {
207 | 		merge_id = merge[i--];
208 | 		for (; old_id < merge_id; old_id = idl[--j])
209 | 			idl[k--] = old_id;
210 | 		idl[k--] = merge_id;
211 | 	}
212 | 	idl[0] = total;
213 | }
214 | 
215 | /* Quicksort + Insertion sort for small arrays */
216 | 
217 | #define SMALL	8
218 | #define	MIDL_SWAP(a,b)	{ itmp=(a); (a)=(b); (b)=itmp; }
219 | 
220 | void
221 | mdb_midl_sort( MDB_IDL ids )
222 | {
223 | 	/* Max possible depth of int-indexed tree * 2 items/level */
224 | 	int istack[sizeof(int)*CHAR_BIT * 2];
225 | 	int i,j,k,l,ir,jstack;
226 | 	MDB_ID a, itmp;
227 | 
228 | 	ir = (int)ids[0];
229 | 	l = 1;
230 | 	jstack = 0;
231 | 	for(;;) {
232 | 		if (ir - l < SMALL) {	/* Insertion sort */
233 | 			for (j=l+1;j<=ir;j++) {
234 | 				a = ids[j];
235 | 				for (i=j-1;i>=1;i--) {
236 | 					if (ids[i] >= a) break;
237 | 					ids[i+1] = ids[i];
238 | 				}
239 | 				ids[i+1] = a;
240 | 			}
241 | 			if (jstack == 0) break;
242 | 			ir = istack[jstack--];
243 | 			l = istack[jstack--];
244 | 		} else {
245 | 			k = (l + ir) >> 1;	/* Choose median of left, center, right */
246 | 			MIDL_SWAP(ids[k], ids[l+1]);
247 | 			if (ids[l] < ids[ir]) {
248 | 				MIDL_SWAP(ids[l], ids[ir]);
249 | 			}
250 | 			if (ids[l+1] < ids[ir]) {
251 | 				MIDL_SWAP(ids[l+1], ids[ir]);
252 | 			}
253 | 			if (ids[l] < ids[l+1]) {
254 | 				MIDL_SWAP(ids[l], ids[l+1]);
255 | 			}
256 | 			i = l+1;
257 | 			j = ir;
258 | 			a = ids[l+1];
259 | 			for(;;) {
260 | 				do i++; while(ids[i] > a);
261 | 				do j--; while(ids[j] < a);
262 | 				if (j < i) break;
263 | 				MIDL_SWAP(ids[i],ids[j]);
264 | 			}
265 | 			ids[l+1] = ids[j];
266 | 			ids[j] = a;
267 | 			jstack += 2;
268 | 			if (ir-i+1 >= j-l) {
269 | 				istack[jstack] = ir;
270 | 				istack[jstack-1] = i;
271 | 				ir = j-1;
272 | 			} else {
273 | 				istack[jstack] = j-1;
274 | 				istack[jstack-1] = l;
275 | 				l = i;
276 | 			}
277 | 		}
278 | 	}
279 | }
280 | 
281 | unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id )
282 | {
283 | 	/*
284 | 	 * binary search of id in ids
285 | 	 * if found, returns position of id
286 | 	 * if not found, returns first position greater than id
287 | 	 */
288 | 	unsigned base = 0;
289 | 	unsigned cursor = 1;
290 | 	int val = 0;
291 | 	unsigned n = (unsigned)ids[0].mid;
292 | 
293 | 	while( 0 < n ) {
294 | 		unsigned pivot = n >> 1;
295 | 		cursor = base + pivot + 1;
296 | 		val = CMP( id, ids[cursor].mid );
297 | 
298 | 		if( val < 0 ) {
299 | 			n = pivot;
300 | 
301 | 		} else if ( val > 0 ) {
302 | 			base = cursor;
303 | 			n -= pivot + 1;
304 | 
305 | 		} else {
306 | 			return cursor;
307 | 		}
308 | 	}
309 | 
310 | 	if( val > 0 ) {
311 | 		++cursor;
312 | 	}
313 | 	return cursor;
314 | }
315 | 
316 | int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id )
317 | {
318 | 	unsigned x, i;
319 | 
320 | 	x = mdb_mid2l_search( ids, id->mid );
321 | 
322 | 	if( x < 1 ) {
323 | 		/* internal error */
324 | 		return -2;
325 | 	}
326 | 
327 | 	if ( x <= ids[0].mid && ids[x].mid == id->mid ) {
328 | 		/* duplicate */
329 | 		return -1;
330 | 	}
331 | 
332 | 	if ( ids[0].mid >= MDB_IDL_UM_MAX ) {
333 | 		/* too big */
334 | 		return -2;
335 | 
336 | 	} else {
337 | 		/* insert id */
338 | 		ids[0].mid++;
339 | 		for (i=(unsigned)ids[0].mid; i>x; i--)
340 | 			ids[i] = ids[i-1];
341 | 		ids[x] = *id;
342 | 	}
343 | 
344 | 	return 0;
345 | }
346 | 
347 | int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id )
348 | {
349 | 	/* Too big? */
350 | 	if (ids[0].mid >= MDB_IDL_UM_MAX) {
351 | 		return -2;
352 | 	}
353 | 	ids[0].mid++;
354 | 	ids[ids[0].mid] = *id;
355 | 	return 0;
356 | }
357 | 
358 | /** @} */
359 | /** @} */
360 | 


--------------------------------------------------------------------------------
/lib/midl.h:
--------------------------------------------------------------------------------
  1 | /**	@file midl.h
  2 |  *	@brief LMDB ID List header file.
  3 |  *
  4 |  *	This file was originally part of back-bdb but has been
  5 |  *	modified for use in libmdb. Most of the macros defined
  6 |  *	in this file are unused, just left over from the original.
  7 |  *
  8 |  *	This file is only used internally in libmdb and its definitions
  9 |  *	are not exposed publicly.
 10 |  */
 11 | /* $OpenLDAP$ */
 12 | /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
 13 |  *
 14 |  * Copyright 2000-2021 The OpenLDAP Foundation.
 15 |  * Portions Copyright 2001-2021 Howard Chu, Symas Corp.
 16 |  * All rights reserved.
 17 |  *
 18 |  * Redistribution and use in source and binary forms, with or without
 19 |  * modification, are permitted only as authorized by the OpenLDAP
 20 |  * Public License.
 21 |  *
 22 |  * A copy of this license is available in the file LICENSE in the
 23 |  * top-level directory of the distribution or, alternatively, at
 24 |  * <http://www.OpenLDAP.org/license.html>.
 25 |  */
 26 | 
 27 | #ifndef _MDB_MIDL_H_
 28 | #define _MDB_MIDL_H_
 29 | 
 30 | #include <stddef.h>
 31 | 
 32 | #ifdef __cplusplus
 33 | extern "C" {
 34 | #endif
 35 | 
 36 | /** @defgroup internal	LMDB Internals
 37 |  *	@{
 38 |  */
 39 | 
 40 | /** @defgroup idls	ID List Management
 41 |  *	@{
 42 |  */
 43 | 	/** A generic unsigned ID number. These were entryIDs in back-bdb.
 44 | 	 *	Preferably it should have the same size as a pointer.
 45 | 	 */
 46 | typedef size_t MDB_ID;
 47 | 
 48 | 	/** An IDL is an ID List, a sorted array of IDs. The first
 49 | 	 * element of the array is a counter for how many actual
 50 | 	 * IDs are in the list. In the original back-bdb code, IDLs are
 51 | 	 * sorted in ascending order. For libmdb IDLs are sorted in
 52 | 	 * descending order.
 53 | 	 */
 54 | typedef MDB_ID *MDB_IDL;
 55 | 
 56 | /* IDL sizes - likely should be even bigger
 57 |  *   limiting factors: sizeof(ID), thread stack size
 58 |  */
 59 | #define	MDB_IDL_LOGN	16	/* DB_SIZE is 2^16, UM_SIZE is 2^17 */
 60 | #define MDB_IDL_DB_SIZE		(1<<MDB_IDL_LOGN)
 61 | #define MDB_IDL_UM_SIZE		(1<<(MDB_IDL_LOGN+1))
 62 | 
 63 | #define MDB_IDL_DB_MAX		(MDB_IDL_DB_SIZE-1)
 64 | #define MDB_IDL_UM_MAX		(MDB_IDL_UM_SIZE-1)
 65 | 
 66 | #define MDB_IDL_SIZEOF(ids)		(((ids)[0]+1) * sizeof(MDB_ID))
 67 | #define MDB_IDL_IS_ZERO(ids) ( (ids)[0] == 0 )
 68 | #define MDB_IDL_CPY( dst, src ) (memcpy( dst, src, MDB_IDL_SIZEOF( src ) ))
 69 | #define MDB_IDL_FIRST( ids )	( (ids)[1] )
 70 | #define MDB_IDL_LAST( ids )		( (ids)[(ids)[0]] )
 71 | 
 72 | 	/** Current max length of an #mdb_midl_alloc()ed IDL */
 73 | #define MDB_IDL_ALLOCLEN( ids )	( (ids)[-1] )
 74 | 
 75 | 	/** Append ID to IDL. The IDL must be big enough. */
 76 | #define mdb_midl_xappend(idl, id) do { \
 77 | 		MDB_ID *xidl = (idl), xlen = ++(xidl[0]); \
 78 | 		xidl[xlen] = (id); \
 79 | 	} while (0)
 80 | 
 81 | 	/** Search for an ID in an IDL.
 82 | 	 * @param[in] ids	The IDL to search.
 83 | 	 * @param[in] id	The ID to search for.
 84 | 	 * @return	The index of the first ID greater than or equal to \b id.
 85 | 	 */
 86 | unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id );
 87 | 
 88 | 	/** Allocate an IDL.
 89 | 	 * Allocates memory for an IDL of the given size.
 90 | 	 * @return	IDL on success, NULL on failure.
 91 | 	 */
 92 | MDB_IDL mdb_midl_alloc(int num);
 93 | 
 94 | 	/** Free an IDL.
 95 | 	 * @param[in] ids	The IDL to free.
 96 | 	 */
 97 | void mdb_midl_free(MDB_IDL ids);
 98 | 
 99 | 	/** Shrink an IDL.
100 | 	 * Return the IDL to the default size if it has grown larger.
101 | 	 * @param[in,out] idp	Address of the IDL to shrink.
102 | 	 */
103 | void mdb_midl_shrink(MDB_IDL *idp);
104 | 
105 | 	/** Make room for num additional elements in an IDL.
106 | 	 * @param[in,out] idp	Address of the IDL.
107 | 	 * @param[in] num	Number of elements to make room for.
108 | 	 * @return	0 on success, ENOMEM on failure.
109 | 	 */
110 | int mdb_midl_need(MDB_IDL *idp, unsigned num);
111 | 
112 | 	/** Append an ID onto an IDL.
113 | 	 * @param[in,out] idp	Address of the IDL to append to.
114 | 	 * @param[in] id	The ID to append.
115 | 	 * @return	0 on success, ENOMEM if the IDL is too large.
116 | 	 */
117 | int mdb_midl_append( MDB_IDL *idp, MDB_ID id );
118 | 
119 | 	/** Append an IDL onto an IDL.
120 | 	 * @param[in,out] idp	Address of the IDL to append to.
121 | 	 * @param[in] app	The IDL to append.
122 | 	 * @return	0 on success, ENOMEM if the IDL is too large.
123 | 	 */
124 | int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app );
125 | 
126 | 	/** Append an ID range onto an IDL.
127 | 	 * @param[in,out] idp	Address of the IDL to append to.
128 | 	 * @param[in] id	The lowest ID to append.
129 | 	 * @param[in] n		Number of IDs to append.
130 | 	 * @return	0 on success, ENOMEM if the IDL is too large.
131 | 	 */
132 | int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n );
133 | 
134 | 	/** Merge an IDL onto an IDL. The destination IDL must be big enough.
135 | 	 * @param[in] idl	The IDL to merge into.
136 | 	 * @param[in] merge	The IDL to merge.
137 | 	 */
138 | void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge );
139 | 
140 | 	/** Sort an IDL.
141 | 	 * @param[in,out] ids	The IDL to sort.
142 | 	 */
143 | void mdb_midl_sort( MDB_IDL ids );
144 | 
145 | 	/** An ID2 is an ID/pointer pair.
146 | 	 */
147 | typedef struct MDB_ID2 {
148 | 	MDB_ID mid;		/**< The ID */
149 | 	void *mptr;		/**< The pointer */
150 | } MDB_ID2;
151 | 
152 | 	/** An ID2L is an ID2 List, a sorted array of ID2s.
153 | 	 * The first element's \b mid member is a count of how many actual
154 | 	 * elements are in the array. The \b mptr member of the first element is unused.
155 | 	 * The array is sorted in ascending order by \b mid.
156 | 	 */
157 | typedef MDB_ID2 *MDB_ID2L;
158 | 
159 | 	/** Search for an ID in an ID2L.
160 | 	 * @param[in] ids	The ID2L to search.
161 | 	 * @param[in] id	The ID to search for.
162 | 	 * @return	The index of the first ID2 whose \b mid member is greater than or equal to \b id.
163 | 	 */
164 | unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id );
165 | 
166 | 
167 | 	/** Insert an ID2 into a ID2L.
168 | 	 * @param[in,out] ids	The ID2L to insert into.
169 | 	 * @param[in] id	The ID2 to insert.
170 | 	 * @return	0 on success, -1 if the ID was already present in the ID2L.
171 | 	 */
172 | int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id );
173 | 
174 | 	/** Append an ID2 into a ID2L.
175 | 	 * @param[in,out] ids	The ID2L to append into.
176 | 	 * @param[in] id	The ID2 to append.
177 | 	 * @return	0 on success, -2 if the ID2L is too big.
178 | 	 */
179 | int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id );
180 | 
181 | /** @} */
182 | /** @} */
183 | #ifdef __cplusplus
184 | }
185 | #endif
186 | #endif	/* _MDB_MIDL_H_ */
187 | 


--------------------------------------------------------------------------------
/lib/py-lmdb/env-copy-txn.patch:
--------------------------------------------------------------------------------
  1 | diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
  2 | index ff03c22..3d6f4cc 100644
  3 | --- a/libraries/liblmdb/lmdb.h
  4 | +++ b/libraries/liblmdb/lmdb.h
  5 | @@ -682,9 +682,14 @@ int  mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd);
  6 |  	 *		consumes more CPU and runs more slowly than the default.
  7 |  	 *		Currently it fails if the environment has suffered a page leak.
  8 |  	 * </ul>
  9 | +	 * @param[in] txn Transaction used for the copy.  If NULL, a temporary
 10 | +	 * transaction will be used.  This is only valid if the #MDB_CP_COMPACT
 11 | +	 * flag is set.
 12 | +	 *
 13 |  	 * @return A non-zero error value on failure and 0 on success.
 14 |  	 */
 15 |  int  mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags);
 16 | +int  mdb_env_copy3(MDB_env *env, const char *path, unsigned int flags, MDB_txn *txn);
 17 |  
 18 |  	/** @brief Copy an LMDB environment to the specified file descriptor,
 19 |  	 *	with options.
 20 | @@ -701,9 +706,13 @@ int  mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags);
 21 |  	 * have already been opened for Write access.
 22 |  	 * @param[in] flags Special options for this operation.
 23 |  	 * See #mdb_env_copy2() for options.
 24 | +	 * @param[in] txn Transaction used for the copy.  If NULL, a temporary
 25 | +	 * transaction will be used.  This is only valid if the #MDB_CP_COMPACT
 26 | +	 * flag is set.
 27 |  	 * @return A non-zero error value on failure and 0 on success.
 28 |  	 */
 29 |  int  mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags);
 30 | +int  mdb_env_copyfd3(MDB_env *env, mdb_filehandle_t fd, unsigned int flags, MDB_txn *txn);
 31 |  
 32 |  	/** @brief Return statistics about the LMDB environment.
 33 |  	 *
 34 | diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
 35 | index deb6779..b5d152c 100644
 36 | --- a/libraries/liblmdb/mdb.c
 37 | +++ b/libraries/liblmdb/mdb.c
 38 | @@ -9366,12 +9366,12 @@ done:
 39 |  
 40 |  	/** Copy environment with compaction. */
 41 |  static int ESECT
 42 | -mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 43 | +mdb_env_copyfd1(MDB_env *env, HANDLE fd, MDB_txn *txn)
 44 |  {
 45 |  	MDB_meta *mm;
 46 |  	MDB_page *mp;
 47 |  	mdb_copy my = {0};
 48 | -	MDB_txn *txn = NULL;
 49 | +	MDB_txn *orig_txn = txn;
 50 |  	pthread_t thr;
 51 |  	pgno_t root, new_root;
 52 |  	int rc = MDB_SUCCESS;
 53 | @@ -9417,9 +9417,11 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 54 |  	if (rc)
 55 |  		goto done;
 56 |  
 57 | -	rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
 58 | -	if (rc)
 59 | -		goto finish;
 60 | +	if (!txn) {
 61 | +		rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
 62 | +		if (rc)
 63 | +			goto finish;
 64 | +	}
 65 |  
 66 |  	mp = (MDB_page *)my.mc_wbuf[0];
 67 |  	memset(mp, 0, NUM_METAS * env->me_psize);
 68 | @@ -9479,7 +9481,8 @@ finish:
 69 |  		my.mc_error = rc;
 70 |  	mdb_env_cthr_toggle(&my, 1 | MDB_EOF);
 71 |  	rc = THREAD_FINISH(thr);
 72 | -	mdb_txn_abort(txn);
 73 | +	if (!orig_txn)
 74 | +		mdb_txn_abort(txn);
 75 |  
 76 |  done:
 77 |  #ifdef _WIN32
 78 | @@ -9596,12 +9599,22 @@ leave:
 79 |  }
 80 |  
 81 |  int ESECT
 82 | -mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags)
 83 | +mdb_env_copyfd3(MDB_env *env, HANDLE fd, unsigned int flags, MDB_txn *txn)
 84 |  {
 85 |  	if (flags & MDB_CP_COMPACT)
 86 | -		return mdb_env_copyfd1(env, fd);
 87 | +		return mdb_env_copyfd1(env, fd, txn);
 88 |  	else
 89 | +	{
 90 | +		if (txn) /* may only use txn with compact */
 91 | +			return EINVAL;
 92 |  		return mdb_env_copyfd0(env, fd);
 93 | +	}
 94 | +}
 95 | +
 96 | +int ESECT
 97 | +mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags)
 98 | +{
 99 | +	return mdb_env_copyfd3(env, fd, flags, NULL);
100 |  }
101 |  
102 |  int ESECT
103 | @@ -9612,6 +9625,12 @@ mdb_env_copyfd(MDB_env *env, HANDLE fd)
104 |  
105 |  int ESECT
106 |  mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
107 | +{
108 | +	return mdb_env_copy3(env, path, flags, NULL);
109 | +}
110 | +
111 | +int ESECT
112 | +mdb_env_copy3(MDB_env *env, const char *path, unsigned int flags, MDB_txn *txn)
113 |  {
114 |  	int rc;
115 |  	MDB_name fname;
116 | @@ -9623,7 +9642,7 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
117 |  		mdb_fname_destroy(fname);
118 |  	}
119 |  	if (rc == MDB_SUCCESS) {
120 | -		rc = mdb_env_copyfd2(env, newfd, flags);
121 | +		rc = mdb_env_copyfd3(env, newfd, flags, txn);
122 |  		if (close(newfd) < 0 && rc == MDB_SUCCESS)
123 |  			rc = ErrCode();
124 |  	}
125 | 


--------------------------------------------------------------------------------
/lib/py-lmdb/preload.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2015 The py-lmdb authors, all rights reserved.
 3 |  *
 4 |  * Redistribution and use in source and binary forms, with or without
 5 |  * modification, are permitted only as authorized by the OpenLDAP
 6 |  * Public License.
 7 |  *
 8 |  * A copy of this license is available in the file LICENSE in the
 9 |  * top-level directory of the distribution or, alternatively, at
10 |  * <http://www.OpenLDAP.org/license.html>.
11 |  *
12 |  * OpenLDAP is a registered trademark of the OpenLDAP Foundation.
13 |  *
14 |  * Individual files and/or contributed packages may be copyright by
15 |  * other parties and/or subject to additional restrictions.
16 |  *
17 |  * This work also contains materials derived from public sources.
18 |  *
19 |  * Additional information about OpenLDAP can be obtained at
20 |  * <http://www.openldap.org/>.
21 |  */
22 | 
23 | #ifndef LMDB_PRELOAD_H
24 | #define LMDB_PRELOAD_H
25 | 
26 | #include <stdlib.h>
27 | 
28 | /**
29 |  * Touch a byte from every page in `x`, causing any read faults necessary for
30 |  * copying the value to occur. This should be called with the GIL released, in
31 |  * order to dramatically decrease the chances of a page fault being taken with
32 |  * the GIL held.
33 |  *
34 |  * We do this since PyMalloc cannot be invoked with the GIL released, and we
35 |  * cannot know the size of the MDB result value before dropping the GIL. This
36 |  * seems the simplest and cheapest compromise to ensuring multithreaded Python
37 |  * apps don't hard stall when dealing with a database larger than RAM.
38 |  */
39 | static void preload(int rc, void *x, size_t size) {
40 |     if(rc == 0) {
41 |         volatile char j;
42 |         size_t i;
43 |         for(i = 0; i < size; i += 4096) {
44 |             j = ((volatile char *)x)[i];
45 |         }
46 |         (void) j; /* -Wunused-variable */
47 |     }
48 | }
49 | 
50 | #endif /* !LMDB_PRELOAD_H */
51 | 


--------------------------------------------------------------------------------
/lib/win32-stdint/stdint.h:
--------------------------------------------------------------------------------
  1 | // ISO C9x  compliant stdint.h for Microsoft Visual Studio
  2 | // Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124
  3 | //
  4 | //  Copyright (c) 2006-2013 Alexander Chemeris
  5 | //
  6 | // Redistribution and use in source and binary forms, with or without
  7 | // modification, are permitted provided that the following conditions are met:
  8 | //
  9 | //   1. Redistributions of source code must retain the above copyright notice,
 10 | //      this list of conditions and the following disclaimer.
 11 | //
 12 | //   2. Redistributions in binary form must reproduce the above copyright
 13 | //      notice, this list of conditions and the following disclaimer in the
 14 | //      documentation and/or other materials provided with the distribution.
 15 | //
 16 | //   3. Neither the name of the product nor the names of its contributors may
 17 | //      be used to endorse or promote products derived from this software
 18 | //      without specific prior written permission.
 19 | //
 20 | // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
 21 | // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 22 | // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
 23 | // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 25 | // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 26 | // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 27 | // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 28 | // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 29 | // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 30 | //
 31 | ///////////////////////////////////////////////////////////////////////////////
 32 | 
 33 | #ifndef _MSC_VER // [
 34 | #error "Use this header only with Microsoft Visual C++ compilers!"
 35 | #endif // _MSC_VER ]
 36 | 
 37 | #ifndef _MSC_STDINT_H_ // [
 38 | #define _MSC_STDINT_H_
 39 | 
 40 | #if _MSC_VER > 1000
 41 | #pragma once
 42 | #endif
 43 | 
 44 | #if _MSC_VER >= 1600 // [
 45 | #include <stdint.h>
 46 | #else // ] _MSC_VER >= 1600 [
 47 | 
 48 | #include <limits.h>
 49 | 
 50 | // For Visual Studio 6 in C++ mode and for many Visual Studio versions when
 51 | // compiling for ARM we should wrap <wchar.h> include with 'extern "C++" {}'
 52 | // or compiler give many errors like this:
 53 | //   error C2733: second C linkage of overloaded function 'wmemchr' not allowed
 54 | #ifdef __cplusplus
 55 | extern "C" {
 56 | #endif
 57 | #  include <wchar.h>
 58 | #ifdef __cplusplus
 59 | }
 60 | #endif
 61 | 
 62 | // Define _W64 macros to mark types changing their size, like intptr_t.
 63 | #ifndef _W64
 64 | #  if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300
 65 | #     define _W64 __w64
 66 | #  else
 67 | #     define _W64
 68 | #  endif
 69 | #endif
 70 | 
 71 | 
 72 | // 7.18.1 Integer types
 73 | 
 74 | // 7.18.1.1 Exact-width integer types
 75 | 
 76 | // Visual Studio 6 and Embedded Visual C++ 4 doesn't
 77 | // realize that, e.g. char has the same size as __int8
 78 | // so we give up on __intX for them.
 79 | #if (_MSC_VER < 1300)
 80 |    typedef signed char       int8_t;
 81 |    typedef signed short      int16_t;
 82 |    typedef signed int        int32_t;
 83 |    typedef unsigned char     uint8_t;
 84 |    typedef unsigned short    uint16_t;
 85 |    typedef unsigned int      uint32_t;
 86 | #else
 87 |    typedef signed __int8     int8_t;
 88 |    typedef signed __int16    int16_t;
 89 |    typedef signed __int32    int32_t;
 90 |    typedef unsigned __int8   uint8_t;
 91 |    typedef unsigned __int16  uint16_t;
 92 |    typedef unsigned __int32  uint32_t;
 93 | #endif
 94 | typedef signed __int64       int64_t;
 95 | typedef unsigned __int64     uint64_t;
 96 | 
 97 | 
 98 | // 7.18.1.2 Minimum-width integer types
 99 | typedef int8_t    int_least8_t;
100 | typedef int16_t   int_least16_t;
101 | typedef int32_t   int_least32_t;
102 | typedef int64_t   int_least64_t;
103 | typedef uint8_t   uint_least8_t;
104 | typedef uint16_t  uint_least16_t;
105 | typedef uint32_t  uint_least32_t;
106 | typedef uint64_t  uint_least64_t;
107 | 
108 | // 7.18.1.3 Fastest minimum-width integer types
109 | typedef int8_t    int_fast8_t;
110 | typedef int16_t   int_fast16_t;
111 | typedef int32_t   int_fast32_t;
112 | typedef int64_t   int_fast64_t;
113 | typedef uint8_t   uint_fast8_t;
114 | typedef uint16_t  uint_fast16_t;
115 | typedef uint32_t  uint_fast32_t;
116 | typedef uint64_t  uint_fast64_t;
117 | 
118 | // 7.18.1.4 Integer types capable of holding object pointers
119 | #ifdef _WIN64 // [
120 |    typedef signed __int64    intptr_t;
121 |    typedef unsigned __int64  uintptr_t;
122 | #else // _WIN64 ][
123 |    typedef _W64 signed int   intptr_t;
124 |    typedef _W64 unsigned int uintptr_t;
125 | #endif // _WIN64 ]
126 | 
127 | // 7.18.1.5 Greatest-width integer types
128 | typedef int64_t   intmax_t;
129 | typedef uint64_t  uintmax_t;
130 | 
131 | 
132 | // 7.18.2 Limits of specified-width integer types
133 | 
134 | #if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [   See footnote 220 at page 257 and footnote 221 at page 259
135 | 
136 | // 7.18.2.1 Limits of exact-width integer types
137 | #define INT8_MIN     ((int8_t)_I8_MIN)
138 | #define INT8_MAX     _I8_MAX
139 | #define INT16_MIN    ((int16_t)_I16_MIN)
140 | #define INT16_MAX    _I16_MAX
141 | #define INT32_MIN    ((int32_t)_I32_MIN)
142 | #define INT32_MAX    _I32_MAX
143 | #define INT64_MIN    ((int64_t)_I64_MIN)
144 | #define INT64_MAX    _I64_MAX
145 | #define UINT8_MAX    _UI8_MAX
146 | #define UINT16_MAX   _UI16_MAX
147 | #define UINT32_MAX   _UI32_MAX
148 | #define UINT64_MAX   _UI64_MAX
149 | 
150 | // 7.18.2.2 Limits of minimum-width integer types
151 | #define INT_LEAST8_MIN    INT8_MIN
152 | #define INT_LEAST8_MAX    INT8_MAX
153 | #define INT_LEAST16_MIN   INT16_MIN
154 | #define INT_LEAST16_MAX   INT16_MAX
155 | #define INT_LEAST32_MIN   INT32_MIN
156 | #define INT_LEAST32_MAX   INT32_MAX
157 | #define INT_LEAST64_MIN   INT64_MIN
158 | #define INT_LEAST64_MAX   INT64_MAX
159 | #define UINT_LEAST8_MAX   UINT8_MAX
160 | #define UINT_LEAST16_MAX  UINT16_MAX
161 | #define UINT_LEAST32_MAX  UINT32_MAX
162 | #define UINT_LEAST64_MAX  UINT64_MAX
163 | 
164 | // 7.18.2.3 Limits of fastest minimum-width integer types
165 | #define INT_FAST8_MIN    INT8_MIN
166 | #define INT_FAST8_MAX    INT8_MAX
167 | #define INT_FAST16_MIN   INT16_MIN
168 | #define INT_FAST16_MAX   INT16_MAX
169 | #define INT_FAST32_MIN   INT32_MIN
170 | #define INT_FAST32_MAX   INT32_MAX
171 | #define INT_FAST64_MIN   INT64_MIN
172 | #define INT_FAST64_MAX   INT64_MAX
173 | #define UINT_FAST8_MAX   UINT8_MAX
174 | #define UINT_FAST16_MAX  UINT16_MAX
175 | #define UINT_FAST32_MAX  UINT32_MAX
176 | #define UINT_FAST64_MAX  UINT64_MAX
177 | 
178 | // 7.18.2.4 Limits of integer types capable of holding object pointers
179 | #ifdef _WIN64 // [
180 | #  define INTPTR_MIN   INT64_MIN
181 | #  define INTPTR_MAX   INT64_MAX
182 | #  define UINTPTR_MAX  UINT64_MAX
183 | #else // _WIN64 ][
184 | #  define INTPTR_MIN   INT32_MIN
185 | #  define INTPTR_MAX   INT32_MAX
186 | #  define UINTPTR_MAX  UINT32_MAX
187 | #endif // _WIN64 ]
188 | 
189 | // 7.18.2.5 Limits of greatest-width integer types
190 | #define INTMAX_MIN   INT64_MIN
191 | #define INTMAX_MAX   INT64_MAX
192 | #define UINTMAX_MAX  UINT64_MAX
193 | 
194 | // 7.18.3 Limits of other integer types
195 | 
196 | #ifdef _WIN64 // [
197 | #  define PTRDIFF_MIN  _I64_MIN
198 | #  define PTRDIFF_MAX  _I64_MAX
199 | #else  // _WIN64 ][
200 | #  define PTRDIFF_MIN  _I32_MIN
201 | #  define PTRDIFF_MAX  _I32_MAX
202 | #endif  // _WIN64 ]
203 | 
204 | #define SIG_ATOMIC_MIN  INT_MIN
205 | #define SIG_ATOMIC_MAX  INT_MAX
206 | 
207 | #ifndef SIZE_MAX // [
208 | #  ifdef _WIN64 // [
209 | #     define SIZE_MAX  _UI64_MAX
210 | #  else // _WIN64 ][
211 | #     define SIZE_MAX  _UI32_MAX
212 | #  endif // _WIN64 ]
213 | #endif // SIZE_MAX ]
214 | 
215 | // WCHAR_MIN and WCHAR_MAX are also defined in <wchar.h>
216 | #ifndef WCHAR_MIN // [
217 | #  define WCHAR_MIN  0
218 | #endif  // WCHAR_MIN ]
219 | #ifndef WCHAR_MAX // [
220 | #  define WCHAR_MAX  _UI16_MAX
221 | #endif  // WCHAR_MAX ]
222 | 
223 | #define WINT_MIN  0
224 | #define WINT_MAX  _UI16_MAX
225 | 
226 | #endif // __STDC_LIMIT_MACROS ]
227 | 
228 | 
229 | // 7.18.4 Limits of other integer types
230 | 
231 | #if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [   See footnote 224 at page 260
232 | 
233 | // 7.18.4.1 Macros for minimum-width integer constants
234 | 
235 | #define INT8_C(val)  val##i8
236 | #define INT16_C(val) val##i16
237 | #define INT32_C(val) val##i32
238 | #define INT64_C(val) val##i64
239 | 
240 | #define UINT8_C(val)  val##ui8
241 | #define UINT16_C(val) val##ui16
242 | #define UINT32_C(val) val##ui32
243 | #define UINT64_C(val) val##ui64
244 | 
245 | // 7.18.4.2 Macros for greatest-width integer constants
246 | // These #ifndef's are needed to prevent collisions with <boost/cstdint.hpp>.
247 | // Check out Issue 9 for the details.
248 | #ifndef INTMAX_C //   [
249 | #  define INTMAX_C   INT64_C
250 | #endif // INTMAX_C    ]
251 | #ifndef UINTMAX_C //  [
252 | #  define UINTMAX_C  UINT64_C
253 | #endif // UINTMAX_C   ]
254 | 
255 | #endif // __STDC_CONSTANT_MACROS ]
256 | 
257 | #endif // _MSC_VER >= 1600 ]
258 | 
259 | #endif // _MSC_STDINT_H_ ]
260 | 


--------------------------------------------------------------------------------
/lib/win32/inttypes.h:
--------------------------------------------------------------------------------
  1 | // ISO C9x  compliant inttypes.h for Microsoft Visual Studio
  2 | // Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124
  3 | //
  4 | //  Copyright (c) 2006-2013 Alexander Chemeris
  5 | //
  6 | // Redistribution and use in source and binary forms, with or without
  7 | // modification, are permitted provided that the following conditions are met:
  8 | //
  9 | //   1. Redistributions of source code must retain the above copyright notice,
 10 | //      this list of conditions and the following disclaimer.
 11 | //
 12 | //   2. Redistributions in binary form must reproduce the above copyright
 13 | //      notice, this list of conditions and the following disclaimer in the
 14 | //      documentation and/or other materials provided with the distribution.
 15 | //
 16 | //   3. Neither the name of the product nor the names of its contributors may
 17 | //      be used to endorse or promote products derived from this software
 18 | //      without specific prior written permission.
 19 | //
 20 | // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
 21 | // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 22 | // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
 23 | // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 25 | // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 26 | // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 27 | // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 28 | // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 29 | // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 30 | //
 31 | ///////////////////////////////////////////////////////////////////////////////
 32 | 
 33 | #ifndef _MSC_VER // [
 34 | #error "Use this header only with Microsoft Visual C++ compilers!"
 35 | #endif // _MSC_VER ]
 36 | 
 37 | #ifndef _MSC_INTTYPES_H_ // [
 38 | #define _MSC_INTTYPES_H_
 39 | 
 40 | #if _MSC_VER > 1000
 41 | #pragma once
 42 | #endif
 43 | 
 44 | #include "stdint.h"
 45 | 
 46 | // 7.8 Format conversion of integer types
 47 | 
 48 | typedef struct {
 49 |    intmax_t quot;
 50 |    intmax_t rem;
 51 | } imaxdiv_t;
 52 | 
 53 | // 7.8.1 Macros for format specifiers
 54 | 
 55 | #if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [   See footnote 185 at page 198
 56 | 
 57 | // The fprintf macros for signed integers are:
 58 | #define PRId8       "d"
 59 | #define PRIi8       "i"
 60 | #define PRIdLEAST8  "d"
 61 | #define PRIiLEAST8  "i"
 62 | #define PRIdFAST8   "d"
 63 | #define PRIiFAST8   "i"
 64 | 
 65 | #define PRId16       "hd"
 66 | #define PRIi16       "hi"
 67 | #define PRIdLEAST16  "hd"
 68 | #define PRIiLEAST16  "hi"
 69 | #define PRIdFAST16   "hd"
 70 | #define PRIiFAST16   "hi"
 71 | 
 72 | #define PRId32       "I32d"
 73 | #define PRIi32       "I32i"
 74 | #define PRIdLEAST32  "I32d"
 75 | #define PRIiLEAST32  "I32i"
 76 | #define PRIdFAST32   "I32d"
 77 | #define PRIiFAST32   "I32i"
 78 | 
 79 | #define PRId64       "I64d"
 80 | #define PRIi64       "I64i"
 81 | #define PRIdLEAST64  "I64d"
 82 | #define PRIiLEAST64  "I64i"
 83 | #define PRIdFAST64   "I64d"
 84 | #define PRIiFAST64   "I64i"
 85 | 
 86 | #define PRIdMAX     "I64d"
 87 | #define PRIiMAX     "I64i"
 88 | 
 89 | #define PRIdPTR     "Id"
 90 | #define PRIiPTR     "Ii"
 91 | 
 92 | // The fprintf macros for unsigned integers are:
 93 | #define PRIo8       "o"
 94 | #define PRIu8       "u"
 95 | #define PRIx8       "x"
 96 | #define PRIX8       "X"
 97 | #define PRIoLEAST8  "o"
 98 | #define PRIuLEAST8  "u"
 99 | #define PRIxLEAST8  "x"
100 | #define PRIXLEAST8  "X"
101 | #define PRIoFAST8   "o"
102 | #define PRIuFAST8   "u"
103 | #define PRIxFAST8   "x"
104 | #define PRIXFAST8   "X"
105 | 
106 | #define PRIo16       "ho"
107 | #define PRIu16       "hu"
108 | #define PRIx16       "hx"
109 | #define PRIX16       "hX"
110 | #define PRIoLEAST16  "ho"
111 | #define PRIuLEAST16  "hu"
112 | #define PRIxLEAST16  "hx"
113 | #define PRIXLEAST16  "hX"
114 | #define PRIoFAST16   "ho"
115 | #define PRIuFAST16   "hu"
116 | #define PRIxFAST16   "hx"
117 | #define PRIXFAST16   "hX"
118 | 
119 | #define PRIo32       "I32o"
120 | #define PRIu32       "I32u"
121 | #define PRIx32       "I32x"
122 | #define PRIX32       "I32X"
123 | #define PRIoLEAST32  "I32o"
124 | #define PRIuLEAST32  "I32u"
125 | #define PRIxLEAST32  "I32x"
126 | #define PRIXLEAST32  "I32X"
127 | #define PRIoFAST32   "I32o"
128 | #define PRIuFAST32   "I32u"
129 | #define PRIxFAST32   "I32x"
130 | #define PRIXFAST32   "I32X"
131 | 
132 | #define PRIo64       "I64o"
133 | #define PRIu64       "I64u"
134 | #define PRIx64       "I64x"
135 | #define PRIX64       "I64X"
136 | #define PRIoLEAST64  "I64o"
137 | #define PRIuLEAST64  "I64u"
138 | #define PRIxLEAST64  "I64x"
139 | #define PRIXLEAST64  "I64X"
140 | #define PRIoFAST64   "I64o"
141 | #define PRIuFAST64   "I64u"
142 | #define PRIxFAST64   "I64x"
143 | #define PRIXFAST64   "I64X"
144 | 
145 | #define PRIoMAX     "I64o"
146 | #define PRIuMAX     "I64u"
147 | #define PRIxMAX     "I64x"
148 | #define PRIXMAX     "I64X"
149 | 
150 | #define PRIoPTR     "Io"
151 | #define PRIuPTR     "Iu"
152 | #define PRIxPTR     "Ix"
153 | #define PRIXPTR     "IX"
154 | 
155 | // The fscanf macros for signed integers are:
156 | #define SCNd8       "d"
157 | #define SCNi8       "i"
158 | #define SCNdLEAST8  "d"
159 | #define SCNiLEAST8  "i"
160 | #define SCNdFAST8   "d"
161 | #define SCNiFAST8   "i"
162 | 
163 | #define SCNd16       "hd"
164 | #define SCNi16       "hi"
165 | #define SCNdLEAST16  "hd"
166 | #define SCNiLEAST16  "hi"
167 | #define SCNdFAST16   "hd"
168 | #define SCNiFAST16   "hi"
169 | 
170 | #define SCNd32       "ld"
171 | #define SCNi32       "li"
172 | #define SCNdLEAST32  "ld"
173 | #define SCNiLEAST32  "li"
174 | #define SCNdFAST32   "ld"
175 | #define SCNiFAST32   "li"
176 | 
177 | #define SCNd64       "I64d"
178 | #define SCNi64       "I64i"
179 | #define SCNdLEAST64  "I64d"
180 | #define SCNiLEAST64  "I64i"
181 | #define SCNdFAST64   "I64d"
182 | #define SCNiFAST64   "I64i"
183 | 
184 | #define SCNdMAX     "I64d"
185 | #define SCNiMAX     "I64i"
186 | 
187 | #ifdef _WIN64 // [
188 | #  define SCNdPTR     "I64d"
189 | #  define SCNiPTR     "I64i"
190 | #else  // _WIN64 ][
191 | #  define SCNdPTR     "ld"
192 | #  define SCNiPTR     "li"
193 | #endif  // _WIN64 ]
194 | 
195 | // The fscanf macros for unsigned integers are:
196 | #define SCNo8       "o"
197 | #define SCNu8       "u"
198 | #define SCNx8       "x"
199 | #define SCNX8       "X"
200 | #define SCNoLEAST8  "o"
201 | #define SCNuLEAST8  "u"
202 | #define SCNxLEAST8  "x"
203 | #define SCNXLEAST8  "X"
204 | #define SCNoFAST8   "o"
205 | #define SCNuFAST8   "u"
206 | #define SCNxFAST8   "x"
207 | #define SCNXFAST8   "X"
208 | 
209 | #define SCNo16       "ho"
210 | #define SCNu16       "hu"
211 | #define SCNx16       "hx"
212 | #define SCNX16       "hX"
213 | #define SCNoLEAST16  "ho"
214 | #define SCNuLEAST16  "hu"
215 | #define SCNxLEAST16  "hx"
216 | #define SCNXLEAST16  "hX"
217 | #define SCNoFAST16   "ho"
218 | #define SCNuFAST16   "hu"
219 | #define SCNxFAST16   "hx"
220 | #define SCNXFAST16   "hX"
221 | 
222 | #define SCNo32       "lo"
223 | #define SCNu32       "lu"
224 | #define SCNx32       "lx"
225 | #define SCNX32       "lX"
226 | #define SCNoLEAST32  "lo"
227 | #define SCNuLEAST32  "lu"
228 | #define SCNxLEAST32  "lx"
229 | #define SCNXLEAST32  "lX"
230 | #define SCNoFAST32   "lo"
231 | #define SCNuFAST32   "lu"
232 | #define SCNxFAST32   "lx"
233 | #define SCNXFAST32   "lX"
234 | 
235 | #define SCNo64       "I64o"
236 | #define SCNu64       "I64u"
237 | #define SCNx64       "I64x"
238 | #define SCNX64       "I64X"
239 | #define SCNoLEAST64  "I64o"
240 | #define SCNuLEAST64  "I64u"
241 | #define SCNxLEAST64  "I64x"
242 | #define SCNXLEAST64  "I64X"
243 | #define SCNoFAST64   "I64o"
244 | #define SCNuFAST64   "I64u"
245 | #define SCNxFAST64   "I64x"
246 | #define SCNXFAST64   "I64X"
247 | 
248 | #define SCNoMAX     "I64o"
249 | #define SCNuMAX     "I64u"
250 | #define SCNxMAX     "I64x"
251 | #define SCNXMAX     "I64X"
252 | 
253 | #ifdef _WIN64 // [
254 | #  define SCNoPTR     "I64o"
255 | #  define SCNuPTR     "I64u"
256 | #  define SCNxPTR     "I64x"
257 | #  define SCNXPTR     "I64X"
258 | #else  // _WIN64 ][
259 | #  define SCNoPTR     "lo"
260 | #  define SCNuPTR     "lu"
261 | #  define SCNxPTR     "lx"
262 | #  define SCNXPTR     "lX"
263 | #endif  // _WIN64 ]
264 | 
265 | #endif // __STDC_FORMAT_MACROS ]
266 | 
267 | // 7.8.2 Functions for greatest-width integer types
268 | 
269 | // 7.8.2.1 The imaxabs function
270 | #define imaxabs _abs64
271 | 
272 | // 7.8.2.2 The imaxdiv function
273 | 
274 | // This is modified version of div() function from Microsoft's div.c found
275 | // in %MSVC.NET%\crt\src\div.c
276 | #ifdef STATIC_IMAXDIV // [
277 | static
278 | #else // STATIC_IMAXDIV ][
279 | _inline
280 | #endif // STATIC_IMAXDIV ]
281 | imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom)
282 | {
283 |    imaxdiv_t result;
284 | 
285 |    result.quot = numer / denom;
286 |    result.rem = numer % denom;
287 | 
288 |    if (numer < 0 && result.rem > 0) {
289 |       // did division wrong; must fix up
290 |       ++result.quot;
291 |       result.rem -= denom;
292 |    }
293 | 
294 |    return result;
295 | }
296 | 
297 | // 7.8.2.3 The strtoimax and strtoumax functions
298 | #define strtoimax _strtoi64
299 | #define strtoumax _strtoui64
300 | 
301 | // 7.8.2.4 The wcstoimax and wcstoumax functions
302 | #define wcstoimax _wcstoi64
303 | #define wcstoumax _wcstoui64
304 | 
305 | 
306 | #endif // _MSC_INTTYPES_H_ ]
307 | 


--------------------------------------------------------------------------------
/lib/win32/unistd.h:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jnwatson/py-lmdb/4806170f185642d5a96b48ec10823a57d5e94080/lib/win32/unistd.h


--------------------------------------------------------------------------------
/lmdb/__init__.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2013-2021 The py-lmdb authors, all rights reserved.
 2 | #
 3 | # Redistribution and use in source and binary forms, with or without
 4 | # modification, are permitted only as authorized by the OpenLDAP
 5 | # Public License.
 6 | #
 7 | # A copy of this license is available in the file LICENSE in the
 8 | # top-level directory of the distribution or, alternatively, at
 9 | # <http://www.OpenLDAP.org/license.html>.
10 | #
11 | # OpenLDAP is a registered trademark of the OpenLDAP Foundation.
12 | #
13 | # Individual files and/or contributed packages may be copyright by
14 | # other parties and/or subject to additional restrictions.
15 | #
16 | # This work also contains materials derived from public sources.
17 | #
18 | # Additional information about OpenLDAP can be obtained at
19 | # <http://www.openldap.org/>.
20 | 
21 | """
22 | Python wrapper for OpenLDAP's "Lightning" MDB database.
23 | 
24 | Please see https://lmdb.readthedocs.io/
25 | """
26 | 
27 | import os
28 | import sys
29 | 
30 | def _reading_docs():
31 |     # Hack: disable speedups while testing or reading docstrings. Don't check
32 |     # for basename for embedded python - variable 'argv' does not exists there or is empty.
33 |     if not(hasattr(sys, 'argv')) or not sys.argv:
34 |         return False
35 | 
36 |     basename = os.path.basename(sys.argv[0])
37 |     return any(x in basename for x in ('sphinx-build', 'pydoc'))
38 | 
39 | try:
40 |     if _reading_docs() or os.getenv('LMDB_FORCE_CFFI') is not None:
41 |         raise ImportError
42 |     from lmdb.cpython import *
43 |     from lmdb.cpython import open
44 |     from lmdb.cpython import __all__
45 | except ImportError:
46 |     if (not _reading_docs()) and os.getenv('LMDB_FORCE_CPYTHON') is not None:
47 |         raise
48 |     from lmdb.cffi import *
49 |     from lmdb.cffi import open
50 |     from lmdb.cffi import __all__
51 |     from lmdb.cffi import __doc__
52 | 
53 | __version__ = '1.7.0.dev0'
54 | 


--------------------------------------------------------------------------------
/lmdb/__main__.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2013-2024 The py-lmdb authors, all rights reserved.
 2 | #
 3 | # Redistribution and use in source and binary forms, with or without
 4 | # modification, are permitted only as authorized by the OpenLDAP
 5 | # Public License.
 6 | #
 7 | # A copy of this license is available in the file LICENSE in the
 8 | # top-level directory of the distribution or, alternatively, at
 9 | # <http://www.OpenLDAP.org/license.html>.
10 | #
11 | # OpenLDAP is a registered trademark of the OpenLDAP Foundation.
12 | #
13 | # Individual files and/or contributed packages may be copyright by
14 | # other parties and/or subject to additional restrictions.
15 | #
16 | # This work also contains materials derived from public sources.
17 | #
18 | # Additional information about OpenLDAP can be obtained at
19 | # <http://www.openldap.org/>.
20 | 
21 | # Hack to support Python >=v2.6 'python -mlmdb'
22 | from __future__ import absolute_import
23 | import lmdb.tool
24 | lmdb.tool.main()
25 | 


--------------------------------------------------------------------------------
/misc/cursor-del-break.c:
--------------------------------------------------------------------------------
 1 | // http://www.openldap.org/its/index.cgi/Software%20Bugs?id=7722
 2 | // gcc -g -I src/py-lmdb/lib -o cursor-del-break cursor-del-break.c
 3 | #include <stdlib.h>
 4 | #include <unistd.h>
 5 | #include <stdio.h>
 6 | #include <string.h>
 7 | #include <ctype.h>
 8 | 
 9 | #include "lmdb.h"
10 | #include "mdb.c"
11 | #include "midl.c"
12 | 
13 | 
14 | void check(int x)
15 | {
16 |     if(x) {
17 |         fprintf(stderr, "eek %s\n", mdb_strerror(x));
18 |         _exit(1);
19 |     }
20 | }
21 | 
22 | #define RECS 2048
23 | #define DB_PATH "/ram/tdb"
24 | 
25 | MDB_dbi dbi;
26 | MDB_txn *txn;
27 | MDB_env *env;
28 | MDB_cursor *c1;
29 | 
30 | char recpattern[256];
31 | MDB_val keyv;
32 | MDB_val valv;
33 | 
34 | void new_txn(void)
35 | {
36 |     if(txn) {
37 |         fprintf(stderr, "commit\n");
38 |         check(mdb_txn_commit(txn));
39 |     }
40 |     check(mdb_txn_begin(env, NULL, 0, &txn));
41 | }
42 | 
43 | int main(void)
44 | {
45 |     check(mdb_env_create(&env));
46 |     check(mdb_env_set_mapsize(env, 1048576UL*1024UL*3UL));
47 |     check(mdb_env_set_maxreaders(env, 126));
48 |     check(mdb_env_set_maxdbs(env, 1));
49 |     if(! access(DB_PATH, X_OK)) {
50 |         system("rm -rf " DB_PATH);
51 |     }
52 |     check(mkdir(DB_PATH, 0777));
53 |     check(mdb_env_open(env, DB_PATH, MDB_MAPASYNC|MDB_NOSYNC|MDB_NOMETASYNC, 0644));
54 |     new_txn();
55 |     check(mdb_dbi_open(txn, NULL, 0, &dbi));
56 | 
57 |     // make pattern
58 |     int i;
59 |     for(i = 0; i < sizeof recpattern; i++) {
60 |         recpattern[i] = i % 256;
61 |     }
62 | 
63 |     for(i = 0; i < RECS; i++) {
64 |         char keybuf[40];
65 |         keyv.mv_size = sprintf(keybuf, "%08x", i);
66 |         keyv.mv_data = keybuf;
67 |         valv.mv_size = sizeof recpattern;
68 |         valv.mv_data = recpattern;
69 |         check(mdb_put(txn, dbi, &keyv, &valv, 0));
70 |     }
71 | 
72 |     new_txn();
73 | 
74 |     check(mdb_cursor_open(txn, dbi, &c1));
75 |     check(mdb_cursor_get(c1, &keyv, &valv, MDB_FIRST));
76 |     check(mdb_del(txn, dbi, &keyv, NULL));
77 | 
78 |     for(i = 1; i < RECS; i++) {
79 |         check(mdb_cursor_get(c1, &keyv, &valv, MDB_NEXT));
80 |         char keybuf[40];
81 |         int sz = sprintf(keybuf, "%08x", i);
82 |         check((!(sz==keyv.mv_size)) || memcmp(keyv.mv_data, keybuf, sz));
83 |         check(memcmp(valv.mv_data, recpattern, sizeof recpattern));
84 |         printf("%d\n", i);
85 |         check(mdb_del(txn, dbi, &keyv, NULL));
86 |     }
87 | 
88 |     new_txn();
89 | }
90 | 


--------------------------------------------------------------------------------
/misc/cursor-iter-bug.c:
--------------------------------------------------------------------------------
 1 | // Demonstrates a bug introduced in upstream 0.9.27.  Fixed in 0.9.29
 2 | // https://bugs.openldap.org/show_bug.cgi?id=9461
 3 | // gcc -g -I ../lib -o cursor-iter-bug cursor-iter-bug.c -lpthread
 4 | // In a dupsort DB, deleting a value before where a cursor is set causes the
 5 | // cursor to lose its place.  mdb_cursor_get with MDB_NEXT will improperly
 6 | // return the current value and not MDB_NOT_FOUND as is correct.
 7 | 
 8 | #include <stdlib.h>
 9 | #include <unistd.h>
10 | #include <stdio.h>
11 | #include <string.h>
12 | #include <ctype.h>
13 | #include <assert.h>
14 | 
15 | #include "lmdb.h"
16 | #include "mdb.c"
17 | #include "midl.c"
18 | 
19 | 
20 | void check(int x)
21 | {
22 |     if(x) {
23 |         fprintf(stderr, "eek %s\n", mdb_strerror(x));
24 |         _exit(1);
25 |     }
26 | }
27 | 
28 | #define DB_PATH "tmp.lmdb"
29 | 
30 | MDB_txn *txn;
31 | MDB_env *env;
32 | 
33 | void new_txn(void)
34 | {
35 |     if(txn) {
36 |         fprintf(stderr, "commit\n");
37 |         check(mdb_txn_commit(txn));
38 |     }
39 |     check(mdb_txn_begin(env, NULL, 0, &txn));
40 | }
41 | 
42 | int main(void)
43 | {
44 | 
45 |     MDB_dbi dbi;
46 |     MDB_cursor *c1;
47 |     MDB_val keyv;
48 |     MDB_val valv;
49 | 
50 |     check(mdb_env_create(&env));
51 |     check(mdb_env_set_mapsize(env, 1048576UL*1024UL*3UL));
52 |     check(mdb_env_set_maxreaders(env, 126));
53 |     check(mdb_env_set_maxdbs(env, 2));
54 |     if(! access(DB_PATH, X_OK)) {
55 |         system("rm -rf " DB_PATH);
56 |     }
57 |     check(mkdir(DB_PATH, 0777));
58 |     check(mdb_env_open(env, DB_PATH, MDB_MAPASYNC|MDB_NOSYNC|MDB_NOMETASYNC, 0644));
59 |     new_txn();
60 |     check(mdb_dbi_open(txn, "db", MDB_DUPSORT | MDB_CREATE, &dbi));
61 | 
62 |     new_txn();
63 |     check(mdb_cursor_open(txn, dbi, &c1));
64 | 
65 |     keyv.mv_size = 2;
66 |     keyv.mv_data = "\x00\x01";
67 |     valv.mv_size = 4;
68 |     valv.mv_data = "hehe";
69 |     check(mdb_cursor_put(c1, &keyv, &valv, 0));
70 | 
71 |     keyv.mv_size = 2;
72 |     keyv.mv_data = "\x00\x02";
73 |     valv.mv_size = 4;
74 |     valv.mv_data = "haha";
75 |     check(mdb_cursor_put(c1, &keyv, &valv, 0));
76 | 
77 |     check(mdb_cursor_get(c1, &keyv, &valv, MDB_SET_KEY));
78 | 
79 |     check(mdb_cursor_get(c1, &keyv, &valv, MDB_GET_CURRENT));
80 |     assert(keyv.mv_size == 2);
81 |     char * key = keyv.mv_data;
82 |     assert(key[0] == 0 && key[1] == 2);
83 | 
84 |     keyv.mv_size = 2;
85 |     keyv.mv_data = "\x00\x01";
86 |     check(mdb_del(txn, dbi, &keyv, NULL));
87 | 
88 |     int rc = mdb_cursor_get(c1, &keyv, &valv, MDB_NEXT);
89 | 
90 |     // Below assertion fails in LMDB 0.9.27 and LMDB 0.9.28 (and passes in 0.9.26)
91 | 
92 |     assert(rc == MDB_NOTFOUND);
93 |     new_txn();
94 | }
95 | 


--------------------------------------------------------------------------------
/misc/cursor_put_pyparse.diff:
--------------------------------------------------------------------------------
 1 | diff --git a/lmdb/cpython.c b/lmdb/cpython.c
 2 | index dd1c8b9..ced5ea3 100644
 3 | --- a/lmdb/cpython.c
 4 | +++ b/lmdb/cpython.c
 5 | @@ -2319,11 +2319,25 @@ cursor_put(CursorObject *self, PyObject *args, PyObject *kwds)
 6 |          {ARG_BOOL, OVERWRITE_S, OFFSET(cursor_put, overwrite)},
 7 |          {ARG_BOOL, APPEND_S, OFFSET(cursor_put, append)}
 8 |      };
 9 | +    static char *keywords[] = {
10 | +        "key", "value", "dupdata", "overwrite", "append", NULL
11 | +    };
12 | +    PyObject *key, *val;
13 |      int flags;
14 |      int rc;
15 |  
16 |      static PyObject *cache = NULL;
17 | -    if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) {
18 | +    if(! self->valid) {
19 | +        return err_invalid();
20 | +    }
21 | +    if(! PyArg_ParseTupleAndKeywords(args, kwds, "OO|iii", keywords,
22 | +                                     &key, &val, &arg.dupdata, &arg.overwrite,
23 | +                                     &arg.append)) {
24 | +        return NULL;
25 | +    }
26 | +
27 | +    if(val_from_buffer(&arg.key, key) ||
28 | +       val_from_buffer(&arg.val, val)) {
29 |          return NULL;
30 |      }
31 |  
32 | 


--------------------------------------------------------------------------------
/misc/gdb.commands:
--------------------------------------------------------------------------------
 1 | # vim:syntax=gdb
 2 | #
 3 | # Print a backtrace if a program crashes. Run using:
 4 | #       gdb -x misc/gdb.commands --args argv0 argv1 ..
 5 | #
 6 | 
 7 | set confirm off
 8 | 
 9 | define hook-stop
10 |     init-if-undefined $_exitcode = 999
11 |     if $_exitcode == 999
12 |         echo Abnormal stop.\n
13 |         backtrace
14 |         quit 2
15 |     else
16 |         echo Normal exit.\n
17 |         quit $_exitcode
18 |     end
19 | end
20 | 
21 | run
22 | 


--------------------------------------------------------------------------------
/misc/its7733.c:
--------------------------------------------------------------------------------
 1 | // http://www.openldap.org/its/index.cgi/Software%20Bugs?id=7733
 2 | // gcc -g -I ../lib -o its7733 its7733.c
 3 | #include <stdlib.h>
 4 | #include <unistd.h>
 5 | #include <stdio.h>
 6 | #include <string.h>
 7 | #include <ctype.h>
 8 | 
 9 | #include "lmdb.h"
10 | #include "mdb.c"
11 | #include "midl.c"
12 | 
13 | 
14 | void check(int x)
15 | {
16 |     if(x) {
17 |         fprintf(stderr, "eek %s\n", mdb_strerror(x));
18 |         _exit(1);
19 |     }
20 | }
21 | 
22 | #define DB_PATH "/ram/tdb"
23 | 
24 | MDB_dbi dbi;
25 | MDB_txn *txn;
26 | MDB_env *env;
27 | MDB_cursor *c1;
28 | 
29 | MDB_val keyv;
30 | MDB_val valv;
31 | 
32 | void new_txn(void)
33 | {
34 |     if(txn) {
35 |         fprintf(stderr, "commit\n");
36 |         check(mdb_txn_commit(txn));
37 |     }
38 |     check(mdb_txn_begin(env, NULL, 0, &txn));
39 | }
40 | 
41 | 
42 | void put(const char *k)
43 | {
44 |     keyv.mv_size = strlen(k);
45 |     keyv.mv_data = k;
46 |     valv.mv_size = 0;
47 |     valv.mv_data = "";
48 |     check(mdb_put(txn, dbi, &keyv, &valv, 0));
49 | }
50 | 
51 | int main(void)
52 | {
53 |     check(mdb_env_create(&env));
54 |     check(mdb_env_set_mapsize(env, 1048576UL*1024UL*3UL));
55 |     check(mdb_env_set_maxreaders(env, 126));
56 |     check(mdb_env_set_maxdbs(env, 1));
57 |     if(! access(DB_PATH, X_OK)) {
58 |         system("rm -rf " DB_PATH);
59 |     }
60 |     check(mkdir(DB_PATH, 0777));
61 |     check(mdb_env_open(env, DB_PATH, MDB_MAPASYNC|MDB_NOSYNC|MDB_NOMETASYNC, 0644));
62 |     new_txn();
63 |     check(mdb_dbi_open(txn, NULL, 0, &dbi));
64 | 
65 |     put("a");
66 |     put("b");
67 |     put("baa");
68 |     put("d");
69 | 
70 |     new_txn();
71 | 
72 |     check(mdb_cursor_open(txn, dbi, &c1));
73 |     check(mdb_cursor_get(c1, &keyv, &valv, MDB_LAST));
74 |     check(mdb_cursor_del(c1, 0));
75 |     check(mdb_cursor_del(c1, 0));
76 |     new_txn();
77 | }
78 | 


--------------------------------------------------------------------------------
/misc/readers_mrb_env.patch:
--------------------------------------------------------------------------------
 1 | Store the address of the MDB_env structure that owns a reader in addition to
 2 | its PID and TID, to allow multiple MDB_envs to be open on the same thread,
 3 | since on mdb_env_close(), LMDB unconditionally obliterates any readers with a
 4 | matching PID. This patch extends the test to (PID, MDB_env*).
 5 | 
 6 | diff --git a/lib/mdb.c b/lib/mdb.c
 7 | index fd0a3b5..f2ebdfa 100644
 8 | --- a/lib/mdb.c
 9 | +++ b/lib/mdb.c
10 | @@ -536,6 +536,8 @@ typedef struct MDB_rxbody {
11 |  	txnid_t		mrb_txnid;
12 |  	/** The process ID of the process owning this reader txn. */
13 |  	MDB_PID_T	mrb_pid;
14 | +	/** MDB_env within the process owning this reader txn. */
15 | +	void *      mrb_env;
16 |  	/** The thread ID of the thread owning this txn. */
17 |  	pthread_t	mrb_tid;
18 |  } MDB_rxbody;
19 | @@ -547,6 +549,7 @@ typedef struct MDB_reader {
20 |  		/** shorthand for mrb_txnid */
21 |  #define	mr_txnid	mru.mrx.mrb_txnid
22 |  #define	mr_pid	mru.mrx.mrb_pid
23 | +#define	mr_env	mru.mrx.mrb_env
24 |  #define	mr_tid	mru.mrx.mrb_tid
25 |  		/** cache line alignment */
26 |  		char pad[(sizeof(MDB_rxbody)+CACHELINE-1) & ~(CACHELINE-1)];
27 | @@ -2285,6 +2288,7 @@ mdb_txn_renew0(MDB_txn *txn)
28 |  					return MDB_READERS_FULL;
29 |  				}
30 |  				ti->mti_readers[i].mr_pid = pid;
31 | +				ti->mti_readers[i].mr_env = env;
32 |  				ti->mti_readers[i].mr_tid = tid;
33 |  				if (i == nr)
34 |  					ti->mti_numreaders = ++nr;
35 | @@ -4254,7 +4258,8 @@ mdb_env_close0(MDB_env *env, int excl)
36 |  		 * me_txkey with its destructor must be disabled first.
37 |  		 */
38 |  		for (i = env->me_numreaders; --i >= 0; )
39 | -			if (env->me_txns->mti_readers[i].mr_pid == pid)
40 | +			if (env->me_txns->mti_readers[i].mr_pid == pid
41 | +				&& env->me_txns->mti_readers[i].mr_env == env)
42 |  				env->me_txns->mti_readers[i].mr_pid = 0;
43 |  #ifdef _WIN32
44 |  		if (env->me_rmutex) {
45 | 


--------------------------------------------------------------------------------
/misc/run_in_vm.py:
--------------------------------------------------------------------------------
  1 | #
  2 | # Copyright 2014 The py-lmdb authors, all rights reserved.
  3 | # 
  4 | # Redistribution and use in source and binary forms, with or without
  5 | # modification, are permitted only as authorized by the OpenLDAP
  6 | # Public License.
  7 | # 
  8 | # A copy of this license is available in the file LICENSE in the
  9 | # top-level directory of the distribution or, alternatively, at
 10 | # <http://www.OpenLDAP.org/license.html>.
 11 | # 
 12 | # OpenLDAP is a registered trademark of the OpenLDAP Foundation.
 13 | # 
 14 | # Individual files and/or contributed packages may be copyright by
 15 | # other parties and/or subject to additional restrictions.
 16 | # 
 17 | # This work also contains materials derived from public sources.
 18 | # 
 19 | # Additional information about OpenLDAP can be obtained at
 20 | # <http://www.openldap.org/>.
 21 | #
 22 | 
 23 | from __future__ import absolute_import
 24 | import atexit
 25 | import json
 26 | import os
 27 | import shutil
 28 | import socket
 29 | import subprocess
 30 | import sys
 31 | import tempfile
 32 | 
 33 | 
 34 | def run(*args):
 35 |     if os.path.exists('build'):
 36 |         shutil.rmtree('build')
 37 |     try:
 38 |         subprocess.check_call(args)
 39 |     except:
 40 |         print '!!! COMMAND WAS:', args
 41 |         raise
 42 | 
 43 | 
 44 | def qmp_write(fp, o):
 45 |     buf = json.dumps(o) + '\n'
 46 |     fp.write(buf.replace('{', '{ '))
 47 | 
 48 | 
 49 | def qmp_read(fp):
 50 |     s = fp.readline()
 51 |     return json.loads(s)
 52 | 
 53 | 
 54 | def qmp_say_hello(fp):
 55 |     assert 'QMP' in qmp_read(fp)
 56 |     qmp_write(fp, {'execute': 'qmp_capabilities'})
 57 |     assert qmp_read(fp)['return'] == {}
 58 | 
 59 | 
 60 | def qmp_command(fp, name, args):
 61 |     qmp_write(fp, {'execute': name, 'arguments': args})
 62 |     while True:
 63 |         o = qmp_read(fp)
 64 |         if 'return' not in o:
 65 |             print 'skip', o
 66 |             continue
 67 |         print 'cmd out', o
 68 |         return o['return']
 69 | 
 70 | 
 71 | def qmp_monitor(fp, cmd):
 72 |     return qmp_command(fp, 'human-monitor-command', {
 73 |         'command-line': cmd
 74 |     })
 75 | 
 76 | 
 77 | def main():
 78 |     vm = sys.argv[1]
 79 |     cmdline = sys.argv[2:]
 80 | 
 81 |     rsock, wsock = socket.socketpair()
 82 |     rfp = rsock.makefile('r+b', 1)
 83 | 
 84 |     qemu_path = '/usr/local/bin/qemu-system-x86_64'
 85 |     qemu_args = ['sudo', qemu_path, '-enable-kvm', '-m', '1024',
 86 |                  '-qmp', 'stdio', '-nographic', '-S',
 87 |                  '-vnc', '127.0.0.1:0',
 88 |                  '-net', 'user,hostfwd=tcp:127.0.0.1:9422-:22',
 89 |                  '-net', 'nic,model=virtio',
 90 |                  '-drive', 'file=%s,if=virtio' % (vm,)]
 91 |     print ' '.join(qemu_args).replace('qmp', 'monitor')
 92 |     exit()
 93 |     proc = subprocess.Popen(qemu_args,
 94 |         stdin=wsock.fileno(), stdout=wsock.fileno()
 95 |         )
 96 | 
 97 |     qmp_say_hello(rfp)
 98 |     assert '' == qmp_monitor(rfp, 'loadvm 1')
 99 |     assert '' == qmp_monitor(rfp, 'cont')
100 |     import time
101 |     time.sleep(100)
102 |     qmp_monitor(rfp, 'quit')
103 | 
104 | if __name__ == '__main__':
105 |     main()
106 | 


--------------------------------------------------------------------------------
/misc/test_monster_acid_trace.diff:
--------------------------------------------------------------------------------
 1 | diff --git a/tests/crash_test.py b/tests/crash_test.py
 2 | index 1ac3b46..3785bd5 100644
 3 | --- a/tests/crash_test.py
 4 | +++ b/tests/crash_test.py
 5 | @@ -173,21 +173,43 @@ class MultiCursorDeleteTest(unittest.TestCase):
 6 |          c2.delete()
 7 |          assert next(c1f) == B('eeee')
 8 |  
 9 | +    def _trace(self, op, *args):
10 | +        bits = []
11 | +        for arg in args:
12 | +            if isinstance(arg, bytes):
13 | +                bits.append(arg.encode('hex'))
14 | +            elif isinstance(arg, bool):
15 | +                bits.append(bytes(int(arg)))
16 | +        self.fp.write('%s %s %s\n' % (self.idx, op, ' '.join(bits)))
17 | +
18 |      def test_monster(self):
19 |          # Generate predictable sequence of sizes.
20 |          rand = random.Random()
21 |          rand.seed(0)
22 |  
23 | +        self.fp = open('trace.out', 'w')
24 | +        self._counter = 0
25 | +        self.idx = 0
26 | +
27 |          txn = self.env.begin(write=True)
28 |          keys = []
29 |          for i in range(20000):
30 |              key = B('%06x' % i)
31 |              val = B('x' * rand.randint(76, 350))
32 | +            self._trace('put', key, val)
33 |              assert txn.put(key, val)
34 |              keys.append(key)
35 |  
36 | +
37 | +        iter_id = self._counter
38 | +        self._counter += 1
39 | +        self._trace('iter', iter_id, '0', False)
40 | +
41 |          deleted = 0
42 |          for key in txn.cursor().iternext(values=False):
43 | +            self._trace('fetch', iter_id)
44 | +            self._trace('yield', iter_id, key, '')
45 | +            self._trace('delete', key)
46 |              assert txn.delete(key), key
47 |              deleted += 1
48 |  
49 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
  1 | #
  2 | # Copyright 2013-2024 The py-lmdb authors, all rights reserved.
  3 | #
  4 | # Redistribution and use in source and binary forms, with or without
  5 | # modification, are permitted only as authorized by the OpenLDAP
  6 | # Public License.
  7 | #
  8 | # A copy of this license is available in the file LICENSE in the
  9 | # top-level directory of the distribution or, alternatively, at
 10 | # <http://www.OpenLDAP.org/license.html>.
 11 | #
 12 | # OpenLDAP is a registered trademark of the OpenLDAP Foundation.
 13 | #
 14 | # Individual files and/or contributed packages may be copyright by
 15 | # other parties and/or subject to additional restrictions.
 16 | #
 17 | # This work also contains materials derived from public sources.
 18 | #
 19 | # Additional information about OpenLDAP can be obtained at
 20 | # <http://www.openldap.org/>.
 21 | #
 22 | 
 23 | from __future__ import absolute_import
 24 | from __future__ import with_statement
 25 | 
 26 | import os
 27 | import sys
 28 | import shutil
 29 | import platform
 30 | 
 31 | from setuptools import Extension
 32 | from setuptools import setup
 33 | 
 34 | try:
 35 |     import memsink
 36 | except ImportError:
 37 |     memsink = None
 38 | 
 39 | 
 40 | if hasattr(platform, 'python_implementation'):
 41 |     use_cpython = platform.python_implementation() == 'CPython'
 42 | else:
 43 |     use_cpython = True
 44 | 
 45 | if os.getenv('LMDB_FORCE_CFFI') is not None:
 46 |     use_cpython = False
 47 | 
 48 | if (3, 0) < sys.version_info[:2] < (3, 5):
 49 |     sys.stderr.write('Error: py-lmdb requires at CPython 3.5\n')
 50 |     raise SystemExit(1)
 51 | 
 52 | #
 53 | # Figure out which LMDB implementation to use.
 54 | #
 55 | 
 56 | if os.getenv('LMDB_INCLUDEDIR'):
 57 |     extra_include_dirs = [os.getenv('LMDB_INCLUDEDIR')]
 58 | else:
 59 |     extra_include_dirs = []
 60 | 
 61 | if os.getenv('LMDB_LIBDIR'):
 62 |     extra_library_dirs = [os.getenv('LMDB_LIBDIR')]
 63 | else:
 64 |     extra_library_dirs = []
 65 | 
 66 | extra_include_dirs += ['lib/py-lmdb']
 67 | extra_compile_args = []
 68 | 
 69 | patch_lmdb_source = False
 70 | 
 71 | if os.getenv('LMDB_FORCE_SYSTEM') is not None:
 72 |     print('py-lmdb: Using system version of liblmdb.')
 73 |     extra_sources = []
 74 |     extra_include_dirs += []
 75 |     libraries = ['lmdb']
 76 | elif os.getenv('LMDB_PURE') is not None:
 77 |     print('py-lmdb: Using bundled unmodified liblmdb; override with LMDB_FORCE_SYSTEM=1.')
 78 |     extra_sources = ['lib/mdb.c', 'lib/midl.c']
 79 |     extra_include_dirs += ['lib']
 80 |     libraries = []
 81 | else:
 82 |     print('py-lmdb: Using bundled liblmdb with py-lmdb patches; override with LMDB_FORCE_SYSTEM=1 or LMDB_PURE=1.')
 83 |     extra_sources = ['build/lib/mdb.c', 'build/lib/midl.c']
 84 |     extra_include_dirs += ['build/lib']
 85 |     extra_compile_args += ['-DHAVE_PATCHED_LMDB=1']
 86 |     libraries = []
 87 |     patch_lmdb_source = True
 88 | 
 89 | if patch_lmdb_source:
 90 |     if sys.platform.startswith('win'):
 91 |         try:
 92 |             import patch_ng as patch
 93 |         except ImportError:
 94 |             raise Exception('Building py-lmdb from source on Windows requires the "patch-ng" python module.')
 95 | 
 96 |     # Clean out any previously patched files
 97 |     dest = 'build' + os.sep + 'lib'
 98 |     try:
 99 |         os.mkdir('build')
100 |     except Exception:
101 |         pass
102 | 
103 |     try:
104 |         shutil.rmtree(dest)
105 |     except Exception:
106 |         pass
107 |     shutil.copytree('lib', dest)
108 | 
109 |     # Copy away the lmdb source then patch it
110 |     if sys.platform.startswith('win'):
111 |         patchfile = 'lib' + os.sep + 'py-lmdb' + os.sep + 'env-copy-txn.patch'
112 |         patchset = patch.fromfile(patchfile)
113 |         rv = patchset.apply(2, root=dest)
114 |         if not rv:
115 |             raise Exception('Applying patch failed')
116 |     else:
117 |         rv = os.system('patch -N -p3 -d build/lib < lib/py-lmdb/env-copy-txn.patch')
118 |         if rv:
119 |             raise Exception('Applying patch failed')
120 | 
121 | # distutils perplexingly forces NDEBUG for package code!
122 | extra_compile_args += ['-UNDEBUG']
123 | 
124 | # Disable some Clang/GCC warnings.
125 | if not os.getenv('LMDB_MAINTAINER'):
126 |     extra_compile_args += ['-w']
127 | 
128 | 
129 | # Microsoft Visual Studio 9 ships with neither inttypes.h, stdint.h, or a sane
130 | # definition for ssize_t, so here we add lib/win32 to the search path, which
131 | # contains emulation header files provided by a third party. We force-include
132 | # Python.h everywhere since it has a portable definition of ssize_t, which
133 | # inttypes.h and stdint.h lack, and to avoid having to modify the LMDB source
134 | # code. Advapi32 is needed for LMDB's use of Windows security APIs.
135 | p = sys.version.find('MSC v.')
136 | msvc_ver = int(sys.version[p + 6: p + 10]) if p != -1 else None
137 | 
138 | if sys.platform.startswith('win'):
139 |     # If running on Visual Studio<=2010 we must provide <stdint.h>. Newer
140 |     # versions provide it out of the box.
141 |     if msvc_ver and not msvc_ver >= 1600:
142 |         extra_include_dirs += ['lib\\win32-stdint']
143 |     extra_include_dirs += ['lib\\win32']
144 |     extra_compile_args += [r'/FIPython.h']
145 |     libraries += ['Advapi32']
146 | 
147 | 
148 | # Capture setup.py configuration for later use by cffi, otherwise the
149 | # configuration may differ, forcing a recompile (and therefore likely compile
150 | # errors). This happens even when `use_cpython` since user might want to
151 | # LMDB_FORCE_CFFI=1 during testing.
152 | with open('lmdb/_config.py', 'w') as fp:
153 |     fp.write('CONFIG = dict(%r)\n\n' % ((
154 |         ('extra_compile_args', extra_compile_args),
155 |         ('extra_sources', extra_sources),
156 |         ('extra_library_dirs', extra_library_dirs),
157 |         ('extra_include_dirs', extra_include_dirs),
158 |         ('libraries', libraries),
159 |     ),))
160 | 
161 | 
162 | if use_cpython:
163 |     print('py-lmdb: Using CPython extension; override with LMDB_FORCE_CFFI=1.')
164 |     install_requires = []
165 |     if memsink:
166 |         extra_compile_args += ['-DHAVE_MEMSINK',
167 |                                '-I' + os.path.dirname(memsink.__file__)]
168 |     ext_modules = [Extension(
169 |         name='cpython',
170 |         sources=['lmdb/cpython.c'] + extra_sources,
171 |         extra_compile_args=extra_compile_args,
172 |         libraries=libraries,
173 |         include_dirs=extra_include_dirs,
174 |         library_dirs=extra_library_dirs
175 |     )]
176 | else:
177 |     print('Using cffi extension.')
178 |     install_requires = ['cffi>=0.8']
179 |     try:
180 |         import lmdb.cffi
181 |         ext_modules = [lmdb.cffi._ffi.verifier.get_extension()]
182 |     except ImportError:
183 |         sys.stderr.write('Could not import lmdb; ensure cffi is installed!\n')
184 |         ext_modules = []
185 | 
186 | def grep_version():
187 |     path = os.path.join(os.path.dirname(__file__), 'lmdb/__init__.py')
188 |     with open(path) as fp:
189 |         for line in fp:
190 |             if line.startswith('__version__'):
191 |                 return eval(line.split()[-1])
192 | 
193 | setup(
194 |     name='lmdb',
195 |     version=grep_version(),
196 |     description="Universal Python binding for the LMDB 'Lightning' Database",
197 |     long_description="Universal Python binding for the LMDB 'Lightning' Database",
198 |     long_description_content_type="text/plain",
199 |     author='David Wilson',
200 |     maintainer='Nic Watson',
201 |     license='OLDAP-2.8',
202 |     url='http://github.com/jnwatson/py-lmdb/',
203 |     packages=['lmdb'],
204 | 
205 |     classifiers=[
206 |         "Programming Language :: Python",
207 |         "Programming Language :: Python :: Implementation :: CPython",
208 |         "Programming Language :: Python :: Implementation :: PyPy",
209 |         "Programming Language :: Python :: 3",
210 |         "Programming Language :: Python :: 3.5",
211 |         "Programming Language :: Python :: 3.6",
212 |         "Programming Language :: Python :: 3.7",
213 |         "Programming Language :: Python :: 3.8",
214 |         "Programming Language :: Python :: 3.9",
215 |         "Programming Language :: Python :: 3.10",
216 |         "Programming Language :: Python :: 3.11",
217 |         "Programming Language :: Python :: 3.12",
218 |         "Programming Language :: Python :: 3.13",
219 |         "Topic :: Database",
220 |         "Topic :: Database :: Database Engines/Servers",
221 |     ],
222 |     ext_package='lmdb',
223 |     ext_modules=ext_modules,
224 |     install_requires=install_requires,
225 | )
226 | 


--------------------------------------------------------------------------------
/tests/crash_test.py:
--------------------------------------------------------------------------------
  1 | #
  2 | # Copyright 2013-2021 The py-lmdb authors, all rights reserved.
  3 | #
  4 | # Redistribution and use in source and binary forms, with or without
  5 | # modification, are permitted only as authorized by the OpenLDAP
  6 | # Public License.
  7 | #
  8 | # A copy of this license is available in the file LICENSE in the
  9 | # top-level directory of the distribution or, alternatively, at
 10 | # <http://www.OpenLDAP.org/license.html>.
 11 | #
 12 | # OpenLDAP is a registered trademark of the OpenLDAP Foundation.
 13 | #
 14 | # Individual files and/or contributed packages may be copyright by
 15 | # other parties and/or subject to additional restrictions.
 16 | #
 17 | # This work also contains materials derived from public sources.
 18 | #
 19 | # Additional information about OpenLDAP can be obtained at
 20 | # <http://www.openldap.org/>.
 21 | #
 22 | 
 23 | # This is not a test suite! More like a collection of triggers for previously
 24 | # observed crashes. Want to contribute to py-lmdb? Please write a test suite!
 25 | #
 26 | # what happens when empty keys/ values passed to various funcs
 27 | # incorrect types
 28 | # try to break cpython arg parsing - too many/few/incorrect args
 29 | # Various efforts to cause Python-level leaks.
 30 | #
 31 | 
 32 | from __future__ import absolute_import
 33 | from __future__ import with_statement
 34 | 
 35 | import sys
 36 | import itertools
 37 | import random
 38 | import unittest
 39 | import multiprocessing
 40 | 
 41 | import lmdb
 42 | import testlib
 43 | 
 44 | from testlib import B
 45 | from testlib import O
 46 | 
 47 | 
 48 | class CrashTest(unittest.TestCase):
 49 |     def tearDown(self):
 50 |         testlib.cleanup()
 51 | 
 52 |     # Various efforts to cause segfaults.
 53 | 
 54 |     def setUp(self):
 55 |         self.path, self.env = testlib.temp_env()
 56 |         with self.env.begin(write=True) as txn:
 57 |             txn.put(B('dave'), B(''))
 58 |             txn.put(B('dave2'), B(''))
 59 | 
 60 |     def testOldCrash(self):
 61 |         txn = self.env.begin()
 62 |         dir(iter(txn.cursor()))
 63 | 
 64 |     def testCloseWithTxn(self):
 65 |         txn = self.env.begin(write=True)
 66 |         self.env.close()
 67 |         self.assertRaises(Exception, (lambda: list(txn.cursor())))
 68 | 
 69 |     def testDoubleClose(self):
 70 |         self.env.close()
 71 |         self.env.close()
 72 | 
 73 |     def testTxnCloseActiveIter(self):
 74 |         with self.env.begin() as txn:
 75 |             it = txn.cursor().iternext()
 76 |         self.assertRaises(Exception, (lambda: list(it)))
 77 | 
 78 |     def testDbCloseActiveIter(self):
 79 |         db = self.env.open_db(key=B('dave3'))
 80 |         with self.env.begin(write=True) as txn:
 81 |             txn.put(B('a'), B('b'), db=db)
 82 |             it = txn.cursor(db=db).iternext()
 83 |         self.assertRaises(Exception, (lambda: list(it)))
 84 | 
 85 | 
 86 | class IteratorTest(unittest.TestCase):
 87 |     def tearDown(self):
 88 |         testlib.cleanup()
 89 | 
 90 |     def setUp(self):
 91 |         self.path, self.env = testlib.temp_env()
 92 |         self.txn = self.env.begin(write=True)
 93 |         self.c = self.txn.cursor()
 94 | 
 95 |     def testEmpty(self):
 96 |         self.assertEqual([], list(self.c))
 97 |         self.assertEqual([], list(self.c.iternext()))
 98 |         self.assertEqual([], list(self.c.iterprev()))
 99 | 
100 |     def testFilled(self):
101 |         testlib.putData(self.txn)
102 |         self.assertEqual(testlib.ITEMS, list(self.c))
103 |         self.assertEqual(testlib.ITEMS, list(self.c))
104 |         self.assertEqual(testlib.ITEMS, list(self.c.iternext()))
105 |         self.assertEqual(testlib.ITEMS[::-1], list(self.txn.cursor().iterprev()))
106 |         self.assertEqual(testlib.ITEMS[::-1], list(self.c.iterprev()))
107 |         self.assertEqual(testlib.ITEMS, list(self.c))
108 | 
109 |     def testFilledSkipForward(self):
110 |         testlib.putData(self.txn)
111 |         self.c.set_range(B('b'))
112 |         self.assertEqual(testlib.ITEMS[1:], list(self.c))
113 | 
114 |     def testFilledSkipReverse(self):
115 |         testlib.putData(self.txn)
116 |         self.c.set_range(B('b'))
117 |         self.assertEqual(testlib.REV_ITEMS[-2:], list(self.c.iterprev()))
118 | 
119 |     def testFilledSkipEof(self):
120 |         testlib.putData(self.txn)
121 |         self.assertEqual(False, self.c.set_range(B('z')))
122 |         self.assertEqual(testlib.REV_ITEMS, list(self.c.iterprev()))
123 | 
124 | 
125 | class BigReverseTest(unittest.TestCase):
126 |     def tearDown(self):
127 |         testlib.cleanup()
128 | 
129 |     # Test for issue with MDB_LAST+MDB_PREV skipping chunks of database.
130 |     def test_big_reverse(self):
131 |         path, env = testlib.temp_env()
132 |         txn = env.begin(write=True)
133 |         keys = [B('%05d' % i) for i in range(0xffff)]
134 |         for k in keys:
135 |             txn.put(k, k, append=True)
136 |         assert list(txn.cursor().iterprev(values=False)) == list(reversed(keys))
137 | 
138 | 
139 | class MultiCursorDeleteTest(unittest.TestCase):
140 |     def tearDown(self):
141 |         testlib.cleanup()
142 | 
143 |     def setUp(self):
144 |         self.path, self.env = testlib.temp_env()
145 | 
146 |     def test1(self):
147 |         """Ensure MDB_NEXT is ignored on `c1' when it was previously positioned
148 |         on the key that `c2' just deleted."""
149 |         txn = self.env.begin(write=True)
150 |         cur = txn.cursor()
151 |         while cur.first():
152 |             cur.delete()
153 | 
154 |         for i in range(1, 10):
155 |             cur.put(O(ord('a') + i) * i, B(''))
156 | 
157 |         c1 = txn.cursor()
158 |         c1f = c1.iternext(values=False)
159 |         while next(c1f) != B('ddd'):
160 |             pass
161 |         c2 = txn.cursor()
162 |         assert c2.set_key(B('ddd'))
163 |         c2.delete()
164 |         assert next(c1f) == B('eeee')
165 | 
166 |     def test_monster(self):
167 |         # Generate predictable sequence of sizes.
168 |         rand = random.Random()
169 |         rand.seed(0)
170 | 
171 |         txn = self.env.begin(write=True)
172 |         keys = []
173 |         for i in range(20000):
174 |             key = B('%06x' % i)
175 |             val = B('x' * rand.randint(76, 350))
176 |             assert txn.put(key, val)
177 |             keys.append(key)
178 | 
179 |         deleted = 0
180 |         for key in txn.cursor().iternext(values=False):
181 |             assert txn.delete(key), key
182 |             deleted += 1
183 | 
184 |         assert deleted == len(keys), deleted
185 | 
186 | 
187 | class TxnFullTest(unittest.TestCase):
188 |     def tearDown(self):
189 |         testlib.cleanup()
190 | 
191 |     def test_17bf75b12eb94d9903cd62329048b146d5313bad(self):
192 |         """
193 |         me_txn0 previously cached MDB_TXN_ERROR permanently. Fixed by
194 |         17bf75b12eb94d9903cd62329048b146d5313bad.
195 |         """
196 |         path, env = testlib.temp_env(map_size=4096 * 9, sync=False, max_spare_txns=0)
197 |         for i in itertools.count():
198 |             try:
199 |                 with env.begin(write=True) as txn:
200 |                     txn.put(B(str(i)), B(str(i)))
201 |             except lmdb.MapFullError:
202 |                 break
203 | 
204 |         # Should not crash with MDB_BAD_TXN:
205 |         with env.begin(write=True) as txn:
206 |             txn.delete(B('1'))
207 | 
208 | 
209 | class EmptyIterTest(unittest.TestCase):
210 |     def tearDown(self):
211 |         testlib.cleanup()
212 | 
213 |     def test_python3_iternext_segfault(self):
214 |         # https://github.com/dw/py-lmdb/issues/105
215 |         _, env = testlib.temp_env()
216 |         txn = env.begin()
217 |         cur = txn.cursor()
218 |         ite = cur.iternext()
219 |         nex = getattr(ite, 'next', getattr(ite, '__next__', None))
220 |         assert nex is not None
221 |         self.assertRaises(StopIteration, nex)
222 | 
223 | 
224 | class MultiputTest(unittest.TestCase):
225 |     def tearDown(self):
226 |         testlib.cleanup()
227 | 
228 |     def test_multiput_segfault(self):
229 |         # http://github.com/jnwatson/py-lmdb/issues/173
230 |         _, env = testlib.temp_env()
231 |         db = env.open_db(b'foo', dupsort=True)
232 |         txn = env.begin(db=db, write=True)
233 |         txn.put(b'a', b'\x00\x00\x00\x00\x00\x00\x00\x00')
234 |         txn.put(b'a', b'\x05')
235 |         txn.put(b'a', b'\t')
236 |         txn.put(b'a', b'\r')
237 |         txn.put(b'a', b'\x11')
238 |         txn.put(b'a', b'\x15')
239 |         txn.put(b'a', b'\x19')
240 |         txn.put(b'a', b'\x1d')
241 |         txn.put(b'a', b'!')
242 |         txn.put(b'a', b'%')
243 |         txn.put(b'a', b')')
244 |         txn.put(b'a', b'-')
245 |         txn.put(b'a', b'1')
246 |         txn.put(b'a', b'5')
247 |         txn.commit()
248 | 
249 | class InvalidArgTest(unittest.TestCase):
250 |     def tearDown(self):
251 |         testlib.cleanup()
252 | 
253 |     def test_duplicate_arg(self):
254 |         # https://github.com/jnwatson/py-lmdb/issues/203
255 |         _, env = testlib.temp_env()
256 |         txn = env.begin(write=True)
257 |         c = txn.cursor()
258 |         self.assertRaises(TypeError, c.get, b'a', key=True)
259 | 
260 | class BadCursorTest(unittest.TestCase):
261 |     def tearDown(self):
262 |         testlib.cleanup()
263 | 
264 |     def test_cursor_open_failure(self):
265 |         '''
266 |         Test the error path for when mdb_cursor_open fails
267 | 
268 |         Note:
269 |             this only would crash if cpython is built with Py_TRACE_REFS
270 |         '''
271 |         # https://github.com/jnwatson/py-lmdb/issues/216
272 |         path, env = testlib.temp_env()
273 |         db = env.open_db(b'db', dupsort=True)
274 |         env.close()
275 |         del env
276 | 
277 |         env = lmdb.open(path, readonly=True, max_dbs=4)
278 |         txn1 = env.begin(write=False)
279 |         db = env.open_db(b'db', dupsort=True, txn=txn1)
280 |         txn2 = env.begin(write=False)
281 |         self.assertRaises(lmdb.InvalidParameterError, txn2.cursor, db=db)
282 | 
283 | MINDBSIZE = 64 * 1024 * 2  # certain ppcle Linux distros have a 64K page size
284 | 
285 | if sys.version_info[:2] >= (3, 4):
286 |     class MapResizeTest(unittest.TestCase):
287 | 
288 |         def tearDown(self):
289 |             testlib.cleanup()
290 | 
291 |         @staticmethod
292 |         def do_resize(path):
293 |             '''
294 |             Increase map size and fill up database, making sure that the root page is no longer
295 |             accessible in the main process.
296 |             '''
297 |             with lmdb.open(path, max_dbs=10, create=False, map_size=MINDBSIZE) as env:
298 |                 env.open_db(b'foo')
299 |                 env.set_mapsize(MINDBSIZE * 2)
300 |                 count = 0
301 |                 try:
302 |                     # Figure out how many keyvals we can enter before we run out of space
303 |                     with env.begin(write=True) as txn:
304 |                         while True:
305 |                             datum = count.to_bytes(4, 'little')
306 |                             txn.put(datum, b'0')
307 |                             count += 1
308 | 
309 |                 except lmdb.MapFullError:
310 |                     # Now put (and commit) just short of that
311 |                     with env.begin(write=True) as txn:
312 |                         for i in range(count - 100):
313 |                             datum = i.to_bytes(4, 'little')
314 |                             txn.put(datum, b'0')
315 |                 else:
316 |                     assert 0
317 | 
318 |         def test_opendb_resize(self):
319 |             '''
320 |             Test that we correctly handle a MDB_MAP_RESIZED in env.open_db.
321 | 
322 |             Would seg fault in cffi implementation
323 |             '''
324 |             mpctx = multiprocessing.get_context('spawn')
325 |             path, env = testlib.temp_env(max_dbs=10, map_size=MINDBSIZE)
326 |             env.close()
327 |             env = lmdb.open(path, max_dbs=10, map_size=MINDBSIZE, readonly=True)
328 |             proc = mpctx.Process(target=self.do_resize, args=(path,))
329 |             proc.start()
330 |             proc.join(5)
331 |             assert proc.exitcode is not None
332 |             self.assertRaises(lmdb.MapResizedError, env.open_db, b'foo')
333 | 
334 | if __name__ == '__main__':
335 |     unittest.main()
336 | 


--------------------------------------------------------------------------------
/tests/cursor_test.py:
--------------------------------------------------------------------------------
  1 | #
  2 | # Copyright 2013-2021 The py-lmdb authors, all rights reserved.
  3 | #
  4 | # Redistribution and use in source and binary forms, with or without
  5 | # modification, are permitted only as authorized by the OpenLDAP
  6 | # Public License.
  7 | #
  8 | # A copy of this license is available in the file LICENSE in the
  9 | # top-level directory of the distribution or, alternatively, at
 10 | # <http://www.OpenLDAP.org/license.html>.
 11 | #
 12 | # OpenLDAP is a registered trademark of the OpenLDAP Foundation.
 13 | #
 14 | # Individual files and/or contributed packages may be copyright by
 15 | # other parties and/or subject to additional restrictions.
 16 | #
 17 | # This work also contains materials derived from public sources.
 18 | #
 19 | # Additional information about OpenLDAP can be obtained at
 20 | # <http://www.openldap.org/>.
 21 | #
 22 | 
 23 | # test delete(dupdata)
 24 | 
 25 | from __future__ import absolute_import
 26 | from __future__ import with_statement
 27 | import sys
 28 | import unittest
 29 | 
 30 | import lmdb
 31 | 
 32 | import testlib
 33 | from testlib import B
 34 | from testlib import BT
 35 | 
 36 | class ContextManagerTest(unittest.TestCase):
 37 |     def tearDown(self):
 38 |         testlib.cleanup()
 39 | 
 40 |     def test_ok(self):
 41 |         path, env = testlib.temp_env()
 42 |         txn = env.begin(write=True)
 43 |         with txn.cursor() as curs:
 44 |             curs.put(B('foo'), B('123'))
 45 |         self.assertRaises(Exception, lambda: curs.get(B('foo')))
 46 | 
 47 |     def test_crash(self):
 48 |         path, env = testlib.temp_env()
 49 |         txn = env.begin(write=True)
 50 | 
 51 |         try:
 52 |             with txn.cursor() as curs:
 53 |                 curs.put(123, 123)
 54 |         except Exception:
 55 |             pass
 56 |         self.assertRaises(Exception, lambda: curs.get(B('foo')))
 57 | 
 58 | 
 59 | class CursorTestBase(unittest.TestCase):
 60 |     def tearDown(self):
 61 |         testlib.cleanup()
 62 | 
 63 |     def setUp(self):
 64 |         self.path, self.env = testlib.temp_env()
 65 |         self.txn = self.env.begin(write=True)
 66 |         self.c = self.txn.cursor()
 67 | 
 68 | 
 69 | class CursorTest(CursorTestBase):
 70 |     def testKeyValueItemEmpty(self):
 71 |         self.assertEqual(B(''), self.c.key())
 72 |         self.assertEqual(B(''), self.c.value())
 73 |         self.assertEqual(BT('', ''), self.c.item())
 74 | 
 75 |     def testFirstLastEmpty(self):
 76 |         self.assertEqual(False, self.c.first())
 77 |         self.assertEqual(False, self.c.last())
 78 | 
 79 |     def testFirstFilled(self):
 80 |         testlib.putData(self.txn)
 81 |         self.assertEqual(True, self.c.first())
 82 |         self.assertEqual(testlib.ITEMS[0], self.c.item())
 83 | 
 84 |     def testLastFilled(self):
 85 |         testlib.putData(self.txn)
 86 |         self.assertEqual(True, self.c.last())
 87 |         self.assertEqual(testlib.ITEMS[-1], self.c.item())
 88 | 
 89 |     def testSetKey(self):
 90 |         self.assertRaises(Exception, (lambda: self.c.set_key(B(''))))
 91 |         self.assertEqual(False, self.c.set_key(B('missing')))
 92 |         testlib.putData(self.txn)
 93 |         self.assertEqual(True, self.c.set_key(B('b')))
 94 |         self.assertEqual(False, self.c.set_key(B('ba')))
 95 | 
 96 |     def testSetRange(self):
 97 |         self.assertEqual(False, self.c.set_range(B('x')))
 98 |         testlib.putData(self.txn)
 99 |         self.assertEqual(False, self.c.set_range(B('x')))
100 |         self.assertEqual(True, self.c.set_range(B('a')))
101 |         self.assertEqual(B('a'), self.c.key())
102 |         self.assertEqual(True, self.c.set_range(B('ba')))
103 |         self.assertEqual(B('baa'), self.c.key())
104 |         self.c.set_range(B(''))
105 |         self.assertEqual(B('a'), self.c.key())
106 | 
107 |     def testDeleteEmpty(self):
108 |         self.assertEqual(False, self.c.delete())
109 | 
110 |     def testDeleteFirst(self):
111 |         testlib.putData(self.txn)
112 |         self.assertEqual(False, self.c.delete())
113 |         self.c.first()
114 |         self.assertEqual(BT('a', ''), self.c.item())
115 |         self.assertEqual(True, self.c.delete())
116 |         self.assertEqual(BT('b', ''), self.c.item())
117 |         self.assertEqual(True, self.c.delete())
118 |         self.assertEqual(BT('baa', ''), self.c.item())
119 |         self.assertEqual(True, self.c.delete())
120 |         self.assertEqual(BT('d', ''), self.c.item())
121 |         self.assertEqual(True, self.c.delete())
122 |         self.assertEqual(BT('', ''), self.c.item())
123 |         self.assertEqual(False, self.c.delete())
124 |         self.assertEqual(BT('', ''), self.c.item())
125 | 
126 |     def testDeleteLast(self):
127 |         testlib.putData(self.txn)
128 |         self.assertEqual(True, self.c.last())
129 |         self.assertEqual(BT('d', ''), self.c.item())
130 |         self.assertEqual(True, self.c.delete())
131 |         self.assertEqual(BT('', ''), self.c.item())
132 |         self.assertEqual(False, self.c.delete())
133 |         self.assertEqual(BT('', ''), self.c.item())
134 | 
135 |     def testCount(self):
136 |         self.assertRaises(Exception, (lambda: self.c.count()))
137 |         testlib.putData(self.txn)
138 |         self.c.first()
139 |         # TODO: complete dup key support.
140 |         #self.assertEqual(1, self.c.count())
141 | 
142 |     def testPut(self):
143 |         pass
144 | 
145 | class CursorTest2(unittest.TestCase):
146 |     def tearDown(self):
147 |         testlib.cleanup()
148 | 
149 |     def setUp(self):
150 |         self.path, self.env = testlib.temp_env()
151 |         self.db = self.env.open_db(b'foo', dupsort=True)
152 |         self.txn = self.env.begin(write=True, db=self.db)
153 |         self.c = self.txn.cursor()
154 | 
155 |     def testIterWithDeletes(self):
156 |         ''' A problem identified in LMDB 0.9.27 '''
157 |         self.c.put(b'\x00\x01', b'hehe', dupdata=True)
158 |         self.c.put(b'\x00\x02', b'haha', dupdata=True)
159 |         self.c.set_key(b'\x00\x02')
160 |         it = self.c.iternext()
161 |         self.assertEqual((b'\x00\x02', b'haha'), next(it))
162 |         self.txn.delete(b'\x00\x01', b'hehe', db=self.db)
163 |         self.assertRaises(StopIteration, next, it)
164 | 
165 | class PutmultiTest(CursorTestBase):
166 |     def test_empty_seq(self):
167 |         consumed, added = self.c.putmulti(())
168 |         assert consumed == added == 0
169 | 
170 |     def test_2list(self):
171 |         l = [BT('a', ''), BT('a', '')]
172 |         consumed, added = self.c.putmulti(l)
173 |         assert consumed == added == 2
174 | 
175 |         li = iter(l)
176 |         consumed, added = self.c.putmulti(li)
177 |         assert consumed == added == 2
178 | 
179 |     def test_2list_preserve(self):
180 |         l = [BT('a', ''), BT('a', '')]
181 |         consumed, added = self.c.putmulti(l, overwrite=False)
182 |         assert consumed == 2
183 |         assert added == 1
184 | 
185 |         assert self.c.set_key(B('a'))
186 |         assert self.c.delete()
187 | 
188 |         li = iter(l)
189 |         consumed, added = self.c.putmulti(li, overwrite=False)
190 |         assert consumed == 2
191 |         assert added == 1
192 | 
193 |     def test_bad_seq1(self):
194 |         self.assertRaises(Exception,
195 |                           lambda: self.c.putmulti(range(2)))
196 | 
197 |     def test_dupsort(self):
198 |         _, env = testlib.temp_env()
199 |         db1 = env.open_db(B('db1'), dupsort=True)
200 |         txn = env.begin(write=True, db=db1)
201 |         with txn.cursor() as c:
202 |             tups = [BT('a', 'value1'), BT('b', 'value1'), BT('b', 'value2')]
203 |             assert (3, 3) == c.putmulti(tups)
204 | 
205 |     def test_dupsort_putmulti_append(self):
206 |         _, env = testlib.temp_env()
207 |         db1 = env.open_db(B('db1'), dupsort=True)
208 |         txn = env.begin(write=True, db=db1)
209 |         with txn.cursor() as c:
210 |             tups = [BT('a', 'value1'), BT('b', 'value1'), BT('b', 'value2')]
211 |             assert (3, 3) == c.putmulti(tups, append=True)
212 | 
213 |     def test_dupsort_put_append(self):
214 |         _, env = testlib.temp_env()
215 |         db1 = env.open_db(B('db1'), dupsort=True)
216 |         txn = env.begin(write=True, db=db1)
217 |         with txn.cursor() as c:
218 |             assert c.put(B('a'), B('value1'), append=True)
219 |             assert c.put(B('b'), B('value1'), append=True)
220 |             assert c.put(B('b'), B('value2'), append=True)
221 | 
222 | class ReplaceTest(CursorTestBase):
223 |     def test_replace(self):
224 |         assert None is self.c.replace(B('a'), B(''))
225 |         assert B('') == self.c.replace(B('a'), B('x'))
226 |         assert B('x') == self.c.replace(B('a'), B('y'))
227 | 
228 | 
229 | class ContextManagerTest2(CursorTestBase):
230 |     def test_enter(self):
231 |         with self.c as c:
232 |             assert c is self.c
233 |             c.put(B('a'), B('a'))
234 |             assert c.get(B('a')) == B('a')
235 |         self.assertRaises(Exception,
236 |             lambda: c.get(B('a')))
237 | 
238 |     def test_exit_success(self):
239 |         with self.txn.cursor() as c:
240 |             c.put(B('a'), B('a'))
241 |         self.assertRaises(Exception,
242 |             lambda: c.get(B('a')))
243 | 
244 |     def test_exit_failure(self):
245 |         try:
246 |             with self.txn.cursor() as c:
247 |                 c.put(B('a'), B('a'))
248 |             raise ValueError
249 |         except ValueError:
250 |             pass
251 |         self.assertRaises(Exception,
252 |             lambda: c.get(B('a')))
253 | 
254 |     def test_close(self):
255 |         self.c.close()
256 |         self.assertRaises(Exception,
257 |             lambda: c.get(B('a')))
258 | 
259 |     def test_double_close(self):
260 |         self.c.close()
261 |         self.c.close()
262 |         self.assertRaises(Exception,
263 |             lambda: self.c.put(B('a'), B('a')))
264 | 
265 | GiB = 1024 * 1024 * 1024
266 | 
267 | class PreloadTest(CursorTestBase):
268 | 
269 |     def setUp(self, redo=False):
270 |         env_args = {'writemap': True, 'map_size': GiB}
271 |         if not redo:
272 |             self.path, self.env = testlib.temp_env(**env_args)
273 |         else:
274 |             self.path, self.env = testlib.temp_env(path=self.path, **env_args)
275 |         self.txn = self.env.begin(write=True)
276 |         self.c = self.txn.cursor()
277 | 
278 |     @unittest.skipIf(not sys.platform.startswith('linux'), "test only works on Linux")
279 |     def test_preload(self):
280 |         """
281 |         Test that reading just the key doesn't prefault the value contents, but
282 |         reading the data does.
283 |         """
284 | 
285 |         import resource
286 |         self.c.put(B('a'), B('a') * (256 * 1024 * 1024))
287 |         self.txn.commit()
288 |         self.env.close()
289 |         # Just reading the data is obviously going to fault the value in.  The
290 |         # point is to fault it in while the GIL is unlocked.  We use the buffers
291 |         # API so that we're not actually copying the data in.  This doesn't
292 |         # actually show that we're prefaulting with the GIL unlocked, but it
293 |         # does prove we prefault at all, and in 2 correct places.
294 |         self.path, self.env = testlib.temp_env(path=self.path, writemap=True)
295 |         self.txn = self.env.begin(write=True, buffers=True)
296 |         self.c = self.txn.cursor()
297 |         minflts_before = resource.getrusage(resource.RUSAGE_SELF)[6]
298 |         self.c.set_key(B('a'))
299 |         assert bytes(self.c.key()) == B('a')
300 |         minflts_after_key = resource.getrusage(resource.RUSAGE_SELF)[6]
301 | 
302 |         self.c.value()
303 |         minflts_after_value = resource.getrusage(resource.RUSAGE_SELF)[6]
304 | 
305 |         epsilon = 60
306 | 
307 |         # Setting the position doesn't prefault the data
308 |         assert minflts_after_key - minflts_before < epsilon
309 | 
310 |         # Getting the value does prefault the data, even if we only get it by pointer
311 |         assert minflts_after_value - minflts_after_key > 1000
312 | 
313 | class CursorReadOnlyTest(unittest.TestCase):
314 |     def tearDown(self):
315 |         testlib.cleanup()
316 | 
317 |     def test_cursor_readonly(self):
318 |         '''
319 |         Tests whether you can open a cursor on a sub-db at all in a read-only environment.
320 |         '''
321 |         path, env = testlib.temp_env(max_dbs=10)
322 |         env.open_db(b'foo')
323 |         env.close()
324 |         with lmdb.open(path, max_dbs=10, readonly=True) as env:
325 |             db2 = env.open_db(b'foo')
326 |             with env.begin(db=db2) as txn:
327 |                 with txn.cursor(db=db2):
328 |                     pass
329 | 
330 | if __name__ == '__main__':
331 |     unittest.main()
332 | 


--------------------------------------------------------------------------------
/tests/getmulti_test.py:
--------------------------------------------------------------------------------
 1 | from __future__ import absolute_import
 2 | from __future__ import with_statement
 3 | import unittest
 4 | 
 5 | import testlib, struct
 6 | from testlib import KEYSFIXED, ITEMS_MULTI_FIXEDKEY
 7 | from testlib import putBigDataMultiFixed
 8 | 
 9 | class GetMultiTestBase(unittest.TestCase):
10 | 
11 |     def tearDown(self):
12 |         testlib.cleanup()
13 | 
14 |     def setUp(self, dupsort=None, dupfixed=None):
15 |         self.db_key = "testdb".encode('utf-8')
16 |         self.path, self.env = testlib.temp_env(max_dbs=1)
17 |         self.txn = self.env.begin(write=True)
18 |         self.db = self.env.open_db(
19 |             key=self.db_key, txn=self.txn,
20 |             dupsort=dupsort,
21 |             dupfixed=dupfixed
22 |             )
23 |         putBigDataMultiFixed(self.txn, db=self.db)
24 |         self.c = self.txn.cursor(db=self.db)
25 | 
26 |     def matchList(self, ls_a, ls_b):
27 |         return ((not (ls_a or ls_b)) or
28 |             (ls_a and ls_b and all(map(lambda x, y: x == y, ls_a, ls_b))))
29 | 
30 | 
31 | class GetMultiTestNoDupsortNoDupfixed(GetMultiTestBase):
32 | 
33 |     ITEMS2_MULTI_NODUP = ITEMS_MULTI_FIXEDKEY[1::2]
34 | 
35 |     def setUp(self, dupsort=False, dupfixed=False):
36 |         super(GetMultiTestNoDupsortNoDupfixed, self).setUp(dupsort=dupsort, dupfixed=dupfixed)
37 | 
38 |     def testGetMulti(self):
39 |         test_list = self.c.getmulti(KEYSFIXED)
40 |         self.assertEqual(self.matchList(test_list, self.ITEMS2_MULTI_NODUP), True)
41 | 
42 | 
43 | class GetMultiTestDupsortNoDupfixed(GetMultiTestBase):
44 | 
45 |     def setUp(self, dupsort=True, dupfixed=False):
46 |         super(GetMultiTestDupsortNoDupfixed, self).setUp(dupsort=dupsort, dupfixed=dupfixed)
47 | 
48 |     def testGetMulti(self):
49 |         test_list = self.c.getmulti(KEYSFIXED, dupdata=True)
50 |         self.assertEqual(self.matchList(test_list, ITEMS_MULTI_FIXEDKEY), True)
51 | 
52 | 
53 | class GetMultiTestDupsortDupfixed(GetMultiTestBase):
54 | 
55 |     def setUp(self, dupsort=True, dupfixed=True):
56 |         super(GetMultiTestDupsortDupfixed, self).setUp(dupsort=dupsort, dupfixed=dupfixed)
57 | 
58 |     def testGetMulti(self):
59 |         test_list = self.c.getmulti(KEYSFIXED, dupdata=True, dupfixed_bytes=1)
60 |         self.assertEqual(self.matchList(test_list, ITEMS_MULTI_FIXEDKEY), True)
61 | 
62 | class GetMultiTestDupsortDupfixedKeyfixed(GetMultiTestBase):
63 | 
64 |     def setUp(self, dupsort=True, dupfixed=True):
65 |         super(GetMultiTestDupsortDupfixedKeyfixed, self).setUp(dupsort=dupsort, dupfixed=dupfixed)
66 | 
67 |     def testGetMulti(self):
68 |         val_bytes = 1
69 |         arr = bytearray(self.c.getmulti(
70 |             KEYSFIXED, dupdata=True,
71 |             dupfixed_bytes=val_bytes, keyfixed=True
72 |         ))
73 |         asserts = []
74 |         for i, kv in enumerate(ITEMS_MULTI_FIXEDKEY):
75 |             key, val = kv
76 |             asserts.extend((
77 |                 struct.pack('b', arr[i*2]) == key,
78 |                 struct.pack('b', arr[i*2+1]) == val
79 |             ))
80 |         self.assertEqual(all(asserts), True)
81 | 
82 | 
83 | if __name__ == '__main__':
84 |     unittest.main()


--------------------------------------------------------------------------------
/tests/iteration_test.py:
--------------------------------------------------------------------------------
  1 | #! /usr/bin/env python
  2 | #
  3 | # Copyright 2013 The py-lmdb authors, all rights reserved.
  4 | #
  5 | # Redistribution and use in source and binary forms, with or without
  6 | # modification, are permitted only as authorized by the OpenLDAP
  7 | # Public License.
  8 | #
  9 | # A copy of this license is available in the file LICENSE in the
 10 | # top-level directory of the distribution or, alternatively, at
 11 | # <http://www.OpenLDAP.org/license.html>.
 12 | #
 13 | # OpenLDAP is a registered trademark of the OpenLDAP Foundation.
 14 | #
 15 | # Individual files and/or contributed packages may be copyright by
 16 | # other parties and/or subject to additional restrictions.
 17 | #
 18 | # This work also contains materials derived from public sources.
 19 | #
 20 | # Additional information about OpenLDAP can be obtained at
 21 | # <http://www.openldap.org/>.
 22 | #
 23 | 
 24 | # test delete(dupdata)
 25 | 
 26 | from __future__ import absolute_import
 27 | from __future__ import with_statement
 28 | import unittest
 29 | 
 30 | import testlib
 31 | from testlib import B
 32 | from testlib import BT
 33 | from testlib import KEYS, ITEMS, KEYS2, ITEMS2
 34 | from testlib import putData, putBigData
 35 | 
 36 | 
 37 | class IterationTestBase(unittest.TestCase):
 38 |     def tearDown(self):
 39 |         testlib.cleanup()
 40 | 
 41 |     def setUp(self):
 42 |         self.path, self.env = testlib.temp_env()  # creates 10 databases
 43 |         self.txn = self.env.begin(write=True)
 44 |         putData(self.txn)
 45 |         self.c = self.txn.cursor()
 46 |         self.empty_entry = (B(''), B(''))
 47 | 
 48 |     def matchList(self, ls_a, ls_b):
 49 |         return all(map(lambda x, y: x == y, ls_a, ls_b))
 50 | 
 51 | 
 52 | class IterationTestBase2(unittest.TestCase):
 53 |     """ This puts more data than its predecessor"""
 54 | 
 55 |     def tearDown(self):
 56 |         testlib.cleanup()
 57 | 
 58 |     def setUp(self):
 59 |         self.path, self.env = testlib.temp_env()  # creates 10 databases
 60 |         self.txn = self.env.begin(write=True)
 61 |         putBigData(self.txn)  # HERE!
 62 |         self.c = self.txn.cursor()
 63 |         self.empty_entry = ('', '')
 64 | 
 65 |     def matchList(self, ls_a, ls_b):
 66 |         return all(map(lambda x, y: x == y, ls_a, ls_b))
 67 | 
 68 | 
 69 | class IterationTest(IterationTestBase):
 70 |     def testFromStart(self):
 71 |         # From start
 72 |         self.c.first()
 73 |         self.assertEqual(self.c.key(), KEYS[0])  # start of db
 74 |         test_list = [i for i in iter(self.c)]
 75 |         self.assertEqual(self.matchList(test_list, ITEMS), True)
 76 |         self.assertEqual(self.c.item(), self.empty_entry)  # end of db
 77 | 
 78 |     def testFromStartWithIternext(self):
 79 |         # From start with iternext
 80 |         self.c.first()
 81 |         self.assertEqual(self.c.key(), KEYS[0])  # start of db
 82 |         test_list = [i for i in self.c.iternext()]
 83 |         # remaining elements in db
 84 |         self.assertEqual(self.matchList(test_list, ITEMS), True)
 85 |         self.assertEqual(self.c.item(), self.empty_entry)  # end of db
 86 | 
 87 |     def testFromStartWithNext(self):
 88 |         # From start with next
 89 |         self.c.first()
 90 |         self.assertEqual(self.c.key(), KEYS[0])  # start of db
 91 |         test_list = []
 92 |         while 1:
 93 |             test_list.append(self.c.item())
 94 |             if not self.c.next():
 95 |                 break
 96 |         self.assertEqual(self.matchList(test_list, ITEMS), True)
 97 | 
 98 |     def testFromExistentKeySetKey(self):
 99 |         self.c.first()
100 |         self.c.set_key(KEYS[1])
101 |         self.assertEqual(self.c.key(), KEYS[1])
102 |         test_list = [i for i in self.c.iternext()]
103 |         self.assertEqual(self.matchList(test_list, ITEMS[1:]), True)
104 | 
105 |     def testFromExistentKeySetRange(self):
106 |         self.c.first()
107 |         self.c.set_range(KEYS[1])
108 |         self.assertEqual(self.c.key(), KEYS[1])
109 |         test_list = [i for i in self.c.iternext()]
110 |         self.assertEqual(self.matchList(test_list, ITEMS[1:]), True)
111 | 
112 |     def testFromNonExistentKeySetRange(self):
113 |         self.c.first()
114 |         self.c.set_range(B('c'))
115 |         self.assertEqual(self.c.key(), B('d'))
116 |         test_list = [i for i in self.c.iternext()]
117 |         test_items = [i for i in ITEMS if i[0] > B('c')]
118 |         self.assertEqual(self.matchList(test_list, test_items), True)
119 | 
120 |     def testFromLastKey(self):
121 |         self.c.last()
122 |         self.assertEqual(self.c.key(), KEYS[-1])
123 |         test_list = [i for i in self.c.iternext()]
124 |         self.assertEqual(self.matchList(test_list, ITEMS[-1:]), True)
125 | 
126 |     def testFromNonExistentKeyPastEnd(self):
127 |         self.c.last()
128 |         self.assertEqual(self.c.key(), KEYS[-1])
129 |         # next() fails, leaving iterator in an unpositioned state.
130 |         self.c.next()
131 |         self.assertEqual(self.c.item(), self.empty_entry)
132 |         # iternext() from an unpositioned state proceeds from start of DB.
133 |         test_list = list(self.c.iternext())
134 |         self.assertEqual(test_list, ITEMS)
135 | 
136 | 
137 | class ReverseIterationTest(IterationTestBase):
138 |     def testFromStartRev(self):
139 |         # From start
140 |         self.c.first()
141 |         self.assertEqual(self.c.key(), KEYS[0])  # start of db
142 |         test_list = [i for i in self.c.iterprev()]
143 |         self.assertEqual(self.matchList(test_list, ITEMS[:1][::-1]), True)
144 |         self.assertEqual(self.c.item(), self.empty_entry)  # very start of db
145 | 
146 |     def testFromExistentKeySetKeyRev(self):
147 |         self.c.first()
148 |         self.c.set_key(KEYS[2])
149 |         self.assertEqual(self.c.key(), KEYS[2])
150 |         test_list = [i for i in self.c.iterprev()]
151 |         self.assertEqual(self.matchList(test_list, ITEMS[:3][::-1]), True)
152 | 
153 |     def testFromExistentKeySetRangeRev(self):
154 |         self.c.first()
155 |         self.c.set_range(KEYS[2])
156 |         self.assertEqual(self.c.key(), KEYS[2])
157 |         test_list = [i for i in self.c.iterprev()]
158 |         self.assertEqual(self.matchList(test_list, ITEMS[:3][::-1]), True)
159 | 
160 |     def testFromNonExistentKeySetRangeRev(self):
161 |         self.c.first()
162 |         self.c.set_range(B('c'))
163 |         self.assertEqual(self.c.key(), B('d'))
164 |         test_list = [i for i in self.c.iterprev()]
165 |         test_items = [i for i in ITEMS if i[0] <= B('d')]
166 |         test_items = test_items[::-1]
167 |         self.assertEqual(self.matchList(test_list, test_items), True)
168 | 
169 |     def testFromLastKeyRev(self):
170 |         self.c.last()
171 |         self.assertEqual(self.c.key(), KEYS[-1])
172 |         test_list = [i for i in self.c.iterprev()]
173 |         self.assertEqual(self.matchList(test_list, ITEMS[::-1]), True)
174 | 
175 |     def testFromLastKeyWithPrevRev(self):
176 |         self.c.last()
177 |         self.assertEqual(self.c.key(), KEYS[-1])  # end of db
178 |         test_list = []
179 |         while 1:
180 |             test_list.append(self.c.item())
181 |             if not self.c.prev():
182 |                 break
183 |         self.assertEqual(self.matchList(test_list, ITEMS[::-1]), True)
184 | 
185 |     def testFromNonExistentKeyPastEndRev(self):
186 |         self.c.first()
187 |         self.assertEqual(self.c.key(), KEYS[0])
188 |         # prev() fails, leaving iterator in an unpositioned state.
189 |         self.c.prev()
190 |         self.assertEqual(self.c.item(), self.empty_entry)
191 |         # iterprev() from an unpositioned state proceeds from end of DB.
192 |         test_list = list(self.c.iterprev())
193 |         self.assertEqual(test_list, ITEMS[::-1])
194 | 
195 | class IterationTestWithDupsBase(unittest.TestCase):
196 |     def tearDown(self):
197 |         testlib.cleanup()
198 | 
199 |     def setUp(self):
200 |         self.path, self.env = testlib.temp_env()
201 |         db = self.env.open_db(B('db1'), dupsort=True)
202 |         self.txn = self.env.begin(db, write=True)
203 |         for _ in range(2):
204 |             putData(self.txn)
205 |         self.c = self.txn.cursor()
206 |         self.empty_entry = ('', '')
207 | 
208 |     def matchList(self, ls_a, ls_b):
209 |         return all(map(lambda x, y: x == y, ls_a, ls_b))
210 | 
211 | 
212 | class IterationTestWithDups(IterationTestWithDupsBase):
213 |     pass
214 | 
215 | 
216 | class SeekIterationTest(IterationTestBase2):
217 |     def testForwardIterationSeek(self):
218 |         self.c.first()
219 |         test_list = []
220 |         for i in self.c.iternext():
221 |             test_list.append(i)
222 |             # skips d and e
223 |             if self.c.key() == B('baa'):
224 |                 self.c.set_key(B('e'))
225 |         test_item = [i for i in ITEMS2 if i[0] not in (B('d'), B('e'))]
226 |         self.assertEqual(test_list, test_item)
227 | 
228 |     def testPutDuringIteration(self):
229 |         self.c.first()
230 |         test_list = []
231 |         c = self.txn.cursor()
232 |         for i in c.iternext():
233 |             test_list.append(i)
234 |             # adds 'i' upon seeing 'e'
235 |             if c.key() == B('e'):
236 |                 self.c.put(B('i'), B(''))
237 |         test_item = ITEMS2 + [(B('i'), B(''))]
238 |         self.assertEqual(test_list, test_item)
239 | 
240 |     def testDeleteDuringIteration(self):
241 |         self.c.first()
242 |         test_list = []
243 |         for i in self.c.iternext():
244 |             # deletes 'e' upon seeing it
245 |             if self.c.key() == B('e'):
246 |                 # Causes 'e' to be deleted, and advances cursor to next
247 |                 # element.
248 |                 self.c.delete()
249 |                 i = self.c.item()
250 |             test_list.append(i)
251 | 
252 |         test_item = [i for i in ITEMS2 if i[0] != B('e')]
253 |         self.assertEqual(test_list, test_item)
254 | 
255 | 
256 | if __name__ == '__main__':
257 |     unittest.main()
258 | 


--------------------------------------------------------------------------------
/tests/package_test.py:
--------------------------------------------------------------------------------
 1 | #
 2 | # Copyright 2013 The py-lmdb authors, all rights reserved.
 3 | #
 4 | # Redistribution and use in source and binary forms, with or without
 5 | # modification, are permitted only as authorized by the OpenLDAP
 6 | # Public License.
 7 | #
 8 | # A copy of this license is available in the file LICENSE in the
 9 | # top-level directory of the distribution or, alternatively, at
10 | # <http://www.OpenLDAP.org/license.html>.
11 | #
12 | # OpenLDAP is a registered trademark of the OpenLDAP Foundation.
13 | #
14 | # Individual files and/or contributed packages may be copyright by
15 | # other parties and/or subject to additional restrictions.
16 | #
17 | # This work also contains materials derived from public sources.
18 | #
19 | # Additional information about OpenLDAP can be obtained at
20 | # <http://www.openldap.org/>.
21 | #
22 | 
23 | from __future__ import absolute_import
24 | import unittest
25 | 
26 | import lmdb
27 | 
28 | 
29 | class PackageExportsTest(unittest.TestCase):
30 |     """
31 |     Ensure the list of exported names matches a predefined list. Designed to
32 |     ensure future interface changes to cffi.py and cpython.c don't break
33 |     consistency of "from lmdb import *".
34 |     """
35 |     def test_exports(self):
36 |         assert sorted(lmdb.__all__) == [
37 |             'BadDbiError',
38 |             'BadRslotError',
39 |             'BadTxnError',
40 |             'BadValsizeError',
41 |             'CorruptedError',
42 |             'Cursor',
43 |             'CursorFullError',
44 |             'DbsFullError',
45 |             'DiskError',
46 |             'Environment',
47 |             'Error',
48 |             'IncompatibleError',
49 |             'InvalidError',
50 |             'InvalidParameterError',
51 |             'KeyExistsError',
52 |             'LockError',
53 |             'MapFullError',
54 |             'MapResizedError',
55 |             'MemoryError',
56 |             'NotFoundError',
57 |             'PageFullError',
58 |             'PageNotFoundError',
59 |             'PanicError',
60 |             'ReadersFullError',
61 |             'ReadonlyError',
62 |             'TlsFullError',
63 |             'Transaction',
64 |             'TxnFullError',
65 |             'VersionMismatchError',
66 |             '_Database',
67 |             'enable_drop_gil',
68 |             'version',
69 |         ]
70 | 


--------------------------------------------------------------------------------
/tests/testlib.py:
--------------------------------------------------------------------------------
  1 | #
  2 | # Copyright 2013 The py-lmdb authors, all rights reserved.
  3 | #
  4 | # Redistribution and use in source and binary forms, with or without
  5 | # modification, are permitted only as authorized by the OpenLDAP
  6 | # Public License.
  7 | #
  8 | # A copy of this license is available in the file LICENSE in the
  9 | # top-level directory of the distribution or, alternatively, at
 10 | # <http://www.OpenLDAP.org/license.html>.
 11 | #
 12 | # OpenLDAP is a registered trademark of the OpenLDAP Foundation.
 13 | #
 14 | # Individual files and/or contributed packages may be copyright by
 15 | # other parties and/or subject to additional restrictions.
 16 | #
 17 | # This work also contains materials derived from public sources.
 18 | #
 19 | # Additional information about OpenLDAP can be obtained at
 20 | # <http://www.openldap.org/>.
 21 | #
 22 | 
 23 | from __future__ import absolute_import
 24 | import atexit
 25 | import gc
 26 | import os
 27 | import shutil
 28 | import stat
 29 | import sys
 30 | import tempfile
 31 | import traceback
 32 | import unittest
 33 | 
 34 | try:
 35 |     import __builtin__
 36 | except ImportError:
 37 |     import builtins as __builtin__
 38 | 
 39 | import lmdb
 40 | 
 41 | _cleanups = []
 42 | 
 43 | def cleanup():
 44 |     while _cleanups:
 45 |         func = _cleanups.pop()
 46 |         try:
 47 |             func()
 48 |         except Exception:
 49 |             traceback.print_exc()
 50 | 
 51 | atexit.register(cleanup)
 52 | 
 53 | class LmdbTest(unittest.TestCase):
 54 |     def tearDown(self):
 55 |         cleanup()
 56 | 
 57 | 
 58 | def temp_dir(create=True):
 59 |     path = tempfile.mkdtemp(prefix='lmdb_test')
 60 |     assert path is not None, 'tempfile.mkdtemp failed'
 61 |     if not create:
 62 |         os.rmdir(path)
 63 |     _cleanups.append(lambda: shutil.rmtree(path, ignore_errors=True))
 64 |     if hasattr(path, 'decode'):
 65 |         path = path.decode(sys.getfilesystemencoding())
 66 |     return path
 67 | 
 68 | 
 69 | def temp_file(create=True):
 70 |     fd, path = tempfile.mkstemp(prefix='lmdb_test')
 71 |     assert path is not None, 'tempfile.mkstemp failed'
 72 |     os.close(fd)
 73 |     if not create:
 74 |         os.unlink(path)
 75 |     _cleanups.append(lambda: os.path.exists(path) and os.unlink(path))
 76 |     pathlock = path + '-lock'
 77 |     _cleanups.append(lambda: os.path.exists(pathlock) and os.unlink(pathlock))
 78 |     if hasattr(path, 'decode'):
 79 |         path = path.decode(sys.getfilesystemencoding())
 80 |     return path
 81 | 
 82 | 
 83 | def temp_env(path=None, max_dbs=10, **kwargs):
 84 |     if not path:
 85 |         path = temp_dir()
 86 |     env = lmdb.open(path, max_dbs=max_dbs, **kwargs)
 87 |     _cleanups.append(env.close)
 88 |     return path, env
 89 | 
 90 | 
 91 | def path_mode(path):
 92 |     return stat.S_IMODE(os.stat(path).st_mode)
 93 | 
 94 | 
 95 | def debug_collect():
 96 |     if hasattr(gc, 'set_debug') and hasattr(gc, 'get_debug'):
 97 |         old = gc.get_debug()
 98 |         gc.set_debug(gc.DEBUG_LEAK)
 99 |         gc.collect()
100 |         gc.set_debug(old)
101 |     else:
102 |         for x in range(10):
103 |             # PyPy doesn't collect objects with __del__ on first attempt.
104 |             gc.collect()
105 | 
106 | UnicodeType = getattr(__builtin__, 'unicode', str)
107 | BytesType = getattr(__builtin__, 'bytes', str)
108 | 
109 | try:
110 |     INT_TYPES = (int, long)
111 | except NameError:
112 |     INT_TYPES = (int,)
113 | 
114 | # B(ascii 'string') -> bytes
115 | try:
116 |     bytes('')     # Python>=2.6, alias for str().
117 |     B = lambda s: s
118 | except TypeError: # Python3.x, requires encoding parameter.
119 |     B = lambda s: bytes(s, 'ascii')
120 | 
121 | # BL('s1', 's2') -> ['bytes1', 'bytes2']
122 | BL = lambda *args: list(map(B, args))
123 | # TS('s1', 's2') -> ('bytes1', 'bytes2')
124 | BT = lambda *args: tuple(B(s) for s in args)
125 | # O(int) -> length-1 bytes
126 | O = lambda arg: B(chr(arg))
127 | # OCT(s) -> parse string as octal
128 | OCT = lambda s: int(s, 8)
129 | 
130 | 
131 | KEYS = BL('a', 'b', 'baa', 'd')
132 | ITEMS = [(k, B('')) for k in KEYS]
133 | REV_ITEMS = ITEMS[::-1]
134 | VALUES = [B('') for k in KEYS]
135 | 
136 | KEYS2 = BL('a', 'b', 'baa', 'd', 'e', 'f', 'g', 'h')
137 | ITEMS2 = [(k, B('')) for k in KEYS2]
138 | REV_ITEMS2 = ITEMS2[::-1]
139 | VALUES2 = [B('') for k in KEYS2]
140 | 
141 | KEYSFIXED = BL('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h')
142 | VALUES_MULTI = [(B('r'), B('s')) for k in KEYSFIXED]
143 | ITEMS_MULTI_FIXEDKEY = [
144 |     (kv[0], v) for kv in list(zip(KEYSFIXED, VALUES_MULTI)) for v in kv[1]
145 |     ]
146 | 
147 | def _put_items(items, t, db=None):
148 |     for k, v in items:
149 |         if db:
150 |             t.put(k, v, db=db)
151 |         else:
152 |             t.put(k, v)
153 | 
154 | 
155 | def putData(t, db=None):
156 |     _put_items(ITEMS, t, db=db)
157 | 
158 | def putBigData(t, db=None):
159 |     _put_items(ITEMS2, t, db=db)
160 | 
161 | def putBigDataMultiFixed(t, db=None):
162 |     _put_items(ITEMS_MULTI_FIXEDKEY, t, db=db)


--------------------------------------------------------------------------------
/tests/tool_test.py:
--------------------------------------------------------------------------------
 1 | #
 2 | # Copyright 2013 The py-lmdb authors, all rights reserved.
 3 | #
 4 | # Redistribution and use in source and binary forms, with or without
 5 | # modification, are permitted only as authorized by the OpenLDAP
 6 | # Public License.
 7 | #
 8 | # A copy of this license is available in the file LICENSE in the
 9 | # top-level directory of the distribution or, alternatively, at
10 | # <http://www.OpenLDAP.org/license.html>.
11 | #
12 | # OpenLDAP is a registered trademark of the OpenLDAP Foundation.
13 | #
14 | # Individual files and/or contributed packages may be copyright by
15 | # other parties and/or subject to additional restrictions.
16 | #
17 | # This work also contains materials derived from public sources.
18 | #
19 | # Additional information about OpenLDAP can be obtained at
20 | # <http://www.openldap.org/>.
21 | #
22 | 
23 | from __future__ import absolute_import
24 | 
25 | import sys
26 | import shlex
27 | import unittest
28 | 
29 | import lmdb
30 | import lmdb.tool
31 | import testlib
32 | 
33 | def call_tool(cmdline):
34 |     if sys.platform == 'win32':
35 |         args = cmdline.split()
36 |     else:
37 |         args = shlex.split(cmdline)
38 |     return lmdb.tool.main(args)
39 | 
40 | class ToolTest(testlib.LmdbTest):
41 |     def test_cmd_get(self):
42 |         frompath, env = testlib.temp_env()
43 |         db = env.open_db(b'subdb')
44 |         with env.begin(write=True, db=db) as txn:
45 |             txn.put(b'foo', b'bar', db=db)
46 |         env.close()
47 |         call_tool('-d subdb get --env %s' % (frompath,))
48 | 
49 |     def test_cmd_rewrite(self):
50 |         frompath, env = testlib.temp_env()
51 |         env.open_db(b'subdb')
52 |         env.close()
53 |         topath = testlib.temp_dir()
54 |         call_tool('rewrite -e %s -E %s subdb' % (frompath, topath))
55 | 
56 | if __name__ == '__main__':
57 |     unittest.main()
58 | 


--------------------------------------------------------------------------------