├── .editorconfig ├── .github └── workflows │ └── tests.yml ├── .gitignore ├── .meta.toml ├── .readthedocs.yaml ├── 3.11.txt ├── CHANGES.rst ├── CONTRIBUTING.md ├── COPYING ├── COPYRIGHT.txt ├── DEVELOPERS.rst ├── HISTORY.rst ├── LICENSE.txt ├── MANIFEST.in ├── README.rst ├── buildout.cfg ├── docs ├── .static │ └── zodb.ico ├── ConflictResolution.rst ├── Makefile ├── README.rst ├── articles │ ├── ZODB-overview.rst │ ├── ZODB1.rst │ ├── ZODB2.rst │ ├── images │ │ └── zeo-diagram.png │ ├── index.rst │ ├── multi-zodb-gc.rst │ └── old-guide │ │ ├── README.txt │ │ ├── TODO.txt │ │ ├── chatter.py │ │ ├── convert_zodb_guide.py │ │ ├── gfdl.rst │ │ ├── index.rst │ │ ├── introduction.rst │ │ ├── links.rst │ │ ├── modules.rst │ │ ├── prog-zodb.rst │ │ ├── transactions.rst │ │ └── zeo.rst ├── changelog.rst ├── collaborations.rst ├── conf.py ├── cross-database-references.rst ├── developers.rst ├── event.rst ├── guide │ ├── index.rst │ ├── install-and-run.rst │ ├── transactions-and-threading.rst │ └── writing-persistent-objects.rst ├── historical_connections.rst ├── index.rst ├── introduction.rst ├── persistentclass.rst ├── reference │ ├── index.rst │ ├── storages.rst │ ├── transaction.rst │ └── zodb.rst ├── requirements.txt ├── tutorial.rst ├── utils.rst ├── zodb.png └── zodb.svg ├── setup.cfg ├── setup.py ├── src └── ZODB │ ├── ActivityMonitor.py │ ├── BaseStorage.py │ ├── ConflictResolution.py │ ├── ConflictResolution.rst │ ├── Connection.py │ ├── DB.py │ ├── DemoStorage.py │ ├── DemoStorage.test │ ├── ExportImport.py │ ├── FileStorage │ ├── FileStorage.py │ ├── __init__.py │ ├── format.py │ ├── fsdump.py │ ├── fsoids.py │ ├── fspack.py │ ├── interfaces.py │ ├── iterator.test │ ├── tests.py │ └── zconfig.txt │ ├── MappingStorage.py │ ├── POSException.py │ ├── UndoLogCompatible.py │ ├── __init__.py │ ├── _compat.py │ ├── blob.py │ ├── broken.py │ ├── component.xml │ ├── config.py │ ├── config.xml │ ├── conversionhack.py │ ├── cross-database-references.rst │ ├── event.py │ ├── fsIndex.py │ ├── fsrecover.py │ ├── fstools.py │ ├── historical_connections.rst │ ├── interfaces.py │ ├── loglevels.py │ ├── mvccadapter.py │ ├── persistentclass.py │ ├── persistentclass.rst │ ├── scripts │ ├── README.txt │ ├── __init__.py │ ├── analyze.py │ ├── checkbtrees.py │ ├── fsoids.py │ ├── fsrefs.py │ ├── fsstats.py │ ├── fstail.py │ ├── fstest.py │ ├── manual_tests │ │ ├── test-checker.fs │ │ └── testfstest.py │ ├── migrate.py │ ├── migrateblobs.py │ ├── netspace.py │ ├── referrers.py │ ├── repozo.py │ ├── space.py │ ├── tests │ │ ├── __init__.py │ │ ├── fstail.txt │ │ ├── referrers.txt │ │ ├── test_doc.py │ │ ├── test_fsdump_fsstats.py │ │ ├── test_fstest.py │ │ └── test_repozo.py │ └── zodbload.py │ ├── serialize.py │ ├── storage.xml │ ├── tests │ ├── BasicStorage.py │ ├── ConflictResolution.py │ ├── Corruption.py │ ├── HistoryStorage.py │ ├── IExternalGC.test │ ├── IteratorStorage.py │ ├── MTStorage.py │ ├── MVCCMappingStorage.py │ ├── MinPO.py │ ├── PackableStorage.py │ ├── PersistentStorage.py │ ├── ReadOnlyStorage.py │ ├── RecoveryStorage.py │ ├── RevisionStorage.py │ ├── StorageTestBase.py │ ├── Synchronization.py │ ├── TransactionalUndoStorage.py │ ├── __init__.py │ ├── blob_basic.txt │ ├── blob_connection.txt │ ├── blob_consume.txt │ ├── blob_importexport.txt │ ├── blob_layout.txt │ ├── blob_packing.txt │ ├── blob_tempdir.txt │ ├── blob_transaction.txt │ ├── blobstorage_packing.txt │ ├── component.xml │ ├── dangle.py │ ├── dbopen.txt │ ├── fix84.rst │ ├── hexstorage.py │ ├── loggingsupport.py │ ├── multidb.txt │ ├── racetest.py │ ├── sampledm.py │ ├── speed.py │ ├── synchronizers.txt │ ├── testActivityMonitor.py │ ├── testBroken.py │ ├── testCache.py │ ├── testConfig.py │ ├── testConnection.py │ ├── testConnectionSavepoint.py │ ├── testConnectionSavepoint.txt │ ├── testDB.py │ ├── testDemoStorage.py │ ├── testFileStorage.py │ ├── testMVCCMappingStorage.py │ ├── testMappingStorage.py │ ├── testPersistentList.py │ ├── testPersistentMapping.py │ ├── testPersistentWeakref.py │ ├── testRecover.py │ ├── testSerialize.py │ ├── testThreadedShutdown.py │ ├── testUtils.py │ ├── testZODB.py │ ├── test_TransactionMetaData.py │ ├── test_cache.py │ ├── test_doctest_files.py │ ├── test_fsdump.py │ ├── test_mvccadapter.py │ ├── test_prefetch.py │ ├── test_racetest.py │ ├── test_storage.py │ ├── testblob.py │ ├── testconflictresolution.py │ ├── testcrossdatabasereferences.py │ ├── testdocumentation.py │ ├── testfsIndex.py │ ├── testfsoids.py │ ├── testhistoricalconnections.py │ ├── testmvcc.py │ ├── testpersistentclass.py │ └── util.py │ ├── transact.py │ ├── utils.py │ ├── utils.rst │ └── valuedoc.py └── tox.ini /.editorconfig: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | # 4 | # EditorConfig Configuration file, for more details see: 5 | # http://EditorConfig.org 6 | # EditorConfig is a convention description, that could be interpreted 7 | # by multiple editors to enforce common coding conventions for specific 8 | # file types 9 | 10 | # top-most EditorConfig file: 11 | # Will ignore other EditorConfig files in Home directory or upper tree level. 12 | root = true 13 | 14 | 15 | [*] # For All Files 16 | # Unix-style newlines with a newline ending every file 17 | end_of_line = lf 18 | insert_final_newline = true 19 | trim_trailing_whitespace = true 20 | # Set default charset 21 | charset = utf-8 22 | # Indent style default 23 | indent_style = space 24 | # Max Line Length - a hard line wrap, should be disabled 25 | max_line_length = off 26 | 27 | [*.{py,cfg,ini}] 28 | # 4 space indentation 29 | indent_size = 4 30 | 31 | [*.{yml,zpt,pt,dtml,zcml}] 32 | # 2 space indentation 33 | indent_size = 2 34 | 35 | [{Makefile,.gitmodules}] 36 | # Tab indentation (no size specified, but view as 4 spaces) 37 | indent_style = tab 38 | indent_size = unset 39 | tab_width = unset 40 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | name: tests 4 | 5 | on: 6 | push: 7 | pull_request: 8 | schedule: 9 | - cron: '0 12 * * 0' # run once a week on Sunday 10 | # Allow to run this workflow manually from the Actions tab 11 | workflow_dispatch: 12 | 13 | jobs: 14 | build: 15 | strategy: 16 | # We want to see all failures: 17 | fail-fast: false 18 | matrix: 19 | os: 20 | - ["ubuntu", "ubuntu-20.04"] 21 | - ["windows", "windows-latest"] 22 | config: 23 | # [Python version, tox env] 24 | - ["3.9", "release-check"] 25 | - ["3.9", "lint"] 26 | - ["3.7", "py37"] 27 | - ["3.8", "py38"] 28 | - ["3.9", "py39"] 29 | - ["3.10", "py310"] 30 | - ["3.11", "py311"] 31 | - ["3.12", "py312"] 32 | - ["pypy-3.10", "pypy3"] 33 | - ["3.9", "docs"] 34 | - ["3.9", "coverage"] 35 | - ["3.10", "py310-pure"] 36 | exclude: 37 | - { os: ["windows", "windows-latest"], config: ["3.9", "release-check"] } 38 | - { os: ["windows", "windows-latest"], config: ["3.9", "lint"] } 39 | - { os: ["windows", "windows-latest"], config: ["3.9", "docs"] } 40 | - { os: ["windows", "windows-latest"], config: ["3.9", "coverage"] } 41 | 42 | runs-on: ${{ matrix.os[1] }} 43 | if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name 44 | name: ${{ matrix.os[0] }}-${{ matrix.config[1] }} 45 | steps: 46 | - uses: actions/checkout@v4 47 | - name: Set up Python 48 | uses: actions/setup-python@v5 49 | with: 50 | python-version: ${{ matrix.config[0] }} 51 | - name: Pip cache 52 | uses: actions/cache@v4 53 | with: 54 | path: ~/.cache/pip 55 | key: ${{ runner.os }}-pip-${{ matrix.config[0] }}-${{ hashFiles('setup.*', 'tox.ini') }} 56 | restore-keys: | 57 | ${{ runner.os }}-pip-${{ matrix.config[0] }}- 58 | ${{ runner.os }}-pip- 59 | - name: Install dependencies 60 | run: | 61 | python -m pip install --upgrade pip 62 | pip install tox 63 | - name: Test 64 | run: tox -e ${{ matrix.config[1] }} 65 | - name: Coverage 66 | if: matrix.config[1] == 'coverage' 67 | run: | 68 | pip install coveralls 69 | coveralls --service=github 70 | env: 71 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | *.dll 4 | *.egg-info/ 5 | *.profraw 6 | *.pyc 7 | *.pyo 8 | *.so 9 | .coverage 10 | .coverage.* 11 | .eggs/ 12 | .installed.cfg 13 | .mr.developer.cfg 14 | .tox/ 15 | .vscode/ 16 | __pycache__/ 17 | bin/ 18 | build/ 19 | coverage.xml 20 | develop-eggs/ 21 | develop/ 22 | dist/ 23 | docs/_build 24 | eggs/ 25 | etc/ 26 | lib/ 27 | lib64 28 | log/ 29 | parts/ 30 | pyvenv.cfg 31 | testing.log 32 | var/ 33 | -------------------------------------------------------------------------------- /.meta.toml: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | [meta] 4 | template = "pure-python" 5 | commit-id = "1404d28b" 6 | 7 | [python] 8 | with-windows = true 9 | with-pypy = true 10 | with-future-python = false 11 | with-docs = true 12 | with-sphinx-doctests = false 13 | with-macos = false 14 | 15 | [tox] 16 | use-flake8 = true 17 | testenv-commands = [ 18 | "zope-testrunner --test-path=src -a 1000 {posargs:-vc}", 19 | ] 20 | testenv-setenv = [ 21 | "ZOPE_INTERFACE_STRICT_IRO=1", 22 | ] 23 | additional-envlist = [ 24 | "py310-pure", 25 | ] 26 | testenv-additional = [ 27 | "", 28 | "[testenv:py310-pure]", 29 | "basepython = python3.10", 30 | "setenv =", 31 | " PURE_PYTHON = 1", 32 | ] 33 | 34 | [coverage] 35 | fail-under = 80 36 | 37 | [flake8] 38 | additional-config = [ 39 | "# F401 imported but unused", 40 | "per-file-ignores =", 41 | " src/ZODB/FileStorage/__init__.py: F401", 42 | " src/ZODB/__init__.py: F401", 43 | ] 44 | 45 | [manifest] 46 | additional-rules = [ 47 | "include *.yaml", 48 | "include COPYING", 49 | "recursive-include docs *.ico", 50 | "recursive-include docs *.png", 51 | "recursive-include docs *.svg", 52 | "recursive-include src *.fs", 53 | "recursive-include src *.rst", 54 | "recursive-include src *.test", 55 | "recursive-include src *.txt", 56 | "recursive-include src *.xml", 57 | ] 58 | 59 | [check-manifest] 60 | additional-ignores = [ 61 | "docs/_build/doctest/*/*/*/*", 62 | "docs/_build/doctest/*/*/*", 63 | "docs/_build/doctest/*/*", 64 | "docs/_build/html/*/*/*/*", 65 | "docs/_build/html/*/*/*", 66 | "docs/_build/html/*/*", 67 | ] 68 | 69 | [github-actions] 70 | additional-config = [ 71 | "- [\"3.10\", \"py310-pure\"]", 72 | ] 73 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | # Read the Docs configuration file 4 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 5 | 6 | # Required 7 | version: 2 8 | 9 | # Set the version of Python and other tools you might need 10 | build: 11 | os: ubuntu-22.04 12 | tools: 13 | python: "3.11" 14 | 15 | # Build documentation in the docs/ directory with Sphinx 16 | sphinx: 17 | configuration: docs/conf.py 18 | 19 | # We recommend specifying your dependencies to enable reproducible builds: 20 | # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 21 | python: 22 | install: 23 | - requirements: docs/requirements.txt 24 | - method: pip 25 | path: . 26 | -------------------------------------------------------------------------------- /3.11.txt: -------------------------------------------------------------------------------- 1 | Wish list for 3.11. 2 | 3 | These aren't promises, but things I'd like to do: 4 | 5 | - ZEO support for loading blobs via HTTP. 6 | 7 | - ZEO cache fix for loadBefore. 8 | 9 | - invalidation events. 10 | 11 | - Make DBs context manager, so in a simple script, one could do: 12 | 13 | with ZEO.DB(someaddr) as connection: 14 | do some things in a transaction. Commit and close at the end. 15 | 16 | - Persistent sets. 17 | 18 | - PxBTrees, persistent objects as keys in BTrees. 19 | 20 | - Compare on persistent references. 21 | 22 | - Python BTrees and persistence. 23 | 24 | - JSONic read-only mode where you can read most objects for which you 25 | don't have classes as long at they have the default getstate. 26 | This might be as simple as using a variation of broken objects. 27 | 28 | - persistent.Object, which handles the common case of a simple object 29 | that just has some data. (The moral equivalent of a JS object. :) 30 | 31 | - API to preload objects. 32 | 33 | Say you know you're going to oterate over an array of objects, you 34 | might signal that intend to use the object with something like:: 35 | 36 | for oject in objects: 37 | object._p_will_use() 38 | 39 | (I think there's an RFC for something like this.) 40 | 41 | For most storages, _p_will_use won't have any effect, but for ZEO, 42 | it could cause a load request to be sent to the server if the object 43 | isn't already loaded or in the zeo cache. This way, you could have 44 | lots of loads in flight at once, mitigating round-trip costs. 45 | 46 | - ZEO cache iterator, to facilitate analysis of cache contents. 47 | 48 | - Update file-storage iterator to expose file position as non-private 49 | var for transactions and database records. 50 | 51 | Expose trans size. 52 | 53 | - Update ZEO ClientStorage to block for soem short time rather than 54 | error on short disconnection. 55 | 56 | - Remove silly ZEO connection backooff by default. 57 | 58 | - Rewrite ZEO connection logic using async IO rather than threads. 59 | 60 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 5 | # Contributing to zopefoundation projects 6 | 7 | The projects under the zopefoundation GitHub organization are open source and 8 | welcome contributions in different forms: 9 | 10 | * bug reports 11 | * code improvements and bug fixes 12 | * documentation improvements 13 | * pull request reviews 14 | 15 | For any changes in the repository besides trivial typo fixes you are required 16 | to sign the contributor agreement. See 17 | https://www.zope.dev/developer/becoming-a-committer.html for details. 18 | 19 | Please visit our [Developer 20 | Guidelines](https://www.zope.dev/developer/guidelines.html) if you'd like to 21 | contribute code changes and our [guidelines for reporting 22 | bugs](https://www.zope.dev/developer/reporting-bugs.html) if you want to file a 23 | bug report. 24 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | See: 2 | 3 | - the copyright notice in: COPYRIGHT.txt 4 | 5 | - The Zope Public License in LICENSE.txt 6 | -------------------------------------------------------------------------------- /COPYRIGHT.txt: -------------------------------------------------------------------------------- 1 | Zope Foundation and Contributors -------------------------------------------------------------------------------- /DEVELOPERS.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | Developers notes 3 | ================ 4 | 5 | Building 6 | ======== 7 | 8 | Bootstrap buildout, if necessary using ``bootstrap.py``:: 9 | 10 | python bootstrap.py 11 | 12 | Run the buildout:: 13 | 14 | bin/buildout 15 | 16 | Testing 17 | ======= 18 | 19 | The ZODB checkouts are `buildouts `_. 20 | When working from a ZODB checkout, first run the bootstrap.py script 21 | to initialize the buildout: 22 | 23 | % python bootstrap.py 24 | 25 | and then use the buildout script to build ZODB and gather the dependencies: 26 | 27 | % bin/buildout 28 | 29 | This creates a test script: 30 | 31 | % bin/test -v 32 | 33 | This command will run all the tests, printing a single dot for each 34 | test. When it finishes, it will print a test summary. The exact 35 | number of tests can vary depending on platform and available 36 | third-party libraries.:: 37 | 38 | Ran 1182 tests in 241.269s 39 | 40 | OK 41 | 42 | The test script has many more options. Use the ``-h`` or ``--help`` 43 | options to see a file list of options. The default test suite omits 44 | several tests that depend on third-party software or that take a long 45 | time to run. To run all the available tests use the ``--all`` option. 46 | Running all the tests takes much longer.:: 47 | 48 | Ran 1561 tests in 1461.557s 49 | 50 | OK 51 | 52 | Our primary development platforms are Linux and Mac OS X. The test 53 | suite should pass without error on these platforms and, hopefully, 54 | Windows, although it can take a long time on Windows -- longer if you 55 | use ZoneAlarm. 56 | 57 | Generating docs 58 | =============== 59 | 60 | cd to the doc directory and:: 61 | 62 | make html 63 | 64 | Contributing 65 | ============ 66 | 67 | Almost any code change should include tests. 68 | 69 | Any change that changes features should include documentation updates. 70 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Zope Public License (ZPL) Version 2.1 2 | 3 | A copyright notice accompanies this license document that identifies the 4 | copyright holders. 5 | 6 | This license has been certified as open source. It has also been designated as 7 | GPL compatible by the Free Software Foundation (FSF). 8 | 9 | Redistribution and use in source and binary forms, with or without 10 | modification, are permitted provided that the following conditions are met: 11 | 12 | 1. Redistributions in source code must retain the accompanying copyright 13 | notice, this list of conditions, and the following disclaimer. 14 | 15 | 2. Redistributions in binary form must reproduce the accompanying copyright 16 | notice, this list of conditions, and the following disclaimer in the 17 | documentation and/or other materials provided with the distribution. 18 | 19 | 3. Names of the copyright holders must not be used to endorse or promote 20 | products derived from this software without prior written permission from the 21 | copyright holders. 22 | 23 | 4. The right to distribute this software or to use it for any purpose does not 24 | give you the right to use Servicemarks (sm) or Trademarks (tm) of the 25 | copyright 26 | holders. Use of them is covered by separate agreement with the copyright 27 | holders. 28 | 29 | 5. If any files are modified, you must cause the modified files to carry 30 | prominent notices stating that you changed the files and the date of any 31 | change. 32 | 33 | Disclaimer 34 | 35 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESSED 36 | OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 37 | OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 38 | EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT, 39 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 41 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 42 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 43 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 44 | EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | include *.md 4 | include *.rst 5 | include *.txt 6 | include buildout.cfg 7 | include tox.ini 8 | 9 | recursive-include docs *.py 10 | recursive-include docs *.rst 11 | recursive-include docs *.txt 12 | recursive-include docs Makefile 13 | 14 | recursive-include src *.py 15 | include *.yaml 16 | include COPYING 17 | recursive-include docs *.ico 18 | recursive-include docs *.png 19 | recursive-include docs *.svg 20 | recursive-include src *.fs 21 | recursive-include src *.rst 22 | recursive-include src *.test 23 | recursive-include src *.txt 24 | recursive-include src *.xml 25 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ======================================= 2 | ZODB, a Python object-oriented database 3 | ======================================= 4 | 5 | .. image:: https://img.shields.io/pypi/v/ZODB.svg 6 | :target: https://pypi.org/project/ZODB/ 7 | :alt: Latest release 8 | 9 | .. image:: https://img.shields.io/pypi/pyversions/ZODB.svg 10 | :target: https://pypi.org/project/ZODB/ 11 | :alt: Supported Python versions 12 | 13 | .. image:: https://github.com/zopefoundation/ZODB/actions/workflows/tests.yml/badge.svg 14 | :target: https://github.com/zopefoundation/ZODB/actions/workflows/tests.yml 15 | :alt: Build status 16 | 17 | .. image:: https://coveralls.io/repos/github/zopefoundation/ZODB/badge.svg 18 | :target: https://coveralls.io/github/zopefoundation/ZODB 19 | :alt: Coverage status 20 | 21 | .. image:: https://readthedocs.org/projects/zodb-docs/badge/?version=latest 22 | :target: https://zodb-docs.readthedocs.io/en/latest/ 23 | :alt: Documentation status 24 | 25 | ZODB provides an object-oriented database for Python that provides a 26 | high-degree of transparency. ZODB runs on Python 3.7 and 27 | above. It also runs on PyPy. 28 | 29 | - no separate language for database operations 30 | 31 | - very little impact on your code to make objects persistent 32 | 33 | - no database mapper that partially hides the database. 34 | 35 | Using an object-relational mapping **is not** like using an 36 | object-oriented database. 37 | 38 | - almost no seam between code and database. 39 | 40 | ZODB is an ACID Transactional database. 41 | 42 | To learn more, visit: https://zodb-docs.readthedocs.io 43 | 44 | The github repository is at https://github.com/zopefoundation/zodb 45 | 46 | If you're interested in contributing to ZODB itself, see the 47 | `developer notes 48 | `_. 49 | -------------------------------------------------------------------------------- /buildout.cfg: -------------------------------------------------------------------------------- 1 | [buildout] 2 | develop = . 3 | parts = 4 | test 5 | coverage-test 6 | scripts 7 | sphinx 8 | 9 | [versions] 10 | # Avoid breakage in 4.4.5: 11 | zope.testrunner = >= 4.4.6 12 | # sphinx_rtd_theme depends on docutils<0.17 13 | docutils = < 0.17 14 | 15 | [versions:python2] 16 | Sphinx = < 2 17 | pygments = < 2.6 18 | sphinxcontrib-websupport = < 1.2 19 | 20 | [test] 21 | recipe = zc.recipe.testrunner 22 | eggs = 23 | ZODB [test] 24 | initialization = 25 | import os, tempfile 26 | try: os.mkdir('tmp') 27 | except: pass 28 | tempfile.tempdir = os.path.abspath('tmp') 29 | defaults = ['--all'] 30 | 31 | [coverage-test] 32 | recipe = zc.recipe.testrunner 33 | working-directory = . 34 | eggs = 35 | ${test:eggs} 36 | coverage 37 | initialization = 38 | import os, tempfile 39 | try: os.mkdir('tmp') 40 | except: pass 41 | tempfile.tempdir = os.path.abspath('tmp') 42 | if 'COVERAGE_PROCESS_START' not in os.environ: os.environ['COVERAGE_PROCESS_START'] = '.coveragerc' 43 | else: import coverage; coverage.process_startup() 44 | defaults = ['--all'] 45 | 46 | [coverage-report] 47 | recipe = zc.recipe.egg 48 | eggs = z3c.coverage 49 | scripts = coveragereport=coverage-report 50 | arguments = ('${buildout:directory}/coverage', 51 | '${buildout:directory}/coverage/report') 52 | 53 | [scripts] 54 | recipe = zc.recipe.egg 55 | eggs = 56 | ${coverage-test:eggs} 57 | interpreter = py 58 | 59 | [sphinx] 60 | recipe = zc.recipe.egg 61 | eggs = 62 | Sphinx 63 | docutils 64 | ZODB 65 | sphinxcontrib_zopeext 66 | j1m.sphinxautozconfig 67 | sphinx_rtd_theme 68 | scripts = 69 | sphinx-build 70 | interpreter = stxpy 71 | -------------------------------------------------------------------------------- /docs/.static/zodb.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zopefoundation/ZODB/f2dc04c998b4e50e5105bc4358140809b8d66b54/docs/.static/zodb.ico -------------------------------------------------------------------------------- /docs/ConflictResolution.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../src/ZODB/ConflictResolution.rst 2 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = ../bin/sphinx-build 7 | PAPER = 8 | 9 | # Internal variables. 10 | PAPEROPT_a4 = -D latex_paper_size=a4 11 | PAPEROPT_letter = -D latex_paper_size=letter 12 | ALLSPHINXOPTS = -d build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 13 | 14 | .PHONY: help clean html web pickle htmlhelp latex changes linkcheck 15 | 16 | help: 17 | @echo "Please use \`make ' where is one of" 18 | @echo " html to make standalone HTML files" 19 | @echo " pickle to make pickle files" 20 | @echo " json to make JSON files" 21 | @echo " htmlhelp to make HTML files and a HTML help project" 22 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 23 | @echo " changes to make an overview over all changed/added/deprecated items" 24 | @echo " linkcheck to check all external links for integrity" 25 | 26 | clean: 27 | -rm -rf build/* 28 | 29 | html: 30 | mkdir -p build/html build/doctrees 31 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) build/html 32 | @echo 33 | @echo "Build finished. The HTML pages are in build/html." 34 | 35 | pickle: 36 | mkdir -p build/pickle build/doctrees 37 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) build/pickle 38 | @echo 39 | @echo "Build finished; now you can process the pickle files." 40 | 41 | web: pickle 42 | 43 | json: 44 | mkdir -p build/json build/doctrees 45 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) build/json 46 | @echo 47 | @echo "Build finished; now you can process the JSON files." 48 | 49 | htmlhelp: 50 | mkdir -p build/htmlhelp build/doctrees 51 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) build/htmlhelp 52 | @echo 53 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 54 | ".hhp project file in build/htmlhelp." 55 | 56 | latex: 57 | mkdir -p build/latex build/doctrees 58 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) build/latex 59 | @echo 60 | @echo "Build finished; the LaTeX files are in build/latex." 61 | @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ 62 | "run these through (pdf)latex." 63 | 64 | changes: 65 | mkdir -p build/changes build/doctrees 66 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) build/changes 67 | @echo 68 | @echo "The overview file is in build/changes." 69 | 70 | linkcheck: 71 | mkdir -p build/linkcheck build/doctrees 72 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) build/linkcheck 73 | @echo 74 | @echo "Link check complete; look for any errors in the above output " \ 75 | "or in build/linkcheck/output.txt." 76 | -------------------------------------------------------------------------------- /docs/README.rst: -------------------------------------------------------------------------------- 1 | ================== 2 | ZODB documentation 3 | ================== 4 | 5 | ``zodbdocs`` is the source documentation for the website https://zodb.readthedocs.io. It 6 | contains all ZODB relevant documentation like "ZODB/ZEO Programming Guide", 7 | some ZODB articles and links to the ZODB release notes. 8 | 9 | 10 | Building the documentation 11 | -------------------------- 12 | 13 | All documentation is formatted as restructured text. To generate HTML using 14 | Sphinx, use the following:: 15 | 16 | python bootstrap.py 17 | ./bin/buildout 18 | make html 19 | -------------------------------------------------------------------------------- /docs/articles/images/zeo-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zopefoundation/ZODB/f2dc04c998b4e50e5105bc4358140809b8d66b54/docs/articles/images/zeo-diagram.png -------------------------------------------------------------------------------- /docs/articles/index.rst: -------------------------------------------------------------------------------- 1 | ZODB articles 2 | ============= 3 | 4 | Contents 5 | -------- 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | ZODB-overview.rst 11 | ZODB1.rst 12 | ZODB2.rst 13 | old-guide/index 14 | multi-zodb-gc.rst 15 | 16 | 17 | Other ZODB Resources 18 | -------------------- 19 | 20 | - IBM developerWorks `Example-driven ZODB 21 | `_ 22 | 23 | - `How To Love ZODB and Forget RDBMS 24 | `_ 25 | 26 | - `Very old ZODB wiki `_ 27 | -------------------------------------------------------------------------------- /docs/articles/old-guide/README.txt: -------------------------------------------------------------------------------- 1 | This directory contains Andrew Kuchling's programmer's guide to ZODB 2 | and ZEO. The tex source was not being updated in the ZODB docs directory 3 | 4 | It was originally taken from Andrew's zodb.sf.net project on 5 | SourceForge. Because the original version is no longer updated, this 6 | version [in the zodb docs dir] is best viewed as an independent fork now. 7 | -------------------------------------------------------------------------------- /docs/articles/old-guide/TODO.txt: -------------------------------------------------------------------------------- 1 | Write section on __setstate__ 2 | Continue working on it 3 | Suppress the full GFDL text in the PDF/PS versions 4 | 5 | -------------------------------------------------------------------------------- /docs/articles/old-guide/chatter.py: -------------------------------------------------------------------------------- 1 | 2 | import sys, time, os, random 3 | 4 | import transaction 5 | from persistent import Persistent 6 | 7 | from ZEO import ClientStorage 8 | import ZODB 9 | from ZODB.POSException import ConflictError 10 | from BTrees import OOBTree 11 | 12 | class ChatSession(Persistent): 13 | 14 | """Class for a chat session. 15 | Messages are stored in a B-tree, indexed by the time the message 16 | was created. (Eventually we'd want to throw messages out, 17 | 18 | add_message(message) -- add a message to the channel 19 | new_messages() -- return new messages since the last call to 20 | this method 21 | 22 | 23 | """ 24 | 25 | def __init__(self, name): 26 | """Initialize new chat session. 27 | name -- the channel's name 28 | """ 29 | 30 | self.name = name 31 | 32 | # Internal attribute: _messages holds all the chat messages. 33 | self._messages = OOBTree.OOBTree() 34 | 35 | 36 | def new_messages(self): 37 | "Return new messages." 38 | 39 | # self._v_last_time is the time of the most recent message 40 | # returned to the user of this class. 41 | if not hasattr(self, '_v_last_time'): 42 | self._v_last_time = 0 43 | 44 | new = [] 45 | T = self._v_last_time 46 | 47 | for T2, message in self._messages.items(): 48 | if T2 > T: 49 | new.append( message ) 50 | self._v_last_time = T2 51 | 52 | return new 53 | 54 | def add_message(self, message): 55 | """Add a message to the channel. 56 | message -- text of the message to be added 57 | """ 58 | 59 | while 1: 60 | try: 61 | now = time.time() 62 | self._messages[ now ] = message 63 | transaction.commit() 64 | except ConflictError: 65 | # Conflict occurred; this process should abort, 66 | # wait for a little bit, then try again. 67 | transaction.abort() 68 | time.sleep(.2) 69 | else: 70 | # No ConflictError exception raised, so break 71 | # out of the enclosing while loop. 72 | break 73 | # end while 74 | 75 | def get_chat_session(conn, channelname): 76 | """Return the chat session for a given channel, creating the session 77 | if required.""" 78 | 79 | # We'll keep a B-tree of sessions, mapping channel names to 80 | # session objects. The B-tree is stored at the ZODB's root under 81 | # the key 'chat_sessions'. 82 | root = conn.root() 83 | if not root.has_key('chat_sessions'): 84 | print 'Creating chat_sessions B-tree' 85 | root['chat_sessions'] = OOBTree.OOBTree() 86 | transaction.commit() 87 | 88 | sessions = root['chat_sessions'] 89 | 90 | # Get a session object corresponding to the channel name, creating 91 | # it if necessary. 92 | if not sessions.has_key( channelname ): 93 | print 'Creating new session:', channelname 94 | sessions[ channelname ] = ChatSession(channelname) 95 | transaction.commit() 96 | 97 | session = sessions[ channelname ] 98 | return session 99 | 100 | 101 | if __name__ == '__main__': 102 | if len(sys.argv) != 2: 103 | print 'Usage: %s ' % sys.argv[0] 104 | sys.exit(0) 105 | 106 | storage = ClientStorage.ClientStorage( ('localhost', 9672) ) 107 | db = ZODB.DB( storage ) 108 | conn = db.open() 109 | 110 | s = session = get_chat_session(conn, sys.argv[1]) 111 | 112 | messages = ['Hi.', 'Hello', 'Me too', "I'M 3L33T!!!!"] 113 | 114 | while 1: 115 | # Send a random message 116 | msg = random.choice(messages) 117 | session.add_message( '%s: pid %i' % (msg,os.getpid() )) 118 | 119 | # Display new messages 120 | for msg in session.new_messages(): 121 | print msg 122 | 123 | # Wait for a few seconds 124 | pause = random.randint( 1, 4 ) 125 | time.sleep( pause ) 126 | -------------------------------------------------------------------------------- /docs/articles/old-guide/convert_zodb_guide.py: -------------------------------------------------------------------------------- 1 | # Use the python docs converter to convert to rst 2 | # Requires http://svn.python.org/projects/doctools/converter 3 | 4 | from converter import restwriter, convert_file 5 | 6 | import sys 7 | import os 8 | 9 | 10 | if __name__ == '__main__': 11 | try: 12 | rootdir = sys.argv[1] 13 | destdir = os.path.abspath(sys.argv[2]) 14 | except IndexError: 15 | print "usage: convert.py docrootdir destdir" 16 | sys.exit() 17 | 18 | os.chdir(rootdir) 19 | 20 | class IncludeRewrite: 21 | def get(self, a, b=None): 22 | if os.path.exists(a + '.tex'): 23 | return a + '.rst' 24 | print "UNKNOWN FILE %s" % a 25 | return a 26 | restwriter.includes_mapping = IncludeRewrite() 27 | 28 | for infile in os.listdir('.'): 29 | if infile.endswith('.tex'): 30 | convert_file(infile, os.path.join(destdir, infile[:-3]+'rst')) 31 | -------------------------------------------------------------------------------- /docs/articles/old-guide/index.rst: -------------------------------------------------------------------------------- 1 | =============================== 2 | Very old ZODB programming guide 3 | =============================== 4 | 5 | 6 | This guide is based heavily on the work of A. M. Kuchling who wrote the 7 | original guide back in 2002 and which was published under the GNU Free 8 | Documentation License, Version 1.1. See the appendix entitled "GNU Free 9 | Documentation License" for more information. 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | introduction.rst 15 | prog-zodb.rst 16 | zeo.rst 17 | transactions.rst 18 | modules.rst 19 | links.rst 20 | gfdl.rst 21 | -------------------------------------------------------------------------------- /docs/articles/old-guide/links.rst: -------------------------------------------------------------------------------- 1 | .. % links.tex 2 | .. % Collection of relevant links 3 | 4 | 5 | Resources 6 | ========= 7 | 8 | Introduction to the Zope Object Database, by Jim Fulton: --- Goes into much 9 | greater detail, explaining advanced uses of the ZODB and how it's actually 10 | implemented. A definitive reference, and highly recommended. --- 11 | ``_ 12 | 13 | Persistent Programing with ZODB, by Jeremy Hylton and Barry Warsaw: --- Slides 14 | for a tutorial presented at the 10th Python conference. Covers much of the same 15 | ground as this guide, with more details in some areas and less in others. --- 16 | ``_ 17 | 18 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGES.rst 2 | 3 | .. include:: ../HISTORY.rst 4 | -------------------------------------------------------------------------------- /docs/cross-database-references.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../src/ZODB/cross-database-references.rst 2 | -------------------------------------------------------------------------------- /docs/developers.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../DEVELOPERS.rst 2 | -------------------------------------------------------------------------------- /docs/event.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | Event support 3 | ============= 4 | 5 | Sometimes, you want to react when ZODB does certain things. In the 6 | past, ZODB provided ad hoc hook functions for this. Going forward, 7 | ZODB will use an event mechanism. ZODB.event.notify is called with 8 | events of interest. 9 | 10 | If zope.event is installed, then ZODB.event.notify is simply an alias 11 | for zope.event. If zope.event isn't installed, then ZODB.event is a 12 | noop. 13 | -------------------------------------------------------------------------------- /docs/guide/index.rst: -------------------------------------------------------------------------------- 1 | ====================== 2 | ZODB programming guide 3 | ====================== 4 | 5 | This guide consists of a collection of topics that should be of 6 | interest to most developers. They're provided in order of importance, 7 | which is also an order from least to most advanced, but they can be 8 | read in any order. 9 | 10 | If you haven't yet, you should read the :ref:`Tutorial `. 11 | 12 | .. toctree:: 13 | :maxdepth: 2 14 | 15 | install-and-run 16 | writing-persistent-objects.rst 17 | transactions-and-threading 18 | 19 | .. todo: 20 | 21 | blobs 22 | packing-and-garbage-collection 23 | multi-databases 24 | blobs 25 | -------------------------------------------------------------------------------- /docs/historical_connections.rst: -------------------------------------------------------------------------------- 1 | 2 | .. include:: ../src/ZODB/historical_connections.rst 3 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ZODB - a native object database for Python 3 | ========================================== 4 | 5 | Because ZODB is an object database: 6 | 7 | - no separate language for database operations 8 | 9 | - very little impact on your code to make objects persistent 10 | 11 | - no database mapper that partially hides the database. 12 | 13 | Using an object-relational mapping **is not** like using an object database. 14 | 15 | - almost no seam between code and database. 16 | 17 | - Relationships between objects are handled very naturally, supporting 18 | complex object graphs without joins. 19 | 20 | Check out the :doc:`tutorial`! 21 | 22 | ZODB runs on Python 2.7 or Python 3.4 and above. It also runs on PyPy. 23 | 24 | Learning more 25 | ============= 26 | 27 | .. toctree:: 28 | :maxdepth: 1 29 | 30 | introduction 31 | tutorial 32 | guide/index 33 | articles/index 34 | ConflictResolution 35 | collaborations 36 | cross-database-references 37 | event 38 | historical_connections 39 | persistentclass 40 | utils 41 | developers 42 | changelog 43 | reference/index 44 | 45 | * `The ZODB Book (in progress) `_ 46 | 47 | 48 | What is the expansion of "ZODB"? 49 | ================================ 50 | 51 | The expansion of "ZODB" is the Z Object Database. But, of course, we 52 | usually just use "ZODB". 53 | 54 | In the past, it was the Zope Object Database, because it was 55 | developed as part of the Zope project. But ZODB doesn't depend on 56 | Zope in any way and is used in many projects that have nothing to do 57 | with Zope. 58 | 59 | 60 | Downloads 61 | ========= 62 | 63 | ZODB is distributed through the `Python Package Index 64 | `_. 65 | 66 | You can install the ZODB using pip command:: 67 | 68 | $ pip install ZODB 69 | 70 | Community and contributing 71 | ========================== 72 | 73 | Discussion occurs on the `ZODB mailing list 74 | `_. (And for the 75 | transaction system on the `transaction list 76 | `_) 77 | 78 | Bug reporting and feature requests are submitted through github issue 79 | trackers for various ZODB components: 80 | 81 | - ZODB `repository `_ 82 | 83 | - persistent `documentation `_ and its `repository `_. 84 | 85 | - transaction `documentation `_ and its `repository `_ 86 | 87 | - BTrees `documentation `_ and their `repository `_ 88 | 89 | - ZEO (client-server framework) `documentation `_ and its `repository `_ 90 | 91 | - relstorage `documentation `_ and its `repository `_ 92 | 93 | - zodburi `documentation `_ and its `repository `_ 94 | 95 | - NEO `documentation `_ and its `repository `_ 96 | 97 | - readonlystorage `repository `_ 98 | 99 | - newt db `documentation `_ and its `repository `_ 100 | 101 | If you'd like to contribute then we'll gladly accept work on documentation, 102 | helping out other developers and users at the mailing list, submitting bugs, 103 | creating proposals and writing code. 104 | 105 | ZODB is a project managed by the Zope Foundation so you can get write access 106 | for contributing directly - check out the foundation's `Zope Developer Information `_. 107 | -------------------------------------------------------------------------------- /docs/persistentclass.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../src/ZODB/persistentclass.rst 2 | -------------------------------------------------------------------------------- /docs/reference/index.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | Reference Documentation 3 | ======================= 4 | 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | zodb 10 | storages 11 | transaction 12 | -------------------------------------------------------------------------------- /docs/reference/transaction.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Transactions 3 | ============ 4 | 5 | Transaction support is provided by the `transaction 6 | `_ package 7 | [#transaction-package-can-be-used-wo-ZODB]_, which is installed 8 | automatically when you install ZODB. There are two important APIs 9 | provided by the transaction package, ``ITransactionManager`` and 10 | ``ITransaction``, described below. 11 | 12 | ITransactionManager 13 | =================== 14 | 15 | .. autointerface:: transaction.interfaces.ITransactionManager 16 | :members: begin, get, commit, abort, doom, isDoomed, savepoint 17 | 18 | ITransaction 19 | ============ 20 | 21 | .. autointerface:: transaction.interfaces.ITransaction 22 | :members: user, description, commit, abort, doom, savepoint, note, 23 | setUser, setExtendedInfo, 24 | addBeforeCommitHook, getBeforeCommitHooks, 25 | addAfterCommitHook, getAfterCommitHooks 26 | 27 | 28 | .. [#transaction-package-can-be-used-wo-ZODB] The :mod:transaction 29 | package is a general purpose package for managing `distributed 30 | transactions 31 | `_ with a 32 | `two-phase commit protocol 33 | `_. It 34 | can and occasionally is used with packages other than ZODB. 35 | -------------------------------------------------------------------------------- /docs/reference/zodb.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | ZODB APIs 3 | ========= 4 | 5 | .. contents:: 6 | 7 | ZODB module functions 8 | ===================== 9 | 10 | .. method:: DB(storage, *args, **kw) 11 | 12 | Create a database. See :py:class:`ZODB.DB`. 13 | 14 | .. autofunction:: ZODB.connection 15 | 16 | Databases 17 | ========= 18 | 19 | .. autoclass:: ZODB.DB 20 | :members: __init__, open, close, pack, 21 | cacheDetail, cacheExtremeDetail, cacheMinimize, 22 | cacheSize, cacheDetailSize, getCacheSize, getCacheSizeBytes, 23 | lastTransaction, getName, getPoolSize, getSize, 24 | getHistoricalCacheSize, getHistoricalCacheSizeBytes, 25 | getHistoricalPoolSize, getHistoricalTimeout, 26 | objectCount, connectionDebugInfo, 27 | setCacheSize, setCacheSizeBytes, 28 | setHistoricalCacheSize, setHistoricalCacheSizeBytes, 29 | setPoolSize, setHistoricalPoolSize, setHistoricalTimeout, 30 | history, 31 | supportsUndo, undoLog, undoInfo, undoMultiple, undo, 32 | transaction, storage 33 | 34 | .. _database-text-configuration: 35 | 36 | Database text configuration 37 | --------------------------- 38 | 39 | Databases are configured with ``zodb`` sections:: 40 | 41 | 42 | cache-size-bytes 100MB 43 | 44 | 45 | 46 | 47 | A ``zodb`` section must have a storage sub-section specifying a 48 | storage and any of the following options: 49 | 50 | .. zconfigsectionkeys:: ZODB component.xml zodb 51 | 52 | .. _multidatabase-text-configuration: 53 | 54 | For a multi-database configuration, use multiple ``zodb`` sections and 55 | give the sections names:: 56 | 57 | 58 | cache-size-bytes 100MB 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | .. -> src 69 | 70 | >>> import ZODB.config 71 | >>> db = ZODB.config.databaseFromString(src) 72 | >>> sorted(db.databases) 73 | ['first', 'second'] 74 | >>> db._cache_size_bytes 75 | 104857600 76 | 77 | When the configuration is loaded, a single database will be returned, 78 | but all of the databases will be available through the returned 79 | database's ``databases`` attribute. 80 | 81 | Connections 82 | =========== 83 | 84 | .. autoclass:: ZODB.Connection.Connection 85 | :members: add, cacheGC, cacheMinimize, close, db, get, 86 | getDebugInfo, get_connection, isReadOnly, oldstate, 87 | onCloseCallback, root, setDebugInfo, sync, 88 | transaction_manager 89 | 90 | TimeStamp (transaction ids) 91 | =========================== 92 | 93 | .. class:: ZODB.TimeStamp.TimeStamp(year, month, day, hour, minute, seconds) 94 | 95 | Create a time-stamp object. Time stamps facilitate the computation 96 | of transaction ids, which are based on times. The arguments are 97 | integers, except for seconds, which may be a floating-point 98 | number. Time stamps have microsecond precision. Time stamps are 99 | implicitly UTC based. 100 | 101 | Time stamps are orderable and hashable. 102 | 103 | .. method:: day() 104 | 105 | Return the time stamp's day. 106 | 107 | .. method:: hour() 108 | 109 | Return the time stamp's hour. 110 | 111 | .. method:: laterThan(other) 112 | 113 | Return a timestamp instance which is later than 'other'. 114 | 115 | If self already qualifies, return self. 116 | 117 | Otherwise, return a new instance one moment later than 'other'. 118 | 119 | .. method:: minute() 120 | 121 | Return the time stamp's minute. 122 | 123 | .. method:: month() 124 | 125 | Return the time stamp's month. 126 | 127 | .. method:: raw() 128 | 129 | Get an 8-byte representation of the time stamp for use in APIs 130 | that require a time stamp. 131 | 132 | .. method:: second() 133 | 134 | Return the time stamp's second. 135 | 136 | .. method:: timeTime() 137 | 138 | Return the time stamp as seconds since the epoc, as used by the 139 | ``time`` module. 140 | 141 | .. method:: year() 142 | 143 | Return the time stamp's year. 144 | 145 | Loading configuration 146 | ===================== 147 | 148 | .. automodule:: ZODB.config 149 | :members: databaseFromString, databaseFromFile, databaseFromURL, 150 | storageFromString, storageFromFile, storageFromURL 151 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # sphinx_rtd_theme requires docutils < 0.19 2 | docutils < 0.19 3 | # sphinxcontrib_zopeext is not compatible with Sphinx 7 4 | Sphinx > 6, < 7 5 | # Silence dependabot claiming a security issue in older versions: 6 | pygments >= 2.7.4 7 | docutils 8 | ZODB 9 | sphinxcontrib_zopeext 10 | j1m.sphinxautozconfig 11 | # Force a recent version 12 | sphinx_rtd_theme > 1 13 | -------------------------------------------------------------------------------- /docs/utils.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../src/ZODB/utils.rst 2 | -------------------------------------------------------------------------------- /docs/zodb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zopefoundation/ZODB/f2dc04c998b4e50e5105bc4358140809b8d66b54/docs/zodb.png -------------------------------------------------------------------------------- /docs/zodb.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ZODB_logo 5 | Created with Sketch. 6 | 7 | 8 | 12 | 13 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | 4 | [flake8] 5 | doctests = 1 6 | # F401 imported but unused 7 | per-file-ignores = 8 | src/ZODB/FileStorage/__init__.py: F401 9 | src/ZODB/__init__.py: F401 10 | 11 | [check-manifest] 12 | ignore = 13 | .editorconfig 14 | .meta.toml 15 | docs/_build/html/_sources/* 16 | docs/_build/doctest/*/*/*/* 17 | docs/_build/doctest/*/*/* 18 | docs/_build/doctest/*/* 19 | docs/_build/html/*/*/*/* 20 | docs/_build/html/*/*/* 21 | docs/_build/html/*/* 22 | 23 | [isort] 24 | force_single_line = True 25 | combine_as_imports = True 26 | sections = FUTURE,STDLIB,THIRDPARTY,ZOPE,FIRSTPARTY,LOCALFOLDER 27 | known_third_party = docutils, pkg_resources, pytz 28 | known_zope = 29 | known_first_party = 30 | default_section = ZOPE 31 | line_length = 79 32 | lines_after_imports = 2 33 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2002, 2003 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | from setuptools import find_packages 15 | from setuptools import setup 16 | 17 | 18 | def read(path): 19 | with open(path) as f: 20 | return f.read() 21 | 22 | 23 | long_description = read("README.rst") + "\n\n" + read("CHANGES.rst") 24 | 25 | 26 | setup( 27 | name="ZODB", 28 | version='6.0.2.dev0', 29 | author="Jim Fulton", 30 | author_email="jim@zope.com", 31 | maintainer="Zope Foundation and Contributors", 32 | maintainer_email="zodb-dev@zope.dev", 33 | keywords="database nosql python zope", 34 | packages=find_packages('src'), 35 | package_dir={'': 'src'}, 36 | url='http://zodb-docs.readthedocs.io', 37 | license="ZPL 2.1", 38 | platforms=["any"], 39 | classifiers=[ 40 | "Intended Audience :: Developers", 41 | "License :: OSI Approved :: Zope Public License", 42 | "Programming Language :: Python", 43 | "Programming Language :: Python :: 3", 44 | "Programming Language :: Python :: 3.7", 45 | "Programming Language :: Python :: 3.8", 46 | "Programming Language :: Python :: 3.9", 47 | "Programming Language :: Python :: 3.10", 48 | "Programming Language :: Python :: 3.11", 49 | "Programming Language :: Python :: 3.12", 50 | "Programming Language :: Python :: Implementation :: CPython", 51 | "Programming Language :: Python :: Implementation :: PyPy", 52 | "Topic :: Database", 53 | "Topic :: Software Development :: Libraries :: Python Modules", 54 | "Operating System :: Microsoft :: Windows", 55 | "Operating System :: Unix", 56 | "Framework :: ZODB", 57 | ], 58 | description=long_description.split('\n', 2)[1], 59 | long_description=long_description, 60 | long_description_content_type='text/x-rst', 61 | extras_require={ 62 | 'test': [ 63 | 'manuel', 64 | 'zope.testing', 65 | 'zope.testrunner >= 4.4.6', 66 | ], 67 | 'docs': [ 68 | 'Sphinx < 7', 69 | 'ZODB', 70 | 'j1m.sphinxautozconfig', 71 | 'sphinx_rtd_theme', 72 | 'sphinxcontrib_zopeext', 73 | ], 74 | }, 75 | install_requires=[ 76 | 'persistent >= 4.4.0', 77 | 'BTrees >= 4.2.0', 78 | 'ZConfig', 79 | 'transaction >= 2.4', 80 | 'zc.lockfile', 81 | 'zope.interface', 82 | 'zodbpickle >= 1.0.1', 83 | ], 84 | zip_safe=False, 85 | entry_points=""" 86 | [console_scripts] 87 | fsdump = ZODB.FileStorage.fsdump:main 88 | fsoids = ZODB.scripts.fsoids:main 89 | fsrefs = ZODB.scripts.fsrefs:main 90 | fstail = ZODB.scripts.fstail:Main 91 | repozo = ZODB.scripts.repozo:main 92 | """, 93 | include_package_data=True, 94 | python_requires='>=3.7', 95 | ) 96 | -------------------------------------------------------------------------------- /src/ZODB/ActivityMonitor.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """ZODB transfer activity monitoring 15 | """ 16 | 17 | import time 18 | 19 | from . import utils 20 | 21 | 22 | class ActivityMonitor: 23 | """ZODB load/store activity monitor 24 | 25 | This simple implementation just keeps a small log in memory 26 | and iterates over the log when getActivityAnalysis() is called. 27 | 28 | It assumes that log entries are added in chronological sequence. 29 | """ 30 | 31 | def __init__(self, history_length=3600): 32 | self.history_length = history_length # Number of seconds 33 | self.log = [] # [(time, loads, stores)] 34 | self.trim_lock = utils.Lock() 35 | 36 | def closedConnection(self, conn): 37 | log = self.log 38 | now = time.time() 39 | loads, stores = conn.getTransferCounts(1) 40 | log.append((now, loads, stores)) 41 | self.trim(now) 42 | 43 | def trim(self, now): 44 | with self.trim_lock: 45 | log = self.log 46 | cutoff = now - self.history_length 47 | n = 0 48 | loglen = len(log) 49 | while n < loglen and log[n][0] < cutoff: 50 | n = n + 1 51 | if n: 52 | del log[:n] 53 | 54 | def setHistoryLength(self, history_length): 55 | self.history_length = history_length 56 | self.trim(time.time()) 57 | 58 | def getHistoryLength(self): 59 | return self.history_length 60 | 61 | def getActivityAnalysis(self, start=0, end=0, divisions=10): 62 | res = [] 63 | now = time.time() 64 | if start == 0: 65 | start = now - self.history_length 66 | if end == 0: 67 | end = now 68 | for n in range(divisions): 69 | res.append({ 70 | 'start': start + (end - start) * n / divisions, 71 | 'end': start + (end - start) * (n + 1) / divisions, 72 | 'loads': 0, 73 | 'stores': 0, 74 | 'connections': 0, 75 | }) 76 | 77 | div = res[0] 78 | div_end = div['end'] 79 | div_index = 0 80 | connections = 0 81 | total_loads = 0 82 | total_stores = 0 83 | for t, loads, stores in self.log: 84 | if t < start: 85 | # We could use a binary search to find the start. 86 | continue 87 | elif t > end: 88 | # We could use a binary search to find the end also. 89 | break 90 | while t > div_end: 91 | div['loads'] = total_loads 92 | div['stores'] = total_stores 93 | div['connections'] = connections 94 | total_loads = 0 95 | total_stores = 0 96 | connections = 0 97 | div_index = div_index + 1 98 | if div_index < divisions: 99 | div = res[div_index] 100 | div_end = div['end'] 101 | connections = connections + 1 102 | total_loads = total_loads + loads 103 | total_stores = total_stores + stores 104 | 105 | div['stores'] = div['stores'] + total_stores 106 | div['loads'] = div['loads'] + total_loads 107 | div['connections'] = div['connections'] + connections 108 | 109 | return res 110 | -------------------------------------------------------------------------------- /src/ZODB/FileStorage/__init__.py: -------------------------------------------------------------------------------- 1 | # this is a package 2 | 3 | from ZODB.FileStorage.FileStorage import FileIterator 4 | from ZODB.FileStorage.FileStorage import FileStorage 5 | from ZODB.FileStorage.FileStorage import Record 6 | from ZODB.FileStorage.FileStorage import TransactionRecord 7 | from ZODB.FileStorage.FileStorage import packed_version 8 | 9 | 10 | # BBB Alias for compatibility 11 | RecordIterator = TransactionRecord 12 | -------------------------------------------------------------------------------- /src/ZODB/FileStorage/fsdump.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2003 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | import struct 15 | 16 | from ZODB.FileStorage import FileIterator 17 | from ZODB.FileStorage.format import DATA_HDR 18 | from ZODB.FileStorage.format import DATA_HDR_LEN 19 | from ZODB.FileStorage.format import TRANS_HDR 20 | from ZODB.FileStorage.format import TRANS_HDR_LEN 21 | from ZODB.TimeStamp import TimeStamp 22 | from ZODB.utils import get_pickle_metadata 23 | from ZODB.utils import u64 24 | 25 | 26 | def fsdump(path, file=None, with_offset=1): 27 | iter = FileIterator(path) 28 | for i, trans in enumerate(iter): 29 | size = trans._tend - trans._tpos 30 | if with_offset: 31 | print(("Trans #%05d tid=%016x size=%d time=%s offset=%d" % 32 | (i, u64(trans.tid), size, 33 | TimeStamp(trans.tid), trans._pos)), file=file) 34 | else: 35 | print(("Trans #%05d tid=%016x size=%d time=%s" % 36 | (i, u64(trans.tid), size, TimeStamp(trans.tid))), file=file) 37 | print((" status=%r user=%r description=%r" % 38 | (trans.status, trans.user, trans.description)), file=file) 39 | 40 | for j, rec in enumerate(trans): 41 | if rec.data is None: 42 | fullclass = "undo or abort of object creation" 43 | size = "" 44 | else: 45 | modname, classname = get_pickle_metadata(rec.data) 46 | size = " size=%d" % len(rec.data) 47 | fullclass = "{}.{}".format(modname, classname) 48 | 49 | if rec.data_txn: 50 | # It would be nice to print the transaction number 51 | # (i) but it would be expensive to keep track of. 52 | bp = " bp=%016x" % u64(rec.data_txn) 53 | else: 54 | bp = "" 55 | 56 | print((" data #%05d oid=%016x%s class=%s%s" % 57 | (j, u64(rec.oid), size, fullclass, bp)), file=file) 58 | iter.close() 59 | 60 | 61 | def fmt(p64): 62 | # Return a nicely formatted string for a packaged 64-bit value 63 | return "%016x" % u64(p64) 64 | 65 | 66 | class Dumper: 67 | """A very verbose dumper for debugging FileStorage problems.""" 68 | 69 | # TODO: Should revise this class to use FileStorageFormatter. 70 | 71 | def __init__(self, path, dest=None): 72 | self.file = open(path, "rb") 73 | self.dest = dest 74 | 75 | def dump(self): 76 | fid = self.file.read(4) 77 | print("*" * 60, file=self.dest) 78 | print("file identifier: %r" % fid, file=self.dest) 79 | while self.dump_txn(): 80 | pass 81 | 82 | def dump_txn(self): 83 | pos = self.file.tell() 84 | h = self.file.read(TRANS_HDR_LEN) 85 | if not h: 86 | return False 87 | tid, tlen, status, ul, dl, el = struct.unpack(TRANS_HDR, h) 88 | end = pos + tlen 89 | print("=" * 60, file=self.dest) 90 | print("offset: %d" % pos, file=self.dest) 91 | print("end pos: %d" % end, file=self.dest) 92 | print("transaction id: %s" % fmt(tid), file=self.dest) 93 | print("trec len: %d" % tlen, file=self.dest) 94 | print("status: %r" % status, file=self.dest) 95 | user = descr = "" 96 | if ul: 97 | user = self.file.read(ul) 98 | if dl: 99 | descr = self.file.read(dl) 100 | if el: 101 | self.file.read(el) 102 | print("user: %r" % user, file=self.dest) 103 | print("description: %r" % descr, file=self.dest) 104 | print("len(extra): %d" % el, file=self.dest) 105 | while self.file.tell() < end: 106 | self.dump_data(pos) 107 | stlen = self.file.read(8) 108 | print("redundant trec len: %d" % u64(stlen), file=self.dest) 109 | return 1 110 | 111 | def dump_data(self, tloc): 112 | pos = self.file.tell() 113 | h = self.file.read(DATA_HDR_LEN) 114 | assert len(h) == DATA_HDR_LEN 115 | oid, revid, prev, tloc, vlen, dlen = struct.unpack(DATA_HDR, h) 116 | print("-" * 60, file=self.dest) 117 | print("offset: %d" % pos, file=self.dest) 118 | print("oid: %s" % fmt(oid), file=self.dest) 119 | print("revid: %s" % fmt(revid), file=self.dest) 120 | print("previous record offset: %d" % prev, file=self.dest) 121 | print("transaction offset: %d" % tloc, file=self.dest) 122 | assert not vlen 123 | print("len(data): %d" % dlen, file=self.dest) 124 | self.file.read(dlen) 125 | if not dlen: 126 | sbp = self.file.read(8) 127 | print("backpointer: %d" % u64(sbp), file=self.dest) 128 | 129 | 130 | def main(): 131 | import sys 132 | fsdump(sys.argv[1]) 133 | 134 | 135 | if __name__ == "__main__": 136 | main() 137 | -------------------------------------------------------------------------------- /src/ZODB/FileStorage/interfaces.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) Zope Corporation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | import zope.interface 15 | 16 | 17 | class IFileStoragePacker(zope.interface.Interface): 18 | 19 | def __call__(storage, referencesf, stop, gc): 20 | r"""Pack the file storage into a new file 21 | 22 | :param FileStorage storage: The storage object to be packed 23 | :param callable referencesf: A function that extracts object 24 | references from a pickle bytes string. This is usually 25 | ``ZODB.serialize.referencesf``. 26 | :param bytes stop: A transaction id representing the time at 27 | which to stop packing. 28 | :param bool gc: A flag indicating whether garbage collection 29 | should be performed. 30 | 31 | The new file will have the same name as the old file with 32 | ``.pack`` appended. (The packer can get the old file name via 33 | storage._file.name.) If blobs are supported, if the storages 34 | blob_dir attribute is not None or empty, then a .removed file 35 | must be created in the blob directory. This file contains records of 36 | the form:: 37 | 38 | (oid+serial).encode('hex')+'\n' 39 | 40 | or, of the form:: 41 | 42 | oid.encode('hex')+'\n' 43 | 44 | If packing is unnecessary, or would not change the file, then 45 | no pack or removed files are created None is returned, 46 | otherwise a tuple is returned with: 47 | 48 | - the size of the packed file, and 49 | 50 | - the packed index 51 | 52 | If and only if packing was necessary (non-None) and there was 53 | no error, then the commit lock must be acquired. In addition, 54 | it is up to FileStorage to: 55 | 56 | - Rename the .pack file, and 57 | 58 | - process the blob_dir/.removed file by removing the blobs 59 | corresponding to the file records. 60 | """ 61 | 62 | 63 | class IFileStorage(zope.interface.Interface): 64 | 65 | packer = zope.interface.Attribute( 66 | "The IFileStoragePacker to be used for packing." 67 | ) 68 | 69 | _file = zope.interface.Attribute( 70 | "The file object used to access the underlying data." 71 | ) 72 | 73 | _lock = zope.interface.Attribute( 74 | "The storage lock." 75 | ) 76 | 77 | _commit_lock = zope.interface.Attribute( 78 | "The storage commit lock." 79 | ) 80 | -------------------------------------------------------------------------------- /src/ZODB/UndoLogCompatible.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """Provide backward compatibility with storages that only have undoLog().""" 15 | 16 | 17 | class UndoLogCompatible: 18 | 19 | def undoInfo(self, first=0, last=-20, specification=None): 20 | if specification: 21 | # filter(desc) returns true iff `desc` is a "superdict" 22 | # of `specification`, meaning that `desc` contains the same 23 | # (key, value) pairs as `specification`, and possibly additional 24 | # (key, value) pairs. Another way to do this might be 25 | # d = desc.copy() 26 | # d.update(specification) 27 | # return d == desc 28 | def filter(desc, spec=specification.items()): 29 | get = desc.get 30 | for k, v in spec: 31 | if get(k, None) != v: 32 | return 0 33 | return 1 34 | else: 35 | filter = None 36 | 37 | return self.undoLog(first, last, filter) 38 | -------------------------------------------------------------------------------- /src/ZODB/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | 15 | import sys 16 | 17 | from persistent import TimeStamp 18 | from persistent import list 19 | from persistent import mapping 20 | 21 | from ZODB.DB import DB 22 | from ZODB.DB import connection 23 | 24 | 25 | # Backward compat for old imports. 26 | sys.modules['ZODB.TimeStamp'] = sys.modules['persistent.TimeStamp'] 27 | sys.modules['ZODB.PersistentMapping'] = sys.modules['persistent.mapping'] 28 | sys.modules['ZODB.PersistentList'] = sys.modules['persistent.list'] 29 | 30 | del mapping, list, sys 31 | -------------------------------------------------------------------------------- /src/ZODB/_compat.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2013 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | 15 | # We can't use stdlib's pickle because of http://bugs.python.org/issue6784 16 | import zodbpickle.pickle 17 | 18 | 19 | _protocol = 3 20 | FILESTORAGE_MAGIC = b"FS30" 21 | HIGHEST_PROTOCOL = 3 22 | 23 | 24 | class Pickler(zodbpickle.pickle.Pickler): 25 | def __init__(self, f, protocol=None): 26 | super().__init__(f, protocol) 27 | 28 | 29 | class Unpickler(zodbpickle.pickle.Unpickler): 30 | def __init__(self, f): 31 | super().__init__(f) 32 | 33 | # Python doesn't allow assignments to find_global, 34 | # instead, find_class can be overridden 35 | find_global = None 36 | 37 | def find_class(self, modulename, name): 38 | if self.find_global is None: 39 | return super().find_class(modulename, name) 40 | return self.find_global(modulename, name) 41 | 42 | 43 | def dump(o, f, protocol=None): 44 | return zodbpickle.pickle.dump(o, f, protocol) 45 | 46 | 47 | def dumps(o, protocol=None): 48 | return zodbpickle.pickle.dumps(o, protocol) 49 | 50 | 51 | def loads(s): 52 | return zodbpickle.pickle.loads(s, encoding='ASCII', errors='bytes') 53 | 54 | 55 | def PersistentPickler(persistent_id, *args, **kwargs): 56 | """ 57 | Returns a :class:`Pickler` that will use the given ``persistent_id`` 58 | to get persistent IDs. The remainder of the arguments are passed to the 59 | Pickler itself. 60 | 61 | This covers the differences between CPython and PyPy/zodbpickle. 62 | """ 63 | p = Pickler(*args, **kwargs) 64 | 65 | # PyPy uses a python implementation of cPickle/zodbpickle. 66 | # We can't really detect `inst_persistent_id` as it is 67 | # a magic attribute that is not readable, but it doesn't hurt to 68 | # simply always assign to persistent_id also 69 | p.persistent_id = persistent_id 70 | return p 71 | 72 | 73 | def PersistentUnpickler(find_global, load_persistent, *args, **kwargs): 74 | """ 75 | Returns a :class:`Unpickler` that will use the given `find_global` function 76 | to locate classes, and the given `load_persistent` function to load 77 | objects from a persistent id. 78 | 79 | This covers the differences between CPython and PyPy/zodbpickle. 80 | """ 81 | unpickler = Unpickler(*args, **kwargs) 82 | if find_global is not None: 83 | unpickler.find_global = find_global 84 | try: 85 | # PyPy, zodbpickle, the non-c-accelerated version 86 | unpickler.find_class = find_global 87 | except AttributeError: 88 | pass 89 | if load_persistent is not None: 90 | unpickler.persistent_load = load_persistent 91 | 92 | return unpickler 93 | 94 | 95 | def ascii_bytes(x): 96 | if isinstance(x, str): 97 | x = x.encode('ascii') 98 | return x 99 | -------------------------------------------------------------------------------- /src/ZODB/config.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /src/ZODB/conversionhack.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | 15 | import persistent.mapping 16 | 17 | 18 | class fixer: 19 | def __of__(self, parent): 20 | def __setstate__(state, self=parent): 21 | self._container = state 22 | del self.__setstate__ 23 | return __setstate__ 24 | 25 | 26 | fixer = fixer() 27 | 28 | 29 | class hack: 30 | pass 31 | 32 | 33 | hack = hack() 34 | 35 | 36 | def __basicnew__(): 37 | r = persistent.mapping.PersistentMapping() 38 | r.__setstate__ = fixer 39 | return r 40 | 41 | 42 | hack.__basicnew__ = __basicnew__ 43 | -------------------------------------------------------------------------------- /src/ZODB/event.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | try: 15 | from zope.event import notify 16 | except ImportError: 17 | def notify(event): 18 | return None 19 | -------------------------------------------------------------------------------- /src/ZODB/loglevels.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """Supplies custom logging levels BLATHER and TRACE. 15 | 16 | $Revision: 1.1 $ 17 | """ 18 | 19 | import logging 20 | 21 | 22 | __all__ = ["BLATHER", "TRACE"] 23 | 24 | # In the days of zLOG, there were 7 standard log levels, and ZODB/ZEO used 25 | # all of them. Here's how they map to the logging package's 5 standard 26 | # levels: 27 | # 28 | # zLOG logging 29 | # ------------- --------------- 30 | # PANIC (300) FATAL, CRITICAL (50) 31 | # ERROR (200) ERROR (40) 32 | # WARNING, PROBLEM (100) WARN (30) 33 | # INFO (0) INFO (20) 34 | # BLATHER (-100) none -- defined here as BLATHER (15) 35 | # DEBUG (-200) DEBUG (10) 36 | # TRACE (-300) none -- defined here as TRACE (5) 37 | # 38 | # TRACE is used by ZEO for extremely verbose trace output, enabled only 39 | # when chasing bottom-level communications bugs. It really should be at 40 | # a lower level than DEBUG. 41 | # 42 | # BLATHER is a harder call, and various instances could probably be folded 43 | # into INFO or DEBUG without real harm. 44 | 45 | BLATHER = 15 46 | TRACE = 5 47 | logging.addLevelName(BLATHER, "BLATHER") 48 | logging.addLevelName(TRACE, "TRACE") 49 | -------------------------------------------------------------------------------- /src/ZODB/scripts/README.txt: -------------------------------------------------------------------------------- 1 | This directory contains a collection of utilities for managing ZODB 2 | databases. Some are more useful than others. If you install ZODB 3 | using distutils ("python setup.py install"), a few of these will be installed. 4 | 5 | Unless otherwise noted, these scripts are invoked with the name of the 6 | Data.fs file as their only argument. Example: checkbtrees.py data.fs. 7 | 8 | 9 | analyze.py -- a transaction analyzer for FileStorage 10 | 11 | Reports on the data in a FileStorage. The report is organized by 12 | class. It shows total data, as well as separate reports for current 13 | and historical revisions of objects. 14 | 15 | 16 | checkbtrees.py -- checks BTrees in a FileStorage for corruption 17 | 18 | Attempts to find all the BTrees contained in a Data.fs, calls their 19 | _check() methods, and runs them through BTrees.check.check(). 20 | 21 | 22 | fsdump.py -- summarize FileStorage contents, one line per revision 23 | 24 | Prints a report of FileStorage contents, with one line for each 25 | transaction and one line for each data record in that transaction. 26 | Includes time stamps, file positions, and class names. 27 | 28 | 29 | fsoids.py -- trace all uses of specified oids in a FileStorage 30 | 31 | For heavy debugging. 32 | A set of oids is specified by text file listing and/or command line. 33 | A report is generated showing all uses of these oids in the database: 34 | all new-revision creation/modifications, all references from all 35 | revisions of other objects, and all creation undos. 36 | 37 | 38 | fstest.py -- simple consistency checker for FileStorage 39 | 40 | usage: fstest.py [-v] data.fs 41 | 42 | The fstest tool will scan all the data in a FileStorage and report an 43 | error if it finds any corrupt transaction data. The tool will print a 44 | message when the first error is detected an exit. 45 | 46 | The tool accepts one or more -v arguments. If a single -v is used, it 47 | will print a line of text for each transaction record it encounters. 48 | If two -v arguments are used, it will also print a line of text for 49 | each object. The objects for a transaction will be printed before the 50 | transaction itself. 51 | 52 | Note: It does not check the consistency of the object pickles. It is 53 | possible for the damage to occur only in the part of the file that 54 | stores object pickles. Those errors will go undetected. 55 | 56 | 57 | space.py -- report space used by objects in a FileStorage 58 | 59 | usage: space.py [-v] data.fs 60 | 61 | This ignores revisions and versions. 62 | 63 | 64 | netspace.py -- hackish attempt to report on size of objects 65 | 66 | usage: netspace.py [-P | -v] data.fs 67 | 68 | -P: do a pack first 69 | -v: print info for all objects, even if a traversal path isn't found 70 | 71 | Traverses objects from the database root and attempts to calculate 72 | size of object, including all reachable subobjects. 73 | 74 | 75 | repozo.py -- incremental backup utility for FileStorage 76 | 77 | Run the script with the -h option to see usage details. 78 | 79 | 80 | timeout.py -- script to test transaction timeout 81 | 82 | usage: timeout.py address delay [storage-name] 83 | 84 | This script connects to a storage, begins a transaction, calls store() 85 | and tpc_vote(), and then sleeps forever. This should trigger the 86 | transaction timeout feature of the server. 87 | 88 | zodbload.py -- exercise ZODB under a heavy synthesized Zope-like load 89 | 90 | See the module docstring for details. Note that this script requires 91 | Zope. New in ZODB3 3.1.4. 92 | 93 | 94 | fsrefs.py -- check FileStorage for dangling references 95 | 96 | 97 | fstail.py -- display the most recent transactions in a FileStorage 98 | 99 | usage: fstail.py [-n nxtn] data.fs 100 | 101 | The most recent ntxn transactions are displayed, to stdout. 102 | Optional argument -n specifies ntxn, and defaults to 10. 103 | 104 | 105 | migrate.py -- do a storage migration and gather statistics 106 | 107 | See the module docstring for details. 108 | -------------------------------------------------------------------------------- /src/ZODB/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | -------------------------------------------------------------------------------- /src/ZODB/scripts/analyze.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Based on a transaction analyzer by Matt Kromer. 4 | 5 | import sys 6 | from io import BytesIO 7 | 8 | from ZODB._compat import PersistentUnpickler 9 | from ZODB.FileStorage import FileStorage 10 | 11 | 12 | class FakeError(Exception): 13 | def __init__(self, module, name): 14 | Exception.__init__(self) 15 | self.module = module 16 | self.name = name 17 | 18 | 19 | def fake_find_class(module, name): 20 | raise FakeError(module, name) 21 | 22 | 23 | def FakeUnpickler(f): 24 | unpickler = PersistentUnpickler(fake_find_class, None, f) 25 | return unpickler 26 | 27 | 28 | class Report: 29 | def __init__(self): 30 | self.OIDMAP = {} 31 | self.TYPEMAP = {} 32 | self.TYPESIZE = {} 33 | self.FREEMAP = {} 34 | self.USEDMAP = {} 35 | self.TIDS = 0 36 | self.OIDS = 0 37 | self.DBYTES = 0 38 | self.COIDS = 0 39 | self.CBYTES = 0 40 | self.FOIDS = 0 41 | self.FBYTES = 0 42 | 43 | 44 | def shorten(s, n): 45 | length = len(s) 46 | if length <= n: 47 | return s 48 | while len(s) + 3 > n: # account for ... 49 | i = s.find(".") 50 | if i == -1: 51 | # In the worst case, just return the rightmost n bytes 52 | return s[-n:] 53 | else: 54 | s = s[i + 1:] 55 | length = len(s) 56 | return "..." + s 57 | 58 | 59 | def report(rep): 60 | print(f"Processed {rep.OIDS} records in {rep.TIDS} transactions") 61 | print(f"Average record size is {rep.DBYTES * 1.0 / rep.OIDS:7.2f} bytes") 62 | print("Average transaction size is" 63 | f" {rep.DBYTES * 1.0 / rep.TIDS:7.2f} bytes") 64 | 65 | print("Types used:") 66 | fmt = "%-46s %7s %9s %6s %7s" 67 | fmtp = "%-46s %7d %9d %5.1f%% %7.2f" # per-class format 68 | fmts = "%46s %7d %8dk %5.1f%% %7.2f" # summary format 69 | print(fmt % ("Class Name", "Count", "TBytes", "Pct", "AvgSize")) 70 | print(fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7)) 71 | typemap = sorted(rep.TYPEMAP) 72 | cumpct = 0.0 73 | for t in typemap: 74 | pct = rep.TYPESIZE[t] * 100.0 / rep.DBYTES 75 | cumpct += pct 76 | print(fmtp % (shorten(t, 46), rep.TYPEMAP[t], rep.TYPESIZE[t], 77 | pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t])) 78 | 79 | print(fmt % ('='*46, '='*7, '='*9, '='*5, '='*7)) 80 | print("%46s %7d %9s %6s %6.2fk" % ( 81 | 'Total Transactions', rep.TIDS, ' ', ' ', 82 | rep.DBYTES * 1.0 / rep.TIDS / 1024.0)) 83 | print(fmts % ('Total Records', rep.OIDS, rep.DBYTES / 1024.0, cumpct, 84 | rep.DBYTES * 1.0 / rep.OIDS)) 85 | 86 | print(fmts % ('Current Objects', rep.COIDS, rep.CBYTES / 1024.0, 87 | rep.CBYTES * 100.0 / rep.DBYTES, 88 | rep.CBYTES * 1.0 / rep.COIDS)) 89 | if rep.FOIDS: 90 | print(fmts % ('Old Objects', rep.FOIDS, rep.FBYTES / 1024.0, 91 | rep.FBYTES * 100.0 / rep.DBYTES, 92 | rep.FBYTES * 1.0 / rep.FOIDS)) 93 | 94 | 95 | def analyze(path): 96 | fs = FileStorage(path, read_only=1) 97 | fsi = fs.iterator() 98 | report = Report() 99 | for txn in fsi: 100 | analyze_trans(report, txn) 101 | return report 102 | 103 | 104 | def analyze_trans(report, txn): 105 | report.TIDS += 1 106 | for rec in txn: 107 | analyze_rec(report, rec) 108 | 109 | 110 | def get_type(record): 111 | try: 112 | unpickled = FakeUnpickler(BytesIO(record.data)).load() 113 | except FakeError as err: 114 | return "{}.{}".format(err.module, err.name) 115 | classinfo = unpickled[0] 116 | if isinstance(classinfo, tuple): 117 | mod, klass = classinfo 118 | return "{}.{}".format(mod, klass) 119 | else: 120 | return str(classinfo) 121 | 122 | 123 | def analyze_rec(report, record): 124 | oid = record.oid 125 | report.OIDS += 1 126 | if record.data is None: 127 | # No pickle -- aborted version or undo of object creation. 128 | return 129 | try: 130 | size = len(record.data) # Ignores various overhead 131 | report.DBYTES += size 132 | if oid not in report.OIDMAP: 133 | type = get_type(record) 134 | report.OIDMAP[oid] = type 135 | report.USEDMAP[oid] = size 136 | report.COIDS += 1 137 | report.CBYTES += size 138 | else: 139 | type = report.OIDMAP[oid] 140 | fsize = report.USEDMAP[oid] 141 | report.FREEMAP[oid] = report.FREEMAP.get(oid, 0) + fsize 142 | report.USEDMAP[oid] = size 143 | report.FOIDS += 1 144 | report.FBYTES += fsize 145 | report.CBYTES += size - fsize 146 | report.TYPEMAP[type] = report.TYPEMAP.get(type, 0) + 1 147 | report.TYPESIZE[type] = report.TYPESIZE.get(type, 0) + size 148 | except Exception as err: 149 | print(err) 150 | 151 | 152 | if __name__ == "__main__": 153 | path = sys.argv[1] 154 | report(analyze(path)) 155 | -------------------------------------------------------------------------------- /src/ZODB/scripts/checkbtrees.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Check the consistency of BTrees in a Data.fs 3 | 4 | usage: checkbtrees.py data.fs 5 | 6 | Try to find all the BTrees in a Data.fs, call their _check() methods, 7 | and run them through BTrees.check.check(). 8 | """ 9 | 10 | from BTrees.check import check 11 | 12 | import ZODB 13 | from ZODB.FileStorage import FileStorage 14 | 15 | 16 | # Set of oids we've already visited. Since the object structure is 17 | # a general graph, this is needed to prevent unbounded paths in the 18 | # presence of cycles. It's also helpful in eliminating redundant 19 | # checking when a BTree is pointed to by many objects. 20 | oids_seen = {} 21 | 22 | # Append (obj, path) to L if and only if obj is a persistent object 23 | # and we haven't seen it before. 24 | 25 | 26 | def add_if_new_persistent(L, obj, path): 27 | getattr(obj, '_', None) # unghostify 28 | if hasattr(obj, '_p_oid'): 29 | oid = obj._p_oid 30 | if oid not in oids_seen: 31 | L.append((obj, path)) 32 | oids_seen[oid] = 1 33 | 34 | 35 | def get_subobjects(obj): 36 | getattr(obj, '_', None) # unghostify 37 | sub = [] 38 | try: 39 | attrs = obj.__dict__.items() 40 | except AttributeError: 41 | attrs = () 42 | for pair in attrs: 43 | sub.append(pair) 44 | 45 | # what if it is a mapping? 46 | try: 47 | items = obj.items() 48 | except AttributeError: 49 | items = () 50 | for k, v in items: 51 | if not isinstance(k, int): 52 | sub.append(("", k)) 53 | if not isinstance(v, int): 54 | sub.append(("[%s]" % repr(k), v)) 55 | 56 | # what if it is a sequence? 57 | i = 0 58 | while 1: 59 | try: 60 | elt = obj[i] 61 | except: # noqa: E722 do not use bare 'except' 62 | break 63 | sub.append(("[%d]" % i, elt)) 64 | i += 1 65 | 66 | return sub 67 | 68 | 69 | def main(fname=None): 70 | if fname is None: 71 | import sys 72 | try: 73 | fname, = sys.argv[1:] 74 | except: # noqa: E722 do not use bare 'except' 75 | print(__doc__) 76 | sys.exit(2) 77 | 78 | fs = FileStorage(fname, read_only=1) 79 | cn = ZODB.DB(fs).open() 80 | rt = cn.root() 81 | todo = [] 82 | add_if_new_persistent(todo, rt, '') 83 | 84 | found = 0 85 | while todo: 86 | obj, path = todo.pop(0) 87 | found += 1 88 | if not path: 89 | print("", repr(obj)) 90 | else: 91 | print(path, repr(obj)) 92 | 93 | mod = str(obj.__class__.__module__) 94 | if mod.startswith("BTrees"): 95 | if hasattr(obj, "_check"): 96 | try: 97 | obj._check() 98 | except AssertionError as msg: 99 | print("*" * 60) 100 | print(msg) 101 | print("*" * 60) 102 | 103 | try: 104 | check(obj) 105 | except AssertionError as msg: 106 | print("*" * 60) 107 | print(msg) 108 | print("*" * 60) 109 | 110 | if found % 100 == 0: 111 | cn.cacheMinimize() 112 | 113 | for k, v in get_subobjects(obj): 114 | if k.startswith('['): 115 | # getitem 116 | newpath = "{}{}".format(path, k) 117 | else: 118 | newpath = "{}.{}".format(path, k) 119 | add_if_new_persistent(todo, v, newpath) 120 | 121 | print("total", len(fs._index), "found", found) 122 | 123 | 124 | if __name__ == "__main__": 125 | main() 126 | -------------------------------------------------------------------------------- /src/ZODB/scripts/fsoids.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | ############################################################################## 4 | # 5 | # Copyright (c) 2004 Zope Foundation and Contributors. 6 | # All Rights Reserved. 7 | # 8 | # This software is subject to the provisions of the Zope Public License, 9 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 10 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 11 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 12 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 13 | # FOR A PARTICULAR PURPOSE 14 | # 15 | ############################################################################## 16 | 17 | """FileStorage oid-tracer. 18 | 19 | usage: fsoids.py [-f oid_file] Data.fs [oid]... 20 | 21 | Display information about all occurrences of specified oids in a FileStorage. 22 | This is meant for heavy debugging. 23 | 24 | This includes all revisions of the oids, all objects referenced by the 25 | oids, and all revisions of all objects referring to the oids. 26 | 27 | If specified, oid_file is an input text file, containing one oid per 28 | line. oids are specified as integers, in any of Python's integer 29 | notations (typically like 0x341a). One or more oids can also be specified 30 | on the command line. 31 | 32 | The output is grouped by oid, from smallest to largest, and sub-grouped 33 | by transaction, from oldest to newest. 34 | 35 | This will not alter the FileStorage, but running against a live FileStorage 36 | is not recommended (spurious error messages may result). 37 | 38 | See testfsoids.py for a tutorial doctest. 39 | """ 40 | 41 | import sys 42 | 43 | from ZODB.FileStorage.fsoids import Tracer 44 | 45 | 46 | def usage(): 47 | print(__doc__) 48 | 49 | 50 | def main(): 51 | import getopt 52 | 53 | try: 54 | opts, args = getopt.getopt(sys.argv[1:], 'f:') 55 | if not args: 56 | usage() 57 | raise ValueError("Must specify a FileStorage") 58 | path = None 59 | for k, v in opts: 60 | if k == '-f': 61 | path = v 62 | except (getopt.error, ValueError): 63 | usage() 64 | raise 65 | 66 | c = Tracer(args[0]) 67 | for oid in args[1:]: 68 | as_int = int(oid, 0) # 0 == auto-detect base 69 | c.register_oids(as_int) 70 | if path is not None: 71 | for line in open(path): 72 | as_int = int(line, 0) 73 | c.register_oids(as_int) 74 | if not c.oids: 75 | raise ValueError("no oids specified") 76 | c.run() 77 | c.report() 78 | 79 | 80 | if __name__ == "__main__": 81 | main() 82 | -------------------------------------------------------------------------------- /src/ZODB/scripts/fstail.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ############################################################################## 3 | # 4 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 5 | # All Rights Reserved. 6 | # 7 | # This software is subject to the provisions of the Zope Public License, 8 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 9 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 10 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 11 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 12 | # FOR A PARTICULAR PURPOSE 13 | # 14 | ############################################################################## 15 | """Tool to dump the last few transactions from a FileStorage.""" 16 | 17 | import binascii 18 | import getopt 19 | import sys 20 | from hashlib import sha1 21 | 22 | from ZODB.fstools import prev_txn 23 | 24 | 25 | def main(path, ntxn): 26 | with open(path, "rb") as f: 27 | f.seek(0, 2) 28 | th = prev_txn(f) 29 | i = ntxn 30 | while th and i > 0: 31 | hash = sha1(th.get_raw_data()).digest() 32 | th.read_meta() 33 | print("{}: hash={}".format(th.get_timestamp(), 34 | binascii.hexlify(hash).decode())) 35 | print("user=%r description=%r length=%d offset=%d (+%d)" 36 | % (th.user, th.descr, th.length, th.get_offset(), len(th))) 37 | print() 38 | th = th.prev_txn() 39 | i -= 1 40 | 41 | 42 | def Main(): 43 | ntxn = 10 44 | opts, args = getopt.getopt(sys.argv[1:], "n:") 45 | path, = args 46 | for k, v in opts: 47 | if k == '-n': 48 | ntxn = int(v) 49 | main(path, ntxn) 50 | 51 | 52 | if __name__ == "__main__": 53 | Main() 54 | -------------------------------------------------------------------------------- /src/ZODB/scripts/manual_tests/test-checker.fs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zopefoundation/ZODB/f2dc04c998b4e50e5105bc4358140809b8d66b54/src/ZODB/scripts/manual_tests/test-checker.fs -------------------------------------------------------------------------------- /src/ZODB/scripts/migrateblobs.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2008 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """A script to migrate a blob directory into a different layout. 15 | """ 16 | 17 | import logging 18 | import optparse 19 | import os 20 | import shutil 21 | 22 | from ZODB.blob import FilesystemHelper 23 | from ZODB.utils import oid_repr 24 | 25 | 26 | # Check if we actually have link 27 | try: 28 | os.link 29 | except AttributeError: 30 | link_or_copy = shutil.copy 31 | else: 32 | def link_or_copy(f1, f2): 33 | try: 34 | os.link(f1, f2) 35 | except OSError: 36 | shutil.copy(f1, f2) 37 | 38 | 39 | def migrate(source, dest, layout): 40 | source_fsh = FilesystemHelper(source) 41 | source_fsh.create() 42 | dest_fsh = FilesystemHelper(dest, layout) 43 | dest_fsh.create() 44 | print("Migrating blob data from `{}` ({}) to `{}` ({})".format( 45 | source, source_fsh.layout_name, dest, dest_fsh.layout_name)) 46 | for oid, path in source_fsh.listOIDs(): 47 | dest_path = dest_fsh.getPathForOID(oid, create=True) 48 | files = os.listdir(path) 49 | for file in files: 50 | source_file = os.path.join(path, file) 51 | dest_file = os.path.join(dest_path, file) 52 | link_or_copy(source_file, dest_file) 53 | print("\tOID: {} - {} files ".format(oid_repr(oid), len(files))) 54 | 55 | 56 | def main(source=None, dest=None, layout="bushy"): 57 | usage = "usage: %prog [options] " 58 | description = ("Create the new directory and migrate all blob " 59 | "data to while using the new for " 60 | "") 61 | 62 | parser = optparse.OptionParser(usage=usage, description=description) 63 | parser.add_option("-l", "--layout", 64 | default=layout, type='choice', 65 | choices=['bushy', 'lawn'], 66 | help="Define the layout to use for the new directory " 67 | "(bushy or lawn). Default: %default") 68 | options, args = parser.parse_args() 69 | 70 | if not len(args) == 2: 71 | parser.error("source and destination must be given") 72 | 73 | logging.getLogger().addHandler(logging.StreamHandler()) 74 | logging.getLogger().setLevel(0) 75 | 76 | source, dest = args 77 | migrate(source, dest, options.layout) 78 | 79 | 80 | if __name__ == '__main__': 81 | main() 82 | -------------------------------------------------------------------------------- /src/ZODB/scripts/netspace.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Report on the net size of objects counting subobjects. 3 | 4 | usage: netspace.py [-P | -v] data.fs 5 | 6 | -P: do a pack first 7 | -v: print info for all objects, even if a traversal path isn't found 8 | """ 9 | 10 | 11 | import ZODB 12 | from ZODB.FileStorage import FileStorage 13 | from ZODB.serialize import referencesf 14 | from ZODB.utils import U64 15 | from ZODB.utils import get_pickle_metadata 16 | from ZODB.utils import load_current 17 | 18 | 19 | def find_paths(root, maxdist): 20 | """Find Python attribute traversal paths for objects to maxdist distance. 21 | 22 | Starting at a root object, traverse attributes up to distance levels 23 | from the root, looking for persistent objects. Return a dict 24 | mapping oids to traversal paths. 25 | 26 | TODO: Assumes that the keys of the root are not themselves 27 | persistent objects. 28 | 29 | TODO: Doesn't traverse containers. 30 | """ 31 | paths = {} 32 | 33 | # Handle the root as a special case because it's a dict 34 | objs = [] 35 | for k, v in root.items(): 36 | oid = getattr(v, '_p_oid', None) 37 | objs.append((k, v, oid, 0)) 38 | 39 | for path, obj, oid, dist in objs: 40 | if oid is not None: 41 | paths[oid] = path 42 | if dist < maxdist: 43 | getattr(obj, 'foo', None) # unghostify 44 | try: 45 | items = obj.__dict__.items() 46 | except AttributeError: 47 | continue 48 | for k, v in items: 49 | oid = getattr(v, '_p_oid', None) 50 | objs.append(("{}.{}".format(path, k), v, oid, dist + 1)) 51 | 52 | return paths 53 | 54 | 55 | def main(path): 56 | fs = FileStorage(path, read_only=1) 57 | if PACK: 58 | fs.pack() 59 | 60 | db = ZODB.DB(fs) 61 | rt = db.open().root() 62 | paths = find_paths(rt, 3) 63 | 64 | def total_size(oid): 65 | cache = {} 66 | cache_size = 1000 67 | 68 | def _total_size(oid, seen): 69 | v = cache.get(oid) 70 | if v is not None: 71 | return v 72 | data, serialno = load_current(fs, oid) 73 | size = len(data) 74 | for suboid in referencesf(data): 75 | if suboid in seen: 76 | continue 77 | seen[suboid] = 1 78 | size += _total_size(suboid, seen) 79 | cache[oid] = size 80 | if len(cache) == cache_size: 81 | cache.popitem() 82 | return size 83 | return _total_size(oid, {}) 84 | 85 | keys = fs._index.keys() 86 | keys.sort() 87 | keys.reverse() 88 | 89 | if not VERBOSE: 90 | # If not running verbosely, don't print an entry for an object 91 | # unless it has an entry in paths. 92 | keys = filter(paths.has_key, keys) 93 | 94 | fmt = "%8s %5d %8d %s %s.%s" 95 | 96 | for oid in keys: 97 | data, serialno = load_current(fs, oid) 98 | mod, klass = get_pickle_metadata(data) 99 | referencesf(data) 100 | path = paths.get(oid, '-') 101 | print(fmt % (U64(oid), len(data), total_size(oid), path, mod, klass)) 102 | 103 | 104 | def Main(): 105 | import getopt 106 | import sys 107 | 108 | global PACK 109 | global VERBOSE 110 | 111 | PACK = 0 112 | VERBOSE = 0 113 | try: 114 | opts, args = getopt.getopt(sys.argv[1:], 'Pv') 115 | path, = args 116 | except getopt.error as err: 117 | print(err) 118 | print(__doc__) 119 | sys.exit(2) 120 | except ValueError: 121 | print("expected one argument, got", len(args)) 122 | print(__doc__) 123 | sys.exit(2) 124 | for o, v in opts: 125 | if o == '-P': 126 | PACK = 1 127 | if o == '-v': 128 | VERBOSE += 1 129 | main(path) 130 | 131 | 132 | if __name__ == "__main__": 133 | Main() 134 | -------------------------------------------------------------------------------- /src/ZODB/scripts/referrers.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2005 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Compute a table of object id referrers 15 | 16 | $Id$ 17 | """ 18 | 19 | from ZODB.serialize import referencesf 20 | 21 | 22 | def referrers(storage): 23 | result = {} 24 | for transaction in storage.iterator(): 25 | for record in transaction: 26 | for oid in referencesf(record.data): 27 | result.setdefault(oid, []).append((record.oid, record.tid)) 28 | return result 29 | -------------------------------------------------------------------------------- /src/ZODB/scripts/space.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Report on the space used by objects in a storage. 3 | 4 | usage: space.py data.fs 5 | 6 | The current implementation only supports FileStorage. 7 | 8 | Current limitations / simplifications: Ignores revisions and versions. 9 | """ 10 | 11 | from operator import itemgetter 12 | 13 | from ZODB.FileStorage import FileStorage 14 | from ZODB.utils import U64 15 | from ZODB.utils import get_pickle_metadata 16 | from ZODB.utils import load_current 17 | 18 | 19 | def run(path, v=0): 20 | fs = FileStorage(path, read_only=1) 21 | # break into the file implementation 22 | if hasattr(fs._index, 'iterkeys'): 23 | iter = fs._index.keys() 24 | else: 25 | iter = fs._index.keys() 26 | totals = {} 27 | for oid in iter: 28 | data, serialno = load_current(fs, oid) 29 | mod, klass = get_pickle_metadata(data) 30 | key = "{}.{}".format(mod, klass) 31 | bytes, count = totals.get(key, (0, 0)) 32 | bytes += len(data) 33 | count += 1 34 | totals[key] = bytes, count 35 | if v: 36 | print("%8s %5d %s" % (U64(oid), len(data), key)) 37 | L = sorted(totals.items(), key=itemgetter(1), reverse=True) 38 | print("Totals per object class:") 39 | for key, (bytes, count) in L: 40 | print("%8d %8d %s" % (count, bytes, key)) 41 | 42 | 43 | def main(): 44 | import getopt 45 | import sys 46 | try: 47 | opts, args = getopt.getopt(sys.argv[1:], "v") 48 | except getopt.error as msg: 49 | print(msg) 50 | print("usage: space.py [-v] Data.fs") 51 | sys.exit(2) 52 | if len(args) != 1: 53 | print("usage: space.py [-v] Data.fs") 54 | sys.exit(2) 55 | v = 0 56 | for o, a in opts: 57 | if o == "-v": 58 | v += 1 59 | path = args[0] 60 | run(path, v) 61 | 62 | 63 | if __name__ == "__main__": 64 | main() 65 | -------------------------------------------------------------------------------- /src/ZODB/scripts/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zopefoundation/ZODB/f2dc04c998b4e50e5105bc4358140809b8d66b54/src/ZODB/scripts/tests/__init__.py -------------------------------------------------------------------------------- /src/ZODB/scripts/tests/fstail.txt: -------------------------------------------------------------------------------- 1 | ==================== 2 | The `fstail` utility 3 | ==================== 4 | 5 | The `fstail` utility shows information for a FileStorage about the last `n` 6 | transactions: 7 | 8 | We have to prepare a FileStorage first: 9 | 10 | >>> from ZODB.FileStorage import FileStorage 11 | >>> from ZODB.DB import DB 12 | >>> import transaction 13 | >>> from tempfile import mktemp 14 | >>> storagefile = mktemp(suffix='.fs') 15 | >>> base_storage = FileStorage(storagefile) 16 | >>> database = DB(base_storage) 17 | >>> connection1 = database.open() 18 | >>> root = connection1.root() 19 | >>> root['foo'] = 1 20 | >>> transaction.commit() 21 | 22 | Now lets have a look at the last transactions of this FileStorage: 23 | 24 | >>> from ZODB.scripts.fstail import main 25 | >>> main(storagefile, 5) 26 | 2007-11-10 15:18:48.543001: hash=b16422d09fabdb45d4e4325e4b42d7d6f021d3c3 27 | user='' description='' length=132 offset=162 (+23) 28 | 29 | 2007-11-10 15:18:48.543001: hash=b16422d09fabdb45d4e4325e4b42d7d6f021d3c3 30 | user='' description='initial database creation' length=150 offset=4 (+48) 31 | 32 | 33 | Now clean up the storage again: 34 | 35 | >>> import os 36 | >>> connection1.close() 37 | >>> base_storage.close() 38 | >>> os.unlink(storagefile) 39 | >>> os.unlink(storagefile+'.index') 40 | >>> os.unlink(storagefile+'.lock') 41 | >>> os.unlink(storagefile+'.tmp') 42 | -------------------------------------------------------------------------------- /src/ZODB/scripts/tests/referrers.txt: -------------------------------------------------------------------------------- 1 | Getting Object Referrers 2 | ======================== 3 | 4 | The referrers module provides a way to get object referrers. It 5 | provides a referrers method that takes an iterable storage object. It 6 | returns a dictionary mapping object ids to lists of referrer object 7 | versions, which each version is a tuple an object id nd serial 8 | nummber. 9 | 10 | To see how this works, we'll create a small database: 11 | 12 | >>> import transaction 13 | >>> from persistent.mapping import PersistentMapping 14 | >>> from ZODB.FileStorage import FileStorage 15 | >>> from ZODB.DB import DB 16 | >>> import os, tempfile 17 | >>> dest = tempfile.mkdtemp() 18 | >>> fs = FileStorage(os.path.join(dest, 'Data.fs')) 19 | >>> db = DB(fs) 20 | >>> conn = db.open() 21 | >>> conn.root()['a'] = PersistentMapping() 22 | >>> conn.root()['b'] = PersistentMapping() 23 | >>> transaction.commit() 24 | >>> roid = conn.root()._p_oid 25 | >>> aoid = conn.root()['a']._p_oid 26 | >>> boid = conn.root()['b']._p_oid 27 | >>> s1 = conn.root()['b']._p_serial 28 | 29 | >>> conn.root()['a']['b'] = conn.root()['b'] 30 | >>> transaction.commit() 31 | >>> s2 = conn.root()['a']._p_serial 32 | 33 | Now we'll get the storage and compute the referrers: 34 | 35 | >>> import ZODB.scripts.referrers 36 | >>> referrers = ZODB.scripts.referrers.referrers(fs) 37 | 38 | >>> referrers[boid] == [(roid, s1), (aoid, s2)] 39 | True 40 | 41 | .. Cleanup 42 | 43 | >>> db.close() 44 | -------------------------------------------------------------------------------- /src/ZODB/scripts/tests/test_doc.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | import doctest 15 | import re 16 | import unittest 17 | 18 | import zope.testing.renormalizing 19 | 20 | import ZODB.tests.util 21 | 22 | 23 | checker = zope.testing.renormalizing.RENormalizing([ 24 | (re.compile( 25 | r'[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+'), 26 | '2007-11-10 15:18:48.543001'), 27 | (re.compile('hash=[0-9a-f]{40}'), 28 | 'hash=b16422d09fabdb45d4e4325e4b42d7d6f021d3c3'), 29 | # Python 3 bytes add a "b". 30 | (re.compile("b('.*?')"), r"\1"), 31 | (re.compile('b(".*?")'), r"\1"), 32 | # Python 3 produces larger pickles, even when we use zodbpickle :( 33 | # this changes all the offsets and sizes in fstail.txt 34 | (re.compile("user='' description='' " 35 | r"length=[0-9]+ offset=[0-9]+ \(\+23\)"), 36 | "user='' description='' " 37 | "length= offset= (+23)"), 38 | (re.compile("user='' description='initial database creation' " 39 | r"length=[0-9]+ offset=4 \(\+48\)"), 40 | "user='' description='initial database creation' " 41 | "length= offset=4 (+48)"), 42 | ]) 43 | 44 | 45 | def test_suite(): 46 | return unittest.TestSuite(( 47 | doctest.DocFileSuite( 48 | 'referrers.txt', 49 | 'fstail.txt', 50 | setUp=ZODB.tests.util.setUp, tearDown=ZODB.tests.util.tearDown, 51 | checker=checker), 52 | )) 53 | -------------------------------------------------------------------------------- /src/ZODB/scripts/tests/test_fsdump_fsstats.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2021 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | 15 | from ZODB import DB 16 | from ZODB.scripts.fsstats import rx_data 17 | from ZODB.scripts.fsstats import rx_txn 18 | from ZODB.tests.util import TestCase 19 | from ZODB.tests.util import run_module_as_script 20 | 21 | 22 | class FsdumpFsstatsTests(TestCase): 23 | def setUp(self): 24 | super().setUp() 25 | # create (empty) storage ``data.fs`` 26 | DB("data.fs").close() 27 | 28 | def test_fsdump(self): 29 | run_module_as_script("ZODB.FileStorage.fsdump", ["data.fs"]) 30 | # verify that ``fsstats`` will understand the output 31 | with open("stdout") as f: 32 | tno = obno = 0 33 | for li in f: 34 | if li.startswith(" data"): 35 | m = rx_data.search(li) 36 | if m is None: 37 | continue 38 | oid, size, klass = m.groups() 39 | int(size) 40 | obno += 1 41 | elif li.startswith("Trans"): 42 | m = rx_txn.search(li) 43 | if not m: 44 | continue 45 | tid, size = m.groups() 46 | size = int(size) 47 | tno += 1 48 | self.assertEqual(tno, 1) 49 | self.assertEqual(obno, 1) 50 | 51 | def test_fsstats(self): 52 | # The ``fsstats`` output is complex 53 | # currently, we just check the first (summary) line 54 | run_module_as_script("ZODB.FileStorage.fsdump", ["data.fs"], 55 | "data.dmp") 56 | run_module_as_script("ZODB.scripts.fsstats", ["data.dmp"]) 57 | with open("stdout") as f: 58 | self.assertEqual(f.readline().strip(), 59 | "Summary: 1 txns, 1 objects, 1 revisions") 60 | -------------------------------------------------------------------------------- /src/ZODB/scripts/tests/test_fstest.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2010 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | import doctest 15 | import unittest 16 | 17 | from zope.testing import setupstack 18 | 19 | import ZODB 20 | 21 | 22 | def test_fstest_verbose(): 23 | r""" 24 | >>> db = ZODB.DB('data.fs') 25 | >>> db.close() 26 | >>> import ZODB.scripts.fstest 27 | >>> ZODB.scripts.fstest.main(['data.fs']) 28 | 29 | >>> ZODB.scripts.fstest.main(['data.fs']) 30 | 31 | >>> ZODB.scripts.fstest.main(['-v', 'data.fs']) 32 | ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE 33 | 4: transaction tid ... #0 34 | no errors detected 35 | 36 | >>> ZODB.scripts.fstest.main(['-vvv', 'data.fs']) 37 | ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE 38 | 52: object oid 0x0000000000000000 #0 39 | 4: transaction tid ... #0 40 | no errors detected 41 | 42 | """ 43 | 44 | 45 | def test_suite(): 46 | return unittest.TestSuite([ 47 | doctest.DocTestSuite('ZODB.scripts.fstest'), 48 | doctest.DocTestSuite(setUp=setupstack.setUpDirectory, 49 | tearDown=setupstack.tearDown), 50 | ]) 51 | -------------------------------------------------------------------------------- /src/ZODB/storage.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 |
4 | 5 | -------------------------------------------------------------------------------- /src/ZODB/tests/Corruption.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Do some minimal tests of data corruption""" 15 | 16 | import os 17 | import random 18 | import stat 19 | 20 | import ZODB.FileStorage 21 | from ZODB.utils import load_current 22 | 23 | from .StorageTestBase import StorageTestBase 24 | 25 | 26 | class FileStorageCorruptTests(StorageTestBase): 27 | 28 | def setUp(self): 29 | StorageTestBase.setUp(self) 30 | self._storage = ZODB.FileStorage.FileStorage('Data.fs', create=1) 31 | 32 | def _do_stores(self): 33 | oids = [] 34 | for i in range(5): 35 | oid = self._storage.new_oid() 36 | revid = self._dostore(oid) 37 | oids.append((oid, revid)) 38 | return oids 39 | 40 | def _check_stores(self, oids): 41 | for oid, revid in oids: 42 | data, s_revid = load_current(self._storage, oid) 43 | self.assertEqual(s_revid, revid) 44 | 45 | def testTruncatedIndex(self): 46 | oids = self._do_stores() 47 | self._close() 48 | 49 | # truncation the index file 50 | self.assertTrue(os.path.exists('Data.fs.index')) 51 | f = open('Data.fs.index', 'rb+') 52 | f.seek(0, 2) 53 | size = f.tell() 54 | f.seek(size // 2) 55 | f.truncate() 56 | f.close() 57 | 58 | self._storage = ZODB.FileStorage.FileStorage('Data.fs') 59 | self._check_stores(oids) 60 | 61 | def testCorruptedIndex(self): 62 | oids = self._do_stores() 63 | self._close() 64 | 65 | # truncation the index file 66 | self.assertTrue(os.path.exists('Data.fs.index')) 67 | size = os.stat('Data.fs.index')[stat.ST_SIZE] 68 | f = open('Data.fs.index', 'rb+') 69 | while f.tell() < size: 70 | f.seek(random.randrange(1, size // 10), 1) 71 | f.write(b'\000') 72 | f.close() 73 | 74 | self._storage = ZODB.FileStorage.FileStorage('Data.fs') 75 | self._check_stores(oids) 76 | -------------------------------------------------------------------------------- /src/ZODB/tests/HistoryStorage.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Run the history() related tests for a storage. 15 | 16 | Any storage that supports the history() method should be able to pass 17 | all these tests. 18 | """ 19 | 20 | import sys 21 | from time import sleep 22 | from time import time 23 | 24 | from ZODB.tests.MinPO import MinPO 25 | 26 | 27 | class HistoryStorage: 28 | def testSimpleHistory(self): 29 | self._checkHistory((11, 12, 13)) 30 | 31 | def _checkHistory(self, data): 32 | start = time() 33 | # Store a couple of revisions of the object 34 | oid = self._storage.new_oid() 35 | self.assertRaises(KeyError, self._storage.history, oid) 36 | revids = [None] 37 | for data in data: 38 | if sys.platform == 'win32': 39 | # time.time() has a precision of 1ms on Windows. 40 | sleep(0.002) 41 | revids.append(self._dostore(oid, revids[-1], MinPO(data))) 42 | revids.reverse() 43 | del revids[-1] 44 | # Now get various snapshots of the object's history 45 | for i in range(1, 1 + len(revids)): 46 | h = self._storage.history(oid, size=i) 47 | self.assertEqual([d['tid'] for d in h], revids[:i]) 48 | # Check results are sorted by timestamp, in descending order. 49 | if sys.platform == 'win32': 50 | # Same as above. This is also required in case this method is 51 | # called several times for the same storage. 52 | sleep(0.002) 53 | a = time() 54 | for d in h: 55 | b = a 56 | a = d['time'] 57 | self.assertLess(a, b) 58 | self.assertLess(start, a) 59 | -------------------------------------------------------------------------------- /src/ZODB/tests/IExternalGC.test: -------------------------------------------------------------------------------- 1 | Storage Support for external GC 2 | =============================== 3 | 4 | A storage that provides IExternalGC supports external garbage 5 | collectors by providing a deleteObject method that transactionally 6 | deletes an object. 7 | 8 | A create_storage function is provided that creates a storage. 9 | 10 | >>> storage = create_storage() 11 | >>> import ZODB.blob, transaction 12 | >>> db = ZODB.DB(storage) 13 | >>> conn = db.open() 14 | >>> conn.root()[0] = conn.root().__class__() 15 | >>> conn.root()[1] = ZODB.blob.Blob(b'some data') 16 | >>> transaction.commit() 17 | >>> oid0 = conn.root()[0]._p_oid 18 | >>> oid1 = conn.root()[1]._p_oid 19 | >>> del conn.root()[0] 20 | >>> del conn.root()[1] 21 | >>> transaction.commit() 22 | 23 | At this point, object 0 and 1 is garbage, but it's still in the storage: 24 | 25 | >>> p0, s0 = storage.load(oid0, '') 26 | >>> p1, s1 = storage.load(oid1, '') 27 | 28 | The storage is configured not to gc on pack, so even if we pack, these 29 | objects won't go away: 30 | 31 | >>> len(storage) 32 | 3 33 | >>> import time 34 | >>> db.pack(time.time()+1) 35 | >>> len(storage) 36 | 3 37 | >>> p0, s0 = storage.load(oid0, '') 38 | >>> p1, s1 = storage.load(oid1, '') 39 | 40 | Now we'll use the new deleteObject API to delete the objects. We can't 41 | go through the database to do this, so we'll have to manage the 42 | transaction ourselves. 43 | 44 | >>> from ZODB.Connection import TransactionMetaData 45 | >>> txn = TransactionMetaData() 46 | >>> storage.tpc_begin(txn) 47 | >>> storage.deleteObject(oid0, s0, txn) 48 | >>> storage.deleteObject(oid1, s1, txn) 49 | >>> _ = storage.tpc_vote(txn) 50 | >>> tid = storage.tpc_finish(txn) 51 | >>> tid == storage.lastTransaction() 52 | True 53 | 54 | Now if we try to load data for the objects, we get a POSKeyError: 55 | 56 | 57 | >>> storage.load(oid0, '') # doctest: +ELLIPSIS 58 | Traceback (most recent call last): 59 | ... 60 | ZODB.POSException.POSKeyError: ... 61 | 62 | >>> storage.load(oid1, '') # doctest: +ELLIPSIS 63 | Traceback (most recent call last): 64 | ... 65 | ZODB.POSException.POSKeyError: ... 66 | 67 | We can still get the data if we load before the time we deleted. 68 | 69 | >>> storage.loadBefore(oid0, conn.root()._p_serial) == (p0, s0, tid) 70 | True 71 | >>> storage.loadBefore(oid1, conn.root()._p_serial) == (p1, s1, tid) 72 | True 73 | >>> with open(storage.loadBlob(oid1, s1)) as fp: fp.read() 74 | 'some data' 75 | 76 | If we pack, however, the old data will be removed and the data will be 77 | gone: 78 | 79 | >>> db.pack(time.time()+1) 80 | >>> len(db.storage) 81 | 1 82 | 83 | >>> time.sleep(.1) 84 | 85 | >>> storage.load(oid0, '') # doctest: +ELLIPSIS 86 | Traceback (most recent call last): 87 | ... 88 | ZODB.POSException.POSKeyError: ... 89 | 90 | >>> storage.load(oid1, '') # doctest: +ELLIPSIS 91 | Traceback (most recent call last): 92 | ... 93 | ZODB.POSException.POSKeyError: ... 94 | 95 | >>> storage.loadBefore(oid0, conn.root()._p_serial) # doctest: +ELLIPSIS 96 | Traceback (most recent call last): 97 | ... 98 | ZODB.POSException.POSKeyError: ... 99 | 100 | >>> storage.loadBefore(oid1, conn.root()._p_serial) # doctest: +ELLIPSIS 101 | Traceback (most recent call last): 102 | ... 103 | ZODB.POSException.POSKeyError: ... 104 | 105 | >>> storage.loadBlob(oid1, s1) # doctest: +ELLIPSIS 106 | Traceback (most recent call last): 107 | ... 108 | ZODB.POSException.POSKeyError: ... 109 | 110 | A conflict error is raised if the serial we provide to deleteObject 111 | isn't current: 112 | 113 | >>> conn.root()[0] = conn.root().__class__() 114 | >>> transaction.commit() 115 | >>> oid = conn.root()[0]._p_oid 116 | >>> bad_serial = conn.root()[0]._p_serial 117 | >>> conn.root()[0].x = 1 118 | >>> transaction.commit() 119 | 120 | >>> txn = TransactionMetaData() 121 | >>> storage.tpc_begin(txn) 122 | >>> storage.deleteObject(oid, bad_serial, txn); storage.tpc_vote(txn) 123 | ... # doctest: +ELLIPSIS 124 | Traceback (most recent call last): 125 | ... 126 | ZODB.POSException.ConflictError: database conflict error ... 127 | 128 | >>> storage.tpc_abort(txn) 129 | 130 | >>> storage.close() 131 | -------------------------------------------------------------------------------- /src/ZODB/tests/MVCCMappingStorage.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) Zope Corporation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """An extension of MappingStorage that depends on polling. 15 | 16 | Each Connection has its own view of the database. Polling updates each 17 | connection's view. 18 | """ 19 | 20 | from zope.interface import implementer 21 | 22 | import ZODB.POSException 23 | import ZODB.utils 24 | from ZODB.interfaces import IMVCCStorage 25 | from ZODB.MappingStorage import MappingStorage 26 | 27 | 28 | @implementer(IMVCCStorage) 29 | class MVCCMappingStorage(MappingStorage): 30 | 31 | def __init__(self, name="MVCC Mapping Storage"): 32 | MappingStorage.__init__(self, name=name) 33 | # _polled_tid contains the transaction ID at the last poll. 34 | self._polled_tid = b'' 35 | self._data_snapshot = None # {oid->(state, tid)} 36 | self._main_lock = self._lock 37 | 38 | def new_instance(self): 39 | """Returns a storage instance that is a view of the same data. 40 | """ 41 | inst = MVCCMappingStorage(name=self.__name__) 42 | # All instances share the same OID data, transaction log, commit lock, 43 | # and OID sequence. 44 | inst._data = self._data 45 | inst._transactions = self._transactions 46 | inst._commit_lock = self._commit_lock 47 | inst.new_oid = self.new_oid 48 | inst.pack = self.pack 49 | inst.loadBefore = self.loadBefore 50 | inst._ltid = self._ltid 51 | inst._main_lock = self._lock 52 | return inst 53 | 54 | @ZODB.utils.locked(MappingStorage.opened) 55 | def sync(self, force=False): 56 | self._data_snapshot = None 57 | 58 | def release(self): 59 | pass 60 | 61 | @ZODB.utils.locked(MappingStorage.opened) 62 | def load(self, oid, version=''): 63 | assert not version, "Versions are not supported" 64 | if self._data_snapshot is None: 65 | self.poll_invalidations() 66 | info = self._data_snapshot.get(oid) 67 | if info: 68 | return info 69 | raise ZODB.POSException.POSKeyError(oid) 70 | 71 | def poll_invalidations(self): 72 | """Poll the storage for changes by other connections. 73 | """ 74 | # prevent changes to _transactions and _data during analysis 75 | with self._main_lock: 76 | if self._transactions: 77 | new_tid = self._transactions.maxKey() 78 | else: 79 | new_tid = ZODB.utils.z64 80 | 81 | # Copy the current data into a snapshot. This is obviously 82 | # very inefficient for large storages, but it's good for 83 | # tests. 84 | self._data_snapshot = {} 85 | for oid, tid_data in self._data.items(): 86 | if tid_data: 87 | tid = tid_data.maxKey() 88 | self._data_snapshot[oid] = tid_data[tid], tid 89 | 90 | if self._polled_tid: 91 | if self._polled_tid not in self._transactions: 92 | # This connection is so old that we can no longer enumerate 93 | # all the changes. 94 | self._polled_tid = new_tid 95 | return None 96 | 97 | changed_oids = set() 98 | for tid, txn in self._transactions.items( 99 | self._polled_tid, new_tid, 100 | excludemin=True, excludemax=False): 101 | if txn.status == 'p': 102 | # This transaction has been packed, so it is no longer 103 | # possible to enumerate all changed oids. 104 | self._polled_tid = new_tid 105 | return None 106 | if tid == self._ltid: 107 | # ignore the transaction committed by this connection 108 | continue 109 | changed_oids.update(txn.data.keys()) 110 | 111 | self._polled_tid = self._ltid = new_tid 112 | return list(changed_oids) 113 | 114 | def tpc_finish(self, transaction, func=lambda tid: None): 115 | self._data_snapshot = None 116 | with self._main_lock: 117 | return MappingStorage.tpc_finish(self, transaction, func) 118 | 119 | def tpc_abort(self, transaction): 120 | self._data_snapshot = None 121 | with self._main_lock: 122 | MappingStorage.tpc_abort(self, transaction) 123 | 124 | def pack(self, t, referencesf, gc=True): 125 | # prevent all concurrent commits during packing 126 | with self._commit_lock: 127 | MappingStorage.pack(self, t, referencesf, gc) 128 | -------------------------------------------------------------------------------- /src/ZODB/tests/MinPO.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """A minimal persistent object to use for tests""" 15 | import functools 16 | 17 | from persistent import Persistent 18 | 19 | 20 | @functools.total_ordering 21 | class MinPO(Persistent): 22 | def __init__(self, value=None): 23 | self.value = value 24 | 25 | def __hash__(self): 26 | return hash(self.value) 27 | 28 | def __eq__(self, aMinPO): 29 | return self.value == aMinPO.value 30 | 31 | def __lt__(self, aMinPO): 32 | return self.value < aMinPO.value 33 | 34 | def __repr__(self): 35 | return "MinPO(%s)" % self.value 36 | -------------------------------------------------------------------------------- /src/ZODB/tests/PersistentStorage.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Test that a storage's values persist across open and close.""" 15 | 16 | from ZODB.utils import load_current 17 | 18 | 19 | class PersistentStorage: 20 | 21 | def testUpdatesPersist(self): 22 | oids = [] 23 | 24 | def new_oid_wrapper(l=oids, new_oid=self._storage.new_oid): # noqa: E741 E501 ambiguous variable name 'l' and line too long 25 | oid = new_oid() 26 | l.append(oid) 27 | return oid 28 | 29 | self._storage.new_oid = new_oid_wrapper 30 | 31 | self._dostore() 32 | oid = self._storage.new_oid() 33 | revid = self._dostore(oid) 34 | oid = self._storage.new_oid() 35 | revid = self._dostore(oid, data=1) 36 | revid = self._dostore(oid, revid, data=2) 37 | self._dostore(oid, revid, data=3) 38 | 39 | # keep copies of all the objects 40 | objects = [] 41 | for oid in oids: 42 | p, s = load_current(self._storage, oid) 43 | objects.append((oid, '', p, s)) 44 | 45 | self._storage.close() 46 | self.open() 47 | 48 | # keep copies of all the objects 49 | for oid, ver, p, s in objects: 50 | _p, _s = load_current(self._storage, oid) 51 | self.assertEqual(p, _p) 52 | self.assertEqual(s, _s) 53 | -------------------------------------------------------------------------------- /src/ZODB/tests/ReadOnlyStorage.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | from ZODB.Connection import TransactionMetaData 15 | from ZODB.POSException import ReadOnlyError 16 | from ZODB.POSException import Unsupported 17 | from ZODB.utils import load_current 18 | 19 | 20 | class ReadOnlyStorage: 21 | 22 | def _create_data(self): 23 | # test a read-only storage that already has some data 24 | self.oids = {} 25 | for i in range(10): 26 | oid = self._storage.new_oid() 27 | revid = self._dostore(oid) 28 | self.oids[oid] = revid 29 | 30 | def _make_readonly(self): 31 | self._storage.close() 32 | self.open(read_only=True) 33 | self.assertTrue(self._storage.isReadOnly()) 34 | 35 | def testReadMethods(self): 36 | self._create_data() 37 | self._make_readonly() 38 | # Note that this doesn't check _all_ read methods. 39 | for oid in self.oids.keys(): 40 | data, revid = load_current(self._storage, oid) 41 | self.assertEqual(revid, self.oids[oid]) 42 | # Storages without revisions may not have loadSerial(). 43 | try: 44 | _data = self._storage.loadSerial(oid, revid) 45 | self.assertEqual(data, _data) 46 | except Unsupported: 47 | pass 48 | 49 | def testWriteMethods(self): 50 | self._make_readonly() 51 | self.assertRaises(ReadOnlyError, self._storage.new_oid) 52 | t = TransactionMetaData() 53 | self.assertRaises(ReadOnlyError, self._storage.tpc_begin, t) 54 | 55 | self.assertRaises(ReadOnlyError, self._storage.store, 56 | b'\000' * 8, None, b'', '', t) 57 | 58 | self.assertRaises(ReadOnlyError, self._storage.undo, 59 | b'\000' * 8, t) 60 | -------------------------------------------------------------------------------- /src/ZODB/tests/Synchronization.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Test the storage's implemenetation of the storage synchronization spec. 15 | 16 | The Synchronization spec 17 | http://www.zope.org/Documentation/Developer/Models/ZODB/ 18 | ZODB_Architecture_Storage_Interface_State_Synchronization_Diag.html 19 | 20 | It specifies two states committing and non-committing. A storage 21 | starts in the non-committing state. tpc_begin() transfers to the 22 | committting state; tpc_abort() and tpc_finish() transfer back to 23 | non-committing. 24 | 25 | Several other methods are only allowed in one state or another. Many 26 | methods allowed only in the committing state require that they apply 27 | to the currently committing transaction. 28 | 29 | The spec is silent on a variety of methods that don't appear to modify 30 | the state, e.g. load(), undoLog(), pack(). It's unclear whether there 31 | is a separate set of synchronization rules that apply to these methods 32 | or if the synchronization is implementation dependent, i.e. only what 33 | is need to guarantee a corrected implementation. 34 | 35 | The synchronization spec is also silent on whether there is any 36 | contract implied with the caller. If the storage can assume that a 37 | single client is single-threaded and that it will not call, e.g., store() 38 | until after it calls tpc_begin(), the implementation can be 39 | substantially simplified. 40 | 41 | New and/or unspecified methods: 42 | 43 | tpc_vote(): handled like tpc_abort 44 | undo(): how's that handled? 45 | 46 | Methods that have nothing to do with committing/non-committing: 47 | load(), loadSerial(), getName(), getSize(), __len__(), history(), 48 | undoLog(), pack(). 49 | 50 | Specific questions: 51 | 52 | The spec & docs say that undo() takes three arguments, the second 53 | being a transaction. If the specified arg isn't the current 54 | transaction, the undo() should raise StorageTransactionError. This 55 | isn't implemented anywhere. It looks like undo can be called at 56 | anytime. 57 | 58 | FileStorage does not allow undo() during a pack. How should this be 59 | tested? Is it a general restriction? 60 | 61 | 62 | 63 | """ 64 | 65 | from ZODB.Connection import TransactionMetaData 66 | from ZODB.POSException import StorageTransactionError 67 | 68 | 69 | OID = "\000" * 8 70 | SERIALNO = "\000" * 8 71 | TID = "\000" * 8 72 | 73 | 74 | class SynchronizedStorage: 75 | 76 | def verifyNotCommitting(self, callable, *args): 77 | self.assertRaises(StorageTransactionError, callable, *args) 78 | 79 | def verifyWrongTrans(self, callable, *args): 80 | t = TransactionMetaData() 81 | self._storage.tpc_begin(t) 82 | self.assertRaises(StorageTransactionError, callable, *args) 83 | self._storage.tpc_abort(t) 84 | 85 | def testStoreNotCommitting(self): 86 | self.verifyNotCommitting(self._storage.store, 87 | OID, SERIALNO, b"", "", TransactionMetaData()) 88 | 89 | def testStoreWrongTrans(self): 90 | self.verifyWrongTrans(self._storage.store, 91 | OID, SERIALNO, b"", "", TransactionMetaData()) 92 | 93 | def testAbortNotCommitting(self): 94 | self._storage.tpc_abort(TransactionMetaData()) 95 | 96 | def testAbortWrongTrans(self): 97 | t = TransactionMetaData() 98 | self._storage.tpc_begin(t) 99 | self._storage.tpc_abort(TransactionMetaData()) 100 | self._storage.tpc_abort(t) 101 | 102 | def testFinishNotCommitting(self): 103 | t = TransactionMetaData() 104 | self.assertRaises(StorageTransactionError, 105 | self._storage.tpc_finish, t) 106 | self._storage.tpc_abort(t) 107 | 108 | def testFinishWrongTrans(self): 109 | t = TransactionMetaData() 110 | self._storage.tpc_begin(t) 111 | self.assertRaises(StorageTransactionError, 112 | self._storage.tpc_finish, TransactionMetaData()) 113 | self._storage.tpc_abort(t) 114 | 115 | def testBeginCommitting(self): 116 | t = TransactionMetaData() 117 | self._storage.tpc_begin(t) 118 | self._storage.tpc_abort(t) 119 | 120 | # TODO: how to check undo? 121 | -------------------------------------------------------------------------------- /src/ZODB/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Having this makes debugging better. 2 | -------------------------------------------------------------------------------- /src/ZODB/tests/blob_connection.txt: -------------------------------------------------------------------------------- 1 | Connection support for Blobs tests 2 | ================================== 3 | 4 | Connections handle Blobs specially. To demonstrate that, we first need a Blob 5 | with some data: 6 | 7 | >>> from ZODB.interfaces import IBlob 8 | >>> from ZODB.blob import Blob 9 | >>> import transaction 10 | >>> blob = Blob() 11 | >>> data = blob.open("w") 12 | >>> _ = data.write(b"I'm a happy Blob.") 13 | >>> data.close() 14 | 15 | We also need a database with a blob supporting storage. (We're going to use 16 | FileStorage rather than MappingStorage here because we will want ``loadBefore`` 17 | for one of our examples.) 18 | 19 | >>> blob_storage = create_storage() 20 | >>> from ZODB.DB import DB 21 | >>> database = DB(blob_storage) 22 | 23 | Putting a Blob into a Connection works like every other object: 24 | 25 | >>> connection = database.open() 26 | >>> root = connection.root() 27 | >>> root['myblob'] = blob 28 | >>> transaction.commit() 29 | 30 | We can also commit a transaction that seats a blob into place without 31 | calling the blob's open method: 32 | 33 | >>> nothing = transaction.begin() 34 | >>> anotherblob = Blob() 35 | >>> root['anotherblob'] = anotherblob 36 | >>> nothing = transaction.commit() 37 | 38 | Getting stuff out of there works similarly: 39 | 40 | >>> transaction2 = transaction.TransactionManager() 41 | >>> connection2 = database.open(transaction_manager=transaction2) 42 | >>> root = connection2.root() 43 | >>> blob2 = root['myblob'] 44 | >>> IBlob.providedBy(blob2) 45 | True 46 | >>> with blob2.open("r") as fp: fp.read() 47 | b"I'm a happy Blob." 48 | >>> transaction2.abort() 49 | 50 | MVCC also works. 51 | 52 | >>> transaction3 = transaction.TransactionManager() 53 | >>> connection3 = database.open(transaction_manager=transaction3) 54 | >>> f = connection.root()['myblob'].open('w') 55 | >>> _ = f.write(b'I am an ecstatic Blob.') 56 | >>> f.close() 57 | >>> transaction.commit() 58 | >>> with connection3.root()['myblob'].open('r') as fp: fp.read() 59 | b"I'm a happy Blob." 60 | 61 | >>> transaction2.abort() 62 | >>> transaction3.abort() 63 | >>> connection2.close() 64 | >>> connection3.close() 65 | 66 | You can't put blobs into a database that has uses a Non-Blob-Storage, though: 67 | 68 | >>> from ZODB.MappingStorage import MappingStorage 69 | >>> no_blob_storage = MappingStorage() 70 | >>> database2 = DB(no_blob_storage) 71 | >>> connection2 = database2.open(transaction_manager=transaction2) 72 | >>> root = connection2.root() 73 | >>> root['myblob'] = Blob() 74 | >>> transaction2.commit() # doctest: +ELLIPSIS 75 | Traceback (most recent call last): 76 | ... 77 | ZODB.POSException.Unsupported: Storing Blobs in ... 78 | 79 | >>> transaction2.abort() 80 | >>> connection2.close() 81 | 82 | After testing this, we don't need the storage directory and databases anymore: 83 | 84 | >>> transaction.abort() 85 | >>> connection.close() 86 | >>> database.close() 87 | >>> database2.close() 88 | >>> blob_storage.close() 89 | -------------------------------------------------------------------------------- /src/ZODB/tests/blob_consume.txt: -------------------------------------------------------------------------------- 1 | Consuming existing files 2 | ======================== 3 | 4 | The ZODB Blob implementation allows to import existing files as Blobs within 5 | an O(1) operation we call `consume`:: 6 | 7 | Let's create a file:: 8 | 9 | >>> to_import = open('to_import', 'wb') 10 | >>> _ = to_import.write(b"I'm a Blob and I feel fine.") 11 | 12 | The file *must* be closed before giving it to consumeFile: 13 | 14 | >>> to_import.close() 15 | 16 | Now, let's consume this file in a blob by specifying it's name:: 17 | 18 | >>> from ZODB.blob import Blob 19 | >>> blob = Blob() 20 | >>> blob.consumeFile('to_import') 21 | 22 | After the consumeFile operation, the original file has been removed: 23 | 24 | >>> import os 25 | >>> os.path.exists('to_import') 26 | False 27 | 28 | We now can call open on the blob and read and write the data:: 29 | 30 | >>> blob_read = blob.open('r') 31 | >>> blob_read.read() 32 | b"I'm a Blob and I feel fine." 33 | >>> blob_read.close() 34 | >>> blob_write = blob.open('w') 35 | >>> _ = blob_write.write(b'I was changed.') 36 | >>> blob_write.close() 37 | 38 | We can not consume a file when there is a reader or writer around for a blob 39 | already:: 40 | 41 | >>> with open('to_import', 'wb') as file: 42 | ... _ = file.write(b'I am another blob.') 43 | >>> blob_read = blob.open('r') 44 | >>> blob.consumeFile('to_import') 45 | Traceback (most recent call last): 46 | ZODB.interfaces.BlobError: Already opened for reading. 47 | >>> blob_read.close() 48 | >>> blob_write = blob.open('w') 49 | >>> blob.consumeFile('to_import') 50 | Traceback (most recent call last): 51 | ZODB.interfaces.BlobError: Already opened for writing. 52 | >>> blob_write.close() 53 | 54 | Now, after closing all readers and writers we can consume files again:: 55 | 56 | >>> blob.consumeFile('to_import') 57 | >>> blob_read = blob.open('r') 58 | >>> blob_read.read() 59 | b'I am another blob.' 60 | 61 | >>> blob_read.close() 62 | 63 | Edge cases 64 | ========== 65 | 66 | There are some edge cases what happens when the link() operation 67 | fails. We simulate this in different states: 68 | 69 | Case 1: We don't have uncommitted data, but the link operation fails. We fall 70 | back to try a copy/remove operation that is successfull:: 71 | 72 | >>> with open('to_import', 'wb') as file: 73 | ... _ = file.write(b'Some data.') 74 | 75 | >>> def failing_rename(f1, f2): 76 | ... if f1 == 'to_import': 77 | ... raise OSError("I can't link.") 78 | ... os_rename(f1, f2) 79 | 80 | >>> blob = Blob() 81 | >>> os_rename = os.rename 82 | >>> os.rename = failing_rename 83 | >>> blob.consumeFile('to_import') 84 | 85 | The blob did not have data before, so it shouldn't have data now:: 86 | 87 | >>> with blob.open('r') as fp: fp.read() 88 | b'Some data.' 89 | 90 | Case 2: We don't have uncommitted data and both the link operation and the 91 | copy fail. The exception will be re-raised and the target file will not 92 | exist:: 93 | 94 | >>> blob = Blob() 95 | >>> import ZODB.utils 96 | >>> utils_cp = ZODB.utils.cp 97 | 98 | >>> def failing_copy(f1, f2): 99 | ... raise OSError("I can't copy.") 100 | 101 | >>> ZODB.utils.cp = failing_copy 102 | >>> with open('to_import', 'wb') as file: 103 | ... _ = file.write(b'Some data.') 104 | >>> blob.consumeFile('to_import') 105 | Traceback (most recent call last): 106 | OSError: I can't copy. 107 | 108 | The blob did not have data before, so it shouldn't have data now:: 109 | 110 | >>> with blob.open('r') as fp: fp.read() 111 | b'' 112 | 113 | Case 3: We have uncommitted data, but the link and the copy operations fail. 114 | The exception will be re-raised and the target file will exist with the 115 | previous uncomitted data:: 116 | 117 | >>> blob = Blob() 118 | >>> with blob.open('w') as blob_writing: 119 | ... _ = blob_writing.write(b'Uncommitted data') 120 | 121 | >>> blob.consumeFile('to_import') 122 | Traceback (most recent call last): 123 | OSError: I can't copy. 124 | 125 | The blob did existed before and had uncommitted data, this shouldn't have 126 | changed:: 127 | 128 | >>> with blob.open('r') as fp: fp.read() 129 | b'Uncommitted data' 130 | 131 | >>> os.rename = os_rename 132 | >>> ZODB.utils.cp = utils_cp 133 | -------------------------------------------------------------------------------- /src/ZODB/tests/blob_importexport.txt: -------------------------------------------------------------------------------- 1 | Import/export support for blob data 2 | =================================== 3 | 4 | Set up: 5 | 6 | >>> import ZODB.blob, transaction 7 | >>> from persistent.mapping import PersistentMapping 8 | 9 | We need an database with an undoing blob supporting storage: 10 | 11 | >>> database1 = ZODB.DB(create_storage('1')) 12 | >>> database2 = ZODB.DB(create_storage('2')) 13 | 14 | Create our root object for database1: 15 | 16 | >>> connection1 = database1.open() 17 | >>> root1 = connection1.root() 18 | 19 | Put a couple blob objects in our database1 and on the filesystem: 20 | 21 | >>> import time, os 22 | >>> nothing = transaction.begin() 23 | >>> data1 = b'x'*100000 24 | >>> blob1 = ZODB.blob.Blob() 25 | >>> with blob1.open('w') as file: 26 | ... _ = file.write(data1) 27 | >>> data2 = b'y'*100000 28 | >>> blob2 = ZODB.blob.Blob() 29 | >>> with blob2.open('w') as file: 30 | ... _ = file.write(data2) 31 | >>> d = PersistentMapping({'blob1':blob1, 'blob2':blob2}) 32 | >>> root1['blobdata'] = d 33 | >>> transaction.commit() 34 | 35 | Export our blobs from a database1 connection: 36 | 37 | >>> conn = root1['blobdata']._p_jar 38 | >>> oid = root1['blobdata']._p_oid 39 | >>> exportfile = 'export' 40 | >>> connection1.exportFile(oid, exportfile).close() 41 | 42 | Import our exported data into database2: 43 | 44 | >>> connection2 = database2.open() 45 | >>> root2 = connection2.root() 46 | >>> nothing = transaction.begin() 47 | >>> data = root2._p_jar.importFile(exportfile) 48 | >>> root2['blobdata'] = data 49 | >>> transaction.commit() 50 | 51 | Make sure our data exists: 52 | 53 | >>> items1 = root1['blobdata'] 54 | >>> items2 = root2['blobdata'] 55 | >>> bool(items1.keys() == items2.keys()) 56 | True 57 | >>> with items1['blob1'].open() as fp1: 58 | ... with items2['blob1'].open() as fp2: 59 | ... fp1.read() == fp2.read() 60 | True 61 | >>> with items1['blob2'].open() as fp1: 62 | ... with items2['blob2'].open() as fp2: 63 | ... fp1.read() == fp2.read() 64 | True 65 | >>> transaction.get().abort() 66 | 67 | .. cleanup 68 | 69 | >>> database1.close() 70 | >>> database2.close() 71 | -------------------------------------------------------------------------------- /src/ZODB/tests/blob_packing.txt: -------------------------------------------------------------------------------- 1 | Packing support for blob data 2 | ============================= 3 | 4 | Set up: 5 | 6 | >>> from ZODB.serialize import referencesf 7 | >>> from ZODB.blob import Blob 8 | >>> from ZODB import utils 9 | >>> from ZODB.DB import DB 10 | >>> import transaction 11 | 12 | A helper method to assure a unique timestamp across multiple platforms: 13 | 14 | >>> from ZODB.tests.testblob import new_time 15 | 16 | UNDOING 17 | ======= 18 | 19 | We need a database with an undoing blob supporting storage: 20 | 21 | >>> blob_storage = create_storage() 22 | >>> database = DB(blob_storage) 23 | 24 | Create our root object: 25 | 26 | >>> connection1 = database.open() 27 | >>> root = connection1.root() 28 | 29 | Put some revisions of a blob object in our database and on the filesystem: 30 | 31 | >>> import os 32 | >>> tids = [] 33 | >>> times = [] 34 | >>> blob = Blob() 35 | 36 | >>> for i in range(5): 37 | ... _ = transaction.begin() 38 | ... times.append(new_time()) 39 | ... with blob.open('w') as file: 40 | ... _ = file.write(b'this is blob data ' + str(i).encode()) 41 | ... if i: 42 | ... tids.append(blob._p_serial) 43 | ... else: 44 | ... root['blob'] = blob 45 | ... transaction.commit() 46 | 47 | >>> blob._p_activate() 48 | >>> tids.append(blob._p_serial) 49 | 50 | >>> oid = root['blob']._p_oid 51 | >>> fns = [ blob_storage.fshelper.getBlobFilename(oid, x) for x in tids ] 52 | >>> [ os.path.exists(x) for x in fns ] # no pack 53 | [True, True, True, True, True] 54 | 55 | Do a pack to the slightly before the first revision was written: 56 | 57 | >>> packtime = times[0] 58 | >>> blob_storage.pack(packtime, referencesf) 59 | >>> [ os.path.exists(x) for x in fns ] 60 | [True, True, True, True, True] 61 | 62 | Do a pack to the slightly before the second revision was written: 63 | 64 | >>> packtime = times[1] 65 | >>> blob_storage.pack(packtime, referencesf) 66 | >>> [ os.path.exists(x) for x in fns ] 67 | [True, True, True, True, True] 68 | 69 | Do a pack to the slightly before the third revision was written: 70 | 71 | >>> packtime = times[2] 72 | >>> blob_storage.pack(packtime, referencesf) 73 | >>> [ os.path.exists(x) for x in fns ] 74 | [False, True, True, True, True] 75 | 76 | Do a pack to the slightly before the fourth revision was written: 77 | 78 | >>> packtime = times[3] 79 | >>> blob_storage.pack(packtime, referencesf) 80 | >>> [ os.path.exists(x) for x in fns ] 81 | [False, False, True, True, True] 82 | 83 | Do a pack to the slightly before the fifth revision was written: 84 | 85 | >>> packtime = times[4] 86 | >>> blob_storage.pack(packtime, referencesf) 87 | >>> [ os.path.exists(x) for x in fns ] 88 | [False, False, False, True, True] 89 | 90 | Do a pack to now: 91 | 92 | >>> packtime = new_time() 93 | >>> blob_storage.pack(packtime, referencesf) 94 | >>> [ os.path.exists(x) for x in fns ] 95 | [False, False, False, False, True] 96 | 97 | Delete the object and do a pack, it should get rid of the most current 98 | revision as well as the entire directory: 99 | 100 | >>> nothing = transaction.begin() 101 | >>> del root['blob'] 102 | >>> transaction.commit() 103 | >>> packtime = new_time() 104 | >>> blob_storage.pack(packtime, referencesf) 105 | >>> [ os.path.exists(x) for x in fns ] 106 | [False, False, False, False, False] 107 | >>> os.path.exists(os.path.split(fns[0])[0]) 108 | False 109 | 110 | Clean up our blob directory and database: 111 | 112 | >>> database.close() 113 | -------------------------------------------------------------------------------- /src/ZODB/tests/blob_tempdir.txt: -------------------------------------------------------------------------------- 1 | ======================================= 2 | Temporary directory handling with blobs 3 | ======================================= 4 | 5 | When creating uncommitted data files for a blob (e.g. by calling 6 | `blob.open('w')`) we need to decide where to create them. The decision depends 7 | on whether the blob is already stored in a database or not. 8 | 9 | Case 1: Blobs that are not in a database yet 10 | ============================================ 11 | 12 | Let's create a new blob and open it for writing:: 13 | 14 | >>> from ZODB.blob import Blob 15 | >>> b = Blob() 16 | >>> w = b.open('w') 17 | 18 | The created file is in the default temporary directory:: 19 | 20 | >>> import tempfile 21 | >>> w.name.startswith(tempfile.gettempdir()) 22 | True 23 | 24 | >>> w.close() 25 | 26 | Case 2: Blobs that are in a database 27 | ==================================== 28 | 29 | For this case we instanciate a blob and add it to a database immediately. 30 | First, we need a datatabase with blob support:: 31 | 32 | >>> from ZODB.MappingStorage import MappingStorage 33 | >>> from ZODB.blob import BlobStorage 34 | >>> from ZODB.DB import DB 35 | >>> import os.path 36 | >>> base_storage = MappingStorage('test') 37 | >>> blob_dir = os.path.abspath('blobs') 38 | >>> blob_storage = BlobStorage(blob_dir, base_storage) 39 | >>> database = DB(blob_storage) 40 | 41 | Now we create a blob and put it in the database. After that we open it for 42 | writing and expect the file to be in the blob temporary directory:: 43 | 44 | >>> blob = Blob() 45 | >>> connection = database.open() 46 | >>> connection.add(blob) 47 | >>> w = blob.open('w') 48 | >>> w.name.startswith(os.path.join(blob_dir, 'tmp')) 49 | True 50 | 51 | >>> w.close() 52 | >>> database.close() 53 | 54 | -------------------------------------------------------------------------------- /src/ZODB/tests/component.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 |
8 | 9 | 14 |
15 | 16 | 17 | -------------------------------------------------------------------------------- /src/ZODB/tests/dangle.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | 15 | """Functional test to produce a dangling reference.""" 16 | 17 | import time 18 | 19 | import transaction 20 | from persistent import Persistent 21 | 22 | from ZODB import DB 23 | from ZODB.FileStorage import FileStorage 24 | 25 | 26 | class P(Persistent): 27 | pass 28 | 29 | 30 | def create_dangling_ref(db): 31 | rt = db.open().root() 32 | 33 | rt[1] = o1 = P() 34 | transaction.get().note("create o1") 35 | transaction.commit() 36 | 37 | rt[2] = o2 = P() 38 | transaction.get().note("create o2") 39 | transaction.commit() 40 | 41 | c = o1.child = P() 42 | transaction.get().note("set child on o1") 43 | transaction.commit() 44 | 45 | o1.child = P() 46 | transaction.get().note("replace child on o1") 47 | transaction.commit() 48 | 49 | time.sleep(2) 50 | # The pack should remove the reference to c, because it is no 51 | # longer referenced from o1. But the object still exists and has 52 | # an oid, so a new commit of it won't create a new object. 53 | db.pack() 54 | 55 | print(repr(c._p_oid)) 56 | o2.child = c 57 | transaction.get().note("set child on o2") 58 | transaction.commit() 59 | 60 | 61 | def main(): 62 | fs = FileStorage("dangle.fs") 63 | db = DB(fs) 64 | create_dangling_ref(db) 65 | db.close() 66 | 67 | 68 | if __name__ == "__main__": 69 | main() 70 | -------------------------------------------------------------------------------- /src/ZODB/tests/fix84.rst: -------------------------------------------------------------------------------- 1 | A change in the way databases were initialized affected tests 2 | ============================================================= 3 | 4 | Originally, databases added root objects by interacting directly with 5 | storages, rather than using connections. As storages transaction 6 | interaction became more complex, interacting directly with storages 7 | let to duplicated code (and buggy) code. 8 | 9 | See: https://github.com/zopefoundation/ZODB/issues/84 10 | 11 | Fixing this had some impacts that affected tests: 12 | 13 | - New databases now have a connection with a single object in it's cache. 14 | This is a very slightly good thing, but it broke some tests expectations. 15 | 16 | - Tests that manipulated time, had their clocks off because of new time calls. 17 | 18 | This led to some test fixes, in many cases adding a mysterious 19 | ``cacheMinimize()`` call. 20 | -------------------------------------------------------------------------------- /src/ZODB/tests/loggingsupport.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Support for testing logging code 15 | 16 | If you want to test that your code generates proper log output, you 17 | can create and install a handler that collects output: 18 | 19 | >>> handler = InstalledHandler('foo.bar') 20 | 21 | The handler is installed into loggers for all of the names passed. In 22 | addition, the logger level is set to 1, which means, log 23 | everything. If you want to log less than everything, you can provide a 24 | level keyword argument. The level setting effects only the named 25 | loggers. 26 | 27 | Then, any log output is collected in the handler: 28 | 29 | >>> logging.getLogger('foo.bar').exception('eek') 30 | >>> logging.getLogger('foo.bar').info('blah blah') 31 | 32 | >>> for record in handler.records: 33 | ... print(record.name, record.levelname) 34 | ... print(' ', record.getMessage()) 35 | foo.bar ERROR 36 | eek 37 | foo.bar INFO 38 | blah blah 39 | 40 | A similar effect can be gotten by just printing the handler: 41 | 42 | >>> print(handler) 43 | foo.bar ERROR 44 | eek 45 | foo.bar INFO 46 | blah blah 47 | 48 | After checking the log output, you need to uninstall the handler: 49 | 50 | >>> handler.uninstall() 51 | 52 | At which point, the handler won't get any more log output. 53 | Let's clear the handler: 54 | 55 | >>> handler.clear() 56 | >>> handler.records 57 | [] 58 | 59 | And then log something: 60 | 61 | >>> logging.getLogger('foo.bar').info('blah') 62 | 63 | and, sure enough, we still have no output: 64 | 65 | >>> handler.records 66 | [] 67 | 68 | $Id: loggingsupport.py 28349 2004-11-06 00:10:32Z tim_one $ 69 | """ 70 | 71 | import logging 72 | 73 | 74 | class Handler(logging.Handler): 75 | 76 | def __init__(self, *names, **kw): 77 | logging.Handler.__init__(self) 78 | self.names = names 79 | self.records = [] 80 | self.setLoggerLevel(**kw) 81 | 82 | def setLoggerLevel(self, level=1): 83 | self.level = level 84 | self.oldlevels = {} 85 | self.olddisabled = {} 86 | 87 | def emit(self, record): 88 | self.records.append(record) 89 | 90 | def clear(self): 91 | del self.records[:] 92 | 93 | def install(self): 94 | for name in self.names: 95 | logger = logging.getLogger(name) 96 | self.oldlevels[name] = logger.level 97 | self.olddisabled[name] = logger.disabled 98 | logger.setLevel(self.level) 99 | logger.disabled = False 100 | logger.addHandler(self) 101 | 102 | def uninstall(self): 103 | for name in self.names: 104 | logger = logging.getLogger(name) 105 | logger.setLevel(self.oldlevels[name]) 106 | logger.disabled = self.olddisabled[name] 107 | logger.removeHandler(self) 108 | 109 | def __str__(self): 110 | return '\n'.join( 111 | [("%s %s\n %s" % 112 | (record.name, record.levelname, 113 | '\n'.join([line 114 | for line in record.getMessage().split('\n') 115 | if line.strip()]) 116 | ) 117 | ) 118 | for record in self.records] 119 | ) 120 | 121 | 122 | class InstalledHandler(Handler): 123 | 124 | def __init__(self, *names): 125 | Handler.__init__(self, *names) 126 | self.install() 127 | -------------------------------------------------------------------------------- /src/ZODB/tests/speed.py: -------------------------------------------------------------------------------- 1 | import getopt 2 | import os 3 | import string 4 | import sys 5 | import time 6 | 7 | import persistent 8 | import transaction 9 | 10 | import ZODB 11 | import ZODB.FileStorage 12 | 13 | 14 | ############################################################################## 15 | # 16 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 17 | # All Rights Reserved. 18 | # 19 | # This software is subject to the provisions of the Zope Public License, 20 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 21 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 22 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 23 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 24 | # FOR A PARTICULAR PURPOSE 25 | # 26 | ############################################################################## 27 | usage = """Test speed of a ZODB storage 28 | 29 | Options: 30 | 31 | -d file The data file to use as input. 32 | The default is this script. 33 | 34 | -n n The number of repititions 35 | 36 | -s module A module that defines a 'Storage' 37 | attribute, which is an open storage. 38 | If not specified, a FileStorage will ne 39 | used. 40 | 41 | -z Test compressing data 42 | 43 | -D Run in debug mode 44 | 45 | -L Test loads as well as stores by minimizing 46 | the cache after eachrun 47 | 48 | -M Output means only 49 | """ 50 | 51 | sys.path.insert(0, os.getcwd()) 52 | 53 | 54 | class P(persistent.Persistent): 55 | pass 56 | 57 | 58 | def main(args): 59 | 60 | opts, args = getopt.getopt(args, 'zd:n:Ds:LM') 61 | z = s = None 62 | data = sys.argv[0] 63 | nrep = 5 64 | minimize = 0 65 | detailed = 1 66 | for o, v in opts: 67 | if o == '-n': 68 | nrep = string.atoi(v) 69 | elif o == '-d': 70 | data = v 71 | elif o == '-s': 72 | s = v 73 | elif o == '-z': 74 | global zlib 75 | import zlib 76 | z = compress 77 | elif o == '-L': 78 | minimize = 1 79 | elif o == '-M': 80 | detailed = 0 81 | elif o == '-D': 82 | os.environ['STUPID_LOG_FILE'] = '' 83 | os.environ['STUPID_LOG_SEVERITY'] = '-999' 84 | 85 | if s: 86 | s = __import__(s, globals(), globals(), ('__doc__',)) 87 | s = s.Storage 88 | else: 89 | s = ZODB.FileStorage.FileStorage('zeo_speed.fs', create=1) 90 | 91 | with open(data) as fp: 92 | data = fp.read() 93 | db = ZODB.DB(s, 94 | # disable cache deactivation 95 | cache_size=4000, 96 | cache_deactivate_after=6000,) 97 | 98 | results = {1: 0, 10: 0, 100: 0, 1000: 0} 99 | for j in range(nrep): 100 | for r in 1, 10, 100, 1000: 101 | t = time.time() 102 | jar = db.open() 103 | transaction.begin() 104 | rt = jar.root() 105 | key = 's%s' % r 106 | if key in rt: 107 | p = rt[key] 108 | else: 109 | rt[key] = p = P() 110 | for i in range(r): 111 | if z is not None: 112 | d = z(data) 113 | else: 114 | d = data 115 | v = getattr(p, str(i), P()) 116 | v.d = d 117 | setattr(p, str(i), v) 118 | transaction.commit() 119 | jar.close() 120 | t = time.time()-t 121 | if detailed: 122 | sys.stderr.write("{}\t{}\t{:.4f}\n".format(j, r, t)) 123 | sys.stdout.flush() 124 | results[r] = results[r]+t 125 | rt = d = p = v = None # release all references 126 | if minimize: 127 | time.sleep(3) 128 | jar.cacheMinimize(3) 129 | 130 | if detailed: 131 | print('-'*24) 132 | for r in 1, 10, 100, 1000: 133 | t = results[r]/nrep 134 | sys.stderr.write("mean:\t{}\t{:.4f}\t{:.4f} (s/o)\n".format(r, t, t/r)) 135 | 136 | db.close() 137 | 138 | 139 | def compress(s): 140 | c = zlib.compressobj() 141 | o = c.compress(s) 142 | return o+c.flush() 143 | 144 | 145 | if __name__ == '__main__': 146 | main(sys.argv[1:]) 147 | -------------------------------------------------------------------------------- /src/ZODB/tests/synchronizers.txt: -------------------------------------------------------------------------------- 1 | ============= 2 | Synchronizers 3 | ============= 4 | 5 | Here are some tests that storage ``sync()`` methods get called at appropriate 6 | times in the life of a transaction. The tested behavior is new in ZODB 3.4. 7 | 8 | First define a lightweight storage with a ``sync()`` method: 9 | 10 | >>> import ZODB 11 | >>> from ZODB.MappingStorage import MappingStorage 12 | >>> import transaction 13 | 14 | >>> class SimpleStorage(MappingStorage): 15 | ... sync_called = False 16 | ... 17 | ... def sync(self, *args): 18 | ... self.sync_called = True 19 | 20 | Make a change locally: 21 | 22 | >>> st = SimpleStorage() 23 | >>> db = ZODB.DB(st) 24 | >>> st.sync_called = False 25 | >>> cn = db.open() 26 | >>> rt = cn.root() 27 | >>> rt['a'] = 1 28 | 29 | Sync isn't called when a connection is opened, even though that 30 | implicitly starts a new transaction: 31 | 32 | >>> st.sync_called 33 | False 34 | 35 | Sync is only called when we explicitly start a new transaction: 36 | 37 | >>> _ = transaction.begin() 38 | 39 | >>> st.sync_called 40 | True 41 | >>> st.sync_called = False 42 | 43 | BTW, calling ``sync()`` on a connection starts a new transaction, which 44 | caused ``sync()`` to be called on the storage: 45 | 46 | >>> cn.sync() 47 | >>> st.sync_called 48 | True 49 | >>> st.sync_called = False 50 | 51 | ``sync()`` is not called by the Connection's ``afterCompletion()`` 52 | hook after the commit completes, because we'll sync when a new 53 | transaction begins: 54 | 55 | >>> transaction.commit() 56 | >>> st.sync_called # False before 3.4 57 | False 58 | 59 | ``sync()`` is also not called by the ``afterCompletion()`` hook after an abort. 60 | 61 | >>> st.sync_called = False 62 | >>> rt['b'] = 2 63 | >>> transaction.abort() 64 | >>> st.sync_called # False before 3.4 65 | False 66 | 67 | And ``sync()`` is called whenever we explicitly start a new transaction, via 68 | the ``newTransaction()`` hook. 69 | 70 | >>> st.sync_called = False 71 | >>> dummy = transaction.begin() 72 | >>> st.sync_called # False before 3.4 73 | True 74 | 75 | Clean up. Closing db isn't enough -- closing a DB doesn't close its 76 | `Connections`. Leaving our `Connection` open here can cause the 77 | ``SimpleStorage.sync()`` method to get called later, during another test, and 78 | our doctest-synthesized module globals no longer exist then. You get a weird 79 | traceback then ;-) 80 | 81 | >>> cn.close() 82 | 83 | As a special case, if a synchronizer registers while a transaction is 84 | in flight, then newTransaction and thus the storage sync method is 85 | called: 86 | 87 | >>> tm = transaction.TransactionManager() 88 | >>> st.sync_called = False 89 | >>> _ = tm.begin() # we're doing this _before_ opening a connection 90 | >>> cn = db.open(transaction_manager=tm) 91 | >>> st.sync_called 92 | True 93 | 94 | >>> cn.close() 95 | >>> db.close() 96 | 97 | -------------------------------------------------------------------------------- /src/ZODB/tests/testActivityMonitor.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Tests of the default activity monitor. 15 | 16 | See ZODB/ActivityMonitor.py 17 | 18 | $Id$ 19 | """ 20 | 21 | import time 22 | import unittest 23 | 24 | from ZODB.ActivityMonitor import ActivityMonitor 25 | 26 | 27 | class FakeConnection: 28 | 29 | loads = 0 30 | stores = 0 31 | 32 | def _transferred(self, loads, stores): 33 | self.loads = self.loads + loads 34 | self.stores = self.stores + stores 35 | 36 | def getTransferCounts(self, clear=0): 37 | res = self.loads, self.stores 38 | if clear: 39 | self.loads = self.stores = 0 40 | return res 41 | 42 | 43 | class Tests(unittest.TestCase): 44 | 45 | def testAddLogEntries(self): 46 | am = ActivityMonitor(history_length=3600) 47 | self.assertEqual(len(am.log), 0) 48 | c = FakeConnection() 49 | c._transferred(1, 2) 50 | am.closedConnection(c) 51 | c._transferred(3, 7) 52 | am.closedConnection(c) 53 | self.assertEqual(len(am.log), 2) 54 | 55 | def testTrim(self): 56 | am = ActivityMonitor(history_length=0.1) 57 | c = FakeConnection() 58 | c._transferred(1, 2) 59 | am.closedConnection(c) 60 | time.sleep(0.2) 61 | c._transferred(3, 7) 62 | am.closedConnection(c) 63 | self.assertTrue(len(am.log) <= 1) 64 | 65 | def testSetHistoryLength(self): 66 | am = ActivityMonitor(history_length=3600) 67 | c = FakeConnection() 68 | c._transferred(1, 2) 69 | am.closedConnection(c) 70 | time.sleep(0.2) 71 | c._transferred(3, 7) 72 | am.closedConnection(c) 73 | self.assertEqual(len(am.log), 2) 74 | am.setHistoryLength(0.1) 75 | self.assertEqual(am.getHistoryLength(), 0.1) 76 | self.assertTrue(len(am.log) <= 1) 77 | 78 | def testActivityAnalysis(self): 79 | am = ActivityMonitor(history_length=3600) 80 | c = FakeConnection() 81 | c._transferred(1, 2) 82 | am.closedConnection(c) 83 | c._transferred(3, 7) 84 | am.closedConnection(c) 85 | res = am.getActivityAnalysis(start=0, end=0, divisions=10) 86 | lastend = 0 87 | for n in range(9): 88 | div = res[n] 89 | self.assertEqual(div['stores'], 0) 90 | self.assertEqual(div['loads'], 0) 91 | self.assertTrue(div['start'] > 0) 92 | self.assertTrue(div['start'] >= lastend) 93 | self.assertTrue(div['start'] < div['end']) 94 | lastend = div['end'] 95 | div = res[9] 96 | self.assertEqual(div['stores'], 9) 97 | self.assertEqual(div['loads'], 4) 98 | self.assertTrue(div['start'] > 0) 99 | self.assertTrue(div['start'] >= lastend) 100 | self.assertTrue(div['start'] < div['end']) 101 | 102 | 103 | def test_suite(): 104 | return unittest.defaultTestLoader.loadTestsFromTestCase(Tests) 105 | -------------------------------------------------------------------------------- /src/ZODB/tests/testBroken.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Test broken-object suppport 15 | """ 16 | 17 | import os 18 | import sys 19 | import unittest 20 | 21 | import persistent 22 | import transaction 23 | 24 | 25 | if os.environ.get('USE_ZOPE_TESTING_DOCTEST'): 26 | from zope.testing.doctest import DocTestSuite 27 | else: 28 | from doctest import DocTestSuite 29 | 30 | from ZODB.tests.util import DB 31 | from ZODB.tests.util import checker 32 | 33 | 34 | def test_integration(): 35 | r"""Test the integration of broken object support with the databse: 36 | 37 | >>> db = DB() 38 | 39 | We'll create a fake module with a class: 40 | 41 | >>> class NotThere(object): 42 | ... Atall = type('Atall', (persistent.Persistent, ), 43 | ... {'__module__': 'ZODB.not.there'}) 44 | 45 | And stuff this into sys.modules to simulate a regular module: 46 | 47 | >>> sys.modules['ZODB.not.there'] = NotThere 48 | >>> sys.modules['ZODB.not'] = NotThere 49 | 50 | Now, we'll create and save an instance, and make sure we can 51 | load it in another connection: 52 | 53 | >>> a = NotThere.Atall() 54 | >>> a.x = 1 55 | >>> conn1 = db.open() 56 | >>> conn1.root()['a'] = a 57 | >>> transaction.commit() 58 | 59 | >>> conn2 = db.open() 60 | >>> a2 = conn2.root()['a'] 61 | >>> a2.__class__ is a.__class__ 62 | True 63 | >>> a2.x 64 | 1 65 | 66 | Now, we'll uninstall the module, simulating having the module 67 | go away: 68 | 69 | >>> del sys.modules['ZODB.not.there'] 70 | 71 | and we'll try to load the object in another connection: 72 | 73 | >>> conn3 = db.open() 74 | >>> a3 = conn3.root()['a'] 75 | >>> a3 # doctest: +NORMALIZE_WHITESPACE 76 | 78 | 79 | >>> a3.__Broken_state__ 80 | {'x': 1} 81 | 82 | Broken objects provide an interface: 83 | 84 | >>> from ZODB.interfaces import IBroken 85 | >>> IBroken.providedBy(a3) 86 | True 87 | 88 | Let's clean up: 89 | 90 | >>> db.close() 91 | >>> del sys.modules['ZODB.not'] 92 | 93 | Cleanup: 94 | 95 | >>> import ZODB.broken 96 | >>> ZODB.broken.broken_cache.clear() 97 | """ 98 | 99 | 100 | def test_suite(): 101 | return unittest.TestSuite(( 102 | DocTestSuite('ZODB.broken', checker=checker), 103 | DocTestSuite(checker=checker), 104 | )) 105 | -------------------------------------------------------------------------------- /src/ZODB/tests/testMappingStorage.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | import unittest 15 | from collections import namedtuple 16 | 17 | import ZODB.MappingStorage 18 | import ZODB.tests.hexstorage 19 | from ZODB.tests import BasicStorage 20 | from ZODB.tests import HistoryStorage 21 | from ZODB.tests import IteratorStorage 22 | from ZODB.tests import MTStorage 23 | from ZODB.tests import PackableStorage 24 | from ZODB.tests import RevisionStorage 25 | from ZODB.tests import StorageTestBase 26 | from ZODB.tests import Synchronization 27 | 28 | 29 | class MappingStorageTests( 30 | StorageTestBase.StorageTestBase, 31 | BasicStorage.BasicStorage, 32 | 33 | HistoryStorage.HistoryStorage, 34 | IteratorStorage.ExtendedIteratorStorage, 35 | IteratorStorage.IteratorStorage, 36 | MTStorage.MTStorage, 37 | PackableStorage.PackableStorageWithOptionalGC, 38 | RevisionStorage.RevisionStorage, 39 | Synchronization.SynchronizedStorage, 40 | ): 41 | 42 | def setUp(self): 43 | StorageTestBase.StorageTestBase.setUp(self, ) 44 | self._storage = ZODB.MappingStorage.MappingStorage() 45 | 46 | def testOversizeNote(self): 47 | # This base class test checks for the common case where a storage 48 | # doesnt support huge transaction metadata. This storage doesnt 49 | # have this limit, so we inhibit this test here. 50 | pass 51 | 52 | def testLoadBeforeUndo(self): 53 | pass # we don't support undo yet 54 | testUndoZombie = testLoadBeforeUndo 55 | 56 | 57 | class MappingStorageHexTests(MappingStorageTests): 58 | 59 | def setUp(self): 60 | StorageTestBase.StorageTestBase.setUp(self, ) 61 | self._storage = ZODB.tests.hexstorage.HexStorage( 62 | ZODB.MappingStorage.MappingStorage()) 63 | 64 | 65 | MockTransaction = namedtuple( 66 | 'transaction', 67 | ['user', 'description', 'extension'] 68 | ) 69 | 70 | 71 | class MappingStorageTransactionRecordTests(unittest.TestCase): 72 | 73 | def setUp(self): 74 | self._transaction_record = ZODB.MappingStorage.TransactionRecord( 75 | 0, 76 | MockTransaction('user', 'description', 'extension'), 77 | '' 78 | ) 79 | 80 | def test_set__extension(self): 81 | self._transaction_record._extension = 'new' 82 | self.assertEqual(self._transaction_record.extension, 'new') 83 | 84 | def test_get__extension(self): 85 | self.assertEqual( 86 | self._transaction_record.extension, 87 | self._transaction_record._extension 88 | ) 89 | -------------------------------------------------------------------------------- /src/ZODB/tests/testPersistentMapping.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Verify that PersistentMapping works with old versions of Zope. 15 | 16 | The comments in PersistentMapping.py address the issue in some detail. 17 | The pickled form of a PersistentMapping must use _container to store 18 | the actual mapping, because old versions of Zope used this attribute. 19 | If the new code doesn't generate pickles that are consistent with the 20 | old code, developers will have a hard time testing the new code. 21 | """ 22 | 23 | import sys 24 | import unittest 25 | 26 | import ZODB 27 | from ZODB.Connection import TransactionMetaData 28 | from ZODB.MappingStorage import MappingStorage 29 | 30 | 31 | # This pickle contains a persistent mapping pickle created from the 32 | # old code. 33 | pickle = ('((U\x0bPersistenceq\x01U\x11PersistentMappingtq\x02Nt.}q\x03U\n' 34 | '_containerq\x04}q\x05U\x07versionq\x06U\x03oldq\x07ss.\n') 35 | 36 | 37 | class PMTests(unittest.TestCase): 38 | 39 | def testOldStyleRoot(self): 40 | # The Persistence module doesn't exist in Zope3's idea of what ZODB 41 | # is, but the global `pickle` references it explicitly. So just 42 | # bail if Persistence isn't available. 43 | try: 44 | import Persistence # noqa: F401 'Persistence' imported but unused 45 | except ImportError: 46 | return 47 | # insert the pickle in place of the root 48 | s = MappingStorage() 49 | t = TransactionMetaData() 50 | s.tpc_begin(t) 51 | s.store('\000' * 8, None, pickle, '', t) 52 | s.tpc_vote(t) 53 | s.tpc_finish(t) 54 | 55 | db = ZODB.DB(s) 56 | # If the root can be loaded successfully, we should be okay. 57 | r = db.open().root() 58 | # But make sure it looks like a new mapping 59 | self.assertTrue(hasattr(r, 'data')) 60 | self.assertTrue(not hasattr(r, '_container')) 61 | 62 | def testBackwardCompat(self): 63 | # Verify that the sanest of the ZODB 3.2 dotted paths still works. 64 | from persistent.mapping import PersistentMapping as newPath 65 | 66 | from ZODB.PersistentMapping import PersistentMapping as oldPath 67 | 68 | self.assertTrue(oldPath is newPath) 69 | 70 | def testBasicOps(self): 71 | from persistent.mapping import PersistentMapping 72 | m = PersistentMapping({'x': 1}, a=2, b=3) 73 | m['name'] = 'bob' 74 | self.assertEqual(m['name'], "bob") 75 | self.assertEqual(m.get('name', 42), "bob") 76 | self.assertTrue('name' in m) 77 | 78 | try: 79 | m['fred'] 80 | except KeyError: 81 | pass 82 | else: 83 | self.fail("expected KeyError") 84 | self.assertTrue('fred' not in m) 85 | self.assertEqual(m.get('fred'), None) 86 | self.assertEqual(m.get('fred', 42), 42) 87 | 88 | keys = sorted(m.keys()) 89 | self.assertEqual(keys, ['a', 'b', 'name', 'x']) 90 | 91 | values = set(m.values()) 92 | self.assertEqual(values, {1, 2, 3, 'bob'}) 93 | 94 | items = sorted(m.items()) 95 | self.assertEqual(items, 96 | [('a', 2), ('b', 3), ('name', 'bob'), ('x', 1)]) 97 | 98 | # PersistentMapping didn't have an __iter__ method before ZODB 3.4.2. 99 | # Check that it plays well now with the Python iteration protocol. 100 | def testIteration(self): 101 | from persistent.mapping import PersistentMapping 102 | m = PersistentMapping({'x': 1}, a=2, b=3) 103 | m['name'] = 'bob' 104 | 105 | def check(keylist): 106 | keylist.sort() 107 | self.assertEqual(keylist, ['a', 'b', 'name', 'x']) 108 | 109 | check(list(m)) 110 | check([key for key in m]) 111 | 112 | i = iter(m) 113 | keylist = [] 114 | while 1: 115 | try: 116 | key = next(i) 117 | except StopIteration: 118 | break 119 | keylist.append(key) 120 | check(keylist) 121 | 122 | 123 | def find_global(modulename, classname): 124 | """Helper for this test suite to get special PersistentMapping""" 125 | 126 | if classname == "PersistentMapping": 127 | class PersistentMapping: 128 | def __setstate__(self, state): 129 | self.__dict__.update(state) 130 | return PersistentMapping 131 | else: 132 | __import__(modulename) 133 | mod = sys.modules[modulename] 134 | return getattr(mod, classname) 135 | -------------------------------------------------------------------------------- /src/ZODB/tests/testThreadedShutdown.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time 3 | 4 | import ZODB 5 | 6 | 7 | class ZODBClientThread(threading.Thread): 8 | 9 | sleep_time = 3 10 | 11 | def __init__(self, db, test): 12 | threading.Thread.__init__(self) 13 | self._exc_info = None 14 | self.daemon = True 15 | self.db = db 16 | self.test = test 17 | self.event = threading.Event() 18 | 19 | def run(self): 20 | conn = self.db.open() 21 | conn.sync() 22 | self.event.set() 23 | time.sleep(self.sleep_time) 24 | 25 | # conn.close calls self.transaction_manager.unregisterSynch(self) 26 | # and this succeeds. 27 | conn.close() 28 | 29 | 30 | class ShutdownTest(ZODB.tests.util.TestCase): 31 | 32 | def setUp(self): 33 | # Our default transaction manager is 34 | # transaction._manager.ThreadTransactionManager 35 | # so no need to set it. 36 | ZODB.tests.util.TestCase.setUp(self) 37 | self._storage = ZODB.FileStorage.FileStorage( 38 | 'ZODBTests.fs', create=1) 39 | self._db = ZODB.DB(self._storage) 40 | 41 | def tearDown(self): 42 | ZODB.tests.util.TestCase.tearDown(self) 43 | 44 | def test_shutdown(self): 45 | client_thread = ZODBClientThread(self._db, self) 46 | client_thread.start() 47 | client_thread.event.wait() 48 | # calls conn._release_resources, that calls conn.close(), 49 | # that calls conn.transaction_manager.unregisterSynch(self), 50 | # but from a different thread, so transaction_manager._synchs 51 | # have different contents. 52 | self._db.close() 53 | 54 | # Be sure not to 'leak' the running thread; if it hasn't 55 | # finished after this, something went very wrong 56 | client_thread.join(client_thread.sleep_time + 10) 57 | if client_thread.is_alive(): 58 | self.fail("Client thread failed to close connection") 59 | -------------------------------------------------------------------------------- /src/ZODB/tests/test_TransactionMetaData.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | import unittest 15 | import warnings 16 | 17 | from .._compat import dumps 18 | from .._compat import loads 19 | from ..Connection import TransactionMetaData 20 | 21 | 22 | class TransactionMetaDataTests(unittest.TestCase): 23 | 24 | def test_basic(self): 25 | extension = dict(foo='FOO') 26 | t = TransactionMetaData('user\x80', 'description\x80', extension) 27 | self.assertEqual(t.user, b'user\xc2\x80') 28 | self.assertEqual(t.description, b'description\xc2\x80') 29 | self.assertEqual(t.extension, extension) 30 | self.assertEqual(loads(t.extension_bytes), extension) 31 | with warnings.catch_warnings(record=True) as w: 32 | warnings.simplefilter("always") 33 | self.assertEqual(t._extension, t.extension) 34 | self.assertEqual(len(w), 1) 35 | self.assertTrue(issubclass(w[-1].category, DeprecationWarning)) 36 | self.assertTrue("_extension is deprecated" in str(w[-1].message)) 37 | 38 | def test_basic_no_encoding(self): 39 | extension = dict(foo='FOO') 40 | extension_bytes = dumps(extension) 41 | t = TransactionMetaData(b'user', b'description', extension_bytes) 42 | self.assertEqual(t.user, b'user') 43 | self.assertEqual(t.description, b'description') 44 | self.assertEqual(t.extension, extension) 45 | with warnings.catch_warnings(): 46 | warnings.simplefilter("ignore") 47 | self.assertEqual(t._extension, t.extension) 48 | self.assertIs(t.extension_bytes, extension_bytes) 49 | 50 | def test_constructor_default_args(self): 51 | t = TransactionMetaData() 52 | self.assertEqual(t.user, b'') 53 | self.assertEqual(t.description, b'') 54 | self.assertEqual(t.extension, {}) 55 | with warnings.catch_warnings(): 56 | warnings.simplefilter("ignore") 57 | self.assertEqual(t._extension, t.extension) 58 | 59 | def test_set_extension(self): 60 | data = {} 61 | t = TransactionMetaData('', '', data) 62 | self.assertEqual(t.user, b'') 63 | self.assertEqual(t.description, b'') 64 | self.assertIs(t.extension, data) 65 | with warnings.catch_warnings(): 66 | warnings.simplefilter("ignore") 67 | self.assertEqual(t._extension, t.extension) 68 | self.assertEqual(t.extension_bytes, b'') 69 | 70 | for name in 'extension', '_extension': 71 | data = {name: name + 'foo'} 72 | setattr(t, name, data) 73 | self.assertIs(t.extension, data) 74 | self.assertIs(t._extension, t.extension) 75 | extension_bytes = t.extension_bytes 76 | self.assertEqual(loads(extension_bytes), data) 77 | empty = {} 78 | setattr(t, name, empty) 79 | self.assertIs(t.extension, empty) 80 | self.assertIs(t._extension, t.extension) 81 | self.assertEqual(t.extension_bytes, b'') 82 | 83 | def test_used_by_connection(self): 84 | import ZODB 85 | from ZODB.MappingStorage import MappingStorage 86 | 87 | class Storage(MappingStorage): 88 | def tpc_begin(self, transaction): 89 | self.test_transaction = transaction 90 | return MappingStorage.tpc_begin(self, transaction) 91 | 92 | storage = Storage() 93 | conn = ZODB.connection(storage) 94 | with conn.transaction_manager as t: 95 | t.user = 'user\x80' 96 | t.description = 'description\x80' 97 | t.setExtendedInfo('foo', 'FOO') 98 | conn.root.x = 1 99 | 100 | t = storage.test_transaction 101 | self.assertEqual(t.__class__, TransactionMetaData) 102 | self.assertEqual(t.user, b'user\xc2\x80') 103 | self.assertEqual(t.description, b'description\xc2\x80') 104 | self.assertEqual(t.extension, dict(foo='FOO')) 105 | 106 | def test_data(self): 107 | t = TransactionMetaData() 108 | 109 | # Can't get data that wasn't set: 110 | with self.assertRaises(KeyError) as c: 111 | t.data(self) 112 | self.assertEqual(c.exception.args, (self,)) 113 | 114 | data = dict(a=1) 115 | t.set_data(self, data) 116 | self.assertEqual(t.data(self), data) 117 | 118 | # Can't get something we haven't stored. 119 | with self.assertRaises(KeyError) as c: 120 | t.data(data) 121 | self.assertEqual(c.exception.args, (data,)) 122 | -------------------------------------------------------------------------------- /src/ZODB/tests/test_doctest_files.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | import doctest 15 | import unittest 16 | 17 | 18 | __test__ = dict( 19 | cross_db_refs_to_blank_db_name=""" 20 | 21 | There was a bug that caused bad refs to be generated is a database 22 | name was blank. 23 | 24 | >>> import ZODB.tests.util, persistent.mapping, transaction 25 | >>> dbs = {} 26 | >>> db1 = ZODB.tests.util.DB(database_name='', databases=dbs) 27 | >>> db2 = ZODB.tests.util.DB(database_name='2', databases=dbs) 28 | >>> conn1 = db1.open() 29 | >>> conn2 = conn1.get_connection('2') 30 | >>> for i in range(10): 31 | ... conn1.root()[i] = persistent.mapping.PersistentMapping() 32 | ... transaction.commit() 33 | >>> conn2.root()[0] = conn1.root()[9] 34 | >>> transaction.commit() 35 | >>> conn2.root()._p_deactivate() 36 | >>> conn2.root()[0] is conn1.root()[9] 37 | True 38 | 39 | >>> list(conn2.root()[0].keys()) 40 | [] 41 | 42 | >>> db2.close() 43 | >>> db1.close() 44 | """, 45 | ) 46 | 47 | 48 | def test_suite(): 49 | suite = unittest.TestSuite() 50 | suite.addTest(doctest.DocFileSuite("dbopen.txt", 51 | "multidb.txt", 52 | "synchronizers.txt", 53 | )) 54 | suite.addTest(doctest.DocTestSuite()) 55 | return suite 56 | -------------------------------------------------------------------------------- /src/ZODB/tests/test_fsdump.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2005 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | r""" 15 | fsdump test 16 | =========== 17 | 18 | Let's get a path to work with first. 19 | 20 | >>> path = 'Data.fs' 21 | 22 | More imports. 23 | 24 | >>> import ZODB 25 | >>> from ZODB.FileStorage import FileStorage 26 | >>> import transaction as txn 27 | >>> from BTrees.OOBTree import OOBTree 28 | >>> from ZODB.FileStorage.fsdump import fsdump # we're testing this 29 | 30 | Create an empty FileStorage. 31 | 32 | >>> st = FileStorage(path) 33 | 34 | For empty DB fsdump() output definitely empty: 35 | 36 | >>> fsdump(path) 37 | 38 | Create a root object and try again: 39 | 40 | >>> db = ZODB.DB(st) # yes, that creates a root object! 41 | >>> fsdump(path) #doctest: +ELLIPSIS 42 | Trans #00000 tid=... time=... offset= 43 | status=' ' user=b'' description=b'initial database creation' 44 | data #00000 oid=0000000000000000 size= class=persistent.mapping.PersistentMapping 45 | 46 | Now we see first transaction with root object. 47 | 48 | Let's add a BTree: 49 | 50 | >>> root = db.open().root() 51 | >>> root['tree'] = OOBTree() 52 | >>> txn.get().note(u'added an OOBTree') 53 | >>> txn.get().commit() 54 | >>> fsdump(path) #doctest: +ELLIPSIS 55 | Trans #00000 tid=... time=... offset= 56 | status=' ' user=b'' description=b'initial database creation' 57 | data #00000 oid=0000000000000000 size= class=persistent.mapping.PersistentMapping 58 | Trans #00001 tid=... time=... offset= 59 | status=' ' user=b'' description=b'added an OOBTree' 60 | data #00000 oid=0000000000000000 size= class=persistent.mapping.PersistentMapping 61 | data #00001 oid=0000000000000001 size= class=BTrees.OOBTree.OOBTree... 62 | 63 | Now we see two transactions and two changed objects. 64 | 65 | Clean up. 66 | 67 | >>> db.close() 68 | """ # noqa: E501 line too long 69 | 70 | import doctest 71 | import re 72 | 73 | import zope.testing.setupstack 74 | from zope.testing import renormalizing 75 | 76 | import ZODB.tests.util 77 | 78 | 79 | checker = renormalizing.RENormalizing([ 80 | # Normalizing this makes diffs easier to read 81 | (re.compile(r'\btid=[0-9a-f]+\b'), 'tid=...'), 82 | (re.compile(r'\b\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d+\b'), '...'), 83 | # Python 3 produces larger pickles, even when we use zodbpickle :( 84 | # this changes all the offsets and sizes 85 | (re.compile(r'\bsize=[0-9]+\b'), 'size='), 86 | (re.compile(r'\boffset=[0-9]+\b'), 'offset='), 87 | ]) 88 | 89 | 90 | def test_suite(): 91 | return doctest.DocTestSuite( 92 | setUp=zope.testing.setupstack.setUpDirectory, 93 | tearDown=ZODB.tests.util.tearDown, 94 | optionflags=doctest.REPORT_NDIFF, 95 | checker=ZODB.tests.util.checker + checker) 96 | -------------------------------------------------------------------------------- /src/ZODB/tests/test_mvccadapter.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2017 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | 15 | import unittest 16 | 17 | from ZODB import mvccadapter 18 | 19 | 20 | class TestBase(unittest.TestCase): 21 | 22 | def test_getattr_does_not_hide_exceptions(self): 23 | class TheException(Exception): 24 | pass 25 | 26 | class RaisesOnAccess: 27 | 28 | @property 29 | def thing(self): 30 | raise TheException() 31 | 32 | base = mvccadapter.Base(RaisesOnAccess()) 33 | base._copy_methods = ('thing',) 34 | 35 | with self.assertRaises(TheException): 36 | getattr(base, 'thing') 37 | 38 | def test_getattr_raises_if_missing(self): 39 | base = mvccadapter.Base(self) 40 | base._copy_methods = ('thing',) 41 | 42 | with self.assertRaises(AttributeError): 43 | getattr(base, 'thing') 44 | 45 | 46 | class TestHistoricalStorageAdapter(unittest.TestCase): 47 | 48 | def test_forwards_release(self): 49 | class Base: 50 | released = False 51 | 52 | def release(self): 53 | self.released = True 54 | 55 | base = Base() 56 | adapter = mvccadapter.HistoricalStorageAdapter(base, None) 57 | 58 | adapter.release() 59 | 60 | self.assertTrue(base.released) 61 | -------------------------------------------------------------------------------- /src/ZODB/tests/test_prefetch.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import ZODB 4 | from ZODB.utils import u64 5 | from ZODB.utils import z64 6 | 7 | from .MVCCMappingStorage import MVCCMappingStorage 8 | 9 | 10 | class PrefetchTests(unittest.TestCase): 11 | 12 | def test_prefetch(self): 13 | db = ZODB.DB(None) 14 | 15 | fetched = [] 16 | 17 | def prefetch(oids, tid): 18 | fetched.append((list(map(u64, oids)), tid)) 19 | 20 | db.storage.prefetch = prefetch 21 | 22 | with db.transaction() as conn: 23 | for i in range(10): 24 | conn.root()[i] = conn.root().__class__() 25 | 26 | conn = db.open() 27 | conn.prefetch(z64) 28 | conn.prefetch([z64]) 29 | conn.prefetch(conn.root()) 30 | 31 | conn.prefetch(z64, (conn.root()[i] for i in range(3)), conn.root()[3]) 32 | 33 | self.assertEqual(fetched, 34 | [([0], conn._storage._start), 35 | ([0], conn._storage._start), 36 | ([0], conn._storage._start), 37 | ([0, 1, 2, 3, 4], conn._storage._start), 38 | ]) 39 | 40 | db.close() 41 | 42 | def test_prefetch_optional(self): 43 | conn = ZODB.connection(None) 44 | conn.prefetch(z64) 45 | conn.prefetch([z64]) 46 | conn.prefetch(conn.root()) 47 | conn.prefetch(z64, [z64]) 48 | conn.prefetch(z64, [z64], conn.root()) 49 | conn.close() 50 | 51 | def test_prefetch_optional_imvcc(self): 52 | conn = ZODB.connection(MVCCMappingStorage()) 53 | conn.prefetch(z64) 54 | conn.prefetch([z64]) 55 | conn.prefetch(conn.root()) 56 | conn.prefetch(z64, [z64]) 57 | conn.prefetch(z64, [z64], conn.root()) 58 | conn.close() 59 | -------------------------------------------------------------------------------- /src/ZODB/tests/test_racetest.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2019 - 2023 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | from time import sleep 15 | from types import SimpleNamespace 16 | from unittest import TestCase 17 | 18 | from .racetest import TestWorkGroup 19 | 20 | 21 | class TestWorkGroupTests(TestCase): 22 | def setUp(self): 23 | self._failed = failed = [] 24 | case_mockup = SimpleNamespace(fail=failed.append) 25 | self.tg = TestWorkGroup(case_mockup) 26 | 27 | @property 28 | def failed(self): 29 | return "\n\n".join(self._failed) 30 | 31 | def test_success(self): 32 | tg = self.tg 33 | tg.go(tg_test_function) 34 | tg.wait(10) 35 | self.assertEqual(self.failed, "") 36 | 37 | def test_failure1(self): 38 | tg = self.tg 39 | tg.go(tg_test_function, T_FAIL) 40 | tg.wait(10) 41 | self.assertEqual(self.failed, "T0 failed") 42 | 43 | def test_failure1_okmany(self): 44 | tg = self.tg 45 | tg.go(tg_test_function, T_SUCCESS) 46 | tg.go(tg_test_function, T_SUCCESS) 47 | tg.go(tg_test_function, T_SUCCESS) 48 | tg.go(tg_test_function, T_FAIL) 49 | tg.wait(10) 50 | self.assertEqual(self.failed, "T3 failed") 51 | 52 | def test_failure_many(self): 53 | tg = self.tg 54 | tg.go(tg_test_function, T_FAIL) 55 | tg.go(tg_test_function, T_SUCCESS) 56 | tg.go(tg_test_function, T_FAIL) 57 | tg.go(tg_test_function, T_SUCCESS) 58 | tg.go(tg_test_function, T_FAIL) 59 | tg.wait(10) 60 | self.assertIn("T0 failed", self.failed) 61 | self.assertIn("T2 failed", self.failed) 62 | self.assertIn("T4 failed", self.failed) 63 | self.assertNotIn("T1 failed", self.failed) 64 | self.assertNotIn("T3 failed", self.failed) 65 | 66 | def test_exception(self): 67 | tg = self.tg 68 | tg.go(tg_test_function, T_EXC) 69 | tg.wait(10) 70 | self.assertIn("Unhandled exception", self.failed) 71 | self.assertIn("in thread T0", self.failed) 72 | 73 | def test_timeout(self): 74 | tg = self.tg 75 | tg.go(tg_test_function, T_SLOW) 76 | tg.wait(0.1) 77 | self.assertEqual(self.failed, 78 | "test did not finish within 0.1 seconds") 79 | 80 | def test_thread_unfinished(self): 81 | tg = self.tg 82 | tg.go(tg_test_function, T_SLOW) 83 | tg.go(tg_test_function, T_SLOW, 2) 84 | tg.go(tg_test_function, T_SLOW, wait_time=2) 85 | tg.wait(0.1) 86 | self.assertEqual(self.failed, 87 | "test did not finish within 0.1 seconds\n\n" 88 | "threads did not finish: ['T2']") 89 | 90 | 91 | T_SUCCESS = 0 92 | T_SLOW = 1 93 | T_EXC = 3 94 | T_FAIL = 4 95 | 96 | 97 | def tg_test_function(tg, tx, mode=T_SUCCESS, waits=1, wait_time=0.2): 98 | if mode == T_SUCCESS: 99 | return 100 | if mode == T_FAIL: 101 | tg.fail("T%d failed" % tx) 102 | return 103 | if mode == T_EXC: 104 | raise ValueError(str(tx)) 105 | assert mode == T_SLOW 106 | while waits: 107 | waits -= 1 108 | if tg.failed(): 109 | return 110 | sleep(wait_time) 111 | -------------------------------------------------------------------------------- /src/ZODB/tests/test_storage.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """A storage used for unittests. 15 | 16 | The primary purpose of this module is to have a minimal multi-version 17 | storage to use for unit tests. MappingStorage isn't sufficient. 18 | Since even a minimal storage has some complexity, we run standard 19 | storage tests against the test storage. 20 | """ 21 | import bisect 22 | 23 | from ZODB import POSException 24 | from ZODB.BaseStorage import BaseStorage 25 | from ZODB.tests import BasicStorage 26 | from ZODB.tests import MTStorage 27 | from ZODB.tests import RevisionStorage 28 | from ZODB.tests import StorageTestBase 29 | from ZODB.tests import Synchronization 30 | from ZODB.utils import z64 31 | 32 | 33 | class Transaction: 34 | """Hold data for current transaction for MinimalMemoryStorage.""" 35 | 36 | def __init__(self, tid): 37 | self.index = {} 38 | self.tid = tid 39 | 40 | def store(self, oid, data): 41 | self.index[(oid, self.tid)] = data 42 | 43 | def cur(self): 44 | return dict.fromkeys([oid for oid, tid in self.index.keys()], self.tid) 45 | 46 | 47 | class MinimalMemoryStorage(BaseStorage): 48 | """Simple in-memory storage that supports revisions. 49 | 50 | This storage is needed to test multi-version concurrency control. 51 | It is similar to MappingStorage, but keeps multiple revisions. It 52 | does not support versions. It doesn't implement operations like 53 | pack(), because they aren't necessary for testing. 54 | """ 55 | 56 | def __init__(self): 57 | super().__init__("name") 58 | # _index maps oid, tid pairs to data records 59 | self._index = {} 60 | # _cur maps oid to current tid 61 | self._cur = {} 62 | 63 | self._ltid = z64 64 | 65 | def isCurrent(self, oid, serial): 66 | return serial == self._cur[oid] 67 | 68 | def hook(self, oid, tid, version): 69 | # A hook for testing 70 | pass 71 | 72 | def __len__(self): 73 | return len(self._index) 74 | 75 | def _clear_temp(self): 76 | pass 77 | 78 | def load(self, oid, version=''): 79 | assert version == '' 80 | with self._lock: 81 | assert not version 82 | tid = self._cur[oid] 83 | self.hook(oid, tid, '') 84 | return self._index[(oid, tid)], tid 85 | 86 | def _begin(self, tid, u, d, e): 87 | self._txn = Transaction(tid) 88 | 89 | def store(self, oid, serial, data, v, txn): 90 | if txn is not self._transaction: 91 | raise POSException.StorageTransactionError(self, txn) 92 | assert not v 93 | if self._cur.get(oid) != serial: 94 | if not (serial is None or self._cur.get(oid) in [None, z64]): 95 | raise POSException.ConflictError( 96 | oid=oid, serials=(self._cur.get(oid), serial), data=data) 97 | self._txn.store(oid, data) 98 | return self._tid 99 | 100 | def _abort(self): 101 | del self._txn 102 | 103 | def _finish(self, tid, u, d, e): 104 | with self._lock: 105 | self._index.update(self._txn.index) 106 | self._cur.update(self._txn.cur()) 107 | self._ltid = self._tid 108 | 109 | def loadBefore(self, the_oid, the_tid): 110 | # It's okay if loadBefore() is really expensive, because this 111 | # storage is just used for testing. 112 | with self._lock: 113 | tids = [tid for oid, tid in self._index if oid == the_oid] 114 | if not tids: 115 | raise KeyError(the_oid) 116 | tids.sort() 117 | i = bisect.bisect_left(tids, the_tid) - 1 118 | if i == -1: 119 | return None 120 | tid = tids[i] 121 | j = i + 1 122 | if j == len(tids): 123 | end_tid = None 124 | else: 125 | end_tid = tids[j] 126 | 127 | self.hook(the_oid, self._cur[the_oid], '') 128 | 129 | return self._index[(the_oid, tid)], tid, end_tid 130 | 131 | def loadSerial(self, oid, serial): 132 | return self._index[(oid, serial)] 133 | 134 | def close(self): 135 | pass 136 | 137 | cleanup = close 138 | 139 | 140 | class MinimalTestSuite(StorageTestBase.StorageTestBase, 141 | BasicStorage.BasicStorage, 142 | MTStorage.MTStorage, 143 | Synchronization.SynchronizedStorage, 144 | RevisionStorage.RevisionStorage, 145 | ): 146 | 147 | def setUp(self): 148 | StorageTestBase.StorageTestBase.setUp(self) 149 | self._storage = MinimalMemoryStorage() 150 | 151 | # we don't implement undo 152 | 153 | def testLoadBeforeUndo(self): 154 | pass 155 | -------------------------------------------------------------------------------- /src/ZODB/tests/testdocumentation.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | import doctest 15 | import os 16 | import unittest 17 | from os.path import join 18 | 19 | import manuel.capture 20 | import manuel.doctest 21 | import manuel.testing 22 | import zope.testing.module 23 | 24 | import ZODB 25 | 26 | 27 | def setUp(test): 28 | test.globs.update( 29 | ZODB=ZODB, 30 | ) 31 | zope.testing.module.setUp(test) 32 | 33 | 34 | def tearDown(test): 35 | zope.testing.module.tearDown(test) 36 | 37 | 38 | def test_suite(): 39 | base, src = os.path.split(os.path.dirname(os.path.dirname(ZODB.__file__))) 40 | assert src == 'src', src 41 | base = join(base, 'docs') 42 | guide = join(base, 'guide') 43 | reference = join(base, 'reference') 44 | 45 | return unittest.TestSuite(( 46 | manuel.testing.TestSuite( 47 | manuel.doctest.Manuel( 48 | optionflags=doctest.IGNORE_EXCEPTION_DETAIL, 49 | ) + manuel.capture.Manuel(), 50 | join(guide, 'writing-persistent-objects.rst'), 51 | join(guide, 'install-and-run.rst'), 52 | join(guide, 'transactions-and-threading.rst'), 53 | join(reference, 'zodb.rst'), 54 | join(reference, 'storages.rst'), 55 | setUp=setUp, tearDown=tearDown, 56 | ), 57 | )) 58 | -------------------------------------------------------------------------------- /src/ZODB/tests/testhistoricalconnections.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2007 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | import manuel.doctest 15 | import manuel.footnote 16 | import manuel.testing 17 | 18 | import ZODB.tests.util 19 | 20 | 21 | def test_suite(): 22 | return manuel.testing.TestSuite( 23 | manuel.doctest.Manuel(checker=ZODB.tests.util.checker) + 24 | manuel.footnote.Manuel(), 25 | '../historical_connections.rst', 26 | setUp=ZODB.tests.util.setUp, tearDown=ZODB.tests.util.tearDown, 27 | ) 28 | -------------------------------------------------------------------------------- /src/ZODB/tests/testpersistentclass.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | import doctest 15 | import sys 16 | import unittest 17 | 18 | import transaction 19 | 20 | import ZODB.persistentclass 21 | import ZODB.tests.util 22 | 23 | 24 | def class_with_circular_ref_to_self(): 25 | """ 26 | It should be possible for a class to reger to itself. 27 | 28 | >>> C = ZODB.persistentclass.PersistentMetaClass('C', (object,), {}) 29 | 30 | >>> C.me = C 31 | >>> db = ZODB.tests.util.DB() 32 | >>> conn = db.open() 33 | >>> conn.root()['C'] = C 34 | >>> transaction.commit() 35 | 36 | >>> conn2 = db.open() 37 | >>> C2 = conn2.root()['C'] 38 | >>> c = C2() 39 | >>> c.__class__.__name__ 40 | 'C' 41 | 42 | """ 43 | 44 | 45 | def test_new_ghost_w_persistent_class(): 46 | """ 47 | Peristent meta classes work with PickleCache.new_ghost: 48 | 49 | >>> import ZODB.persistentclass 50 | 51 | >>> PC = ZODB.persistentclass.PersistentMetaClass('PC', (object,), {}) 52 | 53 | >>> PC._p_oid 54 | >>> PC._p_jar 55 | >>> PC._p_serial 56 | >>> PC._p_changed 57 | False 58 | 59 | >>> import persistent 60 | >>> jar = object() 61 | >>> cache = persistent.PickleCache(jar, 10, 100) 62 | >>> cache.new_ghost(b'1', PC) 63 | 64 | >>> PC._p_oid == b'1' 65 | True 66 | >>> PC._p_jar is jar 67 | True 68 | >>> PC._p_serial 69 | >>> PC._p_changed 70 | False 71 | """ 72 | 73 | # XXX need to update files to get newer testing package 74 | 75 | 76 | class FakeModule: 77 | def __init__(self, name, dict): 78 | self.__dict__ = dict 79 | self.__name__ = name 80 | 81 | 82 | def setUp(test): 83 | ZODB.tests.util.setUp(test) 84 | test.globs['some_database'] = ZODB.tests.util.DB() 85 | module = FakeModule('ZODB.persistentclass_txt', test.globs) 86 | sys.modules[module.__name__] = module 87 | 88 | 89 | def tearDown(test): 90 | test.globs['some_database'].close() 91 | del sys.modules['ZODB.persistentclass_txt'] 92 | ZODB.tests.util.tearDown(test) 93 | 94 | 95 | def test_suite(): 96 | return unittest.TestSuite(( 97 | doctest.DocFileSuite( 98 | "../persistentclass.rst", 99 | setUp=setUp, tearDown=tearDown, 100 | checker=ZODB.tests.util.checker, 101 | optionflags=doctest.ELLIPSIS), 102 | doctest.DocTestSuite(setUp=setUp, tearDown=tearDown), 103 | )) 104 | -------------------------------------------------------------------------------- /src/ZODB/transact.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2003 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """Tools to simplify transactions within applications.""" 15 | 16 | import transaction 17 | 18 | from ZODB.POSException import ConflictError 19 | from ZODB.POSException import ReadConflictError 20 | 21 | 22 | def _commit(note): 23 | t = transaction.get() 24 | if note: 25 | t.note(note) 26 | t.commit() 27 | 28 | 29 | def transact(f, note=None, retries=5): 30 | """Returns transactional version of function argument f. 31 | 32 | Higher-order function that converts a regular function into 33 | a transactional function. The transactional function will 34 | retry up to retries time before giving up. If note, it will 35 | be added to the transaction metadata when it commits. 36 | 37 | The retries occur on ConflictErrors. If some other 38 | TransactionError occurs, the transaction will not be retried. 39 | """ 40 | 41 | # TODO: deal with ZEO disconnected errors? 42 | 43 | def g(*args, **kwargs): 44 | n = retries 45 | while n: 46 | n -= 1 47 | try: 48 | r = f(*args, **kwargs) 49 | except ReadConflictError: 50 | # the only way ReadConflictError can happen here is due to 51 | # simultaneous pack removing objects revision that f could try 52 | # to load. 53 | transaction.abort() 54 | if not n: 55 | raise 56 | continue 57 | try: 58 | _commit(note) 59 | except ConflictError: 60 | transaction.abort() 61 | if not n: 62 | raise 63 | continue 64 | return r 65 | raise RuntimeError("couldn't commit transaction") 66 | return g 67 | -------------------------------------------------------------------------------- /src/ZODB/valuedoc.py: -------------------------------------------------------------------------------- 1 | """Work around an issue with defining class attribute documentation. 2 | 3 | See http://stackoverflow.com/questions/9153473/sphinx-values-for-attributes-reported-as-none/39276413 4 | """ # noqa: E501 line too long 5 | 6 | 7 | class ValueDoc: 8 | 9 | def __init__(self, text): 10 | self.text = text 11 | 12 | def __repr__(self): 13 | return self.text 14 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | [tox] 4 | minversion = 3.18 5 | envlist = 6 | release-check 7 | lint 8 | py37 9 | py38 10 | py39 11 | py310 12 | py311 13 | py312 14 | pypy3 15 | docs 16 | coverage 17 | py310-pure 18 | 19 | [testenv] 20 | usedevelop = true 21 | package = wheel 22 | wheel_build_env = .pkg 23 | deps = 24 | setenv = 25 | ZOPE_INTERFACE_STRICT_IRO=1 26 | py312: VIRTUALENV_PIP=23.1.2 27 | py312: PIP_REQUIRE_VIRTUALENV=0 28 | commands = 29 | zope-testrunner --test-path=src -a 1000 {posargs:-vc} 30 | extras = 31 | test 32 | 33 | [testenv:py310-pure] 34 | basepython = python3.10 35 | setenv = 36 | PURE_PYTHON = 1 37 | [testenv:release-check] 38 | description = ensure that the distribution is ready to release 39 | basepython = python3 40 | skip_install = true 41 | deps = 42 | twine 43 | build 44 | check-manifest 45 | check-python-versions >= 0.20.0 46 | wheel 47 | commands_pre = 48 | commands = 49 | check-manifest 50 | check-python-versions --only setup.py,tox.ini,.github/workflows/tests.yml 51 | python -m build --sdist --no-isolation 52 | twine check dist/* 53 | 54 | [testenv:lint] 55 | basepython = python3 56 | skip_install = true 57 | deps = 58 | isort 59 | flake8 60 | commands = 61 | isort --check-only --diff {toxinidir}/src {toxinidir}/setup.py 62 | flake8 src setup.py 63 | 64 | [testenv:isort-apply] 65 | basepython = python3 66 | skip_install = true 67 | commands_pre = 68 | deps = 69 | isort 70 | commands = 71 | isort {toxinidir}/src {toxinidir}/setup.py [] 72 | 73 | [testenv:docs] 74 | basepython = python3 75 | skip_install = false 76 | extras = 77 | docs 78 | commands_pre = 79 | commands = 80 | sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html 81 | 82 | [testenv:coverage] 83 | basepython = python3 84 | allowlist_externals = 85 | mkdir 86 | deps = 87 | coverage 88 | commands = 89 | mkdir -p {toxinidir}/parts/htmlcov 90 | coverage run -m zope.testrunner --test-path=src {posargs:-vc} 91 | coverage html --ignore-errors 92 | coverage report --ignore-errors --show-missing --fail-under=80 93 | 94 | [coverage:run] 95 | branch = True 96 | source = ZODB 97 | 98 | [coverage:report] 99 | precision = 2 100 | exclude_lines = 101 | pragma: no cover 102 | pragma: nocover 103 | except ImportError: 104 | raise NotImplementedError 105 | if __name__ == '__main__': 106 | self.fail 107 | raise AssertionError 108 | 109 | [coverage:html] 110 | directory = parts/htmlcov 111 | --------------------------------------------------------------------------------