├── src └── ZEO │ ├── asyncio │ ├── __init__.py │ ├── compat.py │ ├── README.rst │ ├── marshal.py │ ├── _smp.pyx │ └── base.py │ ├── scripts │ ├── __init__.py │ ├── tests.py │ ├── timeout.py │ ├── README.txt │ ├── parsezeolog.py │ └── zeoup.py │ ├── version.txt │ ├── tests │ ├── component.xml │ ├── threaded.py │ ├── __init__.py │ ├── server.pem.csr │ ├── test_marshal.py │ ├── test_sync.py │ ├── new_addr.test │ ├── client.pem │ ├── protocols.test │ ├── serverpw.pem │ ├── server.pem │ ├── client_key.pem │ ├── server_key.pem │ ├── serverpw_key.pem │ ├── test_client_credentials.py │ ├── Cache.py │ ├── utils.py │ ├── testTransactionBuffer.py │ ├── dynamic_server_ports.test │ ├── TestThread.py │ ├── racetest.py │ ├── servertesting.py │ ├── client-config.test │ ├── testZEOOptions.py │ ├── zdoptions.test │ ├── stress.py │ ├── testConversionSupport.py │ ├── invalidation-age.txt │ ├── zeo-fan-out.test │ ├── testConfig.py │ ├── drop_cache_rather_than_verify.txt │ ├── ThreadTests.py │ ├── test_client_side_conflict_resolution.py │ ├── testZEOServer.py │ └── zeo_blob_cache.test │ ├── zeoctl.xml │ ├── zeoctl.py │ ├── _compat.py │ ├── schema.xml │ ├── Exceptions.py │ ├── util.py │ ├── shortrepr.py │ ├── zconfig.py │ ├── monitor.py │ ├── __init__.py │ ├── TransactionBuffer.py │ ├── interfaces.py │ ├── nagios.rst │ ├── server.xml │ └── nagios.py ├── COPYRIGHT.txt ├── docs ├── changelog.rst ├── nagios.rst ├── requirements.txt ├── reference.rst ├── index.rst ├── client-cache.rst ├── protocol.rst ├── blob-nfs.rst └── introduction.rst ├── COPYING ├── .gitignore ├── setup.cfg ├── MANIFEST.in ├── log.ini ├── pyproject.toml ├── .readthedocs.yaml ├── .pre-commit-config.yaml ├── CONTRIBUTING.md ├── README.rst ├── .github └── workflows │ ├── pre-commit.yml │ └── tests.yml ├── .editorconfig ├── .meta.toml ├── LICENSE.txt ├── tox.ini └── setup.py /src/ZEO/asyncio/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | -------------------------------------------------------------------------------- /src/ZEO/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | -------------------------------------------------------------------------------- /src/ZEO/version.txt: -------------------------------------------------------------------------------- 1 | 3.7.0b3 2 | -------------------------------------------------------------------------------- /COPYRIGHT.txt: -------------------------------------------------------------------------------- 1 | Zope Foundation and Contributors -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGES.rst 2 | -------------------------------------------------------------------------------- /docs/nagios.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../src/ZEO/nagios.rst 2 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | Sphinx 2 | repoze.sphinx.autointerface 3 | sphinx_rtd_theme > 1 -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | See: 2 | 3 | - the copyright notice in: COPYRIGHT.txt 4 | 5 | - The Zope Public License in LICENSE.txt 6 | -------------------------------------------------------------------------------- /src/ZEO/asyncio/compat.py: -------------------------------------------------------------------------------- 1 | try: 2 | from uvloop import new_event_loop 3 | except ModuleNotFoundError: 4 | from asyncio import new_event_loop 5 | -------------------------------------------------------------------------------- /src/ZEO/tests/component.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 9 |
10 | 11 | 12 | -------------------------------------------------------------------------------- /docs/reference.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Reference 3 | ========= 4 | 5 | Client & server 6 | =============== 7 | 8 | .. autofunction:: ZEO.server 9 | 10 | .. autofunction:: ZEO.client 11 | 12 | .. autoclass:: ZEO.ClientStorage.ClientStorage 13 | :members: 14 | :show-inheritance: 15 | 16 | .. autofunction:: ZEO.DB 17 | 18 | Exceptions 19 | ========== 20 | 21 | .. automodule:: ZEO.Exceptions 22 | :members: 23 | :show-inheritance: 24 | -------------------------------------------------------------------------------- /src/ZEO/tests/threaded.py: -------------------------------------------------------------------------------- 1 | """Test layer for threaded-server tests 2 | 3 | uvloop currently has a bug, 4 | https://github.com/MagicStack/uvloop/issues/39, that causes failure if 5 | multiprocessing and threaded servers are mixed in the same 6 | application, so we isolate the few threaded tests in their own layer. 7 | """ 8 | import ZODB.tests.util 9 | 10 | 11 | threaded_server_tests = ZODB.tests.util.MininalTestLayer( 12 | 'threaded_server_tests') 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | *.dll 4 | *.egg-info/ 5 | *.profraw 6 | *.pyc 7 | *.pyo 8 | *.so 9 | .coverage 10 | .coverage.* 11 | .eggs/ 12 | .installed.cfg 13 | .mr.developer.cfg 14 | .tox/ 15 | .vscode/ 16 | __pycache__/ 17 | bin/ 18 | build/ 19 | coverage.xml 20 | develop-eggs/ 21 | develop/ 22 | dist/ 23 | docs/_build 24 | eggs/ 25 | etc/ 26 | lib/ 27 | lib64 28 | log/ 29 | parts/ 30 | pyvenv.cfg 31 | testing.log 32 | var/ 33 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | 4 | [flake8] 5 | doctests = 1 6 | # E501 line too long 7 | per-file-ignores = 8 | src/ZEO/tests/test_cache.py: E501 9 | src/ZEO/asyncio/compat.py: F401 10 | 11 | [check-manifest] 12 | ignore = 13 | .editorconfig 14 | .meta.toml 15 | docs/_build/html/_sources/* 16 | 17 | [isort] 18 | force_single_line = True 19 | combine_as_imports = True 20 | sections = FUTURE,STDLIB,THIRDPARTY,ZOPE,FIRSTPARTY,LOCALFOLDER 21 | known_third_party = docutils, pkg_resources, pytz 22 | known_zope = 23 | known_first_party = 24 | default_section = ZOPE 25 | line_length = 79 26 | lines_after_imports = 2 27 | -------------------------------------------------------------------------------- /src/ZEO/tests/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | include *.md 4 | include *.rst 5 | include *.txt 6 | include buildout.cfg 7 | include tox.ini 8 | include .pre-commit-config.yaml 9 | 10 | recursive-include docs *.py 11 | recursive-include docs *.rst 12 | recursive-include docs *.txt 13 | recursive-include docs Makefile 14 | 15 | recursive-include src *.py 16 | include *.py 17 | include *.yaml 18 | include COPYING 19 | include log.ini 20 | recursive-include src *.csr 21 | recursive-include src *.pem 22 | recursive-include src *.pyx 23 | recursive-include src *.rst 24 | recursive-include src *.test 25 | recursive-include src *.txt 26 | recursive-include src *.xml 27 | -------------------------------------------------------------------------------- /log.ini: -------------------------------------------------------------------------------- 1 | # This file configures the logging module for the test harness: 2 | # critical errors are logged to testing.log; everything else is 3 | # ignored. 4 | 5 | # Documentation for the file format is at 6 | # http://www.red-dove.com/python_logging.html#config 7 | 8 | [logger_root] 9 | level=CRITICAL 10 | handlers=normal 11 | 12 | [handler_normal] 13 | class=FileHandler 14 | level=NOTSET 15 | formatter=common 16 | args=('testing.log', 'a') 17 | filename=testing.log 18 | mode=a 19 | 20 | [formatter_common] 21 | format=------ 22 | %(asctime)s %(levelname)s %(name)s %(message)s 23 | datefmt=%Y-%m-%dT%H:%M:%S 24 | 25 | [loggers] 26 | keys=root 27 | 28 | [handlers] 29 | keys=normal 30 | 31 | [formatters] 32 | keys=common 33 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | 4 | [build-system] 5 | requires = [ 6 | "setuptools == 75.8.2", 7 | "wheel", 8 | ] 9 | build-backend = "setuptools.build_meta" 10 | 11 | [tool.coverage.run] 12 | branch = true 13 | source = ["ZEO"] 14 | 15 | [tool.coverage.report] 16 | fail_under = 53 17 | precision = 2 18 | ignore_errors = true 19 | show_missing = true 20 | exclude_lines = [ 21 | "pragma: no cover", 22 | "pragma: nocover", 23 | "except ImportError:", 24 | "raise NotImplementedError", 25 | "if __name__ == '__main__':", 26 | "self.fail", 27 | "raise AssertionError", 28 | "raise unittest.Skip", 29 | ] 30 | 31 | [tool.coverage.html] 32 | directory = "parts/htmlcov" 33 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | # Read the Docs configuration file 4 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 5 | 6 | # Required 7 | version: 2 8 | 9 | # Set the version of Python and other tools you might need 10 | build: 11 | os: ubuntu-22.04 12 | tools: 13 | python: "3.11" 14 | 15 | # Build documentation in the docs/ directory with Sphinx 16 | sphinx: 17 | configuration: docs/conf.py 18 | 19 | # We recommend specifying your dependencies to enable reproducible builds: 20 | # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 21 | python: 22 | install: 23 | - requirements: docs/requirements.txt 24 | - method: pip 25 | path: . 26 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | minimum_pre_commit_version: '3.6' 4 | repos: 5 | - repo: https://github.com/pycqa/isort 6 | rev: "6.0.1" 7 | hooks: 8 | - id: isort 9 | - repo: https://github.com/hhatto/autopep8 10 | rev: "v2.3.2" 11 | hooks: 12 | - id: autopep8 13 | args: [--in-place, --aggressive, --aggressive] 14 | - repo: https://github.com/asottile/pyupgrade 15 | rev: v3.19.1 16 | hooks: 17 | - id: pyupgrade 18 | args: [--py39-plus] 19 | - repo: https://github.com/isidentical/teyit 20 | rev: 0.4.3 21 | hooks: 22 | - id: teyit 23 | - repo: https://github.com/PyCQA/flake8 24 | rev: "7.2.0" 25 | hooks: 26 | - id: flake8 27 | additional_dependencies: 28 | - flake8-debugger == 4.1.2 29 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 5 | # Contributing to zopefoundation projects 6 | 7 | The projects under the zopefoundation GitHub organization are open source and 8 | welcome contributions in different forms: 9 | 10 | * bug reports 11 | * code improvements and bug fixes 12 | * documentation improvements 13 | * pull request reviews 14 | 15 | For any changes in the repository besides trivial typo fixes you are required 16 | to sign the contributor agreement. See 17 | https://www.zope.dev/developer/becoming-a-committer.html for details. 18 | 19 | Please visit our [Developer 20 | Guidelines](https://www.zope.dev/developer/guidelines.html) if you'd like to 21 | contribute code changes and our [guidelines for reporting 22 | bugs](https://www.zope.dev/developer/reporting-bugs.html) if you want to file a 23 | bug report. 24 | -------------------------------------------------------------------------------- /src/ZEO/zeoctl.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | This schema describes the configuration of the ZEO storage server 5 | controller. It differs from the schema for the storage server 6 | only in that the "runner" section is required. 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 |
22 | 23 |
24 | 25 | 28 | 29 |
30 | 31 | 32 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============================================================ 2 | ZEO - Single-server client-server database server for ZODB 3 | ============================================================ 4 | 5 | ZEO is a client-server storage for `ZODB `_ for 6 | sharing a single storage among many clients. When you use ZEO, a 7 | lower-level storage, typically a file storage, is opened in the ZEO 8 | server process. Client programs connect to this process using a ZEO 9 | ClientStorage. ZEO provides a consistent view of the database to all 10 | clients. The ZEO client and server communicate using a custom 11 | protocol layered on top of TCP. 12 | 13 | Some alternatives to ZEO: 14 | 15 | - `NEO `_ is a distributed-server 16 | client-server storage. 17 | 18 | - `RelStorage `_ 19 | leverages the RDBMS servers to provide a client-server storage. 20 | 21 | The documentation is available on `readthedocs `_. 22 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yml: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | name: pre-commit 4 | 5 | on: 6 | pull_request: 7 | push: 8 | branches: 9 | - master 10 | # Allow to run this workflow manually from the Actions tab 11 | workflow_dispatch: 12 | 13 | env: 14 | FORCE_COLOR: 1 15 | 16 | jobs: 17 | pre-commit: 18 | permissions: 19 | contents: read 20 | pull-requests: write 21 | name: linting 22 | runs-on: ubuntu-latest 23 | steps: 24 | - uses: actions/checkout@v4 25 | - uses: actions/setup-python@v5 26 | with: 27 | python-version: 3.x 28 | - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd #v3.0.1 29 | with: 30 | extra_args: --all-files --show-diff-on-failure 31 | env: 32 | PRE_COMMIT_COLOR: always 33 | - uses: pre-commit-ci/lite-action@5d6cc0eb514c891a40562a58a8e71576c5c7fb43 #v1.1.0 34 | if: always() 35 | with: 36 | msg: Apply pre-commit code formatting 37 | -------------------------------------------------------------------------------- /src/ZEO/tests/server.pem.csr: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIICsjCCAZoCAQAwbTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlZBMQ0wCwYDVQQK 3 | DARaT0RCMREwDwYDVQQLDAh6b2RiLm9yZzERMA8GA1UEAwwIem9kYi5vcmcxHDAa 4 | BgkqhkiG9w0BCQEWDXpvZGJAem9kYi5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IB 5 | DwAwggEKAoIBAQD0rlGmUNcmllyuQ/7YCNozFc5nUA1ENjvsF1S4DG0qhkhPQxz2 6 | RQ2vhO8tpvgItVzmLDAR+KvvE9I8R7GRmXzwgm6qNZLrZ3XUkWVtTrZWrnhlwRiK 7 | Kae5HWHnN5TMoao1wwTFtOjuNyBKPcWcfjAJR5ZY5QRVqyelp8vU7hWJP0G5rFZc 8 | oPuCegVLODpBuXMI7Z/GSepp0t2jBOafIXkRSLa59ozJjjtqRkULwF/Twaetm0Cw 9 | Q0VEdORl/vpg2SK+834ad+vrSS2IvkewJqsCSNdiql+LxinCuUWSqe5yq/dXNDA5 10 | Cy+73hQlJy6IPlX0wRzafeWmT4itB5oM2YLpAgMBAAGgADANBgkqhkiG9w0BAQsF 11 | AAOCAQEAVzxIqDiv3evn3LsKrE0HcSkWKnValZz0e4iF96qmstLs2NJa+WsV7p/J 12 | Tg8DgUbQ72G9wN10OQld1k06KUd1SEWhOonBX60lGOkqyn6LHassItbwgCmHC5hk 13 | qs7h0b56s/gSnxYvN3tAWiRzNxdudFQBB7Ughy2SRN3ChsNDBuRIsJQN2yZtYjXM 14 | lZb2J7hZChFGD+L/9Cq6oPhUD+l1aFUv8PvU3jInf/IYyvNQJ3qeYRpOcNR4cnyf 15 | 6oRJn2b3ypFF/4F4ZiOb6Qocpcg7qBRRqztr4C2MZuDST4/zIBAHfKlUwD1/uo7A 16 | BdXUUeM1J1Gaf8GRLSvB8AeZg6/ztA== 17 | -----END CERTIFICATE REQUEST----- 18 | -------------------------------------------------------------------------------- /src/ZEO/tests/test_marshal.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from ZEO.asyncio.marshal import encode 4 | from ZEO.asyncio.marshal import pickle_server_decode 5 | 6 | 7 | try: 8 | from ZopeUndo.Prefix import Prefix 9 | except ModuleNotFoundError: 10 | _HAVE_ZOPE_UNDO = False 11 | else: 12 | _HAVE_ZOPE_UNDO = True 13 | 14 | 15 | class MarshalTests(unittest.TestCase): 16 | 17 | @unittest.skipUnless(_HAVE_ZOPE_UNDO, 'ZopeUndo is not installed') 18 | def testServerDecodeZopeUndoFilter(self): 19 | # this is an example (1) of Zope2's arguments for 20 | # undoInfo call. Arguments are encoded by ZEO client 21 | # and decoded by server. The operation must be idempotent. 22 | # (1) https://github.com/zopefoundation/Zope/blob/2.13/src/App/Undo.py#L111 # NOQA: E501 line too long 23 | args = (0, 20, {'user_name': Prefix('test')}) 24 | # test against repr because Prefix __eq__ operator 25 | # doesn't compare Prefix with Prefix but only 26 | # Prefix with strings. see Prefix.__doc__ 27 | self.assertEqual( 28 | repr(pickle_server_decode(encode(*args))), 29 | repr(args) 30 | ) 31 | -------------------------------------------------------------------------------- /src/ZEO/zeoctl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.3 2 | ############################################################################## 3 | # 4 | # Copyright (c) 2005 Zope Foundation and Contributors. 5 | # All Rights Reserved. 6 | # 7 | # This software is subject to the provisions of the Zope Public License, 8 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 9 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 10 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 11 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 12 | # FOR A PARTICULAR PURPOSE 13 | # 14 | ############################################################################## 15 | 16 | """Wrapper script for zdctl.py that causes it to use the ZEO schema.""" 17 | 18 | import os 19 | 20 | import zdaemon.zdctl 21 | 22 | import ZEO 23 | 24 | 25 | # Main program 26 | def main(args=None): 27 | options = zdaemon.zdctl.ZDCtlOptions() 28 | options.schemadir = os.path.dirname(ZEO.__file__) 29 | options.schemafile = "zeoctl.xml" 30 | zdaemon.zdctl.main(args, options) 31 | 32 | 33 | if __name__ == "__main__": 34 | main() 35 | -------------------------------------------------------------------------------- /src/ZEO/scripts/tests.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2004 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | import doctest 15 | import re 16 | import unittest 17 | 18 | from zope.testing import renormalizing 19 | 20 | 21 | def test_suite(): 22 | return unittest.TestSuite(( 23 | doctest.DocFileSuite( 24 | 'zeopack.test', 25 | checker=renormalizing.RENormalizing([ 26 | (re.compile('usage: Usage: '), 'Usage: '), # Py 2.4 27 | (re.compile('options:'), 'Options:'), # Py 2.4 28 | ]), 29 | ), 30 | )) 31 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | # 4 | # EditorConfig Configuration file, for more details see: 5 | # http://EditorConfig.org 6 | # EditorConfig is a convention description, that could be interpreted 7 | # by multiple editors to enforce common coding conventions for specific 8 | # file types 9 | 10 | # top-most EditorConfig file: 11 | # Will ignore other EditorConfig files in Home directory or upper tree level. 12 | root = true 13 | 14 | 15 | [*] # For All Files 16 | # Unix-style newlines with a newline ending every file 17 | end_of_line = lf 18 | insert_final_newline = true 19 | trim_trailing_whitespace = true 20 | # Set default charset 21 | charset = utf-8 22 | # Indent style default 23 | indent_style = space 24 | # Max Line Length - a hard line wrap, should be disabled 25 | max_line_length = off 26 | 27 | [*.{py,cfg,ini}] 28 | # 4 space indentation 29 | indent_size = 4 30 | 31 | [*.{yml,zpt,pt,dtml,zcml}] 32 | # 2 space indentation 33 | indent_size = 2 34 | 35 | [{Makefile,.gitmodules}] 36 | # Tab indentation (no size specified, but view as 4 spaces) 37 | indent_style = tab 38 | indent_size = unset 39 | tab_width = unset 40 | -------------------------------------------------------------------------------- /src/ZEO/tests/test_sync.py: -------------------------------------------------------------------------------- 1 | from zope.testing import setupstack 2 | 3 | from ZEO import _forker as forker 4 | 5 | from .. import client 6 | from .. import server 7 | 8 | 9 | class SyncTests(setupstack.TestCase): 10 | 11 | def instrument(self): 12 | self.__ping_calls = 0 13 | 14 | server = getattr(forker, self.__name + '_server') 15 | 16 | [zs] = getattr(server.server, 'zeo_storages_by_storage_id')['1'] 17 | orig_ping = getattr(zs, 'ping') 18 | 19 | def ping(): 20 | self.__ping_calls += 1 21 | return orig_ping() 22 | 23 | setattr(zs, 'ping', ping) 24 | 25 | def test_server_sync(self): 26 | self.__name = 's%s' % id(self) 27 | addr, stop = server(name=self.__name) 28 | 29 | # By default the client sync method is a noop: 30 | c = client(addr, wait_timeout=2) 31 | self.instrument() 32 | c.sync() 33 | self.assertEqual(self.__ping_calls, 0) 34 | c.close() 35 | 36 | # But if we pass server_sync: 37 | c = client(addr, wait_timeout=2, server_sync=True) 38 | self.instrument() 39 | c.sync() 40 | self.assertEqual(self.__ping_calls, 1) 41 | c.close() 42 | 43 | stop() 44 | -------------------------------------------------------------------------------- /src/ZEO/tests/new_addr.test: -------------------------------------------------------------------------------- 1 | You can change the address(es) of a client storaage. 2 | 3 | We'll start by setting up a server and connecting to it: 4 | 5 | >>> import ZEO, transaction 6 | 7 | >>> addr, stop = ZEO.server(path='test.fs', threaded=False) 8 | >>> conn = ZEO.connection(addr) 9 | >>> client = conn.db().storage 10 | >>> client.is_connected() 11 | True 12 | >>> conn.root() 13 | {} 14 | >>> conn.root.x = 1 15 | >>> transaction.commit() 16 | 17 | Now we'll close the server: 18 | 19 | >>> stop() 20 | 21 | And wait for the connectin to notice it's disconnected: 22 | 23 | >>> wait_until(lambda : not client.is_connected()) 24 | 25 | Now, we'll restart the server: 26 | 27 | >>> addr, stop = ZEO.server(path='test.fs', threaded=False) 28 | 29 | Update with another client: 30 | 31 | >>> conn2 = ZEO.connection(addr) 32 | >>> conn2.root.x += 1 33 | >>> transaction.commit() 34 | 35 | Update the connection and wait for connect: 36 | 37 | >>> client.new_addr(addr) 38 | >>> wait_until(lambda : client.is_connected()) 39 | >>> _ = transaction.begin() 40 | >>> conn.root() 41 | {'x': 2} 42 | 43 | .. cleanup 44 | 45 | >>> conn.close() 46 | >>> conn2.close() 47 | >>> stop() 48 | -------------------------------------------------------------------------------- /src/ZEO/tests/client.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDYjCCAkqgAwIBAgIJAIleRCrCxQqfMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV 3 | BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX 4 | aWRnaXRzIFB0eSBMdGQwIBcNMTkwNDI2MDg1ODA5WhgPMzAwNDEyMTgwODU4MDla 5 | MEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJ 6 | bnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw 7 | ggEKAoIBAQD0rlGmUNcmllyuQ/7YCNozFc5nUA1ENjvsF1S4DG0qhkhPQxz2RQ2v 8 | hO8tpvgItVzmLDAR+KvvE9I8R7GRmXzwgm6qNZLrZ3XUkWVtTrZWrnhlwRiKKae5 9 | HWHnN5TMoao1wwTFtOjuNyBKPcWcfjAJR5ZY5QRVqyelp8vU7hWJP0G5rFZcoPuC 10 | egVLODpBuXMI7Z/GSepp0t2jBOafIXkRSLa59ozJjjtqRkULwF/Twaetm0CwQ0VE 11 | dORl/vpg2SK+834ad+vrSS2IvkewJqsCSNdiql+LxinCuUWSqe5yq/dXNDA5Cy+7 12 | 3hQlJy6IPlX0wRzafeWmT4itB5oM2YLpAgMBAAGjUzBRMB0GA1UdDgQWBBSH6QYi 13 | Dj6xPq3C4dirKpgTN3N/wTAfBgNVHSMEGDAWgBSH6QYiDj6xPq3C4dirKpgTN3N/ 14 | wTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQDneWDUrFmQlCNy 15 | luXVrld4qi+ufpcjl7r/fsA7cTqNN3IWtzCbvbqQ8X2pSjfFkjvdcyqJAZ56Db5Q 16 | sb8F8m9zuTEm3qUlAqkbQb6vhiwwbEjeflPRkx/MyuUvForGgvnKQAB74bApTHo4 17 | u3DTuXN5px6Cc4LgcDHZXRPIr/8v+ZzBjL+csp6HO5ezl7qa5xc9hyXkGm6m/2iE 18 | aosKD1ll3XaBUfR7qyL5T46ccjdmr+w22HXuUQvhpQ4TGBIp3xEn1DDAClfjqH6A 19 | PjIfiwjYyg9PHblx8gmjGyCUp4CgR0pEbetolemdPLm+t7cSB3VNbzKnV04G9x73 20 | zieSaJid 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /src/ZEO/tests/protocols.test: -------------------------------------------------------------------------------- 1 | Test that old protocols are not supported 2 | ========================================= 3 | 4 | ZEO5 client used to support both ZEO5 and ZEO4 servers. Later support for 5 | ZEO5.client-ZEO4.server interoperability was found to be subject to data 6 | corruptions and removed. Here we verify that current ZEO client rejects 7 | connecting to old ZEO server. 8 | 9 | Let's start a Z4 server 10 | 11 | >>> storage_conf = ''' 12 | ... 13 | ... path Data.fs 14 | ... 15 | ... ''' 16 | 17 | >>> addr, stop = start_server( 18 | ... storage_conf, {'msgpack': 'false'}, protocol=b'4') 19 | 20 | A current client should not be able to connect to an old server: 21 | 22 | >>> import ZEO, logging, zope.testing.loggingsupport 23 | >>> hlog = zope.testing.loggingsupport.InstalledHandler( 24 | ... 'ZEO.asyncio.client', level=logging.ERROR) 25 | >>> logging.getLogger('ZEO.asyncio.client').addHandler(hlog) 26 | >>> ZEO.client(addr, client='client', wait_timeout=2) 27 | Traceback (most recent call last): 28 | ... 29 | ClientDisconnected: timed out waiting for connection 30 | 31 | >>> print(hlog) 32 | ZEO.asyncio.client ERROR 33 | Registration or cache validation failed, b'Z4' 34 | 35 | >>> hlog.uninstall() 36 | -------------------------------------------------------------------------------- /src/ZEO/_compat.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """Python versions compatiblity 15 | """ 16 | import platform 17 | import sys 18 | 19 | from zodbpickle.pickle import Unpickler as _Unpickler 20 | 21 | 22 | PYPY = getattr(platform, 'python_implementation', lambda: None)() == 'PyPy' 23 | WIN = sys.platform.startswith('win') 24 | 25 | 26 | class Unpickler(_Unpickler): 27 | # Python doesn't allow assignments to find_global any more, 28 | # instead, find_class can be overridden 29 | 30 | find_global = None 31 | 32 | def find_class(self, modulename, name): 33 | if self.find_global is None: 34 | return super().find_class(modulename, name) 35 | return self.find_global(modulename, name) 36 | -------------------------------------------------------------------------------- /src/ZEO/schema.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | This schema describes the configuration of the ZEO storage server 8 | process. 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
24 | 25 |
26 | 27 | 30 | 31 | One or more storages that are provided by the ZEO server. The 32 | section names are used as the storage names, and must be unique 33 | within each ZEO storage server. Traditionally, these names 34 | represent small integers starting at '1'. 35 | 36 | 37 | 38 |
39 | 40 | 41 | -------------------------------------------------------------------------------- /src/ZEO/tests/serverpw.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDiDCCAnCgAwIBAgIJAKjyf/5aIOS8MA0GCSqGSIb3DQEBBQUAMFgxCzAJBgNV 3 | BAYTAlVTMQswCQYDVQQIDAJWQTENMAsGA1UECgwEWk9EQjERMA8GA1UEAwwIem9k 4 | Yi5vcmcxGjAYBgkqhkiG9w0BCQEWC3B3QHpvZGIub3JnMCAXDTE5MDQyNjA5MjIw 5 | NFoYDzMwMDQxMjE4MDkyMjA0WjBYMQswCQYDVQQGEwJVUzELMAkGA1UECAwCVkEx 6 | DTALBgNVBAoMBFpPREIxETAPBgNVBAMMCHpvZGIub3JnMRowGAYJKoZIhvcNAQkB 7 | Fgtwd0B6b2RiLm9yZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMrD 8 | +LA3UQ911T1BBD46cIm/0bettM4+3jMRul9NE4gdMwTeSsUo8AusYXZor6gGkfHf 9 | 9Nkc3lS9Yr+OYa0Mv2N6QoIBp4BpUgqURJRLoMjFlj4Vo7et7/V9uYE1HPwKefWt 10 | oDD/hhlEM+9uWigfg23mbu6igYAggM9+3d8ieGYZug6gildzucwZUY5ZeUM3kj2g 11 | Wabi9vLgJQzLcD/TCceBO4w3LtOmODPYT/UxEA7JZDs9liIBKzjJxl9qzzTnw7wM 12 | j6itJjK6b9zN5wjPnY/qNvHkta9woNZpzxBsmgc96W2Fi3Fwwd57b42HoxZbrzwI 13 | HOodJ4UB007d32QyIWkCAwEAAaNTMFEwHQYDVR0OBBYEFDui1OC2+2z2rHADglk5 14 | tGOndxhoMB8GA1UdIwQYMBaAFDui1OC2+2z2rHADglk5tGOndxhoMA8GA1UdEwEB 15 | /wQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAJ5muBIXZhAXJ4NM79FBV1BLPvQ2 16 | 2RRczEmNGXN35TTzVx1L/h+M8Jbwow40aN/aVjJN0m8+JkJdQZDgnUX+g9ZLPfxU 17 | aAZY8p2nPvMn0AmOmx/DyXyd05WMj+qskyBioPKR79VmhTqSvycnjqORhToKHtrd 18 | n4CNhJr6j0bnwYmLjH5/6x3py37DeS0G5hvBCBkkAP8Ov5NXIOovnCC0FrkOMvhe 19 | CBFt+J4BU/BUJZHsEj1PMMmGadGZm6UGvWptJQOVxdvFlI3QG1vM+bPP9ve7Tshl 20 | rTWzvhfATatBkjnCcuSI0xHRyStauHl4+e9AHH7PI68H19StXnH746aGce0= 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /src/ZEO/tests/server.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDsjCCApqgAwIBAgIJAMV1O1+aKDW9MA0GCSqGSIb3DQEBBQUAMG0xCzAJBgNV 3 | BAYTAlVTMQswCQYDVQQIDAJWQTENMAsGA1UECgwEWk9EQjERMA8GA1UECwwIem9k 4 | Yi5vcmcxETAPBgNVBAMMCHpvZGIub3JnMRwwGgYJKoZIhvcNAQkBFg16b2RiQHpv 5 | ZGIub3JnMCAXDTE5MDQyNjA5MDEzNloYDzMwMDQxMjE4MDkwMTM2WjBtMQswCQYD 6 | VQQGEwJVUzELMAkGA1UECAwCVkExDTALBgNVBAoMBFpPREIxETAPBgNVBAsMCHpv 7 | ZGIub3JnMREwDwYDVQQDDAh6b2RiLm9yZzEcMBoGCSqGSIb3DQEJARYNem9kYkB6 8 | b2RiLm9yZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAPSuUaZQ1yaW 9 | XK5D/tgI2jMVzmdQDUQ2O+wXVLgMbSqGSE9DHPZFDa+E7y2m+Ai1XOYsMBH4q+8T 10 | 0jxHsZGZfPCCbqo1kutnddSRZW1OtlaueGXBGIopp7kdYec3lMyhqjXDBMW06O43 11 | IEo9xZx+MAlHlljlBFWrJ6Wny9TuFYk/QbmsVlyg+4J6BUs4OkG5cwjtn8ZJ6mnS 12 | 3aME5p8heRFItrn2jMmOO2pGRQvAX9PBp62bQLBDRUR05GX++mDZIr7zfhp36+tJ 13 | LYi+R7AmqwJI12KqX4vGKcK5RZKp7nKr91c0MDkLL7veFCUnLog+VfTBHNp95aZP 14 | iK0HmgzZgukCAwEAAaNTMFEwHQYDVR0OBBYEFIfpBiIOPrE+rcLh2KsqmBM3c3/B 15 | MB8GA1UdIwQYMBaAFIfpBiIOPrE+rcLh2KsqmBM3c3/BMA8GA1UdEwEB/wQFMAMB 16 | Af8wDQYJKoZIhvcNAQEFBQADggEBAGX342qkGd1wR8C0R+o8Dq1Pkq4zBk+eMdkx 17 | C6wZsxkSoRqsFHAoZSxYNXT3xPGwbl/SO8+NzZO9WNRkQukLYUWe4SjdVtLkYJD8 18 | KcFVbDY2dv/CJRxUftNYTDJC2PscNjgaNRdLpyScprqxFx6IwfhepquIjuQj/7s5 19 | 4txou3D+KKvmNMoFdW1dvrsxn38vkhfFgTi5/FczlDL8n3dmLY3w9bOZAz1BTlVh 20 | FAE9mWJpnxRCVGzGOxaMNA1O8CrUfS4uzoUnWRR0uuftrA4VJOOEVOzHHwdDskso 21 | kswYHeWWJfqLUJlf1Sr+6dQ11VONm86+xY/2Xw0lS+zqXAll/x8= 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ========================================================== 2 | ZEO - Single-server client-server database server for ZODB 3 | ========================================================== 4 | 5 | ZEO is a client-server storage for `ZODB `_ for 6 | sharing a single storage among many clients. When you use ZEO, a 7 | lower-level storage, typically a file storage, is opened in the ZEO 8 | server process. Client programs connect to this process using a ZEO 9 | ClientStorage. ZEO provides a consistent view of the database to all 10 | clients. The ZEO client and server communicate using a custom 11 | protocol layered on top of TCP. 12 | 13 | ZEO can be installed with pip like most python packages: 14 | 15 | .. code-block:: bash 16 | 17 | pip install zeo 18 | 19 | Some alternatives to ZEO: 20 | 21 | - `NEO `_ is a distributed-server 22 | client-server storage. 23 | 24 | - `RelStorage `_ 25 | leverages the RDBMS servers to provide a client-server storage. 26 | 27 | Content 28 | ======= 29 | 30 | .. toctree:: 31 | :maxdepth: 2 32 | 33 | introduction 34 | server 35 | clients 36 | protocol 37 | nagios 38 | ordering 39 | client-cache 40 | client-cache-tracing 41 | blob-nfs 42 | changelog 43 | reference 44 | 45 | Indices and tables 46 | ================== 47 | 48 | * :ref:`genindex` 49 | * :ref:`modindex` 50 | * :ref:`search` 51 | -------------------------------------------------------------------------------- /src/ZEO/Exceptions.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """Exceptions for ZEO.""" 15 | 16 | import transaction.interfaces 17 | from ZODB.POSException import StorageError 18 | 19 | 20 | class ClientStorageError(StorageError): 21 | """An error occurred in the ZEO Client Storage. 22 | """ 23 | 24 | 25 | class UnrecognizedResult(ClientStorageError): 26 | """A server call returned an unrecognized result. 27 | """ 28 | 29 | 30 | class ClientDisconnected(ClientStorageError, 31 | transaction.interfaces.TransientError): 32 | """The database storage is disconnected from the storage. 33 | """ 34 | 35 | 36 | class ProtocolError(ClientStorageError): 37 | """A client contacted a server with an incomparible protocol 38 | """ 39 | 40 | 41 | class ServerException(ClientStorageError): 42 | """ 43 | """ 44 | -------------------------------------------------------------------------------- /src/ZEO/tests/client_key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEA9K5RplDXJpZcrkP+2AjaMxXOZ1ANRDY77BdUuAxtKoZIT0Mc 3 | 9kUNr4TvLab4CLVc5iwwEfir7xPSPEexkZl88IJuqjWS62d11JFlbU62Vq54ZcEY 4 | iimnuR1h5zeUzKGqNcMExbTo7jcgSj3FnH4wCUeWWOUEVasnpafL1O4ViT9BuaxW 5 | XKD7gnoFSzg6QblzCO2fxknqadLdowTmnyF5EUi2ufaMyY47akZFC8Bf08GnrZtA 6 | sENFRHTkZf76YNkivvN+Gnfr60ktiL5HsCarAkjXYqpfi8YpwrlFkqnucqv3VzQw 7 | OQsvu94UJScuiD5V9MEc2n3lpk+IrQeaDNmC6QIDAQABAoIBAErM27MvdYabYvv3 8 | V3otwp7pZK8avuOCfPEg9MpLKjhc0tBAYSM8WwG0bvYS3DK1VxAapBtqXQ16jsPU 9 | 2wj61kIkbbZlKGQEvfXc+Rfgf0eikLXywRDDyT2DKQHpcPjZ11IWK2hRdQAWJC3u 10 | EnJT9VVw6BqG8LtL1pQC5wJSQo0xC1sJ/MTr/szLvKRjuYZE7YStpUfV6RYq2KQF 11 | 7Oa9nPKtxlIbDCa7z4S6y5yiusYrSSFilK0pVSU+9789kGNZMLzKbnGu+YSVB/Bx 12 | MLXWRAD8DV9zign255pIU/xI5VKjOwID38JfgdcebV/KeCPu8W6jKKbfUsUCqcjL 13 | YjDtHYECgYEA/SaxUoejMOasHppnsAewy/I+DzMuX+KYztqAnzjsuGwRxmxjYyQe 14 | w7EidinM3WuloJIBZzA9aULmWjSKOfTsuGm+Mokucbbw9jaWVT6Co3kWrHySInhZ 15 | sfTwHKz5ojGBcQD4l06xaVM9utNi6r8wvJijFl5xIsMzc5szEkWs9vkCgYEA9285 16 | bGSAAwzUFHVk1pyLKozM2gOtF5rrAUQlWtNVU6K2tw+MKEGara0f+HFZrJZC9Rh2 17 | HBm2U9PPt/kJ73HErQG+E6n0jfol8TQ3ZKz3tlSxImh0CiaKLnh4ahf7o8zU16nT 18 | XDfu3+Rf11EhORXYfZLmdubfCOD4ZaB2/405N3ECgYEA7b4k0gkoLYi1JJiFwD+4 19 | vhBmUAgVCV/ZeoqiOOZRCnITz3GDdVw6uDXm02o2R8wM5Fu6jZo0UmLNyvGEzyFC 20 | H37PbM6Am7LfYZuqW6w1LClQLfVfmJfGROZvib65QqWTlvj+fbsdyniuhIJ5Z1Tf 21 | BH+kyiEvxyHjdDLRJ9vfsKECgYA8P9MFt7sMAxWpHaS+NUQVyk8fTwHY25oZptRJ 22 | t2fxg49mJ90C+GaHn75HKqKhSb1oHNq1oPUqmEreC0AGE/fGAMSd2SZ5Y83VW9eZ 23 | JhzzQtAXBsQqrJO9GQyJGOnnSrsRAIM800nRLrS/ozupwM4EVb7UeQcaDF2vsVEI 24 | jQS/oQKBgHj26xn7AunX5GS8EYe4GPj4VZehmlnEKONGrPrr25aWkaY4kDJgMLUb 25 | AxwIQHbCMm5TMqIxi5l39/O9dxuuGCkOs37j7C3f3VVFuQW1KKyHem9OClgFDZj3 26 | tEEk1N3NevrH06NlmAHweHMuJXL8mBvM375zH9tSw5mgG0OMRbnG 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /src/ZEO/tests/server_key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEA9K5RplDXJpZcrkP+2AjaMxXOZ1ANRDY77BdUuAxtKoZIT0Mc 3 | 9kUNr4TvLab4CLVc5iwwEfir7xPSPEexkZl88IJuqjWS62d11JFlbU62Vq54ZcEY 4 | iimnuR1h5zeUzKGqNcMExbTo7jcgSj3FnH4wCUeWWOUEVasnpafL1O4ViT9BuaxW 5 | XKD7gnoFSzg6QblzCO2fxknqadLdowTmnyF5EUi2ufaMyY47akZFC8Bf08GnrZtA 6 | sENFRHTkZf76YNkivvN+Gnfr60ktiL5HsCarAkjXYqpfi8YpwrlFkqnucqv3VzQw 7 | OQsvu94UJScuiD5V9MEc2n3lpk+IrQeaDNmC6QIDAQABAoIBAErM27MvdYabYvv3 8 | V3otwp7pZK8avuOCfPEg9MpLKjhc0tBAYSM8WwG0bvYS3DK1VxAapBtqXQ16jsPU 9 | 2wj61kIkbbZlKGQEvfXc+Rfgf0eikLXywRDDyT2DKQHpcPjZ11IWK2hRdQAWJC3u 10 | EnJT9VVw6BqG8LtL1pQC5wJSQo0xC1sJ/MTr/szLvKRjuYZE7YStpUfV6RYq2KQF 11 | 7Oa9nPKtxlIbDCa7z4S6y5yiusYrSSFilK0pVSU+9789kGNZMLzKbnGu+YSVB/Bx 12 | MLXWRAD8DV9zign255pIU/xI5VKjOwID38JfgdcebV/KeCPu8W6jKKbfUsUCqcjL 13 | YjDtHYECgYEA/SaxUoejMOasHppnsAewy/I+DzMuX+KYztqAnzjsuGwRxmxjYyQe 14 | w7EidinM3WuloJIBZzA9aULmWjSKOfTsuGm+Mokucbbw9jaWVT6Co3kWrHySInhZ 15 | sfTwHKz5ojGBcQD4l06xaVM9utNi6r8wvJijFl5xIsMzc5szEkWs9vkCgYEA9285 16 | bGSAAwzUFHVk1pyLKozM2gOtF5rrAUQlWtNVU6K2tw+MKEGara0f+HFZrJZC9Rh2 17 | HBm2U9PPt/kJ73HErQG+E6n0jfol8TQ3ZKz3tlSxImh0CiaKLnh4ahf7o8zU16nT 18 | XDfu3+Rf11EhORXYfZLmdubfCOD4ZaB2/405N3ECgYEA7b4k0gkoLYi1JJiFwD+4 19 | vhBmUAgVCV/ZeoqiOOZRCnITz3GDdVw6uDXm02o2R8wM5Fu6jZo0UmLNyvGEzyFC 20 | H37PbM6Am7LfYZuqW6w1LClQLfVfmJfGROZvib65QqWTlvj+fbsdyniuhIJ5Z1Tf 21 | BH+kyiEvxyHjdDLRJ9vfsKECgYA8P9MFt7sMAxWpHaS+NUQVyk8fTwHY25oZptRJ 22 | t2fxg49mJ90C+GaHn75HKqKhSb1oHNq1oPUqmEreC0AGE/fGAMSd2SZ5Y83VW9eZ 23 | JhzzQtAXBsQqrJO9GQyJGOnnSrsRAIM800nRLrS/ozupwM4EVb7UeQcaDF2vsVEI 24 | jQS/oQKBgHj26xn7AunX5GS8EYe4GPj4VZehmlnEKONGrPrr25aWkaY4kDJgMLUb 25 | AxwIQHbCMm5TMqIxi5l39/O9dxuuGCkOs37j7C3f3VVFuQW1KKyHem9OClgFDZj3 26 | tEEk1N3NevrH06NlmAHweHMuJXL8mBvM375zH9tSw5mgG0OMRbnG 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /src/ZEO/tests/serverpw_key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | Proc-Type: 4,ENCRYPTED 3 | DEK-Info: DES-EDE3-CBC,769B900D03925712 4 | 5 | z5M/XkqEC1+PxJ1T3QrUhGG9fPTBsPxKy8WlwIytbMg0RXS0oJBiFgNYp1ktqzGo 6 | yT+AdTCRR1hNVX8M5HbV3ksUjKxXKCL3+yaaB6JtGbNRr2qTNwosvxD92nKT/hvN 7 | R6rHF6LcO05s8ubs9b9ON/ja7HCx69N5CjBuCbCFHUTlAXkwD9w0ScrxrtfP50EY 8 | FOw6LAqhhzq6/KO7c1SJ7k9LYzakhL+nbw5KM9QgBk4WHlmKLbCZIZ5RWvu0F4s5 9 | n4qk/BcuXIkbYuEv2kH0nDk5eDfA/dj7xZcMMgL5VFymQzaZLYyj4WuQYXu/7JW/ 10 | nM/ZWBkZOMaI3vnPTG1hJ9pgjLjQnjfNA/bGWwbLxjCsPmR8yvZS4v2iqdB6X3Vl 11 | yJ9aV9r8KoU0PJk3x4v2Zp+RQxgrKSaQw59sXptaXAY3NCRR6ohvq4P5X6UB8r5S 12 | vYdoMeVXhX1hzXeMguln7zQInwJhPZqk4wMIV3lTsCqh1eJ7NC2TGCwble+B/ClR 13 | KtzuJBzoYPLw44ltnXPEMmVH1Fkh17+QZFgZRJrKGD9PGOAXmnzudsZ1xX9kNnOM 14 | JLIT/mzKcqkd8S1n1Xi1x7zGA0Ng5xmKGFe8oDokPJucJO1Ou+hbLDmC0DZUGzr1 15 | qqPJ3F/DzZZDTmD/rZF6doPJgFAZvgpVeiWS8/v1qbz/nz13uwXDLjRPgLfcKpmQ 16 | 4R3V4QlgviDilW61VTZnzV9qAOx4fG6+IwWIGBlrJnfsH/fSCDNlAStc6k12zdun 17 | PIIRJBfbEprGig3vRWUoBASReqow1JCN9DaVCX1P27pDKY5oDe+7/HOrQpwhPoya 18 | 2HEwbKeyY0nCcCXbkWGL1bwEUs/PrJv+61rik4KxOWhKpHWkZLzbozELb44jXrJx 19 | e8K8XKz4La2DEjsUYHc31u6T69GBQO9JDEvih15phUWq8ITvDnkHpAg+wYb1JAHD 20 | QcqDtAulMvT/ZGN0h7qdwbHMggEsLgCCVPG4iZ5K4cXsMbePFvQqq+o4FTMF+cM5 21 | 2Dq0wir92U9cH+ooy80LIt5Kp5zqgQZzr73o9MEgwqJocCrx9ZrofKRUmTV+ZU0r 22 | w5mfUM47Ctnqia0UNGx6SUs3CHFDPWPbzrAaqGzSvFhzR1MMoL1/rJzP1VSm3Fk3 23 | ESWkPrg0J8dcQP/ch9MhH8eoQYyA+2q1vClUbeZLAs5KoHxgi6pSkGYqFhshrA+t 24 | 2AIrUPDPPDf0PgRoXJrzdVOiNNY1rzyql+0JqDH6DjCVcAADWY+48p9U2YFTd7Je 25 | DvnZWihwe0qYGn1AKIkvJ4SR3bQg36etrxhMrMl/8lUn2dnT7GFrhjr9HwCpJwa7 26 | 8tv150SrQXt3FXZCHb+RMUgoWZDeksDohPiGzXkPU6kaSviZVnRMslyU4ahWp6vC 27 | 8tYUhb7K6N+is1hYkICNt6zLl2vBDuCDWmiIwopHtnH1kz8bYlp4/GBVaMIgZiCM 28 | gM/7+p4YCc++s2sJiQ9+BqPo0zKm3bbSP+fPpeWefQVte9Jx4S36YXU52HsJxBTN 29 | WUdHABC+aS2A45I12xMNzOJR6VfxnG6f3JLpt3MkUCEg+898vJGope+TJUhD+aJC 30 | -----END RSA PRIVATE KEY----- 31 | -------------------------------------------------------------------------------- /src/ZEO/tests/test_client_credentials.py: -------------------------------------------------------------------------------- 1 | """Clients can pass credentials to a server. 2 | 3 | This is an experimental feature to enable server authentication and 4 | authorization. 5 | """ 6 | import unittest 7 | 8 | from zope.testing import setupstack 9 | 10 | import ZEO.StorageServer 11 | 12 | from .threaded import threaded_server_tests 13 | 14 | 15 | class ClientAuthTests(setupstack.TestCase): 16 | 17 | def setUp(self): 18 | self.setUpDirectory() 19 | self.__register = ZEO.StorageServer.ZEOStorage.register 20 | 21 | def tearDown(self): 22 | ZEO.StorageServer.ZEOStorage.register = self.__register 23 | 24 | def test_passing_credentials(self): 25 | 26 | # First, we'll temporarily swap the storage server register 27 | # method with one that let's is see credentials that were passed: 28 | 29 | creds_log = [] 30 | 31 | def register(zs, storage_id, read_only, credentials=self): 32 | creds_log.append(credentials) 33 | return self.__register(zs, storage_id, read_only) 34 | 35 | ZEO.StorageServer.ZEOStorage.register = register 36 | 37 | # Now start an in process server 38 | addr, stop = ZEO.server() 39 | 40 | # If we connect, without providing credentials, then no 41 | # credentials will be passed to register: 42 | 43 | client = ZEO.client(addr) 44 | 45 | self.assertEqual(creds_log, [self]) 46 | client.close() 47 | creds_log.pop() 48 | 49 | # Even if we pass credentials, they'll be ignored 50 | creds = dict(user='me', password='123') 51 | client = ZEO.client(addr, credentials=creds) 52 | self.assertEqual(creds_log, [self]) 53 | client.close() 54 | 55 | stop() 56 | 57 | 58 | def test_suite(): 59 | suite = unittest.defaultTestLoader.loadTestsFromTestCase(ClientAuthTests) 60 | suite.layer = threaded_server_tests 61 | return suite 62 | -------------------------------------------------------------------------------- /src/ZEO/scripts/timeout.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.3 2 | 3 | """Transaction timeout test script. 4 | 5 | This script connects to a storage, begins a transaction, calls store() 6 | and tpc_vote(), and then sleeps forever. This should trigger the 7 | transaction timeout feature of the server. 8 | 9 | usage: timeout.py address delay [storage-name] 10 | 11 | """ 12 | 13 | import sys 14 | import time 15 | 16 | from ZODB.Connection import TransactionMetaData 17 | from ZODB.tests.MinPO import MinPO 18 | from ZODB.tests.StorageTestBase import zodb_pickle 19 | 20 | from ZEO.ClientStorage import ClientStorage 21 | 22 | 23 | ZERO = '\0' * 8 24 | 25 | 26 | def main(): 27 | if len(sys.argv) not in (3, 4): 28 | sys.stderr.write("Usage: timeout.py address delay [storage-name]\n" % 29 | sys.argv[0]) 30 | sys.exit(2) 31 | 32 | hostport = sys.argv[1] 33 | delay = float(sys.argv[2]) 34 | if sys.argv[3:]: 35 | name = sys.argv[3] 36 | else: 37 | name = "1" 38 | 39 | if "/" in hostport: 40 | address = hostport 41 | else: 42 | if ":" in hostport: 43 | i = hostport.index(":") 44 | host, port = hostport[:i], hostport[i + 1:] 45 | else: 46 | host, port = "", hostport 47 | port = int(port) 48 | address = (host, port) 49 | 50 | print("Connecting to %s..." % repr(address)) 51 | storage = ClientStorage(address, name) 52 | print("Connected. Now starting a transaction...") 53 | 54 | oid = storage.new_oid() 55 | revid = ZERO 56 | data = MinPO("timeout.py") 57 | pickled_data = zodb_pickle(data) 58 | t = TransactionMetaData() 59 | t.user = "timeout.py" 60 | storage.tpc_begin(t) 61 | storage.store(oid, revid, pickled_data, '', t) 62 | print("Stored. Now voting...") 63 | storage.tpc_vote(t) 64 | 65 | print("Voted; now sleeping %s..." % delay) 66 | time.sleep(delay) 67 | print("Done.") 68 | 69 | 70 | if __name__ == "__main__": 71 | main() 72 | -------------------------------------------------------------------------------- /src/ZEO/tests/Cache.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """Tests of the ZEO cache""" 15 | 16 | from ZODB.Connection import TransactionMetaData 17 | from ZODB.tests.MinPO import MinPO 18 | from ZODB.tests.StorageTestBase import zodb_unpickle 19 | 20 | 21 | class TransUndoStorageWithCache: 22 | 23 | def checkUndoInvalidation(self): 24 | oid = self._storage.new_oid() 25 | revid = self._dostore(oid, data=MinPO(23)) 26 | revid = self._dostore(oid, revid=revid, data=MinPO(24)) 27 | revid = self._dostore(oid, revid=revid, data=MinPO(25)) 28 | 29 | info = self._storage.undoInfo() 30 | if not info: 31 | # Preserved this comment, but don't understand it: 32 | # "Perhaps we have an old storage implementation that 33 | # does do the negative nonsense." 34 | info = self._storage.undoInfo(0, 20) 35 | tid = info[0]['id'] 36 | 37 | # Now start an undo transaction 38 | t = TransactionMetaData() 39 | t.note('undo1') 40 | oids = self._begin_undos_vote(t, tid) 41 | 42 | # Make sure this doesn't load invalid data into the cache 43 | self._storage.load(oid, '') 44 | 45 | self._storage.tpc_finish(t) 46 | 47 | [uoid] = oids 48 | assert uoid == oid 49 | data, revid = self._storage.load(oid, '') 50 | obj = zodb_unpickle(data) 51 | assert obj == MinPO(24) 52 | -------------------------------------------------------------------------------- /docs/client-cache.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | ZEO Client Cache 3 | ================ 4 | 5 | The client cache provides a disk based cache for each ZEO client. The 6 | client cache allows reads to be done from local disk rather than by remote 7 | access to the storage server. 8 | 9 | The cache may be persistent or transient. If the cache is persistent, then 10 | the cache file is retained for use after process restarts. A non- 11 | persistent cache uses a temporary file. 12 | 13 | The client cache is managed in a single file, of the specified size. 14 | 15 | The life of the cache is as follows: 16 | 17 | - The cache file is opened (if it already exists), or created and set to 18 | the specified size. 19 | 20 | - Cache records are written to the cache file, as transactions commit 21 | locally, and as data are loaded from the server. 22 | 23 | - Writes are to "the current file position". This is a pointer that 24 | travels around the file, circularly. After a record is written, the 25 | pointer advances to just beyond it. Objects starting at the current 26 | file position are evicted, as needed, to make room for the next record 27 | written. 28 | 29 | A distinct index file is not created, although indexing structures are 30 | maintained in memory while a ClientStorage is running. When a persistent 31 | client cache file is reopened, these indexing structures are recreated 32 | by analyzing the file contents. 33 | 34 | Persistent cache files are created in the directory named in the ``var`` 35 | argument to the ClientStorage, or if ``var`` is None, in the current 36 | working directory. Persistent cache files have names of the form:: 37 | 38 | client-storage.zec 39 | 40 | where: 41 | 42 | client -- the client name, as given by the ClientStorage's ``client`` 43 | argument 44 | 45 | storage -- the storage name, as given by the ClientStorage's ``storage`` 46 | argument; this is typically a string denoting a small integer, 47 | "1" by default 48 | 49 | For example, the cache file for client '8881' and storage 'spam' is named 50 | "8881-spam.zec". 51 | -------------------------------------------------------------------------------- /src/ZEO/util.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """Utilities for setting up the server environment.""" 15 | 16 | import os 17 | 18 | 19 | def parentdir(p, n=1): 20 | """Return the ancestor of p from n levels up.""" 21 | d = p 22 | while n: 23 | d = os.path.dirname(d) 24 | if not d or d == '.': 25 | d = os.getcwd() 26 | n -= 1 27 | return d 28 | 29 | 30 | class Environment: 31 | """Determine location of the Data.fs & ZEO_SERVER.pid files. 32 | 33 | Pass the argv[0] used to start ZEO to the constructor. 34 | 35 | Use the zeo_pid and fs attributes to get the filenames. 36 | """ 37 | 38 | def __init__(self, argv0): 39 | v = os.environ.get("INSTANCE_HOME") 40 | if v is None: 41 | # looking for a Zope/var directory assuming that this code 42 | # is installed in Zope/lib/python/ZEO 43 | p = parentdir(argv0, 4) 44 | if os.path.isdir(os.path.join(p, "var")): 45 | v = p 46 | else: 47 | v = os.getcwd() 48 | self.home = v 49 | self.var = os.path.join(v, "var") 50 | if not os.path.isdir(self.var): 51 | self.var = self.home 52 | 53 | pid = os.environ.get("ZEO_SERVER_PID") 54 | if pid is None: 55 | pid = os.path.join(self.var, "ZEO_SERVER.pid") 56 | 57 | self.zeo_pid = pid 58 | self.fs = os.path.join(self.var, "Data.fs") 59 | -------------------------------------------------------------------------------- /.meta.toml: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | [meta] 4 | template = "pure-python" 5 | commit-id = "09d4ab7e" 6 | 7 | [python] 8 | with-windows = false 9 | with-pypy = true 10 | with-future-python = false 11 | with-docs = true 12 | with-sphinx-doctests = false 13 | with-macos = false 14 | 15 | [tox] 16 | testenv-commands = [ 17 | "# Run unit tests first.", 18 | "zope-testrunner -u --test-path=src -a 1000 {posargs:-vc}", 19 | "# Only run functional tests if unit tests pass.", 20 | "zope-testrunner -f --test-path=src -a 1000 {posargs:-vc}", 21 | ] 22 | testenv-deps = [ 23 | "!zodbmaster: ZODB >= 4.2.0b1", 24 | "zodbmaster: git+https://github.com/zopefoundation/ZODB.git@master\\#egg=ZODB", 25 | "uvloop: uvloop", 26 | ] 27 | testenv-setenv = [ 28 | "msgpack1: ZEO_MSGPACK=1", 29 | ] 30 | 31 | [flake8] 32 | additional-config = [ 33 | "# E501 line too long", 34 | "per-file-ignores =", 35 | " src/ZEO/tests/test_cache.py: E501", 36 | " src/ZEO/asyncio/compat.py: F401", 37 | ] 38 | 39 | [coverage] 40 | fail-under = 53 41 | 42 | [manifest] 43 | additional-rules = [ 44 | "include *.py", 45 | "include *.yaml", 46 | "include COPYING", 47 | "include log.ini", 48 | "recursive-include src *.csr", 49 | "recursive-include src *.pem", 50 | "recursive-include src *.pyx", 51 | "recursive-include src *.rst", 52 | "recursive-include src *.test", 53 | "recursive-include src *.txt", 54 | "recursive-include src *.xml", 55 | ] 56 | 57 | [github-actions] 58 | additional-config = [ 59 | "- [\"3.9\", \"py39-msgpack1\"]", 60 | "- [\"3.9\", \"py39-uvloop\"]", 61 | "- [\"3.10\", \"py310-msgpack1\"]", 62 | "- [\"3.10\", \"py310-uvloop\"]", 63 | "- [\"3.11\", \"py311-msgpack1\"]", 64 | "- [\"3.11\", \"py311-uvloop\"]", 65 | "- [\"3.12\", \"py312-msgpack1\"]", 66 | "- [\"3.12\", \"py312-uvloop\"]", 67 | "- [\"3.13\", \"py313-msgpack1\"]", 68 | "- [\"3.13\", \"py313-uvloop\"]", 69 | "- [\"3.13\", \"py313-zodbmaster\"]", 70 | "- [\"pypy-3.10\", \"pypy3-msgpack1\"]", 71 | ] 72 | -------------------------------------------------------------------------------- /src/ZEO/tests/utils.py: -------------------------------------------------------------------------------- 1 | """Testing helpers 2 | """ 3 | import ZEO.StorageServer 4 | 5 | from ..asyncio.server import best_protocol_version 6 | 7 | 8 | class ServerProtocol: 9 | 10 | method = ('register', ) 11 | 12 | def __init__(self, zs, 13 | protocol_version=best_protocol_version, 14 | addr='test-address'): 15 | self.calls = [] 16 | self.addr = addr 17 | self.zs = zs 18 | self.protocol_version = protocol_version 19 | zs.notify_connected(self) 20 | 21 | closed = False 22 | 23 | def close(self): 24 | if not self.closed: 25 | self.closed = True 26 | self.zs.notify_disconnected() 27 | 28 | def call_soon_threadsafe(self, func, *args): 29 | func(*args) 30 | 31 | def async_(self, *args): 32 | self.calls.append(args) 33 | 34 | async_threadsafe = async_ 35 | 36 | 37 | class StorageServer: 38 | """Create a client interface to a StorageServer. 39 | 40 | This is for testing StorageServer. It interacts with the storgr 41 | server through its network interface, but without creating a 42 | network connection. 43 | """ 44 | 45 | def __init__(self, test, storage, 46 | protocol_version=b'Z' + best_protocol_version, 47 | **kw): 48 | self.test = test 49 | self.storage_server = ZEO.StorageServer.StorageServer( 50 | None, {'1': storage}, **kw) 51 | self.zs = self.storage_server.create_client_handler() 52 | self.protocol = ServerProtocol(self.zs, 53 | protocol_version=protocol_version) 54 | self.zs.register('1', kw.get('read_only', False)) 55 | 56 | def assert_calls(self, test, *argss): 57 | if argss: 58 | for args in argss: 59 | test.assertEqual(self.protocol.calls.pop(0), args) 60 | else: 61 | test.assertEqual(self.protocol.calls, ()) 62 | 63 | def unpack_result(self, result): 64 | """For methods that return Result objects, unwrap the results 65 | """ 66 | result, callback = result.args 67 | callback() 68 | return result 69 | -------------------------------------------------------------------------------- /src/ZEO/scripts/README.txt: -------------------------------------------------------------------------------- 1 | This directory contains a collection of utilities for working with 2 | ZEO. Some are more useful than others. If you install ZODB using 3 | distutils ("python setup.py install"), some of these will be 4 | installed. 5 | 6 | Unless otherwise noted, these scripts are invoked with the name of the 7 | Data.fs file as their only argument. Example: checkbtrees.py data.fs. 8 | 9 | 10 | parsezeolog.py -- parse BLATHER logs from ZEO server 11 | 12 | This script may be obsolete. It has not been tested against the 13 | current log output of the ZEO server. 14 | 15 | Reports on the time and size of transactions committed by a ZEO 16 | server, by inspecting log messages at BLATHER level. 17 | 18 | 19 | 20 | timeout.py -- script to test transaction timeout 21 | 22 | usage: timeout.py address delay [storage-name] 23 | 24 | This script connects to a storage, begins a transaction, calls store() 25 | and tpc_vote(), and then sleeps forever. This should trigger the 26 | transaction timeout feature of the server. 27 | 28 | 29 | zeopack.py -- pack a ZEO server 30 | 31 | The script connects to a server and calls pack() on a specific 32 | storage. See the script for usage details. 33 | 34 | 35 | zeoreplay.py -- experimental script to replay transactions from a ZEO log 36 | 37 | Like parsezeolog.py, this may be obsolete because it was written 38 | against an earlier version of the ZEO server. See the script for 39 | usage details. 40 | 41 | 42 | zeoup.py 43 | 44 | usage: zeoup.py [options] 45 | 46 | The test will connect to a ZEO server, load the root object, and 47 | attempt to update the zeoup counter in the root. It will report 48 | success if it updates to counter or if it gets a ConflictError. A 49 | ConflictError is considered a success, because the client was able to 50 | start a transaction. 51 | 52 | See the script for details about the options. 53 | 54 | 55 | 56 | zeoserverlog.py -- analyze ZEO server log for performance statistics 57 | 58 | See the module docstring for details; there are a large number of 59 | options. New in ZODB3 3.1.4. 60 | 61 | 62 | zeoqueue.py -- report number of clients currently waiting in the ZEO queue 63 | 64 | See the module docstring for details. 65 | -------------------------------------------------------------------------------- /src/ZEO/shortrepr.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | 15 | REPR_LIMIT = 60 16 | 17 | 18 | def short_repr(obj): 19 | """Return an object repr limited to REPR_LIMIT bytes.""" 20 | 21 | # Some of the objects being repr'd are large strings. A lot of memory 22 | # would be wasted to repr them and then truncate, so they are treated 23 | # specially in this function. 24 | # Also handle short repr of a tuple containing a long string. 25 | 26 | # This strategy works well for arguments to StorageServer methods. 27 | # The oid is usually first and will get included in its entirety. 28 | # The pickle is near the beginning, too, and you can often fit the 29 | # module name in the pickle. 30 | 31 | if isinstance(obj, str): 32 | if len(obj) > REPR_LIMIT: 33 | r = repr(obj[:REPR_LIMIT]) 34 | else: 35 | r = repr(obj) 36 | if len(r) > REPR_LIMIT: 37 | r = r[:REPR_LIMIT - 4] + '...' + r[-1] 38 | return r 39 | elif isinstance(obj, (list, tuple)): 40 | elts = [] 41 | size = 0 42 | for elt in obj: 43 | r = short_repr(elt) 44 | elts.append(r) 45 | size += len(r) 46 | if size > REPR_LIMIT: 47 | break 48 | if isinstance(obj, tuple): 49 | r = "(%s)" % (", ".join(elts)) 50 | else: 51 | r = "[%s]" % (", ".join(elts)) 52 | else: 53 | r = repr(obj) 54 | if len(r) > REPR_LIMIT: 55 | return r[:REPR_LIMIT] + '...' 56 | else: 57 | return r 58 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Zope Public License (ZPL) Version 2.1 2 | 3 | A copyright notice accompanies this license document that identifies the 4 | copyright holders. 5 | 6 | This license has been certified as open source. It has also been designated as 7 | GPL compatible by the Free Software Foundation (FSF). 8 | 9 | Redistribution and use in source and binary forms, with or without 10 | modification, are permitted provided that the following conditions are met: 11 | 12 | 1. Redistributions in source code must retain the accompanying copyright 13 | notice, this list of conditions, and the following disclaimer. 14 | 15 | 2. Redistributions in binary form must reproduce the accompanying copyright 16 | notice, this list of conditions, and the following disclaimer in the 17 | documentation and/or other materials provided with the distribution. 18 | 19 | 3. Names of the copyright holders must not be used to endorse or promote 20 | products derived from this software without prior written permission from the 21 | copyright holders. 22 | 23 | 4. The right to distribute this software or to use it for any purpose does not 24 | give you the right to use Servicemarks (sm) or Trademarks (tm) of the 25 | copyright 26 | holders. Use of them is covered by separate agreement with the copyright 27 | holders. 28 | 29 | 5. If any files are modified, you must cause the modified files to carry 30 | prominent notices stating that you changed the files and the date of any 31 | change. 32 | 33 | Disclaimer 34 | 35 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESSED 36 | OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 37 | OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 38 | EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT, 39 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 41 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 42 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 43 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 44 | EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 | -------------------------------------------------------------------------------- /src/ZEO/tests/testTransactionBuffer.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | import random 15 | import unittest 16 | 17 | from ZEO.TransactionBuffer import TransactionBuffer 18 | 19 | 20 | def random_string(size): 21 | """Return a random string of size size.""" 22 | lst = [chr(random.randrange(256)) for i in range(size)] 23 | return "".join(lst) 24 | 25 | 26 | def new_store_data(): 27 | """Return arbitrary data to use as argument to store() method.""" 28 | return random_string(8), random_string(random.randrange(1000)) 29 | 30 | 31 | def store(tbuf, resolved=False): 32 | data = new_store_data() 33 | tbuf.store(*data) 34 | if resolved: 35 | tbuf.server_resolve(data[0]) 36 | return data 37 | 38 | 39 | class TransBufTests(unittest.TestCase): 40 | 41 | def checkTypicalUsage(self): 42 | tbuf = TransactionBuffer(0) 43 | store(tbuf) 44 | store(tbuf) 45 | for o in tbuf: 46 | pass 47 | tbuf.close() 48 | 49 | def checkOrderPreserved(self): 50 | tbuf = TransactionBuffer(0) 51 | data = [] 52 | for i in range(10): 53 | data.append((store(tbuf), False)) 54 | data.append((store(tbuf, True), True)) 55 | 56 | for i, (oid, d, resolved) in enumerate(tbuf): 57 | self.assertEqual((oid, d), data[i][0]) 58 | self.assertEqual(resolved, data[i][1]) 59 | tbuf.close() 60 | 61 | 62 | def test_suite(): 63 | test_loader = unittest.TestLoader() 64 | test_loader.testMethodPrefix = 'check' 65 | return test_loader.loadTestsFromTestCase(TransBufTests) 66 | -------------------------------------------------------------------------------- /src/ZEO/tests/dynamic_server_ports.test: -------------------------------------------------------------------------------- 1 | The storage server can be told to bind to port 0, allowing the OS to 2 | pick a port dynamically. For this to be useful, there needs to be a 3 | way to tell someone. For this reason, the server posts events to 4 | ZODB.notify. 5 | 6 | >>> import ZODB.event 7 | >>> old_notify = ZODB.event.notify 8 | 9 | >>> last_event = None 10 | >>> def notify(event): 11 | ... global last_event 12 | ... last_event = event 13 | >>> ZODB.event.notify = notify 14 | 15 | Now, let's start a server and verify that we get a serving event: 16 | 17 | >>> import ZEO 18 | >>> addr, stop = ZEO.server() 19 | 20 | >>> isinstance(last_event, ZEO.StorageServer.Serving) 21 | True 22 | 23 | >>> last_event.address == addr 24 | True 25 | 26 | >>> server = last_event.server 27 | >>> server.addr == addr 28 | True 29 | 30 | Let's make sure we can connect. 31 | 32 | >>> client = ZEO.client(last_event.address).close() 33 | 34 | If we close the server, we'll get a closed event: 35 | 36 | >>> stop() 37 | >>> isinstance(last_event, ZEO.StorageServer.Closed) 38 | True 39 | >>> last_event.server is server 40 | True 41 | 42 | If we pass an empty string as the host part of the server address, we 43 | can't really assign a single address, so the server addr attribute is 44 | left alone: 45 | 46 | >>> addr, stop = ZEO.server(port=('', 0)) 47 | >>> isinstance(last_event, ZEO.StorageServer.Serving) 48 | True 49 | 50 | >>> last_event.address[1] > 0 51 | True 52 | 53 | >>> last_event.server.addr 54 | ('', 0) 55 | 56 | >>> stop() 57 | 58 | The runzeo module provides some process support, including getting the 59 | server configuration via a ZConfig configuration file. To spell a 60 | dynamic port using ZConfig, you'd use a hostname by itself. In this 61 | case, ZConfig passes None as the port. 62 | 63 | >>> import ZEO.runzeo 64 | >>> with open('conf', 'w') as f: 65 | ... _ = f.write(""" 66 | ... 67 | ... address 127.0.0.1 68 | ... 69 | ... 70 | ... 71 | ... """) 72 | >>> options = ZEO.runzeo.ZEOOptions() 73 | >>> options.realize('-C conf'.split()) 74 | >>> options.address 75 | ('127.0.0.1', None) 76 | 77 | >>> rs = ZEO.runzeo.ZEOServer(options) 78 | >>> rs.check_socket() 79 | >>> options.address 80 | ('127.0.0.1', 0) 81 | 82 | 83 | .. cleanup 84 | 85 | >>> ZODB.event.notify = old_notify 86 | -------------------------------------------------------------------------------- /src/ZEO/tests/TestThread.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """A Thread base class for use with unittest.""" 15 | import sys 16 | import threading 17 | 18 | 19 | class TestThread(threading.Thread): 20 | """Base class for defining threads that run from unittest. 21 | 22 | The subclass should define a testrun() method instead of a run() 23 | method. 24 | 25 | Call cleanup() when the test is done with the thread, instead of join(). 26 | If the thread exits with an uncaught exception, it's captured and 27 | re-raised when cleanup() is called. cleanup() should be called by 28 | the main thread! Trying to tell unittest that a test failed from 29 | another thread creates a nightmare of timing-depending cascading 30 | failures and missed errors (tracebacks that show up on the screen, 31 | but don't cause unittest to believe the test failed). 32 | 33 | cleanup() also joins the thread. If the thread ended without raising 34 | an uncaught exception, and the join doesn't succeed in the timeout 35 | period, then the test is made to fail with a "Thread still alive" 36 | message. 37 | """ 38 | 39 | def __init__(self, testcase): 40 | threading.Thread.__init__(self) 41 | # In case this thread hangs, don't stop Python from exiting. 42 | self.daemon = True 43 | self._exc_info = None 44 | self._testcase = testcase 45 | 46 | def run(self): 47 | try: 48 | self.testrun() 49 | except: # NOQA: E722 blank except 50 | self._exc_info = sys.exc_info() 51 | 52 | def cleanup(self, timeout=15): 53 | self.join(timeout) 54 | if self._exc_info: 55 | et, ev, etb = self._exc_info 56 | if ev is None: 57 | ev = et() 58 | raise ev.with_traceback(etb) 59 | if self.is_alive(): 60 | self._testcase.fail("Thread did not finish: %s" % self) 61 | -------------------------------------------------------------------------------- /docs/protocol.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ZEO Network Protocol (sans authentication) 3 | ========================================== 4 | 5 | This document describes the ZEO network protocol. It assumes that the 6 | optional authentication protocol isn't used. At the lowest 7 | level, the protocol consists of sized messages. All communication 8 | between the client and server consists of sized messages. A sized 9 | message consists of a 4-byte unsigned big-endian content length, 10 | followed by the content. There are two subprotocols, for protocol 11 | negotiation, and for normal operation. The normal operation protocol 12 | is a basic RPC protocol. 13 | 14 | In the protocol negotiation phase, the server sends a protocol 15 | identifier to the client. The client chooses a protocol to use to the 16 | server. The client or the server can fail if it doesn't like the 17 | protocol string sent by the other party. After sending their protocol 18 | strings, the client and server switch to RPC mode. 19 | 20 | The RPC protocol uses messages that are pickled tuples consisting of: 21 | 22 | message_id 23 | The message id is used to match replies with requests, allowing 24 | multiple outstanding synchronous requests. 25 | 26 | async_flag 27 | An integer 0 for a regular (2-way) request and 1 for a one-way 28 | request. Two-way requests have a reply. One way requests don't. 29 | ZRS tries to use as many one-way requests as possible to avoid 30 | network round trips. 31 | 32 | name 33 | The name of a method to call. If this is the special string 34 | ".reply", then the message is interpreted as a return from a 35 | synchronous call. 36 | 37 | args 38 | A tuple of positional arguments or returned values. 39 | 40 | After making a connection and negotiating the protocol, the following 41 | interactions occur: 42 | 43 | - The client requests the authentication protocol by calling 44 | getAuthProtocol. For this discussion, we'll assume the server 45 | returns None. Note that if the server doesn't require 46 | authentication, this step is optional. 47 | 48 | - The client calls register passing a storage identifier and a 49 | read-only flag. The server doesn't return a value, but it may raise 50 | an exception either if the storage doesn't exist, or if the 51 | storage is readonly and the read-only flag passed by the client is 52 | false. 53 | 54 | At this point, the client and server send each other messages as 55 | needed. The client may make regular or one-way calls to the 56 | server. The server sends replies and one-way calls to the client. 57 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | name: tests 4 | 5 | on: 6 | push: 7 | pull_request: 8 | schedule: 9 | - cron: '0 12 * * 0' # run once a week on Sunday 10 | # Allow to run this workflow manually from the Actions tab 11 | workflow_dispatch: 12 | 13 | jobs: 14 | build: 15 | permissions: 16 | contents: read 17 | pull-requests: write 18 | strategy: 19 | # We want to see all failures: 20 | fail-fast: false 21 | matrix: 22 | os: 23 | - ["ubuntu", "ubuntu-latest"] 24 | config: 25 | # [Python version, tox env] 26 | - ["3.11", "release-check"] 27 | - ["3.9", "py39"] 28 | - ["3.10", "py310"] 29 | - ["3.11", "py311"] 30 | - ["3.12", "py312"] 31 | - ["3.13", "py313"] 32 | - ["pypy-3.10", "pypy3"] 33 | - ["3.11", "docs"] 34 | - ["3.11", "coverage"] 35 | - ["3.9", "py39-msgpack1"] 36 | - ["3.9", "py39-uvloop"] 37 | - ["3.10", "py310-msgpack1"] 38 | - ["3.10", "py310-uvloop"] 39 | - ["3.11", "py311-msgpack1"] 40 | - ["3.11", "py311-uvloop"] 41 | - ["3.12", "py312-msgpack1"] 42 | - ["3.12", "py312-uvloop"] 43 | - ["3.13", "py313-msgpack1"] 44 | - ["3.13", "py313-uvloop"] 45 | - ["3.13", "py313-zodbmaster"] 46 | - ["pypy-3.10", "pypy3-msgpack1"] 47 | 48 | runs-on: ${{ matrix.os[1] }} 49 | if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name 50 | name: ${{ matrix.config[1] }} 51 | steps: 52 | - uses: actions/checkout@v4 53 | with: 54 | persist-credentials: false 55 | - name: Install uv + caching 56 | uses: astral-sh/setup-uv@v6 57 | with: 58 | enable-cache: true 59 | cache-dependency-glob: | 60 | setup.* 61 | tox.ini 62 | python-version: ${{ matrix.matrix.config[0] }} 63 | github-token: ${{ secrets.GITHUB_TOKEN }} 64 | - name: Test 65 | if: ${{ !startsWith(runner.os, 'Mac') }} 66 | run: uvx --with tox-uv tox -e ${{ matrix.config[1] }} 67 | - name: Test (macOS) 68 | if: ${{ startsWith(runner.os, 'Mac') }} 69 | run: uvx --with tox-uv tox -e ${{ matrix.config[1] }}-universal2 70 | - name: Coverage 71 | if: matrix.config[1] == 'coverage' 72 | run: | 73 | uvx coveralls --service=github 74 | env: 75 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 76 | -------------------------------------------------------------------------------- /src/ZEO/tests/racetest.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2022 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """Module racetest provides infrastructure for tests that catch concurrency 15 | problems in ZEO. 16 | 17 | It complements ZODB.tests.racetest . 18 | """ 19 | 20 | import time 21 | 22 | import zope.interface 23 | 24 | 25 | # LoadDelayedStorage wraps base storage in injects delays after load* 26 | # operations. 27 | # 28 | # It is useful to catch concurrency problems due to races in between load and 29 | # other storage events, e.g. invalidations. 30 | # 31 | # See e.g. https://github.com/zopefoundation/ZEO/issues/209 for example where 32 | # injecting such delays was effective to catch bugs that were otherwise hard to 33 | # reproduce. 34 | class LoadDelayedStorage: 35 | 36 | def __init__(self, base, tdelay=0.01): 37 | self.base = base 38 | self.tdelay = tdelay 39 | zope.interface.directlyProvides(self, zope.interface.providedBy(base)) 40 | 41 | def _delay(self): 42 | time.sleep(self.tdelay) 43 | 44 | def __getattr__(self, name): 45 | return getattr(self.base, name) 46 | 47 | def __len__(self): # ^^^ __getattr__ does not forward __ 48 | return len(self.base) # because they are mangled 49 | 50 | def load(self, *argv, **kw): 51 | _ = self.base.load(*argv, **kw) 52 | self._delay() 53 | return _ 54 | 55 | def loadBefore(self, *argv, **kw): 56 | _ = self.base.loadBefore(*argv, **kw) 57 | self._delay() 58 | return _ 59 | 60 | def loadSerial(self, *argv, **kw): 61 | _ = self.base.loadSerial(*argv, **kw) 62 | self._delay() 63 | return _ 64 | 65 | 66 | class ZConfigLoadDelayed: 67 | 68 | _factory = LoadDelayedStorage 69 | 70 | def __init__(self, config): 71 | self.config = config 72 | self.name = config.getSectionName() 73 | 74 | def open(self): 75 | base = self.config.base.open() 76 | return self._factory(base) 77 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Generated from: 2 | # https://github.com/zopefoundation/meta/tree/master/config/pure-python 3 | [tox] 4 | minversion = 3.18 5 | envlist = 6 | release-check 7 | lint 8 | py39 9 | py310 10 | py311 11 | py312 12 | py313 13 | pypy3 14 | docs 15 | coverage 16 | 17 | [testenv] 18 | usedevelop = true 19 | package = wheel 20 | wheel_build_env = .pkg 21 | deps = 22 | setuptools == 75.8.2 23 | !zodbmaster: ZODB >= 4.2.0b1 24 | zodbmaster: git+https://github.com/zopefoundation/ZODB.git@master\#egg=ZODB 25 | uvloop: uvloop 26 | setenv = 27 | msgpack1: ZEO_MSGPACK=1 28 | commands = 29 | # Run unit tests first. 30 | zope-testrunner -u --test-path=src -a 1000 {posargs:-vc} 31 | # Only run functional tests if unit tests pass. 32 | zope-testrunner -f --test-path=src -a 1000 {posargs:-vc} 33 | extras = 34 | test 35 | 36 | [testenv:setuptools-latest] 37 | basepython = python3 38 | deps = 39 | git+https://github.com/pypa/setuptools.git\#egg=setuptools 40 | !zodbmaster: ZODB >= 4.2.0b1 41 | zodbmaster: git+https://github.com/zopefoundation/ZODB.git@master\#egg=ZODB 42 | uvloop: uvloop 43 | 44 | [testenv:release-check] 45 | description = ensure that the distribution is ready to release 46 | basepython = python3 47 | skip_install = true 48 | deps = 49 | setuptools == 75.8.2 50 | wheel 51 | twine 52 | build 53 | check-manifest 54 | check-python-versions >= 0.20.0 55 | wheel 56 | commands_pre = 57 | commands = 58 | check-manifest 59 | check-python-versions --only setup.py,tox.ini,.github/workflows/tests.yml 60 | python -m build --sdist --no-isolation 61 | twine check dist/* 62 | 63 | [testenv:lint] 64 | description = This env runs all linters configured in .pre-commit-config.yaml 65 | basepython = python3 66 | skip_install = true 67 | deps = 68 | pre-commit 69 | commands_pre = 70 | commands = 71 | pre-commit run --all-files --show-diff-on-failure 72 | 73 | [testenv:docs] 74 | basepython = python3 75 | skip_install = false 76 | extras = 77 | docs 78 | commands_pre = 79 | commands = 80 | sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html 81 | 82 | [testenv:coverage] 83 | basepython = python3 84 | allowlist_externals = 85 | mkdir 86 | deps = 87 | coverage[toml] 88 | !zodbmaster: ZODB >= 4.2.0b1 89 | zodbmaster: git+https://github.com/zopefoundation/ZODB.git@master\#egg=ZODB 90 | uvloop: uvloop 91 | commands = 92 | mkdir -p {toxinidir}/parts/htmlcov 93 | coverage run -m zope.testrunner --test-path=src {posargs:-vc} 94 | coverage html 95 | coverage report 96 | -------------------------------------------------------------------------------- /src/ZEO/zconfig.py: -------------------------------------------------------------------------------- 1 | """SSL configuration support 2 | """ 3 | import os 4 | 5 | 6 | def ssl_config(section, server): 7 | import ssl 8 | 9 | cafile = capath = None 10 | auth = section.authenticate 11 | if auth: 12 | if os.path.isdir(auth): 13 | capath = auth 14 | elif auth != 'DYNAMIC': 15 | cafile = auth 16 | 17 | context = ssl.create_default_context( 18 | ssl.Purpose.CLIENT_AUTH if server else ssl.Purpose.SERVER_AUTH, 19 | cafile=cafile, capath=capath) 20 | 21 | if not auth: 22 | assert not server 23 | context.load_default_certs() 24 | 25 | if section.certificate: 26 | password = section.password_function 27 | if password: 28 | module, name = password.rsplit('.', 1) 29 | module = __import__(module, globals(), locals(), ['*'], 0) 30 | password = getattr(module, name) 31 | context.load_cert_chain(section.certificate, section.key, password) 32 | 33 | context.verify_mode = ssl.CERT_REQUIRED 34 | 35 | context.verify_flags |= ssl.VERIFY_X509_STRICT | ( 36 | context.cert_store_stats()['crl'] and ssl.VERIFY_CRL_CHECK_LEAF) 37 | 38 | if server: 39 | context.check_hostname = False 40 | return context 41 | 42 | context.check_hostname = section.check_hostname 43 | 44 | return context, section.server_hostname 45 | 46 | 47 | def server_ssl(section): 48 | return ssl_config(section, True) 49 | 50 | 51 | def client_ssl(section): 52 | return ssl_config(section, False) 53 | 54 | 55 | class ClientStorageConfig: 56 | 57 | def __init__(self, config): 58 | self.config = config 59 | self.name = config.getSectionName() 60 | 61 | def open(self): 62 | from ZEO.ClientStorage import ClientStorage 63 | 64 | # config.server is a multikey of socket-connection-address values 65 | # where the value is a socket family, address tuple. 66 | config = self.config 67 | 68 | addresses = [server.address for server in config.server] 69 | options = {} 70 | if config.blob_cache_size is not None: 71 | options['blob_cache_size'] = config.blob_cache_size 72 | if config.blob_cache_size_check is not None: 73 | options['blob_cache_size_check'] = config.blob_cache_size_check 74 | if config.client_label is not None: 75 | options['client_label'] = config.client_label 76 | 77 | ssl = config.ssl 78 | if ssl: 79 | options['ssl'] = ssl[0] 80 | options['ssl_server_hostname'] = ssl[1] 81 | 82 | return ClientStorage( 83 | addresses, 84 | blob_dir=config.blob_dir, 85 | shared_blob_dir=config.shared_blob_dir, 86 | storage=config.storage, 87 | cache_size=config.cache_size, 88 | cache=config.cache_path, 89 | name=config.name, 90 | read_only=config.read_only, 91 | read_only_fallback=config.read_only_fallback, 92 | server_sync=config.server_sync, 93 | wait_timeout=config.wait_timeout, 94 | **options) 95 | -------------------------------------------------------------------------------- /docs/blob-nfs.rst: -------------------------------------------------------------------------------- 1 | =========================================== 2 | How to use NFS to make Blobs more efficient 3 | =========================================== 4 | 5 | :Author: Christian Theune 6 | 7 | Overview 8 | ======== 9 | 10 | When handling blobs, the biggest goal is to avoid writing operations that 11 | require the blob data to be transferred using up IO resources. 12 | 13 | When bringing a blob into the system, at least one O(N) operation has to 14 | happen, e.g. when the blob is uploaded via a network server. The blob should 15 | be extracted as a file on the final storage volume as early as possible, 16 | avoiding further copies. 17 | 18 | In a ZEO setup, all data is stored on a networked server and passed to it 19 | using zrpc. This is a major problem for handling blobs, because it will lock 20 | all transactions from committing when storing a single large blob. As a 21 | default, this mechanism works but is not recommended for high-volume 22 | installations. 23 | 24 | Shared filesystem 25 | ================= 26 | 27 | The solution for the transfer problem is to setup various storage parameters 28 | so that blobs are always handled on a single volume that is shared via network 29 | between ZEO servers and clients. 30 | 31 | Step 1: Setup a writable shared filesystem for ZEO server and client 32 | -------------------------------------------------------------------- 33 | 34 | On the ZEO server, create two directories on the volume that will be used by 35 | this setup (assume the volume is accessible via $SERVER/): 36 | 37 | - $SERVER/blobs 38 | 39 | - $SERVER/tmp 40 | 41 | Then export the $SERVER directory using a shared network filesystem like NFS. 42 | Make sure it's writable by the ZEO clients. 43 | 44 | Assume the exported directory is available on the client as $CLIENT. 45 | 46 | Step 2: Application temporary directories 47 | ----------------------------------------- 48 | 49 | Applications (i.e. Zope) will put uploaded data in a temporary directory 50 | first. Adjust your TMPDIR, TMP or TEMP environment variable to point to the 51 | shared filesystem: 52 | 53 | $ export TMPDIR=$CLIENT/tmp 54 | 55 | Step 3: ZEO client caches 56 | ------------------------- 57 | 58 | Edit the file `zope.conf` on the ZEO client and adjust the configuration of 59 | the `zeoclient` storage with two new variables:: 60 | 61 | blob-dir = $CLIENT/blobs 62 | blob-cache-writable = yes 63 | 64 | Step 4: ZEO server 65 | ------------------ 66 | 67 | Edit the file `zeo.conf` on the ZEO server to configure the blob directory. 68 | Assuming the published storage of the ZEO server is a file storage, then the 69 | configuration should look like this:: 70 | 71 | 72 | 73 | path $INSTANCE/var/Data.fs 74 | 75 | blob-dir $SERVER/blobs 76 | 77 | 78 | (Remember to manually replace $SERVER and $CLIENT with the exported directory 79 | as accessible by either the ZEO server or the ZEO client.) 80 | 81 | Conclusion 82 | ---------- 83 | 84 | At this point, after restarting your ZEO server and clients, the blob 85 | directory will be shared and a minimum amount of IO will occur when working 86 | with blobs. 87 | -------------------------------------------------------------------------------- /src/ZEO/tests/servertesting.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | 15 | # Testing the current ZEO implementation is rather hard due to the 16 | # architecture, which mixes concerns, especially between application 17 | # and networking. Still, it's not as bad as it could be. 18 | 19 | # The 2 most important classes in the architecture are ZEOStorage and 20 | # StorageServer. A ZEOStorage is created for each client connection. 21 | # The StorageServer maintains data shared or needed for coordination 22 | # among clients. 23 | 24 | # The other important part of the architecture is connections. 25 | # Connections are used by ZEOStorages to send messages or return data 26 | # to clients. 27 | 28 | # Here, we'll try to provide some testing infrastructure to isolate 29 | # servers from the network. 30 | 31 | import ZODB.MappingStorage 32 | 33 | import ZEO.asyncio.tests 34 | import ZEO.StorageServer 35 | 36 | 37 | class StorageServer(ZEO.StorageServer.StorageServer): 38 | def __init__(self, addr='test_addr', storages=None, **kw): 39 | if storages is None: 40 | storages = {'1': ZODB.MappingStorage.MappingStorage()} 41 | ZEO.StorageServer.StorageServer.__init__(self, addr, storages, **kw) 42 | 43 | def close(self): 44 | if self.__closed: 45 | return 46 | # instances are typically not run in their own thread 47 | # therefore, the loop usually does not run and the 48 | # normal ``close`` does not work. 49 | loop = self.get_loop() 50 | if loop.is_running(): 51 | return super().close() 52 | loop.call_soon_threadsafe(super().close) 53 | loop.run_forever() # will stop automatically 54 | loop.close() 55 | 56 | def get_loop(self): 57 | return self.acceptor.event_loop # might not work for ``MTAcceptor`` 58 | 59 | 60 | def client(server, name='client'): 61 | zs = ZEO.StorageServer.ZEOStorage(server) 62 | protocol = ZEO.asyncio.tests.server_protocol( 63 | False, zs, protocol_version=b'Z5', addr='test-addr-%s' % name) 64 | 65 | # ``server_protocol`` uses its own testing loop (not 66 | # that of *server*). As a consequence, ``protocol.close`` 67 | # does not work correctly. 68 | # In addition, the artificial loop needs to get closed. 69 | pr_close = protocol.close 70 | 71 | def close(*args, **kw): 72 | pr_close() 73 | zs.notify_disconnected() 74 | protocol.loop.close() 75 | del protocol.close # break reference cycle 76 | 77 | protocol.close = close # install proper closing 78 | zs.notify_connected(protocol) 79 | zs.register('1', 0) 80 | return zs 81 | -------------------------------------------------------------------------------- /src/ZEO/monitor.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2003 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """Monitor behavior of ZEO server and record statistics. 15 | """ 16 | 17 | import time 18 | 19 | 20 | class StorageStats: 21 | """Per-storage usage statistics.""" 22 | 23 | def __init__(self, connections=None): 24 | self.connections = connections 25 | self.loads = 0 26 | self.stores = 0 27 | self.commits = 0 28 | self.aborts = 0 29 | self.active_txns = 0 30 | self.lock_time = None 31 | self.conflicts = 0 32 | self.conflicts_resolved = 0 33 | self.start = time.ctime() 34 | 35 | @property 36 | def clients(self): 37 | return len(self.connections) 38 | 39 | def parse(self, s): 40 | # parse the dump format 41 | lines = s.split("\n") 42 | for line in lines: 43 | field, value = line.split(":", 1) 44 | if field == "Server started": 45 | self.start = value 46 | elif field == "Clients": 47 | # Hack because we use this both on the server and on 48 | # the client where there are no connections. 49 | self.connections = [0] * int(value) 50 | elif field == "Clients verifying": 51 | self.verifying_clients = int(value) 52 | elif field == "Active transactions": 53 | self.active_txns = int(value) 54 | elif field == "Commit lock held for": 55 | # This assumes 56 | self.lock_time = time.time() - int(value) 57 | elif field == "Commits": 58 | self.commits = int(value) 59 | elif field == "Aborts": 60 | self.aborts = int(value) 61 | elif field == "Loads": 62 | self.loads = int(value) 63 | elif field == "Stores": 64 | self.stores = int(value) 65 | elif field == "Conflicts": 66 | self.conflicts = int(value) 67 | elif field == "Conflicts resolved": 68 | self.conflicts_resolved = int(value) 69 | 70 | def dump(self, f): 71 | print("Server started:", self.start, file=f) 72 | print("Clients:", self.clients, file=f) 73 | print("Clients verifying:", self.verifying_clients, file=f) 74 | print("Active transactions:", self.active_txns, file=f) 75 | if self.lock_time: 76 | howlong = time.time() - self.lock_time 77 | print("Commit lock held for:", int(howlong), file=f) 78 | print("Commits:", self.commits, file=f) 79 | print("Aborts:", self.aborts, file=f) 80 | print("Loads:", self.loads, file=f) 81 | print("Stores:", self.stores, file=f) 82 | print("Conflicts:", self.conflicts, file=f) 83 | print("Conflicts resolved:", self.conflicts_resolved, file=f) 84 | -------------------------------------------------------------------------------- /src/ZEO/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """ZEO -- Zope Enterprise Objects. 15 | 16 | See the file README.txt in this directory for an overview. 17 | 18 | ZEO is now part of ZODB; ZODB's home on the web is 19 | 20 | http://wiki.zope.org/ZODB 21 | 22 | """ 23 | 24 | 25 | def client(*args, **kw): 26 | """ 27 | Shortcut for :class:`ZEO.ClientStorage.ClientStorage`. 28 | """ 29 | import ZEO.ClientStorage 30 | return ZEO.ClientStorage.ClientStorage(*args, **kw) 31 | 32 | 33 | def DB(*args, **kw): 34 | """ 35 | Shortcut for creating a :class:`ZODB.DB` using a ZEO :func:`~ZEO.client`. 36 | """ 37 | s = client(*args, **kw) 38 | try: 39 | import ZODB 40 | return ZODB.DB(s) 41 | except Exception: 42 | s.close() 43 | raise 44 | 45 | 46 | def connection(*args, **kw): 47 | db = DB(*args, **kw) 48 | try: 49 | return db.open_then_close_db_when_connection_closes() 50 | except Exception: 51 | db.close() 52 | raise 53 | 54 | 55 | def server(path=None, blob_dir=None, storage_conf=None, zeo_conf=None, 56 | port=0, threaded=True, **kw): 57 | """Convenience function to start a server for interactive exploration 58 | 59 | This fuction starts a ZEO server, given a storage configuration or 60 | a file-storage path and blob directory. You can also supply a ZEO 61 | configuration string or a port. If neither a ZEO port or 62 | configuration is supplied, a port is chosen randomly. 63 | 64 | The server address and a stop function are returned. The address 65 | can be passed to ZEO.ClientStorage.ClientStorage or ZEO.DB to 66 | create a client to the server. The stop function can be called 67 | without arguments to stop the server. 68 | 69 | Arguments: 70 | 71 | path 72 | A file-storage path. This argument is ignored if a storage 73 | configuration is supplied. 74 | 75 | blob_dir 76 | A blob directory path. This argument is ignored if a storage 77 | configuration is supplied. 78 | 79 | storage_conf 80 | A storage configuration string. If none is supplied, then at 81 | least a file-storage path must be supplied and the storage 82 | configuration will be generated from the file-storage path and 83 | the blob directory. 84 | 85 | zeo_conf 86 | A ZEO server configuration string. 87 | 88 | port 89 | If no ZEO configuration is supplied, the one will be computed 90 | from the port. If no port is supplied, one will be chosedn 91 | dynamically. 92 | 93 | """ 94 | import ZEO._forker as forker 95 | if storage_conf is None and path is None: 96 | storage_conf = '\n' 97 | 98 | return forker.start_zeo_server( 99 | storage_conf, zeo_conf, port, keep=True, path=path, 100 | blob_dir=blob_dir, suicide=False, threaded=threaded, **kw) 101 | -------------------------------------------------------------------------------- /src/ZEO/tests/client-config.test: -------------------------------------------------------------------------------- 1 | ZEO Client Configuration 2 | ======================== 3 | 4 | Here we'll describe (and test) the various ZEO Client configuration 5 | options. To facilitate this, we'l start a server that our client can 6 | connect to: 7 | 8 | >>> addr, _ = start_server(blob_dir='server-blobs') 9 | 10 | The simplest client configuration specified a server address: 11 | 12 | >>> import ZODB.config 13 | >>> storage = ZODB.config.storageFromString(""" 14 | ... 15 | ... server %s:%s 16 | ... 17 | ... """ % addr) 18 | 19 | >>> storage.getName(), storage.__class__.__name__ 20 | ... # doctest: +ELLIPSIS 21 | ("[('127.0.0.1', ...)] (connected)", 'ClientStorage') 22 | 23 | >>> storage.blob_dir 24 | >>> storage._storage 25 | '1' 26 | >>> storage._cache.maxsize 27 | 20971520 28 | >>> storage._cache.path 29 | >>> storage._is_read_only 30 | False 31 | >>> storage._read_only_fallback 32 | False 33 | >>> storage._blob_cache_size 34 | 35 | >>> storage.close() 36 | 37 | >>> storage = ZODB.config.storageFromString(""" 38 | ... 39 | ... server %s:%s 40 | ... blob-dir blobs 41 | ... storage 2 42 | ... cache-size 100 43 | ... name bob 44 | ... client cache 45 | ... read-only true 46 | ... drop-cache-rather-verify true 47 | ... blob-cache-size 1000MB 48 | ... blob-cache-size-check 10 49 | ... wait false 50 | ... 51 | ... """ % addr) 52 | 53 | 54 | >>> storage.getName(), storage.__class__.__name__ 55 | ('bob (disconnected)', 'ClientStorage') 56 | 57 | >>> storage.blob_dir 58 | 'blobs' 59 | >>> storage._storage 60 | '2' 61 | >>> storage._cache.maxsize 62 | 100 63 | >>> import os 64 | >>> storage._cache.path == os.path.abspath('cache-2.zec') 65 | True 66 | 67 | >>> storage._is_read_only 68 | True 69 | >>> storage._read_only_fallback 70 | False 71 | >>> storage._blob_cache_size 72 | 1048576000 73 | 74 | >>> print(storage._blob_cache_size_check) 75 | 104857600 76 | 77 | In isolated runs, the ``close`` below failed with a timeout. 78 | Set up a log handler to get at the associated log message. 79 | Note that we are interested only in this specific log message 80 | (from `ZEO.asyncio.client.ClientIO.close_co`). 81 | Occasionally (when the server response for the access to the 82 | non existing storage `2` is processed sufficiently fast), 83 | there will we other error log entries; we ignore them. 84 | 85 | Note: ``InstalledHandler`` changes the logger's level. 86 | Therefore, we cannot set ``level`` to ``ERROR`` even though 87 | we are only interested in error messages for this test (it would discard 88 | other log messages important for the timeout analysis). 89 | Instead, we initially request the ``DEBUG`` level 90 | and later set the handler's level to ``ERROR``. 91 | The main logfile might get unusual entries as a side effect. 92 | 93 | >>> import logging 94 | >>> from zope.testing.loggingsupport import InstalledHandler 95 | >>> handler = InstalledHandler("ZEO.asyncio.client", level=logging.DEBUG) 96 | >>> handler.setLevel(logging.ERROR) 97 | 98 | >>> storage.close() 99 | 100 | Check log records 101 | 102 | >>> def check_log_records(records): 103 | ... records = [r.message for r in records] 104 | ... return any("as a bug" in r for r in records) and records or [] 105 | >>> check_log_records(handler.records) 106 | [] 107 | 108 | Cleanup 109 | 110 | >>> handler.uninstall() 111 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2002, 2003 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | 15 | from setuptools import find_packages 16 | from setuptools import setup 17 | 18 | 19 | version = '6.2.dev0' 20 | 21 | install_requires = [ 22 | 'ZODB >= 5.1.1', 23 | 'transaction >= 2.0.3', 24 | 'persistent >= 4.1.0', 25 | 'zc.lockfile', 26 | 'ZConfig', 27 | 'zdaemon', 28 | 'zope.interface', 29 | ] 30 | 31 | tests_require = [ 32 | # We rely on implementation details of 33 | # test mocks. See https://github.com/zopefoundation/ZODB/pull/222 34 | 'ZConfig', 35 | 'ZODB >= 5.5.1', 36 | 'ZopeUndo', 37 | 'zope.testing', 38 | 'transaction', 39 | 'msgpack', 40 | 'zdaemon', 41 | 'zope.testrunner', 42 | ] 43 | 44 | 45 | long_description = ( 46 | open('README.rst').read() 47 | + '\n' + 48 | open('CHANGES.rst').read() 49 | ) 50 | 51 | setup(name="ZEO", 52 | version=version, 53 | description=long_description.split('\n', 2)[1], 54 | long_description=long_description, 55 | url='https://github.com/zopefoundation/ZEO', 56 | author="Zope Foundation and Contributors", 57 | author_email="zodb@googlegroups.com", 58 | keywords=['database', 'zodb'], 59 | packages=find_packages('src'), 60 | package_dir={'': 'src'}, 61 | license="ZPL-2.1", 62 | platforms=["any"], 63 | classifiers=[ 64 | "Intended Audience :: Developers", 65 | "License :: OSI Approved :: Zope Public License", 66 | "Programming Language :: Python :: 3", 67 | "Programming Language :: Python :: 3.9", 68 | "Programming Language :: Python :: 3.10", 69 | "Programming Language :: Python :: 3.11", 70 | "Programming Language :: Python :: 3.12", 71 | "Programming Language :: Python :: 3.13", 72 | "Programming Language :: Python :: Implementation :: CPython", 73 | "Programming Language :: Python :: Implementation :: PyPy", 74 | "Topic :: Database", 75 | "Topic :: Software Development :: Libraries :: Python Modules", 76 | "Operating System :: Microsoft :: Windows", 77 | "Operating System :: Unix", 78 | "Framework :: ZODB", 79 | ], 80 | extras_require={ 81 | 'test': tests_require, 82 | 'uvloop': [ 83 | 'uvloop >=0.5.1' 84 | ], 85 | 'msgpack': [ 86 | 'msgpack-python' 87 | ], 88 | 'docs': [ 89 | 'Sphinx', 90 | 'repoze.sphinx.autointerface', 91 | 'sphinx_rtd_theme', 92 | ], 93 | }, 94 | install_requires=install_requires, 95 | zip_safe=False, 96 | entry_points=""" 97 | [console_scripts] 98 | zeopack = ZEO.scripts.zeopack:main 99 | runzeo = ZEO.runzeo:main 100 | zeoctl = ZEO.zeoctl:main 101 | zeo-nagios = ZEO.nagios:main 102 | """, 103 | include_package_data=True, 104 | python_requires='>=3.9', 105 | ) 106 | -------------------------------------------------------------------------------- /src/ZEO/TransactionBuffer.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """A TransactionBuffer store transaction updates until commit or abort. 15 | 16 | A transaction may generate enough data that it is not practical to 17 | always hold pending updates in memory. Instead, a TransactionBuffer 18 | is used to store the data until a commit or abort. 19 | """ 20 | 21 | # A faster implementation might store trans data in memory until it 22 | # reaches a certain size. 23 | 24 | import tempfile 25 | 26 | from zodbpickle.pickle import Pickler 27 | from zodbpickle.pickle import Unpickler 28 | 29 | 30 | class TransactionBuffer: 31 | 32 | # The TransactionBuffer is used by client storage to hold update 33 | # data until the tpc_finish(). It is only used by a single 34 | # thread, because only one thread can be in the two-phase commit 35 | # at one time. 36 | 37 | def __init__(self, connection_generation): 38 | self.connection_generation = connection_generation 39 | self.file = tempfile.TemporaryFile(suffix=".tbuf") 40 | self.count = 0 41 | self.size = 0 42 | self.blobs = [] 43 | # It's safe to use a fast pickler because the only objects 44 | # stored are builtin types -- strings or None. 45 | self.pickler = Pickler(self.file, 1) 46 | self.pickler.fast = 1 47 | self.server_resolved = set() # {oid} 48 | self.client_resolved = {} # {oid -> buffer_record_number} 49 | self.exception = None 50 | 51 | def close(self): 52 | self.file.close() 53 | 54 | def store(self, oid, data): 55 | """Store oid, version, data for later retrieval""" 56 | self.pickler.dump((oid, data)) 57 | self.count += 1 58 | # Estimate per-record cache size 59 | self.size = self.size + (data and len(data) or 0) + 31 60 | 61 | def resolve(self, oid, data): 62 | """Record client-resolved data 63 | """ 64 | self.store(oid, data) 65 | self.client_resolved[oid] = self.count - 1 66 | 67 | def server_resolve(self, oid): 68 | self.server_resolved.add(oid) 69 | 70 | def storeBlob(self, oid, blobfilename): 71 | self.blobs.append((oid, blobfilename)) 72 | 73 | def __iter__(self): 74 | self.file.seek(0) 75 | unpickler = Unpickler(self.file) 76 | server_resolved = self.server_resolved 77 | client_resolved = self.client_resolved 78 | 79 | # Gaaaa, this is awkward. There can be entries in serials that 80 | # aren't in the buffer, because undo. Entries can be repeated 81 | # in the buffer, because ZODB. (Maybe this is a bug now, but 82 | # it may be a feature later. 83 | 84 | seen = set() 85 | for i in range(self.count): 86 | oid, data = unpickler.load() 87 | if client_resolved.get(oid, i) == i: 88 | seen.add(oid) 89 | yield oid, data, oid in server_resolved 90 | 91 | # We may have leftover oids because undo 92 | for oid in server_resolved: 93 | if oid not in seen: 94 | yield oid, None, True 95 | -------------------------------------------------------------------------------- /src/ZEO/interfaces.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2006 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | 15 | import zope.interface 16 | 17 | 18 | class StaleCache: 19 | """A ZEO cache is stale and requires verification. 20 | """ 21 | 22 | def __init__(self, storage): 23 | self.storage = storage 24 | 25 | 26 | class IClientCache(zope.interface.Interface): 27 | """Client cache interface. 28 | 29 | Note that caches need to be thread safe. 30 | """ 31 | 32 | def close(): 33 | """Close the cache 34 | """ 35 | 36 | def load(oid): 37 | """Get current data for object 38 | 39 | Returns data and serial, or None. 40 | """ 41 | 42 | def __len__(): 43 | """Retirn the number of items in the cache. 44 | """ 45 | 46 | def store(oid, start_tid, end_tid, data): 47 | """Store data for the object 48 | 49 | The start_tid is the transaction that committed this data. 50 | 51 | The end_tid is the tid of the next transaction that modified 52 | the objects, or None if this is the current version. 53 | """ 54 | 55 | def loadBefore(oid, tid): 56 | """Load the data for the object last modified before the tid 57 | 58 | Returns the data, and start and end tids. 59 | """ 60 | 61 | def invalidate(oid, tid): 62 | """Invalidate data for the object 63 | 64 | If ``tid`` is None, forget all knowledge of `oid`. (``tid`` 65 | can be None only for invalidations generated by startup cache 66 | verification.) 67 | 68 | If ``tid`` isn't None, and we had current data for ``oid``, 69 | stop believing we have current data, and mark the data we had 70 | as being valid only up to `tid`. In all other cases, do 71 | nothing. 72 | """ 73 | 74 | def getLastTid(): 75 | """Get the last tid seen by the cache 76 | 77 | This is the cached last tid we've seen from the server. 78 | 79 | This method may be called from multiple threads. (It's assumed 80 | to be trivial.) 81 | """ 82 | 83 | def setLastTid(tid): 84 | """Save the last tid sent by the server 85 | """ 86 | 87 | def clear(): 88 | """Clear/empty the cache 89 | """ 90 | 91 | 92 | class IServeable(zope.interface.Interface): 93 | """Interface provided by storages that can be served by ZEO 94 | """ 95 | 96 | def tpc_transaction(): 97 | """The current transaction being committed. 98 | 99 | If a storage is participating in a two-phase commit, then 100 | return the transaction (object) being committed. Otherwise 101 | return None. 102 | """ 103 | 104 | def lastInvalidations(size): 105 | """Get recent transaction invalidations 106 | 107 | This method is optional and is used to get invalidations 108 | performed by the most recent transactions. 109 | 110 | An iterable of up to size entries must be returned, where each 111 | entry is a transaction id and a sequence of object-id/empty-string 112 | pairs describing the objects written by the 113 | transaction, in chronological order. 114 | """ 115 | -------------------------------------------------------------------------------- /src/ZEO/asyncio/README.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | asyncio-based networking for ZEO 3 | ================================ 4 | 5 | This package provides the networking interface for ZEO. It provides a 6 | somewhat RPC-like API. 7 | 8 | Notes 9 | ===== 10 | 11 | Sending data immediately: asyncio vs asyncore 12 | -------------------------------------------- 13 | 14 | The previous ZEO networking implementation used the ``asyncore`` library. 15 | When writing with asyncore, writes were done only from the event loop. 16 | This meant that when sending data, code would have to "wake up" the 17 | event loop, typically after adding data to some sort of output buffer. 18 | 19 | Asyncio takes an entirely different and saner approach. When an 20 | application wants to send data, it writes to a transport. All 21 | interactions with a transport (in a correct application) are from the 22 | same thread, which is also the thread running any event loop. 23 | Transports are always either idle or sending data. When idle, the 24 | transport writes to the outout socket immediately. If not all data 25 | isn't sent, then it buffers it and becomes sending. If a transport is 26 | sending, then we know that the socket isn't ready for more data, so 27 | ``write`` can just buffer the data. There's no point in waking up the 28 | event loop, because the socket will do so when it's ready for more 29 | data. 30 | 31 | An exception to the paragraph above occurs when operations cross 32 | threads, as occures for most client operations and when a transaction 33 | commits on the server and results have to be sent to other clients. In 34 | these cases, a call_soon_threadsafe method is used which queues an 35 | operation and has to wake up an event loop to process it. 36 | 37 | Server threading 38 | ---------------- 39 | 40 | ZEO server implementation always uses single networking thread that serves all 41 | clients. In other words the server is single-threaded. 42 | 43 | Historically ZEO switched to a multi-threaded implementation several years ago 44 | because it was found to improve performance for large databases using 45 | magnetic disks. Because client threads are always working on behalf of 46 | a single client, there's not really an issue with making blocking 47 | calls, such as executing slow I/O operations. 48 | 49 | Initially, the asyncio-based implementation used a multi-threaded 50 | server. A simple thread accepted connections and handed accepted 51 | sockets to ``create_connection``. This became a problem when SSL was 52 | added because ``create_connection`` sets up SSL conections as client 53 | connections, and doesn't provide an option to create server 54 | connections. 55 | 56 | In response, Jim created an ``asyncio.Server``-based implementation. 57 | This required using a single thread. This was a pretty trivial 58 | change, however, it led to the tests becoming unstable to the point 59 | that it was impossible to run all tests without some failing. One 60 | test was broken due to a ``asyncio.Server`` `bug 61 | `_. It's unclear whether the test 62 | instability is due to ``asyncio.Server`` problems or due to latent 63 | test (or ZEO) bugs, but even after beating the tests mostly into 64 | submission, tests failures are more likely when using 65 | ``asyncio.Server``. Beatings will continue. 66 | 67 | While fighting test failures using ``asyncio.Server``, the 68 | multi-threaded implementation was updated to use a monkey patch to 69 | allow it to create SSL server connections. Aside from the real risk of a 70 | monkey patch, this works very well. 71 | 72 | Both implementations seemed to perform about the same. 73 | 74 | Over the time single-threaded server mode became the default, and, given that 75 | multi-threaded implementation did not provide any speed advantages, it was 76 | eventually deprecated and scheduled for removal to ease maintenance burden. 77 | 78 | Finally, the multi-threaded server mode was removed, when it was found that this 79 | mode had concurrency bugs that lead to data corruptions. See `issue 209 80 | ` for details. 81 | -------------------------------------------------------------------------------- /src/ZEO/scripts/parsezeolog.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.3 2 | 3 | """Parse the BLATHER logging generated by ZEO2. 4 | 5 | An example of the log format is: 6 | 2002-04-15T13:05:29 BLATHER(-100) ZEO Server storea(3235680, [714], 235339406490168806) ('10.0.26.30', 45514) # NOQA: E501 line too long 7 | """ 8 | 9 | import re 10 | import time 11 | 12 | 13 | rx_time = re.compile(r'(\d\d\d\d-\d\d-\d\d)T(\d\d:\d\d:\d\d)') 14 | 15 | 16 | def parse_time(line): 17 | """Return the time portion of a zLOG line in seconds or None.""" 18 | mo = rx_time.match(line) 19 | if mo is None: 20 | return None 21 | date, time_ = mo.group(1, 2) 22 | date_l = [int(elt) for elt in date.split('-')] 23 | time_l = [int(elt) for elt in time_.split(':')] 24 | return int(time.mktime(date_l + time_l + [0, 0, 0])) 25 | 26 | 27 | rx_meth = re.compile(r"zrpc:\d+ calling (\w+)\((.*)") 28 | 29 | 30 | def parse_method(line): 31 | pass 32 | 33 | 34 | def parse_line(line): 35 | """Parse a log entry and return time, method info, and client.""" 36 | t = parse_time(line) 37 | if t is None: 38 | return None, None 39 | mo = rx_meth.search(line) 40 | if mo is None: 41 | return None, None 42 | meth_name = mo.group(1) 43 | meth_args = mo.group(2).strip() 44 | if meth_args.endswith(')'): 45 | meth_args = meth_args[:-1] 46 | meth_args = [s.strip() for s in meth_args.split(",")] 47 | m = meth_name, tuple(meth_args) 48 | return t, m 49 | 50 | 51 | class TStats: 52 | 53 | counter = 1 54 | 55 | def __init__(self): 56 | self.id = TStats.counter 57 | TStats.counter += 1 58 | 59 | fields = ("time", "vote", "done", "user", "path") 60 | fmt = "%-24s %5s %5s %-15s %s" 61 | hdr = fmt % fields 62 | 63 | def report(self): 64 | """Print a report about the transaction""" 65 | if hasattr(self, "vote"): 66 | d_vote = self.vote - self.begin 67 | else: 68 | d_vote = "*" 69 | if hasattr(self, "finish"): 70 | d_finish = self.finish - self.begin 71 | else: 72 | d_finish = "*" 73 | print(self.fmt % (time.ctime(self.begin), d_vote, d_finish, 74 | self.user, self.url)) 75 | 76 | 77 | class TransactionParser: 78 | 79 | def __init__(self): 80 | self.txns = {} 81 | self.skipped = 0 82 | 83 | def parse(self, line): 84 | t, m = parse_line(line) 85 | if t is None: 86 | return 87 | name = m[0] 88 | meth = getattr(self, name, None) 89 | if meth is not None: 90 | meth(t, m[1]) 91 | 92 | def tpc_begin(self, time, args): 93 | t = TStats() 94 | t.begin = time 95 | t.user = args[1] 96 | t.url = args[2] 97 | t.objects = [] 98 | tid = eval(args[0]) 99 | self.txns[tid] = t 100 | 101 | def get_txn(self, args): 102 | tid = eval(args[0]) 103 | try: 104 | return self.txns[tid] 105 | except KeyError: 106 | print("uknown tid", repr(tid)) 107 | return None 108 | 109 | def tpc_finish(self, time, args): 110 | t = self.get_txn(args) 111 | if t is None: 112 | return 113 | t.finish = time 114 | 115 | def vote(self, time, args): 116 | t = self.get_txn(args) 117 | if t is None: 118 | return 119 | t.vote = time 120 | 121 | def get_txns(self): 122 | L = [(t.id, t) for t in self.txns.values()] 123 | L.sort() 124 | return [t for (id, t) in L] 125 | 126 | 127 | if __name__ == "__main__": 128 | import fileinput 129 | 130 | p = TransactionParser() 131 | i = 0 132 | for line in fileinput.input(): 133 | i += 1 134 | try: 135 | p.parse(line) 136 | except: # NOQA: E722 bare except 137 | print("line", i) 138 | raise 139 | print("Transaction: %d" % len(p.txns)) 140 | print(TStats.hdr) 141 | for txn in p.get_txns(): 142 | txn.report() 143 | -------------------------------------------------------------------------------- /src/ZEO/tests/testZEOOptions.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | 15 | """Test suite for ZEO.runzeo.ZEOOptions.""" 16 | 17 | import os 18 | import tempfile 19 | 20 | import ZODB.config 21 | from zdaemon.tests.testzdoptions import TestZDOptions 22 | 23 | from ZEO.runzeo import ZEOOptions 24 | 25 | 26 | # When a hostname isn't specified in a socket binding address, ZConfig 27 | # supplies the empty string. 28 | DEFAULT_BINDING_HOST = "" 29 | 30 | 31 | class TestZEOOptions(TestZDOptions): 32 | 33 | OptionsClass = ZEOOptions 34 | 35 | input_args = ["-f", "Data.fs", "-a", "5555"] 36 | output_opts = [("-f", "Data.fs"), ("-a", "5555")] 37 | output_args = [] 38 | 39 | configdata = """ 40 | 41 | address 5555 42 | 43 | 44 | path Data.fs 45 | 46 | """ 47 | 48 | def setUp(self): 49 | self.tempfilename = tempfile.mktemp() 50 | with open(self.tempfilename, "w") as f: 51 | f.write(self.configdata) 52 | 53 | def tearDown(self): 54 | try: 55 | os.remove(self.tempfilename) 56 | except OSError: 57 | pass 58 | 59 | def test_configure(self): 60 | # Hide the base class test_configure 61 | pass 62 | 63 | def test_default_help(self): pass # disable silly test w spurious failures 64 | 65 | def test_defaults_with_schema(self): 66 | options = self.OptionsClass() 67 | options.realize(["-C", self.tempfilename]) 68 | self.assertEqual(options.address, (DEFAULT_BINDING_HOST, 5555)) 69 | self.assertEqual(len(options.storages), 1) 70 | opener = options.storages[0] 71 | self.assertEqual(opener.name, "fs") 72 | self.assertEqual(opener.__class__, ZODB.config.FileStorage) 73 | self.assertEqual(options.read_only, 0) 74 | self.assertEqual(options.transaction_timeout, None) 75 | self.assertEqual(options.invalidation_queue_size, 100) 76 | 77 | def test_defaults_without_schema(self): 78 | options = self.OptionsClass() 79 | options.realize(["-a", "5555", "-f", "Data.fs"]) 80 | self.assertEqual(options.address, (DEFAULT_BINDING_HOST, 5555)) 81 | self.assertEqual(len(options.storages), 1) 82 | opener = options.storages[0] 83 | self.assertEqual(opener.name, "1") 84 | self.assertEqual(opener.__class__, ZODB.config.FileStorage) 85 | self.assertEqual(opener.config.path, "Data.fs") 86 | self.assertEqual(options.read_only, 0) 87 | self.assertEqual(options.transaction_timeout, None) 88 | self.assertEqual(options.invalidation_queue_size, 100) 89 | 90 | def test_commandline_overrides(self): 91 | options = self.OptionsClass() 92 | options.realize(["-C", self.tempfilename, 93 | "-a", "6666", "-f", "Wisdom.fs"]) 94 | self.assertEqual(options.address, (DEFAULT_BINDING_HOST, 6666)) 95 | self.assertEqual(len(options.storages), 1) 96 | opener = options.storages[0] 97 | self.assertEqual(opener.__class__, ZODB.config.FileStorage) 98 | self.assertEqual(opener.config.path, "Wisdom.fs") 99 | self.assertEqual(options.read_only, 0) 100 | self.assertEqual(options.transaction_timeout, None) 101 | self.assertEqual(options.invalidation_queue_size, 100) 102 | 103 | 104 | del TestZDOptions # don't run ZDaemon tests 105 | -------------------------------------------------------------------------------- /src/ZEO/tests/zdoptions.test: -------------------------------------------------------------------------------- 1 | Minimal test of Server Options Handling 2 | ======================================= 3 | 4 | This is initially motivated by a desire to remove the requirement of 5 | specifying a storage name when there is only one storage. 6 | 7 | Storage Names 8 | ------------- 9 | 10 | It is an error not to specify any storages: 11 | 12 | >>> import sys, ZEO.runzeo 13 | >>> from io import StringIO 14 | >>> stderr = sys.stderr 15 | 16 | >>> with open('config', 'w') as f: 17 | ... _ = f.write(""" 18 | ... 19 | ... address 8100 20 | ... 21 | ... """) 22 | 23 | >>> sys.stderr = StringIO() 24 | >>> options = ZEO.runzeo.ZEOOptions() 25 | >>> options.realize('-C config'.split()) 26 | Traceback (most recent call last): 27 | ... 28 | SystemExit: 2 29 | 30 | >>> print(sys.stderr.getvalue()) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE 31 | Error: not enough values for section type 'zodb.storage'; 32 | 0 found, 1 required 33 | ... 34 | 35 | 36 | But we can specify a storage without a name: 37 | 38 | >>> with open('config', 'w') as f: 39 | ... _ = f.write(""" 40 | ... 41 | ... address 8100 42 | ... 43 | ... 44 | ... 45 | ... """) 46 | >>> options = ZEO.runzeo.ZEOOptions() 47 | >>> options.realize('-C config'.split()) 48 | >>> [storage.name for storage in options.storages] 49 | ['1'] 50 | 51 | We can't have multiple unnamed storages: 52 | 53 | >>> sys.stderr = StringIO() 54 | >>> with open('config', 'w') as f: 55 | ... _ = f.write(""" 56 | ... 57 | ... address 8100 58 | ... 59 | ... 60 | ... 61 | ... 62 | ... 63 | ... """) 64 | >>> options = ZEO.runzeo.ZEOOptions() 65 | >>> options.realize('-C config'.split()) 66 | Traceback (most recent call last): 67 | ... 68 | SystemExit: 2 69 | 70 | >>> print(sys.stderr.getvalue()) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE 71 | Error: No more than one storage may be unnamed. 72 | ... 73 | 74 | Or an unnamed storage and one named '1': 75 | 76 | >>> sys.stderr = StringIO() 77 | >>> with open('config', 'w') as f: 78 | ... _ = f.write(""" 79 | ... 80 | ... address 8100 81 | ... 82 | ... 83 | ... 84 | ... 85 | ... 86 | ... """) 87 | >>> options = ZEO.runzeo.ZEOOptions() 88 | >>> options.realize('-C config'.split()) 89 | Traceback (most recent call last): 90 | ... 91 | SystemExit: 2 92 | 93 | >>> print(sys.stderr.getvalue()) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE 94 | Error: Can't have an unnamed storage and a storage named 1. 95 | ... 96 | 97 | But we can have multiple storages: 98 | 99 | >>> with open('config', 'w') as f: 100 | ... _ = f.write(""" 101 | ... 102 | ... address 8100 103 | ... 104 | ... 105 | ... 106 | ... 107 | ... 108 | ... """) 109 | >>> options = ZEO.runzeo.ZEOOptions() 110 | >>> options.realize('-C config'.split()) 111 | >>> [storage.name for storage in options.storages] 112 | ['x', 'y'] 113 | 114 | As long as the names are unique: 115 | 116 | >>> sys.stderr = StringIO() 117 | >>> with open('config', 'w') as f: 118 | ... _ = f.write(""" 119 | ... 120 | ... address 8100 121 | ... 122 | ... 123 | ... 124 | ... 125 | ... 126 | ... """) 127 | >>> options = ZEO.runzeo.ZEOOptions() 128 | >>> options.realize('-C config'.split()) 129 | Traceback (most recent call last): 130 | ... 131 | SystemExit: 2 132 | 133 | >>> print(sys.stderr.getvalue()) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE 134 | Error: section names must not be re-used within the same container:'1' 135 | ... 136 | 137 | .. Cleanup ===================================================== 138 | 139 | >>> sys.stderr = stderr 140 | -------------------------------------------------------------------------------- /src/ZEO/tests/stress.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """A ZEO client-server stress test to look for leaks. 15 | 16 | The stress test should run in an infinite loop and should involve 17 | multiple connections. 18 | """ 19 | # TODO: This code is currently broken. 20 | 21 | import os 22 | import random 23 | 24 | import transaction 25 | import ZODB 26 | from ZODB.MappingStorage import MappingStorage 27 | from ZODB.tests import MinPO 28 | 29 | from ZEO.ClientStorage import ClientStorage 30 | from ZEO.tests import forker 31 | 32 | 33 | NUM_TRANSACTIONS_PER_CONN = 10 34 | NUM_CONNECTIONS = 10 35 | NUM_ROOTS = 20 36 | MAX_DEPTH = 20 37 | MIN_OBJSIZE = 128 38 | MAX_OBJSIZE = 2048 39 | 40 | 41 | def an_object(): 42 | """Return an object suitable for a PersistentMapping key""" 43 | size = random.randrange(MIN_OBJSIZE, MAX_OBJSIZE) 44 | if os.path.exists("/dev/urandom"): 45 | fp = open("/dev/urandom") 46 | buf = fp.read(size) 47 | fp.close() 48 | return buf 49 | else: 50 | fp = open(MinPO.__file__) 51 | lst = list(fp.read(size)) 52 | fp.close() 53 | random.shuffle(lst) 54 | return "".join(lst) 55 | 56 | 57 | def setup(cn): 58 | """Initialize the database with some objects""" 59 | root = cn.root() 60 | for i in range(NUM_ROOTS): 61 | prev = an_object() 62 | for j in range(random.randrange(1, MAX_DEPTH)): 63 | o = MinPO.MinPO(prev) 64 | prev = o 65 | root[an_object()] = o 66 | transaction.commit() 67 | cn.close() 68 | 69 | 70 | def work(cn): 71 | """Do some work with a transaction""" 72 | cn.sync() 73 | root = cn.root() 74 | obj = random.choice(root.values()) 75 | # walk down to the bottom 76 | while not isinstance(obj.value, str): 77 | obj = obj.value 78 | obj.value = an_object() 79 | transaction.commit() 80 | 81 | 82 | def main(): 83 | # Yuck! Need to cleanup forker so that the API is consistent 84 | # across Unix and Windows, at least if that's possible. 85 | if os.name == "nt": 86 | zaddr, tport, pid = forker.start_zeo_server('MappingStorage', ()) 87 | 88 | def exitserver(): 89 | import socket 90 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 91 | s.connect(tport) 92 | s.close() 93 | else: 94 | zaddr = '', random.randrange(20000, 30000) 95 | pid, exitobj = forker.start_zeo_server(MappingStorage(), zaddr) 96 | 97 | def exitserver(): 98 | exitobj.close() 99 | 100 | while 1: 101 | pid = start_child(zaddr) 102 | print("started", pid) 103 | os.waitpid(pid, 0) 104 | 105 | exitserver() 106 | 107 | 108 | def start_child(zaddr): 109 | 110 | pid = os.fork() 111 | if pid != 0: 112 | return pid 113 | try: 114 | _start_child(zaddr) 115 | finally: 116 | os._exit(0) 117 | 118 | 119 | def _start_child(zaddr): 120 | storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5, wait=1) 121 | db = ZODB.DB(storage, pool_size=NUM_CONNECTIONS) 122 | setup(db.open()) 123 | conns = [] 124 | conn_count = 0 125 | 126 | for i in range(NUM_CONNECTIONS): 127 | c = db.open() 128 | c.__count = 0 129 | conns.append(c) 130 | conn_count += 1 131 | 132 | while conn_count < 25: 133 | c = random.choice(conns) 134 | if c.__count > NUM_TRANSACTIONS_PER_CONN: 135 | conns.remove(c) 136 | c.close() 137 | conn_count += 1 138 | c = db.open() 139 | c.__count = 0 140 | conns.append(c) 141 | else: 142 | c.__count += 1 143 | work(c) 144 | 145 | 146 | if __name__ == "__main__": 147 | main() 148 | -------------------------------------------------------------------------------- /src/ZEO/tests/testConversionSupport.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2006 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | import doctest 15 | 16 | 17 | class FakeStorageBase: 18 | 19 | def __getattr__(self, name): 20 | if name in ('getTid', 'history', 'load', 'loadSerial', 21 | 'lastTransaction', 'getSize', 'getName', 'supportsUndo', 22 | 'tpc_transaction'): 23 | return lambda *a, **k: None 24 | raise AttributeError(name) 25 | 26 | def isReadOnly(self): 27 | return False 28 | 29 | def __len__(self): 30 | return 4 31 | 32 | 33 | class FakeStorage(FakeStorageBase): 34 | 35 | def record_iternext(self, next=None): 36 | if next is None: 37 | next = '0' 38 | next = str(int(next) + 1) 39 | oid = next 40 | if next == '4': 41 | next = None 42 | 43 | return oid, oid * 8, 'data ' + oid, next 44 | 45 | 46 | class FakeServer: 47 | storages = { 48 | '1': FakeStorage(), 49 | '2': FakeStorageBase(), 50 | } 51 | lock_managers = storages 52 | 53 | def register_connection(*args): 54 | return None, None 55 | 56 | client_conflict_resolution = False 57 | 58 | 59 | class FakeConnection: 60 | protocol_version = b'Z5' 61 | addr = 'test' 62 | 63 | def call_soon_threadsafe(f, *a): 64 | return f(*a) 65 | 66 | async_ = async_threadsafe = None 67 | 68 | 69 | def test_server_record_iternext(): 70 | """ 71 | 72 | On the server, record_iternext calls are simply delegated to the 73 | underlying storage. 74 | 75 | >>> import ZEO.StorageServer 76 | 77 | >>> zeo = ZEO.StorageServer.ZEOStorage(FakeServer(), False) 78 | >>> zeo.notify_connected(FakeConnection()) 79 | >>> zeo.register('1', False) 80 | 81 | >>> next = None 82 | >>> while 1: 83 | ... oid, serial, data, next = zeo.record_iternext(next) 84 | ... print(oid) 85 | ... if next is None: 86 | ... break 87 | 1 88 | 2 89 | 3 90 | 4 91 | 92 | The storage info also reflects the fact that record_iternext is supported. 93 | 94 | >>> zeo.get_info()['supports_record_iternext'] 95 | True 96 | 97 | >>> zeo = ZEO.StorageServer.ZEOStorage(FakeServer(), False) 98 | >>> zeo.notify_connected(FakeConnection()) 99 | >>> zeo.register('2', False) 100 | 101 | >>> zeo.get_info()['supports_record_iternext'] 102 | False 103 | 104 | """ 105 | 106 | 107 | def test_client_record_iternext(): 108 | """Test client storage delegation to the network client 109 | 110 | The client simply delegates record_iternext calls to it's server stub. 111 | 112 | There's really no decent way to test ZEO without running too much crazy 113 | stuff. I'd rather do a lame test than a really lame test, so here goes. 114 | 115 | First, fake out the connection manager so we can make a connection: 116 | 117 | >>> import ZEO 118 | 119 | >>> class Client(ZEO.asyncio.testing.ClientRunner): 120 | ... 121 | ... def record_iternext(self, next=None): 122 | ... if next == None: 123 | ... next = '0' 124 | ... next = str(int(next) + 1) 125 | ... oid = next 126 | ... if next == '4': 127 | ... next = None 128 | ... 129 | ... return oid, oid*8, 'data ' + oid, next 130 | ... 131 | 132 | >>> client = ZEO.client( 133 | ... '', wait=False, _client_factory=Client) 134 | 135 | Now we'll have our way with it's private _server attr: 136 | 137 | >>> next = None 138 | >>> while 1: 139 | ... oid, serial, data, next = client.record_iternext(next) 140 | ... print(oid) 141 | ... if next is None: 142 | ... break 143 | 1 144 | 2 145 | 3 146 | 4 147 | >>> client.close() 148 | 149 | """ 150 | 151 | 152 | def test_suite(): 153 | return doctest.DocTestSuite() 154 | -------------------------------------------------------------------------------- /src/ZEO/nagios.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | ZEO Nagios plugin 3 | ================= 4 | 5 | ZEO includes a script that provides a nagios monitor plugin: 6 | 7 | >>> import time 8 | >>> import importlib.metadata 9 | >>> try: 10 | ... nagios = importlib.metadata.entry_points( 11 | ... group='console_scripts')['zeo-nagios'].load() 12 | ... except TypeError: # PY39 13 | ... from ZEO.nagios import main as nagios 14 | 15 | In it's simplest form, the script just checks if it can get status: 16 | 17 | >>> import ZEO 18 | >>> addr, stop = ZEO.server('test.fs', threaded=False) 19 | >>> saddr = ':'.join(map(str, addr)) # (host, port) -> host:port 20 | 21 | >>> nagios([saddr]) 22 | Empty storage '1' 23 | 1 24 | 25 | The storage was empty. In that case, the monitor warned as much. 26 | 27 | Let's add some data: 28 | 29 | >>> ZEO.DB(addr).close() 30 | >>> nagios([saddr]) 31 | OK 32 | 33 | If we stop the server, we'll error: 34 | 35 | >>> stop() 36 | >>> nagios([saddr]) 37 | Can't connect [Errno 61] Connection refused 38 | 2 39 | 40 | Metrics 41 | ======= 42 | 43 | The monitor will optionally output server metric data. There are 2 44 | kinds of metrics it can output, level and rate metric. If we use the 45 | -m/--output-metrics option, we'll just get rate metrics: 46 | 47 | >>> addr, stop = ZEO.server('test.fs', threaded=False) 48 | >>> saddr = ':'.join(map(str, addr)) # (host, port) -> host:port 49 | >>> nagios([saddr, '-m']) 50 | OK|active_txns=0 51 | | connections=0 52 | waiting=0 53 | 54 | We only got the metrics that are levels, like current number of 55 | connections. If we want rate metrics, we need to be able to save 56 | values from run to run. We need to use the -s/--status-path option to 57 | specify the name of a file for status information: 58 | 59 | >>> nagios([saddr, '-m', '-sstatus']) 60 | OK|active_txns=0 61 | | connections=0 62 | waiting=0 63 | 64 | We still didn't get any rate metrics, because we've only run once. 65 | Let's actually do something with the database and then make another 66 | sample. 67 | 68 | >>> db = ZEO.DB(addr) 69 | >>> nagios([saddr, '-m', '-sstatus']) 70 | OK|active_txns=0 71 | | connections=1 72 | waiting=0 73 | aborts=0.0 74 | commits=0.0 75 | conflicts=0.0 76 | conflicts_resolved=0.0 77 | loads=81.226297803 78 | stores=0.0 79 | 80 | Note that this time, we saw that there was a connection. 81 | 82 | The ZEO.nagios module provides a check function that can be used by 83 | other monitors (e.g. that get address data from ZooKeeper). It takes: 84 | 85 | - Address string, 86 | 87 | - Metrics flag. 88 | 89 | - Status file name (or None), and 90 | 91 | - Time units for rate metrics 92 | 93 | :: 94 | 95 | >>> import ZEO.nagios 96 | >>> ZEO.nagios.check(saddr, True, 'status', 'seconds') 97 | OK|active_txns=0 98 | | connections=1 99 | waiting=0 100 | aborts=0.0 101 | commits=0.0 102 | conflicts=0.0 103 | conflicts_resolved=0.0 104 | loads=0.0 105 | stores=0.0 106 | 107 | >>> db.close() 108 | >>> stop() 109 | 110 | Multi-storage servers 111 | ===================== 112 | 113 | A ZEO server can host multiple servers. (This is a feature that will 114 | likely be dropped in the future.) When this is the case, the monitor 115 | profixes metrics with a storage id. 116 | 117 | >>> addr, stop = ZEO.server( 118 | ... storage_conf = """ 119 | ... 120 | ... 121 | ... 122 | ... 123 | ... """, threaded=False) 124 | >>> saddr = ':'.join(map(str, addr)) # (host, port) -> host:port 125 | >>> nagios([saddr, '-m', '-sstatus']) 126 | Empty storage 'first'|first:active_txns=0 127 | Empty storage 'second' 128 | | first:connections=0 129 | first:waiting=0 130 | second:active_txns=0 131 | second:connections=0 132 | second:waiting=0 133 | 1 134 | >>> nagios([saddr, '-m', '-sstatus']) 135 | Empty storage 'first'|first:active_txns=0 136 | Empty storage 'second' 137 | | first:connections=0 138 | first:waiting=0 139 | second:active_txns=0 140 | second:connections=0 141 | second:waiting=0 142 | first:aborts=0.0 143 | first:commits=0.0 144 | first:conflicts=0.0 145 | first:conflicts_resolved=0.0 146 | first:loads=42.42 147 | first:stores=0.0 148 | second:aborts=0.0 149 | second:commits=0.0 150 | second:conflicts=0.0 151 | second:conflicts_resolved=0.0 152 | second:loads=42.42 153 | second:stores=0.0 154 | 1 155 | 156 | >>> stop() 157 | -------------------------------------------------------------------------------- /src/ZEO/tests/invalidation-age.txt: -------------------------------------------------------------------------------- 1 | Invalidation age 2 | ================ 3 | 4 | When a ZEO client with a non-empty cache connects to the server, it 5 | needs to verify whether the data in its cache is current. It does 6 | this in one of 2 ways: 7 | 8 | quick verification 9 | It gets a list of invalidations from the server since the last 10 | transaction the client has seen and applies those to it's disk and 11 | in-memory caches. This is only possible if there haven't been too 12 | many transactions since the client was last connected. 13 | 14 | full verification 15 | If quick verification isn't possible, the client iterates through 16 | it's disk cache asking the server to verify whether each current 17 | entry is valid. 18 | 19 | Unfortunately, for large caches, full verification is soooooo not 20 | quick that it is impractical. Quick verificatioin is highly 21 | desireable. 22 | 23 | To support quick verification, the server keeps a list of recent 24 | invalidations. The size of this list is controlled by the 25 | invalidation_queue_size parameter. If there is a lot of database 26 | activity, the size might need to be quite large to support having 27 | clients be disconnected for more than a few minutes. A very large 28 | invalidation queue size can use a lot of memory. 29 | 30 | To suppliment the invalidation queue, you can also specify an 31 | invalidation_age parameter. When a client connects and presents the 32 | last transaction id it has seen, we first check to see if the 33 | invalidation queue has that transaction id. It it does, then we send 34 | all transactions since that id. Otherwise, we check to see if the 35 | difference between storage's last transaction id and the given id is 36 | less than or equal to the invalidation age. If it is, then we iterate 37 | over the storage, starting with the given id, to get the invalidations 38 | since the given id. 39 | 40 | NOTE: This assumes that iterating from a point near the "end" of a 41 | database is inexpensive. Don't use this option for a storage for which 42 | that is not the case. 43 | 44 | Here's an example. We set up a server, using an 45 | invalidation-queue-size of 5: 46 | 47 | >>> addr, admin = start_server(zeo_conf=dict(invalidation_queue_size=5), 48 | ... keep=True) 49 | 50 | Now, we'll open a client with a persistent cache, set up some data, 51 | and then close client: 52 | 53 | >>> import ZEO, transaction 54 | >>> db = ZEO.DB(addr, client='test') 55 | >>> conn = db.open() 56 | >>> for i in range(9): 57 | ... conn.root()[i] = conn.root().__class__() 58 | ... conn.root()[i].x = 0 59 | >>> transaction.commit() 60 | >>> db.close() 61 | 62 | We'll open another client, and commit some transactions: 63 | 64 | >>> db = ZEO.DB(addr) 65 | >>> conn = db.open() 66 | >>> import transaction 67 | >>> for i in range(2): 68 | ... conn.root()[i].x = 1 69 | ... transaction.commit() 70 | >>> db.close() 71 | 72 | If we reopen the first client, we'll do quick verification. 73 | 74 | >>> db = ZEO.DB(addr, client='test') # doctest: +ELLIPSIS 75 | >>> db._storage._server.client.verify_result 76 | 'quick verification' 77 | 78 | >>> [v.x for v in db.open().root().values()] 79 | [1, 1, 0, 0, 0, 0, 0, 0, 0] 80 | 81 | Now, if we disconnect and commit more than 5 transactions, we'll see 82 | that we had to clear the cache: 83 | 84 | >>> db.close() 85 | >>> db = ZEO.DB(addr) 86 | >>> conn = db.open() 87 | >>> import transaction 88 | >>> for i in range(9): 89 | ... conn.root()[i].x = 2 90 | ... transaction.commit() 91 | >>> db.close() 92 | 93 | >>> db = ZEO.DB(addr, client='test') 94 | >>> db._storage._server.client.verify_result 95 | 'cache too old, clearing' 96 | 97 | >>> [v.x for v in db.open().root().values()] 98 | [2, 2, 2, 2, 2, 2, 2, 2, 2] 99 | 100 | >>> db.close() 101 | 102 | But if we restart the server with invalidation-age set, we can 103 | do quick verification: 104 | 105 | >>> stop_server(admin) 106 | >>> addr, admin = start_server(zeo_conf=dict(invalidation_queue_size=5, 107 | ... invalidation_age=100)) 108 | >>> db = ZEO.DB(addr) 109 | >>> conn = db.open() 110 | >>> import transaction 111 | >>> for i in range(9): 112 | ... conn.root()[i].x = 3 113 | ... transaction.commit() 114 | >>> db.close() 115 | 116 | 117 | >>> db = ZEO.DB(addr, client='test') # doctest: +ELLIPSIS 118 | >>> db._storage._server.client.verify_result 119 | 'quick verification' 120 | 121 | >>> [v.x for v in db.open().root().values()] 122 | [3, 3, 3, 3, 3, 3, 3, 3, 3] 123 | 124 | >>> db.close() 125 | -------------------------------------------------------------------------------- /src/ZEO/scripts/zeoup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.3 2 | 3 | """Make sure a ZEO server is running. 4 | 5 | usage: zeoup.py [options] 6 | 7 | The test will connect to a ZEO server, load the root object, and attempt to 8 | update the zeoup counter in the root. It will report success if it updates 9 | the counter or if it gets a ConflictError. A ConflictError is considered a 10 | success, because the client was able to start a transaction. 11 | 12 | Options: 13 | 14 | -p port -- port to connect to 15 | 16 | -h host -- host to connect to (default is current host) 17 | 18 | -S storage -- storage name (default '1') 19 | 20 | -U path -- Unix-domain socket to connect to 21 | 22 | --nowrite -- Do not update the zeoup counter. 23 | 24 | -1 -- Connect to a ZEO 1.0 server. 25 | 26 | You must specify either -p and -h or -U. 27 | """ 28 | 29 | import getopt 30 | import logging 31 | import socket 32 | import sys 33 | import time 34 | 35 | import transaction 36 | import ZODB 37 | from persistent.mapping import PersistentMapping 38 | from ZODB.POSException import ConflictError 39 | from ZODB.tests.MinPO import MinPO 40 | 41 | from ZEO.ClientStorage import ClientStorage 42 | 43 | 44 | ZEO_VERSION = 2 45 | 46 | 47 | def setup_logging(): 48 | # Set up logging to stderr which will show messages originating 49 | # at severity ERROR or higher. 50 | root = logging.getLogger() 51 | root.setLevel(logging.ERROR) 52 | fmt = logging.Formatter( 53 | "------\n%(asctime)s %(levelname)s %(name)s %(message)s", 54 | "%Y-%m-%dT%H:%M:%S") 55 | handler = logging.StreamHandler() 56 | handler.setFormatter(fmt) 57 | root.addHandler(handler) 58 | 59 | 60 | def check_server(addr, storage, write): 61 | t0 = time.time() 62 | if ZEO_VERSION == 2: 63 | # TODO: should do retries w/ exponential backoff. 64 | cs = ClientStorage(addr, storage=storage, wait=0, 65 | read_only=(not write)) 66 | else: 67 | cs = ClientStorage(addr, storage=storage, debug=1, 68 | wait_for_server_on_startup=1) 69 | # _startup() is an artifact of the way ZEO 1.0 works. The 70 | # ClientStorage doesn't get fully initialized until registerDB() 71 | # is called. The only thing we care about, though, is that 72 | # registerDB() calls _startup(). 73 | 74 | if write: 75 | db = ZODB.DB(cs) 76 | cn = db.open() 77 | root = cn.root() 78 | try: 79 | # We store the data in a special `monitor' dict under the root, 80 | # where other tools may also store such heartbeat and bookkeeping 81 | # type data. 82 | monitor = root.get('monitor') 83 | if monitor is None: 84 | monitor = root['monitor'] = PersistentMapping() 85 | obj = monitor['zeoup'] = monitor.get('zeoup', MinPO(0)) 86 | obj.value += 1 87 | transaction.commit() 88 | except ConflictError: 89 | pass 90 | cn.close() 91 | db.close() 92 | else: 93 | data, serial = cs.load("\0\0\0\0\0\0\0\0", "") 94 | cs.close() 95 | t1 = time.time() 96 | print("Elapsed time: %.2f" % (t1 - t0)) 97 | 98 | 99 | def usage(exit=1): 100 | print(__doc__) 101 | print(" ".join(sys.argv)) 102 | sys.exit(exit) 103 | 104 | 105 | def main(): 106 | host = None 107 | port = None 108 | unix = None 109 | write = 1 110 | storage = '1' 111 | try: 112 | opts, args = getopt.getopt(sys.argv[1:], 'p:h:U:S:1', 113 | ['nowrite']) 114 | for o, a in opts: 115 | if o == '-p': 116 | port = int(a) 117 | elif o == '-h': 118 | host = a 119 | elif o == '-U': 120 | unix = a 121 | elif o == '-S': 122 | storage = a 123 | elif o == '--nowrite': 124 | write = 0 125 | elif o == '-1': 126 | ZEO_VERSION = 1 # NOQA: F841 unused variable 127 | except Exception as err: 128 | s = str(err) 129 | if s: 130 | s = ": " + s 131 | print(err.__class__.__name__ + s) 132 | usage() 133 | 134 | if unix is not None: 135 | addr = unix 136 | else: 137 | if host is None: 138 | host = socket.gethostname() 139 | if port is None: 140 | usage() 141 | addr = host, port 142 | 143 | setup_logging() 144 | check_server(addr, storage, write) 145 | 146 | 147 | if __name__ == "__main__": 148 | try: 149 | main() 150 | except SystemExit: 151 | raise 152 | except Exception as err: 153 | s = str(err) 154 | if s: 155 | s = ": " + s 156 | print(err.__class__.__name__ + s) 157 | sys.exit(1) 158 | -------------------------------------------------------------------------------- /src/ZEO/tests/zeo-fan-out.test: -------------------------------------------------------------------------------- 1 | ZEO Fan Out 2 | =========== 3 | 4 | We should be able to set up ZEO servers with ZEO clients. Let's see 5 | if we can make it work. 6 | 7 | We'll use some helper functions. The first is a helper that starts 8 | ZEO servers for us and another one that picks ports. 9 | 10 | We'll start the first server: 11 | 12 | >>> (_, port0), adminaddr0 = start_server( 13 | ... '\npath fs\nblob-dir blobs\n', keep=1) 14 | 15 | Then we'll start 2 others that use this one: 16 | 17 | >>> addr1, _ = start_server( 18 | ... '\nserver %s\nblob-dir b1\n' % port0) 19 | >>> addr2, _ = start_server( 20 | ... '\nserver %s\nblob-dir b2\n' % port0) 21 | 22 | 23 | Now, let's create some client storages that connect to these: 24 | 25 | >>> import os, ZEO, ZODB.blob, ZODB.POSException, transaction 26 | 27 | >>> db0 = ZEO.DB(port0, blob_dir='cb0') 28 | >>> db1 = ZEO.DB(addr1, blob_dir='cb1') 29 | >>> tm1 = transaction.TransactionManager() 30 | >>> c1 = db1.open(transaction_manager=tm1) 31 | >>> r1 = c1.root() 32 | >>> r1 33 | {} 34 | 35 | >>> db2 = ZEO.DB(addr2, blob_dir='cb2') 36 | >>> tm2 = transaction.TransactionManager() 37 | >>> c2 = db2.open(transaction_manager=tm2) 38 | >>> r2 = c2.root() 39 | >>> r2 40 | {} 41 | 42 | If we update c1, we'll eventually see the change in c2: 43 | 44 | >>> import persistent.mapping 45 | 46 | >>> r1[1] = persistent.mapping.PersistentMapping() 47 | >>> r1[1].v = 1000 48 | >>> r1[2] = persistent.mapping.PersistentMapping() 49 | >>> r1[2].v = -1000 50 | >>> r1[3] = ZODB.blob.Blob(b'x'*4111222) 51 | >>> for i in range(1000, 2000): 52 | ... r1[i] = persistent.mapping.PersistentMapping() 53 | ... r1[i].v = 0 54 | >>> tm1.commit() 55 | >>> blob_id = r1[3]._p_oid, r1[1]._p_serial 56 | 57 | >>> import time 58 | >>> for i in range(100): 59 | ... t = tm2.begin() 60 | ... if 1 in r2: 61 | ... break 62 | ... time.sleep(0.01) 63 | >>> tm2.abort() 64 | 65 | 66 | >>> r2[1].v 67 | 1000 68 | 69 | >>> r2[2].v 70 | -1000 71 | 72 | Now, let's see if we can break it. :) 73 | 74 | >>> def f(): 75 | ... c = db1.open(transaction.TransactionManager()) 76 | ... r = c.root() 77 | ... i = 0 78 | ... while i < 100: 79 | ... r[1].v -= 1 80 | ... r[2].v += 1 81 | ... try: 82 | ... c.transaction_manager.commit() 83 | ... i += 1 84 | ... except ZODB.POSException.ConflictError: 85 | ... c.transaction_manager.abort() 86 | ... c.close() 87 | 88 | >>> import threading 89 | >>> threadf = threading.Thread(target=f) 90 | >>> threadg = threading.Thread(target=f) 91 | >>> threadf.start() 92 | 93 | >>> threadg.start() 94 | 95 | >>> s2 = db2.storage 96 | >>> start_time = time.time() 97 | >>> import os 98 | >>> from ZEO.ClientStorage import _lock_blob 99 | >>> while time.time() - start_time < 999: 100 | ... t = tm2.begin() 101 | ... if r2[1].v + r2[2].v: 102 | ... print('oops', r2[1], r2[2]) 103 | ... if r2[1].v == 800: 104 | ... break # we caught up 105 | ... path = s2.fshelper.getBlobFilename(*blob_id) 106 | ... if os.path.exists(path): 107 | ... ZODB.blob.remove_committed(path) 108 | ... _ = s2.fshelper.createPathForOID(blob_id[0]) 109 | ... blob_lock = _lock_blob(path) 110 | ... try: 111 | ... future = s2._call('sendBlob', *blob_id, wait=False) 112 | ... future.result(2) # wait at most 2s for completion 113 | ... finally: 114 | ... blob_lock.close() 115 | ... else: print('Dang') 116 | 117 | >>> threadf.join() 118 | 119 | >>> threadg.join() 120 | 121 | If we shutdown and restart the source server, the variables will be 122 | invalidated: 123 | 124 | >>> stop_server(adminaddr0) 125 | >>> _ = start_server('\npath fs\n\n', 126 | ... port=port0) 127 | >>> time.sleep(1) # get past startup / verification 128 | 129 | >>> for i in range(1000): 130 | ... c1.sync() 131 | ... c2.sync() 132 | ... if ( 133 | ... (r1[1]._p_changed is None) 134 | ... and 135 | ... (r1[2]._p_changed is None) 136 | ... and 137 | ... (r2[1]._p_changed is None) 138 | ... and 139 | ... (r2[2]._p_changed is None) 140 | ... ): 141 | ... print('Cool') 142 | ... break 143 | ... time.sleep(0.01) 144 | ... else: 145 | ... print('Dang') 146 | Cool 147 | 148 | Cleanup: 149 | 150 | >>> db0.close() 151 | >>> db1.close() 152 | >>> db2.close() 153 | -------------------------------------------------------------------------------- /src/ZEO/tests/testConfig.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2003 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | 15 | import unittest 16 | 17 | from ZODB.config import storageFromString 18 | from zope.testing import setupstack 19 | 20 | from .forker import start_zeo_server 21 | from .threaded import threaded_server_tests 22 | 23 | 24 | class ZEOConfigTestBase(setupstack.TestCase): 25 | 26 | setUp = setupstack.setUpDirectory 27 | 28 | def start_server(self, settings='', **kw): 29 | 30 | for name, value in kw.items(): 31 | settings += '\n{} {}\n'.format(name.replace('_', '-'), value) 32 | 33 | zeo_conf = """ 34 | 35 | address 127.0.0.1:0 36 | %s 37 | 38 | """ % settings 39 | return start_zeo_server("\n\n", 40 | zeo_conf, threaded=True) 41 | 42 | def start_client(self, addr, settings='', **kw): 43 | settings += '\nserver %s:%s\n' % addr 44 | for name, value in kw.items(): 45 | settings += '\n{} {}\n'.format(name.replace('_', '-'), value) 46 | return storageFromString( 47 | """ 48 | %import ZEO 49 | 50 | 51 | {} 52 | 53 | """.format(settings)) 54 | 55 | def _client_assertions(self, client, addr, 56 | connected=True, 57 | cache_size=20 * (1 << 20), 58 | cache_path=None, 59 | blob_dir=None, 60 | shared_blob_dir=False, 61 | blob_cache_size=None, 62 | blob_cache_size_check=10, 63 | read_only=False, 64 | read_only_fallback=False, 65 | server_sync=False, 66 | wait_timeout=30, 67 | client_label=None, 68 | storage='1', 69 | name=None): 70 | self.assertEqual(client.is_connected(), connected) 71 | self.assertEqual(client._addr, [addr]) 72 | self.assertEqual(client._cache.maxsize, cache_size) 73 | 74 | self.assertEqual(client._cache.path, cache_path) 75 | self.assertEqual(client.blob_dir, blob_dir) 76 | self.assertEqual(client.shared_blob_dir, shared_blob_dir) 77 | self.assertEqual(client._blob_cache_size, blob_cache_size) 78 | if blob_cache_size: 79 | self.assertEqual(client._blob_cache_size_check, 80 | blob_cache_size * blob_cache_size_check // 100) 81 | self.assertEqual(client._is_read_only, read_only) 82 | self.assertEqual(client._read_only_fallback, read_only_fallback) 83 | self.assertEqual(client._server.timeout, wait_timeout) 84 | self.assertEqual(client._client_label, client_label) 85 | self.assertEqual(client._storage, storage) 86 | self.assertEqual(client.__name__, 87 | name if name is not None else str(client._addr)) 88 | 89 | 90 | class ZEOConfigTest(ZEOConfigTestBase): 91 | 92 | def test_default_zeo_config(self, **client_settings): 93 | addr, stop = self.start_server() 94 | 95 | client = self.start_client(addr, **client_settings) 96 | self._client_assertions(client, addr, **client_settings) 97 | 98 | client.close() 99 | stop() 100 | 101 | def test_client_variations(self): 102 | 103 | for name, value in dict(cache_size=4200, 104 | cache_path='test', 105 | blob_dir='blobs', 106 | blob_cache_size=424242, 107 | read_only=True, 108 | read_only_fallback=True, 109 | server_sync=True, 110 | wait_timeout=33, 111 | client_label='test_client', 112 | name='Test', 113 | ).items(): 114 | params = {name: value} 115 | self.test_default_zeo_config(**params) 116 | 117 | def test_blob_cache_size_check(self): 118 | self.test_default_zeo_config(blob_cache_size=424242, 119 | blob_cache_size_check=50) 120 | 121 | 122 | def test_suite(): 123 | suite = unittest.defaultTestLoader.loadTestsFromTestCase(ZEOConfigTest) 124 | suite.layer = threaded_server_tests 125 | return suite 126 | -------------------------------------------------------------------------------- /src/ZEO/server.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | The full path to an SSL certificate file. 8 | 9 | 10 | 11 | 12 | 13 | The full path to an SSL key file for the server certificate. 14 | 15 | 16 | 17 | 18 | 19 | Dotted name of importable function for retrieving a password 20 | for the client certificate key. 21 | 22 | 23 | 24 | 25 | 26 | Path to a file or directory containing client certificates to 27 | be authenticated. This can also be - or SIGNED to require 28 | signed client certificates. 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 |
37 | 38 | 39 | The content of a ZEO section describe operational parameters 40 | of a ZEO server except for the storage(s) to be served. 41 | 42 | 43 | 45 | 46 | The address at which the server should listen. This can be in 47 | the form 'host:port' to signify a TCP/IP connection or a 48 | pathname string to signify a Unix domain socket connection (at 49 | least one '/' is required). A hostname may be a DNS name or a 50 | dotted IP address. If the hostname is omitted, the platform's 51 | default behavior is used when binding the listening socket ('' 52 | is passed to socket.bind() as the hostname portion of the 53 | address). 54 | 55 | 56 | 57 | 60 | 61 | Flag indicating whether the server should operate in read-only 62 | mode. Defaults to false. Note that even if the server is 63 | operating in writable mode, individual storages may still be 64 | read-only. But if the server is in read-only mode, no write 65 | operations are allowed, even if the storages are writable. Note 66 | that pack() is considered a read-only operation. 67 | 68 | 69 | 70 | 73 | 74 | The storage server keeps a queue of the objects modified by the 75 | last N transactions, where N == invalidation_queue_size. This 76 | queue is used to speed client cache verification when a client 77 | disconnects for a short period of time. 78 | 79 | 80 | 81 | 82 | 83 | The maximum age of a client for which quick-verification 84 | invalidations will be provided by iterating over the served 85 | storage. This option should only be used if the served storage 86 | supports efficient iteration from a starting point near the 87 | end of the transaction history (e.g. end of file). 88 | 89 | 90 | 91 | 93 | 94 | The maximum amount of time to wait for a transaction to commit 95 | after acquiring the storage lock, specified in seconds. If the 96 | transaction takes too long, the client connection will be closed 97 | and the transaction aborted. 98 | 99 | 100 | 101 | 103 | 104 | The full path to the file in which to write the ZEO server's Process ID 105 | at startup. If omitted, $INSTANCE/var/ZEO.pid is used. 106 | 107 | $INSTANCE/var/ZEO.pid (or $clienthome/ZEO.pid) 108 | 109 | 110 | 112 | 113 | Flag indicating whether the server should return conflict 114 | errors to the client, for resolution there. 115 | 116 | 117 | 118 | 119 | 120 | Use msgpack to serialize and de-serialize ZEO protocol messages. 121 | 122 | An advantage of using msgpack for ZEO communication is that 123 | it's a tiny bit faster. 124 | 125 | msgpack can also be enabled by setting the ``ZEO_MSGPACK`` 126 | environment to a non-empty string. 127 | 128 | 129 | 130 | 131 | 132 | 133 | -------------------------------------------------------------------------------- /src/ZEO/tests/drop_cache_rather_than_verify.txt: -------------------------------------------------------------------------------- 1 | Avoiding cache verifification 2 | ============================= 3 | 4 | For large databases it is common to also use very large ZEO cache 5 | files. If a client has beed disconnected for too long, the server 6 | can't play back missing invalidations. In this case, the cache is 7 | cleared. When this happens, a ZEO.interfaces.StaleCache event is 8 | published, largely for backward compatibility. 9 | 10 | ClientStorage used to provide an option to drop it's cache rather than 11 | doing verification. This is now the only behavior. Cache 12 | verification is no longer supported. 13 | 14 | - Invalidates all object caches 15 | 16 | - Drops or clears it's client cache. (The end result is that the cache 17 | is working but empty.) 18 | 19 | - Logs a CRITICAL message. 20 | 21 | Here's an example that shows that this is actually what happens. 22 | 23 | Start a server, create a client to it and commit some data 24 | 25 | >>> addr, admin = start_server(keep=1) 26 | >>> import ZEO, transaction 27 | >>> db = ZEO.DB(addr, client='cache', name='test') 28 | >>> wait_connected(db.storage) 29 | >>> conn = db.open() 30 | >>> conn.root()[1] = conn.root().__class__() 31 | >>> conn.root()[1].x = 1 32 | >>> transaction.commit() 33 | >>> len(db.storage._cache) 34 | 3 35 | 36 | Now, we'll stop the server and restart with a different address: 37 | 38 | >>> stop_server(admin) 39 | >>> addr2, admin = start_server(keep=1) 40 | 41 | And create another client and write some data to it: 42 | 43 | >>> db2 = ZEO.DB(addr2) 44 | >>> wait_connected(db2.storage) 45 | >>> conn2 = db2.open() 46 | >>> for i in range(5): 47 | ... conn2.root()[1].x += 1 48 | ... transaction.commit() 49 | >>> db2.close() 50 | >>> stop_server(admin) 51 | 52 | Now, we'll restart the server. Before we do that, we'll capture 53 | logging and event data: 54 | 55 | >>> import logging, zope.testing.loggingsupport, ZODB.event 56 | >>> handler = zope.testing.loggingsupport.InstalledHandler( 57 | ... 'ZEO', level=logging.ERROR) 58 | >>> events = [] 59 | >>> def event_handler(e): 60 | ... if hasattr(e, 'storage'): 61 | ... events.append(( 62 | ... len(e.storage._cache), str(handler), e.__class__.__name__)) 63 | 64 | >>> old_notify = ZODB.event.notify 65 | >>> ZODB.event.notify = event_handler 66 | 67 | Note that the event handler is saving away the length of the cache and 68 | the state of the log handler. We'll use this to show that the event 69 | is generated before the cache is dropped or the message is logged. 70 | 71 | Now, we'll restart the server on the original address: 72 | 73 | >>> _, admin = start_server(zeo_conf=dict(invalidation_queue_size=1), 74 | ... addr=addr, keep=1) 75 | 76 | >>> wait_connected(db.storage) 77 | 78 | Now, let's verify our assertions above: 79 | 80 | - Publishes a stale-cache event. 81 | 82 | >>> for e in events: 83 | ... print(e) 84 | (3, '', 'StaleCache') 85 | 86 | Note that the length of the cache when the event handler was 87 | called waa non-zero. This is because the cache wasn't cleared 88 | yet. Similarly, the dropping-cache message hasn't been logged 89 | yet. 90 | 91 | >>> del events[:] 92 | 93 | - Drops or clears it's client cache. (The end result is that the cache 94 | is working but empty.) 95 | 96 | >>> len(db.storage._cache) 97 | 0 98 | 99 | - Invalidates all object caches 100 | 101 | >>> transaction.abort() 102 | >>> conn.root()._p_changed 103 | 104 | - Logs a CRITICAL message. 105 | 106 | >>> print(handler) # doctest: +ELLIPSIS 107 | ZEO... CRITICAL 108 | test dropping stale cache 109 | 110 | >>> handler.clear() 111 | 112 | If we access the root object, it'll be loaded from the server: 113 | 114 | >>> conn.root()[1].x 115 | 6 116 | 117 | Similarly, if we simply disconnect the client, and write data from 118 | another client: 119 | 120 | >>> db.close() 121 | 122 | >>> db2 = ZEO.DB(addr) 123 | >>> wait_connected(db2.storage) 124 | >>> conn2 = db2.open() 125 | >>> for i in range(5): 126 | ... conn2.root()[1].x += 1 127 | ... transaction.commit() 128 | >>> db2.close() 129 | 130 | >>> db = ZEO.DB(addr, drop_cache_rather_verify=True, client='cache', 131 | ... name='test') 132 | >>> wait_connected(db.storage) 133 | 134 | 135 | - Drops or clears it's client cache. (The end result is that the cache 136 | is working but empty.) 137 | 138 | >>> len(db.storage._cache) <= 1 139 | True 140 | 141 | (When a database is created, it checks to make sure the root object is 142 | in the database, which is why we get 1, rather than 0 objects in the cache.) 143 | 144 | - Publishes a stale-cache event. 145 | 146 | >>> for e in events: 147 | ... print(e) 148 | (2, '', 'StaleCache') 149 | 150 | >>> del events[:] 151 | 152 | - Logs a CRITICAL message. 153 | 154 | >>> print(handler) # doctest: +ELLIPSIS 155 | ZEO... CRITICAL 156 | test dropping stale cache 157 | 158 | >>> handler.clear() 159 | 160 | If we access the root object, it'll be loaded from the server: 161 | 162 | >>> conn = db.open() 163 | >>> conn.root()[1].x 164 | 11 165 | 166 | .. Cleanup 167 | 168 | >>> db.close() 169 | >>> handler.uninstall() 170 | >>> ZODB.event.notify = old_notify 171 | -------------------------------------------------------------------------------- /src/ZEO/nagios.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2011 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | """%prog [options] address 15 | 16 | Where the address is an IPV6 address of the form: [addr]:port, an IPV4 17 | address of the form: addr:port, or the name of a unix-domain socket file. 18 | """ 19 | import json 20 | import optparse 21 | import os 22 | import re 23 | import socket 24 | import struct 25 | import sys 26 | import time 27 | 28 | 29 | NO_TRANSACTION = '0' * 16 30 | 31 | nodiff_names = 'active_txns connections waiting'.split() 32 | diff_names = 'aborts commits conflicts conflicts_resolved loads stores'.split() 33 | 34 | per_times = dict(seconds=1.0, minutes=60.0, hours=3600.0, days=86400.0) 35 | 36 | 37 | def new_metric(metrics, storage_id, name, value): 38 | if storage_id == '1': 39 | label = name 40 | else: 41 | if ' ' in storage_id: 42 | label = f"'{storage_id}:{name}'" 43 | else: 44 | label = f'{storage_id}:{name}' 45 | metrics.append(f'{label}={value}') 46 | 47 | 48 | def result(messages, metrics=(), status=None): 49 | if metrics: 50 | messages[0] += '|' + metrics[0] 51 | if len(metrics) > 1: 52 | messages.append('| ' + '\n '.join(metrics[1:])) 53 | print('\n'.join(messages)) 54 | return status 55 | 56 | 57 | def error(message): 58 | return result((message, ), (), 2) 59 | 60 | 61 | def warn(message): 62 | return result((message, ), (), 1) 63 | 64 | 65 | def check(addr, output_metrics, status, per): 66 | m = re.match(r'\[(\S+)\]:(\d+)$', addr) 67 | if m: 68 | addr = m.group(1), int(m.group(2)) 69 | s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) 70 | else: 71 | m = re.match(r'(\S+):(\d+)$', addr) 72 | if m: 73 | addr = m.group(1), int(m.group(2)) 74 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 75 | else: 76 | s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 77 | try: 78 | s.connect(addr) 79 | except OSError as err: 80 | s.close() 81 | return error("Can't connect %s" % err) 82 | 83 | s.sendall(b'\x00\x00\x00\x04ruok') 84 | proto = s.recv(struct.unpack(">I", s.recv(4))[0]) # NOQA: F841 unused 85 | datas = s.recv(struct.unpack(">I", s.recv(4))[0]) 86 | s.close() 87 | data = json.loads(datas.decode("ascii")) 88 | if not data: 89 | return warn("No storages") 90 | 91 | metrics = [] 92 | messages = [] 93 | level = 0 94 | if output_metrics: 95 | for storage_id, sdata in sorted(data.items()): 96 | for name in nodiff_names: 97 | new_metric(metrics, storage_id, name, sdata[name]) 98 | 99 | if status: 100 | now = time.time() 101 | if os.path.exists(status): 102 | dt = now - os.stat(status).st_mtime 103 | if dt > 0: # sanity :) 104 | with open(status) as f: # Read previous 105 | old = json.loads(f.read()) 106 | dt /= per_times[per] 107 | for storage_id, sdata in sorted(data.items()): 108 | sdata['sameple-time'] = now 109 | if storage_id in old: 110 | sold = old[storage_id] 111 | for name in diff_names: 112 | v = (sdata[name] - sold[name]) / dt 113 | new_metric(metrics, storage_id, name, v) 114 | with open(status, 'w') as f: # save current 115 | f.write(json.dumps(data)) 116 | 117 | for storage_id, sdata in sorted(data.items()): 118 | if sdata['last-transaction'] == NO_TRANSACTION: 119 | messages.append("Empty storage %r" % storage_id) 120 | level = max(level, 1) 121 | if not messages: 122 | messages.append('OK') 123 | return result(messages, metrics, level or None) 124 | 125 | 126 | def main(args=None): 127 | if args is None: 128 | args = sys.argv[1:] 129 | 130 | parser = optparse.OptionParser(__doc__) 131 | parser.add_option( 132 | '-m', '--output-metrics', action="store_true", 133 | help="Output metrics.", 134 | ) 135 | parser.add_option( 136 | '-s', '--status-path', 137 | help="Path to status file, needed to get rate metrics", 138 | ) 139 | parser.add_option( 140 | '-u', '--time-units', type='choice', default='minutes', 141 | choices=['seconds', 'minutes', 'hours', 'days'], 142 | help="Time unit for rate metrics", 143 | ) 144 | (options, args) = parser.parse_args(args) 145 | [addr] = args 146 | return check( 147 | addr, options.output_metrics, options.status_path, options.time_units) 148 | 149 | 150 | if __name__ == '__main__': 151 | main() 152 | -------------------------------------------------------------------------------- /src/ZEO/tests/ThreadTests.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """Compromising positions involving threads.""" 15 | 16 | import threading 17 | 18 | from ZODB.Connection import TransactionMetaData 19 | from ZODB.tests.StorageTestBase import MinPO 20 | from ZODB.tests.StorageTestBase import zodb_pickle 21 | 22 | import ZEO.Exceptions 23 | 24 | 25 | ZERO = '\0' * 8 26 | 27 | 28 | class BasicThread(threading.Thread): 29 | def __init__(self, storage, doNextEvent, threadStartedEvent): 30 | self.storage = storage 31 | self.trans = TransactionMetaData() 32 | self.doNextEvent = doNextEvent 33 | self.threadStartedEvent = threadStartedEvent 34 | self.gotValueError = 0 35 | self.gotDisconnected = 0 36 | threading.Thread.__init__(self) 37 | self.daemon = True 38 | 39 | def join(self): 40 | threading.Thread.join(self, 10) 41 | assert not self.is_alive() 42 | 43 | 44 | class GetsThroughVoteThread(BasicThread): 45 | # This thread gets partially through a transaction before it turns 46 | # execution over to another thread. We're trying to establish that a 47 | # tpc_finish() after a storage has been closed by another thread will get 48 | # a ClientStorageError error. 49 | # 50 | # This class gets does a tpc_begin(), store(), tpc_vote() and is waiting 51 | # to do the tpc_finish() when the other thread closes the storage. 52 | def run(self): 53 | self.storage.tpc_begin(self.trans) 54 | oid = self.storage.new_oid() 55 | self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans) 56 | self.storage.tpc_vote(self.trans) 57 | self.threadStartedEvent.set() 58 | self.doNextEvent.wait(10) 59 | try: 60 | self.storage.tpc_finish(self.trans) 61 | except ZEO.Exceptions.ClientStorageError: 62 | self.gotValueError = 1 63 | self.storage.tpc_abort(self.trans) 64 | 65 | 66 | class GetsThroughBeginThread(BasicThread): 67 | # This class is like the above except that it is intended to be run when 68 | # another thread is already in a tpc_begin(). Thus, this thread will 69 | # block in the tpc_begin until another thread closes the storage. When 70 | # that happens, this one will get disconnected too. 71 | def run(self): 72 | try: 73 | self.storage.tpc_begin(self.trans) 74 | except ZEO.Exceptions.ClientStorageError: 75 | self.gotValueError = 1 76 | 77 | 78 | class ThreadTests: 79 | # Thread 1 should start a transaction, but not get all the way through it. 80 | # Main thread should close the connection. Thread 1 should then get 81 | # disconnected. 82 | def checkDisconnectedOnThread2Close(self): 83 | doNextEvent = threading.Event() 84 | threadStartedEvent = threading.Event() 85 | thread1 = GetsThroughVoteThread(self._storage, 86 | doNextEvent, threadStartedEvent) 87 | thread1.start() 88 | threadStartedEvent.wait(10) 89 | self._storage.close() 90 | doNextEvent.set() 91 | thread1.join() 92 | self.assertEqual(thread1.gotValueError, 1) 93 | 94 | # Thread 1 should start a transaction, but not get all the way through 95 | # it. While thread 1 is in the middle of the transaction, a second thread 96 | # should start a transaction, and it will block in the tcp_begin() -- 97 | # because thread 1 has acquired the lock in its tpc_begin(). Now the main 98 | # thread closes the storage and both sub-threads should get disconnected. 99 | def checkSecondBeginFails(self): 100 | doNextEvent = threading.Event() 101 | threadStartedEvent = threading.Event() 102 | thread1 = GetsThroughVoteThread(self._storage, 103 | doNextEvent, threadStartedEvent) 104 | thread2 = GetsThroughBeginThread(self._storage, 105 | doNextEvent, threadStartedEvent) 106 | thread1.start() 107 | threadStartedEvent.wait(1) 108 | thread2.start() 109 | self._storage.close() 110 | doNextEvent.set() 111 | thread1.join() 112 | thread2.join() 113 | self.assertEqual(thread1.gotValueError, 1) 114 | self.assertEqual(thread2.gotValueError, 1) 115 | 116 | # Run a bunch of threads doing small and large stores in parallel 117 | def checkMTStores(self): 118 | threads = [] 119 | for i in range(5): 120 | t = threading.Thread(target=self.mtstorehelper) 121 | threads.append(t) 122 | t.start() 123 | for t in threads: 124 | t.join(30) 125 | for i in threads: 126 | self.assertFalse(t.is_alive()) 127 | 128 | # Helper for checkMTStores 129 | def mtstorehelper(self): 130 | objs = [] 131 | for i in range(10): 132 | objs.append(MinPO("X" * 200000)) 133 | objs.append(MinPO("X")) 134 | for obj in objs: 135 | self._dostore(data=obj) 136 | -------------------------------------------------------------------------------- /src/ZEO/asyncio/marshal.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright (c) 2001, 2002 Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE 12 | # 13 | ############################################################################## 14 | """Support for marshaling ZEO messages 15 | 16 | Not to be confused with marshaling objects in ZODB. 17 | 18 | We currently use pickle. In the future, we may use a 19 | Python-independent format, or possibly a minimal pickle subset. 20 | """ 21 | 22 | import logging 23 | from io import BytesIO 24 | 25 | from zodbpickle.pickle import Pickler 26 | 27 | from .._compat import Unpickler 28 | from ..shortrepr import short_repr 29 | 30 | 31 | logger = logging.getLogger(__name__) 32 | 33 | 34 | def encoder(protocol, server=False): 35 | """Return a non-thread-safe encoder 36 | """ 37 | 38 | if protocol[:1] == b'M': 39 | from msgpack import packb 40 | 41 | default = server_default if server else None 42 | 43 | def encode(*args): 44 | return packb( 45 | args, use_bin_type=True, default=default) 46 | 47 | return encode 48 | else: 49 | assert protocol[:1] == b'Z' 50 | 51 | f = BytesIO() 52 | getvalue = f.getvalue 53 | seek = f.seek 54 | truncate = f.truncate 55 | pickler = Pickler(f, 3) 56 | pickler.fast = 1 57 | dump = pickler.dump 58 | 59 | def encode(*args): 60 | seek(0) 61 | truncate() 62 | dump(args) 63 | return getvalue() 64 | 65 | return encode 66 | 67 | 68 | def encode(*args): 69 | 70 | return encoder(b'Z')(*args) 71 | 72 | 73 | def decoder(protocol): 74 | if protocol[:1] == b'M': 75 | from msgpack import unpackb 76 | 77 | def msgpack_decode(data): 78 | """Decodes msg and returns its parts""" 79 | return unpackb(data, raw=False, use_list=False) 80 | 81 | return msgpack_decode 82 | else: 83 | assert protocol[:1] == b'Z' 84 | return pickle_decode 85 | 86 | 87 | def pickle_decode(msg): 88 | """Decodes msg and returns its parts""" 89 | unpickler = Unpickler(BytesIO(msg)) 90 | unpickler.find_global = find_global 91 | try: 92 | # PyPy, zodbpickle, the non-c-accelerated version 93 | unpickler.find_class = find_global 94 | except AttributeError: 95 | pass 96 | try: 97 | return unpickler.load() # msgid, flags, name, args 98 | except: # NOQA: E722 bare except 99 | logger.error("can't decode message: %s" % short_repr(msg)) 100 | raise 101 | 102 | 103 | def server_decoder(protocol): 104 | if protocol[:1] == b'M': 105 | return decoder(protocol) 106 | else: 107 | assert protocol[:1] == b'Z' 108 | return pickle_server_decode 109 | 110 | 111 | def pickle_server_decode(msg): 112 | """Decodes msg and returns its parts""" 113 | unpickler = Unpickler(BytesIO(msg)) 114 | unpickler.find_global = server_find_global 115 | try: 116 | # PyPy, zodbpickle, the non-c-accelerated version 117 | unpickler.find_class = server_find_global 118 | except AttributeError: 119 | pass 120 | 121 | try: 122 | return unpickler.load() # msgid, flags, name, args 123 | except: # NOQA: E722 bare except 124 | logger.error("can't decode message: %s" % short_repr(msg)) 125 | raise 126 | 127 | 128 | def server_default(obj): 129 | if isinstance(obj, Exception): 130 | return reduce_exception(obj) 131 | else: 132 | return obj 133 | 134 | 135 | def reduce_exception(exc): 136 | class_ = exc.__class__ 137 | class_ = f'{class_.__module__}.{class_.__name__}' 138 | return class_, exc.__dict__ or exc.args 139 | 140 | 141 | _globals = globals() 142 | _silly = ('__doc__',) 143 | 144 | exception_type_type = type(Exception) 145 | 146 | _SAFE_MODULE_NAMES = ( 147 | 'ZopeUndo.Prefix', 'zodbpickle', 148 | 'builtins', 'copy_reg', '__builtin__', 149 | ) 150 | 151 | 152 | def find_global(module, name): 153 | """Helper for message unpickler""" 154 | try: 155 | m = __import__(module, _globals, _globals, _silly) 156 | except ImportError as msg: 157 | raise ImportError(f'import error {module}: {msg}') 158 | 159 | try: 160 | r = getattr(m, name) 161 | except AttributeError: 162 | raise ImportError(f'module {module} has no global {name}') 163 | 164 | safe = getattr(r, '__no_side_effects__', 0) 165 | if safe: 166 | return r 167 | 168 | # TODO: is there a better way to do this? 169 | if isinstance(r, exception_type_type) and issubclass(r, Exception): 170 | return r 171 | 172 | raise ImportError(f'Unsafe global: {module}.{name}') 173 | 174 | 175 | def server_find_global(module, name): 176 | """Helper for message unpickler""" 177 | if module not in _SAFE_MODULE_NAMES: 178 | raise ImportError(f'Module not allowed: {module}') 179 | 180 | try: 181 | m = __import__(module, _globals, _globals, _silly) 182 | except ImportError as msg: 183 | raise ImportError(f'import error {module}: {msg}') 184 | 185 | try: 186 | r = getattr(m, name) 187 | except AttributeError: 188 | raise ImportError(f'module {module} has no global {name}') 189 | 190 | return r 191 | -------------------------------------------------------------------------------- /src/ZEO/asyncio/_smp.pyx: -------------------------------------------------------------------------------- 1 | # cython: language_level=3, boundscheck=False, wraparound=False 2 | 3 | """``cython`` implementation for ``SizedMessageProtocol``.""" 4 | from cpython.bytes cimport PyBytes_FromStringAndSize 5 | from libc.string cimport memcpy 6 | 7 | import logging 8 | import struct 9 | 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | cdef enum ReadState: 15 | process_size = 1 16 | process_message = 2 17 | closed = 0 18 | 19 | cdef object pack = struct.pack 20 | cdef object unpack = struct.unpack 21 | 22 | 23 | cdef class SizedMessageProtocol: 24 | cdef object __closed 25 | cdef public object receive # callback for received messages 26 | cdef public object connection_lost_called 27 | 28 | def __init__(self, receive): 29 | self.receive = receive 30 | self.__closed = self.connection_lost_called = False 31 | 32 | def set_receive(self, receive): 33 | self.receive = receive 34 | 35 | cdef public object transport 36 | 37 | def close(self): 38 | if self.__closed: 39 | return 40 | self.__closed = True 41 | self.eof_received() 42 | self.transport.close() 43 | # break reference cycles 44 | self.transport = self.receive = self.writelines = None 45 | 46 | # output 47 | cdef bint paused 48 | cdef list output # buffer 49 | cdef object writelines 50 | 51 | cdef _write_message(self, bytes message): 52 | self.writelines((pack(">I", len(message)), message)) 53 | 54 | def write_message(self, message): 55 | if self.paused: 56 | self.output.append(message) 57 | else: 58 | self._write_message(message) 59 | 60 | def write_message_iter(self, message_iter): 61 | it = iter(message_iter) 62 | if self.paused: 63 | self.output.append(it) 64 | return 65 | for message in it: 66 | self._write_message(message) 67 | if self.paused: 68 | self.output.append(it) 69 | return 70 | 71 | # protocol responsibilities 72 | def pause_writing(self): 73 | self.paused = 1 74 | 75 | def resume_writing(self): 76 | self.paused = 0 77 | cdef list output = self.output 78 | while output and not self.paused: 79 | message = output.pop(0) 80 | if type(message) is bytes: 81 | self._write_message(message) 82 | else: 83 | it = message 84 | for message in it: 85 | self._write_message(message) 86 | if self.paused: 87 | self.output.insert(0, it) 88 | return 89 | 90 | # input 91 | cdef ReadState read_state # current read state 92 | cdef unsigned read_wanted # wanted data size 93 | cdef unsigned received_count # received unprocessed bytes 94 | cdef list chunk_buffer # received data chunks 95 | cdef unsigned chunk_index # unprocessed index in 1. chunk 96 | 97 | # protocol responsibilities 98 | def data_received(self, bytes data): 99 | self.chunk_buffer.append(data) 100 | self.received_count += len(data) 101 | cdef unsigned wanted = self.read_wanted 102 | cdef bytes target 103 | cdef unsigned char *tv 104 | cdef unsigned tvi 105 | cdef bytes chunk 106 | cdef const unsigned char *cv 107 | cdef unsigned ci 108 | cdef unsigned unprocessed, use, i 109 | while self.read_state and self.read_wanted <= self.received_count: 110 | wanted = self.read_wanted 111 | tv = target = PyBytes_FromStringAndSize( NULL, wanted) 112 | tvi = 0 113 | while wanted: 114 | cv = chunk = self.chunk_buffer[0] 115 | ci = self.chunk_index 116 | unprocessed = len(chunk) - ci 117 | if unprocessed > wanted: 118 | use = wanted 119 | self.chunk_index += wanted 120 | else: 121 | use = unprocessed 122 | self.chunk_buffer.pop(0) 123 | self.chunk_index = 0 124 | if use <= 4: 125 | for i in range(use): 126 | tv[tvi + i] = cv[ci + i] 127 | else: 128 | memcpy(&tv[tvi], &cv[ci], use) 129 | tvi += use 130 | wanted -= use 131 | self.received_count -= self.read_wanted 132 | if self.read_state == process_size: 133 | self.read_state = process_message 134 | self.read_wanted = unpack(">I", target)[0] 135 | else: # read_state == process_message 136 | try: 137 | self.receive(target) 138 | except Exception: 139 | logger.exception("Processing message `%r` failed" 140 | % target) 141 | if self.read_state: # not yet closed 142 | self.read_state = process_size 143 | self.read_wanted = 4 144 | 145 | def connection_made(self, transport): 146 | self.transport = transport 147 | self.writelines = transport.writelines 148 | self.paused = 0 149 | self.output = [] 150 | self.read_state = process_size 151 | self.read_wanted = 4 152 | self.received_count = 0 153 | self.chunk_buffer = [] 154 | self.chunk_index = 0 155 | 156 | 157 | def connection_lost(self, exc): 158 | self.connection_lost_called = True 159 | if self.__closed: 160 | return 161 | self.transport.close() 162 | 163 | def eof_received(self): 164 | self.read_state = closed 165 | -------------------------------------------------------------------------------- /src/ZEO/tests/test_client_side_conflict_resolution.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import tempfile 4 | 5 | import zope.testing.setupstack 6 | from BTrees.Length import Length 7 | from ZODB import serialize 8 | from ZODB.broken import find_global 9 | from ZODB.DemoStorage import DemoStorage 10 | from ZODB.utils import maxtid 11 | from ZODB.utils import z64 12 | 13 | import ZEO 14 | 15 | from .utils import StorageServer 16 | 17 | 18 | class Var: 19 | def __eq__(self, other): 20 | self.value = other 21 | return True 22 | 23 | 24 | class ClientSideConflictResolutionTests(zope.testing.setupstack.TestCase): 25 | 26 | def test_server_side(self): 27 | # First, verify default conflict resolution. 28 | server = StorageServer(self, DemoStorage()) 29 | zs = server.zs 30 | 31 | reader = serialize.ObjectReader( 32 | factory=lambda conn, *args: find_global(*args)) 33 | writer = serialize.ObjectWriter() 34 | ob = Length(0) 35 | ob._p_oid = z64 36 | 37 | # 2 non-conflicting transactions: 38 | 39 | zs.tpc_begin(1, '', '', {}) 40 | zs.storea(ob._p_oid, z64, writer.serialize(ob), 1) 41 | self.assertEqual(zs.vote(1), []) 42 | tid1 = server.unpack_result(zs.tpc_finish(1)) 43 | server.assert_calls(self, ('info', {'length': 1, 'size': Var()})) 44 | 45 | ob.change(1) 46 | zs.tpc_begin(2, '', '', {}) 47 | zs.storea(ob._p_oid, tid1, writer.serialize(ob), 2) 48 | self.assertEqual(zs.vote(2), []) 49 | tid2 = server.unpack_result(zs.tpc_finish(2)) 50 | server.assert_calls(self, ('info', {'size': Var(), 'length': 1})) 51 | 52 | # Now, a cnflicting one: 53 | zs.tpc_begin(3, '', '', {}) 54 | zs.storea(ob._p_oid, tid1, writer.serialize(ob), 3) 55 | 56 | # Vote returns the object id, indicating that a conflict was resolved. 57 | self.assertEqual(zs.vote(3), [ob._p_oid]) 58 | tid3 = server.unpack_result(zs.tpc_finish(3)) 59 | 60 | p, serial, next_serial = zs.loadBefore(ob._p_oid, maxtid) 61 | self.assertEqual((serial, next_serial), (tid3, None)) 62 | self.assertEqual(reader.getClassName(p), 'BTrees.Length.Length') 63 | self.assertEqual(reader.getState(p), 2) 64 | 65 | # Now, we'll create a server that expects the client to 66 | # resolve conflicts: 67 | 68 | server = StorageServer( 69 | self, DemoStorage(), client_conflict_resolution=True) 70 | zs = server.zs 71 | 72 | # 2 non-conflicting transactions: 73 | 74 | zs.tpc_begin(1, '', '', {}) 75 | zs.storea(ob._p_oid, z64, writer.serialize(ob), 1) 76 | self.assertEqual(zs.vote(1), []) 77 | tid1 = server.unpack_result(zs.tpc_finish(1)) 78 | server.assert_calls(self, ('info', {'size': Var(), 'length': 1})) 79 | 80 | ob.change(1) 81 | zs.tpc_begin(2, '', '', {}) 82 | zs.storea(ob._p_oid, tid1, writer.serialize(ob), 2) 83 | self.assertEqual(zs.vote(2), []) 84 | tid2 = server.unpack_result(zs.tpc_finish(2)) 85 | server.assert_calls(self, ('info', {'length': 1, 'size': Var()})) 86 | 87 | # Now, a conflicting one: 88 | zs.tpc_begin(3, '', '', {}) 89 | zs.storea(ob._p_oid, tid1, writer.serialize(ob), 3) 90 | 91 | # Vote returns an object, indicating that a conflict was not resolved. 92 | self.assertEqual( 93 | zs.vote(3), 94 | [dict(oid=ob._p_oid, 95 | serials=(tid2, tid1), 96 | data=writer.serialize(ob), 97 | )], 98 | ) 99 | 100 | # Now, it's up to the client to resolve the conflict. It can 101 | # do this by making another store call. In this call, we use 102 | # tid2 as the starting tid: 103 | ob.change(1) 104 | zs.storea(ob._p_oid, tid2, writer.serialize(ob), 3) 105 | self.assertEqual(zs.vote(3), []) 106 | tid3 = server.unpack_result(zs.tpc_finish(3)) 107 | server.assert_calls(self, ('info', {'size': Var(), 'length': 1})) 108 | 109 | p, serial, next_serial = zs.loadBefore(ob._p_oid, maxtid) 110 | self.assertEqual((serial, next_serial), (tid3, None)) 111 | self.assertEqual(reader.getClassName(p), 'BTrees.Length.Length') 112 | self.assertEqual(reader.getState(p), 3) 113 | 114 | def test_client_side(self): 115 | # First, traditional: 116 | path = tempfile.mkdtemp(prefix='zeo-test-') 117 | self.addCleanup(shutil.rmtree, path) 118 | addr, stop = ZEO.server(os.path.join(path, 'data.fs'), threaded=False) 119 | db = ZEO.DB(addr, wait_timeout=2) 120 | with db.transaction() as conn: 121 | conn.root.len = Length(0) 122 | conn2 = db.open() 123 | conn2.root.len.change(1) 124 | with db.transaction() as conn: 125 | conn.root.len.change(1) 126 | 127 | conn2.transaction_manager.commit() 128 | 129 | self.assertEqual(conn2.root.len.value, 2) 130 | 131 | db.close() 132 | stop() 133 | 134 | # Now, do conflict resolution on the client. 135 | addr2, stop = ZEO.server( 136 | storage_conf='\n\n', 137 | zeo_conf=dict(client_conflict_resolution=True), 138 | threaded=False, 139 | ) 140 | 141 | db = ZEO.DB(addr2) 142 | with db.transaction() as conn: 143 | conn.root.len = Length(0) 144 | conn2 = db.open() 145 | conn2.root.len.change(1) 146 | with db.transaction() as conn: 147 | conn.root.len.change(1) 148 | 149 | self.assertEqual(conn2.root.len.value, 1) 150 | conn2.transaction_manager.commit() 151 | 152 | self.assertEqual(conn2.root.len.value, 2) 153 | 154 | db.close() 155 | stop() 156 | -------------------------------------------------------------------------------- /src/ZEO/tests/testZEOServer.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest import mock 3 | 4 | from ZEO.runzeo import ZEOServer 5 | 6 | 7 | class TestStorageServer: 8 | 9 | def __init__(self, fail_create_server): 10 | self.called = [] 11 | if fail_create_server: 12 | raise RuntimeError() 13 | 14 | def close(self): 15 | self.called.append("close") 16 | 17 | 18 | class TestZEOServer(ZEOServer): 19 | 20 | def __init__(self, fail_create_server=False, fail_loop_forever=False): 21 | ZEOServer.__init__(self, None) 22 | self.called = [] 23 | self.fail_create_server = fail_create_server 24 | self.fail_loop_forever = fail_loop_forever 25 | 26 | def setup_default_logging(self): 27 | self.called.append("setup_default_logging") 28 | 29 | def check_socket(self): 30 | self.called.append("check_socket") 31 | 32 | def clear_socket(self): 33 | self.called.append("clear_socket") 34 | 35 | def make_pidfile(self): 36 | self.called.append("make_pidfile") 37 | 38 | def open_storages(self): 39 | self.called.append("open_storages") 40 | 41 | def setup_signals(self): 42 | self.called.append("setup_signals") 43 | 44 | def create_server(self): 45 | self.called.append("create_server") 46 | self.server = TestStorageServer(self.fail_create_server) 47 | 48 | def loop_forever(self): 49 | self.called.append("loop_forever") 50 | if self.fail_loop_forever: 51 | raise RuntimeError() 52 | 53 | def close_server(self): 54 | self.called.append("close_server") 55 | ZEOServer.close_server(self) 56 | 57 | def remove_pidfile(self): 58 | self.called.append("remove_pidfile") 59 | 60 | 61 | class AttributeErrorTests(unittest.TestCase): 62 | 63 | def testFailCreateServer(self): 64 | # 65 | # Fix AttributeError: 'ZEOServer' object has no attribute 66 | # 'server' in ZEOServer.main 67 | # 68 | # Demonstrate the AttributeError 69 | zeo = TestZEOServer(fail_create_server=True) 70 | self.assertRaises(RuntimeError, zeo.main) 71 | 72 | 73 | class CloseServerTests(unittest.TestCase): 74 | 75 | def testCallSequence(self): 76 | # The close_server hook is called after loop_forever 77 | # has returned 78 | zeo = TestZEOServer() 79 | zeo.main() 80 | self.assertEqual(zeo.called, [ 81 | "setup_default_logging", 82 | "check_socket", 83 | "clear_socket", 84 | "make_pidfile", 85 | "open_storages", 86 | "setup_signals", 87 | "create_server", 88 | "loop_forever", 89 | "close_server", # New 90 | "clear_socket", 91 | "remove_pidfile", 92 | ]) 93 | # The default implementation closes the storage server 94 | self.assertEqual(hasattr(zeo, "server"), True) 95 | self.assertEqual(zeo.server.called, ["close"]) 96 | 97 | def testFailLoopForever(self): 98 | # The close_server hook is called if loop_forever exits 99 | # with an exception 100 | zeo = TestZEOServer(fail_loop_forever=True) 101 | self.assertRaises(RuntimeError, zeo.main) 102 | self.assertEqual(zeo.called, [ 103 | "setup_default_logging", 104 | "check_socket", 105 | "clear_socket", 106 | "make_pidfile", 107 | "open_storages", 108 | "setup_signals", 109 | "create_server", 110 | "loop_forever", 111 | "close_server", 112 | "clear_socket", 113 | "remove_pidfile", 114 | ]) 115 | # The storage server has been closed 116 | self.assertEqual(hasattr(zeo, "server"), True) 117 | self.assertEqual(zeo.server.called, ["close"]) 118 | 119 | def testFailCreateServer(self): 120 | # The close_server hook is called if create_server exits 121 | # with an exception 122 | zeo = TestZEOServer(fail_create_server=True) 123 | self.assertRaises(RuntimeError, zeo.main) 124 | self.assertEqual(zeo.called, [ 125 | "setup_default_logging", 126 | "check_socket", 127 | "clear_socket", 128 | "make_pidfile", 129 | "open_storages", 130 | "setup_signals", 131 | "create_server", 132 | "close_server", 133 | "clear_socket", 134 | "remove_pidfile", 135 | ]) 136 | # The server attribute is present but None 137 | self.assertEqual(hasattr(zeo, "server"), True) 138 | self.assertEqual(zeo.server, None) 139 | 140 | 141 | @mock.patch('os.unlink') 142 | class TestZEOServerSocket(unittest.TestCase): 143 | 144 | def _unlinked(self, unlink, options): 145 | server = ZEOServer(options) 146 | server.clear_socket() 147 | unlink.assert_called_once() 148 | 149 | def _not_unlinked(self, unlink, options): 150 | server = ZEOServer(options) 151 | server.clear_socket() 152 | unlink.assert_not_called() 153 | 154 | def test_clear_with_native_str(self, unlink): 155 | class Options: 156 | address = "a str that does not exist" 157 | self._unlinked(unlink, Options) 158 | 159 | def test_clear_with_unicode_str(self, unlink): 160 | class Options: 161 | address = "a str that does not exist" 162 | self._unlinked(unlink, Options) 163 | 164 | def test_clear_with_bytes(self, unlink): 165 | class Options: 166 | address = b'a byte str that does not exist' 167 | 168 | self._not_unlinked(unlink, Options) 169 | 170 | def test_clear_with_tuple(self, unlink): 171 | class Options: 172 | address = ('abc', 1) 173 | self._not_unlinked(unlink, Options) 174 | -------------------------------------------------------------------------------- /src/ZEO/asyncio/base.py: -------------------------------------------------------------------------------- 1 | """ZEO Protocol. 2 | 3 | A ZEO protocol instance can be used as a connection. 4 | It exchanges ``bytes`` messages. 5 | Messages are sent via the methods 6 | ``write_message`` (send a single message) and 7 | ``write_message_iter`` (send the messages generated by an iterator). 8 | Received messages are reported via callbacks. 9 | Messages are received in the same order as they have been written; 10 | especially, the messages wrote with ``write_message_iter`` 11 | are received as contiguous messages. 12 | 13 | The first message transmits the protocol version. 14 | Its callback is ``finish_connection``. 15 | The first byte of the protocol version message identifies 16 | an encoding type; the remaining bytes specify the version. 17 | ``finish_connection`` is expected to set up 18 | methods ``encode`` and ``decode`` corresponding to the 19 | encoding type. 20 | 21 | Followup messages carry encoded tuples 22 | *msgid*, *async_flag*, *name*, *args* 23 | representing either calls (synchronous or asynchronous) or replies. 24 | Their callback is ``message_received``. 25 | 26 | ZEO protocol instances can be used concurrently from coroutines (executed 27 | in the same thread). 28 | They are not thread safe. 29 | 30 | The ZEO protocol sits on top of a sized message protocol. 31 | 32 | The ZEO protocol has client and server variants. 33 | """ 34 | import logging 35 | from asyncio import Protocol 36 | 37 | from .smp import SizedMessageProtocol 38 | 39 | 40 | logger = logging.getLogger(__name__) 41 | 42 | 43 | class ZEOBaseProtocol(Protocol): 44 | """ZEO protocol base class for the common features.""" 45 | 46 | protocol_version = None 47 | 48 | def __init__(self, loop, name): 49 | self.loop = loop 50 | self.name = name 51 | 52 | # API -- defined in ``connection_made`` 53 | # write_message(message) 54 | # write_message_iter(message_iter) 55 | def call_async(self, method, args): 56 | """call method named *method* asynchronously with *args*.""" 57 | self.write_message(self.encode(0, True, method, args)) 58 | 59 | def call_async_iter(self, it): 60 | self.write_message_iter(self.encode(0, True, method, args) 61 | for method, args in it) 62 | 63 | def get_peername(self): 64 | return self.sm_protocol.transport.get_extra_info('peername') 65 | 66 | def protocol_factory(self): 67 | return self 68 | 69 | closing = None # ``None`` or closed future 70 | sm_protocol = None 71 | 72 | def close(self): 73 | """schedule closing, return closed future.""" 74 | # with ``asyncio``, ``close`` only schedules the closing; 75 | # close completion is signalled via a call to ``connection_lost``. 76 | closing = self.closing 77 | if closing is None: 78 | closing = self.closing = self.loop.create_future() 79 | # can get closed before ``sm_protocol`` set up 80 | if self.sm_protocol is not None: 81 | # will eventually cause ``connection_lost`` 82 | self.sm_protocol.close() 83 | else: 84 | closing.set_result(True) 85 | elif self.sm_protocol is not None: 86 | self.sm_protocol.close() # no problem if repeated 87 | return closing 88 | 89 | def __repr__(self): 90 | cls = self.__class__ 91 | return f'{cls.__module__}.{cls.__name__}({self.name})' 92 | 93 | # to be defined by deriving classes 94 | # def finish_connection(protocol_version_message) 95 | # def message_received(message) 96 | 97 | # ``Protocol`` responsibilities -- defined in ``connection_made`` 98 | # data_received 99 | # eof_received 100 | # pause_writing 101 | # resume_writing 102 | def connection_made(self, transport): 103 | logger.info("Connected %s", self) 104 | # set up lower level sized message protocol 105 | # creates reference cycle 106 | smp = self.sm_protocol = SizedMessageProtocol(self._first_message) 107 | smp.connection_made(transport) # takes over ``transport`` 108 | self.data_received = smp.data_received 109 | self.eof_received = smp.eof_received 110 | self.pause_writing = smp.pause_writing 111 | self.resume_writing = smp.resume_writing 112 | self.write_message = smp.write_message 113 | self.write_message_iter = smp.write_message_iter 114 | 115 | # In real life ``connection_lost`` is only called by 116 | # the transport and ``asyncio.Protocol`` guarantees that 117 | # it is called exactly once (if ``connection_made`` has 118 | # been called) or not at all. 119 | # Some tests, however, call ``connection_lost`` themselves. 120 | # The following attribute helps to ensure that ``connection_lost`` 121 | # is called exactly once. 122 | connection_lost_called = False 123 | 124 | def connection_lost(self, exc): 125 | """The call signals close completion.""" 126 | self.connection_lost_called = True 127 | self.sm_protocol.connection_lost(exc) 128 | closing = self.closing 129 | if closing is None: 130 | closing = self.closing = self.loop.create_future() 131 | if not closing.done(): 132 | closing.set_result(True) 133 | 134 | # internal 135 | def _first_message(self, protocol_version): 136 | self.sm_protocol.set_receive(self.message_received) 137 | self.finish_connection(protocol_version) 138 | 139 | # ``uvloop`` workaround 140 | # We define ``data_received`` in ``connection_made``. 141 | # ``uvloop``, however, caches ``protocol.data_received`` before 142 | # it calls ``connection_made`` - at a consequence, data is not 143 | # received 144 | # The method below is overridden in ``connection_made``. 145 | def data_received(self, data): 146 | self.data_received(data) # not an infinite loop, because overridden 147 | -------------------------------------------------------------------------------- /docs/introduction.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Introduction 3 | ============ 4 | 5 | There are several features that affect the behavior of 6 | ZEO. This section describes how a few of these features 7 | work. Subsequent sections describe how to configure every option. 8 | 9 | Client cache 10 | ============ 11 | 12 | Each ZEO client keeps an on-disk cache of recently used data records 13 | to avoid fetching those records from the server each time they are 14 | requested. It is usually faster to read the objects from disk than it 15 | is to fetch them over the network. The cache can also provide 16 | read-only copies of objects during server outages. 17 | 18 | The cache may be persistent or transient. If the cache is persistent, 19 | then the cache files are retained for use after process restarts. A 20 | non-persistent cache uses temporary files that are removed when the 21 | client storage is closed. 22 | 23 | The client cache size is configured when the ClientStorage is created. 24 | The default size is 20MB, but the right size depends entirely on the 25 | particular database. Setting the cache size too small can hurt 26 | performance, but in most cases making it too big just wastes disk 27 | space. 28 | 29 | ZEO uses invalidations for cache consistency. Every time an object is 30 | modified, the server sends a message to each client informing it of 31 | the change. The client will discard the object from its cache when it 32 | receives an invalidation. (It's actually a little more complicated, 33 | but we won't get into that here.) 34 | 35 | Each time a client connects to a server, it must verify that its cache 36 | contents are still valid. (It did not receive any invalidation 37 | messages while it was disconnected.) This involves asking the server 38 | to replay invalidations it missed. If it's been disconnected too long, 39 | it discards its cache. 40 | 41 | 42 | Invalidation queue 43 | ================== 44 | 45 | The ZEO server keeps a queue of recent invalidation messages in 46 | memory. When a client connects to the server, it sends the timestamp 47 | of the most recent invalidation message it has received. If that 48 | message is still in the invalidation queue, then the server sends the 49 | client all the missing invalidations. 50 | 51 | The default size of the invalidation queue is 100. If the 52 | invalidation queue is larger, it will be more likely that a client 53 | that reconnects will be able to verify its cache using the queue. On 54 | the other hand, a large queue uses more memory on the server to store 55 | the message. Invalidation messages tend to be small, perhaps a few 56 | hundred bytes each on average; it depends on the number of objects 57 | modified by a transaction. 58 | 59 | You can also provide an invalidation age when configuring the 60 | server. In this case, if the invalidation queue is too small, but a 61 | client has been disconnected for a time interval that is less than the 62 | invalidation age, then invalidations are replayed by iterating over 63 | the lower-level storage on the server. If the age is too high, and 64 | clients are disconnected for a long time, then this can put a lot of 65 | load on the server. 66 | 67 | Transaction timeouts 68 | ==================== 69 | 70 | A ZEO server can be configured to timeout a transaction if it takes 71 | too long to complete. Only a single transaction can commit at a time; 72 | so if one transaction takes too long, all other clients will be 73 | delayed waiting for it. In the extreme, a client can hang during the 74 | commit process. If the client hangs, the server will be unable to 75 | commit other transactions until it restarts. A well-behaved client 76 | will not hang, but the server can be configured with a transaction 77 | timeout to guard against bugs that cause a client to hang. 78 | 79 | If any transaction exceeds the timeout threshold, the client's 80 | connection to the server will be closed and the transaction aborted. 81 | Once the transaction is aborted, the server can start processing other 82 | client's requests. Most transactions should take very little time to 83 | commit. The timer begins for a transaction after all the data has 84 | been sent to the server. At this point, the cost of commit should be 85 | dominated by the cost of writing data to disk; it should be unusual 86 | for a commit to take longer than 1 second. A transaction timeout of 87 | 30 seconds should tolerate heavy load and slow communications between 88 | client and server, while guarding against hung servers. 89 | 90 | When a transaction times out, the client can be left in an awkward 91 | position. If the timeout occurs during the second phase of the two 92 | phase commit, the client will log a panic message. This should only 93 | cause problems if the client transaction involved multiple storages. 94 | If it did, it is possible that some storages committed the client 95 | changes and others did not. 96 | 97 | Connection management 98 | ===================== 99 | 100 | A ZEO client manages its connection to the ZEO server. If it loses 101 | the connection, it attempts to reconnect. While 102 | it is disconnected, it can satisfy some reads by using its cache. 103 | 104 | The client can be configured with multiple server addresses. In this 105 | case, it assumes that each server has identical content and will use 106 | any server that is available. It is possible to configure the client 107 | to accept a read-only connection to one of these servers if no 108 | read-write connection is available. If it has a read-only connection, 109 | it will continue to poll for a read-write connection. 110 | 111 | If a single address resolves to multiple IPv4 or IPv6 addresses, 112 | the client will connect to an arbitrary of these addresses. 113 | 114 | SSL 115 | === 116 | 117 | ZEO supports the use of SSL connections between servers and clients, 118 | including certificate authentication. We're still understanding use 119 | cases for this, so details of operation may change. 120 | -------------------------------------------------------------------------------- /src/ZEO/tests/zeo_blob_cache.test: -------------------------------------------------------------------------------- 1 | ZEO caching of blob data 2 | ======================== 3 | 4 | ZEO supports 2 modes for providing clients access to blob data: 5 | 6 | shared 7 | Blob data are shared via a network file system. The client shares 8 | a common blob directory with the server. 9 | 10 | non-shared 11 | Blob data are loaded from the storage server and cached locally. 12 | A maximum size for the blob data can be set and data are removed 13 | when the size is exceeded. 14 | 15 | In this test, we'll demonstrate that blobs data are removed from a ZEO 16 | cache when the amount of data stored exceeds a given limit. 17 | 18 | Let's start by setting up some data: 19 | 20 | >>> addr, _ = start_server(blob_dir='server-blobs') 21 | 22 | We'll also create a client. 23 | 24 | >>> import ZEO 25 | >>> db = ZEO.DB(addr, blob_dir='blobs', blob_cache_size=3000) 26 | 27 | Here, we passed a blob_cache_size parameter, which specifies a target 28 | blob cache size. This is not a hard limit, but rather a target. It 29 | defaults to a very large value. We also passed a blob_cache_size_check 30 | option. The blob_cache_size_check option specifies the number of 31 | bytes, as a percent of the target that can be written or downloaded 32 | from the server before the cache size is checked. The 33 | blob_cache_size_check option defaults to 100. We passed 10, to check 34 | after writing 10% of the target size. 35 | 36 | .. We're going to wait for any threads we started to finish, so... 37 | 38 | >>> import threading 39 | >>> old_threads = list(threading.enumerate()) 40 | 41 | We want to check for name collections in the blob cache dir. We'll try 42 | to provoke name collections by reducing the number of cache directory 43 | subdirectories. 44 | 45 | >>> import ZEO.ClientStorage 46 | >>> orig_blob_cache_layout_size = ZEO.ClientStorage.BlobCacheLayout.size 47 | >>> ZEO.ClientStorage.BlobCacheLayout.size = 11 48 | 49 | Now, let's write some data: 50 | 51 | >>> import ZODB.blob, transaction, time 52 | >>> conn = db.open() 53 | >>> for i in range(1, 101): 54 | ... conn.root()[i] = ZODB.blob.Blob() 55 | ... with conn.root()[i].open('w') as f: 56 | ... w = f.write((chr(i)*100).encode('ascii')) 57 | >>> transaction.commit() 58 | 59 | We've committed 10000 bytes of data, but our target size is 3000. We 60 | expect to have not much more than the target size in the cache blob 61 | directory. 62 | 63 | >>> import os 64 | >>> def cache_size(d): 65 | ... size = 0 66 | ... for base, dirs, files in os.walk(d): 67 | ... for f in files: 68 | ... if f.endswith('.blob'): 69 | ... try: 70 | ... size += os.stat(os.path.join(base, f)).st_size 71 | ... except OSError: 72 | ... if os.path.exists(os.path.join(base, f)): 73 | ... raise 74 | ... return size 75 | 76 | >>> def check(): 77 | ... return cache_size('blobs') < 5000 78 | >>> def onfail(): 79 | ... return cache_size('blobs') 80 | 81 | >>> from ZEO.tests.forker import wait_until 82 | >>> wait_until("size is reduced", check, 99, onfail) 83 | 84 | If we read all of the blobs, data will be downloaded again, as 85 | necessary, but the cache size will remain not much bigger than the 86 | target: 87 | 88 | >>> for i in range(1, 101): 89 | ... with conn.root()[i].open() as f: 90 | ... data = f.read() 91 | ... if data != (chr(i)*100).encode('ascii'): 92 | ... print('bad data', repr(chr(i)), repr(data)) 93 | 94 | >>> wait_until("size is reduced", check, 99, onfail) 95 | 96 | >>> for i in range(1, 101): 97 | ... with conn.root()[i].open() as f: 98 | ... data = f.read() 99 | ... if data != (chr(i)*100).encode('ascii'): 100 | ... print('bad data', repr(chr(i)), repr(data)) 101 | 102 | >>> for i in range(1, 101): 103 | ... with conn.root()[i].open('c') as f: 104 | ... data = f.read() 105 | ... if data != (chr(i)*100).encode('ascii'): 106 | ... print('bad data', repr(chr(i)), repr(data)) 107 | 108 | >>> wait_until("size is reduced", check, 99, onfail) 109 | 110 | Now let see if we can stress things a bit. We'll create many clients 111 | and get them to pound on the blobs all at once to see if we can 112 | provoke problems: 113 | 114 | >>> import threading, random 115 | >>> def run(): 116 | ... db = ZEO.DB(addr, blob_dir='blobs', blob_cache_size=4000) 117 | ... conn = db.open() 118 | ... for i in range(300): 119 | ... time.sleep(0) 120 | ... i = random.randint(1, 100) 121 | ... with conn.root()[i].open() as f: 122 | ... data = f.read() 123 | ... if data != (chr(i)*100).encode('ascii'): 124 | ... print('bad data', repr(chr(i)), repr(data)) 125 | ... i = random.randint(1, 100) 126 | ... with conn.root()[i].open('c') as f: 127 | ... data = f.read() 128 | ... if data != (chr(i)*100).encode('ascii'): 129 | ... print('bad data', repr(chr(i)), repr(data)) 130 | ... db.close() 131 | 132 | >>> threads = [threading.Thread(target=run) for i in range(10)] 133 | >>> for thread in threads: 134 | ... thread.daemon = True 135 | >>> for thread in threads: 136 | ... thread.start() 137 | >>> for thread in threads: 138 | ... thread.join(99) 139 | ... if thread.is_alive(): 140 | ... print("Can't join thread.") 141 | 142 | >>> wait_until("size is reduced", check, 99, onfail) 143 | 144 | .. cleanup 145 | 146 | >>> for thread in threading.enumerate(): 147 | ... if thread not in old_threads: 148 | ... thread.join(33) 149 | 150 | >>> db.close() 151 | >>> ZEO.ClientStorage.BlobCacheLayout.size = orig_blob_cache_layout_size 152 | --------------------------------------------------------------------------------