├── .clog.toml ├── .coveragerc ├── .flake8 ├── .git-blame-ignore-revs ├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.md │ └── enhancement.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml └── workflows │ ├── release.yml │ └── testing.yml ├── .gitignore ├── .hound.yml ├── .readthedocs.yaml ├── CHANGES.md ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── constraints.txt ├── docs ├── Makefile ├── api.rst ├── api │ ├── client.rst │ ├── exceptions.rst │ ├── handlers │ │ ├── eventlet.rst │ │ ├── gevent.rst │ │ ├── threading.rst │ │ └── utils.rst │ ├── interfaces.rst │ ├── protocol │ │ └── states.rst │ ├── recipe │ │ ├── barrier.rst │ │ ├── cache.rst │ │ ├── counter.rst │ │ ├── election.rst │ │ ├── lease.rst │ │ ├── lock.rst │ │ ├── partitioner.rst │ │ ├── party.rst │ │ ├── queue.rst │ │ └── watchers.rst │ ├── retry.rst │ ├── security.rst │ └── testing.rst ├── async_usage.rst ├── basic_usage.rst ├── changelog.rst ├── conf.py ├── contributing.rst ├── glossary.rst ├── implementation.rst ├── index.rst ├── install.rst ├── make.bat └── testing.rst ├── ensure-zookeeper-env.sh ├── init_krb5.sh ├── kazoo ├── __init__.py ├── client.py ├── exceptions.py ├── handlers │ ├── __init__.py │ ├── eventlet.py │ ├── gevent.py │ ├── threading.py │ └── utils.py ├── hosts.py ├── interfaces.py ├── loggingsupport.py ├── protocol │ ├── __init__.py │ ├── connection.py │ ├── paths.py │ ├── serialization.py │ └── states.py ├── recipe │ ├── __init__.py │ ├── barrier.py │ ├── cache.py │ ├── counter.py │ ├── election.py │ ├── lease.py │ ├── lock.py │ ├── partitioner.py │ ├── party.py │ ├── queue.py │ └── watchers.py ├── retry.py ├── security.py ├── testing │ ├── __init__.py │ ├── common.py │ └── harness.py ├── tests │ ├── __init__.py │ ├── conftest.py │ ├── test_barrier.py │ ├── test_build.py │ ├── test_cache.py │ ├── test_client.py │ ├── test_connection.py │ ├── test_counter.py │ ├── test_election.py │ ├── test_eventlet_handler.py │ ├── test_exceptions.py │ ├── test_gevent_handler.py │ ├── test_hosts.py │ ├── test_interrupt.py │ ├── test_lease.py │ ├── test_lock.py │ ├── test_partitioner.py │ ├── test_party.py │ ├── test_paths.py │ ├── test_queue.py │ ├── test_retry.py │ ├── test_sasl.py │ ├── test_security.py │ ├── test_selectors_select.py │ ├── test_threading_handler.py │ ├── test_utils.py │ ├── test_watchers.py │ └── util.py └── version.py ├── pyproject.toml ├── run_failure.py ├── setup.cfg ├── setup.py └── tox.ini /.clog.toml: -------------------------------------------------------------------------------- 1 | [clog] 2 | repository = "https://github.com/python-zk/kazoo" 3 | changelog = "CHANGES.md" 4 | from-latest-tag = true 5 | link-style = "github" 6 | 7 | [sections] 8 | Refactor = ["refactor"] 9 | Test = ["test"] 10 | Doc = ["docs"] 11 | Chore = ["chore"] 12 | Features = ["feat", "feature"] 13 | "Bug Fixes" = ["fix", "bug"] 14 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | include = 3 | kazoo/* 4 | omit = 5 | kazoo/tests/* 6 | kazoo/testing/* 7 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | builtins = _ 3 | exclude = 4 | .git, 5 | __pycache__, 6 | .venv/,venv/, 7 | .tox/, 8 | build/,dist/,*egg, 9 | docs/conf.py, 10 | zookeeper/ 11 | # See black's documentation for E203 12 | max-line-length = 79 13 | extend-ignore = BLK100,E203 14 | 15 | -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # Commit ignored from blame 2 | # git config blame.ignoreRevsFile .git-blame-ignore-revs 3 | 4 | # Reformat using black 22.10.0 5 | 686717629f71c66d39ab0352c605c73eace5bd1f 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: Report a bug encountered while using Kazoo 4 | 5 | --- 6 | 7 | 8 | 9 | 10 | ## Expected Behavior 11 | 12 | 13 | ## Actual Behavior 14 | 15 | 16 | ## Snippet to Reproduce the Problem 17 | 18 | ```COPY/PASTE the snippet here (omit any sensitive information)``` 19 | 20 | ## Logs with logging in DEBUG mode 21 | 22 | ```COPY/PASTE the result of the snippet here (omit any sensitive information)``` 23 | 24 | ## Specifications 25 | 26 | - Kazoo version: 27 | - Result of `pip list` command: 28 | - Zookeeper version: 29 | - Zookeeper configuration: put here any useful ZK configuration (authentication, encryption, number of ZK members, number of (concurrent?) clients, Java version, krb5 version, etc.) 30 | - Python version: 31 | - OS: 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/enhancement.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Enhancement Request 3 | about: Suggest an enhancement for Kazoo 4 | 5 | --- 6 | 7 | 8 | 9 | ## What would you like to be added 10 | 11 | 12 | ## Why is this needed 13 | 14 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Fixes # 2 | 3 | ## Why is this needed? 4 | 5 | ## Proposed Changes 6 | 7 | - change 1 8 | - change 2 9 | - ... 10 | 11 | ## Does this PR introduce any breaking change? 12 | 13 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Kazoo Awesome Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | build-and-release: 10 | name: Build and release Kazoo to Pypi 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Handle the code 14 | uses: actions/checkout@v4 15 | 16 | - name: Set up Python 3.12 17 | uses: actions/setup-python@v5 18 | with: 19 | python-version: "3.12" 20 | 21 | - name: Install pypa/build 22 | run: >- 23 | python -m 24 | pip install 25 | build 26 | --user 27 | 28 | - name: Build a binary wheel and a source tarball 29 | run: >- 30 | python -m 31 | build 32 | -C--global-option=egg_info 33 | -C--global-option=--tag-build="" 34 | --sdist 35 | --wheel 36 | --outdir dist/ 37 | . 38 | 39 | - name: Publish Kazoo to PyPI 40 | if: startsWith(github.ref, 'refs/tags') 41 | uses: pypa/gh-action-pypi-publish@master 42 | with: 43 | password: ${{ secrets.PYPI_API_TOKEN }} 44 | -------------------------------------------------------------------------------- /.github/workflows/testing.yml: -------------------------------------------------------------------------------- 1 | name: Kazoo Awesome Testing 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - release/* 8 | pull_request: 9 | branches: 10 | - master 11 | - release/* 12 | 13 | jobs: 14 | validate: 15 | name: Code Validation 16 | 17 | runs-on: ubuntu-latest 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | 22 | - uses: actions/setup-python@v5 23 | with: 24 | python-version: "3.12" 25 | 26 | - name: Handle pip cache 27 | uses: actions/cache@v4 28 | with: 29 | path: ~/.cache/pip 30 | key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} 31 | restore-keys: | 32 | ${{ runner.os }}-pip- 33 | 34 | - name: Install required dependencies 35 | run: | 36 | python3 -m pip install --upgrade pip 37 | pip install tox 38 | 39 | - name: Code check 40 | run: tox -e ${TOX_VENV} 41 | env: 42 | TOX_VENV: black,pep8,mypy 43 | 44 | test: 45 | needs: [validate] 46 | 47 | name: > 48 | Linux - Test Python ${{ matrix.python-version }}, 49 | ZK ${{ matrix.zk-version }} 50 | runs-on: ubuntu-latest 51 | 52 | strategy: 53 | fail-fast: false 54 | matrix: 55 | python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.10-v7.3.15"] 56 | zk-version: ["3.6.4", "3.7.2", "3.8.3", "3.9.1"] 57 | include: 58 | - python-version: "3.8" 59 | tox-env: py38 60 | - python-version: "3.9" 61 | tox-env: py39 62 | - python-version: "3.10" 63 | tox-env: py310 64 | - python-version: "3.11" 65 | tox-env: py311 66 | - python-version: "3.12" 67 | tox-env: py312 68 | - python-version: "pypy-3.10-v7.3.15" 69 | tox-env: pypy3 70 | steps: 71 | - uses: actions/checkout@v4 72 | 73 | - name: Set up Python ${{ matrix.python-version }} 74 | uses: actions/setup-python@v5 75 | with: 76 | python-version: ${{ matrix.python-version }} 77 | 78 | - name: Handle pip cache 79 | uses: actions/cache@v4 80 | with: 81 | path: ~/.cache/pip 82 | key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} 83 | restore-keys: | 84 | ${{ runner.os }}-pip- 85 | 86 | - name: Handle ZK installation cache 87 | uses: actions/cache@v4 88 | with: 89 | path: zookeeper 90 | key: ${{ runner.os }}-zookeeper 91 | restore-keys: | 92 | ${{ runner.os }}-zookeeper 93 | 94 | - name: Install required dependencies 95 | run: | 96 | sudo apt-get -y install libevent-dev krb5-kdc krb5-admin-server libkrb5-dev 97 | python3 -m pip install --upgrade pip 98 | pip install tox 99 | 100 | - name: Test with tox 101 | run: tox -e ${TOX_VENV} 102 | env: 103 | TOX_VENV: ${{ format('{0}-{1}', matrix.tox-env, 'gevent-eventlet-sasl') }} 104 | ZOOKEEPER_VERSION: ${{ matrix.zk-version }} 105 | # TODO: can be removed once tests for ZK 3.4 are removed 106 | ZOOKEEPER_PREFIX: "${{ !contains(matrix.zk-version, '3.4') && 'apache-' || '' }}" 107 | ZOOKEEPER_SUFFIX: "${{ !contains(matrix.zk-version, '3.4') && '-bin' || '' }}" 108 | ZOOKEEPER_LIB: "${{ !contains(matrix.zk-version, '3.4') && 'lib' || '' }}" 109 | 110 | - name: Publish Codecov report 111 | uses: codecov/codecov-action@v4 112 | 113 | test_windows: 114 | needs: [validate] 115 | name: Windows - Sanity test using a single version of Python and ZK 116 | 117 | runs-on: windows-latest 118 | steps: 119 | - uses: actions/checkout@v4 120 | 121 | - uses: actions/setup-python@v5 122 | with: 123 | python-version: "3.12" 124 | 125 | - name: Handle pip cache 126 | uses: actions/cache@v4 127 | with: 128 | path: ~/.cache/pip 129 | key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} 130 | restore-keys: | 131 | ${{ runner.os }}-pip- 132 | 133 | - name: Handle ZK installation cache 134 | uses: actions/cache@v4 135 | with: 136 | path: zookeeper 137 | key: ${{ runner.os }}-zookeeper 138 | restore-keys: | 139 | ${{ runner.os }}-zookeeper 140 | 141 | # https://github.com/actions/setup-java 142 | - name: Setup Java 143 | uses: actions/setup-java@v4 144 | with: 145 | distribution: 'temurin' 146 | java-version: '17' 147 | 148 | - name: Install required dependencies 149 | run: | 150 | python3 -m pip install --upgrade pip 151 | pip install tox 152 | 153 | - name: Test with tox 154 | run: tox -e py310 155 | env: 156 | ZOOKEEPER_VERSION: 3.9.1 157 | ZOOKEEPER_PREFIX: "apache-" 158 | ZOOKEEPER_SUFFIX: "-bin" 159 | ZOOKEEPER_LIB: "lib" 160 | shell: bash 161 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg 2 | *.egg-info 3 | *.komodo* 4 | *.kpf 5 | *.log 6 | *.pid 7 | *.pyc 8 | *.swp 9 | *.*.swp 10 | .*.*.swo 11 | *~ 12 | bin/ 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | docs/_build 17 | dropin.cache 18 | eggs/ 19 | include 20 | lib/ 21 | lib-python/ 22 | lib_pypy/ 23 | man/ 24 | parts/ 25 | share/ 26 | site-packages/ 27 | zookeeper/ 28 | .coverage 29 | .idea 30 | .project 31 | .pydevproject 32 | .tox 33 | venv 34 | /.settings 35 | /.metadata 36 | 37 | !.gitignore 38 | !.git-blame-ignore-revs 39 | 40 | -------------------------------------------------------------------------------- /.hound.yml: -------------------------------------------------------------------------------- 1 | fail_on_violations: true 2 | python: 3 | enabled: true 4 | config_file: .flake8 5 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: "ubuntu-lts-latest" 11 | tools: 12 | python: "latest" 13 | # You can also specify other tool versions: 14 | # nodejs: "19" 15 | # rust: "1.64" 16 | # golang: "1.19" 17 | 18 | # Build documentation in the docs/ directory with Sphinx 19 | sphinx: 20 | configuration: docs/conf.py 21 | 22 | # If using Sphinx, optionally build your docs in additional formats such as PDF 23 | formats: [] 24 | # - pdf 25 | 26 | # Optionally declare the Python requirements required to build your docs 27 | python: 28 | install: 29 | - method: pip 30 | path: . 31 | extra_requirements: 32 | - docs 33 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contribution Guidelines 2 | 3 | We gladly accept outside contributions. We use our 4 | [Github issue tracker](https://github.com/python-zk/kazoo/issues) 5 | for both discussions and talking about new features or bugs. You can 6 | also fork the project and sent us a pull request. If you have a more 7 | general topic to discuss, the 8 | [user@zookeeper.apache.org](https://zookeeper.apache.org/lists.html) 9 | mailing list is a good place to do so. You can sometimes find us on 10 | IRC in the 11 | [#zookeeper channel on freenode](https://zookeeper.apache.org/irc.html). 12 | 13 | [See the README](/README.rst) for contact information. 14 | 15 | ## Development 16 | 17 | If you want to work on the code and send us a 18 | [pull request](https://help.github.com/articles/using-pull-requests), 19 | first fork the repository on github to your own account. Then clone 20 | your new repository and run the build scripts: 21 | 22 | ``` 23 | git clone git@github.com:/kazoo.git 24 | cd kazoo 25 | make 26 | ``` 27 | 28 | You need a supported version of Python installed and available as `python` 29 | in your shell. To run Zookeeper you also need a Java runtime (JRE or JDK). 30 | Please refer to the Zookeeper documentation for compatible Java versions for 31 | each Zookeeper version. To run tests, you need to have `tox`, the Python 32 | testing tool, installed in your shell. 33 | 34 | You can run all the tests by calling: 35 | 36 | ``` 37 | make test 38 | ``` 39 | 40 | Or to run individual tests: 41 | 42 | ``` 43 | export ZOOKEEPER_PATH=//bin/zookeeper/ 44 | bin/pytest -v kazoo/tests/test_client.py::TestClient::test_create 45 | ``` 46 | 47 | The pytest test runner allows you to filter by test module, class or 48 | individual test method. 49 | 50 | If you made changes to the documentation, you can build it locally: 51 | 52 | ``` 53 | make html 54 | ``` 55 | 56 | And then open `./docs/_build/html/index.html` in a web browser to 57 | verify the correct rendering. 58 | 59 | 60 | ## Bug Reports 61 | 62 | You can file issues here on GitHub. Please try to include as much information as 63 | you can and under what conditions you saw the issue. 64 | 65 | ## Adding Recipes 66 | 67 | New recipes are welcome, however they should include the status/maintainer 68 | RST information so its clear who is maintaining the recipe. This means 69 | that if you submit a recipe for inclusion with Kazoo, you should be ready 70 | to support/maintain it, and address bugs that may be found. 71 | 72 | Ideally a recipe should have at least two maintainers. 73 | 74 | ## Sending Pull Requests 75 | 76 | Patches should be submitted as pull requests (PR). 77 | 78 | Before submitting a PR: 79 | - Your code must run and pass all the automated tests before you submit your PR 80 | for review. "Work in progress" pull requests are allowed to be submitted, but 81 | should be clearly labeled as such and should not be merged until all tests 82 | pass and the code has been reviewed. 83 | - Your patch should include new tests that cover your changes. It is your and 84 | your reviewer's responsibility to ensure your patch includes adequate tests. 85 | 86 | When submitting a PR: 87 | - You agree to license your code under the project's open source license 88 | ([APL 2.0](/LICENSE)). 89 | - Base your branch off the current `master`. 90 | - Add both your code and new tests if relevant. 91 | - Sign your git commit. 92 | - Run the test suite to make sure your code passes linting and tests. 93 | - Ensure your changes do not reduce code coverage of the test suite. 94 | - Please do not include merge commits in pull requests; include only commits 95 | with the new relevant code. 96 | 97 | 98 | ## Code Review 99 | 100 | This project is production Mozilla code and subject to our [engineering practices and quality standards](https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Committing_Rules_and_Responsibilities). Every patch must be peer reviewed. 101 | 102 | ## Git Commit Guidelines 103 | 104 | We loosely follow the [Angular commit guidelines](https://github.com/angular/angular.js/blob/master/CONTRIBUTING.md#type) 105 | of `(scope): ` where `type` must be one of: 106 | 107 | * **feat**: A new feature 108 | * **fix**: A bug fix 109 | * **docs**: Documentation only changes 110 | * **style**: Changes that do not affect the meaning of the code (white-space, formatting, missing 111 | semi-colons, etc) 112 | * **refactor**: A code change that neither fixes a bug or adds a feature 113 | * **perf**: A code change that improves performance 114 | * **test**: Adding missing tests 115 | * **chore**: Changes to the build process or auxiliary tools and libraries such as documentation 116 | generation 117 | 118 | Scope may be left off if none of these components are applicable: 119 | 120 | * **core**: Core client/connection handling 121 | * **recipe**: Changes/Fixes/Additions to recipes 122 | 123 | ### Subject 124 | 125 | The subject contains succinct description of the change: 126 | 127 | * use the imperative, present tense: "change" not "changed" nor "changes" 128 | * don't capitalize first letter 129 | * no dot (.) at the end 130 | 131 | ### Body 132 | 133 | In order to maintain a reference to the context of the commit, add 134 | `closes #` if it closes a related issue or `issue #` 135 | if it's a partial fix. 136 | 137 | You can also write a detailed description of the commit. Just as in the 138 | **subject**, use the imperative, present tense: "change" not "changed" nor 139 | "changes". Please include the motivation for the change and contrast this with 140 | previous behavior. 141 | 142 | ### Footer 143 | 144 | The footer should contain any information about **Breaking Changes** and is also 145 | the place to reference GitHub issues that this commit **Closes**. 146 | 147 | ### Example 148 | 149 | A properly formatted commit message should look like: 150 | 151 | ``` 152 | feat(core): add tasty cookies to the client handler 153 | 154 | Properly formatted commit messages provide understandable history and 155 | documentation. This patch will provide a delicious cookie when all tests have 156 | passed and the commit message is properly formatted. 157 | 158 | BREAKING CHANGE: This patch requires developer to lower expectations about 159 | what "delicious" and "cookie" may mean. Some sadness may result. 160 | 161 | Closes #3.14, #9.75 162 | ``` 163 | 164 | # Legal 165 | 166 | Currently we don't have any legal contributor agreement, so code 167 | ownership stays with the original authors. 168 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | global-exclude *pyc *pyo __pycache__ 2 | # Git files 3 | exclude .gitignore 4 | # CI/CD files 5 | exclude .travis.yml.bak 6 | exclude .clog.toml 7 | prune .github 8 | 9 | exclude Makefile 10 | exclude run_failure.py 11 | 12 | include CHANGES.md 13 | include CONTRIBUTING.md 14 | include README.md 15 | include LICENSE 16 | include MANIFEST.in 17 | 18 | include tox.ini 19 | 20 | recursive-include kazoo * 21 | recursive-include docs * 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | HERE = $(shell pwd) 2 | BIN = $(HERE)/bin 3 | PYTHON = $(BIN)/python 4 | INSTALL = $(BIN)/pip install 5 | TOX_VENV ?= py37 6 | BUILD_DIRS = bin build include lib lib64 man share 7 | 8 | PYTHON_EXE = $(shell [ -f $(PYTHON) ] && echo $(PYTHON) || echo python) 9 | PYPY = $(shell $(PYTHON_EXE) -c "import sys; print(getattr(sys, 'pypy_version_info', False) and 'yes' or 'no')") 10 | CI ?= false 11 | CI_PYTHON_VERSION ?= $(shell $(PYTHON_EXE) -c "import sys; print('.'.join([str(s) for s in sys.version_info][:2]))") 12 | 13 | GREENLET_SUPPORTED = yes 14 | ifeq ($(findstring 3.,$(CI_PYTHON_VERSION)), 3.) 15 | GREENLET_SUPPORTED = no 16 | VENV_CMD = $(PYTHON_EXE) -m venv . 17 | else 18 | VENV_CMD = $(PYTHON_EXE) -m virtualenv . 19 | endif 20 | ifeq ($(PYPY),yes) 21 | GREENLET_SUPPORTED = no 22 | endif 23 | 24 | .PHONY: all build clean test 25 | 26 | all: build 27 | 28 | $(PYTHON): 29 | $(VENV_CMD) 30 | 31 | build: $(PYTHON) 32 | ifeq ($(GREENLET_SUPPORTED),yes) 33 | $(INSTALL) -U -r requirements_eventlet.txt 34 | $(INSTALL) -U -r requirements_gevent.txt 35 | endif 36 | ifneq ($(CI), true) 37 | $(INSTALL) -U -r requirements_sphinx.txt 38 | endif 39 | $(INSTALL) -U -r requirements.txt 40 | $(PYTHON) setup.py develop 41 | $(INSTALL) kazoo[test] 42 | 43 | clean: 44 | rm -rf $(BUILD_DIRS) 45 | 46 | test: 47 | tox -e$(TOX_VENV) 48 | 49 | html: 50 | cd docs && \ 51 | make html 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Kazoo 2 | ===== 3 | 4 | [![Kazoo Awesome Testing](https://github.com/python-zk/kazoo/actions/workflows/testing.yml/badge.svg?branch=master)](https://github.com/python-zk/kazoo/actions/workflows/testing.yml?query=branch%3Amaster) 5 | [![Latest Version](https://img.shields.io/pypi/v/kazoo.svg)](https://pypi.org/project/kazoo/) 6 | [![Reviewed by Hound](https://img.shields.io/badge/Reviewed_by-Hound-8E64B0.svg)](https://houndci.com) 7 | 8 | `kazoo` implements a higher level API to [Apache 9 | Zookeeper](http://zookeeper.apache.org/) for Python clients. 10 | 11 | See [the full docs](http://kazoo.rtfd.org/) for more information. 12 | 13 | License 14 | ------- 15 | 16 | `kazoo` is offered under the Apache License 2.0. 17 | 18 | Authors 19 | ------- 20 | 21 | `kazoo` started under the [Nimbus 22 | Project](http://www.nimbusproject.org/) and through collaboration with 23 | the open-source community has been merged with code from 24 | [Mozilla](http://www.mozilla.org/) and the [Zope 25 | Corporation](http://zope.com/). It has since gathered an active 26 | community of over one hundred contributors. 27 | -------------------------------------------------------------------------------- /constraints.txt: -------------------------------------------------------------------------------- 1 | # Consistent testing environment. 2 | black==22.10.0 3 | coverage==6.3.2 4 | flake8==5.0.2 5 | objgraph==3.5.0 6 | pytest==6.2.5 7 | pytest-cov==3.0.0 8 | pytest-timeout==2.2.0 9 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = ../bin/sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help 18 | help: 19 | @echo "Please use \`make ' where is one of" 20 | @egrep '^\.PHONY: [a-zA-Z_-]+ .*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = "(: |##)"}; {printf "\033[36m%-30s\033[0m %s\n", $$2, $$3}' 21 | 22 | .PHONY: clean 23 | clean: 24 | -rm -rf $(BUILDDIR)/* 25 | 26 | .PHONY: html ## to make standalone HTML files 27 | html: 28 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 29 | @echo 30 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 31 | 32 | .PHONY: dirhtml ## to make HTML files named index.html in directories 33 | dirhtml: 34 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 35 | @echo 36 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 37 | 38 | .PHONY: singlehtml ## to make a single large HTML file 39 | singlehtml: 40 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 41 | @echo 42 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 43 | 44 | .PHONY: pickle ## to make pickle files 45 | pickle: 46 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 47 | @echo 48 | @echo "Build finished; now you can process the pickle files." 49 | 50 | .PHONY: json ## to make JSON files 51 | json: 52 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 53 | @echo 54 | @echo "Build finished; now you can process the JSON files." 55 | 56 | .PHONY: htmlhelp ## to make HTML files and a HTML help project 57 | htmlhelp: 58 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 59 | @echo 60 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 61 | ".hhp project file in $(BUILDDIR)/htmlhelp." 62 | 63 | .PHONY: qthelp ## to make HTML files and a qthelp project 64 | qthelp: 65 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 66 | @echo 67 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 68 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 69 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/kazoo.qhcp" 70 | @echo "To view the help file:" 71 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/kazoo.qhc" 72 | 73 | .PHONY: devhelp ## to make HTML files and a Devhelp project 74 | devhelp: 75 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 76 | @echo 77 | @echo "Build finished." 78 | @echo "To view the help file:" 79 | @echo "# mkdir -p $$HOME/.local/share/devhelp/kazoo" 80 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/kazoo" 81 | @echo "# devhelp" 82 | 83 | .PHONY: epub ## to make an epub 84 | epub: 85 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 86 | @echo 87 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 88 | 89 | .PHONY: latex ## to make LaTeX files, you can set PAPER=a4 or PAPER=letter 90 | latex: 91 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 92 | @echo 93 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 94 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 95 | "(use \`make latexpdf' here to do that automatically)." 96 | 97 | .PHONY: latexpdf ## to make LaTeX files and run them through pdflatex 98 | latexpdf: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo "Running LaTeX files through pdflatex..." 101 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 102 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 103 | 104 | .PHONY: text ## to make text files 105 | text: 106 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 107 | @echo 108 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 109 | 110 | .PHONY: man ## to make manual pages 111 | man: 112 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 113 | @echo 114 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 115 | 116 | .PHONY: texinfo ## to make Texinfo files 117 | texinfo: 118 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 119 | @echo 120 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 121 | @echo "Run \`make' in that directory to run these through makeinfo" \ 122 | "(use \`make info' here to do that automatically)." 123 | 124 | .PHONY: info ## to make Texinfo files and run them through makeinfo 125 | info: 126 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 127 | @echo "Running Texinfo files through makeinfo..." 128 | make -C $(BUILDDIR)/texinfo info 129 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 130 | 131 | .PHONY: gettext ## to make PO message catalogs 132 | gettext: 133 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 134 | @echo 135 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 136 | 137 | .PHONY: changes ## to make an overview of all changed/added/deprecated items 138 | changes: 139 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 140 | @echo 141 | @echo "The overview file is in $(BUILDDIR)/changes." 142 | 143 | .PHONY: linkcheck ## to check all external links for integrity 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | .PHONY: doctest ## to run all doctests embedded in the documentation (if enabled) 151 | doctest: 152 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 153 | @echo "Testing of doctests in the sources finished, look at the " \ 154 | "results in $(BUILDDIR)/doctest/output.txt." 155 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | API Documentation 2 | ================= 3 | 4 | Comprehensive reference material for every public API exposed by 5 | `kazoo` is available within this chapter. The API documentation is 6 | organized alphabetically by module name. 7 | 8 | .. toctree:: 9 | :maxdepth: 1 10 | 11 | api/client 12 | api/exceptions 13 | api/handlers/gevent 14 | api/handlers/threading 15 | api/handlers/utils 16 | api/interfaces 17 | api/protocol/states 18 | api/recipe/barrier 19 | api/recipe/cache 20 | api/recipe/counter 21 | api/recipe/election 22 | api/recipe/lease 23 | api/recipe/lock 24 | api/recipe/partitioner 25 | api/recipe/party 26 | api/recipe/queue 27 | api/recipe/watchers 28 | api/retry 29 | api/security 30 | api/testing 31 | -------------------------------------------------------------------------------- /docs/api/client.rst: -------------------------------------------------------------------------------- 1 | .. _client_module: 2 | 3 | :mod:`kazoo.client` 4 | ---------------------------- 5 | 6 | .. automodule:: kazoo.client 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: KazooClient() 12 | :members: 13 | :member-order: bysource 14 | 15 | .. automethod:: __init__ 16 | 17 | .. attribute:: handler 18 | 19 | The :class:`~kazoo.interfaces.IHandler` strategy used by this 20 | client. Gives access to appropriate synchronization objects. 21 | 22 | .. method:: retry(func, *args, **kwargs) 23 | 24 | Runs the given function with the provided arguments, retrying if it 25 | fails because the ZooKeeper connection is lost, 26 | see :ref:`retrying_commands`. 27 | 28 | .. attribute:: state 29 | 30 | A :class:`~kazoo.protocol.states.KazooState` attribute indicating 31 | the current higher-level connection state. 32 | 33 | .. note:: 34 | 35 | Up to version 2.6.1, requests could only be submitted 36 | in the CONNECTED state. Requests submitted while 37 | SUSPENDED would immediately raise a 38 | :exc:`~kazoo.exceptions.SessionExpiredError`. This 39 | was problematic, as sessions are usually recovered on 40 | reconnect. 41 | 42 | Kazoo now simply queues requests submitted in the 43 | SUSPENDED state, expecting a recovery. This matches 44 | the behavior of the Java and C clients. 45 | 46 | Requests submitted in a LOST state still fail 47 | immediately with the corresponding exception. 48 | 49 | See: 50 | 51 | * https://github.com/python-zk/kazoo/issues/374 and 52 | * https://github.com/python-zk/kazoo/pull/570 53 | 54 | .. autoclass:: TransactionRequest 55 | :members: 56 | :member-order: bysource 57 | -------------------------------------------------------------------------------- /docs/api/exceptions.rst: -------------------------------------------------------------------------------- 1 | .. _exceptions_module: 2 | 3 | :mod:`kazoo.exceptions` 4 | ----------------------- 5 | 6 | .. automodule:: kazoo.exceptions 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoexception:: KazooException 12 | 13 | .. autoexception:: ZookeeperError 14 | 15 | .. autoexception:: AuthFailedError 16 | 17 | .. autoexception:: BadVersionError 18 | 19 | .. autoexception:: ConfigurationError 20 | 21 | .. autoexception:: InvalidACLError 22 | 23 | .. autoexception:: LockTimeout 24 | 25 | .. autoexception:: NoChildrenForEphemeralsError 26 | 27 | .. autoexception:: NodeExistsError 28 | 29 | .. autoexception:: NoNodeError 30 | 31 | .. autoexception:: NotEmptyError 32 | 33 | Private API 34 | +++++++++++ 35 | 36 | .. autoexception:: APIError 37 | 38 | .. autoexception:: BadArgumentsError 39 | 40 | .. autoexception:: CancelledError 41 | 42 | .. autoexception:: ConnectionDropped 43 | 44 | .. autoexception:: ConnectionClosedError 45 | 46 | .. autoexception:: ConnectionLoss 47 | 48 | .. autoexception:: DataInconsistency 49 | 50 | .. autoexception:: MarshallingError 51 | 52 | .. autoexception:: NoAuthError 53 | 54 | .. autoexception:: NotReadOnlyCallError 55 | 56 | .. autoexception:: InvalidCallbackError 57 | 58 | .. autoexception:: OperationTimeoutError 59 | 60 | .. autoexception:: RolledBackError 61 | 62 | .. autoexception:: RuntimeInconsistency 63 | 64 | .. autoexception:: SessionExpiredError 65 | 66 | .. autoexception:: SessionMovedError 67 | 68 | .. autoexception:: SystemZookeeperError 69 | 70 | .. autoexception:: UnimplementedError 71 | 72 | .. autoexception:: WriterNotClosedException 73 | 74 | .. autoexception:: ZookeeperStoppedError 75 | -------------------------------------------------------------------------------- /docs/api/handlers/eventlet.rst: -------------------------------------------------------------------------------- 1 | .. _eventlet_handler_module: 2 | 3 | :mod:`kazoo.handlers.eventlet` 4 | ---------------------------- 5 | 6 | .. automodule:: kazoo.handlers.eventlet 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: SequentialEventletHandler 12 | :members: 13 | 14 | Private API 15 | +++++++++++ 16 | 17 | .. autoclass:: AsyncResult 18 | :members: 19 | -------------------------------------------------------------------------------- /docs/api/handlers/gevent.rst: -------------------------------------------------------------------------------- 1 | .. _gevent_handler_module: 2 | 3 | :mod:`kazoo.handlers.gevent` 4 | ---------------------------- 5 | 6 | .. automodule:: kazoo.handlers.gevent 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: SequentialGeventHandler 12 | :members: 13 | 14 | Private API 15 | +++++++++++ 16 | 17 | .. autoclass:: AsyncResult 18 | :members: 19 | -------------------------------------------------------------------------------- /docs/api/handlers/threading.rst: -------------------------------------------------------------------------------- 1 | .. _thread_handler_module: 2 | 3 | :mod:`kazoo.handlers.threading` 4 | ------------------------------- 5 | 6 | .. automodule:: kazoo.handlers.threading 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: SequentialThreadingHandler 12 | :members: 13 | 14 | Private API 15 | +++++++++++ 16 | 17 | .. autoclass:: AsyncResult 18 | :members: 19 | 20 | .. autoexception:: TimeoutError 21 | -------------------------------------------------------------------------------- /docs/api/handlers/utils.rst: -------------------------------------------------------------------------------- 1 | .. _utils_module: 2 | 3 | :mod:`kazoo.handlers.utils` 4 | --------------------------- 5 | 6 | .. automodule:: kazoo.handlers.utils 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autofunction:: capture_exceptions 12 | .. autofunction:: wrap 13 | 14 | Private API 15 | +++++++++++ 16 | 17 | .. autofunction:: create_socket_pair 18 | .. autofunction:: create_tcp_socket 19 | -------------------------------------------------------------------------------- /docs/api/interfaces.rst: -------------------------------------------------------------------------------- 1 | .. _interfaces_module: 2 | 3 | :mod:`kazoo.interfaces` 4 | ---------------------------- 5 | 6 | .. automodule:: kazoo.interfaces 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | :class:`IHandler` implementations should be created by the developer to be 12 | passed into :class:`~kazoo.client.KazooClient` during instantiation for the 13 | preferred callback handling. 14 | 15 | If the developer needs to use objects implementing the :class:`IAsyncResult` 16 | interface, the :meth:`IHandler.async_result` method must be used instead of 17 | instantiating one directly. 18 | 19 | .. autoclass:: IHandler 20 | :members: 21 | 22 | Private API 23 | +++++++++++ 24 | 25 | The :class:`IAsyncResult` documents the proper implementation for providing 26 | a value that results from a Zookeeper completion callback. Since the 27 | :class:`~kazoo.client.KazooClient` returns an :class:`IAsyncResult` object 28 | instead of taking a completion callback for async functions, developers 29 | wishing to have their own callback called should use the 30 | :meth:`IAsyncResult.rawlink` method. 31 | 32 | .. autoclass:: IAsyncResult 33 | :members: 34 | -------------------------------------------------------------------------------- /docs/api/protocol/states.rst: -------------------------------------------------------------------------------- 1 | .. _states_module: 2 | 3 | :mod:`kazoo.protocol.states` 4 | ---------------------------- 5 | 6 | .. automodule:: kazoo.protocol.states 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: EventType 12 | 13 | .. autoclass:: KazooState 14 | 15 | .. autoclass:: KeeperState 16 | 17 | .. autoclass:: WatchedEvent 18 | 19 | .. autoclass:: ZnodeStat 20 | 21 | Private API 22 | +++++++++++ 23 | 24 | .. autoclass:: Callback 25 | -------------------------------------------------------------------------------- /docs/api/recipe/barrier.rst: -------------------------------------------------------------------------------- 1 | .. _barrier_module: 2 | 3 | :mod:`kazoo.recipe.barrier` 4 | ---------------------------- 5 | 6 | .. automodule:: kazoo.recipe.barrier 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: Barrier 12 | :members: 13 | 14 | .. automethod:: __init__ 15 | 16 | .. autoclass:: DoubleBarrier 17 | :members: 18 | 19 | .. automethod:: __init__ 20 | -------------------------------------------------------------------------------- /docs/api/recipe/cache.rst: -------------------------------------------------------------------------------- 1 | .. _cache_module: 2 | 3 | :mod:`kazoo.recipe.cache` 4 | ---------------------------- 5 | 6 | .. automodule:: kazoo.recipe.cache 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: TreeCache 12 | 13 | .. automethod:: start 14 | .. automethod:: close 15 | .. automethod:: listen 16 | .. automethod:: listen_fault 17 | .. automethod:: get_data 18 | .. automethod:: get_children 19 | 20 | .. autoclass:: TreeEvent 21 | :members: 22 | :show-inheritance: 23 | 24 | .. autoclass:: NodeData 25 | :members: 26 | :show-inheritance: 27 | -------------------------------------------------------------------------------- /docs/api/recipe/counter.rst: -------------------------------------------------------------------------------- 1 | .. _counter_module: 2 | 3 | :mod:`kazoo.recipe.counter` 4 | --------------------------- 5 | 6 | .. automodule:: kazoo.recipe.counter 7 | 8 | .. versionadded:: 0.7 9 | The Counter class. 10 | 11 | Public API 12 | ++++++++++ 13 | 14 | .. autoclass:: Counter 15 | :members: 16 | 17 | .. automethod:: __init__ 18 | .. automethod:: __add__ 19 | .. automethod:: __sub__ 20 | -------------------------------------------------------------------------------- /docs/api/recipe/election.rst: -------------------------------------------------------------------------------- 1 | .. _election_module: 2 | 3 | :mod:`kazoo.recipe.election` 4 | ---------------------------- 5 | 6 | .. automodule:: kazoo.recipe.election 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: Election 12 | :members: 13 | 14 | .. automethod:: __init__ 15 | -------------------------------------------------------------------------------- /docs/api/recipe/lease.rst: -------------------------------------------------------------------------------- 1 | .. _lease_module: 2 | 3 | :mod:`kazoo.recipe.lease` 4 | ---------------------------- 5 | 6 | .. automodule:: kazoo.recipe.lease 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: NonBlockingLease 12 | :members: 13 | 14 | .. automethod:: __init__ 15 | 16 | .. autoclass:: MultiNonBlockingLease 17 | :members: 18 | 19 | .. automethod:: __init__ 20 | -------------------------------------------------------------------------------- /docs/api/recipe/lock.rst: -------------------------------------------------------------------------------- 1 | .. _lock_module: 2 | 3 | :mod:`kazoo.recipe.lock` 4 | ---------------------------- 5 | 6 | .. automodule:: kazoo.recipe.lock 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: Lock 12 | :members: 13 | 14 | .. automethod:: __init__ 15 | 16 | .. autoclass:: ReadLock 17 | :members: 18 | :inherited-members: 19 | 20 | .. automethod:: __init__ 21 | 22 | .. autoclass:: WriteLock 23 | :members: 24 | :inherited-members: 25 | 26 | .. automethod:: __init__ 27 | 28 | .. autoclass:: Semaphore 29 | :members: 30 | 31 | .. automethod:: __init__ 32 | -------------------------------------------------------------------------------- /docs/api/recipe/partitioner.rst: -------------------------------------------------------------------------------- 1 | .. _partitioner_module: 2 | 3 | :mod:`kazoo.recipe.partitioner` 4 | ------------------------------- 5 | 6 | .. automodule:: kazoo.recipe.partitioner 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: SetPartitioner 12 | :members: 13 | 14 | .. automethod:: __init__ 15 | 16 | .. autoclass:: PartitionState 17 | -------------------------------------------------------------------------------- /docs/api/recipe/party.rst: -------------------------------------------------------------------------------- 1 | .. _party_module: 2 | 3 | :mod:`kazoo.recipe.party` 4 | ------------------------- 5 | 6 | .. automodule:: kazoo.recipe.party 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: Party 12 | :members: 13 | :inherited-members: 14 | 15 | .. automethod:: __init__ 16 | .. automethod:: __iter__ 17 | .. automethod:: __len__ 18 | 19 | .. autoclass:: ShallowParty 20 | :members: 21 | :inherited-members: 22 | 23 | .. automethod:: __init__ 24 | .. automethod:: __iter__ 25 | .. automethod:: __len__ 26 | -------------------------------------------------------------------------------- /docs/api/recipe/queue.rst: -------------------------------------------------------------------------------- 1 | .. _queue_module: 2 | 3 | :mod:`kazoo.recipe.queue` 4 | ------------------------- 5 | 6 | .. automodule:: kazoo.recipe.queue 7 | 8 | .. versionadded:: 0.6 9 | The Queue class. 10 | 11 | .. versionadded:: 1.0 12 | The LockingQueue class. 13 | 14 | Public API 15 | ++++++++++ 16 | 17 | .. autoclass:: Queue 18 | :members: 19 | :inherited-members: 20 | 21 | .. automethod:: __init__ 22 | .. automethod:: __len__ 23 | 24 | .. autoclass:: LockingQueue 25 | :members: 26 | :inherited-members: 27 | 28 | .. automethod:: __init__ 29 | .. automethod:: __len__ 30 | -------------------------------------------------------------------------------- /docs/api/recipe/watchers.rst: -------------------------------------------------------------------------------- 1 | .. _watchers_module: 2 | 3 | :mod:`kazoo.recipe.watchers` 4 | ---------------------------- 5 | 6 | .. automodule:: kazoo.recipe.watchers 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: DataWatch 12 | :members: 13 | 14 | .. automethod:: __init__ 15 | 16 | .. automethod:: __call__ 17 | 18 | 19 | .. autoclass:: ChildrenWatch 20 | :members: 21 | 22 | .. automethod:: __init__ 23 | 24 | .. automethod:: __call__ 25 | 26 | .. autoclass:: PatientChildrenWatch 27 | :members: 28 | 29 | .. automethod:: __init__ 30 | -------------------------------------------------------------------------------- /docs/api/retry.rst: -------------------------------------------------------------------------------- 1 | .. _retry_module: 2 | 3 | :mod:`kazoo.retry` 4 | ---------------------------- 5 | 6 | .. automodule:: kazoo.retry 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: KazooRetry 12 | :members: 13 | :member-order: bysource 14 | 15 | .. automethod:: __init__ 16 | 17 | .. automethod:: __call__ 18 | 19 | .. autoexception:: ForceRetryError 20 | 21 | .. autoexception:: RetryFailedError 22 | 23 | .. autoexception:: InterruptedError 24 | -------------------------------------------------------------------------------- /docs/api/security.rst: -------------------------------------------------------------------------------- 1 | .. _security_module: 2 | 3 | :mod:`kazoo.security` 4 | ---------------------------- 5 | 6 | .. automodule:: kazoo.security 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: ACL 12 | 13 | .. autoclass:: Id 14 | 15 | .. autofunction:: make_digest_acl 16 | 17 | Private API 18 | +++++++++++ 19 | 20 | .. autofunction:: make_acl 21 | 22 | .. autofunction:: make_digest_acl_credential 23 | 24 | .. autoclass: ACLPermission 25 | -------------------------------------------------------------------------------- /docs/api/testing.rst: -------------------------------------------------------------------------------- 1 | .. _testing_harness_module: 2 | 3 | :mod:`kazoo.testing.harness` 4 | ---------------------------- 5 | 6 | .. automodule:: kazoo.testing.harness 7 | 8 | Public API 9 | ++++++++++ 10 | 11 | .. autoclass:: KazooTestHarness 12 | .. autoclass:: KazooTestCase 13 | -------------------------------------------------------------------------------- /docs/async_usage.rst: -------------------------------------------------------------------------------- 1 | .. _async_usage: 2 | 3 | ================== 4 | Asynchronous Usage 5 | ================== 6 | 7 | The asynchronous Kazoo API relies on the 8 | :class:`~kazoo.interfaces.IAsyncResult` object which is returned by all the 9 | asynchronous methods. Callbacks can be added with the 10 | :meth:`~kazoo.interfaces.IAsyncResult.rawlink` method which works in a 11 | consistent manner whether threads or an asynchronous framework like gevent is 12 | used. 13 | 14 | Kazoo utilizes a pluggable :class:`~kazoo.interfaces.IHandler` interface which 15 | abstracts the callback system to ensure it works consistently. 16 | 17 | Connection Handling 18 | =================== 19 | 20 | Creating a connection: 21 | 22 | .. code-block:: python 23 | 24 | from kazoo.client import KazooClient 25 | from kazoo.handlers.gevent import SequentialGeventHandler 26 | 27 | zk = KazooClient(handler=SequentialGeventHandler()) 28 | 29 | # returns immediately 30 | event = zk.start_async() 31 | 32 | # Wait for 30 seconds and see if we're connected 33 | event.wait(timeout=30) 34 | 35 | if not zk.connected: 36 | # Not connected, stop trying to connect 37 | zk.stop() 38 | raise Exception("Unable to connect.") 39 | 40 | In this example, the `wait` method is used on the event object returned by the 41 | :meth:`~kazoo.client.KazooClient.start_async` method. A timeout is **always** 42 | used because its possible that we might never connect and that should be 43 | handled gracefully. 44 | 45 | The :class:`~kazoo.handlers.gevent.SequentialGeventHandler` is used when you 46 | want to use gevent (and 47 | :class:`~kazoo.handlers.eventlet.SequentialEventletHandler` when eventlet is 48 | used). Kazoo doesn't rely on gevents/eventlet monkey patching and requires 49 | that you pass in the appropriate handler, the default handler is 50 | :class:`~kazoo.handlers.threading.SequentialThreadingHandler`. 51 | 52 | Asynchronous Callbacks 53 | ====================== 54 | 55 | All kazoo `_async` methods except for 56 | :meth:`~kazoo.client.KazooClient.start_async` return an 57 | :class:`~kazoo.interfaces.IAsyncResult` instance. These instances allow 58 | you to see when a result is ready, or chain one or more callback 59 | functions to the result that will be called when it's ready. 60 | 61 | The callback function will be passed the 62 | :class:`~kazoo.interfaces.IAsyncResult` instance and should call the 63 | :meth:`~kazoo.interfaces.IAsyncResult.get` method on it to retrieve 64 | the value. This call could result in an exception being raised 65 | if the asynchronous function encountered an error. It should be caught 66 | and handled appropriately. 67 | 68 | Example: 69 | 70 | .. code-block:: python 71 | 72 | import sys 73 | 74 | from kazoo.exceptions import ConnectionLossException 75 | from kazoo.exceptions import NoAuthException 76 | 77 | def my_callback(async_obj): 78 | try: 79 | children = async_obj.get() 80 | do_something(children) 81 | except (ConnectionLossException, NoAuthException): 82 | sys.exit(1) 83 | 84 | # Both these statements return immediately, the second sets a callback 85 | # that will be run when get_children_async has its return value 86 | async_obj = zk.get_children_async("/some/node") 87 | async_obj.rawlink(my_callback) 88 | 89 | Zookeeper CRUD 90 | ============== 91 | 92 | The following CRUD methods all work the same as their synchronous counterparts 93 | except that they return an :class:`~kazoo.interfaces.IAsyncResult` object. 94 | 95 | Creating Method: 96 | 97 | * :meth:`~kazoo.client.KazooClient.create_async` 98 | 99 | Reading Methods: 100 | 101 | * :meth:`~kazoo.client.KazooClient.exists_async` 102 | * :meth:`~kazoo.client.KazooClient.get_async` 103 | * :meth:`~kazoo.client.KazooClient.get_children_async` 104 | 105 | Updating Methods: 106 | 107 | * :meth:`~kazoo.client.KazooClient.set_async` 108 | 109 | Deleting Methods: 110 | 111 | * :meth:`~kazoo.client.KazooClient.delete_async` 112 | 113 | The :meth:`~kazoo.client.KazooClient.ensure_path` has no asynchronous 114 | counterpart at the moment nor can the 115 | :meth:`~kazoo.client.KazooClient.delete_async` method do recursive deletes. 116 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGES.rst 2 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /docs/glossary.rst: -------------------------------------------------------------------------------- 1 | .. _glossary: 2 | 3 | Glossary 4 | ======== 5 | 6 | 7 | .. glossary:: 8 | 9 | Zookeeper 10 | `Apache Zookeeper `_ is a centralized 11 | service for maintaining configuration information, naming, providing 12 | distributed synchronization, and providing group services. 13 | -------------------------------------------------------------------------------- /docs/implementation.rst: -------------------------------------------------------------------------------- 1 | .. _implementation_details: 2 | 3 | ====================== 4 | Implementation Details 5 | ====================== 6 | 7 | Up to version 0.3 kazoo used the Python bindings to the Zookeeper C library. 8 | Unfortunately those bindings are fairly buggy and required a fair share of 9 | weird workarounds to interface with the native OS thread used in those 10 | bindings. 11 | 12 | Starting with version 0.4 kazoo implements the entire Zookeeper wire protocol 13 | itself in pure Python. Doing so removed the need for the workarounds and made 14 | it much easier to implement the features missing in the C bindings. 15 | 16 | Handlers 17 | ======== 18 | 19 | Both the Kazoo handlers run 3 separate queues to help alleviate deadlock issues 20 | and ensure consistent execution order regardless of environment. The 21 | :class:`~kazoo.handlers.gevent.SequentialGeventHandler` runs a separate 22 | greenlet for each queue that processes the callbacks queued in order. The 23 | :class:`~kazoo.handlers.threading.SequentialThreadingHandler` runs a separate 24 | thread for each queue that processes the callbacks queued in order (thus the 25 | naming scheme which notes they are sequential in anticipation that there could 26 | be handlers shipped in the future which don't make this guarantee). 27 | 28 | Callbacks are queued by type, the 3 types being: 29 | 30 | 1. Session events (State changes, registered listener functions) 31 | 2. Watch events (Watch callbacks, DataWatch, and ChildrenWatch functions) 32 | 3. Completion callbacks (Functions chained to 33 | :class:`~kazoo.interfaces.IAsyncResult` objects) 34 | 35 | This ensures that calls can be made to Zookeeper from any callback **except for 36 | a state listener** without worrying that critical session events will be 37 | blocked. 38 | 39 | .. warning:: 40 | 41 | Its important to remember that if you write code that blocks in one of 42 | these functions then no queued functions of that type will be executed 43 | until the code stops blocking. If your code might block, it should run 44 | itself in a separate greenlet/thread so that the other callbacks can 45 | run. 46 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ===== 2 | kazoo 3 | ===== 4 | 5 | Kazoo is a Python library designed to make working with :term:`Zookeeper` a 6 | more hassle-free experience that is less prone to errors. 7 | 8 | Kazoo features: 9 | 10 | * A wide range of recipe implementations, like Lock, Election or Queue 11 | * Data and Children Watchers 12 | * Simplified Zookeeper connection state tracking 13 | * Unified asynchronous API for use with greenlets or threads 14 | * Support for `gevent`_ >= 1.2 15 | * Support for `eventlet`_ 16 | * Support for Zookeeper 3.3, 3.4, and 3.5 servers 17 | * Integrated testing helpers for Zookeeper clusters 18 | * Pure-Python based implementation of the wire protocol, avoiding all the 19 | memory leaks, lacking features, and debugging madness of the C library 20 | 21 | Kazoo is heavily inspired by `Netflix Curator`_ simplifications and helpers. 22 | 23 | .. note:: 24 | 25 | You should be familiar with Zookeeper and have read the `Zookeeper 26 | Programmers Guide`_ before using `kazoo`. 27 | 28 | Reference Docs 29 | ============== 30 | 31 | .. toctree:: 32 | :maxdepth: 1 33 | 34 | install 35 | basic_usage 36 | async_usage 37 | implementation 38 | testing 39 | api 40 | Changelog 41 | Contributing 42 | 43 | Why 44 | === 45 | 46 | Using :term:`Zookeeper` in a safe manner can be difficult due to the variety of 47 | edge-cases in :term:`Zookeeper` and other bugs that have been present in the 48 | Python C binding. Due to how the C library utilizes a separate C thread for 49 | :term:`Zookeeper` communication some libraries like `gevent`_ (or `eventlet`_) 50 | also don't work properly by default. 51 | 52 | By utilizing a pure Python implementation, Kazoo handles all of these 53 | cases and provides a new asynchronous API which is consistent when 54 | using threads or `gevent`_ (or `eventlet`_) greenlets. 55 | 56 | Source Code 57 | =========== 58 | 59 | All source code is available on `github under kazoo `_. 61 | 62 | Bugs/Support 63 | ============ 64 | 65 | Bugs should be reported on the `kazoo github issue tracker 66 | `_. 67 | 68 | The developers of ``kazoo`` can frequently be found on the Freenode IRC 69 | network in the `\#zookeeper`_ channel. 70 | 71 | For general discussions and support questions, please use the 72 | `python-zk `_ mailing list 73 | hosted on Google Groups. 74 | 75 | Indices and tables 76 | ================== 77 | 78 | * :ref:`genindex` 79 | * :ref:`modindex` 80 | * :ref:`glossary` 81 | 82 | .. toctree:: 83 | :hidden: 84 | 85 | glossary 86 | 87 | License 88 | ======= 89 | 90 | ``kazoo`` is offered under the Apache License 2.0. 91 | 92 | Authors 93 | ======= 94 | 95 | ``kazoo`` started under the `Nimbus Project`_ and through collaboration with 96 | the open-source community has been merged with code from `Mozilla`_ and the 97 | `Zope Corporation`_. It has since gathered an active community of over two 98 | dozen contributors from a variety of companies (twitter, mozilla, yahoo! and 99 | others). 100 | 101 | .. _Apache Zookeeper: http://zookeeper.apache.org/ 102 | .. _Zookeeper Programmers Guide: https://zookeeper.apache.org/doc/current/zookeeperProgrammers.html 103 | .. _Zookeeper Recipes: http://zookeeper.apache.org/doc/current/recipes.html#sc_recoverableSharedLocks 104 | .. _Nimbus Project: http://www.nimbusproject.org/ 105 | .. _Zope Corporation: http://zope.com/ 106 | .. _Mozilla: http://www.mozilla.org/ 107 | .. _Netflix Curator: https://github.com/Netflix/curator 108 | .. _gevent: http://gevent.org/ 109 | .. _eventlet: http://eventlet.net/ 110 | .. _\#zookeeper: irc://chat.freenode.net/zookeeper 111 | -------------------------------------------------------------------------------- /docs/install.rst: -------------------------------------------------------------------------------- 1 | .. _install: 2 | 3 | ========== 4 | Installing 5 | ========== 6 | 7 | kazoo can be installed via ``pip``: 8 | 9 | .. code-block:: bash 10 | 11 | $ pip install kazoo 12 | 13 | Kazoo implements the Zookeeper protocol in pure Python, so you don't need 14 | any Python Zookeeper C bindings installed. 15 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. linkcheck to check all external links for integrity 37 | echo. doctest to run all doctests embedded in the documentation if enabled 38 | goto end 39 | ) 40 | 41 | if "%1" == "clean" ( 42 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 43 | del /q /s %BUILDDIR%\* 44 | goto end 45 | ) 46 | 47 | if "%1" == "html" ( 48 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 49 | if errorlevel 1 exit /b 1 50 | echo. 51 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 52 | goto end 53 | ) 54 | 55 | if "%1" == "dirhtml" ( 56 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 57 | if errorlevel 1 exit /b 1 58 | echo. 59 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 60 | goto end 61 | ) 62 | 63 | if "%1" == "singlehtml" ( 64 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 68 | goto end 69 | ) 70 | 71 | if "%1" == "pickle" ( 72 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished; now you can process the pickle files. 76 | goto end 77 | ) 78 | 79 | if "%1" == "json" ( 80 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished; now you can process the JSON files. 84 | goto end 85 | ) 86 | 87 | if "%1" == "htmlhelp" ( 88 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can run HTML Help Workshop with the ^ 92 | .hhp project file in %BUILDDIR%/htmlhelp. 93 | goto end 94 | ) 95 | 96 | if "%1" == "qthelp" ( 97 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 98 | if errorlevel 1 exit /b 1 99 | echo. 100 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 101 | .qhcp project file in %BUILDDIR%/qthelp, like this: 102 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\kazoo.qhcp 103 | echo.To view the help file: 104 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\kazoo.ghc 105 | goto end 106 | ) 107 | 108 | if "%1" == "devhelp" ( 109 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 110 | if errorlevel 1 exit /b 1 111 | echo. 112 | echo.Build finished. 113 | goto end 114 | ) 115 | 116 | if "%1" == "epub" ( 117 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 118 | if errorlevel 1 exit /b 1 119 | echo. 120 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 121 | goto end 122 | ) 123 | 124 | if "%1" == "latex" ( 125 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 129 | goto end 130 | ) 131 | 132 | if "%1" == "text" ( 133 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The text files are in %BUILDDIR%/text. 137 | goto end 138 | ) 139 | 140 | if "%1" == "man" ( 141 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 145 | goto end 146 | ) 147 | 148 | if "%1" == "texinfo" ( 149 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 150 | if errorlevel 1 exit /b 1 151 | echo. 152 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 153 | goto end 154 | ) 155 | 156 | if "%1" == "gettext" ( 157 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 158 | if errorlevel 1 exit /b 1 159 | echo. 160 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 161 | goto end 162 | ) 163 | 164 | if "%1" == "changes" ( 165 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 166 | if errorlevel 1 exit /b 1 167 | echo. 168 | echo.The overview file is in %BUILDDIR%/changes. 169 | goto end 170 | ) 171 | 172 | if "%1" == "linkcheck" ( 173 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 174 | if errorlevel 1 exit /b 1 175 | echo. 176 | echo.Link check complete; look for any errors in the above output ^ 177 | or in %BUILDDIR%/linkcheck/output.txt. 178 | goto end 179 | ) 180 | 181 | if "%1" == "doctest" ( 182 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 183 | if errorlevel 1 exit /b 1 184 | echo. 185 | echo.Testing of doctests in the sources finished, look at the ^ 186 | results in %BUILDDIR%/doctest/output.txt. 187 | goto end 188 | ) 189 | 190 | :end 191 | -------------------------------------------------------------------------------- /docs/testing.rst: -------------------------------------------------------------------------------- 1 | .. _testing: 2 | 3 | ======= 4 | Testing 5 | ======= 6 | 7 | Kazoo has several test harnesses used internally for its own tests that are 8 | exposed as public API's for use in your own tests for common Zookeeper cluster 9 | management and session testing. They can be mixed in with your own `unittest` 10 | or `pytest` tests along with a `mock` object that allows you to force specific 11 | `KazooClient` commands to fail in various ways. 12 | 13 | The test harness needs to be able to find the Zookeeper Java libraries. You 14 | need to specify an environment variable called `ZOOKEEPER_PATH` and point it 15 | to their location, for example `/usr/share/java`. The directory should contain 16 | a `zookeeper-*.jar` and a `lib` directory containing at least a `log4j-*.jar`. 17 | 18 | If your Java setup is complex, you may also override our classpath mechanism 19 | completely by specifying an environment variable called `ZOOKEEPER_CLASSPATH`. 20 | If provided, it will be used unmodified as the Java classpath for Zookeeper. 21 | 22 | You can specify an optional `ZOOKEEPER_PORT_OFFSET` environment variable to 23 | influence the ports the cluster is using. By default the offset is 20000 and 24 | a cluster with three members will use ports 20000, 20010 and 20020. 25 | 26 | 27 | Kazoo Test Harness 28 | ================== 29 | 30 | The :class:`~kazoo.testing.harness.KazooTestHarness` can be used directly or 31 | mixed in with your test code. 32 | 33 | Example: 34 | 35 | .. code-block:: python 36 | 37 | from kazoo.testing import KazooTestHarness 38 | 39 | class MyTest(KazooTestHarness): 40 | def setUp(self): 41 | self.setup_zookeeper() 42 | 43 | def tearDown(self): 44 | self.teardown_zookeeper() 45 | 46 | def testmycode(self): 47 | self.client.ensure_path('/test/path') 48 | result = self.client.get('/test/path') 49 | ... 50 | 51 | 52 | Kazoo Test Case 53 | =============== 54 | 55 | The :class:`~kazoo.testing.harness.KazooTestCase` is complete test case that 56 | is equivalent to the mixin setup of 57 | :class:`~kazoo.testing.harness.KazooTestHarness`. An equivalent test to the 58 | one above: 59 | 60 | .. code-block:: python 61 | 62 | from kazoo.testing import KazooTestCase 63 | 64 | class MyTest(KazooTestCase): 65 | def testmycode(self): 66 | self.client.ensure_path('/test/path') 67 | result = self.client.get('/test/path') 68 | ... 69 | 70 | Zake 71 | ==== 72 | 73 | For those that do not need (or desire) to setup a Zookeeper cluster to test 74 | integration with kazoo there is also a library called 75 | `zake `_. Contributions to 76 | `Zake's github repository `_ are welcome. 77 | 78 | Zake can be used to provide a *mock client* to layers of your application that 79 | interact with kazoo (using the same client interface) during testing to allow 80 | for introspection of what was stored, which watchers are active (and more) 81 | after your test of your application code has finished. 82 | -------------------------------------------------------------------------------- /ensure-zookeeper-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | HERE=`pwd` 6 | ZOO_BASE_DIR="$HERE/zookeeper" 7 | ZOOKEEPER_VERSION=${ZOOKEEPER_VERSION:-3.4.14} 8 | ZOOKEEPER_PATH="$ZOO_BASE_DIR/$ZOOKEEPER_VERSION" 9 | ZOOKEEPER_PREFIX=${ZOOKEEPER_PREFIX} 10 | ZOOKEEPER_SUFFIX=${ZOOKEEPER_SUFFIX} 11 | ZOO_MIRROR_URL="https://archive.apache.org/dist" 12 | 13 | 14 | function download_zookeeper(){ 15 | mkdir -p $ZOO_BASE_DIR 16 | cd $ZOO_BASE_DIR 17 | ZOOKEEPER_DOWNLOAD_URL=${ZOO_MIRROR_URL}/zookeeper/zookeeper-${ZOOKEEPER_VERSION}/${ZOOKEEPER_PREFIX}zookeeper-${ZOOKEEPER_VERSION}${ZOOKEEPER_SUFFIX}.tar.gz 18 | echo "Will download ZK from ${ZOOKEEPER_DOWNLOAD_URL}" 19 | (curl --silent -L -C - $ZOOKEEPER_DOWNLOAD_URL | tar -zx) || (echo "Failed downloading ZK from ${ZOOKEEPER_DOWNLOAD_URL}" && exit 1) 20 | mv ${ZOOKEEPER_PREFIX}zookeeper-${ZOOKEEPER_VERSION}${ZOOKEEPER_SUFFIX} $ZOOKEEPER_VERSION 21 | chmod a+x $ZOOKEEPER_PATH/bin/zkServer.sh 22 | } 23 | 24 | if [ ! -d "$ZOOKEEPER_PATH" ]; then 25 | download_zookeeper 26 | echo "Downloaded zookeeper $ZOOKEEPER_VERSION to $ZOOKEEPER_PATH" 27 | else 28 | echo "Already downloaded zookeeper $ZOOKEEPER_VERSION to $ZOOKEEPER_PATH" 29 | fi 30 | 31 | # Used as install_path when starting ZK 32 | export ZOOKEEPER_PATH="${ZOOKEEPER_PATH}/${ZOOKEEPER_LIB}" 33 | cd $HERE 34 | 35 | # Yield execution to venv command 36 | 37 | exec $* 38 | -------------------------------------------------------------------------------- /init_krb5.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | KRB5KDC=$(which krb5kdc || true) 6 | KDB5_UTIL=$(which kdb5_util || true) 7 | KADMIN=$(which kadmin.local || true) 8 | 9 | if [ $# -lt 2 ]; then 10 | echo "Usage $0 TARGET_DIR CMD..." 11 | exit 1 12 | fi 13 | WRK_DIR=$1 14 | shift 15 | 16 | # Check installed packages 17 | if [ -z ${KRB5KDC:+x} ] || [ -z ${KDB5_UTIL:+x} ] || [ -z ${KADMIN:+x} ]; then 18 | echo "Missing Kerberos utilities, skipping environment setup." 19 | exec $@ 20 | fi 21 | 22 | if [ -e ${WRK_DIR} ]; then 23 | echo "Working directory kdc already exists!" 24 | exit 1 25 | fi 26 | 27 | WRK_DIR=$(readlink -f ${WRK_DIR}) 28 | KDC_DIR="${WRK_DIR}/krb5kdc" 29 | 30 | ############################################################################### 31 | # Cleanup handlers 32 | 33 | function kdclogs { 34 | echo "Kerberos environment logs:" 35 | tail -v -n50 ${KDC_DIR}/*.log 36 | } 37 | 38 | function killkdc { 39 | if [ -e ${KDC_DIR}/kdc.pid ]; then 40 | echo "Terminating KDC server listening on ${KDC_PORT}..." 41 | kill -TERM $(cat ${KDC_DIR}/kdc.pid) 42 | fi 43 | rm -vfr ${WRK_DIR} 44 | } 45 | trap killkdc EXIT 46 | trap kdclogs ERR 47 | 48 | ############################################################################### 49 | export KRB5_TEST_ENV=${WRK_DIR} 50 | export KRB5_CONFIG=${WRK_DIR}/krb5.conf 51 | 52 | KDC_PORT=$((${RANDOM}+1024)) 53 | mkdir -vp ${WRK_DIR} 54 | mkdir -vp ${KDC_DIR} 55 | 56 | cat <${WRK_DIR}/krb5.conf 57 | [logging] 58 | default = FILE:${KDC_DIR}/krb5libs.log 59 | kdc = FILE:${KDC_DIR}/krb5kdc.log 60 | admin_server = FILE:${KDC_DIR}/kadmind.log 61 | 62 | [libdefaults] 63 | dns_lookup_realm = false 64 | ticket_lifetime = 24h 65 | renew_lifetime = 7d 66 | forwardable = true 67 | rdns = false 68 | default_realm = KAZOOTEST.ORG 69 | default_tkt_enctypes=aes128-cts-hmac-sha1-96 70 | default_tgs_enctypes=aes128-cts-hmac-sha1-96 71 | #default_ccache_name = KEYRING:persistent:%{uid} 72 | 73 | [realms] 74 | KAZOOTEST.ORG = { 75 | database_name = ${KDC_DIR}/principal 76 | admin_keytab = FILE:${KDC_DIR}/kadm5.keytab 77 | key_stash_file = ${KDC_DIR}/stash 78 | kdc_listen = 127.0.0.1:${KDC_PORT} 79 | kdc_tcp_listen = 127.0.0.1:${KDC_PORT} 80 | kdc = 127.0.0.1:${KDC_PORT} 81 | kdc_ports = ${KDC_PORT} 82 | kdc_tcp_ports = "" 83 | default_domain = KAZOOTEST.ORG 84 | } 85 | 86 | [domain_realm] 87 | .kazootest.org = KAZOOTEST.ORG 88 | kazootest.org = KAZOOTEST.ORG 89 | EOF 90 | 91 | cat <i", old)[0] if old != b"" else self.default 96 | else: 97 | old = old.decode("ascii") if old != b"" else self.default 98 | version = stat.version 99 | data = self.default_type(old) 100 | return data, version 101 | 102 | @property 103 | def value(self): 104 | return self._value()[0] 105 | 106 | def _change(self, value): 107 | if not isinstance(value, self.default_type): 108 | raise TypeError("invalid type for value change") 109 | self.client.retry(self._inner_change, value) 110 | return self 111 | 112 | def _inner_change(self, value): 113 | self.pre_value, version = self._value() 114 | post_value = self.pre_value + value 115 | if self.support_curator: 116 | data = struct.pack(">i", post_value) 117 | else: 118 | data = repr(post_value).encode("ascii") 119 | try: 120 | self.client.set(self.path, data, version=version) 121 | except BadVersionError: # pragma: nocover 122 | self.post_value = None 123 | raise ForceRetryError() 124 | self.post_value = post_value 125 | 126 | def __add__(self, value): 127 | """Add value to counter.""" 128 | return self._change(value) 129 | 130 | def __sub__(self, value): 131 | """Subtract value from counter.""" 132 | return self._change(-value) 133 | -------------------------------------------------------------------------------- /kazoo/recipe/election.py: -------------------------------------------------------------------------------- 1 | """ZooKeeper Leader Elections 2 | 3 | :Maintainer: None 4 | :Status: Unknown 5 | 6 | """ 7 | from kazoo.exceptions import CancelledError 8 | 9 | 10 | class Election(object): 11 | """Kazoo Basic Leader Election 12 | 13 | Example usage with a :class:`~kazoo.client.KazooClient` instance:: 14 | 15 | zk = KazooClient() 16 | zk.start() 17 | election = zk.Election("/electionpath", "my-identifier") 18 | 19 | # blocks until the election is won, then calls 20 | # my_leader_function() 21 | election.run(my_leader_function) 22 | 23 | """ 24 | 25 | def __init__(self, client, path, identifier=None): 26 | """Create a Kazoo Leader Election 27 | 28 | :param client: A :class:`~kazoo.client.KazooClient` instance. 29 | :param path: The election path to use. 30 | :param identifier: Name to use for this lock contender. This 31 | can be useful for querying to see who the 32 | current lock contenders are. 33 | 34 | """ 35 | self.lock = client.Lock(path, identifier) 36 | 37 | def run(self, func, *args, **kwargs): 38 | """Contend for the leadership 39 | 40 | This call will block until either this contender is cancelled 41 | or this contender wins the election and the provided leadership 42 | function subsequently returns or fails. 43 | 44 | :param func: A function to be called if/when the election is 45 | won. 46 | :param args: Arguments to leadership function. 47 | :param kwargs: Keyword arguments to leadership function. 48 | 49 | """ 50 | if not callable(func): 51 | raise ValueError("leader function is not callable") 52 | 53 | try: 54 | with self.lock: 55 | func(*args, **kwargs) 56 | 57 | except CancelledError: 58 | pass 59 | 60 | def cancel(self): 61 | """Cancel participation in the election 62 | 63 | .. note:: 64 | 65 | If this contender has already been elected leader, this 66 | method will not interrupt the leadership function. 67 | 68 | """ 69 | self.lock.cancel() 70 | 71 | def contenders(self): 72 | """Return an ordered list of the current contenders in the 73 | election 74 | 75 | .. note:: 76 | 77 | If the contenders did not set an identifier, it will appear 78 | as a blank string. 79 | 80 | """ 81 | return self.lock.contenders() 82 | -------------------------------------------------------------------------------- /kazoo/recipe/lease.py: -------------------------------------------------------------------------------- 1 | """Zookeeper lease implementations 2 | 3 | :Maintainer: Lars Albertsson 4 | :Maintainer: Jyrki Pulliainen 5 | :Status: Beta 6 | 7 | """ 8 | import datetime 9 | import json 10 | import socket 11 | 12 | from kazoo.exceptions import CancelledError 13 | 14 | 15 | class NonBlockingLease(object): 16 | """Exclusive lease that does not block. 17 | 18 | An exclusive lease ensures that only one client at a time owns the lease. 19 | The client may renew the lease without losing it by obtaining a new lease 20 | with the same path and same identity. The lease object evaluates to True 21 | if the lease was obtained. 22 | 23 | A common use case is a situation where a task should only run on a single 24 | host. In this case, the clients that did not obtain the lease should exit 25 | without performing the protected task. 26 | 27 | The lease stores time stamps using client clocks, and will therefore only 28 | work if client clocks are roughly synchronised. It uses UTC, and works 29 | across time zones and daylight savings. 30 | 31 | Example usage: with a :class:`~kazoo.client.KazooClient` instance:: 32 | 33 | zk = KazooClient() 34 | zk.start() 35 | # Hold lease over an hour in order to keep job on same machine, 36 | # with failover if it dies. 37 | lease = zk.NonBlockingLease( 38 | "/db_leases/hourly_cleanup", datetime.timedelta(minutes = 70), 39 | identifier = "DB hourly cleanup on " + socket.gethostname()) 40 | if lease: 41 | do_hourly_database_cleanup() 42 | """ 43 | 44 | # Bump when storage format changes 45 | _version = 1 46 | _date_format = "%Y-%m-%dT%H:%M:%S" 47 | _byte_encoding = "utf-8" 48 | 49 | def __init__( 50 | self, 51 | client, 52 | path, 53 | duration, 54 | identifier=None, 55 | utcnow=datetime.datetime.utcnow, 56 | ): 57 | """Create a non-blocking lease. 58 | 59 | :param client: A :class:`~kazoo.client.KazooClient` instance. 60 | :param path: The lease path to use. 61 | :param duration: Duration during which the lease is reserved. A 62 | :class:`~datetime.timedelta` instance. 63 | :param identifier: Unique name to use for this lease holder. Reuse in 64 | order to renew the lease. Defaults to 65 | :meth:`socket.gethostname()`. 66 | :param utcnow: Clock function, by default returning 67 | :meth:`datetime.datetime.utcnow()`. Used for testing. 68 | 69 | """ 70 | ident = identifier or socket.gethostname() 71 | self.obtained = False 72 | self._attempt_obtaining(client, path, duration, ident, utcnow) 73 | 74 | def _attempt_obtaining(self, client, path, duration, ident, utcnow): 75 | client.ensure_path(path) 76 | holder_path = path + "/lease_holder" 77 | lock = client.Lock(path, ident) 78 | try: 79 | with lock: 80 | now = utcnow() 81 | if client.exists(holder_path): 82 | raw, _ = client.get(holder_path) 83 | data = self._decode(raw) 84 | if data["version"] != self._version: 85 | # We need an upgrade, let someone else take the lease 86 | return 87 | current_end = datetime.datetime.strptime( 88 | data["end"], self._date_format 89 | ) 90 | if data["holder"] != ident and now < current_end: 91 | # Another client is still holding the lease 92 | return 93 | client.delete(holder_path) 94 | end_lease = (now + duration).strftime(self._date_format) 95 | new_data = { 96 | "version": self._version, 97 | "holder": ident, 98 | "end": end_lease, 99 | } 100 | client.create(holder_path, self._encode(new_data)) 101 | self.obtained = True 102 | 103 | except CancelledError: 104 | pass 105 | 106 | def _encode(self, data_dict): 107 | return json.dumps(data_dict).encode(self._byte_encoding) 108 | 109 | def _decode(self, raw): 110 | return json.loads(raw.decode(self._byte_encoding)) 111 | 112 | # Python 2.x 113 | def __nonzero__(self): 114 | return self.obtained 115 | 116 | # Python 3.x 117 | def __bool__(self): 118 | return self.obtained 119 | 120 | 121 | class MultiNonBlockingLease(object): 122 | """Exclusive lease for multiple clients. 123 | 124 | This type of lease is useful when a limited set of hosts should run a 125 | particular task. It will attempt to obtain leases trying a sequence of 126 | ZooKeeper lease paths. 127 | 128 | :param client: A :class:`~kazoo.client.KazooClient` instance. 129 | :param count: Number of host leases allowed. 130 | :param path: ZooKeeper path under which lease files are stored. 131 | :param duration: Duration during which the lease is reserved. A 132 | :class:`~datetime.timedelta` instance. 133 | :param identifier: Unique name to use for this lease holder. Reuse in order 134 | to renew the lease. 135 | Defaults do :meth:`socket.gethostname()`. 136 | :param utcnow: Clock function, by default returning 137 | :meth:`datetime.datetime.utcnow()`. Used for testing. 138 | 139 | """ 140 | 141 | def __init__( 142 | self, 143 | client, 144 | count, 145 | path, 146 | duration, 147 | identifier=None, 148 | utcnow=datetime.datetime.utcnow, 149 | ): 150 | self.obtained = False 151 | for num in range(count): 152 | ls = NonBlockingLease( 153 | client, 154 | "%s/%d" % (path, num), 155 | duration, 156 | identifier=identifier, 157 | utcnow=utcnow, 158 | ) 159 | if ls: 160 | self.obtained = True 161 | break 162 | 163 | # Python 2.x 164 | def __nonzero__(self): 165 | return self.obtained 166 | 167 | # Python 3.x 168 | def __bool__(self): 169 | return self.obtained 170 | -------------------------------------------------------------------------------- /kazoo/recipe/party.py: -------------------------------------------------------------------------------- 1 | """Party 2 | 3 | :Maintainer: Ben Bangert 4 | :Status: Production 5 | 6 | A Zookeeper pool of party members. The :class:`Party` object can be 7 | used for determining members of a party. 8 | 9 | """ 10 | import uuid 11 | 12 | from kazoo.exceptions import NodeExistsError, NoNodeError 13 | 14 | 15 | class BaseParty(object): 16 | """Base implementation of a party.""" 17 | 18 | def __init__(self, client, path, identifier=None): 19 | """ 20 | :param client: A :class:`~kazoo.client.KazooClient` instance. 21 | :param path: The party path to use. 22 | :param identifier: An identifier to use for this member of the 23 | party when participating. 24 | 25 | """ 26 | self.client = client 27 | self.path = path 28 | self.data = str(identifier or "").encode("utf-8") 29 | self.ensured_path = False 30 | self.participating = False 31 | 32 | def _ensure_parent(self): 33 | if not self.ensured_path: 34 | # make sure our parent node exists 35 | self.client.ensure_path(self.path) 36 | self.ensured_path = True 37 | 38 | def join(self): 39 | """Join the party""" 40 | return self.client.retry(self._inner_join) 41 | 42 | def _inner_join(self): 43 | self._ensure_parent() 44 | try: 45 | self.client.create(self.create_path, self.data, ephemeral=True) 46 | self.participating = True 47 | except NodeExistsError: 48 | # node was already created, perhaps we are recovering from a 49 | # suspended connection 50 | self.participating = True 51 | 52 | def leave(self): 53 | """Leave the party""" 54 | self.participating = False 55 | return self.client.retry(self._inner_leave) 56 | 57 | def _inner_leave(self): 58 | try: 59 | self.client.delete(self.create_path) 60 | except NoNodeError: 61 | return False 62 | return True 63 | 64 | def __len__(self): 65 | """Return a count of participating clients""" 66 | self._ensure_parent() 67 | return len(self._get_children()) 68 | 69 | def _get_children(self): 70 | return self.client.retry(self.client.get_children, self.path) 71 | 72 | 73 | class Party(BaseParty): 74 | """Simple pool of participating processes""" 75 | 76 | _NODE_NAME = "__party__" 77 | 78 | def __init__(self, client, path, identifier=None): 79 | BaseParty.__init__(self, client, path, identifier=identifier) 80 | self.node = uuid.uuid4().hex + self._NODE_NAME 81 | self.create_path = self.path + "/" + self.node 82 | 83 | def __iter__(self): 84 | """Get a list of participating clients' data values""" 85 | self._ensure_parent() 86 | children = self._get_children() 87 | for child in children: 88 | try: 89 | d, _ = self.client.retry( 90 | self.client.get, self.path + "/" + child 91 | ) 92 | yield d.decode("utf-8") 93 | except NoNodeError: # pragma: nocover 94 | pass 95 | 96 | def _get_children(self): 97 | children = BaseParty._get_children(self) 98 | return [c for c in children if self._NODE_NAME in c] 99 | 100 | 101 | class ShallowParty(BaseParty): 102 | """Simple shallow pool of participating processes 103 | 104 | This differs from the :class:`Party` as the identifier is used in 105 | the name of the party node itself, rather than the data. This 106 | places some restrictions on the length as it must be a valid 107 | Zookeeper node (an alphanumeric string), but reduces the overhead 108 | of getting a list of participants to a single Zookeeper call. 109 | 110 | """ 111 | 112 | def __init__(self, client, path, identifier=None): 113 | BaseParty.__init__(self, client, path, identifier=identifier) 114 | self.node = "-".join([uuid.uuid4().hex, self.data.decode("utf-8")]) 115 | self.create_path = self.path + "/" + self.node 116 | 117 | def __iter__(self): 118 | """Get a list of participating clients' identifiers""" 119 | self._ensure_parent() 120 | children = self._get_children() 121 | for child in children: 122 | yield child[child.find("-") + 1 :] 123 | -------------------------------------------------------------------------------- /kazoo/retry.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import random 3 | import time 4 | 5 | from kazoo.exceptions import ( 6 | ConnectionClosedError, 7 | ConnectionLoss, 8 | KazooException, 9 | OperationTimeoutError, 10 | SessionExpiredError, 11 | ) 12 | 13 | 14 | log = logging.getLogger(__name__) 15 | 16 | 17 | class ForceRetryError(Exception): 18 | """Raised when some recipe logic wants to force a retry.""" 19 | 20 | 21 | class RetryFailedError(KazooException): 22 | """Raised when retrying an operation ultimately failed, after 23 | retrying the maximum number of attempts. 24 | """ 25 | 26 | 27 | class InterruptedError(RetryFailedError): 28 | """Raised when the retry is forcibly interrupted by the interrupt 29 | function""" 30 | 31 | 32 | class KazooRetry(object): 33 | """Helper for retrying a method in the face of retry-able 34 | exceptions""" 35 | 36 | RETRY_EXCEPTIONS = (ConnectionLoss, OperationTimeoutError, ForceRetryError) 37 | 38 | EXPIRED_EXCEPTIONS = (SessionExpiredError,) 39 | 40 | def __init__( 41 | self, 42 | max_tries=1, 43 | delay=0.1, 44 | backoff=2, 45 | max_jitter=0.4, 46 | max_delay=60.0, 47 | ignore_expire=True, 48 | sleep_func=time.sleep, 49 | deadline=None, 50 | interrupt=None, 51 | ): 52 | """Create a :class:`KazooRetry` instance for retrying function 53 | calls. 54 | 55 | :param max_tries: How many times to retry the command. -1 means 56 | infinite tries. 57 | :param delay: Initial delay between retry attempts. 58 | :param backoff: Backoff multiplier between retry attempts. 59 | Defaults to 2 for exponential backoff. 60 | :param max_jitter: Percentage of jitter to apply to each retry's delay 61 | to ensure all clients to do not hammer the server 62 | at the same time. Between 0.0 and 1.0. 63 | :param max_delay: Maximum delay in seconds, regardless of other 64 | backoff settings. Defaults to one minute. 65 | :param ignore_expire: 66 | Whether a session expiration should be ignored and treated 67 | as a retry-able command. 68 | :param interrupt: 69 | Function that will be called with no args that may return 70 | True if the retry should be ceased immediately. This will 71 | be called no more than every 0.1 seconds during a wait 72 | between retries. 73 | 74 | """ 75 | self.max_tries = max_tries 76 | self.delay = delay 77 | self.backoff = backoff 78 | # Ensure max_jitter is in (0, 1) 79 | self.max_jitter = max(min(max_jitter, 1.0), 0.0) 80 | self.max_delay = float(max_delay) 81 | self._attempts = 0 82 | self._cur_delay = delay 83 | self.deadline = deadline 84 | self._cur_stoptime = None 85 | self.sleep_func = sleep_func 86 | self.retry_exceptions = self.RETRY_EXCEPTIONS 87 | self.interrupt = interrupt 88 | if ignore_expire: 89 | self.retry_exceptions += self.EXPIRED_EXCEPTIONS 90 | 91 | def reset(self): 92 | """Reset the attempt counter""" 93 | self._attempts = 0 94 | self._cur_delay = self.delay 95 | self._cur_stoptime = None 96 | 97 | def copy(self): 98 | """Return a clone of this retry manager""" 99 | obj = KazooRetry( 100 | max_tries=self.max_tries, 101 | delay=self.delay, 102 | backoff=self.backoff, 103 | max_jitter=self.max_jitter, 104 | max_delay=self.max_delay, 105 | sleep_func=self.sleep_func, 106 | deadline=self.deadline, 107 | interrupt=self.interrupt, 108 | ) 109 | obj.retry_exceptions = self.retry_exceptions 110 | return obj 111 | 112 | def __call__(self, func, *args, **kwargs): 113 | """Call a function with arguments until it completes without 114 | throwing a Kazoo exception 115 | 116 | :param func: Function to call 117 | :param args: Positional arguments to call the function with 118 | :params kwargs: Keyword arguments to call the function with 119 | 120 | The function will be called until it doesn't throw one of the 121 | retryable exceptions (ConnectionLoss, OperationTimeout, or 122 | ForceRetryError), and optionally retrying on session 123 | expiration. 124 | 125 | """ 126 | self.reset() 127 | 128 | while True: 129 | try: 130 | if self.deadline is not None and self._cur_stoptime is None: 131 | self._cur_stoptime = time.monotonic() + self.deadline 132 | return func(*args, **kwargs) 133 | except ConnectionClosedError: 134 | raise 135 | except self.retry_exceptions: 136 | self._attempts += 1 137 | # Note: max_tries == -1 means infinite tries. 138 | if self._attempts == self.max_tries: 139 | raise RetryFailedError("Too many retry attempts") 140 | jitter = random.uniform( 141 | 1.0 - self.max_jitter, 1.0 + self.max_jitter 142 | ) 143 | sleeptime = self._cur_delay * jitter 144 | 145 | if ( 146 | self._cur_stoptime is not None 147 | and time.monotonic() + sleeptime >= self._cur_stoptime 148 | ): 149 | raise RetryFailedError("Exceeded retry deadline") 150 | 151 | if self.interrupt: 152 | remain_time = sleeptime 153 | while remain_time > 0: 154 | # Break the time period down and sleep for no 155 | # longer than 0.1 before calling the interrupt 156 | self.sleep_func(min(0.1, remain_time)) 157 | remain_time -= 0.1 158 | if self.interrupt(): 159 | raise InterruptedError() 160 | else: 161 | self.sleep_func(sleeptime) 162 | self._cur_delay = min(sleeptime * self.backoff, self.max_delay) 163 | -------------------------------------------------------------------------------- /kazoo/security.py: -------------------------------------------------------------------------------- 1 | """Kazoo Security""" 2 | from base64 import b64encode 3 | from collections import namedtuple 4 | import hashlib 5 | 6 | 7 | # Represents a Zookeeper ID and ACL object 8 | Id = namedtuple("Id", "scheme id") 9 | 10 | 11 | class ACL(namedtuple("ACL", "perms id")): 12 | """An ACL for a Zookeeper Node 13 | 14 | An ACL object is created by using an :class:`Id` object along with 15 | a :class:`Permissions` setting. For convenience, 16 | :meth:`make_digest_acl` should be used to create an ACL object with 17 | the desired scheme, id, and permissions. 18 | """ 19 | 20 | @property 21 | def acl_list(self): 22 | perms = [] 23 | if self.perms & Permissions.ALL == Permissions.ALL: 24 | perms.append("ALL") 25 | return perms 26 | if self.perms & Permissions.READ == Permissions.READ: 27 | perms.append("READ") 28 | if self.perms & Permissions.WRITE == Permissions.WRITE: 29 | perms.append("WRITE") 30 | if self.perms & Permissions.CREATE == Permissions.CREATE: 31 | perms.append("CREATE") 32 | if self.perms & Permissions.DELETE == Permissions.DELETE: 33 | perms.append("DELETE") 34 | if self.perms & Permissions.ADMIN == Permissions.ADMIN: 35 | perms.append("ADMIN") 36 | return perms 37 | 38 | def __repr__(self): 39 | return "ACL(perms=%r, acl_list=%s, id=%r)" % ( 40 | self.perms, 41 | self.acl_list, 42 | self.id, 43 | ) 44 | 45 | 46 | class Permissions(object): 47 | READ = 1 48 | WRITE = 2 49 | CREATE = 4 50 | DELETE = 8 51 | ADMIN = 16 52 | ALL = 31 53 | 54 | 55 | # Shortcuts for common Ids 56 | ANYONE_ID_UNSAFE = Id("world", "anyone") 57 | AUTH_IDS = Id("auth", "") 58 | 59 | # Shortcuts for common ACLs 60 | OPEN_ACL_UNSAFE = [ACL(Permissions.ALL, ANYONE_ID_UNSAFE)] 61 | CREATOR_ALL_ACL = [ACL(Permissions.ALL, AUTH_IDS)] 62 | READ_ACL_UNSAFE = [ACL(Permissions.READ, ANYONE_ID_UNSAFE)] 63 | 64 | 65 | def make_digest_acl_credential(username, password): 66 | """Create a SHA1 digest credential. 67 | 68 | .. note:: 69 | 70 | This function uses UTF-8 to encode non-ASCII codepoints, 71 | whereas ZooKeeper uses the "default locale" for decoding. It 72 | may be a good idea to start the JVM with `-Dfile.encoding=UTF-8` 73 | in non-UTF-8 locales. 74 | See: https://github.com/python-zk/kazoo/pull/584 75 | 76 | """ 77 | credential = username.encode("utf-8") + b":" + password.encode("utf-8") 78 | cred_hash = b64encode(hashlib.sha1(credential).digest()).strip() 79 | return username + ":" + cred_hash.decode("utf-8") 80 | 81 | 82 | def make_acl( 83 | scheme, 84 | credential, 85 | read=False, 86 | write=False, 87 | create=False, 88 | delete=False, 89 | admin=False, 90 | all=False, 91 | ): 92 | """Given a scheme and credential, return an :class:`ACL` object 93 | appropriate for use with Kazoo. 94 | 95 | :param scheme: The scheme to use. I.e. `digest`. 96 | :param credential: 97 | A colon separated username, password. The password should be 98 | hashed with the `scheme` specified. The 99 | :meth:`make_digest_acl_credential` method will create and 100 | return a credential appropriate for use with the `digest` 101 | scheme. 102 | :param write: Write permission. 103 | :type write: bool 104 | :param create: Create permission. 105 | :type create: bool 106 | :param delete: Delete permission. 107 | :type delete: bool 108 | :param admin: Admin permission. 109 | :type admin: bool 110 | :param all: All permissions. 111 | :type all: bool 112 | 113 | :rtype: :class:`ACL` 114 | 115 | """ 116 | if all: 117 | permissions = Permissions.ALL 118 | else: 119 | permissions = 0 120 | if read: 121 | permissions |= Permissions.READ 122 | if write: 123 | permissions |= Permissions.WRITE 124 | if create: 125 | permissions |= Permissions.CREATE 126 | if delete: 127 | permissions |= Permissions.DELETE 128 | if admin: 129 | permissions |= Permissions.ADMIN 130 | return ACL(permissions, Id(scheme, credential)) 131 | 132 | 133 | def make_digest_acl( 134 | username, 135 | password, 136 | read=False, 137 | write=False, 138 | create=False, 139 | delete=False, 140 | admin=False, 141 | all=False, 142 | ): 143 | """Create a digest ACL for Zookeeper with the given permissions 144 | 145 | This method combines :meth:`make_digest_acl_credential` and 146 | :meth:`make_acl` to create an :class:`ACL` object appropriate for 147 | use with Kazoo's ACL methods. 148 | 149 | :param username: Username to use for the ACL. 150 | :param password: A plain-text password to hash. 151 | :param write: Write permission. 152 | :type write: bool 153 | :param create: Create permission. 154 | :type create: bool 155 | :param delete: Delete permission. 156 | :type delete: bool 157 | :param admin: Admin permission. 158 | :type admin: bool 159 | :param all: All permissions. 160 | :type all: bool 161 | 162 | :rtype: :class:`ACL` 163 | 164 | """ 165 | cred = make_digest_acl_credential(username, password) 166 | return make_acl( 167 | "digest", 168 | cred, 169 | read=read, 170 | write=write, 171 | create=create, 172 | delete=delete, 173 | admin=admin, 174 | all=all, 175 | ) 176 | -------------------------------------------------------------------------------- /kazoo/testing/__init__.py: -------------------------------------------------------------------------------- 1 | from kazoo.testing.harness import KazooTestCase, KazooTestHarness 2 | 3 | 4 | __all__ = ( 5 | "KazooTestHarness", 6 | "KazooTestCase", 7 | ) 8 | -------------------------------------------------------------------------------- /kazoo/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/python-zk/kazoo/686d770f39f4ac59b1b40b565fe0dc72e69d511c/kazoo/tests/__init__.py -------------------------------------------------------------------------------- /kazoo/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | log = logging.getLogger(__name__) 4 | 5 | 6 | def pytest_exception_interact(node, call, report): 7 | try: 8 | cluster = node._testcase.cluster 9 | log.error("Zookeeper cluster logs:") 10 | for logs in cluster.get_logs(): 11 | log.error(logs) 12 | except Exception: 13 | log.exception("Cannot get ZK logs:") 14 | -------------------------------------------------------------------------------- /kazoo/tests/test_barrier.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | from kazoo.testing import KazooTestCase 4 | 5 | 6 | class KazooBarrierTests(KazooTestCase): 7 | def test_barrier_not_exist(self): 8 | b = self.client.Barrier("/some/path") 9 | assert b.wait() 10 | 11 | def test_barrier_exists(self): 12 | b = self.client.Barrier("/some/path") 13 | b.create() 14 | assert not b.wait(0) 15 | b.remove() 16 | assert b.wait() 17 | 18 | def test_remove_nonexistent_barrier(self): 19 | b = self.client.Barrier("/some/path") 20 | assert not b.remove() 21 | 22 | 23 | class KazooDoubleBarrierTests(KazooTestCase): 24 | def test_basic_barrier(self): 25 | b = self.client.DoubleBarrier("/some/path", 1) 26 | assert not b.participating 27 | b.enter() 28 | assert b.participating 29 | b.leave() 30 | assert not b.participating 31 | 32 | def test_two_barrier(self): 33 | av = threading.Event() 34 | ev = threading.Event() 35 | bv = threading.Event() 36 | release_all = threading.Event() 37 | b1 = self.client.DoubleBarrier("/some/path", 2) 38 | b2 = self.client.DoubleBarrier("/some/path", 2) 39 | 40 | def make_barrier_one(): 41 | b1.enter() 42 | ev.set() 43 | release_all.wait() 44 | b1.leave() 45 | ev.set() 46 | 47 | def make_barrier_two(): 48 | bv.wait() 49 | b2.enter() 50 | av.set() 51 | release_all.wait() 52 | b2.leave() 53 | av.set() 54 | 55 | # Spin up both of them 56 | t1 = threading.Thread(target=make_barrier_one) 57 | t1.start() 58 | t2 = threading.Thread(target=make_barrier_two) 59 | t2.start() 60 | 61 | assert not b1.participating 62 | assert not b2.participating 63 | 64 | bv.set() 65 | av.wait() 66 | ev.wait() 67 | assert b1.participating 68 | assert b2.participating 69 | 70 | av.clear() 71 | ev.clear() 72 | 73 | release_all.set() 74 | av.wait() 75 | ev.wait() 76 | assert not b1.participating 77 | assert not b2.participating 78 | t1.join() 79 | t2.join() 80 | 81 | def test_three_barrier(self): 82 | av = threading.Event() 83 | ev = threading.Event() 84 | bv = threading.Event() 85 | release_all = threading.Event() 86 | b1 = self.client.DoubleBarrier("/some/path", 3) 87 | b2 = self.client.DoubleBarrier("/some/path", 3) 88 | b3 = self.client.DoubleBarrier("/some/path", 3) 89 | 90 | def make_barrier_one(): 91 | b1.enter() 92 | ev.set() 93 | release_all.wait() 94 | b1.leave() 95 | ev.set() 96 | 97 | def make_barrier_two(): 98 | bv.wait() 99 | b2.enter() 100 | av.set() 101 | release_all.wait() 102 | b2.leave() 103 | av.set() 104 | 105 | # Spin up both of them 106 | t1 = threading.Thread(target=make_barrier_one) 107 | t1.start() 108 | t2 = threading.Thread(target=make_barrier_two) 109 | t2.start() 110 | 111 | assert not b1.participating 112 | assert not b2.participating 113 | 114 | bv.set() 115 | assert not b1.participating 116 | assert not b2.participating 117 | b3.enter() 118 | ev.wait() 119 | av.wait() 120 | 121 | assert b1.participating 122 | assert b2.participating 123 | assert b3.participating 124 | 125 | av.clear() 126 | ev.clear() 127 | 128 | release_all.set() 129 | b3.leave() 130 | av.wait() 131 | ev.wait() 132 | assert not b1.participating 133 | assert not b2.participating 134 | assert not b3.participating 135 | t1.join() 136 | t2.join() 137 | 138 | def test_barrier_existing_parent_node(self): 139 | b = self.client.DoubleBarrier("/some/path", 1) 140 | assert b.participating is False 141 | self.client.create("/some", ephemeral=True) 142 | # the barrier cannot create children under an ephemeral node 143 | b.enter() 144 | assert b.participating is False 145 | 146 | def test_barrier_existing_node(self): 147 | b = self.client.DoubleBarrier("/some", 1) 148 | assert b.participating is False 149 | self.client.ensure_path(b.path) 150 | self.client.create(b.create_path, ephemeral=True) 151 | # the barrier will re-use an existing node 152 | b.enter() 153 | assert b.participating is True 154 | b.leave() 155 | -------------------------------------------------------------------------------- /kazoo/tests/test_build.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | from kazoo.testing import KazooTestCase 6 | 7 | 8 | class TestBuildEnvironment(KazooTestCase): 9 | def setUp(self): 10 | KazooTestCase.setUp(self) 11 | if not os.environ.get("CI"): 12 | pytest.skip("Only run build config tests on CI.") 13 | 14 | def test_zookeeper_version(self): 15 | server_version = self.client.server_version() 16 | server_version = ".".join([str(i) for i in server_version]) 17 | env_version = os.environ.get("ZOOKEEPER_VERSION") 18 | if env_version: 19 | if "-" in env_version: 20 | # Ignore pre-release markers like -alpha 21 | env_version = env_version.split("-")[0] 22 | assert env_version == server_version 23 | -------------------------------------------------------------------------------- /kazoo/tests/test_counter.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | import pytest 4 | 5 | from kazoo.testing import KazooTestCase 6 | 7 | 8 | class KazooCounterTests(KazooTestCase): 9 | def _makeOne(self, **kw): 10 | path = "/" + uuid.uuid4().hex 11 | return self.client.Counter(path, **kw) 12 | 13 | def test_int_counter(self): 14 | counter = self._makeOne() 15 | assert counter.value == 0 16 | counter += 2 17 | counter + 1 18 | assert counter.value == 3 19 | counter -= 3 20 | counter - 1 21 | assert counter.value == -1 22 | 23 | def test_int_curator_counter(self): 24 | counter = self._makeOne(support_curator=True) 25 | assert counter.value == 0 26 | counter += 2 27 | counter + 1 28 | assert counter.value == 3 29 | counter -= 3 30 | counter - 1 31 | assert counter.value == -1 32 | counter += 1 33 | counter += 2147483647 34 | assert counter.value == 2147483647 35 | counter -= 2147483647 36 | counter -= 2147483647 37 | assert counter.value == -2147483647 38 | 39 | def test_float_counter(self): 40 | counter = self._makeOne(default=0.0) 41 | assert counter.value == 0.0 42 | counter += 2.1 43 | assert counter.value == 2.1 44 | counter -= 3.1 45 | assert counter.value == -1.0 46 | 47 | def test_errors(self): 48 | counter = self._makeOne() 49 | with pytest.raises(TypeError): 50 | counter.__add__(2.1) 51 | with pytest.raises(TypeError): 52 | counter.__add__(b"a") 53 | with pytest.raises(TypeError): 54 | counter = self._makeOne(default=0.0, support_curator=True) 55 | 56 | def test_pre_post_values(self): 57 | counter = self._makeOne() 58 | assert counter.value == 0 59 | assert counter.pre_value is None 60 | assert counter.post_value is None 61 | counter += 2 62 | assert counter.pre_value == 0 63 | assert counter.post_value == 2 64 | counter -= 3 65 | assert counter.pre_value == 2 66 | assert counter.post_value == -1 67 | -------------------------------------------------------------------------------- /kazoo/tests/test_election.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | import sys 3 | import threading 4 | 5 | import pytest 6 | 7 | from kazoo.testing import KazooTestCase 8 | from kazoo.tests.util import wait 9 | 10 | 11 | class UniqueError(Exception): 12 | """Error raised only by test leader function""" 13 | 14 | 15 | class KazooElectionTests(KazooTestCase): 16 | def setUp(self): 17 | super(KazooElectionTests, self).setUp() 18 | self.path = "/" + uuid.uuid4().hex 19 | 20 | self.condition = threading.Condition() 21 | 22 | # election contenders set these when elected. The exit event is set by 23 | # the test to make the leader exit. 24 | self.leader_id = None 25 | self.exit_event = None 26 | 27 | # tests set this before the event to make the leader raise an error 28 | self.raise_exception = False 29 | 30 | # set by a worker thread when an unexpected error is hit. 31 | # better way to do this? 32 | self.thread_exc_info = None 33 | 34 | def _spawn_contender(self, contender_id, election): 35 | thread = threading.Thread( 36 | target=self._election_thread, args=(contender_id, election) 37 | ) 38 | thread.daemon = True 39 | thread.start() 40 | return thread 41 | 42 | def _election_thread(self, contender_id, election): 43 | try: 44 | election.run(self._leader_func, contender_id) 45 | except UniqueError: 46 | if not self.raise_exception: 47 | self.thread_exc_info = sys.exc_info() 48 | except Exception: 49 | self.thread_exc_info = sys.exc_info() 50 | else: 51 | if self.raise_exception: 52 | e = Exception("expected leader func to raise exception") 53 | self.thread_exc_info = (Exception, e, None) 54 | 55 | def _leader_func(self, name): 56 | exit_event = threading.Event() 57 | with self.condition: 58 | self.exit_event = exit_event 59 | self.leader_id = name 60 | self.condition.notify_all() 61 | 62 | exit_event.wait(45) 63 | if self.raise_exception: 64 | raise UniqueError("expected error in the leader function") 65 | 66 | def _check_thread_error(self): 67 | if self.thread_exc_info: 68 | t, o, tb = self.thread_exc_info 69 | raise t(o) 70 | 71 | def test_election(self): 72 | elections = {} 73 | threads = {} 74 | for _ in range(3): 75 | contender = "c" + uuid.uuid4().hex 76 | elections[contender] = self.client.Election(self.path, contender) 77 | threads[contender] = self._spawn_contender( 78 | contender, elections[contender] 79 | ) 80 | 81 | # wait for a leader to be elected 82 | times = 0 83 | with self.condition: 84 | while not self.leader_id: 85 | self.condition.wait(5) 86 | times += 1 87 | if times > 5: 88 | raise Exception( 89 | "Still not a leader: lid: %s", self.leader_id 90 | ) 91 | 92 | election = self.client.Election(self.path) 93 | 94 | # make sure all contenders are in the pool 95 | wait(lambda: len(election.contenders()) == len(elections)) 96 | contenders = election.contenders() 97 | 98 | assert set(contenders) == set(elections.keys()) 99 | 100 | # first one in list should be leader 101 | first_leader = contenders[0] 102 | assert first_leader == self.leader_id 103 | 104 | # tell second one to cancel election. should never get elected. 105 | elections[contenders[1]].cancel() 106 | 107 | # make leader exit. third contender should be elected. 108 | self.exit_event.set() 109 | with self.condition: 110 | while self.leader_id == first_leader: 111 | self.condition.wait(45) 112 | assert self.leader_id == contenders[2] 113 | self._check_thread_error() 114 | 115 | # make first contender re-enter the race 116 | threads[first_leader].join() 117 | threads[first_leader] = self._spawn_contender( 118 | first_leader, elections[first_leader] 119 | ) 120 | 121 | # contender set should now be the current leader plus the first leader 122 | wait(lambda: len(election.contenders()) == 2) 123 | contenders = election.contenders() 124 | assert set(contenders), set([self.leader_id == first_leader]) 125 | 126 | # make current leader raise an exception. first should be reelected 127 | self.raise_exception = True 128 | self.exit_event.set() 129 | with self.condition: 130 | while self.leader_id != first_leader: 131 | self.condition.wait(45) 132 | assert self.leader_id == first_leader 133 | self._check_thread_error() 134 | 135 | self.exit_event.set() 136 | for thread in threads.values(): 137 | thread.join() 138 | self._check_thread_error() 139 | 140 | def test_bad_func(self): 141 | election = self.client.Election(self.path) 142 | with pytest.raises(ValueError): 143 | election.run("not a callable") 144 | -------------------------------------------------------------------------------- /kazoo/tests/test_eventlet_handler.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import unittest 3 | 4 | import pytest 5 | 6 | from kazoo.client import KazooClient 7 | from kazoo.handlers import utils 8 | from kazoo.protocol import states as kazoo_states 9 | from kazoo.tests import test_client 10 | from kazoo.tests import test_lock 11 | from kazoo.tests import util as test_util 12 | 13 | try: 14 | import eventlet 15 | from eventlet.green import threading 16 | from kazoo.handlers import eventlet as eventlet_handler 17 | 18 | EVENTLET_HANDLER_AVAILABLE = True 19 | except ImportError: 20 | EVENTLET_HANDLER_AVAILABLE = False 21 | 22 | 23 | @contextlib.contextmanager 24 | def start_stop_one(handler=None): 25 | if not handler: 26 | handler = eventlet_handler.SequentialEventletHandler() 27 | handler.start() 28 | try: 29 | yield handler 30 | finally: 31 | handler.stop() 32 | 33 | 34 | class TestEventletHandler(unittest.TestCase): 35 | def setUp(self): 36 | if not EVENTLET_HANDLER_AVAILABLE: 37 | pytest.skip("eventlet handler not available.") 38 | super(TestEventletHandler, self).setUp() 39 | 40 | def test_started(self): 41 | with start_stop_one() as handler: 42 | assert handler.running is True 43 | assert len(handler._workers) != 0 44 | assert handler.running is False 45 | assert len(handler._workers) == 0 46 | 47 | def test_spawn(self): 48 | captures = [] 49 | 50 | def cb(): 51 | captures.append(1) 52 | 53 | with start_stop_one() as handler: 54 | handler.spawn(cb) 55 | 56 | assert len(captures) == 1 57 | 58 | def test_dispatch(self): 59 | captures = [] 60 | 61 | def cb(): 62 | captures.append(1) 63 | 64 | with start_stop_one() as handler: 65 | handler.dispatch_callback(kazoo_states.Callback("watch", cb, [])) 66 | 67 | assert len(captures) == 1 68 | 69 | def test_async_link(self): 70 | captures = [] 71 | 72 | def cb(handler): 73 | captures.append(handler) 74 | 75 | with start_stop_one() as handler: 76 | r = handler.async_result() 77 | r.rawlink(cb) 78 | r.set(2) 79 | 80 | assert len(captures) == 1 81 | assert r.get() == 2 82 | 83 | def test_timeout_raising(self): 84 | handler = eventlet_handler.SequentialEventletHandler() 85 | 86 | with pytest.raises(handler.timeout_exception): 87 | raise handler.timeout_exception("This is a timeout") 88 | 89 | def test_async_ok(self): 90 | captures = [] 91 | 92 | def delayed(): 93 | captures.append(1) 94 | return 1 95 | 96 | def after_delayed(handler): 97 | captures.append(handler) 98 | 99 | with start_stop_one() as handler: 100 | r = handler.async_result() 101 | r.rawlink(after_delayed) 102 | w = handler.spawn(utils.wrap(r)(delayed)) 103 | w.join() 104 | 105 | assert len(captures) == 2 106 | assert captures[0] == 1 107 | assert r.get() == 1 108 | 109 | def test_get_with_no_block(self): 110 | handler = eventlet_handler.SequentialEventletHandler() 111 | 112 | with start_stop_one(handler): 113 | r = handler.async_result() 114 | 115 | with pytest.raises(handler.timeout_exception): 116 | r.get(block=False) 117 | r.set(1) 118 | assert r.get() == 1 119 | 120 | def test_async_exception(self): 121 | def broken(): 122 | raise IOError("Failed") 123 | 124 | with start_stop_one() as handler: 125 | r = handler.async_result() 126 | w = handler.spawn(utils.wrap(r)(broken)) 127 | w.join() 128 | 129 | assert r.successful() is False 130 | with pytest.raises(IOError): 131 | r.get() 132 | 133 | def test_huge_file_descriptor(self): 134 | try: 135 | import resource 136 | except ImportError: 137 | self.skipTest("resource module unavailable on this platform") 138 | from eventlet.green import socket 139 | from kazoo.handlers.utils import create_tcp_socket 140 | 141 | try: 142 | resource.setrlimit(resource.RLIMIT_NOFILE, (4096, 4096)) 143 | except (ValueError, resource.error): 144 | self.skipTest("couldn't raise fd limit high enough") 145 | fd = 0 146 | socks = [] 147 | while fd < 4000: 148 | sock = create_tcp_socket(socket) 149 | fd = sock.fileno() 150 | socks.append(sock) 151 | with start_stop_one() as h: 152 | h.start() 153 | h.select(socks, [], [], 0) 154 | h.stop() 155 | for sock in socks: 156 | sock.close() 157 | 158 | 159 | class TestEventletClient(test_client.TestClient): 160 | def setUp(self): 161 | if not EVENTLET_HANDLER_AVAILABLE: 162 | pytest.skip("eventlet handler not available.") 163 | super(TestEventletClient, self).setUp() 164 | 165 | @staticmethod 166 | def make_event(): 167 | return threading.Event() 168 | 169 | @staticmethod 170 | def make_condition(): 171 | return threading.Condition() 172 | 173 | def _makeOne(self, *args): 174 | return eventlet_handler.SequentialEventletHandler(*args) 175 | 176 | def _get_client(self, **kwargs): 177 | kwargs["handler"] = self._makeOne() 178 | return KazooClient(self.hosts, **kwargs) 179 | 180 | 181 | class TestEventletSemaphore(test_lock.TestSemaphore): 182 | def setUp(self): 183 | if not EVENTLET_HANDLER_AVAILABLE: 184 | pytest.skip("eventlet handler not available.") 185 | super(TestEventletSemaphore, self).setUp() 186 | 187 | @staticmethod 188 | def make_condition(): 189 | return threading.Condition() 190 | 191 | @staticmethod 192 | def make_event(): 193 | return threading.Event() 194 | 195 | @staticmethod 196 | def make_thread(*args, **kwargs): 197 | return threading.Thread(*args, **kwargs) 198 | 199 | def _makeOne(self, *args): 200 | return eventlet_handler.SequentialEventletHandler(*args) 201 | 202 | def _get_client(self, **kwargs): 203 | kwargs["handler"] = self._makeOne() 204 | c = KazooClient(self.hosts, **kwargs) 205 | try: 206 | self._clients.append(c) 207 | except AttributeError: 208 | self._client = [c] 209 | return c 210 | 211 | 212 | class TestEventletLock(test_lock.KazooLockTests): 213 | def setUp(self): 214 | if not EVENTLET_HANDLER_AVAILABLE: 215 | pytest.skip("eventlet handler not available.") 216 | super(TestEventletLock, self).setUp() 217 | 218 | @staticmethod 219 | def make_condition(): 220 | return threading.Condition() 221 | 222 | @staticmethod 223 | def make_event(): 224 | return threading.Event() 225 | 226 | @staticmethod 227 | def make_thread(*args, **kwargs): 228 | return threading.Thread(*args, **kwargs) 229 | 230 | @staticmethod 231 | def make_wait(): 232 | return test_util.Wait(getsleep=(lambda: eventlet.sleep)) 233 | 234 | def _makeOne(self, *args): 235 | return eventlet_handler.SequentialEventletHandler(*args) 236 | 237 | def _get_client(self, **kwargs): 238 | kwargs["handler"] = self._makeOne() 239 | c = KazooClient(self.hosts, **kwargs) 240 | try: 241 | self._clients.append(c) 242 | except AttributeError: 243 | self._client = [c] 244 | return c 245 | -------------------------------------------------------------------------------- /kazoo/tests/test_exceptions.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import pytest 4 | 5 | 6 | class ExceptionsTestCase(TestCase): 7 | def _get(self): 8 | from kazoo import exceptions 9 | 10 | return exceptions 11 | 12 | def test_backwards_alias(self): 13 | module = self._get() 14 | assert hasattr(module, "NoNodeException") 15 | assert module.NoNodeException is module.NoNodeError 16 | 17 | def test_exceptions_code(self): 18 | module = self._get() 19 | exc_8 = module.EXCEPTIONS[-8] 20 | assert isinstance(exc_8(), module.BadArgumentsError) 21 | 22 | def test_invalid_code(self): 23 | module = self._get() 24 | with pytest.raises(RuntimeError): 25 | module.EXCEPTIONS.__getitem__(666) 26 | 27 | def test_exceptions_construction(self): 28 | module = self._get() 29 | exc = module.EXCEPTIONS[-101]() 30 | assert type(exc) is module.NoNodeError 31 | assert exc.args == () 32 | -------------------------------------------------------------------------------- /kazoo/tests/test_gevent_handler.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sys 3 | 4 | import pytest 5 | 6 | from kazoo.client import KazooClient 7 | from kazoo.exceptions import NoNodeError 8 | from kazoo.protocol.states import Callback 9 | from kazoo.testing import KazooTestCase 10 | from kazoo.tests import test_client 11 | 12 | 13 | @pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") 14 | class TestGeventHandler(unittest.TestCase): 15 | def setUp(self): 16 | try: 17 | import gevent # NOQA 18 | except ImportError: 19 | pytest.skip("gevent not available.") 20 | 21 | def _makeOne(self, *args): 22 | from kazoo.handlers.gevent import SequentialGeventHandler 23 | 24 | return SequentialGeventHandler(*args) 25 | 26 | def _getAsync(self, *args): 27 | from kazoo.handlers.gevent import AsyncResult 28 | 29 | return AsyncResult 30 | 31 | def _getEvent(self): 32 | from gevent.event import Event 33 | 34 | return Event 35 | 36 | def test_proper_threading(self): 37 | h = self._makeOne() 38 | h.start() 39 | assert isinstance(h.event_object(), self._getEvent()) 40 | 41 | def test_matching_async(self): 42 | h = self._makeOne() 43 | h.start() 44 | async_handler = self._getAsync() 45 | assert isinstance(h.async_result(), async_handler) 46 | 47 | def test_exception_raising(self): 48 | h = self._makeOne() 49 | 50 | with pytest.raises(h.timeout_exception): 51 | raise h.timeout_exception("This is a timeout") 52 | 53 | def test_exception_in_queue(self): 54 | h = self._makeOne() 55 | h.start() 56 | ev = self._getEvent()() 57 | 58 | def func(): 59 | ev.set() 60 | raise ValueError("bang") 61 | 62 | call1 = Callback("completion", func, ()) 63 | h.dispatch_callback(call1) 64 | ev.wait() 65 | 66 | def test_queue_empty_exception(self): 67 | from gevent.queue import Empty 68 | 69 | h = self._makeOne() 70 | h.start() 71 | ev = self._getEvent()() 72 | 73 | def func(): 74 | ev.set() 75 | raise Empty() 76 | 77 | call1 = Callback("completion", func, ()) 78 | h.dispatch_callback(call1) 79 | ev.wait() 80 | 81 | 82 | @pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") 83 | class TestBasicGeventClient(KazooTestCase): 84 | def setUp(self): 85 | try: 86 | import gevent # NOQA 87 | except ImportError: 88 | pytest.skip("gevent not available.") 89 | KazooTestCase.setUp(self) 90 | 91 | def _makeOne(self, *args): 92 | from kazoo.handlers.gevent import SequentialGeventHandler 93 | 94 | return SequentialGeventHandler(*args) 95 | 96 | def _getEvent(self): 97 | from gevent.event import Event 98 | 99 | return Event 100 | 101 | def test_start(self): 102 | client = self._get_client(handler=self._makeOne()) 103 | client.start() 104 | assert client.state == "CONNECTED" 105 | client.stop() 106 | 107 | def test_start_stop_double(self): 108 | client = self._get_client(handler=self._makeOne()) 109 | client.start() 110 | assert client.state == "CONNECTED" 111 | client.handler.start() 112 | client.handler.stop() 113 | client.stop() 114 | 115 | def test_basic_commands(self): 116 | client = self._get_client(handler=self._makeOne()) 117 | client.start() 118 | assert client.state == "CONNECTED" 119 | client.create("/anode", b"fred") 120 | assert client.get("/anode")[0] == b"fred" 121 | assert client.delete("/anode") 122 | assert client.exists("/anode") is None 123 | client.stop() 124 | 125 | def test_failures(self): 126 | client = self._get_client(handler=self._makeOne()) 127 | client.start() 128 | with pytest.raises(NoNodeError): 129 | client.get("/none") 130 | client.stop() 131 | 132 | def test_data_watcher(self): 133 | client = self._get_client(handler=self._makeOne()) 134 | client.start() 135 | client.ensure_path("/some/node") 136 | ev = self._getEvent()() 137 | 138 | @client.DataWatch("/some/node") 139 | def changed(d, stat): 140 | ev.set() 141 | 142 | ev.wait() 143 | ev.clear() 144 | client.set("/some/node", b"newvalue") 145 | ev.wait() 146 | client.stop() 147 | 148 | def test_huge_file_descriptor(self): 149 | import resource 150 | from gevent import socket 151 | from kazoo.handlers.utils import create_tcp_socket 152 | 153 | try: 154 | resource.setrlimit(resource.RLIMIT_NOFILE, (4096, 4096)) 155 | except (ValueError, resource.error): 156 | self.skipTest("couldn't raise fd limit high enough") 157 | fd = 0 158 | socks = [] 159 | while fd < 4000: 160 | sock = create_tcp_socket(socket) 161 | fd = sock.fileno() 162 | socks.append(sock) 163 | h = self._makeOne() 164 | h.start() 165 | h.select(socks, [], [], 0) 166 | h.stop() 167 | for sock in socks: 168 | sock.close() 169 | 170 | 171 | @pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") 172 | class TestGeventClient(test_client.TestClient): 173 | def setUp(self): 174 | try: 175 | import gevent # NOQA 176 | except ImportError: 177 | pytest.skip("gevent not available.") 178 | KazooTestCase.setUp(self) 179 | 180 | def _makeOne(self, *args): 181 | from kazoo.handlers.gevent import SequentialGeventHandler 182 | 183 | return SequentialGeventHandler(*args) 184 | 185 | def _get_client(self, **kwargs): 186 | kwargs["handler"] = self._makeOne() 187 | return KazooClient(self.hosts, **kwargs) 188 | -------------------------------------------------------------------------------- /kazoo/tests/test_hosts.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from kazoo.hosts import collect_hosts 4 | 5 | 6 | class HostsTestCase(TestCase): 7 | def test_ipv4(self): 8 | hosts, chroot = collect_hosts( 9 | "127.0.0.1:2181, 192.168.1.2:2181, \ 10 | 132.254.111.10:2181" 11 | ) 12 | assert hosts == [ 13 | ("127.0.0.1", 2181), 14 | ("192.168.1.2", 2181), 15 | ("132.254.111.10", 2181), 16 | ] 17 | assert chroot is None 18 | 19 | hosts, chroot = collect_hosts( 20 | ["127.0.0.1:2181", "192.168.1.2:2181", "132.254.111.10:2181"] 21 | ) 22 | assert hosts == [ 23 | ("127.0.0.1", 2181), 24 | ("192.168.1.2", 2181), 25 | ("132.254.111.10", 2181), 26 | ] 27 | assert chroot is None 28 | 29 | def test_ipv6(self): 30 | hosts, chroot = collect_hosts("[fe80::200:5aee:feaa:20a2]:2181") 31 | assert hosts == [("fe80::200:5aee:feaa:20a2", 2181)] 32 | assert chroot is None 33 | 34 | hosts, chroot = collect_hosts(["[fe80::200:5aee:feaa:20a2]:2181"]) 35 | assert hosts == [("fe80::200:5aee:feaa:20a2", 2181)] 36 | assert chroot is None 37 | 38 | def test_hosts_list(self): 39 | hosts, chroot = collect_hosts("zk01:2181, zk02:2181, zk03:2181") 40 | expected1 = [("zk01", 2181), ("zk02", 2181), ("zk03", 2181)] 41 | assert hosts == expected1 42 | assert chroot is None 43 | 44 | hosts, chroot = collect_hosts(["zk01:2181", "zk02:2181", "zk03:2181"]) 45 | assert hosts == expected1 46 | assert chroot is None 47 | 48 | expected2 = "/test" 49 | hosts, chroot = collect_hosts("zk01:2181, zk02:2181, zk03:2181/test") 50 | assert hosts == expected1 51 | assert chroot == expected2 52 | 53 | hosts, chroot = collect_hosts( 54 | ["zk01:2181", "zk02:2181", "zk03:2181", "/test"] 55 | ) 56 | assert hosts == expected1 57 | assert chroot == expected2 58 | -------------------------------------------------------------------------------- /kazoo/tests/test_interrupt.py: -------------------------------------------------------------------------------- 1 | import os 2 | from sys import platform 3 | 4 | import pytest 5 | 6 | from kazoo.testing import KazooTestCase 7 | 8 | 9 | class KazooInterruptTests(KazooTestCase): 10 | def test_interrupted_systemcall(self): 11 | """ 12 | Make sure interrupted system calls don't break the world, since we 13 | can't control what all signals our connection thread will get 14 | """ 15 | if "linux" not in platform: 16 | pytest.skip( 17 | "Unable to reproduce error case on non-linux platforms" 18 | ) 19 | 20 | path = "interrupt_test" 21 | value = b"1" 22 | self.client.create(path, value) 23 | 24 | # set the euid to the current process' euid. 25 | # glibc sends SIGRT to all children, which will interrupt the 26 | # system call 27 | os.seteuid(os.geteuid()) 28 | 29 | # basic sanity test that it worked alright 30 | assert self.client.get(path)[0] == value 31 | -------------------------------------------------------------------------------- /kazoo/tests/test_party.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from kazoo.testing import KazooTestCase 4 | 5 | 6 | class KazooPartyTests(KazooTestCase): 7 | def setUp(self): 8 | super(KazooPartyTests, self).setUp() 9 | self.path = "/" + uuid.uuid4().hex 10 | 11 | def test_party(self): 12 | parties = [self.client.Party(self.path, "p%s" % i) for i in range(5)] 13 | 14 | one_party = parties[0] 15 | 16 | assert list(one_party) == [] 17 | assert len(one_party) == 0 18 | 19 | participants = set() 20 | for party in parties: 21 | party.join() 22 | participants.add(party.data.decode("utf-8")) 23 | 24 | assert set(party) == participants 25 | assert len(party) == len(participants) 26 | 27 | for party in parties: 28 | party.leave() 29 | participants.remove(party.data.decode("utf-8")) 30 | 31 | assert set(party) == participants 32 | assert len(party) == len(participants) 33 | 34 | def test_party_reuse_node(self): 35 | party = self.client.Party(self.path, "p1") 36 | self.client.ensure_path(self.path) 37 | self.client.create(party.create_path) 38 | party.join() 39 | assert party.participating is True 40 | party.leave() 41 | assert party.participating is False 42 | assert len(party) == 0 43 | 44 | def test_party_vanishing_node(self): 45 | party = self.client.Party(self.path, "p1") 46 | party.join() 47 | assert party.participating is True 48 | self.client.delete(party.create_path) 49 | party.leave() 50 | assert party.participating is False 51 | assert len(party) == 0 52 | 53 | 54 | class KazooShallowPartyTests(KazooTestCase): 55 | def setUp(self): 56 | super(KazooShallowPartyTests, self).setUp() 57 | self.path = "/" + uuid.uuid4().hex 58 | 59 | def test_party(self): 60 | parties = [ 61 | self.client.ShallowParty(self.path, "p%s" % i) for i in range(5) 62 | ] 63 | 64 | one_party = parties[0] 65 | 66 | assert list(one_party) == [] 67 | assert len(one_party) == 0 68 | 69 | participants = set() 70 | for party in parties: 71 | party.join() 72 | participants.add(party.data.decode("utf-8")) 73 | 74 | assert set(party) == participants 75 | assert len(party) == len(participants) 76 | 77 | for party in parties: 78 | party.leave() 79 | participants.remove(party.data.decode("utf-8")) 80 | 81 | assert set(party) == participants 82 | assert len(party) == len(participants) 83 | -------------------------------------------------------------------------------- /kazoo/tests/test_paths.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from unittest import TestCase 3 | 4 | import pytest 5 | 6 | from kazoo.protocol import paths 7 | 8 | 9 | if sys.version_info > (3,): # pragma: nocover 10 | 11 | def u(s): 12 | return s 13 | 14 | else: # pragma: nocover 15 | 16 | def u(s): 17 | return unicode(s, "unicode_escape") # noqa 18 | 19 | 20 | class NormPathTestCase(TestCase): 21 | def test_normpath(self): 22 | assert paths.normpath("/a/b") == "/a/b" 23 | 24 | def test_normpath_empty(self): 25 | assert paths.normpath("") == "" 26 | 27 | def test_normpath_unicode(self): 28 | assert paths.normpath(u("/\xe4/b")) == u("/\xe4/b") 29 | 30 | def test_normpath_dots(self): 31 | assert paths.normpath("/a./b../c") == "/a./b../c" 32 | 33 | def test_normpath_slash(self): 34 | assert paths.normpath("/") == "/" 35 | 36 | def test_normpath_multiple_slashes(self): 37 | assert paths.normpath("//") == "/" 38 | assert paths.normpath("//a/b") == "/a/b" 39 | assert paths.normpath("/a//b//") == "/a/b" 40 | assert paths.normpath("//a////b///c/") == "/a/b/c" 41 | 42 | def test_normpath_relative(self): 43 | with pytest.raises(ValueError): 44 | paths.normpath("./a/b") 45 | with pytest.raises(ValueError): 46 | paths.normpath("/a/../b") 47 | 48 | def test_normpath_trailing(self): 49 | assert paths.normpath("/", trailing=True) == "/" 50 | 51 | 52 | class JoinTestCase(TestCase): 53 | def test_join(self): 54 | assert paths.join("/a") == "/a" 55 | assert paths.join("/a", "b/") == "/a/b/" 56 | assert paths.join("/a", "b", "c") == "/a/b/c" 57 | 58 | def test_join_empty(self): 59 | assert paths.join("") == "" 60 | assert paths.join("", "a", "b") == "a/b" 61 | assert paths.join("/a", "", "b/", "c") == "/a/b/c" 62 | 63 | def test_join_absolute(self): 64 | assert paths.join("/a/b", "/c") == "/c" 65 | 66 | 67 | class IsAbsTestCase(TestCase): 68 | def test_isabs(self): 69 | assert paths.isabs("/") is True 70 | assert paths.isabs("/a") is True 71 | assert paths.isabs("/a//b/c") is True 72 | assert paths.isabs("//a/b") is True 73 | 74 | def test_isabs_false(self): 75 | assert paths.isabs("") is False 76 | assert paths.isabs("a/") is False 77 | assert paths.isabs("a/../") is False 78 | 79 | 80 | class BaseNameTestCase(TestCase): 81 | def test_basename(self): 82 | assert paths.basename("") == "" 83 | assert paths.basename("/") == "" 84 | assert paths.basename("//a") == "a" 85 | assert paths.basename("//a/") == "" 86 | assert paths.basename("/a/b.//c..") == "c.." 87 | 88 | 89 | class PrefixRootTestCase(TestCase): 90 | def test_prefix_root(self): 91 | assert paths._prefix_root("/a/", "b/c") == "/a/b/c" 92 | assert paths._prefix_root("/a/b", "c/d") == "/a/b/c/d" 93 | assert paths._prefix_root("/a", "/b/c") == "/a/b/c" 94 | assert paths._prefix_root("/a", "//b/c.") == "/a/b/c." 95 | 96 | 97 | class NormRootTestCase(TestCase): 98 | def test_norm_root(self): 99 | assert paths._norm_root("") == "/" 100 | assert paths._norm_root("/") == "/" 101 | assert paths._norm_root("//a") == "/a" 102 | assert paths._norm_root("//a./b") == "/a./b" 103 | -------------------------------------------------------------------------------- /kazoo/tests/test_queue.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | import pytest 4 | 5 | from kazoo.testing import KazooTestCase 6 | from kazoo.tests.util import CI_ZK_VERSION 7 | 8 | 9 | class KazooQueueTests(KazooTestCase): 10 | def _makeOne(self): 11 | path = "/" + uuid.uuid4().hex 12 | return self.client.Queue(path) 13 | 14 | def test_queue_validation(self): 15 | queue = self._makeOne() 16 | with pytest.raises(TypeError): 17 | queue.put({}) 18 | with pytest.raises(TypeError): 19 | queue.put(b"one", b"100") 20 | with pytest.raises(TypeError): 21 | queue.put(b"one", 10.0) 22 | with pytest.raises(ValueError): 23 | queue.put(b"one", -100) 24 | with pytest.raises(ValueError): 25 | queue.put(b"one", 100000) 26 | 27 | def test_empty_queue(self): 28 | queue = self._makeOne() 29 | assert len(queue) == 0 30 | assert queue.get() is None 31 | assert len(queue) == 0 32 | 33 | def test_queue(self): 34 | queue = self._makeOne() 35 | queue.put(b"one") 36 | queue.put(b"two") 37 | queue.put(b"three") 38 | assert len(queue) == 3 39 | 40 | assert queue.get() == b"one" 41 | assert queue.get() == b"two" 42 | assert queue.get() == b"three" 43 | assert len(queue) == 0 44 | 45 | def test_priority(self): 46 | queue = self._makeOne() 47 | queue.put(b"four", priority=101) 48 | queue.put(b"one", priority=0) 49 | queue.put(b"two", priority=0) 50 | queue.put(b"three", priority=10) 51 | 52 | assert queue.get() == b"one" 53 | assert queue.get() == b"two" 54 | assert queue.get() == b"three" 55 | assert queue.get() == b"four" 56 | 57 | 58 | class KazooLockingQueueTests(KazooTestCase): 59 | def setUp(self): 60 | KazooTestCase.setUp(self) 61 | skip = False 62 | if CI_ZK_VERSION and CI_ZK_VERSION < (3, 4): 63 | skip = True 64 | elif CI_ZK_VERSION and CI_ZK_VERSION >= (3, 4): 65 | skip = False 66 | else: 67 | ver = self.client.server_version() 68 | if ver[1] < 4: 69 | skip = True 70 | if skip: 71 | pytest.skip("Must use Zookeeper 3.4 or above") 72 | 73 | def _makeOne(self): 74 | path = "/" + uuid.uuid4().hex 75 | return self.client.LockingQueue(path) 76 | 77 | def test_queue_validation(self): 78 | queue = self._makeOne() 79 | with pytest.raises(TypeError): 80 | queue.put({}) 81 | with pytest.raises(TypeError): 82 | queue.put(b"one", b"100") 83 | with pytest.raises(TypeError): 84 | queue.put(b"one", 10.0) 85 | with pytest.raises(ValueError): 86 | queue.put(b"one", -100) 87 | with pytest.raises(ValueError): 88 | queue.put(b"one", 100000) 89 | with pytest.raises(TypeError): 90 | queue.put_all({}) 91 | with pytest.raises(TypeError): 92 | queue.put_all([{}]) 93 | with pytest.raises(TypeError): 94 | queue.put_all([b"one"], b"100") 95 | with pytest.raises(TypeError): 96 | queue.put_all([b"one"], 10.0) 97 | with pytest.raises(ValueError): 98 | queue.put_all([b"one"], -100) 99 | with pytest.raises(ValueError): 100 | queue.put_all([b"one"], 100000) 101 | 102 | def test_empty_queue(self): 103 | queue = self._makeOne() 104 | assert len(queue) == 0 105 | assert queue.get(0) is None 106 | assert len(queue) == 0 107 | 108 | def test_queue(self): 109 | queue = self._makeOne() 110 | queue.put(b"one") 111 | queue.put_all([b"two", b"three"]) 112 | assert len(queue) == 3 113 | 114 | assert not queue.consume() 115 | assert not queue.holds_lock() 116 | assert queue.get(1) == b"one" 117 | assert queue.holds_lock() 118 | # Without consuming, should return the same element 119 | assert queue.get(1) == b"one" 120 | assert queue.consume() 121 | assert not queue.holds_lock() 122 | assert queue.get(1) == b"two" 123 | assert queue.holds_lock() 124 | assert queue.consume() 125 | assert not queue.holds_lock() 126 | assert queue.get(1) == b"three" 127 | assert queue.holds_lock() 128 | assert queue.consume() 129 | assert not queue.holds_lock() 130 | assert not queue.consume() 131 | assert len(queue) == 0 132 | 133 | def test_consume(self): 134 | queue = self._makeOne() 135 | 136 | queue.put(b"one") 137 | assert not queue.consume() 138 | queue.get(0.1) 139 | assert queue.consume() 140 | assert not queue.consume() 141 | 142 | def test_release(self): 143 | queue = self._makeOne() 144 | 145 | queue.put(b"one") 146 | assert queue.get(1) == b"one" 147 | assert queue.holds_lock() 148 | assert queue.release() 149 | assert not queue.holds_lock() 150 | assert queue.get(1) == b"one" 151 | assert queue.consume() 152 | assert not queue.release() 153 | assert len(queue) == 0 154 | 155 | def test_holds_lock(self): 156 | queue = self._makeOne() 157 | 158 | assert not queue.holds_lock() 159 | queue.put(b"one") 160 | queue.get(0.1) 161 | assert queue.holds_lock() 162 | queue.consume() 163 | assert not queue.holds_lock() 164 | 165 | def test_priority(self): 166 | queue = self._makeOne() 167 | queue.put(b"four", priority=101) 168 | queue.put(b"one", priority=0) 169 | queue.put(b"two", priority=0) 170 | queue.put(b"three", priority=10) 171 | 172 | assert queue.get(1) == b"one" 173 | assert queue.consume() 174 | assert queue.get(1) == b"two" 175 | assert queue.consume() 176 | assert queue.get(1) == b"three" 177 | assert queue.consume() 178 | assert queue.get(1) == b"four" 179 | assert queue.consume() 180 | 181 | def test_concurrent_execution(self): 182 | queue = self._makeOne() 183 | value1 = [] 184 | value2 = [] 185 | value3 = [] 186 | event1 = self.client.handler.event_object() 187 | event2 = self.client.handler.event_object() 188 | event3 = self.client.handler.event_object() 189 | 190 | def get_concurrently(value, event): 191 | q = self.client.LockingQueue(queue.path) 192 | value.append(q.get(0.1)) 193 | event.set() 194 | 195 | self.client.handler.spawn(get_concurrently, value1, event1) 196 | self.client.handler.spawn(get_concurrently, value2, event2) 197 | self.client.handler.spawn(get_concurrently, value3, event3) 198 | queue.put(b"one") 199 | event1.wait(0.2) 200 | event2.wait(0.2) 201 | event3.wait(0.2) 202 | 203 | result = value1 + value2 + value3 204 | assert result.count(b"one") == 1 205 | assert result.count(None) == 2 206 | -------------------------------------------------------------------------------- /kazoo/tests/test_retry.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | import pytest 4 | 5 | from kazoo import exceptions as ke 6 | from kazoo import retry as kr 7 | 8 | 9 | def _make_retry(*args, **kwargs): 10 | """Return a KazooRetry instance with a dummy sleep function.""" 11 | 12 | def _sleep_func(_time): 13 | pass 14 | 15 | return kr.KazooRetry(*args, sleep_func=_sleep_func, **kwargs) 16 | 17 | 18 | def _make_try_func(times=1): 19 | """Returns a function that raises ForceRetryError `times` time before 20 | returning None. 21 | """ 22 | callmock = mock.Mock( 23 | side_effect=[kr.ForceRetryError("Failed!")] * times + [None], 24 | ) 25 | return callmock 26 | 27 | 28 | def test_call(): 29 | retry = _make_retry(delay=0, max_tries=2) 30 | func = _make_try_func() 31 | retry(func, "foo", bar="baz") 32 | assert func.call_args_list == [ 33 | mock.call("foo", bar="baz"), 34 | mock.call("foo", bar="baz"), 35 | ] 36 | 37 | 38 | def test_reset(): 39 | retry = _make_retry(delay=0, max_tries=2) 40 | func = _make_try_func() 41 | retry(func) 42 | assert ( 43 | func.call_count == retry._attempts + 1 == 2 44 | ), "Called 2 times, failed _attempts 1, succeeded 1" 45 | retry.reset() 46 | assert retry._attempts == 0 47 | 48 | 49 | def test_too_many_tries(): 50 | retry = _make_retry(delay=0, max_tries=10) 51 | func = _make_try_func(times=999) 52 | with pytest.raises(kr.RetryFailedError): 53 | retry(func) 54 | assert ( 55 | func.call_count == retry._attempts == 10 56 | ), "Called 10 times, failed _attempts 10" 57 | 58 | 59 | def test_maximum_delay(): 60 | retry = _make_retry(delay=10, max_tries=100, max_jitter=0) 61 | func = _make_try_func(times=2) 62 | retry(func) 63 | assert func.call_count == 3, "Called 3 times, 2 failed _attempts" 64 | assert retry._cur_delay == 10 * 2**2, "Normal exponential backoff" 65 | retry.reset() 66 | func = _make_try_func(times=10) 67 | retry(func) 68 | assert func.call_count == 11, "Called 11 times, 10 failed _attempts" 69 | assert retry._cur_delay == 60, "Delay capped by maximum" 70 | # gevent's sleep function is picky about the type 71 | assert isinstance(retry._cur_delay, float) 72 | 73 | 74 | def test_copy(): 75 | retry = _make_retry() 76 | rcopy = retry.copy() 77 | assert rcopy is not retry 78 | assert rcopy.sleep_func is retry.sleep_func 79 | 80 | 81 | def test_connection_closed(): 82 | retry = _make_retry() 83 | 84 | def testit(): 85 | raise ke.ConnectionClosedError 86 | 87 | with pytest.raises(ke.ConnectionClosedError): 88 | retry(testit) 89 | 90 | 91 | def test_session_expired(): 92 | retry = _make_retry(max_tries=1) 93 | 94 | def testit(): 95 | raise ke.SessionExpiredError 96 | 97 | with pytest.raises(kr.RetryFailedError): 98 | retry(testit) 99 | 100 | retry = _make_retry(max_tries=1, ignore_expire=False) 101 | 102 | with pytest.raises(ke.SessionExpiredError): 103 | retry(testit) 104 | -------------------------------------------------------------------------------- /kazoo/tests/test_sasl.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import time 4 | 5 | import pytest 6 | 7 | from kazoo.testing import KazooTestHarness 8 | from kazoo.exceptions import ( 9 | AuthFailedError, 10 | NoAuthError, 11 | ) 12 | from kazoo.tests.util import CI_ZK_VERSION 13 | 14 | 15 | class TestLegacySASLDigestAuthentication(KazooTestHarness): 16 | def setUp(self): 17 | try: 18 | import puresasl # NOQA 19 | except ImportError: 20 | pytest.skip("PureSASL not available.") 21 | 22 | os.environ["ZOOKEEPER_JAAS_AUTH"] = "digest" 23 | self.setup_zookeeper() 24 | 25 | if CI_ZK_VERSION: 26 | version = CI_ZK_VERSION 27 | else: 28 | version = self.client.server_version() 29 | if not version or version < (3, 4): 30 | pytest.skip("Must use Zookeeper 3.4 or above") 31 | 32 | def tearDown(self): 33 | self.teardown_zookeeper() 34 | os.environ.pop("ZOOKEEPER_JAAS_AUTH", None) 35 | 36 | def test_connect_sasl_auth(self): 37 | from kazoo.security import make_acl 38 | 39 | username = "jaasuser" 40 | password = "jaas_password" 41 | 42 | acl = make_acl("sasl", credential=username, all=True) 43 | 44 | sasl_auth = "%s:%s" % (username, password) 45 | client = self._get_client(auth_data=[("sasl", sasl_auth)]) 46 | 47 | client.start() 48 | try: 49 | client.create("/1", acl=(acl,)) 50 | # give ZK a chance to copy data to other node 51 | time.sleep(0.1) 52 | with pytest.raises(NoAuthError): 53 | self.client.get("/1") 54 | finally: 55 | client.delete("/1") 56 | client.stop() 57 | client.close() 58 | 59 | def test_invalid_sasl_auth(self): 60 | client = self._get_client(auth_data=[("sasl", "baduser:badpassword")]) 61 | with pytest.raises(AuthFailedError): 62 | client.start() 63 | 64 | 65 | class TestSASLDigestAuthentication(KazooTestHarness): 66 | def setUp(self): 67 | try: 68 | import puresasl # NOQA 69 | except ImportError: 70 | pytest.skip("PureSASL not available.") 71 | 72 | os.environ["ZOOKEEPER_JAAS_AUTH"] = "digest" 73 | self.setup_zookeeper() 74 | 75 | if CI_ZK_VERSION: 76 | version = CI_ZK_VERSION 77 | else: 78 | version = self.client.server_version() 79 | if not version or version < (3, 4): 80 | pytest.skip("Must use Zookeeper 3.4 or above") 81 | 82 | def tearDown(self): 83 | self.teardown_zookeeper() 84 | os.environ.pop("ZOOKEEPER_JAAS_AUTH", None) 85 | 86 | def test_connect_sasl_auth(self): 87 | from kazoo.security import make_acl 88 | 89 | username = "jaasuser" 90 | password = "jaas_password" 91 | 92 | acl = make_acl("sasl", credential=username, all=True) 93 | 94 | client = self._get_client( 95 | sasl_options={ 96 | "mechanism": "DIGEST-MD5", 97 | "username": username, 98 | "password": password, 99 | } 100 | ) 101 | client.start() 102 | try: 103 | client.create("/1", acl=(acl,)) 104 | # give ZK a chance to copy data to other node 105 | time.sleep(0.1) 106 | with pytest.raises(NoAuthError): 107 | self.client.get("/1") 108 | finally: 109 | client.delete("/1") 110 | client.stop() 111 | client.close() 112 | 113 | def test_invalid_sasl_auth(self): 114 | client = self._get_client( 115 | sasl_options={ 116 | "mechanism": "DIGEST-MD5", 117 | "username": "baduser", 118 | "password": "badpassword", 119 | } 120 | ) 121 | with pytest.raises(AuthFailedError): 122 | client.start() 123 | 124 | 125 | class TestSASLGSSAPIAuthentication(KazooTestHarness): 126 | def setUp(self): 127 | try: 128 | import puresasl # NOQA 129 | except ImportError: 130 | pytest.skip("PureSASL not available.") 131 | try: 132 | import kerberos # NOQA 133 | except ImportError: 134 | pytest.skip("Kerberos support not available.") 135 | if not os.environ.get("KRB5_TEST_ENV"): 136 | pytest.skip("Test Kerberos environ not setup.") 137 | 138 | os.environ["ZOOKEEPER_JAAS_AUTH"] = "gssapi" 139 | self.setup_zookeeper() 140 | 141 | if CI_ZK_VERSION: 142 | version = CI_ZK_VERSION 143 | else: 144 | version = self.client.server_version() 145 | if not version or version < (3, 4): 146 | pytest.skip("Must use Zookeeper 3.4 or above") 147 | 148 | def tearDown(self): 149 | self.teardown_zookeeper() 150 | os.environ.pop("ZOOKEEPER_JAAS_AUTH", None) 151 | 152 | def test_connect_gssapi_auth(self): 153 | from kazoo.security import make_acl 154 | 155 | # Ensure we have a client ticket 156 | subprocess.check_call( 157 | [ 158 | "kinit", 159 | "-kt", 160 | os.path.expandvars("${KRB5_TEST_ENV}/client.keytab"), 161 | "client", 162 | ] 163 | ) 164 | 165 | acl = make_acl("sasl", credential="client@KAZOOTEST.ORG", all=True) 166 | 167 | client = self._get_client(sasl_options={"mechanism": "GSSAPI"}) 168 | client.start() 169 | try: 170 | client.create("/1", acl=(acl,)) 171 | # give ZK a chance to copy data to other node 172 | time.sleep(0.1) 173 | with pytest.raises(NoAuthError): 174 | self.client.get("/1") 175 | finally: 176 | client.delete("/1") 177 | client.stop() 178 | client.close() 179 | 180 | def test_invalid_gssapi_auth(self): 181 | # Request a post-datated ticket, so that it is currently invalid. 182 | subprocess.check_call( 183 | [ 184 | "kinit", 185 | "-kt", 186 | os.path.expandvars("${KRB5_TEST_ENV}/client.keytab"), 187 | "-s", 188 | "30min", 189 | "client", 190 | ] 191 | ) 192 | 193 | client = self._get_client(sasl_options={"mechanism": "GSSAPI"}) 194 | with pytest.raises(AuthFailedError): 195 | client.start() 196 | -------------------------------------------------------------------------------- /kazoo/tests/test_security.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from kazoo.security import Permissions 4 | 5 | 6 | class TestACL(unittest.TestCase): 7 | def _makeOne(self, *args, **kwargs): 8 | from kazoo.security import make_acl 9 | 10 | return make_acl(*args, **kwargs) 11 | 12 | def test_read_acl(self): 13 | acl = self._makeOne("digest", ":", read=True) 14 | assert acl.perms & Permissions.READ == Permissions.READ 15 | 16 | def test_all_perms(self): 17 | acl = self._makeOne( 18 | "digest", 19 | ":", 20 | read=True, 21 | write=True, 22 | create=True, 23 | delete=True, 24 | admin=True, 25 | ) 26 | for perm in [ 27 | Permissions.READ, 28 | Permissions.CREATE, 29 | Permissions.WRITE, 30 | Permissions.DELETE, 31 | Permissions.ADMIN, 32 | ]: 33 | assert acl.perms & perm == perm 34 | 35 | def test_perm_listing(self): 36 | from kazoo.security import ACL 37 | 38 | f = ACL(15, "fred") 39 | assert "READ" in f.acl_list 40 | assert "WRITE" in f.acl_list 41 | assert "CREATE" in f.acl_list 42 | assert "DELETE" in f.acl_list 43 | 44 | f = ACL(16, "fred") 45 | assert "ADMIN" in f.acl_list 46 | 47 | f = ACL(31, "george") 48 | assert "ALL" in f.acl_list 49 | 50 | def test_perm_repr(self): 51 | from kazoo.security import ACL 52 | 53 | f = ACL(16, "fred") 54 | assert "ACL(perms=16, acl_list=['ADMIN']" in repr(f) 55 | -------------------------------------------------------------------------------- /kazoo/tests/test_selectors_select.py: -------------------------------------------------------------------------------- 1 | """ 2 | The official python select function test case copied from python source 3 | to test the selector_select function. 4 | """ 5 | 6 | import os 7 | import socket 8 | import sys 9 | import unittest 10 | 11 | from kazoo.handlers.utils import selector_select 12 | 13 | select = selector_select 14 | 15 | 16 | @unittest.skipIf( 17 | (sys.platform[:3] == "win"), "can't easily test on this system" 18 | ) 19 | class SelectTestCase(unittest.TestCase): 20 | class Nope: 21 | pass 22 | 23 | class Almost: 24 | def fileno(self): 25 | return "fileno" 26 | 27 | def test_error_conditions(self): 28 | self.assertRaises(TypeError, select, 1, 2, 3) 29 | self.assertRaises(TypeError, select, [self.Nope()], [], []) 30 | self.assertRaises(TypeError, select, [self.Almost()], [], []) 31 | self.assertRaises(TypeError, select, [], [], [], "not a number") 32 | self.assertRaises(ValueError, select, [], [], [], -1) 33 | 34 | # Issue #12367: http://www.freebsd.org/cgi/query-pr.cgi?pr=kern/155606 35 | @unittest.skipIf( 36 | sys.platform.startswith("freebsd"), 37 | "skip because of a FreeBSD bug: kern/155606", 38 | ) 39 | def test_errno(self): 40 | with open(__file__, "rb") as fp: 41 | fd = fp.fileno() 42 | fp.close() 43 | self.assertRaises(ValueError, select, [fd], [], [], 0) 44 | 45 | def test_returned_list_identity(self): 46 | # See issue #8329 47 | r, w, x = select([], [], [], 1) 48 | self.assertIsNot(r, w) 49 | self.assertIsNot(r, x) 50 | self.assertIsNot(w, x) 51 | 52 | def test_select(self): 53 | cmd = "for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done" 54 | p = os.popen(cmd, "r") 55 | for tout in (0, 1, 2, 4, 8, 16) + (None,) * 10: 56 | rfd, wfd, xfd = select([p], [], [], tout) 57 | if (rfd, wfd, xfd) == ([], [], []): 58 | continue 59 | if (rfd, wfd, xfd) == ([p], [], []): 60 | line = p.readline() 61 | if not line: 62 | break 63 | continue 64 | self.fail("Unexpected return values from select():", rfd, wfd, xfd) 65 | p.close() 66 | 67 | # Issue 16230: Crash on select resized list 68 | def test_select_mutated(self): 69 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: 70 | a = [] 71 | 72 | class F: 73 | def fileno(self): 74 | del a[-1] 75 | return s.fileno() 76 | 77 | a[:] = [F()] * 10 78 | self.assertEqual(select([], a, []), ([], a[:5], [])) 79 | 80 | 81 | if __name__ == "__main__": 82 | unittest.main() 83 | -------------------------------------------------------------------------------- /kazoo/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import patch 3 | 4 | import pytest 5 | 6 | try: 7 | from kazoo.handlers.eventlet import green_socket as socket 8 | 9 | EVENTLET_HANDLER_AVAILABLE = True 10 | except ImportError: 11 | EVENTLET_HANDLER_AVAILABLE = False 12 | 13 | 14 | class TestCreateTCPConnection(unittest.TestCase): 15 | def test_timeout_arg(self): 16 | from kazoo.handlers import utils 17 | from kazoo.handlers.utils import create_tcp_connection, socket, time 18 | 19 | with patch.object(socket, "create_connection") as create_connection: 20 | with patch.object(utils, "_set_default_tcpsock_options"): 21 | # Ensure a gap between calls to time.time() does not result in 22 | # create_connection being called with a negative timeout 23 | # argument. 24 | with patch.object(time, "time", side_effect=range(10)): 25 | create_tcp_connection( 26 | socket, ("127.0.0.1", 2181), timeout=1.5 27 | ) 28 | 29 | for call_args in create_connection.call_args_list: 30 | timeout = call_args[0][1] 31 | assert timeout >= 0, "socket timeout must be nonnegative" 32 | 33 | def test_ssl_server_hostname(self): 34 | from kazoo.handlers import utils 35 | from kazoo.handlers.utils import create_tcp_connection, socket, ssl 36 | 37 | with patch.object(utils, "_set_default_tcpsock_options"): 38 | with patch.object(ssl.SSLContext, "wrap_socket") as wrap_socket: 39 | create_tcp_connection( 40 | socket, 41 | ("127.0.0.1", 2181), 42 | timeout=1.5, 43 | hostname="fakehostname", 44 | use_ssl=True, 45 | ) 46 | 47 | for call_args in wrap_socket.call_args_list: 48 | server_hostname = call_args[1]["server_hostname"] 49 | assert server_hostname == "fakehostname" 50 | 51 | def test_ssl_server_check_hostname(self): 52 | from kazoo.handlers import utils 53 | from kazoo.handlers.utils import create_tcp_connection, socket, ssl 54 | 55 | with patch.object(utils, "_set_default_tcpsock_options"): 56 | with patch.object( 57 | ssl.SSLContext, "wrap_socket", autospec=True 58 | ) as wrap_socket: 59 | create_tcp_connection( 60 | socket, 61 | ("127.0.0.1", 2181), 62 | timeout=1.5, 63 | hostname="fakehostname", 64 | use_ssl=True, 65 | check_hostname=True, 66 | ) 67 | 68 | for call_args in wrap_socket.call_args_list: 69 | ssl_context = call_args[0][0] 70 | assert ssl_context.check_hostname 71 | 72 | def test_ssl_server_check_hostname_config_validation(self): 73 | from kazoo.handlers.utils import create_tcp_connection, socket 74 | 75 | with pytest.raises(ValueError): 76 | create_tcp_connection( 77 | socket, 78 | ("127.0.0.1", 2181), 79 | timeout=1.5, 80 | hostname="fakehostname", 81 | use_ssl=True, 82 | verify_certs=False, 83 | check_hostname=True, 84 | ) 85 | 86 | def test_timeout_arg_eventlet(self): 87 | if not EVENTLET_HANDLER_AVAILABLE: 88 | pytest.skip("eventlet handler not available.") 89 | 90 | from kazoo.handlers import utils 91 | from kazoo.handlers.utils import create_tcp_connection, time 92 | 93 | with patch.object(socket, "create_connection") as create_connection: 94 | with patch.object(utils, "_set_default_tcpsock_options"): 95 | # Ensure a gap between calls to time.time() does not result in 96 | # create_connection being called with a negative timeout 97 | # argument. 98 | with patch.object(time, "time", side_effect=range(10)): 99 | create_tcp_connection( 100 | socket, ("127.0.0.1", 2181), timeout=1.5 101 | ) 102 | 103 | for call_args in create_connection.call_args_list: 104 | timeout = call_args[0][1] 105 | assert timeout >= 0, "socket timeout must be nonnegative" 106 | 107 | def test_slow_connect(self): 108 | # Currently, create_tcp_connection will raise a socket timeout if it 109 | # takes longer than the specified "timeout" to create a connection. 110 | # In the future, "timeout" might affect only the created socket and not 111 | # the time it takes to create it. 112 | from kazoo.handlers.utils import create_tcp_connection, socket, time 113 | 114 | # Simulate a second passing between calls to check the current time. 115 | with patch.object(time, "time", side_effect=range(10)): 116 | with pytest.raises(socket.error): 117 | create_tcp_connection(socket, ("127.0.0.1", 2181), timeout=0.5) 118 | 119 | def test_negative_timeout(self): 120 | from kazoo.handlers.utils import create_tcp_connection, socket 121 | 122 | with pytest.raises(socket.error): 123 | create_tcp_connection(socket, ("127.0.0.1", 2181), timeout=-1) 124 | 125 | def test_zero_timeout(self): 126 | # Rather than pass '0' through as a timeout to 127 | # socket.create_connection, create_tcp_connection should raise 128 | # socket.error. This is because the socket library treats '0' as an 129 | # indicator to create a non-blocking socket. 130 | from kazoo.handlers.utils import create_tcp_connection, socket, time 131 | 132 | # Simulate no time passing between calls to check the current time. 133 | with patch.object(time, "time", return_value=time.time()): 134 | with pytest.raises(socket.error): 135 | create_tcp_connection(socket, ("127.0.0.1", 2181), timeout=0) 136 | -------------------------------------------------------------------------------- /kazoo/tests/util.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # 3 | # Copyright Zope Foundation and Contributors. 4 | # All Rights Reserved. 5 | # 6 | # This software is subject to the provisions of the Zope Public License, 7 | # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. 8 | # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED 9 | # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 10 | # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS 11 | # FOR A PARTICULAR PURPOSE. 12 | # 13 | ############################################################################## 14 | 15 | import logging 16 | import os 17 | import time 18 | 19 | CI = os.environ.get("CI", False) 20 | CI_ZK_VERSION = CI and os.environ.get("ZOOKEEPER_VERSION", None) 21 | if CI_ZK_VERSION: 22 | if "-" in CI_ZK_VERSION: 23 | # Ignore pre-release markers like -alpha 24 | CI_ZK_VERSION = CI_ZK_VERSION.split("-")[0] 25 | CI_ZK_VERSION = tuple([int(n) for n in CI_ZK_VERSION.split(".")]) 26 | 27 | 28 | class Handler(logging.Handler): 29 | def __init__(self, *names, **kw): 30 | logging.Handler.__init__(self) 31 | self.names = names 32 | self.records = [] 33 | self.setLoggerLevel(**kw) 34 | 35 | def setLoggerLevel(self, level=1): 36 | self.level = level 37 | self.oldlevels = {} 38 | 39 | def emit(self, record): 40 | self.records.append(record) 41 | 42 | def clear(self): 43 | del self.records[:] 44 | 45 | def install(self): 46 | for name in self.names: 47 | logger = logging.getLogger(name) 48 | self.oldlevels[name] = logger.level 49 | logger.setLevel(self.level) 50 | logger.addHandler(self) 51 | 52 | def uninstall(self): 53 | for name in self.names: 54 | logger = logging.getLogger(name) 55 | logger.setLevel(self.oldlevels[name]) 56 | logger.removeHandler(self) 57 | 58 | def __str__(self): 59 | return "\n".join( 60 | [ 61 | ( 62 | "%s %s\n %s" 63 | % ( 64 | record.name, 65 | record.levelname, 66 | "\n".join( 67 | [ 68 | line 69 | for line in record.getMessage().split("\n") 70 | if line.strip() 71 | ] 72 | ), 73 | ) 74 | ) 75 | for record in self.records 76 | ] 77 | ) 78 | 79 | 80 | class InstalledHandler(Handler): 81 | def __init__(self, *names, **kw): 82 | Handler.__init__(self, *names, **kw) 83 | self.install() 84 | 85 | 86 | class Wait(object): 87 | class TimeOutWaitingFor(Exception): 88 | "A test condition timed out" 89 | 90 | timeout = 9 91 | wait = 0.01 92 | 93 | def __init__( 94 | self, 95 | timeout=None, 96 | wait=None, 97 | exception=None, 98 | getnow=(lambda: time.monotonic), 99 | getsleep=(lambda: time.sleep), 100 | ): 101 | if timeout is not None: 102 | self.timeout = timeout 103 | 104 | if wait is not None: 105 | self.wait = wait 106 | 107 | if exception is not None: 108 | self.TimeOutWaitingFor = exception 109 | 110 | self.getnow = getnow 111 | self.getsleep = getsleep 112 | 113 | def __call__(self, func=None, timeout=None, wait=None, message=None): 114 | if func is None: 115 | return lambda func: self(func, timeout, wait, message) 116 | 117 | if func(): 118 | return 119 | 120 | now = self.getnow() 121 | sleep = self.getsleep() 122 | if timeout is None: 123 | timeout = self.timeout 124 | if wait is None: 125 | wait = self.wait 126 | wait = float(wait) 127 | 128 | deadline = now() + timeout 129 | while 1: 130 | sleep(wait) 131 | if func(): 132 | return 133 | if now() > deadline: 134 | raise self.TimeOutWaitingFor( 135 | message or func.__doc__ or func.__name__ 136 | ) 137 | 138 | 139 | wait = Wait() 140 | -------------------------------------------------------------------------------- /kazoo/version.py: -------------------------------------------------------------------------------- 1 | __version__ = "2.10.0" 2 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | build-backend = 'setuptools.build_meta' 3 | requires = [ 4 | 'setuptools >= 46.4.0', 5 | ] 6 | 7 | [tool.black] 8 | line-length = 79 9 | target-version = ['py37', 'py38', 'py39', 'py310'] 10 | include = '\.pyi?$' 11 | # 'extend-exclude' excludes files or directories in addition to the defaults 12 | # A regex preceded with ^/ will apply only to files and directories 13 | # in the root of the project. 14 | # ( 15 | # ^/foo.py # exclude a file named foo.py in the root of the project. 16 | # | .*_pb2.py # exclude autogenerated Protocol Buffer files anywhere in the 17 | # ) 18 | # # project. 19 | extend-exclude = ''' 20 | ''' 21 | 22 | [tool.pytest.ini_options] 23 | addopts = "-ra -v --color=yes" 24 | log_cli = true 25 | log_cli_date_format = "%Y-%m-%d %H:%M:%S" 26 | log_cli_format = "%(asctime)s %(levelname)s %(message)s" 27 | log_cli_level = "INFO" 28 | # Per-test timeout in seconds 29 | timeout = 180 30 | 31 | [tool.mypy] 32 | 33 | # For details on each flag, please see the mypy documentation at: 34 | # https://mypy.readthedocs.io/en/stable/config_file.html#config-file 35 | 36 | # Note: The order of flags listed here should match the order used in mypy's 37 | # documentation to make it easier to find the documentation for each flag. 38 | 39 | # Import Discovery 40 | ignore_missing_imports = false 41 | 42 | # Disallow dynamic typing 43 | disallow_any_unimported = true 44 | disallow_any_expr = false 45 | disallow_any_decorated = true 46 | disallow_any_explicit = true 47 | disallow_any_generics = true 48 | disallow_subclassing_any = true 49 | 50 | # Untyped definitions and calls 51 | disallow_untyped_calls = true 52 | disallow_untyped_defs = true 53 | disallow_incomplete_defs = true 54 | check_untyped_defs = true 55 | disallow_untyped_decorators = true 56 | 57 | # None and Optional handling 58 | implicit_optional = false 59 | strict_optional = true 60 | 61 | # Configuring warnings 62 | warn_redundant_casts = true 63 | warn_unused_ignores = true 64 | warn_no_return = true 65 | warn_return_any = true 66 | warn_unreachable = true 67 | 68 | # Miscellaneous strictness flags 69 | allow_untyped_globals = false 70 | allow_redefinition = false 71 | local_partial_types = true 72 | implicit_reexport = false 73 | strict_concatenate = true 74 | strict_equality = true 75 | strict = true 76 | 77 | # Configuring error messages 78 | show_error_context = true 79 | show_column_numbers = true 80 | hide_error_codes = false 81 | pretty = true 82 | color_output = true 83 | error_summary = true 84 | show_absolute_path = true 85 | 86 | # Miscellaneous 87 | warn_unused_configs = true 88 | verbosity = 0 89 | 90 | # FIXME: As type annotations are introduced, please remove the appropriate 91 | # ignore_errors flag below. New modules should NOT be added here! 92 | 93 | [[tool.mypy.overrides]] 94 | module = [ 95 | 'kazoo.client', 96 | 'kazoo.exceptions', 97 | 'kazoo.handlers.eventlet', 98 | 'kazoo.handlers.gevent', 99 | 'kazoo.handlers.threading', 100 | 'kazoo.handlers.utils', 101 | 'kazoo.hosts', 102 | 'kazoo.interfaces', 103 | 'kazoo.loggingsupport', 104 | 'kazoo.protocol.connection', 105 | 'kazoo.protocol.paths', 106 | 'kazoo.protocol.serialization', 107 | 'kazoo.protocol.states', 108 | 'kazoo.recipe.barrier', 109 | 'kazoo.recipe.cache', 110 | 'kazoo.recipe.counter', 111 | 'kazoo.recipe.election', 112 | 'kazoo.recipe.lease', 113 | 'kazoo.recipe.lock', 114 | 'kazoo.recipe.partitioner', 115 | 'kazoo.recipe.party', 116 | 'kazoo.recipe.queue', 117 | 'kazoo.recipe.watchers', 118 | 'kazoo.retry', 119 | 'kazoo.security', 120 | 'kazoo.testing.common', 121 | 'kazoo.testing.harness', 122 | 'kazoo.tests.conftest', 123 | 'kazoo.tests.test_barrier', 124 | 'kazoo.tests.test_build', 125 | 'kazoo.tests.test_cache', 126 | 'kazoo.tests.test_client', 127 | 'kazoo.tests.test_connection', 128 | 'kazoo.tests.test_counter', 129 | 'kazoo.tests.test_election', 130 | 'kazoo.tests.test_eventlet_handler', 131 | 'kazoo.tests.test_exceptions', 132 | 'kazoo.tests.test_gevent_handler', 133 | 'kazoo.tests.test_hosts', 134 | 'kazoo.tests.test_interrupt', 135 | 'kazoo.tests.test_lease', 136 | 'kazoo.tests.test_lock', 137 | 'kazoo.tests.test_partitioner', 138 | 'kazoo.tests.test_party', 139 | 'kazoo.tests.test_paths', 140 | 'kazoo.tests.test_queue', 141 | 'kazoo.tests.test_retry', 142 | 'kazoo.tests.test_sasl', 143 | 'kazoo.tests.test_security', 144 | 'kazoo.tests.test_selectors_select', 145 | 'kazoo.tests.test_threading_handler', 146 | 'kazoo.tests.test_utils', 147 | 'kazoo.tests.test_watchers', 148 | 'kazoo.tests.util', 149 | 'kazoo.version' 150 | ] 151 | ignore_errors = true 152 | -------------------------------------------------------------------------------- /run_failure.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | 5 | def test(arg): 6 | return os.system("bin/pytest -v %s" % arg) 7 | 8 | 9 | def main(args): 10 | if not args: 11 | print( 12 | "Run as bin/python run_failure.py , for example: \n" 13 | "bin/python run_failure.py " 14 | "kazoo.tests.test_watchers:KazooChildrenWatcherTests" 15 | ) 16 | return 17 | arg = args[0] 18 | i = 0 19 | while 1: 20 | i += 1 21 | print("Run number: %s" % i) 22 | ret = test(arg) 23 | if ret != 0: 24 | break 25 | 26 | 27 | if __name__ == "__main__": 28 | main(sys.argv[1:]) 29 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = kazoo 3 | version = attr: kazoo.version.__version__ 4 | author = Kazoo team 5 | author_email = python-zk@googlegroups.com 6 | url = https://kazoo.readthedocs.io 7 | description = "Higher Level Zookeeper Client" 8 | long_description = file: README.md, CHANGES.md 9 | long_description_content_type = text/markdown 10 | license = Apache 2.0 11 | license_files = 12 | LICENSE 13 | platform = any 14 | keywords = zookeeper, lock, leader, configuration 15 | classifiers = 16 | Development Status :: 5 - Production/Stable 17 | License :: OSI Approved :: Apache Software License 18 | Intended Audience :: Developers 19 | Operating System :: OS Independent 20 | Programming Language :: Python 21 | Programming Language :: Python :: 3 22 | Programming Language :: Python :: 3.8 23 | Programming Language :: Python :: 3.9 24 | Programming Language :: Python :: 3.10 25 | Programming Language :: Python :: 3.11 26 | Programming Language :: Python :: 3.12 27 | Programming Language :: Python :: Implementation :: CPython 28 | Programming Language :: Python :: Implementation :: PyPy 29 | Topic :: Communications 30 | Topic :: System :: Distributed Computing 31 | Topic :: System :: Networking 32 | project_urls = 33 | Documentation = https://kazoo.readthedocs.io 34 | Changelog = https://github.com/python-zk/kazoo/releases 35 | Source = https://github.com/python-zk/kazoo 36 | Bug Tracker = https://github.com/python-zk/kazoo/issues 37 | 38 | 39 | [options] 40 | zip_safe = false 41 | include_package_data = true 42 | packages = find: 43 | 44 | [aliases] 45 | release = sdist bdist_wheel 46 | 47 | [egg_info] 48 | tag_build = dev 49 | 50 | [bdist_wheel] 51 | universal = true 52 | 53 | [options.extras_require] 54 | dev = 55 | flake8 56 | 57 | test = 58 | objgraph 59 | pytest 60 | pytest-cov 61 | pytest-timeout 62 | gevent>=1.2 ; implementation_name!='pypy' 63 | eventlet>=0.17.1 ; implementation_name!='pypy' 64 | pyjks 65 | pyopenssl 66 | 67 | eventlet = 68 | eventlet>=0.17.1 69 | 70 | gevent = 71 | gevent>=1.2 72 | 73 | sasl = 74 | pure_sasl>=0.5.1 75 | 76 | docs = 77 | Sphinx>=1.2.2 78 | sphinx-autodoc-typehints>=1 79 | 80 | typing = 81 | mypy>=0.991 82 | 83 | alldeps = 84 | %(dev)s 85 | %(eventlet)s 86 | %(gevent)s 87 | %(sasl)s 88 | %(docs)s 89 | %(typing)s 90 | 91 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | setuptools.setup() 4 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | minversion = 4.3.4 3 | requires= 4 | virtualenv>=20.7.2 5 | skip_missing_interpreters=True 6 | envlist = 7 | pep8,black,mypy, 8 | gevent,eventlet,sasl, 9 | docs, 10 | pypy3 11 | isolated_build = true 12 | 13 | [testenv] 14 | wheel = True 15 | wheel_build_env = build 16 | install_command = pip install -c{toxinidir}/constraints.txt {opts} {packages} 17 | passenv = 18 | CI 19 | TOX_* 20 | CI_* 21 | ZOOKEEPER_* 22 | setenv = 23 | pypy3: PYPY=1 24 | extras = 25 | test 26 | docs: docs 27 | gevent: gevent 28 | eventlet: eventlet 29 | sasl: sasl 30 | deps = 31 | sasl: kerberos 32 | allowlist_externals = 33 | {toxinidir}/ensure-zookeeper-env.sh 34 | {toxinidir}/init_krb5.sh 35 | bash 36 | commands = 37 | bash \ 38 | sasl: {toxinidir}/init_krb5.sh {envtmpdir}/kerberos \ 39 | {toxinidir}/ensure-zookeeper-env.sh \ 40 | pytest {posargs: -ra -v --cov-report=xml --cov=kazoo kazoo/tests} 41 | 42 | [testenv:build] 43 | 44 | [testenv:pep8] 45 | basepython = python3 46 | extras = alldeps 47 | deps = 48 | flake8 49 | usedevelop = True 50 | commands = flake8 {posargs} {toxinidir}/kazoo 51 | 52 | [testenv:black] 53 | basepython = python3 54 | extras = 55 | deps = 56 | black 57 | usedevelop = True 58 | commands = black --check {posargs: {toxinidir}/kazoo {toxinidir}/kazoo} 59 | 60 | [testenv:mypy] 61 | basepython = python3 62 | extras = alldeps 63 | deps = 64 | mypy 65 | mypy: types-mock 66 | usedevelop = True 67 | commands = mypy --config-file {toxinidir}/pyproject.toml {toxinidir}/kazoo 68 | --------------------------------------------------------------------------------