├── .bandit.yaml ├── .coveragerc ├── .flake8 ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ └── python-package.yml ├── .gitignore ├── .isort.cfg ├── .safety-policy.yml ├── .vscode └── settings.json ├── CODE_OF_CONDUCT.md ├── LICENSE ├── Makefile ├── README.md ├── mypy.ini ├── pottery ├── __init__.py ├── aionextid.py ├── aioredlock.py ├── annotations.py ├── base.py ├── bloom.py ├── cache.py ├── counter.py ├── deque.py ├── dict.py ├── exceptions.py ├── executor.py ├── hyper.py ├── list.py ├── monkey.py ├── nextid.py ├── py.typed ├── queue.py ├── redlock.py ├── set.py └── timer.py ├── pytest.ini ├── requirements-to-freeze.txt ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── conftest.py ├── test_aionextid.py ├── test_aioredlock.py ├── test_base.py ├── test_bloom.py ├── test_cache.py ├── test_counter.py ├── test_deque.py ├── test_dict.py ├── test_doctests.py ├── test_executor.py ├── test_hyper.py ├── test_list.py ├── test_monkey.py ├── test_nextid.py ├── test_queue.py ├── test_redlock.py ├── test_set.py └── test_timer.py /.bandit.yaml: -------------------------------------------------------------------------------- 1 | assert_used: 2 | skips: ['*/test_*.py'] 3 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # .coveragerc # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | [run] 20 | branch = True 21 | source = . 22 | omit = 23 | venv/* 24 | setup.py 25 | 26 | [report] 27 | exclude_lines = 28 | pragma: no cover 29 | if __name__ == .__main__.: 30 | raise NotImplementedError 31 | show_missing = True 32 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # .flake8 # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | [flake8] 20 | 21 | # Legend 22 | # ------ 23 | # E126: continuation line over-indented for hanging indent 24 | # E127: continuation line over-indented for visual indent 25 | # E226: missing whitespace around arithmetic operator 26 | # E302: expected 2 blank lines, found 1 27 | # E305: expected 2 blank lines after class or function definition, found 1 28 | # E402: module level import not at top of file 29 | # E501: line too long (80 > 79 characters) 30 | # E711: comparison to None should be 'if cond is None:' 31 | # E713: test for membership should be 'not in' 32 | # F401: '.monkey' imported but unused 33 | # W503: line break before binary operator 34 | 35 | ignore = E226,E501,W503 36 | 37 | per-file-ignores = 38 | pottery/__init__.py:E402 39 | pottery/base.py:F401 40 | pottery/exceptions.py:E302 41 | pottery/monkey.py:E302,E305,E402 42 | pottery/redlock.py:E127 43 | tests/*.py:E126,E127,E711,E713 44 | tests/test_redis.py:E711,F401 45 | tests/test_source.py:F401 46 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Import '...' 16 | 2. Instantiate '....' 17 | 3. Call method '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Environment (please complete the following information):** 24 | - OS: [e.g. macOS, Linux] 25 | - Python version [e.g. 3.9.7] 26 | - Redis version [e.g. 6.2.5] 27 | 28 | **Additional context** 29 | Add any other context about the problem here. 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Python package 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | python-version: ['3.10', '3.11', '3.12', '3.13'] 18 | services: 19 | redis: 20 | image: redis 21 | options: >- 22 | --health-cmd "redis-cli ping" 23 | --health-interval 10s 24 | --health-timeout 5s 25 | --health-retries 5 26 | ports: 27 | - 6379:6379 28 | 29 | steps: 30 | - uses: actions/checkout@v4 31 | - name: Set up Python ${{ matrix.python-version }} 32 | uses: actions/setup-python@v5 33 | with: 34 | python-version: ${{ matrix.python-version }} 35 | cache: 'pip' 36 | - name: Install dependencies 37 | run: | 38 | python -m pip install --upgrade pip setuptools wheel 39 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 40 | - name: Test with pytest 41 | run: pytest --verbose --cov-config=.coveragerc --cov=pottery --cov=tests 42 | - name: Type check with Mypy 43 | run: mypy 44 | - name: Lint with Flake8 and isort 45 | run: | 46 | flake8 *\.py pottery/*\.py tests/*\.py --count --max-complexity=10 --statistics 47 | isort *\.py pottery/*\.py tests/*\.py --check-only --diff 48 | - name: Check for security vulnerabilities with Bandit 49 | run: | 50 | bandit --recursive pottery 51 | - name: Check for security vulnerabilities with Safety 52 | uses: pyupio/safety-action@v1 53 | with: 54 | api-key: ${{ secrets.SAFETY_API_KEY }} 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | 91 | 92 | 93 | # Raj's additions 94 | dump.rdb 95 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # .isort.cfg # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | [settings] 20 | skip=monkey.py 21 | force_single_line=True 22 | lines_after_imports=2 23 | -------------------------------------------------------------------------------- /.safety-policy.yml: -------------------------------------------------------------------------------- 1 | version: '3.0' 2 | 3 | scanning-settings: 4 | max-depth: 6 5 | exclude: [] 6 | include-files: [] 7 | system: 8 | targets: [] 9 | 10 | 11 | report: 12 | dependency-vulnerabilities: 13 | enabled: true 14 | auto-ignore-in-report: 15 | python: 16 | environment-results: true 17 | unpinned-requirements: true 18 | cvss-severity: [] 19 | vulnerabilities: 20 | 70612: 21 | reason: "We don't render Jinja templates" 22 | expires: '2024-12-31' 23 | 24 | 25 | fail-scan-with-exit-code: 26 | dependency-vulnerabilities: 27 | enabled: true 28 | fail-on-any-of: 29 | cvss-severity: 30 | - high 31 | - critical 32 | - medium 33 | exploitability: 34 | - high 35 | - critical 36 | - medium 37 | 38 | security-updates: 39 | dependency-vulnerabilities: 40 | auto-security-updates-limit: 41 | - patch 42 | 43 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.testing.unittestArgs": [ 3 | "-v", 4 | "-s", 5 | "./tests", 6 | "-p", 7 | "test_*.py" 8 | ], 9 | "python.testing.pytestEnabled": true, 10 | "python.testing.unittestEnabled": false, 11 | "python.linting.enabled": true, 12 | "python.linting.lintOnSave": true, 13 | "python.linting.mypyEnabled": true, 14 | "python.linting.flake8Enabled": true, 15 | "python.linting.banditEnabled": true, 16 | "python.linting.banditArgs": [ 17 | "--configfile", 18 | ".bandit.yaml" 19 | ], 20 | } -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | 2 | # Contributor Covenant Code of Conduct 3 | 4 | ## Our Pledge 5 | 6 | We as members, contributors, and leaders pledge to make participation in our 7 | community a harassment-free experience for everyone, regardless of age, body 8 | size, visible or invisible disability, ethnicity, sex characteristics, gender 9 | identity and expression, level of experience, education, socio-economic status, 10 | nationality, personal appearance, race, religion, or sexual identity 11 | and orientation. 12 | 13 | We pledge to act and interact in ways that contribute to an open, welcoming, 14 | diverse, inclusive, and healthy community. 15 | 16 | ## Our Standards 17 | 18 | Examples of behavior that contributes to a positive environment for our 19 | community include: 20 | 21 | * Demonstrating empathy and kindness toward other people 22 | * Being respectful of differing opinions, viewpoints, and experiences 23 | * Giving and gracefully accepting constructive feedback 24 | * Accepting responsibility and apologizing to those affected by our mistakes, 25 | and learning from the experience 26 | * Focusing on what is best not just for us as individuals, but for the 27 | overall community 28 | 29 | Examples of unacceptable behavior include: 30 | 31 | * The use of sexualized language or imagery, and sexual attention or 32 | advances of any kind 33 | * Trolling, insulting or derogatory comments, and personal or political attacks 34 | * Public or private harassment 35 | * Publishing others' private information, such as a physical or email 36 | address, without their explicit permission 37 | * Other conduct which could reasonably be considered inappropriate in a 38 | professional setting 39 | 40 | ## Enforcement Responsibilities 41 | 42 | Community leaders are responsible for clarifying and enforcing our standards of 43 | acceptable behavior and will take appropriate and fair corrective action in 44 | response to any behavior that they deem inappropriate, threatening, offensive, 45 | or harmful. 46 | 47 | Community leaders have the right and responsibility to remove, edit, or reject 48 | comments, commits, code, wiki edits, issues, and other contributions that are 49 | not aligned to this Code of Conduct, and will communicate reasons for moderation 50 | decisions when appropriate. 51 | 52 | ## Scope 53 | 54 | This Code of Conduct applies within all community spaces, and also applies when 55 | an individual is officially representing the community in public spaces. 56 | Examples of representing our community include using an official e-mail address, 57 | posting via an official social media account, or acting as an appointed 58 | representative at an online or offline event. 59 | 60 | ## Enforcement 61 | 62 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 63 | reported to the community leaders responsible for enforcement at coraline c/o 64 | idolhands dot com. 65 | 66 | All complaints will be reviewed and investigated promptly and fairly. 67 | 68 | All community leaders are obligated to respect the privacy and security of the 69 | reporter of any incident. 70 | 71 | ## Enforcement Guidelines 72 | 73 | Community leaders will follow these Community Impact Guidelines in determining 74 | the consequences for any action they deem in violation of this Code of Conduct: 75 | 76 | ### 1. Correction 77 | 78 | **Community Impact**: Use of inappropriate language or other behavior deemed 79 | unprofessional or unwelcome in the community. 80 | 81 | **Consequence**: A private, written warning from community leaders, providing 82 | clarity around the nature of the violation and an explanation of why the 83 | behavior was inappropriate. A public apology may be requested. 84 | 85 | ### 2. Warning 86 | 87 | **Community Impact**: A violation through a single incident or series 88 | of actions. 89 | 90 | **Consequence**: A warning with consequences for continued behavior. No 91 | interaction with the people involved, including unsolicited interaction with 92 | those enforcing the Code of Conduct, for a specified period of time. This 93 | includes avoiding interactions in community spaces as well as external channels 94 | like social media. Violating these terms may lead to a temporary or 95 | permanent ban. 96 | 97 | ### 3. Temporary Ban 98 | 99 | **Community Impact**: A serious violation of community standards, including 100 | sustained inappropriate behavior. 101 | 102 | **Consequence**: A temporary ban from any sort of interaction or public 103 | communication with the community for a specified period of time. No public or 104 | private interaction with the people involved, including unsolicited interaction 105 | with those enforcing the Code of Conduct, is allowed during this period. 106 | Violating these terms may lead to a permanent ban. 107 | 108 | ### 4. Permanent Ban 109 | 110 | **Community Impact**: Demonstrating a pattern of violation of community 111 | standards, including sustained inappropriate behavior, harassment of an 112 | individual, or aggression toward or disparagement of classes of individuals. 113 | 114 | **Consequence**: A permanent ban from any sort of public interaction within 115 | the community. 116 | 117 | ## Attribution 118 | 119 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 120 | version 2.0, available at 121 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 122 | 123 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 124 | enforcement ladder](https://github.com/mozilla/diversity). 125 | 126 | [homepage]: https://www.contributor-covenant.org 127 | 128 | For answers to common questions about this code of conduct, see the FAQ at 129 | https://www.contributor-covenant.org/faq. Translations are available at 130 | https://www.contributor-covenant.org/translations. 131 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # Makefile # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | # Determine this Makefile's path. Be sure to place the following line before 20 | # include directives, if any. 21 | THIS_FILE := $(lastword $(MAKEFILE_LIST)) 22 | 23 | 24 | venv ?= venv 25 | 26 | init upgrade: formulae := {openssl,readline,xz,redis} 27 | python upgrade: version ?= 3.13.3 28 | upgrade: requirements ?= requirements-to-freeze.txt 29 | delete-keys: pattern ?= tmp:* 30 | 31 | 32 | .PHONY: install 33 | install: init python 34 | 35 | 36 | .PHONY: init 37 | init: 38 | -xcode-select --install 39 | # command -v brew >/dev/null 2>&1 || \ 40 | # ruby -e "$$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" 41 | brew analytics regenerate-uuid 42 | brew analytics off 43 | -brew install $(formulae) 44 | -git clone https://github.com/pyenv/pyenv.git ~/.pyenv 45 | 46 | .PHONY: python 47 | python: 48 | cd ~/.pyenv/plugins/python-build/../.. && git pull 49 | CFLAGS="-I$(shell brew --prefix openssl)/include -I$(shell brew --prefix readline)/include -g -O2" \ 50 | LDFLAGS="-L$(shell brew --prefix openssl)/lib -L$(shell brew --prefix readline)/lib" \ 51 | pyenv install --skip-existing $(version) 52 | pyenv rehash 53 | @$(MAKE) --makefile=$(THIS_FILE) upgrade recursive=true requirements=requirements.txt 54 | 55 | .PHONY: upgrade 56 | upgrade: 57 | -brew update 58 | -brew upgrade $(formulae) 59 | brew cleanup 60 | ifneq ($(recursive),) 61 | rm -rf $(venv) 62 | ~/.pyenv/versions/$(version)/bin/python3 -m venv $(venv) 63 | endif 64 | source $(venv)/bin/activate && \ 65 | pip3 install --upgrade --no-cache-dir pip setuptools wheel && \ 66 | pip3 install --requirement $(requirements) --upgrade --no-cache-dir && \ 67 | pip3 freeze > requirements.txt 68 | git status 69 | git diff 70 | 71 | 72 | .PHONY: test 73 | test: 74 | $(eval $@_SOURCE_FILES := $(shell find . -name '*.py' -not -path './.git/*' -not -path './build/*' -not -path './dist/*' -not -path './pottery.egg-info/*' -not -path './venv/*')) 75 | source $(venv)/bin/activate && \ 76 | pytest --verbose --cov-config=.coveragerc --cov=pottery --cov=tests && \ 77 | echo Running static type checks && \ 78 | mypy && \ 79 | echo Running Flake8 on $($@_SOURCE_FILES) && \ 80 | flake8 $($@_SOURCE_FILES) --count --max-complexity=10 --statistics && \ 81 | echo Running isort on $($@_SOURCE_FILES) && \ 82 | isort $($@_SOURCE_FILES) --check-only --diff && \ 83 | bandit --recursive pottery && \ 84 | safety scan 85 | 86 | 87 | .PHONY: release 88 | release: 89 | rm -f dist/* 90 | source $(venv)/bin/activate && \ 91 | python3 setup.py sdist && \ 92 | python3 setup.py bdist_wheel && \ 93 | twine upload dist/* 94 | 95 | 96 | # Usage: 97 | # make pattern="tmp:*" delete-keys 98 | .PHONY: delete-keys 99 | delete-keys: 100 | redis-cli --scan --pattern "$(pattern)" | xargs redis-cli del 101 | 102 | .PHONY: clean 103 | clean: 104 | rm -rf {$(venv),pottery/__pycache__,tests/__pycache__,.coverage,.mypy_cache,pottery.egg-info,build,dist} 105 | 106 | 107 | .PHONY: lines-of-code 108 | lines-of-code: 109 | find . -name '*.py' -not -path "./.git/*" -not -path "./venv/*" -not -path "./build/*" | xargs wc -l 110 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # mypy.ini # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | 20 | [mypy] 21 | files = *.py,pottery/,tests/ 22 | disallow_incomplete_defs = True 23 | disallow_untyped_calls = True 24 | disallow_untyped_decorators = True 25 | disallow_any_unimported = True 26 | disallow_subclassing_any = True 27 | warn_unreachable = True 28 | warn_return_any = True 29 | warn_redundant_casts = True 30 | 31 | 32 | 33 | [mypy-mmh3] 34 | ignore_missing_imports = True 35 | 36 | [mypy-py] 37 | ignore_missing_imports = True 38 | 39 | [mypy-setuptools] 40 | ignore_missing_imports = True 41 | -------------------------------------------------------------------------------- /pottery/__init__.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # __init__.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | from typing import Tuple 20 | 21 | # TODO: When we drop support for Python 3.7, change the following import to: 22 | # from typing import Final 23 | from typing_extensions import Final 24 | 25 | from .counter import RedisCounter 26 | from .deque import RedisDeque 27 | from .dict import RedisDict 28 | from .list import RedisList 29 | from .queue import RedisSimpleQueue 30 | from .set import RedisSet 31 | 32 | 33 | from .monkey import PotteryEncoder # isort: skip 34 | 35 | from .exceptions import PotteryError # isort:skip 36 | from .exceptions import KeyExistsError # isort:skip 37 | from .exceptions import RandomKeyError # isort:skip 38 | from .exceptions import QueueEmptyError # isort:skip 39 | from .exceptions import PrimitiveError # isort:skip 40 | from .exceptions import QuorumIsImpossible # isort:skip 41 | from .exceptions import QuorumNotAchieved # isort:skip 42 | from .exceptions import TooManyExtensions # isort:skip 43 | from .exceptions import ExtendUnlockedLock # isort:skip 44 | from .exceptions import ReleaseUnlockedLock # isort:skip 45 | from .exceptions import PotteryWarning # isort:skip 46 | from .exceptions import InefficientAccessWarning # isort:skip 47 | 48 | from .cache import CachedOrderedDict # isort:skip 49 | from .cache import redis_cache # isort:skip 50 | from .aionextid import AIONextID # isort:skip 51 | from .nextid import NextID # isort:skip 52 | from .aioredlock import AIORedlock # isort:skip 53 | from .redlock import Redlock # isort:skip 54 | from .redlock import synchronize # isort:skip 55 | from .timer import ContextTimer # isort:skip 56 | 57 | 58 | from .bloom import BloomFilter # isort:skip 59 | from .hyper import HyperLogLog # isort:skip 60 | 61 | 62 | __all__: Final[Tuple[str, ...]] = ( 63 | 'PotteryEncoder', 64 | 65 | 'PotteryError', 66 | 'KeyExistsError', 67 | 'RandomKeyError', 68 | 'QueueEmptyError', 69 | 'PrimitiveError', 70 | 'QuorumIsImpossible', 71 | 'QuorumNotAchieved', 72 | 'TooManyExtensions', 73 | 'ExtendUnlockedLock', 74 | 'ReleaseUnlockedLock', 75 | 'PotteryWarning', 76 | 'InefficientAccessWarning', 77 | 78 | 'CachedOrderedDict', 79 | 'redis_cache', 80 | 'AIONextID', 81 | 'NextID', 82 | 'AIORedlock', 83 | 'Redlock', 84 | 'synchronize', 85 | 'ContextTimer', 86 | 87 | 'RedisCounter', 88 | 'RedisDeque', 89 | 'RedisDict', 90 | 'RedisList', 91 | 'RedisSimpleQueue', 92 | 'RedisSet', 93 | 'BloomFilter', 94 | 'HyperLogLog', 95 | ) 96 | -------------------------------------------------------------------------------- /pottery/aionextid.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # aionextid.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 'Asynchronous distributed Redis-powered monotonically increasing ID generator.' 18 | 19 | 20 | # TODO: Remove the following import after deferred evaluation of annotations 21 | # because the default. 22 | # 1. https://docs.python.org/3/whatsnew/3.7.html#whatsnew37-pep563 23 | # 2. https://www.python.org/dev/peps/pep-0563/ 24 | # 3. https://www.python.org/dev/peps/pep-0649/ 25 | from __future__ import annotations 26 | 27 | import asyncio 28 | import contextlib 29 | from typing import Any 30 | from typing import ClassVar 31 | from typing import Iterable 32 | 33 | from redis import RedisError 34 | from redis.asyncio import Redis as AIORedis 35 | 36 | from .base import AIOPrimitive 37 | from .base import logger 38 | from .exceptions import QuorumNotAchieved 39 | from .nextid import NextID 40 | from .nextid import Scripts 41 | 42 | 43 | class AIONextID(Scripts, AIOPrimitive): 44 | 'Async distributed Redis-powered monotonically increasing ID generator.' 45 | 46 | __slots__ = ('num_tries',) 47 | 48 | _KEY_PREFIX: ClassVar[str] = NextID._KEY_PREFIX 49 | 50 | def __init__(self, 51 | *, 52 | key: str = 'current', 53 | masters: Iterable[AIORedis] = frozenset(), 54 | num_tries: int = NextID._NUM_TRIES, 55 | ) -> None: 56 | 'Initialize an AIONextID ID generator.' 57 | super().__init__(key=key, masters=masters) 58 | self.num_tries = num_tries 59 | 60 | def __aiter__(self) -> AIONextID: 61 | return self # pragma: no cover 62 | 63 | async def __anext__(self) -> int: 64 | for _ in range(self.num_tries): 65 | with contextlib.suppress(QuorumNotAchieved): 66 | next_id = await self.__get_current_ids() + 1 67 | await self.__set_current_ids(next_id) 68 | return next_id 69 | raise QuorumNotAchieved(self.key, self.masters) 70 | 71 | async def __get_current_id(self, master: AIORedis) -> Any | None: 72 | current_id = await master.get(self.key) 73 | return current_id 74 | 75 | async def __set_current_id(self, master: AIORedis, value: int) -> bool: 76 | current_id: int | None = await self._set_id_script( # type: ignore 77 | keys=(self.key,), 78 | args=(value,), 79 | client=master, 80 | ) 81 | return current_id == value 82 | 83 | async def __reset_current_id(self, master: AIORedis) -> None: 84 | await master.delete(self.key) 85 | 86 | async def __get_current_ids(self) -> int: 87 | current_ids, redis_errors = [], [] 88 | coros = [self.__get_current_id(master) for master in self.masters] # type: ignore 89 | for coro in asyncio.as_completed(coros): # type: ignore 90 | try: 91 | current_id = int(await coro or b'0') 92 | except RedisError as error: 93 | redis_errors.append(error) 94 | logger.exception( 95 | '%s.__get_current_ids() caught %s', 96 | self.__class__.__qualname__, 97 | error.__class__.__qualname__, 98 | ) 99 | else: 100 | current_ids.append(current_id) 101 | if len(current_ids) > len(self.masters) // 2: 102 | return max(current_ids) 103 | raise QuorumNotAchieved( 104 | self.key, 105 | self.masters, 106 | redis_errors=redis_errors, 107 | ) 108 | 109 | async def __set_current_ids(self, value: int) -> None: 110 | num_masters_set, redis_errors = 0, [] 111 | coros = [self.__set_current_id(master, value) for master in self.masters] # type: ignore 112 | for coro in asyncio.as_completed(coros): 113 | try: 114 | num_masters_set += await coro 115 | except RedisError as error: 116 | redis_errors.append(error) 117 | logger.exception( 118 | '%s.__set_current_ids() caught %s', 119 | self.__class__.__qualname__, 120 | error.__class__.__qualname__, 121 | ) 122 | if num_masters_set > len(self.masters) // 2: 123 | return 124 | raise QuorumNotAchieved( 125 | self.key, 126 | self.masters, 127 | redis_errors=redis_errors, 128 | ) 129 | 130 | async def reset(self) -> None: 131 | num_masters_reset, redis_errors = 0, [] 132 | coros = [self.__reset_current_id(master) for master in self.masters] # type: ignore 133 | for coro in asyncio.as_completed(coros): 134 | try: 135 | await coro 136 | except RedisError as error: 137 | redis_errors.append(error) 138 | logger.exception( 139 | '%s.reset() caught %s', 140 | self.__class__.__qualname__, 141 | error.__class__.__qualname__, 142 | ) 143 | else: 144 | num_masters_reset += 1 145 | if num_masters_reset > len(self.masters) // 2: 146 | return 147 | raise QuorumNotAchieved( 148 | self.key, 149 | self.masters, 150 | redis_errors=redis_errors, 151 | ) 152 | 153 | def __repr__(self) -> str: 154 | return f'<{self.__class__.__qualname__} key={self.key}>' 155 | -------------------------------------------------------------------------------- /pottery/annotations.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # annotations.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | from typing import Any 20 | from typing import Callable 21 | from typing import Dict 22 | from typing import List 23 | from typing import TypeVar 24 | from typing import Union 25 | 26 | 27 | # A function that receives *args and **kwargs, and returns anything. Useful 28 | # for annotating decorators. 29 | F = TypeVar('F', bound=Callable[..., Any]) 30 | 31 | 32 | JSONTypes = Union[None, bool, int, float, str, List[Any], Dict[str, Any]] 33 | RedisValues = Union[bytes, str, float, int] 34 | -------------------------------------------------------------------------------- /pottery/counter.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # counter.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | # TODO: Remove the following import after deferred evaluation of annotations 20 | # because the default. 21 | # 1. https://docs.python.org/3/whatsnew/3.7.html#whatsnew37-pep563 22 | # 2. https://www.python.org/dev/peps/pep-0563/ 23 | # 3. https://www.python.org/dev/peps/pep-0649/ 24 | from __future__ import annotations 25 | 26 | import collections 27 | import contextlib 28 | import itertools 29 | from typing import Callable 30 | from typing import Iterable 31 | from typing import List 32 | from typing import Tuple 33 | from typing import Union 34 | from typing import cast 35 | 36 | from redis.client import Pipeline 37 | from typing_extensions import Counter 38 | 39 | from .annotations import JSONTypes 40 | from .dict import RedisDict 41 | 42 | 43 | InitIter = Iterable[JSONTypes] 44 | InitArg = Union[InitIter, Counter] 45 | 46 | 47 | class RedisCounter(RedisDict, collections.Counter): 48 | 'Redis-backed container compatible with collections.Counter.' 49 | 50 | # Method overrides: 51 | 52 | def _populate(self, # type: ignore 53 | pipeline: Pipeline, 54 | arg: InitArg = tuple(), 55 | *, 56 | sign: int = +1, 57 | **kwargs: int, 58 | ) -> None: 59 | dict_ = {} 60 | if isinstance(arg, collections.abc.Mapping): 61 | items = arg.items() 62 | for key, value in items: 63 | dict_[key] = sign * value 64 | else: 65 | for key in arg: 66 | value = dict_.get(key, self[key]) 67 | dict_[key] = value + sign 68 | 69 | for key, value in kwargs.items(): 70 | if dict_.get(key, 0) == 0: 71 | original = self[key] 72 | else: # pragma: no cover 73 | original = dict_[key] 74 | dict_[key] = original + sign * value 75 | 76 | dict_ = {key: self[key] + value for key, value in dict_.items()} 77 | encoded_dict = self._encode_dict(dict_) 78 | if encoded_dict: 79 | pipeline.multi() # Available since Redis 1.2.0 80 | # Available since Redis 2.0.0: 81 | pipeline.hset(self.key, mapping=encoded_dict) # type: ignore 82 | 83 | # Preserve the Open-Closed Principle with name mangling. 84 | # https://youtu.be/miGolgp9xq8?t=2086 85 | # https://stackoverflow.com/a/38534939 86 | __populate = _populate 87 | 88 | def update(self, arg: InitArg = tuple(), **kwargs: int) -> None: # type: ignore 89 | 'Like dict.update() but add counts instead of replacing them. O(n)' 90 | with self._watch(arg) as pipeline: 91 | self.__populate(pipeline, arg, sign=+1, **kwargs) 92 | 93 | def subtract(self, arg: InitArg = tuple(), **kwargs: int) -> None: # type: ignore 94 | 'Like dict.update() but subtracts counts instead of replacing them. O(n)' 95 | with self._watch(arg) as pipeline: 96 | self.__populate(pipeline, arg, sign=-1, **kwargs) 97 | 98 | def __getitem__(self, key: JSONTypes) -> int: 99 | 'c.__getitem__(key) <==> c.get(key, 0). O(1)' 100 | try: 101 | value = cast(int, super().__getitem__(key)) 102 | except KeyError: 103 | value = super().__missing__(key) # type: ignore 104 | return value 105 | 106 | def __delitem__(self, key: JSONTypes) -> None: # type: ignore 107 | 'c.__delitem__(key) <==> del c[key]. O(1)' 108 | with contextlib.suppress(KeyError): 109 | super().__delitem__(key) 110 | 111 | def __repr__(self) -> str: 112 | 'Return the string representation of the RedisCounter. O(n)' 113 | items = self.__most_common() 114 | pairs = (f"'{key}': {value}" for key, value in items) 115 | repr_ = ', '.join(pairs) 116 | return self.__class__.__qualname__ + '{' + repr_ + '}' 117 | 118 | def to_counter(self) -> Counter[JSONTypes]: 119 | 'Convert a RedisCounter into a plain Python collections.Counter.' 120 | return collections.Counter(self) 121 | 122 | __to_counter = to_counter 123 | 124 | def __math_op(self, 125 | other: Counter[JSONTypes], 126 | *, 127 | method: Callable[[Counter[JSONTypes], Counter[JSONTypes]], Counter[JSONTypes]], 128 | ) -> Counter[JSONTypes]: 129 | with self._watch(other): 130 | counter = self.__to_counter() 131 | return method(counter, other) 132 | 133 | def __add__(self, other: Counter[JSONTypes]) -> Counter[JSONTypes]: # type: ignore 134 | "Return the addition our counts to other's counts, but keep only counts > 0. O(n)" 135 | return self.__math_op(other, method=collections.Counter.__add__) 136 | 137 | def __sub__(self, other: Counter[JSONTypes]) -> Counter[JSONTypes]: 138 | "Return the subtraction other's counts from our counts, but keep only counts > 0. O(n)" 139 | return self.__math_op(other, method=collections.Counter.__sub__) 140 | 141 | def __or__(self, other: Counter[JSONTypes]) -> Counter[JSONTypes]: # type: ignore 142 | "Return the max of our counts vs. other's counts (union), but keep only counts > 0. O(n)" 143 | return self.__math_op(other, method=collections.Counter.__or__) 144 | 145 | def __and__(self, other: Counter[JSONTypes]) -> Counter[JSONTypes]: 146 | "Return the min of our counts vs. other's counts (intersection) but keep only counts > 0. O(n)" 147 | return self.__math_op(other, method=collections.Counter.__and__) 148 | 149 | def __unary_op(self, 150 | *, 151 | test_func: Callable[[int], bool], 152 | modifier_func: Callable[[int], int], 153 | ) -> Counter[JSONTypes]: 154 | with self._watch(): 155 | counter: Counter[JSONTypes] = collections.Counter() 156 | for key, value in self.__to_counter().items(): 157 | if test_func(value): 158 | counter[key] = modifier_func(value) 159 | return counter 160 | 161 | def __pos__(self) -> Counter[JSONTypes]: 162 | 'Return our counts > 0. O(n)' 163 | return self.__unary_op( 164 | test_func=lambda x: x > 0, 165 | modifier_func=lambda x: x, 166 | ) 167 | 168 | def __neg__(self) -> Counter[JSONTypes]: 169 | 'Return the absolute value of our counts < 0. O(n)' 170 | return self.__unary_op( 171 | test_func=lambda x: x < 0, 172 | modifier_func=lambda x: -x, 173 | ) 174 | 175 | def __imath_op(self, 176 | other: Counter[JSONTypes], 177 | *, 178 | sign: int = +1, 179 | ) -> RedisCounter: 180 | with self._watch(other) as pipeline: 181 | try: 182 | other_items = cast(RedisCounter, other).to_counter().items() 183 | except AttributeError: 184 | other_items = other.items() 185 | to_set = {k: self[k] + sign * v for k, v in other_items} 186 | to_del = {k for k, v in to_set.items() if v <= 0} 187 | to_del.update( 188 | k for k, v in self.items() if k not in to_set and v <= 0 189 | ) 190 | encoded_to_set = { 191 | self._encode(k): self._encode(v) for k, v in to_set.items() if v 192 | } 193 | encoded_to_del = {self._encode(k) for k in to_del} 194 | if encoded_to_set or encoded_to_del: 195 | pipeline.multi() # Available since Redis 1.2.0 196 | if encoded_to_set: 197 | # Available since Redis 2.0.0: 198 | pipeline.hset(self.key, mapping=encoded_to_set) # type: ignore 199 | if encoded_to_del: 200 | pipeline.hdel(self.key, *encoded_to_del) # Available since Redis 2.0.0 201 | return self 202 | 203 | def __iadd__(self, other: Counter[JSONTypes]) -> Counter[JSONTypes]: # type: ignore 204 | 'Same as __add__(), but in-place. O(n)' 205 | return self.__imath_op(other, sign=+1) 206 | 207 | def __isub__(self, other: Counter[JSONTypes]) -> Counter[JSONTypes]: # type: ignore 208 | 'Same as __sub__(), but in-place. O(n)' 209 | return self.__imath_op(other, sign=-1) 210 | 211 | def __iset_op(self, 212 | other: Counter[JSONTypes], 213 | *, 214 | method: Callable[[int, int], bool], 215 | ) -> RedisCounter: 216 | with self._watch(other) as pipeline: 217 | self_counter = self.__to_counter() 218 | try: 219 | other_counter = cast(RedisCounter, other).to_counter() 220 | except AttributeError: 221 | other_counter = other 222 | to_set, to_del = {}, set() 223 | for k in itertools.chain(self_counter, other_counter): 224 | if method(self_counter[k], other_counter[k]): 225 | to_set[k] = self_counter[k] 226 | else: 227 | to_set[k] = other_counter[k] 228 | if to_set[k] <= 0: 229 | del to_set[k] 230 | to_del.add(k) 231 | if to_set or to_del: 232 | pipeline.multi() # Available since Redis 1.2.0 233 | if to_set: 234 | encoded_to_set = { 235 | self._encode(k): self._encode(v) for k, v in to_set.items() 236 | } 237 | # Available since Redis 2.0.0: 238 | pipeline.hset(self.key, mapping=encoded_to_set) # type: ignore 239 | if to_del: 240 | encoded_to_del = {self._encode(k) for k in to_del} 241 | pipeline.hdel(self.key, *encoded_to_del) # Available since Redis 2.0.0 242 | return self 243 | 244 | def __ior__(self, other: Counter[JSONTypes]) -> Counter[JSONTypes]: # type: ignore 245 | 'Same as __or__(), but in-place. O(n)' 246 | return self.__iset_op(other, method=int.__gt__) 247 | 248 | def __iand__(self, other: Counter[JSONTypes]) -> Counter[JSONTypes]: # type: ignore 249 | 'Same as __and__(), but in-place. O(n)' 250 | return self.__iset_op(other, method=int.__lt__) 251 | 252 | def most_common(self, 253 | n: int | None = None, 254 | ) -> List[Tuple[JSONTypes, int]]: 255 | counter = self.__to_counter() 256 | return counter.most_common(n=n) 257 | 258 | __most_common = most_common 259 | -------------------------------------------------------------------------------- /pottery/deque.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # deque.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | # TODO: When we drop support for Python 3.9, remove the following import. We 20 | # only need it for X | Y union type annotations as of 2022-01-29. 21 | from __future__ import annotations 22 | 23 | import collections 24 | import warnings 25 | from typing import Iterable 26 | from typing import Tuple 27 | from typing import cast 28 | 29 | from redis import Redis 30 | from redis.client import Pipeline 31 | 32 | from .annotations import JSONTypes 33 | from .exceptions import InefficientAccessWarning 34 | from .list import RedisList 35 | 36 | 37 | class RedisDeque(RedisList, collections.deque): # type: ignore 38 | 'Redis-backed container compatible with collections.deque.' 39 | 40 | # Overrides: 41 | 42 | _ALLOWED_TO_EQUAL = collections.deque 43 | 44 | def __init__(self, 45 | iterable: Iterable[JSONTypes] = tuple(), 46 | maxlen: int | None = None, 47 | *, 48 | redis: Redis | None = None, 49 | key: str = '', 50 | ) -> None: 51 | 'Initialize the RedisDeque. O(n)' 52 | if maxlen is not None and not isinstance(maxlen, int): 53 | raise TypeError('an integer is required') 54 | self._maxlen = maxlen 55 | super().__init__(iterable, redis=redis, key=key) 56 | if not iterable and self.maxlen is not None and len(self) > self.maxlen: 57 | raise IndexError( 58 | f'persistent {self.__class__.__qualname__} beyond its maximum size' 59 | ) 60 | 61 | def _populate(self, 62 | pipeline: Pipeline, 63 | iterable: Iterable[JSONTypes] = tuple(), 64 | ) -> None: 65 | if self.maxlen is not None: 66 | if self.maxlen: 67 | iterable = tuple(iterable)[-self.maxlen:] 68 | else: 69 | # self.maxlen == 0. Populate the RedisDeque with an empty 70 | # iterable. 71 | iterable = tuple() 72 | super()._populate(pipeline, iterable) 73 | 74 | @property 75 | def maxlen(self) -> int | None: 76 | return self._maxlen 77 | 78 | @maxlen.setter 79 | def maxlen(self, value: int) -> None: 80 | raise AttributeError( 81 | f"attribute 'maxlen' of '{self.__class__.__qualname__}' objects is not " 82 | 'writable' 83 | ) 84 | 85 | def insert(self, index: int, value: JSONTypes) -> None: 86 | 'Insert an element into the RedisDeque before the given index. O(n)' 87 | with self._watch() as pipeline: 88 | current_length = cast(int, pipeline.llen(self.key)) # Available since Redis 1.0.0 89 | if self.maxlen is not None and current_length >= self.maxlen: 90 | raise IndexError( 91 | f'{self.__class__.__qualname__} already at its maximum size' 92 | ) 93 | super()._insert(index, value, pipeline=pipeline) 94 | 95 | def append(self, value: JSONTypes) -> None: 96 | 'Add an element to the right side of the RedisDeque. O(1)' 97 | self.__extend((value,), right=True) 98 | 99 | def appendleft(self, value: JSONTypes) -> None: 100 | 'Add an element to the left side of the RedisDeque. O(1)' 101 | self.__extend((value,), right=False) 102 | 103 | def extend(self, values: Iterable[JSONTypes]) -> None: 104 | 'Extend the RedisDeque by appending elements from the iterable. O(1)' 105 | self.__extend(values, right=True) 106 | 107 | def extendleft(self, values: Iterable[JSONTypes]) -> None: 108 | '''Extend the RedisDeque by prepending elements from the iterable. O(1) 109 | 110 | Note the order in which the elements are prepended from the iterable: 111 | 112 | >>> d = RedisDeque() 113 | >>> d.extendleft('abc') 114 | >>> d 115 | RedisDeque(['c', 'b', 'a']) 116 | ''' 117 | self.__extend(values, right=False) 118 | 119 | def __extend(self, 120 | values: Iterable[JSONTypes], 121 | *, 122 | right: bool = True, 123 | ) -> None: 124 | with self._watch(values) as pipeline: 125 | push_method_name = 'rpush' if right else 'lpush' 126 | encoded_values = [self._encode(value) for value in values] 127 | len_ = cast(int, pipeline.llen(self.key)) + len(encoded_values) # Available since Redis 1.0.0 128 | trim_indices: Tuple[int, int] | Tuple = tuple() 129 | if self.maxlen is not None and len_ >= self.maxlen: 130 | trim_indices = (len_-self.maxlen, len_) if right else (0, self.maxlen-1) 131 | 132 | pipeline.multi() # Available since Redis 1.2.0 133 | push_method = getattr(pipeline, push_method_name) 134 | push_method(self.key, *encoded_values) 135 | if trim_indices: 136 | pipeline.ltrim(self.key, *trim_indices) # Available since Redis 1.0.0 137 | 138 | def pop(self) -> JSONTypes: # type: ignore 139 | return super().pop() 140 | 141 | def popleft(self) -> JSONTypes: 142 | return super().pop(0) 143 | 144 | def rotate(self, n: int = 1) -> None: 145 | '''Rotate the RedisDeque n steps to the right (default n=1). O(n) 146 | 147 | If n is negative, rotates left. 148 | ''' 149 | if not isinstance(n, int): 150 | raise TypeError( 151 | f"'{n.__class__.__qualname__}' object cannot be interpreted " 152 | 'as an integer' 153 | ) 154 | if n == 0: 155 | # Rotating 0 steps is a no-op. 156 | return 157 | 158 | with self._watch() as pipeline: 159 | if not self: 160 | # Rotating an empty RedisDeque is a no-op. 161 | return 162 | 163 | push_method_name = 'lpush' if n > 0 else 'rpush' # Available since Redis 1.0.0 164 | values = self[-n:][::-1] if n > 0 else self[:-n] 165 | encoded_values = (self._encode(value) for value in values) 166 | trim_indices = (0, len(self)-1) if n > 0 else (-n, len(self)-1-n) 167 | 168 | pipeline.multi() # Available since Redis 1.2.0 169 | push_method = getattr(pipeline, push_method_name) 170 | push_method(self.key, *encoded_values) 171 | pipeline.ltrim(self.key, *trim_indices) # Available since Redis 1.0.0 172 | 173 | # Methods required for Raj's sanity: 174 | 175 | def __bool__(self) -> bool: 176 | 'Whether the RedisDeque contains any elements. O(1)' 177 | return bool(len(self)) 178 | 179 | def __repr__(self) -> str: 180 | 'Return the string representation of the RedisDeque. O(n)' 181 | warnings.warn( 182 | cast(str, InefficientAccessWarning.__doc__), 183 | InefficientAccessWarning, 184 | ) 185 | encoded_values = self.redis.lrange(self.key, 0, -1) # Available since Redis 1.0.0 186 | values = [self._decode(value) for value in encoded_values] 187 | repr = self.__class__.__qualname__ + '(' + str(values) 188 | if self.maxlen is not None: 189 | repr += f', maxlen={self.maxlen}' 190 | repr += ')' 191 | return repr 192 | -------------------------------------------------------------------------------- /pottery/dict.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # dict.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | # TODO: When we drop support for Python 3.9, remove the following import. We 20 | # only need it for X | Y union type annotations as of 2022-01-29. 21 | from __future__ import annotations 22 | 23 | import collections.abc 24 | import itertools 25 | import warnings 26 | from typing import Any 27 | from typing import Dict 28 | from typing import Generator 29 | from typing import Iterable 30 | from typing import Mapping 31 | from typing import Tuple 32 | from typing import Union 33 | from typing import cast 34 | 35 | from redis import Redis 36 | from redis.client import Pipeline 37 | 38 | from .annotations import JSONTypes 39 | from .base import Container 40 | from .base import Iterable_ 41 | from .exceptions import InefficientAccessWarning 42 | from .exceptions import KeyExistsError 43 | 44 | 45 | InitMap = Mapping[JSONTypes, JSONTypes] 46 | InitItem = Tuple[JSONTypes, JSONTypes] 47 | InitIter = Iterable[InitItem] 48 | InitArg = Union[InitMap, InitIter] 49 | 50 | 51 | class RedisDict(Container, Iterable_, collections.abc.MutableMapping): 52 | 'Redis-backed container compatible with Python dicts.' 53 | 54 | def __init__(self, 55 | arg: InitArg = tuple(), 56 | *, 57 | redis: Redis | None = None, 58 | key: str = '', 59 | **kwargs: JSONTypes, 60 | ) -> None: 61 | 'Initialize the RedisDict. O(n)' 62 | super().__init__(redis=redis, key=key) 63 | if arg or kwargs: 64 | with self._watch(arg) as pipeline: 65 | if pipeline.exists(self.key): # Available since Redis 1.0.0 66 | raise KeyExistsError(self.redis, self.key) 67 | self._populate(pipeline, arg, **kwargs) 68 | 69 | def _populate(self, 70 | pipeline: Pipeline, 71 | arg: InitArg = tuple(), 72 | **kwargs: JSONTypes, 73 | ) -> None: 74 | if isinstance(arg, collections.abc.Mapping): 75 | arg = arg.items() 76 | items = itertools.chain(arg, kwargs.items()) 77 | dict_ = dict(items) 78 | encoded_dict = self.__encode_dict(dict_) 79 | if encoded_dict: 80 | if len(encoded_dict) > 1: 81 | warnings.warn( 82 | cast(str, InefficientAccessWarning.__doc__), 83 | InefficientAccessWarning, 84 | ) 85 | pipeline.multi() # Available since Redis 1.2.0 86 | # Available since Redis 2.0.0: 87 | pipeline.hset(self.key, mapping=encoded_dict) # type: ignore 88 | 89 | # Preserve the Open-Closed Principle with name mangling. 90 | # https://youtu.be/miGolgp9xq8?t=2086 91 | # https://stackoverflow.com/a/38534939 92 | __populate = _populate 93 | 94 | def _encode_dict(self, dict_: Mapping[JSONTypes, JSONTypes]) -> Dict[str, str]: 95 | encoded_dict = { 96 | self._encode(key): self._encode(value) 97 | for key, value in dict_.items() 98 | } 99 | return encoded_dict 100 | 101 | __encode_dict = _encode_dict 102 | 103 | # Methods required by collections.abc.MutableMapping: 104 | 105 | def __getitem__(self, key: JSONTypes) -> JSONTypes: 106 | 'd.__getitem__(key) <==> d[key]. O(1)' 107 | encoded_key = self._encode(key) 108 | encoded_value = self.redis.hget(self.key, encoded_key) # Available since Redis 2.0.0 109 | if encoded_value is None: 110 | raise KeyError(key) 111 | value = self._decode(encoded_value) 112 | return value 113 | 114 | def __setitem__(self, key: JSONTypes, value: JSONTypes) -> None: 115 | 'd.__setitem__(key, value) <==> d[key] = value. O(1)' 116 | encoded_key = self._encode(key) 117 | encoded_value = self._encode(value) 118 | self.redis.hset(self.key, encoded_key, encoded_value) # Available since Redis 2.0.0 119 | 120 | def __delitem__(self, key: JSONTypes) -> None: 121 | 'd.__delitem__(key) <==> del d[key]. O(1)' 122 | encoded_key = self._encode(key) 123 | if not self.redis.hdel(self.key, encoded_key): # Available since Redis 2.0.0 124 | raise KeyError(key) 125 | 126 | def __iter__(self) -> Generator[JSONTypes, None, None]: 127 | warnings.warn( 128 | cast(str, InefficientAccessWarning.__doc__), 129 | InefficientAccessWarning, 130 | ) 131 | encoded_items = self.redis.hscan_iter(self.key) # Available since Redis 2.8.0 132 | keys = (self._decode(key) for key, _ in encoded_items) 133 | yield from keys 134 | 135 | def __len__(self) -> int: 136 | 'Return the number of items in the RedisDict. O(1)' 137 | return self.redis.hlen(self.key) # Available since Redis 2.0.0 138 | 139 | # Methods required for Raj's sanity: 140 | 141 | def __repr__(self) -> str: 142 | 'Return the string representation of the RedisDict. O(n)' 143 | return f'{self.__class__.__qualname__}{self.__to_dict()}' 144 | 145 | # Method overrides: 146 | 147 | # From collections.abc.MutableMapping: 148 | def update(self, arg: InitArg = tuple(), **kwargs: JSONTypes) -> None: # type: ignore 149 | with self._watch(arg) as pipeline: 150 | self.__populate(pipeline, arg, **kwargs) 151 | 152 | # From collections.abc.Mapping: 153 | def __contains__(self, key: Any) -> bool: 154 | 'd.__contains__(key) <==> key in d. O(1)' 155 | try: 156 | encoded_key = self._encode(key) 157 | except TypeError: 158 | return False 159 | return self.redis.hexists(self.key, encoded_key) # Available since Redis 2.0.0 160 | 161 | def to_dict(self) -> Dict[JSONTypes, JSONTypes]: 162 | 'Convert a RedisDict into a plain Python dict.' 163 | encoded_items = self.redis.hgetall(self.key).items() # Available since Redis 2.0.0 164 | if encoded_items: 165 | warnings.warn( 166 | cast(str, InefficientAccessWarning.__doc__), 167 | InefficientAccessWarning, 168 | ) 169 | dict_ = { 170 | self._decode(encoded_key): self._decode(encoded_value) 171 | for encoded_key, encoded_value in encoded_items 172 | } 173 | return dict_ 174 | 175 | __to_dict = to_dict 176 | -------------------------------------------------------------------------------- /pottery/exceptions.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # exceptions.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | # TODO: When we drop support for Python 3.9, remove the following import. We 20 | # only need it for X | Y union type annotations as of 2022-01-29. 21 | from __future__ import annotations 22 | 23 | from dataclasses import dataclass 24 | from queue import Empty 25 | from typing import Iterable 26 | 27 | from redis import Redis 28 | from redis import RedisError 29 | 30 | 31 | @dataclass 32 | class PotteryError(Exception): 33 | 'Base exception class for Pottery containers.' 34 | 35 | redis: Redis 36 | key: str | None = None 37 | 38 | class KeyExistsError(PotteryError): 39 | 'Initializing a container on a Redis key that already exists.' 40 | 41 | class RandomKeyError(PotteryError, RuntimeError): 42 | "Can't create a random Redis key; all of our attempts already exist." 43 | 44 | class QueueEmptyError(PotteryError, Empty): 45 | 'Non-blocking .get() or .get_nowait() called on RedisQueue which is empty.' 46 | 47 | 48 | @dataclass 49 | class PrimitiveError(Exception): 50 | 'Base exception class for distributed primitives.' 51 | 52 | key: str 53 | masters: Iterable[Redis] 54 | redis_errors: Iterable[RedisError] = tuple() 55 | 56 | class QuorumNotAchieved(PrimitiveError, RuntimeError): 57 | 'Consensus-based algorithm could not achieve quorum.' 58 | 59 | class TooManyExtensions(PrimitiveError, RuntimeError): 60 | 'Redlock has been extended too many times.' 61 | 62 | class ExtendUnlockedLock(PrimitiveError, RuntimeError): 63 | 'Attempting to extend an unlocked Redlock.' 64 | 65 | class ReleaseUnlockedLock(PrimitiveError, RuntimeError): 66 | 'Attempting to release an unlocked Redlock.' 67 | 68 | class QuorumIsImpossible(PrimitiveError, RuntimeError): 69 | 'Too many Redis masters threw RedisErrors; quorum can not be achieved.' 70 | 71 | 72 | class PotteryWarning(Warning): 73 | 'Base warning class for Pottery containers.' 74 | 75 | class InefficientAccessWarning(PotteryWarning): 76 | 'Doing an O(n) Redis operation.' 77 | -------------------------------------------------------------------------------- /pottery/executor.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # executor.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | # TODO: When we drop support for Python 3.9, remove the following import. We 20 | # only need it for X | Y union type annotations as of 2022-01-29. 21 | from __future__ import annotations 22 | 23 | import concurrent.futures 24 | from types import TracebackType 25 | from typing import Type 26 | 27 | from typing_extensions import Literal 28 | 29 | 30 | class BailOutExecutor(concurrent.futures.ThreadPoolExecutor): 31 | '''ThreadPoolExecutor subclass that doesn't wait for futures on .__exit__(). 32 | 33 | The beating heart of all consensus based distributed algorithms is to 34 | scatter a computation across multiple nodes, then to gather their results, 35 | then to evaluate whether quorum is achieved. 36 | 37 | In some cases, quorum requires gathering all of the nodes' results (e.g., 38 | interrogating all nodes for a maximum value for a variable). 39 | 40 | But in other cases, quorum requires gathering only n // 2 + 1 nodes' 41 | results (e.g., figuring out if > 50% of nodes believe that I'm the owner of 42 | a lock). 43 | 44 | In the latter case, the desired behavior is for the executor to bail out 45 | early returning control to the main thread as soon as quorum is achieved, 46 | while still allowing pending in-flight futures to complete in backgound 47 | threads. Python's ThreadPoolExecutor's .__exit__() method waits for 48 | pending futures to complete before returning control to the main thread, 49 | preventing bail out: 50 | https://github.com/python/cpython/blob/212337369a64aa96d8b370f39b70113078ad0020/Lib/concurrent/futures/_base.py 51 | https://docs.python.org/3.9/library/concurrent.futures.html#concurrent.futures.Executor.shutdown 52 | 53 | This subclass overrides .__exit__() to not wait for pending futures to 54 | complete before returning control to the main thread, allowing bail out. 55 | ''' 56 | 57 | def __exit__(self, 58 | exc_type: Type[BaseException] | None, 59 | exc_value: BaseException | None, 60 | exc_traceback: TracebackType | None, 61 | ) -> Literal[False]: 62 | self.shutdown(wait=False) 63 | return False 64 | -------------------------------------------------------------------------------- /pottery/hyper.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # hyper.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | # TODO: Remove the following import after deferred evaluation of annotations 20 | # because the default. 21 | # 1. https://docs.python.org/3/whatsnew/3.7.html#whatsnew37-pep563 22 | # 2. https://www.python.org/dev/peps/pep-0563/ 23 | # 3. https://www.python.org/dev/peps/pep-0649/ 24 | from __future__ import annotations 25 | 26 | import uuid 27 | import warnings 28 | from typing import Generator 29 | from typing import Iterable 30 | from typing import List 31 | from typing import cast 32 | 33 | from redis import Redis 34 | 35 | from .annotations import JSONTypes 36 | from .annotations import RedisValues 37 | from .base import Container 38 | from .base import random_key 39 | from .exceptions import InefficientAccessWarning 40 | 41 | 42 | class HyperLogLog(Container): 43 | '''Redis-backed HyperLogLog with a Pythonic API. 44 | 45 | HyperLogLogs are an interesting data structure designed to answer the 46 | question, "How many distinct elements have I seen?"; but not the questions, 47 | "Have I seen this element before?" or "What are all of the elements that 48 | I've seen before?" So think of HyperLogLogs as Python sets that you can add 49 | elements to and get the length of; but that you might not want to use to 50 | test element membership, and can't iterate through, or get elements out of. 51 | 52 | HyperLogLogs are probabilistic, which means that they/re accurate within a 53 | margin of error up to 2%. However, they can reasonably accurately estimate 54 | the cardinality (size) of vast datasets (like the number of unique Google 55 | searches issued in a day) with a tiny amount of storage (1.5 KB). 56 | 57 | Wikipedia article: 58 | https://en.wikipedia.org/wiki/HyperLogLog 59 | 60 | antirez's blog post: 61 | http://antirez.com/news/75 62 | 63 | Riak blog post: 64 | https://riak.com/posts/technical/what-in-the-hell-is-hyperloglog/index.html?p=13169.html 65 | 66 | Create a HyperLogLog and clean up Redis before the doctest: 67 | 68 | >>> google_searches = HyperLogLog(key='google-searches') 69 | >>> google_searches.clear() 70 | 71 | Insert an element into the HyperLogLog: 72 | 73 | >>> google_searches.add('sonic the hedgehog video game') 74 | 75 | See how many elements we've inserted into the HyperLogLog: 76 | 77 | >>> len(google_searches) 78 | 1 79 | 80 | Insert multiple elements into the HyperLogLog: 81 | 82 | >>> google_searches.update({ 83 | ... 'google in 1998', 84 | ... 'minesweeper', 85 | ... 'joey tribbiani', 86 | ... 'wizard of oz', 87 | ... 'rgb to hex', 88 | ... 'pac-man', 89 | ... 'breathing exercise', 90 | ... 'do a barrel roll', 91 | ... 'snake', 92 | ... }) 93 | >>> len(google_searches) 94 | 10 95 | 96 | Test for element membership in the HyperLogLog: 97 | 98 | >>> 'joey tribbiani' in google_searches 99 | True 100 | >>> 'jennifer aniston' in google_searches 101 | False 102 | 103 | Remove all of the elements from the HyperLogLog: 104 | 105 | >>> google_searches.clear() 106 | ''' 107 | 108 | def __init__(self, 109 | iterable: Iterable[RedisValues] = frozenset(), 110 | *, 111 | redis: Redis | None = None, 112 | key: str = '', 113 | ) -> None: 114 | '''Initialize the HyperLogLog. O(n) 115 | 116 | Here, n is the number of elements in iterable that you want to insert 117 | into this HyperLogLog. 118 | ''' 119 | super().__init__(redis=redis, key=key) 120 | self.__update(iterable) 121 | 122 | def add(self, value: RedisValues) -> None: 123 | 'Add an element to the HyperLogLog. O(1)' 124 | self.__update({value}) 125 | 126 | def update(self, *objs: HyperLogLog | Iterable[RedisValues]) -> None: 127 | other_hll_keys: List[str] = [] 128 | encoded_values: List[str] = [] 129 | with self._watch(objs) as pipeline: 130 | for obj in objs: 131 | if isinstance(obj, self.__class__): 132 | if not self._same_redis(obj): 133 | raise RuntimeError( 134 | f"can't update {self} with {obj} as they live on " 135 | "different Redis instances/databases" 136 | ) 137 | other_hll_keys.append(obj.key) 138 | else: 139 | for value in cast(Iterable[JSONTypes], obj): 140 | encoded_value = self._encode(value) 141 | encoded_values.append(encoded_value) 142 | 143 | if other_hll_keys or len(encoded_values) > 1: 144 | warnings.warn( 145 | cast(str, InefficientAccessWarning.__doc__), 146 | InefficientAccessWarning, 147 | ) 148 | 149 | pipeline.multi() # Available since Redis 1.2.0 150 | pipeline.pfmerge(self.key, *other_hll_keys) # Available since Redis 2.8.9 151 | pipeline.pfadd(self.key, *encoded_values) # Available since Redis 2.8.9 152 | 153 | # Preserve the Open-Closed Principle with name mangling. 154 | # https://youtu.be/miGolgp9xq8?t=2086 155 | # https://stackoverflow.com/a/38534939 156 | __update = update 157 | 158 | def union(self, 159 | *objs: HyperLogLog | Iterable[RedisValues], 160 | redis: Redis | None = None, 161 | key: str = '', 162 | ) -> HyperLogLog: 163 | new_hll = self.__class__(redis=redis, key=key) 164 | new_hll.update(self, *objs) 165 | return new_hll 166 | 167 | def __len__(self) -> int: 168 | '''Return the approximate number of elements in the HyperLogLog. O(1) 169 | 170 | Please note that this method returns an approximation, not an exact 171 | value, though it's quite accurate. 172 | ''' 173 | return self.redis.pfcount(self.key) # Available since Redis 2.8.9 174 | 175 | def __contains__(self, value: JSONTypes) -> bool: 176 | '''hll.__contains__(element) <==> element in hll. O(1) 177 | 178 | Please note that this method *may* return false positives, but *never* 179 | returns false negatives. This means that if `element in hll` evaluates 180 | to True, then you *may* have inserted the element into the HyperLogLog. 181 | But if `element in hll` evaluates to False, then you *must not* have 182 | inserted it. 183 | ''' 184 | return next(self.__contains_many(value)) 185 | 186 | def contains_many(self, *values: JSONTypes) -> Generator[bool, None, None]: 187 | '''Yield whether this HyperLogLog contains multiple elements. O(n) 188 | 189 | Please note that this method *may* produce false positives, but *never* 190 | produces false negatives. This means that if .contains_many() yields 191 | True, then you *may* have inserted the element into the HyperLogLog. 192 | But if .contains_many() yields False, then you *must not* have inserted 193 | it. 194 | ''' 195 | if len(values) > 1: 196 | warnings.warn( 197 | cast(str, InefficientAccessWarning.__doc__), 198 | InefficientAccessWarning, 199 | ) 200 | 201 | encoded_values = [] 202 | for value in values: 203 | try: 204 | encoded_value = self._encode(value) 205 | except TypeError: 206 | encoded_value = str(uuid.uuid4()) 207 | encoded_values.append(encoded_value) 208 | 209 | with self._watch() as pipeline: 210 | tmp_hll_key = random_key(redis=pipeline) 211 | pipeline.multi() # Available since Redis 1.2.0 212 | for encoded_value in encoded_values: 213 | pipeline.copy(self.key, tmp_hll_key) # Available since Redis 6.2.0 214 | pipeline.pfadd(tmp_hll_key, encoded_value) # Available since Redis 2.8.9 215 | pipeline.unlink(tmp_hll_key) # Available since Redis 4.0.0 216 | # Pluck out the results of the pipeline.pfadd() commands. Ignore 217 | # the results of the enclosing pipeline.copy() and pipeline.unlink() 218 | # commands. 219 | cardinalities_changed = pipeline.execute()[1::3] # Available since Redis 1.2.0 220 | 221 | for cardinality_changed in cardinalities_changed: 222 | yield not cardinality_changed 223 | 224 | __contains_many = contains_many 225 | 226 | def __repr__(self) -> str: 227 | 'Return the string representation of the HyperLogLog. O(1)' 228 | return f'<{self.__class__.__qualname__} key={self.key} len={len(self)}>' 229 | -------------------------------------------------------------------------------- /pottery/monkey.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # monkey.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 'Monkey patches.' 18 | 19 | 20 | # TODO: When we drop support for Python 3.9, remove the following import. We 21 | # only need it for X | Y union type annotations as of 2022-01-29. 22 | from __future__ import annotations 23 | 24 | import logging 25 | from typing import Final 26 | 27 | 28 | logger: Final[logging.Logger] = logging.getLogger('pottery') 29 | logger.addHandler(logging.NullHandler()) 30 | 31 | 32 | import functools # isort: skip 33 | import json # isort: skip 34 | from typing import Any # isort: skip 35 | from typing import Callable # isort: skip 36 | 37 | class PotteryEncoder(json.JSONEncoder): 38 | 'Custom JSON encoder that can serialize Pottery containers.' 39 | 40 | def default(self, o: Any) -> Any: 41 | from pottery.base import Container 42 | if isinstance(o, Container): 43 | if hasattr(o, 'to_dict'): 44 | return o.to_dict() # type: ignore 45 | if hasattr(o, 'to_list'): # pragma: no cover 46 | return o.to_list() # type: ignore 47 | return super().default(o) 48 | 49 | def _decorate_dumps(func: Callable[..., str]) -> Callable[..., str]: 50 | 'Decorate json.dumps() to use PotteryEncoder by default.' 51 | @functools.wraps(func) 52 | def wrapper(*args: Any, 53 | cls: type[json.JSONEncoder] = PotteryEncoder, 54 | **kwargs: Any, 55 | ) -> str: 56 | return func(*args, cls=cls, **kwargs) 57 | return wrapper 58 | 59 | json.dumps = _decorate_dumps(json.dumps) 60 | 61 | logger.info( 62 | 'Monkey patched json.dumps() to be able to JSONify Pottery containers by ' 63 | 'default' 64 | ) 65 | -------------------------------------------------------------------------------- /pottery/nextid.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # nextid.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | '''Distributed Redis-powered monotonically increasing ID generator. 18 | 19 | Rationale and algorithm description: 20 | http://antirez.com/news/102 21 | 22 | Lua scripting: 23 | https://github.com/andymccurdy/redis-py#lua-scripting 24 | ''' 25 | 26 | 27 | # TODO: Remove the following import after deferred evaluation of annotations 28 | # because the default. 29 | # 1. https://docs.python.org/3/whatsnew/3.7.html#whatsnew37-pep563 30 | # 2. https://www.python.org/dev/peps/pep-0563/ 31 | # 3. https://www.python.org/dev/peps/pep-0649/ 32 | from __future__ import annotations 33 | 34 | import concurrent.futures 35 | import contextlib 36 | from typing import ClassVar 37 | from typing import Iterable 38 | from typing import List 39 | from typing import Tuple 40 | from typing import Type 41 | from typing import cast 42 | 43 | from redis import Redis 44 | from redis import RedisError 45 | from redis.asyncio import Redis as AIORedis 46 | from redis.commands.core import Script 47 | 48 | from .base import Primitive 49 | from .base import logger 50 | from .exceptions import QuorumIsImpossible 51 | from .exceptions import QuorumNotAchieved 52 | from .executor import BailOutExecutor 53 | 54 | 55 | class Scripts: 56 | '''Parent class to define/register Lua scripts for Redis. 57 | 58 | Note that we only have to register these Lua scripts once -- so we do it on 59 | the first instantiation of NextID. 60 | ''' 61 | 62 | __slots__: Tuple[str, ...] = tuple() 63 | 64 | _set_id_script: ClassVar[Script | None] = None 65 | 66 | def __init__(self, 67 | *, 68 | key: str = 'current', 69 | masters: Iterable[Redis | AIORedis] = frozenset(), 70 | raise_on_redis_errors: bool = False, 71 | ) -> None: 72 | super().__init__( # type: ignore 73 | key=key, 74 | masters=masters, 75 | raise_on_redis_errors=raise_on_redis_errors, 76 | ) 77 | self.__register_set_id_script() 78 | 79 | # Preserve the Open-Closed Principle with name mangling. 80 | # https://youtu.be/miGolgp9xq8?t=2086 81 | # https://stackoverflow.com/a/38534939 82 | def __register_set_id_script(self) -> None: 83 | if self._set_id_script is None: 84 | class_name = self.__class__.__qualname__ 85 | logger.info('Registering %s._set_id_script', class_name) 86 | master = next(iter(self.masters)) # type: ignore 87 | # Available since Redis 2.6.0: 88 | self.__class__._set_id_script = master.register_script(''' 89 | local curr = tonumber(redis.call('get', KEYS[1])) 90 | local next = tonumber(ARGV[1]) 91 | if curr == nil or curr < next then 92 | redis.call('set', KEYS[1], next) 93 | return next 94 | else 95 | return nil 96 | end 97 | ''') 98 | 99 | 100 | class NextID(Scripts, Primitive): 101 | '''Distributed Redis-powered monotonically increasing ID generator. 102 | 103 | This algorithm safely and reliably produces monotonically increasing IDs 104 | across threads, processes, and even machines, without a single point of 105 | failure. Two caveats: 106 | 107 | 1. If many clients are generating IDs concurrently, then there may be 108 | "holes" in the sequence of IDs (e.g.: 1, 2, 6, 10, 11, 21, ...). 109 | 110 | 2. This algorithm scales to about 5,000 IDs per second (with 5 Redis 111 | masters). If you need IDs faster than that, then you may want to 112 | consider other techniques. 113 | 114 | Rationale and algorithm description: 115 | http://antirez.com/news/102 116 | 117 | Clean up Redis for the doctest: 118 | 119 | >>> from redis import Redis 120 | >>> redis = Redis() 121 | >>> redis.delete('nextid:tweet-ids') in {0, 1} 122 | True 123 | 124 | Usage: 125 | 126 | >>> tweet_ids_1 = NextID(key='tweet-ids', masters={redis}) 127 | >>> tweet_ids_2 = NextID(key='tweet-ids', masters={redis}) 128 | >>> next(tweet_ids_1) 129 | 1 130 | >>> next(tweet_ids_2) 131 | 2 132 | >>> next(tweet_ids_1) 133 | 3 134 | >>> tweet_ids_1.reset() 135 | >>> next(tweet_ids_1) 136 | 1 137 | ''' 138 | 139 | __slots__ = ('num_tries',) 140 | 141 | _KEY_PREFIX: ClassVar[str] = 'nextid' 142 | _NUM_TRIES: ClassVar[int] = 3 143 | 144 | def __init__(self, 145 | *, 146 | key: str = 'current', 147 | masters: Iterable[Redis] = frozenset(), 148 | raise_on_redis_errors: bool = False, 149 | num_tries: int = _NUM_TRIES, 150 | ) -> None: 151 | '''Initialize a NextID ID generator. 152 | 153 | Keyword arguments: 154 | key -- a string that identifies your ID sequence (e.g., 'tweets') 155 | masters -- the Redis clients used to achieve quorum for this ID 156 | generator 157 | raise_on_redis_errors -- whether to raise the QuorumIsImplssible 158 | exception when too many Redis masters throw errors 159 | num_tries -- the number of times to try to achieve quorum before 160 | giving up and raising the QuorumNotAchieved exception 161 | ''' 162 | super().__init__( 163 | key=key, 164 | masters=masters, 165 | raise_on_redis_errors=raise_on_redis_errors, 166 | ) 167 | self.num_tries = num_tries 168 | 169 | def __iter__(self) -> NextID: 170 | return self 171 | 172 | def __next__(self) -> int: 173 | suppressable_errors: List[Type[BaseException]] = [QuorumNotAchieved] 174 | if not self.raise_on_redis_errors: 175 | suppressable_errors.append(QuorumIsImpossible) 176 | for _ in range(self.num_tries): 177 | with contextlib.suppress(*suppressable_errors): 178 | next_id = self.__current_id + 1 179 | self.__current_id = next_id 180 | return next_id 181 | raise QuorumNotAchieved(self.key, self.masters) 182 | 183 | @property 184 | def __current_id(self) -> int: 185 | with BailOutExecutor() as executor: 186 | futures = set() 187 | for master in self.masters: 188 | future = executor.submit(master.get, self.key) 189 | futures.add(future) 190 | 191 | current_ids, redis_errors = [], [] 192 | for future in concurrent.futures.as_completed(futures): 193 | try: 194 | current_id = int(future.result() or b'0') 195 | except RedisError as error: 196 | redis_errors.append(error) 197 | logger.exception( 198 | '%s.__current_id() getter caught %s', 199 | self.__class__.__qualname__, 200 | error.__class__.__qualname__, 201 | ) 202 | else: 203 | current_ids.append(current_id) 204 | if len(current_ids) > len(self.masters) // 2: # pragma: no cover 205 | return max(current_ids) 206 | 207 | self._check_enough_masters_up(None, redis_errors) 208 | raise QuorumNotAchieved( 209 | self.key, 210 | self.masters, 211 | redis_errors=redis_errors, 212 | ) 213 | 214 | @__current_id.setter 215 | def __current_id(self, value: int) -> None: 216 | with BailOutExecutor() as executor: 217 | futures = set() 218 | for master in self.masters: 219 | future = executor.submit( 220 | cast(Script, self._set_id_script), 221 | keys=(self.key,), 222 | args=(value,), 223 | client=master, 224 | ) 225 | futures.add(future) 226 | 227 | num_masters_set, redis_errors = 0, [] 228 | for future in concurrent.futures.as_completed(futures): 229 | try: 230 | num_masters_set += future.result() == value 231 | except RedisError as error: 232 | redis_errors.append(error) 233 | logger.exception( 234 | '%s.__current_id() setter caught %s', 235 | self.__class__.__qualname__, 236 | error.__class__.__qualname__, 237 | ) 238 | else: 239 | if num_masters_set > len(self.masters) // 2: # pragma: no cover 240 | return 241 | 242 | self._check_enough_masters_up(None, redis_errors) 243 | raise QuorumNotAchieved( 244 | self.key, 245 | self.masters, 246 | redis_errors=redis_errors, 247 | ) 248 | 249 | def reset(self) -> None: 250 | 'Reset the ID generator to 0.' 251 | with concurrent.futures.ThreadPoolExecutor() as executor: 252 | futures = set() 253 | for master in self.masters: 254 | future = executor.submit(master.delete, self.key) 255 | futures.add(future) 256 | 257 | num_masters_reset, redis_errors = 0, [] 258 | for future in concurrent.futures.as_completed(futures): 259 | try: 260 | future.result() 261 | except RedisError as error: 262 | redis_errors.append(error) 263 | logger.exception( 264 | '%s.reset() caught %s', 265 | self.__class__.__qualname__, 266 | error.__class__.__qualname__, 267 | ) 268 | else: 269 | num_masters_reset += 1 270 | if num_masters_reset == len(self.masters): # pragma: no cover 271 | return 272 | 273 | self._check_enough_masters_up(None, redis_errors) 274 | raise QuorumNotAchieved( 275 | self.key, 276 | self.masters, 277 | redis_errors=redis_errors, 278 | ) 279 | 280 | def __repr__(self) -> str: 281 | return f'<{self.__class__.__qualname__} key={self.key}>' 282 | -------------------------------------------------------------------------------- /pottery/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brainix/pottery/16080101a94e35725efeae88e72501d91a36cb96/pottery/py.typed -------------------------------------------------------------------------------- /pottery/queue.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # queue.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | # TODO: When we drop support for Python 3.9, remove the following import. We 20 | # only need it for X | Y union type annotations as of 2022-01-29. 21 | from __future__ import annotations 22 | 23 | import math 24 | import random 25 | import time 26 | from typing import ClassVar 27 | from typing import Tuple 28 | from typing import cast 29 | 30 | from redis import WatchError 31 | 32 | from .annotations import JSONTypes 33 | from .base import Container 34 | from .exceptions import QueueEmptyError 35 | from .timer import ContextTimer 36 | 37 | 38 | class RedisSimpleQueue(Container): 39 | RETRY_DELAY: ClassVar[int] = 200 40 | 41 | def qsize(self) -> int: 42 | '''Return the size of the queue. O(1) 43 | 44 | Be aware that there's a potential race condition here where the queue 45 | changes before you use the result of .qsize(). 46 | ''' 47 | return self.redis.xlen(self.key) # Available since Redis 5.0.0 48 | 49 | # Preserve the Open-Closed Principle with name mangling. 50 | # https://youtu.be/miGolgp9xq8?t=2086 51 | # https://stackoverflow.com/a/38534939 52 | __qsize = qsize 53 | 54 | def empty(self) -> bool: 55 | '''Return True if the queue is empty; False otherwise. O(1) 56 | 57 | This method is likely to be removed at some point. Use `.qsize() == 0` 58 | as a direct substitute, but be aware that either approach risks a race 59 | condition where the queue grows before you use the result of .empty() or 60 | .qsize(). 61 | ''' 62 | return self.__qsize() == 0 63 | 64 | def put(self, 65 | item: JSONTypes, 66 | block: bool = True, 67 | timeout: float | None = None, 68 | ) -> None: 69 | '''Put the item on the queue. O(1) 70 | 71 | The optional block and timeout arguments are ignored, as this method 72 | never blocks. They are provided for compatibility with the queue.Queue 73 | class. 74 | ''' 75 | encoded_item = self._encode(item) 76 | self.redis.xadd(self.key, {'item': encoded_item}, id='*') # Available since Redis 5.0.0 77 | 78 | __put = put 79 | 80 | def put_nowait(self, item: JSONTypes) -> None: 81 | '''Put an item into the queue without blocking. O(1) 82 | 83 | This is exactly equivalent to `.put(item)` and is provided for 84 | compatibility with the queue.Queue class. 85 | ''' 86 | return self.__put(item, block=False) 87 | 88 | def get(self, 89 | block: bool = True, 90 | timeout: float | None = None, 91 | ) -> JSONTypes: 92 | '''Remove and return an item from the queue. O(1) 93 | 94 | If optional args block is True and timeout is None (the default), block 95 | if necessary until an item is available. If timeout is a non-negative 96 | number, block at most timeout seconds and raise the QueueEmptyError 97 | exception if no item becomes available within that time. Otherwise 98 | (block is False), return an item if one is immediately available, else 99 | raise the QueueEmptyError exception (timeout is ignored in this case). 100 | ''' 101 | redis_block = (timeout or 0.0) if block else 0.0 102 | redis_block = math.floor(redis_block) 103 | with ContextTimer() as timer: 104 | while True: 105 | try: 106 | item = self.__remove_and_return(redis_block) 107 | return item 108 | except (WatchError, IndexError): 109 | if not block or timer.elapsed() / 1000 >= (timeout or 0): 110 | raise QueueEmptyError(redis=self.redis, key=self.key) 111 | delay = random.uniform(0, self.RETRY_DELAY/1000) # nosec 112 | time.sleep(delay) 113 | 114 | __get = get 115 | 116 | def __remove_and_return(self, redis_block: int) -> JSONTypes: 117 | with self._watch() as pipeline: 118 | # XXX: The following line raises WatchError after the socket timeout 119 | # if the RedisQueue is empty and we're not blocking. This feels 120 | # like a bug in redis-py? 121 | returned_value = pipeline.xread( # Available since Redis 5.0.0 122 | {self.key: 0}, 123 | count=1, 124 | block=redis_block, 125 | ) 126 | # The following line raises IndexError if the RedisQueue is empty 127 | # and we're blocking. 128 | id_, dict_ = cast(Tuple[bytes, dict], returned_value[0][1][0]) 129 | pipeline.multi() # Available since Redis 1.2.0 130 | pipeline.xdel(self.key, id_) # Available since Redis 5.0.0 131 | encoded_item = dict_[b'item'] 132 | item = self._decode(encoded_item) 133 | return item 134 | 135 | def get_nowait(self) -> JSONTypes: 136 | '''Remove and return an item from the queue without blocking. O(1) 137 | 138 | Get an item if one is immediately available. Otherwise raise the 139 | QueueEmptyError exception. 140 | 141 | This is exactly equivalent to `.get(block=False)` and is provided for 142 | compatibility with the queue.Queue class. 143 | ''' 144 | return self.__get(block=False) 145 | -------------------------------------------------------------------------------- /pottery/timer.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # timer.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 'Measure the execution time of small code snippets.' 18 | 19 | 20 | # TODO: Remove the following import after deferred evaluation of annotations 21 | # because the default. 22 | # 1. https://docs.python.org/3/whatsnew/3.7.html#whatsnew37-pep563 23 | # 2. https://www.python.org/dev/peps/pep-0563/ 24 | # 3. https://www.python.org/dev/peps/pep-0649/ 25 | from __future__ import annotations 26 | 27 | import timeit 28 | from types import TracebackType 29 | from typing import Type 30 | from typing import overload 31 | 32 | from typing_extensions import Literal 33 | 34 | 35 | class ContextTimer: 36 | '''Measure the execution time of small code snippets. 37 | 38 | Note that ContextTimer measures wall (real-world) time, not CPU time; and 39 | that .elapsed() returns time in milliseconds. 40 | 41 | You can use ContextTimer stand-alone... 42 | 43 | >>> import time 44 | >>> timer = ContextTimer() 45 | >>> timer.start() 46 | >>> time.sleep(0.1) 47 | >>> 100 <= timer.elapsed() < 200 48 | True 49 | >>> timer.stop() 50 | >>> time.sleep(0.1) 51 | >>> 100 <= timer.elapsed() < 200 52 | True 53 | 54 | ...or as a context manager: 55 | 56 | >>> tests = [] 57 | >>> with ContextTimer() as timer: 58 | ... time.sleep(0.1) 59 | ... tests.append(100 <= timer.elapsed() < 200) 60 | >>> time.sleep(0.1) 61 | >>> tests.append(100 <= timer.elapsed() < 200) 62 | >>> tests 63 | [True, True] 64 | ''' 65 | 66 | __slots__ = ('_started', '_stopped') 67 | 68 | def __init__(self) -> None: 69 | self._started = 0.0 70 | self._stopped = 0.0 71 | 72 | def __enter__(self) -> ContextTimer: 73 | self.__start() 74 | return self 75 | 76 | @overload 77 | def __exit__(self, 78 | exc_type: None, 79 | exc_value: None, 80 | exc_traceback: None, 81 | ) -> Literal[False]: 82 | raise NotImplementedError 83 | 84 | @overload 85 | def __exit__(self, 86 | exc_type: Type[BaseException], 87 | exc_value: BaseException, 88 | exc_traceback: TracebackType, 89 | ) -> Literal[False]: 90 | raise NotImplementedError 91 | 92 | def __exit__(self, 93 | exc_type: Type[BaseException] | None, 94 | exc_value: BaseException | None, 95 | exc_traceback: TracebackType | None, 96 | ) -> Literal[False]: 97 | self.__stop() 98 | return False 99 | 100 | def start(self) -> None: 101 | if self._stopped: 102 | raise RuntimeError('timer has already been stopped') 103 | elif self._started: 104 | raise RuntimeError('timer has already been started') 105 | else: 106 | self._started = timeit.default_timer() 107 | 108 | # Preserve the Open-Closed Principle with name mangling. 109 | # https://youtu.be/miGolgp9xq8?t=2086 110 | # https://stackoverflow.com/a/38534939 111 | __start = start 112 | 113 | def stop(self) -> None: 114 | if self._stopped: 115 | raise RuntimeError('timer has already been stopped') 116 | elif self._started: 117 | self._stopped = timeit.default_timer() 118 | else: 119 | raise RuntimeError("timer hasn't yet been started") 120 | 121 | __stop = stop 122 | 123 | def elapsed(self) -> int: 124 | if self._started: 125 | stopped_or_current = self._stopped or timeit.default_timer() 126 | elapsed = stopped_or_current - self._started 127 | return round(elapsed * 1000) 128 | else: 129 | raise RuntimeError("timer hasn't yet been started") 130 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # pytest.ini # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | 20 | [pytest] 21 | asyncio_mode = auto 22 | -------------------------------------------------------------------------------- /requirements-to-freeze.txt: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # requirements-to-freeze.txt # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | # There's a bug in redis-py 3.4.0 that prevents connecting to Redis with 20 | # authentication (a username and password). For more info: 21 | # https://github.com/andymccurdy/redis-py/issues/1278 22 | # 23 | # We need redis-py 4.2.0rc1 or later for aioredis. For more info: 24 | # https://github.com/aio-libs/aioredis-py/tree/19be499015a8cf32580e937cbfd711fd48489eca#-aioredis-is-now-in-redis-py-420rc1- 25 | redis>=4.2.0rc1 26 | hiredis 27 | mmh3 28 | typing_extensions 29 | 30 | pytest 31 | pytest-asyncio 32 | pytest-cov 33 | uvloop 34 | mypy 35 | types-redis 36 | 37 | flake8 38 | isort 39 | bandit 40 | safety 41 | 42 | twine 43 | 44 | 45 | # We don't need Requests at the top-level. However, it's pulled in from 46 | # something else, and there's a security vulnerability in the version that it 47 | # pulls in. For more info: 48 | # https://nvd.nist.gov/vuln/detail/CVE-2018-18074 49 | requests>=2.20.0 50 | 51 | # We don't need urllib3 at the top-level. However, it's pulled in from 52 | # something else, and there's a security vulnerability in the version that it 53 | # pulls in. For more info: 54 | # https://nvd.nist.gov/vuln/detail/CVE-2018-20060 55 | # https://nvd.nist.gov/vuln/detail/CVE-2019-11324 56 | urllib3>=1.24.2 57 | 58 | # We don't need docutils at the top-level. However, it's pulled in from 59 | # something else, and recent docutils doesn't support Python 3.8. 60 | docutils==0.20.1 61 | 62 | # We don't need pyjwt at the top-level. However, it's pulled in from something 63 | # else, and there's a security vulnerability in the version that it pulls in. 64 | # For more info: 65 | # https://data.safetycli.com/p/pypi/pyjwt/eda/? 66 | pyjwt>=2.10.1 67 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | annotated-types==0.7.0 2 | anyio==4.9.0 3 | Authlib==1.6.0 4 | bandit==1.8.3 5 | certifi==2025.4.26 6 | cffi==1.17.1 7 | charset-normalizer==3.4.2 8 | click==8.1.8 9 | coverage==7.8.2 10 | cryptography==45.0.3 11 | docutils==0.20.1 12 | dparse==0.6.4 13 | filelock==3.16.1 14 | flake8==7.2.0 15 | h11==0.16.0 16 | hiredis==3.2.1 17 | httpcore==1.0.9 18 | httpx==0.28.1 19 | id==1.5.0 20 | idna==3.10 21 | iniconfig==2.1.0 22 | isort==6.0.1 23 | jaraco.classes==3.4.0 24 | jaraco.context==6.0.1 25 | jaraco.functools==4.1.0 26 | Jinja2==3.1.6 27 | joblib==1.5.1 28 | keyring==25.6.0 29 | markdown-it-py==3.0.0 30 | MarkupSafe==3.0.2 31 | marshmallow==4.0.0 32 | mccabe==0.7.0 33 | mdurl==0.1.2 34 | mmh3==5.1.0 35 | more-itertools==10.7.0 36 | mypy==1.15.0 37 | mypy_extensions==1.1.0 38 | nh3==0.2.21 39 | nltk==3.9.1 40 | packaging==25.0 41 | pbr==6.1.1 42 | pluggy==1.6.0 43 | psutil==6.1.1 44 | pycodestyle==2.13.0 45 | pycparser==2.22 46 | pydantic==2.9.2 47 | pydantic_core==2.23.4 48 | pyflakes==3.3.2 49 | Pygments==2.19.1 50 | PyJWT==2.10.1 51 | pytest==8.3.5 52 | pytest-asyncio==1.0.0 53 | pytest-cov==6.1.1 54 | PyYAML==6.0.2 55 | readme_renderer==43.0 56 | redis==6.2.0 57 | regex==2024.11.6 58 | requests==2.32.3 59 | requests-toolbelt==1.0.0 60 | rfc3986==2.0.0 61 | rich==14.0.0 62 | ruamel.yaml==0.18.11 63 | ruamel.yaml.clib==0.2.12 64 | safety==3.5.1 65 | safety-schemas==0.0.14 66 | setuptools==80.9.0 67 | shellingham==1.5.4 68 | sniffio==1.3.1 69 | stevedore==5.4.1 70 | tenacity==9.1.2 71 | tomlkit==0.13.2 72 | tqdm==4.67.1 73 | twine==6.1.0 74 | typer==0.16.0 75 | types-cffi==1.17.0.20250523 76 | types-pyOpenSSL==24.1.0.20240722 77 | types-redis==4.6.0.20241004 78 | types-setuptools==80.8.0.20250521 79 | typing_extensions==4.13.2 80 | urllib3==2.4.0 81 | uvloop==0.21.0 82 | wheel==0.45.1 83 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # setup.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | '''Redis for Humans. 18 | 19 | Redis is awesome, but Redis commands are not always intuitive. Pottery is a 20 | Pythonic way to access Redis. If you know how to use Python dicts, then you 21 | already know how to use Pottery. Pottery is useful for accessing Redis more 22 | easily, and also for implementing microservice resilience patterns; and it has 23 | been battle tested in production at scale. 24 | ''' 25 | 26 | 27 | import pathlib 28 | 29 | from setuptools import find_packages 30 | from setuptools import setup 31 | 32 | 33 | __title__ = 'pottery' 34 | __version__ = '3.0.1' 35 | __description__ = __doc__.split(sep='\n\n', maxsplit=1)[0] 36 | __url__ = 'https://github.com/brainix/pottery' 37 | __author__ = 'Rajiv Bakulesh Shah' 38 | __author_email__ = 'brainix@gmail.com' 39 | __keywords__ = 'Redis client persistent storage' 40 | __copyright__ = f'Copyright © 2015-2025, {__author__}, original author.' 41 | 42 | 43 | _package_dir = pathlib.Path(__file__).parent 44 | _long_description = (_package_dir / 'README.md').read_text() 45 | 46 | 47 | setup( 48 | version=__version__, 49 | description=__description__, 50 | long_description=_long_description, 51 | long_description_content_type='text/markdown', 52 | url=__url__, 53 | author=__author__, 54 | author_email=__author_email__, 55 | classifiers=[ 56 | 'Intended Audience :: Developers', 57 | 'Development Status :: 4 - Beta', 58 | 'Topic :: Database :: Front-Ends', 59 | 'Topic :: System :: Distributed Computing', 60 | 'Topic :: Utilities', 61 | 'Programming Language :: Python :: 3 :: Only', 62 | 'Programming Language :: Python :: 3.10', 63 | 'Programming Language :: Python :: 3.11', 64 | 'Programming Language :: Python :: 3.12', 65 | 'Programming Language :: Python :: 3.13', 66 | 'Framework :: AsyncIO', 67 | 'Typing :: Typed', 68 | ], 69 | keywords=__keywords__, 70 | python_requires='>=3.10, <4', 71 | install_requires=('redis>=4.2.0rc1', 'mmh3', 'typing_extensions'), 72 | extras_require={}, 73 | packages=find_packages(exclude=('contrib', 'docs', 'tests*')), 74 | package_data={'pottery': ['py.typed']}, 75 | data_files=[], 76 | entry_points={}, 77 | ) 78 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # __init__.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # conftest.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | import random 20 | import warnings 21 | # TODO: When we drop support for Python 3.9, change the following import to: 22 | # from collections.abc import AsyncGenerator 23 | from typing import AsyncGenerator 24 | from typing import Generator 25 | 26 | import pytest 27 | import uvloop 28 | from redis import Redis 29 | from redis.asyncio import Redis as AIORedis 30 | 31 | from pottery import PotteryWarning 32 | 33 | 34 | @pytest.fixture(autouse=True) 35 | def install_uvloop() -> None: 36 | uvloop.install() 37 | 38 | 39 | @pytest.fixture(autouse=True) 40 | def filter_warnings() -> None: 41 | warnings.filterwarnings('ignore', category=PotteryWarning) 42 | 43 | 44 | @pytest.fixture(scope='session') 45 | def redis_url() -> str: 46 | redis_db = random.randint(1, 15) # nosec 47 | return f'redis://localhost:6379/{redis_db}' 48 | 49 | 50 | @pytest.fixture 51 | def redis(redis_url: str) -> Generator[Redis, None, None]: 52 | redis_client = Redis.from_url(redis_url, socket_timeout=1) 53 | redis_client.flushdb() 54 | yield redis_client 55 | redis_client.flushdb() 56 | 57 | 58 | @pytest.fixture 59 | async def aioredis(redis_url: str) -> AsyncGenerator[AIORedis, None]: # type: ignore 60 | redis_client = AIORedis.from_url(redis_url, socket_timeout=1) 61 | await redis_client.flushdb() 62 | yield redis_client 63 | await redis_client.flushdb() 64 | -------------------------------------------------------------------------------- /tests/test_aionextid.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # test_aioredlock.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 'Async distributed Redis-powered monotonically increasing ID generator tests.' 18 | 19 | 20 | import asyncio 21 | import contextlib 22 | import unittest.mock 23 | 24 | import pytest 25 | from redis.asyncio import Redis as AIORedis 26 | from redis.commands.core import AsyncScript 27 | from redis.exceptions import TimeoutError 28 | 29 | from pottery import AIONextID 30 | from pottery import QuorumNotAchieved 31 | 32 | 33 | # TODO: When we drop support for Python 3.9, delete the following definition of 34 | # aiter(). 35 | try: 36 | aiter # type: ignore 37 | except NameError: # pragma: no cover 38 | def aiter(iterable): 39 | return iterable.__aiter__() 40 | 41 | # TODO: When we drop support for Python 3.9, delete the following definition of 42 | # anext(). 43 | try: 44 | anext # type: ignore 45 | except NameError: # pragma: no cover 46 | # I got this anext() definition from here: 47 | # https://github.com/python/cpython/blob/f4c03484da59049eb62a9bf7777b963e2267d187/Lib/test/test_asyncgen.py#L52 48 | _NO_DEFAULT = object() 49 | 50 | def anext(iterator, default=_NO_DEFAULT): 51 | try: 52 | __anext__ = type(iterator).__anext__ 53 | except AttributeError: 54 | raise TypeError(f'{iterator!r} is not an async iterator') 55 | if default is _NO_DEFAULT: 56 | return __anext__(iterator) 57 | 58 | async def anext_impl(): 59 | try: 60 | return await __anext__(iterator) 61 | except StopAsyncIteration: 62 | return default 63 | return anext_impl() 64 | 65 | 66 | @pytest.fixture 67 | def aioids(aioredis: AIORedis) -> AIONextID: # type: ignore 68 | return AIONextID(masters={aioredis}) 69 | 70 | 71 | async def test_aionextid(aioids: AIONextID) -> None: 72 | for expected in range(1, 10): 73 | got = await anext(aioids) # type: ignore 74 | assert got == expected, f'expected {expected}, got {got}' 75 | 76 | 77 | async def test_reset(aioids: AIONextID) -> None: 78 | assert await anext(aioids) == 1 # type: ignore 79 | await aioids.reset() 80 | assert await anext(aioids) == 1 # type: ignore 81 | 82 | 83 | @pytest.mark.parametrize('num_aioids', range(1, 6)) 84 | async def test_contention(num_aioids: int) -> None: 85 | dbs = range(1, 6) 86 | urls = [f'redis://localhost:6379/{db}' for db in dbs] 87 | masters = [AIORedis.from_url(url, socket_timeout=1) for url in urls] 88 | aioids = [AIONextID(key='tweet-ids', masters=masters) for _ in range(num_aioids)] 89 | 90 | try: 91 | coros = [anext(aioids[id_gen]) for id_gen in range(num_aioids)] # type: ignore 92 | tasks = [asyncio.create_task(coro) for coro in coros] 93 | done, _ = await asyncio.wait(tasks) 94 | results = [] 95 | with contextlib.suppress(QuorumNotAchieved): 96 | for task in done: 97 | results.append(task.result()) 98 | assert len(results) == len(set(results)) 99 | # To see the following output, issue: 100 | # $ source venv/bin/activate; pytest -rP tests/test_aionextid.py::test_contention; deactivate 101 | print(f'{num_aioids} aioids, {results} IDs') 102 | 103 | finally: 104 | # Clean up for the next unit test run. 105 | await aioids[0].reset() 106 | 107 | 108 | def test_slots(aioids: AIONextID) -> None: 109 | with pytest.raises(AttributeError): 110 | aioids.__dict__ 111 | 112 | 113 | def test_aiter(aioids: AIONextID) -> None: 114 | assert aiter(aioids) is aioids # type: ignore 115 | 116 | 117 | async def test_anext_quorumnotachieved(aioids: AIONextID) -> None: 118 | aioredis = next(iter(aioids.masters)) 119 | with pytest.raises(QuorumNotAchieved), \ 120 | unittest.mock.patch.object(aioredis, 'get') as get: 121 | get.side_effect = TimeoutError 122 | await anext(aioids) # type: ignore 123 | 124 | with pytest.raises(QuorumNotAchieved), \ 125 | unittest.mock.patch.object(AsyncScript, '__call__') as __call__: 126 | __call__.side_effect = TimeoutError 127 | await anext(aioids) # type: ignore 128 | 129 | 130 | async def test_reset_quorumnotachieved(aioids: AIONextID) -> None: 131 | aioredis = next(iter(aioids.masters)) 132 | with pytest.raises(QuorumNotAchieved), \ 133 | unittest.mock.patch.object(aioredis, 'delete') as delete: 134 | delete.side_effect = TimeoutError 135 | await aioids.reset() 136 | 137 | 138 | def test_repr(aioids: AIONextID) -> None: 139 | assert repr(aioids) == '' 140 | -------------------------------------------------------------------------------- /tests/test_aioredlock.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # test_aioredlock.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 'Asynchronous distributed Redis-powered lock tests.' 18 | 19 | 20 | import asyncio 21 | import unittest.mock 22 | 23 | import pytest 24 | from redis.asyncio import Redis as AIORedis 25 | from redis.commands.core import AsyncScript 26 | from redis.exceptions import TimeoutError 27 | 28 | from pottery import AIORedlock 29 | from pottery import ExtendUnlockedLock 30 | from pottery import QuorumNotAchieved 31 | from pottery import Redlock 32 | from pottery import ReleaseUnlockedLock 33 | from pottery.exceptions import TooManyExtensions 34 | 35 | 36 | @pytest.fixture 37 | def aioredlock(aioredis: AIORedis) -> AIORedlock: # type: ignore 38 | return AIORedlock(masters={aioredis}, key='shower') 39 | 40 | 41 | async def test_locked_acquire_and_release(aioredlock: AIORedlock) -> None: 42 | assert not await aioredlock.locked() 43 | assert await aioredlock.acquire() 44 | assert await aioredlock.locked() 45 | await aioredlock.release() 46 | assert not await aioredlock.locked() 47 | with pytest.raises(ReleaseUnlockedLock): 48 | await aioredlock.release() 49 | 50 | 51 | async def test_extend(aioredlock: AIORedlock) -> None: 52 | with pytest.raises(ExtendUnlockedLock): 53 | await aioredlock.extend() 54 | assert await aioredlock.acquire() 55 | for extension_num in range(Redlock._NUM_EXTENSIONS): 56 | await aioredlock.extend() 57 | with pytest.raises(TooManyExtensions): 58 | await aioredlock.extend() 59 | 60 | 61 | async def test_context_manager(aioredlock: AIORedlock) -> None: 62 | assert not await aioredlock.locked() 63 | async with aioredlock: 64 | assert await aioredlock.locked() 65 | assert not await aioredlock.locked() 66 | 67 | 68 | async def test_context_manager_extend(aioredlock: AIORedlock) -> None: 69 | with pytest.raises(ExtendUnlockedLock): 70 | await aioredlock.extend() 71 | async with aioredlock: 72 | for extension_num in range(Redlock._NUM_EXTENSIONS): 73 | await aioredlock.extend() 74 | with pytest.raises(TooManyExtensions): 75 | await aioredlock.extend() 76 | 77 | 78 | async def test_acquire_fails_within_auto_release_time(aioredlock: AIORedlock) -> None: 79 | aioredlock.auto_release_time = .001 80 | assert not await aioredlock.acquire(blocking=False) 81 | 82 | 83 | async def test_context_manager_fails_within_auto_release_time(aioredlock: AIORedlock) -> None: 84 | aioredlock.auto_release_time = .001 85 | aioredlock.context_manager_blocking = False 86 | with pytest.raises(QuorumNotAchieved): 87 | async with aioredlock: # pragma: no cover 88 | ... 89 | 90 | 91 | async def test_acquire_and_time_out(aioredlock: AIORedlock) -> None: 92 | aioredlock.auto_release_time = 1 93 | assert not await aioredlock.locked() 94 | assert await aioredlock.acquire() 95 | assert await aioredlock.locked() 96 | await asyncio.sleep(aioredlock.auto_release_time) 97 | assert not await aioredlock.locked() 98 | 99 | 100 | async def test_context_manager_time_out_before_exit(aioredlock: AIORedlock) -> None: 101 | aioredlock.auto_release_time = 1 102 | with pytest.raises(ReleaseUnlockedLock): 103 | async with aioredlock: 104 | await asyncio.sleep(aioredlock.auto_release_time * 2) 105 | assert not await aioredlock.locked() 106 | 107 | 108 | async def test_context_manager_release_before_exit(aioredlock: AIORedlock) -> None: 109 | with pytest.raises(ReleaseUnlockedLock): 110 | async with aioredlock: 111 | await aioredlock.release() 112 | 113 | 114 | def test_context_manager_nonblocking_with_timeout(aioredis: AIORedis) -> None: # type: ignore 115 | with pytest.raises(ValueError): 116 | AIORedlock( 117 | masters={aioredis}, 118 | key='shower', 119 | auto_release_time=.2, 120 | context_manager_blocking=False, 121 | context_manager_timeout=.1 122 | ) 123 | 124 | 125 | async def test_acquire_nonblocking_with_timeout(aioredlock: AIORedlock) -> None: 126 | with pytest.raises(ValueError): 127 | await aioredlock.acquire(blocking=False, timeout=.1) 128 | 129 | 130 | async def test_acquire_rediserror(aioredlock: AIORedlock) -> None: 131 | aioredis = next(iter(aioredlock.masters)) 132 | with unittest.mock.patch.object(aioredis, 'set') as set: 133 | set.side_effect = TimeoutError 134 | assert not await aioredlock.acquire(blocking=False) 135 | 136 | 137 | async def test_locked_rediserror(aioredlock: AIORedlock) -> None: 138 | async with aioredlock: 139 | assert await aioredlock.locked() 140 | with unittest.mock.patch.object(AsyncScript, '__call__') as __call__: 141 | __call__.side_effect = TimeoutError 142 | assert not await aioredlock.locked() 143 | 144 | 145 | async def test_extend_rediserror(aioredlock: AIORedlock) -> None: 146 | async with aioredlock: 147 | await aioredlock.extend() 148 | with unittest.mock.patch.object(AsyncScript, '__call__') as __call__: 149 | __call__.side_effect = TimeoutError 150 | with pytest.raises(ExtendUnlockedLock): 151 | await aioredlock.extend() 152 | 153 | 154 | async def test_release_rediserror(aioredlock: AIORedlock) -> None: 155 | with unittest.mock.patch.object(AsyncScript, '__call__') as __call__: 156 | __call__.side_effect = TimeoutError 157 | await aioredlock.acquire() 158 | with pytest.raises(ReleaseUnlockedLock): 159 | await aioredlock.release() 160 | 161 | 162 | async def test_enqueued(aioredlock: AIORedlock) -> None: 163 | aioredlock.auto_release_time = .2 164 | aioredis = next(iter(aioredlock.masters)) 165 | aioredlock2 = AIORedlock(masters={aioredis}, key='shower', auto_release_time=.2) # type: ignore 166 | 167 | await aioredlock.acquire() 168 | # aioredlock2 is enqueued until self.aioredlock is automatically released: 169 | assert await aioredlock2.acquire() 170 | 171 | await aioredlock.acquire() 172 | # aioredlock2 is enqueued until the acquire timeout has expired: 173 | assert not await aioredlock2.acquire(timeout=0.1) 174 | 175 | 176 | @pytest.mark.parametrize('num_locks', range(1, 11)) 177 | async def test_contention(num_locks: int) -> None: 178 | dbs = range(1, 6) 179 | urls = [f'redis://localhost:6379/{db}' for db in dbs] 180 | masters = [AIORedis.from_url(url, socket_timeout=1) for url in urls] 181 | locks = [AIORedlock(key='shower', masters=masters, auto_release_time=.2) for _ in range(num_locks)] 182 | 183 | try: 184 | coros = [lock.acquire(blocking=False) for lock in locks] 185 | tasks = [asyncio.create_task(coro) for coro in coros] 186 | done, _ = await asyncio.wait(tasks) 187 | results = [task.result() for task in done] 188 | num_unlocked = results.count(False) 189 | num_locked = results.count(True) 190 | assert num_locks-1 <= num_unlocked <= num_locks 191 | assert 0 <= num_locked <= 1 192 | # To see the following output, issue: 193 | # $ source venv/bin/activate; pytest -rP tests/test_aioredlock.py::test_contention; deactivate 194 | print(f'{num_locks} locks, {num_unlocked} unlocked, {num_locked} locked') 195 | 196 | finally: 197 | # Clean up for the next unit test run. 198 | coros = [lock.release() for lock in locks] # type: ignore 199 | tasks = [asyncio.create_task(coro) for coro in coros] 200 | done, _ = await asyncio.wait(tasks) 201 | [task.exception() for task in done] 202 | 203 | 204 | def test_slots(aioredlock: AIORedlock) -> None: 205 | with pytest.raises(AttributeError): 206 | aioredlock.__dict__ 207 | 208 | 209 | def test_repr(aioredlock: AIORedlock) -> None: 210 | assert repr(aioredlock) == '' 211 | -------------------------------------------------------------------------------- /tests/test_base.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # test_base.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | import gc 20 | import unittest.mock 21 | from typing import Generator 22 | 23 | import pytest 24 | from redis import Redis 25 | 26 | from pottery import RandomKeyError 27 | from pottery import RedisDict 28 | from pottery.base import Iterable_ 29 | from pottery.base import Primitive 30 | from pottery.base import _Comparable 31 | from pottery.base import _Pipelined 32 | from pottery.base import random_key 33 | 34 | 35 | class TestRandomKey: 36 | @staticmethod 37 | def test_random_key_raises_typeerror_for_invalid_num_tries(redis: Redis) -> None: 38 | with pytest.raises(TypeError): 39 | random_key(redis=redis, num_tries=3.0) # type: ignore 40 | 41 | @staticmethod 42 | def test_random_key_raises_valueerror_for_invalid_num_tries(redis: Redis) -> None: 43 | with pytest.raises(ValueError): 44 | random_key(redis=redis, num_tries=-1) 45 | 46 | @staticmethod 47 | def test_random_key_raises_randomkeyerror_when_no_tries_left(redis: Redis) -> None: 48 | with pytest.raises(RandomKeyError), \ 49 | unittest.mock.patch.object(redis, 'exists') as exists: 50 | exists.return_value = True 51 | random_key(redis=redis) 52 | 53 | 54 | class TestCommon: 55 | @staticmethod 56 | def test_out_of_scope(redis: Redis) -> None: 57 | def scope() -> str: 58 | raj = RedisDict(redis=redis, hobby='music', vegetarian=True) 59 | assert redis.exists(raj.key) 60 | return raj.key 61 | 62 | key = scope() 63 | gc.collect() 64 | assert not redis.exists(key) 65 | 66 | @staticmethod 67 | def test_del(redis: Redis) -> None: 68 | raj = RedisDict(redis=redis, key='pottery:raj', hobby='music', vegetarian=True) 69 | nilika = RedisDict(redis=redis, key='pottery:nilika', hobby='music', vegetarian=True) 70 | luvh = RedisDict(redis=redis, key='luvh', hobby='bullying', vegetarian=False) 71 | 72 | with unittest.mock.patch.object(redis, 'unlink') as unlink: 73 | del raj 74 | gc.collect() 75 | unlink.assert_called_with('pottery:raj') 76 | unlink.reset_mock() 77 | 78 | del nilika 79 | unlink.assert_called_with('pottery:nilika') 80 | unlink.reset_mock() 81 | 82 | del luvh 83 | unlink.assert_not_called() 84 | 85 | @staticmethod 86 | def test_eq(redis: Redis) -> None: 87 | raj = RedisDict(redis=redis, key='pottery:raj', hobby='music', vegetarian=True) 88 | nilika = RedisDict(redis=redis, key='pottery:nilika', hobby='music', vegetarian=True) 89 | luvh = RedisDict(redis=redis, key='luvh', hobby='bullying', vegetarian=False) 90 | 91 | assert raj == raj 92 | assert raj == nilika 93 | assert raj == {'hobby': 'music', 'vegetarian': True} 94 | assert not raj == luvh 95 | assert not raj == None 96 | 97 | @staticmethod 98 | def test_ne(redis: Redis) -> None: 99 | raj = RedisDict(redis=redis, key='pottery:raj', hobby='music', vegetarian=True) 100 | nilika = RedisDict(redis=redis, key='pottery:nilika', hobby='music', vegetarian=True) 101 | luvh = RedisDict(redis=redis, key='luvh', hobby='bullying', vegetarian=False) 102 | 103 | assert not raj != raj 104 | assert not raj != nilika 105 | assert not raj != {'hobby': 'music', 'vegetarian': True} 106 | assert raj != luvh 107 | assert raj != None 108 | 109 | @staticmethod 110 | def test_randomkeyerror_raised(redis: Redis) -> None: 111 | raj = RedisDict(redis=redis, key='pottery:raj', hobby='music', vegetarian=True) 112 | 113 | with pytest.raises(RandomKeyError), \ 114 | unittest.mock.patch.object(raj.redis, 'exists') as exists: 115 | exists.return_value = True 116 | raj._random_key() 117 | 118 | @staticmethod 119 | def test_randomkeyerror_repr(redis: Redis) -> None: 120 | raj = RedisDict(redis=redis, key='pottery:raj', hobby='music', vegetarian=True) 121 | 122 | with unittest.mock.patch.object(raj.redis, 'exists') as exists: 123 | exists.return_value = True 124 | try: 125 | raj._random_key() 126 | except RandomKeyError as wtf: 127 | redis_db = redis.get_connection_kwargs()['db'] # type: ignore 128 | assert repr(wtf) == ( 129 | f'RandomKeyError(redis=)>)>, key=None)' 130 | ) 131 | else: # pragma: no cover 132 | pytest.fail(reason='RandomKeyError not raised') 133 | 134 | 135 | class TestEncodable: 136 | @staticmethod 137 | @pytest.fixture 138 | def decoded_redis(redis_url: str) -> Generator[Redis, None, None]: 139 | redis = Redis.from_url(redis_url, socket_timeout=1, decode_responses=True) 140 | redis.flushdb() 141 | yield redis 142 | redis.flushdb() 143 | 144 | @staticmethod 145 | def test_decoded_responses(decoded_redis: Redis) -> None: 146 | 'Ensure that Pottery still works if the Redis client decodes responses.' 147 | tel = RedisDict({'jack': 4098, 'sape': 4139}, redis=decoded_redis) # type: ignore 148 | 149 | # Ensure that repr(tel) does not raise this exception: 150 | # 151 | # Traceback (most recent call last): 152 | # File "/Users/rajiv.shah/Documents/Code/pottery/tests/test_base.py", line 139, in test_decoded_responses 153 | # repr(tel) 154 | # File "/Users/rajiv.shah/Documents/Code/pottery/pottery/dict.py", line 116, in __repr__ 155 | # dict_ = {self._decode(key): self._decode(value) for key, value in items} 156 | # File "/Users/rajiv.shah/Documents/Code/pottery/pottery/dict.py", line 116, in 157 | # dict_ = {self._decode(key): self._decode(value) for key, value in items} 158 | # File "/Users/rajiv.shah/Documents/Code/pottery/pottery/base.py", line 154, in _decode 159 | # decoded: JSONTypes = json.loads(value.decode()) 160 | # AttributeError: 'str' object has no attribute 'decode' 161 | repr(tel) 162 | 163 | 164 | class TestPipelined: 165 | @staticmethod 166 | def test_abc_cant_be_instantiated(): 167 | with pytest.raises(TypeError): 168 | _Pipelined() 169 | 170 | 171 | class TestComparable: 172 | @staticmethod 173 | def test_abc_cant_be_instantiated() -> None: 174 | with pytest.raises(TypeError): 175 | _Comparable() # type: ignore 176 | 177 | 178 | class TestIterable: 179 | @staticmethod 180 | def test_abc_cant_be_instantiated() -> None: 181 | with pytest.raises(TypeError): 182 | Iterable_() # type: ignore 183 | 184 | @staticmethod 185 | def test_iter(redis: Redis) -> None: 186 | garbage = RedisDict(redis=redis) 187 | for num in range(1024): 188 | garbage[num] = num 189 | assert set(iter(garbage)) == set(range(1024)) 190 | 191 | 192 | class TestPrimitive: 193 | @staticmethod 194 | def test_abc_cant_be_instantiated() -> None: 195 | with pytest.raises(TypeError): 196 | Primitive(key='abc') # type: ignore 197 | -------------------------------------------------------------------------------- /tests/test_counter.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # test_counter.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | '''These tests come from these examples: 18 | https://docs.python.org/3/library/collections.html#counter-objects 19 | ''' 20 | 21 | 22 | import collections 23 | from typing import Counter 24 | 25 | from redis import Redis 26 | 27 | from pottery import RedisCounter 28 | from pottery.base import _Common 29 | 30 | 31 | def test_basic_usage(redis: Redis) -> None: 32 | c = RedisCounter(redis=redis) 33 | for word in ('red', 'blue', 'red', 'green', 'blue', 'blue'): 34 | c[word] += 1 35 | assert c == collections.Counter(blue=3, red=2, green=1) 36 | 37 | 38 | def test_init(redis: Redis) -> None: 39 | c = RedisCounter(redis=redis) 40 | assert c == collections.Counter() 41 | 42 | 43 | def test_init_with_iterable(redis: Redis) -> None: 44 | c = RedisCounter('gallahad', redis=redis) # type: ignore 45 | assert c == collections.Counter(a=3, l=2, g=1, h=1, d=1) # NoQA: E741 46 | 47 | 48 | def test_init_with_mapping(redis: Redis) -> None: 49 | c = RedisCounter({'red': 4, 'blue': 2}, redis=redis) # type: ignore 50 | assert c == collections.Counter(red=4, blue=2) 51 | 52 | 53 | def test_init_with_kwargs(redis: Redis) -> None: 54 | c = RedisCounter(redis=redis, cats=4, dogs=8) 55 | assert c == collections.Counter(dogs=8, cats=4) 56 | 57 | 58 | def test_missing_element_doesnt_raise_keyerror(redis: Redis) -> None: 59 | c = RedisCounter(('eggs', 'ham'), redis=redis) # type: ignore 60 | assert c['bacon'] == 0 61 | 62 | 63 | def test_setting_0_adds_item(redis: Redis) -> None: 64 | c = RedisCounter(('eggs', 'ham'), redis=redis) # type: ignore 65 | assert set(c) == {'eggs', 'ham'} 66 | c['sausage'] = 0 67 | assert set(c) == {'eggs', 'ham', 'sausage'} 68 | 69 | 70 | def test_setting_0_count_doesnt_remove_item(redis: Redis) -> None: 71 | c = RedisCounter(('eggs', 'ham'), redis=redis) # type: ignore 72 | c['ham'] = 0 73 | assert set(c) == {'eggs', 'ham'} 74 | 75 | 76 | def test_del_removes_item(redis: Redis) -> None: 77 | c = RedisCounter(('eggs', 'ham'), redis=redis) # type: ignore 78 | c['sausage'] = 0 79 | assert set(c) == {'eggs', 'ham', 'sausage'} 80 | del c['sausage'] 81 | assert set(c) == {'eggs', 'ham'} 82 | del c['ham'] 83 | assert set(c) == {'eggs'} 84 | 85 | 86 | def test_elements(redis: Redis) -> None: 87 | c = RedisCounter(redis=redis, a=4, b=2, c=0, d=-2) 88 | assert sorted(c.elements()) == ['a', 'a', 'a', 'a', 'b', 'b'] 89 | 90 | 91 | def test_most_common(redis: Redis) -> None: 92 | c = RedisCounter('abracadabra', redis=redis) # type: ignore 93 | assert sorted(c.most_common(3)) == [('a', 5), ('b', 2), ('r', 2)] 94 | 95 | 96 | def test_update_with_empty_dict(redis: Redis) -> None: 97 | c = RedisCounter(redis=redis, foo=1) 98 | c.update({}) 99 | assert isinstance(c, RedisCounter) 100 | assert c == collections.Counter(foo=1) 101 | 102 | 103 | def test_update_with_overlapping_dict(redis: Redis) -> None: 104 | c = RedisCounter(redis=redis, foo=1, bar=1) 105 | c.update({'bar': 1, 'baz': 3, 'qux': 4}) 106 | assert isinstance(c, RedisCounter) 107 | assert c == collections.Counter(foo=1, bar=2, baz=3, qux=4) 108 | 109 | 110 | def test_subtract(redis: Redis) -> None: 111 | c = RedisCounter(redis=redis, a=4, b=2, c=0, d=-2) 112 | d = RedisCounter(redis=redis, a=1, b=2, c=3, d=4) 113 | c.subtract(d) 114 | assert isinstance(c, RedisCounter) 115 | assert c == collections.Counter(a=3, b=0, c=-3, d=-6) 116 | 117 | 118 | def test_repr(redis: Redis) -> None: 119 | 'Test RedisCounter.__repr__()' 120 | c = RedisCounter(('eggs', 'ham'), redis=redis) # type: ignore 121 | assert repr(c) in { 122 | "RedisCounter{'eggs': 1, 'ham': 1}", 123 | "RedisCounter{'ham': 1, 'eggs': 1}", 124 | } 125 | 126 | 127 | def test_make_counter(redis: Redis) -> None: 128 | 'Test RedisCounter._make_counter()' 129 | kwargs = {str(element): element for element in range(1000)} 130 | c = RedisCounter(redis=redis, **kwargs).to_counter() # type: ignore 131 | assert c == collections.Counter(**kwargs) 132 | 133 | 134 | def test_add(redis: Redis) -> None: 135 | 'Test RedisCounter.__add__()' 136 | c = RedisCounter(redis=redis, a=3, b=1) 137 | d = RedisCounter(redis=redis, a=1, b=2) 138 | e = c + d 139 | assert isinstance(e, collections.Counter) 140 | assert e == collections.Counter(a=4, b=3) 141 | 142 | 143 | def test_sub(redis: Redis) -> None: 144 | 'Test RedisCounter.__sub__()' 145 | c = RedisCounter(redis=redis, a=3, b=1) 146 | d = RedisCounter(redis=redis, a=1, b=2) 147 | e = c - d 148 | assert isinstance(e, collections.Counter) 149 | assert e == collections.Counter(a=2) 150 | 151 | 152 | def test_or(redis: Redis) -> None: 153 | 'Test RedisCounter.__or__()' 154 | c = RedisCounter(redis=redis, a=3, b=1) 155 | d = RedisCounter(redis=redis, a=1, b=2) 156 | e = c | d 157 | assert isinstance(e, collections.Counter) 158 | assert e == collections.Counter(a=3, b=2) 159 | 160 | 161 | def test_and(redis: Redis) -> None: 162 | 'Test RedisCounter.__and__()' 163 | c = RedisCounter(redis=redis, a=3, b=1) 164 | d = RedisCounter(redis=redis, a=1, b=2) 165 | e = c & d 166 | assert isinstance(e, collections.Counter) 167 | assert e == collections.Counter(a=1, b=1) 168 | 169 | 170 | def test_pos(redis: Redis) -> None: 171 | 'Test RedisCounter.__pos__()' 172 | c = RedisCounter(redis=redis, foo=-2, bar=-1, baz=0, qux=1) 173 | assert isinstance(+c, collections.Counter) 174 | assert +c == collections.Counter(qux=1) 175 | 176 | 177 | def test_neg(redis: Redis) -> None: 178 | 'Test RedisCounter.__neg__()' 179 | c = RedisCounter(redis=redis, foo=-2, bar=-1, baz=0, qux=1) 180 | assert isinstance(-c, collections.Counter) 181 | assert -c == collections.Counter(foo=2, bar=1) 182 | 183 | 184 | def test_in_place_add_with_empty_counter(redis: Redis) -> None: 185 | 'Test RedisCounter.__iadd__() with an empty counter' 186 | c = RedisCounter(redis=redis, a=1, b=2) 187 | d = RedisCounter(redis=redis) 188 | c += d # type: ignore 189 | assert isinstance(c, RedisCounter) 190 | assert c == collections.Counter(a=1, b=2) 191 | 192 | 193 | def test_in_place_add_with_overlapping_counter(redis: Redis) -> None: 194 | 'Test RedisCounter.__iadd__() with a counter with overlapping keys' 195 | c = RedisCounter(redis=redis, a=4, b=2, c=0, d=-2) 196 | d = RedisCounter(redis=redis, a=1, b=2, c=3, d=4) 197 | c += d # type: ignore 198 | assert isinstance(c, RedisCounter) 199 | assert c == collections.Counter(a=5, b=4, c=3, d=2) 200 | 201 | 202 | def test_in_place_add_removes_zeroes(redis: Redis) -> None: 203 | c = RedisCounter(redis=redis, a=4, b=2, c=0, d=-2) 204 | d: Counter[str] = collections.Counter(a=-4, b=-2, c=0, d=2) 205 | c += d # type: ignore 206 | assert isinstance(c, RedisCounter) 207 | assert c == collections.Counter() 208 | 209 | 210 | def test_in_place_subtract(redis: Redis) -> None: 211 | 'Test RedisCounter.__isub__()' 212 | c = RedisCounter(redis=redis, a=4, b=2, c=0, d=-2) 213 | d = RedisCounter(redis=redis, a=1, b=2, c=3, d=4) 214 | c -= d # type: ignore 215 | assert isinstance(c, RedisCounter) 216 | assert c == collections.Counter(a=3) 217 | 218 | 219 | def test_in_place_or_with_two_empty_counters(redis: Redis) -> None: 220 | 'Test RedisCounter.__ior__() with two empty counters' 221 | c = RedisCounter(redis=redis) 222 | d = RedisCounter(redis=redis) 223 | c |= d # type: ignore 224 | assert isinstance(c, RedisCounter) 225 | assert c == collections.Counter() 226 | 227 | 228 | def test_in_place_or_with_two_overlapping_counters(redis: Redis) -> None: 229 | 'Test RedisCounter.__ior__() with two counters with overlapping keys' 230 | c = RedisCounter(redis=redis, a=4, b=2, c=0, d=-2) 231 | d: Counter[str] = collections.Counter(a=1, b=2, c=3, d=4) 232 | c |= d # type: ignore 233 | assert isinstance(c, RedisCounter) 234 | assert c == collections.Counter(a=4, b=2, c=3, d=4) 235 | 236 | 237 | def test_in_place_and(redis: Redis) -> None: 238 | 'Test RedisCounter.__iand__()' 239 | c = RedisCounter(redis=redis, a=4, b=2, c=0, d=-2) 240 | d = RedisCounter(redis=redis, a=1, b=2, c=3, d=4) 241 | c &= d # type: ignore 242 | assert isinstance(c, RedisCounter) 243 | assert c == collections.Counter(a=1, b=2) 244 | 245 | 246 | def test_in_place_and_results_in_empty_counter(redis: Redis) -> None: 247 | c = RedisCounter(redis=redis, a=4, b=2) 248 | d = RedisCounter(redis=redis, c=3, d=4) 249 | c &= d # type: ignore 250 | assert isinstance(c, RedisCounter) 251 | assert c == collections.Counter() 252 | 253 | 254 | def test_method_resolution_order() -> None: 255 | # We need for _Common to come ahead of collections.Counter in the 256 | # inheritance chain. Because when we instantiate a RedisCounter, we 257 | # need to hit _Common's .__init__() first, which *doesn't* delegate to 258 | # super().__init__(), which *prevents* collections.Counter's 259 | # .__init__() from getting hit; and this is the correct behavior. 260 | # 261 | # RedisCounter inherits from collections.Counter for method reuse; not 262 | # to initialize a collections.Counter and store key/value pairs in 263 | # memory. 264 | # 265 | # Inspired by Raymond Hettinger's excellent Python's super() considered 266 | # super! 267 | # https://rhettinger.wordpress.com/2011/05/26/super-considered-super/ 268 | position = RedisCounter.mro().index 269 | assert position(_Common) < position(collections.Counter) 270 | -------------------------------------------------------------------------------- /tests/test_deque.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # test_deque.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | '''These tests come from these examples: 18 | https://docs.python.org/3/library/collections.html#collections.deque 19 | ''' 20 | 21 | 22 | import collections 23 | import itertools 24 | import unittest.mock 25 | from typing import Any 26 | from typing import Generator 27 | from typing import Iterable 28 | from typing import cast 29 | 30 | import pytest 31 | from redis import Redis 32 | 33 | from pottery import RedisDeque 34 | from pottery import RedisList 35 | from pottery.base import Container 36 | 37 | 38 | def test_basic_usage(redis: Redis) -> None: 39 | d = RedisDeque('ghi', redis=redis) 40 | assert d == collections.deque(['g', 'h', 'i']) 41 | 42 | d.append('j') 43 | d.appendleft('f') 44 | assert d == collections.deque(['f', 'g', 'h', 'i', 'j']) 45 | 46 | assert d.pop() == 'j' 47 | assert d.popleft() == 'f' 48 | assert d == collections.deque(['g', 'h', 'i']) 49 | assert d[0] == 'g' 50 | assert d[-1] == 'i' 51 | 52 | assert list(reversed(d)) == ['i', 'h', 'g'] 53 | assert 'h' in d 54 | d.extend('jkl') 55 | assert d == collections.deque(['g', 'h', 'i', 'j', 'k', 'l']) 56 | d.rotate(1) 57 | assert d == collections.deque(['l', 'g', 'h', 'i', 'j', 'k']) 58 | d.rotate(-1) 59 | assert d == collections.deque(['g', 'h', 'i', 'j', 'k', 'l']) 60 | 61 | assert RedisDeque(reversed(d), redis=redis) == collections.deque(['l', 'k', 'j', 'i', 'h', 'g']) 62 | d.clear() 63 | with pytest.raises(IndexError): 64 | d.pop() 65 | 66 | d.extendleft('abc') 67 | assert d == collections.deque(['c', 'b', 'a']) 68 | 69 | 70 | def test_init_with_wrong_type_maxlen(redis: Redis) -> None: 71 | with unittest.mock.patch.object(Container, '__del__') as delete, \ 72 | pytest.raises(TypeError): 73 | delete.return_value = None 74 | RedisDeque(redis=redis, maxlen='2') # type: ignore 75 | 76 | 77 | def test_init_with_maxlen(redis: Redis) -> None: 78 | d = RedisDeque([1, 2, 3, 4, 5, 6], redis=redis, maxlen=3) 79 | assert d == collections.deque([4, 5, 6]) 80 | 81 | d = RedisDeque([1, 2, 3, 4, 5, 6], redis=redis, maxlen=0) 82 | assert d == collections.deque() 83 | 84 | 85 | def test_persistent_deque_bigger_than_maxlen(redis: Redis) -> None: 86 | d1 = RedisDeque('ghi', redis=redis) 87 | d1 # Workaround for Pyflakes. :-( 88 | with pytest.raises(IndexError): 89 | RedisDeque(redis=redis, key=d1.key, maxlen=0) 90 | 91 | 92 | def test_maxlen_not_writable(redis: Redis) -> None: 93 | d = RedisDeque(redis=redis) 94 | with pytest.raises(AttributeError): 95 | d.maxlen = 2 96 | 97 | 98 | def test_insert_into_full(redis: Redis) -> None: 99 | d = RedisDeque('gh', redis=redis, maxlen=3) 100 | d.insert(len(d), 'i') 101 | assert d == collections.deque(['g', 'h', 'i']) 102 | 103 | with pytest.raises(IndexError): 104 | d.insert(len(d), 'j') 105 | 106 | 107 | def test_append_trims_when_full(redis: Redis) -> None: 108 | d = RedisDeque('gh', redis=redis, maxlen=3) 109 | d.append('i') 110 | assert d == collections.deque(['g', 'h', 'i']) 111 | d.append('j') 112 | assert d == collections.deque(['h', 'i', 'j']) 113 | d.appendleft('g') 114 | assert d == collections.deque(['g', 'h', 'i']) 115 | 116 | 117 | def test_extend(redis: Redis) -> None: 118 | d = RedisDeque('ghi', redis=redis, maxlen=4) 119 | d.extend('jkl') 120 | assert d == collections.deque(['i', 'j', 'k', 'l']) 121 | d.extendleft('hg') 122 | assert d == collections.deque(['g', 'h', 'i', 'j']) 123 | 124 | 125 | def test_popleft_from_empty(redis: Redis) -> None: 126 | d = RedisDeque(redis=redis) 127 | with pytest.raises(IndexError): 128 | d.popleft() 129 | 130 | 131 | @pytest.mark.parametrize('invalid_steps', (None, 'a', 0.5)) 132 | def test_invalid_rotating(redis: Redis, invalid_steps: Any) -> None: 133 | d = RedisDeque(('g', 'h', 'i', 'j', 'k', 'l'), redis=redis) 134 | with pytest.raises(TypeError): 135 | d.rotate(invalid_steps) 136 | 137 | 138 | def test_rotate_zero_steps(redis: Redis) -> None: 139 | 'Rotating 0 steps is a no-op' 140 | d = RedisDeque(('g', 'h', 'i', 'j', 'k', 'l'), redis=redis) 141 | d.rotate(0) 142 | assert d == collections.deque(['g', 'h', 'i', 'j', 'k', 'l']) 143 | 144 | 145 | def test_rotate_empty_deque(redis: Redis) -> None: 146 | 'Rotating an empty RedisDeque is a no-op' 147 | d = RedisDeque(redis=redis) 148 | d.rotate(2) 149 | assert d == collections.deque() 150 | 151 | 152 | def test_rotate_right(redis: Redis) -> None: 153 | 'A positive number rotates a RedisDeque right' 154 | # I got this example from here: 155 | # https://pymotw.com/2/collections/deque.html#rotating 156 | d = RedisDeque(range(10), redis=redis) 157 | d.rotate(2) 158 | assert d == collections.deque([8, 9, 0, 1, 2, 3, 4, 5, 6, 7]) 159 | 160 | 161 | def test_rotate_left(redis: Redis) -> None: 162 | 'A negative number rotates a RedisDeque left' 163 | # I got this example from here: 164 | # https://pymotw.com/2/collections/deque.html#rotating 165 | d = RedisDeque(range(10), redis=redis) 166 | d.rotate(-2) 167 | assert d == collections.deque([2, 3, 4, 5, 6, 7, 8, 9, 0, 1]) 168 | 169 | 170 | def test_moving_average(redis: Redis) -> None: 171 | 'Test RedisDeque-based moving average' 172 | 173 | # I got this recipe from here: 174 | # https://docs.python.org/3.9/library/collections.html#deque-recipes 175 | def moving_average(iterable: Iterable[int], n: int = 3) -> Generator[float, None, None]: 176 | it = iter(iterable) 177 | d = RedisDeque(itertools.islice(it, n-1), redis=redis) 178 | d.appendleft(0) 179 | s = sum(d) 180 | for elem in it: 181 | s += elem - cast(int, d.popleft()) 182 | d.append(elem) 183 | yield s / n 184 | 185 | seq = list(moving_average([40, 30, 50, 46, 39, 44])) 186 | assert seq == [40.0, 42.0, 45.0, 43.0] 187 | 188 | 189 | def test_delete_nth(redis: Redis) -> None: 190 | 'Recipe for deleting the nth element from a RedisDeque' 191 | d = RedisDeque(('g', 'h', 'i', 'j', 'k', 'l'), redis=redis) 192 | 193 | # Delete the 3rd element in the deque, or the 'j'. I got this recipe 194 | # from here: 195 | # https://docs.python.org/3.9/library/collections.html#deque-recipes 196 | d.rotate(-3) 197 | e = d.popleft() 198 | d.rotate(3) 199 | assert e == 'j' 200 | assert d == collections.deque(['g', 'h', 'i', 'k', 'l']) 201 | 202 | 203 | def test_truthiness(redis: Redis) -> None: 204 | d = RedisDeque('ghi', redis=redis) 205 | assert bool(d) 206 | d.clear() 207 | assert not bool(d) 208 | 209 | 210 | def test_repr(redis: Redis) -> None: 211 | d = RedisDeque(redis=redis) 212 | assert repr(d) == 'RedisDeque([])' 213 | 214 | d = RedisDeque('ghi', redis=redis) 215 | assert repr(d) == "RedisDeque(['g', 'h', 'i'])" 216 | 217 | d = RedisDeque(redis=redis, maxlen=2) 218 | assert repr(d) == 'RedisDeque([], maxlen=2)' 219 | 220 | d = RedisDeque('ghi', redis=redis, maxlen=2) 221 | assert repr(d) == "RedisDeque(['h', 'i'], maxlen=2)" 222 | 223 | 224 | def test_eq_redislist_same_redis_key(redis: Redis) -> None: 225 | deque = RedisDeque('ghi', redis=redis) 226 | list_ = RedisList(redis=redis, key=deque.key) 227 | assert not deque == list_ 228 | assert deque != list_ 229 | -------------------------------------------------------------------------------- /tests/test_dict.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # test_dict.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | '''These tests come from these examples: 18 | https://docs.python.org/3/tutorial/datastructures.html#dictionaries 19 | ''' 20 | 21 | 22 | import collections 23 | import json 24 | 25 | import pytest 26 | from redis import Redis 27 | 28 | from pottery import KeyExistsError 29 | from pottery import RedisDict 30 | 31 | 32 | def test_keyexistserror_raised(redis: Redis) -> None: 33 | d = RedisDict( 34 | redis=redis, 35 | key='pottery:tel', 36 | sape=4139, 37 | guido=4127, 38 | jack=4098, 39 | ) 40 | d # Workaround for Pyflakes. :-( 41 | with pytest.raises(KeyExistsError): 42 | RedisDict( 43 | redis=redis, 44 | key='pottery:tel', 45 | sape=4139, 46 | guido=4127, 47 | jack=4098, 48 | ) 49 | 50 | 51 | def test_keyexistserror_repr(redis: Redis) -> None: 52 | d = RedisDict( 53 | redis=redis, 54 | key='pottery:tel', 55 | sape=4139, 56 | guido=4127, 57 | jack=4098, 58 | ) 59 | d # Workaround for Pyflakes. :-( 60 | try: 61 | RedisDict( 62 | redis=redis, 63 | key='pottery:tel', 64 | sape=4139, 65 | guido=4127, 66 | jack=4098, 67 | ) 68 | except KeyExistsError as wtf: 69 | redis_db = redis.get_connection_kwargs()['db'] # type: ignore 70 | assert repr(wtf) == ( 71 | f"KeyExistsError(redis=)>)>, key='pottery:tel')" 72 | ) 73 | else: # pragma: no cover 74 | pytest.fail(reason='KeyExistsError not raised') 75 | 76 | 77 | def test_basic_usage(redis: Redis) -> None: 78 | tel = RedisDict(redis=redis, jack=4098, sape=4139) 79 | tel['guido'] = 4127 80 | assert tel == {'sape': 4139, 'guido': 4127, 'jack': 4098} 81 | assert tel['jack'] == 4098 82 | del tel['sape'] 83 | tel['irv'] = 4127 84 | assert tel == {'guido': 4127, 'irv': 4127, 'jack': 4098} 85 | assert sorted(tel) == ['guido', 'irv', 'jack'] 86 | assert 'guido' in tel 87 | assert not 'jack' not in tel 88 | 89 | 90 | def test_init_with_key_value_pairs(redis: Redis) -> None: 91 | d = RedisDict([('sape', 4139), ('guido', 4127), ('jack', 4098)], redis=redis) 92 | assert d == {'sape': 4139, 'jack': 4098, 'guido': 4127} 93 | 94 | 95 | def test_init_with_kwargs(redis: Redis) -> None: 96 | d = RedisDict(redis=redis, sape=4139, guido=4127, jack=4098) 97 | assert d == {'sape': 4139, 'jack': 4098, 'guido': 4127} 98 | 99 | # The following tests come from these examples: 100 | # https://docs.python.org/3.4/library/stdtypes.html#mapping-types-dict 101 | 102 | 103 | def test_more_construction_options(redis: Redis) -> None: 104 | a = RedisDict(redis=redis, one=1, two=2, three=3) 105 | b = {'one': 1, 'two': 2, 'three': 3} 106 | c = RedisDict(zip(['one', 'two', 'three'], [1, 2, 3]), redis=redis) 107 | d = RedisDict([('two', 2), ('one', 1), ('three', 3)], redis=redis) 108 | e = RedisDict({'three': 3, 'one': 1, 'two': 2}, redis=redis) # type: ignore 109 | assert a == b == c == d == e 110 | 111 | 112 | def test_len(redis: Redis) -> None: 113 | a = RedisDict(redis=redis) 114 | assert len(a) == 0 115 | a = RedisDict(redis=redis, one=1, two=2, three=3) 116 | assert len(a) == 3 117 | a['four'] = 4 118 | assert len(a) == 4 119 | del a['four'] 120 | assert len(a) == 3 121 | 122 | 123 | def test_repr(redis: Redis) -> None: 124 | a = RedisDict(redis=redis, one=1, two=2) 125 | assert repr(a) in { 126 | "RedisDict{'one': 1, 'two': 2}", 127 | "RedisDict{'two': 2, 'one': 1}", 128 | } 129 | 130 | 131 | def test_update(redis: Redis) -> None: 132 | a = RedisDict(redis=redis, one=1, two=2, three=3) 133 | a.update() 134 | assert a == {'one': 1, 'two': 2, 'three': 3} 135 | 136 | a.update({'four': 4, 'five': 5}) # type: ignore 137 | assert a == {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5} 138 | 139 | a.update((('six', 6), ('seven', 7))) 140 | assert a == { 141 | 'one': 1, 142 | 'two': 2, 143 | 'three': 3, 144 | 'four': 4, 145 | 'five': 5, 146 | 'six': 6, 147 | 'seven': 7, 148 | } 149 | 150 | a.update(eight=8, nine=9) 151 | assert a == { 152 | 'one': 1, 153 | 'two': 2, 154 | 'three': 3, 155 | 'four': 4, 156 | 'five': 5, 157 | 'six': 6, 158 | 'seven': 7, 159 | 'eight': 8, 160 | 'nine': 9, 161 | } 162 | 163 | 164 | def test_keyerror(redis: Redis) -> None: 165 | a = RedisDict(redis=redis, one=1, two=2, three=3) 166 | assert a['one'] == 1 167 | assert a['two'] == 2 168 | assert a['three'] == 3 169 | with pytest.raises(KeyError): 170 | a['four'] 171 | 172 | 173 | def test_key_assignment(redis: Redis) -> None: 174 | a = RedisDict(redis=redis, one=1, two=2, three=2) 175 | assert a['three'] == 2 176 | a['three'] = 3 177 | assert a['three'] == 3 178 | a['four'] = 4 179 | assert a['four'] == 4 180 | 181 | 182 | def test_key_deletion(redis: Redis) -> None: 183 | a = RedisDict(redis=redis, one=1, two=2, three=3) 184 | assert sorted(a) == ['one', 'three', 'two'] 185 | a['four'] = 4 186 | assert sorted(a) == ['four', 'one', 'three', 'two'] 187 | with pytest.raises(KeyError): 188 | del a['five'] 189 | del a['four'] 190 | assert sorted(a) == ['one', 'three', 'two'] 191 | del a['three'] 192 | assert sorted(a) == ['one', 'two'] 193 | del a['two'] 194 | assert sorted(a) == ['one'] 195 | del a['one'] 196 | assert sorted(a) == [] 197 | with pytest.raises(KeyError): 198 | del a['one'] 199 | 200 | 201 | def test_key_membership(redis: Redis) -> None: 202 | a = RedisDict(redis=redis, one=1, two=2, three=3) 203 | assert 'one' in a 204 | assert 'four' not in a 205 | assert not 'four' in a 206 | a['four'] = 4 207 | assert 'four' in a 208 | del a['four'] 209 | assert 'four' not in a 210 | assert not 'four' in a 211 | 212 | 213 | def test_clear(redis: Redis) -> None: 214 | a = RedisDict(redis=redis, one=1, two=2, three=3) 215 | assert sorted(a) == ['one', 'three', 'two'] 216 | assert a.clear() is None # type: ignore 217 | assert sorted(a) == [] 218 | assert a.clear() is None # type: ignore 219 | assert sorted(a) == [] 220 | 221 | 222 | def test_get(redis: Redis) -> None: 223 | a = RedisDict(redis=redis, one=1, two=2, three=3) 224 | assert a.get('one') == 1 225 | assert a.get('one', 42) == 1 226 | assert a.get('two') == 2 227 | assert a.get('two', 42) == 2 228 | assert a.get('three') == 3 229 | assert a.get('three', 42) == 3 230 | assert a.get('four') is None 231 | assert a.get('four', 42) == 42 232 | a['four'] = 4 233 | assert a.get('four') == 4 234 | assert a.get('four', 42) == 4 235 | del a['four'] 236 | assert a.get('four') is None 237 | assert a.get('four', 42) == 42 238 | 239 | 240 | def test_items(redis: Redis) -> None: 241 | a = RedisDict(redis=redis, one=1, two=2, three=3) 242 | assert isinstance(a.items(), collections.abc.ItemsView) 243 | assert len(a) == 3 244 | assert set(a.items()) == {('one', 1), ('two', 2), ('three', 3)} 245 | assert ('one', 1) in a.items() 246 | assert ('four', 4) not in a.items() 247 | 248 | 249 | def test_keys(redis: Redis) -> None: 250 | a = RedisDict(redis=redis, one=1, two=2, three=3) 251 | assert isinstance(a.keys(), collections.abc.KeysView) 252 | assert len(a) == 3 253 | assert set(a.keys()) == {'one', 'two', 'three'} 254 | assert 'one' in a.keys() 255 | assert 'four' not in a.keys() 256 | 257 | 258 | def test_values(redis: Redis) -> None: 259 | a = RedisDict(redis=redis, one=1, two=2, three=3) 260 | assert isinstance(a.values(), collections.abc.ValuesView) 261 | assert len(a) == 3 262 | assert set(a.values()) == {1, 2, 3} 263 | assert 1 in a.values() 264 | assert 4 not in a.values() 265 | 266 | 267 | def test_membership_for_non_jsonifyable_element(redis: Redis) -> None: 268 | redis_dict = RedisDict(redis=redis) 269 | assert not BaseException in redis_dict 270 | 271 | 272 | def test_json_dumps(redis: Redis) -> None: 273 | a = RedisDict(redis=redis, one=1, two=2, three=3) 274 | assert json.dumps(a) == '{"one": 1, "two": 2, "three": 3}' 275 | 276 | 277 | def test_eq_same_redis_database_and_key(redis: Redis) -> None: 278 | a = RedisDict(redis=redis, one=1, two=2, three=3) 279 | b = RedisDict(redis=a.redis, key=a.key) 280 | assert a == b 281 | -------------------------------------------------------------------------------- /tests/test_doctests.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # test_doctests.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | import doctest 20 | import importlib 21 | import pathlib 22 | from types import ModuleType 23 | from typing import Generator 24 | 25 | import pytest 26 | from redis import Redis 27 | 28 | 29 | def modules() -> Generator[ModuleType, None, None]: 30 | tests_dir = pathlib.Path(__file__).parent 31 | package_dir = tests_dir.parent 32 | source_dir = package_dir / 'pottery' 33 | source_files = source_dir.glob('**/*.py') 34 | for source_file in source_files: 35 | relative_path = source_file.relative_to(package_dir) 36 | parts = list(relative_path.parts) 37 | parts[-1] = source_file.stem 38 | module_name = '.'.join(parts) 39 | module = importlib.import_module(module_name) 40 | yield module 41 | 42 | 43 | @pytest.mark.parametrize('module', modules()) 44 | def test_modules(module: ModuleType) -> None: 45 | 'Run doctests in modules and confirm that they are not science fiction' 46 | results = doctest.testmod(m=module) 47 | assert not results.failed 48 | 49 | 50 | @pytest.fixture 51 | def flush_redis() -> Generator[None, None, None]: 52 | redis = Redis.from_url('redis://localhost:6379/1') 53 | redis.flushdb() 54 | yield 55 | redis.flushdb() 56 | 57 | 58 | @pytest.mark.usefixtures('flush_redis') 59 | def test_readme() -> None: 60 | 'Run doctests in README.md and confirm that they are not fake news' 61 | tests_dir = pathlib.Path(__file__).parent 62 | package_dir = tests_dir.parent 63 | readme = str(package_dir / 'README.md') 64 | results = doctest.testfile(readme, module_relative=False) 65 | assert not results.failed 66 | -------------------------------------------------------------------------------- /tests/test_executor.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # test_executor.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | import concurrent.futures 20 | import time 21 | 22 | from pottery.executor import BailOutExecutor 23 | 24 | 25 | def test_threadpoolexecutor() -> None: 26 | 'ThreadPoolExecutor waits for futures to complete on .__exit__()' 27 | with concurrent.futures.ThreadPoolExecutor() as executor: 28 | future = executor.submit(time.sleep, 0.1) 29 | assert not future.running() 30 | 31 | with concurrent.futures.ThreadPoolExecutor() as executor: 32 | future1 = executor.submit(time.sleep, 0.1) 33 | future2 = executor.submit(time.sleep, 0.2) 34 | future1.result() 35 | assert not future1.running() 36 | assert not future2.running() 37 | 38 | 39 | def test_bailoutexecutor() -> None: 40 | 'BailOutExecutor does not wait for futures to complete on .__exit__()' 41 | with BailOutExecutor() as executor: 42 | future = executor.submit(time.sleep, 0.1) 43 | assert future.running() 44 | time.sleep(0.15) 45 | assert not future.running() 46 | 47 | with BailOutExecutor() as executor: 48 | future1 = executor.submit(time.sleep, 0.1) 49 | future2 = executor.submit(time.sleep, 0.2) 50 | future1.result() 51 | assert not future1.running() 52 | assert future2.running() 53 | time.sleep(0.15) 54 | assert not future2.running() 55 | -------------------------------------------------------------------------------- /tests/test_hyper.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # test_hyper.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | import uuid 20 | 21 | import pytest 22 | from redis import Redis 23 | 24 | from pottery import HyperLogLog 25 | 26 | 27 | def test_init_without_iterable(redis: Redis) -> None: 28 | hll = HyperLogLog(redis=redis) 29 | assert len(hll) == 0 30 | 31 | 32 | def test_init_with_iterable(redis: Redis) -> None: 33 | hll = HyperLogLog({'foo', 'bar', 'zap', 'a'}, redis=redis) 34 | assert len(hll) == 4 35 | 36 | 37 | def test_add(redis: Redis) -> None: 38 | hll = HyperLogLog(redis=redis) 39 | hll.add('foo') 40 | assert len(hll) == 1 41 | 42 | hll.add('bar') 43 | assert len(hll) == 2 44 | 45 | hll.add('zap') 46 | assert len(hll) == 3 47 | 48 | hll.add('a') 49 | assert len(hll) == 4 50 | 51 | hll.add('a') 52 | assert len(hll) == 4 53 | 54 | hll.add('b') 55 | assert len(hll) == 5 56 | 57 | hll.add('c') 58 | assert len(hll) == 6 59 | 60 | hll.add('foo') 61 | assert len(hll) == 6 62 | 63 | 64 | def test_update(redis: Redis) -> None: 65 | hll1 = HyperLogLog({'foo', 'bar', 'zap', 'a'}, redis=redis) 66 | hll2 = HyperLogLog({'a', 'b', 'c', 'foo'}, redis=redis) 67 | hll1.update(hll2) 68 | assert len(hll1) == 6 69 | 70 | hll1 = HyperLogLog({'foo', 'bar', 'zap', 'a'}, redis=redis) 71 | hll1.update({'b', 'c', 'd', 'foo'}) 72 | assert len(hll1) == 7 73 | 74 | hll1 = HyperLogLog({'foo', 'bar', 'zap', 'a'}, redis=redis) 75 | hll1.update(hll2, {'b', 'c', 'd', 'baz'}) 76 | assert len(hll1) == 8 77 | 78 | 79 | def test_update_different_redis_instances(redis: Redis) -> None: 80 | hll1 = HyperLogLog({'foo', 'bar', 'zap', 'a'}, redis=redis) 81 | hll2 = HyperLogLog(redis=Redis()) 82 | with pytest.raises(RuntimeError): 83 | hll1.update(hll2) 84 | 85 | 86 | def test_union(redis: Redis) -> None: 87 | hll1 = HyperLogLog({'foo', 'bar', 'zap', 'a'}, redis=redis) 88 | hll2 = HyperLogLog({'a', 'b', 'c', 'foo'}, redis=redis) 89 | assert len(hll1.union(hll2, redis=redis)) == 6 90 | assert len(hll1.union({'b', 'c', 'd', 'foo'}, redis=redis)) == 7 91 | assert len(hll1.union(hll2, {'b', 'c', 'd', 'baz'}, redis=redis)) == 8 92 | 93 | 94 | @pytest.mark.parametrize('metasyntactic_variable', ('foo', 'bar')) 95 | def test_contains_metasyntactic_variables(redis: Redis, metasyntactic_variable: str) -> None: 96 | metasyntactic_variables = HyperLogLog({'foo', 'bar', 'zap', 'a'}, redis=redis) 97 | assert metasyntactic_variable in metasyntactic_variables 98 | 99 | 100 | @pytest.mark.parametrize('metasyntactic_variable', ('baz', 'qux')) 101 | def test_doesnt_contain_metasyntactic_variables(redis: Redis, metasyntactic_variable: str) -> None: 102 | metasyntactic_variables = HyperLogLog({'foo', 'bar', 'zap', 'a'}, redis=redis) 103 | assert metasyntactic_variable not in metasyntactic_variables 104 | 105 | 106 | def test_contains_many_uuids(redis: Redis) -> None: 107 | NUM_ELEMENTS = 5000 108 | known_uuids, unknown_uuids = [], [] 109 | generate_uuid = lambda: str(uuid.uuid4()) # NoQA: E731 110 | for _ in range(NUM_ELEMENTS): 111 | known_uuids.append(generate_uuid()) # type: ignore 112 | unknown_uuids.append(generate_uuid()) # type: ignore 113 | uuid_hll = HyperLogLog(known_uuids, redis=redis) 114 | num_known_contained = sum(uuid_hll.contains_many(*known_uuids)) 115 | num_unknown_contained = sum(uuid_hll.contains_many(*unknown_uuids)) 116 | assert num_known_contained == NUM_ELEMENTS 117 | assert num_unknown_contained <= NUM_ELEMENTS * 0.25, \ 118 | f'{num_unknown_contained} is not <= {NUM_ELEMENTS * 0.25}' 119 | 120 | 121 | def test_membership_for_non_jsonifyable_element(redis: Redis) -> None: 122 | hll = HyperLogLog(redis=redis) 123 | assert not BaseException in hll # type: ignore 124 | 125 | 126 | def test_repr(redis: Redis) -> None: 127 | 'Test HyperLogLog.__repr__()' 128 | hll = HyperLogLog({'foo', 'bar', 'zap', 'a'}, redis=redis, key='hll') 129 | assert repr(hll) == '' 130 | -------------------------------------------------------------------------------- /tests/test_list.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # test_list.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | '''These tests come from these examples: 18 | https://docs.python.org/3/tutorial/introduction.html#lists 19 | https://docs.python.org/3/tutorial/datastructures.html#more-on-lists 20 | ''' 21 | 22 | 23 | import json 24 | from typing import Any 25 | 26 | import pytest 27 | from redis import Redis 28 | 29 | from pottery import KeyExistsError 30 | from pottery import RedisDeque 31 | from pottery import RedisList 32 | 33 | 34 | KEY = 'squares' 35 | 36 | 37 | def test_indexerror(redis: Redis) -> None: 38 | list_ = RedisList(redis=redis) 39 | with pytest.raises(IndexError): 40 | list_[0] = 'raj' # type: ignore 41 | 42 | 43 | def test_keyexistserror(redis: Redis) -> None: 44 | squares = RedisList([1, 4, 9, 16, 25], redis=redis, key=KEY) 45 | squares # Workaround for Pyflakes. :-( 46 | with pytest.raises(KeyExistsError): 47 | RedisList([1, 4, 9, 16, 25], redis=redis, key=KEY) 48 | 49 | 50 | def test_init_empty_list(redis: Redis) -> None: 51 | squares = RedisList(redis=redis, key=KEY) 52 | assert squares == [] 53 | 54 | 55 | def test_basic_usage(redis: Redis) -> None: 56 | squares = RedisList([1, 4, 9, 16, 25], redis=redis) 57 | assert squares == [1, 4, 9, 16, 25] 58 | assert squares[0] == 1 59 | assert squares[-1] == 25 60 | assert squares[-3:] == [9, 16, 25] 61 | assert squares[:] == [1, 4, 9, 16, 25] 62 | assert squares + [36, 49, 64, 81, 100] == \ 63 | [1, 4, 9, 16, 25, 36, 49, 64, 81, 100] 64 | 65 | 66 | def test_mutability_and_append(redis: Redis) -> None: 67 | cubes = RedisList([1, 8, 27, 65, 125], redis=redis) 68 | cubes[3] = 64 # type: ignore 69 | assert cubes == [1, 8, 27, 64, 125] 70 | cubes.append(216) 71 | cubes.append(7**3) 72 | assert cubes == [1, 8, 27, 64, 125, 216, 343] 73 | 74 | 75 | def test_slicing(redis: Redis) -> None: 76 | letters = RedisList(['a', 'b', 'c', 'd', 'e', 'f', 'g'], redis=redis) 77 | assert letters == ['a', 'b', 'c', 'd', 'e', 'f', 'g'] 78 | assert letters[2:5] == ['c', 'd', 'e'] 79 | assert letters[2:5:2] == ['c', 'e'] 80 | assert letters[2:5:3] == ['c'] 81 | assert letters[2:5:4] == ['c'] 82 | letters[2:5] = ['C', 'D', 'E'] # type: ignore 83 | assert letters == ['a', 'b', 'C', 'D', 'E', 'f', 'g'] 84 | letters[2:5:2] = [None, None] # type: ignore 85 | assert letters == ['a', 'b', None, 'D', None, 'f', 'g'] 86 | letters[2:5] = [] # type: ignore 87 | assert letters == ['a', 'b', 'f', 'g'] 88 | letters[:] = [] # type: ignore 89 | assert letters == [] 90 | 91 | 92 | def test_len(redis: Redis) -> None: 93 | letters = RedisList(['a', 'b', 'c', 'd'], redis=redis) 94 | assert len(letters) == 4 95 | 96 | 97 | def test_nesting(redis: Redis) -> None: 98 | a = ['a', 'b', 'c'] 99 | n = [1, 2, 3] 100 | x = RedisList([a, n], redis=redis) 101 | assert x == [['a', 'b', 'c'], [1, 2, 3]] 102 | assert x[0] == ['a', 'b', 'c'] 103 | assert x[0][1] == 'b' 104 | 105 | 106 | def test_more_on_lists(redis: Redis) -> None: 107 | a = RedisList([66.25, 333, 333, 1, 1234.5], redis=redis) 108 | assert (a.count(333), a.count(66.25), a.count('x')) == (2, 1, 0) 109 | a.insert(2, -1) 110 | a.append(333) 111 | assert a == [66.25, 333, -1, 333, 1, 1234.5, 333] 112 | assert a.index(333) == 1 113 | a.remove(333) 114 | assert a == [66.25, -1, 333, 1, 1234.5, 333] 115 | a.reverse() 116 | assert a == [333, 1234.5, 1, 333, -1, 66.25] 117 | a.sort() 118 | assert a == [-1, 1, 66.25, 333, 333, 1234.5] 119 | assert a.pop() == 1234.5 120 | assert a == [-1, 1, 66.25, 333, 333] 121 | 122 | 123 | def test_using_list_as_stack(redis: Redis) -> None: 124 | stack = RedisList([3, 4, 5], redis=redis) 125 | stack.append(6) 126 | stack.append(7) 127 | assert stack == [3, 4, 5, 6, 7] 128 | assert stack.pop() == 7 129 | assert stack == [3, 4, 5, 6] 130 | assert stack.pop() == 6 131 | assert stack.pop() == 5 132 | assert stack == [3, 4] 133 | 134 | 135 | def test_del(redis: Redis) -> None: 136 | a = RedisList([-1, 1, 66.25, 333, 333, 1234.5], redis=redis) 137 | del a[0] # type: ignore 138 | assert a == [1, 66.25, 333, 333, 1234.5] 139 | del a[2:4] # type: ignore 140 | assert a == [1, 66.25, 1234.5] 141 | del a[:0] # type: ignore 142 | assert a == [1, 66.25, 1234.5] 143 | del a[:] # type: ignore 144 | assert a == [] 145 | 146 | 147 | def test_insert_left(redis: Redis) -> None: 148 | squares = RedisList([9, 16, 25], redis=redis) 149 | squares.insert(-1, 4) 150 | assert squares == [4, 9, 16, 25] 151 | squares.insert(0, 1) 152 | assert squares == [1, 4, 9, 16, 25] 153 | 154 | 155 | def test_insert_middle(redis: Redis) -> None: 156 | nums = RedisList([0, 0, 0, 0], redis=redis) 157 | nums.insert(2, 2) 158 | assert nums == [0, 0, 2, 0, 0] 159 | 160 | 161 | def test_insert_right(redis: Redis) -> None: 162 | squares = RedisList([1, 4, 9], redis=redis) 163 | squares.insert(100, 16) 164 | squares.insert(100, 25) 165 | assert squares == [1, 4, 9, 16, 25] 166 | 167 | 168 | def test_extend(redis: Redis) -> None: 169 | squares = RedisList([1, 4, 9], redis=redis) 170 | squares.extend([16, 25]) 171 | assert squares == [1, 4, 9, 16, 25] 172 | 173 | 174 | def test_sort(redis: Redis) -> None: 175 | squares = RedisList({1, 4, 9, 16, 25}, redis=redis) 176 | squares.sort() 177 | assert squares == [1, 4, 9, 16, 25] 178 | 179 | squares.sort(reverse=True) 180 | assert squares == [25, 16, 9, 4, 1] 181 | 182 | with pytest.raises(NotImplementedError): 183 | squares.sort(key=str) # type: ignore 184 | 185 | 186 | def test_eq_redisdeque_same_redis_key(redis: Redis) -> None: 187 | list_ = RedisList([1, 4, 9, 16, 25], redis=redis, key=KEY) 188 | deque = RedisDeque(redis=redis, key=KEY) 189 | assert not list_ == deque 190 | assert list_ != deque 191 | 192 | 193 | def test_eq_same_object(redis: Redis) -> None: 194 | squares = RedisList([1, 4, 9, 16, 25], redis=redis, key=KEY) 195 | assert squares == squares 196 | assert not squares != squares 197 | 198 | 199 | def test_eq_same_redis_instance_and_key(redis: Redis) -> None: 200 | squares1 = RedisList([1, 4, 9, 16, 25], redis=redis, key=KEY) 201 | squares2 = RedisList(redis=redis, key=KEY) 202 | assert squares1 == squares2 203 | assert not squares1 != squares2 204 | 205 | 206 | def test_eq_same_redis_instance_different_keys(redis: Redis) -> None: 207 | key1 = 'squares1' 208 | key2 = 'squares2' 209 | squares1 = RedisList([1, 4, 9, 16, 25], redis=redis, key=key1) 210 | squares2 = RedisList([1, 4, 9, 16, 25], redis=redis, key=key2) 211 | assert squares1 == squares2 212 | assert not squares1 != squares2 213 | 214 | 215 | def test_eq_different_lengths(redis: Redis) -> None: 216 | squares1 = RedisList([1, 4, 9, 16, 25], redis=redis) 217 | squares2 = [1, 4, 9, 16, 25, 36] 218 | assert not squares1 == squares2 219 | assert squares1 != squares2 220 | 221 | 222 | def test_eq_different_items(redis: Redis) -> None: 223 | squares1 = RedisList([1, 4, 9, 16, 25], redis=redis) 224 | squares2 = [4, 9, 16, 25, 36] 225 | assert not squares1 == squares2 226 | assert squares1 != squares2 227 | 228 | 229 | def test_eq_unordered_collection(redis: Redis) -> None: 230 | squares1 = RedisList([1], redis=redis) 231 | squares2 = {1} 232 | assert not squares1 == squares2 233 | assert squares1 != squares2 234 | 235 | 236 | def test_eq_immutable_sequence(redis: Redis) -> None: 237 | squares1 = RedisList([1, 4, 9, 16, 25], redis=redis) 238 | squares2 = (1, 4, 9, 16, 25) 239 | assert not squares1 == squares2 240 | assert squares1 != squares2 241 | 242 | 243 | def test_eq_typeerror(redis: Redis) -> None: 244 | squares = RedisList([1, 4, 9, 16, 25], redis=redis) 245 | assert not squares == None 246 | assert squares != None 247 | 248 | 249 | def test_repr(redis: Redis) -> None: 250 | squares = RedisList([1, 4, 9, 16, 25], redis=redis) 251 | assert repr(squares) == 'RedisList[1, 4, 9, 16, 25]' 252 | 253 | 254 | def test_pop_out_of_range(redis: Redis) -> None: 255 | squares = RedisList([1, 4, 9, 16, 25], redis=redis) 256 | with pytest.raises(IndexError): 257 | squares.pop(len(squares)) 258 | 259 | 260 | def test_pop_index(redis: Redis) -> None: 261 | metasyntactic = RedisList( 262 | ['foo', 'bar', 'baz', 'qux', 'quux', 'corge', 'grault', 'garply', 'waldo', 'fred', 'plugh', 'xyzzy', 'thud'], 263 | redis=redis, 264 | ) 265 | assert metasyntactic.pop(1) == 'bar' 266 | 267 | 268 | def test_remove_nonexistent(redis: Redis) -> None: 269 | metasyntactic = RedisList( 270 | ['foo', 'bar', 'baz', 'qux', 'quux', 'corge', 'grault', 'garply', 'waldo', 'fred', 'plugh', 'xyzzy', 'thud'], 271 | redis=redis, 272 | ) 273 | with pytest.raises(ValueError): 274 | metasyntactic.remove('raj') 275 | 276 | 277 | def test_json_dumps(redis: Redis) -> None: 278 | metasyntactic = RedisList( 279 | ['foo', 'bar', 'baz', 'qux', 'quux', 'corge', 'grault', 'garply', 'waldo', 'fred', 'plugh', 'xyzzy', 'thud'], 280 | redis=redis, 281 | ) 282 | assert json.dumps(metasyntactic) == ( 283 | '["foo", "bar", "baz", "qux", "quux", "corge", "grault", "garply", ' 284 | '"waldo", "fred", "plugh", "xyzzy", "thud"]' 285 | ) 286 | 287 | 288 | @pytest.mark.parametrize('invalid_slice', (None, 'a', 0.5)) 289 | def test_invalid_slicing(redis: Redis, invalid_slice: Any) -> None: 290 | letters = RedisList(['a', 'b', 'c', 'd'], redis=redis) 291 | with pytest.raises(TypeError): 292 | letters[invalid_slice] 293 | 294 | 295 | def test_extended_slicing(redis: Redis) -> None: 296 | python_list = [1, 2, 3, 4, 5] 297 | redis_list = RedisList(python_list, redis=redis) 298 | assert redis_list[len(redis_list)-1:3-1:-1] == python_list[len(python_list)-1:3-1:-1] 299 | 300 | 301 | def test_slice_notation(redis: Redis) -> None: 302 | # I got these examples from: 303 | # https://railsware.com/blog/python-for-machine-learning-indexing-and-slicing-for-lists-tuples-strings-and-other-sequential-types/#Slice_Notation 304 | nums = RedisList([10, 20, 30, 40, 50, 60, 70, 80, 90], redis=redis) 305 | assert nums[2:7] == [30, 40, 50, 60, 70] 306 | assert nums[0:4] == [10, 20, 30, 40] 307 | assert nums[:5] == [10, 20, 30, 40, 50] 308 | assert nums[-3:] == [70, 80, 90] 309 | assert nums[1:-1] == [20, 30, 40, 50, 60, 70, 80] 310 | assert nums[-3:8] == [70, 80] 311 | assert nums[-5:-1] == [50, 60, 70, 80] 312 | assert nums[:-2] == [10, 20, 30, 40, 50, 60, 70] 313 | assert nums[::2] == [10, 30, 50, 70, 90] 314 | assert nums[1::2] == [20, 40, 60, 80] 315 | assert nums[1:-3:2] == [20, 40, 60] 316 | assert nums[::-1] == [90, 80, 70, 60, 50, 40, 30, 20, 10] 317 | assert nums[-2::-1] == [80, 70, 60, 50, 40, 30, 20, 10] 318 | assert nums[-2:1:-1] == [80, 70, 60, 50, 40, 30] 319 | assert nums[-2:1:-3] == [80, 50] 320 | 321 | 322 | def test_invalid_slice_assignment(redis: Redis) -> None: 323 | nums = RedisList([10, 20, 30, 40, 50, 60, 70, 80, 90], redis=redis) 324 | with pytest.raises(TypeError): 325 | nums[:] = 10 # type: ignore 326 | -------------------------------------------------------------------------------- /tests/test_monkey.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # test_monkey.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | import json 20 | 21 | import pytest 22 | from redis import Redis 23 | 24 | from pottery import RedisDict 25 | from pottery import RedisList 26 | 27 | 28 | def test_typeerror_not_jsonifyable() -> None: 29 | "Ensure json.dumps() raises TypeError for objs that can't be serialized" 30 | try: 31 | json.dumps(object()) 32 | except TypeError as error: 33 | assert str(error) == 'Object of type object is not JSON serializable' 34 | 35 | 36 | def test_dict() -> None: 37 | 'Ensure that json.dumps() can serialize a dict' 38 | assert json.dumps({}) == '{}' 39 | 40 | 41 | def test_redisdict(redis: Redis) -> None: 42 | 'Ensure that json.dumps() can serialize a RedisDict' 43 | dict_ = RedisDict(redis=redis) 44 | assert json.dumps(dict_) == '{}' 45 | 46 | 47 | def test_list() -> None: 48 | 'Ensure that json.dumps() can serialize a list' 49 | assert json.dumps([]) == '[]' 50 | 51 | 52 | def test_redislist(redis: Redis) -> None: 53 | 'Ensure that json.dumps() can serialize a RedisList' 54 | list_ = RedisList(redis=redis) 55 | assert json.dumps(list_) == '[]' 56 | 57 | 58 | def test_json_encoder(redis: Redis) -> None: 59 | 'Ensure that we can pass in the cls keyword argument to json.dumps()' 60 | dict_ = RedisDict(redis=redis) 61 | with pytest.raises(TypeError): 62 | json.dumps(dict_, cls=None) 63 | -------------------------------------------------------------------------------- /tests/test_nextid.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # test_nextid.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 'Distributed Redis-powered monotonically increasing ID generator tests.' 18 | 19 | 20 | import concurrent.futures 21 | import contextlib 22 | import unittest.mock 23 | from typing import Generator 24 | 25 | import pytest 26 | from redis import Redis 27 | from redis.commands.core import Script 28 | from redis.exceptions import TimeoutError 29 | 30 | from pottery import NextID 31 | from pottery import QuorumIsImpossible 32 | from pottery import QuorumNotAchieved 33 | 34 | 35 | @pytest.fixture 36 | def ids(redis: Redis) -> Generator[NextID, None, None]: 37 | redis.unlink('nextid:current') 38 | yield NextID(masters={redis}) 39 | redis.unlink('nextid:current') 40 | 41 | 42 | def test_nextid(ids: NextID) -> None: 43 | for id_ in range(1, 10): 44 | assert next(ids) == id_ 45 | 46 | 47 | def test_iter(ids: NextID) -> None: 48 | assert iter(ids) is ids 49 | 50 | 51 | def test_reset(ids: NextID) -> None: 52 | ids.reset() 53 | for redis in ids.masters: 54 | assert not redis.exists(ids.key) 55 | 56 | assert next(ids) == 1 57 | for redis in ids.masters: 58 | assert redis.exists(ids.key) 59 | 60 | ids.reset() 61 | for redis in ids.masters: 62 | assert not redis.exists(ids.key) 63 | 64 | assert next(ids) == 1 65 | for redis in ids.masters: 66 | assert redis.exists(ids.key) 67 | 68 | 69 | @pytest.mark.parametrize('num_ids', range(1, 6)) 70 | def test_contention(num_ids: int) -> None: 71 | dbs = range(1, 6) 72 | urls = [f'redis://localhost:6379/{db}' for db in dbs] 73 | masters = [Redis.from_url(url, socket_timeout=1) for url in urls] 74 | ids = [NextID(key='tweet-ids', masters=masters) for _ in range(num_ids)] 75 | 76 | try: 77 | results = [] 78 | with concurrent.futures.ThreadPoolExecutor() as executor: 79 | futures = [executor.submit(next, i) for i in ids] # type: ignore 80 | for future in concurrent.futures.as_completed(futures): 81 | with contextlib.suppress(QuorumNotAchieved): 82 | result = future.result() 83 | results.append(result) 84 | assert len(results) == len(set(results)) 85 | # To see the following output, issue: 86 | # $ source venv/bin/activate; pytest -rP tests/test_nextid.py::test_contention; deactivate 87 | print(f'{num_ids} ids, {results} IDs') 88 | 89 | finally: 90 | ids[0].reset() 91 | 92 | 93 | def test_repr(ids: NextID) -> None: 94 | assert repr(ids) == '' 95 | 96 | 97 | def test_slots(ids: NextID) -> None: 98 | with pytest.raises(AttributeError): 99 | ids.__dict__ 100 | 101 | 102 | def test_next_quorumnotachieved(ids: NextID) -> None: 103 | with pytest.raises(QuorumNotAchieved), \ 104 | unittest.mock.patch.object(next(iter(ids.masters)), 'get') as get: 105 | get.side_effect = TimeoutError 106 | next(ids) 107 | 108 | with pytest.raises(QuorumNotAchieved), \ 109 | unittest.mock.patch.object(Script, '__call__') as __call__: 110 | __call__.side_effect = TimeoutError 111 | next(ids) 112 | 113 | 114 | def test_next_quorumisimpossible(redis: Redis) -> None: 115 | ids = NextID(masters={redis}, raise_on_redis_errors=True) 116 | with pytest.raises(QuorumIsImpossible), \ 117 | unittest.mock.patch.object(next(iter(ids.masters)), 'get') as get: 118 | get.side_effect = TimeoutError 119 | next(ids) 120 | 121 | with pytest.raises(QuorumIsImpossible), \ 122 | unittest.mock.patch.object(Script, '__call__') as __call__: 123 | __call__.side_effect = TimeoutError 124 | next(ids) 125 | 126 | 127 | def test_reset_quorumnotachieved(ids: NextID) -> None: 128 | with pytest.raises(QuorumNotAchieved), \ 129 | unittest.mock.patch.object(next(iter(ids.masters)), 'delete') as delete: 130 | delete.side_effect = TimeoutError 131 | ids.reset() 132 | 133 | 134 | def test_reset_quorumisimpossible(redis: Redis) -> None: 135 | ids = NextID(masters={redis}, raise_on_redis_errors=True) 136 | with pytest.raises(QuorumIsImpossible), \ 137 | unittest.mock.patch.object(next(iter(ids.masters)), 'delete') as delete: 138 | delete.side_effect = TimeoutError 139 | ids.reset() 140 | -------------------------------------------------------------------------------- /tests/test_queue.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # test_queue.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | import pytest 20 | from redis import Redis 21 | 22 | from pottery import ContextTimer 23 | from pottery import QueueEmptyError 24 | from pottery import RedisSimpleQueue 25 | 26 | 27 | @pytest.fixture 28 | def queue(redis: Redis) -> RedisSimpleQueue: 29 | return RedisSimpleQueue(redis=redis) 30 | 31 | 32 | def test_put(queue: RedisSimpleQueue) -> None: 33 | assert queue.qsize() == 0 34 | assert queue.empty() 35 | 36 | for num in range(1, 6): 37 | queue.put(num) 38 | assert queue.qsize() == num 39 | assert not queue.empty() 40 | 41 | 42 | def test_put_nowait(queue: RedisSimpleQueue) -> None: 43 | assert queue.qsize() == 0 44 | assert queue.empty() 45 | 46 | for num in range(1, 6): 47 | queue.put_nowait(num) 48 | assert queue.qsize() == num 49 | assert not queue.empty() 50 | 51 | 52 | def test_get(queue: RedisSimpleQueue) -> None: 53 | with pytest.raises(QueueEmptyError): 54 | queue.get() 55 | 56 | for num in range(1, 6): 57 | queue.put(num) 58 | assert queue.get() == num 59 | assert queue.qsize() == 0 60 | assert queue.empty() 61 | 62 | with pytest.raises(QueueEmptyError): 63 | queue.get() 64 | 65 | 66 | def test_get_nowait(queue: RedisSimpleQueue) -> None: 67 | with pytest.raises(QueueEmptyError): 68 | queue.get_nowait() 69 | 70 | for num in range(1, 6): 71 | queue.put(num) 72 | 73 | for num in range(1, 6): 74 | assert queue.get_nowait() == num 75 | assert queue.qsize() == 5 - num 76 | assert queue.empty() == (num == 5) 77 | 78 | with pytest.raises(QueueEmptyError): 79 | queue.get_nowait() 80 | 81 | 82 | def test_get_timeout(queue: RedisSimpleQueue) -> None: 83 | timeout = 1 84 | with pytest.raises(QueueEmptyError), ContextTimer() as timer: 85 | queue.get(timeout=1) 86 | assert timer.elapsed() / 1000 >= timeout 87 | -------------------------------------------------------------------------------- /tests/test_timer.py: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- # 2 | # test_timer.py # 3 | # # 4 | # Copyright © 2015-2025, Rajiv Bakulesh Shah, original author. # 5 | # # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); # 7 | # you may not use this file except in compliance with the License. # 8 | # You may obtain a copy of the License at: # 9 | # http://www.apache.org/licenses/LICENSE-2.0 # 10 | # # 11 | # Unless required by applicable law or agreed to in writing, software # 12 | # distributed under the License is distributed on an "AS IS" BASIS, # 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 14 | # See the License for the specific language governing permissions and # 15 | # limitations under the License. # 16 | # --------------------------------------------------------------------------- # 17 | 18 | 19 | import time 20 | 21 | import pytest 22 | 23 | from pottery import ContextTimer 24 | 25 | 26 | @pytest.fixture 27 | def timer() -> ContextTimer: 28 | return ContextTimer() 29 | 30 | 31 | def confirm_elapsed(timer: ContextTimer, expected: int) -> None: 32 | ACCURACY = 50 # in milliseconds 33 | elapsed = timer.elapsed() 34 | assert elapsed >= expected, f'elapsed ({elapsed}) is not >= expected ({expected})' 35 | assert elapsed < expected + ACCURACY, f'elapsed ({elapsed}) is not < expected ({expected + ACCURACY})' 36 | 37 | 38 | def test_start_stop_and_elapsed(timer: ContextTimer) -> None: 39 | # timer hasn't been started 40 | with pytest.raises(RuntimeError): 41 | timer.elapsed() 42 | with pytest.raises(RuntimeError): 43 | timer.stop() 44 | 45 | # timer has been started but not stopped 46 | timer.start() 47 | with pytest.raises(RuntimeError): 48 | timer.start() 49 | time.sleep(0.1) 50 | confirm_elapsed(timer, 1*100) 51 | timer.stop() 52 | 53 | # timer has been stopped 54 | with pytest.raises(RuntimeError): 55 | timer.start() 56 | time.sleep(0.1) 57 | confirm_elapsed(timer, 1*100) 58 | with pytest.raises(RuntimeError): 59 | timer.stop() 60 | 61 | 62 | def test_context_manager(timer: ContextTimer) -> None: 63 | with timer: 64 | confirm_elapsed(timer, 0) 65 | for iteration in range(1, 3): 66 | time.sleep(0.1) 67 | confirm_elapsed(timer, iteration*100) 68 | confirm_elapsed(timer, iteration*100) 69 | time.sleep(0.1) 70 | confirm_elapsed(timer, iteration*100) 71 | 72 | with pytest.raises(RuntimeError), timer: # pragma: no cover 73 | ... 74 | --------------------------------------------------------------------------------