├── .github └── workflows │ ├── ci.yml │ ├── linting.yml │ ├── mypy.yml │ └── release.yml ├── .gitignore ├── .readthedocs.yml ├── LICENSE ├── README.md ├── docs ├── Makefile ├── conf.py ├── configuration.md ├── examples.md ├── index.rst ├── introduction.md ├── logo.webp ├── make.bat ├── pged.md ├── setup.md └── strategies.md ├── pyproject.toml ├── src └── pgcachewatch │ ├── __init__.py │ ├── __main__.py │ ├── cli.py │ ├── decorators.py │ ├── listeners.py │ ├── logconfig.py │ ├── models.py │ ├── pg_event_distributor.py │ ├── queries.py │ ├── strategies.py │ └── utils.py └── tests ├── conftest.py ├── db ├── Dockerfile └── init_db.sh ├── test_decoraters.py ├── test_fastapi.py ├── test_integration.py ├── test_listeners.py ├── test_pg_event_distributor.py ├── test_strategies.py └── test_utils.py /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Build and test pgcachewatch 2 | 3 | on: 4 | pull_request: 5 | branches: [main] 6 | push: 7 | branches: [main] 8 | 9 | jobs: 10 | ci: 11 | strategy: 12 | fail-fast: false 13 | matrix: 14 | python-version: ["3.10", "3.11", "3.12"] 15 | postgres-version: ["14", "15", "16"] 16 | os: [ubuntu-latest] 17 | 18 | name: PY ${{ matrix.python-version }} on ${{ matrix.os }} using PG ${{ matrix.postgres-version }} 19 | runs-on: ${{ matrix.os }} 20 | 21 | steps: 22 | - uses: actions/checkout@v4 23 | 24 | - name: Set up Python 25 | uses: actions/setup-python@v4 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | 29 | - name: Build custom PostgreSQL Docker image 30 | run: | 31 | docker build \ 32 | --build-arg POSTGRES_VERSION=${{ matrix.postgres-version }} \ 33 | -t custom-postgres:latest tests/db/ 34 | env: 35 | POSTGRES_VERSION: ${{ matrix.postgres-version }} 36 | 37 | - name: Start PostgreSQL container 38 | run: | 39 | docker run -d --network host --name postgres custom-postgres:latest 40 | 41 | - name: Install pgcachewatch 42 | run: | 43 | pip install pip -U 44 | pip install .[dev] 45 | 46 | - name: Wait for PostgreSQL to become ready 47 | run: | 48 | for i in {1..10}; do 49 | docker exec postgres pg_isready && break 50 | sleep 5 51 | done 52 | 53 | - name: Check PostgreSQL Container Logs 54 | run: docker logs postgres 55 | 56 | - name: Full test 57 | run: pytest -v 58 | 59 | check: 60 | name: Check test matrix passed. 61 | needs: ci 62 | runs-on: ubuntu-latest 63 | steps: 64 | - name: Check status 65 | run: echo "All tests passed; ready to merge." 66 | -------------------------------------------------------------------------------- /.github/workflows/linting.yml: -------------------------------------------------------------------------------- 1 | name: linting 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | linting: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | python-version: ["3.10"] 11 | steps: 12 | - uses: actions/checkout@v4 13 | - name: Set up python 14 | uses: actions/setup-python@v4 15 | - name: Install dev-env. 16 | run: | 17 | pip install -U pip 18 | pip install ".[dev]" 19 | - name: Ruff check 20 | if: ${{ always() }} 21 | run: ruff check . 22 | - name: Ruff format 23 | if: ${{ always() }} 24 | run: ruff format . --check 25 | -------------------------------------------------------------------------------- /.github/workflows/mypy.yml: -------------------------------------------------------------------------------- 1 | name: Mypy 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | mypy: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | python-version: ["3.10"] 11 | steps: 12 | - uses: actions/checkout@v4 13 | 14 | - name: Set up python 15 | uses: actions/setup-python@v4 16 | with: 17 | python-version: ${{ matrix.python-version }} 18 | 19 | - name: Install dev-env. 20 | run: | 21 | pip install -U pip 22 | pip install ".[dev]" 23 | 24 | - name: Mypy 25 | run: mypy . 26 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Publish Python distribution to PyPI 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | build-and-publish: 9 | name: Publish Python distribution to PyPI. 10 | runs-on: ubuntu-latest 11 | 12 | environment: 13 | name: release 14 | url: https://pypi.org/project/PGCacheWatch/ 15 | 16 | permissions: 17 | id-token: write 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | with: 22 | fetch-depth: 0 # Ensures tags are also fetched 23 | 24 | - name: Set up Python 3.10. 25 | uses: actions/setup-python@v4 26 | with: 27 | python-version: "3.10" 28 | 29 | - name: Install release dependencies. 30 | run: python3 -m pip install build twine setuptools_scm 31 | 32 | - name: Build package. 33 | run: python3 -m build . --sdist --wheel --outdir dist/ 34 | 35 | - uses: pypa/gh-action-pypi-publish@release/v1 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | 132 | # vscode project settings 133 | .vscode/ 134 | 135 | # Dynamic version 136 | src/pgcachewatch/_version.py 137 | .DS_Store 138 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: "ubuntu-22.04" 5 | tools: 6 | python: "3.11" 7 | 8 | sphinx: 9 | configuration: docs/conf.py 10 | 11 | python: 12 | install: 13 | - method: pip 14 | path: .[docs] 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## 🚀 PGCacheWatch - Supercharge Your Caching Strategy 🚀 2 | [![CI](https://github.com/janbjorge/PGCacheWatch/actions/workflows/ci.yml/badge.svg)](https://github.com/janbjorge/PGCacheWatch/actions/workflows/ci.yml?query=branch%3Amain) 3 | [![pypi](https://img.shields.io/pypi/v/PGCacheWatch.svg)](https://pypi.python.org/pypi/PGCacheWatch) 4 | [![downloads](https://static.pepy.tech/badge/PGCacheWatch/month)](https://pepy.tech/project/PGCacheWatch) 5 | [![versions](https://img.shields.io/pypi/pyversions/PGCacheWatch.svg)](https://github.com/janbjorge/PGCacheWatch) 6 | 7 | --- 8 | 📚 **Documentation**: [Explore the Docs 📖](https://pgcachewatch.readthedocs.io/en/latest/) 9 | 10 | 🔍 **Source Code**: [View on GitHub 💾](https://github.com/janbjorge/PGCacheWatch/) 11 | 12 | --- 13 | PGCacheWatch is the a Python library designed to propel your applications into a new realm of efficiency with real-time PostgreSQL event notifications for cache invalidation. Wave goodbye to stale data and hello to seamless cache management, bolstered performance powered by the robust backbone of PostgreSQL. 14 | 15 | ## Example with FastAPI 16 | PGCacheWatch integrates with FastAPI, empowering you to keep your application's data fresh and consistent by dynamically invalidating cache in line with database updates. 17 | 18 | ```python 19 | import contextlib 20 | import typing 21 | 22 | import asyncpg 23 | from fastapi import FastAPI 24 | from pgcachewatch import decorators, listeners, models, strategies 25 | 26 | # Initialize a PGEventQueue listener to listen for database events. 27 | listener = listeners.PGEventQueue() 28 | 29 | @contextlib.asynccontextmanager 30 | async def app_setup_teardown(_: FastAPI) -> typing.AsyncGenerator[None, None]: 31 | """ 32 | Asynchronous context manager for FastAPI app setup and teardown. 33 | 34 | This context manager is used to establish and close the database connection 35 | at the start and end of the FastAPI application lifecycle, respectively. 36 | """ 37 | # Establish a database connection using asyncpg. 38 | conn = await asyncpg.connect() 39 | # Connect the listener to the database using the specified channel. 40 | await listener.connect(conn) 41 | 42 | try: 43 | yield 44 | finally: 45 | await conn.close() # Ensure the database connection is closed on app teardown. 46 | 47 | # Create an instance of FastAPI, specifying the app setup and teardown actions. 48 | APP = FastAPI(lifespan=app_setup_teardown) 49 | 50 | # Decorate the cached_query function with cache invalidation logic. 51 | @decorators.cache( 52 | strategy=strategies.Greedy( 53 | listener=listener, 54 | # Invalidate the cache only for 'update' operations on the database. 55 | predicate=lambda x: x.operation == "update", 56 | ) 57 | ) 58 | async def cached_query(user_id: int) -> dict[str, str]: 59 | """ 60 | Simulates a database query that benefits from cache invalidation. 61 | 62 | This function is decorated to use PGCacheWatch's cache invalidation, ensuring 63 | that the data returned is up-to-date following any relevant 'update' operations 64 | on the database. 65 | """ 66 | # Return a mock data response. 67 | return {"data": "query result"} 68 | 69 | # Define a FastAPI route to fetch data, utilizing the cached_query function. 70 | @APP.get("/data") 71 | async def get_data(user_id: int) -> dict: 72 | """ 73 | This endpoint uses the cached_query function to return data, demonstrating 74 | how cache invalidation can be integrated into a web application route. 75 | """ 76 | # Fetch and return the data using the cached query function. 77 | return await cached_query(user_id) 78 | ``` 79 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | project = "PGCacheWatch" 2 | copyright = "2024, JeeyBee" 3 | author = "JeeyBee" 4 | extensions = ["myst_parser"] 5 | templates_path = ["_templates"] 6 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 7 | html_static_path = ["_static"] 8 | pygments_style = "sphinx" 9 | html_theme = "sphinx_rtd_theme" 10 | -------------------------------------------------------------------------------- /docs/configuration.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | Configuring PGCacheWatch for optimal operation with PostgreSQL involves careful consideration of how you establish and manage your database connections, particularly when using asyncpg for asynchronous communication. Here's a refined approach to configuration, taking into account the nuances of connection pooling, listener persistence, and environmental configuration options. 3 | 4 | ### Using Connection Pools with PGCacheWatch 5 | 6 | While asyncpg’s connection pooling is a powerful feature for managing database connections efficiently, it requires careful handling when used with LISTEN/NOTIFY channels due to the nature of persistent listeners. 7 | 8 | #### Persistent Listeners 9 | When a connection from the pool is used to set up LISTEN commands for notifications, it's important to keep that connection dedicated to listening. Returning the connection to the pool would remove the listeners, as the connection could be reused for other database operations, disrupting the notification flow. 10 | 11 | #### Dedicated Listener Connection 12 | To maintain persistent LISTEN operations, establish a dedicated connection outside the pool specifically for this purpose. This ensures that the NOTIFY listeners remain active throughout the application's lifecycle. 13 | 14 | ```python 15 | # Dedicated listener connection 16 | listener_conn = await asyncpg.connect(dsn="postgres://user:password@localhost/dbname") 17 | 18 | # Connection pool for other database operations 19 | pool = await asyncpg.create_pool(dsn="postgres://user:password@localhost/dbname") 20 | ``` 21 | 22 | ### Best Practices for Configuration 23 | 24 | - Security: Always use secure methods (like environment variables or secret management tools) to store and access database credentials, avoiding hard-coded values. 25 | - Connection Pooling: Utilize connection pooling for handling database operations but maintain a separate, dedicated connection for LISTEN/NOTIFY to ensure continuous event listening. 26 | - SSL Configuration: Enable SSL for database connections in production environments to secure data in transit. This can be configured via connection parameters or environment variables. 27 | -------------------------------------------------------------------------------- /docs/examples.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | PGCacheWatch offers a versatile approach to leveraging PostgreSQL's NOTIFY/LISTEN capabilities for real-time event handling and cache invalidation in Python applications. Below are a few detailed examples of how PGCacheWatch can be integrated into different scenarios, providing solutions to common problems encountered in web development and data management. 4 | 5 | ## Automating User Data Enrichment 6 | 7 | ### Challenge 8 | When new users register on a platform, it's common to need additional information that isn't immediately provided at the time of sign-up. Fetching and updating this information in real-time can enhance user profiles and improve the user experience. 9 | 10 | ### Approach 11 | This example demonstrates using PGCacheWatch for real-time user data enrichment upon new registrations. By listening for new user events, the system can asynchronously fetch additional information from external services and update the user's profile without manual intervention. 12 | 13 | ```python 14 | import asyncio 15 | import asyncpg 16 | from pgcachewatch import listeners, models 17 | 18 | # Process new user events for data enrichment 19 | async def process_new_user_event() -> None: 20 | ... 21 | 22 | 23 | # Main listener function for new user events 24 | async def listen_for_new_users() -> None: 25 | conn = await asyncpg.connect() 26 | listener = listeners.PGEventQueue() 27 | await listener.connect(conn) 28 | 29 | try: 30 | print("Listening for new user events...") 31 | while event := await listener.get(): 32 | if event.operation == "insert": 33 | await process_new_user_event() 34 | finally: 35 | await conn.close() 36 | 37 | if __name__ == "__main__": 38 | asyncio.run(listen_for_new_users()) 39 | ``` 40 | 41 | ## Integrating with FastAPI 42 | 43 | ### Challenge 44 | In web applications, ensuring data returned to the user is fresh and consistent with the database can be challenging, especially after updates. Traditional cache invalidation strategies often result in stale data or unnecessary database queries. 45 | 46 | ### Approach 47 | This example shows how to integrate PGCacheWatch with FastAPI to dynamically invalidate cache following database changes. This ensures data freshness and consistency by invalidating the cache only when relevant database changes occur, thus optimizing performance and user experience. 48 | 49 | ```python 50 | import contextlib 51 | import typing 52 | import asyncpg 53 | from fastapi import FastAPI 54 | from pgcachewatch import decorators, listeners, models, strategies 55 | 56 | listener = listeners.PGEventQueue() 57 | 58 | @contextlib.asynccontextmanager 59 | async def app_setup_teardown(_: FastAPI) -> typing.AsyncGenerator[None, None]: 60 | conn = await asyncpg.connect() 61 | await listener.connect(conn) 62 | yield 63 | await conn.close() 64 | 65 | APP = FastAPI(lifespan=app_setup_teardown) 66 | 67 | @decorators.cache(strategy=strategies.Greedy(listener=listener, predicate=lambda x: x.operation == "update")) 68 | async def cached_query(user_id: int) -> dict[str, str]: 69 | return {"data": "query result"} 70 | 71 | @APP.get("/data") 72 | async def get_data(user_id: int) -> dict: 73 | return await cached_query(user_id) 74 | ``` -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to PGCacheWatch's documentation! 2 | ======================================== 3 | 4 | .. figure:: logo.webp 5 | :alt: Logo 6 | :width: 500 7 | :align: center 8 | 9 | PGCacheWatch is a Python library that enhances applications with real-time PostgreSQL event notifications, enabling efficient cache invalidation. It leverages the existing PostgreSQL infrastructure to simplify cache management while ensuring performance and data consistency. 10 | 11 | The repository is hosted on `github `_ 12 | 13 | .. toctree:: 14 | :maxdepth: 1 15 | :caption: Contents: 16 | 17 | introduction 18 | setup 19 | configuration 20 | strategies 21 | pged 22 | examples 23 | -------------------------------------------------------------------------------- /docs/introduction.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | PGCacheWatch is a Python library designed to enhance applications by providing real-time PostgreSQL event notifications, facilitating efficient cache invalidation. It leverages PostgreSQL's existing NOTIFY/LISTEN infrastructure to simplify cache management, ensuring both high performance and data consistency. This document serves as an introduction to PGCacheWatch, covering key features, cache invalidation strategies, and the setup process. 4 | 5 | ## Interaction with PostgreSQL via LISTEN/NOTIFY 6 | 7 | PGCacheWatch utilizes PostgreSQL's LISTEN/NOTIFY mechanism to receive real-time notifications of database changes, enabling applications to invalidate cached data promptly and efficiently. This process involves the following steps: 8 | 9 | 1. **Setting up NOTIFY triggers in PostgreSQL**: Triggers are configured on the database to emit NOTIFY signals upon specified events (e.g., insert, update, delete operations). These triggers call a function that issues the NOTIFY command with a payload containing event details. 10 | 11 | 2. **Listening for notifications in Python**: PGCacheWatch establishes a connection to PostgreSQL and listens on the specified channel(s) for notifications. This is achieved through the asyncpg library, allowing for asynchronous, non-blocking database communication. 12 | 13 | 3. **Processing received notifications**: Upon receiving a notification, PGCacheWatch parses the payload, constructs an event object, and enqueues it for processing. This mechanism enables the application to react to database changes in real-time, ensuring the cache remains up-to-date. 14 | 15 | 4. **Cache Invalidation Strategies**: Depending on the application's requirements, different strategies can be employed to invalidate the cache. These strategies (Greedy, Windowed, Timed) offer varying trade-offs between immediacy and performance, allowing developers to choose the most appropriate approach based on their specific needs. 16 | 17 | ## Example Usage 18 | 19 | The following example demonstrates how to set up a PostgreSQL event queue in PGCacheWatch, connect to a PostgreSQL channel, and listen for events: 20 | 21 | ```python 22 | import asyncio 23 | import asyncpg 24 | 25 | from pgcachewatch.listeners import PGEventQueue 26 | from pgcachewatch.models import PGChannel 27 | 28 | async def main(): 29 | conn = await asyncpg.connect(dsn='postgres://user:password@localhost/dbname') 30 | event_queue = PGEventQueue() 31 | await event_queue.connect(conn) 32 | 33 | try: 34 | print('Listening for events...') 35 | while True: 36 | event = await event_queue.get() 37 | print(f'Received event: {event}') 38 | finally: 39 | await conn.close() 40 | 41 | if __name__ == '__main__': 42 | asyncio.run(main()) 43 | ``` -------------------------------------------------------------------------------- /docs/logo.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janbjorge/PGCacheWatch/9f7e3979ac81b2c55e846ed56638862d6e68743c/docs/logo.webp -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/pged.md: -------------------------------------------------------------------------------- 1 | ## PG Event Distributor 2 | 3 | The PG event distributor, is designed to enhance PGCacheWatch by enabling efficient distribution of PostgreSQL notifications to multiple clients. This service acts as a middleware, receiving notifications from PostgreSQL and broadcasting them to connected clients via WebSockets. This "fan-out" effect ensures real-time cache invalidation across all clients with minimal database connections. 4 | 5 | ### Key Benefits: 6 | 7 | - **Scalability**: Handles numerous clients without increasing load on the PostgreSQL server. 8 | - **Efficiency**: Reduces the need for multiple direct connections to PostgreSQL for NOTIFY/LISTEN. 9 | - **Real-time**: Ensures immediate cache invalidation across services upon database changes. 10 | 11 | ### Illustration: 12 | 13 | ``` 14 | +-------------------+ 15 | | PostgreSQL DB | 16 | | - NOTIFY on event | 17 | +---------+---------+ 18 | | 19 | | NOTIFY 20 | | 21 | +---------v-------------+ 22 | | PG Event Distributor | 23 | | Service | 24 | | - Fan-out NOTIFY | 25 | +---------+-------------+ 26 | | 27 | +-------------+-------------+ 28 | | | 29 | +-------v-------+ +-------v-------+ 30 | | WebSocket | | WebSocket | 31 | | Client 1 | | Client N | 32 | | - Invalidate | | - Invalidate | 33 | | Cache | | Cache | 34 | +---------------+ +---------------+ 35 | ``` 36 | 37 | To leverage the PG Event Distributor within your PGCacheWatch setup, ensure it's running and accessible by your application. Configure PGCacheWatch to connect to the PG Event Distributor instead of directly to PostgreSQL for notifications. This setup amplifies the effectiveness of your cache invalidation strategy by ensuring timely updates across all client caches with optimized resource usage. 38 | 39 | ### Running the PG Event Distributor 40 | To start the PG Event Distributor service, use the following command in your terminal. This command utilizes uvicorn, an ASGI server, to run the service defined in the `pgcachewatch.pg_event_distributor:main` module. The --factory flag is used to indicate that uvicorn should call the provided application factory function to get the ASGI application instance. 41 | 42 | ```bash 43 | uvicorn pgcachewatch.pg_event_distributor:main --factory 44 | ``` 45 | -------------------------------------------------------------------------------- /docs/setup.md: -------------------------------------------------------------------------------- 1 | # Setup and CLI Usage 2 | 3 | Setting up PGCacheWatch involves installing the package, configuring your PostgreSQL database for NOTIFY/LISTEN, and utilizing the provided Command Line Interface (CLI) for managing database triggers and functions. This section outlines the steps to get started and explains the functionalities of the CLI tool. 4 | 5 | ### Installation 6 | 7 | Begin by installing PGCacheWatch using pip: 8 | 9 | ```bash 10 | pip install pgcachewatch 11 | ``` 12 | 13 | This command installs PGCacheWatch along with its dependencies, including `asyncpg` for asynchronous database communication. 14 | 15 | ### Configuring PostgreSQL for NOTIFY/LISTEN 16 | 17 | PGCacheWatch leverages PostgreSQL's NOTIFY/LISTEN mechanism to receive real-time notifications about database changes. To use this feature, you must set up triggers and functions in your database that emit NOTIFY signals on data changes. 18 | 19 | ### Using the PGCacheWatch CLI 20 | 21 | The PGCacheWatch CLI simplifies the process of setting up and managing the necessary database objects for NOTIFY/LISTEN. Here's an overview of the CLI commands and their purposes: 22 | 23 | #### Install Command 24 | Sets up triggers and functions on specified tables to emit NOTIFY signals. This is crucial for initializing PGCacheWatch's event listening capabilities. 25 | 26 | ```bash 27 | python3 -m pgcachewatch install 28 | ``` 29 | ``: Specify one or more table names to set up NOTIFY triggers. The CLI will generate and execute the SQL necessary to create these database objects. 30 | 31 | #### Uninstall Command 32 | Removes the triggers and functions created by the install command, cleaning up the database objects associated with PGCacheWatch. 33 | 34 | ```bash 35 | python3 -m pgcachewatch uninstall 36 | ``` 37 | 38 | ### Best Practices 39 | 40 | - **Testing**: Before applying changes in a production environment, test the CLI commands in a development or staging database to ensure they work as expected. 41 | - **Backup**: Always back up your database before making schema changes, including installing or uninstalling triggers and functions. 42 | - **Documentation**: Keep documentation of the custom options used (channel names, function names, etc.) for future reference or maintenance tasks. 43 | 44 | The PGCacheWatch CLI tool is designed to facilitate the initial setup process, making it easier to integrate real-time PostgreSQL notifications into your applications. By following the above steps and utilizing the CLI, you can streamline the management of database triggers and functions necessary for effective cache invalidation. 45 | -------------------------------------------------------------------------------- /docs/strategies.md: -------------------------------------------------------------------------------- 1 | # Cache Invalidation Strategies 2 | 3 | The cache invalidation strategies provided by PGCacheWatch play a crucial role in maintaining the balance between data freshness and system efficiency. Below are detailed descriptions and ASCII art visualizations for each strategy. 4 | 5 | ## Greedy Strategy 6 | The Greedy strategy is the most straightforward approach, where the cache is invalidated immediately upon any database event that affects the cached data. This strategy ensures the highest level of data freshness by aggressively keeping the cache up-to-date, making it ideal for applications where the accuracy and timeliness of data are paramount. 7 | 8 | ### Visualization 9 | In this visualization, each database event—be it an insert, update, or delete—triggers an immediate cache invalidation, signifying the 'Greedy' nature of this strategy. 10 | ``` 11 | Event Stream: | Insert | Update | Delete | 12 | |--------|--------|--------| 13 | Cache State: Invalidate -> Invalidate -> Invalidate 14 | ``` 15 | 16 | ## Windowed Strategy 17 | The Windowed strategy collects events over a defined period or counts and invalidates the cache only when the specified threshold is reached. This batching approach minimizes the overhead of frequent cache invalidations, making it a suitable choice for applications where slight data staleness is acceptable. The strategy effectively balances system performance with data freshness by reducing the load on the system. 18 | 19 | ### Visualization 20 | Here, the cache is not invalidated at every event. Instead, it waits until a window of either 5 events or 30 seconds is reached before performing a single invalidation, illustrating the 'Windowed' strategy's approach to balancing performance with freshness. 21 | 22 | ``` 23 | Event Stream: | Insert | Insert | Update | ... (5 events within 20 seconds) | 24 | Cache State: Invalidate once after 5th event 25 | ``` 26 | 27 | ## Timed Strategy 28 | The Timed strategy involves invalidating the cache at predetermined time intervals, regardless of the database activity. This strategy provides a predictable pattern of cache invalidation, making it best suited for applications with less dynamic data or where slight delays in data updates are acceptable. The Timed strategy optimizes cache management by ensuring periodic refreshes without the need for monitoring specific database events. 29 | 30 | ### Visualization 31 | In this scenario, cache invalidation occurs strictly based on time intervals (every 10 minutes in this example), regardless of when database events occur. This visualization highlights the 'Timed' strategy's focus on time-based cache refreshes rather than event-driven invalidations. 32 | ``` 33 | Event Stream: | Insert | 10 minutes | Update | 34 | Cache State: Invalidate -> Wait -> Invalidate -> Wait 35 | ``` 36 | 37 | ## Choosing the Right Strategy 38 | Selecting the appropriate cache invalidation strategy requires a thorough assessment of your application's specific needs regarding data freshness, performance implications, and the frequency of data changes. Each strategy offers distinct advantages and trade-offs, making it essential to align the choice with your application's operational requirements and objectives. 39 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | build-backend = "setuptools.build_meta" 3 | requires = ["pip>=22.0.0", "setuptools", "setuptools_scm", "wheel"] 4 | 5 | [project] 6 | authors = [{ name = "janbjorge"}] 7 | description = "A Python library for real-time PostgreSQL event-driven cache invalidation." 8 | dynamic = ["version"] 9 | license = { text = "Apache 2.0" } 10 | name = "PGCacheWatch" 11 | readme = "README.md" 12 | requires-python = ">=3.10" 13 | 14 | classifiers = [ 15 | "Development Status :: 5 - Production/Stable", 16 | "Environment :: Other Environment", 17 | "Environment :: Web Environment", 18 | "Framework :: AsyncIO", 19 | "Intended Audience :: Developers", 20 | "License :: OSI Approved :: Apache Software License", 21 | "Natural Language :: English", 22 | "Operating System :: OS Independent", 23 | "Programming Language :: Python :: 3.10", 24 | "Programming Language :: Python :: 3.11", 25 | "Programming Language :: Python :: 3.12", 26 | "Programming Language :: Python", 27 | "Topic :: Database", 28 | "Topic :: Software Development :: Libraries :: Python Modules", 29 | "Topic :: Utilities", 30 | ] 31 | dependencies = [ 32 | "asyncpg>=0.27.0", 33 | "pydantic>=2.0.0", 34 | "websockets>=12.0.0", 35 | ] 36 | 37 | [project.urls] 38 | Documentation = "https://github.com/janbjorge/pgcachewatch/" 39 | Homepage = "https://github.com/janbjorge/pgcachewatch/" 40 | Issues = "https://github.com/janbjorge/pgcachewatch/issues" 41 | Repository = "https://github.com/janbjorge/pgcachewatch/" 42 | 43 | [project.optional-dependencies] 44 | dev = [ 45 | "asyncpg-stubs", 46 | "fastapi", 47 | "httpx", 48 | "mypy-extensions", 49 | "mypy", 50 | "pytest-asyncio", 51 | "pytest", 52 | "ruff", 53 | "uvicorn", 54 | ] 55 | docs = [ 56 | "myst-parser", 57 | "sphinx", 58 | "sphinx-rtd-theme", 59 | ] 60 | 61 | [tool.setuptools_scm] 62 | write_to = "src/pgcachewatch/_version.py" 63 | 64 | [tool.ruff] 65 | line-length = 88 66 | [tool.ruff.lint] 67 | select = [ 68 | "C", 69 | "E", 70 | "F", 71 | "I", 72 | "PIE", 73 | "Q", 74 | "RET", 75 | "RSE", 76 | "SIM", 77 | "W", 78 | "C90", 79 | ] 80 | [tool.ruff.lint.isort] 81 | combine-as-imports = true 82 | 83 | [tool.mypy] 84 | disallow_untyped_defs = true 85 | exclude = "^(build|docs)" 86 | extra_checks = true 87 | ignore_missing_imports = true 88 | plugins = ["pydantic.mypy"] 89 | python_version = "3.10" 90 | strict_equality = true 91 | warn_redundant_casts = true 92 | warn_unused_configs = true 93 | warn_unused_ignores = true 94 | 95 | [tool.pytest.ini_options] 96 | asyncio_mode = "auto" -------------------------------------------------------------------------------- /src/pgcachewatch/__init__.py: -------------------------------------------------------------------------------- 1 | try: 2 | from ._version import version as __version__ 3 | except ImportError: 4 | __version__ = "unknown" 5 | -------------------------------------------------------------------------------- /src/pgcachewatch/__main__.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from pgcachewatch import cli 4 | 5 | if __name__ == "__main__": 6 | asyncio.run(cli.main()) 7 | -------------------------------------------------------------------------------- /src/pgcachewatch/cli.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import sys 4 | 5 | import asyncpg 6 | 7 | from pgcachewatch import models, queries 8 | 9 | 10 | def cliparser() -> argparse.Namespace: 11 | common_arguments = argparse.ArgumentParser( 12 | add_help=False, 13 | prog="pgcachewatch", 14 | ) 15 | common_arguments.add_argument( 16 | "--channel-name", 17 | default=models.DEFAULT_PG_CHANNE, 18 | help=( 19 | "The PGNotify channel that will be used by pgcachewatch to listen " 20 | "for changes on tables, this should be uniq to pgcachewatch clients." 21 | ), 22 | ) 23 | common_arguments.add_argument( 24 | "--function-name", 25 | default="fn_pgcachewatch_table_change", 26 | help=( 27 | "The prefix of the postgres 'helper function' that emits " 28 | "the on change evnets." 29 | ), 30 | ) 31 | common_arguments.add_argument( 32 | "--trigger-name", 33 | default="tg_pgcachewatch_table_change", 34 | help="All triggers installed on tables will start with this prefix.", 35 | ) 36 | common_arguments.add_argument( 37 | "--commit", 38 | action="store_true", 39 | help="Commit changes to database.", 40 | ) 41 | 42 | common_arguments.add_argument( 43 | "--pg-dsn", 44 | help=( 45 | "Connection string in the libpq URI format, including host, port, user, " 46 | "database, password, passfile, and SSL options. Must be properly quoted; " 47 | "IPv6 addresses must be in brackets. " 48 | "Example: postgres://user:pass@host:port/database. Defaults to PGDSN " 49 | "environment variable if set." 50 | ), 51 | default=os.environ.get("PGDSN"), 52 | ) 53 | 54 | common_arguments.add_argument( 55 | "--pg-host", 56 | help=( 57 | "Database host address, which can be an IP or domain name. " 58 | "Defaults to PGHOST environment variable if set." 59 | ), 60 | default=os.environ.get("PGHOST"), 61 | ) 62 | 63 | common_arguments.add_argument( 64 | "--pg-port", 65 | help=( 66 | "Port number for the server host Defaults to PGPORT environment variable " 67 | "or 5432 if not set." 68 | ), 69 | default=os.environ.get("PGPORT", "5432"), 70 | ) 71 | 72 | common_arguments.add_argument( 73 | "--pg-user", 74 | help=( 75 | "Database role for authentication. Defaults to PGUSER environment " 76 | "variable if set." 77 | ), 78 | default=os.environ.get("PGUSER"), 79 | ) 80 | 81 | common_arguments.add_argument( 82 | "--pg-database", 83 | help=( 84 | "Name of the database to connect to. Defaults to PGDATABASE environment " 85 | "variable if set." 86 | ), 87 | default=os.environ.get("PGDATABASE"), 88 | ) 89 | 90 | common_arguments.add_argument( 91 | "--pg-password", 92 | help=( 93 | "Password for authentication. Defaults to PGPASSWORD " 94 | "environment variable if set" 95 | ), 96 | default=os.environ.get("PGPASSWORD"), 97 | ) 98 | 99 | parser = argparse.ArgumentParser( 100 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 101 | prog="pgcachewatch", 102 | ) 103 | 104 | subparsers = parser.add_subparsers(dest="command", required=True) 105 | 106 | install = subparsers.add_parser( 107 | "install", 108 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 109 | parents=[common_arguments], 110 | ) 111 | install.add_argument("tables", nargs=argparse.ONE_OR_MORE) 112 | 113 | subparsers.add_parser( 114 | "uninstall", 115 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 116 | parents=[common_arguments], 117 | ) 118 | 119 | return parser.parse_args() 120 | 121 | 122 | async def main() -> None: 123 | parsed = cliparser() 124 | 125 | pg_fn_name = f"{parsed.function_name}_{parsed.channel_name}" 126 | pg_tg_name = f"{parsed.trigger_name}_{parsed.channel_name}" 127 | 128 | async with asyncpg.create_pool( 129 | parsed.pg_dsn, 130 | database=parsed.pg_database, 131 | password=parsed.pg_password, 132 | port=parsed.pg_port, 133 | user=parsed.pg_user, 134 | host=parsed.pg_host, 135 | min_size=0, 136 | max_size=1, 137 | ) as pool: 138 | match parsed.command: 139 | case "install": 140 | install = "\n".join( 141 | [ 142 | queries.create_notify_function( 143 | channel_name=parsed.channel_name, 144 | function_name=pg_fn_name, 145 | ) 146 | ] 147 | + [ 148 | queries.create_after_change_trigger( 149 | trigger_name=pg_tg_name, 150 | table_name=table, 151 | function_name=pg_fn_name, 152 | ) 153 | for table in parsed.tables 154 | ] 155 | ) 156 | 157 | print(install, flush=True) 158 | 159 | if parsed.commit: 160 | await pool.execute(install) 161 | else: 162 | print( 163 | "::: Use '--commit' to write changes to db. :::", 164 | file=sys.stderr, 165 | ) 166 | 167 | case "uninstall": 168 | trigger_names = await pool.fetch( 169 | queries.fetch_trigger_names(pg_tg_name), 170 | ) 171 | combined = "\n".join( 172 | ( 173 | "\n".join( 174 | queries.drop_trigger(t["trigger_name"], t["table"]) 175 | for t in trigger_names 176 | ), 177 | queries.drop_function(pg_fn_name), 178 | ) 179 | ) 180 | print(combined, flush=True) 181 | if parsed.commit: 182 | await pool.execute(combined) 183 | else: 184 | print( 185 | "::: Use '--commit' to write changes to db. :::", 186 | file=sys.stderr, 187 | ) 188 | -------------------------------------------------------------------------------- /src/pgcachewatch/decorators.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from functools import _make_key as make_key 3 | from typing import Awaitable, Callable, Hashable, Literal, TypeVar 4 | 5 | from typing_extensions import ParamSpec 6 | 7 | from pgcachewatch import strategies 8 | from pgcachewatch.logconfig import logger 9 | 10 | P = ParamSpec("P") 11 | T = TypeVar("T") 12 | 13 | 14 | def cache( 15 | strategy: strategies.Strategy, 16 | statistics_callback: Callable[[Literal["hit", "miss"]], None] = lambda _: None, 17 | ) -> Callable[[Callable[P, Awaitable[T]]], Callable[P, Awaitable[T]]]: 18 | """ 19 | Decorator for caching asynchronous function calls based on provided 20 | caching strategy. 21 | 22 | This decorator leverages an asynchronous caching strategy to manage cache 23 | entries and ensure efficient data retrieval. The cache is keyed by the 24 | function's arguments, with support for both positional and keyword arguments. 25 | It provides mechanisms for cache invalidation and supports concurrent access by 26 | utilizing asyncio.Future for pending results, effectively preventing cache 27 | stampedes. 28 | 29 | The decorator ensures that: 30 | - If the connection is unhealthy, caching is bypassed, and the 31 | function is executed directly. 32 | - The cache is cleared based on signals from the caching 33 | strategy, indicating data invalidation needs. 34 | - Cache entries are created or retrieved based on the unique call 35 | signature of the decorated function. 36 | - Cache hits and misses are logged and can trigger custom actions 37 | via the statistics_callback. 38 | 39 | Note: This decorator is intended for use with asynchronous functions. 40 | """ 41 | 42 | def outer(fn: Callable[P, Awaitable[T]]) -> Callable[P, Awaitable[T]]: 43 | cached = dict[Hashable, asyncio.Future[T]]() 44 | 45 | async def inner(*args: P.args, **kwargs: P.kwargs) -> T: 46 | # If db-conn is down, disable cache. 47 | if not strategy.connection_healthy(): 48 | logger.critical("Database connection is closed, caching disabled.") 49 | return await fn(*args, **kwargs) 50 | 51 | # Clear cache if we have a event from 52 | # the database the instructs us to clear. 53 | if strategy.clear(): 54 | logger.debug("Cache clear") 55 | cached.clear() 56 | 57 | key = make_key(args, kwargs, typed=False) 58 | 59 | try: 60 | waiter = cached[key] 61 | except KeyError: 62 | # Cache miss 63 | logger.debug("Cache miss") 64 | statistics_callback("miss") 65 | else: 66 | # Cache hit 67 | logger.debug("Cache hit") 68 | statistics_callback("hit") 69 | return await waiter 70 | 71 | # Initialize Future to prevent cache stampedes. 72 | cached[key] = waiter = asyncio.Future[T]() 73 | 74 | try: 75 | # # Attempt to compute result and set for waiter 76 | waiter.set_result(await fn(*args, **kwargs)) 77 | except Exception as e: 78 | # Remove key from cache on failure. 79 | cached.pop(key, None) 80 | # Propagate exception to all awaiting the future. 81 | waiter.set_exception(e) 82 | 83 | return await waiter 84 | 85 | return inner 86 | 87 | return outer 88 | -------------------------------------------------------------------------------- /src/pgcachewatch/listeners.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import datetime 3 | import json 4 | from typing import Callable, Protocol 5 | 6 | import asyncpg 7 | import websockets 8 | 9 | from . import models 10 | from .logconfig import logger 11 | 12 | 13 | def _critical_termination_listener(*_: object, **__: object) -> None: 14 | # Must be defined in the global namespace, as ayncpg keeps 15 | # a set of functions to call. This this will now happen once as 16 | # all instance will point to the same function. 17 | logger.critical("Connection is closed / terminated.") 18 | 19 | 20 | def create_event_inserter( 21 | queue: asyncio.Queue[models.Event], 22 | max_latency: datetime.timedelta, 23 | ) -> Callable[ 24 | [ 25 | models.PGChannel, 26 | str | bytes | bytearray, 27 | ], 28 | None, 29 | ]: 30 | """ 31 | Creates a callable that parses JSON payloads into `models.Event` 32 | objects and inserts them into a queue. If the event's latency 33 | exceeds the specified maximum, it logs a warning. Errors during 34 | parsing or inserting are logged as exceptions. 35 | """ 36 | 37 | def parse_and_insert( 38 | channel: models.PGChannel, 39 | payload: str | bytes | bytearray, 40 | ) -> None: 41 | """ 42 | Parses a JSON payload and inserts it into the queue as an `models.Event` object. 43 | """ 44 | try: 45 | event_data = json.loads(payload) 46 | 47 | # Add or overwrite channel key with the current channel 48 | event_data["channel"] = channel 49 | 50 | parsed_event = models.Event.model_validate(event_data) 51 | 52 | except Exception: 53 | logger.exception( 54 | "Failed to parse payload: `%s`.", 55 | payload, 56 | ) 57 | return 58 | 59 | if parsed_event.latency > max_latency: 60 | logger.warning( 61 | "Event latency (%s) exceeds maximum (%s): `%s` from `%s`.", 62 | parsed_event.latency, 63 | max_latency, 64 | parsed_event, 65 | channel, 66 | ) 67 | else: 68 | logger.info( 69 | "Inserting event into queue: `%s` from `%s`.", 70 | parsed_event, 71 | channel, 72 | ) 73 | 74 | try: 75 | queue.put_nowait(parsed_event) 76 | except Exception: 77 | logger.exception( 78 | "Unexpected error inserting event into queue: `%s`.", 79 | parsed_event, 80 | ) 81 | 82 | return parse_and_insert 83 | 84 | 85 | class EventQueueProtocol(Protocol): 86 | """ 87 | Protocol for an event queue interface. 88 | 89 | Specifies the required methods for an event queue to check the connection health 90 | and to retrieve events without waiting. Implementing classes must provide concrete 91 | implementations of these methods to ensure compatibility with the event handling 92 | system. 93 | """ 94 | 95 | def connection_healthy(self) -> bool: 96 | """ 97 | Checks if the connection is healthy. 98 | 99 | This method should return True if the connection to the underlying service 100 | (e.g., database, message broker) is active and healthy, False otherwise. 101 | 102 | Returns: 103 | bool: True if the connection is healthy, False otherwise. 104 | """ 105 | raise NotImplementedError 106 | 107 | def get_nowait(self) -> models.Event: 108 | """ 109 | Retrieves an event from the queue without waiting. 110 | 111 | Attempts to immediately retrieve an event from the queue. If no event is 112 | available, this method should raise an appropriate exception (e.g., QueueEmpty). 113 | 114 | Returns: 115 | models.Event: The event retrieved from the queue. 116 | 117 | Raises: 118 | QueueEmpty: If no event is available in the queue to retrieve. 119 | """ 120 | raise NotImplementedError 121 | 122 | 123 | class PGEventQueue(asyncio.Queue[models.Event]): 124 | """ 125 | A PostgreSQL event queue that listens to a specified 126 | channel and stores incoming events. 127 | """ 128 | 129 | def __init__( 130 | self, 131 | max_size: int = 0, 132 | max_latency: datetime.timedelta = datetime.timedelta(milliseconds=500), 133 | ) -> None: 134 | super().__init__(maxsize=max_size) 135 | self._pg_channel: None | models.PGChannel = None 136 | self._pg_connection: None | asyncpg.Connection = None 137 | self._max_latency = max_latency 138 | 139 | async def connect( 140 | self, 141 | connection: asyncpg.Connection, 142 | channel: models.PGChannel = models.DEFAULT_PG_CHANNE, 143 | ) -> None: 144 | """ 145 | Asynchronously connects the PGEventQueue to a specified 146 | PostgreSQL channel and connection. 147 | 148 | This method establishes a listener on a PostgreSQL channel 149 | using the provided connection. It is designed to be called 150 | once per PGEventQueue instance to ensure a one-to-one relationship 151 | between the event queue and a database channel. If an attempt is 152 | made to connect a PGEventQueue instance to more than one channel 153 | or connection, a RuntimeError is raised to enforce this constraint. 154 | 155 | Parameters: 156 | - connection: asyncpg.Connection 157 | The asyncpg connection object to be used for listening to database events. 158 | - channel: models.PGChannel 159 | The database channel to listen on for events. 160 | 161 | Raises: 162 | - RuntimeError: If the PGEventQueue is already connected to a 163 | channel or connection. 164 | 165 | Usage: 166 | ```python 167 | await pg_event_queue.connect( 168 | connection=your_asyncpg_connection, 169 | channel=your_pg_channel, 170 | ) 171 | ``` 172 | """ 173 | if self._pg_channel or self._pg_connection: 174 | raise RuntimeError( 175 | "PGEventQueue instance is already connected to a channel and/or " 176 | "connection. Only supports one channel and connection per " 177 | "PGEventQueue instance." 178 | ) 179 | 180 | self._pg_channel = channel 181 | self._pg_connection = connection 182 | self._pg_connection.add_termination_listener(_critical_termination_listener) 183 | 184 | event_handler = create_event_inserter(self, self._max_latency) 185 | await self._pg_connection.add_listener( 186 | self._pg_channel, 187 | lambda *x: event_handler(self._pg_channel, x[-1]), 188 | ) 189 | 190 | def connection_healthy(self) -> bool: 191 | return bool(self._pg_connection and not self._pg_connection.is_closed()) 192 | 193 | 194 | class WSEventQueue(asyncio.Queue[models.Event]): 195 | def __init__( 196 | self, 197 | max_size: int = 0, 198 | max_latency: datetime.timedelta = datetime.timedelta(milliseconds=500), 199 | ) -> None: 200 | super().__init__(maxsize=max_size) 201 | self._max_latency = max_latency 202 | self._handler_task: asyncio.Task | None = None 203 | self._ws: websockets.WebSocketClientProtocol | None = None 204 | 205 | async def connect( 206 | self, 207 | ws: websockets.WebSocketClientProtocol, 208 | channel: models.PGChannel = models.DEFAULT_PG_CHANNE, 209 | ) -> None: 210 | async def _handler(ws: websockets.WebSocketClientProtocol) -> None: 211 | event_handler = create_event_inserter(self, self._max_latency) 212 | while True: 213 | try: 214 | event_handler(self._pg_channel, await ws.recv()) 215 | except websockets.ConnectionClosedOK: 216 | break 217 | 218 | if self._handler_task is not None: 219 | raise RuntimeError( 220 | "WSEventQueue instance is already connected to a channel and/or " 221 | "connection. Only supports one channel and connection per " 222 | "WSEventQueue instance." 223 | ) 224 | 225 | self._ws = ws 226 | self._pg_channel = channel 227 | self._handler_task = asyncio.create_task(_handler(ws)) 228 | self._handler_task.add_done_callback(_critical_termination_listener) 229 | 230 | def connection_healthy(self) -> bool: 231 | task_ok = bool(self._handler_task and not self._handler_task.done()) 232 | ws_ok = bool(self._ws and not self._ws.closed) 233 | return task_ok and ws_ok 234 | -------------------------------------------------------------------------------- /src/pgcachewatch/logconfig.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Final 3 | 4 | logger: Final = logging.getLogger("pgcachewatch") 5 | logger.addHandler(logging.NullHandler()) 6 | -------------------------------------------------------------------------------- /src/pgcachewatch/models.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from typing import Literal, NewType 3 | 4 | import pydantic 5 | 6 | OPERATIONS = Literal[ 7 | "insert", 8 | "update", 9 | "delete", 10 | "truncate", 11 | ] 12 | 13 | PGChannel = NewType( 14 | "PGChannel", 15 | str, 16 | ) 17 | 18 | DEFAULT_PG_CHANNE = PGChannel("ch_pgcachewatch_table_change") 19 | 20 | 21 | class DeadlineSetting(pydantic.BaseModel): 22 | """ 23 | A data class representing settings for a deadline. 24 | 25 | Attributes: 26 | max_iter: Maximum number of iterations allowed. 27 | max_time: Maximum time allowed as a timedelta object. 28 | """ 29 | 30 | max_iter: int = pydantic.Field(gt=0, default=1_000) 31 | max_time: datetime.timedelta = pydantic.Field( 32 | default=datetime.timedelta(milliseconds=1) 33 | ) 34 | 35 | @pydantic.model_validator(mode="after") 36 | def _max_time_gt_zero(self) -> "DeadlineSetting": 37 | if self.max_time <= datetime.timedelta(seconds=0): 38 | raise ValueError("max_time must be greater than zero") 39 | return self 40 | 41 | 42 | class Event(pydantic.BaseModel): 43 | """ 44 | A class representing an event in a PostgreSQL channel. 45 | 46 | Attributes: 47 | channel: The PostgreSQL channel the event belongs to. 48 | operation: The type of operation performed (insert, update or delete). 49 | sent_at: The timestamp when the event was sent. 50 | table: The table the event is associated with. 51 | received_at: The timestamp when the event was received. 52 | """ 53 | 54 | channel: PGChannel 55 | operation: OPERATIONS 56 | sent_at: pydantic.AwareDatetime 57 | table: str 58 | received_at: pydantic.AwareDatetime = pydantic.Field( 59 | init=False, 60 | default_factory=lambda: datetime.datetime.now( 61 | tz=datetime.timezone.utc, 62 | ), 63 | ) 64 | 65 | @property 66 | def latency(self) -> datetime.timedelta: 67 | """ 68 | Calculate the latency between when the event was sent and received. 69 | """ 70 | return self.received_at - self.sent_at 71 | -------------------------------------------------------------------------------- /src/pgcachewatch/pg_event_distributor.py: -------------------------------------------------------------------------------- 1 | """ 2 | Facilitates WebSocket clients subscribing to PostgreSQL notifications via 3 | a single connection. Reduces PostgreSQL server load by sharing one connection 4 | among multiple clients 5 | 6 | Usage example: 7 | `uvicorn pgcachewatch.pg_event_distributor:main --factory` 8 | """ 9 | 10 | import asyncio 11 | from contextlib import asynccontextmanager 12 | from typing import AsyncGenerator 13 | 14 | import asyncpg 15 | from fastapi import Depends, FastAPI, Response, WebSocket, WebSocketDisconnect 16 | 17 | 18 | @asynccontextmanager 19 | async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: 20 | """ 21 | Manages the applications database conncetion. 22 | """ 23 | app.state.pg_connection = conn = await asyncpg.connect() 24 | try: 25 | yield 26 | finally: 27 | await conn.close() 28 | 29 | 30 | def get_pg_connection(req: WebSocket) -> asyncpg.Connection: 31 | """ 32 | Retrieves PostgreSQL connection from app state for FastAPI endpoints. 33 | """ 34 | assert isinstance( 35 | conn := req.app.state.pg_connection, 36 | asyncpg.Connection, 37 | ) 38 | return conn 39 | 40 | 41 | def main() -> FastAPI: 42 | """ 43 | Configures FastAPI app with PostgreSQL connection and WebSocket 44 | endpoint for PUB/SUB. 45 | """ 46 | 47 | app = FastAPI(lifespan=lifespan) 48 | 49 | @app.get("/up") 50 | async def up() -> Response: 51 | return Response() 52 | 53 | @app.websocket("/pgpubsub/{channel}") 54 | async def pubsub_proxy( 55 | websocket: WebSocket, 56 | channel: str, 57 | conn: asyncpg.Connection = Depends(get_pg_connection), 58 | ) -> None: 59 | """ 60 | Forwards messages from a PostgreSQL channel to WebSocket clients. 61 | """ 62 | 63 | await websocket.accept() 64 | que = asyncio.Queue[str]() 65 | 66 | async def putter( 67 | connection: asyncpg.Connection, 68 | pid: int, 69 | channel: str, 70 | payload: str, 71 | ) -> None: 72 | """ 73 | Enqueues message payloads for forwarding to WebSocket clients on new 74 | publication. 75 | """ 76 | await que.put(payload) 77 | 78 | await conn.add_listener(channel, putter) # type: ignore[arg-type] 79 | 80 | try: 81 | while True: 82 | await websocket.send_text(await que.get()) 83 | except WebSocketDisconnect: 84 | await conn.remove_listener(channel, putter) # type: ignore[arg-type] 85 | 86 | return app 87 | -------------------------------------------------------------------------------- /src/pgcachewatch/queries.py: -------------------------------------------------------------------------------- 1 | def create_notify_function( 2 | channel_name: str, 3 | function_name: str, 4 | ) -> str: 5 | return f""" 6 | CREATE OR REPLACE FUNCTION {function_name}() RETURNS TRIGGER AS $$ 7 | BEGIN 8 | PERFORM pg_notify( 9 | '{channel_name}', 10 | json_build_object( 11 | 'operation', lower(TG_OP), 12 | 'table', TG_TABLE_NAME, 13 | 'sent_at', NOW() 14 | )::text); 15 | RETURN NEW; 16 | END; 17 | $$ LANGUAGE plpgsql; 18 | """ 19 | 20 | 21 | def create_after_change_trigger( 22 | trigger_name: str, 23 | table_name: str, 24 | function_name: str, 25 | ) -> str: 26 | return f""" 27 | CREATE OR REPLACE TRIGGER {trigger_name} 28 | AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON {table_name} 29 | EXECUTE FUNCTION {function_name}(); 30 | """ 31 | 32 | 33 | def fetch_trigger_names(prefix: str) -> str: 34 | return f""" 35 | SELECT 36 | event_object_table AS table, 37 | trigger_name 38 | FROM 39 | information_schema.triggers 40 | WHERE 41 | trigger_name LIKE '{prefix}%' 42 | """ 43 | 44 | 45 | def drop_trigger(trigger_name: str, table: str) -> str: 46 | return f"""DROP TRIGGER IF EXISTS {trigger_name} ON {table};""" 47 | 48 | 49 | def drop_function(name: str) -> str: 50 | return f"""DROP FUNCTION IF EXISTS {name}();""" 51 | -------------------------------------------------------------------------------- /src/pgcachewatch/strategies.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import datetime 3 | from typing import Callable, Protocol 4 | 5 | from . import listeners, models, utils 6 | 7 | 8 | class Strategy(Protocol): 9 | """ 10 | A protocol defining the clear method for different strategies. 11 | """ 12 | 13 | def clear(self) -> bool: 14 | raise NotImplementedError 15 | 16 | def connection_healthy(self) -> bool: 17 | raise NotImplementedError 18 | 19 | 20 | class Greedy(Strategy): 21 | """ 22 | A strategy that clears events based on a predicate until a deadline is reached. 23 | """ 24 | 25 | def __init__( 26 | self, 27 | listener: listeners.EventQueueProtocol, 28 | settings: models.DeadlineSetting = models.DeadlineSetting(), 29 | predicate: Callable[[models.Event], bool] = bool, 30 | ) -> None: 31 | super().__init__() 32 | self._listener = listener 33 | self._predicate = predicate 34 | self._settings = settings 35 | 36 | def connection_healthy(self) -> bool: 37 | return self._listener.connection_healthy() 38 | 39 | def clear(self) -> bool: 40 | for current in utils.pick_until_deadline( 41 | self._listener, 42 | settings=self._settings, 43 | ): 44 | if self._predicate(current): 45 | return True 46 | return False 47 | 48 | 49 | class Windowed(Strategy): 50 | """ 51 | A strategy that clears events when a specified sequence 52 | of operations occurs within a window. 53 | """ 54 | 55 | def __init__( 56 | self, 57 | listener: listeners.EventQueueProtocol, 58 | window: list[models.OPERATIONS], 59 | settings: models.DeadlineSetting = models.DeadlineSetting(), 60 | ) -> None: 61 | super().__init__() 62 | self._listener = listener 63 | self._settings = settings 64 | self._events = collections.deque[models.OPERATIONS](maxlen=len(window)) 65 | self._window = collections.deque[models.OPERATIONS](window, maxlen=len(window)) 66 | 67 | def connection_healthy(self) -> bool: 68 | return self._listener.connection_healthy() 69 | 70 | def clear(self) -> bool: 71 | for current in utils.pick_until_deadline( 72 | self._listener, 73 | settings=self._settings, 74 | ): 75 | self._events.append(current.operation) 76 | if self._window == self._events: 77 | return True 78 | return False 79 | 80 | 81 | class Timed(Strategy): 82 | """ 83 | A strategy that clears events based on a specified time interval between events. 84 | """ 85 | 86 | def __init__( 87 | self, 88 | listener: listeners.EventQueueProtocol, 89 | timedelta: datetime.timedelta, 90 | settings: models.DeadlineSetting = models.DeadlineSetting(), 91 | ) -> None: 92 | super().__init__() 93 | self._listener = listener 94 | self._timedelta = timedelta 95 | self._settings = settings 96 | self._previous = datetime.datetime.now(tz=datetime.timezone.utc) 97 | 98 | def connection_healthy(self) -> bool: 99 | return self._listener.connection_healthy() 100 | 101 | def clear(self) -> bool: 102 | for current in utils.pick_until_deadline( 103 | queue=self._listener, 104 | settings=self._settings, 105 | ): 106 | if current.sent_at - self._previous > self._timedelta: 107 | self._previous = current.sent_at 108 | return True 109 | return False 110 | -------------------------------------------------------------------------------- /src/pgcachewatch/utils.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import datetime 3 | from typing import Generator 4 | 5 | import asyncpg 6 | 7 | from pgcachewatch import listeners, models 8 | 9 | 10 | async def emit_event( 11 | conn: asyncpg.Connection | asyncpg.Pool, 12 | event: models.Event, 13 | ) -> None: 14 | """ 15 | Emit an event to the specified PostgreSQL channel. 16 | """ 17 | await conn.execute( 18 | "SELECT pg_notify($1, $2)", 19 | event.channel, 20 | event.model_dump_json(), 21 | ) 22 | 23 | 24 | def pick_until_deadline( 25 | queue: listeners.EventQueueProtocol, 26 | settings: models.DeadlineSetting, 27 | ) -> Generator[models.Event, None, None]: 28 | """ 29 | Yield events from the queue until the deadline is reached or queue is empty. 30 | """ 31 | 32 | deadline = datetime.datetime.now() + settings.max_time 33 | iter_cnt = 0 34 | 35 | while settings.max_iter > iter_cnt and deadline > datetime.datetime.now(): 36 | try: 37 | yield queue.get_nowait() 38 | except asyncio.QueueEmpty: 39 | return 40 | iter_cnt += 1 41 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import os 3 | from contextlib import suppress 4 | from datetime import datetime, timedelta 5 | from subprocess import PIPE, Popen 6 | from typing import AsyncGenerator 7 | 8 | import asyncpg 9 | import httpx 10 | import pytest 11 | 12 | 13 | def pgb_address() -> str: 14 | return "127.0.0.1:8000" 15 | 16 | 17 | async def pg_event_distributor_isup() -> bool: 18 | timeout = timedelta(seconds=1) 19 | deadline = datetime.now() + timeout 20 | 21 | async with httpx.AsyncClient(base_url=f"http://{pgb_address()}") as client: 22 | while datetime.now() < deadline: 23 | with suppress(httpx.ConnectError): 24 | if (await client.get("/up")).is_success: 25 | return True 26 | await asyncio.sleep(0.001) 27 | 28 | raise RuntimeError("Isup timeout") 29 | 30 | 31 | @pytest.fixture(scope="function") 32 | async def pgconn() -> AsyncGenerator[asyncpg.Connection, None]: 33 | conn = await asyncpg.connect() 34 | try: 35 | yield conn 36 | finally: 37 | await conn.close() 38 | 39 | 40 | @pytest.fixture(scope="function") 41 | async def pgpool() -> AsyncGenerator[asyncpg.Pool, None]: 42 | async with asyncpg.create_pool() as pool: 43 | yield pool 44 | 45 | 46 | @pytest.fixture(scope="function", autouse=True) 47 | def set_pg_envs(monkeypatch: pytest.MonkeyPatch) -> None: 48 | Unset = object() 49 | 50 | if os.environ.get("PGHOST", Unset) is Unset: 51 | monkeypatch.setenv("PGHOST", "localhost") 52 | 53 | if os.environ.get("PGUSER", Unset) is Unset: 54 | monkeypatch.setenv("PGUSER", "testuser") 55 | 56 | if os.environ.get("PGPASSWORD", Unset) is Unset: 57 | monkeypatch.setenv("PGPASSWORD", "testpassword") 58 | 59 | if os.environ.get("PGDATABASE", Unset) is Unset: 60 | monkeypatch.setenv("PGDATABASE", "testdb") 61 | 62 | 63 | @pytest.fixture(scope="function") 64 | async def pgedapp() -> AsyncGenerator[Popen, None]: 65 | with Popen( 66 | "uvicorn pgcachewatch.pg_event_distributor:main --factory".split(), 67 | stderr=PIPE, 68 | stdout=PIPE, 69 | ) as p: 70 | await pg_event_distributor_isup() 71 | try: 72 | yield p 73 | finally: 74 | p.kill() 75 | p.wait() 76 | -------------------------------------------------------------------------------- /tests/db/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG POSTGRES_VERSION 2 | FROM postgres:${POSTGRES_VERSION} 3 | ENV POSTGRES_USER=testuser 4 | ENV POSTGRES_PASSWORD=testpassword 5 | 6 | # Copy the combined database setup script into the container 7 | COPY ./init_db.sh /docker-entrypoint-initdb.d/init_db.sh 8 | 9 | # Ensure the script is executable 10 | RUN chmod +x /docker-entrypoint-initdb.d/init_db.sh 11 | -------------------------------------------------------------------------------- /tests/db/init_db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # Create the testdb database 5 | psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL 6 | CREATE DATABASE testdb; 7 | EOSQL 8 | 9 | # Connect to testdb and set up the sysconf table 10 | psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname testdb <<-EOSQL 11 | CREATE TABLE sysconf ( 12 | key VARCHAR(255) PRIMARY KEY, 13 | value TEXT NOT NULL 14 | ); 15 | INSERT INTO sysconf (key, value) VALUES 16 | ('app_name', 'MyApplication'), 17 | ('app_version', '1.0.0'), 18 | ('maintenance_mode', 'false'), 19 | ('updated_at', '2024-02-19 23:01:54.609243+00'); 20 | EOSQL 21 | -------------------------------------------------------------------------------- /tests/test_decoraters.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import collections 3 | import datetime 4 | from typing import NoReturn 5 | 6 | import asyncpg 7 | import pytest 8 | from pgcachewatch import decorators, listeners, models, strategies 9 | 10 | 11 | @pytest.mark.parametrize("N", (1, 2, 4, 16, 64)) 12 | async def test_greedy_cache_decorator(N: int, pgconn: asyncpg.Connection) -> None: 13 | statistics = collections.Counter[str]() 14 | listener = listeners.PGEventQueue() 15 | await listener.connect(pgconn, models.PGChannel("test_cache_decorator")) 16 | 17 | @decorators.cache( 18 | strategy=strategies.Greedy(listener=listener), 19 | statistics_callback=lambda x: statistics.update([x]), 20 | ) 21 | async def now() -> datetime.datetime: 22 | return datetime.datetime.now() 23 | 24 | nows = set(await asyncio.gather(*[now() for _ in range(N)])) 25 | assert len(nows) == 1 26 | 27 | assert statistics["hit"] == N - 1 28 | assert statistics["miss"] == 1 29 | 30 | 31 | @pytest.mark.parametrize("N", (1, 2, 4, 16, 64)) 32 | async def test_greedy_cache_decorator_connection_closed( 33 | N: int, 34 | pgconn: asyncpg.Connection, 35 | ) -> None: 36 | listener = listeners.PGEventQueue() 37 | await listener.connect( 38 | pgconn, 39 | models.PGChannel("test_greedy_cache_decorator_connection_closed"), 40 | ) 41 | await pgconn.close() 42 | 43 | @decorators.cache(strategy=strategies.Greedy(listener=listener)) 44 | async def now() -> datetime.datetime: 45 | return datetime.datetime.now() 46 | 47 | nows = await asyncio.gather(*[now() for _ in range(N)]) 48 | assert len(set(nows)) == N 49 | 50 | 51 | @pytest.mark.parametrize("N", (1, 2, 4, 16, 64)) 52 | async def test_greedy_cache_decorator_exceptions( 53 | N: int, 54 | pgconn: asyncpg.Connection, 55 | ) -> None: 56 | listener = listeners.PGEventQueue() 57 | await listener.connect( 58 | pgconn, 59 | models.PGChannel("test_greedy_cache_decorator_exceptions"), 60 | ) 61 | 62 | @decorators.cache(strategy=strategies.Greedy(listener=listener)) 63 | async def raise_runtime_error() -> NoReturn: 64 | raise RuntimeError 65 | 66 | for _ in range(N): 67 | with pytest.raises(RuntimeError): 68 | await raise_runtime_error() 69 | 70 | exceptions = await asyncio.gather( 71 | *[raise_runtime_error() for _ in range(N)], 72 | return_exceptions=True, 73 | ) 74 | assert len(exceptions) == N 75 | assert all(isinstance(exc, RuntimeError) for exc in exceptions) 76 | 77 | 78 | @pytest.mark.parametrize("N", (1, 2, 4, 16, 64)) 79 | async def test_greedy_cache_identity( 80 | N: int, 81 | pgconn: asyncpg.Connection, 82 | ) -> None: 83 | statistics = collections.Counter[str]() 84 | listener = listeners.PGEventQueue() 85 | await listener.connect( 86 | pgconn, 87 | models.PGChannel("test_greedy_cache_decorator_exceptions"), 88 | ) 89 | 90 | @decorators.cache( 91 | strategy=strategies.Greedy(listener=listener), 92 | statistics_callback=lambda x: statistics.update([x]), 93 | ) 94 | async def identity(x: int) -> int: 95 | return x 96 | 97 | results = await asyncio.gather(*[identity(n) for n in range(N)]) 98 | 99 | assert sorted(results) == list(range(N)) 100 | assert statistics["miss"] == N 101 | assert statistics["hit"] == 0 102 | 103 | 104 | @pytest.mark.parametrize("N", (1, 2, 4, 16, 64)) 105 | async def test_greedy_cache_sleepy( 106 | N: int, 107 | pgconn: asyncpg.Connection, 108 | ) -> None: 109 | statistics = collections.Counter[str]() 110 | listener = listeners.PGEventQueue() 111 | await listener.connect( 112 | pgconn, 113 | models.PGChannel("test_greedy_cache_decorator_exceptions"), 114 | ) 115 | 116 | @decorators.cache( 117 | strategy=strategies.Greedy(listener=listener), 118 | statistics_callback=lambda x: statistics.update([x]), 119 | ) 120 | async def now() -> datetime.datetime: 121 | await asyncio.sleep(0.01) 122 | return datetime.datetime.now() 123 | 124 | results = await asyncio.gather(*[now() for _ in range(N)]) 125 | 126 | assert len(set(results)) == 1 127 | assert statistics["miss"] == 1 128 | assert statistics["hit"] == N - 1 129 | -------------------------------------------------------------------------------- /tests/test_fastapi.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import datetime 3 | 4 | import asyncpg 5 | import fastapi 6 | import pytest 7 | from fastapi.testclient import TestClient 8 | from pgcachewatch import decorators, listeners, models, strategies, utils 9 | 10 | 11 | async def fastapitestapp( 12 | channel: models.PGChannel, 13 | pgconn: asyncpg.Connection, 14 | ) -> fastapi.FastAPI: 15 | app = fastapi.FastAPI() 16 | 17 | listener = listeners.PGEventQueue() 18 | await listener.connect(pgconn, channel) 19 | 20 | @decorators.cache(strategy=strategies.Greedy(listener=listener)) 21 | async def slow_db_read() -> dict: 22 | await asyncio.sleep(0.1) # sim. a slow db-query. 23 | return {"now": datetime.datetime.now().isoformat()} 24 | 25 | @app.get("/sysconf") 26 | async def sysconf() -> dict[str, str]: 27 | return await slow_db_read() 28 | 29 | return app 30 | 31 | 32 | @pytest.mark.parametrize("N", (2, 4, 16)) 33 | async def test_fastapi( 34 | N: int, 35 | pgconn: asyncpg.Connection, 36 | ) -> None: 37 | # No cache invalidation evnets emitted, all timestamps should be the same. 38 | tc = TestClient( 39 | await fastapitestapp( 40 | models.PGChannel("test_fastapi"), 41 | pgconn, 42 | ) 43 | ) 44 | responses = set[str](tc.get("/sysconf").json()["now"] for _ in range(N)) 45 | assert len(responses) == 1 46 | 47 | 48 | @pytest.mark.parametrize("N", (4, 8, 16)) 49 | async def test_fastapi_invalidate_cache( 50 | N: int, 51 | pgconn: asyncpg.Connection, 52 | ) -> None: 53 | # Emits one cache invalidation event per call, number of uniq timestamps 54 | # should equal the number of calls(N). 55 | 56 | channel = models.PGChannel(f"test_fastapi_invalidate_cache_{N}") 57 | tc = TestClient(await fastapitestapp(channel, pgconn)) 58 | 59 | responses = set[str]() 60 | for _ in range(N): 61 | responses.add(tc.get("/sysconf").json()["now"]) 62 | await utils.emit_event( 63 | conn=pgconn, 64 | event=models.Event( 65 | channel=channel, 66 | operation="update", 67 | sent_at=datetime.datetime.now(tz=datetime.timezone.utc), 68 | table="placeholder", 69 | ), 70 | ) 71 | await asyncio.sleep(0.01) # allow some time for the evnet to propegate. 72 | assert len(responses) == N 73 | -------------------------------------------------------------------------------- /tests/test_integration.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import collections 3 | import datetime 4 | 5 | import asyncpg 6 | import pytest 7 | from pgcachewatch import cli, decorators, listeners, strategies 8 | 9 | 10 | def utcnow() -> datetime.datetime: 11 | return datetime.datetime.now(tz=datetime.timezone.utc) 12 | 13 | 14 | async def test_1_install_triggers( 15 | monkeypatch: pytest.MonkeyPatch, 16 | pgconn: asyncpg.Connection, 17 | ) -> None: 18 | monkeypatch.setattr( 19 | "sys.argv", 20 | ["pgcachewatch", "install", "sysconf", "--commit"], 21 | ) 22 | await cli.main() 23 | assert ( 24 | len( 25 | await pgconn.fetch( 26 | cli.queries.fetch_trigger_names(cli.cliparser().trigger_name) 27 | ) 28 | ) 29 | == 3 30 | ) 31 | 32 | 33 | @pytest.mark.parametrize("N", (2, 8, 16)) 34 | async def test_2_caching( 35 | N: int, 36 | pgconn: asyncpg.Connection, 37 | pgpool: asyncpg.Pool, 38 | ) -> None: 39 | statistics = collections.Counter[str]() 40 | listener = listeners.PGEventQueue() 41 | await listener.connect(pgconn) 42 | 43 | cnt = 0 44 | 45 | @decorators.cache( 46 | strategy=strategies.Greedy(listener=listener), 47 | statistics_callback=lambda x: statistics.update([x]), 48 | ) 49 | async def fetch_sysconf() -> list: 50 | nonlocal cnt 51 | cnt += 1 52 | return await pgpool.fetch("SELECT * FROM sysconf") 53 | 54 | await asyncio.gather(*[fetch_sysconf() for _ in range(N)]) 55 | 56 | # Give a bit of leeway due IO network io. 57 | await asyncio.sleep(0.1) 58 | 59 | assert cnt == 1 60 | assert statistics["miss"] == 1 61 | assert statistics["hit"] == N - 1 62 | 63 | 64 | async def test_3_cache_invalidation_update( 65 | pgconn: asyncpg.Connection, 66 | pgpool: asyncpg.Pool, 67 | ) -> None: 68 | statistics = collections.Counter[str]() 69 | listener = listeners.PGEventQueue() 70 | await listener.connect(pgconn) 71 | 72 | @decorators.cache( 73 | strategy=strategies.Greedy(listener=listener), 74 | statistics_callback=lambda x: statistics.update([x]), 75 | ) 76 | async def fetch_sysconf() -> list: 77 | return await pgpool.fetch("SELECT * FROM sysconf") 78 | 79 | async def blast() -> list: 80 | before = await fetch_sysconf() 81 | while (rv := await fetch_sysconf()) == before: 82 | await asyncio.sleep(0.001) 83 | return rv 84 | 85 | blast_task = asyncio.create_task(blast()) 86 | await pgpool.execute( 87 | "UPDATE sysconf set value = $1 where key = 'updated_at'", 88 | utcnow().isoformat(), 89 | ) 90 | await asyncio.wait_for(blast_task, 1) 91 | # First fetch and update 92 | assert statistics["miss"] == 2 93 | 94 | 95 | async def test_3_cache_invalidation_insert( 96 | pgconn: asyncpg.Connection, 97 | pgpool: asyncpg.Pool, 98 | ) -> None: 99 | statistics = collections.Counter[str]() 100 | listener = listeners.PGEventQueue() 101 | await listener.connect(pgconn) 102 | 103 | @decorators.cache( 104 | strategy=strategies.Greedy(listener=listener), 105 | statistics_callback=lambda x: statistics.update([x]), 106 | ) 107 | async def fetch_sysconf() -> list: 108 | return await pgpool.fetch("SELECT * FROM sysconf") 109 | 110 | async def blast() -> list: 111 | before = await fetch_sysconf() 112 | while (rv := await fetch_sysconf()) == before: 113 | await asyncio.sleep(0.001) 114 | return rv 115 | 116 | blast_task = asyncio.create_task(blast()) 117 | await pgpool.execute( 118 | "INSERT INTO sysconf (key, value) VALUES ($1, $2);", 119 | utcnow().isoformat(), 120 | utcnow().isoformat(), 121 | ) 122 | await asyncio.wait_for(blast_task, 1) 123 | # First fetch and insert 124 | assert statistics["miss"] == 2 125 | 126 | 127 | async def test_3_cache_invalidation_delete( 128 | pgconn: asyncpg.Connection, 129 | pgpool: asyncpg.Pool, 130 | ) -> None: 131 | statistics = collections.Counter[str]() 132 | listener = listeners.PGEventQueue() 133 | await listener.connect(pgconn) 134 | 135 | @decorators.cache( 136 | strategy=strategies.Greedy(listener=listener), 137 | statistics_callback=lambda x: statistics.update([x]), 138 | ) 139 | async def fetch_sysconf() -> list: 140 | return await pgpool.fetch("SELECT * FROM sysconf") 141 | 142 | async def blast() -> list: 143 | before = await fetch_sysconf() 144 | while (rv := await fetch_sysconf()) == before: 145 | await asyncio.sleep(0.001) 146 | return rv 147 | 148 | blast_task = asyncio.create_task(blast()) 149 | await pgpool.execute( 150 | "DELETE FROM sysconf WHERE key ~ '^\\d{4}-\\d{2}-\\d{2}';", 151 | ) 152 | await asyncio.wait_for(blast_task, 1) 153 | # First fetch and insert 154 | assert statistics["miss"] == 2 155 | 156 | 157 | async def test_4_uninstall_triggers( 158 | monkeypatch: pytest.MonkeyPatch, 159 | pgconn: asyncpg.Connection, 160 | ) -> None: 161 | monkeypatch.setattr( 162 | "sys.argv", 163 | ["pgcachewatch", "uninstall", "--commit"], 164 | ) 165 | await cli.main() 166 | assert ( 167 | len( 168 | await pgconn.fetch( 169 | cli.queries.fetch_trigger_names(cli.cliparser().trigger_name) 170 | ) 171 | ) 172 | == 0 173 | ) 174 | -------------------------------------------------------------------------------- /tests/test_listeners.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import datetime 3 | from subprocess import Popen 4 | from typing import get_args 5 | 6 | import asyncpg 7 | import pytest 8 | import websockets 9 | from conftest import pgb_address 10 | from pgcachewatch import listeners, models, utils 11 | 12 | 13 | @pytest.mark.parametrize("N", (1, 8, 32)) 14 | @pytest.mark.parametrize("operation", get_args(models.OPERATIONS)) 15 | async def test_eventqueue_and_pglistner( 16 | N: int, 17 | operation: models.OPERATIONS, 18 | pgconn: asyncpg.Connection, 19 | pgpool: asyncpg.Pool, 20 | ) -> None: 21 | channel = models.PGChannel(f"test_eventqueue_and_pglistner_{N}_{operation}") 22 | listener = listeners.PGEventQueue() 23 | await listener.connect(pgconn, channel) 24 | 25 | to_emit = [ 26 | models.Event( 27 | channel=models.PGChannel(channel), 28 | operation=operation, 29 | sent_at=datetime.datetime.now(tz=datetime.timezone.utc), 30 | table="", 31 | ) 32 | for _ in range(N) 33 | ] 34 | await asyncio.gather(*[utils.emit_event(pgpool, e) for e in to_emit]) 35 | 36 | # Give a bit of leeway due IO network io. 37 | await asyncio.sleep(0.1) 38 | 39 | assert listener.qsize() == N 40 | 41 | # Due to use of gather(...) order can not be assumed, need to sort. 42 | assert to_emit == sorted( 43 | (listener.get_nowait() for _ in range(N)), 44 | key=lambda x: x.sent_at, 45 | ) 46 | 47 | 48 | @pytest.mark.parametrize("N", (1, 8, 32)) 49 | @pytest.mark.parametrize("operation", get_args(models.OPERATIONS)) 50 | async def test_eventqueue_and_wslistner( 51 | pgedapp: Popen, 52 | N: int, 53 | operation: models.OPERATIONS, 54 | pgpool: asyncpg.Pool, 55 | ) -> None: 56 | channel = models.PGChannel(f"test_eventqueue_and_pglistner_{N}_{operation}") 57 | listener = listeners.WSEventQueue() 58 | 59 | async with websockets.connect(f"ws://{pgb_address()}/pgpubsub/{channel}") as ws: 60 | await listener.connect(ws, channel) 61 | 62 | to_emit = [ 63 | models.Event( 64 | channel=models.PGChannel(channel), 65 | operation=operation, 66 | sent_at=datetime.datetime.now(tz=datetime.timezone.utc), 67 | table="", 68 | ) 69 | for _ in range(N) 70 | ] 71 | 72 | await asyncio.gather(*[utils.emit_event(pgpool, e) for e in to_emit]) 73 | 74 | # Give a bit of leeway due IO network io. 75 | await asyncio.sleep(0.1) 76 | 77 | assert listener.qsize() == N 78 | 79 | # Due to use of gather(...) order can not be assumed, need to sort. 80 | assert to_emit == sorted( 81 | (listener.get_nowait() for _ in range(N)), 82 | key=lambda x: x.sent_at, 83 | ) 84 | -------------------------------------------------------------------------------- /tests/test_pg_event_distributor.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from datetime import datetime, timezone 3 | from subprocess import Popen 4 | from typing import get_args 5 | 6 | import asyncpg 7 | import pytest 8 | import websockets 9 | from conftest import pg_event_distributor_isup, pgb_address 10 | from pgcachewatch import listeners, models, utils 11 | 12 | 13 | async def test_up_endpoint(pgedapp: Popen) -> None: 14 | assert await pg_event_distributor_isup() 15 | 16 | 17 | @pytest.mark.parametrize("operation", get_args(models.OPERATIONS)) 18 | @pytest.mark.parametrize("N", (1, 8)) 19 | async def test_ws_broadcast( 20 | pgedapp: Popen, 21 | N: int, 22 | pgpool: asyncpg.Pool, 23 | operation: models.OPERATIONS, 24 | channel: models.PGChannel = models.PGChannel("test_ws_broadcast"), 25 | ) -> None: 26 | async with websockets.connect(f"ws://{pgb_address()}/pgpubsub/{channel}") as ws: 27 | to_emit = [ 28 | models.Event( 29 | channel=models.PGChannel(channel), 30 | operation=operation, 31 | sent_at=datetime.now(tz=timezone.utc), 32 | table="", 33 | ) 34 | for _ in range(N) 35 | ] 36 | await asyncio.gather(*[utils.emit_event(pgpool, e) for e in to_emit]) 37 | 38 | # Give a bit of leeway due IO network io. 39 | await asyncio.sleep(0.1) 40 | received = [models.Event.model_validate_json(x) for x in ws.messages] 41 | 42 | # Due to use of gather(...) order can be assumed, need to sort. 43 | assert to_emit == sorted(received, key=lambda x: x.sent_at) 44 | 45 | 46 | @pytest.mark.parametrize("operation", get_args(models.OPERATIONS)) 47 | @pytest.mark.parametrize("N", (1, 8, 16)) 48 | async def test_put_ws_event_queue( 49 | N: int, 50 | operation: models.OPERATIONS, 51 | channel: models.PGChannel = models.PGChannel("test_put_ws_event_queue"), 52 | ) -> None: 53 | que = listeners.WSEventQueue() 54 | for _ in range(N): 55 | que.put_nowait( 56 | models.Event( 57 | channel=models.PGChannel(channel), 58 | operation=operation, 59 | sent_at=datetime.now(tz=timezone.utc), 60 | table="", 61 | ) 62 | ) 63 | 64 | assert que.qsize() == N 65 | 66 | que = listeners.WSEventQueue() 67 | for _ in range(N): 68 | await que.put( 69 | models.Event( 70 | channel=models.PGChannel(channel), 71 | operation=operation, 72 | sent_at=datetime.now(tz=timezone.utc), 73 | table="", 74 | ) 75 | ) 76 | assert que.qsize() == N 77 | 78 | que = listeners.WSEventQueue() 79 | await asyncio.gather( 80 | *[ 81 | que.put( 82 | models.Event( 83 | channel=models.PGChannel(channel), 84 | operation=operation, 85 | sent_at=datetime.now(tz=timezone.utc), 86 | table="", 87 | ) 88 | ) 89 | for _ in range(N) 90 | ] 91 | ) 92 | 93 | # Give a bit of leeway due IO network io. 94 | await asyncio.sleep(0.1) 95 | 96 | assert que.qsize() == N 97 | 98 | 99 | @pytest.mark.parametrize("operation", get_args(models.OPERATIONS)) 100 | @pytest.mark.parametrize("N", (1, 64)) 101 | async def test_put_on_event_ws_event_queue( 102 | pgedapp: Popen, 103 | N: int, 104 | pgpool: asyncpg.Pool, 105 | operation: models.OPERATIONS, 106 | channel: models.PGChannel = models.PGChannel("test_put_on_event_ws_event_queue"), 107 | ) -> None: 108 | async with websockets.connect(f"ws://{pgb_address()}/pgpubsub/{channel}") as ws: 109 | lisn = listeners.WSEventQueue() 110 | await lisn.connect(ws, channel) 111 | 112 | to_emit = [ 113 | models.Event( 114 | channel=models.PGChannel(channel), 115 | operation=operation, 116 | sent_at=datetime.now(tz=timezone.utc), 117 | table="", 118 | ) 119 | for _ in range(N) 120 | ] 121 | 122 | await asyncio.gather(*[utils.emit_event(pgpool, e) for e in to_emit]) 123 | 124 | # Give a bit of leeway due IO network io. 125 | await asyncio.sleep(0.1) 126 | 127 | assert lisn.qsize() == N 128 | # Due to use of gather(...) order can be assumed, need to sort. 129 | assert to_emit == sorted( 130 | (lisn.get_nowait() for _ in range(N)), 131 | key=lambda x: x.sent_at, 132 | ) 133 | 134 | 135 | async def test_ws_event_queue_connection_healthy( 136 | pgedapp: Popen, 137 | channel: models.PGChannel = models.PGChannel( 138 | "test_ws_event_queue_connection_healthy" 139 | ), 140 | ) -> None: 141 | async with websockets.connect(f"ws://{pgb_address()}/pgpubsub/{channel}") as ws: 142 | lisn = listeners.WSEventQueue() 143 | await lisn.connect(ws, channel) 144 | assert lisn.connection_healthy() 145 | 146 | assert not lisn.connection_healthy() 147 | -------------------------------------------------------------------------------- /tests/test_strategies.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import datetime 3 | 4 | import asyncpg 5 | import pytest 6 | from pgcachewatch import listeners, models, strategies 7 | 8 | 9 | @pytest.mark.parametrize("N", (4, 16, 64)) 10 | async def test_greedy_strategy(N: int, pgconn: asyncpg.Connection) -> None: 11 | channel = models.PGChannel("test_greedy_strategy") 12 | 13 | listener = listeners.PGEventQueue() 14 | await listener.connect(pgconn, channel) 15 | 16 | strategy = strategies.Greedy( 17 | listener=listener, 18 | predicate=lambda e: e.operation == "insert", 19 | ) 20 | 21 | for _ in range(N): 22 | await listener.put( 23 | models.Event( 24 | operation="insert", 25 | sent_at=datetime.datetime.now(tz=datetime.timezone.utc), 26 | table="placeholder", 27 | channel=channel, 28 | ) 29 | ) 30 | assert strategy.clear() 31 | 32 | for _ in range(N): 33 | await listener.put( 34 | models.Event( 35 | operation="update", 36 | sent_at=datetime.datetime.now(tz=datetime.timezone.utc), 37 | table="placeholder", 38 | channel=channel, 39 | ) 40 | ) 41 | assert not strategy.clear() 42 | 43 | for _ in range(N): 44 | assert not strategy.clear() 45 | 46 | 47 | @pytest.mark.parametrize("N", (4, 16, 64)) 48 | async def test_windowed_strategy( 49 | N: int, 50 | pgconn: asyncpg.Connection, 51 | ) -> None: 52 | channel = models.PGChannel("test_windowed_strategy") 53 | listener = listeners.PGEventQueue() 54 | await listener.connect(pgconn, channel) 55 | strategy = strategies.Windowed( 56 | listener=listener, window=["insert", "update", "delete"] 57 | ) 58 | 59 | # Right pattern insert -> update -> delete 60 | for _ in range(N): 61 | await listener.put( 62 | models.Event( 63 | channel=channel, 64 | operation="insert", 65 | sent_at=datetime.datetime.now(tz=datetime.timezone.utc), 66 | table="placeholder", 67 | ) 68 | ) 69 | await listener.put( 70 | models.Event( 71 | channel=channel, 72 | operation="update", 73 | sent_at=datetime.datetime.now(tz=datetime.timezone.utc), 74 | table="placeholder", 75 | ) 76 | ) 77 | await listener.put( 78 | models.Event( 79 | channel=channel, 80 | operation="delete", 81 | sent_at=datetime.datetime.now(tz=datetime.timezone.utc), 82 | table="placeholder", 83 | ) 84 | ) 85 | assert strategy.clear() 86 | 87 | # Falsy patteren, chain of nothing. 88 | for _ in range(N): 89 | assert not strategy.clear() 90 | 91 | # Falsy patteren, chain of inserts. 92 | for _ in range(N): 93 | await listener.put( 94 | models.Event( 95 | channel=channel, 96 | operation="insert", 97 | sent_at=datetime.datetime.now(tz=datetime.timezone.utc), 98 | table="placeholder", 99 | ) 100 | ) 101 | assert not strategy.clear() 102 | 103 | 104 | @pytest.mark.parametrize("N", (4, 16, 64)) 105 | @pytest.mark.parametrize( 106 | "dt", 107 | ( 108 | datetime.timedelta(milliseconds=5), 109 | datetime.timedelta(milliseconds=10), 110 | ), 111 | ) 112 | async def test_timed_strategy( 113 | dt: datetime.timedelta, 114 | N: int, 115 | pgconn: asyncpg.Connection, 116 | ) -> None: 117 | channel = models.PGChannel("test_timed_strategy") 118 | listener = listeners.PGEventQueue() 119 | await listener.connect(pgconn, channel) 120 | strategy = strategies.Timed(listener=listener, timedelta=dt) 121 | 122 | # Bursed spaced out accoring to min dt req. to trigger a refresh. 123 | for _ in range(N): 124 | await asyncio.sleep(dt.total_seconds()) 125 | await listener.put( 126 | models.Event( 127 | channel=channel, 128 | operation="insert", 129 | sent_at=datetime.datetime.now(tz=datetime.timezone.utc), 130 | table="placeholder", 131 | ) 132 | ) 133 | assert strategy.clear() 134 | 135 | # Bursets to close to trigger a refresh. 136 | for _ in range(N): 137 | await listener.put( 138 | models.Event( 139 | channel=channel, 140 | operation="insert", 141 | sent_at=datetime.datetime.now(tz=datetime.timezone.utc), 142 | table="placeholder", 143 | ) 144 | ) 145 | assert not strategy.clear() 146 | 147 | # No evnets, no clear. 148 | for _ in range(N): 149 | assert not strategy.clear() 150 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import datetime 3 | import time 4 | from typing import get_args 5 | 6 | import asyncpg 7 | import pytest 8 | from pgcachewatch import listeners, models, utils 9 | 10 | 11 | @pytest.mark.parametrize("N", (1, 2, 8)) 12 | @pytest.mark.parametrize("operation", get_args(models.OPERATIONS)) 13 | async def test_emit_event( 14 | N: int, 15 | operation: models.OPERATIONS, 16 | pgconn: asyncpg.Connection, 17 | pgpool: asyncpg.Pool, 18 | ) -> None: 19 | channel = "test_emit_event" 20 | listener = listeners.PGEventQueue() 21 | await listener.connect(pgconn, models.PGChannel(channel)) 22 | await asyncio.gather( 23 | *[ 24 | utils.emit_event( 25 | pgpool, 26 | models.Event( 27 | channel=channel, 28 | operation=operation, 29 | sent_at=datetime.datetime.now(tz=datetime.timezone.utc), 30 | table="placeholder", 31 | ), 32 | ) 33 | for _ in range(N) 34 | ] 35 | ) 36 | 37 | # Give a bit of leeway due IO network io. 38 | await asyncio.sleep(0.1) 39 | 40 | assert listener.qsize() == N 41 | events = [listener.get_nowait() for _ in range(N)] 42 | assert len(events) == N 43 | assert [e.operation for e in events].count(operation) == N 44 | 45 | 46 | @pytest.mark.parametrize("max_iter", (100, 200, 500)) 47 | async def test_pick_until_deadline_max_iter( 48 | max_iter: int, 49 | pgconn: asyncpg.Connection, 50 | ) -> None: 51 | channel = "test_pick_until_deadline_max_iter" 52 | listener = listeners.PGEventQueue() 53 | await listener.connect(pgconn, models.PGChannel(channel)) 54 | 55 | items = list(range(max_iter * 2)) 56 | for item in items: 57 | listener.put_nowait(item) # type: ignore 58 | 59 | assert listener.qsize() == len(items) 60 | assert ( 61 | len( 62 | list( 63 | utils.pick_until_deadline( 64 | listener, 65 | settings=models.DeadlineSetting( 66 | max_iter=max_iter, 67 | max_time=datetime.timedelta(days=1), 68 | ), 69 | ) 70 | ) 71 | ) 72 | == max_iter 73 | ) 74 | 75 | 76 | @pytest.mark.parametrize( 77 | "max_time", 78 | ( 79 | datetime.timedelta(milliseconds=25), 80 | datetime.timedelta(milliseconds=50), 81 | datetime.timedelta(milliseconds=100), 82 | ), 83 | ) 84 | async def test_pick_until_deadline_max_time( 85 | max_time: datetime.timedelta, 86 | monkeypatch: pytest.MonkeyPatch, 87 | pgconn: asyncpg.Connection, 88 | ) -> None: 89 | channel = "test_pick_until_deadline_max_time" 90 | listener = listeners.PGEventQueue() 91 | await listener.connect(pgconn, models.PGChannel(channel)) 92 | 93 | x = -1 94 | 95 | def always_get_noawit() -> int: 96 | nonlocal x 97 | x += 1 98 | return x 99 | 100 | monkeypatch.setattr(listener, "get_nowait", always_get_noawit) 101 | 102 | start = time.perf_counter() 103 | list( 104 | utils.pick_until_deadline( 105 | listener, 106 | settings=models.DeadlineSetting( 107 | max_iter=1_000_000_000, 108 | max_time=max_time, 109 | ), 110 | ) 111 | ) 112 | end = time.perf_counter() 113 | assert end - start >= max_time.total_seconds() 114 | --------------------------------------------------------------------------------