├── test ├── __init__.py ├── unit │ ├── __init__.py │ ├── test_cache_keys.py │ ├── test_session.py │ ├── test_response.py │ └── test_base_backend.py ├── server │ ├── gunicorn-cfg.py │ ├── __init__.py │ └── Dockerfile ├── integration │ ├── __init__.py │ ├── test_redis.py │ ├── test_memory.py │ ├── test_dynamodb.py │ ├── test_mongodb.py │ ├── test_filesystem.py │ ├── base_storage_test.py │ └── test_sqlite.py └── conftest.py ├── aiohttp_client_cache ├── py.typed ├── __init__.py ├── backends │ ├── __init__.py │ ├── mongodb.py │ ├── redis.py │ ├── filesystem.py │ ├── dynamodb.py │ └── sqlite.py ├── cache_keys.py ├── signatures.py ├── session.py ├── response.py └── cache_control.py ├── .prettierrc ├── docs ├── history.md ├── contributors.md ├── contributing.md ├── Dockerfile ├── docker-compose.yml ├── index.md ├── _templates │ └── apidoc │ │ └── module.rst.jinja ├── _static │ ├── collapsible_container.css │ ├── collapsible_container.js │ └── table.css ├── aiohttp_client_cache.session.md ├── reference.md ├── related_projects.md ├── examples.md ├── security.md ├── conf.py ├── backends.md └── user_guide.md ├── .github ├── ISSUE_TEMPLATE │ ├── question.md │ ├── feature.md │ └── bug.md ├── dependabot.yml └── workflows │ ├── deploy.yml │ └── build.yml ├── MANIFEST.in ├── examples ├── README.md ├── github.py ├── url_patterns.py ├── log_requests.py └── precache.py ├── .gitignore ├── .readthedocs.yml ├── dragonflydb.yaml ├── .pre-commit-config.yaml ├── CONTRIBUTORS.md ├── LICENSE ├── docker-compose.yml ├── noxfile.py ├── pyproject.toml ├── README.md ├── CONTRIBUTING.md └── HISTORY.md /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /aiohttp_client_cache/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true 3 | } 4 | -------------------------------------------------------------------------------- /docs/history.md: -------------------------------------------------------------------------------- 1 | ```{include} ../HISTORY.md 2 | 3 | ``` 4 | -------------------------------------------------------------------------------- /docs/contributors.md: -------------------------------------------------------------------------------- 1 | ```{include} ../CONTRIBUTORS.md 2 | 3 | ``` 4 | -------------------------------------------------------------------------------- /docs/contributing.md: -------------------------------------------------------------------------------- 1 | (contributing)= 2 | 3 | ```{include} ../CONTRIBUTING.md 4 | 5 | ``` 6 | -------------------------------------------------------------------------------- /test/server/gunicorn-cfg.py: -------------------------------------------------------------------------------- 1 | bind = '0.0.0.0:8181' 2 | workers = 1 3 | accesslog = '-' 4 | loglevel = 'info' 5 | capture_output = True 6 | enable_stdio_inheritance = True 7 | -------------------------------------------------------------------------------- /test/integration/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa: F401 2 | from test.integration.base_backend_test import BaseBackendTest 3 | from test.integration.base_storage_test import BaseStorageTest 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U00002754 Question" 3 | about: 'Question about how to use aiohttp-client-cache or other project details' 4 | title: '' 5 | labels: question 6 | --- 7 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.md 2 | include *.yml 3 | include LICENSE 4 | include aiohttp_client_cache/py.typed 5 | graft docs 6 | graft examples 7 | graft test 8 | prune docs/_* 9 | global-exclude *.py[co] 10 | -------------------------------------------------------------------------------- /docs/Dockerfile: -------------------------------------------------------------------------------- 1 | # Readthedocs build container with project dependencies pre-installed 2 | FROM readthedocs/build:8.0 3 | COPY . /src/ 4 | RUN pip3 install -U /src/[docs,backends] 5 | ENTRYPOINT ["/bin/bash"] 6 | -------------------------------------------------------------------------------- /aiohttp_client_cache/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.12.4' 2 | 3 | # flake8: noqa: F401, F403 4 | from aiohttp_client_cache.backends import * 5 | from aiohttp_client_cache.response import CachedResponse 6 | from aiohttp_client_cache.session import CachedSession 7 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # aiohttp-client-cache examples 2 | 3 | This folder contains some complete examples for using the main features of aiohttp-client-cache. 4 | These are also viewable on [readthedocs](https://aiohttp-client-cache.readthedocs.io/en/latest/examples.html). 5 | -------------------------------------------------------------------------------- /docs/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | services: 4 | readthedocs: 5 | container_name: readthedocs 6 | build: 7 | context: .. 8 | dockerfile: docs/Dockerfile 9 | network: host 10 | user: '1000' 11 | tty: true 12 | volumes: 13 | - '..:/home/docs/project' 14 | working_dir: '/home/docs/project/docs' 15 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | (index-page)= 2 | 3 | ```{include} ../README.md 4 | 5 | ``` 6 | 7 | # Contents 8 | 9 | ```{toctree} 10 | :maxdepth: 2 11 | 12 | user_guide 13 | backends 14 | security 15 | examples 16 | reference 17 | ``` 18 | 19 | # Project Info 20 | 21 | ```{toctree} 22 | :maxdepth: 2 23 | 24 | related_projects 25 | contributing 26 | contributors 27 | history 28 | ``` 29 | -------------------------------------------------------------------------------- /docs/_templates/apidoc/module.rst.jinja: -------------------------------------------------------------------------------- 1 | {{- basename.split('.')[-1] | e | heading }} 2 | 3 | Summary 4 | ------- 5 | .. automodsumm:: {{ qualname }} 6 | :classes-only: 7 | 8 | .. automodsumm:: {{ qualname }} 9 | :functions-only: 10 | 11 | Module Contents 12 | --------------- 13 | .. automodule:: {{ qualname }} 14 | {%- for option in automodule_options %} 15 | :{{ option }}: 16 | {%- endfor %} 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | *.sqlite 3 | *.egg 4 | *.egg-info 5 | .idea/ 6 | .venv 7 | .vim/ 8 | .vscode/ 9 | build/ 10 | dist/ 11 | downloads/ 12 | venv/ 13 | 14 | # Test / coverage reports 15 | .coverage 16 | .coverage.* 17 | .mypy_cache/ 18 | .tox 19 | test-reports/ 20 | 21 | # Sphinx 22 | docs/_build/ 23 | docs/modules/ 24 | docs/requirements.txt 25 | 26 | # JS 27 | node_modules/ 28 | package.json 29 | package-lock.json 30 | -------------------------------------------------------------------------------- /docs/_static/collapsible_container.css: -------------------------------------------------------------------------------- 1 | /* Adapted from: https://github.com/plone/training/blob/master/_static/custom.css */ 2 | 3 | .toggle .admonition-title { 4 | display: block; 5 | clear: both; 6 | cursor: pointer; 7 | } 8 | 9 | .toggle .admonition-title:after { 10 | content: ' ▶'; 11 | } 12 | 13 | .toggle .admonition-title.open:after { 14 | content: ' ▼'; 15 | } 16 | 17 | .toggle p:last-child { 18 | margin-bottom: 0; 19 | } 20 | -------------------------------------------------------------------------------- /docs/_static/collapsible_container.js: -------------------------------------------------------------------------------- 1 | // Taken from: https://github.com/plone/training/blob/master/_templates/page.html 2 | 3 | $(document).ready(function () { 4 | $('.toggle > *').hide(); 5 | $('.toggle .admonition-title').show(); 6 | $('.toggle .admonition-title').click(function () { 7 | $(this).parent().children().not('.admonition-title').toggle(400); 8 | $(this).parent().children('.admonition-title').toggleClass('open'); 9 | }); 10 | }); 11 | -------------------------------------------------------------------------------- /docs/aiohttp_client_cache.session.md: -------------------------------------------------------------------------------- 1 | # session 2 | 3 | ## Summary 4 | 5 | ```{eval-rst} 6 | .. automodsumm:: aiohttp_client_cache.session 7 | :classes-only: 8 | ``` 9 | 10 | ## Module Contents 11 | 12 | ```{eval-rst} 13 | .. autoclass:: aiohttp_client_cache.session.CachedSession 14 | :members: _request, disabled, delete_expired_responses 15 | :show-inheritance: 16 | ``` 17 | 18 | ```{eval-rst} 19 | .. autoclass:: aiohttp_client_cache.session.CacheMixin 20 | ``` 21 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: 'pip' 4 | directory: '/' 5 | schedule: 6 | interval: 'weekly' 7 | day: 'saturday' 8 | time: '16:00' 9 | # Only create PRs for major and minor releases 10 | ignore: 11 | - dependency-name: '*' 12 | update-types: ['version-update:semver-patch'] 13 | - package-ecosystem: 'github-actions' 14 | directory: '/' 15 | schedule: 16 | interval: 'weekly' 17 | day: 'saturday' 18 | time: '16:00' 19 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file 2 | version: 2 3 | 4 | sphinx: 5 | builder: html 6 | configuration: docs/conf.py 7 | 8 | build: 9 | os: 'ubuntu-24.04' 10 | tools: 11 | python: '3.12' 12 | jobs: 13 | # Use uv to export optional + documentation dependencies 14 | post_create_environment: 15 | - pip install uv 16 | - uv export -q --no-dev --group docs --no-emit-project -o docs/requirements.txt 17 | python: 18 | install: 19 | - method: pip 20 | path: . 21 | - requirements: docs/requirements.txt 22 | -------------------------------------------------------------------------------- /docs/_static/table.css: -------------------------------------------------------------------------------- 1 | /* Slightly modified table styles from Furo theme*/ 2 | 3 | table.docutils { 4 | border-radius: 0.2rem; 5 | border-spacing: 0; 6 | border-collapse: collapse; 7 | box-shadow: 8 | 0 2px 2px 0 rgba(0, 0, 0, 0.14), 9 | 0 1px 5px 0 rgba(0, 0, 0, 0.12), 10 | 0 3px 1px -2px rgba(0, 0, 0, 0.2); 11 | } 12 | 13 | table.docutils td { 14 | text-align: left; 15 | } 16 | table.docutils tr { 17 | transition: background-color 0.125s; 18 | } 19 | table.docutils tr:hover { 20 | background-color: rgba(0, 0, 0, 0.035); 21 | box-shadow: inset 0 0.05rem 0 #fff; 22 | } 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F680 Feature request" 3 | about: 'Request a new feature or other non-bugfix improvement' 4 | title: 'Feature request:' 5 | labels: enhancement 6 | --- 7 | 8 | ### Feature description 9 | 10 | ### Use case 11 | 12 | _Is there a specific goal that this would help you accomplish, or can you provide any other context about how you would like to use this feature?_ 13 | 14 | ### Workarounds 15 | 16 | _Is there an existing workaround to accomplish this?_ 17 | 18 | ### Plan to implement 19 | 20 | _Are you interested in submitting a PR to implement this?_ 21 | -------------------------------------------------------------------------------- /test/server/__init__.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from hashlib import md5 3 | 4 | from flask import Flask, request 5 | 6 | app = Flask(__name__) 7 | 8 | 9 | @app.route('/cache/') 10 | def cache(interval): 11 | """Cacheable endpoint that uses an ETag that resets every `interval` seconds""" 12 | now = datetime.now() 13 | server_etag = md5(str(now.second // interval).encode()).hexdigest() 14 | request_etag = request.headers.get('If-None-Match') 15 | if request_etag == server_etag: 16 | return 'NOT MODIFIED', 304 17 | else: 18 | return 'OK', 200, {'ETag': server_etag} 19 | -------------------------------------------------------------------------------- /dragonflydb.yaml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | services: 3 | dragonfly: 4 | image: 'docker.dragonflydb.io/dragonflydb/dragonfly' 5 | # NOTE: Disabled because this will not work with a rootless Docker. 6 | #ulimits: 7 | # memlock: -1 8 | ports: 9 | - '6379:6379' 10 | # For better performance, consider `host` mode instead `port` to avoid docker NAT. 11 | # `host` mode is NOT currently supported in Swarm Mode. 12 | # https://docs.docker.com/compose/compose-file/compose-file-v3/#network_mode 13 | # network_mode: "host" 14 | volumes: 15 | - dragonflydata:/data 16 | volumes: 17 | dragonflydata: 18 | -------------------------------------------------------------------------------- /test/server/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.13-slim 2 | 3 | WORKDIR /app 4 | 5 | ENV PIP_DEFAULT_TIMEOUT=100 \ 6 | PIP_DISABLE_PIP_VERSION_CHECK=1 \ 7 | PIP_NO_CACHE_DIR=1 8 | 9 | COPY test ./test 10 | COPY pyproject.toml uv.lock README.md ./ 11 | 12 | # Install uv 13 | COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv 14 | 15 | # Install dependencies 16 | RUN uv sync --group test-server --no-dev 17 | 18 | # set environment variables 19 | ENV PYTHONDONTWRITEBYTECODE=1 20 | ENV PYTHONUNBUFFERED=1 21 | ENV PYTHONPATH="/app/test" 22 | 23 | ENTRYPOINT ["uv", "run", "gunicorn", "server:app", "-c", "test/server/gunicorn-cfg.py"] 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F41E Bug report" 3 | about: 'Reporting a bug or other unexpected behavior' 4 | title: '' 5 | labels: bug 6 | --- 7 | 8 | ### The problem 9 | 10 | _A description of what the bug is, including a complete traceback (if applicable)_ 11 | 12 | ### Expected behavior 13 | 14 | _A description of what you expected to happen_ 15 | 16 | ### Steps to reproduce the behavior 17 | 18 | _With a complete code example, if possible_ 19 | 20 | ### Workarounds 21 | 22 | _Is there an existing workaround for this issue?_ 23 | 24 | ### Environment 25 | 26 | - aiohttp-client-cache version: [e.g. `0.3.0`] 27 | - Python version: [e.g. `3.9`] 28 | - Platform: [e.g. Debian 10] 29 | -------------------------------------------------------------------------------- /docs/reference.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | (api-reference)= 4 | 5 | # API Reference 6 | 7 | This section documents all the public interfaces of aiohttp_client_cache. 8 | 9 | ## Main Modules 10 | 11 | ```{toctree} 12 | :titlesonly: true 13 | aiohttp_client_cache.session.rst 14 | modules/aiohttp_client_cache.response.rst 15 | ``` 16 | 17 | ## Backend Modules 18 | 19 | ```{toctree} 20 | :glob: true 21 | :titlesonly: true 22 | modules/aiohttp_client_cache.backends.* 23 | ``` 24 | 25 | ## Utility Modules 26 | 27 | ```{toctree} 28 | :titlesonly: true 29 | modules/aiohttp_client_cache.cache_control.rst 30 | modules/aiohttp_client_cache.cache_keys.rst 31 | ``` 32 | -------------------------------------------------------------------------------- /docs/related_projects.md: -------------------------------------------------------------------------------- 1 | # Related Projects 2 | 3 | Other python cache projects you may want to check out: 4 | 5 | - [aiohttp-cache](https://github.com/cr0hn/aiohttp-cache): A server-side async HTTP cache for the 6 | `aiohttp` web server 7 | - [diskcache](https://github.com/grantjenks/python-diskcache): A general-purpose (not HTTP-specific) 8 | file-based cache built on SQLite 9 | - [aiocache](https://github.com/aio-libs/aiocache): General-purpose (not HTTP-specific) async cache 10 | backends 11 | - [requests-cache](https://github.com/reclosedev/requests-cache): An HTTP cache for the `requests` library 12 | - [CacheControl](https://github.com/ionrock/cachecontrol): An HTTP cache for `requests` that caches 13 | according to uses HTTP headers and status codes 14 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v6.0.0 4 | hooks: 5 | - id: check-toml 6 | - id: check-yaml 7 | - id: end-of-file-fixer 8 | - id: mixed-line-ending 9 | - id: trailing-whitespace 10 | - repo: https://github.com/astral-sh/ruff-pre-commit 11 | rev: v0.13.2 12 | hooks: 13 | - id: ruff 14 | args: [--fix] 15 | - id: ruff-format 16 | - repo: https://github.com/pre-commit/mirrors-mypy 17 | rev: v1.18.2 18 | hooks: 19 | - id: mypy 20 | files: aiohttp_client_cache 21 | additional_dependencies: [attrs, aiohttp, types-aiofiles, types-redis] 22 | - repo: https://github.com/pre-commit/mirrors-prettier 23 | rev: v4.0.0-alpha.8 24 | hooks: 25 | - id: prettier 26 | - repo: https://github.com/crate-ci/typos 27 | rev: v1.37.1 28 | hooks: 29 | - id: typos 30 | -------------------------------------------------------------------------------- /CONTRIBUTORS.md: -------------------------------------------------------------------------------- 1 | # Contributors 2 | 3 | Thanks to the following individuals for contributing to `aiohttp-client-cache`: 4 | 5 | - [0hax](https://github.com/0hax) 6 | - [Alessio Locatelli](https://github.com/alessio-locatelli) 7 | - [Andrew Kursar](https://github.com/akursar) 8 | - [Austin Raney](https://github.com/aaraney) 9 | - [Cristÿ Constantin](https://github.com/croqaz) 10 | - [Damian Fajfer](https://github.com/fajfer) 11 | - [Fredrik Bergroth](https://github.com/fbergroth) 12 | - [Ilias Tsatiris](https://github.com/iliastsa) 13 | - [James Braza](https://github.com/jamesbraza) 14 | - [Jamim](https://github.com/Jamim) 15 | - [Joe Bergeron](https://github.com/Jophish) 16 | - [Kirk Hansen](https://github.com/kirkhansen) 17 | - [Layday](https://github.com/layday) 18 | - [MrChuw](https://github.com/MrChuw) 19 | - [Project D.D.](https://github.com/RozeFound) 20 | - [Roman Haritonov](https://github.com/reclosedev) 21 | - [Thomas Neidhart](https://github.com/netomi) 22 | -------------------------------------------------------------------------------- /docs/examples.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | Following are some complete examples to demonstrate some of the features of aiohttp-client-cache. 4 | These can also be found in the 5 | [examples/](https://github.com/requests-cache/aiohttp-client-cache/tree/main/examples) folder on GitHub. 6 | 7 | ## Expiration based on URL patterns 8 | 9 | ```{include} ../examples/url_patterns.py 10 | :start-line: 3 11 | :end-line: 4 12 | ``` 13 | 14 | :::{admonition} Example code 15 | :class: toggle 16 | 17 | ```{literalinclude} ../examples/url_patterns.py 18 | :lines: 6- 19 | ``` 20 | 21 | ::: 22 | 23 | ## Precaching site links 24 | 25 | ```{include} ../examples/precache.py 26 | :start-line: 2 27 | :end-line: 16 28 | ``` 29 | 30 | :::{admonition} Example code 31 | :class: toggle 32 | 33 | ```{literalinclude} ../examples/precache.py 34 | :lines: 18- 35 | ``` 36 | 37 | ::: 38 | 39 | ## Logging requests 40 | 41 | ```{include} ../examples/log_requests.py 42 | :start-line: 2 43 | :end-line: 3 44 | ``` 45 | 46 | :::{admonition} Example code 47 | :class: toggle 48 | 49 | ```{literalinclude} ../examples/log_requests.py 50 | :lines: 5- 51 | ``` 52 | 53 | ::: 54 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Jordan Cook 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test/integration/test_redis.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | 5 | import pytest 6 | from redis.asyncio import from_url 7 | 8 | from aiohttp_client_cache.backends.redis import DEFAULT_ADDRESS, RedisBackend, RedisCache 9 | from test.integration import BaseBackendTest, BaseStorageTest 10 | 11 | 12 | def is_db_running(): 13 | """Test if a Redis server is running locally on the default port""" 14 | 15 | async def get_db_info(): 16 | client = await from_url(DEFAULT_ADDRESS) 17 | await client.info() 18 | await client.aclose() # type: ignore[attr-defined] 19 | 20 | try: 21 | asyncio.run(get_db_info()) 22 | return True 23 | except OSError as e: 24 | print(e) 25 | return False 26 | 27 | 28 | pytestmark = [ 29 | pytest.mark.asyncio, 30 | pytest.mark.skipif(not is_db_running(), reason='Redis server required for integration tests'), 31 | ] 32 | 33 | 34 | class TestRedisCache(BaseStorageTest): 35 | storage_class = RedisCache 36 | picklable = True 37 | 38 | 39 | class TestRedisBackend(BaseBackendTest): 40 | backend_class = RedisBackend 41 | -------------------------------------------------------------------------------- /examples/github.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # fmt: off 3 | """ 4 | An example of making conditional requests to the GitHub Rest API` 5 | """ 6 | 7 | import asyncio 8 | import logging 9 | 10 | from aiohttp_client_cache import CachedSession, FileBackend 11 | 12 | CACHE_DIR = "~/.cache/aiohttp-requests" 13 | 14 | 15 | async def main(): 16 | cache = FileBackend(cache_name=CACHE_DIR, use_temp=True) 17 | await cache.clear() 18 | 19 | org = "requests-cache" 20 | url = f"https://api.github.com/orgs/{org}/repos" 21 | 22 | # we make 2 requests for the same resource (list of all repos of the requests-cache organization) 23 | # the second request refreshes the cached response with the remote server 24 | # the debug output should illustrate that the cached response gets refreshed 25 | async with CachedSession(cache=cache) as session: 26 | response = await session.get(url) 27 | print(f"url = {response.url}, status = {response.status}, " 28 | f"ratelimit-used = {response.headers['x-ratelimit-used']}") 29 | 30 | await asyncio.sleep(1) 31 | 32 | response = await session.get(url, refresh=True) 33 | print(f"url = {response.url}, status = {response.status}, " 34 | f"ratelimit-used = {response.headers['x-ratelimit-used']}") 35 | 36 | 37 | if __name__ == "__main__": 38 | logging.basicConfig(level=logging.INFO) 39 | logging.getLogger("aiohttp_client_cache").setLevel(logging.DEBUG) 40 | asyncio.run(main()) 41 | -------------------------------------------------------------------------------- /examples/url_patterns.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # fmt: off 3 | """ 4 | An example of setting expiration based on {ref}`user_guide:url patterns` 5 | """ 6 | import asyncio 7 | from datetime import timedelta 8 | 9 | from aiohttp_client_cache import CachedSession, SQLiteBackend 10 | 11 | default_expire_after = 60 * 60 # By default, cached responses expire in an hour 12 | urls_expire_after = { 13 | 'httpbin.org/image': timedelta(days=7), # Requests for this base URL will expire in a week 14 | '*.fillmurray.com': -1, # Requests matching this pattern will never expire 15 | } 16 | urls = [ 17 | 'https://httpbin.org/get', # Will expire in an hour 18 | 'https://httpbin.org/image/jpeg', # Will expire in a week 19 | 'http://www.fillmurray.com/460/300', # Will never expire 20 | ] 21 | 22 | 23 | async def main(): 24 | cache = SQLiteBackend( 25 | cache_name='~/.cache/aiohttp-requests.db', 26 | expire_after=default_expire_after, 27 | urls_expire_after=urls_expire_after, 28 | ) 29 | 30 | async with CachedSession(cache=cache) as session: 31 | tasks = [asyncio.create_task(session.get(url)) for url in urls] 32 | return await asyncio.gather(*tasks) 33 | 34 | 35 | if __name__ == "__main__": 36 | original_responses = asyncio.run(main()) 37 | cached_responses = asyncio.run(main()) 38 | for response in cached_responses: 39 | expires = response.expires.isoformat() if response.expires else 'Never' 40 | print(f'{response.url}: {expires}') 41 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Containers needed to test all backend services locally 2 | services: 3 | httpbin: 4 | image: kennethreitz/httpbin 5 | container_name: httpbin 6 | ports: 7 | # Use an unprivileged port to support running the Docker daemon as a non-root user (Rootless mode). 8 | # See https://docs.docker.com/engine/security/rootless/#networking-errors 9 | - ${HTTPBIN_CUSTOM_PORT:-8080}:80 10 | 11 | httpbin-custom: 12 | container_name: httpbin-custom 13 | build: 14 | context: . 15 | dockerfile: test/server/Dockerfile 16 | ports: 17 | - '8181:8181' 18 | 19 | dynamodb: 20 | image: amazon/dynamodb-local 21 | hostname: dynamodb-local 22 | container_name: dynamodb-local 23 | ports: 24 | - 8000:8000 25 | command: '-jar DynamoDBLocal.jar -inMemory' 26 | environment: 27 | AWS_ACCESS_KEY_ID: 'placeholder' 28 | AWS_SECRET_ACCESS_KEY: 'placeholder' 29 | working_dir: /home/dynamodblocal 30 | 31 | mongo: 32 | image: mongo 33 | container_name: mongo 34 | environment: 35 | MONGO_INITDB_DATABASE: aiohttp_client_cache_pytest 36 | ports: 37 | - 27017:27017 38 | volumes: 39 | - 'mongodb_data:/data/db' 40 | 41 | redis: 42 | image: docker.io/bitnami/redis 43 | container_name: redis 44 | environment: 45 | ALLOW_EMPTY_PASSWORD: 'yes' 46 | ports: 47 | - 6379:6379 48 | volumes: 49 | - 'redis_data:/bitnami/redis/data' 50 | 51 | volumes: 52 | mongodb_data: 53 | driver: local 54 | redis_data: 55 | driver: local 56 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Deploy 2 | 3 | on: 4 | push: 5 | tags: ['*'] 6 | workflow_dispatch: 7 | inputs: 8 | pre-release-suffix: 9 | description: 'Version suffix for pre-releases ("a", "b", "rc", etc.)' 10 | required: false 11 | default: 'dev' 12 | pre-release-version: 13 | description: 'Version number for pre-releases; defaults to build number' 14 | required: false 15 | default: '' 16 | 17 | jobs: 18 | # Deploy stable builds on tags only, and pre-release builds from manual trigger ("workflow_dispatch") 19 | release: 20 | runs-on: ubuntu-latest 21 | permissions: 22 | id-token: write 23 | steps: 24 | - uses: actions/checkout@v5 25 | - uses: astral-sh/setup-uv@v7 26 | 27 | - name: Set pre-release version 28 | if: ${{ !startsWith(github.ref, 'refs/tags/v') }} 29 | env: 30 | pre-release-suffix: ${{ github.event.inputs.pre-release-suffix || 'dev' }} 31 | pre-release-version: ${{ github.event.inputs.pre-release-version || github.run_number }} 32 | run: | 33 | PKG_VERSION=$(uvx --from=toml-cli toml get --toml-path=pyproject.toml project.version) 34 | DEV_VERSION=$PKG_VERSION.${{ env.pre-release-suffix }}${{ env.pre-release-version }} 35 | echo "Setting pre-release version to $DEV_VERSION" 36 | uvx --from=toml-cli toml set --toml-path=pyproject.toml project.version $DEV_VERSION 37 | 38 | - name: Build package distributions 39 | run: uv build 40 | - name: Publish package distributions to PyPI 41 | uses: pypa/gh-action-pypi-publish@release/v1 42 | -------------------------------------------------------------------------------- /test/integration/test_memory.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from aiohttp_client_cache.backends.base import CacheBackend, DictCache 4 | from test.conftest import httpbin 5 | from test.integration import BaseBackendTest, BaseStorageTest 6 | 7 | 8 | class TestMemoryBackend(BaseBackendTest): 9 | """Run tests for CacheBackend base class, which uses in-memory caching by default""" 10 | 11 | backend_class = CacheBackend 12 | 13 | async def test_content_reset(self): 14 | """Test that cached response content can be read multiple times (without consuming and 15 | re-reading the same file-like object) 16 | """ 17 | url = httpbin('get') 18 | async with self.init_session() as session: 19 | original_response = await session.get(url) 20 | original_content = await original_response.read() 21 | 22 | cached_response_1 = await session.get(url) 23 | content_1 = await cached_response_1.read() 24 | cached_response_2 = await session.get(url) 25 | content_2 = await cached_response_2.read() 26 | assert content_1 == content_2 == original_content 27 | 28 | async def test_without_contextmanager(self): 29 | """Test that the cache backend can be safely used without the CachedSession contextmanager. 30 | An "unclosed ClientSession" warning is expected here, however. 31 | """ 32 | session = await self._init_session() 33 | await session.get(httpbin('get')) 34 | del session 35 | 36 | # Serialization tests don't apply to in-memory cache 37 | async def test_serializer__pickle(self): 38 | pass 39 | 40 | async def test_serializer__itsdangerous(self): 41 | pass 42 | 43 | 44 | class TestMemoryCache(BaseStorageTest): 45 | storage_class = DictCache 46 | -------------------------------------------------------------------------------- /test/integration/test_dynamodb.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from os import urandom 4 | from typing import Any 5 | 6 | import pytest 7 | 8 | from aiohttp_client_cache.backends.dynamodb import MAX_ITEM_SIZE, DynamoDBBackend, DynamoDbCache 9 | from test.integration import BaseBackendTest, BaseStorageTest 10 | 11 | AWS_OPTIONS = { 12 | 'endpoint_url': 'http://localhost:8000', 13 | 'region_name': 'us-east-1', 14 | 'aws_access_key_id': 'placeholder', 15 | 'aws_secret_access_key': 'placeholder', 16 | } 17 | 18 | 19 | def is_dynamodb_running(): 20 | """Test if a DynamoDB service is running locally""" 21 | import boto3 22 | 23 | try: 24 | client = boto3.client('dynamodb', **AWS_OPTIONS) 25 | client.describe_limits() 26 | return True 27 | except OSError: 28 | return False 29 | 30 | 31 | pytestmark = [ 32 | pytest.mark.asyncio, 33 | pytest.mark.skipif( 34 | not is_dynamodb_running(), reason='local DynamoDB service required for integration tests' 35 | ), 36 | ] 37 | 38 | 39 | class TestDynamoDbCache(BaseStorageTest): 40 | storage_class = DynamoDbCache 41 | picklable = True 42 | init_kwargs = { 43 | 'create_if_not_exists': True, 44 | 'key_attr_name': 'k', 45 | 'val_attr_name': 'v', 46 | **AWS_OPTIONS, 47 | } 48 | 49 | async def test_write_oversized_item(self): 50 | """If an item exceeds DynamoDB's max item size, expect it to not be written to the cache""" 51 | data = urandom(MAX_ITEM_SIZE + 1) 52 | async with self.init_cache(self.storage_class) as cache: 53 | await cache.write('key', data) 54 | assert await cache.contains('key') is False 55 | 56 | 57 | class TestDynamoDBBackend(BaseBackendTest): 58 | backend_class = DynamoDBBackend 59 | init_kwargs: dict[str, Any] = {'create_if_not_exists': True, **AWS_OPTIONS} 60 | -------------------------------------------------------------------------------- /examples/log_requests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | An example of testing the cache to prove that it's not making more requests than expected. 4 | """ 5 | 6 | import asyncio 7 | from contextlib import asynccontextmanager 8 | from logging import basicConfig, getLogger 9 | from unittest.mock import patch 10 | 11 | from aiohttp import ClientSession 12 | 13 | from aiohttp_client_cache import CachedResponse, CachedSession, SQLiteBackend 14 | 15 | basicConfig(level='INFO') 16 | logger = getLogger('aiohttp_client_cache.examples') 17 | # Uncomment for more verbose debug output 18 | # getLogger('aiohttp_client_cache').setLevel('DEBUG') 19 | 20 | 21 | @asynccontextmanager 22 | async def log_requests(): 23 | """Context manager that mocks and logs all non-cached requests""" 24 | 25 | async def mock_response(*args, **kwargs): 26 | return CachedResponse(method='GET', reason='OK', status=200, url='url', version='1.1') 27 | 28 | with patch.object(ClientSession, '_request', side_effect=mock_response) as mock_request: 29 | async with CachedSession(cache=SQLiteBackend('cache-test.sqlite')) as session: 30 | await session.cache.clear() 31 | yield session 32 | cached_responses = [v async for v in session.cache.responses.values()] 33 | 34 | logger.debug('All calls to ClientSession._request():') 35 | logger.debug(mock_request.mock_calls) 36 | 37 | logger.info(f'Responses cached: {len(cached_responses)}') 38 | logger.info(f'Requests sent: {mock_request.call_count}') 39 | 40 | 41 | async def main(): 42 | """Example usage; replace with any other requests you want to test""" 43 | async with log_requests() as session: 44 | for i in range(10): 45 | response = await session.get('http://httpbin.org/get') 46 | logger.debug(f'Response {i}: {type(response).__name__}') 47 | 48 | 49 | if __name__ == '__main__': 50 | asyncio.run(main()) 51 | -------------------------------------------------------------------------------- /test/integration/test_mongodb.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | 5 | import pytest 6 | from pymongo import MongoClient 7 | from pymongo.errors import ConnectionFailure 8 | 9 | from aiohttp_client_cache.backends.mongodb import MongoDBBackend, MongoDBCache, MongoDBPickleCache 10 | from test.integration import BaseBackendTest, BaseStorageTest 11 | 12 | 13 | def is_db_running(): 14 | """Test if a MongoDB server is running locally on the default port""" 15 | try: 16 | client = MongoClient(serverSelectionTimeoutMS=200) 17 | client.server_info() 18 | return True 19 | except ConnectionFailure: 20 | return False 21 | 22 | 23 | pytestmark = [ 24 | pytest.mark.asyncio, 25 | pytest.mark.skipif(not is_db_running(), reason='MongoDB server required for integration tests'), 26 | ] 27 | 28 | 29 | class TestMongoDBCache(BaseStorageTest): 30 | storage_class = MongoDBCache 31 | 32 | async def test_connection_kwargs(self): 33 | loop = asyncio.get_running_loop() 34 | async with self.init_cache(self.storage_class, host='127.0.0.1', io_loop=loop) as cache: 35 | assert cache.connection.address == ('127.0.0.1', 27017) 36 | assert cache.connection.io_loop is loop 37 | 38 | async def test_values_many(self): 39 | # If some entries are missing the "data" field for some reason, they 40 | # should not be returned with the results. 41 | async with self.init_cache(self.storage_class) as cache: 42 | await cache.collection.insert_many({'data': f'value_{i}'} for i in range(10)) 43 | await cache.collection.insert_many({'not_data': f'value_{i}'} for i in range(10)) 44 | actual_results = [v async for v in cache.values()] 45 | assert actual_results == [f'value_{i}' for i in range(10)] 46 | 47 | 48 | class TestMongoDBPickleCache(TestMongoDBCache): 49 | storage_class = MongoDBPickleCache 50 | picklable = True 51 | 52 | 53 | class TestMongoDBBackend(BaseBackendTest): 54 | backend_class = MongoDBBackend 55 | -------------------------------------------------------------------------------- /aiohttp_client_cache/backends/__init__.py: -------------------------------------------------------------------------------- 1 | from inspect import Parameter, signature 2 | from logging import getLogger 3 | from typing import Callable 4 | 5 | from aiohttp_client_cache.backends.base import ( # noqa: F401 6 | BaseCache, 7 | CacheBackend, 8 | DictCache, 9 | ResponseOrKey, 10 | ) 11 | 12 | logger = getLogger(__name__) 13 | 14 | 15 | def get_placeholder_backend(original_exception): 16 | """This creates a placeholder type for a backend class that does not have dependencies 17 | installed. This allows delaying the ImportError until init is called, rather then when imported. 18 | """ 19 | 20 | class PlaceholderBackend: 21 | def __init__(*args, **kwargs): 22 | logger.error('Dependencies are not installed for this backend') 23 | raise original_exception 24 | 25 | return PlaceholderBackend 26 | 27 | 28 | def get_valid_kwargs(func: Callable, kwargs: dict, accept_varkwargs: bool = True) -> dict: 29 | """Get the subset of non-None ``kwargs`` that are valid params for ``func``""" 30 | params = signature(func).parameters 31 | 32 | # If func accepts variable keyword arguments (**kwargs), all are valid 33 | if accept_varkwargs and any(p.kind is Parameter.VAR_KEYWORD for p in params.values()): 34 | return kwargs 35 | 36 | return {k: v for k, v in kwargs.items() if k in params.keys() and v is not None} 37 | 38 | 39 | # Import all backends for which dependencies are installed 40 | try: 41 | from aiohttp_client_cache.backends.dynamodb import DynamoDBBackend 42 | except ImportError as e: 43 | DynamoDBBackend = get_placeholder_backend(e) # type: ignore 44 | try: 45 | from aiohttp_client_cache.backends.filesystem import FileBackend 46 | except ImportError as e: 47 | FileBackend = get_placeholder_backend(e) # type: ignore 48 | try: 49 | from aiohttp_client_cache.backends.mongodb import MongoDBBackend 50 | except ImportError as e: 51 | MongoDBBackend = get_placeholder_backend(e) # type: ignore 52 | try: 53 | from aiohttp_client_cache.backends.redis import RedisBackend 54 | except ImportError as e: 55 | RedisBackend = get_placeholder_backend(e) # type: ignore 56 | try: 57 | from aiohttp_client_cache.backends.sqlite import SQLiteBackend 58 | except ImportError as e: 59 | SQLiteBackend = get_placeholder_backend(e) # type: ignore 60 | -------------------------------------------------------------------------------- /test/integration/test_filesystem.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import AsyncIterator 4 | from contextlib import asynccontextmanager 5 | from os.path import isfile 6 | from shutil import rmtree 7 | from tempfile import gettempdir 8 | 9 | from aiohttp_client_cache.backends.filesystem import FileBackend, FileCache 10 | from aiohttp_client_cache.session import CachedSession 11 | from test.conftest import CACHE_NAME 12 | from test.integration import BaseBackendTest, BaseStorageTest 13 | 14 | 15 | class TestFileCache(BaseStorageTest): 16 | storage_class = FileCache 17 | picklable = True 18 | 19 | @asynccontextmanager 20 | async def init_cache(self, index=0, **kwargs) -> AsyncIterator[FileCache]: # type: ignore[override] 21 | cache = self.storage_class(f'{CACHE_NAME}_{index}', use_temp=True, **kwargs) 22 | await cache.clear() 23 | yield cache 24 | await cache.close() 25 | 26 | @classmethod 27 | def teardown_class(cls): 28 | rmtree(CACHE_NAME, ignore_errors=True) 29 | 30 | async def test_use_temp(self): 31 | relative_path = self.storage_class(CACHE_NAME).cache_dir 32 | temp_path = self.storage_class(CACHE_NAME, use_temp=True).cache_dir 33 | assert not relative_path.startswith(gettempdir()) 34 | assert temp_path.startswith(gettempdir()) 35 | 36 | async def test_paths(self): 37 | async with self.init_cache() as cache: 38 | for i in range(10): 39 | await cache.write(f'key_{i}', f'value_{i}') 40 | 41 | assert len([p async for p in cache.paths()]) == 10 42 | async for path in cache.paths(): 43 | assert isfile(path) 44 | 45 | # TODO 46 | async def test_write_error(self): 47 | pass 48 | 49 | 50 | class TestFileBackend(BaseBackendTest): 51 | backend_class = FileBackend 52 | init_kwargs = {'use_temp': True} 53 | 54 | async def test_redirect_cache_path(self): 55 | async with self.init_session() as session: 56 | assert isinstance(session, CachedSession) 57 | 58 | cache = session.cache 59 | assert isinstance(cache, FileBackend) 60 | 61 | cache_dir = cache.responses.cache_dir # type: ignore[attr-defined] 62 | redirects_file = cache.redirects.filename # type: ignore[attr-defined] 63 | 64 | assert redirects_file.startswith(cache_dir) 65 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | tags: ['*'] 7 | pull_request: 8 | branches: [main] 9 | workflow_dispatch: 10 | env: 11 | LATEST_PY_VERSION: '3.14' 12 | COVERAGE_ARGS: '--cov --cov-report=term --cov-report=xml' 13 | XDIST_ARGS: '--numprocesses=auto --dist=loadfile' 14 | 15 | jobs: 16 | test: 17 | runs-on: ubuntu-latest 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | python-version: 22 | - '3.14' 23 | - '3.13' 24 | - '3.12' 25 | - '3.11' 26 | - '3.10' 27 | - '3.9' 28 | services: 29 | nginx: 30 | image: kennethreitz/httpbin 31 | ports: 32 | - 8080:80 33 | 34 | steps: 35 | # Install dependencies, with caching 36 | - uses: actions/checkout@v5 37 | - uses: astral-sh/setup-uv@v7 38 | with: 39 | enable-cache: true 40 | cache-dependency-glob: uv.lock 41 | - name: Install dependencies 42 | run: | 43 | uv python install ${{ matrix.python-version }} 44 | uv sync --all-extras --group test-server 45 | 46 | # Start integration test databases 47 | - uses: supercharge/mongodb-github-action@1.12.0 48 | with: 49 | mongodb-version: '5.0' 50 | - uses: supercharge/redis-github-action@1.8.0 51 | with: 52 | redis-version: '6' 53 | - uses: rrainn/dynamodb-action@v4.0.0 54 | - name: Run custom test server 55 | run: | 56 | cd test && uv run gunicorn -D -c server/gunicorn-cfg.py server:app 57 | 58 | # Run tests with coverage report 59 | - name: Run tests 60 | run: | 61 | uv run pytest -rs test/unit ${{ env.XDIST_ARGS }} ${{ env.COVERAGE_ARGS }} 62 | uv run pytest -rs test/integration --cov-append ${{ env.XDIST_ARGS }} ${{ env.COVERAGE_ARGS }} 63 | 64 | # Latest python version: send coverage report to codecov 65 | - name: 'Upload coverage report to Codecov' 66 | if: ${{ matrix.python-version == env.LATEST_PY_VERSION }} 67 | uses: codecov/codecov-action@v5 68 | 69 | # Run code analysis checks 70 | analyze: 71 | runs-on: ubuntu-latest 72 | steps: 73 | - uses: actions/checkout@v5 74 | - uses: actions/setup-python@v6 75 | with: 76 | python-version: ${{ env.LATEST_PY_VERSION }} 77 | - name: Run style checks and linting via pre-commit hooks 78 | uses: pre-commit/action@v3.0.0 79 | -------------------------------------------------------------------------------- /test/conftest.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from collections.abc import AsyncIterator 3 | from contextlib import asynccontextmanager 4 | from datetime import datetime 5 | from os import getenv 6 | from tempfile import NamedTemporaryFile 7 | 8 | import pytest 9 | 10 | from aiohttp_client_cache import CachedResponse, CachedSession, SQLiteBackend 11 | 12 | ALL_METHODS = ['GET', 'HEAD', 'OPTIONS', 'POST', 'PUT', 'PATCH', 'DELETE'] 13 | CACHE_NAME = 'pytest_cache' 14 | HTTPBIN_METHODS = ['GET', 'POST', 'PUT', 'PATCH', 'DELETE'] 15 | HTTPBIN_FORMATS = [ 16 | 'brotli', 17 | 'deflate', 18 | 'deny', 19 | 'encoding/utf8', 20 | 'gzip', 21 | 'html', 22 | 'image/jpeg', 23 | 'image/png', 24 | 'image/svg', 25 | 'image/webp', 26 | 'json', 27 | 'robots.txt', 28 | 'xml', 29 | ] 30 | 31 | HTTPDATE_STR = 'Fri, 16 APR 2021 21:13:00 GMT' 32 | HTTPDATE_DATETIME = datetime(2021, 4, 16, 21, 13) 33 | 34 | 35 | # Configure logging for pytest session 36 | logging.basicConfig(level='INFO') 37 | # logging.getLogger('aiohttp_client_cache').setLevel('DEBUG') 38 | 39 | 40 | def from_cache(*responses) -> bool: 41 | """Indicate whether one or more responses came from the cache""" 42 | return all(isinstance(response, CachedResponse) for response in responses) 43 | 44 | 45 | def httpbin(path: str = ''): 46 | """Get the url for either a local or remote httpbin instance""" 47 | base_url = getenv('HTTPBIN_URL', 'http://localhost:8080/') 48 | return base_url + path 49 | 50 | 51 | def httpbin_custom(path: str = ''): 52 | """Get the url for a local httpbin_custom instance""" 53 | base_url = 'http://localhost:8181/' 54 | return base_url + path 55 | 56 | 57 | @pytest.fixture(scope='function') 58 | async def tempfile_session(): 59 | """:py:func:`.get_tempfile_session` as a pytest fixture""" 60 | async with get_tempfile_session() as session: 61 | yield session 62 | 63 | 64 | @asynccontextmanager 65 | async def get_tempfile_session(**kwargs) -> AsyncIterator[CachedSession]: 66 | """Get a CachedSession using a temporary SQLite db""" 67 | with NamedTemporaryFile(suffix='.db') as temp: 68 | cache = SQLiteBackend(cache_name=temp.name, allowed_methods=ALL_METHODS, **kwargs) 69 | async with CachedSession(cache=cache) as session: 70 | yield session 71 | 72 | 73 | def assert_delta_approx_equal(dt1: datetime, dt2: datetime, target_delta, threshold_seconds=2): 74 | """Assert that the given datetimes are approximately ``target_delta`` seconds apart""" 75 | diff_in_seconds = (dt2 - dt1).total_seconds() 76 | assert abs(diff_in_seconds - target_delta) <= threshold_seconds 77 | -------------------------------------------------------------------------------- /docs/security.md: -------------------------------------------------------------------------------- 1 | (security)= 2 | 3 | # Security 4 | 5 | ## Pickle Vulnerabilities 6 | 7 | :::{warning} 8 | The python `pickle` module has [known security vulnerabilities](https://docs.python.org/3/library/pickle.html), 9 | potentially leading to code execution when deserializing data. 10 | ::: 11 | 12 | This means it should only be used to deserialize data that you trust hasn't been tampered with. 13 | Since this isn't always possible, aiohttp-client-cache can optionally use 14 | [itsdangerous](https://itsdangerous.palletsprojects.com) to add a layer of security around these operations. 15 | It works by signing serialized data with a secret key that you control. Then, if the data is tampered 16 | with, the signature check fails and raises an error. 17 | 18 | ## Creating and Storing a Secret Key 19 | 20 | To enable this behavior, first create a secret key, which can be any `str` or `bytes` object. 21 | 22 | One common pattern for handling this is to store it wherever you store the rest of your credentials 23 | ([Linux keyring](https://itsfoss.com/ubuntu-keyring), 24 | [macOS keychain](https://support.apple.com/guide/mac-help/use-keychains-to-store-passwords-mchlf375f392/mac), 25 | [password database](https://keepassxc.org), etc.), 26 | set it in an environment variable, and then read it in your application: 27 | 28 | ```python 29 | >>> import os 30 | >>> secret_key = os.environ['SECRET_KEY'] 31 | ``` 32 | 33 | Alternatively, you can use the [keyring](https://keyring.readthedocs.io) package to read the key 34 | directly: 35 | 36 | ```python 37 | >>> import keyring 38 | >>> secret_key = keyring.get_password('aiohttp-example', 'secret_key') 39 | ``` 40 | 41 | ## Signing Cached Responses 42 | 43 | Once you have your key, just pass it to {py:class}`.CachedSession` or {py:func}`.install_cache` to start using it: 44 | 45 | ```python 46 | >>> from aiohttp_client_cache import CachedSession, RedisBackend 47 | 48 | >>> cache = RedisBackend(secret_key=secret_key) 49 | >>> async with CachedSession(cache=cache) as session: 50 | >>> await session.get('https://httpbin.org/get') 51 | ``` 52 | 53 | You can verify that it's working by modifying the cached item (_without_ your key): 54 | 55 | ```python 56 | >>> cache_2 = RedisBackend(secret_key='a different key') 57 | >>> async with CachedSession(cache=cache) as session_2: 58 | >>> cache_key = list(await session_2.cache.responses.keys())[0] 59 | >>> await session_2.cache.responses.write(cache_key, 'exploit!') 60 | ``` 61 | 62 | Then, if you try to get that cached response again (_with_ your key), you will get an error: 63 | 64 | ```python 65 | >>> async with CachedSession(cache=cache) as session: 66 | >>> await session.get('https://httpbin.org/get') 67 | BadSignature: Signature b'iFNmzdUOSw5vqrR9Cb_wfI1EoZ8' does not match 68 | ``` 69 | -------------------------------------------------------------------------------- /test/unit/test_cache_keys.py: -------------------------------------------------------------------------------- 1 | """The cache_keys module is mostly covered indirectly via other tests. 2 | This just contains tests for some extra edge cases not covered elsewhere. 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | from copy import copy 8 | 9 | import pytest 10 | from multidict import MultiDict 11 | 12 | from aiohttp_client_cache.cache_keys import create_key 13 | 14 | 15 | @pytest.mark.parametrize( 16 | 'url, params', 17 | [ 18 | ('https://example.com?foo=bar¶m=1', None), 19 | ('https://example.com?foo=bar¶m=1', {}), 20 | ('https://example.com?foo=bar¶m=1&', {}), 21 | ('https://example.com?param=1&foo=bar', {}), 22 | ('https://example.com?param=1', {'foo': 'bar'}), 23 | ('https://example.com?foo=bar', {'param': '1'}), 24 | ('https://example.com', {'param': '1', 'foo': 'bar'}), 25 | ('https://example.com', {'foo': 'bar', 'param': '1'}), 26 | ('https://example.com', {'foo': 'bar', 'param': 1}), 27 | ('https://example.com?', {'foo': 'bar', 'param': '1'}), 28 | ('https://example.com?', (('foo', 'bar'), ('param', '1'))), 29 | ], 30 | ) 31 | def test_normalize_url_params(url, params): 32 | """All of these variations should produce the same cache key""" 33 | original_params = copy(params) if params is not None else params 34 | cache_key = 'e93c762132a09fb2398beafee0ed2e9f4240ad941e905581631b9ac9e70ab40e' 35 | assert create_key('GET', url, params=params) == cache_key 36 | assert original_params == params # Make sure we didn't modify the original params object 37 | 38 | 39 | @pytest.mark.parametrize( 40 | 'url, params', 41 | [ 42 | ('https://example.com?param1=value1¶m1=value2', {}), 43 | ('https://example.com?param1=value1', {'param1': 'value2'}), 44 | ('https://example.com', (('param1', 'value1'), ('param1', 'value2'))), 45 | ('https://example.com', MultiDict((('param1', 'value1'), ('param1', 'value2')))), 46 | ], 47 | ) 48 | def test_encode_duplicate_params(url, params): 49 | """All means of providing request params with duplicate parameter names should result in a 50 | cache key distinct from a request with only one of that parameter name. 51 | """ 52 | assert ( 53 | create_key('GET', url, params=params) 54 | != create_key('GET', 'http://url.com?param1=value1') 55 | != create_key('GET', 'http://url.com?param1=value2') 56 | ) 57 | 58 | 59 | @pytest.mark.parametrize('field', ['data', 'json']) 60 | @pytest.mark.parametrize('body', [{'foo': 'bar'}, '{"foo": "bar"}', b'{"foo": "bar"}']) 61 | def test_encode_request_body(body, field): 62 | """Request body should be handled correctly whether it's a dict or already serialized""" 63 | cache_key = create_key('GET', 'https://example.com', **{field: body}) 64 | assert isinstance(cache_key, str) 65 | -------------------------------------------------------------------------------- /examples/precache.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | An example that fetches and caches the content of a given web page, and all links found on that page 4 | 5 | Usage: `./precache.py ` 6 | 7 | Example: 8 | ```bash 9 | $ # Run twice and note stats before and after 10 | $ ./precache.py https://www.nytimes.com 11 | Found 102 links 12 | Completed run in 6.195 seconds and cached 53.570 MB 13 | $ ./precache.py https://www.nytimes.com 14 | Found 102 links 15 | Completed run in 0.436 seconds and cached 0.000 MB 16 | ``` 17 | """ 18 | 19 | import asyncio 20 | import re 21 | import sys 22 | import time 23 | import urllib.parse 24 | from contextlib import contextmanager 25 | from os.path import getsize 26 | 27 | from aiohttp_client_cache import CachedSession, SQLiteBackend 28 | 29 | CACHE_NAME = 'precache' 30 | DEFAULT_URL = 'https://www.nytimes.com' 31 | HREF_PATTERN = re.compile(r'href="(.*?)"') 32 | 33 | 34 | async def precache_page_links(parent_url): 35 | """Fetch and cache the content of a given web page and all links found on that page""" 36 | async with CachedSession(cache=SQLiteBackend()) as session: 37 | urls = await get_page_links(session, parent_url) 38 | 39 | tasks = [asyncio.create_task(cache_url(session, url)) for url in urls] 40 | responses = await asyncio.gather(*tasks) 41 | 42 | return responses 43 | 44 | 45 | async def get_page_links(session, url): 46 | """Get all links found in the HTML of the given web page""" 47 | print(f'Finding all links on page: {url}') 48 | links = set() 49 | response = await session.get(url) 50 | response.raise_for_status() 51 | html = await response.text() 52 | 53 | for link in HREF_PATTERN.findall(html): 54 | try: 55 | links.add(urllib.parse.urljoin(url, link)) 56 | except Exception as e: 57 | print(f'Failed to add link: {link}') 58 | print(e) 59 | 60 | print(f'Found {len(links)} links') 61 | return links 62 | 63 | 64 | async def cache_url(session, url): 65 | try: 66 | return await session.get(url) 67 | except Exception as e: 68 | print(e) 69 | return None 70 | 71 | 72 | def get_cache_bytes(): 73 | """Get the current size of the cache, in bytes""" 74 | try: 75 | return getsize(f'{CACHE_NAME}.sqlite') 76 | except Exception: 77 | return 0 78 | 79 | 80 | @contextmanager 81 | def measure_cache(): 82 | """Measure time elapsed and size of added cache content""" 83 | start_time = time.perf_counter() 84 | start_bytes = get_cache_bytes() 85 | yield 86 | 87 | elapsed_time = time.perf_counter() - start_time 88 | cached_bytes = (get_cache_bytes() - start_bytes) / 1024 / 1024 89 | print(f'Completed run in {elapsed_time:0.3f} seconds and cached {cached_bytes:0.3f} MB') 90 | 91 | 92 | if __name__ == '__main__': 93 | parent_url = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_URL 94 | with measure_cache(): 95 | asyncio.run(precache_page_links(parent_url)) 96 | -------------------------------------------------------------------------------- /aiohttp_client_cache/cache_keys.py: -------------------------------------------------------------------------------- 1 | """Functions for creating keys used for cache requests""" 2 | 3 | from __future__ import annotations 4 | 5 | import hashlib 6 | from collections.abc import Iterable, Mapping, Sequence 7 | from typing import Any, Union 8 | 9 | from aiohttp.typedefs import StrOrURL 10 | from multidict import MultiDict 11 | from url_normalize import url_normalize 12 | from yarl import URL 13 | 14 | RequestParams = Union[Mapping, Sequence, str] 15 | 16 | 17 | def create_key( 18 | method: str, 19 | url: StrOrURL, 20 | params: RequestParams | None = None, 21 | data: dict | None = None, 22 | json: dict | None = None, 23 | headers: dict | None = None, 24 | include_headers: bool = False, 25 | ignored_params: Iterable[str] | None = None, 26 | **kwargs, 27 | ) -> str: 28 | """Create a unique cache key based on request details""" 29 | # Normalize and filter all relevant pieces of request data 30 | norm_url = normalize_url_params(url, params) 31 | if ignored_params: 32 | filtered_params = filter_ignored_params(norm_url.query, ignored_params) 33 | norm_url = norm_url.with_query(filtered_params) 34 | headers = filter_ignored_params(headers, ignored_params) 35 | data = filter_ignored_params(data, ignored_params) 36 | json = filter_ignored_params(json, ignored_params) 37 | 38 | # Create a hash based on the normalized and filtered request 39 | key = hashlib.sha256() 40 | key.update(method.upper().encode()) 41 | key.update(str(norm_url).encode()) 42 | key.update(encode_dict(data)) 43 | key.update(encode_dict(json)) 44 | if include_headers: 45 | key.update(encode_dict(headers)) 46 | return key.hexdigest() 47 | 48 | 49 | def filter_ignored_params(data, ignored_params: Iterable[str]): 50 | """Remove any ignored params from an object, if it's dict-like""" 51 | if not isinstance(data, Mapping) or not ignored_params: 52 | return data 53 | return MultiDict(((k, v) for k, v in data.items() if k not in ignored_params)) 54 | 55 | 56 | def normalize_url_params(url: StrOrURL, params: RequestParams | None = None) -> URL: 57 | """Normalize any combination of request parameter formats that aiohttp accepts""" 58 | if isinstance(url, str): 59 | url = URL(url) 60 | 61 | # Handle trailing empty param 62 | norm_params = MultiDict([i for i in url.query.items() if i != ('', '')]) 63 | 64 | # Combine `params` argument with URL query string if needed 65 | if params: 66 | norm_params.extend(url.with_query(params).query) 67 | 68 | # Sort params, apply additional normalization, and convert back to URL object 69 | url = url.with_query(sorted(norm_params.items())) 70 | return URL(url_normalize(str(url))) 71 | 72 | 73 | def encode_dict(data: Any) -> bytes: 74 | if not data: 75 | return b'' 76 | if isinstance(data, bytes): 77 | return data 78 | elif not isinstance(data, Mapping): 79 | return str(data).encode() 80 | item_pairs = [f'{k}={v}' for k, v in sorted((data or {}).items())] 81 | return '&'.join(item_pairs).encode() 82 | -------------------------------------------------------------------------------- /noxfile.py: -------------------------------------------------------------------------------- 1 | """Notes: 2 | * 'test' command: nox will use uv.lock to determine dependency versions 3 | * 'lint' command: tools and environments are managed by pre-commit 4 | * All other commands: the current environment will be used instead of creating new ones 5 | """ 6 | 7 | from os.path import join 8 | from pathlib import Path 9 | from shutil import rmtree 10 | 11 | import nox 12 | 13 | nox.options.reuse_existing_virtualenvs = True 14 | nox.options.sessions = ['lint', 'cov'] 15 | 16 | DOCS_DIR = Path('docs') 17 | LIVE_DOCS_PORT = 8181 18 | LIVE_DOCS_IGNORE = ['*.pyc', '*.tmp', join('**', 'modules', '*')] 19 | LIVE_DOCS_WATCH = ['aiohttp_client_cache', 'examples'] 20 | CLEAN_DIRS = ['dist', 'build', DOCS_DIR / '_build', DOCS_DIR / 'modules'] 21 | 22 | TEST_DIR = Path('test') 23 | UNIT_TESTS = TEST_DIR / 'unit' 24 | INTEGRATION_TESTS = TEST_DIR / 'integration' 25 | COVERAGE_ARGS = ( 26 | '--cov --cov-report=term --cov-report=html' # Generate HTML + stdout coverage report 27 | ) 28 | XDIST_ARGS = '--numprocesses=auto --dist=loadfile' # Run tests in parallel, grouped by test module 29 | 30 | 31 | def install_deps(session): 32 | """Install project and test dependencies into a test-specific virtualenv using uv""" 33 | session.env['UV_PROJECT_ENVIRONMENT'] = session.virtualenv.location 34 | session.run_install( 35 | 'uv', 36 | 'sync', 37 | '--frozen', 38 | '--all-extras', 39 | ) 40 | 41 | 42 | @nox.session(python=['3.9', '3.10', '3.11', '3.12', '3.13', '3.14']) 43 | def test(session): 44 | """Run tests for a specific python version""" 45 | test_paths = session.posargs or [UNIT_TESTS] 46 | install_deps(session) 47 | cmd = f'pytest -rs {XDIST_ARGS}' 48 | session.run(*cmd.split(' '), *test_paths) 49 | 50 | 51 | @nox.session(python=False) 52 | def clean(session): 53 | """Clean up temporary build + documentation files""" 54 | for dir in CLEAN_DIRS: 55 | print(f'Removing {dir}') 56 | rmtree(dir, ignore_errors=True) # type: ignore[arg-type] 57 | 58 | 59 | @nox.session(python=False, name='cov') 60 | def coverage(session): 61 | """Run tests and generate coverage report""" 62 | cmd_1 = f'pytest {UNIT_TESTS} -rs {XDIST_ARGS} {COVERAGE_ARGS}' 63 | cmd_2 = f'pytest {INTEGRATION_TESTS} -rs {XDIST_ARGS} {COVERAGE_ARGS} --cov-append' 64 | session.run(*cmd_1.split(' ')) 65 | session.run(*cmd_2.split(' ')) 66 | 67 | 68 | @nox.session(python=False) 69 | def docs(session): 70 | """Build Sphinx documentation""" 71 | cmd = 'sphinx-build docs docs/_build/html -j auto' 72 | session.run(*cmd.split(' ')) 73 | 74 | 75 | @nox.session(python=False) 76 | def livedocs(session): 77 | """Auto-build docs with live reload in browser. 78 | Add `--open` to also open the browser after starting. 79 | """ 80 | args = ['-a'] 81 | args += [f'--watch {pattern}' for pattern in LIVE_DOCS_WATCH] 82 | args += [f'--ignore {pattern}' for pattern in LIVE_DOCS_IGNORE] 83 | args += [f'--port {LIVE_DOCS_PORT}', '--host 0.0.0.0', '-j auto'] 84 | if session.posargs == ['open']: 85 | args.append('--open-browser') 86 | 87 | clean(session) 88 | cmd = 'sphinx-autobuild docs docs/_build/html ' + ' '.join(args) 89 | session.run(*cmd.split(' ')) 90 | 91 | 92 | @nox.session(python=False) 93 | def lint(session): 94 | """Run linters and code formatters via pre-commit""" 95 | session.run('pre-commit', 'run', '--all-files') 96 | -------------------------------------------------------------------------------- /aiohttp_client_cache/backends/mongodb.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import AsyncIterable 4 | from typing import Any 5 | 6 | from motor.motor_asyncio import AsyncIOMotorClient 7 | from pymongo import MongoClient 8 | 9 | from aiohttp_client_cache.backends import BaseCache, CacheBackend, ResponseOrKey, get_valid_kwargs 10 | 11 | 12 | class MongoDBBackend(CacheBackend): 13 | """Async cache backend for `MongoDB `_ 14 | 15 | Notes: 16 | * Requires `motor `_ 17 | * Accepts keyword arguments for :py:class:`pymongo.MongoClient` 18 | 19 | Args: 20 | cache_name: Database name 21 | connection: Optional client object to use instead of creating a new one 22 | kwargs: Additional keyword arguments for :py:class:`.CacheBackend` or backend connection 23 | """ 24 | 25 | def __init__( 26 | self, 27 | cache_name: str = 'aiohttp-cache', 28 | connection: AsyncIOMotorClient = None, 29 | **kwargs: Any, 30 | ): 31 | super().__init__(cache_name=cache_name, **kwargs) 32 | self.responses = MongoDBPickleCache(cache_name, 'responses', connection, **kwargs) 33 | self.redirects = MongoDBCache(cache_name, 'redirects', self.responses.connection, **kwargs) 34 | 35 | 36 | class MongoDBCache(BaseCache): 37 | """An async interface for caching objects in MongoDB 38 | 39 | Args: 40 | db_name: database name (be careful with production databases) 41 | collection_name: collection name 42 | connection: MongoDB connection instance to use instead of creating a new one 43 | kwargs: Additional keyword args for :py:class:`~motor.motor_asyncio.AsyncIOMotorClient` 44 | """ 45 | 46 | def __init__( 47 | self, 48 | db_name: str, 49 | collection_name: str, 50 | connection: AsyncIOMotorClient = None, 51 | **kwargs: Any, 52 | ): 53 | super().__init__(**kwargs) 54 | 55 | # Motor accepts the same arguments as pymongo, plus one additional argument 56 | connection_kwargs = get_valid_kwargs(MongoClient.__init__, kwargs, accept_varkwargs=False) 57 | if kwargs.get('io_loop'): 58 | connection_kwargs['io_loop'] = kwargs.pop('io_loop') 59 | 60 | self.connection = connection or AsyncIOMotorClient(**connection_kwargs) 61 | self.db = self.connection[db_name] 62 | self.collection = self.db[collection_name] 63 | 64 | async def clear(self): 65 | await self.collection.drop() 66 | 67 | async def contains(self, key: str) -> bool: 68 | return bool(await self.collection.find_one({'_id': key}, projection={'_id': True})) 69 | 70 | async def bulk_delete(self, keys: set): 71 | spec = {'_id': {'$in': list(keys)}} 72 | await self.collection.delete_many(spec) 73 | 74 | async def delete(self, key: str): 75 | spec = {'_id': key} 76 | await self.collection.delete_one(spec) 77 | 78 | async def keys(self) -> AsyncIterable[str]: 79 | async for doc in self.collection.find({}, {'_id': True}): 80 | yield doc['_id'] 81 | 82 | async def read(self, key: str) -> ResponseOrKey: 83 | doc = await self.collection.find_one({'_id': key}, projection={'_id': False, 'data': True}) 84 | try: 85 | return doc['data'] 86 | except TypeError: 87 | return None 88 | 89 | async def size(self) -> int: 90 | return await self.collection.count_documents({}) 91 | 92 | async def values(self) -> AsyncIterable[ResponseOrKey]: 93 | async for doc in self.collection.find( 94 | {'data': {'$exists': True}}, projection={'_id': False, 'data': True} 95 | ): 96 | yield doc['data'] 97 | 98 | async def write(self, key: str, item: ResponseOrKey): 99 | update = {'$set': {'data': item}} 100 | await self.collection.update_one({'_id': key}, update, upsert=True) 101 | 102 | 103 | class MongoDBPickleCache(MongoDBCache): 104 | """Same as :py:class:`MongoDBCache`, but pickles values before saving""" 105 | 106 | async def read(self, key): 107 | return self.deserialize(await super().read(key)) 108 | 109 | async def write(self, key, item): 110 | await super().write(key, self.serialize(item)) 111 | 112 | async def values(self) -> AsyncIterable[ResponseOrKey]: 113 | async for doc in self.collection.find({'data': {'$exists': True}}): 114 | yield self.deserialize(doc['data']) 115 | -------------------------------------------------------------------------------- /aiohttp_client_cache/backends/redis.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import AsyncIterable 4 | from typing import Any 5 | 6 | from redis.asyncio import Redis, from_url 7 | 8 | from aiohttp_client_cache.backends import BaseCache, CacheBackend, ResponseOrKey, get_valid_kwargs 9 | 10 | DEFAULT_ADDRESS = 'redis://localhost' 11 | 12 | 13 | class RedisBackend(CacheBackend): 14 | """Async cache backend for `Redis `_ 15 | 16 | Notes: 17 | * Requires `redis-py `_ 18 | * Accepts keyword arguments for :py:class:`redis.asyncio.client.Redis` 19 | 20 | Args: 21 | cache_name: Used as a namespace (prefix for hash key) 22 | address: Redis server URI 23 | kwargs: Additional keyword arguments for :py:class:`.CacheBackend` or backend connection 24 | """ 25 | 26 | def __init__( 27 | self, cache_name: str = 'aiohttp-cache', address: str = DEFAULT_ADDRESS, **kwargs: Any 28 | ): 29 | super().__init__(cache_name=cache_name, **kwargs) 30 | self.responses = RedisCache(cache_name, 'responses', address=address, **kwargs) 31 | self.redirects = RedisCache(cache_name, 'redirects', address=address, **kwargs) 32 | 33 | 34 | class RedisCache(BaseCache): 35 | """An async interface for caching objects in Redis. 36 | 37 | Args: 38 | namespace: namespace to use 39 | collection_name: name of the hash map stored in redis 40 | connection: An existing connection object to reuse instead of creating a new one 41 | address: Address of Redis server 42 | kwargs: Additional keyword arguments for :py:class:`aioredis.Redis` 43 | 44 | Note: The hash key name on the redis server will be ``namespace:collection_name``. 45 | """ 46 | 47 | def __init__( 48 | self, 49 | namespace: str, 50 | collection_name: str, 51 | address: str = DEFAULT_ADDRESS, 52 | connection: Redis | None = None, 53 | **kwargs: Any, 54 | ): 55 | # Pop off BaseCache kwargs and use the rest as Redis connection kwargs 56 | super().__init__(**kwargs) 57 | self.address = address 58 | self._connection = connection 59 | self.connection_kwargs = get_valid_kwargs(Redis.__init__, kwargs) 60 | self.hash_key = f'{namespace}:{collection_name}' 61 | 62 | async def get_connection(self): 63 | """Lazy-initialize redis connection""" 64 | if not self._connection: 65 | self._connection = await from_url(self.address, **self.connection_kwargs) 66 | return self._connection 67 | 68 | async def close(self): 69 | if self._connection: 70 | await self._connection.aclose() # type: ignore[attr-defined] 71 | self._connection = None 72 | 73 | async def clear(self): 74 | connection = await self.get_connection() 75 | async for key in self.keys(): 76 | await connection.hdel(self.hash_key, key) 77 | 78 | async def contains(self, key: str) -> bool: 79 | connection = await self.get_connection() 80 | return await connection.hexists(self.hash_key, key) 81 | 82 | async def bulk_delete(self, keys: set): 83 | """Requires redis version >=2.4""" 84 | connection = await self.get_connection() 85 | await connection.hdel(self.hash_key, *keys) 86 | 87 | async def delete(self, key: str): 88 | connection = await self.get_connection() 89 | await connection.hdel(self.hash_key, key) 90 | 91 | async def keys(self) -> AsyncIterable[str]: 92 | connection = await self.get_connection() 93 | for k in await connection.hkeys(self.hash_key): 94 | yield k.decode() 95 | 96 | async def read(self, key: str) -> ResponseOrKey: 97 | connection = await self.get_connection() 98 | result = await connection.hget(self.hash_key, key) 99 | return self.deserialize(result) 100 | 101 | async def size(self) -> int: 102 | connection = await self.get_connection() 103 | return await connection.hlen(self.hash_key) 104 | 105 | async def values(self) -> AsyncIterable[ResponseOrKey]: 106 | connection = await self.get_connection() 107 | for v in await connection.hvals(self.hash_key): 108 | yield self.deserialize(v) 109 | 110 | async def write(self, key: str, item: ResponseOrKey): 111 | connection = await self.get_connection() 112 | await connection.hset( 113 | self.hash_key, 114 | key, 115 | self.serialize(item), 116 | ) 117 | -------------------------------------------------------------------------------- /test/integration/base_storage_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import AsyncIterator 4 | from contextlib import asynccontextmanager 5 | from datetime import datetime 6 | from typing import Any, TypeVar 7 | 8 | import pytest 9 | 10 | from aiohttp_client_cache.backends.sqlite import BaseCache 11 | from test.conftest import CACHE_NAME 12 | 13 | pytestmark = pytest.mark.asyncio 14 | picklable_test_data = {'key_1': 'item_1', 'key_2': datetime(2021, 8, 14), 'key_3': 3.141592654} 15 | str_test_data = {f'key_{i}': f'item_{i}' for i in range(10)} 16 | 17 | BaseCacheT = TypeVar('BaseCacheT', bound=BaseCache) 18 | 19 | 20 | class BaseStorageTest: 21 | """Base class for testing cache storage dict-like interfaces""" 22 | 23 | init_kwargs: dict = {} 24 | picklable: bool = False 25 | test_data: dict[str, Any] = picklable_test_data 26 | storage_class: type[BaseCache] 27 | 28 | @asynccontextmanager 29 | async def init_cache( 30 | self, storage_class: type[BaseCacheT] | None = None, index=0, **kwargs 31 | ) -> AsyncIterator[BaseCacheT]: 32 | self.test_data = picklable_test_data if self.picklable else str_test_data 33 | cache_class = storage_class or self.storage_class 34 | assert cache_class 35 | cache = cache_class(CACHE_NAME, f'table_{index}', **self.init_kwargs, **kwargs) 36 | await cache.clear() 37 | yield cache # type: ignore[misc] 38 | await cache.close() 39 | 40 | async def test_write_read(self): 41 | async with self.init_cache() as cache: # type: ignore[var-annotated] 42 | # Test write(), contains(), and size() 43 | for k, v in self.test_data.items(): 44 | await cache.write(k, v) 45 | assert await cache.contains(k) is True 46 | assert await cache.size() == len(self.test_data) 47 | 48 | # Test read() 49 | for k, v in self.test_data.items(): 50 | assert await cache.read(k) == v 51 | 52 | async def test_missing_key(self): 53 | async with self.init_cache() as cache: # type: ignore[var-annotated] 54 | assert await cache.contains('nonexistent_key') is False 55 | assert await cache.read('nonexistent_key') is None 56 | 57 | async def test_delete(self): 58 | async with self.init_cache() as cache: # type: ignore[var-annotated] 59 | await cache.write('do_not_delete', 'value') 60 | for k, v in self.test_data.items(): 61 | await cache.write(k, v) 62 | 63 | for k in self.test_data.keys(): 64 | await cache.delete(k) 65 | assert await cache.contains(k) is False 66 | 67 | assert await cache.read('do_not_delete') == 'value' 68 | 69 | async def test_bulk_delete(self): 70 | async with self.init_cache() as cache: # type: ignore[var-annotated] 71 | await cache.write('do_not_delete', 'value') 72 | for k, v in self.test_data.items(): 73 | await cache.write(k, v) 74 | 75 | await cache.bulk_delete(self.test_data.keys()) 76 | 77 | for k in self.test_data.keys(): 78 | assert await cache.contains(k) is False 79 | 80 | async def test_bulk_delete_ignores_nonexistent_keys(self): 81 | async with self.init_cache() as cache: # type: ignore[var-annotated] 82 | await cache.bulk_delete(self.test_data.keys()) 83 | 84 | async def test_keys_values(self): 85 | test_data = {f'key_{i}': f'value_{i}' for i in range(20)} 86 | 87 | # test keys() and values() 88 | async with self.init_cache() as cache: # type: ignore[var-annotated] 89 | assert [k async for k in cache.keys()] == [] 90 | assert [v async for v in cache.values()] == [] 91 | 92 | for k, v in test_data.items(): 93 | await cache.write(k, v) 94 | 95 | assert sorted([k async for k in cache.keys()]) == sorted(test_data.keys()) 96 | assert sorted([v async for v in cache.values()]) == sorted(test_data.values()) 97 | 98 | async def test_size(self): 99 | async with self.init_cache() as cache: # type: ignore[var-annotated] 100 | assert await cache.size() == 0 101 | for k, v in self.test_data.items(): 102 | await cache.write(k, v) 103 | 104 | assert await cache.size() == len(self.test_data) 105 | 106 | async def test_clear(self): 107 | async with self.init_cache() as cache: # type: ignore[var-annotated] 108 | for k, v in self.test_data.items(): 109 | await cache.write(k, v) 110 | 111 | await cache.clear() 112 | assert await cache.size() == 0 113 | assert {k async for k in cache.keys()} == set() 114 | assert {v async for v in cache.values()} == set() 115 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "aiohttp-client-cache" 3 | version = "0.14.2" 4 | description = "Persistent cache for aiohttp requests" 5 | authors = [{name = "Jordan Cook"}, {name = "Alessio Locatelli"}] 6 | license = {text = "MIT License"} 7 | readme = "README.md" 8 | keywords = ["aiohttp", "async", "asyncio", "cache", "cache-backends", "client", "http", 9 | "persistence", "requests", "sqlite", "redis", "mongodb", "dynamodb", "dragonflydb"] 10 | classifiers = [ 11 | "Development Status :: 4 - Beta", 12 | "Framework :: AsyncIO", 13 | "Framework :: aiohttp", 14 | "Intended Audience :: Developers", 15 | "Topic :: Software Development :: Libraries :: Python Modules", 16 | "Typing :: Typed", 17 | ] 18 | requires-python = ">=3.9" 19 | dependencies = [ 20 | "aiohttp >=3.8", 21 | "attrs >=21.2", 22 | "itsdangerous >=2.0", 23 | "url-normalize >=2.2", 24 | "typing-extensions >=4 ; python_version<='3.10'", # For Self type addition 25 | ] 26 | 27 | [project.optional-dependencies] 28 | # Optional backend dependencies 29 | all = ["aioboto3>=9.0", "aiobotocore>=2.0", "aiofiles>=0.6.0", "aiosqlite>=0.20", "motor>=3.1", "redis>=4.2"] 30 | dynamodb = ["aioboto3>=9.0", "aiobotocore>=2.0"] 31 | filesystem = ["aiofiles>=0.6.0", "aiosqlite>=0.20"] 32 | mongodb = ["motor>=3.1"] 33 | redis = ["redis>=4.2"] 34 | sqlite = ["aiosqlite>=0.20"] 35 | 36 | [dependency-groups] 37 | # Development dependencies 38 | dev = [ 39 | # For unit + integration tests 40 | "async-timeout >=4.0", 41 | "brotli >=1.0", 42 | "faker >=30.0", 43 | "pytest >=8.4", 44 | "pytest-aiohttp >=1.1", 45 | "pytest-asyncio >=1.2", 46 | "pytest-clarity >=1.0", 47 | "pytest-cov >=6.2", 48 | "pytest-xdist >=3.6", 49 | # For convenience in local development; additional tools are managed by pre-commit 50 | "nox >=2022.11", 51 | "pre-commit >=4.2", 52 | "types-aiofiles >=0.1.7", 53 | ] 54 | 55 | # Documentation dependencies needed for Readthedocs builds 56 | docs = [ 57 | "furo >=2025.9; python_version>='3.11'", 58 | "linkify-it-py >=2.0; python_version>='3.11'", 59 | "markdown-it-py >=2.2; python_version>='3.11'", 60 | "myst-parser >=3.0; python_version>='3.11'", 61 | "python-forge >=18.6; python_version>='3.11'", 62 | "sphinx ~=8.2; python_version>='3.11'", 63 | "sphinx-automodapi >=0.18,<0.21; python_version>='3.11'", 64 | "sphinx-autodoc-typehints >=2.4; python_version>='3.11'", 65 | "sphinx-copybutton >=0.5; python_version>='3.11'", 66 | "sphinx-inline-tabs >=2023.4; python_version>='3.11'", 67 | "sphinxcontrib-apidoc >=0.3; python_version>='3.11'", 68 | ] 69 | 70 | # Test server dependencies 71 | test-server = [ 72 | "flask>=2.0", 73 | "gunicorn>=21.2", 74 | ] 75 | 76 | [project.urls] 77 | Homepage = "https://github.com/requests-cache/aiohttp-client-cache" 78 | Repository = "https://github.com/requests-cache/aiohttp-client-cache" 79 | Documentation = "https://aiohttp-client-cache.readthedocs.io" 80 | 81 | [build-system] 82 | requires = ["hatchling"] 83 | build-backend = "hatchling.build" 84 | 85 | [tool.hatch.build.targets.wheel] 86 | packages = ["aiohttp_client_cache"] 87 | 88 | [tool.hatch.build.targets.sdist] 89 | include = [ 90 | "*.md", 91 | "*.yml", 92 | "aiohttp_client_cache/", 93 | "docs/", 94 | "examples/", 95 | "test/", 96 | ] 97 | 98 | [tool.coverage.html] 99 | directory = 'test-reports' 100 | 101 | [tool.coverage.run] 102 | branch = true 103 | source = ['aiohttp_client_cache'] 104 | omit = [ 105 | 'aiohttp_client_cache/__init__.py', 106 | 'aiohttp_client_cache/backends/__init__.py', 107 | 'aiohttp_client_cache/signatures.py', 108 | ] 109 | 110 | [tool.coverage.report] 111 | exclude_lines = [ 112 | 'pragma: no cover', 113 | 'if TYPE_CHECKING:', 114 | ] 115 | 116 | [tool.mypy] 117 | python_version = "3.9" 118 | ignore_missing_imports = true 119 | warn_redundant_casts = true 120 | warn_unused_ignores = true 121 | warn_unreachable = true 122 | show_error_codes = true 123 | show_column_numbers = true 124 | check_untyped_defs=true 125 | pretty = true 126 | 127 | [tool.pytest.ini_options] 128 | asyncio_mode = "auto" 129 | asyncio_default_fixture_loop_scope = "function" 130 | 131 | [tool.ruff] 132 | line-length = 100 133 | output-format = 'grouped' 134 | target-version = 'py39' 135 | 136 | [tool.ruff.format] 137 | quote-style = 'single' 138 | 139 | [tool.ruff.lint] 140 | select = ['B', 'C4', 'C90', 'E', 'F', 'I', 'UP'] 141 | ignore = ['B023', 'B027'] 142 | 143 | [tool.ruff.lint.isort] 144 | known-first-party = ['test'] 145 | 146 | # Wrap lines to 100 chars, but don't error on unwrappable lines until 120 chars 147 | [tool.ruff.lint.pycodestyle] 148 | max-line-length = 120 149 | 150 | [tool.typos] 151 | files.extend-exclude = ["CONTRIBUTORS.md"] 152 | -------------------------------------------------------------------------------- /aiohttp_client_cache/backends/filesystem.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import AsyncIterable 4 | from contextlib import contextmanager 5 | from os import listdir, makedirs 6 | from os.path import abspath, expanduser, isabs, isfile, join 7 | from pathlib import Path 8 | from pickle import PickleError 9 | from shutil import rmtree 10 | from tempfile import gettempdir 11 | from typing import Any 12 | 13 | import aiofiles 14 | import aiofiles.os 15 | 16 | from aiohttp_client_cache.backends import BaseCache, CacheBackend, ResponseOrKey 17 | from aiohttp_client_cache.backends.sqlite import SQLiteCache 18 | 19 | 20 | class FileBackend(CacheBackend): 21 | """Backend that stores cached responses as files on the local filesystem. 22 | 23 | Notes: 24 | * Requires `aiofiles `_ and `aiosqlite `_. 25 | * Response paths will be in the format ``/responses/``. 26 | * Redirects are stored in a SQLite database, located at ``/redirects.sqlite``. 27 | 28 | Args: 29 | cache_name: Base directory for cache files 30 | use_temp: Store cache files in a temp directory (e.g., ``/tmp/http_cache/``). 31 | Note: if ``cache_name`` is an absolute path, this option will be ignored. 32 | autoclose: Close any active backend connections when the session is closed 33 | kwargs: Additional keyword arguments for :py:class:`.CacheBackend` 34 | """ 35 | 36 | def __init__( 37 | self, 38 | cache_name: Path | str = 'http_cache', 39 | use_temp: bool = False, 40 | autoclose: bool = True, 41 | **kwargs: Any, 42 | ): 43 | super().__init__(autoclose=autoclose, **kwargs) 44 | self.responses = FileCache(cache_name, use_temp=use_temp, **kwargs) 45 | db_path = join(self.responses.cache_dir, 'redirects.sqlite') 46 | self.redirects = SQLiteCache(db_path, 'redirects', **kwargs) 47 | 48 | 49 | class FileCache(BaseCache): 50 | """A dictionary-like interface to files on the local filesystem""" 51 | 52 | def __init__(self, cache_name, use_temp: bool = False, **kwargs: Any): 53 | super().__init__(**kwargs) 54 | self.cache_dir = _get_cache_dir(cache_name, use_temp) 55 | 56 | @contextmanager 57 | def _try_io(self, ignore_errors: bool = True): 58 | """Attempt an I/O operation, and either ignore errors or re-raise them as KeyErrors""" 59 | try: 60 | yield 61 | except (OSError, PickleError): 62 | if not ignore_errors: 63 | raise 64 | 65 | def _join(self, key): 66 | return join(self.cache_dir, str(key)) 67 | 68 | async def clear(self): 69 | """Note: Currently this is a blocking operation""" 70 | with self._try_io(): 71 | rmtree(self.cache_dir) 72 | makedirs(self.cache_dir) 73 | 74 | async def contains(self, key: str) -> bool: 75 | return isfile(self._join(key)) 76 | 77 | async def read(self, key: str) -> ResponseOrKey: 78 | with self._try_io(False): 79 | path = self._join(key) 80 | if await aiofiles.os.path.exists(path): 81 | async with aiofiles.open(self._join(key), 'rb') as f: 82 | return self.deserialize(await f.read()) 83 | else: 84 | return None 85 | 86 | async def bulk_delete(self, keys: set): 87 | for key in keys: 88 | await self.delete(key) 89 | 90 | async def delete(self, key: str): 91 | with self._try_io(): 92 | await aiofiles.os.remove(self._join(key)) 93 | 94 | async def write(self, key: str, value: ResponseOrKey): 95 | with self._try_io(ignore_errors=False): 96 | async with aiofiles.open(self._join(key), 'wb') as f: 97 | await f.write(self.serialize(value) or b'') 98 | 99 | async def keys(self) -> AsyncIterable[str]: 100 | for filename in filter(lambda fn: not fn.endswith('.sqlite'), listdir(self.cache_dir)): 101 | yield filename 102 | 103 | async def size(self) -> int: 104 | return len([k async for k in self.keys()]) 105 | 106 | async def values(self) -> AsyncIterable[ResponseOrKey]: 107 | async for key in self.keys(): 108 | yield await self.read(key) 109 | 110 | async def paths(self): 111 | """Get file paths to all cached responses""" 112 | async for key in self.keys(): 113 | yield self._join(key) 114 | 115 | 116 | def _get_cache_dir(cache_dir: Path | str, use_temp: bool) -> str: 117 | # Save to a temp directory, if specified 118 | if use_temp and not isabs(cache_dir): 119 | cache_dir = join(gettempdir(), cache_dir, 'responses') 120 | 121 | # Expand relative and user paths (~/*), and make sure parent dirs exist 122 | cache_dir = abspath(expanduser(str(cache_dir))) 123 | makedirs(cache_dir, exist_ok=True) 124 | return cache_dir 125 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Sphinx documentation build configuration file 2 | import sys 3 | from os.path import abspath, dirname, join 4 | 5 | DOCS_DIR = abspath(dirname(__file__)) 6 | PROJECT_DIR = dirname(DOCS_DIR) 7 | PACKAGE_DIR = join(PROJECT_DIR, 'aiohttp_client_cache') 8 | 9 | # Add project path so we can import our package 10 | sys.path.insert(0, PROJECT_DIR) 11 | from aiohttp_client_cache import __version__ # noqa 12 | 13 | # General project info 14 | project = 'aiohttp-client-cache' 15 | needs_sphinx = '5.0' 16 | version = release = __version__ 17 | 18 | # General source info 19 | master_doc = 'index' 20 | source_suffix = ['.rst', '.md'] 21 | html_static_path = ['_static'] 22 | templates_path = ['_templates'] 23 | 24 | # Sphinx extension modules 25 | extensions = [ 26 | 'sphinx.ext.autodoc', 27 | 'sphinx.ext.autosectionlabel', 28 | 'sphinx.ext.autosummary', 29 | 'sphinx.ext.intersphinx', 30 | 'sphinx.ext.napoleon', 31 | 'sphinx_autodoc_typehints', 32 | 'sphinx_automodapi.automodapi', 33 | 'sphinx_automodapi.smart_resolver', 34 | 'sphinx_copybutton', 35 | 'sphinx_inline_tabs', 36 | 'sphinxcontrib.apidoc', 37 | 'myst_parser', 38 | ] 39 | 40 | # Enable automatic links to other projects' Sphinx docs 41 | intersphinx_mapping = { 42 | 'aioboto3': ('https://aioboto3.readthedocs.io/en/latest/', None), 43 | 'aiohttp': ('https://docs.aiohttp.org/en/stable/', None), 44 | 'aiosqlite': ('https://aiosqlite.omnilib.dev/en/latest/', None), 45 | 'botocore': ('http://botocore.readthedocs.io/en/latest/', None), 46 | 'boto3': ('https://boto3.amazonaws.com/v1/documentation/api/latest', None), 47 | 'motor': ('https://motor.readthedocs.io/en/stable/', None), 48 | 'pymongo': ('https://pymongo.readthedocs.io/en/stable/', None), 49 | 'python': ('https://docs.python.org/3', None), 50 | 'redis': ('https://redis-py.readthedocs.io/en/stable/', None), 51 | 'yarl': ('https://yarl.aio-libs.org/en/latest/', None), 52 | } 53 | 54 | # MyST extensions 55 | myst_enable_extensions = [ 56 | 'colon_fence', 57 | 'html_image', 58 | 'linkify', 59 | 'replacements', 60 | 'smartquotes', 61 | ] 62 | 63 | # Exclude modules with manually formatted docs and documentation utility modules 64 | exclude_patterns = [ 65 | '_build', 66 | 'modules/aiohttp_client_cache.rst', 67 | 'modules/aiohttp_client_cache.backends.rst', 68 | 'modules/aiohttp_client_cache.session.rst', 69 | 'modules/aiohttp_client_cache.signatures.rst', 70 | ] 71 | 72 | # napoleon settings 73 | napoleon_google_docstring = True 74 | napoleon_include_private_with_doc = False 75 | napoleon_include_special_with_doc = False 76 | napoleon_use_param = True 77 | 78 | # Strip prompt text when copying code blocks with copy button 79 | copybutton_prompt_text = r'>>> |\.\.\. |\$ ' 80 | copybutton_prompt_is_regexp = True 81 | 82 | # Move type hint info to function description instead of signature 83 | autodoc_typehints = 'description' 84 | always_document_param_types = True 85 | set_type_checking_flag = False 86 | 87 | # Use apidoc to auto-generate rst sources 88 | apidoc_excluded_paths = ['forge_utils.py'] 89 | apidoc_module_dir = PACKAGE_DIR 90 | apidoc_module_first = True 91 | apidoc_output_dir = 'modules' 92 | apidoc_separate_modules = True 93 | apidoc_template_dir = '_templates/apidoc' 94 | apidoc_toc_file = False 95 | add_module_names = False 96 | 97 | # Options for automodapi and autosectionlabel 98 | automodsumm_inherited_members = False 99 | autosectionlabel_prefix_document = True 100 | numpydoc_show_class_members = False 101 | 102 | # HTML general settings 103 | # html_favicon = join('images', 'favicon.ico') 104 | html_js_files = ['collapsible_container.js'] 105 | html_css_files = ['collapsible_container.css', 'table.css'] 106 | html_show_sphinx = False 107 | pygments_style = 'friendly' 108 | pygments_dark_style = 'material' 109 | 110 | # HTML theme settings 111 | html_theme = 'furo' 112 | html_theme_options = { 113 | # 'light_css_variables': { 114 | # 'color-brand-primary': '#00766c', # MD light-blue-600; light #64d8cb | med #26a69a 115 | # 'color-brand-content': '#006db3', # MD teal-400; light #63ccff | med #039be5 116 | # }, 117 | # 'dark_css_variables': { 118 | # 'color-brand-primary': '#64d8cb', 119 | # 'color-brand-content': '#63ccff', 120 | # }, 121 | 'sidebar_hide_name': False, 122 | } 123 | 124 | 125 | def setup(app): 126 | """Run some additional steps after the Sphinx builder is initialized""" 127 | app.connect('builder-inited', patch_automodapi) 128 | 129 | 130 | def patch_automodapi(app): 131 | """Monkey-patch the automodapi extension to exclude imported members 132 | 133 | See: https://github.com/astropy/sphinx-automodapi/issues/119 134 | """ 135 | from sphinx_automodapi import automodsumm 136 | from sphinx_automodapi.utils import find_mod_objs 137 | 138 | def find_local_mod_objs(*args, **kwargs): 139 | kwargs['onlylocals'] = True 140 | return find_mod_objs(*args, **kwargs) 141 | 142 | automodsumm.find_mod_objs = find_local_mod_objs 143 | -------------------------------------------------------------------------------- /docs/backends.md: -------------------------------------------------------------------------------- 1 | (backends)= 2 | 3 | # Cache Backends 4 | 5 | ## Backend Classes 6 | 7 | Several cache backends are included, which can be selected using the `cache` parameter for 8 | {py:class}`.CachedSession`: 9 | 10 | ```{eval-rst} 11 | .. autosummary:: 12 | :nosignatures: 13 | 14 | aiohttp_client_cache.backends.base.CacheBackend 15 | aiohttp_client_cache.backends.dynamodb.DynamoDBBackend 16 | aiohttp_client_cache.backends.filesystem.FileBackend 17 | aiohttp_client_cache.backends.mongodb.MongoDBBackend 18 | aiohttp_client_cache.backends.redis.RedisBackend 19 | aiohttp_client_cache.backends.sqlite.SQLiteBackend 20 | ``` 21 | 22 | Usage example: 23 | 24 | ```python 25 | >>> from aiohttp_client_cache import CachedSession, RedisBackend 26 | >>> 27 | >>> async with CachedSession(cache=RedisBackend()) as session: 28 | ... await session.get('http://httpbin.org/get') 29 | ``` 30 | 31 | See {ref}`api-reference` for backend-specific usage details. 32 | 33 | ## Backend Cache Name 34 | 35 | The `cache_name` parameter will be used as follows depending on the backend: 36 | 37 | - DynamoDb: Table name 38 | - Filesystem: Cache directory 39 | - MongoDb: Database name 40 | - Redis: Namespace, meaning all keys will be prefixed with `':'` 41 | - SQLite: Database path; user paths are allowed, e.g `~/.cache/my_cache.sqlite` 42 | 43 | ## Backend-Specific Arguments 44 | 45 | When initializing a {py:class}`.CacheBackend`, you can provide any valid keyword arguments for the 46 | backend's internal connection class or function. 47 | 48 | For example, with {py:class}`.SQLiteBackend`, you can pass arguments accepted by 49 | {py:func}`sqlite3.connect`: 50 | 51 | ```python 52 | >>> cache = SQLiteBackend( 53 | ... timeout=2.5, 54 | ... uri='file://home/user/.cache/aiohttp-cache.db?mode=ro&cache=private', 55 | ... ) 56 | ``` 57 | 58 | ## Custom Backends 59 | 60 | If the built-in backends don't suit your needs, you can create your own by making subclasses of 61 | {py:class}`.CacheBackend` and {py:class}`.BaseCache`: 62 | 63 | ```python 64 | >>> from aiohttp_client_cache import CachedSession 65 | >>> from aiohttp_client_cache.backends import BaseCache, BaseStorage 66 | 67 | >>> class CustomCache(BaseCache): 68 | ... """Wrapper for higher-level cache operations. In most cases, the only thing you need 69 | ... to specify here is which storage class(es) to use. 70 | ... """ 71 | ... def __init__(self, **kwargs): 72 | ... super().__init__(**kwargs) 73 | ... self.redirects = CustomStorage(**kwargs) 74 | ... self.responses = CustomStorage(**kwargs) 75 | 76 | >>> class CustomStorage(BaseStorage): 77 | ... """interface for lower-level backend storage operations""" 78 | ... def __init__(self, **kwargs): 79 | ... super().__init__(**kwargs) 80 | ... 81 | ... async def contains(self, key: str) -> bool: 82 | ... """Check if a key is stored in the cache""" 83 | ... 84 | ... async def clear(self): 85 | ... """Delete all items from the cache""" 86 | ... 87 | ... async def delete(self, key: str): 88 | ... """Delete an item from the cache""" 89 | ... 90 | ... async def keys(self) -> AsyncIterable[str]: 91 | ... """Get all keys stored in the cache""" 92 | ... 93 | ... async def read(self, key: str) -> ResponseOrKey: 94 | ... """Read anitem from the cache""" 95 | ... 96 | ... async def size(self) -> int: 97 | ... """Get the number of items in the cache""" 98 | ... 99 | ... def values(self) -> AsyncIterable[ResponseOrKey]: 100 | ... """Get all values stored in the cache""" 101 | ... 102 | ... async def write(self, key: str, item: ResponseOrKey): 103 | ... """Write an item to the cache""" 104 | ``` 105 | 106 | You can then use your custom backend in a {py:class}`.CachedSession` with the `cache` parameter: 107 | 108 | ```python 109 | >>> session = CachedSession(cache=CustomCache()) 110 | ``` 111 | 112 | ## Can I reuse a cache backend instance across multiple `CachedSession` instances? 113 | 114 | First of all, read the following warning in the [`aiohttp` documentation](https://docs.aiohttp.org/en/stable/client_quickstart.html#make-a-request) to make sure you need multiple `CachedSession` or `Session`: 115 | 116 | > Don’t create a session per request. Most likely you need a session per application which performs all requests together. 117 | > 118 | > More complex cases may require a session per site, e.g. one for Github and other one for Facebook APIs. Anyway making a session for every request is a very bad idea. 119 | > 120 | > A session contains a connection pool inside. Connection reusage and keep-alive (both are on by default) may speed up total performance. 121 | 122 | It depends on your application design, but you have at least three options: 123 | 124 | - Create a cache instance per `CachedSession`: 125 | 126 | ```py 127 | github_api = CachedSession(SQLiteBackend()) 128 | gitlab_api = CachedSession(SQLiteBackend()) 129 | ``` 130 | 131 | - Create a single cache instance and use `autoclose=False`: 132 | 133 | ```py 134 | cache_backend = CacheBackend() 135 | 136 | sessions_pool = [...] # Manage multiple `Cachedsession` with a single cached backend. 137 | 138 | # Make requests... 139 | 140 | cache_backend.close() 141 | ``` 142 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # aiohttp-client-cache 2 | 3 | [![Build status](https://github.com/requests-cache/aiohttp-client-cache/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/requests-cache/aiohttp-client-cache/actions) 4 | [![Documentation Status](https://img.shields.io/readthedocs/aiohttp-client-cache/latest?label=docs)](https://aiohttp-client-cache.readthedocs.io/en/stable/) 5 | [![Codecov](https://codecov.io/gh/requests-cache/aiohttp-client-cache/branch/main/graph/badge.svg?token=I6PNLYTILM)](https://codecov.io/gh/requests-cache/aiohttp-client-cache) 6 | [![PyPI](https://img.shields.io/pypi/v/aiohttp-client-cache?color=blue)](https://pypi.org/project/aiohttp-client-cache) 7 | [![Conda](https://img.shields.io/conda/vn/conda-forge/aiohttp-client-cache?color=blue)](https://anaconda.org/conda-forge/aiohttp-client-cache) 8 | [![PyPI - Python Versions](https://img.shields.io/pypi/pyversions/aiohttp-client-cache)](https://pypi.org/project/aiohttp-client-cache) 9 | [![PyPI - Format](https://img.shields.io/pypi/format/aiohttp-client-cache?color=blue)](https://pypi.org/project/aiohttp-client-cache) 10 | 11 | **aiohttp-client-cache** is an async persistent cache for [aiohttp](https://docs.aiohttp.org) 12 | client requests, based on [requests-cache](https://github.com/reclosedev/requests-cache). 13 | 14 | # Features 15 | 16 | - **Ease of use:** Use as a [drop-in replacement](https://aiohttp-client-cache.readthedocs.io/en/stable/user_guide.html) 17 | for `aiohttp.ClientSession` 18 | - **Customization:** Works out of the box with little to no config, but with plenty of options 19 | available for customizing cache 20 | [expiration](https://aiohttp-client-cache.readthedocs.io/en/stable/user_guide.html#cache-expiration) 21 | and other [behavior](https://aiohttp-client-cache.readthedocs.io/en/stable/user_guide.html#cache-options) 22 | - **Persistence:** Includes several [storage backends](https://aiohttp-client-cache.readthedocs.io/en/stable/backends.html): 23 | SQLite, DynamoDB, MongoDB, DragonflyDB and Redis. 24 | 25 | # Quickstart 26 | 27 | First, install with pip (python 3.9+ required): 28 | 29 | ```bash 30 | pip install aiohttp-client-cache[all] 31 | ``` 32 | 33 | **Note:** 34 | Adding `[all]` will install optional dependencies for all supported backends. When adding this 35 | library to your application, you can include only the dependencies you actually need; see individual 36 | backend docs and [pyproject.toml](https://github.com/requests-cache/aiohttp-client-cache/blob/main/pyproject.toml) 37 | for details. 38 | 39 | ## Basic Usage 40 | 41 | Next, use [aiohttp_client_cache.CachedSession](https://aiohttp-client-cache.readthedocs.io/en/stable/modules/aiohttp_client_cache.session.html#aiohttp_client_cache.session.CachedSession) 42 | in place of [aiohttp.ClientSession](https://docs.aiohttp.org/en/stable/client_reference.html#aiohttp.ClientSession). 43 | To briefly demonstrate how to use it: 44 | 45 | **Replace this:** 46 | 47 | ```python 48 | from aiohttp import ClientSession 49 | 50 | async with ClientSession() as session: 51 | await session.get('http://httpbin.org/delay/1') 52 | ``` 53 | 54 | **With this:** 55 | 56 | ```python 57 | from aiohttp_client_cache import CachedSession, SQLiteBackend 58 | 59 | async with CachedSession(cache=SQLiteBackend('demo_cache')) as session: 60 | await session.get('http://httpbin.org/delay/1') 61 | ``` 62 | 63 | The URL in this example adds a delay of 1 second, simulating a slow or rate-limited website. 64 | With caching, the response will be fetched once, saved to `demo_cache.sqlite`, and subsequent 65 | requests will return the cached response near-instantly. 66 | 67 | ## Configuration 68 | 69 | Several options are available to customize caching behavior. This example demonstrates a few of them: 70 | 71 | ```python 72 | # fmt: off 73 | from aiohttp_client_cache import SQLiteBackend 74 | 75 | cache = SQLiteBackend( 76 | cache_name='~/.cache/aiohttp-requests.db', # For SQLite, this will be used as the filename 77 | expire_after=60*60, # By default, cached responses expire in an hour 78 | urls_expire_after={'*.fillmurray.com': -1}, # Requests for any subdomain on this site will never expire 79 | allowed_codes=(200, 418), # Cache responses with these status codes 80 | allowed_methods=['GET', 'POST'], # Cache requests with these HTTP methods 81 | include_headers=True, # Cache requests with different headers separately 82 | ignored_params=['auth_token'], # Keep using the cached response even if this param changes 83 | timeout=2.5, # Connection timeout for SQLite backend 84 | ) 85 | ``` 86 | 87 | # More Info 88 | 89 | To learn more, see: 90 | 91 | - [User Guide](https://aiohttp-client-cache.readthedocs.io/en/stable/user_guide.html) 92 | - [Cache Backends](https://aiohttp-client-cache.readthedocs.io/en/stable/backends.html) 93 | - [API Reference](https://aiohttp-client-cache.readthedocs.io/en/stable/reference.html) 94 | - [Examples](https://aiohttp-client-cache.readthedocs.io/en/stable/examples.html) 95 | 96 | # Feedback 97 | 98 | If there is a feature you want, if you've discovered a bug, or if you have other general feedback, please 99 | [create an issue](https://github.com/requests-cache/aiohttp-client-cache/issues/new/choose) for it! 100 | -------------------------------------------------------------------------------- /aiohttp_client_cache/signatures.py: -------------------------------------------------------------------------------- 1 | """Utilities for dynamically modifying function signatures. 2 | 3 | The purpose of this is to generate thorough documentation of arguments without a ton of 4 | copy-pasted text. This applies to both Sphinx docs hosted on readthedocs, as well as autocompletion, 5 | type checking, etc. within an IDE or other editor. 6 | 7 | Currently this is used to add the following backend-specific connection details: 8 | 9 | * Function signatures 10 | * Type annotations 11 | * Argument docs 12 | """ 13 | 14 | from __future__ import annotations 15 | 16 | import inspect 17 | import re 18 | from logging import getLogger 19 | from typing import Callable 20 | 21 | AUTOMETHOD_INIT = '.. automethod:: __init__' 22 | logger = getLogger(__name__) 23 | 24 | 25 | def extend_signature(super_function: Callable, *extra_functions: Callable) -> Callable: 26 | """A function decorator that modifies the target function's signature and docstring with: 27 | 28 | * Params from superclass 29 | * Params from target function 30 | * (optional) Params from additional functions called by the target function 31 | 32 | This also makes ``forge`` optional. If it's not installed, or if there's a problem with 33 | modifying a signature, this will just log the error and return the function with its original 34 | signature. 35 | """ 36 | 37 | def wrapper(target_function: Callable): 38 | try: 39 | target_function = copy_docstrings(target_function, super_function, *extra_functions) 40 | revision = get_combined_revision(target_function, super_function, *extra_functions) 41 | return revision(target_function) 42 | except Exception as e: 43 | logger.debug(e) 44 | return target_function 45 | 46 | return wrapper 47 | 48 | 49 | def get_combined_revision(*functions: Callable): 50 | """Combine the parameters of all revisions into a single revision""" 51 | import forge 52 | 53 | params = {} 54 | for func in functions: 55 | params.update(forge.copy(func).signature.parameters) 56 | params = deduplicate_kwargs(params) 57 | return forge.sign(*params.values()) 58 | 59 | 60 | def deduplicate_kwargs(params: dict) -> dict: 61 | """If a list of params contains one or more variadic keyword args (e.g., ``**kwargs``), 62 | ensure there are no duplicates and move it to the end. 63 | """ 64 | import forge 65 | 66 | # Check for kwargs by param type instead of by name 67 | has_var_kwargs = False 68 | for k, v in params.copy().items(): 69 | if v.kind == inspect.Parameter.VAR_KEYWORD: 70 | has_var_kwargs = True 71 | params.pop(k) 72 | 73 | # If it was present, add kwargs as the last param 74 | if has_var_kwargs: 75 | params.update(forge.kwargs) 76 | return params 77 | 78 | 79 | def copy_docstrings(target_function: Callable, *template_functions: Callable) -> Callable: 80 | """Copy 'Args' documentation from one or more template functions to a target function. 81 | Assumes Google-style docstrings. 82 | 83 | Args: 84 | target_function: Function to modify 85 | template_functions: Functions containing docstrings to apply to ``target_function`` 86 | 87 | Returns: 88 | Target function with modified docstring 89 | """ 90 | # Start with initial function description 91 | docstring, args_section, return_section = _split_docstring(target_function.__doc__) 92 | 93 | # Combine and insert 'Args' section 94 | args_sections = [args_section] 95 | args_sections += [_split_docstring(func.__doc__)[1] for func in template_functions] 96 | docstring += '\n\nArgs:\n' + _combine_args_sections(*args_sections) 97 | 98 | # Insert 'Returns' section, if present 99 | if return_section: 100 | docstring += f'\n\nReturns:\n {return_section}' 101 | 102 | target_function.__doc__ = docstring 103 | return target_function 104 | 105 | 106 | def _split_docstring(docstring: str | None = None) -> tuple[str, str, str]: 107 | """Split a docstring into the following sections, if present: 108 | 109 | * Function summary 110 | * Argument descriptions 111 | * Return value description 112 | """ 113 | summary = docstring or '' 114 | args_section = return_section = '' 115 | if 'Returns:' in summary: 116 | summary, return_section = summary.split('Returns:') 117 | if 'Args:' in summary: 118 | summary, args_section = summary.split('Args:') 119 | 120 | def fmt(chunk): 121 | return inspect.cleandoc(chunk.strip()) 122 | 123 | return fmt(summary), fmt(args_section), fmt(return_section) 124 | 125 | 126 | def _combine_args_sections(*args_sections: str) -> str: 127 | """Combine 'Args' sections from multiple functions into one section, removing any duplicates""" 128 | # Ensure one line per arg 129 | args_section = '\n'.join(args_sections).strip() 130 | args_section = re.sub('\n\\s+', ' ', args_section) 131 | 132 | # Split into key-value pairs to remove any duplicates; if so, keep the first one 133 | args: dict[str, str] = {} 134 | for line in args_section.splitlines(): 135 | k, v = line.split(':', 1) 136 | args.setdefault(k.strip(), v.strip()) 137 | 138 | return '\n'.join([f' {k}: {v}' for k, v in args.items()]) 139 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guide 2 | 3 | ## Dev Installation 4 | 5 | To set up for local development (requires [uv](https://docs.astral.sh/uv/getting-started/installation/)): 6 | 7 | ```sh 8 | $ git clone https://github.com/requests-cache/aiohttp-client-cache 9 | $ cd aiohttp-client-cache 10 | $ uv sync --frozen --all-extras --all-groups 11 | ``` 12 | 13 | ## Pre-commit hooks 14 | 15 | CI jobs will run code style checks, type checks, linting, etc. If you would like to run these same 16 | checks locally, you can use [pre-commit](https://github.com/pre-commit/pre-commit). 17 | This is optional but recommended. 18 | 19 | To install pre-commit hooks: 20 | 21 | ```sh 22 | pre-commit install 23 | ``` 24 | 25 | To manually run checks on all files: 26 | 27 | ```sh 28 | pre-commit run --all-files 29 | # Alternative alias with nox: 30 | nox -e lint 31 | ``` 32 | 33 | To disable pre-commit hooks: 34 | 35 | ```sh 36 | pre-commit uninstall 37 | ``` 38 | 39 | ## Testing 40 | 41 | ### Test Layout 42 | 43 | Tests are divided into unit and integration tests: 44 | 45 | - Unit tests can be run without any additional setup, and **don't depend on any external services** 46 | - Integration tests **depend on additional services**, which are easiest to run using Docker 47 | (see Integration Tests section below) 48 | 49 | ### Running Tests 50 | 51 | - Run `uv run pytest` to run all tests 52 | - Run `uv run pytest test/unit` to run only unit tests 53 | - Run `uv run pytest test/integration` to run only integration tests 54 | 55 | For CI jobs (including PRs), these tests will be run for each supported python version. 56 | You can use [nox](https://nox.thea.codes) to do this locally, if needed: 57 | 58 | ```sh 59 | nox -e test 60 | ``` 61 | 62 | Or to run tests for a specific python version: 63 | 64 | ```sh 65 | nox -e test-3.10 66 | ``` 67 | 68 | To generate a coverage report: 69 | 70 | ```sh 71 | nox -e cov 72 | ``` 73 | 74 | See `nox --list` for a full list of available commands. 75 | 76 | ### Integration Test Containers 77 | 78 | A live web server and backend databases are required to run integration tests, and docker-compose 79 | config is included to make this easier. First, [install docker](https://docs.docker.com/get-docker/) 80 | and [install docker-compose](https://docs.docker.com/compose/install/). 81 | 82 | Then, run: 83 | 84 | ```sh 85 | docker-compose up -d 86 | uv run pytest test/integration 87 | ``` 88 | 89 | To test DragonflyDB you need to stop a Redis container (if running) and run `docker compose -f dragonflydb.yaml up`. 90 | No other changes are required, you can run related tests with e.g. `uv run pytest test -k redis`. 91 | 92 | ## Documentation 93 | 94 | [Sphinx](http://www.sphinx-doc.org/en/master/) is used to generate documentation. To build the docs locally: 95 | 96 | ```sh 97 | $ nox -e docs 98 | ``` 99 | 100 | To preview: 101 | 102 | ```sh 103 | # MacOS: 104 | $ open docs/_build/index.html 105 | # Linux: 106 | $ xdg-open docs/_build/html/index.html 107 | ``` 108 | 109 | ### Readthedocs 110 | 111 | Documentation is automatically built and published by Readthedocs whenever code is merged into the 112 | `main` branch. 113 | 114 | Sometimes, there are differences in the Readthedocs build environment that can cause builds to 115 | succeed locally but fail remotely. To help debug this, you can use the 116 | [readthedocs/build](https://github.com/readthedocs/readthedocs-docker-images) container to build 117 | the docs. A configured build container is included in `docker-compose.yml` to simplify this. 118 | 119 | Run with: 120 | 121 | ```sh 122 | docker compose up -d --build 123 | docker exec readthedocs make all 124 | ``` 125 | 126 | ## Pull Requests 127 | 128 | Here are some general guidelines for submitting a pull request: 129 | 130 | - If the changes are trivial, just briefly explain the changes in the PR description. 131 | - Otherwise, please submit an issue describing the proposed change prior to submitting a PR. 132 | - Add unit test coverage for your changes 133 | - If your changes add or modify user-facing behavior, add documentation describing those changes 134 | - Submit the PR to be merged into the `main` branch. 135 | 136 | ## Notes for Maintainers 137 | 138 | ### Releases 139 | 140 | - Releases are built and published to PyPI based on **git tags.** 141 | - [Milestones](https://github.com/requests-cache/aiohttp-client-cache/milestones) will be used to track 142 | progress on major and minor releases. 143 | - GitHub Actions will build and deploy packages to PyPI on tagged commits 144 | on the `main` branch. 145 | 146 | Release steps: 147 | 148 | - Update the version in both `pyproject.toml` and `aiohttp_client_cache/__init__.py` 149 | - Make sure the release notes in `HISTORY.md` are up to date 150 | - Push a new tag, e.g.: `git tag v0.1.0 && git push origin v0.1.0` 151 | - This will trigger a deployment. Verify that this completes successfully and that the new version can be installed from pypi with `pip install` 152 | - A [readthedocs build](https://readthedocs.org/projects/aiohttp-client-cache/builds/) will be triggered by the new tag. Verify that this completes successfully. 153 | 154 | Downstream builds: 155 | 156 | - We also maintain a [Conda package](https://anaconda.org/conda-forge/aiohttp-client-cache), which is automatically built and published by conda-forge whenever a new release is published to PyPI. The [feedstock repo](https://github.com/conda-forge/aiohttp-client-cache-feedstock) only needs to be updated manually if there are changes to dependencies. 157 | - For reference: [repology](https://repology.org/project/python:aiohttp-client-cache) lists additional downstream packages maintained by other developers. 158 | 159 | ### Code Layout 160 | 161 | Here is a brief overview of the main classes and modules. See [API Reference](https://aiohttp-client-cache.readthedocs.io/en/latest/reference.html) for more complete documentation. 162 | 163 | - `session.CacheMixin`, `session.CachedSession`: A mixin and wrapper class, respectively, for `aiohttp.ClientSession`. There is little logic here except wrapping `ClientSession._request()` with caching behavior. 164 | - `response.CachedResponse`: A wrapper class built from an `aiohttp.ClientResponse`, with additional cache-related info. This is what is serialized and persisted to the cache. 165 | - `backends.base.CacheBackend`: Most of the caching logic lives here, including saving and retrieving responses. It contains two `BaseCache` objects for storing responses and redirects, respectively. 166 | - `backends.base.BaseCache`: Base class for lower-level storage operations, overridden by individual backends. 167 | - Other modules under `backends.*`: Backend implementations that subclass `CacheBackend` + `BaseCache` 168 | - `cache_control`: Utilities for determining cache expiration and other cache actions 169 | - `cache_keys`: Utilities for creating cache keys 170 | -------------------------------------------------------------------------------- /test/unit/test_session.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from http.cookies import SimpleCookie 4 | from unittest.mock import AsyncMock, MagicMock, patch 5 | 6 | import pytest 7 | from aiohttp import ClientResponse 8 | from yarl import URL 9 | 10 | from aiohttp_client_cache.backends import CacheBackend 11 | from aiohttp_client_cache.response import CachedResponse 12 | from aiohttp_client_cache.session import CachedSession, CacheMixin, ClientSession 13 | 14 | pytestmark = [pytest.mark.asyncio] 15 | 16 | 17 | FakeCachedResponse = CachedResponse(method='GET', reason='OK', status=200, url='url', version='1.1') 18 | FakeClientResponse = ClientResponse( 19 | method='GET', 20 | url=URL('http://example.com'), 21 | writer=None, 22 | continue100=None, 23 | timer=None, # type: ignore[arg-type] 24 | request_info=None, # type: ignore[arg-type] 25 | traces=None, # type: ignore[arg-type] 26 | # loop=asyncio.get_event_loop(), 27 | loop=MagicMock(), 28 | session=None, # type: ignore[arg-type] 29 | ) 30 | 31 | 32 | async def test_session__init_kwargs(): 33 | cookie_jar = MagicMock() 34 | base_url = 'https://test.com' 35 | 36 | async with CachedSession( 37 | cache=MagicMock(spec=CacheBackend), base_url=base_url, cookie_jar=cookie_jar 38 | ) as session: 39 | assert session._base_url == URL(base_url) 40 | assert session._cookie_jar is cookie_jar 41 | 42 | 43 | async def test_custom_session__init_kwargs(): 44 | """Ensure ClientSession kwargs are passed through even with a custom class with modified init 45 | signature 46 | """ 47 | 48 | class CustomSession(CachedSession, ClientSession): 49 | def __init__(self, *args, **kwargs): 50 | super().__init__(*args, **kwargs) 51 | 52 | cookie_jar = MagicMock() 53 | base_url = 'https://test.com' 54 | 55 | async with CustomSession( 56 | cache=MagicMock(spec=CacheBackend), base_url=base_url, cookie_jar=cookie_jar 57 | ) as session: 58 | assert session._base_url == URL(base_url) 59 | assert session._cookie_jar is cookie_jar 60 | 61 | 62 | async def test_session__init_posarg(): 63 | base_url = 'https://test.com' 64 | async with CachedSession(base_url, cache=MagicMock(spec=CacheBackend)) as session: 65 | assert session._base_url == URL(base_url) 66 | 67 | 68 | @patch.object(ClientSession, '_request', return_value=FakeCachedResponse) 69 | async def test_session__cache_hit(mock_request): 70 | cache = MagicMock(spec=CacheBackend) 71 | response = AsyncMock(is_expired=False, url=URL('https://test.com')) 72 | cache.request.return_value = response 73 | 74 | async with CachedSession(cache=cache) as session: 75 | await session.get('http://test.url') 76 | 77 | assert mock_request.called is False 78 | 79 | 80 | @patch.object(ClientSession, '_request', return_value=FakeCachedResponse) 81 | async def test_session__cache_expired_or_invalid(mock_request): 82 | cache = MagicMock(spec=CacheBackend) 83 | cache.request.return_value = None 84 | 85 | async with CachedSession(cache=cache) as session: 86 | await session.get('http://test.url') 87 | 88 | assert mock_request.called is True 89 | 90 | 91 | @patch.object(ClientSession, '_request', return_value=FakeCachedResponse) 92 | async def test_session__cache_miss(mock_request): 93 | cache = MagicMock(spec=CacheBackend) 94 | cache.request.return_value = None 95 | 96 | async with CachedSession(cache=cache) as session: 97 | await session.get('http://test.url') 98 | 99 | assert mock_request.called is True 100 | 101 | 102 | @patch.object(ClientSession, '_request', return_value=FakeCachedResponse) 103 | async def test_session__request_expire_after(mock_request): 104 | cache = MagicMock(spec=CacheBackend) 105 | cache.request.return_value = None 106 | 107 | async with CachedSession(cache=cache) as session: 108 | await session.get('http://test.url', expire_after=10) 109 | 110 | assert mock_request.called is True 111 | assert 'expire_after' not in mock_request.call_args 112 | 113 | 114 | @patch.object(ClientSession, '_request', return_value=FakeClientResponse) 115 | async def test_session__default_attrs(mock_request): 116 | cache = MagicMock(spec=CacheBackend) 117 | cache.request.return_value = None 118 | 119 | async with CachedSession(cache=cache) as session: 120 | response = await session.get('http://test.url') 121 | 122 | assert response.from_cache is False and response.is_expired is False 123 | 124 | 125 | @pytest.mark.parametrize( 126 | 'params', 127 | [ 128 | {'param': 'value'}, # Dict of strings 129 | {'param': 4.2}, # Dict of floats 130 | (('param', 'value'),), # Tuple of (key, value) pairs 131 | 'param', # string 132 | ], 133 | ) 134 | @patch.object(ClientSession, '_request', return_value=FakeClientResponse) 135 | async def test_all_param_types(mock_request, params) -> None: 136 | """Ensure that CachedSession.request() accepts all the same parameter types as aiohttp""" 137 | cache = MagicMock(spec=CacheBackend) 138 | cache.request.return_value = None 139 | 140 | async with CachedSession(cache=cache) as session: 141 | response = await session.get('http://test.url', params=params) 142 | 143 | assert response.from_cache is False 144 | 145 | 146 | @patch.object(ClientSession, '_request', return_value=FakeCachedResponse) 147 | async def test_session__cookies(mock_request): 148 | cache = MagicMock(spec=CacheBackend) 149 | response = AsyncMock( 150 | is_expired=False, 151 | url=URL('https://test.com'), 152 | cookies=SimpleCookie({'test_cookie': 'value'}), 153 | ) 154 | cache.request.return_value = response 155 | 156 | async with CachedSession(cache=cache) as session: 157 | session.cookie_jar.clear() 158 | await session.get('http://test.url') 159 | cookies = session.cookie_jar.filter_cookies('https://test.com') 160 | 161 | assert cookies['test_cookie'].value == 'value' 162 | 163 | 164 | @patch.object(ClientSession, '_request', return_value=FakeCachedResponse) 165 | async def test_session__empty_cookies(mock_request): 166 | """Previous versions didn't set cookies if they were empty. Just make sure it doesn't explode.""" 167 | cache = MagicMock(spec=CacheBackend) 168 | response = AsyncMock(is_expired=False, url=URL('https://test.com'), cookies=None) 169 | cache.request.return_value = response 170 | 171 | async with CachedSession(cache=cache) as session: 172 | session.cookie_jar.clear() 173 | await session.get('http://test.url') 174 | assert not session.cookie_jar.filter_cookies('https://test.com') 175 | 176 | 177 | @patch.object(ClientSession, '_request', return_value=FakeCachedResponse) 178 | async def test_mixin(mock_request): 179 | """Ensure that CacheMixin can be used as a mixin with a custom session class""" 180 | 181 | class CustomSession(CacheMixin, ClientSession): 182 | pass 183 | 184 | cache = MagicMock(spec=CacheBackend) 185 | response = AsyncMock(is_expired=False, url=URL('https://test.com')) 186 | cache.request.return_value = response 187 | 188 | async with CustomSession(cache=cache) as session: 189 | await session.get('http://test.url') 190 | 191 | assert mock_request.called is False 192 | 193 | 194 | @patch.object(ClientSession, '_request', return_value=FakeCachedResponse) 195 | async def test_session__cache_include_headers(mock_request): 196 | async with CachedSession(cache=CacheBackend(include_headers=True)) as session: 197 | await session.get('https://test.com') 198 | 199 | assert mock_request.called is True 200 | mock_request.called = False 201 | 202 | session.headers.update({'key': 'value'}) 203 | await session.get('https://test.com') 204 | 205 | assert mock_request.called is True 206 | -------------------------------------------------------------------------------- /test/integration/test_sqlite.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | import os 5 | import sqlite3 6 | from contextlib import asynccontextmanager 7 | from tempfile import gettempdir 8 | from unittest.mock import AsyncMock, MagicMock, patch 9 | 10 | import aiosqlite 11 | import pytest 12 | 13 | from aiohttp_client_cache.backends.sqlite import SQLiteBackend, SQLiteCache, SQLitePickleCache 14 | from test.conftest import CACHE_NAME, httpbin 15 | from test.integration import BaseBackendTest, BaseStorageTest 16 | 17 | pytestmark = pytest.mark.asyncio 18 | 19 | 20 | class TestSQLiteCache(BaseStorageTest): 21 | init_kwargs = {'use_temp': True} 22 | storage_class = SQLiteCache 23 | 24 | @classmethod 25 | def teardown_class(cls): 26 | try: 27 | os.unlink(f'{CACHE_NAME}.sqlite') 28 | except Exception: 29 | pass 30 | 31 | async def test_use_temp(self): 32 | relative_path = self.storage_class(CACHE_NAME).filename 33 | temp_path = self.storage_class(CACHE_NAME, use_temp=True).filename 34 | assert not relative_path.startswith(gettempdir()) 35 | assert temp_path.startswith(gettempdir()) 36 | 37 | async def test_bulk_commit(self): 38 | async with self.init_cache(self.storage_class) as cache: 39 | async with cache.bulk_commit(): 40 | pass 41 | 42 | n_items = 1000 43 | async with cache.bulk_commit(): 44 | for i in range(n_items): 45 | await cache.write(f'key_{i}', f'value_{i}') 46 | 47 | keys = {k async for k in cache.keys()} 48 | values = {v async for v in cache.values()} 49 | assert keys == {f'key_{i}' for i in range(n_items)} 50 | assert values == {f'value_{i}' for i in range(n_items)} 51 | 52 | async def test_concurrent_bulk_commit(self): 53 | """Multiple concurrent bulk commits should not interfere with each other""" 54 | mock_connection = AsyncMock(spec=aiosqlite.Connection) 55 | mock_connection._connection = MagicMock(spec=sqlite3.Connection) 56 | # Note: this AsyncMock behavior is implicit in python 3.13+ 57 | mock_connection.commit = AsyncMock() 58 | mock_connection.execute = AsyncMock() 59 | 60 | @asynccontextmanager 61 | async def bulk_commit_ctx(): 62 | async with self.init_cache(self.storage_class) as cache: 63 | cache._connection = mock_connection 64 | 65 | async def bulk_commit_items(n_items): 66 | async with cache.bulk_commit(): 67 | for i in range(n_items): 68 | await cache.write(f'key_{n_items}_{i}', f'value_{i}') 69 | 70 | yield bulk_commit_items 71 | 72 | async with bulk_commit_ctx() as bulk_commit_items: 73 | tasks = [asyncio.create_task(bulk_commit_items(n)) for n in [10, 100, 1000, 10000]] 74 | await asyncio.gather(*tasks) 75 | assert mock_connection.commit.call_count == 4 76 | 77 | async def test_fast_save(self): 78 | async with ( 79 | self.init_cache(self.storage_class, index=1, fast_save=True) as cache_1, 80 | self.init_cache(self.storage_class, index=2, fast_save=True) as cache_2, 81 | ): 82 | for i in range(1000): 83 | await cache_1.write(i, i) # type: ignore[arg-type] 84 | await cache_2.write(i, i) # type: ignore[arg-type] 85 | 86 | keys_1 = {k async for k in cache_1.keys()} 87 | keys_2 = {k async for k in cache_2.keys()} 88 | values_1 = {v async for v in cache_1.values()} 89 | values_2 = {v async for v in cache_2.values()} 90 | assert keys_1 == keys_2 == set(range(1000)) 91 | assert values_1 == values_2 == set(range(1000)) 92 | 93 | @patch('sqlite3.connect') 94 | async def test_connection_kwargs(self, mock_sqlite): 95 | """A spot check to make sure optional connection kwargs gets passed to connection""" 96 | async with self.init_cache(self.storage_class, timeout=0.5, invalid_kwarg='???') as cache: 97 | mock_sqlite.assert_called_with(cache.filename, timeout=0.5) 98 | 99 | async def test_close(self): 100 | async with self.init_cache(self.storage_class) as cache: 101 | async with cache.get_connection(): 102 | pass 103 | await cache.close() 104 | await cache.close() # Closing again should be a no-op 105 | assert cache._connection is None 106 | 107 | async def test_failed_thread_close(self): 108 | """If closing the connection thread fails, it should log a warning and continue""" 109 | async with self.init_cache(self.storage_class) as cache: 110 | async with cache.get_connection(): 111 | pass 112 | with patch.object(aiosqlite.Connection, '_stop_running', side_effect=AttributeError): 113 | del cache 114 | 115 | # TODO: Tests for unimplemented features 116 | # async def test_chunked_bulk_delete(self): 117 | # """When deleting more items than SQLite can handle in a single statement, it should be 118 | # chunked into multiple smaller statements 119 | # """ 120 | # # Populate the cache with more items than can fit in a single delete statement 121 | # cache = await self.init_cache() 122 | # async with cache.bulk_commit(): 123 | # for i in range(2000): 124 | # await cache.write(f'key_{i}', f'value_{i}') 125 | 126 | # keys = {k async for k in cache.keys()} 127 | 128 | # # First pass to ensure that bulk_delete is split across three statements 129 | # with patch.object(cache, 'connection') as mock_connection: 130 | # con = mock_connection().__enter__.return_value 131 | # cache.bulk_delete(keys) 132 | # assert con.execute.call_count == 3 133 | 134 | # # Second pass to actually delete keys and make sure it doesn't explode 135 | # await cache.bulk_delete(keys) 136 | # assert await cache.size() == 0 137 | 138 | # async def test_noop(self): 139 | # async def do_noop_bulk(cache): 140 | # async with cache.bulk_commit(): 141 | # pass 142 | # del cache 143 | 144 | # cache = await self.init_cache() 145 | # thread = Thread(target=do_noop_bulk, args=(cache,)) 146 | # thread.start() 147 | # thread.join() 148 | 149 | # # make sure connection is not closed by the thread 150 | # await cache.write('key_1', 'value_1') 151 | # assert {k async for k in cache.keys()} == {'key_1'} 152 | 153 | 154 | class TestSQLitePickleCache(BaseStorageTest): 155 | init_kwargs = {'use_temp': True} 156 | picklable = True 157 | storage_class = SQLitePickleCache 158 | 159 | 160 | class TestSQLiteBackend(BaseBackendTest): 161 | backend_class = SQLiteBackend 162 | init_kwargs = {'use_temp': True} 163 | 164 | async def test_shared_connection(self, *args, **kwargs): 165 | async with self.init_session() as session: 166 | assert session.cache.responses._lock is session.cache.redirects._lock 167 | await session.get(httpbin('get')) 168 | assert session.cache.responses._connection is session.cache.redirects._connection 169 | assert isinstance(session.cache.responses._connection._connection, sqlite3.Connection) 170 | 171 | async def test_autoclose__default(self): 172 | """By default, the backend should be closed when the session is closed""" 173 | 174 | async with self.init_session() as session: 175 | mock_close = MagicMock(wraps=session.cache.close) 176 | session.cache.close = mock_close # type: ignore[method-assign] 177 | 178 | await session.get(httpbin('get')) 179 | mock_close.assert_called_once() 180 | -------------------------------------------------------------------------------- /aiohttp_client_cache/backends/dynamodb.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import AsyncIterable 4 | from contextlib import asynccontextmanager 5 | from logging import getLogger 6 | from typing import Any 7 | 8 | import aioboto3 9 | from aioboto3.session import ResourceCreatorContext 10 | from aioboto3.session import Session as AWSSession 11 | from botocore.exceptions import ClientError 12 | 13 | from aiohttp_client_cache.backends import BaseCache, CacheBackend, ResponseOrKey, get_valid_kwargs 14 | 15 | logger = getLogger(__name__) 16 | MAX_ITEM_SIZE = 400000 # 400KB 17 | 18 | 19 | class DynamoDBBackend(CacheBackend): 20 | """Async cache backend for `DynamoDB `_ 21 | 22 | Notes: 23 | * Requires `aioboto3 `_ 24 | * Accepts keyword arguments for :py:meth:`boto3.session.Session.client` 25 | * See `DynamoDB Service Resource 26 | `_ 27 | for more usage details. 28 | * DynamoDB has a maximum item size of 400KB. If an item exceeds this size, it will not be 29 | written to the cache. 30 | 31 | Args: 32 | cache_name: Table name to use 33 | key_attr_name: The name of the field to use for keys in the DynamoDB document 34 | val_attr_name: The name of the field to use for values in the DynamoDB document 35 | create_if_not_exists: Whether or not to attempt to create the DynamoDB table 36 | context: An existing `ResourceCreatorContext `_ 37 | to reuse instead of creating a new one 38 | kwargs: Additional keyword arguments for :py:class:`.CacheBackend` or backend connection 39 | """ 40 | 41 | def __init__( 42 | self, 43 | cache_name: str = 'aiohttp-cache', 44 | key_attr_name: str = 'k', 45 | val_attr_name: str = 'v', 46 | create_if_not_exists: bool = False, 47 | context: ResourceCreatorContext | None = None, 48 | **kwargs: Any, 49 | ): 50 | super().__init__(cache_name=cache_name, **kwargs) 51 | self.responses = DynamoDbCache( 52 | cache_name, 53 | 'resp', 54 | key_attr_name, 55 | val_attr_name, 56 | create_if_not_exists, 57 | context=context, 58 | **kwargs, 59 | ) 60 | self.redirects = DynamoDbCache( 61 | cache_name, 62 | 'redir', 63 | key_attr_name, 64 | val_attr_name, 65 | create_if_not_exists, 66 | context=self.responses.context, 67 | **kwargs, 68 | ) 69 | 70 | 71 | class DynamoDbCache(BaseCache): 72 | """An async interface for caching objects in a DynamoDB key-store 73 | 74 | The actual key name on the dynamodb server will be ``namespace:key``. 75 | In order to deal with how dynamodb stores data/keys, all values must be serialized. 76 | """ 77 | 78 | def __init__( 79 | self, 80 | table_name: str, 81 | namespace: str, 82 | key_attr_name: str = 'k', 83 | val_attr_name: str = 'v', 84 | create_if_not_exists: bool = False, 85 | context: ResourceCreatorContext = None, 86 | **kwargs: Any, 87 | ): 88 | super().__init__(**kwargs) 89 | self.table_name = table_name 90 | self.namespace = namespace 91 | self.key_attr_name = key_attr_name 92 | self.val_attr_name = val_attr_name 93 | self.create_if_not_exists = create_if_not_exists 94 | 95 | resource_kwargs = get_valid_kwargs(AWSSession.resource, kwargs) 96 | self.context = context or aioboto3.Session().resource('dynamodb', **resource_kwargs) 97 | self._table = None 98 | 99 | @asynccontextmanager 100 | async def get_connection(self): 101 | # Re-use the service resource if it's already been created 102 | if self.context.cls: 103 | yield self.context.cls 104 | else: 105 | yield await self.context.__aenter__() 106 | 107 | async def get_table(self): 108 | if not self._table: 109 | async with self.get_connection() as conn: 110 | if self.create_if_not_exists: 111 | self._table = await self._create_table(conn) 112 | else: 113 | self._table = await conn.Table(self.table_name) 114 | return self._table 115 | 116 | async def _create_table(self, conn): 117 | table = await conn.Table(self.table_name) 118 | 119 | try: 120 | await conn.create_table( 121 | AttributeDefinitions=[{'AttributeName': self.key_attr_name, 'AttributeType': 'S'}], 122 | TableName=self.table_name, 123 | KeySchema=[{'AttributeName': self.key_attr_name, 'KeyType': 'HASH'}], 124 | BillingMode='PAY_PER_REQUEST', 125 | ) 126 | await table.wait_until_exists() 127 | except ClientError as e: 128 | if e.response['Error']['Code'] != 'ResourceInUseException': 129 | raise 130 | 131 | return table 132 | 133 | def _doc(self, key) -> dict: 134 | return {self.key_attr_name: f'{self.namespace}:{key}'} 135 | 136 | async def _scan(self) -> AsyncIterable[dict]: 137 | table = await self.get_table() 138 | paginator = table.meta.client.get_paginator('scan') 139 | iterator = paginator.paginate( 140 | TableName=table.name, 141 | Select='ALL_ATTRIBUTES', 142 | FilterExpression=f'begins_with({self.key_attr_name}, :namespace)', 143 | ExpressionAttributeValues={':namespace': f'{self.namespace}:'}, 144 | ) 145 | async for result in iterator: 146 | for item in result['Items']: 147 | yield item 148 | 149 | async def bulk_delete(self, keys: set) -> None: 150 | table = await self.get_table() 151 | async with table.batch_writer() as dynamo_writer: 152 | for key in keys: 153 | doc = self._doc(key) 154 | await dynamo_writer.delete_item(Key=doc) 155 | 156 | async def delete(self, key: str) -> None: 157 | doc = self._doc(key) 158 | table = await self.get_table() 159 | await table.delete_item(Key=doc) 160 | 161 | async def read(self, key: str) -> ResponseOrKey: 162 | table = await self.get_table() 163 | response = await table.get_item(Key=self._doc(key), ProjectionExpression=self.val_attr_name) 164 | item = response.get('Item') 165 | if item: 166 | return self.deserialize(item[self.val_attr_name].value) 167 | return None 168 | 169 | async def write(self, key: str, item: ResponseOrKey) -> None: 170 | item = self.serialize(item) 171 | if len(item or b'') > MAX_ITEM_SIZE: 172 | logger.warning( 173 | f'Item size exceeds maximum for DynamoDB ({MAX_ITEM_SIZE}); skipping write' 174 | ) 175 | return 176 | 177 | table = await self.get_table() 178 | doc = self._doc(key) 179 | doc[self.val_attr_name] = item 180 | await table.put_item(Item=doc) 181 | 182 | async def clear(self) -> None: 183 | async for key in self.keys(): 184 | await self.delete(key) 185 | 186 | async def contains(self, key: str) -> bool: 187 | resp = await self.read(key) 188 | return resp is not None 189 | 190 | async def keys(self) -> AsyncIterable[str]: 191 | len_prefix = len(self.namespace) + 1 192 | async for item in self._scan(): 193 | yield item[self.key_attr_name][len_prefix:] 194 | 195 | async def size(self) -> int: 196 | return len([i async for i in self._scan()]) 197 | 198 | async def values(self) -> AsyncIterable[ResponseOrKey]: 199 | async for item in self._scan(): 200 | yield self.deserialize(item[self.val_attr_name].value) 201 | -------------------------------------------------------------------------------- /test/unit/test_response.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from datetime import datetime, timedelta, timezone 4 | from unittest import mock 5 | 6 | import pytest 7 | from aiohttp import ClientResponseError, web 8 | from multidict import MultiDictProxy 9 | from yarl import URL 10 | 11 | from aiohttp_client_cache.cache_control import utcnow 12 | from aiohttp_client_cache.response import CachedResponse, RequestInfo, UnsupportedExpiresError 13 | 14 | 15 | async def get_test_response(client_factory, url='/', headers=None, **kwargs): 16 | app = web.Application() 17 | app.router.add_route('GET', '/valid_url', mock_handler) 18 | app.router.add_route('GET', '/json', json_mock_handler) 19 | app.router.add_route('GET', '/empty_content', empty_mock_handler) 20 | app.router.add_route('GET', '/null_content', null_mock_handler) 21 | client = await client_factory(app) 22 | client_response = await client.get(url, headers=headers) 23 | 24 | return await CachedResponse.from_client_response(client_response, **kwargs) 25 | 26 | 27 | async def mock_handler(request): 28 | return web.Response( 29 | body=b'Hello, world', 30 | headers={ 31 | 'Link': '; rel="preconnect"', 32 | 'Content-Disposition': 'attachment; name="test-param"; filename="img.jpg"', 33 | 'Content-Type': 'text/plain; charset=utf-8', 34 | 'Content-Length': '12', 35 | }, 36 | ) 37 | 38 | 39 | async def json_mock_handler(request): 40 | return web.Response(body=b'{"key": "value"}') 41 | 42 | 43 | # Note: Empty string vs null can trigger different corner cases 44 | async def empty_mock_handler(request): 45 | return web.Response(body=b' ') 46 | 47 | 48 | async def null_mock_handler(request): 49 | return web.Response(body=None) 50 | 51 | 52 | async def test_basic_attrs(aiohttp_client): 53 | response = await get_test_response(aiohttp_client) 54 | 55 | assert response.method == 'GET' 56 | assert response.reason == 'Not Found' 57 | assert response.status == 404 58 | assert isinstance(response.url, URL) 59 | assert response.encoding == 'utf-8' 60 | assert response.headers['Content-Type'] == 'text/plain; charset=utf-8' 61 | assert await response.text() == '404: Not Found' 62 | assert response.history == () 63 | assert response._released is True 64 | 65 | 66 | @mock.patch('aiohttp_client_cache.response.utcnow') 67 | async def test_is_expired(mock_utcnow, aiohttp_client): 68 | mock_utcnow.return_value = utcnow() 69 | expires = utcnow() + timedelta(seconds=0.02) 70 | 71 | response = await get_test_response(aiohttp_client, expires=expires) 72 | 73 | assert response.expires == expires 74 | assert response.is_expired is False 75 | 76 | mock_utcnow.return_value += timedelta(0.02) 77 | assert response.is_expired is True 78 | 79 | 80 | async def test_is_expired__invalid(aiohttp_client): 81 | with pytest.raises(AttributeError, match="'str' object has no attribute 'tzinfo'"): 82 | await get_test_response(aiohttp_client, expires='asdf') 83 | with pytest.raises(UnsupportedExpiresError, match='Expected a naive datetime'): 84 | await get_test_response(aiohttp_client, expires=datetime.now(timezone.utc)) 85 | 86 | 87 | async def test_content_disposition(aiohttp_client): 88 | response = await get_test_response(aiohttp_client, '/valid_url') 89 | assert response.content_disposition.type == 'attachment' 90 | assert response.content_disposition.filename == 'img.jpg' 91 | assert response.content_disposition.parameters.get('name') == 'test-param' 92 | 93 | 94 | async def test_encoding(aiohttp_client): 95 | response = await get_test_response(aiohttp_client) 96 | assert response.encoding == response.get_encoding() == 'utf-8' 97 | 98 | 99 | async def test_request_info(aiohttp_client): 100 | response = await get_test_response( 101 | aiohttp_client, '/valid_url', headers={'Custom-Header': 'value'} 102 | ) 103 | request_info = response.request_info 104 | 105 | assert isinstance(request_info, RequestInfo) 106 | assert request_info.method == 'GET' 107 | assert request_info.url == request_info.real_url 108 | assert str(request_info.url).endswith('/valid_url') 109 | assert request_info.headers['Custom-Header'] == 'value' 110 | 111 | 112 | async def test_headers(aiohttp_client): 113 | response = await get_test_response(aiohttp_client) 114 | raw_headers = dict(response.raw_headers) 115 | 116 | assert b'Content-Type' in raw_headers and b'Content-Length' in raw_headers 117 | assert 'Content-Type' in response.headers and 'Content-Length' in response.headers 118 | assert response._headers == response.headers 119 | with pytest.raises(TypeError): 120 | response.headers['key'] = 'value' 121 | 122 | 123 | async def test_headers__mixin_attributes(aiohttp_client): 124 | response = await get_test_response(aiohttp_client, '/valid_url') 125 | assert response.charset == 'utf-8' 126 | assert response.content_length == 12 127 | assert response.content_type == 'text/plain' 128 | 129 | 130 | async def test_headers__case_insensitive_multidict(aiohttp_client): 131 | """Headers should be case-insensitive and allow multiple values""" 132 | response = await get_test_response(aiohttp_client) 133 | response.raw_headers += ((b'Cache-Control', b'public'),) 134 | response.raw_headers += ((b'Cache-Control', b'max-age=360'),) 135 | 136 | assert response.headers['Cache-Control'] == 'public' 137 | assert response.headers.get('Cache-Control') == 'public' 138 | assert response.headers.get('CACHE-CONTROL') == 'public' 139 | assert set(response.headers.getall('Cache-Control')) == {'max-age=360', 'public'} 140 | 141 | 142 | async def test_links(aiohttp_client): 143 | response = await get_test_response(aiohttp_client, '/valid_url') 144 | expected_links = [('preconnect', [('rel', 'preconnect'), ('url', 'https://example.com')])] 145 | assert response._links == expected_links 146 | assert isinstance(response.links, MultiDictProxy) 147 | assert response.links['preconnect']['url'] == URL('https://example.com') 148 | 149 | 150 | # TODO 151 | async def test_history(aiohttp_client): 152 | pass 153 | 154 | 155 | async def test_json(aiohttp_client): 156 | response = await get_test_response(aiohttp_client, '/json') 157 | assert await response.json() == {'key': 'value'} 158 | 159 | 160 | async def test_json__empty_content(aiohttp_client): 161 | response = await get_test_response(aiohttp_client, '/empty_content') 162 | assert await response.json() is None 163 | 164 | 165 | async def test_json__null_content(aiohttp_client): 166 | response = await get_test_response(aiohttp_client, '/null_content') 167 | assert await response.json() is None 168 | 169 | 170 | async def test_json__non_json_content(aiohttp_client): 171 | response = await get_test_response(aiohttp_client) 172 | with pytest.raises(ValueError): 173 | await response.json() 174 | 175 | 176 | async def test_raise_for_status__200(aiohttp_client): 177 | response = await get_test_response(aiohttp_client, '/valid_url') 178 | assert not response.raise_for_status() 179 | assert response.ok is True 180 | 181 | 182 | async def test_raise_for_status__404(aiohttp_client): 183 | response = await get_test_response(aiohttp_client, '/invalid_url') 184 | with pytest.raises(ClientResponseError): 185 | response.raise_for_status() 186 | assert response.ok is False 187 | 188 | 189 | async def test_text(aiohttp_client): 190 | response = await get_test_response(aiohttp_client) 191 | assert await response.text() == '404: Not Found' 192 | 193 | 194 | async def test_read(aiohttp_client): 195 | response = await get_test_response(aiohttp_client) 196 | assert await response.read() == b'404: Not Found' 197 | 198 | 199 | async def test_no_ops(aiohttp_client): 200 | # Just make sure CachedResponse doesn't explode if extra ClientResponse methods are called 201 | response = await get_test_response(aiohttp_client) 202 | 203 | await response.start() 204 | response.release() 205 | response.close() 206 | assert response.closed is True 207 | await response.wait_for_close() 208 | await response.terminate() 209 | assert response.connection is None 210 | assert response._released is True 211 | -------------------------------------------------------------------------------- /test/unit/test_base_backend.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import pickle 4 | from unittest.mock import MagicMock, patch 5 | 6 | import pytest 7 | 8 | from aiohttp_client_cache import CachedResponse 9 | from aiohttp_client_cache.backends import CacheBackend, DictCache, get_placeholder_backend 10 | 11 | TEST_URL = 'https://test.com' 12 | 13 | 14 | def get_mock_response(**kwargs): 15 | response_kwargs = { 16 | 'url': TEST_URL, 17 | 'method': 'GET', 18 | 'status': 200, 19 | 'is_expired': False, 20 | '_released': True, 21 | } 22 | response_kwargs.update(kwargs) 23 | return MagicMock(spec=CachedResponse, **response_kwargs) 24 | 25 | 26 | def test_get_placeholder_backend(): 27 | class TestBackend: 28 | def __init__(self): 29 | import nonexistent_module # noqa: F401 30 | 31 | try: 32 | TestBackend() 33 | except ImportError as e: 34 | placeholder = get_placeholder_backend(e) 35 | 36 | # Initializing the placeholder class should re-raise the original ImportError 37 | with pytest.raises(ImportError): 38 | placeholder() 39 | 40 | 41 | async def test_get_response__cache_response_hit(): 42 | cache = CacheBackend() 43 | mock_response = get_mock_response() 44 | await cache.responses.write('request-key', mock_response) 45 | 46 | response = await cache.get_response('request-key') 47 | assert response == mock_response 48 | 49 | 50 | async def test_get_response__cache_redirect_hit(): 51 | # Set up a cache with a couple cached items and a redirect 52 | cache = CacheBackend() 53 | mock_response = get_mock_response() 54 | await cache.responses.write('request-key', mock_response) 55 | await cache.redirects.write('redirect-key', 'request-key') 56 | 57 | response = await cache.get_response('redirect-key') 58 | assert response == mock_response 59 | 60 | 61 | @patch.object(CacheBackend, 'delete') 62 | async def test_get_response__cache_miss(mock_delete): 63 | cache = CacheBackend() 64 | 65 | response_1 = await cache.get_response('nonexistent-key') 66 | assert response_1 is None 67 | mock_delete.assert_not_called() 68 | 69 | 70 | @patch.object(CacheBackend, 'delete') 71 | @patch.object(CacheBackend, 'is_cacheable', return_value=False) 72 | async def test_get_response__cache_expired(mock_is_cacheable, mock_delete): 73 | cache = CacheBackend() 74 | mock_response = get_mock_response(is_expired=True) 75 | await cache.responses.write('request-key', mock_response) 76 | 77 | response = await cache.get_response('request-key') 78 | assert response is None 79 | mock_delete.assert_called_with('request-key') 80 | 81 | 82 | @pytest.mark.parametrize('error_type', [AttributeError, KeyError, TypeError, pickle.PickleError]) 83 | @patch.object(CacheBackend, 'delete') 84 | @patch.object(DictCache, 'read') 85 | async def test_get_response__cache_invalid(mock_read, mock_delete, error_type): 86 | cache = CacheBackend() 87 | mock_read.side_effect = error_type 88 | mock_response = get_mock_response() 89 | await cache.responses.write('request-key', mock_response) 90 | 91 | response = await cache.get_response('request-key') 92 | assert response is None 93 | mock_delete.assert_not_called() 94 | 95 | 96 | @patch.object(DictCache, 'read', return_value=object()) 97 | async def test_get_response__quiet_serde_error(mock_read): 98 | """Test for a quiet deserialization error in which no errors are raised but attributes are 99 | missing 100 | """ 101 | cache = CacheBackend() 102 | mock_response = get_mock_response() 103 | await cache.responses.write('request-key', mock_response) 104 | 105 | response = await cache.get_response('request-key') 106 | assert response is None 107 | 108 | 109 | async def test_save_response(): 110 | cache = CacheBackend() 111 | mock_response = get_mock_response() 112 | mock_response.history = [MagicMock(method='GET', url='test')] 113 | redirect_key = cache.create_key('GET', 'test') 114 | 115 | await cache.save_response(mock_response, 'key') 116 | cached_response = await cache.responses.read('key') 117 | assert cached_response and isinstance(cached_response, CachedResponse) 118 | assert await cache.redirects.read(redirect_key) == 'key' 119 | 120 | 121 | async def test_save_response__manual_save(): 122 | """Manually save a response with no cache key provided""" 123 | cache = CacheBackend() 124 | mock_response = get_mock_response() 125 | 126 | await cache.save_response(mock_response) 127 | cached_response = [r async for r in cache.responses.values()][0] 128 | assert cached_response and isinstance(cached_response, CachedResponse) 129 | 130 | 131 | async def test_clear(): 132 | cache = CacheBackend() 133 | await cache.responses.write('key', 'value') 134 | await cache.redirects.write('key', 'value') 135 | await cache.clear() 136 | 137 | assert await cache.responses.size() == 0 138 | assert await cache.redirects.size() == 0 139 | 140 | 141 | async def test_delete(): 142 | cache = CacheBackend() 143 | mock_response = get_mock_response() 144 | mock_response.history = [MagicMock(method='GET', url='test')] 145 | redirect_key = cache.create_key('GET', 'test') 146 | 147 | await cache.responses.write('key', mock_response) 148 | await cache.redirects.write(redirect_key, 'key') 149 | await cache.redirects.write('some_other_redirect', 'key') 150 | 151 | await cache.delete('key') 152 | assert await cache.responses.size() == 0 153 | assert await cache.redirects.size() == 1 154 | 155 | 156 | async def test_delete_expired_responses(): 157 | cache = CacheBackend() 158 | await cache.responses.write('request-key-1', get_mock_response(is_expired=False)) 159 | await cache.responses.write('request-key-2', get_mock_response(is_expired=True)) 160 | 161 | assert await cache.responses.size() == 2 162 | await cache.delete_expired_responses() 163 | assert await cache.responses.size() == 1 164 | 165 | 166 | async def test_delete_url(): 167 | cache = CacheBackend() 168 | mock_response = await CachedResponse.from_client_response(get_mock_response()) 169 | cache_key = cache.create_key('GET', TEST_URL, params={'param': 'value'}) 170 | 171 | await cache.responses.write(cache_key, mock_response) 172 | assert await cache.responses.size() == 1 173 | await cache.delete_url(TEST_URL, params={'param': 'value'}) 174 | assert await cache.responses.size() == 0 175 | 176 | 177 | async def test_has_url(): 178 | cache = CacheBackend() 179 | mock_response = await CachedResponse.from_client_response(get_mock_response()) 180 | cache_key = cache.create_key('GET', TEST_URL, params={'param': 'value'}) 181 | 182 | await cache.responses.write(cache_key, mock_response) 183 | assert await cache.has_url(TEST_URL, params={'param': 'value'}) 184 | assert not await cache.has_url('https://test.com/some_other_path') 185 | 186 | 187 | @patch('aiohttp_client_cache.backends.base.create_key') 188 | async def test_create_key(mock_create_key): 189 | """Actual logic is in cache_keys module; just test to make sure it gets called correctly""" 190 | headers = {'key': 'value'} 191 | ignored_params = ['ignored'] 192 | cache = CacheBackend(include_headers=True, ignored_params=ignored_params) 193 | 194 | cache.create_key('GET', 'https://test.com', headers=headers) 195 | mock_create_key.assert_called_with( 196 | 'GET', 197 | 'https://test.com', 198 | include_headers=True, 199 | ignored_params=set(ignored_params), 200 | headers=headers, 201 | ) 202 | 203 | 204 | async def test_get_urls(): 205 | cache = CacheBackend() 206 | for i in range(7): 207 | mock_response = get_mock_response(url=f'https://test.com/{i}') 208 | await cache.responses.write(f'request-key-{i}', mock_response) 209 | 210 | urls = {url async for url in cache.get_urls()} 211 | assert urls == {f'https://test.com/{i}' for i in range(7)} 212 | 213 | 214 | @pytest.mark.parametrize( 215 | 'method, status, disabled, expired, filter_return, expected_result', 216 | [ 217 | ('GET', 200, False, False, True, True), 218 | ('DELETE', 200, False, False, True, False), 219 | ('GET', 502, False, False, True, False), 220 | ('GET', 200, True, False, True, False), 221 | ('GET', 200, False, True, True, False), 222 | ('GET', 200, False, False, False, False), 223 | ], 224 | ) 225 | async def test_is_cacheable(method, status, disabled, expired, filter_return, expected_result): 226 | mock_response = get_mock_response( 227 | method=method, 228 | status=status, 229 | is_expired=expired, 230 | ) 231 | cache = CacheBackend() 232 | cache.filter_fn = lambda x: filter_return 233 | cache.disabled = disabled 234 | assert await cache.is_cacheable(mock_response) is expected_result 235 | 236 | 237 | @pytest.mark.parametrize( 238 | 'method, status, disabled, expired, body, expected_result', 239 | [ 240 | ('GET', 200, False, False, '{"content": "...", "success": true}', True), 241 | ('DELETE', 200, True, False, '{"content": "...", "success": true}', False), 242 | ('DELETE', 200, True, False, '{"content": "...", "success": false}', False), 243 | ('DELETE', 200, False, False, '{"content": "...", "success": true}', False), 244 | ('GET', 200, True, False, '{"content": "...", "success": false}', False), 245 | ('GET', 200, False, True, '{"content": "...", "success": true}', False), 246 | ], 247 | ) 248 | async def test_is_cacheable_inspect(method, status, disabled, expired, body, expected_result): 249 | async def filter(resp): 250 | json_resp = await resp.json() 251 | 252 | return json_resp['success'] 253 | 254 | mock_response = get_mock_response(method=method, status=status, is_expired=expired, _body=body) 255 | 256 | cache = CacheBackend() 257 | cache.filter_fn = filter 258 | cache.disabled = disabled 259 | assert await cache.is_cacheable(mock_response) is expected_result 260 | -------------------------------------------------------------------------------- /aiohttp_client_cache/session.py: -------------------------------------------------------------------------------- 1 | """Core functions for cache configuration""" 2 | 3 | from __future__ import annotations 4 | 5 | import sys 6 | import warnings 7 | from asyncio import Lock 8 | from contextlib import asynccontextmanager 9 | from functools import lru_cache 10 | from logging import getLogger 11 | from typing import TYPE_CHECKING, cast 12 | 13 | from aiohttp import ClientSession 14 | from aiohttp.typedefs import StrOrURL 15 | 16 | from aiohttp_client_cache.backends import CacheBackend, get_valid_kwargs 17 | from aiohttp_client_cache.cache_control import CacheActions, ExpirationTime, compose_refresh_headers 18 | from aiohttp_client_cache.response import AnyResponse, CachedResponse, set_response_defaults 19 | from aiohttp_client_cache.signatures import extend_signature 20 | 21 | if TYPE_CHECKING: 22 | from aiohttp.client import _RequestContextManager, _RequestOptions 23 | from typing_extensions import Unpack 24 | 25 | class _ExpandedRequestOptions(_RequestOptions, total=False): 26 | expire_after: ExpirationTime 27 | 28 | MIXIN_BASE = ClientSession 29 | 30 | else: 31 | MIXIN_BASE = object 32 | 33 | logger = getLogger(__name__) 34 | 35 | if sys.version_info >= (3, 10): 36 | from contextlib import nullcontext 37 | else: 38 | from contextlib import AbstractAsyncContextManager 39 | 40 | class nullcontext(AbstractAsyncContextManager): 41 | async def __aexit__(self, *excinfo): 42 | pass 43 | 44 | 45 | if sys.version_info >= (3, 11): 46 | from typing import Self 47 | else: 48 | from typing_extensions import Self 49 | 50 | 51 | @lru_cache(maxsize=16384) 52 | def _get_lock(_: int, __: str) -> Lock: 53 | return Lock() 54 | 55 | 56 | class CacheMixin(MIXIN_BASE): 57 | """A mixin class for :py:class:`aiohttp.ClientSession` that adds caching support""" 58 | 59 | @extend_signature(ClientSession.__init__) 60 | def __init__( 61 | self, 62 | base_url: StrOrURL | None = None, 63 | *, 64 | cache: CacheBackend | None = None, 65 | **kwargs, 66 | ): 67 | self.cache = cache or CacheBackend() 68 | self._null_lock = nullcontext() 69 | 70 | # Pass along any valid kwargs for ClientSession (or custom session superclass) 71 | session_kwargs = get_valid_kwargs(super().__init__, {**kwargs, 'base_url': base_url}) 72 | super().__init__(**session_kwargs) 73 | 74 | @extend_signature(ClientSession._request) 75 | async def _request( 76 | self, 77 | method: str, 78 | str_or_url: StrOrURL, 79 | expire_after: ExpirationTime = None, 80 | refresh: bool = False, 81 | **kwargs, 82 | ) -> CachedResponse: 83 | """Wrapper around :py:meth:`.SessionClient._request` that adds caching""" 84 | # Attempt to fetch cached response 85 | headers = self._prepare_headers(kwargs.get('headers', None)) 86 | kwargs['headers'] = headers 87 | key = self.cache.create_key(method, str_or_url, **kwargs) 88 | actions = self.cache.create_cache_actions( 89 | key, str_or_url, expire_after=expire_after, refresh=refresh, **kwargs 90 | ) 91 | 92 | if actions.skip_read: 93 | lock: Lock | nullcontext = self._null_lock 94 | else: 95 | lock = _get_lock(id(self), key) 96 | 97 | async with lock: 98 | response = await self.cache.request(actions) 99 | 100 | def restore_cookies(r): 101 | self.cookie_jar.update_cookies(r.cookies or {}, r.url) 102 | for redirect in r.history: 103 | self.cookie_jar.update_cookies(redirect.cookies or {}, redirect.url) 104 | 105 | if actions.revalidate and response: 106 | from_cache, new_response = await self._refresh_cached_response( 107 | method, str_or_url, response, actions, **kwargs 108 | ) 109 | if not from_cache: 110 | return set_response_defaults(new_response) 111 | else: 112 | restore_cookies(new_response) 113 | return cast(CachedResponse, new_response) 114 | 115 | # Restore any cached cookies to the session 116 | if response: 117 | restore_cookies(response) 118 | return response 119 | # If the response was missing or expired, send and cache a new request 120 | else: 121 | if actions.skip_read: 122 | logger.debug(f'Reading from cache was skipped; making request to {str_or_url}') 123 | else: 124 | logger.debug(f'Cached response not found; making request to {str_or_url}') 125 | new_response = await super()._request(method, str_or_url, **kwargs) 126 | actions.update_from_response(new_response) 127 | if await self.cache.is_cacheable(new_response, actions): 128 | await self.cache.save_response(new_response, actions.key, actions.expires) 129 | return set_response_defaults(new_response) 130 | 131 | async def _refresh_cached_response( 132 | self, 133 | method: str, 134 | str_or_url: StrOrURL, 135 | cached_response: CachedResponse, 136 | actions: CacheActions, 137 | **kwargs, 138 | ) -> tuple[bool, AnyResponse]: 139 | """Checks if the cached response is still valid using conditional requests if supported""" 140 | 141 | # check whether we can do a conditional request, 142 | # i.e. if the necessary headers are present (ETag, Last-Modified) 143 | conditional_request_supported, refresh_headers = compose_refresh_headers( 144 | kwargs.get('headers'), cached_response.headers 145 | ) 146 | 147 | if conditional_request_supported: 148 | logger.debug(f'Refreshing cached response; making request to {str_or_url}') 149 | kwargs['headers'] = refresh_headers 150 | refreshed_response = await super()._request(method, str_or_url, **kwargs) 151 | 152 | if refreshed_response.status == 304: 153 | logger.debug('Cached response not modified; returning cached response') 154 | return True, cached_response 155 | else: 156 | actions.update_from_response(refreshed_response) 157 | if await self.cache.is_cacheable(refreshed_response, actions): 158 | logger.debug('Cached response refreshed; updating cache') 159 | await self.cache.save_response(refreshed_response, actions.key, actions.expires) 160 | else: 161 | logger.debug('Cached response refreshed; deleting from cache') 162 | await self.cache.delete(actions.key) 163 | 164 | return False, refreshed_response 165 | else: 166 | logger.debug( 167 | 'Conditional requests not supported, no ETag or Last-Modified headers present; ' 168 | 'returning cached response' 169 | ) 170 | return True, cached_response 171 | 172 | async def close(self): 173 | """Close both aiohttp connector and any backend connection(s) on contextmanager exit""" 174 | await super().close() 175 | await self.cache._close_if_enabled() 176 | 177 | @asynccontextmanager 178 | async def disabled(self): 179 | """Temporarily disable the cache 180 | 181 | Example: 182 | 183 | >>> async with CachedSession() as session: 184 | >>> await session.get('http://httpbin.org/ip') 185 | >>> async with session.disabled(): 186 | >>> # Will return a new response, not a cached one 187 | >>> await session.get('http://httpbin.org/ip') 188 | """ 189 | token = self.cache._disabled.set(True) 190 | yield 191 | self.cache._disabled.reset(token) 192 | 193 | async def delete_expired_responses(self): 194 | """Remove all expired responses from the cache""" 195 | await self.cache.delete_expired_responses() 196 | 197 | # The version check is unnecessary but since aiohttp has done it we're forced into it too. 198 | if sys.version_info >= (3, 11) and TYPE_CHECKING: 199 | 200 | def get( 201 | self, 202 | url: StrOrURL, 203 | **kwargs: Unpack[_ExpandedRequestOptions], 204 | ) -> _RequestContextManager: ... 205 | 206 | def options( 207 | self, 208 | url: StrOrURL, 209 | **kwargs: Unpack[_ExpandedRequestOptions], 210 | ) -> _RequestContextManager: ... 211 | 212 | def head( 213 | self, 214 | url: StrOrURL, 215 | **kwargs: Unpack[_ExpandedRequestOptions], 216 | ) -> _RequestContextManager: ... 217 | 218 | def post( 219 | self, 220 | url: StrOrURL, 221 | **kwargs: Unpack[_ExpandedRequestOptions], 222 | ) -> _RequestContextManager: ... 223 | 224 | def put( 225 | self, 226 | url: StrOrURL, 227 | **kwargs: Unpack[_ExpandedRequestOptions], 228 | ) -> _RequestContextManager: ... 229 | 230 | def patch( 231 | self, 232 | url: StrOrURL, 233 | **kwargs: Unpack[_ExpandedRequestOptions], 234 | ) -> _RequestContextManager: ... 235 | 236 | def delete( 237 | self, 238 | url: StrOrURL, 239 | **kwargs: Unpack[_ExpandedRequestOptions], 240 | ) -> _RequestContextManager: ... 241 | 242 | 243 | # Ignore aiohttp warning: "Inheritance from ClientSession is discouraged" 244 | # Since only _request() is overridden, there is minimal chance of breakage, but still possible 245 | with warnings.catch_warnings(): 246 | warnings.simplefilter('ignore') 247 | 248 | class CachedSession(CacheMixin, ClientSession): 249 | """A drop-in replacement for :py:class:`aiohttp.ClientSession` that adds caching support 250 | 251 | Args: 252 | cache: A cache backend object. See :py:mod:`aiohttp_client_cache.backends` for 253 | options. If not provided, an in-memory cache will be used. 254 | """ 255 | 256 | async def __aenter__(self) -> Self: 257 | return self 258 | -------------------------------------------------------------------------------- /HISTORY.md: -------------------------------------------------------------------------------- 1 | # History 2 | 3 | ## 0.14.2 (2025-10-30) 4 | 5 | - Fixed `include_headers` to apply to `CachedSession.headers` 6 | - Added support for python 3.14 7 | - Packaging and project config are now handled with [uv](https://docs.astral.sh/uv/). For users, installation still works the same. For developers, see [Contributing Guide](https://aiohttp-client-cache.readthedocs.io/en/stable/contributing.html) for details. 8 | 9 | ## 0.14.1 (2025-10-01) 10 | 11 | - Update type annotation for `CachedSession.__aenter__` with `Self` type 12 | - Fix behavior of `CachedSession.disabled()` when using multiple session instances 13 | 14 | ## 0.14.0 (2025-09-22) 15 | 16 | - Fixed a race condition in SQLite backend that could lead to `sqlite3.OperationalError: database is locked` errors. 17 | - Fixed an issue where `CachedResponse.request_info` contained response headers instead of request headers. 18 | - Removed a redundant Python 4 constraint so now our users do not need to write `<4` in their "pyproject.toml" files. 19 | 20 | ## 0.13.0 (2025-04-08) 21 | 22 | - Fixed `CachedResponse.read()` to be consistent with `ClientResponse.read()` by allowing to call `read()` multiple times. (#289) 23 | - Now a warning is raised when a cache backend is accessed after disconnecting (after exiting the `CachedSession` context manager). (#241) 24 | - Fixed compatibility with `url-normalize` 2.0 (#326) 25 | - Dropped Python 3.8 support due to the EOL. 26 | 27 | ## 0.12.4 (2024-10-30) 28 | 29 | - Fixed a bug that allowed users to use `save_response()` and `from_client_response()` with an incorrect `expires` argument without throwing any warnings or errors. 30 | 31 | ## 0.12.3 (2024-10-04) 32 | 33 | - Revert some changes from `v0.12.0`, and add alternative fix for compatibility with aiohttp 3.10.6+ 34 | 35 | ## 0.12.2 (2024-10-02) 36 | 37 | - Fixed a regression in `v0.12.0` when the `request_info` property was unavailable on a cached response. (!260) 38 | 39 | ## 0.12.1 (2024-10-02) 40 | 41 | - Fixed `get_encoding()` access after unpickling. (#256) 42 | 43 | ## 0.12.0 (2024-10-01) 44 | 45 | - Add support for Python 3.13 46 | - Fix `CachedResponse.is_expired` check to consider any errors as "expired". (!252) 47 | - Fix compatibility with aiohttp 3.10.6+ (#251) 48 | - Now `CachedResponse` inherits from the `aiohttp.ClientResponse`. 49 | 50 | ## 0.11.1 (2024-08-01) 51 | 52 | - Fix compatibility with aiosqlite 0.20 53 | - Add complete type hints for `CachedSession.get()`, `post()`, etc. for compatibility with aiohttp 3.10 54 | - Remove usage of `datetime.utcnow()` (deprecated in python 3.12) 55 | 56 | ## 0.11.0 (2024-02-08) 57 | 58 | - Add support for Python 3.12. 59 | - Add a Docker Compose file with [DragonflyDB](https://www.dragonflydb.io/) service that can be used as a Redis drop-in replacement. 60 | - Add minor performance improvements for MongoDB backend. (!203) 61 | 62 | ## Deprecations and Removals 63 | 64 | - Drop support for Python 3.7. 65 | 66 | ## 0.10.0 (2023-10-30) 67 | 68 | - Add support for conditional requests with `ETag` and `Last-Modified` 69 | - If a DynamoDB item exceeds the max size (400KB), skip writing to the cache and log a warning instead of raising an error 70 | - Add `CachedResponse.closed` attribute for compatibility with `aiohttp.ClientResponse` 71 | - Close `aiosqlite` thread if it's still running when session object is deleted 72 | - Move redirects cache for `FileBackend` into same directory as cached response files 73 | - Fix issue in which `CachedSession.disabled()` prevents only cache read but not write 74 | 75 | ## 0.9.1 (2023-09-20) 76 | 77 | - Remove unintended optional dependencies in both PyPI and conda-forge packages 78 | 79 | ## 0.9.0 (2023-09-19) 80 | 81 | - Add compatibility with Sentry python SDK 82 | - Add `autoclose` option to `CacheBackend` to close backend connections when the session context exits. 83 | - Enabled by default for SQLite backend, and disabled by default for other backends. 84 | - `python-forge` is no longer required and is now an optional dependency 85 | - Fix reading response content multiple times for memory backend 86 | 87 | ## 0.8.2 (2023-07-14) 88 | 89 | - Add some missing type annotations to backend classes 90 | - Fix passing connection parameters to MongoDB backend 91 | - Revert closing backend connections on session context exit 92 | - Fix `CachedResponse.close()` method to match `ClientResponse.close()` 93 | 94 | ## 0.8.1 (2023-01-05) 95 | 96 | - For SQLite backend, close database connection on `ClientSession` context exit 97 | 98 | ## 0.8.0 (2022-12-29) 99 | 100 | - Lazily initialize and reuse SQLite connection objects 101 | - Fix `AttributeError` when using a response cached with an older version of `attrs` 102 | - Fix concurrent usage of `SQLiteCache.bulk_commit()` 103 | - Add `fast_save` option for `SQLiteCache` (`PRAGMA` setting to improve write performance, with some tradeoffs) 104 | 105 | ## 0.7.3 (2022-07-31) 106 | 107 | - Remove upper version constraint for `attrs` dependency 108 | 109 | ## 0.7.2 (2022-07-13) 110 | 111 | - Fix `TypeError` bug when using `expire_after` param with `CachedSession._request()` 112 | 113 | ## 0.7.1 (2022-06-22) 114 | 115 | - Fix possible deadlock with `SQLiteCache.init_db()` and `clear()` 116 | 117 | ## 0.7.0 (2022-05-21) 118 | 119 | [See all issues & PRs for v0.7](https://github.com/requests-cache/aiohttp-client-cache/milestone/6?closed=1) 120 | 121 | - Support manually saving a response to the cache with `CachedSession.cache.save_response()` 122 | - Add compatibility with aioboto3 0.9+ 123 | - Migrate to redis-py 4.2+ (merged with aioredis) 124 | - Add missing `aiosqlite` dependency for filesystem backend 125 | - Add missing `CachedResponse` properties derived from headers: 126 | - `charset` 127 | - `content_length` 128 | - `content_type` 129 | - Add support for async filter functions 130 | - Move repo to [requests-cache](https://github.com/requests-cache) organization 131 | 132 | ## 0.6.1 (2022-02-13) 133 | 134 | - Migrate to aioredis 2.0 135 | - Fix issue with restoring empty session cookies 136 | 137 | ## 0.6.0 (2022-02-12) 138 | 139 | [See all issues & PRs for v0.6](https://github.com/requests-cache/aiohttp-client-cache/milestone/5?closed=1) 140 | 141 | - Add a `bulk_delete()` method for all backends to improve performance of `delete_expired_responses()` 142 | - Update session cookies after fetching cached responses with cookies 143 | - Update session cookies after fetching cached responses with _redirects_ with cookies 144 | - Add support for additional request parameter types that `aiohttp` accepts: 145 | - Strings 146 | - `(key, value)` sequences 147 | - Non-`dict` `Mapping` objects 148 | - Fix URL normalization for `MultiDict` objects with duplicate keys 149 | - E.g., so `http://url.com?foo=bar&foo=baz` is cached separately from `http://url.com?foo=bar` 150 | - Update `ignored_params` to also apply to headers (if `include_headers=True`) 151 | 152 | ## 0.5.2 (2021-11-03) 153 | 154 | - Fix compatibility with aiohttp 3.8 155 | 156 | ## 0.5.1 (2021-09-10) 157 | 158 | - Fix issue with request params duplicated from request URL 159 | 160 | ## 0.5.0 (2021-09-01) 161 | 162 | [See all issues & PRs for v0.5](https://github.com/requests-cache/aiohttp-client-cache/milestone/4?closed=1) 163 | 164 | - Add a filesystem backend 165 | - Add support for streaming requests 166 | - Add `RedisBackend.close()` method 167 | - Add `MongoDBPickleCache.values()` method that deserializes items 168 | - Allow `BaseCache.has_url()` and `delete_url()` to take all the same parameters as `create_key()` 169 | - Improve normalization for variations of URLs & request parameters 170 | - Fix handling of request body when it has already been serialized 171 | - Fix bug enabling Cache-Control support by default 172 | - Add some missing no-op methods to `CachedResponse` for compatibility with `ClientResponse` 173 | 174 | --- 175 | 176 | ## 0.4.3 (2021-07-27) 177 | 178 | - Fix bug in which response header `Expires` was used for cache expiration even with `cache_control=False` 179 | - Fix bug in which HTTP dates parsed from response headers weren't converted to UTC 180 | - Add handling for invalid timestamps in `CachedResponse.is_expired` 181 | 182 | ## 0.4.2 (2021-07-26) 183 | 184 | - Fix handling of `CachedResponse.encoding` when the response body is `None` 185 | 186 | ## 0.4.1 (2021-07-09) 187 | 188 | - Fix initialziation of `SQLiteBackend` so it can be created outside main event loop 189 | 190 | ## 0.4.0 (2021-05-12) 191 | 192 | [See all issues & PRs for v0.4](https://github.com/requests-cache/aiohttp-client-cache/milestone/3?closed=1) 193 | 194 | - Add optional support for the following **request** headers: 195 | - `Cache-Control: max-age` 196 | - `Cache-Control: no-cache` 197 | - `Cache-Control: no-store` 198 | - Add optional support for the following **response** headers: 199 | - `Cache-Control: max-age` 200 | - `Cache-Control: no-store` 201 | - `Expires` 202 | - Add support for HTTP timestamps (RFC 5322) in `expire_after` parameters 203 | - Add a `use_temp` option to `SQLiteBackend` to use a tempfile 204 | - Published package on [conda-forge](https://anaconda.org/conda-forge/aiohttp-client-cache) 205 | 206 | ## 0.3.0 (2021-04-09) 207 | 208 | [See all issues & PRs for v0.3](https://github.com/requests-cache/aiohttp-client-cache/milestone/2?closed=1) 209 | 210 | - Add async implementation of DynamoDb backend 211 | - Add support for expiration for individual requests 212 | - Add support for expiration based on URL patterns 213 | - Add support for serializing/deserializing `ClientSession.links` 214 | - Add case-insensitive response headers for compatibility with aiohttp.ClientResponse.headers 215 | - Add optional integration with `itsdangerous` for safer serialization 216 | - Add `CacheBackend.get_urls()` to get all urls currently in the cache 217 | - Add some default attributes (`from_cache, is_expired`, etc.) to returned ClientResponse objects 218 | - Allow passing all backend-specific connection kwargs via CacheBackend 219 | - Add support for `json` request body 220 | - Convert all `keys()` and `values()` methods into async generators 221 | - Fix serialization of Content-Disposition 222 | - Fix filtering ignored parameters for request body (`data` and `json`) 223 | - Add user guide, more examples, and other project docs 224 | 225 | ## 0.2.0 (2021-02-28) 226 | 227 | [See all issues & PRs for v0.2](https://github.com/requests-cache/aiohttp-client-cache/milestone/1?closed=1) 228 | 229 | - Refactor SQLite backend to use `aiosqlite` for async cache operations 230 | - Refactor MongoDB backend to use `motor` for async cache operations 231 | - Refactor Redis backend to use `aiosqlite` for async cache operations 232 | - Add integration tests and `docker-compose` for local test servers 233 | 234 | ## 0.1.0 (2020-11-14) 235 | 236 | - Initial fork from [`requests-cache`](https://github.com/reclosedev/requests-cache) 237 | - First pass at a general refactor and conversion from `requests` to `aiohttp` 238 | - Basic features are functional, but some backends do not actually operate asynchronously 239 | -------------------------------------------------------------------------------- /aiohttp_client_cache/response.py: -------------------------------------------------------------------------------- 1 | # TODO: CachedResponse may be better as a non-slotted subclass of ClientResponse. 2 | # Will look into this when working on issue #67. 3 | from __future__ import annotations 4 | 5 | import asyncio 6 | import json 7 | import sys 8 | from collections.abc import Mapping 9 | from datetime import datetime 10 | from functools import singledispatch 11 | from http.cookies import SimpleCookie 12 | from logging import getLogger 13 | from typing import Any, Optional, Union 14 | from unittest.mock import Mock 15 | 16 | import attr 17 | from aiohttp import ClientResponse, ClientResponseError, hdrs, multipart 18 | from aiohttp.client_reqrep import ContentDisposition, MappingProxyType, RequestInfo 19 | from aiohttp.helpers import HeadersMixin 20 | from aiohttp.streams import StreamReader 21 | from aiohttp.typedefs import RawHeaders, StrOrURL 22 | from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy 23 | from yarl import URL 24 | 25 | from aiohttp_client_cache.cache_control import utcnow 26 | 27 | if sys.version_info >= (3, 11): 28 | from typing import Self 29 | else: 30 | from typing_extensions import Self 31 | 32 | # CachedResponse attributes to not copy directly from ClientResponse 33 | EXCLUDE_ATTRS = { 34 | '_body', 35 | '_content', 36 | '_links', 37 | 'created_at', 38 | 'encoding', 39 | 'expires', 40 | 'history', 41 | 'last_used', 42 | 'real_url', 43 | 'request_info', 44 | 'request_raw_headers', 45 | } 46 | 47 | # Default attributes to add to ClientResponse objects 48 | CACHED_RESPONSE_DEFAULTS = { 49 | 'created_at': None, 50 | 'expires': None, 51 | 'from_cache': False, 52 | 'is_expired': False, 53 | } 54 | 55 | JsonResponse = Optional[dict[str, Any]] 56 | DictItems = list[tuple[str, str]] 57 | LinkItems = list[tuple[str, DictItems]] 58 | LinkMultiDict = MultiDictProxy[MultiDictProxy[Union[str, URL]]] 59 | 60 | logger = getLogger(__name__) 61 | 62 | 63 | class UnsupportedExpiresError(Exception): 64 | def __init__(self, expires: datetime): 65 | super().__init__(f'Expected a naive datetime, but got {expires}.') 66 | 67 | 68 | @attr.s(slots=True) 69 | class CachedResponse(HeadersMixin): 70 | """A dataclass containing cached response information, used for serialization. 71 | It will mostly behave the same as a :py:class:`aiohttp.ClientResponse` that has been read, 72 | with some additional cache-related info. 73 | """ 74 | 75 | method: str = attr.ib() 76 | reason: str = attr.ib() 77 | status: int = attr.ib() 78 | url: URL = attr.ib(converter=URL) 79 | version: str = attr.ib() 80 | _body: Any = attr.ib(default=None) 81 | _content: StreamReader | None = attr.ib(default=None) 82 | _links: LinkItems = attr.ib(factory=list) 83 | cookies: SimpleCookie = attr.ib(factory=SimpleCookie) 84 | created_at: datetime = attr.ib(factory=utcnow) 85 | encoding: str = attr.ib(default='utf-8') 86 | expires: datetime | None = attr.ib(default=None) 87 | raw_headers: RawHeaders = attr.ib(factory=tuple) 88 | request_raw_headers: RawHeaders = attr.ib(factory=tuple) 89 | real_url: StrOrURL = attr.ib(default=None) 90 | history: tuple = attr.ib(factory=tuple) 91 | last_used: datetime = attr.ib(factory=utcnow) 92 | 93 | @classmethod 94 | async def from_client_response( 95 | cls, client_response: ClientResponse, expires: datetime | None = None 96 | ): 97 | """Convert a ClientResponse into a CachedResponse""" 98 | if expires is not None and expires.tzinfo is not None: 99 | # An unrecoverable error (wrong library low-level API usage). 100 | raise UnsupportedExpiresError(expires) 101 | 102 | # Copy most attributes over as is 103 | copy_attrs = set(attr.fields_dict(cls).keys()) - EXCLUDE_ATTRS 104 | response = cls(**{k: getattr(client_response, k) for k in copy_attrs}) 105 | 106 | # Read response content, and reset StreamReader on original response 107 | if not client_response._released: 108 | await client_response.read() 109 | response._body = client_response._body 110 | client_response.content = CachedStreamReader(client_response._body) 111 | 112 | # Set remaining attributes individually. 113 | response.expires = expires 114 | response.links = client_response.links 115 | response.real_url = client_response.request_info.real_url 116 | response.request_raw_headers = tuple( 117 | (k.encode('utf-8'), v.encode('utf-8')) 118 | for k, v in client_response.request_info.headers.items() 119 | ) 120 | 121 | # The encoding may be unset even if the response has been read, and 122 | # get_encoding() does not handle certain edge cases like an empty response body 123 | try: 124 | response.encoding = client_response.get_encoding() 125 | except (RuntimeError, TypeError): 126 | pass 127 | 128 | if client_response.history: 129 | response.history = ( 130 | *[await cls.from_client_response(r) for r in client_response.history], 131 | ) 132 | return response 133 | 134 | @property 135 | def content(self) -> StreamReader: 136 | if self._content is None: 137 | self._content = CachedStreamReader(self._body) 138 | return self._content 139 | 140 | @content.setter 141 | def content(self, value: StreamReader): 142 | self._content = value 143 | 144 | @property 145 | def content_disposition(self) -> ContentDisposition | None: 146 | """Get Content-Disposition headers, if any""" 147 | raw = self.headers.get(hdrs.CONTENT_DISPOSITION) 148 | if raw is None: 149 | return None 150 | disposition_type, params_dct = multipart.parse_content_disposition(raw) 151 | params = MappingProxyType(params_dct) 152 | filename = multipart.content_disposition_filename(params) 153 | return ContentDisposition(disposition_type, params, filename) 154 | 155 | @property 156 | def from_cache(self): 157 | return True 158 | 159 | @property 160 | def _headers(self) -> CIMultiDictProxy[str]: # type: ignore[override] 161 | return self.headers 162 | 163 | @property 164 | def headers(self) -> CIMultiDictProxy[str]: 165 | """Get headers as an immutable, case-insensitive multidict from raw headers""" 166 | 167 | def decode_header(header): 168 | """Decode an individual (key, value) pair""" 169 | return ( 170 | header[0].decode('utf-8', 'surrogateescape'), 171 | header[1].decode('utf-8', 'surrogateescape'), 172 | ) 173 | 174 | return CIMultiDictProxy(CIMultiDict([decode_header(h) for h in self.raw_headers])) 175 | 176 | @property 177 | def host(self) -> str: 178 | return self.url.host or '' 179 | 180 | @property 181 | def is_expired(self) -> bool: 182 | """Determine if this cached response is expired""" 183 | return self.expires is not None and utcnow() > self.expires 184 | 185 | @property 186 | def links(self) -> MultiDictProxy: 187 | """Convert stored links into the format returned by :attr:`ClientResponse.links`""" 188 | items = [(k, _to_url_multidict(v)) for k, v in self._links] 189 | return MultiDictProxy(MultiDict([(k, MultiDictProxy(v)) for k, v in items])) 190 | 191 | @links.setter 192 | def links(self, value: Mapping): 193 | self._links = [(k, _to_str_tuples(v)) for k, v in value.items()] 194 | 195 | @property 196 | def ok(self) -> bool: 197 | """Returns ``True`` if ``status`` is less than ``400``, ``False`` if not""" 198 | return self.status < 400 199 | 200 | @property 201 | def request_headers(self) -> CIMultiDictProxy[str]: 202 | """Get request headers as an immutable, case-insensitive multidict from raw headers""" 203 | 204 | def decode_header(header): 205 | """Decode an individual (key, value) pair""" 206 | return ( 207 | header[0].decode('utf-8', 'surrogateescape'), 208 | header[1].decode('utf-8', 'surrogateescape'), 209 | ) 210 | 211 | return CIMultiDictProxy(CIMultiDict([decode_header(h) for h in self.request_raw_headers])) 212 | 213 | @property 214 | def request_info(self) -> RequestInfo: 215 | return RequestInfo( 216 | url=URL(self.url), 217 | method=self.method, 218 | headers=self.request_headers, 219 | real_url=URL(str(self.real_url)), 220 | ) 221 | 222 | def get_encoding(self): 223 | return self.encoding 224 | 225 | async def json(self, encoding: str | None = None, **kwargs) -> dict[str, Any] | None: 226 | """Read and decode JSON response""" 227 | stripped = self._body.strip() 228 | if not stripped: 229 | return None 230 | return json.loads(stripped.decode(encoding or self.encoding)) 231 | 232 | def raise_for_status(self): 233 | if self.status >= 400: 234 | raise ClientResponseError( 235 | self.request_info, 236 | self.history, 237 | status=self.status, 238 | message=self.reason, 239 | headers=self.headers, 240 | ) 241 | 242 | async def read(self) -> bytes: 243 | """Read response payload.""" 244 | # We are ready to return the body because `read()` 245 | # was called on a `CachedResponse` creation. 246 | return self._body 247 | 248 | def reset(self): 249 | """Reset the stream reader to re-read a streamed response""" 250 | self._content = None 251 | 252 | async def text(self, encoding: str | None = None, errors: str = 'strict') -> str: 253 | """Read response payload and decode""" 254 | return self._body.decode(encoding or self.encoding, errors=errors) 255 | 256 | # No-op/placeholder properties and methods that don't apply to a CachedResponse, but provide 257 | # compatibility with aiohttp.ClientResponse 258 | # ---------- 259 | 260 | @property 261 | def _released(self): 262 | return True 263 | 264 | @property 265 | def connection(self): 266 | return None 267 | 268 | async def __aenter__(self) -> Self: 269 | return self 270 | 271 | async def __aexit__(self, *exc: Any) -> None: 272 | pass 273 | 274 | @property 275 | def closed(self) -> bool: 276 | return True 277 | 278 | def close(self): 279 | pass 280 | 281 | async def wait_for_close(self): 282 | pass 283 | 284 | def release(self): 285 | pass 286 | 287 | async def start(self): 288 | pass 289 | 290 | async def terminate(self): 291 | pass 292 | 293 | 294 | class CachedStreamReader(StreamReader): 295 | """A StreamReader loaded from previously consumed response content. This feeds cached data into 296 | the stream so it can support all the same behavior as the original stream: async iteration, 297 | chunked reads, etc. 298 | """ 299 | 300 | def __init__(self, body: bytes | None = None): 301 | body = body or b'' 302 | protocol = Mock(_reading_paused=False) 303 | super().__init__(protocol, limit=len(body), loop=asyncio.get_event_loop()) 304 | self.feed_data(body) 305 | self.feed_eof() 306 | 307 | 308 | AnyResponse = Union[ClientResponse, CachedResponse] 309 | 310 | 311 | @singledispatch 312 | def set_response_defaults(response): 313 | raise NotImplementedError 314 | 315 | 316 | @set_response_defaults.register 317 | def _(response: CachedResponse) -> CachedResponse: 318 | return response 319 | 320 | 321 | @set_response_defaults.register 322 | def _(response: ClientResponse) -> ClientResponse: 323 | """Set some default CachedResponse values on a ClientResponse object, so they can be 324 | expected to always be present 325 | """ 326 | for k, v in CACHED_RESPONSE_DEFAULTS.items(): 327 | setattr(response, k, v) 328 | return response 329 | 330 | 331 | def _to_str_tuples(data: Mapping) -> DictItems: 332 | return [(k, str(v)) for k, v in data.items()] 333 | 334 | 335 | def _to_url_multidict(data: DictItems) -> MultiDict: 336 | return MultiDict([(k, URL(url)) for k, url in data]) 337 | -------------------------------------------------------------------------------- /aiohttp_client_cache/backends/sqlite.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | import functools 5 | import sqlite3 6 | import warnings 7 | from collections.abc import AsyncIterable, AsyncIterator 8 | from contextlib import asynccontextmanager 9 | from contextvars import ContextVar 10 | from logging import getLogger 11 | from os import makedirs 12 | from os.path import abspath, basename, dirname, expanduser, isabs, join 13 | from pathlib import Path 14 | from tempfile import gettempdir 15 | from typing import Any 16 | 17 | import aiosqlite 18 | 19 | from aiohttp_client_cache.backends import BaseCache, CacheBackend, ResponseOrKey, get_valid_kwargs 20 | 21 | bulk_commit_var: ContextVar[bool] = ContextVar('bulk_commit', default=False) 22 | logger = getLogger(__name__) 23 | 24 | 25 | closed_session_warning = functools.partial( 26 | warnings.warn, 27 | 'Cache access after closing the `Cachedsession` context manager ' 28 | + 'is discouraged and can be forbidden in the future to prevent ' 29 | + 'errors related to a closed database connection. Use `autoclose=False` ' 30 | + 'if you are managing the cache backend connection on your side or ' 31 | + 'reusing a single cache class instance in multiple `CachedSession`.', 32 | stacklevel=2, 33 | ) 34 | 35 | 36 | class SQLiteBackend(CacheBackend): 37 | """Async cache backend for `SQLite `_ 38 | 39 | Notes: 40 | * Requires `aiosqlite `_ 41 | * Accepts keyword arguments for :py:func:`sqlite3.connect` / :py:func:`aiosqlite.connect` 42 | * The path to the database file will be ```` (or ``.sqlite`` if no 43 | file extension is specified) 44 | 45 | Args: 46 | cache_name: Database filename 47 | use_temp: Store database in a temp directory (e.g., ``/tmp/http_cache.sqlite``). 48 | Note: if ``cache_name`` is an absolute path, this option will be ignored. 49 | fast_save: Increase cache write performance, but with the possibility of data loss. See 50 | `pragma: synchronous `_ for 51 | details. 52 | autoclose: Close any active backend connections when the session is closed 53 | kwargs: Additional keyword arguments for :py:class:`.CacheBackend` or backend connection 54 | """ 55 | 56 | def __init__( 57 | self, 58 | cache_name: str = 'aiohttp-cache', 59 | use_temp: bool = False, 60 | fast_save: bool = False, 61 | autoclose: bool = True, 62 | **kwargs: Any, 63 | ): 64 | super().__init__(cache_name=cache_name, autoclose=autoclose, **kwargs) 65 | self.responses = SQLitePickleCache( 66 | cache_name, 'responses', use_temp=use_temp, fast_save=fast_save, **kwargs 67 | ) 68 | self.redirects = SQLiteCache( 69 | cache_name, 70 | 'redirects', 71 | use_temp=use_temp, 72 | connection=self.responses._connection, 73 | lock=self.responses._lock, 74 | **kwargs, 75 | ) 76 | 77 | 78 | class SQLiteCache(BaseCache): 79 | """An async interface for caching objects in a SQLite database. 80 | 81 | Example: 82 | 83 | >>> # Store data in two tables under the 'testdb' database 84 | >>> d1 = SQLiteCache('testdb', 'table1') 85 | >>> d2 = SQLiteCache('testdb', 'table2') 86 | 87 | Args: 88 | filename: Database filename 89 | table_name: Table name 90 | use_temp: Store database in a temp directory (e.g., ``/tmp/http_cache.sqlite``). 91 | Note: if ``cache_name`` is an absolute path, this option will be ignored. 92 | connection: Existing aiosqlite connection to reuse 93 | lock: Existing async lock to reuse 94 | kwargs: Additional keyword arguments for :py:func:`sqlite3.connect` 95 | """ 96 | 97 | def __init__( 98 | self, 99 | filename: str, 100 | table_name: str = 'aiohttp-cache', 101 | use_temp: bool = False, 102 | fast_save: bool = False, 103 | connection: aiosqlite.Connection | None = None, 104 | lock: asyncio.Lock | None = None, 105 | **kwargs: Any, 106 | ): 107 | super().__init__(**kwargs) 108 | self.fast_save = fast_save 109 | self.filename = _get_cache_filename(filename, use_temp) 110 | self.table_name = table_name 111 | 112 | # Create a connection object, but delay actually connecting until the first request 113 | connection_kwargs = get_valid_kwargs(sqlite_template, kwargs) 114 | self._connection = connection or aiosqlite.connect(self.filename, **connection_kwargs) 115 | self._lock = lock or asyncio.Lock() 116 | self._initialized = False 117 | 118 | @asynccontextmanager 119 | async def get_connection(self, commit: bool = False) -> AsyncIterator[aiosqlite.Connection]: 120 | """Wrapper around aiosqlite connection to ensure it and the database are initialized""" 121 | # Note: aiosqlite.Connection is a Thread subclass. Awaiting it will start the thread, 122 | # set an internal _connection attribute (sqlite3.Connection), and return itself. 123 | # Doing this here delays the actual connection until the first request, and allows sharing 124 | # the connection object between multiple SQLiteCache instances. 125 | async with self._lock: 126 | if self._connection._connection is None: 127 | self._connection = await self._connection 128 | # If reusing an existing connection, database may not be initialized yet 129 | if not self._initialized: 130 | await self._init_db() 131 | 132 | yield self._connection 133 | 134 | if self._closed: 135 | closed_session_warning() 136 | if commit and not bulk_commit_var.get(): 137 | await self._connection.commit() 138 | 139 | async def _init_db(self): 140 | """Initialize the database, if it hasn't already been""" 141 | if self.fast_save: 142 | await self._connection.execute('PRAGMA synchronous = 0;') 143 | await self._connection.execute( 144 | f'CREATE TABLE IF NOT EXISTS `{self.table_name}` (key PRIMARY KEY, value)' 145 | ) 146 | self._initialized = True 147 | 148 | def __del__(self): 149 | """If the aiosqlite connection is still open when this object is deleted, force its thread 150 | to close by stopping its internal queue. This is basically a last resort to avoid hanging 151 | the application if this backend is used without the CachedSession contextmanager. 152 | 153 | Note: Since this uses internal attributes, it has the potential to break in future versions 154 | of aiosqlite. 155 | """ 156 | if self._connection is not None: 157 | try: 158 | self._connection._stop_running() 159 | except (AttributeError, TypeError): 160 | logger.warning('Could not close SQLite connection thread', exc_info=True) 161 | self._connection = None 162 | 163 | @asynccontextmanager 164 | async def bulk_commit(self): 165 | """Contextmanager to more efficiently write a large number of records at once 166 | 167 | Example: 168 | 169 | >>> cache = SQLiteCache('test') 170 | >>> async with cache.bulk_commit(): 171 | ... for i in range(1000): 172 | ... await cache.write(f'key_{i}', str(i)) 173 | 174 | """ 175 | bulk_commit_var.set(True) 176 | try: 177 | yield 178 | await self._connection.commit() 179 | finally: 180 | bulk_commit_var.set(False) 181 | 182 | async def clear(self): 183 | async with self.get_connection(commit=True) as db, self._lock: 184 | await db.execute(f'DROP TABLE `{self.table_name}`') 185 | await db.execute('VACUUM') 186 | await self._init_db() 187 | 188 | async def close(self): 189 | """Close any open connections""" 190 | self._closed = True 191 | async with self._lock: 192 | if self._connection is not None: 193 | await self._connection.close() 194 | self._connection = None 195 | 196 | async def contains(self, key: str) -> bool: 197 | async with self.get_connection() as db: 198 | cursor = await db.execute( 199 | f'SELECT COUNT(*) FROM `{self.table_name}` WHERE key=?', (key,) 200 | ) 201 | row = await cursor.fetchone() 202 | return bool(row[0]) if row else False 203 | 204 | async def bulk_delete(self, keys: set): 205 | async with self.get_connection(commit=True) as db: 206 | placeholders = ', '.join('?' for _ in keys) 207 | await db.execute( 208 | f'DELETE FROM `{self.table_name}` WHERE key IN ({placeholders})', 209 | tuple(keys), 210 | ) 211 | 212 | async def delete(self, key: str): 213 | async with self.get_connection(commit=True) as db: 214 | await db.execute(f'DELETE FROM `{self.table_name}` WHERE key=?', (key,)) 215 | 216 | async def keys(self) -> AsyncIterable[str]: 217 | async with self.get_connection() as db: 218 | async with db.execute(f'SELECT key FROM `{self.table_name}`') as cursor: 219 | async for row in cursor: 220 | yield row[0] 221 | 222 | async def read(self, key: str) -> ResponseOrKey: 223 | async with self.get_connection() as db: 224 | if self._closed: 225 | closed_session_warning() 226 | cursor = await db.execute(f'SELECT value FROM `{self.table_name}` WHERE key=?', (key,)) 227 | row = await cursor.fetchone() 228 | return row[0] if row else None 229 | 230 | async def size(self) -> int: 231 | async with self.get_connection() as db: 232 | cursor = await db.execute(f'SELECT COUNT(key) FROM `{self.table_name}`') 233 | row = await cursor.fetchone() 234 | return row[0] if row else 0 235 | 236 | async def values(self) -> AsyncIterable[ResponseOrKey]: 237 | async with self.get_connection() as db: 238 | async with db.execute(f'SELECT value FROM `{self.table_name}`') as cursor: 239 | async for row in cursor: 240 | yield row[0] 241 | 242 | async def write(self, key: str, item: ResponseOrKey | sqlite3.Binary): 243 | async with self.get_connection(commit=True) as db: 244 | await db.execute( 245 | f'INSERT OR REPLACE INTO `{self.table_name}` (key,value) VALUES (?,?)', 246 | (key, item), 247 | ) 248 | 249 | 250 | class SQLitePickleCache(SQLiteCache): 251 | """Same as :py:class:`SqliteCache`, but pickles values before saving""" 252 | 253 | async def read(self, key: str) -> ResponseOrKey: 254 | return self.deserialize(await super().read(key)) 255 | 256 | async def values(self) -> AsyncIterable[ResponseOrKey]: 257 | async with self.get_connection() as db: 258 | async with db.execute(f'select value from `{self.table_name}`') as cursor: 259 | async for row in cursor: 260 | yield self.deserialize(row[0]) 261 | 262 | async def write(self, key, item): 263 | await super().write(key, sqlite3.Binary(self.serialize(item))) # type: ignore[arg-type] 264 | 265 | 266 | def sqlite_template( 267 | timeout: float = 5.0, 268 | detect_types: int = 0, 269 | isolation_level: str | None = None, 270 | check_same_thread: bool = True, 271 | factory: type | None = None, 272 | cached_statements: int = 100, 273 | uri: bool = False, 274 | ): 275 | """Template function to get an accurate function signature for :py:func:`sqlite3.connect`""" 276 | 277 | 278 | def _get_cache_filename(filename: Path | str, use_temp: bool) -> str: 279 | """Get resolved path for database file""" 280 | # Save to a temp directory, if specified 281 | if use_temp and not isabs(filename): 282 | filename = join(gettempdir(), filename) 283 | 284 | # Expand relative and user paths (~/*), and add file extension if not specified 285 | filename = abspath(expanduser(str(filename))) 286 | if '.' not in basename(filename): 287 | filename += '.sqlite' 288 | 289 | # Make sure parent dirs exist 290 | makedirs(dirname(filename), exist_ok=True) 291 | return filename 292 | -------------------------------------------------------------------------------- /aiohttp_client_cache/cache_control.py: -------------------------------------------------------------------------------- 1 | """Utilities for determining cache expiration and other cache actions""" 2 | 3 | from __future__ import annotations 4 | 5 | import sys 6 | from collections.abc import Mapping 7 | from datetime import datetime, timedelta, timezone 8 | from email.utils import parsedate_to_datetime 9 | from fnmatch import fnmatch 10 | from functools import singledispatch 11 | from itertools import chain 12 | from logging import getLogger 13 | from typing import Any, NoReturn, Union 14 | 15 | from aiohttp import ClientResponse 16 | from aiohttp.typedefs import StrOrURL 17 | from attr import define, field 18 | from multidict import CIMultiDict 19 | 20 | # Value that may be set by either Cache-Control headers or CacheBackend params to disable caching 21 | DO_NOT_CACHE = 0 22 | 23 | # Currently supported Cache-Control directives 24 | CACHE_DIRECTIVES = ['max-age', 'no-cache', 'no-store'] 25 | 26 | # All cache-related headers, for logging/reference; not all are supported 27 | REQUEST_CACHE_HEADERS = [ 28 | 'Cache-Control', 29 | 'If-Unmodified-Since', 30 | 'If-Modified-Since', 31 | 'If-Match', 32 | 'If-None-Match', 33 | ] 34 | RESPONSE_CACHE_HEADERS = ['Cache-Control', 'ETag', 'Expires', 'Age'] 35 | 36 | CacheDirective = tuple[str, Union[None, int, bool]] 37 | ExpirationTime = Union[None, int, float, str, datetime, timedelta] 38 | ExpirationPatterns = dict[str, ExpirationTime] 39 | logger = getLogger(__name__) 40 | 41 | 42 | @define() 43 | class CacheActions: 44 | """A dataclass that contains info on specific actions to take for a given cache item. 45 | This is determined by a combination of CacheBackend settings and request + response headers. 46 | If multiple sources are provided, they will be used in the following order of precedence: 47 | 48 | 1. Cache-Control request headers (if enabled) 49 | 2. Cache-Control response headers (if enabled) 50 | 3. Per-request expiration 51 | 4. Per-URL expiration 52 | 5. Per-session expiration 53 | """ 54 | 55 | cache_control: bool = field(default=False) 56 | expire_after: ExpirationTime = field(default=None) 57 | key: str = field(default=None) 58 | revalidate: bool = field(default=False) # Note: Revalidation is not currently implemented 59 | skip_read: bool = field(default=False) 60 | skip_write: bool = field(default=False) 61 | 62 | @classmethod 63 | def from_request( 64 | cls, 65 | key: str, 66 | cache_control: bool = False, 67 | cache_disabled: bool = False, 68 | refresh: bool = False, 69 | headers: Mapping | None = None, 70 | **kwargs, 71 | ): 72 | """Initialize from request info and CacheBackend settings""" 73 | if cache_disabled: 74 | return cls(key=key, skip_read=True, skip_write=True) 75 | else: 76 | headers = headers or {} 77 | if cache_control and has_cache_headers(headers): 78 | return cls.from_headers(key, headers) 79 | else: 80 | return cls.from_settings( 81 | key, cache_control=cache_control, refresh=refresh, **kwargs 82 | ) 83 | 84 | @classmethod 85 | def from_headers(cls, key: str, headers: Mapping): 86 | """Initialize from request headers""" 87 | directives = get_cache_directives(headers) 88 | do_not_cache = directives.get('max-age') == DO_NOT_CACHE 89 | return cls( 90 | cache_control=True, 91 | key=key, 92 | expire_after=directives.get('max-age'), 93 | skip_read=do_not_cache or 'no-store' in directives or 'no-cache' in directives, 94 | skip_write=do_not_cache or 'no-store' in directives, 95 | revalidate=False, 96 | ) 97 | 98 | @classmethod 99 | def from_settings( 100 | cls, 101 | key: str, 102 | url: StrOrURL, 103 | cache_control: bool = False, 104 | refresh: bool = False, 105 | request_expire_after: ExpirationTime = None, 106 | session_expire_after: ExpirationTime = None, 107 | urls_expire_after: ExpirationPatterns | None = None, 108 | **kwargs, 109 | ): 110 | """Initialize from CacheBackend settings""" 111 | # Check expire_after values in order of precedence 112 | expire_after = coalesce( 113 | request_expire_after, 114 | get_url_expiration(url, urls_expire_after), 115 | session_expire_after, 116 | ) 117 | 118 | do_not_cache = expire_after == DO_NOT_CACHE 119 | return cls( 120 | cache_control=cache_control, 121 | key=key, 122 | expire_after=expire_after, 123 | skip_read=do_not_cache, 124 | skip_write=do_not_cache, 125 | revalidate=refresh and not do_not_cache, 126 | ) 127 | 128 | @property 129 | def expires(self) -> datetime | None: 130 | """Convert the user/header-provided expiration value to a datetime""" 131 | return get_expiration_datetime(self.expire_after) 132 | 133 | def update_from_response(self, response: ClientResponse): 134 | """Update expiration + actions based on response headers, if not previously set by request""" 135 | if not self.cache_control: 136 | return 137 | 138 | directives = get_cache_directives(response.headers) 139 | do_not_cache = directives.get('max-age') == DO_NOT_CACHE 140 | self.expire_after = coalesce( 141 | self.expires, directives.get('max-age'), directives.get('expires') 142 | ) 143 | self.skip_write = self.skip_write or do_not_cache or 'no-store' in directives 144 | self.revalidate = self.revalidate or do_not_cache 145 | 146 | 147 | def coalesce(*values: Any, default=None) -> Any: 148 | """Get the first non-``None`` value in a list of values""" 149 | return next((v for v in values if v is not None), default) 150 | 151 | 152 | def get_expiration_datetime(expire_after: ExpirationTime) -> datetime | None: 153 | """Convert an expiration value in any supported format to an absolute datetime""" 154 | logger.debug(f'Determining expiration time based on: {expire_after}') 155 | if isinstance(expire_after, str): 156 | expire_after = parse_http_date(expire_after) 157 | if expire_after is None or expire_after == -1: 158 | return None 159 | if isinstance(expire_after, datetime): 160 | return convert_to_utc_naive(expire_after) 161 | 162 | if not isinstance(expire_after, timedelta): 163 | assert isinstance(expire_after, (int, float)) 164 | expire_after = timedelta(seconds=expire_after) 165 | return utcnow() + expire_after 166 | 167 | 168 | def get_cache_directives(headers: Mapping) -> dict: 169 | """Get all Cache-Control directives, and handle multiple headers and comma-separated lists""" 170 | if not headers: 171 | return {} 172 | if not hasattr(headers, 'getall'): 173 | headers = CIMultiDict(headers) 174 | 175 | header_values = headers.getall('Cache-Control', []) 176 | cache_directives = [v.split(',') for v in header_values if v] 177 | cache_directives = list(chain.from_iterable(cache_directives)) 178 | kv_directives = dict([split_kv_directive(value) for value in cache_directives]) 179 | 180 | if 'Expires' in headers: 181 | kv_directives['expires'] = headers.getone('Expires') # type: ignore 182 | return kv_directives 183 | 184 | 185 | def get_url_expiration( 186 | url: StrOrURL, urls_expire_after: ExpirationPatterns | None = None 187 | ) -> ExpirationTime: 188 | """Check for a matching per-URL expiration, if any""" 189 | for pattern, expire_after in (urls_expire_after or {}).items(): 190 | if url_match(url, pattern): 191 | logger.debug(f'URL {url} matched pattern "{pattern}": {expire_after}') 192 | return expire_after 193 | return None 194 | 195 | 196 | def has_cache_headers(headers: Mapping) -> bool: 197 | """Determine if headers contain cache directives **that we currently support**""" 198 | ci_headers = CIMultiDict(headers) 199 | cache_control = ','.join(ci_headers.getall('Cache-Control', [])) 200 | return any([d in cache_control for d in CACHE_DIRECTIVES] + [bool(headers.get('Expires'))]) 201 | 202 | 203 | def compose_refresh_headers( 204 | request_headers: Mapping | None, cached_headers: Mapping 205 | ) -> tuple[bool, Mapping]: 206 | """Returns headers containing directives for conditional requests if the cached headers support it""" 207 | refresh_headers = dict(request_headers) if request_headers is not None else {} 208 | conditional_request_supported = False 209 | 210 | if 'ETag' in cached_headers: 211 | refresh_headers['If-None-Match'] = cached_headers['ETag'] 212 | conditional_request_supported = True 213 | 214 | if 'Last-Modified' in cached_headers: 215 | refresh_headers['If-Modified-Since'] = cached_headers['Last-Modified'] 216 | conditional_request_supported = True 217 | 218 | return conditional_request_supported, refresh_headers 219 | 220 | 221 | if sys.version_info >= (3, 10): 222 | 223 | def parse_http_date(value: str) -> datetime | None: 224 | """Attempt to parse an HTTP (RFC 5322-compatible) timestamp""" 225 | try: 226 | return parsedate_to_datetime(value) 227 | except ValueError: 228 | logger.debug(f'Failed to parse timestamp: {value}') 229 | return None 230 | 231 | else: # pragma: no cover 232 | 233 | def parse_http_date(value: str) -> datetime | None: 234 | """Attempt to parse an HTTP (RFC 5322-compatible) timestamp""" 235 | try: 236 | return parsedate_to_datetime(value) 237 | except (ValueError, TypeError): 238 | logger.debug(f'Failed to parse timestamp: {value}') 239 | return None 240 | 241 | 242 | def split_kv_directive(header_value: str) -> CacheDirective: 243 | """Split a cache directive into a ``(header_value, int)`` key-value pair, if possible; 244 | otherwise just ``(header_value, True)``. 245 | """ 246 | header_value = header_value.strip() 247 | if '=' in header_value: 248 | k, v = header_value.split('=', 1) 249 | return k, try_int(v) 250 | else: 251 | return header_value, True 252 | 253 | 254 | def convert_to_utc_naive(dt: datetime): 255 | """All internal datetimes are UTC and timezone-naive. Convert any user/header-provided 256 | datetimes to the same format. 257 | """ 258 | if dt.tzinfo: 259 | dt.astimezone(timezone.utc) 260 | dt = dt.replace(tzinfo=None) 261 | return dt 262 | 263 | 264 | # TODO: This could be replaced with timezone-aware datetimes, but this will cause problems with 265 | # existing cache data. It would be best to do this at the same time as a release that includes 266 | # changes to request matching logic (i.e., new cache keys). 267 | def utcnow() -> datetime: 268 | """Get the current time in UTC, as a timezone-naive datetime""" 269 | return datetime.now(timezone.utc).replace(tzinfo=None) 270 | 271 | 272 | @singledispatch 273 | def try_int(value): 274 | raise NotImplementedError 275 | 276 | 277 | @try_int.register 278 | def _(value: None) -> None: 279 | return value 280 | 281 | 282 | @try_int.register 283 | def _(value: int) -> int: 284 | return value 285 | 286 | 287 | @try_int.register 288 | def _(value: float) -> NoReturn: 289 | # Make sure that we do not inadvertently process a supertype of `int`. 290 | raise TypeError 291 | 292 | 293 | @try_int.register 294 | def _(value: bool) -> NoReturn: 295 | # Make sure that we do not inadvertently process a supertype of `int`. 296 | raise TypeError 297 | 298 | 299 | @try_int.register 300 | def _(value: str): 301 | try: 302 | return int(value) 303 | except ValueError: 304 | return None 305 | 306 | 307 | def url_match(url: StrOrURL, pattern: StrOrURL) -> bool: 308 | """Determine if a URL matches a pattern 309 | 310 | Args: 311 | url: URL to test. Its base URL (without protocol) will be used. 312 | pattern: Glob pattern to match against. A recursive wildcard will be added if not present 313 | 314 | Example: 315 | >>> url_match('https://httpbin.org/delay/1', 'httpbin.org/delay') 316 | True 317 | >>> url_match('https://httpbin.org/stream/1', 'httpbin.org/*/1') 318 | True 319 | >>> url_match('https://httpbin.org/stream/2', 'httpbin.org/*/1') 320 | False 321 | """ 322 | if not url: 323 | return False 324 | url = str(url).split('://')[-1] 325 | pattern = str(pattern).split('://')[-1].rstrip('*') + '**' 326 | return fnmatch(url, pattern) 327 | -------------------------------------------------------------------------------- /docs/user_guide.md: -------------------------------------------------------------------------------- 1 | # User Guide 2 | 3 | This section covers the main features of aiohttp-client-cache. 4 | 5 | ## Installation 6 | 7 | Install with pip: 8 | 9 | ```bash 10 | pip install aiohttp-client-cache 11 | ``` 12 | 13 | Or with conda, if you prefer: 14 | 15 | ```bash 16 | conda install -c conda-forge aiohttp-client-cache 17 | ``` 18 | 19 | ### Requirements 20 | 21 | - Requires python 3.9+. 22 | - You may need additional dependencies depending on which backend you want to use. To install with 23 | extra dependencies for all supported {ref}`backends`: 24 | ```bash 25 | pip install aiohttp-client-cache[all] 26 | ``` 27 | 28 | ### Optional Setup Steps 29 | 30 | - See {ref}`security` for recommended setup steps for more secure cache serialization. 31 | - See {ref}`Contributing Guide ` for setup steps for local development. 32 | 33 | ## General Usage 34 | 35 | {py:class}`.CachedSession` can be used as a drop-in replacement for {py:class}`aiohttp.ClientSession`. 36 | Basic usage looks like this: 37 | 38 | ```python 39 | >>> from aiohttp_client_cache import CachedSession 40 | >>> 41 | >>> async with CachedSession() as session: 42 | >>> await session.get('http://httpbin.org/delay/1') 43 | ``` 44 | 45 | Any {py:class}`~aiohttp.ClientSession` method can be used (but see {ref}`user_guide:http methods` section 46 | below for config details): 47 | 48 | ```python 49 | >>> await session.request('GET', 'http://httpbin.org/get') 50 | >>> await session.head('http://httpbin.org/get') 51 | ``` 52 | 53 | Caching can be temporarily disabled with {py:meth}`.CachedSession.disabled`: 54 | 55 | ```python 56 | >>> async with session.disabled(): 57 | ... await session.get('http://httpbin.org/get') 58 | ``` 59 | 60 | The best way to clean up your cache is through {ref}`user_guide:cache expiration`, but you can also 61 | clear out everything at once with {py:meth}`.CacheBackend.clear`: 62 | 63 | ```python 64 | >>> await session.cache.clear() 65 | ``` 66 | 67 | ## Cache Options 68 | 69 | A number of options are available to modify which responses are cached and how they are cached. 70 | 71 | ### HTTP Methods 72 | 73 | By default, only GET and HEAD requests are cached. To cache additional HTTP methods, specify them 74 | with `allowed_methods`. For example, caching POST requests can be used to ensure you don't send 75 | the same data multiple times: 76 | 77 | ```python 78 | >>> cache = SQLiteBackend(allowed_methods=('GET', 'POST')) 79 | >>> async with CachedSession(cache=cache) as session: 80 | >>> await session.post('http://httpbin.org/post', json={'param': 'value'}) 81 | ``` 82 | 83 | ### Status Codes 84 | 85 | By default, only responses with a 200 status code are cached. To cache additional status codes, 86 | specify them with `allowed_codes`" 87 | 88 | ```python 89 | >>> cache = SQLiteBackend(allowed_codes=(200, 418)) 90 | >>> async with CachedSession(cache=cache) as session: 91 | >>> await session.get('http://httpbin.org/teapot') 92 | ``` 93 | 94 | ### Request Parameters 95 | 96 | By default, all request parameters are taken into account when caching responses. In some cases, 97 | there may be request parameters that don't affect the response data, for example authentication tokens 98 | or credentials. If you want to ignore specific parameters, specify them with `ignored_parameters`: 99 | 100 | ```python 101 | >>> cache = SQLiteBackend(ignored_parameters=['auth-token']) 102 | >>> async with CachedSession(cache=cache) as session: 103 | >>> # Only the first request will be sent 104 | >>> await session.get('http://httpbin.org/get', params={'auth-token': '2F63E5DF4F44'}) 105 | >>> await session.get('http://httpbin.org/get', params={'auth-token': 'D9FAEB3449D3'}) 106 | ``` 107 | 108 | ### Request Headers 109 | 110 | In some cases, different headers may result in different response data, so you may want to cache 111 | them separately. To enable this, use `include_headers`: 112 | 113 | ```python 114 | >>> cache = SQLiteBackend(include_headers=True) 115 | >>> async with CachedSession(cache=cache) as session: 116 | >>> # Both of these requests will be sent and cached separately 117 | >>> await session.get('http://httpbin.org/headers', {'Accept': 'text/plain'}) 118 | >>> await session.get('http://httpbin.org/headers', {'Accept': 'application/json'}) 119 | ``` 120 | 121 | ## Cache Expiration 122 | 123 | By default, cached responses will be stored indefinitely. You can initialize the cache with an 124 | `expire_after` value to specify how long responses will be cached. 125 | 126 | ### Expiration Values 127 | 128 | `expire_after` can be any of the following: 129 | 130 | - `-1`: Never expire 131 | - `0` Expire immediately, e.g. skip writing to the cache 132 | - A positive number (in seconds) 133 | - A {py:class}`~datetime.timedelta` 134 | - A {py:class}`~datetime.datetime` 135 | 136 | Examples: 137 | 138 | ```python 139 | >>> # Set expiration for the session using a value in seconds 140 | >>> cache = SQLiteBackend(expire_after=360) 141 | 142 | >>> # To specify a different unit of time, use a timedelta 143 | >>> from datetime import timedelta 144 | >>> cache = SQLiteBackend(expire_after=timedelta(days=30)) 145 | 146 | >>> # Update an existing session to disable expiration (i.e., store indefinitely) 147 | >>> session.expire_after = -1 148 | ``` 149 | 150 | ### URL Patterns 151 | 152 | You can use `urls_expire_after` to set different expiration values for different requests, based on 153 | URL glob patterns. This allows you to customize caching based on what you know about the resources 154 | you're requesting. For example, you might request one resource that gets updated frequently, another 155 | that changes infrequently, and another that never changes. Example: 156 | 157 | ```python 158 | >>> cache = SQLiteBackend( 159 | ... urls_expire_after={ 160 | ... '*.site_1.com': 30, 161 | ... 'site_2.com/resource_1': 60 * 2, 162 | ... 'site_2.com/resource_2': 60 * 60 * 24, 163 | ... 'site_2.com/static': -1, 164 | ... } 165 | ... ) 166 | ``` 167 | 168 | **Notes:** 169 | 170 | - `urls_expire_after` should be a dict in the format `{'pattern': expire_after}` 171 | - `expire_after` accepts the same types as `CacheBackend.expire_after` 172 | - Patterns will match request **base URLs**, so the pattern `site.com/resource/` is equivalent to 173 | `http*://site.com/resource/**` 174 | - If there is more than one match, the first match will be used in the order they are defined 175 | - If no patterns match a request, `CacheBackend.expire_after` will be used as a default. 176 | 177 | ### Cache-Control 178 | 179 | :::{warning} 180 | This is **not** intended to be a thorough or strict implementation of header-based HTTP caching, 181 | e.g. according to RFC 2616. 182 | ::: 183 | 184 | Optional support is included for a simplified subset of 185 | [Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control) 186 | and other cache headers in both requests and responses. To enable this behavior, use the 187 | `cache_control` backend option: 188 | 189 | ```python 190 | >>> cache = SQLiteBackend(cache_control=True) 191 | ``` 192 | 193 | **Supported request headers:** 194 | 195 | - `Cache-Control: max-age`: Used as the expiration time in seconds 196 | - `Cache-Control: no-cache`: Skips reading response data from the cache 197 | - `Cache-Control: no-store`: Skips reading and writing response data from/to the cache 198 | 199 | **Supported response headers:** 200 | 201 | - `Cache-Control: max-age`: Used as the expiration time in seconds 202 | - `Cache-Control: no-store` Skips writing response data to the cache 203 | - `Expires`: Used as an absolute expiration time 204 | 205 | **Notes:** 206 | 207 | - Unlike a browser or proxy cache, `max-age=0` does not currently clear previously cached responses. 208 | - If enabled, Cache-Control directives will take priority over any other `expire_after` value. 209 | See {ref}`user_guide:expiration precedence` for the full order of precedence. 210 | 211 | ### Removing Expired Responses 212 | 213 | For better performance, expired responses won't be removed immediately, but will be removed 214 | (or replaced) the next time they are requested. To manually clear all expired responses, use 215 | {py:meth}`.CachedSession.delete_expired_responses`: 216 | 217 | ```python 218 | >>> session.delete_expired_responses() 219 | ``` 220 | 221 | You can also apply a different `expire_after` to previously cached responses, which will 222 | revalidate the cache with the new expiration time: 223 | 224 | ```python 225 | >>> session.delete_expired_responses(expire_after=timedelta(days=30)) 226 | ``` 227 | 228 | ### Expiration Precedence 229 | 230 | Expiration can be set on a per-session, per-URL, or per-request basis, in addition to cache 231 | headers. When there are multiple values provided for a given request, the following order of 232 | precedence is used: 233 | 234 | 1. Cache-Control request headers (if enabled) 235 | 2. Cache-Control response headers (if enabled) 236 | 3. Per-request expiration (`expire_after` argument for {py:meth}`.CachedSession.request`) 237 | 4. Per-URL expiration (`urls_expire_after` argument for {py:class}`.CachedSession`) 238 | 5. Per-session expiration (`expire_after` argument for {py:class}`.CacheBackend`) 239 | 240 | ## Cache Inspection 241 | 242 | Here are some ways to get additional information out of the cache session, backend, and responses: 243 | 244 | ### Response Attributes 245 | 246 | The following attributes are available on both cached and new responses returned from {py:class}`.CachedSession`: 247 | 248 | - `from_cache`: indicates if the response came from the cache 249 | - `created_at`: {py:class}`~datetime.datetime` of when the cached response was created or last updated 250 | - `expires`: {py:class}`~datetime.datetime` after which the cached response will expire 251 | - `is_expired`: indicates if the cached response is expired (if an old response was returned due to a request error) 252 | 253 | Examples: 254 | 255 | ```python 256 | >>> from aiohttp_client_cache import CachedSession 257 | >>> session = CachedSession(expire_after=timedelta(days=1)) 258 | 259 | >>> # Placeholders are added for non-cached responses 260 | >>> r = await session.get('http://httpbin.org/get') 261 | >>> print(r.from_cache, r.created_at, r.expires, r.is_expired) 262 | False None None None 263 | 264 | >>> # Values will be populated for cached responses 265 | >>> r = await session.get('http://httpbin.org/get') 266 | >>> print(r.from_cache, r.created_at, r.expires, r.is_expired) 267 | True 2021-01-01 18:00:00 2021-01-02 18:00:00 False 268 | ``` 269 | 270 | ### Cache Contents 271 | 272 | You can use {py:meth}`.CachedSession.cache.get_urls` to see all URLs currently in the cache: 273 | 274 | ```python 275 | >>> async for url in session.cache.get_urls(): 276 | ... print(url) 277 | ['https://httpbin.org/get', 'https://httpbin.org/stream/100'] 278 | ``` 279 | 280 | If needed, you can get more details on cached responses via `CachedSession.cache.responses`, which 281 | is a interface to the cache backend. See {py:class}`.CachedResponse` for a full list of 282 | attributes available. 283 | 284 | For example, if you wanted to to see all URLs requested with a specific method: 285 | 286 | ```python 287 | >>> post_urls = [ 288 | >>> response.url async for response in session.cache.responses.values() 289 | >>> if response.method == 'POST' 290 | >>> ] 291 | ``` 292 | 293 | You can also inspect `CachedSession.cache.redirects`, which maps redirect URLs to keys of the 294 | responses they redirect to. 295 | 296 | ## Other Cache Features 297 | 298 | ### Custom Response Filtering 299 | 300 | If you need more advanced behavior for determining what to cache, you can provide a custom filtering 301 | function via the `filter_fn` param. This can by any function or coroutine that takes a 302 | {py:class}`aiohttp.ClientResponse` object and returns a boolean indicating whether or not that 303 | response should be cached. It will be applied to both new responses (on write) and previously cached 304 | responses (on read). Example: 305 | 306 | ```python 307 | >>> from sys import getsizeof 308 | >>> from aiohttp_client_cache import CachedSession, SQLiteCache 309 | >>> 310 | >>> async def filter_by_size(response): 311 | >>> """Don't cache responses with a body over 1 MB""" 312 | >>> return getsizeof(response._body) <= 1024 * 1024 313 | >>> 314 | >>> cache = SQLiteCache(filter_fn=filter_by_size) 315 | ``` 316 | 317 | ### Library Compatibility 318 | 319 | This library works by extending `aiohttp.ClientSession`, and there are other libraries out there 320 | that do the same. For that reason a mixin class is included, so you can create a custom class with 321 | behavior from multiple `aiohttp`-based libraries: 322 | 323 | ```python 324 | >>> from aiohttp import ClientSession 325 | >>> from aiohttp_client_cache import CacheMixin 326 | >>> from some_other_library import CustomMixin 327 | >>> 328 | >>> class CustomSession(CacheMixin, CustomMixin, ClientSession): 329 | ... """Session with features from both aiohttp_client_cache and some_other_library""" 330 | ``` 331 | --------------------------------------------------------------------------------