├── tests ├── __init__.py ├── test_retry_options.py ├── app.py └── test_client.py ├── aiohttp_retry ├── py.typed ├── __init__.py ├── types.py ├── retry_options.py └── client.py ├── requirements.txt ├── .codespellrc ├── AUTHORS ├── .gitignore ├── requirements_ci.txt ├── .github └── workflows │ ├── codespell.yml │ ├── publish-to-pypi.yml │ └── python-package.yml ├── setup.py ├── LICENSE ├── pyproject.toml └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /aiohttp_retry/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp 2 | yarl 3 | -------------------------------------------------------------------------------- /.codespellrc: -------------------------------------------------------------------------------- 1 | [codespell] 2 | skip = .git 3 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Dmitry Inyutin 2 | -------------------------------------------------------------------------------- /aiohttp_retry/__init__.py: -------------------------------------------------------------------------------- 1 | from .client import * # noqa: F403 2 | from .retry_options import * # noqa: F403 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | venv/ 3 | env/ 4 | __pycache__/ 5 | .mypy_cache/ 6 | .pytest_cache/ 7 | 8 | aiohttp_retry.egg-info/ 9 | build/ 10 | dist/ 11 | var/ 12 | -------------------------------------------------------------------------------- /requirements_ci.txt: -------------------------------------------------------------------------------- 1 | mypy==1.4.1 # last version that support python 3.7 2 | pytest-aiohttp==1.0.5 3 | pytest==7.4.4 # last version that support python 3.7 4 | ruff==0.7.1 5 | -------------------------------------------------------------------------------- /aiohttp_retry/types.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | from aiohttp import ClientSession 4 | 5 | from .client import RetryClient 6 | 7 | ClientType = Union[ClientSession, RetryClient] 8 | -------------------------------------------------------------------------------- /.github/workflows/codespell.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Codespell 3 | 4 | on: 5 | push: 6 | branches: [master] 7 | pull_request: 8 | branches: [master] 9 | 10 | jobs: 11 | codespell: 12 | name: Check for spelling errors 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v3 18 | - name: Codespell 19 | uses: codespell-project/actions-codespell@v1 20 | -------------------------------------------------------------------------------- /.github/workflows/publish-to-pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish package to PyPI 2 | 3 | on: 4 | workflow_dispatch: 5 | release: 6 | types: [ released ] 7 | 8 | 9 | jobs: 10 | build-n-publish: 11 | name: Build and publish Python package to PyPI 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@master 15 | - name: Set up Python 3.7 16 | uses: actions/setup-python@v1 17 | with: 18 | python-version: 3.7 19 | 20 | - name: Install pypa/build 21 | run: >- 22 | python3 -m pip install --user --upgrade setuptools wheel 23 | - name: Build a binary wheel and a source tarball 24 | run: >- 25 | python3 setup.py sdist bdist_wheel 26 | 27 | - name: Publish distribution 📦 to PyPI 28 | if: startsWith(github.ref, 'refs/tags') 29 | uses: pypa/gh-action-pypi-publish@master 30 | with: 31 | password: ${{ secrets.PYPI_API_TOKEN }} 32 | -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | name: Test package 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | test_package: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] 15 | 16 | steps: 17 | - uses: actions/checkout@master 18 | 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@master 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install -r requirements_ci.txt 28 | 29 | - name: Test with ruff 30 | run: | 31 | ruff check . 32 | ruff format --diff . 33 | 34 | - name: Test with mypy 35 | run: mypy -m aiohttp_retry 36 | 37 | - name: Test with pytest 38 | env: 39 | PYTHONPATH: . 40 | run: pytest 41 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | with open("README.md", encoding="utf-8") as fh: 4 | long_description = fh.read() 5 | 6 | 7 | setup( 8 | name="aiohttp_retry", 9 | version="2.9.1", 10 | description="Simple retry client for aiohttp", 11 | long_description=long_description, 12 | long_description_content_type="text/markdown", 13 | classifiers=[ 14 | "Programming Language :: Python :: 3", 15 | "License :: OSI Approved :: MIT License", 16 | "Operating System :: OS Independent", 17 | ], 18 | keywords="aiohttp retry client", 19 | author="Dmitry Inyutin", 20 | author_email="inyutin.da@gmail.com", 21 | url="https://github.com/inyutin/aiohttp_retry", 22 | license="MIT", 23 | include_package_data=True, 24 | packages=find_packages(exclude=["tests", "tests.*"]), 25 | platforms=["any"], 26 | python_requires=">=3.7", 27 | install_requires=[ 28 | "aiohttp", 29 | ], 30 | package_data={ 31 | "aiohttp_retry": ["py.typed"], 32 | }, 33 | ) 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) 2020 aiohttp_retry Authors 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 5 | 6 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 9 | -------------------------------------------------------------------------------- /tests/test_retry_options.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from aiohttp_retry import ( 4 | ExponentialRetry, 5 | FibonacciRetry, 6 | JitterRetry, 7 | ListRetry, 8 | RandomRetry, 9 | ) 10 | 11 | 12 | def test_exponential_retry() -> None: 13 | retry = ExponentialRetry(attempts=10) 14 | timeouts = [retry.get_timeout(x) for x in range(10)] 15 | assert timeouts == [0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 30.0] 16 | 17 | 18 | def test_random_retry() -> None: 19 | retry = RandomRetry(attempts=10, random_func=random.Random(0).random) 20 | timeouts = [round(retry.get_timeout(x), 2) for x in range(10)] 21 | assert timeouts == [2.55, 2.3, 1.32, 0.85, 1.58, 1.27, 2.37, 0.98, 1.48, 1.79] 22 | 23 | 24 | def test_list_retry() -> None: 25 | expected = [1.2, 2.1, 3.4, 4.3, 4.5, 5.4, 5.6, 6.5, 6.7, 7.6] 26 | retry = ListRetry(expected) 27 | timeouts = [retry.get_timeout(x) for x in range(10)] 28 | assert timeouts == expected 29 | 30 | 31 | def test_fibonacci_retry() -> None: 32 | retry = FibonacciRetry(attempts=10, multiplier=2, max_timeout=60) 33 | timeouts = [retry.get_timeout(x) for x in range(10)] 34 | assert timeouts == [4.0, 6.0, 10.0, 16.0, 26.0, 42.0, 60, 60, 60, 60] 35 | 36 | 37 | def test_jitter_retry() -> None: 38 | random.seed(10) 39 | retry = JitterRetry(attempts=10) 40 | timeouts = [retry.get_timeout(x) for x in range(10)] 41 | assert len(timeouts) == 10 42 | 43 | expected = [ 44 | 1.4, 45 | 0.9, 46 | 1.7, 47 | 0.9, 48 | 4.2, 49 | 5.9, 50 | 8.1, 51 | 12.9, 52 | 26.6, 53 | 30.4, 54 | ] 55 | for idx, timeout in enumerate(timeouts): 56 | assert abs(timeout - expected[idx]) < 0.1 57 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.mypy] 2 | disallow_untyped_calls = true 3 | disallow_untyped_defs = true 4 | disallow_incomplete_defs = true 5 | check_untyped_defs = true 6 | disallow_untyped_decorators = true 7 | 8 | warn_redundant_casts = true 9 | warn_unused_ignores = true 10 | warn_no_return = true 11 | warn_return_any = true 12 | warn_unreachable = true 13 | 14 | [tool.ruff] 15 | indent-width = 4 16 | line-length = 120 17 | target-version = "py37" 18 | 19 | [tool.ruff.lint] 20 | select = ["ALL"] 21 | ignore = [ 22 | "ANN401", # Dynamically typed expressions (typing.Any) are disallowed in `**kwargs` 23 | "COM812", # Missing trailing comma in Python 3.6+ 24 | "D100", # Missing docstring in public module 25 | "D101", # Missing docstring in public class 26 | "D102", # Missing docstring in public method 27 | "D103", # Missing docstring in public function 28 | "D104", # Missing docstring in public package 29 | "D105", # Missing docstring in magic method 30 | "D107", # Missing docstring in __init__ 31 | "D203", # 1 blank line required before class docstring 32 | "G004", # Logging statement uses string formatting 33 | "D211", # No blank lines before class docstring 34 | "D213", # Multi-line docstring summary should start at the second line 35 | "FBT001", # Boolean positional arg in function call 36 | "FBT002", # Boolean positional arg in function definition 37 | "ISC001", # Implicit string concatenation 38 | "PTH123", # open() should be replaced by Path.open() 39 | "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes 40 | "PLR0913", # Too many arguments in function definition 41 | ] 42 | 43 | fixable = ["ALL"] 44 | unfixable = ["FBT002"] 45 | 46 | [tool.ruff.lint.per-file-ignores] 47 | "__init__.py" = ["E402"] 48 | "**/{tests}/*" = [ 49 | "ARG001", # Unused function argument 50 | "E722", # Do not use bare except 51 | "INP001", # File is part of an implicit namespace package 52 | "PLR2004", # Magic value used in comparison 53 | "PT011", # pytest.raises() should be used as a context manager 54 | "S101", # Use of assert detected 55 | "SLF001", # Private member accessed 56 | ] 57 | 58 | [tool.pytest.ini_options] 59 | asyncio_mode = "auto" 60 | -------------------------------------------------------------------------------- /tests/app.py: -------------------------------------------------------------------------------- 1 | from aiohttp import web 2 | 3 | 4 | class App: 5 | def __init__(self) -> None: 6 | self.counter = 0 7 | 8 | app = web.Application() 9 | app.router.add_get("/ping", self.ping_handler) 10 | app.router.add_get("/internal_error", self.internal_error_handler) 11 | app.router.add_get("/not_found_error", self.not_found_error_handler) 12 | app.router.add_get("/sometimes_error", self.sometimes_error) 13 | app.router.add_get("/sometimes_json", self.sometimes_json) 14 | app.router.add_get("/check_headers", self.check_headers) 15 | app.router.add_get("/with_auth", self.with_auth) 16 | 17 | app.router.add_options("/options_handler", self.ping_handler) 18 | app.router.add_head("/head_handler", self.ping_handler) 19 | app.router.add_post("/post_handler", self.ping_handler) 20 | app.router.add_put("/put_handler", self.ping_handler) 21 | app.router.add_patch("/patch_handler", self.ping_handler) 22 | app.router.add_delete("/delete_handler", self.ping_handler) 23 | 24 | self._web_app = app 25 | 26 | async def ping_handler(self, _: web.Request) -> web.Response: 27 | self.counter += 1 28 | return web.Response(text="Ok!", status=200) 29 | 30 | async def internal_error_handler(self, _: web.Request) -> web.Response: 31 | self.counter += 1 32 | raise web.HTTPInternalServerError 33 | 34 | async def not_found_error_handler(self, _: web.Request) -> web.Response: 35 | self.counter += 1 36 | raise web.HTTPNotFound 37 | 38 | async def sometimes_error(self, _: web.Request) -> web.Response: 39 | self.counter += 1 40 | if self.counter == 3: 41 | return web.Response(text="Ok!", status=200) 42 | 43 | raise web.HTTPInternalServerError 44 | 45 | async def sometimes_json(self, _: web.Request) -> web.Response: 46 | self.counter += 1 47 | if self.counter == 3: 48 | return web.json_response(data={"status": "Ok!"}, status=200) 49 | 50 | return web.Response(text="Ok!", status=200) 51 | 52 | async def check_headers(self, request: web.Request) -> web.Response: 53 | self.counter += 1 54 | if request.headers.get("correct_headers") != "True": 55 | raise web.HTTPNotAcceptable 56 | 57 | return web.Response(text="Ok!", status=200) 58 | 59 | async def with_auth(self, request: web.Request) -> web.Response: 60 | self.counter += 1 61 | 62 | # BasicAuth("username", "password") # noqa: ERA001 63 | if request.headers.get("Authorization") != "Basic dXNlcm5hbWU6cGFzc3dvcmQ=": 64 | return web.Response(text="incorrect auth", status=403) 65 | return web.Response(text="Ok!", status=200) 66 | 67 | @property 68 | def web_app(self) -> web.Application: 69 | return self._web_app 70 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Simple aiohttp retry client 2 | 3 | Python 3.7 or higher. 4 | 5 | **Install**: `pip install aiohttp-retry`. 6 | 7 | [!["Buy Me A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://www.buymeacoffee.com/inyutin) 8 | 9 | 10 | ### Breaking API changes 11 | - Everything between [2.7.0 - 2.8.3) is yanked. 12 | There is a bug with evaluate_response_callback, it led to infinite retries 13 | 14 | - 2.8.0 is incorrect and yanked. 15 | https://github.com/inyutin/aiohttp_retry/issues/79 16 | 17 | - Since 2.5.6 this is a new parameter in ```get_timeout``` func called "response". 18 | If you have defined your own ```RetryOptions```, you should add this param into it. 19 | Issue about this: https://github.com/inyutin/aiohttp_retry/issues/59 20 | 21 | ### Examples of usage: 22 | ```python 23 | from aiohttp_retry import RetryClient, ExponentialRetry 24 | 25 | async def main(): 26 | retry_options = ExponentialRetry(attempts=1) 27 | retry_client = RetryClient(raise_for_status=False, retry_options=retry_options) 28 | async with retry_client.get('https://ya.ru') as response: 29 | print(response.status) 30 | 31 | await retry_client.close() 32 | ``` 33 | 34 | ```python 35 | from aiohttp import ClientSession 36 | from aiohttp_retry import RetryClient 37 | 38 | async def main(): 39 | client_session = ClientSession() 40 | retry_client = RetryClient(client_session=client_session) 41 | async with retry_client.get('https://ya.ru') as response: 42 | print(response.status) 43 | 44 | await client_session.close() 45 | ``` 46 | 47 | ```python 48 | from aiohttp_retry import RetryClient, RandomRetry 49 | 50 | async def main(): 51 | retry_options = RandomRetry(attempts=1) 52 | retry_client = RetryClient(raise_for_status=False, retry_options=retry_options) 53 | 54 | response = await retry_client.get('/ping') 55 | print(response.status) 56 | 57 | await retry_client.close() 58 | ``` 59 | 60 | ```python 61 | from aiohttp_retry import RetryClient 62 | 63 | async def main(): 64 | async with RetryClient() as client: 65 | async with client.get('https://ya.ru') as response: 66 | print(response.status) 67 | ``` 68 | 69 | You can change parameters between attempts by passing multiple requests params: 70 | ```python 71 | from aiohttp_retry import RetryClient, RequestParams, ExponentialRetry 72 | 73 | async def main(): 74 | retry_client = RetryClient(raise_for_status=False) 75 | 76 | async with retry_client.requests( 77 | params_list=[ 78 | RequestParams( 79 | method='GET', 80 | url='https://ya.ru', 81 | ), 82 | RequestParams( 83 | method='GET', 84 | url='https://ya.ru', 85 | headers={'some_header': 'some_value'}, 86 | ), 87 | ] 88 | ) as response: 89 | print(response.status) 90 | 91 | await retry_client.close() 92 | ``` 93 | 94 | You can also add some logic, F.E. logging, on failures by using trace mechanic. 95 | ```python 96 | import logging 97 | import sys 98 | from types import SimpleNamespace 99 | 100 | from aiohttp import ClientSession, TraceConfig, TraceRequestStartParams 101 | 102 | from aiohttp_retry import RetryClient, ExponentialRetry 103 | 104 | 105 | handler = logging.StreamHandler(sys.stdout) 106 | logging.basicConfig(handlers=[handler]) 107 | logger = logging.getLogger(__name__) 108 | retry_options = ExponentialRetry(attempts=2) 109 | 110 | 111 | async def on_request_start( 112 | session: ClientSession, 113 | trace_config_ctx: SimpleNamespace, 114 | params: TraceRequestStartParams, 115 | ) -> None: 116 | current_attempt = trace_config_ctx.trace_request_ctx['current_attempt'] 117 | if retry_options.attempts <= current_attempt: 118 | logger.warning('Wow! We are in last attempt') 119 | 120 | 121 | async def main(): 122 | trace_config = TraceConfig() 123 | trace_config.on_request_start.append(on_request_start) 124 | retry_client = RetryClient(retry_options=retry_options, trace_configs=[trace_config]) 125 | 126 | response = await retry_client.get('https://httpstat.us/503', ssl=False) 127 | print(response.status) 128 | 129 | await retry_client.close() 130 | ``` 131 | Look tests for more examples. \ 132 | **Be aware: last request returns as it is.** 133 | **If the last request ended with exception, that this exception will be raised from RetryClient request** 134 | 135 | ### Documentation 136 | `RetryClient` takes the same arguments as ClientSession[[docs](https://docs.aiohttp.org/en/stable/client_reference.html)] \ 137 | `RetryClient` has methods: 138 | - request 139 | - get 140 | - options 141 | - head 142 | - post 143 | - put 144 | - patch 145 | - put 146 | - delete 147 | 148 | They are same as for `ClientSession`, but take one possible additional argument: 149 | ```python 150 | class RetryOptionsBase: 151 | def __init__( 152 | self, 153 | attempts: int = 3, # How many times we should retry 154 | statuses: Iterable[int] | None = None, # On which statuses we should retry 155 | exceptions: Iterable[type[Exception]] | None = None, # On which exceptions we should retry, by default on all 156 | retry_all_server_errors: bool = True, # If should retry all 500 errors or not 157 | # a callback that will run on response to decide if retry 158 | evaluate_response_callback: EvaluateResponseCallbackType | None = None, 159 | ): 160 | ... 161 | 162 | @abc.abstractmethod 163 | def get_timeout(self, attempt: int, response: Optional[Response] = None) -> float: 164 | raise NotImplementedError 165 | 166 | ``` 167 | You can specify `RetryOptions` both for `RetryClient` and it's methods. 168 | `RetryOptions` in methods override `RetryOptions` defined in `RetryClient` constructor. 169 | 170 | **Important**: by default all 5xx responses are retried + statuses you specified as ```statuses``` param 171 | If you will pass ```retry_all_server_errors=False``` than you can manually set what 5xx errors to retry. 172 | 173 | You can define your own timeouts logic or use: 174 | - ```ExponentialRetry``` with exponential backoff 175 | - ```RandomRetry``` for random backoff 176 | - ```ListRetry``` with backoff you predefine by list 177 | - ```FibonacciRetry``` with backoff that looks like fibonacci sequence 178 | - ```JitterRetry``` exponential retry with a bit of randomness 179 | 180 | **Important**: you can proceed server response as an parameter for calculating next timeout. 181 | However this response can be None, server didn't make a response or you have set up ```raise_for_status=True``` 182 | Look here for an example: https://github.com/inyutin/aiohttp_retry/issues/59 183 | 184 | Additionally, you can specify ```evaluate_response_callback```. It receive a ```ClientResponse``` and decide to retry or not by returning a bool. 185 | It can be useful, if server API sometimes response with malformed data. 186 | 187 | #### Request Trace Context 188 | `RetryClient` add *current attempt number* to `request_trace_ctx` (see examples, 189 | for more info see [aiohttp doc](https://docs.aiohttp.org/en/stable/client_advanced.html#aiohttp-client-tracing)). 190 | 191 | ### Change parameters between retries 192 | `RetryClient` also has a method called `requests`. This method should be used if you want to make requests with different params. 193 | ```python 194 | @dataclass 195 | class RequestParams: 196 | method: str 197 | url: _RAW_URL_TYPE 198 | headers: dict[str, Any] | None = None 199 | trace_request_ctx: dict[str, Any] | None = None 200 | kwargs: dict[str, Any] | None = None 201 | ``` 202 | 203 | ```python 204 | def requests( 205 | self, 206 | params_list: list[RequestParams], 207 | retry_options: RetryOptionsBase | None = None, 208 | raise_for_status: bool | None = None, 209 | ) -> _RequestContext: 210 | ``` 211 | 212 | You can find an example of usage above or in tests. 213 | But basically `RequestParams` is a structure to define params for `ClientSession.request` func. 214 | `method`, `url`, `headers` `trace_request_ctx` defined outside kwargs, because they are popular. 215 | 216 | There is also an old way to change URL between retries by specifying ```url``` as list of urls. Example: 217 | ```python 218 | from aiohttp_retry import RetryClient 219 | 220 | retry_client = RetryClient() 221 | async with retry_client.get(url=['/internal_error', '/ping']) as response: 222 | text = await response.text() 223 | assert response.status == 200 224 | assert text == 'Ok!' 225 | 226 | await retry_client.close() 227 | ``` 228 | 229 | In this example we request ```/interval_error```, fail and then successfully request ```/ping```. 230 | If you specify less urls than ```attempts``` number in ```RetryOptions```, ```RetryClient``` will request last url at last attempts. 231 | This means that in example above we would request ```/ping``` once again in case of failure. 232 | 233 | ### Types 234 | 235 | `aiohttp_retry` is a typed project. It should be fully compatible with mypy. 236 | 237 | It also introduce one special type: 238 | ``` 239 | ClientType = Union[ClientSession, RetryClient] 240 | ``` 241 | 242 | This type can be imported by ```from aiohttp_retry.types import ClientType``` 243 | -------------------------------------------------------------------------------- /aiohttp_retry/retry_options.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import abc 4 | import random 5 | from typing import Any, Awaitable, Callable, Iterable 6 | from warnings import warn 7 | 8 | from aiohttp import ClientResponse 9 | 10 | EvaluateResponseCallbackType = Callable[[ClientResponse], Awaitable[bool]] 11 | 12 | 13 | class RetryOptionsBase: 14 | def __init__( 15 | self, 16 | attempts: int = 3, # How many times we should retry 17 | statuses: Iterable[int] | None = None, # On which statuses we should retry 18 | exceptions: Iterable[type[Exception]] | None = None, # On which exceptions we should retry, by default on all 19 | methods: Iterable[str] | None = None, # On which HTTP methods we should retry 20 | retry_all_server_errors: bool = True, # If should retry all 500 errors or not 21 | # a callback that will run on response to decide if retry 22 | evaluate_response_callback: EvaluateResponseCallbackType | None = None, 23 | ) -> None: 24 | self.attempts: int = attempts 25 | if statuses is None: 26 | statuses = set() 27 | self.statuses: Iterable[int] = statuses 28 | 29 | if exceptions is None: 30 | exceptions = set() 31 | self.exceptions: Iterable[type[Exception]] = exceptions 32 | 33 | if methods is None: 34 | methods = {"HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE", "POST", "CONNECT", "PATCH"} 35 | self.methods: Iterable[str] = {method.upper() for method in methods} 36 | 37 | self.retry_all_server_errors = retry_all_server_errors 38 | self.evaluate_response_callback = evaluate_response_callback 39 | 40 | @abc.abstractmethod 41 | def get_timeout(self, attempt: int, response: ClientResponse | None = None) -> float: 42 | raise NotImplementedError 43 | 44 | 45 | class ExponentialRetry(RetryOptionsBase): 46 | def __init__( 47 | self, 48 | attempts: int = 3, # How many times we should retry 49 | start_timeout: float = 0.1, # Base timeout time, then it exponentially grow 50 | max_timeout: float = 30.0, # Max possible timeout between tries 51 | factor: float = 2.0, # How much we increase timeout each time 52 | statuses: set[int] | None = None, # On which statuses we should retry 53 | exceptions: set[type[Exception]] | None = None, # On which exceptions we should retry 54 | methods: set[str] | None = None, # On which HTTP methods we should retry 55 | retry_all_server_errors: bool = True, 56 | evaluate_response_callback: EvaluateResponseCallbackType | None = None, 57 | ) -> None: 58 | super().__init__( 59 | attempts=attempts, 60 | statuses=statuses, 61 | exceptions=exceptions, 62 | methods=methods, 63 | retry_all_server_errors=retry_all_server_errors, 64 | evaluate_response_callback=evaluate_response_callback, 65 | ) 66 | 67 | self._start_timeout: float = start_timeout 68 | self._max_timeout: float = max_timeout 69 | self._factor: float = factor 70 | 71 | def get_timeout( 72 | self, 73 | attempt: int, 74 | response: ClientResponse | None = None, # noqa: ARG002 75 | ) -> float: 76 | """Return timeout with exponential backoff.""" 77 | timeout = self._start_timeout * (self._factor**attempt) 78 | return min(timeout, self._max_timeout) 79 | 80 | 81 | def RetryOptions(*args: Any, **kwargs: Any) -> ExponentialRetry: # noqa: N802 82 | warn("RetryOptions is deprecated, use ExponentialRetry", stacklevel=1) 83 | return ExponentialRetry(*args, **kwargs) 84 | 85 | 86 | class RandomRetry(RetryOptionsBase): 87 | def __init__( 88 | self, 89 | attempts: int = 3, # How many times we should retry 90 | statuses: Iterable[int] | None = None, # On which statuses we should retry 91 | exceptions: Iterable[type[Exception]] | None = None, # On which exceptions we should retry 92 | methods: Iterable[str] | None = None, # On which HTTP methods we should retry 93 | min_timeout: float = 0.1, # Minimum possible timeout 94 | max_timeout: float = 3.0, # Maximum possible timeout between tries 95 | random_func: Callable[[], float] = random.random, # Random number generator 96 | retry_all_server_errors: bool = True, 97 | evaluate_response_callback: EvaluateResponseCallbackType | None = None, 98 | ) -> None: 99 | super().__init__( 100 | attempts=attempts, 101 | statuses=statuses, 102 | exceptions=exceptions, 103 | methods=methods, 104 | retry_all_server_errors=retry_all_server_errors, 105 | evaluate_response_callback=evaluate_response_callback, 106 | ) 107 | 108 | self.attempts: int = attempts 109 | self.min_timeout: float = min_timeout 110 | self.max_timeout: float = max_timeout 111 | self.random = random_func 112 | 113 | def get_timeout( 114 | self, 115 | attempt: int, # noqa: ARG002 116 | response: ClientResponse | None = None, # noqa: ARG002 117 | ) -> float: 118 | """Generate random timeouts.""" 119 | return self.min_timeout + self.random() * (self.max_timeout - self.min_timeout) 120 | 121 | 122 | class ListRetry(RetryOptionsBase): 123 | def __init__( 124 | self, 125 | timeouts: list[float], 126 | statuses: Iterable[int] | None = None, # On which statuses we should retry 127 | exceptions: Iterable[type[Exception]] | None = None, # On which exceptions we should retry 128 | methods: Iterable[str] | None = None, # On which HTTP methods we should retry 129 | retry_all_server_errors: bool = True, 130 | evaluate_response_callback: EvaluateResponseCallbackType | None = None, 131 | ) -> None: 132 | super().__init__( 133 | attempts=len(timeouts), 134 | statuses=statuses, 135 | exceptions=exceptions, 136 | methods=methods, 137 | retry_all_server_errors=retry_all_server_errors, 138 | evaluate_response_callback=evaluate_response_callback, 139 | ) 140 | self.timeouts = timeouts 141 | 142 | def get_timeout( 143 | self, 144 | attempt: int, 145 | response: ClientResponse | None = None, # noqa: ARG002 146 | ) -> float: 147 | """Timeouts from a defined list.""" 148 | return self.timeouts[attempt] 149 | 150 | 151 | class FibonacciRetry(RetryOptionsBase): 152 | def __init__( 153 | self, 154 | attempts: int = 3, 155 | multiplier: float = 1.0, 156 | statuses: Iterable[int] | None = None, 157 | exceptions: Iterable[type[Exception]] | None = None, 158 | methods: Iterable[str] | None = None, 159 | max_timeout: float = 3.0, # Maximum possible timeout between tries 160 | retry_all_server_errors: bool = True, 161 | evaluate_response_callback: EvaluateResponseCallbackType | None = None, 162 | ) -> None: 163 | super().__init__( 164 | attempts=attempts, 165 | statuses=statuses, 166 | exceptions=exceptions, 167 | methods=methods, 168 | retry_all_server_errors=retry_all_server_errors, 169 | evaluate_response_callback=evaluate_response_callback, 170 | ) 171 | 172 | self.max_timeout = max_timeout 173 | self.multiplier = multiplier 174 | self.prev_step = 1.0 175 | self.current_step = 1.0 176 | 177 | def get_timeout( 178 | self, 179 | attempt: int, # noqa: ARG002 180 | response: ClientResponse | None = None, # noqa: ARG002 181 | ) -> float: 182 | new_current_step = self.prev_step + self.current_step 183 | self.prev_step = self.current_step 184 | self.current_step = new_current_step 185 | 186 | return min(self.multiplier * new_current_step, self.max_timeout) 187 | 188 | 189 | class JitterRetry(ExponentialRetry): 190 | """https://github.com/inyutin/aiohttp_retry/issues/44.""" 191 | 192 | def __init__( 193 | self, 194 | attempts: int = 3, # How many times we should retry 195 | start_timeout: float = 0.1, # Base timeout time, then it exponentially grow 196 | max_timeout: float = 30.0, # Max possible timeout between tries 197 | factor: float = 2.0, # How much we increase timeout each time 198 | statuses: set[int] | None = None, # On which statuses we should retry 199 | exceptions: set[type[Exception]] | None = None, # On which exceptions we should retry 200 | methods: set[str] | None = None, # On which HTTP methods we should retry 201 | random_interval_size: float = 2.0, # size of interval for random component 202 | retry_all_server_errors: bool = True, 203 | evaluate_response_callback: EvaluateResponseCallbackType | None = None, 204 | ) -> None: 205 | super().__init__( 206 | attempts=attempts, 207 | start_timeout=start_timeout, 208 | max_timeout=max_timeout, 209 | factor=factor, 210 | statuses=statuses, 211 | exceptions=exceptions, 212 | methods=methods, 213 | retry_all_server_errors=retry_all_server_errors, 214 | evaluate_response_callback=evaluate_response_callback, 215 | ) 216 | 217 | self._start_timeout: float = start_timeout 218 | self._max_timeout: float = max_timeout 219 | self._factor: float = factor 220 | self._random_interval_size = random_interval_size 221 | 222 | def get_timeout( 223 | self, 224 | attempt: int, 225 | response: ClientResponse | None = None, # noqa: ARG002 226 | ) -> float: 227 | timeout: float = super().get_timeout(attempt) + random.uniform(0, self._random_interval_size) ** self._factor 228 | return timeout 229 | -------------------------------------------------------------------------------- /aiohttp_retry/client.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | import logging 5 | import sys 6 | from abc import abstractmethod 7 | from dataclasses import dataclass 8 | from typing import ( 9 | TYPE_CHECKING, 10 | Any, 11 | Awaitable, 12 | Callable, 13 | Generator, 14 | List, 15 | Tuple, 16 | Union, 17 | ) 18 | 19 | from aiohttp import ClientResponse, ClientSession, hdrs 20 | from aiohttp.typedefs import StrOrURL 21 | from yarl import URL as YARL_URL 22 | 23 | from .retry_options import ExponentialRetry, RetryOptionsBase 24 | 25 | _MIN_SERVER_ERROR_STATUS = 500 26 | 27 | if TYPE_CHECKING: 28 | from types import TracebackType 29 | 30 | if sys.version_info >= (3, 8): 31 | from typing import Protocol 32 | else: 33 | from typing_extensions import Protocol 34 | 35 | 36 | class _Logger(Protocol): 37 | """_Logger defines which methods logger object should have.""" 38 | 39 | @abstractmethod 40 | def debug(self, msg: str, *args: Any, **kwargs: Any) -> None: 41 | pass 42 | 43 | @abstractmethod 44 | def warning(self, msg: str, *args: Any, **kwargs: Any) -> None: 45 | pass 46 | 47 | @abstractmethod 48 | def exception(self, msg: str, *args: Any, **kwargs: Any) -> None: 49 | pass 50 | 51 | 52 | # url itself or list of urls for changing between retries 53 | _RAW_URL_TYPE = Union[StrOrURL, YARL_URL] 54 | _URL_TYPE = Union[_RAW_URL_TYPE, List[_RAW_URL_TYPE], Tuple[_RAW_URL_TYPE, ...]] 55 | _LoggerType = Union[_Logger, logging.Logger] 56 | 57 | RequestFunc = Callable[..., Awaitable[ClientResponse]] 58 | 59 | 60 | @dataclass 61 | class RequestParams: 62 | method: str 63 | url: _RAW_URL_TYPE 64 | headers: dict[str, Any] | None = None 65 | trace_request_ctx: dict[str, Any] | None = None 66 | kwargs: dict[str, Any] | None = None 67 | 68 | 69 | class _RequestContext: 70 | def __init__( 71 | self, 72 | request_func: RequestFunc, 73 | params_list: list[RequestParams], 74 | logger: _LoggerType, 75 | retry_options: RetryOptionsBase, 76 | raise_for_status: bool = False, 77 | ) -> None: 78 | assert len(params_list) > 0 # noqa: S101 79 | 80 | self._request_func = request_func 81 | self._params_list = params_list 82 | self._logger = logger 83 | self._retry_options = retry_options 84 | self._raise_for_status = raise_for_status 85 | 86 | self._response: ClientResponse | None = None 87 | 88 | async def _is_skip_retry(self, current_attempt: int, response: ClientResponse) -> bool: 89 | if current_attempt == self._retry_options.attempts: 90 | return True 91 | 92 | if response.method.upper() not in self._retry_options.methods: 93 | return True 94 | 95 | if response.status >= _MIN_SERVER_ERROR_STATUS and self._retry_options.retry_all_server_errors: 96 | return False 97 | 98 | if response.status in self._retry_options.statuses: 99 | return False 100 | 101 | if self._retry_options.evaluate_response_callback is None: 102 | return True 103 | 104 | return await self._retry_options.evaluate_response_callback(response) 105 | 106 | async def _do_request(self) -> ClientResponse: 107 | current_attempt = 0 108 | 109 | while True: 110 | self._logger.debug(f"Attempt {current_attempt+1} out of {self._retry_options.attempts}") 111 | 112 | current_attempt += 1 113 | try: 114 | try: 115 | params = self._params_list[current_attempt - 1] 116 | except IndexError: 117 | params = self._params_list[-1] 118 | 119 | response: ClientResponse = await self._request_func( 120 | params.method, 121 | params.url, 122 | headers=params.headers, 123 | trace_request_ctx={ 124 | "current_attempt": current_attempt, 125 | **(params.trace_request_ctx or {}), 126 | }, 127 | **(params.kwargs or {}), 128 | ) 129 | 130 | debug_message = f"Retrying after response code: {response.status}" 131 | skip_retry = await self._is_skip_retry(current_attempt, response) 132 | 133 | if skip_retry: 134 | if self._raise_for_status: 135 | response.raise_for_status() 136 | self._response = response 137 | return self._response 138 | retry_wait = self._retry_options.get_timeout(attempt=current_attempt, response=response) 139 | 140 | except Exception as e: 141 | if current_attempt >= self._retry_options.attempts: 142 | raise 143 | 144 | is_exc_valid = any(isinstance(e, exc) for exc in self._retry_options.exceptions) 145 | if not is_exc_valid: 146 | raise 147 | 148 | debug_message = f"Retrying after exception: {e!r}" 149 | retry_wait = self._retry_options.get_timeout(attempt=current_attempt, response=None) 150 | 151 | self._logger.debug(debug_message) 152 | await asyncio.sleep(retry_wait) 153 | 154 | def __await__(self) -> Generator[Any, None, ClientResponse]: 155 | return self.__aenter__().__await__() 156 | 157 | async def __aenter__(self) -> ClientResponse: 158 | return await self._do_request() 159 | 160 | async def __aexit__( 161 | self, 162 | exc_type: type[BaseException] | None, 163 | exc_val: BaseException | None, 164 | exc_tb: TracebackType | None, 165 | ) -> None: 166 | if self._response is not None and not self._response.closed: 167 | self._response.close() 168 | 169 | 170 | def _url_to_urls(url: _URL_TYPE) -> tuple[StrOrURL, ...]: 171 | if isinstance(url, (str, YARL_URL)): 172 | return (url,) 173 | 174 | if isinstance(url, list): 175 | urls = tuple(url) 176 | elif isinstance(url, tuple): 177 | urls = url 178 | else: 179 | msg = "you can pass url only by str or list/tuple" # type: ignore[unreachable] 180 | raise ValueError(msg) # noqa: TRY004 181 | 182 | if len(urls) == 0: 183 | msg = "you can pass url by str or list/tuple with attempts count size" 184 | raise ValueError(msg) 185 | 186 | return urls 187 | 188 | 189 | class RetryClient: 190 | def __init__( 191 | self, 192 | client_session: ClientSession | None = None, 193 | logger: _LoggerType | None = None, 194 | retry_options: RetryOptionsBase | None = None, 195 | raise_for_status: bool = False, 196 | *args: Any, 197 | **kwargs: Any, 198 | ) -> None: 199 | if client_session is not None: 200 | client = client_session 201 | closed = None 202 | else: 203 | client = ClientSession(*args, **kwargs) 204 | closed = False 205 | 206 | self._client = client 207 | self._closed = closed 208 | 209 | self._logger: _LoggerType = logger or logging.getLogger("aiohttp_retry") 210 | self._retry_options: RetryOptionsBase = retry_options or ExponentialRetry() 211 | self._raise_for_status = raise_for_status 212 | 213 | @property 214 | def retry_options(self) -> RetryOptionsBase: 215 | return self._retry_options 216 | 217 | def requests( 218 | self, 219 | params_list: list[RequestParams], 220 | retry_options: RetryOptionsBase | None = None, 221 | raise_for_status: bool | None = None, 222 | ) -> _RequestContext: 223 | return self._make_requests( 224 | params_list=params_list, 225 | retry_options=retry_options, 226 | raise_for_status=raise_for_status, 227 | ) 228 | 229 | def request( 230 | self, 231 | method: str, 232 | url: StrOrURL, 233 | retry_options: RetryOptionsBase | None = None, 234 | raise_for_status: bool | None = None, 235 | **kwargs: Any, 236 | ) -> _RequestContext: 237 | return self._make_request( 238 | method=method, 239 | url=url, 240 | retry_options=retry_options, 241 | raise_for_status=raise_for_status, 242 | **kwargs, 243 | ) 244 | 245 | def get( 246 | self, 247 | url: _URL_TYPE, 248 | retry_options: RetryOptionsBase | None = None, 249 | raise_for_status: bool | None = None, 250 | **kwargs: Any, 251 | ) -> _RequestContext: 252 | return self._make_request( 253 | method=hdrs.METH_GET, 254 | url=url, 255 | retry_options=retry_options, 256 | raise_for_status=raise_for_status, 257 | **kwargs, 258 | ) 259 | 260 | def options( 261 | self, 262 | url: _URL_TYPE, 263 | retry_options: RetryOptionsBase | None = None, 264 | raise_for_status: bool | None = None, 265 | **kwargs: Any, 266 | ) -> _RequestContext: 267 | return self._make_request( 268 | method=hdrs.METH_OPTIONS, 269 | url=url, 270 | retry_options=retry_options, 271 | raise_for_status=raise_for_status, 272 | **kwargs, 273 | ) 274 | 275 | def head( 276 | self, 277 | url: _URL_TYPE, 278 | retry_options: RetryOptionsBase | None = None, 279 | raise_for_status: bool | None = None, 280 | **kwargs: Any, 281 | ) -> _RequestContext: 282 | return self._make_request( 283 | method=hdrs.METH_HEAD, 284 | url=url, 285 | retry_options=retry_options, 286 | raise_for_status=raise_for_status, 287 | **kwargs, 288 | ) 289 | 290 | def post( 291 | self, 292 | url: _URL_TYPE, 293 | retry_options: RetryOptionsBase | None = None, 294 | raise_for_status: bool | None = None, 295 | **kwargs: Any, 296 | ) -> _RequestContext: 297 | return self._make_request( 298 | method=hdrs.METH_POST, 299 | url=url, 300 | retry_options=retry_options, 301 | raise_for_status=raise_for_status, 302 | **kwargs, 303 | ) 304 | 305 | def put( 306 | self, 307 | url: _URL_TYPE, 308 | retry_options: RetryOptionsBase | None = None, 309 | raise_for_status: bool | None = None, 310 | **kwargs: Any, 311 | ) -> _RequestContext: 312 | return self._make_request( 313 | method=hdrs.METH_PUT, 314 | url=url, 315 | retry_options=retry_options, 316 | raise_for_status=raise_for_status, 317 | **kwargs, 318 | ) 319 | 320 | def patch( 321 | self, 322 | url: _URL_TYPE, 323 | retry_options: RetryOptionsBase | None = None, 324 | raise_for_status: bool | None = None, 325 | **kwargs: Any, 326 | ) -> _RequestContext: 327 | return self._make_request( 328 | method=hdrs.METH_PATCH, 329 | url=url, 330 | retry_options=retry_options, 331 | raise_for_status=raise_for_status, 332 | **kwargs, 333 | ) 334 | 335 | def delete( 336 | self, 337 | url: _URL_TYPE, 338 | retry_options: RetryOptionsBase | None = None, 339 | raise_for_status: bool | None = None, 340 | **kwargs: Any, 341 | ) -> _RequestContext: 342 | return self._make_request( 343 | method=hdrs.METH_DELETE, 344 | url=url, 345 | retry_options=retry_options, 346 | raise_for_status=raise_for_status, 347 | **kwargs, 348 | ) 349 | 350 | async def close(self) -> None: 351 | await self._client.close() 352 | self._closed = True 353 | 354 | def _make_request( 355 | self, 356 | method: str, 357 | url: _URL_TYPE, 358 | retry_options: RetryOptionsBase | None = None, 359 | raise_for_status: bool | None = None, 360 | **kwargs: Any, 361 | ) -> _RequestContext: 362 | url_list = _url_to_urls(url) 363 | params_list = [ 364 | RequestParams( 365 | method=method, 366 | url=url, 367 | headers=kwargs.pop("headers", {}), 368 | trace_request_ctx=kwargs.pop("trace_request_ctx", None), 369 | kwargs=kwargs, 370 | ) 371 | for url in url_list 372 | ] 373 | 374 | return self._make_requests( 375 | params_list=params_list, 376 | retry_options=retry_options, 377 | raise_for_status=raise_for_status, 378 | ) 379 | 380 | def _make_requests( 381 | self, 382 | params_list: list[RequestParams], 383 | retry_options: RetryOptionsBase | None = None, 384 | raise_for_status: bool | None = None, 385 | ) -> _RequestContext: 386 | if retry_options is None: 387 | retry_options = self._retry_options 388 | if raise_for_status is None: 389 | raise_for_status = self._raise_for_status 390 | return _RequestContext( 391 | request_func=self._client.request, 392 | params_list=params_list, 393 | logger=self._logger, 394 | retry_options=retry_options, 395 | raise_for_status=raise_for_status, 396 | ) 397 | 398 | async def __aenter__(self) -> RetryClient: # noqa: PYI034 399 | return self 400 | 401 | async def __aexit__( 402 | self, 403 | exc_type: type[BaseException] | None, 404 | exc_val: BaseException | None, 405 | exc_tb: TracebackType | None, 406 | ) -> None: 407 | await self.close() 408 | 409 | def __del__(self) -> None: 410 | if getattr(self, "_closed", None) is None: 411 | # in case object was not initialized (__init__ raised an exception) 412 | return 413 | 414 | if not self._closed: 415 | self._logger.warning("Aiohttp retry client was not closed") 416 | -------------------------------------------------------------------------------- /tests/test_client.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from types import SimpleNamespace 4 | from typing import TYPE_CHECKING 5 | 6 | import pytest 7 | from aiohttp import ( 8 | BasicAuth, 9 | ClientResponse, 10 | ClientResponseError, 11 | ClientSession, 12 | TraceConfig, 13 | TraceRequestStartParams, 14 | hdrs, 15 | ) 16 | from yarl import URL 17 | 18 | from aiohttp_retry import ExponentialRetry, ListRetry, RetryClient 19 | from aiohttp_retry.client import RequestParams 20 | from tests.app import App 21 | 22 | if TYPE_CHECKING: 23 | import pytest_aiohttp.plugin 24 | 25 | from aiohttp_retry.retry_options import RetryOptionsBase 26 | 27 | 28 | async def get_retry_client_and_test_app_for_test( 29 | aiohttp_client: pytest_aiohttp.plugin.AiohttpClient, 30 | raise_for_status: bool = False, 31 | retry_options: RetryOptionsBase | None = None, 32 | ) -> tuple[RetryClient, App]: 33 | test_app = App() 34 | app = test_app.web_app() 35 | client = await aiohttp_client(app) 36 | 37 | retry_client = RetryClient(client_session=client, retry_options=retry_options, raise_for_status=raise_for_status) 38 | return retry_client, test_app 39 | 40 | 41 | async def test_hello(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 42 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) 43 | async with retry_client.get("/ping") as response: 44 | text = await response.text() 45 | assert response.status == 200 46 | assert text == "Ok!" 47 | 48 | assert test_app.counter == 1 49 | 50 | await retry_client.close() 51 | 52 | 53 | async def test_hello_by_request(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 54 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) 55 | async with retry_client.request(method=hdrs.METH_GET, url="/ping") as response: 56 | text = await response.text() 57 | assert response.status == 200 58 | assert text == "Ok!" 59 | 60 | assert test_app.counter == 1 61 | 62 | await retry_client.close() 63 | 64 | 65 | async def test_hello_with_context(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 66 | test_app = App() 67 | app = test_app.web_app() 68 | client = await aiohttp_client(app) 69 | async with RetryClient() as retry_client: 70 | retry_client._client = client 71 | async with retry_client.get("/ping") as response: 72 | text = await response.text() 73 | assert response.status == 200 74 | assert text == "Ok!" 75 | 76 | assert test_app.counter == 1 77 | 78 | 79 | async def test_internal_error(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 80 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) 81 | retry_options = ExponentialRetry(attempts=5) 82 | async with retry_client.get("/internal_error", retry_options) as response: 83 | assert response.status == 500 84 | assert test_app.counter == 5 85 | 86 | await retry_client.close() 87 | 88 | 89 | async def test_not_found_error(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 90 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) 91 | retry_options = ExponentialRetry(attempts=5, statuses={404}) 92 | async with retry_client.get("/not_found_error", retry_options) as response: 93 | assert response.status == 404 94 | assert test_app.counter == 5 95 | 96 | await retry_client.close() 97 | 98 | 99 | async def test_sometimes_error(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 100 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) 101 | retry_options = ExponentialRetry(attempts=5) 102 | async with retry_client.get("/sometimes_error", retry_options) as response: 103 | text = await response.text() 104 | assert response.status == 200 105 | assert text == "Ok!" 106 | 107 | assert test_app.counter == 3 108 | 109 | await retry_client.close() 110 | 111 | 112 | async def test_sometimes_error_with_raise_for_status(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 113 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client, raise_for_status=True) 114 | retry_options = ExponentialRetry(attempts=5, exceptions={ClientResponseError}) 115 | async with retry_client.get("/sometimes_error", retry_options) as response: 116 | text = await response.text() 117 | assert response.status == 200 118 | assert text == "Ok!" 119 | 120 | assert test_app.counter == 3 121 | 122 | await retry_client.close() 123 | 124 | 125 | async def test_override_options(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 126 | retry_client, test_app = await get_retry_client_and_test_app_for_test( 127 | aiohttp_client, 128 | retry_options=ExponentialRetry(attempts=1), 129 | ) 130 | retry_options = ExponentialRetry(attempts=5) 131 | async with retry_client.get("/sometimes_error", retry_options) as response: 132 | text = await response.text() 133 | assert response.status == 200 134 | assert text == "Ok!" 135 | 136 | assert test_app.counter == 3 137 | 138 | await retry_client.close() 139 | 140 | 141 | async def test_hello_awaitable(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 142 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) 143 | response = await retry_client.get("/ping") 144 | text = await response.text() 145 | assert response.status == 200 146 | assert text == "Ok!" 147 | 148 | assert test_app.counter == 1 149 | 150 | await retry_client.close() 151 | 152 | 153 | async def test_add_trace_request_ctx(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 154 | actual_request_contexts = [] 155 | 156 | async def on_request_start( 157 | _: ClientSession, 158 | trace_config_ctx: SimpleNamespace, 159 | __: TraceRequestStartParams, 160 | ) -> None: 161 | actual_request_contexts.append(trace_config_ctx) 162 | 163 | test_app = App() 164 | 165 | trace_config = TraceConfig() 166 | trace_config.on_request_start.append(on_request_start) 167 | 168 | retry_client = RetryClient() 169 | retry_client._client = await aiohttp_client(test_app.web_app(), trace_configs=[trace_config]) 170 | 171 | async with retry_client.get("/sometimes_error", trace_request_ctx={"foo": "bar"}): 172 | assert test_app.counter == 3 173 | 174 | assert actual_request_contexts == [ 175 | SimpleNamespace( 176 | trace_request_ctx={ 177 | "foo": "bar", 178 | "current_attempt": i + 1, 179 | }, 180 | ) 181 | for i in range(3) 182 | ] 183 | 184 | 185 | @pytest.mark.parametrize("attempts", [2, 3]) 186 | async def test_change_urls_in_request(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient, attempts: int) -> None: 187 | retry_client, test_app = await get_retry_client_and_test_app_for_test( 188 | aiohttp_client, 189 | retry_options=ExponentialRetry(attempts=attempts), 190 | ) 191 | async with retry_client.get(url=["/internal_error", "/ping"]) as response: 192 | text = await response.text() 193 | assert response.status == 200 194 | assert text == "Ok!" 195 | 196 | assert test_app.counter == 2 197 | 198 | await retry_client.close() 199 | 200 | 201 | @pytest.mark.parametrize("attempts", [2, 3]) 202 | async def test_change_urls_as_tuple_in_request( 203 | aiohttp_client: pytest_aiohttp.plugin.AiohttpClient, attempts: int 204 | ) -> None: 205 | retry_client, test_app = await get_retry_client_and_test_app_for_test( 206 | aiohttp_client, 207 | retry_options=ExponentialRetry(attempts=attempts), 208 | ) 209 | async with retry_client.get(url=("/internal_error", "/ping")) as response: 210 | text = await response.text() 211 | assert response.status == 200 212 | assert text == "Ok!" 213 | 214 | assert test_app.counter == 2 215 | 216 | await retry_client.close() 217 | 218 | 219 | @pytest.mark.parametrize("url", [{"/ping", "/internal_error"}, []]) 220 | async def test_pass_bad_urls(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient, url: list | set) -> None: 221 | retry_client, _ = await get_retry_client_and_test_app_for_test(aiohttp_client) 222 | with pytest.raises(ValueError): 223 | async with retry_client.get(url=url): 224 | pass 225 | 226 | await retry_client.close() 227 | 228 | 229 | @pytest.mark.parametrize( 230 | ("url", "method"), 231 | [ 232 | ("/options_handler", "options"), 233 | ("/head_handler", "head"), 234 | ("/post_handler", "post"), 235 | ("/put_handler", "put"), 236 | ("/patch_handler", "patch"), 237 | ("/delete_handler", "delete"), 238 | ], 239 | ) 240 | async def test_methods(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient, url: str, method: str) -> None: 241 | retry_client, _ = await get_retry_client_and_test_app_for_test(aiohttp_client) 242 | method_func = getattr(retry_client, method) 243 | async with method_func(url) as response: 244 | assert response.method.lower() == method 245 | 246 | await retry_client.close() 247 | 248 | 249 | async def test_not_found_error_with_retry_client_raise_for_status( 250 | aiohttp_client: pytest_aiohttp.plugin.AiohttpClient, 251 | ) -> None: 252 | test_app = App() 253 | app = test_app.web_app 254 | 255 | client = await aiohttp_client(app) 256 | retry_client = RetryClient(raise_for_status=True) 257 | retry_client._client = client 258 | 259 | retry_options = ExponentialRetry(attempts=5, statuses={404}) 260 | override_response = retry_client.get("/not_found_error", retry_options, raise_for_status=False) 261 | assert not override_response._raise_for_status 262 | response = retry_client.get("/not_found_error", retry_options) 263 | assert response._raise_for_status 264 | 265 | try: 266 | async with response: 267 | pass 268 | except ClientResponseError as exc: 269 | assert exc.status == 404 # noqa: PT017 270 | assert test_app.counter == 5 271 | else: 272 | msg = "Expected ClientResponseError not raised" 273 | raise AssertionError(msg) 274 | 275 | await retry_client.close() 276 | await client.close() 277 | 278 | 279 | async def test_request(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 280 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) 281 | async with retry_client.request(hdrs.METH_GET, "/ping") as response: 282 | text = await response.text() 283 | assert response.status == 200 284 | assert text == "Ok!" 285 | 286 | assert test_app.counter == 1 287 | 288 | await retry_client.close() 289 | 290 | 291 | async def test_url_as_yarl(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 292 | """https://github.com/inyutin/aiohttp_retry/issues/41.""" 293 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) 294 | async with retry_client.get(URL("/ping")) as response: 295 | text = await response.text() 296 | assert response.status == 200 297 | assert text == "Ok!" 298 | 299 | assert test_app.counter == 1 300 | 301 | await retry_client.close() 302 | 303 | 304 | async def test_change_client_retry_options(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 305 | retry_options = ExponentialRetry(attempts=5) 306 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client, retry_options=retry_options) 307 | 308 | # first time with 5 attempts is okay 309 | async with retry_client.get("/sometimes_error") as response: 310 | text = await response.text() 311 | assert response.status == 200 312 | assert text == "Ok!" 313 | 314 | assert test_app.counter == 3 315 | 316 | test_app.counter = 0 317 | retry_client.retry_options.attempts = 2 318 | 319 | # second time with 5 attempts is error 320 | async with retry_client.get("/sometimes_error") as response: 321 | text = await response.text() 322 | assert response.status == 500 323 | assert test_app.counter == 2 324 | 325 | await retry_client.close() 326 | 327 | 328 | async def test_not_retry_server_errors(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 329 | retry_options = ExponentialRetry(retry_all_server_errors=False) 330 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) 331 | async with retry_client.get("/internal_error", retry_options) as response: 332 | assert response.status == 500 333 | assert test_app.counter == 1 334 | 335 | await retry_client.close() 336 | 337 | 338 | async def test_list_retry_works_for_multiple_attempts(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 339 | retry_options = ListRetry(timeouts=[0] * 3) 340 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) 341 | 342 | async with retry_client.get("/internal_error", retry_options) as response: 343 | assert response.status == 500 344 | assert test_app.counter == 3 345 | 346 | await retry_client.close() 347 | 348 | 349 | async def test_dont_retry_if_not_in_retry_methods(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 350 | retry_client, test_app = await get_retry_client_and_test_app_for_test( 351 | aiohttp_client, 352 | retry_options=ExponentialRetry(), # try on all methods by default 353 | ) 354 | 355 | async with retry_client.get("/internal_error") as response: 356 | assert response.status == 500 357 | assert test_app.counter == 3 358 | 359 | await retry_client.close() 360 | 361 | retry_client, test_app = await get_retry_client_and_test_app_for_test( 362 | aiohttp_client, 363 | retry_options=ExponentialRetry(methods={"POST"}), # try on only POST method 364 | ) 365 | 366 | async with retry_client.get("/internal_error") as response: 367 | assert response.status == 500 368 | assert test_app.counter == 1 369 | 370 | await retry_client.close() 371 | 372 | 373 | async def test_implicit_client(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 374 | # check that if client not passed that it created implicitly 375 | test_app = App() 376 | 377 | retry_client = RetryClient() 378 | assert retry_client._client is not None 379 | 380 | retry_client._client = await aiohttp_client(test_app.web_app()) 381 | async with retry_client.get("/ping") as response: 382 | assert response.status == 200 383 | 384 | await retry_client.close() 385 | 386 | 387 | async def test_evaluate_response_callback(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 388 | async def evaluate_response(response: ClientResponse) -> bool: 389 | try: 390 | await response.json() 391 | except: 392 | return False 393 | return True 394 | 395 | retry_options = ExponentialRetry(attempts=5, evaluate_response_callback=evaluate_response) 396 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client, retry_options=retry_options) 397 | 398 | async with retry_client.get("/sometimes_json") as response: 399 | body = await response.json() 400 | assert response.status == 200 401 | assert body == {"status": "Ok!"} 402 | 403 | assert test_app.counter == 3 404 | 405 | 406 | async def test_multiply_urls_by_requests(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 407 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) 408 | async with retry_client.requests( 409 | params_list=[ 410 | RequestParams( 411 | method="GET", 412 | url="/internal_error", 413 | ), 414 | RequestParams( 415 | method="GET", 416 | url="/ping", 417 | ), 418 | ], 419 | ) as response: 420 | text = await response.text() 421 | assert response.status == 200 422 | assert text == "Ok!" 423 | 424 | assert test_app.counter == 2 425 | 426 | await retry_client.close() 427 | 428 | 429 | async def test_multiply_methods_by_requests(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 430 | retry_options = ExponentialRetry(statuses={405}) # method not allowed 431 | retry_client, _ = await get_retry_client_and_test_app_for_test(aiohttp_client, retry_options=retry_options) 432 | async with retry_client.requests( 433 | params_list=[ 434 | RequestParams( 435 | method="POST", 436 | url="/ping", 437 | ), 438 | RequestParams( 439 | method="GET", 440 | url="/ping", 441 | ), 442 | ], 443 | ) as response: 444 | text = await response.text() 445 | assert response.status == 200 446 | assert text == "Ok!" 447 | 448 | await retry_client.close() 449 | 450 | 451 | async def test_change_headers(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 452 | retry_options = ExponentialRetry(statuses={406}) 453 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client, retry_options=retry_options) 454 | async with retry_client.requests( 455 | params_list=[ 456 | RequestParams( 457 | method="GET", 458 | url="/check_headers", 459 | ), 460 | RequestParams( 461 | method="GET", 462 | url="/check_headers", 463 | headers={"correct_headers": "True"}, 464 | ), 465 | ], 466 | ) as response: 467 | text = await response.text() 468 | assert response.status == 200 469 | assert text == "Ok!" 470 | 471 | assert test_app.counter == 2 472 | 473 | await retry_client.close() 474 | 475 | 476 | async def test_additional_params(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 477 | # https://github.com/inyutin/aiohttp_retry/issues/79 478 | auth = BasicAuth("username", "password") 479 | retry_client, _ = await get_retry_client_and_test_app_for_test(aiohttp_client) 480 | async with retry_client.request(hdrs.METH_GET, "/with_auth", auth=auth) as response: 481 | text = await response.text() 482 | assert response.status == 200 483 | assert text == "Ok!" 484 | 485 | await retry_client.close() 486 | 487 | 488 | async def test_request_headers(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 489 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) 490 | async with retry_client.get(url="/check_headers", headers={"correct_headers": "True"}) as response: 491 | text = await response.text() 492 | assert response.status == 200 493 | assert text == "Ok!" 494 | 495 | assert test_app.counter == 1 496 | 497 | await retry_client.close() 498 | 499 | 500 | async def test_list_retry_all_failed(aiohttp_client: pytest_aiohttp.plugin.AiohttpClient) -> None: 501 | # there was a specific bug 502 | async def evaluate_response(response: ClientResponse) -> bool: 503 | return False 504 | 505 | retry_options = ListRetry(timeouts=[1] * 3, statuses={403}, evaluate_response_callback=evaluate_response) 506 | retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) 507 | 508 | async with retry_client.get("/with_auth", retry_options=retry_options) as response: 509 | assert response.status == 403 510 | assert test_app.counter == 3 511 | 512 | await retry_client.close() 513 | --------------------------------------------------------------------------------