├── src ├── gracy │ ├── py.typed │ ├── _general.py │ ├── _configs.py │ ├── replays │ │ ├── storages │ │ │ ├── _sqlite_schema.py │ │ │ ├── sqlite.py │ │ │ ├── _base.py │ │ │ └── pymongo.py │ │ └── _wrappers.py │ ├── _validators.py │ ├── __init__.py │ ├── _types.py │ ├── _reports │ │ ├── _models.py │ │ ├── _builders.py │ │ └── _printers.py │ ├── _paginator.py │ ├── _loggers.py │ ├── exceptions.py │ ├── common_hooks.py │ └── _models.py └── tests │ ├── test_generators.py │ ├── test_gracy_httpx.py │ ├── generate_test_db.py │ ├── test_namespaces.py │ ├── test_parsers.py │ ├── test_loggers.py │ ├── conftest.py │ ├── test_validators.py │ ├── test_hooks.py │ └── test_retry.py ├── img ├── logo.png ├── report-rich-example.png └── report-plotly-example.png ├── .gitignore ├── .gracy └── pokeapi.sqlite3 ├── .flake8 ├── .github ├── FUNDING.yml └── workflows │ ├── release.yml │ └── ci.yml ├── docker-compose.yml ├── .pre-commit-config.yaml ├── todo.md ├── .vscode └── settings.json ├── examples ├── httpbin_post.py ├── memory.py ├── pokestarwarsapi.py ├── pokeapi_retry.py ├── pokeapi_namespaces.py ├── pokeapi.py ├── pokeapi_replay_mongo.py ├── pokeapi_replay.py ├── pokeapi_throttle.py └── pokeapi_limit_concurrency.py ├── LICENSE ├── pyproject.toml └── CHANGELOG.md /src/gracy/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guilatrova/gracy/HEAD/img/logo.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.pyc 3 | *.log 4 | .DS_Store 5 | dist/ 6 | .mongo 7 | .venv/ 8 | -------------------------------------------------------------------------------- /.gracy/pokeapi.sqlite3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guilatrova/gracy/HEAD/.gracy/pokeapi.sqlite3 -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length=120 3 | extend-ignore=E203 4 | exclude= 5 | __init__.py 6 | -------------------------------------------------------------------------------- /img/report-rich-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guilatrova/gracy/HEAD/img/report-rich-example.png -------------------------------------------------------------------------------- /img/report-plotly-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guilatrova/gracy/HEAD/img/report-plotly-example.png -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: guilatrova 2 | custom: https://www.paypal.com/donate/?business=SUQKVABPUHUUQ&no_recurring=0&item_name=Thank+you+very+much+for+considering+supporting+my+work.+%E2%9D%A4%EF%B8%8F+It+keeps+me+motivated+to+keep+producing+value+for+you.¤cy_code=USD 3 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | 3 | services: 4 | gracy_mongo: 5 | image: mongo:latest 6 | restart: always 7 | environment: 8 | MONGO_INITDB_ROOT_USERNAME: root 9 | MONGO_INITDB_ROOT_PASSWORD: example 10 | ports: 11 | - 27017:27017 12 | volumes: 13 | - .mongo/data:/data/db 14 | -------------------------------------------------------------------------------- /src/gracy/_general.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import typing as t 4 | 5 | VALID_BUILD_REQUEST_KEYS = { 6 | "content", 7 | "data", 8 | "files", 9 | "json", 10 | "params", 11 | "headers", 12 | "cookies", 13 | "timeout", 14 | "extensions", 15 | } 16 | """ 17 | There're some kwargs that are handled by httpx request, but only a few are properly handled by https build_request. 18 | Defined in httpx._client:322 19 | """ 20 | 21 | 22 | def extract_request_kwargs(kwargs: dict[str, t.Any]) -> dict[str, t.Any]: 23 | return {k: v for k, v in kwargs.items() if k in VALID_BUILD_REQUEST_KEYS} 24 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.4.0 4 | hooks: 5 | - id: end-of-file-fixer 6 | - id: trailing-whitespace 7 | - repo: https://github.com/astral-sh/ruff-pre-commit 8 | rev: v0.1.5 9 | hooks: 10 | - id: ruff 11 | args: [--fix, --exit-non-zero-on-fix] 12 | - id: ruff-format 13 | - repo: local 14 | hooks: 15 | - id: pyright 16 | name: pyright 17 | language: system 18 | types: [python] 19 | entry: "poetry run pyright" 20 | require_serial: true # use require_serial so that script is only called once per commit 21 | verbose: true # print the number of files as a sanity-check 22 | -------------------------------------------------------------------------------- /todo.md: -------------------------------------------------------------------------------- 1 | # Todo 2 | - [x] Strict status code 3 | - [x] Allowed status code 4 | - [x] Retry 5 | - [x] Retry but pass 6 | - [x] Retry logging 7 | - [x] Metrics (% of successful calls) 8 | - [x] Status codes 9 | - [x] % per status code 10 | - [x] avg ms taken to execute 11 | - [x] Parsing 12 | - [x] default = lambda r: r.json() 13 | - [x] 200 = lambda r: r.text() 14 | - [x] 404 = None 15 | - [x] 401 = InadequatePermissions 16 | - [x] Throttle 17 | - [x] URL regex support 18 | - [ ] Authorization 19 | - [ ] Validate if token is still valid 20 | - [ ] Auto refresh 21 | - [ ] Docs 22 | - [x] Readme 23 | - [x] Methods without `_` 24 | - [ ] Contributing 25 | - [ ] Allow to specify status ranges 26 | - [x] Custom exception for parsing errors 27 | - [x] Replay/Record payloads 28 | - [x] SQLite 29 | - [ ] Mongo 30 | - [ ] Custom Storage 31 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.analysis.typeCheckingMode": "basic", 3 | "editor.formatOnSave": true, 4 | "[python]": { 5 | "editor.rulers": [ 6 | 120 7 | ], 8 | "editor.formatOnSave": true, 9 | "editor.defaultFormatter": "charliermarsh.ruff", 10 | "editor.codeActionsOnSave": { 11 | "source.fixAll.ruff": "explicit", 12 | "source.organizeImports.ruff": "explicit" 13 | }, 14 | }, 15 | "python.testing.pytestArgs": [ 16 | "src/tests" 17 | ], 18 | "python.testing.unittestEnabled": false, 19 | "python.testing.pytestEnabled": true, 20 | "files.exclude": { 21 | "**/.git": true, 22 | "**/.svn": true, 23 | "**/.hg": true, 24 | "**/CVS": true, 25 | "**/__pycache__": true, 26 | ".mypy_cache": true, 27 | ".pytest_cache": true, 28 | "**/.DS_Store": true, 29 | "**/Thumbs.db": true, 30 | ".venv/**": true, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/gracy/_configs.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from contextlib import contextmanager 4 | from contextvars import ContextVar 5 | 6 | from ._models import GracyConfig 7 | 8 | custom_config_context: ContextVar[GracyConfig | None] = ContextVar( 9 | "gracy_context", default=None 10 | ) 11 | within_hook_context: ContextVar[bool] = ContextVar("within_hook_context", default=False) 12 | 13 | 14 | @contextmanager 15 | def custom_gracy_config(config: GracyConfig): 16 | token = custom_config_context.set(config) 17 | 18 | try: 19 | yield 20 | finally: 21 | try: 22 | custom_config_context.reset(token) 23 | except Exception: 24 | pass # Best effort 25 | 26 | 27 | @contextmanager 28 | def within_hook(): 29 | token = within_hook_context.set(True) 30 | 31 | try: 32 | yield 33 | finally: 34 | within_hook_context.reset(token) 35 | -------------------------------------------------------------------------------- /examples/httpbin_post.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | import httpx 5 | import typing as t 6 | 7 | from gracy import Gracy 8 | 9 | 10 | class GracefulHttpbin(Gracy[str]): 11 | class Config: 12 | BASE_URL = "https://httpbin.org/" 13 | 14 | async def post_json_example(self): 15 | res = await self.post( 16 | "post", None, json={"test": "json"}, headers={"header1": "1"} 17 | ) 18 | return res 19 | 20 | async def post_data_example(self): 21 | res = await self.post("post", None, data="data", headers={"header2": "2"}) 22 | return res 23 | 24 | 25 | async def main(): 26 | api = GracefulHttpbin() 27 | 28 | json_res = t.cast(httpx.Response, await api.post_json_example()) 29 | data_res = t.cast(httpx.Response, await api.post_data_example()) 30 | 31 | print(json_res.json()) 32 | print("-" * 100) 33 | print(data_res.json()) 34 | 35 | 36 | if __name__ == "__main__": 37 | asyncio.run(main()) 38 | -------------------------------------------------------------------------------- /src/tests/test_generators.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import pytest 4 | import typing as t 5 | 6 | from gracy import Gracy, graceful_generator 7 | from tests.conftest import REPLAY, PokeApiEndpoint 8 | 9 | 10 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 11 | class Config: 12 | BASE_URL = "https://pokeapi.co/api/v2/" 13 | 14 | @graceful_generator(parser={"default": lambda r: r.json()}) 15 | async def get_2_yield_graceful(self): 16 | names = ["charmander", "pikachu"] 17 | 18 | for name in names: 19 | r = await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 20 | yield r 21 | 22 | 23 | @pytest.fixture() 24 | def make_pokeapi(): 25 | def factory(): 26 | Gracy.dangerously_reset_report() 27 | return GracefulPokeAPI(REPLAY) 28 | 29 | return factory 30 | 31 | 32 | async def test_pokemon_ok_json(make_pokeapi: t.Callable[[], GracefulPokeAPI]): 33 | pokeapi = make_pokeapi() 34 | count = 0 35 | 36 | async for _ in pokeapi.get_2_yield_graceful(): 37 | count += 1 38 | 39 | assert count == 2 40 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Semantic Release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | release: 10 | runs-on: ubuntu-latest 11 | concurrency: release 12 | 13 | steps: 14 | - uses: actions/checkout@v2 15 | with: 16 | fetch-depth: 0 17 | token: ${{ secrets.GRACY_GITHUB_TOKEN }} 18 | 19 | - name: Install poetry 20 | shell: bash 21 | run: pipx install poetry 22 | 23 | - name: Set up Python 3.10 24 | uses: actions/setup-python@v4 25 | with: 26 | python-version: "3.10" 27 | cache: poetry 28 | 29 | - name: Install dependencies 30 | run: poetry install --with=dev 31 | 32 | - name: Python Semantic Release 33 | run: | 34 | git config --global user.name "github-actions" 35 | git config --global user.email "action@github.com" 36 | 37 | poetry run semantic-release publish -D commit_author="github-actions " 38 | env: 39 | GH_TOKEN: ${{secrets.GRACY_GITHUB_TOKEN}} 40 | PYPI_TOKEN: ${{secrets.PYPI_TOKEN}} 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2021 Guilherme Latrova 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/gracy/replays/storages/_sqlite_schema.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import typing as t 4 | 5 | TABLE_NAME: t.Final = "gracy_recordings" 6 | 7 | CREATE_RECORDINGS_TABLE: t.Final = f""" 8 | CREATE TABLE {TABLE_NAME}( 9 | url VARCHAR(255) NOT NULL, 10 | method VARCHAR(20) NOT NULL, 11 | request_body BLOB NULL, 12 | response BLOB NOT NULL, 13 | updated_at DATETIME NOT NULL 14 | ) 15 | """ 16 | 17 | INDEX_RECORDINGS_TABLE: t.Final = f""" 18 | CREATE UNIQUE INDEX idx_gracy_request 19 | ON {TABLE_NAME}(url, method, request_body) 20 | """ 21 | 22 | INDEX_RECORDINGS_TABLE_WITHOUT_REQUEST_BODY: t.Final = f""" 23 | CREATE UNIQUE INDEX idx_gracy_request_empty_req_body 24 | ON {TABLE_NAME}(url, method) 25 | WHERE request_body IS NULL 26 | """ 27 | 28 | INSERT_RECORDING_BASE: t.Final = f""" 29 | INSERT OR REPLACE INTO {TABLE_NAME} 30 | VALUES (?, ?, ?, ?, ?) 31 | """ 32 | 33 | FIND_REQUEST_WITH_REQ_BODY: t.Final = f""" 34 | SELECT response, updated_at FROM {TABLE_NAME} 35 | WHERE 36 | url = ? AND 37 | method = ? AND 38 | request_body = ? 39 | """ 40 | 41 | FIND_REQUEST_WITHOUT_REQ_BODY: t.Final = f""" 42 | SELECT response, updated_at FROM {TABLE_NAME} 43 | WHERE 44 | url = ? AND 45 | method = ? AND 46 | request_body IS NULL 47 | """ 48 | -------------------------------------------------------------------------------- /examples/memory.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import httpx 4 | from dataclasses import dataclass 5 | from time import sleep 6 | 7 | from gracy import ( 8 | BaseEndpoint, 9 | Gracy, 10 | GracyRequestContext, 11 | ) 12 | from gracy.exceptions import GracyUserDefinedException 13 | 14 | 15 | class PokemonNotFound(GracyUserDefinedException): 16 | BASE_MESSAGE = "Unable to find a pokemon with the name [{NAME}] at {URL} due to {STATUS} status" 17 | 18 | def _format_message( 19 | self, request_context: GracyRequestContext, response: httpx.Response 20 | ) -> str: 21 | format_args = self._build_default_args() 22 | name = request_context.endpoint_args.get("NAME", "Unknown") 23 | return self.BASE_MESSAGE.format(NAME=name, **format_args) 24 | 25 | 26 | class PokeApiEndpoint(BaseEndpoint): 27 | GET_POKEMON = "/pokemon/{NAME}" 28 | GET_GENERATION = "/generation/{ID}" 29 | 30 | 31 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 32 | class Config: 33 | BASE_URL = "https://pokeapi.co/api/v2/" 34 | 35 | pass 36 | 37 | 38 | @dataclass 39 | class Test: 40 | pass 41 | 42 | 43 | def main(): 44 | while True: 45 | GracefulPokeAPI() 46 | sleep(1) 47 | 48 | 49 | if __name__ == "__main__": 50 | main() 51 | -------------------------------------------------------------------------------- /src/tests/test_gracy_httpx.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import pytest 4 | import typing as t 5 | from http import HTTPStatus 6 | 7 | from gracy import GracefulRetry, Gracy, GracyConfig 8 | from tests.conftest import PRESENT_POKEMON_NAME, REPLAY, PokeApiEndpoint 9 | 10 | RETRY: t.Final = GracefulRetry( 11 | delay=0.001, 12 | max_attempts=2, 13 | retry_on={HTTPStatus.NOT_FOUND}, 14 | behavior="break", 15 | ) 16 | 17 | 18 | @pytest.fixture() 19 | def make_pokeapi(): 20 | def factory(): 21 | Gracy.dangerously_reset_report() 22 | return GracefulPokeAPI(REPLAY) 23 | 24 | return factory 25 | 26 | 27 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 28 | class Config: 29 | BASE_URL = "https://pokeapi.co/api/v2/" 30 | SETTINGS = GracyConfig( 31 | retry=RETRY, 32 | allowed_status_code={HTTPStatus.NOT_FOUND}, 33 | parser={HTTPStatus.NOT_FOUND: None}, 34 | ) 35 | 36 | 37 | MAKE_POKEAPI_TYPE = t.Callable[[], GracefulPokeAPI] 38 | 39 | 40 | async def test_pass_kwargs(make_pokeapi: MAKE_POKEAPI_TYPE): 41 | pokeapi = make_pokeapi() 42 | 43 | await pokeapi.get( 44 | PokeApiEndpoint.GET_POKEMON, 45 | dict(NAME=PRESENT_POKEMON_NAME), 46 | follow_redirects=True, 47 | ) 48 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - "docs/**" 7 | - "*.md" 8 | 9 | pull_request: 10 | paths-ignore: 11 | - "docs/**" 12 | - "*.md" 13 | 14 | jobs: 15 | test: 16 | # We want to run on external PRs, but not on our own internal PRs as they'll be run 17 | # by the push to the branch. Without this if check, checks are duplicated since 18 | # internal PRs match both the push and pull_request events. 19 | if: 20 | github.event_name == 'push' || github.event.pull_request.head.repo.full_name != 21 | github.repository 22 | 23 | strategy: 24 | fail-fast: false 25 | matrix: 26 | python-version: [3.8, 3.9, "3.10", 3.11] 27 | os: [ubuntu-latest] #, macOS-latest, windows-latest] 28 | 29 | runs-on: ${{ matrix.os }} 30 | 31 | steps: 32 | - uses: actions/checkout@v2 33 | 34 | - name: Install poetry 35 | shell: bash 36 | run: pipx install poetry 37 | 38 | - name: Set up Python 39 | uses: actions/setup-python@v4 40 | with: 41 | python-version: ${{ matrix.python-version }} 42 | cache: poetry 43 | # key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} 44 | 45 | - name: Install dependencies 46 | run: poetry install --with=dev 47 | 48 | - name: Lint 49 | uses: pre-commit/action@v3.0.0 50 | 51 | - name: Unit tests 52 | run: poetry run pytest -vvv 53 | -------------------------------------------------------------------------------- /src/gracy/_validators.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import typing as t 4 | 5 | import httpx 6 | 7 | from ._models import GracefulValidator 8 | from .exceptions import NonOkResponse, UnexpectedResponse 9 | 10 | 11 | class DefaultValidator(GracefulValidator): 12 | def check(self, response: httpx.Response) -> None: 13 | if response.is_success: 14 | return None 15 | 16 | raise NonOkResponse(str(response.url), response) 17 | 18 | 19 | class StrictStatusValidator(GracefulValidator): 20 | def __init__(self, status_code: t.Union[int, t.Iterable[int]]) -> None: 21 | if isinstance(status_code, t.Iterable): 22 | self._status_codes = status_code 23 | else: 24 | self._status_codes = {status_code} 25 | 26 | def check(self, response: httpx.Response) -> None: 27 | if response.status_code in self._status_codes: 28 | return None 29 | 30 | raise UnexpectedResponse(str(response.url), response, self._status_codes) 31 | 32 | 33 | class AllowedStatusValidator(GracefulValidator): 34 | def __init__(self, status_code: t.Union[int, t.Iterable[int]]) -> None: 35 | if isinstance(status_code, t.Iterable): 36 | self._status_codes = status_code 37 | else: 38 | self._status_codes = {status_code} 39 | 40 | def check(self, response: httpx.Response) -> None: 41 | if response.is_success: 42 | return None 43 | 44 | if response.status_code in self._status_codes: 45 | return None 46 | 47 | raise NonOkResponse(str(response.url), response) 48 | -------------------------------------------------------------------------------- /examples/pokestarwarsapi.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | from http import HTTPStatus 5 | 6 | from gracy import BaseEndpoint, Gracy, LogEvent, LogLevel, graceful 7 | 8 | 9 | class PokeApiEndpoint(BaseEndpoint): 10 | GET_POKEMON = "/pokemon/{NAME}" 11 | 12 | 13 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 14 | class Config: 15 | BASE_URL = "https://pokeapi.co/api/v2/" 16 | 17 | @graceful( 18 | strict_status_code={HTTPStatus.OK}, 19 | log_request=LogEvent(LogLevel.INFO), 20 | parser={ 21 | "default": lambda r: r.json()["name"], 22 | HTTPStatus.NOT_FOUND: None, 23 | }, 24 | ) 25 | async def get_pokemon(self, name: str): 26 | return await self.get[str](PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 27 | 28 | 29 | class StarWarsAPI(Gracy[str]): 30 | class Config: 31 | BASE_URL = "https://swapi.dev/api/" 32 | 33 | @graceful( 34 | strict_status_code=HTTPStatus.OK, 35 | log_request=LogEvent(LogLevel.INFO), 36 | parser={"default": lambda r: r.json()["name"]}, 37 | ) 38 | async def get_person(self, person_id: int): 39 | return await self.get[str]("people/{PERSON_ID}", {"PERSON_ID": str(person_id)}) 40 | 41 | 42 | pokeapi = GracefulPokeAPI() 43 | swapi = StarWarsAPI() 44 | 45 | 46 | async def main(): 47 | try: 48 | pk: str | None = await pokeapi.get_pokemon("pikachu") 49 | sw: str = await swapi.get_person(1) 50 | 51 | print("PK: result of get_pokemon:", pk) 52 | print("SW: result of get_person:", sw) 53 | 54 | finally: 55 | pokeapi.report_status("rich") 56 | 57 | 58 | asyncio.run(main()) 59 | -------------------------------------------------------------------------------- /src/tests/generate_test_db.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | import typing as t 5 | 6 | from gracy import BaseEndpoint, Gracy, GracyReplay 7 | from gracy.replays.storages.sqlite import SQLiteReplayStorage 8 | 9 | 10 | class PokeApiEndpoint(BaseEndpoint): 11 | GET_POKEMON = "/pokemon/{NAME}" 12 | GET_BERRY = "/berry/{NAME}" 13 | GET_GENERATION = "/generation/{ID}" 14 | 15 | 16 | class GracefulPokeAPIRecorder(Gracy[PokeApiEndpoint]): 17 | class Config: 18 | BASE_URL = "https://pokeapi.co/api/v2/" 19 | 20 | def __init__(self) -> None: 21 | record_mode: t.Final = GracyReplay( 22 | "record", 23 | SQLiteReplayStorage("pokeapi.sqlite3"), 24 | ) 25 | 26 | super().__init__(record_mode) 27 | 28 | async def get_pokemon(self, name: str): 29 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 30 | 31 | async def get_berry(self, name: str): 32 | return await self.get(PokeApiEndpoint.GET_BERRY, {"NAME": name}) 33 | 34 | async def get_generation(self, gen: int): 35 | return await self.get(PokeApiEndpoint.GET_GENERATION, {"ID": str(gen)}) 36 | 37 | 38 | async def main(): 39 | pokeapi = GracefulPokeAPIRecorder() 40 | poke_names = {"pikachu", "elekid", "charmander", "blaziken", "hitmonchan"} 41 | 42 | try: 43 | get_pokemons = [ 44 | asyncio.create_task(pokeapi.get_pokemon(name)) for name in poke_names 45 | ] 46 | get_gens = [ 47 | asyncio.create_task(pokeapi.get_generation(gen_id)) 48 | for gen_id in range(1, 3) 49 | ] 50 | get_berries = [asyncio.create_task(pokeapi.get_berry("cheri"))] 51 | 52 | await asyncio.gather(*(get_pokemons + get_gens + get_berries)) 53 | 54 | finally: 55 | pokeapi.report_status("rich") 56 | 57 | 58 | if __name__ == "__main__": 59 | asyncio.run(main()) 60 | -------------------------------------------------------------------------------- /examples/pokeapi_retry.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | from http import HTTPStatus 5 | 6 | from gracy import ( 7 | BaseEndpoint, 8 | GracefulRetry, 9 | Gracy, 10 | GracyReplay, 11 | LogEvent, 12 | LogLevel, 13 | graceful, 14 | ) 15 | from gracy.replays.storages.sqlite import SQLiteReplayStorage 16 | 17 | retry = GracefulRetry( 18 | delay=1, 19 | max_attempts=3, 20 | delay_modifier=1.2, 21 | retry_on=None, 22 | log_before=LogEvent(LogLevel.WARNING), 23 | log_after=LogEvent(LogLevel.WARNING), 24 | log_exhausted=LogEvent(LogLevel.CRITICAL), 25 | behavior="pass", 26 | ) 27 | 28 | 29 | class ServerIsOutError(Exception): 30 | pass 31 | 32 | 33 | class PokeApiEndpoint(BaseEndpoint): 34 | GET_POKEMON = "/pokemon/{NAME}" 35 | 36 | 37 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 38 | class Config: 39 | BASE_URL = "https://pokeapi.co/api/v2/" 40 | 41 | @graceful( 42 | strict_status_code={HTTPStatus.OK}, 43 | retry=retry, 44 | log_errors=LogEvent(LogLevel.ERROR), 45 | parser={ 46 | "default": lambda r: r.json()["name"], 47 | HTTPStatus.NOT_FOUND: None, 48 | HTTPStatus.INTERNAL_SERVER_ERROR: ServerIsOutError, 49 | }, 50 | ) 51 | async def get_pokemon(self, name: str): 52 | return await self.get[str](PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 53 | 54 | 55 | record = GracyReplay("record", SQLiteReplayStorage("pokeapi.sqlite3")) 56 | pokeapi = GracefulPokeAPI(record) 57 | 58 | 59 | async def main(): 60 | try: 61 | p1: str | None = await pokeapi.get_pokemon("pikachu") # 1 req = 200 62 | print("P1: result of get_pokemon:", p1) 63 | 64 | p2: str | None = await pokeapi.get_pokemon("doesnt-exist") # 1+3 req = 404 65 | print("P2: result of get_pokemon:", p2) 66 | 67 | finally: 68 | pokeapi.report_status("rich") 69 | 70 | 71 | asyncio.run(main()) 72 | -------------------------------------------------------------------------------- /src/gracy/__init__.py: -------------------------------------------------------------------------------- 1 | """Gracefully manage your API interactions""" 2 | from __future__ import annotations 3 | 4 | import logging 5 | 6 | from . import common_hooks, exceptions, replays 7 | from ._core import Gracy, GracyNamespace, graceful, graceful_generator 8 | from ._models import ( 9 | DEFAULT_CONFIG, 10 | BaseEndpoint, 11 | ConcurrentRequestLimit, 12 | GracefulRetry, 13 | GracefulRetryState, 14 | GracefulThrottle, 15 | GracefulValidator, 16 | GracyConfig, 17 | GracyRequestContext, 18 | LogEvent, 19 | LogLevel, 20 | OverrideRetryOn, 21 | ThrottleRule, 22 | ) 23 | from ._paginator import GracyOffsetPaginator, GracyPaginator 24 | from ._reports._models import GracyAggregatedRequest, GracyAggregatedTotal, GracyReport 25 | from ._types import generated_parsed_response, parsed_response 26 | from .replays.storages._base import GracyReplay, GracyReplayStorage, ReplayLogEvent 27 | 28 | logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") 29 | 30 | __version__ = "1.34.0" 31 | 32 | __all__ = [ 33 | "exceptions", 34 | # Core 35 | "Gracy", 36 | "GracyNamespace", 37 | "graceful", 38 | "graceful_generator", 39 | # Paginatior 40 | "GracyPaginator", 41 | "GracyOffsetPaginator", 42 | # Models 43 | "BaseEndpoint", 44 | "GracefulRetry", 45 | "OverrideRetryOn", 46 | "GracefulRetryState", 47 | "GracefulValidator", 48 | "GracyRequestContext", 49 | "LogEvent", 50 | "LogLevel", 51 | "GracefulThrottle", 52 | "ThrottleRule", 53 | "GracyConfig", 54 | "DEFAULT_CONFIG", 55 | "ConcurrentRequestLimit", 56 | # Replays 57 | "replays", 58 | "GracyReplay", 59 | "GracyReplayStorage", 60 | "ReplayLogEvent", 61 | # Reports 62 | "GracyReport", 63 | "GracyAggregatedTotal", 64 | "GracyAggregatedRequest", 65 | # Hooks 66 | "common_hooks", 67 | # Types 68 | "parsed_response", 69 | "generated_parsed_response", 70 | ] 71 | -------------------------------------------------------------------------------- /src/tests/test_namespaces.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import pytest 4 | import typing as t 5 | from http import HTTPStatus 6 | 7 | from gracy import GracefulRetry, Gracy, GracyConfig, GracyNamespace 8 | from tests.conftest import ( 9 | PRESENT_BERRY_NAME, 10 | PRESENT_POKEMON_NAME, 11 | REPLAY, 12 | PokeApiEndpoint, 13 | assert_muiti_endpoints_requests_made, 14 | ) 15 | 16 | RETRY: t.Final = GracefulRetry( 17 | delay=0.001, 18 | max_attempts=2, 19 | retry_on={HTTPStatus.NOT_FOUND}, 20 | behavior="break", 21 | ) 22 | 23 | 24 | class PokemonNamespace(GracyNamespace[PokeApiEndpoint]): 25 | async def get_one(self, name: str): 26 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 27 | 28 | 29 | class BerryNamespace(GracyNamespace[PokeApiEndpoint]): 30 | async def get_one(self, name: str): 31 | return await self.get(PokeApiEndpoint.GET_BERRY, {"NAME": name}) 32 | 33 | 34 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 35 | class Config: 36 | BASE_URL = "https://pokeapi.co/api/v2/" 37 | SETTINGS = GracyConfig( 38 | retry=RETRY, 39 | allowed_status_code={HTTPStatus.NOT_FOUND}, 40 | parser={HTTPStatus.NOT_FOUND: None}, 41 | ) 42 | 43 | berry: BerryNamespace 44 | pokemon: PokemonNamespace 45 | 46 | 47 | @pytest.fixture() 48 | def make_pokeapi(): 49 | def factory(): 50 | Gracy.dangerously_reset_report() 51 | return GracefulPokeAPI(REPLAY) 52 | 53 | return factory 54 | 55 | 56 | MAKE_POKEAPI_TYPE = t.Callable[[], GracefulPokeAPI] 57 | 58 | 59 | async def test_get_from_namespaces(make_pokeapi: MAKE_POKEAPI_TYPE): 60 | pokeapi = make_pokeapi() 61 | 62 | await pokeapi.pokemon.get_one(PRESENT_POKEMON_NAME) 63 | await pokeapi.berry.get_one(PRESENT_BERRY_NAME) 64 | 65 | EXPECTED_ENDPOINTS = 2 66 | EXPECTED_REQUESTS = (1, 1) 67 | 68 | assert_muiti_endpoints_requests_made( 69 | pokeapi, EXPECTED_ENDPOINTS, *EXPECTED_REQUESTS 70 | ) 71 | -------------------------------------------------------------------------------- /src/gracy/_types.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import httpx 4 | import sys 5 | import typing as t 6 | from http import HTTPStatus 7 | 8 | from typing_extensions import deprecated 9 | 10 | if sys.version_info >= (3, 10): 11 | from typing import ParamSpec 12 | else: 13 | from typing_extensions import ParamSpec 14 | 15 | 16 | class Unset: 17 | """ 18 | The default "unset" state indicates that whatever default is set on the 19 | client should be used. This is different to setting `None`, which 20 | explicitly disables the parameter, possibly overriding a client default. 21 | """ 22 | 23 | def __bool__(self): 24 | return False 25 | 26 | 27 | PARSER_KEY = t.Union[HTTPStatus, int, t.Literal["default"]] 28 | PARSER_VALUE = t.Union[t.Type[Exception], t.Callable[[httpx.Response], t.Any], None] 29 | PARSER_TYPE = t.Union[t.Dict[PARSER_KEY, PARSER_VALUE], Unset, None] 30 | 31 | UNSET_VALUE: t.Final = Unset() 32 | 33 | 34 | P = ParamSpec("P") 35 | T = t.TypeVar("T") 36 | 37 | 38 | @deprecated("Use typed http methods instead e.g. `self.get[DesiredType]()`") 39 | def parsed_response(return_type: t.Type[T]): # type: ignore 40 | def _decorated( 41 | func: t.Callable[P, t.Any] 42 | ) -> t.Callable[P, t.Coroutine[t.Any, t.Any, T]]: 43 | async def _gracy_method(*args: P.args, **kwargs: P.kwargs) -> T: 44 | return await func(*args, **kwargs) 45 | 46 | return _gracy_method 47 | 48 | return _decorated 49 | 50 | 51 | @deprecated("Use typed http methods instead e.g. `self.get[DesiredType]()`") 52 | def generated_parsed_response(return_type: t.Type[T]): # type: ignore 53 | def _decorated( 54 | func: t.Callable[P, t.AsyncGenerator[t.Any, t.Any]] 55 | ) -> t.Callable[P, t.AsyncGenerator[T, t.Any]]: 56 | async def _gracy_method( 57 | *args: P.args, **kwargs: P.kwargs 58 | ) -> t.AsyncGenerator[T, t.Any]: 59 | async for i in func(*args, **kwargs): 60 | yield i 61 | 62 | return _gracy_method 63 | 64 | return _decorated 65 | -------------------------------------------------------------------------------- /src/tests/test_parsers.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import httpx 4 | import pytest 5 | import typing as t 6 | from http import HTTPStatus 7 | 8 | from gracy import Gracy, GracyConfig, graceful 9 | from gracy.exceptions import GracyParseFailed 10 | from tests.conftest import ( 11 | MISSING_NAME, 12 | PRESENT_POKEMON_NAME, 13 | REPLAY, 14 | PokeApiEndpoint, 15 | assert_one_request_made, 16 | ) 17 | 18 | 19 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 20 | class Config: 21 | BASE_URL = "https://pokeapi.co/api/v2/" 22 | SETTINGS = GracyConfig(allowed_status_code=HTTPStatus.NOT_FOUND) 23 | 24 | @graceful(parser={"default": lambda r: r.json()}) 25 | async def get_pokemon(self, name: str): 26 | return await self.get[t.Dict[str, t.Any]]( 27 | PokeApiEndpoint.GET_POKEMON, {"NAME": name} 28 | ) 29 | 30 | @graceful(parser={HTTPStatus.NOT_FOUND: lambda r: None}) 31 | async def get_pokemon_not_found_as_none(self, name: str): 32 | return await self.get[t.Optional[httpx.Response]]( 33 | PokeApiEndpoint.GET_POKEMON, {"NAME": name} 34 | ) 35 | 36 | 37 | @pytest.fixture() 38 | def make_pokeapi(): 39 | def factory(): 40 | Gracy.dangerously_reset_report() 41 | return GracefulPokeAPI(REPLAY) 42 | 43 | return factory 44 | 45 | 46 | async def test_pokemon_ok_json(make_pokeapi: t.Callable[[], GracefulPokeAPI]): 47 | pokeapi = make_pokeapi() 48 | 49 | result: dict[str, t.Any] = await pokeapi.get_pokemon(PRESENT_POKEMON_NAME) 50 | 51 | assert isinstance(result, dict) 52 | assert "name" in result 53 | assert result["name"] == PRESENT_POKEMON_NAME 54 | assert_one_request_made(pokeapi) 55 | 56 | 57 | async def test_pokemon_bad_json(make_pokeapi: t.Callable[[], GracefulPokeAPI]): 58 | pokeapi = make_pokeapi() 59 | 60 | with pytest.raises(GracyParseFailed): 61 | await pokeapi.get_pokemon(MISSING_NAME) 62 | 63 | assert_one_request_made(pokeapi) 64 | 65 | 66 | async def test_pokemon_not_found_as_none(make_pokeapi: t.Callable[[], GracefulPokeAPI]): 67 | pokeapi = make_pokeapi() 68 | 69 | result = await pokeapi.get_pokemon_not_found_as_none(MISSING_NAME) 70 | 71 | assert result is None 72 | assert_one_request_made(pokeapi) 73 | -------------------------------------------------------------------------------- /src/tests/test_loggers.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import httpx 4 | import logging 5 | import pytest 6 | import typing as t 7 | 8 | from gracy import GracefulValidator, Gracy, GracyConfig, LogEvent, LogLevel 9 | from gracy.exceptions import NonOkResponse 10 | from tests.conftest import MISSING_NAME, PRESENT_POKEMON_NAME, REPLAY, PokeApiEndpoint 11 | 12 | 13 | class CustomValidator(GracefulValidator): 14 | def check(self, response: httpx.Response) -> None: 15 | if response.json()["order"] != 47: 16 | raise ValueError("Pokemon #order should be 47") # noqa: TRY003 17 | 18 | 19 | def assert_log(record: logging.LogRecord, expected_event: LogEvent): 20 | assert record.levelno == expected_event.level 21 | assert record.message == expected_event.custom_message # No formatting set 22 | 23 | 24 | # NOTE: captest only captures >=warning 25 | ON_REQUEST: t.Final = LogEvent(LogLevel.WARNING, "LOG_REQUEST") 26 | ON_RESPONSE: t.Final = LogEvent(LogLevel.ERROR, "LOG_RESPONSE") 27 | ON_ERROR: t.Final = LogEvent(LogLevel.CRITICAL, "LOG_ERROR") 28 | 29 | 30 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 31 | class Config: 32 | BASE_URL = "https://pokeapi.co/api/v2/" 33 | SETTINGS = GracyConfig( 34 | log_request=ON_REQUEST, 35 | log_response=ON_RESPONSE, 36 | log_errors=ON_ERROR, 37 | ) 38 | 39 | async def get_pokemon(self, name: str): 40 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 41 | 42 | 43 | @pytest.fixture() 44 | def pokeapi(): 45 | Gracy.dangerously_reset_report() 46 | return GracefulPokeAPI(REPLAY) 47 | 48 | 49 | async def test_pokemon_log_request_response( 50 | pokeapi: GracefulPokeAPI, caplog: pytest.LogCaptureFixture 51 | ): 52 | await pokeapi.get_pokemon(PRESENT_POKEMON_NAME) 53 | 54 | assert len(caplog.records) == 2 55 | assert_log(caplog.records[0], ON_REQUEST) 56 | assert_log(caplog.records[1], ON_RESPONSE) 57 | 58 | 59 | async def test_pokemon_log_request_response_error( 60 | pokeapi: GracefulPokeAPI, caplog: pytest.LogCaptureFixture 61 | ): 62 | with pytest.raises(NonOkResponse): 63 | await pokeapi.get_pokemon(MISSING_NAME) 64 | 65 | assert len(caplog.records) == 3 66 | assert_log(caplog.records[0], ON_REQUEST) 67 | assert_log(caplog.records[1], ON_RESPONSE) 68 | assert_log(caplog.records[2], ON_ERROR) 69 | -------------------------------------------------------------------------------- /src/gracy/replays/_wrappers.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import typing as t 4 | from functools import wraps 5 | 6 | import httpx 7 | 8 | from gracy._general import extract_request_kwargs 9 | from gracy.exceptions import GracyReplayRequestNotFound 10 | 11 | from .storages._base import GracyReplay 12 | 13 | httpx_func_type = t.Callable[..., t.Awaitable[httpx.Response]] 14 | 15 | 16 | def record_mode(replay: GracyReplay, httpx_request_func: httpx_func_type): 17 | @wraps(httpx_request_func) 18 | async def _wrapper(*args: t.Any, **kwargs: t.Any): 19 | httpx_response = await httpx_request_func(*args, **kwargs) 20 | await replay.storage.record(httpx_response) 21 | replay.inc_record() 22 | 23 | return httpx_response 24 | 25 | return _wrapper 26 | 27 | 28 | def replay_mode( 29 | replay: GracyReplay, client: httpx.AsyncClient, httpx_request_func: httpx_func_type 30 | ): 31 | @wraps(httpx_request_func) 32 | async def _wrapper(*args: t.Any, **kwargs: t.Any): 33 | request_kwargs = extract_request_kwargs(kwargs) 34 | request = client.build_request(*args, **request_kwargs) 35 | 36 | stored_response = await replay.storage.load( 37 | request, 38 | replay.discard_replays_older_than, 39 | replay.discard_bad_responses, 40 | ) 41 | replay.inc_replay() 42 | 43 | return stored_response 44 | 45 | return _wrapper 46 | 47 | 48 | def smart_replay_mode( 49 | replay: GracyReplay, client: httpx.AsyncClient, httpx_request_func: httpx_func_type 50 | ): 51 | @wraps(httpx_request_func) 52 | async def _wrapper(*args: t.Any, **kwargs: t.Any): 53 | request_kwargs = extract_request_kwargs(kwargs) 54 | request = client.build_request(*args, **request_kwargs) 55 | 56 | try: 57 | stored_response = await replay.storage.load( 58 | request, 59 | replay.discard_replays_older_than, 60 | replay.discard_bad_responses, 61 | ) 62 | 63 | except GracyReplayRequestNotFound: 64 | httpx_response = await httpx_request_func(*args, **kwargs) 65 | await replay.storage.record(httpx_response) 66 | response = httpx_response 67 | replay.inc_record() 68 | 69 | else: 70 | response = stored_response 71 | replay.inc_replay() 72 | 73 | return response 74 | 75 | return _wrapper 76 | -------------------------------------------------------------------------------- /src/tests/conftest.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import typing as t 4 | 5 | import httpx 6 | 7 | from gracy import BaseEndpoint, Gracy, GracyReplay 8 | from gracy.replays.storages.sqlite import SQLiteReplayStorage 9 | 10 | MISSING_NAME: t.Final = "doesnt-exist" 11 | """Should match what we recorded previously to successfully replay""" 12 | 13 | PRESENT_POKEMON_NAME: t.Final = "charmander" 14 | """Should match what we recorded previously to successfully replay""" 15 | 16 | PRESENT_BERRY_NAME: t.Final = "cheri" 17 | """Should match what we recorded previously to successfully replay""" 18 | 19 | REPLAY: t.Final = GracyReplay("replay", SQLiteReplayStorage("pokeapi.sqlite3")) 20 | 21 | 22 | class FakeReplayStorage(SQLiteReplayStorage): 23 | """Completely ignores the request defined to return a response matching the urls in the order specified""" 24 | 25 | def __init__(self, force_urls: t.List[str]) -> None: 26 | self._force_urls = force_urls 27 | self._response_idx = 0 28 | super().__init__("pokeapi.sqlite3") 29 | 30 | def _find_record(self, request: httpx.Request): 31 | cur = self._con.cursor() 32 | url = self._force_urls[self._response_idx] 33 | self._response_idx += 1 34 | 35 | cur.execute( 36 | """ 37 | SELECT response, updated_at FROM gracy_recordings 38 | WHERE 39 | url = ?""", 40 | (url,), 41 | ) 42 | 43 | return cur.fetchone() 44 | 45 | 46 | class PokeApiEndpoint(BaseEndpoint): 47 | GET_POKEMON = "/pokemon/{NAME}" 48 | GET_BERRY = "/berry/{NAME}" 49 | 50 | 51 | def assert_one_request_made(gracy_api: Gracy[PokeApiEndpoint]): 52 | report = gracy_api.get_report() 53 | assert len(report.requests) == 1 54 | 55 | 56 | def assert_requests_made( 57 | gracy_api: Gracy[PokeApiEndpoint], total_requests: int, endpoints_count: int = 1 58 | ): 59 | report = gracy_api.get_report() 60 | 61 | assert len(report.requests) == endpoints_count 62 | assert report.requests[0].total_requests == total_requests 63 | 64 | 65 | def assert_muiti_endpoints_requests_made( 66 | gracy_api: Gracy[PokeApiEndpoint], 67 | endpoints_count: int, 68 | *total_requests: int, 69 | ): 70 | report = gracy_api.get_report() 71 | 72 | assert len(report.requests) == endpoints_count 73 | 74 | for i, expected_total in enumerate(total_requests): 75 | assert report.requests[i].total_requests == expected_total 76 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "gracy" 3 | version = "1.34.0" 4 | description = "Gracefully manage your API interactions" 5 | authors = ["Guilherme Latrova "] 6 | license = "MIT" 7 | keywords = ["api", "throttling", "http", "https", "async", "retry"] 8 | readme = "README.md" 9 | homepage = "https://github.com/guilatrova/gracy" 10 | repository = "https://github.com/guilatrova/gracy" 11 | include = ["LICENSE", "py.typed"] 12 | classifiers = [ 13 | "Development Status :: 4 - Beta", 14 | "Environment :: Web Environment", 15 | "Framework :: AsyncIO", 16 | "Intended Audience :: Developers", 17 | "License :: OSI Approved :: BSD License", 18 | "Operating System :: OS Independent", 19 | "Programming Language :: Python :: 3", 20 | "Programming Language :: Python :: 3 :: Only", 21 | "Programming Language :: Python :: 3.8", 22 | "Programming Language :: Python :: 3.9", 23 | "Programming Language :: Python :: 3.10", 24 | "Programming Language :: Python :: 3.11", 25 | "Topic :: Internet :: WWW/HTTP", 26 | ] 27 | packages = [{ include = "gracy", from = "src" }] 28 | 29 | [tool.poetry.urls] 30 | "Changelog" = "https://github.com/guilatrova/gracy/blob/main/CHANGELOG.md" 31 | 32 | 33 | [tool.semantic_release] 34 | version_variable = ["src/gracy/__init__.py:__version__"] 35 | version_toml = ["pyproject.toml:tool.poetry.version"] 36 | branch = "main" 37 | upload_to_pypi = true 38 | upload_to_release = true 39 | build_command = "pip install poetry && poetry build" 40 | 41 | [tool.poetry.dependencies] 42 | python = ">=3.8.1,<4.0" 43 | httpx = ">=0.23.0" 44 | rich = { version = "*", optional = true } 45 | pymongo = { version = "*", optional = true } 46 | typing-extensions = "^4.9.0" 47 | # It should be python = "<3.10" if we didn't use the 'deprecated' import from PEP 702 48 | 49 | [tool.poetry.group.dev.dependencies] 50 | python-semantic-release = "^7.33.0" 51 | pre-commit = "^3.5.0" 52 | rich = "^13.2.0" 53 | pymongo = "^4.3.3" 54 | pytest = "^7.2.1" 55 | pytest-asyncio = "^0.20.3" 56 | ruff = "^0.1.6" 57 | pyright = "^1.1.351" 58 | 59 | [tool.poetry.extras] 60 | rich = ["rich"] 61 | pymongo = ["pymongo"] 62 | plotly = ["plotly", "pandas"] 63 | 64 | [tool.ruff.lint.isort] 65 | extra-standard-library = ["pytest", "httpx"] 66 | required-imports = ["from __future__ import annotations"] 67 | 68 | # https://microsoft.github.io/pyright/#/configuration 69 | [tool.pyright] 70 | include = ["src"] 71 | pythonVersion = "3.8" 72 | pythonPlatform = "All" 73 | reportMissingImports = "warning" 74 | reportIncompatibleVariableOverride = "none" 75 | 76 | [tool.pytest.ini_options] 77 | asyncio_mode = "auto" 78 | testpaths = "src/tests" 79 | 80 | [build-system] 81 | requires = ["poetry-core>=1.0.0"] 82 | build-backend = "poetry.core.masonry.api" 83 | -------------------------------------------------------------------------------- /examples/pokeapi_namespaces.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | import typing as t 5 | from http import HTTPStatus 6 | 7 | from gracy import ( 8 | BaseEndpoint, 9 | Gracy, 10 | GracyConfig, 11 | GracyNamespace, 12 | LogEvent, 13 | LogLevel, 14 | ) 15 | from rich import print 16 | 17 | RESP_TYPE = t.Union[t.Dict[str, t.Any], None] 18 | 19 | 20 | class PokeApiEndpoint(BaseEndpoint): 21 | BERRY = "/berry/{KEY}" 22 | BERRY_FLAVOR = "/berry-flavor/{KEY}" 23 | BERRY_FIRMNESS = "/berry-firmness/{KEY}" 24 | 25 | POKEMON = "/pokemon/{KEY}" 26 | POKEMON_COLOR = "/pokemon-color/{KEY}" 27 | POKEMON_FORM = "/pokemon-form/{KEY}" 28 | 29 | 30 | class PokeApiBerryNamespace(GracyNamespace[PokeApiEndpoint]): 31 | async def get_this(self, name_or_id: t.Union[str, int]): 32 | return await self.get[RESP_TYPE]( 33 | PokeApiEndpoint.BERRY, dict(KEY=str(name_or_id)) 34 | ) 35 | 36 | async def get_flavor(self, name_or_id: t.Union[str, int]): 37 | return await self.get[RESP_TYPE]( 38 | PokeApiEndpoint.BERRY_FLAVOR, dict(KEY=str(name_or_id)) 39 | ) 40 | 41 | async def get_firmness(self, name_or_id: t.Union[str, int]): 42 | return await self.get[RESP_TYPE]( 43 | PokeApiEndpoint.BERRY_FIRMNESS, dict(KEY=str(name_or_id)) 44 | ) 45 | 46 | 47 | class PokeApiPokemonNamespace(GracyNamespace[PokeApiEndpoint]): 48 | async def get_this(self, name_or_id: t.Union[str, int]): 49 | return await self.get[RESP_TYPE]( 50 | PokeApiEndpoint.POKEMON, dict(KEY=str(name_or_id)) 51 | ) 52 | 53 | async def get_color(self, name_or_id: t.Union[str, int]): 54 | return await self.get[RESP_TYPE]( 55 | PokeApiEndpoint.POKEMON_COLOR, dict(KEY=str(name_or_id)) 56 | ) 57 | 58 | async def get_form(self, name_or_id: t.Union[str, int]): 59 | return await self.get[RESP_TYPE]( 60 | PokeApiEndpoint.POKEMON_FORM, dict(KEY=str(name_or_id)) 61 | ) 62 | 63 | 64 | class PokeApi(Gracy[PokeApiEndpoint]): 65 | class Config: 66 | BASE_URL = "https://pokeapi.co/api/v2/" 67 | REQUEST_TIMEOUT = 5.0 68 | SETTINGS = GracyConfig( 69 | parser={ 70 | HTTPStatus.OK: lambda resp: resp.json(), 71 | HTTPStatus.NOT_FOUND: None, 72 | }, 73 | allowed_status_code=HTTPStatus.NOT_FOUND, 74 | log_errors=LogEvent(LogLevel.ERROR), 75 | ) 76 | 77 | berry: PokeApiBerryNamespace 78 | pokemon: PokeApiPokemonNamespace 79 | 80 | 81 | async def main(): 82 | api = PokeApi() 83 | 84 | berry = api.berry.get_this("cheri") 85 | berry_flavor = api.berry.get_flavor("spicy") 86 | pikachu = api.pokemon.get_this("pikachu") 87 | black = api.pokemon.get_color("black") 88 | 89 | results = await asyncio.gather(berry, berry_flavor, pikachu, black) 90 | 91 | for content in results: 92 | print(content) 93 | 94 | api.report_status("rich") 95 | 96 | 97 | if __name__ == "__main__": 98 | asyncio.run(main()) 99 | -------------------------------------------------------------------------------- /examples/pokeapi.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | import httpx 5 | import typing as t 6 | from http import HTTPStatus 7 | 8 | from gracy import ( 9 | BaseEndpoint, 10 | GracefulRetry, 11 | Gracy, 12 | GracyRequestContext, 13 | LogEvent, 14 | LogLevel, 15 | graceful, 16 | ) 17 | from gracy.exceptions import GracyUserDefinedException 18 | 19 | retry = GracefulRetry( 20 | delay=1, 21 | max_attempts=3, 22 | delay_modifier=1.5, 23 | retry_on=None, 24 | log_before=LogEvent(LogLevel.WARNING), 25 | log_after=LogEvent(LogLevel.WARNING), 26 | log_exhausted=LogEvent(LogLevel.CRITICAL), 27 | behavior="pass", 28 | ) 29 | 30 | 31 | class PokemonNotFound(GracyUserDefinedException): 32 | BASE_MESSAGE = "Unable to find a pokemon with the name [{NAME}] at {URL} due to {STATUS} status" 33 | 34 | def _format_message( 35 | self, request_context: GracyRequestContext, response: httpx.Response 36 | ) -> str: 37 | format_args = self._build_default_args() 38 | name = request_context.endpoint_args.get("NAME", "Unknown") 39 | return self.BASE_MESSAGE.format(NAME=name, **format_args) 40 | 41 | 42 | class ServerIsOutError(Exception): 43 | pass 44 | 45 | 46 | class PokeApiEndpoint(BaseEndpoint): 47 | GET_POKEMON = "/pokemon/{NAME}" 48 | GET_GENERATION = "/generation/{ID}" 49 | 50 | 51 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 52 | class Config: 53 | BASE_URL = "https://pokeapi.co/api/v2/" 54 | 55 | @graceful( 56 | strict_status_code={HTTPStatus.OK}, 57 | retry=retry, 58 | log_request=LogEvent(LogLevel.WARNING), 59 | log_errors=LogEvent( 60 | LogLevel.ERROR, 61 | lambda r: "Request failed with {STATUS}" 62 | f" and it was {'' if r.is_redirect else 'NOT'} redirected" 63 | if r 64 | else "", 65 | ), 66 | parser={ 67 | "default": lambda r: r.json()["name"], 68 | HTTPStatus.NOT_FOUND: PokemonNotFound, 69 | HTTPStatus.INTERNAL_SERVER_ERROR: ServerIsOutError, 70 | }, 71 | ) 72 | async def get_pokemon(self, name: str): 73 | self.get 74 | 75 | return await self.get[t.Optional[str]]( 76 | PokeApiEndpoint.GET_POKEMON, {"NAME": name} 77 | ) 78 | 79 | async def get_generation(self, gen: int): 80 | return await self.get(PokeApiEndpoint.GET_GENERATION, {"ID": str(gen)}) 81 | 82 | 83 | pokeapi = GracefulPokeAPI() 84 | pokeapi_two = GracefulPokeAPI() 85 | 86 | 87 | async def main(): 88 | try: 89 | p1 = await pokeapi.get_pokemon("pikachu") 90 | 91 | try: 92 | p2 = await pokeapi_two.get_pokemon("doesnt-exist") 93 | except PokemonNotFound as ex: 94 | p2 = str(ex) 95 | 96 | await pokeapi.get_generation(1) 97 | 98 | print("P1: result of get_pokemon:", p1) 99 | print("P2: result of get_pokemon:", p2) 100 | 101 | finally: 102 | pokeapi.report_status("list") 103 | 104 | 105 | asyncio.run(main()) 106 | -------------------------------------------------------------------------------- /examples/pokeapi_replay_mongo.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | import httpx 5 | from http import HTTPStatus 6 | 7 | from gracy import ( 8 | BaseEndpoint, 9 | GracefulRetry, 10 | Gracy, 11 | GracyReplay, 12 | GracyRequestContext, 13 | LogEvent, 14 | LogLevel, 15 | graceful, 16 | ) 17 | from gracy.exceptions import GracyUserDefinedException 18 | from gracy.replays.storages.pymongo import MongoCredentials, MongoReplayStorage 19 | 20 | retry = GracefulRetry( 21 | delay=1, 22 | max_attempts=3, 23 | delay_modifier=1.5, 24 | retry_on=None, 25 | log_before=LogEvent(LogLevel.WARNING), 26 | log_after=LogEvent(LogLevel.WARNING), 27 | log_exhausted=LogEvent(LogLevel.CRITICAL), 28 | behavior="pass", 29 | ) 30 | 31 | mongo_container = MongoCredentials( 32 | host="localhost", username="root", password="example" 33 | ) 34 | record_mode = GracyReplay("record", MongoReplayStorage(mongo_container)) 35 | replay_mode = GracyReplay("replay", MongoReplayStorage(mongo_container)) 36 | 37 | 38 | class PokemonNotFound(GracyUserDefinedException): 39 | BASE_MESSAGE = "Unable to find a pokemon with the name [{NAME}] at {URL} due to {STATUS} status" 40 | 41 | def _format_message( 42 | self, request_context: GracyRequestContext, response: httpx.Response 43 | ) -> str: 44 | format_args = self._build_default_args() 45 | name = request_context.endpoint_args.get("NAME", "Unknown") 46 | return self.BASE_MESSAGE.format(NAME=name, **format_args) 47 | 48 | 49 | class PokeApiEndpoint(BaseEndpoint): 50 | GET_POKEMON = "/pokemon/{NAME}" 51 | GET_GENERATION = "/generation/{ID}" 52 | 53 | 54 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 55 | class Config: 56 | BASE_URL = "https://pokeapi.co/api/v2/" 57 | 58 | @graceful( 59 | strict_status_code={HTTPStatus.OK}, 60 | retry=retry, 61 | log_errors=LogEvent( 62 | LogLevel.ERROR, 63 | lambda r: "Request failed with {STATUS}" 64 | f" and it was {'' if r.is_redirect else 'NOT'} redirected" 65 | if r 66 | else "", 67 | ), 68 | parser={ 69 | "default": lambda r: r.json()["name"], 70 | HTTPStatus.NOT_FOUND: PokemonNotFound, 71 | HTTPStatus.INTERNAL_SERVER_ERROR: PokemonNotFound, 72 | }, 73 | ) 74 | async def get_pokemon(self, name: str): 75 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 76 | 77 | async def get_generation(self, gen: int): 78 | return await self.get(PokeApiEndpoint.GET_GENERATION, {"ID": str(gen)}) 79 | 80 | 81 | async def main(replay_mode: GracyReplay): 82 | pokeapi = GracefulPokeAPI(replay_mode) 83 | poke_names = {"pikachu", "elekid", "charmander", "blaziken", "hitmonchan"} 84 | 85 | try: 86 | get_pokemons = [ 87 | asyncio.create_task(pokeapi.get_pokemon(name)) for name in poke_names 88 | ] 89 | get_gens = [ 90 | asyncio.create_task(pokeapi.get_generation(gen_id)) 91 | for gen_id in range(1, 3) 92 | ] 93 | 94 | await asyncio.gather(*(get_pokemons + get_gens)) 95 | 96 | finally: 97 | pokeapi.report_status("rich") 98 | print("-" * 100) 99 | pokeapi.report_status("list") 100 | print("-" * 100) 101 | pokeapi.report_status("logger") 102 | 103 | 104 | asyncio.run(main(replay_mode)) 105 | -------------------------------------------------------------------------------- /examples/pokeapi_replay.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | import httpx 5 | from http import HTTPStatus 6 | 7 | from gracy import ( 8 | BaseEndpoint, 9 | GracefulRetry, 10 | Gracy, 11 | GracyReplay, 12 | GracyRequestContext, 13 | LogEvent, 14 | LogLevel, 15 | ReplayLogEvent, 16 | graceful, 17 | ) 18 | from gracy.exceptions import GracyUserDefinedException 19 | from gracy.replays.storages.sqlite import SQLiteReplayStorage 20 | 21 | retry = GracefulRetry( 22 | delay=1, 23 | max_attempts=3, 24 | delay_modifier=1.5, 25 | retry_on=None, 26 | log_before=LogEvent(LogLevel.WARNING), 27 | log_after=LogEvent(LogLevel.WARNING), 28 | log_exhausted=LogEvent(LogLevel.CRITICAL), 29 | behavior="pass", 30 | ) 31 | 32 | record_mode = GracyReplay( 33 | "record", 34 | SQLiteReplayStorage("pokeapi.sqlite3"), 35 | ) 36 | replay_mode = GracyReplay( 37 | "replay", 38 | SQLiteReplayStorage("pokeapi.sqlite3"), 39 | log_replay=ReplayLogEvent(LogLevel.WARNING, frequency=1), 40 | ) 41 | 42 | 43 | class PokemonNotFound(GracyUserDefinedException): 44 | BASE_MESSAGE = "Unable to find a pokemon with the name [{NAME}] at {URL} due to {STATUS} status" 45 | 46 | def _format_message( 47 | self, request_context: GracyRequestContext, response: httpx.Response 48 | ) -> str: 49 | format_args = self._build_default_args() 50 | name = request_context.endpoint_args.get("NAME", "Unknown") 51 | return self.BASE_MESSAGE.format(NAME=name, **format_args) 52 | 53 | 54 | class PokeApiEndpoint(BaseEndpoint): 55 | GET_POKEMON = "/pokemon/{NAME}" 56 | GET_GENERATION = "/generation/{ID}" 57 | 58 | 59 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 60 | class Config: 61 | BASE_URL = "https://pokeapi.co/api/v2/" 62 | 63 | @graceful( 64 | strict_status_code={HTTPStatus.OK}, 65 | retry=retry, 66 | log_errors=LogEvent( 67 | LogLevel.ERROR, 68 | lambda r: "Request failed with {STATUS}" 69 | f" and it was {'' if r.is_redirect else 'NOT'} redirected" 70 | if r 71 | else "", 72 | ), 73 | log_response=LogEvent(LogLevel.INFO), 74 | parser={ 75 | "default": lambda r: r.json()["name"], 76 | HTTPStatus.NOT_FOUND: PokemonNotFound, 77 | HTTPStatus.INTERNAL_SERVER_ERROR: PokemonNotFound, 78 | }, 79 | ) 80 | async def get_pokemon(self, name: str): 81 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 82 | 83 | async def get_generation(self, gen: int): 84 | return await self.get(PokeApiEndpoint.GET_GENERATION, {"ID": str(gen)}) 85 | 86 | 87 | async def main(replay_mode: GracyReplay): 88 | pokeapi = GracefulPokeAPI(replay_mode) 89 | poke_names = {"pikachu", "elekid", "charmander", "blaziken", "hitmonchan"} 90 | 91 | try: 92 | get_pokemons = [ 93 | asyncio.create_task(pokeapi.get_pokemon(name)) for name in poke_names 94 | ] 95 | get_gens = [ 96 | asyncio.create_task(pokeapi.get_generation(gen_id)) 97 | for gen_id in range(1, 3) 98 | ] 99 | 100 | await asyncio.gather(*(get_pokemons + get_gens)) 101 | 102 | finally: 103 | pokeapi.report_status("rich") 104 | print("-" * 100) 105 | pokeapi.report_status("list") 106 | print("-" * 100) 107 | pokeapi.report_status("logger") 108 | 109 | 110 | asyncio.run(main(replay_mode)) 111 | -------------------------------------------------------------------------------- /src/gracy/_reports/_models.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import httpx 4 | import typing as t 5 | from dataclasses import dataclass, field 6 | from statistics import mean 7 | 8 | from .._models import RequestTimeline 9 | from ..replays.storages._base import GracyReplay 10 | 11 | 12 | @dataclass(frozen=True) 13 | class GracyRequestResult: 14 | __slots__ = ("uurl", "response") 15 | 16 | uurl: str 17 | response: httpx.Response | Exception 18 | 19 | 20 | @dataclass 21 | class GracyRequestCounters: 22 | throttles: int = 0 23 | retries: int = 0 24 | replays: int = 0 25 | 26 | 27 | @dataclass 28 | class ReportGenericAggregatedRequest: 29 | uurl: str 30 | """unformatted url""" 31 | 32 | total_requests: int 33 | 34 | resp_2xx: int 35 | resp_3xx: int 36 | resp_4xx: int 37 | resp_5xx: int 38 | reqs_aborted: int 39 | 40 | retries: int 41 | throttles: int 42 | replays: int 43 | 44 | max_latency: float 45 | 46 | @property 47 | def success_rate(self) -> float: 48 | if self.total_requests: 49 | return (self.resp_2xx / self.total_requests) * 100 50 | 51 | return 0 52 | 53 | @property 54 | def failed_rate(self) -> float: 55 | if self.total_requests: 56 | return 100.00 - self.success_rate 57 | 58 | return 0 59 | 60 | 61 | @dataclass 62 | class GracyAggregatedRequest(ReportGenericAggregatedRequest): 63 | avg_latency: float = 0 64 | req_rate_per_sec: float = 0 65 | 66 | 67 | @dataclass 68 | class GracyAggregatedTotal(ReportGenericAggregatedRequest): 69 | all_avg_latency: list[float] = field(default_factory=list) 70 | all_req_rates: list[float] = field(default_factory=list) 71 | 72 | @property 73 | def avg_latency(self) -> float: 74 | entries = self.all_avg_latency or [0] 75 | return mean(entries) 76 | 77 | @property 78 | def req_rate_per_sec(self) -> float: 79 | entries = self.all_req_rates or [0] 80 | return mean(entries) 81 | 82 | def increment_result(self, row: GracyAggregatedRequest) -> None: 83 | self.total_requests += row.total_requests 84 | self.resp_2xx += row.resp_2xx 85 | self.resp_3xx += row.resp_3xx 86 | self.resp_4xx += row.resp_4xx 87 | self.resp_5xx += row.resp_5xx 88 | self.reqs_aborted += row.reqs_aborted 89 | self.throttles += row.throttles 90 | self.retries += row.retries 91 | self.replays += row.replays 92 | 93 | self.all_avg_latency.append(row.avg_latency) 94 | if row.req_rate_per_sec > 0: 95 | self.all_req_rates.append(row.req_rate_per_sec) 96 | 97 | 98 | class GracyReport: 99 | def __init__( 100 | self, 101 | replay_settings: t.Optional[GracyReplay], 102 | requests_timeline: t.Dict[str, t.List[RequestTimeline]], 103 | ) -> None: 104 | self.requests: list[GracyAggregatedRequest | GracyAggregatedTotal] = [] 105 | self.total = GracyAggregatedTotal( 106 | "TOTAL", # serves as title 107 | total_requests=0, 108 | resp_2xx=0, 109 | resp_3xx=0, 110 | resp_4xx=0, 111 | resp_5xx=0, 112 | reqs_aborted=0, 113 | retries=0, 114 | throttles=0, 115 | replays=0, 116 | max_latency=0, 117 | ) 118 | self.replay_settings = replay_settings 119 | self.requests_timeline = requests_timeline 120 | 121 | def add_request(self, request: GracyAggregatedRequest) -> None: 122 | self.requests.append(request) 123 | self.total.increment_result(request) 124 | -------------------------------------------------------------------------------- /src/tests/test_validators.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import httpx 4 | import pytest 5 | import typing as t 6 | from http import HTTPStatus 7 | 8 | from gracy import GracefulValidator, Gracy, graceful 9 | from gracy.exceptions import NonOkResponse, UnexpectedResponse 10 | from tests.conftest import ( 11 | MISSING_NAME, 12 | PRESENT_POKEMON_NAME, 13 | REPLAY, 14 | PokeApiEndpoint, 15 | assert_one_request_made, 16 | ) 17 | 18 | 19 | class CustomValidator(GracefulValidator): 20 | def check(self, response: httpx.Response) -> None: 21 | if response.json()["order"] != 47: 22 | raise ValueError("Pokemon #order should be 47") # noqa: TRY003 23 | 24 | 25 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 26 | class Config: 27 | BASE_URL = "https://pokeapi.co/api/v2/" 28 | 29 | async def get_pokemon(self, name: str): 30 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 31 | 32 | @graceful(strict_status_code=HTTPStatus.INTERNAL_SERVER_ERROR) 33 | async def get_pokemon_with_wrong_strict_status(self, name: str): 34 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 35 | 36 | @graceful(strict_status_code=HTTPStatus.OK) 37 | async def get_pokemon_with_correct_strict_status(self, name: str): 38 | return await self.get[httpx.Response]( 39 | PokeApiEndpoint.GET_POKEMON, {"NAME": name} 40 | ) 41 | 42 | @graceful(allowed_status_code=HTTPStatus.NOT_FOUND) 43 | async def get_pokemon_allow_404(self, name: str): 44 | return await self.get[httpx.Response]( 45 | PokeApiEndpoint.GET_POKEMON, {"NAME": name} 46 | ) 47 | 48 | 49 | @pytest.fixture() 50 | def make_pokeapi(): 51 | def factory(): 52 | Gracy.dangerously_reset_report() 53 | return GracefulPokeAPI(REPLAY) 54 | 55 | return factory 56 | 57 | 58 | async def test_pokemon_ok_default(make_pokeapi: t.Callable[[], GracefulPokeAPI]): 59 | pokeapi = make_pokeapi() 60 | 61 | result = t.cast(httpx.Response, await pokeapi.get_pokemon(PRESENT_POKEMON_NAME)) 62 | 63 | assert result.status_code == HTTPStatus.OK 64 | 65 | assert_one_request_made(pokeapi) 66 | 67 | 68 | async def test_pokemon_not_found_default(make_pokeapi: t.Callable[[], GracefulPokeAPI]): 69 | pokeapi = make_pokeapi() 70 | 71 | try: 72 | _ = await pokeapi.get_pokemon(MISSING_NAME) 73 | 74 | except NonOkResponse as ex: 75 | assert ex.response.status_code == HTTPStatus.NOT_FOUND 76 | 77 | else: 78 | pytest.fail("NonOkResponse was expected") 79 | 80 | assert_one_request_made(pokeapi) 81 | 82 | 83 | async def test_pokemon_strict_status_fail( 84 | make_pokeapi: t.Callable[[], GracefulPokeAPI] 85 | ): 86 | pokeapi = make_pokeapi() 87 | 88 | try: 89 | _ = await pokeapi.get_pokemon_with_wrong_strict_status(PRESENT_POKEMON_NAME) 90 | 91 | except UnexpectedResponse as ex: 92 | assert ex.response.status_code == HTTPStatus.OK 93 | 94 | else: 95 | pytest.fail("UnexpectedResponse was expected") 96 | 97 | assert_one_request_made(pokeapi) 98 | 99 | 100 | async def test_pokemon_strict_status_success( 101 | make_pokeapi: t.Callable[[], GracefulPokeAPI] 102 | ): 103 | pokeapi = make_pokeapi() 104 | 105 | result = await pokeapi.get_pokemon_with_correct_strict_status(PRESENT_POKEMON_NAME) 106 | 107 | assert result.status_code == HTTPStatus.OK 108 | assert_one_request_made(pokeapi) 109 | 110 | 111 | async def test_pokemon_allow_404(make_pokeapi: t.Callable[[], GracefulPokeAPI]): 112 | pokeapi = make_pokeapi() 113 | 114 | result = await pokeapi.get_pokemon_allow_404(MISSING_NAME) 115 | 116 | assert result.status_code == HTTPStatus.NOT_FOUND 117 | assert_one_request_made(pokeapi) 118 | -------------------------------------------------------------------------------- /src/gracy/replays/storages/sqlite.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import httpx 4 | import logging 5 | import pickle 6 | import sqlite3 7 | import typing as t 8 | from dataclasses import dataclass 9 | from datetime import datetime 10 | from pathlib import Path 11 | 12 | from gracy.exceptions import GracyReplayRequestNotFound 13 | 14 | from . import _sqlite_schema as schema 15 | from ._base import GracyReplayStorage 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | @dataclass 21 | class GracyRecording: 22 | url: str 23 | method: str 24 | 25 | request_body: bytes | None 26 | response: bytes 27 | 28 | updated_at: datetime 29 | 30 | 31 | class SQLiteReplayStorage(GracyReplayStorage): 32 | def __init__( 33 | self, db_name: str = "gracy-records.sqlite3", dir: str = ".gracy" 34 | ) -> None: 35 | self.db_dir = Path(dir) 36 | self.db_file = self.db_dir / db_name 37 | self._con: sqlite3.Connection = None # type: ignore 38 | 39 | def _create_db(self) -> None: 40 | logger.info("Creating Gracy Replay sqlite database") 41 | con = sqlite3.connect(str(self.db_file)) 42 | cur = con.cursor() 43 | 44 | cur.execute(schema.CREATE_RECORDINGS_TABLE) 45 | cur.execute(schema.INDEX_RECORDINGS_TABLE) 46 | cur.execute(schema.INDEX_RECORDINGS_TABLE_WITHOUT_REQUEST_BODY) 47 | 48 | def _insert_into_db(self, recording: GracyRecording) -> None: 49 | cur = self._con.cursor() 50 | 51 | params = ( 52 | recording.url, 53 | recording.method, 54 | recording.request_body, 55 | recording.response, 56 | datetime.now(), 57 | ) 58 | cur.execute(schema.INSERT_RECORDING_BASE, params) 59 | self._con.commit() 60 | 61 | def prepare(self) -> None: 62 | self.db_dir.mkdir(parents=True, exist_ok=True) 63 | if self.db_file.exists() is False: 64 | self._create_db() 65 | 66 | self._con = sqlite3.connect(str(self.db_file)) 67 | 68 | async def record(self, response: httpx.Response) -> None: 69 | response_serialized = pickle.dumps(response) 70 | 71 | recording = GracyRecording( 72 | str(response.url), 73 | str(response.request.method), 74 | response.request.content or None, 75 | response_serialized, 76 | datetime.now(), 77 | ) 78 | 79 | self._insert_into_db(recording) 80 | 81 | def _find_record(self, request: httpx.Request): 82 | cur = self._con.cursor() 83 | params: t.Iterable[str | bytes] 84 | 85 | if bool(request.content): 86 | params = (str(request.url), request.method, request.content) 87 | cur.execute(schema.FIND_REQUEST_WITH_REQ_BODY, params) 88 | else: 89 | params = (str(request.url), request.method) 90 | cur.execute(schema.FIND_REQUEST_WITHOUT_REQ_BODY, params) 91 | 92 | fetch_res = cur.fetchone() 93 | return fetch_res 94 | 95 | async def find_replay( 96 | self, request: httpx.Request, discard_before: datetime | None 97 | ) -> t.Any | None: 98 | fetch_res = self._find_record(request) 99 | if fetch_res is None: 100 | return None 101 | 102 | updated_at: datetime = fetch_res[1] 103 | if discard_before and updated_at < discard_before: 104 | return None 105 | 106 | return fetch_res 107 | 108 | async def _load( 109 | self, request: httpx.Request, discard_before: datetime | None 110 | ) -> httpx.Response: 111 | fetch_res = await self.find_replay(request, discard_before) 112 | 113 | if fetch_res is None: 114 | raise GracyReplayRequestNotFound(request) 115 | 116 | serialized_response: bytes = fetch_res[0] 117 | response: httpx.Response = pickle.loads(serialized_response) 118 | 119 | return response 120 | -------------------------------------------------------------------------------- /examples/pokeapi_throttle.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | import time 5 | import typing as t 6 | from datetime import timedelta 7 | from http import HTTPStatus 8 | 9 | from gracy import ( 10 | BaseEndpoint, 11 | GracefulRetry, 12 | GracefulThrottle, 13 | Gracy, 14 | GracyConfig, 15 | LogEvent, 16 | LogLevel, 17 | ThrottleRule, 18 | graceful, 19 | ) 20 | from rich import print 21 | 22 | RETRY = GracefulRetry( 23 | delay=0, # Force throttling to work 24 | max_attempts=3, 25 | retry_on=None, 26 | log_after=LogEvent(LogLevel.WARNING), 27 | log_exhausted=LogEvent(LogLevel.CRITICAL), 28 | behavior="pass", 29 | ) 30 | 31 | THROTTLE_RULE = ThrottleRule(r".*", 4, timedelta(seconds=2)) 32 | 33 | 34 | class PokeApiEndpoint(BaseEndpoint): 35 | GET_POKEMON = "/pokemon/{NAME}" 36 | GET_GENERATION = "/generation/{ID}" 37 | 38 | 39 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 40 | class Config: 41 | BASE_URL = "https://pokeapi.co/api/v2/" 42 | SETTINGS = GracyConfig( 43 | strict_status_code={HTTPStatus.OK}, 44 | retry=RETRY, 45 | parser={ 46 | "default": lambda r: r.json(), 47 | HTTPStatus.NOT_FOUND: None, 48 | }, 49 | throttling=GracefulThrottle( 50 | rules=THROTTLE_RULE, 51 | log_limit_reached=LogEvent(LogLevel.ERROR), 52 | log_wait_over=LogEvent(LogLevel.WARNING), 53 | ), 54 | ) 55 | 56 | @graceful( 57 | parser={"default": lambda r: r.json()["order"], HTTPStatus.NOT_FOUND: None} 58 | ) 59 | async def get_pokemon(self, name: str): 60 | val = t.cast( 61 | t.Optional[str], await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 62 | ) 63 | 64 | if val: 65 | print(f"{name} is #{val} in the pokedex") 66 | else: 67 | print(f"{name} was not found") 68 | 69 | async def get_generation(self, gen: int): 70 | return await self.get(PokeApiEndpoint.GET_GENERATION, {"ID": str(gen)}) 71 | 72 | 73 | pokeapi = GracefulPokeAPI() 74 | 75 | 76 | async def main(): 77 | pokemon_names = [ 78 | "bulbasaur", 79 | "charmander", 80 | "squirtle", 81 | "pikachu", 82 | "jigglypuff", 83 | "mewtwo", 84 | "gyarados", 85 | "dragonite", 86 | "mew", 87 | "chikorita", 88 | "cyndaquil", 89 | "totodile", 90 | "pichu", 91 | "togepi", 92 | "ampharos", 93 | "typhlosion", 94 | "feraligatr", 95 | "espeon", 96 | "umbreon", 97 | "lugia", 98 | "ho-oh", 99 | "treecko", 100 | "torchic", 101 | "mudkip", 102 | "gardevoir", 103 | "sceptile", 104 | "blaziken", 105 | "swampert", 106 | "rayquaza", 107 | "latias", 108 | "latios", 109 | "lucario", 110 | "garchomp", 111 | "darkrai", 112 | "giratina", # (1) this fails, so good to test retry 113 | "arceus", 114 | "snivy", 115 | "tepig", 116 | "oshawott", 117 | "zekrom", 118 | "reshiram", 119 | "victini", 120 | "chespin", 121 | "fennekin", 122 | "froakie", 123 | "xerneas", 124 | "yveltal", 125 | "zygarde", # (2) this fails, so good to test retry 126 | "decidueye", 127 | "incineroar", 128 | ] 129 | # pokemon_names = pokemon_names[:10] 130 | print( 131 | f"Will query {len(pokemon_names)} pokemons concurrently - {str(THROTTLE_RULE)}" 132 | ) 133 | 134 | try: 135 | start = time.time() 136 | 137 | pokemon_reqs = [ 138 | asyncio.create_task(pokeapi.get_pokemon(name)) for name in pokemon_names 139 | ] 140 | gen_reqs = [ 141 | asyncio.create_task(pokeapi.get_generation(gen)) for gen in range(1, 4) 142 | ] 143 | 144 | await asyncio.gather(*pokemon_reqs, *gen_reqs) 145 | elapsed = time.time() - start 146 | print(f"All requests took {timedelta(seconds=elapsed)}s to finish") 147 | 148 | finally: 149 | pokeapi.report_status("rich") 150 | pokeapi.report_status("list") 151 | pokeapi._throttle_controller.debug_print() # type: ignore 152 | 153 | 154 | asyncio.run(main()) 155 | -------------------------------------------------------------------------------- /src/gracy/replays/storages/_base.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | import typing as t 5 | from abc import ABC, abstractmethod 6 | from dataclasses import dataclass 7 | from datetime import datetime 8 | 9 | import httpx 10 | 11 | from gracy.exceptions import GracyReplayRequestNotFound 12 | 13 | from ..._loggers import DefaultLogMessage, do_log 14 | from ..._models import LogEvent 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | REPLAY_FLAG: t.Final = "_gracy_replayed" 20 | 21 | 22 | def is_replay(resp: httpx.Response) -> bool: 23 | return getattr(resp, REPLAY_FLAG, False) 24 | 25 | 26 | class GracyReplayStorage(ABC): 27 | def prepare(self) -> None: 28 | """(Optional) Executed upon API instance creation.""" 29 | pass 30 | 31 | @abstractmethod 32 | async def record(self, response: httpx.Response) -> None: 33 | """Logic to store the response object. Note the httpx.Response has request data""" 34 | pass 35 | 36 | @abstractmethod 37 | async def find_replay( 38 | self, request: httpx.Request, discard_before: datetime | None 39 | ) -> t.Any | None: 40 | pass 41 | 42 | @abstractmethod 43 | async def _load( 44 | self, request: httpx.Request, discard_before: datetime | None 45 | ) -> httpx.Response: 46 | """Logic to load a response object based on the request. Raises `GracyReplayRequestNotFound` if missing""" 47 | pass 48 | 49 | async def load( 50 | self, 51 | request: httpx.Request, 52 | discard_before: datetime | None, 53 | discard_bad_responses: bool = False, 54 | ) -> httpx.Response: 55 | """Logic to load a response object based on the request. Raises `GracyReplayRequestNotFound` if missing""" 56 | resp = await self._load(request, discard_before) 57 | setattr(resp, REPLAY_FLAG, True) 58 | 59 | if discard_bad_responses and resp.is_success is False: 60 | raise GracyReplayRequestNotFound(request) 61 | 62 | return resp 63 | 64 | def flush(self) -> None: 65 | """(Optional) Executed during close (preferably once all requests were made).""" 66 | pass 67 | 68 | 69 | @dataclass 70 | class ReplayLogEvent(LogEvent): 71 | frequency: int = 1_000 72 | """Defines how often to log when request is recorded/replayed""" 73 | 74 | 75 | @dataclass 76 | class GracyReplay: 77 | mode: t.Literal["record", "replay", "smart-replay"] 78 | """ 79 | `record`: Will record all requests made to the API 80 | `replay`: Will read all responses from the defined storage 81 | `smart-replay`: Will read all responses (like `replay`), but if it's missing it will `record` for future replays 82 | """ 83 | 84 | storage: GracyReplayStorage 85 | """Where to read/write requests and responses""" 86 | 87 | discard_replays_older_than: datetime | None = None 88 | """If set, Gracy will treat all replays older than defined value as not found""" 89 | 90 | discard_bad_responses: bool = False 91 | """If set True, then Gracy will discard bad requests (e.g. non 2xx)""" 92 | 93 | disable_throttling: bool = False 94 | """Only applicable to `smart-replay` and `replay` modes. If a replay exists then don't throttle the request""" 95 | 96 | display_report: bool = True 97 | """Whether to display records made and replays made to the final report""" 98 | 99 | log_record: ReplayLogEvent | None = None 100 | """Whether to log and how often to upon recording requests. The only available placeholder is `RECORDED_COUNT`""" 101 | 102 | log_replay: ReplayLogEvent | None = None 103 | """Whether to log and how often to upon replaying requests. The only available placeholder is `REPLAYED_COUNT`""" 104 | 105 | records_made: int = 0 106 | replays_made: int = 0 107 | 108 | async def has_replay(self, request: httpx.Request) -> bool: 109 | replay = await self.storage.find_replay( 110 | request, self.discard_replays_older_than 111 | ) 112 | return bool(replay) 113 | 114 | def inc_record(self): 115 | self.records_made += 1 116 | 117 | if log_ev := self.log_record: 118 | if self.records_made % log_ev.frequency == 0: 119 | args = dict(RECORDED_COUNT=f"{self.records_made:,}") 120 | do_log(log_ev, DefaultLogMessage.REPLAY_RECORDED, args) 121 | 122 | def inc_replay(self): 123 | self.replays_made += 1 124 | 125 | if log_ev := self.log_replay: 126 | if self.replays_made % log_ev.frequency == 0: 127 | args = dict(REPLAYED_COUNT=f"{self.replays_made:,}") 128 | do_log(log_ev, DefaultLogMessage.REPLAY_REPLAYED, args) 129 | -------------------------------------------------------------------------------- /src/gracy/_paginator.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import typing as t 4 | 5 | RESP_T = t.TypeVar("RESP_T") 6 | TOKEN_T = t.TypeVar("TOKEN_T") 7 | 8 | 9 | class GracyPaginator(t.Generic[RESP_T, TOKEN_T]): 10 | def __init__( 11 | self, 12 | gracy_func: t.Callable[..., t.Awaitable[RESP_T]], 13 | has_next: t.Callable[[t.Optional[RESP_T]], bool], 14 | initial_token: TOKEN_T, 15 | page_size: int = 20, 16 | get_next_token: t.Optional[t.Callable[[RESP_T, TOKEN_T], TOKEN_T]] = None, 17 | get_prev_token: t.Optional[t.Callable[[TOKEN_T], TOKEN_T]] = None, 18 | prepare_params: t.Optional[ 19 | t.Callable[[TOKEN_T, int], t.Dict[str, t.Any]] 20 | ] = None, 21 | has_prev: t.Optional[t.Callable[[TOKEN_T], bool]] = None, 22 | ): 23 | self.has_next = has_next 24 | 25 | self._endpoint_func = gracy_func 26 | self._custom_has_prev = has_prev 27 | self._prepare_endpoint_params = prepare_params 28 | self._get_next_token = get_next_token 29 | self._get_prev_token = get_prev_token 30 | 31 | self._token = initial_token 32 | self._page_size = page_size 33 | self._cur_resp: t.Optional[RESP_T] = None 34 | 35 | def _prepare_params( 36 | self, 37 | token: TOKEN_T, 38 | page_size: int, 39 | ) -> t.Dict[str, t.Any]: 40 | if self._prepare_endpoint_params: 41 | return self._prepare_endpoint_params(token, page_size) 42 | 43 | params = dict(token=token, limit=self._page_size) 44 | return params 45 | 46 | async def _fetch_page(self) -> RESP_T: 47 | params = self._prepare_params(self._token, page_size=20) 48 | self._cur_resp = await self._endpoint_func(**params) 49 | return self._cur_resp 50 | 51 | def has_prev(self, token: TOKEN_T) -> bool: 52 | if self._custom_has_prev: 53 | return self._custom_has_prev(token) 54 | 55 | return False 56 | 57 | def _calculate_next_token(self, resp: RESP_T, token: TOKEN_T) -> TOKEN_T: 58 | if self._get_next_token: 59 | return self._get_next_token(resp, token) 60 | 61 | raise NotImplementedError("GracyPaginator requires you to setup get_next_token") # noqa: TRY003 62 | 63 | def _calculate_prev_token(self, token: TOKEN_T) -> TOKEN_T: 64 | if self._get_prev_token: 65 | return self._get_prev_token(token) 66 | 67 | raise NotImplementedError("GracyPaginator requires you to setup get_prev_token") # noqa: TRY003 68 | 69 | def set_page(self, token: TOKEN_T) -> None: 70 | self._token = token 71 | 72 | async def next_page(self) -> RESP_T | None: 73 | if not self.has_next(self._cur_resp): 74 | return None 75 | 76 | page_result = await self._fetch_page() 77 | 78 | self._token = self._calculate_next_token(page_result, self._token) 79 | 80 | return page_result 81 | 82 | async def prev_page(self): 83 | if not self.has_prev(self._token): 84 | return None 85 | 86 | self._token = self._calculate_prev_token(self._token) 87 | 88 | page_result = await self._fetch_page() 89 | 90 | return page_result 91 | 92 | def __aiter__(self): 93 | return self 94 | 95 | async def __anext__(self): 96 | page = await self.next_page() 97 | if page is None: 98 | raise StopAsyncIteration 99 | return page 100 | 101 | 102 | class GracyOffsetPaginator(t.Generic[RESP_T], GracyPaginator[RESP_T, int]): 103 | def __init__( 104 | self, 105 | gracy_func: t.Callable[..., t.Awaitable[RESP_T]], 106 | has_next: t.Callable[[RESP_T | None], bool], 107 | page_size: int = 20, 108 | prepare_params: t.Callable[[int, int], t.Dict[str, t.Any]] | None = None, 109 | has_prev: t.Callable[[int], bool] | None = None, 110 | ): 111 | super().__init__( 112 | gracy_func, 113 | has_next, 114 | initial_token=0, 115 | page_size=page_size, 116 | prepare_params=prepare_params, 117 | has_prev=has_prev, 118 | ) 119 | 120 | def _prepare_params( 121 | self, 122 | token: int, 123 | page_size: int, 124 | ) -> t.Dict[str, t.Any]: 125 | if self._prepare_endpoint_params: 126 | return self._prepare_endpoint_params(token, page_size) 127 | 128 | params = dict(offset=token, limit=self._page_size) 129 | return params 130 | 131 | def has_prev(self, token: int) -> bool: 132 | if self._custom_has_prev: 133 | return self._custom_has_prev(token) 134 | 135 | return token > 0 136 | 137 | def _calculate_next_token(self, resp: RESP_T, token: int) -> int: 138 | return token + self._page_size 139 | 140 | def _calculate_prev_token(self, token: int) -> int: 141 | return token - self._page_size 142 | -------------------------------------------------------------------------------- /examples/pokeapi_limit_concurrency.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | import logging 5 | import time 6 | from datetime import timedelta 7 | from http import HTTPStatus 8 | 9 | from rich import print 10 | from rich.logging import RichHandler 11 | 12 | logging.basicConfig( 13 | level=logging.INFO, 14 | format="%(message)s", 15 | datefmt="[%X]", 16 | handlers=[RichHandler()], 17 | ) 18 | 19 | from gracy import ( # noqa: E402 20 | BaseEndpoint, 21 | ConcurrentRequestLimit, 22 | GracefulRetry, 23 | Gracy, 24 | GracyConfig, 25 | LogEvent, 26 | LogLevel, 27 | graceful, 28 | ) 29 | 30 | CONCURRENCY = ( 31 | ConcurrentRequestLimit( 32 | 2, 33 | limit_per_uurl=False, 34 | log_limit_reached=LogEvent( 35 | LogLevel.ERROR, 36 | custom_message="{URL} hit {CONCURRENT_REQUESTS} ongoing concurrent request", 37 | ), 38 | log_limit_freed=LogEvent(LogLevel.INFO, "{URL} is free to request"), 39 | ), 40 | ) 41 | 42 | RETRY = GracefulRetry( 43 | delay=0, # Force throttling to work 44 | max_attempts=3, 45 | retry_on=None, 46 | log_after=LogEvent(LogLevel.INFO), 47 | log_exhausted=LogEvent(LogLevel.ERROR), 48 | behavior="pass", 49 | ) 50 | 51 | 52 | class PokeApiEndpoint(BaseEndpoint): 53 | GET_POKEMON = "/pokemon/{NAME}" 54 | GET_GENERATION = "/generation/{ID}" 55 | 56 | 57 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 58 | class Config: 59 | BASE_URL = "https://pokeapi.co/api/v2/" 60 | SETTINGS = GracyConfig( 61 | strict_status_code={HTTPStatus.OK}, 62 | retry=RETRY, 63 | concurrent_requests=CONCURRENCY, 64 | parser={ 65 | "default": lambda r: r.json(), 66 | HTTPStatus.NOT_FOUND: None, 67 | }, 68 | ) 69 | 70 | @graceful( 71 | parser={"default": lambda r: r.json()["order"], HTTPStatus.NOT_FOUND: None} 72 | ) 73 | async def get_pokemon(self, name: str): 74 | await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 75 | 76 | async def get_generation(self, gen: int): 77 | return await self.get(PokeApiEndpoint.GET_GENERATION, {"ID": str(gen)}) 78 | 79 | @graceful(parser={"default": lambda r: r}) 80 | async def slow_req(self, s: int): 81 | await self.get("https://httpbin.org/delay/{DELAY}", dict(DELAY=str(s))) 82 | 83 | 84 | pokeapi = GracefulPokeAPI() 85 | 86 | 87 | async def main(): 88 | pokemon_names = [ 89 | "bulbasaur", 90 | "charmander", 91 | "squirtle", 92 | "pikachu", 93 | "jigglypuff", 94 | "mewtwo", 95 | "gyarados", 96 | "dragonite", 97 | "mew", 98 | "chikorita", 99 | "cyndaquil", 100 | "totodile", 101 | "pichu", 102 | "togepi", 103 | "ampharos", 104 | "typhlosion", 105 | "feraligatr", 106 | "espeon", 107 | "umbreon", 108 | "lugia", 109 | "ho-oh", 110 | "treecko", 111 | "torchic", 112 | "mudkip", 113 | "gardevoir", 114 | "sceptile", 115 | "blaziken", 116 | "swampert", 117 | "rayquaza", 118 | "latias", 119 | "latios", 120 | "lucario", 121 | "garchomp", 122 | "darkrai", 123 | "giratina", # (1) this fails, so good to test retry 124 | "arceus", 125 | "snivy", 126 | "tepig", 127 | "oshawott", 128 | "zekrom", 129 | "reshiram", 130 | "victini", 131 | "chespin", 132 | "fennekin", 133 | "froakie", 134 | "xerneas", 135 | "yveltal", 136 | "zygarde", # (2) this fails, so good to test retry 137 | "decidueye", 138 | "incineroar", 139 | ] 140 | # pokemon_names = pokemon_names[:10] 141 | 142 | try: 143 | start = time.time() 144 | 145 | pokemon_reqs = [ 146 | asyncio.create_task(pokeapi.get_pokemon(name)) 147 | for name in pokemon_names[:10] 148 | ] 149 | 150 | slow_reqs = [asyncio.create_task(pokeapi.slow_req(s)) for s in range(3)] 151 | 152 | pokemon_reqs += [ 153 | asyncio.create_task(pokeapi.get_pokemon(name)) 154 | for name in pokemon_names[10:20] 155 | ] 156 | 157 | slow_reqs += [asyncio.create_task(pokeapi.slow_req(s)) for s in range(3)] 158 | 159 | pokemon_reqs += [ 160 | asyncio.create_task(pokeapi.get_pokemon(name)) 161 | for name in pokemon_names[20:] 162 | ] 163 | 164 | gen_reqs = [ 165 | asyncio.create_task(pokeapi.get_generation(gen)) for gen in range(1, 4) 166 | ] 167 | 168 | await asyncio.gather(*pokemon_reqs, *gen_reqs, *slow_reqs) 169 | 170 | await pokeapi.get_pokemon("hitmonchan") 171 | 172 | elapsed = time.time() - start 173 | print(f"All requests took {timedelta(seconds=elapsed)}s to finish") 174 | 175 | finally: 176 | plotly = pokeapi.report_status("plotly") 177 | plotly.show() 178 | 179 | 180 | asyncio.run(main()) 181 | -------------------------------------------------------------------------------- /src/gracy/replays/storages/pymongo.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import httpx 4 | import json 5 | import pickle 6 | import typing as t 7 | from dataclasses import asdict, dataclass 8 | from datetime import datetime 9 | from threading import Lock 10 | 11 | from gracy.exceptions import GracyReplayRequestNotFound 12 | 13 | from ._base import GracyReplayStorage 14 | 15 | try: 16 | import pymongo 17 | except ModuleNotFoundError: 18 | pass 19 | 20 | 21 | @dataclass 22 | class MongoCredentials: 23 | host: str | None = None 24 | """Can be a full URI""" 25 | port: int = 27017 26 | username: str | None = None 27 | password: str | None = None 28 | 29 | 30 | class MongoReplayDocument(t.TypedDict): 31 | url: str 32 | method: str 33 | request_body: bytes | None 34 | response: bytes 35 | response_content: dict[str, t.Any] | str | None 36 | """Useful for debugging since Mongo supports unstructured data""" 37 | updated_at: datetime 38 | 39 | 40 | def get_unique_keys_from_doc( 41 | replay_doc: MongoReplayDocument, 42 | ) -> t.Dict[str, bytes | None | str]: 43 | return { 44 | "url": replay_doc["url"], 45 | "method": replay_doc["method"], 46 | "request_body": replay_doc["request_body"], 47 | } 48 | 49 | 50 | def get_unique_keys_from_request( 51 | request: httpx.Request, 52 | ) -> t.Dict[str, bytes | None | str]: 53 | return { 54 | "url": str(request.url), 55 | "method": request.method, 56 | "request_body": request.content or None, 57 | } 58 | 59 | 60 | batch_lock = Lock() 61 | 62 | 63 | class MongoReplayStorage(GracyReplayStorage): 64 | def __init__( 65 | self, 66 | creds: MongoCredentials, 67 | database_name: str = "gracy", 68 | collection_name: str = "gracy-replay", 69 | batch_size: int | None = None, 70 | ) -> None: 71 | creds_kwargs = asdict(creds) 72 | 73 | client = pymongo.MongoClient(**creds_kwargs, document_class=MongoReplayDocument) # pyright: ignore[reportPossiblyUnboundVariable] 74 | mongo_db = client[database_name] 75 | self._collection = mongo_db[collection_name] 76 | self._batch = batch_size 77 | self._batch_ops: t.List[pymongo.ReplaceOne[MongoReplayDocument]] = [] 78 | 79 | def _flush_batch(self) -> None: 80 | if self._batch_ops: 81 | with batch_lock: 82 | self._collection.bulk_write(self._batch_ops) # type: ignore 83 | self._batch_ops = [] 84 | 85 | def _create_or_batch(self, doc: MongoReplayDocument) -> None: 86 | filter = get_unique_keys_from_doc(doc) 87 | if self._batch and self._batch > 1: 88 | with batch_lock: 89 | self._batch_ops.append(pymongo.ReplaceOne(filter, doc, upsert=True)) # pyright: ignore[reportPossiblyUnboundVariable] 90 | 91 | if len(self._batch_ops) >= self._batch: 92 | self._flush_batch() 93 | 94 | else: 95 | self._collection.replace_one(filter, doc, upsert=True) 96 | 97 | def prepare(self) -> None: 98 | self._collection.create_index( 99 | [("url", 1), ("method", 1), ("request_body", 1)], 100 | background=True, 101 | unique=True, 102 | ) 103 | 104 | async def record(self, response: httpx.Response) -> None: 105 | response_serialized = pickle.dumps(response) 106 | 107 | response_content = response.text or None 108 | content_type = response.headers.get("Content-Type") 109 | if content_type and "json" in content_type: 110 | try: 111 | jsonified_content = response.json() 112 | 113 | except json.decoder.JSONDecodeError: 114 | pass 115 | 116 | else: 117 | response_content = jsonified_content 118 | 119 | doc = MongoReplayDocument( 120 | url=str(response.url), 121 | method=str(response.request.method), 122 | request_body=response.request.content or None, 123 | response=response_serialized, 124 | response_content=response_content, 125 | updated_at=datetime.now(), 126 | ) 127 | 128 | self._create_or_batch(doc) 129 | 130 | async def find_replay( 131 | self, request: httpx.Request, discard_before: datetime | None 132 | ) -> MongoReplayDocument | None: 133 | filter = get_unique_keys_from_request(request) 134 | doc = self._collection.find_one(filter) 135 | 136 | if doc is None: 137 | return None 138 | 139 | if discard_before and doc["updated_at"] < discard_before: 140 | return None 141 | 142 | return doc 143 | 144 | async def _load( 145 | self, request: httpx.Request, discard_before: datetime | None 146 | ) -> httpx.Response: 147 | doc = await self.find_replay(request, discard_before) 148 | 149 | if doc is None: 150 | raise GracyReplayRequestNotFound(request) 151 | 152 | serialized_response = doc["response"] 153 | response: httpx.Response = pickle.loads(serialized_response) 154 | 155 | return response 156 | 157 | def flush(self) -> None: 158 | self._flush_batch() 159 | -------------------------------------------------------------------------------- /src/tests/test_hooks.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import httpx 4 | import pytest 5 | import typing as t 6 | from http import HTTPStatus 7 | from unittest.mock import patch 8 | 9 | from gracy import ( 10 | GracefulRetry, 11 | GracefulRetryState, 12 | Gracy, 13 | GracyConfig, 14 | GracyRequestContext, 15 | ) 16 | from gracy.exceptions import GracyRequestFailed 17 | from tests.conftest import ( 18 | MISSING_NAME, 19 | PRESENT_POKEMON_NAME, 20 | REPLAY, 21 | PokeApiEndpoint, 22 | assert_requests_made, 23 | ) 24 | 25 | RETRY: t.Final = GracefulRetry( 26 | delay=0.001, 27 | max_attempts=2, 28 | retry_on={HTTPStatus.NOT_FOUND}, 29 | behavior="break", 30 | ) 31 | 32 | 33 | @pytest.fixture() 34 | def make_pokeapi(): 35 | def factory(): 36 | Gracy.dangerously_reset_report() 37 | return GracefulPokeAPI(REPLAY) 38 | 39 | return factory 40 | 41 | 42 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 43 | class Config: 44 | BASE_URL = "https://pokeapi.co/api/v2/" 45 | SETTINGS = GracyConfig( 46 | retry=RETRY, 47 | allowed_status_code={HTTPStatus.NOT_FOUND}, 48 | parser={HTTPStatus.NOT_FOUND: None}, 49 | ) 50 | 51 | def __init__(self, *args: t.Any, **kwargs: t.Any) -> None: 52 | self.before_count = 0 53 | 54 | self.after_status_counter = t.DefaultDict[int, int](int) 55 | self.after_aborts = 0 56 | self.after_retries_counter = 0 57 | 58 | super().__init__(*args, **kwargs) 59 | 60 | async def before(self, context: GracyRequestContext): 61 | self.before_count += 1 62 | 63 | async def after( 64 | self, 65 | context: GracyRequestContext, 66 | response_or_exc: httpx.Response | Exception, 67 | retry_state: GracefulRetryState | None, 68 | ): 69 | if retry_state: 70 | self.after_retries_counter += 1 71 | 72 | if isinstance(response_or_exc, httpx.Response): 73 | self.after_status_counter[response_or_exc.status_code] += 1 74 | else: 75 | self.after_aborts += 1 76 | 77 | async def get_pokemon(self, name: str): 78 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 79 | 80 | 81 | class GracefulPokeAPIWithRequestHooks(GracefulPokeAPI): 82 | async def before(self, context: GracyRequestContext): 83 | await super().before(context) 84 | # This shouldn't re-trigger any hook! 85 | await self.get_pokemon(PRESENT_POKEMON_NAME) 86 | 87 | async def after( 88 | self, 89 | context: GracyRequestContext, 90 | response_or_exc: httpx.Response | Exception, 91 | retry_state: GracefulRetryState | None, 92 | ): 93 | await super().after(context, response_or_exc, retry_state) 94 | # This shouldn't re-trigger any hook! 95 | await self.get_pokemon(PRESENT_POKEMON_NAME) 96 | 97 | 98 | MAKE_POKEAPI_TYPE = t.Callable[[], GracefulPokeAPI] 99 | 100 | 101 | async def test_before_hook_counts(make_pokeapi: MAKE_POKEAPI_TYPE): 102 | pokeapi = make_pokeapi() 103 | 104 | assert pokeapi.before_count == 0 105 | await pokeapi.get(PokeApiEndpoint.GET_POKEMON, dict(NAME=PRESENT_POKEMON_NAME)) 106 | assert pokeapi.before_count == 1 107 | await pokeapi.get(PokeApiEndpoint.GET_POKEMON, dict(NAME=PRESENT_POKEMON_NAME)) 108 | assert pokeapi.before_count == 2 109 | 110 | 111 | async def test_after_hook_counts_statuses(make_pokeapi: MAKE_POKEAPI_TYPE): 112 | pokeapi = make_pokeapi() 113 | 114 | assert pokeapi.after_status_counter[HTTPStatus.OK] == 0 115 | assert pokeapi.after_status_counter[HTTPStatus.NOT_FOUND] == 0 116 | 117 | await pokeapi.get( 118 | PokeApiEndpoint.GET_POKEMON, dict(NAME=PRESENT_POKEMON_NAME) 119 | ) # 200 120 | await pokeapi.get( 121 | PokeApiEndpoint.GET_POKEMON, dict(NAME=PRESENT_POKEMON_NAME) 122 | ) # 200 123 | await pokeapi.get( 124 | PokeApiEndpoint.GET_POKEMON, dict(NAME=MISSING_NAME) 125 | ) # 404 + retry 2x 126 | await pokeapi.get( 127 | PokeApiEndpoint.GET_POKEMON, dict(NAME=MISSING_NAME) 128 | ) # 404 + retry 2x 129 | 130 | assert pokeapi.after_status_counter[HTTPStatus.OK] == 2 131 | assert pokeapi.after_status_counter[HTTPStatus.NOT_FOUND] == 6 132 | assert pokeapi.after_retries_counter == 4 133 | 134 | 135 | async def test_after_hook_counts_aborts(): 136 | Gracy.dangerously_reset_report() 137 | pokeapi = GracefulPokeAPI() 138 | 139 | class SomeRequestException(Exception): 140 | pass 141 | 142 | mock: t.Any 143 | with patch.object(pokeapi, "_client", autospec=True) as mock: 144 | mock.request.side_effect = SomeRequestException("Request failed") 145 | 146 | with pytest.raises(GracyRequestFailed): 147 | await pokeapi.get_pokemon(PRESENT_POKEMON_NAME) 148 | 149 | assert pokeapi.after_status_counter[HTTPStatus.OK] == 0 150 | assert pokeapi.after_status_counter[HTTPStatus.NOT_FOUND] == 0 151 | assert pokeapi.after_retries_counter == 0 152 | assert pokeapi.after_aborts == 1 153 | 154 | 155 | async def test_hook_has_no_recursion(): 156 | Gracy.dangerously_reset_report() 157 | pokeapi = GracefulPokeAPIWithRequestHooks(REPLAY) 158 | 159 | EXPECTED_REQS: t.Final = 1 + 2 # This + Before hook + After hook 160 | await pokeapi.get_pokemon(PRESENT_POKEMON_NAME) 161 | 162 | assert_requests_made(pokeapi, EXPECTED_REQS) 163 | 164 | 165 | async def test_hook_with_retries_has_no_recursion(): 166 | Gracy.dangerously_reset_report() 167 | pokeapi = GracefulPokeAPIWithRequestHooks(REPLAY) 168 | 169 | # (1 This + 2 Retries) + 2 hooks for each (3) 170 | EXPECTED_REQS: t.Final = (1 + 2) + (2 * 3) 171 | await pokeapi.get_pokemon(MISSING_NAME) 172 | 173 | assert pokeapi.before_count == 3 174 | assert pokeapi.after_status_counter[HTTPStatus.NOT_FOUND] == 3 175 | assert pokeapi.after_retries_counter == 2 176 | assert_requests_made(pokeapi, EXPECTED_REQS) 177 | -------------------------------------------------------------------------------- /src/gracy/_loggers.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import httpx 4 | import logging 5 | import typing as t 6 | from enum import Enum 7 | 8 | from ._models import GracefulRetryState, GracyRequestContext, LogEvent, ThrottleRule 9 | 10 | logger = logging.getLogger("gracy") 11 | 12 | 13 | def is_replay(resp: httpx.Response) -> bool: 14 | return getattr(resp, "_gracy_replayed", False) 15 | 16 | 17 | class SafeDict(t.Dict[str, str]): 18 | def __missing__(self, key: str): 19 | return "{" + key + "}" 20 | 21 | 22 | class DefaultLogMessage(str, Enum): 23 | BEFORE = "Request on {URL} is ongoing" 24 | AFTER = "{REPLAY}[{METHOD}] {URL} returned {STATUS}" 25 | ERRORS = "[{METHOD}] {URL} returned a bad status ({STATUS})" 26 | 27 | THROTTLE_HIT = "{URL} hit {THROTTLE_LIMIT} reqs/{THROTTLE_TIME_RANGE}" 28 | THROTTLE_DONE = "Done waiting {THROTTLE_TIME}s to hit {URL}" 29 | 30 | RETRY_BEFORE = ( 31 | "GracefulRetry: {URL} will wait {RETRY_DELAY}s before next attempt due to " 32 | "{RETRY_CAUSE} ({CUR_ATTEMPT} out of {MAX_ATTEMPT})" 33 | ) 34 | RETRY_AFTER = ( 35 | "GracefulRetry: {URL} replied {STATUS} ({CUR_ATTEMPT} out of {MAX_ATTEMPT})" 36 | ) 37 | RETRY_EXHAUSTED = "GracefulRetry: {URL} exhausted the maximum attempts of {MAX_ATTEMPT} due to {RETRY_CAUSE}" 38 | 39 | REPLAY_RECORDED = "Gracy Replay: Recorded {RECORDED_COUNT} requests" 40 | REPLAY_REPLAYED = "Gracy Replay: Replayed {REPLAYED_COUNT} requests" 41 | 42 | CONCURRENT_REQUEST_LIMIT_HIT = ( 43 | "{UURL} hit {CONCURRENT_REQUESTS} ongoing concurrent requests" 44 | ) 45 | CONCURRENT_REQUEST_LIMIT_FREED = "{UURL} concurrency has been freed" 46 | 47 | 48 | def do_log( 49 | logevent: LogEvent, 50 | defaultmsg: str, 51 | format_args: dict[str, t.Any], 52 | response: httpx.Response | None = None, 53 | ): 54 | # Let's protect ourselves against potential customizations with undefined {key} 55 | safe_format_args = SafeDict(**format_args) 56 | 57 | if logevent.custom_message: 58 | if isinstance(logevent.custom_message, str): 59 | message = logevent.custom_message.format_map(safe_format_args) 60 | else: 61 | message = logevent.custom_message(response).format_map(safe_format_args) 62 | else: 63 | message = defaultmsg.format_map(safe_format_args) 64 | 65 | logger.log(logevent.level, message, extra=format_args) 66 | 67 | 68 | def extract_base_format_args(request_context: GracyRequestContext) -> dict[str, str]: 69 | return dict( 70 | URL=request_context.url, 71 | ENDPOINT=request_context.endpoint, 72 | UURL=request_context.unformatted_url, 73 | UENDPOINT=request_context.unformatted_endpoint, 74 | METHOD=request_context.method, 75 | ) 76 | 77 | 78 | def extract_response_format_args(response: httpx.Response | None) -> dict[str, str]: 79 | status_code = response.status_code if response else "ABORTED" 80 | elapsed = response.elapsed if response else "UNKNOWN" 81 | 82 | if response and is_replay(response): 83 | replayed = "TRUE" 84 | replayed_str = "REPLAYED" 85 | else: 86 | replayed = "FALSE" 87 | replayed_str = "" 88 | 89 | return dict( 90 | STATUS=str(status_code), 91 | ELAPSED=str(elapsed), 92 | IS_REPLAY=replayed, 93 | REPLAY=replayed_str, 94 | ) 95 | 96 | 97 | def process_log_before_request( 98 | logevent: LogEvent, request_context: GracyRequestContext 99 | ) -> None: 100 | format_args = extract_base_format_args(request_context) 101 | do_log(logevent, DefaultLogMessage.BEFORE, format_args) 102 | 103 | 104 | def process_log_throttle( 105 | logevent: LogEvent, 106 | default_message: str, 107 | await_time: float, 108 | rule: ThrottleRule, 109 | request_context: GracyRequestContext, 110 | ): 111 | format_args = dict( 112 | **extract_base_format_args(request_context), 113 | THROTTLE_TIME=await_time, 114 | THROTTLE_LIMIT=rule.max_requests, 115 | THROTTLE_TIME_RANGE=rule.readable_time_range, 116 | ) 117 | 118 | do_log(logevent, default_message, format_args) 119 | 120 | 121 | def process_log_retry( 122 | logevent: LogEvent, 123 | defaultmsg: str, 124 | request_context: GracyRequestContext, 125 | state: GracefulRetryState, 126 | response: httpx.Response | None = None, 127 | ): 128 | maybe_response_args: dict[str, str] = {} 129 | if response: 130 | maybe_response_args = extract_response_format_args(response) 131 | 132 | format_args = dict( 133 | **extract_base_format_args(request_context), 134 | **maybe_response_args, 135 | RETRY_DELAY=state.delay, 136 | RETRY_CAUSE=state.cause, 137 | CUR_ATTEMPT=state.cur_attempt, 138 | MAX_ATTEMPT=state.max_attempts, 139 | ) 140 | 141 | do_log(logevent, defaultmsg, format_args, response) 142 | 143 | 144 | def process_log_after_request( 145 | logevent: LogEvent, 146 | defaultmsg: str, 147 | request_context: GracyRequestContext, 148 | response: httpx.Response | None, 149 | ) -> None: 150 | format_args: dict[str, str] = dict( 151 | **extract_base_format_args(request_context), 152 | **extract_response_format_args(response), 153 | ) 154 | 155 | do_log(logevent, defaultmsg, format_args, response) 156 | 157 | 158 | def process_log_concurrency_limit( 159 | logevent: LogEvent, count: int, request_context: GracyRequestContext 160 | ): 161 | format_args: t.Dict[str, str] = dict( 162 | CONCURRENT_REQUESTS=f"{count:,}", 163 | **extract_base_format_args(request_context), 164 | ) 165 | 166 | do_log(logevent, DefaultLogMessage.CONCURRENT_REQUEST_LIMIT_HIT, format_args) 167 | 168 | 169 | def process_log_concurrency_freed( 170 | logevent: LogEvent, request_context: GracyRequestContext 171 | ): 172 | format_args: t.Dict[str, str] = dict( 173 | **extract_base_format_args(request_context), 174 | ) 175 | do_log(logevent, DefaultLogMessage.CONCURRENT_REQUEST_LIMIT_FREED, format_args) 176 | -------------------------------------------------------------------------------- /src/gracy/_reports/_builders.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import httpx 4 | import re 5 | import typing as t 6 | from collections import defaultdict 7 | from statistics import mean 8 | 9 | from .._models import GracyRequestContext, RequestTimeline, ThrottleController 10 | from ..replays.storages._base import GracyReplay, is_replay 11 | from ._models import ( 12 | GracyAggregatedRequest, 13 | GracyReport, 14 | GracyRequestCounters, 15 | GracyRequestResult, 16 | ) 17 | 18 | ANY_REGEX: t.Final = r".+" 19 | 20 | REQUEST_ERROR_STATUS: t.Final = 0 21 | REQUEST_SUM_KEY = t.Union[int, t.Literal["total", "retries", "throttles", "replays", 0]] 22 | REQUEST_SUM_PER_STATUS_TYPE = t.Dict[str, t.DefaultDict[REQUEST_SUM_KEY, int]] 23 | 24 | 25 | class ReportBuilder: 26 | def __init__(self) -> None: 27 | self._results: t.List[GracyRequestResult] = [] 28 | self._counters = t.DefaultDict[str, GracyRequestCounters](GracyRequestCounters) 29 | self._request_history = t.DefaultDict[str, t.List[RequestTimeline]](list) 30 | 31 | def track( 32 | self, 33 | request_context: GracyRequestContext, 34 | response_or_exc: t.Union[httpx.Response, Exception], 35 | request_start: float, 36 | ): 37 | self._results.append( 38 | GracyRequestResult(request_context.unformatted_url, response_or_exc) 39 | ) 40 | if isinstance(response_or_exc, httpx.Response): 41 | if is_replay(response_or_exc): 42 | self._replayed(request_context) 43 | 44 | request_entry = RequestTimeline.build(request_start, response_or_exc) 45 | self._request_history[request_context.unformatted_url].append(request_entry) 46 | 47 | def retried(self, request_context: GracyRequestContext): 48 | self._counters[request_context.unformatted_url].retries += 1 49 | 50 | def throttled(self, request_context: GracyRequestContext): 51 | self._counters[request_context.unformatted_url].throttles += 1 52 | 53 | def _replayed(self, request_context: GracyRequestContext): 54 | self._counters[request_context.unformatted_url].replays += 1 55 | 56 | def _calculate_req_rate_for_url( 57 | self, unformatted_url: str, throttle_controller: ThrottleController 58 | ) -> float: 59 | pattern = re.compile(re.sub(r"{(\w+)}", ANY_REGEX, unformatted_url)) 60 | rate = throttle_controller.calculate_requests_per_sec(pattern) 61 | return rate 62 | 63 | def build( 64 | self, 65 | throttle_controller: ThrottleController, 66 | replay_settings: GracyReplay | None, 67 | ) -> GracyReport: 68 | requests_by_uurl = t.DefaultDict[ 69 | str, t.Set[t.Union[httpx.Response, Exception]] 70 | ](set) 71 | requests_sum: REQUEST_SUM_PER_STATUS_TYPE = defaultdict( 72 | lambda: defaultdict(int) 73 | ) 74 | 75 | for result in self._results: 76 | requests_by_uurl[result.uurl].add(result.response) 77 | requests_sum[result.uurl]["total"] += 1 78 | if isinstance(result.response, httpx.Response): 79 | requests_sum[result.uurl][result.response.status_code] += 1 80 | else: 81 | requests_sum[result.uurl][REQUEST_ERROR_STATUS] += 1 82 | 83 | for uurl, counters in self._counters.items(): 84 | requests_sum[uurl]["throttles"] = counters.throttles 85 | requests_sum[uurl]["retries"] = counters.retries 86 | requests_sum[uurl]["replays"] = counters.replays 87 | 88 | requests_sum = dict( 89 | sorted( 90 | requests_sum.items(), key=lambda item: item[1]["total"], reverse=True 91 | ) 92 | ) 93 | 94 | report = GracyReport(replay_settings, self._request_history) 95 | 96 | for uurl, data in requests_sum.items(): 97 | all_requests = { 98 | req for req in requests_by_uurl[uurl] if isinstance(req, httpx.Response) 99 | } 100 | 101 | total_requests = data["total"] 102 | url_latency = [r.elapsed.total_seconds() for r in all_requests] 103 | 104 | # Rate 105 | # Use min to handle scenarios like: 106 | # 10 reqs in a 2 millisecond window would produce a number >1,000 leading the user to think that we're 107 | # producing 1,000 requests which isn't true. 108 | rate = min( 109 | self._calculate_req_rate_for_url(uurl, throttle_controller), 110 | total_requests, 111 | ) 112 | 113 | resp_2xx = 0 114 | resp_3xx = 0 115 | resp_4xx = 0 116 | resp_5xx = 0 117 | aborted = 0 118 | retries = 0 119 | throttles = 0 120 | replays = 0 121 | 122 | for maybe_status, count in data.items(): 123 | if maybe_status == "total": 124 | continue 125 | 126 | if maybe_status == REQUEST_ERROR_STATUS: 127 | aborted += count 128 | continue 129 | 130 | if maybe_status == "throttles": 131 | throttles += count 132 | continue 133 | 134 | if maybe_status == "retries": 135 | retries += count 136 | continue 137 | 138 | if maybe_status == "replays": 139 | replays += count 140 | continue 141 | 142 | status = maybe_status 143 | if 200 <= status < 300: 144 | resp_2xx += count 145 | elif 300 <= status < 400: 146 | resp_3xx += count 147 | elif 400 <= status < 500: 148 | resp_4xx += count 149 | elif 500 <= status: 150 | resp_5xx += count 151 | 152 | report_request = GracyAggregatedRequest( 153 | uurl, 154 | total_requests, 155 | # Responses 156 | resp_2xx=resp_2xx, 157 | resp_3xx=resp_3xx, 158 | resp_4xx=resp_4xx, 159 | resp_5xx=resp_5xx, 160 | reqs_aborted=aborted, 161 | retries=retries, 162 | throttles=throttles, 163 | replays=replays, 164 | # General 165 | avg_latency=mean(url_latency) if url_latency else 0, 166 | max_latency=max(url_latency) if url_latency else 0, 167 | req_rate_per_sec=rate, 168 | ) 169 | 170 | report.add_request(report_request) 171 | 172 | return report 173 | -------------------------------------------------------------------------------- /src/gracy/exceptions.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import httpx 4 | import typing as t 5 | from abc import ABC, abstractmethod 6 | 7 | from ._models import GracyRequestContext 8 | 9 | REDUCE_PICKABLE_RETURN = t.Tuple[t.Type[Exception], t.Tuple[t.Any, ...]] 10 | 11 | 12 | class GracyException(Exception, ABC): 13 | @abstractmethod 14 | def __reduce__(self) -> REDUCE_PICKABLE_RETURN: 15 | """ 16 | `__reduce__` is required to avoid Gracy from breaking in different 17 | environments that pickles the results (e.g. inside ThreadPools). 18 | 19 | More context: https://stackoverflow.com/a/36342588/2811539 20 | """ 21 | pass 22 | 23 | 24 | class GracyRequestFailed(GracyException): 25 | """ 26 | Sometimes the httpx's request fails for whatever reason (TCP, SSL, etc errors), this 27 | is a wrapper exception so the client can be easily "retried" for any failed requests. 28 | 29 | NOTE: Consider that failed requests means NO RESPONSE because the request never completed 30 | 31 | Maybe this would be an `ExceptionGroup` if Gracy ever deprecates Python < 3.11 32 | """ 33 | 34 | def __init__(self, context: GracyRequestContext, original_exc: Exception) -> None: 35 | self.original_exc = original_exc 36 | self.request_context = context 37 | 38 | original_exc_name = self._get_exc_name(original_exc) 39 | 40 | super().__init__( 41 | f"The request for [{context.method}] {context.url} never got a response due to {original_exc_name} " 42 | ) 43 | 44 | # Inspired by https://stackoverflow.com/a/54716092/2811539 45 | # We include the original exception as part of the stack trace by doing that. 46 | self.__cause__ = original_exc 47 | self.__context__ = original_exc 48 | 49 | @staticmethod 50 | def _get_exc_name(exc: Exception) -> str: 51 | """ 52 | Formats the exception as "module.ClassType" 53 | 54 | e.g. httpx.ReadTimeout 55 | """ 56 | exc_type = type(exc) 57 | 58 | module = exc_type.__module__ 59 | if module is not None and module != "__main__": 60 | return module + "." + exc_type.__qualname__ 61 | 62 | return exc_type.__qualname__ 63 | 64 | def __reduce__(self) -> REDUCE_PICKABLE_RETURN: 65 | return (GracyRequestFailed, (self.request_context, self.original_exc)) 66 | 67 | 68 | class GracyParseFailed(GracyException): 69 | def __init__(self, response: httpx.Response) -> None: 70 | msg = ( 71 | f"Unable to parse result from [{response.request.method}] {response.url} ({response.status_code}). " 72 | f"Response content is: {response.text}" 73 | ) 74 | 75 | self.url = response.request.url 76 | self.response = response 77 | 78 | super().__init__(msg) 79 | 80 | def __reduce__(self) -> REDUCE_PICKABLE_RETURN: 81 | return (GracyParseFailed, (self.response,)) 82 | 83 | 84 | class BadResponse(GracyException): 85 | def __init__( 86 | self, 87 | message: str | None, 88 | url: str, 89 | response: httpx.Response, 90 | expected: str | int | t.Iterable[int], 91 | ) -> None: 92 | self.url = url 93 | self.response = response 94 | 95 | self._args = ( 96 | message, 97 | url, 98 | response, 99 | expected, 100 | ) 101 | 102 | if isinstance(expected, str): 103 | expectedstr = expected 104 | elif isinstance(expected, int): 105 | expectedstr = str(expected) 106 | else: 107 | expectedstr = ", ".join([str(s) for s in expected]) 108 | 109 | curmsg = ( 110 | message 111 | or f"{url} raised {response.status_code}, but it was expecting {expectedstr}" 112 | ) 113 | 114 | super().__init__(curmsg) 115 | 116 | def __reduce__(self) -> REDUCE_PICKABLE_RETURN: 117 | return (BadResponse, self._args) 118 | 119 | 120 | class UnexpectedResponse(BadResponse): 121 | def __init__( 122 | self, url: str, response: httpx.Response, expected: str | int | t.Iterable[int] 123 | ) -> None: 124 | super().__init__(None, url, response, expected) 125 | 126 | self.url = url 127 | self.response = response 128 | self.expected = expected 129 | 130 | def __reduce__(self) -> REDUCE_PICKABLE_RETURN: 131 | return (UnexpectedResponse, (self.url, self.response, self.expected)) 132 | 133 | 134 | class NonOkResponse(BadResponse): 135 | def __init__(self, url: str, response: httpx.Response) -> None: 136 | super().__init__(None, url, response, "any successful status code") 137 | 138 | self.arg1 = url 139 | self.arg2 = response 140 | 141 | def __reduce__(self) -> REDUCE_PICKABLE_RETURN: 142 | return (NonOkResponse, (self.arg1, self.arg2)) 143 | 144 | 145 | class GracyUserDefinedException(GracyException): 146 | BASE_MESSAGE: str = "[{METHOD}] {URL} returned {}" 147 | 148 | def __init__( 149 | self, request_context: GracyRequestContext, response: httpx.Response 150 | ) -> None: 151 | self._request_context = request_context 152 | self._response = response 153 | super().__init__(self._format_message(request_context, response)) 154 | 155 | def _build_default_args(self) -> dict[str, t.Any]: 156 | request_context = self._request_context 157 | 158 | return dict( 159 | # Context 160 | ENDPOINT=request_context.endpoint, 161 | UURL=request_context.unformatted_url, 162 | UENDPOINT=request_context.unformatted_endpoint, 163 | # Response 164 | URL=self.response.request.url, 165 | METHOD=self.response.request.method, 166 | STATUS=self.response.status_code, 167 | ELAPSED=self.response.elapsed, 168 | ) 169 | 170 | def _format_message( 171 | self, request_context: GracyRequestContext, response: httpx.Response 172 | ) -> str: 173 | format_args = self._build_default_args() 174 | return self.BASE_MESSAGE.format(**format_args) 175 | 176 | @property 177 | def url(self): 178 | return self._request_context.url 179 | 180 | @property 181 | def endpoint(self): 182 | return self._request_context.endpoint 183 | 184 | @property 185 | def response(self): 186 | return self._response 187 | 188 | def __reduce__(self) -> REDUCE_PICKABLE_RETURN: 189 | return (GracyUserDefinedException, (self._request_context, self._response)) 190 | 191 | 192 | class GracyReplayRequestNotFound(GracyException): 193 | def __init__(self, request: httpx.Request) -> None: 194 | self.request = request 195 | 196 | msg = f"Gracy was unable to replay {request.method} {request.url} - did you forget to record it?" 197 | super().__init__(msg) 198 | 199 | def __reduce__(self) -> REDUCE_PICKABLE_RETURN: 200 | return (GracyReplayRequestNotFound, (self.request,)) 201 | -------------------------------------------------------------------------------- /src/gracy/common_hooks.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | import logging 5 | import typing as t 6 | from asyncio import Lock 7 | from dataclasses import dataclass 8 | from datetime import datetime 9 | from http import HTTPStatus 10 | 11 | import httpx 12 | 13 | from ._loggers import do_log, extract_base_format_args, extract_response_format_args 14 | from ._models import GracyRequestContext, LogEvent 15 | from ._reports._builders import ReportBuilder 16 | from .replays.storages._base import is_replay 17 | 18 | logger = logging.getLogger("gracy") 19 | 20 | 21 | @dataclass 22 | class HookResult: 23 | executed: bool 24 | awaited: float = 0 25 | dry_run: bool = False 26 | 27 | 28 | class HttpHeaderRetryAfterBackOffHook: 29 | """ 30 | Provides two methods `before()` and `after()` to be used as hooks by Gracy. 31 | 32 | This hook checks for 429 (TOO MANY REQUESTS), and then reads the 33 | `retry-after` header (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After). 34 | 35 | If the value is set, then Gracy pauses **ALL** client requests until the time is over. 36 | This behavior can be modified to happen on a per-endpoint basis if `lock_per_endpoint` is True. 37 | 38 | ### ⚠️ Retry 39 | This doesn't replace `GracefulRetry`. 40 | Make sure you implement a proper retry logic, otherwise the 429 will break the client. 41 | 42 | ### Throttling 43 | If you pass the reporter, it will count every await as "throttled" for that UURL. 44 | 45 | ### Processor 46 | You can optionally pass in a lambda, that can be used to modify/increase the wait time from the header. 47 | 48 | ### Log Event 49 | You can optionally define a log event. 50 | It provides the response and the context, but also `RETRY_AFTER` that contains the header value. 51 | `RETRY_AFTER_ACTUAL_WAIT` is also available in case you modify the original value. 52 | """ 53 | 54 | DEFAULT_LOG_MESSAGE: t.Final = ( 55 | "[{METHOD}] {URL} requested to wait for {RETRY_AFTER}s" 56 | ) 57 | ALL_CLIENT_LOCK: t.Final = "CLIENT" 58 | 59 | def __init__( 60 | self, 61 | reporter: ReportBuilder | None = None, 62 | lock_per_endpoint: bool = False, 63 | log_event: LogEvent | None = None, 64 | seconds_processor: None | t.Callable[[float], float] = None, 65 | *, 66 | dry_run: bool = False, 67 | ) -> None: 68 | self._reporter = reporter 69 | self._lock_per_endpoint = lock_per_endpoint 70 | self._lock_manager = t.DefaultDict[str, Lock](Lock) 71 | self._log_event = log_event 72 | self._processor = seconds_processor or (lambda x: x) 73 | self._dry_run = dry_run 74 | 75 | def _process_log( 76 | self, 77 | request_context: GracyRequestContext, 78 | response: httpx.Response, 79 | retry_after: float, 80 | actual_wait: float, 81 | ) -> None: 82 | if event := self._log_event: 83 | format_args: t.Dict[str, str] = dict( 84 | **extract_base_format_args(request_context), 85 | **extract_response_format_args(response), 86 | RETRY_AFTER=str(retry_after), 87 | RETRY_AFTER_ACTUAL_WAIT=str(actual_wait), 88 | ) 89 | 90 | do_log(event, self.DEFAULT_LOG_MESSAGE, format_args, response) 91 | 92 | def _parse_retry_after_as_seconds(self, response: httpx.Response) -> float: 93 | retry_after_value = response.headers.get("retry-after") 94 | 95 | if retry_after_value is None: 96 | return 0 97 | 98 | if retry_after_value.isdigit(): 99 | return int(retry_after_value) 100 | 101 | try: 102 | # It might be a date as: Wed, 21 Oct 2015 07:28:00 GMT 103 | date_time = datetime.strptime(retry_after_value, "%a, %d %b %Y %H:%M:%S %Z") 104 | date_as_seconds = (date_time - datetime.now()).total_seconds() 105 | 106 | except Exception: 107 | logger.exception( 108 | f"Unable to parse {retry_after_value} as date within {type(self).__name__}" 109 | ) 110 | return 0 111 | 112 | else: 113 | return date_as_seconds 114 | 115 | async def before(self, context: GracyRequestContext) -> HookResult: 116 | return HookResult(False) 117 | 118 | async def after( 119 | self, 120 | context: GracyRequestContext, 121 | response_or_exc: httpx.Response | Exception, 122 | ) -> HookResult: 123 | if ( 124 | isinstance(response_or_exc, httpx.Response) 125 | and response_or_exc.status_code == HTTPStatus.TOO_MANY_REQUESTS 126 | ): 127 | if is_replay(response_or_exc): 128 | return HookResult(executed=False, dry_run=self._dry_run) 129 | 130 | retry_after_seconds = self._parse_retry_after_as_seconds(response_or_exc) 131 | actual_wait = self._processor(retry_after_seconds) 132 | 133 | if retry_after_seconds > 0: 134 | lock_name = ( 135 | context.unformatted_url 136 | if self._lock_per_endpoint 137 | else self.ALL_CLIENT_LOCK 138 | ) 139 | 140 | async with self._lock_manager[lock_name]: 141 | self._process_log( 142 | context, response_or_exc, retry_after_seconds, actual_wait 143 | ) 144 | 145 | if self._reporter: 146 | self._reporter.throttled(context) 147 | 148 | if self._dry_run is False: 149 | await asyncio.sleep(actual_wait) 150 | 151 | return HookResult(True, actual_wait, self._dry_run) 152 | 153 | return HookResult(False, dry_run=self._dry_run) 154 | 155 | 156 | class RateLimitBackOffHook: 157 | """ 158 | Provides two methods `before()` and `after()` to be used as hooks by Gracy. 159 | 160 | This hook checks for 429 (TOO MANY REQUESTS) and locks requests for an arbitrary amount of time defined by you. 161 | 162 | If the value is set, then Gracy pauses **ALL** client requests until the time is over. 163 | This behavior can be modified to happen on a per-endpoint basis if `lock_per_endpoint` is True. 164 | 165 | ### ⚠️ Retry 166 | This doesn't replace `GracefulRetry`. 167 | Make sure you implement a proper retry logic, otherwise the 429 will break the client. 168 | 169 | ### Throttling 170 | If you pass the reporter, it will count every await as "throttled" for that UURL. 171 | 172 | ### Log Event 173 | You can optionally define a log event. 174 | It provides the response and the context, but also `WAIT_TIME` that contains the wait value. 175 | """ 176 | 177 | DEFAULT_LOG_MESSAGE: t.Final = ( 178 | "[{METHOD}] {UENDPOINT} got rate limited, waiting for {WAIT_TIME}s" 179 | ) 180 | ALL_CLIENT_LOCK: t.Final = "CLIENT" 181 | 182 | def __init__( 183 | self, 184 | delay: float, 185 | reporter: ReportBuilder | None = None, 186 | lock_per_endpoint: bool = False, 187 | log_event: LogEvent | None = None, 188 | *, 189 | dry_run: bool = False, 190 | ) -> None: 191 | self._reporter = reporter 192 | self._lock_per_endpoint = lock_per_endpoint 193 | self._lock_manager = t.DefaultDict[str, Lock](Lock) 194 | self._log_event = log_event 195 | self._delay = delay 196 | self._dry_run = dry_run 197 | 198 | def _process_log( 199 | self, request_context: GracyRequestContext, response: httpx.Response 200 | ) -> None: 201 | if event := self._log_event: 202 | format_args: t.Dict[str, str] = dict( 203 | **extract_base_format_args(request_context), 204 | **extract_response_format_args(response), 205 | WAIT_TIME=str(self._delay), 206 | ) 207 | 208 | do_log(event, self.DEFAULT_LOG_MESSAGE, format_args, response) 209 | 210 | async def before(self, context: GracyRequestContext) -> HookResult: 211 | return HookResult(False) 212 | 213 | async def after( 214 | self, 215 | context: GracyRequestContext, 216 | response_or_exc: httpx.Response | Exception, 217 | ) -> HookResult: 218 | if ( 219 | isinstance(response_or_exc, httpx.Response) 220 | and response_or_exc.status_code == HTTPStatus.TOO_MANY_REQUESTS 221 | ): 222 | if is_replay(response_or_exc): 223 | return HookResult(executed=False, dry_run=self._dry_run) 224 | 225 | lock_name = ( 226 | context.unformatted_url 227 | if self._lock_per_endpoint 228 | else self.ALL_CLIENT_LOCK 229 | ) 230 | 231 | async with self._lock_manager[lock_name]: 232 | self._process_log(context, response_or_exc) 233 | 234 | if self._reporter: 235 | self._reporter.throttled(context) 236 | 237 | if self._dry_run is False: 238 | await asyncio.sleep(self._delay) 239 | 240 | return HookResult(True, self._delay, self._dry_run) 241 | 242 | return HookResult(False, dry_run=self._dry_run) 243 | -------------------------------------------------------------------------------- /src/gracy/_reports/_printers.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | import typing as t 5 | from abc import ABC, abstractmethod 6 | from datetime import datetime 7 | 8 | from ..replays.storages._base import GracyReplay 9 | from ._models import GracyAggregatedTotal, GracyReport 10 | 11 | logger = logging.getLogger("gracy") 12 | 13 | PRINTERS = t.Literal["rich", "list", "logger", "plotly"] 14 | 15 | 16 | class Titles: 17 | url: t.Final = "URL" 18 | total_requests: t.Final = "Total Reqs (#)" 19 | success_rate: t.Final = "Success (%)" 20 | failed_rate: t.Final = "Fail (%)" 21 | avg_latency: t.Final = "Avg Latency (s)" 22 | max_latency: t.Final = "Max Latency (s)" 23 | resp_2xx: t.Final = "2xx Resps" 24 | resp_3xx: t.Final = "3xx Resps" 25 | resp_4xx: t.Final = "4xx Resps" 26 | resp_5xx: t.Final = "5xx Resps" 27 | reqs_aborted: t.Final = "Aborts" 28 | retries: t.Final = "Retries" 29 | throttles: t.Final = "Throttles" 30 | replays: t.Final = "Replays" 31 | req_rate_per_sec: t.Final = "Avg Reqs/sec" 32 | 33 | 34 | def _getreplays_warn(replay_settings: GracyReplay | None) -> str: 35 | res = "" 36 | if replay_settings and replay_settings.display_report: 37 | if replay_settings.records_made: 38 | res = f"{replay_settings.records_made:,} Requests Recorded" 39 | 40 | if replay_settings.replays_made: 41 | if res: 42 | res += " / " 43 | 44 | res += f"{replay_settings.replays_made:,} Requests Replayed" 45 | 46 | if res: 47 | return f"({res})" 48 | 49 | return res 50 | 51 | 52 | def _format_value( 53 | val: float, 54 | color: str | None = None, 55 | isset_color: str | None = None, 56 | precision: int = 2, 57 | bold: bool = False, 58 | prefix: str = "", 59 | suffix: str = "", 60 | padprefix: int = 0, 61 | ) -> str: 62 | cur = f"{prefix.rjust(padprefix)}{val:,.{precision}f}{suffix}" 63 | 64 | if bold: 65 | cur = f"[bold]{cur}[/bold]" 66 | 67 | if val and isset_color: 68 | cur = f"[{isset_color}]{cur}[/{isset_color}]" 69 | elif color: 70 | cur = f"[{color}]{cur}[/{color}]" 71 | 72 | return cur 73 | 74 | 75 | def _format_int( 76 | val: int, 77 | color: str | None = None, 78 | isset_color: str | None = None, 79 | bold: bool = False, 80 | prefix: str = "", 81 | suffix: str = "", 82 | padprefix: int = 0, 83 | ) -> str: 84 | cur = f"{prefix.rjust(padprefix)}{val:,}{suffix}" 85 | 86 | if bold: 87 | cur = f"[bold]{cur}[/bold]" 88 | 89 | if val and isset_color: 90 | cur = f"[{isset_color}]{cur}[/{isset_color}]" 91 | elif color: 92 | cur = f"[{color}]{cur}[/{color}]" 93 | 94 | return cur 95 | 96 | 97 | def _print_header(report: GracyReport): 98 | print(" ____") 99 | print(" / ___|_ __ __ _ ___ _ _") 100 | print(" | | _| '__/ _` |/ __| | | |") 101 | print(" | |_| | | | (_| | (__| |_| |") 102 | print(" \\____|_| \\__,_|\\___|\\__, |") 103 | print( 104 | f" |___/ Requests Summary Report {_getreplays_warn(report.replay_settings)}" 105 | ) 106 | 107 | 108 | class BasePrinter(ABC): 109 | @abstractmethod 110 | def print_report(self, report: GracyReport) -> t.Any: 111 | pass 112 | 113 | 114 | class RichPrinter(BasePrinter): 115 | def print_report(self, report: GracyReport) -> None: 116 | # Dynamic import so we don't have to require it as dependency 117 | from rich.console import Console 118 | from rich.table import Table 119 | 120 | in_replay_mode = ( 121 | report.replay_settings and report.replay_settings.display_report 122 | ) 123 | 124 | console = Console() 125 | title_warn = ( 126 | f"[yellow]{_getreplays_warn(report.replay_settings)}[/yellow]" 127 | if in_replay_mode 128 | else "" 129 | ) 130 | table = Table(title=f"Gracy Requests Summary {title_warn}") 131 | 132 | table.add_column(Titles.url, overflow="fold") 133 | table.add_column(Titles.total_requests, justify="right") 134 | table.add_column(Titles.success_rate, justify="right") 135 | table.add_column(Titles.failed_rate, justify="right") 136 | table.add_column(Titles.avg_latency, justify="right") 137 | table.add_column(Titles.max_latency, justify="right") 138 | table.add_column(Titles.resp_2xx, justify="right") 139 | table.add_column(Titles.resp_3xx, justify="right") 140 | table.add_column(Titles.resp_4xx, justify="right") 141 | table.add_column(Titles.resp_5xx, justify="right") 142 | table.add_column(Titles.reqs_aborted, justify="right") 143 | table.add_column(Titles.retries, justify="right") 144 | table.add_column(Titles.throttles, justify="right") 145 | 146 | if in_replay_mode: 147 | table.add_column(Titles.replays, justify="right") 148 | 149 | table.add_column(Titles.req_rate_per_sec, justify="right") 150 | 151 | rows = report.requests 152 | report.total.uurl = f"[bold]{report.total.uurl}[/bold]" 153 | rows.append(report.total) 154 | 155 | for idx, request_row in enumerate(rows): 156 | is_last_line_before_footer = idx < len(rows) - 1 and isinstance( 157 | rows[idx + 1], GracyAggregatedTotal 158 | ) 159 | 160 | row_values: tuple[str, ...] = ( 161 | _format_int(request_row.total_requests, bold=True), 162 | _format_value(request_row.success_rate, "green", suffix="%"), 163 | _format_value( 164 | request_row.failed_rate, None, "red", bold=True, suffix="%" 165 | ), 166 | _format_value(request_row.avg_latency), 167 | _format_value(request_row.max_latency), 168 | _format_int(request_row.resp_2xx), 169 | _format_int(request_row.resp_3xx), 170 | _format_int(request_row.resp_4xx, isset_color="red"), 171 | _format_int(request_row.resp_5xx, isset_color="red"), 172 | _format_int(request_row.reqs_aborted, isset_color="red"), 173 | _format_int(request_row.retries, isset_color="yellow"), 174 | _format_int(request_row.throttles, isset_color="yellow"), 175 | ) 176 | 177 | if in_replay_mode: 178 | row_values = ( 179 | *row_values, 180 | _format_int(request_row.replays, isset_color="yellow"), 181 | ) 182 | 183 | table.add_row( 184 | request_row.uurl, 185 | *row_values, 186 | _format_value( 187 | request_row.req_rate_per_sec, precision=1, suffix=" reqs/s" 188 | ), 189 | end_section=is_last_line_before_footer, 190 | ) 191 | 192 | console.print(table) 193 | 194 | 195 | class PlotlyPrinter(BasePrinter): 196 | def print_report(self, report: GracyReport): 197 | # Dynamic import so we don't have to require it as dependency 198 | import pandas as pd # pyright: ignore[reportMissingImports] 199 | import plotly.express as px 200 | 201 | df = pd.DataFrame( 202 | [ 203 | dict( 204 | Uurl=uurl, 205 | Url=entry.url, 206 | Start=datetime.utcfromtimestamp(entry.start), 207 | Finish=datetime.utcfromtimestamp(entry.end), 208 | ) 209 | for uurl, requests in report.requests_timeline.items() 210 | for entry in requests 211 | ] 212 | ) 213 | 214 | fig = px.timeline( 215 | df, 216 | x_start="Start", 217 | x_end="Finish", 218 | y="Uurl", 219 | color="Url", 220 | ) 221 | 222 | fig.update_yaxes(autorange="reversed") 223 | fig.update_xaxes(tickformat="%H:%M:%S.%f") 224 | fig.update_layout(barmode="group") 225 | 226 | return fig 227 | 228 | 229 | class ListPrinter(BasePrinter): 230 | def print_report(self, report: GracyReport) -> None: 231 | _print_header(report) 232 | 233 | entries = report.requests 234 | entries.append(report.total) 235 | in_replay_mode = ( 236 | report.replay_settings and report.replay_settings.display_report 237 | ) 238 | 239 | PAD_PREFIX: t.Final = 20 240 | 241 | for idx, entry in enumerate(entries, 1): 242 | title = entry.uurl if idx == len(entries) else f"{idx}. {entry.uurl}" 243 | print(f"\n\n{title}") 244 | 245 | print( 246 | _format_int( 247 | entry.total_requests, 248 | padprefix=PAD_PREFIX, 249 | prefix=f"{Titles.total_requests}: ", 250 | ) 251 | ) 252 | print( 253 | _format_value( 254 | entry.success_rate, 255 | padprefix=PAD_PREFIX, 256 | prefix=f"{Titles.success_rate}: ", 257 | suffix="%", 258 | ) 259 | ) 260 | print( 261 | _format_value( 262 | entry.failed_rate, 263 | padprefix=PAD_PREFIX, 264 | prefix=f"{Titles.failed_rate}: ", 265 | suffix="%", 266 | ) 267 | ) 268 | print( 269 | _format_value( 270 | entry.avg_latency, 271 | padprefix=PAD_PREFIX, 272 | prefix=f"{Titles.avg_latency}: ", 273 | ) 274 | ) 275 | print( 276 | _format_value( 277 | entry.max_latency, 278 | padprefix=PAD_PREFIX, 279 | prefix=f"{Titles.max_latency}: ", 280 | ) 281 | ) 282 | print( 283 | _format_int( 284 | entry.resp_2xx, padprefix=PAD_PREFIX, prefix=f"{Titles.resp_2xx}: " 285 | ) 286 | ) 287 | print( 288 | _format_int( 289 | entry.resp_3xx, padprefix=PAD_PREFIX, prefix=f"{Titles.resp_3xx}: " 290 | ) 291 | ) 292 | print( 293 | _format_int( 294 | entry.resp_4xx, padprefix=PAD_PREFIX, prefix=f"{Titles.resp_4xx}: " 295 | ) 296 | ) 297 | print( 298 | _format_int( 299 | entry.resp_5xx, padprefix=PAD_PREFIX, prefix=f"{Titles.resp_5xx}: " 300 | ) 301 | ) 302 | print( 303 | _format_int( 304 | entry.reqs_aborted, 305 | padprefix=PAD_PREFIX, 306 | prefix=f"{Titles.reqs_aborted}: ", 307 | ) 308 | ) 309 | print( 310 | _format_int( 311 | entry.retries, padprefix=PAD_PREFIX, prefix=f"{Titles.retries}: " 312 | ) 313 | ) 314 | print( 315 | _format_int( 316 | entry.throttles, 317 | padprefix=PAD_PREFIX, 318 | prefix=f"{Titles.throttles}: ", 319 | ) 320 | ) 321 | 322 | if in_replay_mode: 323 | print( 324 | _format_int( 325 | entry.replays, 326 | padprefix=PAD_PREFIX, 327 | prefix=f"{Titles.replays}: ", 328 | ) 329 | ) 330 | 331 | print( 332 | _format_value( 333 | entry.req_rate_per_sec, 334 | precision=1, 335 | padprefix=PAD_PREFIX, 336 | prefix=f"{Titles.req_rate_per_sec}: ", 337 | suffix=" reqs/s", 338 | ) 339 | ) 340 | 341 | 342 | class LoggerPrinter(BasePrinter): 343 | def print_report(self, report: GracyReport) -> None: 344 | # the first entry should be the most frequent URL hit 345 | if not report.requests: 346 | logger.warning("No requests were triggered") 347 | return 348 | 349 | first_entry, *_ = report.requests 350 | total = report.total 351 | 352 | logger.info( 353 | f"Gracy tracked that '{first_entry.uurl}' was hit {_format_int(first_entry.total_requests)} time(s) " 354 | f"with a success rate of {_format_value(first_entry.success_rate, suffix='%')}, " 355 | f"avg latency of {_format_value(first_entry.avg_latency)}s, " 356 | f"and a rate of {_format_value(first_entry.req_rate_per_sec, precision=1, suffix=' reqs/s')}." 357 | ) 358 | 359 | logger.info( 360 | f"Gracy tracked a total of {_format_int(total.total_requests)} requests " 361 | f"with a success rate of {_format_value(total.success_rate, suffix='%')}, " 362 | f"avg latency of {_format_value(total.avg_latency)}s, " 363 | f"and a rate of {_format_value(total.req_rate_per_sec, precision=1, suffix=' reqs/s')}." 364 | ) 365 | 366 | if replay := report.replay_settings: 367 | if replay.mode == "record": 368 | logger.info("All requests were recorded with GracyReplay") 369 | else: 370 | logger.warning( 371 | "All requests were REPLAYED (no HTTP interaction) with GracyReplay" 372 | ) 373 | 374 | 375 | def print_report(report: GracyReport, method: PRINTERS) -> t.Any: 376 | printer: t.Optional[BasePrinter] = None 377 | if method == "rich": 378 | printer = RichPrinter() 379 | elif method == "list": 380 | printer = ListPrinter() 381 | elif method == "logger": 382 | printer = LoggerPrinter() 383 | elif method == "plotly": 384 | printer = PlotlyPrinter() 385 | 386 | if printer: 387 | return printer.print_report(report) 388 | -------------------------------------------------------------------------------- /src/tests/test_retry.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import httpx 4 | import logging 5 | import pytest 6 | import typing as t 7 | from http import HTTPStatus 8 | from unittest.mock import patch 9 | 10 | from gracy import ( 11 | GracefulRetry, 12 | GracefulValidator, 13 | Gracy, 14 | GracyConfig, 15 | GracyReplay, 16 | LogEvent, 17 | LogLevel, 18 | OverrideRetryOn, 19 | graceful, 20 | ) 21 | from gracy.exceptions import GracyRequestFailed, NonOkResponse 22 | from tests.conftest import ( 23 | MISSING_NAME, 24 | PRESENT_POKEMON_NAME, 25 | REPLAY, 26 | FakeReplayStorage, 27 | PokeApiEndpoint, 28 | assert_requests_made, 29 | ) 30 | 31 | RETRY: t.Final = GracefulRetry( 32 | delay=0.001, 33 | max_attempts=0, 34 | retry_on={HTTPStatus.NOT_FOUND, ValueError}, 35 | behavior="pass", 36 | ) 37 | """NOTE: Max attempts will be patched later in fixture""" 38 | 39 | RETRY_ON_NONE: t.Final = GracefulRetry( 40 | delay=0.001, max_attempts=1, retry_on=None, behavior="pass" 41 | ) 42 | 43 | RETRY_LOG_BEFORE = LogEvent(LogLevel.WARNING, "LOG_BEFORE") 44 | RETRY_LOG_AFTER = LogEvent(LogLevel.ERROR, "LOG_AFTER") 45 | RETRY_LOG_EXHAUSTED = LogEvent(LogLevel.CRITICAL, "LOG_EXHAUSTED") 46 | 47 | RETRY_3_TIMES_LOG: t.Final = GracefulRetry( 48 | delay=0.001, 49 | max_attempts=3, 50 | retry_on=HTTPStatus.NOT_FOUND, 51 | log_before=RETRY_LOG_BEFORE, 52 | log_after=RETRY_LOG_AFTER, 53 | log_exhausted=RETRY_LOG_EXHAUSTED, 54 | ) 55 | 56 | RETRY_3_TIMES_OVERRIDE_PLACEHOLDER_LOG: t.Final = GracefulRetry( 57 | delay=90, # Will be overriden 58 | max_attempts=3, 59 | retry_on=HTTPStatus.NOT_FOUND, 60 | overrides={HTTPStatus.NOT_FOUND: OverrideRetryOn(delay=0.001)}, 61 | log_before=LogEvent(LogLevel.WARNING, "BEFORE: {RETRY_DELAY} {RETRY_CAUSE}"), 62 | log_after=LogEvent(LogLevel.WARNING, "AFTER: {RETRY_CAUSE}"), 63 | ) 64 | 65 | 66 | def assert_log(record: logging.LogRecord, expected_event: LogEvent): 67 | assert record.levelno == expected_event.level 68 | assert record.message == expected_event.custom_message # No formatting set 69 | 70 | 71 | class CustomValidator(GracefulValidator): 72 | def check(self, response: httpx.Response) -> None: 73 | if response.json()["order"] != 47: 74 | raise ValueError("Pokemon #order should be 47") # noqa: TRY003 75 | 76 | 77 | class GracefulPokeAPI(Gracy[PokeApiEndpoint]): 78 | class Config: 79 | BASE_URL = "https://pokeapi.co/api/v2/" 80 | SETTINGS = GracyConfig( 81 | retry=RETRY, 82 | allowed_status_code={HTTPStatus.NOT_FOUND}, 83 | parser={HTTPStatus.NOT_FOUND: None}, 84 | ) 85 | 86 | async def get_pokemon(self, name: str): 87 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 88 | 89 | @graceful(retry=None) 90 | async def get_pokemon_without_retry(self, name: str): 91 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 92 | 93 | @graceful(retry=None, parser=None, allowed_status_code=None) 94 | async def get_pokemon_without_retry_or_parser(self, name: str): 95 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 96 | 97 | @graceful(strict_status_code={HTTPStatus.OK}) 98 | async def get_pokemon_with_strict_status(self, name: str): 99 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 100 | 101 | @graceful(allowed_status_code=None, parser={"default": lambda r: r.json()}) 102 | async def get_pokemon_without_allowed_status(self, name: str): 103 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 104 | 105 | @graceful(validators=CustomValidator()) 106 | async def get_pokemon_with_custom_validator(self, name: str): 107 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 108 | 109 | @graceful(retry=RETRY_ON_NONE) 110 | async def get_pokemon_with_retry_on_none(self, name: str): 111 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 112 | 113 | @graceful(retry=RETRY_ON_NONE, validators=CustomValidator()) 114 | async def get_pokemon_with_retry_on_none_and_validator(self, name: str): 115 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 116 | 117 | @graceful(retry=RETRY_3_TIMES_LOG, allowed_status_code=None) 118 | async def get_pokemon_with_log_retry_3_times(self, name: str): 119 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 120 | 121 | @graceful(retry=RETRY_3_TIMES_OVERRIDE_PLACEHOLDER_LOG, allowed_status_code=None) 122 | async def get_pokemon_with_retry_overriden_log_placeholder(self, name: str): 123 | return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) 124 | 125 | 126 | @pytest.fixture() 127 | def make_pokeapi(): 128 | def factory( 129 | max_attempts: int, break_or_pass: str = "pass", replay_enabled: bool = True 130 | ): 131 | Gracy.dangerously_reset_report() 132 | 133 | api = GracefulPokeAPI(REPLAY) if replay_enabled else GracefulPokeAPI() 134 | api._base_config.retry.max_attempts = max_attempts # type: ignore 135 | api._base_config.retry.behavior = break_or_pass # type: ignore 136 | 137 | return api 138 | 139 | return factory 140 | 141 | 142 | @pytest.fixture() 143 | def make_flaky_pokeapi(): 144 | def factory( 145 | flaky_requests: int, 146 | max_attempts: int | None = None, 147 | break_or_pass: str = "break", 148 | ): 149 | Gracy.dangerously_reset_report() 150 | 151 | force_urls = ( 152 | ["https://pokeapi.co/api/v2/pokemon/doesnt-exist"] * flaky_requests 153 | ) + (["https://pokeapi.co/api/v2/pokemon/charmander"]) 154 | mock_storage = FakeReplayStorage(force_urls) 155 | fake_replay = GracyReplay("replay", mock_storage) 156 | 157 | api = GracefulPokeAPI(fake_replay) 158 | if max_attempts: 159 | api._base_config.retry.max_attempts = max_attempts # type: ignore 160 | api._base_config.retry.behavior = break_or_pass # type: ignore 161 | 162 | return api 163 | 164 | return factory 165 | 166 | 167 | class PokeApiFactory(t.Protocol): 168 | def __call__( 169 | self, 170 | max_attempts: int, 171 | break_or_pass: str = "pass", 172 | replay_enabled: bool = True, 173 | ) -> GracefulPokeAPI: 174 | ... 175 | 176 | 177 | class FlakyPokeApiFactory(t.Protocol): 178 | def __call__( 179 | self, 180 | flaky_requests: int, 181 | max_attempts: int | None = None, 182 | break_or_pass: str = "pass", 183 | ) -> GracefulPokeAPI: 184 | ... 185 | 186 | 187 | async def test_ensure_replay_is_enabled(make_pokeapi: PokeApiFactory): 188 | pokeapi = make_pokeapi(0) 189 | result = await pokeapi.get_pokemon(MISSING_NAME) 190 | report = pokeapi.get_report() 191 | 192 | assert result is None 193 | assert report.replay_settings is not None 194 | assert report.replay_settings.mode == "replay" 195 | assert len(report.requests) == 1 196 | assert report.requests[0].total_requests == 1 197 | 198 | 199 | @pytest.mark.parametrize("max_retries", [2, 4, 6]) 200 | async def test_pokemon_not_found(max_retries: int, make_pokeapi: PokeApiFactory): 201 | EXPECTED_REQS: t.Final = 1 + max_retries # First request + Retries (2) = 3 requests 202 | 203 | pokeapi = make_pokeapi(max_retries) 204 | result = await pokeapi.get_pokemon(MISSING_NAME) 205 | 206 | assert result is None 207 | assert_requests_made(pokeapi, EXPECTED_REQS) 208 | 209 | 210 | @pytest.mark.parametrize("max_retries", [2, 4, 6]) 211 | async def test_pokemon_not_found_without_allowed( 212 | max_retries: int, make_pokeapi: t.Callable[[int, str], GracefulPokeAPI] 213 | ): 214 | EXPECTED_REQS: t.Final = 1 + max_retries # First request + Retries (2) = 3 requests 215 | 216 | pokeapi = make_pokeapi(max_retries, "break") 217 | 218 | with pytest.raises(NonOkResponse): 219 | await pokeapi.get_pokemon_without_allowed_status(MISSING_NAME) 220 | 221 | assert_requests_made(pokeapi, EXPECTED_REQS) 222 | 223 | 224 | @pytest.mark.parametrize("max_retries", [2, 4, 6]) 225 | async def test_pokemon_not_found_with_strict_status( 226 | max_retries: int, make_pokeapi: PokeApiFactory 227 | ): 228 | EXPECTED_REQS: t.Final = 1 + max_retries # First request + Retries (2) = 3 requests 229 | 230 | pokeapi = make_pokeapi(max_retries) 231 | result = await pokeapi.get_pokemon_with_strict_status(MISSING_NAME) 232 | 233 | assert result is None 234 | assert_requests_made(pokeapi, EXPECTED_REQS) 235 | 236 | 237 | async def test_pokemon_with_bad_parser_break_wont_run(make_pokeapi: PokeApiFactory): 238 | MAX_RETRIES: t.Final = 2 239 | EXPECTED_REQS: t.Final = 1 + MAX_RETRIES # First request + Retries (2) = 3 requests 240 | 241 | pokeapi = make_pokeapi(MAX_RETRIES, "break") 242 | 243 | with pytest.raises(NonOkResponse): 244 | await pokeapi.get_pokemon_without_allowed_status(MISSING_NAME) 245 | 246 | assert_requests_made(pokeapi, EXPECTED_REQS) 247 | 248 | 249 | async def test_retry_with_failing_custom_validation(make_pokeapi: PokeApiFactory): 250 | MAX_RETRIES: t.Final = 2 251 | EXPECTED_REQS: t.Final = 1 + MAX_RETRIES # First request + Retries (2) = 3 requests 252 | 253 | pokeapi = make_pokeapi(MAX_RETRIES) 254 | result = await pokeapi.get_pokemon_with_custom_validator(PRESENT_POKEMON_NAME) 255 | 256 | assert result is not None 257 | 258 | assert_requests_made(pokeapi, EXPECTED_REQS) 259 | 260 | 261 | async def test_failing_without_retry(make_pokeapi: PokeApiFactory): 262 | EXPECTED_REQS: t.Final = 1 263 | 264 | pokeapi = make_pokeapi(0) # Won't have effect 265 | 266 | result = await pokeapi.get_pokemon_without_retry(MISSING_NAME) 267 | 268 | assert result is None 269 | assert_requests_made(pokeapi, EXPECTED_REQS) 270 | 271 | 272 | async def test_failing_without_retry_or_parser(make_pokeapi: PokeApiFactory): 273 | EXPECTED_REQS: t.Final = 1 274 | 275 | pokeapi = make_pokeapi(0) # Won't have effect 276 | 277 | with pytest.raises(NonOkResponse): 278 | await pokeapi.get_pokemon_without_retry_or_parser(MISSING_NAME) 279 | 280 | assert_requests_made(pokeapi, EXPECTED_REQS) 281 | 282 | 283 | async def test_retry_none_for_successful_request(make_pokeapi: PokeApiFactory): 284 | EXPECTED_REQS: t.Final = 1 285 | 286 | pokeapi = make_pokeapi(0) # Won't have effect 287 | 288 | result = await pokeapi.get_pokemon_with_retry_on_none(PRESENT_POKEMON_NAME) 289 | 290 | assert result is not None 291 | assert_requests_made(pokeapi, EXPECTED_REQS) 292 | 293 | 294 | async def test_retry_none_for_failing_request(make_pokeapi: PokeApiFactory): 295 | EXPECTED_REQS: t.Final = 2 296 | 297 | pokeapi = make_pokeapi(0) # Won't have effect 298 | 299 | result = await pokeapi.get_pokemon_with_retry_on_none(MISSING_NAME) 300 | 301 | assert result is None 302 | assert_requests_made(pokeapi, EXPECTED_REQS) 303 | 304 | 305 | async def test_retry_none_for_failing_validator(make_pokeapi: PokeApiFactory): 306 | EXPECTED_REQS: t.Final = 2 307 | 308 | pokeapi = make_pokeapi(0) # Won't have effect 309 | 310 | response = await pokeapi.get_pokemon_with_retry_on_none_and_validator( 311 | PRESENT_POKEMON_NAME 312 | ) 313 | 314 | assert response is not None 315 | assert_requests_made(pokeapi, EXPECTED_REQS) 316 | 317 | 318 | async def test_retry_eventually_recovers(make_flaky_pokeapi: FlakyPokeApiFactory): 319 | RETRY_ATTEMPTS: t.Final = 4 320 | EXPECTED_REQS: t.Final = 1 + RETRY_ATTEMPTS 321 | 322 | # Scenario: 1 + 3 Retry attemps fail + Last attempt works 323 | pokeapi = make_flaky_pokeapi(4, RETRY_ATTEMPTS) 324 | 325 | result = await pokeapi.get_pokemon(PRESENT_POKEMON_NAME) 326 | 327 | # Test 328 | assert result is not None 329 | assert_requests_made(pokeapi, EXPECTED_REQS) 330 | 331 | 332 | async def test_retry_eventually_recovers_with_strict( 333 | make_flaky_pokeapi: FlakyPokeApiFactory, 334 | ): 335 | RETRY_ATTEMPTS: t.Final = 4 336 | EXPECTED_REQS: t.Final = 1 + RETRY_ATTEMPTS 337 | 338 | # Scenario: 1 + 3 Retry attempts fail + last attempt works 339 | pokeapi = make_flaky_pokeapi(4, RETRY_ATTEMPTS) 340 | 341 | result = await pokeapi.get_pokemon_with_strict_status(PRESENT_POKEMON_NAME) 342 | 343 | # Test 344 | assert result is not None 345 | assert_requests_made(pokeapi, EXPECTED_REQS) 346 | 347 | 348 | async def test_retry_logs( 349 | make_flaky_pokeapi: FlakyPokeApiFactory, caplog: pytest.LogCaptureFixture 350 | ): 351 | FLAKY_REQUESTS: t.Final = 3 352 | EXPECTED_REQS: t.Final = FLAKY_REQUESTS + 1 353 | 354 | pokeapi = make_flaky_pokeapi(FLAKY_REQUESTS) 355 | 356 | result = await pokeapi.get_pokemon_with_log_retry_3_times(PRESENT_POKEMON_NAME) 357 | 358 | # Test 359 | assert result is not None 360 | assert_requests_made(pokeapi, EXPECTED_REQS) 361 | 362 | assert len(caplog.records) == 6 363 | assert_log(caplog.records[0], RETRY_LOG_BEFORE) 364 | assert_log(caplog.records[1], RETRY_LOG_AFTER) 365 | assert_log(caplog.records[2], RETRY_LOG_BEFORE) 366 | assert_log(caplog.records[3], RETRY_LOG_AFTER) 367 | assert_log(caplog.records[4], RETRY_LOG_BEFORE) 368 | assert_log(caplog.records[5], RETRY_LOG_AFTER) 369 | 370 | 371 | async def test_retry_logs_fail_reason( 372 | make_flaky_pokeapi: FlakyPokeApiFactory, caplog: pytest.LogCaptureFixture 373 | ): 374 | FLAKY_REQUESTS: t.Final = 2 375 | EXPECTED_REQS: t.Final = FLAKY_REQUESTS + 1 376 | 377 | pokeapi = make_flaky_pokeapi(FLAKY_REQUESTS) 378 | 379 | result = await pokeapi.get_pokemon_with_retry_overriden_log_placeholder( 380 | PRESENT_POKEMON_NAME 381 | ) 382 | 383 | # Test 384 | assert result is not None 385 | assert_requests_made(pokeapi, EXPECTED_REQS) 386 | 387 | assert len(caplog.records) == 4 388 | assert caplog.records[0].message == "BEFORE: 0.001 [Bad Status Code: 404]" 389 | assert caplog.records[1].message == "AFTER: [Bad Status Code: 404]" 390 | assert caplog.records[2].message == "BEFORE: 0.001 [Bad Status Code: 404]" 391 | assert caplog.records[3].message == "AFTER: SUCCESSFUL" 392 | 393 | 394 | async def test_retry_logs_exhausts( 395 | make_pokeapi: PokeApiFactory, caplog: pytest.LogCaptureFixture 396 | ): 397 | EXPECTED_REQS: t.Final = 3 + 1 # Retry's value from graceful + 1 398 | 399 | pokeapi = make_pokeapi(0) # Won't take effect due to @graceful 400 | 401 | with pytest.raises(NonOkResponse): 402 | await pokeapi.get_pokemon_with_log_retry_3_times(MISSING_NAME) 403 | 404 | # Test 405 | assert_requests_made(pokeapi, EXPECTED_REQS) 406 | 407 | assert len(caplog.records) == 7 408 | assert_log(caplog.records[0], RETRY_LOG_BEFORE) 409 | assert_log(caplog.records[1], RETRY_LOG_AFTER) 410 | assert_log(caplog.records[2], RETRY_LOG_BEFORE) 411 | assert_log(caplog.records[3], RETRY_LOG_AFTER) 412 | assert_log(caplog.records[4], RETRY_LOG_BEFORE) 413 | assert_log(caplog.records[5], RETRY_LOG_AFTER) 414 | assert_log(caplog.records[6], RETRY_LOG_EXHAUSTED) 415 | 416 | 417 | async def test_retry_without_replay_request_without_response_generic( 418 | make_pokeapi: PokeApiFactory, 419 | ): 420 | EXPECTED_REQS: t.Final = 3 + 1 421 | 422 | class SomeRequestException(Exception): 423 | pass 424 | 425 | # Regardless of replay being disabled, no request will be triggered as we're mocking httpx 426 | pokeapi = make_pokeapi(3, break_or_pass="break", replay_enabled=False) 427 | pokeapi._base_config.retry.retry_on.add(GracyRequestFailed) # type: ignore 428 | 429 | mock: t.Any 430 | with patch.object(pokeapi, "_client", autospec=True) as mock: 431 | mock.request.side_effect = SomeRequestException("Request failed") 432 | 433 | with pytest.raises(GracyRequestFailed): 434 | await pokeapi.get_pokemon(PRESENT_POKEMON_NAME) 435 | 436 | assert_requests_made(pokeapi, EXPECTED_REQS) 437 | -------------------------------------------------------------------------------- /src/gracy/_models.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | import copy 5 | import httpx 6 | import inspect 7 | import itertools 8 | import logging 9 | import re 10 | import typing as t 11 | from abc import ABC, abstractmethod 12 | from contextlib import contextmanager 13 | from dataclasses import dataclass 14 | from datetime import datetime, timedelta 15 | from enum import Enum, IntEnum 16 | from http import HTTPStatus 17 | from threading import Lock 18 | 19 | from ._types import PARSER_TYPE, UNSET_VALUE, Unset 20 | 21 | 22 | class LogLevel(IntEnum): 23 | CRITICAL = logging.CRITICAL 24 | ERROR = logging.ERROR 25 | WARNING = logging.WARNING 26 | INFO = logging.INFO 27 | DEBUG = logging.DEBUG 28 | NOTSET = logging.NOTSET 29 | 30 | 31 | @dataclass 32 | class LogEvent: 33 | level: LogLevel 34 | custom_message: t.Callable[[httpx.Response | None], str] | str | None = None 35 | """You can add some placeholders to be injected in the log. 36 | 37 | e.g. 38 | - `{URL} executed` 39 | - `API replied {STATUS} and took {ELAPSED}` 40 | - `{METHOD} {URL} returned {STATUS}` 41 | - `Becareful because {URL} is flaky` 42 | 43 | Placeholders may change depending on the context. Check the docs to see all available placeholder. 44 | """ 45 | 46 | 47 | LOG_EVENT_TYPE = t.Union[None, Unset, LogEvent] 48 | 49 | 50 | class GracefulRetryState: 51 | cur_attempt: int = 0 52 | success: bool = False 53 | 54 | last_exc: Exception | None = None 55 | last_response: httpx.Response | None 56 | 57 | def __init__(self, retry_config: GracefulRetry) -> None: 58 | self._retry_config = retry_config 59 | self._delay = retry_config.delay 60 | self._override_delay: float | None = None 61 | 62 | @property 63 | def delay(self) -> float: 64 | if self._override_delay is not None: 65 | return self._override_delay 66 | 67 | return self._delay 68 | 69 | @property 70 | def failed(self) -> bool: 71 | return not self.success 72 | 73 | @property 74 | def max_attempts(self): 75 | return self._retry_config.max_attempts 76 | 77 | @property 78 | def can_retry(self): 79 | return self.cur_attempt <= self.max_attempts 80 | 81 | @property 82 | def cant_retry(self): 83 | return not self.can_retry 84 | 85 | @property 86 | def cause(self) -> str: 87 | """Describes why the Retry was triggered""" 88 | 89 | # Importing here to avoid cyclic imports 90 | from gracy.exceptions import ( 91 | GracyRequestFailed, 92 | GracyUserDefinedException, 93 | NonOkResponse, 94 | UnexpectedResponse, 95 | ) 96 | 97 | if self.success: 98 | return "SUCCESSFUL" 99 | 100 | exc = self.last_exc 101 | 102 | if self.last_response: 103 | if isinstance(exc, NonOkResponse) or isinstance(exc, UnexpectedResponse): 104 | return f"[Bad Status Code: {self.last_response.status_code}]" 105 | 106 | if isinstance(exc, GracyUserDefinedException): 107 | return f"[User Error: {type(exc).__name__}]" 108 | 109 | if exc: 110 | if isinstance(exc, GracyRequestFailed): 111 | return f"[Request Error: {type(exc.original_exc).__name__}]" 112 | 113 | return f"[{type(exc).__name__}]" 114 | 115 | # This final block is unlikely to ever happen 116 | resp = t.cast(httpx.Response, self.last_response) 117 | return f"[Bad Status Code: {resp.status_code}]" 118 | 119 | def increment(self, response: httpx.Response | None): 120 | self.cur_attempt += 1 121 | 122 | if self.cur_attempt > 1: 123 | self._delay *= self._retry_config.delay_modifier 124 | 125 | self._override_delay = None 126 | if ( 127 | response 128 | and self._retry_config.overrides 129 | and self._retry_config.overrides.get(response.status_code) 130 | ): 131 | self._override_delay = self._retry_config.overrides[ 132 | response.status_code 133 | ].delay 134 | 135 | 136 | STATUS_OR_EXCEPTION = t.Union[int, t.Type[Exception]] 137 | 138 | 139 | @dataclass 140 | class OverrideRetryOn: 141 | delay: float 142 | 143 | 144 | @dataclass 145 | class GracefulRetry: 146 | delay: float 147 | max_attempts: int 148 | 149 | delay_modifier: float = 1 150 | retry_on: STATUS_OR_EXCEPTION | t.Iterable[STATUS_OR_EXCEPTION] | None = None 151 | log_before: LogEvent | None = None 152 | log_after: LogEvent | None = None 153 | log_exhausted: LogEvent | None = None 154 | behavior: t.Literal["break", "pass"] = "break" 155 | overrides: t.Union[t.Dict[int, OverrideRetryOn], None] = None 156 | 157 | def needs_retry(self, response_result: int) -> bool: 158 | if self.retry_on is None: 159 | return True 160 | 161 | retry_on_status = self.retry_on 162 | if not isinstance(retry_on_status, t.Iterable): 163 | retry_on_status = {retry_on_status} 164 | 165 | return response_result in retry_on_status 166 | 167 | def create_state( 168 | self, result: httpx.Response | None, exc: Exception | None 169 | ) -> GracefulRetryState: 170 | state = GracefulRetryState(self) 171 | state.last_response = result 172 | state.last_exc = exc 173 | return state 174 | 175 | 176 | class ThrottleRule: 177 | url_pattern: t.Pattern[str] 178 | """ 179 | Which URLs do you want to account for this? 180 | e.g. 181 | Strict values: 182 | - `"https://myapi.com/endpoint"` 183 | 184 | Regex values: 185 | - `"https://.*"` 186 | - `"http(s)?://myapi.com/.*"` 187 | """ 188 | 189 | max_requests: int 190 | """ 191 | How many requests should be run `per_time_range` 192 | """ 193 | 194 | per_time_range: timedelta 195 | """ 196 | Used in combination with `max_requests` to measure throttle 197 | """ 198 | 199 | def __init__( 200 | self, 201 | url_pattern: str, 202 | max_requests: int, 203 | per_time_range: timedelta = timedelta(seconds=1), 204 | ) -> None: 205 | self.url_pattern = re.compile(url_pattern) 206 | self.max_requests = max_requests 207 | self.per_time_range = per_time_range 208 | 209 | if isinstance(max_requests, float): 210 | raise TypeError(f"{max_requests=} should be an integer") 211 | 212 | @property 213 | def readable_time_range(self) -> str: 214 | seconds = self.per_time_range.total_seconds() 215 | periods = { 216 | ("hour", 3600), 217 | ("minute", 60), 218 | ("second", 1), 219 | } 220 | 221 | parts: list[str] = [] 222 | for period_name, period_seconds in periods: 223 | if seconds >= period_seconds: 224 | period_value, seconds = divmod(seconds, period_seconds) 225 | if period_value == 1: 226 | parts.append(period_name) 227 | else: 228 | parts.append(f"{int(period_value)} {period_name}s") 229 | 230 | if seconds < 1: 231 | break 232 | 233 | if len(parts) == 1: 234 | return parts[0] 235 | else: 236 | return ", ".join(parts[:-1]) + " and " + parts[-1] 237 | 238 | def __str__(self) -> str: 239 | return f"{self.max_requests} requests per {self.readable_time_range} for URLs matching {self.url_pattern}" 240 | 241 | def calculate_await_time(self, controller: ThrottleController) -> float: 242 | """ 243 | Checks current reqs/second and awaits if limit is reached. 244 | Returns whether limit was hit or not. 245 | """ 246 | rate_limit = self.max_requests 247 | cur_rate = controller.calculate_requests_per_rule( 248 | self.url_pattern, self.per_time_range 249 | ) 250 | 251 | if cur_rate >= rate_limit: 252 | time_diff = (rate_limit - cur_rate) or 1 253 | waiting_time = self.per_time_range.total_seconds() / time_diff 254 | return waiting_time 255 | 256 | return 0.0 257 | 258 | 259 | class ThrottleLocker: 260 | def __init__(self) -> None: 261 | self._regex_lock = t.DefaultDict[t.Pattern[str], Lock](Lock) 262 | self._generic_lock = Lock() 263 | 264 | @contextmanager 265 | def lock_rule(self, rule: ThrottleRule): 266 | with self._regex_lock[rule.url_pattern] as lock: 267 | yield lock 268 | 269 | @contextmanager 270 | def lock_check(self): 271 | with self._generic_lock as lock: 272 | yield lock 273 | 274 | def is_rule_throttled(self, rule: ThrottleRule) -> bool: 275 | return self._regex_lock[rule.url_pattern].locked() 276 | 277 | 278 | THROTTLE_LOCKER: t.Final = ThrottleLocker() 279 | 280 | 281 | class GracefulThrottle: 282 | rules: list[ThrottleRule] = [] 283 | log_limit_reached: LogEvent | None = None 284 | log_wait_over: LogEvent | None = None 285 | 286 | def __init__( 287 | self, 288 | rules: list[ThrottleRule] | ThrottleRule, 289 | log_limit_reached: LogEvent | None = None, 290 | log_wait_over: LogEvent | None = None, 291 | ) -> None: 292 | self.rules = rules if isinstance(rules, t.Iterable) else [rules] 293 | self.log_limit_reached = log_limit_reached 294 | self.log_wait_over = log_wait_over 295 | 296 | 297 | class ThrottleController: 298 | def __init__(self) -> None: 299 | self._control = t.DefaultDict[str, t.List[datetime]](list) 300 | 301 | def init_request(self, request_context: GracyRequestContext): 302 | with THROTTLE_LOCKER.lock_check(): 303 | self._control[request_context.url].append( 304 | datetime.now() 305 | ) # This should always keep it sorted asc 306 | 307 | def calculate_requests_per_rule( 308 | self, url_pattern: t.Pattern[str], range: timedelta 309 | ) -> float: 310 | with THROTTLE_LOCKER.lock_check(): 311 | past_time_window = datetime.now() - range 312 | request_rate = 0.0 313 | 314 | request_times = sorted( 315 | itertools.chain( 316 | *[ 317 | started_ats 318 | for url, started_ats in self._control.items() 319 | if url_pattern.match(url) 320 | ], 321 | ), 322 | reverse=True, 323 | ) 324 | 325 | req_idx = 0 326 | total_reqs = len(request_times) 327 | while req_idx < total_reqs: 328 | # e.g. Limit 4 requests per 2 seconds, now is 09:55 329 | # request_time=09:54 >= past_time_window=09:53 330 | if request_times[req_idx] >= past_time_window: 331 | request_rate += 1 332 | else: 333 | # Because it's sorted desc there's no need to keep iterating 334 | return request_rate 335 | 336 | req_idx += 1 337 | 338 | return request_rate 339 | 340 | def calculate_requests_per_sec(self, url_pattern: t.Pattern[str]) -> float: 341 | with THROTTLE_LOCKER.lock_check(): 342 | requests_per_second = 0.0 343 | coalesced_started_ats = sorted( 344 | itertools.chain( 345 | *[ 346 | started_ats 347 | for url, started_ats in self._control.items() 348 | if url_pattern.match(url) 349 | ] 350 | ) 351 | ) 352 | 353 | if coalesced_started_ats: 354 | # Best effort to measure rate if we just performed 1 request 355 | last = ( 356 | coalesced_started_ats[-1] 357 | if len(coalesced_started_ats) > 1 358 | else datetime.now() 359 | ) 360 | start = coalesced_started_ats[0] 361 | elapsed = last - start 362 | 363 | if elapsed.seconds > 0: 364 | requests_per_second = len(coalesced_started_ats) / elapsed.seconds 365 | 366 | return requests_per_second 367 | 368 | def debug_print(self): 369 | # Intended only for local development 370 | from rich.console import Console 371 | from rich.table import Table 372 | 373 | console = Console() 374 | table = Table(title="Throttling Summary") 375 | table.add_column("URL", overflow="fold") 376 | table.add_column("Count", justify="right") 377 | table.add_column("Times", justify="right") 378 | 379 | for url, times in self._control.items(): 380 | human_times = [time.strftime("%H:%M:%S.%f") for time in times] 381 | table.add_row(url, f"{len(times):,}", f"[yellow]{human_times}[/yellow]") 382 | 383 | console.print(table) 384 | 385 | 386 | class GracefulValidator(ABC): 387 | """ 388 | Run `check` raises exceptions in case it's not passing. 389 | """ 390 | 391 | @abstractmethod 392 | def check(self, response: httpx.Response) -> None: 393 | """Returns `None` to pass or raise exception""" 394 | pass 395 | 396 | 397 | @dataclass 398 | class RequestTimeline: 399 | url: str 400 | start: float 401 | end: float 402 | 403 | @classmethod 404 | def build(cls, start: float, resp: httpx.Response): 405 | end = start + resp.elapsed.total_seconds() 406 | 407 | return cls( 408 | url=str(resp.url), 409 | start=start, 410 | end=end, 411 | ) 412 | 413 | 414 | @dataclass 415 | class ConcurrentRequestLimit: 416 | """ 417 | Limits how many concurrent calls for a specific endpoint can be active. 418 | 419 | e.g. If you limit 10 requests to the endpoing /xyz 420 | """ 421 | 422 | limit: int 423 | uurl_pattern: t.Pattern[str] = re.compile(".*") 424 | blocking_args: t.Optional[t.Iterable[str]] = None 425 | """ 426 | Combine endpoint args to decide whether to limit. 427 | Optional, leaving it blank means that any request to the endpoint will be blocked 428 | """ 429 | 430 | limit_per_uurl: bool = True 431 | """ 432 | Whether Gracy should limit requests per UURL or the whole api. 433 | 434 | If True, UURLs will be grouped, so: 435 | 436 | Limit = 1 437 | #1. GET /test/{VALUE} (0/1) - RUNNING 438 | #3. GET /another/{VALUE} (0/1) - RUNNING 439 | #2. GET /test/{VALUE} (1/1) - WAITING [Grouped with #1] 440 | 441 | If False, every UURL will be matched, so: 442 | 443 | Limit = 1 444 | #1. GET /test/{VALUE} (0/1) - RUNNING 445 | #3. GET /another/{VALUE} (0/1) - WAITING [Grouped with ALL] 446 | #2. GET /test/{VALUE} (1/1) - WAITING [Grouped with ALL] 447 | 448 | """ 449 | 450 | log_limit_reached: LOG_EVENT_TYPE = None 451 | """ 452 | Log event for the first time the limit is reached. 453 | It's only triggered again if the limit slows down. 454 | """ 455 | 456 | log_limit_freed: LOG_EVENT_TYPE = None 457 | 458 | def __post_init__(self): 459 | self._arg_semaphore_map: t.Dict[ 460 | t.Tuple[str, ...], asyncio.BoundedSemaphore 461 | ] = {} 462 | 463 | def _get_blocking_key( 464 | self, request_context: GracyRequestContext 465 | ) -> t.Tuple[str, ...]: 466 | uurl_arg = request_context.unformatted_url if self.limit_per_uurl else "global" 467 | args: t.List[str] = [] 468 | 469 | if self.blocking_args: 470 | args = [ 471 | request_context.endpoint_args.get(arg, "") for arg in self.blocking_args 472 | ] 473 | 474 | return (uurl_arg, *args) 475 | 476 | def get_semaphore( 477 | self, request_context: GracyRequestContext 478 | ) -> asyncio.BoundedSemaphore: 479 | key = self._get_blocking_key(request_context) 480 | 481 | if key not in self._arg_semaphore_map: 482 | self._arg_semaphore_map[key] = asyncio.BoundedSemaphore(self.limit) 483 | 484 | return self._arg_semaphore_map[key] 485 | 486 | 487 | CONCURRENT_REQUEST_TYPE = t.Union[ 488 | t.Iterable[ConcurrentRequestLimit], ConcurrentRequestLimit, None, Unset 489 | ] 490 | 491 | 492 | @dataclass 493 | class GracyConfig: 494 | log_request: LOG_EVENT_TYPE = UNSET_VALUE 495 | log_response: LOG_EVENT_TYPE = UNSET_VALUE 496 | log_errors: LOG_EVENT_TYPE = UNSET_VALUE 497 | 498 | retry: GracefulRetry | None | Unset = UNSET_VALUE 499 | 500 | strict_status_code: t.Iterable[HTTPStatus] | HTTPStatus | None | Unset = UNSET_VALUE 501 | """Strictly enforces only one or many HTTP Status code to be considered as successful. 502 | 503 | e.g. Setting it to 201 would raise exceptions for both 204 or 200""" 504 | 505 | allowed_status_code: t.Iterable[ 506 | HTTPStatus 507 | ] | HTTPStatus | None | Unset = UNSET_VALUE 508 | """Adds one or many HTTP Status code that would normally be considered an error 509 | 510 | e.g. 404 would consider any 200-299 and 404 as successful. 511 | 512 | NOTE: `strict_status_code` takes precedence. 513 | """ 514 | 515 | validators: t.Iterable[ 516 | GracefulValidator 517 | ] | GracefulValidator | None | Unset = UNSET_VALUE 518 | """Adds one or many validators to be run for the response to decide whether it was successful or not. 519 | 520 | NOTE: `strict_status_code` or `allowed_status_code` are executed before. 521 | If none is set, it will first check whether the response has a successful code. 522 | """ 523 | 524 | parser: PARSER_TYPE = UNSET_VALUE 525 | """ 526 | Tell Gracy how to deal with the responses for you. 527 | 528 | Examples: 529 | - `"default": lambda response: response.json()` 530 | - `HTTPStatus.OK: lambda response: response.json()["ok_data"]` 531 | - `HTTPStatus.NOT_FOUND: None` 532 | - `HTTPStatus.INTERNAL_SERVER_ERROR: UserDefinedServerException` 533 | """ 534 | 535 | throttling: GracefulThrottle | None | Unset = UNSET_VALUE 536 | 537 | concurrent_requests: CONCURRENT_REQUEST_TYPE = UNSET_VALUE 538 | 539 | def should_retry( 540 | self, response: httpx.Response | None, req_or_validation_exc: Exception | None 541 | ) -> bool: 542 | """Only checks if given status requires retry. Does not consider attempts.""" 543 | 544 | if self.has_retry: 545 | retry = t.cast(GracefulRetry, self.retry) 546 | 547 | retry_on: t.Iterable[STATUS_OR_EXCEPTION] 548 | if ( 549 | not isinstance(retry.retry_on, t.Iterable) 550 | and retry.retry_on is not None 551 | ): 552 | retry_on = [retry.retry_on] 553 | elif retry.retry_on is None: 554 | retry_on = [] 555 | else: 556 | retry_on = retry.retry_on 557 | 558 | if response is None: 559 | if retry.retry_on is None: 560 | return True 561 | 562 | for maybe_exc in retry_on: 563 | if inspect.isclass(maybe_exc): 564 | if isinstance(req_or_validation_exc, maybe_exc): 565 | return True 566 | 567 | # Importing here to avoid cyclic imports 568 | from .exceptions import GracyRequestFailed 569 | 570 | if isinstance( 571 | req_or_validation_exc, GracyRequestFailed 572 | ) and isinstance(req_or_validation_exc.original_exc, maybe_exc): 573 | return True 574 | 575 | return False 576 | 577 | response_status = response.status_code 578 | 579 | if retry.retry_on is None: 580 | if req_or_validation_exc or response.is_success is False: 581 | return True 582 | 583 | if isinstance(retry.retry_on, t.Iterable): 584 | if response_status in retry.retry_on: 585 | return True 586 | 587 | for maybe_exc in retry.retry_on: 588 | if inspect.isclass(maybe_exc) and isinstance( 589 | req_or_validation_exc, maybe_exc 590 | ): 591 | return True 592 | 593 | elif inspect.isclass(retry.retry_on): 594 | return isinstance(req_or_validation_exc, retry.retry_on) 595 | 596 | else: 597 | return retry.retry_on == response_status 598 | 599 | return False 600 | 601 | @property 602 | def has_retry(self) -> bool: 603 | return self.retry is not None and self.retry != UNSET_VALUE 604 | 605 | @classmethod 606 | def merge_config(cls, base: GracyConfig, modifier: GracyConfig): 607 | new_obj = copy.copy(base) 608 | 609 | for key, value in vars(modifier).items(): 610 | if getattr(base, key) == UNSET_VALUE: 611 | setattr(new_obj, key, value) 612 | elif value != UNSET_VALUE: 613 | setattr(new_obj, key, value) 614 | 615 | return new_obj 616 | 617 | def get_concurrent_limit( 618 | self, context: GracyRequestContext 619 | ) -> t.Optional[ConcurrentRequestLimit]: 620 | if ( 621 | isinstance(self.concurrent_requests, Unset) 622 | or self.concurrent_requests is None 623 | ): 624 | return None 625 | 626 | if isinstance(self.concurrent_requests, ConcurrentRequestLimit): 627 | if self.concurrent_requests.uurl_pattern.match(context.unformatted_url): 628 | return self.concurrent_requests 629 | 630 | return None 631 | 632 | for rule in self.concurrent_requests: 633 | if rule.uurl_pattern.match(context.unformatted_url): 634 | return rule 635 | 636 | return None 637 | 638 | 639 | DEFAULT_CONFIG: t.Final = GracyConfig( 640 | log_request=None, 641 | log_response=None, 642 | log_errors=LogEvent(LogLevel.ERROR), 643 | strict_status_code=None, 644 | allowed_status_code=None, 645 | retry=None, 646 | ) 647 | 648 | 649 | class BaseEndpoint(str, Enum): 650 | def __str__(self) -> str: 651 | return self.value 652 | 653 | 654 | Endpoint = t.TypeVar("Endpoint", bound=t.Union[BaseEndpoint, str]) # , default=str) 655 | 656 | 657 | class GracefulRequest: 658 | request: httpx.Request 659 | request_func: t.Callable[..., t.Awaitable[httpx.Response]] 660 | """Can't use coroutine because we need to retrigger it during retries, and coro can't be awaited twice""" 661 | args: tuple[t.Any, ...] 662 | kwargs: dict[str, t.Any] 663 | 664 | def __init__( 665 | self, 666 | request: httpx.Request, 667 | request_func: t.Callable[..., t.Awaitable[httpx.Response]], 668 | *args: t.Any, 669 | **kwargs: t.Any, 670 | ) -> None: 671 | self.request = request 672 | self.request_func = request_func 673 | self.args = args 674 | self.kwargs = kwargs 675 | 676 | def __call__(self) -> t.Awaitable[httpx.Response]: 677 | return self.request_func(*self.args, **self.kwargs) 678 | 679 | 680 | class GracyRequestContext: 681 | def __init__( 682 | self, 683 | method: str, 684 | base_url: str, 685 | endpoint: str, 686 | endpoint_args: t.Union[t.Dict[str, str], None], 687 | active_config: GracyConfig, 688 | ) -> None: 689 | if base_url.endswith("/"): 690 | base_url = base_url[:-1] 691 | 692 | final_endpoint = endpoint.format(**endpoint_args) if endpoint_args else endpoint 693 | 694 | self.endpoint_args = endpoint_args or {} 695 | self.endpoint = final_endpoint 696 | self.unformatted_endpoint = endpoint 697 | 698 | self.url = f"{base_url}{self.endpoint}" 699 | self.unformatted_url = f"{base_url}{self.unformatted_endpoint}" 700 | 701 | self.method = method 702 | self._active_config = active_config 703 | 704 | @property 705 | def active_config(self) -> GracyConfig: 706 | return self._active_config 707 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | 4 | 5 | ## v1.34.0 (2024-11-27) 6 | 7 | ### Feature 8 | 9 | * Garbage collector improvements ([`36e537e`](https://github.com/guilatrova/gracy/commit/36e537ec85fd12c273a74945cf7962cbdd7494bd)) 10 | 11 | ## v1.33.1 (2024-10-07) 12 | 13 | ### Fix 14 | 15 | * Don't stop if ctx token reset fails ([`4ddad79`](https://github.com/guilatrova/gracy/commit/4ddad79627a5fd528a199d8bf6e58906d75cbec1)) 16 | 17 | ## v1.33.0 (2024-02-20) 18 | 19 | ### Feature 20 | 21 | * Enhance response typing ([`95f8c16`](https://github.com/guilatrova/gracy/commit/95f8c162de7ab64a925bf008e81e11b138426320)) 22 | 23 | ### Fix 24 | 25 | * Add typing_extensions to support deprecated ([`f6b2b7b`](https://github.com/guilatrova/gracy/commit/f6b2b7b1baf7a39c5f6f21cced7aeeeac23cbfe7)) 26 | 27 | ### Documentation 28 | 29 | * Update on dynamic type hinting ([`6d7975d`](https://github.com/guilatrova/gracy/commit/6d7975d1b383d040f2c960d1dc13482530b81571)) 30 | 31 | ## v1.32.2 (2024-01-31) 32 | ### Fix 33 | 34 | * Define correct typing ([`7c889fe`](https://github.com/guilatrova/gracy/commit/7c889fe782e7705843d84ec6dc1b3f0098e779e0)) 35 | 36 | ## v1.32.1 (2024-01-15) 37 | ### Fix 38 | 39 | * **retry:** Verify original exc for failed requests ([`31e79d7`](https://github.com/guilatrova/gracy/commit/31e79d75e6bbeedfdbbc7f69f2c254017c21e22c)) 40 | * Format original exception name properly ([`56254b6`](https://github.com/guilatrova/gracy/commit/56254b6e99325b550d3a33512918ab6bb0a56b9a)) 41 | 42 | ## v1.32.0 (2023-12-09) 43 | ### Feature 44 | 45 | * Improve concurrency logging ([`5292faf`](https://github.com/guilatrova/gracy/commit/5292faf9eff54e9290ab2db836677f44c15b4673)) 46 | * Add plotly printer ([`5b8a460`](https://github.com/guilatrova/gracy/commit/5b8a460ffeb8a363b2dfcbef95b3b6134c1f205a)) 47 | * Track requests timeline ([`9c30ba2`](https://github.com/guilatrova/gracy/commit/9c30ba2d84a9485d44530df9b82a7bcaf04cd01e)) 48 | * Replace manual impl with semaphore ([`57df089`](https://github.com/guilatrova/gracy/commit/57df08936d9f4a808cddd3857058f0f23ba3350b)) 49 | 50 | ### Documentation 51 | 52 | * Add explanation ([`b1ea801`](https://github.com/guilatrova/gracy/commit/b1ea80122d6915a207e95780928918c3253d532a)) 53 | * Explain concurrent request limit ([`63efb40`](https://github.com/guilatrova/gracy/commit/63efb4039a77c697510a6eb9c47c0fd3319e0017)) 54 | * Update badges/description ([`157daaf`](https://github.com/guilatrova/gracy/commit/157daafc3f12b4fa6641a5287688cd632a2ea851)) 55 | 56 | ## v1.31.0 (2023-12-03) 57 | ### Feature 58 | 59 | * Auto instantiate namespaces ([`3d26863`](https://github.com/guilatrova/gracy/commit/3d26863444a2a959a0ca6b4665c4791547acfbec)) 60 | 61 | ## v1.30.0 (2023-12-02) 62 | ### Feature 63 | 64 | * Add paginator ([`f8a1db8`](https://github.com/guilatrova/gracy/commit/f8a1db8108b2662c88c182679a71f2b4bb4476fd)) 65 | 66 | ## v1.29.0 (2023-12-02) 67 | ### Feature 68 | 69 | * Add generated_parsed_response ([`d9add8e`](https://github.com/guilatrova/gracy/commit/d9add8e208c54f72a08df5bfd4b66192c7aba751)) 70 | 71 | ### Fix 72 | 73 | * Specify parsed response type properly ([`2ef4566`](https://github.com/guilatrova/gracy/commit/2ef4566069e8f9a6bec129f393eb8b12c946b139)) 74 | 75 | ### Documentation 76 | 77 | * Explain parsed_response ([`792fd32`](https://github.com/guilatrova/gracy/commit/792fd32af1bed7ed2584f166082c6ca35d9cdaae)) 78 | 79 | ## v1.28.1 (2023-11-30) 80 | ### Fix 81 | 82 | * Resolve deploy issue ([`2ae3717`](https://github.com/guilatrova/gracy/commit/2ae3717aa4620e114e750b01eb363f49b0f0fd97)) 83 | 84 | ## v1.28.0 (2023-11-30) 85 | ### Feature 86 | 87 | * Introduce namespaces ([`67a70f3`](https://github.com/guilatrova/gracy/commit/67a70f3550a77a37d309fcb65800b1b6759fb8f8)) 88 | 89 | ### Documentation 90 | 91 | * Write about namespaces ([`468e205`](https://github.com/guilatrova/gracy/commit/468e205fddb1166770bddb4c8c241668b27d03a7)) 92 | 93 | ## v1.27.1 (2023-08-08) 94 | ### Fix 95 | 96 | * Use Union for python 3.8 ([`02415ac`](https://github.com/guilatrova/gracy/commit/02415aca8009e7bcae70f93b0c0b54c5a7d61473)) 97 | 98 | ## v1.27.0 (2023-08-08) 99 | ### Feature 100 | 101 | * Add concurrent calls ([`69976d3`](https://github.com/guilatrova/gracy/commit/69976d36d25fe45ae7b6cefe5e8800bf926aa4f8)) 102 | 103 | ### Documentation 104 | 105 | * Add note about concurrent requests ([`487dbed`](https://github.com/guilatrova/gracy/commit/487dbedbf707dbe38caf4529be1488ad72a0f2d9)) 106 | 107 | ## v1.26.0 (2023-08-06) 108 | ### Feature 109 | 110 | * Count ongoing requests ([`9495d4e`](https://github.com/guilatrova/gracy/commit/9495d4ed98839562fc470da2964d370638b6f727)) 111 | 112 | ### Fix 113 | 114 | * Use int over HttpStatus enum ([`abebe93`](https://github.com/guilatrova/gracy/commit/abebe938191b760f64490477abad595259537c79)) 115 | 116 | ### Documentation 117 | 118 | * Add common hooks ([`3a85897`](https://github.com/guilatrova/gracy/commit/3a85897baf65222e156e9762cd3f09d8bc35e2e4)) 119 | 120 | ## v1.25.0 (2023-06-30) 121 | ### Feature 122 | 123 | * **replays:** Support log events for replays ([`f9c6b80`](https://github.com/guilatrova/gracy/commit/f9c6b8071302b2262b2973830e7c3d378ae64288)) 124 | 125 | ### Fix 126 | 127 | * **reports:** Hide replays if display disabled ([`68692e0`](https://github.com/guilatrova/gracy/commit/68692e0e94ed1ec1d81ea343787abadd34ce3340)) 128 | 129 | ## v1.24.1 (2023-06-30) 130 | ### Fix 131 | 132 | * **replays:** Pass discard flag ([`2083a52`](https://github.com/guilatrova/gracy/commit/2083a524ed515758ccf1ff6b814d41eafd64ebf5)) 133 | 134 | ## v1.24.0 (2023-06-30) 135 | ### Feature 136 | 137 | * **replays:** Allow to discard bad status ([`9926ae3`](https://github.com/guilatrova/gracy/commit/9926ae3282ab9632261af204734cbfcf2fce721e)) 138 | * **loggers:** Add REPLAY placeholder ([`911a9ec`](https://github.com/guilatrova/gracy/commit/911a9ecdcb83bf877350b96d325047cea80edbd5)) 139 | * **reports:** Show replays ([`fc82bd7`](https://github.com/guilatrova/gracy/commit/fc82bd7322f35a61506bc2b42bc53c51d0ae1029)) 140 | * Track replays per request ([`e7322a3`](https://github.com/guilatrova/gracy/commit/e7322a3b31aa55d31e93e71cda43ba1ee55c53c2)) 141 | * **hooks:** Don't wait if replayed ([`785959f`](https://github.com/guilatrova/gracy/commit/785959f92a8bcacba9d7523a15c7682294bf2a00)) 142 | * **replays:** Add flag when replayed ([`8ae8394`](https://github.com/guilatrova/gracy/commit/8ae839489e338b8fbde18e96771dc7765587a6b2)) 143 | 144 | ## v1.23.0 (2023-06-28) 145 | ### Feature 146 | 147 | * **hooks:** Make reporter optional + dryrun ([`ea0ec00`](https://github.com/guilatrova/gracy/commit/ea0ec007ff5cbd19d1371fe157e9d563d50230bb)) 148 | * **hooks:** Add RateLimitBackOffHook ([`d941c68`](https://github.com/guilatrova/gracy/commit/d941c6880494c41171792fe54b9b6189bba964b9)) 149 | * Allow to modify retry-after ([`a5f043d`](https://github.com/guilatrova/gracy/commit/a5f043dc4a93fa2ade28967d8b019e483d52c6f7)) 150 | * **hooks:** Return hook result ([`ee8f102`](https://github.com/guilatrova/gracy/commit/ee8f10235bced2ed6fe8af1abe219e51950d943f)) 151 | 152 | ### Fix 153 | 154 | * **retry:** Validate log correctly ([`dbddcf5`](https://github.com/guilatrova/gracy/commit/dbddcf5f881f2873f3149bd25e4f06ad930b6c51)) 155 | 156 | ## v1.22.0 (2023-06-27) 157 | ### Feature 158 | 159 | * **retry:** Add RETRY_CAUSE to log ([`3bf1c72`](https://github.com/guilatrova/gracy/commit/3bf1c72776874b3b77f5620ce57e34a912e0a197)) 160 | * **retry:** Add cause ([`2b468fd`](https://github.com/guilatrova/gracy/commit/2b468fde1c1a8d3374268c54814bdfb4f992f191)) 161 | 162 | ### Fix 163 | 164 | * **retry:** Check override for status ([`1b9383d`](https://github.com/guilatrova/gracy/commit/1b9383d0e340c3a422e87f69c17ab1e097ca7112)) 165 | * Make it compatible with py3.8 ([`96e3e61`](https://github.com/guilatrova/gracy/commit/96e3e610eee324047c63adcddd5a6fa534cb51a8)) 166 | * **retry:** Set last rep for overriden retry ([`d0241e5`](https://github.com/guilatrova/gracy/commit/d0241e54a4f6cbd5f0b5541366cae3c3a64eb6c3)) 167 | * Typo and wrong attrs ([`07d30e0`](https://github.com/guilatrova/gracy/commit/07d30e09a4767084bbfcca5654682486b579bb42)) 168 | 169 | ## v1.21.1 (2023-06-26) 170 | ### Fix 171 | 172 | * Expose OverrideRetryOn ([`0f14782`](https://github.com/guilatrova/gracy/commit/0f14782aa870efcec829cd9efb2bc72ea101305e)) 173 | 174 | ## v1.21.0 (2023-06-26) 175 | ### Feature 176 | 177 | * **hooks:** Implement flag for diff locks ([`dcce610`](https://github.com/guilatrova/gracy/commit/dcce6104f1129ee4710b50fc034c1a8629f57cf2)) 178 | 179 | ## v1.20.0 (2023-06-26) 180 | ### Feature 181 | 182 | * **hooks:** Handle dates for retry_after header ([`3feef65`](https://github.com/guilatrova/gracy/commit/3feef652b1de9aa32b551c54307965f24a92e1b7)) 183 | * **hooks:** Add a common hook ([`04734ad`](https://github.com/guilatrova/gracy/commit/04734adcbea3ecc3cde93e1330f58992bb20b3be)) 184 | * **retry:** Implement retry override ([`0045413`](https://github.com/guilatrova/gracy/commit/004541385ca1956d6d3cc3c4807ec09cf9999177)) 185 | 186 | ## v1.19.0 (2023-06-11) 187 | ### Feature 188 | 189 | * Include cause/context to the req exception ([`b2a96f1`](https://github.com/guilatrova/gracy/commit/b2a96f1caba229ef8ae2b102f160b107bde1f9dd)) 190 | 191 | ## v1.18.0 (2023-06-10) 192 | ### Feature 193 | 194 | * Create request failed exc ([`8087365`](https://github.com/guilatrova/gracy/commit/8087365ebbefa1de8801fc232553e1a5f5d954b5)) 195 | 196 | ### Fix 197 | 198 | * **replays:** Handle content type none ([`0229218`](https://github.com/guilatrova/gracy/commit/02292182f99d925ea3cb3d916ba940489a6984e0)) 199 | 200 | ## v1.17.3 (2023-05-31) 201 | ### Fix 202 | 203 | * **deps:** Requires httpx>=0.23 ([`c3fc7c0`](https://github.com/guilatrova/gracy/commit/c3fc7c08387204eee825a5b6bd17d10d41706d78)) 204 | 205 | ## v1.17.2 (2023-05-18) 206 | ### Fix 207 | * Don't pass every kwargs to build_request ([`192838e`](https://github.com/guilatrova/gracy/commit/192838ed0dada8f0dbf30aa7a15a42efe7ad50b0)) 208 | 209 | ## v1.17.1 (2023-05-17) 210 | ### Fix 211 | * Support py38+ ([`1ac2f56`](https://github.com/guilatrova/gracy/commit/1ac2f56741094e5c0b4586093f8fa8cb26720d02)) 212 | 213 | ## v1.17.0 (2023-05-13) 214 | ### Feature 215 | * Count/display data about replays ([`658b5f5`](https://github.com/guilatrova/gracy/commit/658b5f52adf34d5d6e577abc7e6b9cd33a745919)) 216 | * Implement skip throttle ([`efb259c`](https://github.com/guilatrova/gracy/commit/efb259c151ee0807af325901155b67184de062e2)) 217 | 218 | ## v1.16.0 (2023-05-13) 219 | ### Feature 220 | * Pass retry state to after hook ([`218e510`](https://github.com/guilatrova/gracy/commit/218e510d84a3acf1fbee84141ec5f7123672cd6b)) 221 | * Implement gracy hooks ([`243dddb`](https://github.com/guilatrova/gracy/commit/243dddbd58845cbc92a0b84eaf44c612a125daf5)) 222 | * Implement hooks ([`7a88e4c`](https://github.com/guilatrova/gracy/commit/7a88e4c93e3517b9ed095f179582f0fb2809e48a)) 223 | 224 | ### Fix 225 | * Resolve recursion for hooks ([`e1be02b`](https://github.com/guilatrova/gracy/commit/e1be02bb20395c207353de2ea3bae8d839a34c03)) 226 | 227 | ### Documentation 228 | * **hooks:** Add example about hook ([`3cb52db`](https://github.com/guilatrova/gracy/commit/3cb52db7600213252bb36d6de8442bd487fd57b7)) 229 | 230 | ## v1.15.0 (2023-05-11) 231 | ### Feature 232 | * Show 'Aborts' as title ([`8485409`](https://github.com/guilatrova/gracy/commit/8485409e899e5d4591754ad62e35cfa4a128f124)) 233 | * **reports:** Show retries/throttles ([`f6de12a`](https://github.com/guilatrova/gracy/commit/f6de12a51a95b7c0ac8d0302004a3ad8c0d2e146)) 234 | 235 | ## v1.14.0 (2023-05-11) 236 | ### Feature 237 | * Default safe format + retry status code ([`5d7f834`](https://github.com/guilatrova/gracy/commit/5d7f834db146284813341d55979e25b373855606)) 238 | * Display aborted requests ([`67ac1ed`](https://github.com/guilatrova/gracy/commit/67ac1ed103248a8f65890826fc6732ec20adb683)) 239 | 240 | ### Documentation 241 | * Add note about graceful request ([`7e14c80`](https://github.com/guilatrova/gracy/commit/7e14c80205bd56df9297d2a169c3529397b4f05a)) 242 | 243 | ## v1.13.0 (2023-05-10) 244 | ### Feature 245 | * Track broken requests ([`e40d8b8`](https://github.com/guilatrova/gracy/commit/e40d8b8774c86f766c69c0cd8f0d5d5b65f09d0f)) 246 | * Capture broken requests (without a response) ([`bf0ac44`](https://github.com/guilatrova/gracy/commit/bf0ac44e87f96d7acc41f7f0e63411ac0f113a67)) 247 | 248 | ## v1.12.0 (2023-05-04) 249 | ### Feature 250 | * Improve decorator typing ([`72233d6`](https://github.com/guilatrova/gracy/commit/72233d60dd84cfddf2778b585b1260833f357c1e)) 251 | 252 | ## v1.11.4 (2023-05-04) 253 | ### Fix 254 | * Add support for `graceful_generator` ([`22ecf9a`](https://github.com/guilatrova/gracy/commit/22ecf9ac91064fcc4288f38ff73a77f4e165b98d)) 255 | 256 | ## v1.11.3 (2023-03-24) 257 | ### Fix 258 | * Make exception pickable ([`16d6a62`](https://github.com/guilatrova/gracy/commit/16d6a6248fd46a565c411743a3bf0f74dac94363)) 259 | 260 | ### Documentation 261 | * Show custom request timeout ([`e2a069b`](https://github.com/guilatrova/gracy/commit/e2a069b46a01cbcbf5bd2a9507d7d25505ecbd83)) 262 | 263 | ## v1.11.2 (2023-03-03) 264 | ### Fix 265 | * Log exhausted when appropriate ([`8c5d622`](https://github.com/guilatrova/gracy/commit/8c5d622fef7aa6dd2514cfaaf867445f56d7b04a)) 266 | * Retry considers last validation result ([`595177f`](https://github.com/guilatrova/gracy/commit/595177f50e396f4ca7b2dcc1c8ed535928a0aca7)) 267 | * Handle retry edge case ([`077e6f4`](https://github.com/guilatrova/gracy/commit/077e6f49d80cb6d886c31aa010a4f814a6953445)) 268 | * Retry result is used as response ([`8687156`](https://github.com/guilatrova/gracy/commit/8687156991058fa24043dc39658f0a12377a21f6)) 269 | 270 | ### Documentation 271 | * Add httpbin example ([`1babd10`](https://github.com/guilatrova/gracy/commit/1babd1098a46c4d0bc24ed228d76bb094260ad5e)) 272 | 273 | ## v1.11.1 (2023-02-23) 274 | ### Fix 275 | * **retry:** Don't retry when successful ([`b334c22`](https://github.com/guilatrova/gracy/commit/b334c227a4a8a688029130c736118b6dcb4f8f3b)) 276 | * **pymongo:** Adjust filter ([`5ee9f0c`](https://github.com/guilatrova/gracy/commit/5ee9f0c6aa523530929bd69d19a9ff637c46705c)) 277 | * **pymongo:** Use correct methods/kwargs ([`4a191d8`](https://github.com/guilatrova/gracy/commit/4a191d81e083772add036bc3d9d5937ccbf6d31c)) 278 | 279 | ### Documentation 280 | * Update examples ([`26420da`](https://github.com/guilatrova/gracy/commit/26420da78776862a0cb7569b5f64b610ed212ff6)) 281 | 282 | ## v1.11.0 (2023-02-23) 283 | ### Feature 284 | * Enable config debugging flag ([`07c6339`](https://github.com/guilatrova/gracy/commit/07c633923a20343329aa884ddc109f3cde0e5be0)) 285 | 286 | ## v1.10.1 (2023-02-23) 287 | ### Fix 288 | * Error log ([`6f63941`](https://github.com/guilatrova/gracy/commit/6f6394181ed024f738605a4743af2eea788ce4f7)) 289 | 290 | ## v1.10.0 (2023-02-22) 291 | ### Feature 292 | * Allow custom validators ([`50818f8`](https://github.com/guilatrova/gracy/commit/50818f89fe2a03800fde18fa38686a04853cb54a)) 293 | 294 | ### Fix 295 | * Implement proper validate/retry/parse logic ([`0b2fa75`](https://github.com/guilatrova/gracy/commit/0b2fa75228c9340efb8595fee801c0cfa3303619)) 296 | * Raise exception correctly ([`10a90b5`](https://github.com/guilatrova/gracy/commit/10a90b5159a2fce3e24c1bfac7f4b9e0cb58d059)) 297 | 298 | ### Documentation 299 | * Add exception details to retry params ([`8d69234`](https://github.com/guilatrova/gracy/commit/8d692346369b5c83d05e746ec1b7e9f924d02cbd)) 300 | * Enhance custom validator example ([`d5e02eb`](https://github.com/guilatrova/gracy/commit/d5e02eb032739639f9ceb655b5b88c39f8c9a0f6)) 301 | * Add validators ([`e3e8fa6`](https://github.com/guilatrova/gracy/commit/e3e8fa672e5f95d02f60dc3af762b6e6cd189d4d)) 302 | 303 | ## v1.9.1 (2023-02-21) 304 | ### Fix 305 | * Create tuples ([`f648f85`](https://github.com/guilatrova/gracy/commit/f648f85a5787b2cd86934051640e666815fe5864)) 306 | 307 | ## v1.9.0 (2023-02-21) 308 | ### Feature 309 | * Make exceptions pickable ([`5ab62c5`](https://github.com/guilatrova/gracy/commit/5ab62c59ac273078e7a1ef3122e76bf0c6901e70)) 310 | 311 | ### Documentation 312 | * Reword ([`0ca061f`](https://github.com/guilatrova/gracy/commit/0ca061f1b1e73c73b01808e2d9f0258f03e0fefa)) 313 | * Add a emoji ([`8da07ae`](https://github.com/guilatrova/gracy/commit/8da07aecd8da6642edf01a94475ff49f297c1886)) 314 | * Reword ([`a54f1f7`](https://github.com/guilatrova/gracy/commit/a54f1f7bac2b7a5fb52485b31c746e58734066d0)) 315 | * Reorder logging customization ([`f6d9d76`](https://github.com/guilatrova/gracy/commit/f6d9d765daee63e7e863426519f8acda5bc2c5f0)) 316 | 317 | ## v1.8.1 (2023-02-17) 318 | ### Fix 319 | * Retry logic triggers only once ([`0fc2358`](https://github.com/guilatrova/gracy/commit/0fc2358b1631eacc0587a59afe1d21b419f8679e)) 320 | 321 | ## v1.8.0 (2023-02-17) 322 | ### Feature 323 | * Calculate throttling await properly ([`ba520e0`](https://github.com/guilatrova/gracy/commit/ba520e034bab88b2b5a258473f8a2ba7ff7c5879)) 324 | * Lock throttling logs properly ([`a8ebd69`](https://github.com/guilatrova/gracy/commit/a8ebd69df0e5184a6a806870a12888c202ba37d8)) 325 | * Prevent floats for max_requests ([`b9aed74`](https://github.com/guilatrova/gracy/commit/b9aed746bdfcd672920baeb047cf02b31e146503)) 326 | * Format rule time range ([`514cbae`](https://github.com/guilatrova/gracy/commit/514cbaeeb2d02de12f60a62e8285ce0ba1ad0437)) 327 | * Allow custom time windows for throttling ([`7fc35f0`](https://github.com/guilatrova/gracy/commit/7fc35f09e4a5e8df50a746cf95d112b08d4dd9bc)) 328 | 329 | ### Fix 330 | * Correct kwargs ([`0db5925`](https://github.com/guilatrova/gracy/commit/0db59254081d479a20c411ab346cad605e3a2efb)) 331 | 332 | ### Documentation 333 | * Add `THROTTLE_TIME_RANGE` ([`299c200`](https://github.com/guilatrova/gracy/commit/299c2008b5da43e7a52035dc285375b0b1dfc093)) 334 | * **throttling:** Add timedelta example ([`74c20ef`](https://github.com/guilatrova/gracy/commit/74c20ef91c521165b72c999c7212268ca83ec7cc)) 335 | * Enhance throttling example ([`200b3c5`](https://github.com/guilatrova/gracy/commit/200b3c5adac8a16f3af002d56f2e3c8b84f3f0d3)) 336 | 337 | ## v1.7.1 (2023-02-14) 338 | ### Fix 339 | * **retry:** Remove duplicated default msg ([`963d7e8`](https://github.com/guilatrova/gracy/commit/963d7e8237a85c5f5692a01d7a3d1c0eb733b752)) 340 | 341 | ### Documentation 342 | * Fix reports/replay order ([`b4ddf79`](https://github.com/guilatrova/gracy/commit/b4ddf792fe29ae49e981fda5b1fca0bec4aca0f9)) 343 | 344 | ## v1.7.0 (2023-02-12) 345 | ### Feature 346 | * Handle missing replays ([`4395b83`](https://github.com/guilatrova/gracy/commit/4395b832cd9f75a88d696d5cba2eb7bd9f7ce61d)) 347 | * Report show replay mode ([`b488975`](https://github.com/guilatrova/gracy/commit/b4889755c75c3f3a14507b27b3d57ba243b5c828)) 348 | * Implement replay load w/ sqlite ([`4fa4cf6`](https://github.com/guilatrova/gracy/commit/4fa4cf6983ed64d82560d47c813bfeb4cfa5ed66)) 349 | * Implement replay (store only) w/ sqlite ([`797c2b9`](https://github.com/guilatrova/gracy/commit/797c2b95334f5ebfd9b17555278b1be44b7eeef2)) 350 | 351 | ### Fix 352 | * Handle 0 requests for logger printer ([`09e471c`](https://github.com/guilatrova/gracy/commit/09e471c791e34c9b30427c6903bb19c8c25338aa)) 353 | 354 | ### Documentation 355 | * Add details about custom replay storage ([`f03407f`](https://github.com/guilatrova/gracy/commit/f03407fbd66a40850d679541b0616fc7847c8b5c)) 356 | * Add brief explanation about replay ([`edd1a24`](https://github.com/guilatrova/gracy/commit/edd1a24fb255d8ed23288f277769b923e6af218b)) 357 | 358 | ## v1.6.1 (2023-02-11) 359 | ### Fix 360 | * Gracy supports Python >=3.8 ([`a3623a9`](https://github.com/guilatrova/gracy/commit/a3623a98a7459dcba3dc78ca11917be5c6c5a82d)) 361 | 362 | ## v1.6.0 (2023-02-07) 363 | ### Feature 364 | * Handle parsing failures ([`ac48952`](https://github.com/guilatrova/gracy/commit/ac489522a98412d65b85ac3317dbe6083d8819ad)) 365 | 366 | ### Documentation 367 | * Fix syntax ([`9996b39`](https://github.com/guilatrova/gracy/commit/9996b39f505d3221f2e63d78bd311e90f2608349)) 368 | 369 | ## v1.5.0 (2023-02-05) 370 | ### Feature 371 | * Protect lambda custom msg from unknown keys ([`d6da853`](https://github.com/guilatrova/gracy/commit/d6da8536d1b561fd606d2911749d99309aa92460)) 372 | * Implement lambda for loggers ([`e7d9248`](https://github.com/guilatrova/gracy/commit/e7d9248475ce9dab92913cc7fa7eb6554c9676d7)) 373 | 374 | ### Fix 375 | * Use correct typing for coroutine ([`65296cd`](https://github.com/guilatrova/gracy/commit/65296cdddf925126ea47e591f7def242b0e6b6da)) 376 | 377 | ### Documentation 378 | * Add report examples ([`269810c`](https://github.com/guilatrova/gracy/commit/269810c4d205e5356672287f08c3d34d3bc0c3f0)) 379 | 380 | ## v1.4.0 (2023-02-05) 381 | ### Feature 382 | * Implement the logger printer ([`40298f5`](https://github.com/guilatrova/gracy/commit/40298f5204a499730f93d2d79bbfed43dc754b0c)) 383 | * Implement the list printer ([`9adee2d`](https://github.com/guilatrova/gracy/commit/9adee2d9ea78a569ab1541724b86fb73b06a4f2e)) 384 | * Split rich as optional dep ([`ae169df`](https://github.com/guilatrova/gracy/commit/ae169df066871d4095b95c032e7ec06b85ab3249)) 385 | 386 | ### Documentation 387 | * Fix bad information ([`e1a6746`](https://github.com/guilatrova/gracy/commit/e1a67466a9403dc87719cde9079a0f2b0ed7b16f)) 388 | * Fix bad syntax example ([`116b9bf`](https://github.com/guilatrova/gracy/commit/116b9bf0e1ed6fabdb9e5d365ade7d92ab8d3429)) 389 | 390 | ## v1.3.0 (2023-02-01) 391 | ### Feature 392 | * Use locks for throttled requests ([`b2db6a7`](https://github.com/guilatrova/gracy/commit/b2db6a760b097b27142f17bf533d760e4e99605c)) 393 | 394 | ### Fix 395 | * Throttling/allowed not working ([`cb0251b`](https://github.com/guilatrova/gracy/commit/cb0251b49c43f9376783e6f457073410f6d326a1)) 396 | 397 | ## v1.2.1 (2023-02-01) 398 | ### Fix 399 | * Handle scenarios for just 1 request per url ([`f4f799b`](https://github.com/guilatrova/gracy/commit/f4f799bbc03ae318fba69dd299fb423800a18651)) 400 | 401 | ## v1.2.0 (2023-02-01) 402 | ### Feature 403 | * Simplify req/s rate to the user ([`1b428c7`](https://github.com/guilatrova/gracy/commit/1b428c788f192e0e23c49b27d9a46438d20d230a)) 404 | * Include req rate in report ([`e387a25`](https://github.com/guilatrova/gracy/commit/e387a25f831a27f031ebc1625ac642beb3895678)) 405 | * Clear base urls with ending slash ([`51fb8ee`](https://github.com/guilatrova/gracy/commit/51fb8ee9e369eecd951fb31da92edc3317e63483)) 406 | * Implement retry logging ([`f2d3238`](https://github.com/guilatrova/gracy/commit/f2d3238830bbda163b8b55f874f2ae7ecb11d6df)) 407 | 408 | ### Fix 409 | * Consider retry is unset ([`0ca1ed9`](https://github.com/guilatrova/gracy/commit/0ca1ed9e65faa8e1e7efd024a7264dbc328a3259)) 410 | * Retry must start with 1 ([`3e3e750`](https://github.com/guilatrova/gracy/commit/3e3e75003092bca7f4181c17b68a873ec77c31d1)) 411 | 412 | ### Documentation 413 | * Fix download badge ([`22a9d7a`](https://github.com/guilatrova/gracy/commit/22a9d7a132b86c6da084b6f59ddba74f64814238)) 414 | * Improve examples ([`4ca1f7d`](https://github.com/guilatrova/gracy/commit/4ca1f7df80b6b1bba9f255983a6be5b906b09a85)) 415 | * Add new placeholders ([`8eba619`](https://github.com/guilatrova/gracy/commit/8eba619dd73544861960b0a9a381fe97d2c5468f)) 416 | * Add some notes for custom exceptions ([`225f008`](https://github.com/guilatrova/gracy/commit/225f00828697d8a611bb596e1f3119570a1b363e)) 417 | 418 | ## v1.1.0 (2023-01-30) 419 | ### Feature 420 | * Change api to be public ([`3b0c828`](https://github.com/guilatrova/gracy/commit/3b0c8281c3e164d9a7f01770c698fa825afe562a)) 421 | 422 | ### Documentation 423 | * Fix examples/info ([`0193f11`](https://github.com/guilatrova/gracy/commit/0193f112807f4621f5fd35acc9fbec32c4a2554c)) 424 | 425 | ## v1.0.0 (2023-01-30) 426 | ### Feature 427 | * Drop python 3.7 support ([`0f69e5b`](https://github.com/guilatrova/gracy/commit/0f69e5be00f8202ea2aa98b71630ae167c6431f1)) 428 | 429 | ### Breaking 430 | * drop python 3.7 support ([`0f69e5b`](https://github.com/guilatrova/gracy/commit/0f69e5be00f8202ea2aa98b71630ae167c6431f1)) 431 | 432 | ### Documentation 433 | * Add remaining sections ([`4335b5a`](https://github.com/guilatrova/gracy/commit/4335b5a3313a56c36b7b54c9ec44a07b2e6b4bd0)) 434 | * Add throttling ([`6fc9583`](https://github.com/guilatrova/gracy/commit/6fc958328fcbc5304e745c29918f8ffb2f8fa1a4)) 435 | * Add retry ([`aa8a828`](https://github.com/guilatrova/gracy/commit/aa8a82844a8c77f99897512d23b01eb216b8e0ff)) 436 | * Add credits/settings section ([`113bf48`](https://github.com/guilatrova/gracy/commit/113bf4886ae50418ddaef62d6f4880171f98240f)) 437 | * Write about parsing ([`c133cda`](https://github.com/guilatrova/gracy/commit/c133cda6444058861a5129db5da0a4fd7a12965e)) 438 | * Remove colspans ([`3ef5fd7`](https://github.com/guilatrova/gracy/commit/3ef5fd77dbec659144a034405c815fa5a060d747)) 439 | * Add logging details ([`09e923c`](https://github.com/guilatrova/gracy/commit/09e923cb9bb14b858f8c6ab975fb50ffab8fd42a)) 440 | * Fix badge ([`fea301a`](https://github.com/guilatrova/gracy/commit/fea301a63db98398101ae796f3a14f35882922f7)) 441 | * Add empty topics ([`887b46c`](https://github.com/guilatrova/gracy/commit/887b46ca3a61d20fcc942e18868a159ffaded0f1)) 442 | * Improve top description ([`e745403`](https://github.com/guilatrova/gracy/commit/e745403116483c651ebcd9f7e26fe99ab468ad03)) 443 | 444 | ## v0.6.0 (2023-01-29) 445 | ### Feature 446 | * Implement throttling ([`8691045`](https://github.com/guilatrova/gracy/commit/869104595b7c6954ea31b159e89a1efe8028215c)) 447 | 448 | ### Fix 449 | * **throttling:** Resolve bugs ([`4c41326`](https://github.com/guilatrova/gracy/commit/4c4132608b61256b8949dcbc46558641bccceedf)) 450 | * **throttling:** Handle some scenarios ([`f9d4fbc`](https://github.com/guilatrova/gracy/commit/f9d4fbc5c2e6e378cdfdd7dc8a930852f9620477)) 451 | 452 | ### Documentation 453 | * Improve prop description ([`27f9e01`](https://github.com/guilatrova/gracy/commit/27f9e01dd5004827a8df471034138ad1bf18b10c)) 454 | 455 | ## v0.5.0 (2023-01-29) 456 | ### Feature 457 | * Implement custom exceptions ([`2d89ebd`](https://github.com/guilatrova/gracy/commit/2d89ebd4c862c60bfc816774c3102c8e9e43ed2a)) 458 | * Implement retry pass ([`45e8ce6`](https://github.com/guilatrova/gracy/commit/45e8ce6124127ef69f5a9704a6ae0dc4a48d1f45)) 459 | 460 | ## v0.4.0 (2023-01-29) 461 | ### Feature 462 | * Implement parser ([`ab48cd9`](https://github.com/guilatrova/gracy/commit/ab48cd937cfa37e4455260defa94a8d41620f878)) 463 | 464 | ### Documentation 465 | * Add custom logo ([`19f6bf8`](https://github.com/guilatrova/gracy/commit/19f6bf86b4daf68ee50908cf2833912b0f3de852)) 466 | 467 | ## v0.3.0 (2023-01-29) 468 | ### Feature 469 | * Improve client customization ([`1372b4f`](https://github.com/guilatrova/gracy/commit/1372b4fb9ba7fc6d2c9b8f5e3064f4e2c9fd9ab5)) 470 | 471 | ## v0.2.0 (2023-01-28) 472 | ### Feature 473 | * Calculate footer totals ([`eb77c71`](https://github.com/guilatrova/gracy/commit/eb77c7138c50511cb1d4edfbd7c6f77b52ca6989)) 474 | * Sort table by requests made desc ([`fced5eb`](https://github.com/guilatrova/gracy/commit/fced5eb47dcc87abc97f2d91b1905140ad4d65d9)) 475 | * Add custom color to status ([`9964723`](https://github.com/guilatrova/gracy/commit/99647237155aa0f8b6d236ffbaa71d6d616c4ea7)) 476 | * Fold correct column ([`4a0bff0`](https://github.com/guilatrova/gracy/commit/4a0bff08a67e3c549897cac3ce9ec6d94603c2e7)) 477 | 478 | ## v0.1.0 (2023-01-28) 479 | ### Feature 480 | * Fold url column ([`a4b0ed0`](https://github.com/guilatrova/gracy/commit/a4b0ed0c1b2fe2b313c113e9ecdb6020b2f949a4)) 481 | * Display status range in metrics ([`8d01476`](https://github.com/guilatrova/gracy/commit/8d0147613c83c708064d05033ba1e7a24d3fa6cf)) 482 | * Add custom color to failed requests ([`65c9ab7`](https://github.com/guilatrova/gracy/commit/65c9ab7c2db0f90b1a3f48c4ab74eb2d3a96dd42)) 483 | * Add rich table to display metrics ([`44944f7`](https://github.com/guilatrova/gracy/commit/44944f7874f474fc33b4f532260a65720df0c051)) 484 | * Implement logs ([`9caee55`](https://github.com/guilatrova/gracy/commit/9caee5576f9d8cf3f9a17429b54e5dd26df9fb15)) 485 | * Add stub for report ([`b394afe`](https://github.com/guilatrova/gracy/commit/b394afe66a5fadb3c4831f2ceb75842b717465b4)) 486 | * Narrow down retry logic ([`e444281`](https://github.com/guilatrova/gracy/commit/e444281be9f0e8d9752e0ae847a768fddd1c1586)) 487 | * Make gracy async ([`5edacca`](https://github.com/guilatrova/gracy/commit/5edacca8781c02b7046a636020a7847faf716e8e)) 488 | * Implement retry ([`f0a794a`](https://github.com/guilatrova/gracy/commit/f0a794a40a6d351b02b516fa3a0004798a0710c2)) 489 | * Implement strict/allowed status code ([`171688b`](https://github.com/guilatrova/gracy/commit/171688b591c0c88b825f5ff1590f55c5cf0e1a9d)) 490 | 491 | ### Fix 492 | * Use enum value for _str_ ([`345464f`](https://github.com/guilatrova/gracy/commit/345464f44a48d864d5a39e56dfadf94f6f55da16)) 493 | 494 | ### Documentation 495 | * Reword some stuff ([`546f3fc`](https://github.com/guilatrova/gracy/commit/546f3fc6188c312196c9ca69a5fb80e172b6738f)) 496 | * Slightly improve readme ([`8a56b3d`](https://github.com/guilatrova/gracy/commit/8a56b3d961cf3ad343d7c95412ce49184e914608)) 497 | * Fill with some gracy stuff ([`8183d26`](https://github.com/guilatrova/gracy/commit/8183d2686f8f3a4cdfc50bf8e13465edfc54ef6d)) 498 | --------------------------------------------------------------------------------