├── .editorconfig ├── .github ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE │ ├── 01-bug-report.yml │ ├── 02-question.yml │ ├── 03-feature-request.yml │ └── config.yml ├── PULL_REQUEST_TEMPLATE.md ├── SUPPORT.md └── workflows │ ├── cron.yml │ ├── main.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── compose.yaml ├── docs ├── .pages ├── api.md ├── index.md └── tutorial.md ├── mkdocs.yml ├── pyproject.toml ├── src └── httpx_limiter │ ├── __init__.py │ ├── abstract_rate_limiter_repository.py │ ├── async_multi_rate_limited_transport.py │ ├── async_rate_limited_transport.py │ ├── py.typed │ └── rate.py └── tests ├── acceptance ├── test_async_multi_rate_limit.py └── test_async_rate_limit.py ├── conftest.py └── unit ├── .keep ├── test_abstract_rate_limiter_repository.py ├── test_async_rate_limited_transport.py └── test_rate.py /.editorconfig: -------------------------------------------------------------------------------- 1 | # Editor Configuration (http://editorconfig.org) 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | indent_style = space 7 | indent_size = 4 8 | end_of_line = lf 9 | insert_final_newline = true 10 | trim_trailing_whitespace = true 11 | max_line_length = 88 12 | 13 | [*.{json,yml,yaml}] 14 | indent_size = 2 15 | 16 | [*.{md,rst}] 17 | trim_trailing_whitespace = false 18 | 19 | [Makefile] 20 | indent_style = tab 21 | -------------------------------------------------------------------------------- /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | - Demonstrating empathy and kindness toward other people 21 | - Being respectful of differing opinions, viewpoints, and experiences 22 | - Giving and gracefully accepting constructive feedback 23 | - Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | - Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | - The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | - Trolling, insulting or derogatory comments, and personal or political attacks 33 | - Public or private harassment 34 | - Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | - Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | \[INSERT CONTACT METHOD\]. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | For answers to common questions about this code of conduct, see the FAQ at 125 | https://www.contributor-covenant.org/faq. Translations are available at 126 | https://www.contributor-covenant.org/translations. 127 | 128 | [homepage]: https://www.contributor-covenant.org 129 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Contributions are welcome, and they are greatly appreciated! Every little bit 4 | helps, and credit will always be given. 5 | 6 | ## Example Contributions 7 | 8 | You can contribute in many ways, for example: 9 | 10 | * [Report bugs](#report-bugs) 11 | * [Fix Bugs](#fix-bugs) 12 | * [Implement Features](#implement-features) 13 | * [Write Documentation](#write-documentation) 14 | * [Submit Feedback](#submit-feedback) 15 | 16 | ### Report Bugs 17 | 18 | Report bugs at https://github.com/Midnighter/httpx-limiter/issues. 19 | 20 | **If you are reporting a bug, please follow the template guidelines. The more 21 | detailed your report, the easier and thus faster we can help you.** 22 | 23 | ### Fix Bugs 24 | 25 | Look through the GitHub issues for bugs. Anything labelled with `bug` and `help 26 | wanted` is open to whoever wants to implement it. When you decide to work on 27 | such an issue, please [assign yourself to 28 | it](https://docs.github.com/en/issues/tracking-your-work-with-issues/assigning-issues-and-pull-requests-to-other-github-users) 29 | and add a comment that you'll be working on that, too. If you see another issue 30 | without the `help wanted` label, just post a comment, the maintainers are 31 | usually happy for any support that they can get. 32 | 33 | ### Implement Features 34 | 35 | Look through the GitHub issues for features. Anything labelled with 36 | `enhancement` and `help wanted` is open to whoever wants to implement it. As for 37 | [fixing bugs](#fix-bugs), please [assign yourself to the 38 | issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/assigning-issues-and-pull-requests-to-other-github-users) 39 | and add a comment that you'll be working on that, too. If another enhancement 40 | catches your fancy, but it doesn't have the `help wanted` label, just post a 41 | comment, the maintainers are usually happy for any support that they can get. 42 | 43 | ### Write Documentation 44 | 45 | HTTPX Limiter could always use more documentation, whether as 46 | part of the official documentation, in docstrings, or even on the web in blog 47 | posts, articles, and such. Just [open an issue](https://github.com/Midnighter/ 48 | httpx-limiter/issues) to let us know what you will be working on 49 | so that we can provide you with guidance. 50 | 51 | ### Submit Feedback 52 | 53 | The best way to send feedback is to file an issue at https://github.com/ 54 | Midnighter/httpx-limiter/issues. If your feedback fits the format of one of 55 | the issue templates, please use that. Remember that this is a volunteer-driven 56 | project and everybody has limited time. 57 | 58 | ## Get Started! 59 | 60 | Ready to contribute? Here's how to set up HTTPX Limiter for 61 | local development. 62 | 63 | 1. Fork the https://github.com/Midnighter/httpx-limiter 64 | repository on GitHub. 65 | 2. Clone your fork locally 66 | 67 | ```shell 68 | git clone git@github.com:your_name_here/httpx-limiter.git 69 | ``` 70 | 71 | 3. [Install hatch](https://hatch.pypa.io/latest/install/). 72 | 73 | 4. Create a branch for local development using the default branch (typically `main`) 74 | as a starting 75 | point. Use `fix` or `feat` as a prefix for your branch name. 76 | 77 | ```shell 78 | git checkout main 79 | git checkout -b fix-name-of-your-bugfix 80 | ``` 81 | 82 | Now you can make your changes locally. 83 | 84 | 5. When you're done making changes, apply the quality assurance tools and check 85 | that your changes pass our test suite. This is all included with tox 86 | 87 | ```shell 88 | hatch run test:run 89 | ``` 90 | 91 | 6. Commit your changes and push your branch to GitHub. Please use [semantic 92 | commit messages](https://www.conventionalcommits.org/). 93 | 94 | ```shell 95 | git add . 96 | git commit -m "fix: summarize your changes" 97 | git push -u origin fix-name-of-your-bugfix 98 | ``` 99 | 100 | 7. Open the link displayed in the message when pushing your new branch in order 101 | to submit a pull request. 102 | 103 | ### Pull Request Guidelines 104 | 105 | Before you submit a pull request, check that it meets these guidelines: 106 | 107 | 1. The pull request should include tests. 108 | 2. If the pull request adds functionality, the docs should be updated. Put your 109 | new functionality into a function with a docstring. 110 | 3. Your pull request will automatically be checked by the full test suite. 111 | It needs to pass all of them before it can be considered for merging. 112 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/01-bug-report.yml: -------------------------------------------------------------------------------- 1 | name: 🐞 Bug report 2 | description: Report a problem to help improve this project 3 | title: "[BUG] " 4 | labels: [bug, triage] 5 | body: 6 | - type: checkboxes 7 | attributes: 8 | label: Is there an existing issue for this? 9 | description: Please search to see if an issue already exists for the bug you encountered. 10 | options: 11 | - label: I have searched the existing issues 12 | required: true 13 | 14 | - type: textarea 15 | attributes: 16 | label: Problem description 17 | description: | 18 | A concise description of what you're experiencing. 19 | 20 | Please explain: 21 | 22 | * **what** you tried to achieve, 23 | * **how** you went about it (referring to the code sample), and 24 | * **why** the current behaviour is a problem and what output you expected instead. 25 | validations: 26 | required: false 27 | 28 | - type: textarea 29 | attributes: 30 | label: Code sample 31 | description: > 32 | Create a [minimal, complete, verifiable example](https://stackoverflow.com/help/mcve). 33 | Please, paste your code between the ``` tickmarks below or link to a [gist](https://gist.github.com/). 34 | value: | 35 | Code run: 36 | 37 | ```python 38 | ``` 39 | 40 | Traceback: 41 | 42 | ```text 43 | ``` 44 | validations: 45 | required: false 46 | 47 | - type: textarea 48 | attributes: 49 | label: Environment 50 | description: > 51 | Please paste the output of running `depinfo --markdown httpx_limiter` 52 | in your environment between the `details` tags below. 53 | value: | 54 |
55 | 56 |
57 | validations: 58 | required: true 59 | 60 | - type: textarea 61 | attributes: 62 | label: Anything else? 63 | description: | 64 | Links? References? Anything that will give us more context about the issue you are encountering! 65 | 66 | Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. 67 | validations: 68 | required: false 69 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/02-question.yml: -------------------------------------------------------------------------------- 1 | name: Question 2 | description: Ask a question 3 | title: "[Question] " 4 | labels: [question] 5 | body: 6 | - type: checkboxes 7 | attributes: 8 | label: Checklist 9 | description: > 10 | To help keep this issue tracker clean and focused, please make sure that you have 11 | tried **all** of the following resources before submitting your question. 12 | options: 13 | - label: I searched the [documentation](https://httpx-limiter.readthedocs.io). 14 | required: true 15 | - label: I looked through existing [discussion topics](https://github.com/Midnighter/httpx-limiter/discussions). 16 | required: true 17 | - label: I looked for [similar issues](https://github.com/Midnighter/httpx-limiter/issues). 18 | required: true 19 | - label: I looked up my question/problem in a search engine. 20 | required: true 21 | 22 | - type: textarea 23 | attributes: 24 | label: Question 25 | description: Please ask your question 26 | validations: 27 | required: true 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/03-feature-request.yml: -------------------------------------------------------------------------------- 1 | name: Feature request 2 | description: Suggest an idea for this project 3 | title: "[Feature] " 4 | labels: [enhancement] 5 | body: 6 | - type: checkboxes 7 | attributes: 8 | label: Checklist 9 | description: > 10 | Please make sure you check all these items before submitting your feature request. 11 | options: 12 | - label: There are [no similar issues or pull requests](https://github.com/Midnighter/httpx-limiter/issues) for this yet. 13 | required: true 14 | 15 | - type: textarea 16 | attributes: 17 | label: Problem 18 | description: > 19 | A clear and concise description of what you are trying to achieve. 20 | placeholder: > 21 | "I want to be able to [...] but I can't because [...]". 22 | validations: 23 | required: false 24 | 25 | - type: textarea 26 | attributes: 27 | label: Solution 28 | description: > 29 | A clear and concise description of what you would want to happen. 30 | For API changes, try to provide a code snippet of what you would like the new API to look like. 31 | validations: 32 | required: false 33 | 34 | - type: textarea 35 | attributes: 36 | label: Alternatives 37 | description: > 38 | Please describe any alternative solutions or features you've considered to solve 39 | your problem and why they didn't help. 40 | validations: 41 | required: false 42 | 43 | - type: textarea 44 | attributes: 45 | label: Anything else? 46 | description: > 47 | Provide any additional context, screenshots, tracebacks, etc. about the feature here. 48 | validations: 49 | required: false 50 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | contact_links: 3 | - name: Community Support 4 | url: https://github.com/Midnighter/httpx-limiter/discussions 5 | about: Please ask and answer questions here. 6 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | - [ ] fix #(issue number) 2 | - [ ] description of feature/fix 3 | - [ ] tests added/passed 4 | - [ ] add an entry to the [changelog](../CHANGELOG.md) 5 | -------------------------------------------------------------------------------- /.github/SUPPORT.md: -------------------------------------------------------------------------------- 1 | # Support 2 | 3 | * Start a [discussion topic](https://github.com/Midnighter/httpx-limiter/discussions) 4 | * Email the authors or maintainers 5 | -------------------------------------------------------------------------------- /.github/workflows/cron.yml: -------------------------------------------------------------------------------- 1 | name: Cron Test 2 | 3 | on: 4 | schedule: 5 | # Run every Tuesday at 10:30. 6 | - cron: '30 10 * * 2' 7 | 8 | jobs: 9 | prerequisites: 10 | uses: ./.github/workflows/test.yml 11 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - dev 8 | pull_request: 9 | branches: 10 | - main 11 | - dev 12 | 13 | jobs: 14 | prerequisites: 15 | uses: ./.github/workflows/test.yml 16 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: CD 2 | 3 | on: 4 | push: 5 | tags: 6 | - '[0-9]+.[0-9]+.[0-9]+' 7 | - '[0-9]+.[0-9]+.[0-9]+(a|b|rc|post|dev)[0-9]+' 8 | 9 | jobs: 10 | prerequisites: 11 | uses: ./.github/workflows/test.yml 12 | 13 | release: 14 | needs: [prerequisites] 15 | strategy: 16 | matrix: 17 | os: [ubuntu-latest] 18 | python-version: ["3.12"] 19 | runs-on: ${{ matrix.os }} 20 | permissions: 21 | # Write permissions are needed to create OIDC tokens. 22 | id-token: write 23 | # Write permissions are needed to make GitHub releases. 24 | contents: write 25 | 26 | steps: 27 | - uses: actions/checkout@v4 28 | 29 | - name: Set up Python ${{ matrix.python-version }} 30 | uses: actions/setup-python@v5 31 | with: 32 | python-version: ${{ matrix.python-version }} 33 | 34 | - name: Install hatch 35 | uses: pypa/hatch@install 36 | 37 | - name: Build package 38 | run: hatch build 39 | 40 | # We rely on a trusted publisher configuration being present on PyPI, 41 | # see https://docs.pypi.org/trusted-publishers/. 42 | - name: Publish to PyPI 43 | uses: pypa/gh-action-pypi-publish@release/v1 44 | 45 | - name: GH release 46 | uses: softprops/action-gh-release@v2 47 | with: 48 | body: > 49 | Please see 50 | https://github.com/${{ github.repository }}/blob/${{ github.ref_name }}/CHANGELOG.md 51 | for the full release notes. 52 | draft: false 53 | prerelease: false 54 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test Suite 2 | 3 | on: 4 | workflow_dispatch: {} 5 | workflow_call: {} 6 | 7 | jobs: 8 | lint: 9 | runs-on: ${{ matrix.os }} 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | os: [ubuntu-latest] 14 | python-version: ["3.12"] 15 | 16 | steps: 17 | - uses: actions/checkout@v4 18 | 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v5 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | 24 | - name: Install hatch 25 | uses: pypa/hatch@install 26 | 27 | - name: Check code style 28 | run: hatch run style:code 29 | 30 | - name: Check docstrings 31 | run: hatch run style:docstrings 32 | 33 | - name: Build documentation 34 | run: hatch run docs:build 35 | 36 | unit-tests: 37 | runs-on: ${{ matrix.os }} 38 | strategy: 39 | fail-fast: false 40 | matrix: 41 | os: [ubuntu-latest, macos-latest, windows-latest] 42 | python-version: ["3.10", "3.13"] 43 | 44 | steps: 45 | - uses: actions/checkout@v4 46 | 47 | - name: Set up Python ${{ matrix.python-version }} 48 | uses: actions/setup-python@v5 49 | with: 50 | python-version: ${{ matrix.python-version }} 51 | 52 | - name: Install hatch 53 | uses: pypa/hatch@install 54 | 55 | # - name: Check installation 56 | # if: matrix.os != 'macos-latest' 57 | # run: hatch run install:check 58 | 59 | - name: Check dependencies 60 | run: hatch run audit:check 61 | 62 | - name: Check types 63 | run: hatch run types:check 64 | 65 | - name: Run unit tests 66 | run: hatch run +py=${{ matrix.python-version }} unit-tests:run -- --cov=httpx_limiter --cov-report=term-missing --cov-report=xml 67 | 68 | - name: Report coverage 69 | shell: bash 70 | run: bash <(curl -s https://codecov.io/bash) 71 | 72 | acceptance-tests: 73 | runs-on: ${{ matrix.os }} 74 | strategy: 75 | fail-fast: false 76 | matrix: 77 | os: [ubuntu-latest] 78 | python-version: ["3.13"] 79 | 80 | steps: 81 | - uses: actions/checkout@v4 82 | 83 | - name: Set up Python ${{ matrix.python-version }} 84 | uses: actions/setup-python@v5 85 | with: 86 | python-version: ${{ matrix.python-version }} 87 | 88 | - name: Install hatch 89 | uses: pypa/hatch@install 90 | 91 | - name: Start rate-limited APIs 92 | uses: hoverkraft-tech/compose-action@v2 93 | with: 94 | compose-file: "compose.yaml" 95 | 96 | - name: Run acceptance tests 97 | run: hatch run +py=${{ matrix.python-version }} acceptance-tests:run -- --cov=httpx_limiter --cov-report=term-missing --cov-report=xml 98 | 99 | - name: Report coverage 100 | shell: bash 101 | run: bash <(curl -s https://codecov.io/bash) 102 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### Linux template 2 | *~ 3 | 4 | # temporary files which can be created if a process still has a handle open of a deleted file 5 | .fuse_hidden* 6 | 7 | # KDE directory preferences 8 | .directory 9 | 10 | # Linux trash folder which might appear on any partition or disk 11 | .Trash-* 12 | 13 | # .nfs files are created when an open file is removed but is still being accessed 14 | .nfs* 15 | 16 | ### Windows template 17 | # Windows thumbnail cache files 18 | Thumbs.db 19 | Thumbs.db:encryptable 20 | ehthumbs.db 21 | ehthumbs_vista.db 22 | 23 | # Dump file 24 | *.stackdump 25 | 26 | # Folder config file 27 | [Dd]esktop.ini 28 | 29 | # Recycle Bin used on file shares 30 | $RECYCLE.BIN/ 31 | 32 | # Windows Installer files 33 | *.cab 34 | *.msi 35 | *.msix 36 | *.msm 37 | *.msp 38 | 39 | # Windows shortcuts 40 | *.lnk 41 | 42 | ### JupyterNotebooks template 43 | # gitignore template for Jupyter Notebooks 44 | # website: http://jupyter.org/ 45 | 46 | .ipynb_checkpoints 47 | */.ipynb_checkpoints/* 48 | 49 | # IPython 50 | profile_default/ 51 | ipython_config.py 52 | 53 | # Remove previous ipynb_checkpoints 54 | # git rm -r .ipynb_checkpoints/ 55 | 56 | ### macOS template 57 | # General 58 | .DS_Store 59 | .AppleDouble 60 | .LSOverride 61 | 62 | # Icon must end with two \r 63 | Icon 64 | 65 | # Thumbnails 66 | ._* 67 | 68 | # Files that might appear in the root of a volume 69 | .DocumentRevisions-V100 70 | .fseventsd 71 | .Spotlight-V100 72 | .TemporaryItems 73 | .Trashes 74 | .VolumeIcon.icns 75 | .com.apple.timemachine.donotpresent 76 | 77 | # Directories potentially created on remote AFP share 78 | .AppleDB 79 | .AppleDesktop 80 | Network Trash Folder 81 | Temporary Items 82 | .apdisk 83 | 84 | ### Python template 85 | # Byte-compiled / optimized / DLL files 86 | __pycache__/ 87 | *.py[cod] 88 | *$py.class 89 | 90 | # C extensions 91 | *.so 92 | 93 | # Distribution / packaging 94 | .Python 95 | build/ 96 | develop-eggs/ 97 | dist/ 98 | downloads/ 99 | eggs/ 100 | .eggs/ 101 | lib/ 102 | lib64/ 103 | parts/ 104 | sdist/ 105 | var/ 106 | wheels/ 107 | share/python-wheels/ 108 | *.egg-info/ 109 | .installed.cfg 110 | *.egg 111 | MANIFEST 112 | 113 | # PyInstaller 114 | # Usually these files are written by a python script from a template 115 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 116 | *.manifest 117 | *.spec 118 | 119 | # Installer logs 120 | pip-log.txt 121 | pip-delete-this-directory.txt 122 | 123 | # Unit test / coverage reports 124 | htmlcov/ 125 | .tox/ 126 | .nox/ 127 | .coverage 128 | .coverage.* 129 | .cache 130 | nosetests.xml 131 | coverage.xml 132 | *.cover 133 | *.py,cover 134 | .hypothesis/ 135 | .pytest_cache/ 136 | cover/ 137 | 138 | # Translations 139 | *.mo 140 | *.pot 141 | 142 | # Django stuff: 143 | *.log 144 | local_settings.py 145 | db.sqlite3 146 | db.sqlite3-journal 147 | 148 | # Flask stuff: 149 | instance/ 150 | .webassets-cache 151 | 152 | # Scrapy stuff: 153 | .scrapy 154 | 155 | # Sphinx documentation 156 | docs/_build/ 157 | 158 | # PyBuilder 159 | .pybuilder/ 160 | target/ 161 | 162 | # Jupyter Notebook 163 | 164 | # IPython 165 | 166 | # pyenv 167 | # For a library or package, you might want to ignore these files since the code is 168 | # intended to run in multiple environments; otherwise, check them in: 169 | # .python-version 170 | 171 | # pipenv 172 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 173 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 174 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 175 | # install all needed dependencies. 176 | #Pipfile.lock 177 | 178 | # poetry 179 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 180 | # This is especially recommended for binary packages to ensure reproducibility, and is more 181 | # commonly ignored for libraries. 182 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 183 | #poetry.lock 184 | 185 | # pdm 186 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 187 | #pdm.lock 188 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 189 | # in version control. 190 | # https://pdm.fming.dev/#use-with-ide 191 | .pdm.toml 192 | 193 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 194 | __pypackages__/ 195 | 196 | # Celery stuff 197 | celerybeat-schedule 198 | celerybeat.pid 199 | 200 | # SageMath parsed files 201 | *.sage.py 202 | 203 | # Environments 204 | .env 205 | .venv 206 | env/ 207 | venv/ 208 | ENV/ 209 | env.bak/ 210 | venv.bak/ 211 | 212 | # Spyder project settings 213 | .spyderproject 214 | .spyproject 215 | 216 | # Rope project settings 217 | .ropeproject 218 | 219 | # mkdocs documentation 220 | /site 221 | 222 | # mypy 223 | .mypy_cache/ 224 | .dmypy.json 225 | dmypy.json 226 | 227 | # Pyre type checker 228 | .pyre/ 229 | 230 | # pytype static type analyzer 231 | .pytype/ 232 | 233 | # Cython debug symbols 234 | cython_debug/ 235 | 236 | # PyCharm 237 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 238 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 239 | # and can be added to the global gitignore or merged into this file. For a more nuclear 240 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 241 | #.idea/ 242 | 243 | # Hatch-VCS 244 | _version.py 245 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v5.0.0 4 | hooks: 5 | - id: check-case-conflict 6 | - id: check-ast 7 | - id: check-docstring-first 8 | - id: check-executables-have-shebangs 9 | - id: check-added-large-files 10 | - id: check-case-conflict 11 | - id: check-merge-conflict 12 | - id: check-json 13 | - id: check-toml 14 | - id: check-yaml 15 | - id: debug-statements 16 | - id: end-of-file-fixer 17 | - id: trailing-whitespace 18 | - repo: https://github.com/astral-sh/ruff-pre-commit 19 | rev: v0.9.4 20 | hooks: 21 | - id: ruff 22 | args: ["--fix", "--exit-non-zero-on-fix"] 23 | - id: ruff-format 24 | - repo: https://github.com/jsh9/pydoclint 25 | rev: 0.6.0 26 | hooks: 27 | - id: pydoclint 28 | - repo: https://github.com/python-jsonschema/check-jsonschema 29 | rev: 0.31.1 30 | hooks: 31 | 32 | - id: check-github-workflows 33 | 34 | - id: check-readthedocs 35 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | version: 2 4 | 5 | build: 6 | os: ubuntu-lts-latest 7 | tools: 8 | python: "3.12" 9 | commands: 10 | - pip install --upgrade pip setuptools wheel 11 | - pip install hatch 12 | - hatch run docs:build -- --strict --site-dir "${READTHEDOCS_OUTPUT}/html" 13 | 14 | mkdocs: 15 | configuration: mkdocs.yml 16 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | 10 | ## [0.3.0] - (2025-05-10) 11 | 12 | ### Added 13 | 14 | - Added an `AbstractRateLimiterRepository` and `AsyncMultiRateLimitedTransport` allowing 15 | for multiple rate limits based on the request. 16 | 17 | ## [0.2.0] - (2025-04-04) 18 | 19 | ### Added 20 | 21 | - Added a `Rate` value object to the public interface. 22 | 23 | ### Changed 24 | 25 | - Replaced the `limiter` dependency with `aiolimiter`. 26 | 27 | ## [0.1.0] - (2024-11-22) 28 | 29 | - First release 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | https://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HTTPX Limiter 2 | 3 | | | | 4 | |---|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 5 | | Package | [![Latest PyPI Version](https://img.shields.io/pypi/v/httpx-limiter.svg)](https://pypi.org/project/httpx-limiter/) [![Supported Python Versions](https://img.shields.io/pypi/pyversions/httpx-limiter.svg)](https://pypi.org/project/httpx-limiter/) | 6 | | Meta | [![Apache-2.0](https://img.shields.io/pypi/l/httpx-limiter.svg)](LICENSE) [![Code of Conduct](https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg)](.github/CODE_OF_CONDUCT.md) [![Checked with mypy](https://www.mypy-lang.org/static/mypy_badge.svg)](https://mypy-lang.org/) [![Code Style Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) [![Linting: Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) | 7 | | Automation | [![CI](https://github.com/Midnighter/httpx-limiter/actions/workflows/main.yml/badge.svg)](https://github.com/Midnighter/httpx-limiter/actions/workflows/main.yml) | 8 | 9 | _A lightweight package that provides rate-limited httpx transports._ 10 | 11 | ## Installation 12 | 13 | The package is published on [PyPI](https://pypi.org/project/httpx-limiter/). 14 | Install it, for example, with 15 | 16 | ```sh 17 | pip install httpx-limiter 18 | ``` 19 | 20 | ## Tutorial 21 | 22 | You can limit the number of requests made by an HTTPX client using the 23 | transports provided in this package. That is useful in situations when you need 24 | to make a large number of asynchronous requests against endpoints that implement 25 | a rate limit. 26 | 27 | ### Single Rate Limit 28 | 29 | The simplest use case is to apply a single rate limit to all requests. If you 30 | want to be able to make twenty requests per second, for example, use the 31 | following code: 32 | 33 | ```python 34 | import httpx 35 | from httpx_limiter import AsyncRateLimitedTransport, Rate 36 | 37 | async def main(): 38 | async with httpx.AsyncClient( 39 | transport=AsyncRateLimitedTransport.create(rate=Rate.create(magnitude=20)), 40 | ) as client: 41 | response = await client.get("https://httpbin.org") 42 | ``` 43 | 44 | > [!IMPORTANT] 45 | > Due to limitations in the design of the underlying [leaky 46 | > bucket](https://en.wikipedia.org/wiki/Leaky_bucket) implementation, which is 47 | > used to implement the rate limiting, the magnitude of the rate is also the 48 | > maximum capacity of the bucket. That means, if you set a rate that is larger 49 | > than one, a burst of requests equal to that capacity will be allowed. If you 50 | > do not want to allow any bursts, set the magnitude to one, but the duration to 51 | > the inverse of your desired rate. If you want to allow twenty requests per 52 | > second, for example, set the magnitude to 1 and the duration to 0.05 seconds. 53 | > 54 | > ```python 55 | > Rate.create(magnitude=1, duration=1/20) 56 | > ``` 57 | 58 | ### Multiple Rate Limits 59 | 60 | For more advanced use cases, you can apply different rate limits based on a 61 | concrete implementation of the `AbstractRateLimiterRepository`. There are two 62 | relevant methods that both get passed the current request. One method needs to 63 | identify which rate limit to apply, and the other method sets the rate limit 64 | itself. See the following example: 65 | 66 | ```python 67 | import httpx 68 | from httpx_limiter import ( 69 | AbstractRateLimiterRepository, 70 | AsyncMultiRateLimitedTransport, 71 | Rate 72 | ) 73 | 74 | class DomainBasedRateLimiterRepository(AbstractRateLimiterRepository): 75 | """Apply different rate limits based on the domain being requested.""" 76 | 77 | def get_identifier(self, request: httpx.Request) -> str: 78 | """Return the domain as the identifier for rate limiting.""" 79 | return request.url.host 80 | 81 | def get_rate(self, request: httpx.Request) -> Rate: 82 | """Apply the same, but independent rate limit to each domain.""" 83 | return Rate.create(magnitude=25) 84 | 85 | client = httpx.AsyncClient( 86 | transport=AsyncMultiRateLimitedTransport.create( 87 | repository=DomainBasedRateLimiterRepository(), 88 | ), 89 | ) 90 | ``` 91 | 92 | > [!TIP] 93 | > You are free to ignore the request parameter and use global information like 94 | > the time of day to determine the rate limit. 95 | 96 | ```python 97 | from datetime import datetime, timezone 98 | 99 | import httpx 100 | from httpx_limiter import AbstractRateLimiterRepository, Rate 101 | 102 | class DayNightRateLimiterRepository(AbstractRateLimiterRepository): 103 | """Apply different rate limits based on the time of day.""" 104 | 105 | def get_identifier(self, _: httpx.Request) -> str: 106 | """Identify whether it is currently day or night.""" 107 | if 6 <= datetime.now(tz=timezone.utc).hour < 18: 108 | return "day" 109 | 110 | return "night" 111 | 112 | def get_rate(self, _: httpx.Request) -> Rate: 113 | """Apply different rate limits during the day or night.""" 114 | if self.get_identifier(_) == "day": 115 | return Rate.create(magnitude=10) 116 | 117 | return Rate.create(magnitude=100) 118 | ``` 119 | 120 | ## Copyright 121 | 122 | - Copyright © 2024, 2025 Moritz E. Beber. 123 | - Free software distributed under the [Apache Software License 2.0](./LICENSE). 124 | -------------------------------------------------------------------------------- /compose.yaml: -------------------------------------------------------------------------------- 1 | networks: 2 | public: 3 | name: public 4 | socket: 5 | internal: true 6 | 7 | services: 8 | socket-proxy: 9 | image: tecnativa/docker-socket-proxy:latest 10 | networks: 11 | - socket 12 | volumes: 13 | - /var/run/docker.sock:/var/run/docker.sock:ro 14 | environment: 15 | NETWORKS: 1 16 | CONTAINERS: 1 17 | 18 | gateway: 19 | command: > 20 | --accessLog=true 21 | --api.insecure=true 22 | --log.level=INFO 23 | --accessLog.format=json 24 | --entryPoints.web.address=:80 25 | --providers.docker=true 26 | --providers.docker.endpoint=tcp://socket-proxy:2375 27 | --providers.docker.exposedByDefault=false 28 | --providers.docker.network=public 29 | image: traefik:3 30 | networks: 31 | - public 32 | - socket 33 | - default 34 | ports: 35 | - target: 80 36 | published: 80 37 | mode: host 38 | - target: 8080 39 | published: 8080 40 | mode: host 41 | 42 | httpbin: 43 | image: kennethreitz/httpbin:latest 44 | labels: 45 | - "traefik.enable=true" 46 | - "traefik.http.routers.httpbin.rule=Host(`httpbin.localhost`)" 47 | - "traefik.http.routers.httpbin.entrypoints=web" 48 | - "traefik.http.services.httpbin.loadbalancer.server.port=80" 49 | - "traefik.http.middlewares.httpbin-limiter.ratelimit.average=20" 50 | - "traefik.http.middlewares.httpbin-limiter.ratelimit.burst=1" 51 | - "traefik.http.routers.httpbin.middlewares=httpbin-limiter@docker" 52 | networks: 53 | - public 54 | 55 | fast: 56 | image: kennethreitz/httpbin:latest 57 | labels: 58 | - "traefik.enable=true" 59 | - "traefik.http.routers.fast.rule=Host(`fast.localhost`)" 60 | - "traefik.http.routers.fast.entrypoints=web" 61 | - "traefik.http.services.fast.loadbalancer.server.port=80" 62 | - "traefik.http.middlewares.fast-limiter.ratelimit.average=40" 63 | - "traefik.http.middlewares.fast-limiter.ratelimit.burst=1" 64 | - "traefik.http.routers.fast.middlewares=fast-limiter@docker" 65 | networks: 66 | - public 67 | -------------------------------------------------------------------------------- /docs/.pages: -------------------------------------------------------------------------------- 1 | nav: 2 | - Welcome: index.md 3 | - Tutorial: tutorial.md 4 | - Reference API: api.md 5 | -------------------------------------------------------------------------------- /docs/api.md: -------------------------------------------------------------------------------- 1 | The following classes are exposed to the user: 2 | 3 | * [`Rate`][httpx_limiter.Rate] 4 | * [`AsyncRateLimitedTransport`][httpx_limiter.AsyncRateLimitedTransport] 5 | * [`AbstractRateLimiterRepository`][httpx_limiter.AbstractRateLimiterRepository] 6 | * [`AsyncMultiRateLimitedTransport`][httpx_limiter.AsyncMultiRateLimitedTransport] 7 | 8 | ::: httpx_limiter 9 | options: 10 | members: 11 | - Rate 12 | - AsyncRateLimitedTransport 13 | - AbstractRateLimiterRepository 14 | - AsyncMultiRateLimitedTransport 15 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # HTTPX Limiter 2 | 3 | A lightweight package that provides rate-limited httpx transports. 4 | 5 | ## Installation 6 | 7 | The package is published on [PyPI](https://pypi.org/project/httpx-limiter/). 8 | Install it, for example, with 9 | 10 | ```sh 11 | pip install httpx-limiter 12 | ``` 13 | 14 | ## Copyright 15 | 16 | - Copyright © 2024, 2025 Moritz E. Beber. 17 | - Free software distributed under the [Apache Software License 18 | 2.0](https://www.apache.org/licenses/LICENSE-2.0.html). 19 | 20 | -------------------------------------------------------------------------------- /docs/tutorial.md: -------------------------------------------------------------------------------- 1 | You can limit the number of requests made by an HTTPX client using the 2 | transports provided in this package. That is useful in situations when you need 3 | to make a large number of asynchronous requests against endpoints that implement 4 | a rate limit. 5 | 6 | ### Single Rate Limit 7 | 8 | The simplest use case is to apply a single rate limit to all requests. If you 9 | want to be able to make twenty requests per second, for example, use the 10 | following code: 11 | 12 | ```python 13 | import httpx 14 | from httpx_limiter import AsyncRateLimitedTransport, Rate 15 | 16 | async def main(): 17 | async with httpx.AsyncClient( 18 | transport=AsyncRateLimitedTransport.create(rate=Rate.create(magnitude=20)), 19 | ) as client: 20 | response = await client.get("https://httpbin.org") 21 | ``` 22 | 23 | !!! warning 24 | 25 | Due to limitations in the design of the underlying [leaky 26 | bucket](https://en.wikipedia.org/wiki/Leaky_bucket) implementation, which is 27 | used to implement the rate limiting, the magnitude of **the rate is also the 28 | maximum capacity of the bucket**. That means, if you **set a rate that is 29 | larger than one, a burst of requests equal to that capacity will be 30 | allowed**. If you do not want to allow any bursts, set the magnitude to one, 31 | but the duration to the inverse of your desired rate. If you want to allow 32 | twenty requests per second, for example, set the magnitude to 1 and the 33 | duration to 0.05 seconds. 34 | 35 | 36 | ```python 37 | Rate.create(magnitude=1, duration=1/20) 38 | ``` 39 | 40 | 41 | ### Multiple Rate Limits 42 | 43 | For more advanced use cases, you can apply different rate limits based on a 44 | concrete implementation of the 45 | [`AbstractRateLimiterRepository`][httpx_limiter.AbstractRateLimiterRepository]. 46 | There are two relevant methods that both get passed the current request. One 47 | method needs to identify which rate limit to apply, and the other method sets 48 | the rate limit itself. See the following example: 49 | 50 | ```python 51 | import httpx 52 | from httpx_limiter import ( 53 | AbstractRateLimiterRepository, 54 | AsyncMultiRateLimitedTransport, 55 | Rate 56 | ) 57 | 58 | 59 | class DomainBasedRateLimiterRepository(AbstractRateLimiterRepository): 60 | """Apply different rate limits based on the domain being requested.""" 61 | 62 | def get_identifier(self, request: httpx.Request) -> str: 63 | """Return the domain as the identifier for rate limiting.""" 64 | return request.url.host 65 | 66 | def get_rate(self, request: httpx.Request) -> Rate: 67 | """Apply the same, but independent rate limit to each domain.""" 68 | return Rate.create(magnitude=25) 69 | 70 | 71 | client = httpx.AsyncClient( 72 | transport=AsyncMultiRateLimitedTransport.create( 73 | repository=DomainBasedRateLimiterRepository(), 74 | ), 75 | ) 76 | ``` 77 | 78 | !!! tip 79 | 80 | You are free to ignore the request parameter and use global information like 81 | the time of day to determine the rate limit. 82 | 83 | ```python 84 | from datetime import datetime, timezone 85 | 86 | import httpx 87 | from httpx_limiter import AbstractRateLimiterRepository, Rate 88 | 89 | 90 | class DayNightRateLimiterRepository(AbstractRateLimiterRepository): 91 | """Apply different rate limits based on the time of day.""" 92 | 93 | def get_identifier(self, _: httpx.Request) -> str: 94 | """Identify whether it is currently day or night.""" 95 | if 6 <= datetime.now(tz=timezone.utc).hour < 18: 96 | return "day" 97 | 98 | return "night" 99 | 100 | def get_rate(self, _: httpx.Request) -> Rate: 101 | """Apply different rate limits during the day or night.""" 102 | if self.get_identifier(_) == "day": 103 | return Rate.create(magnitude=10) 104 | 105 | return Rate.create(magnitude=100) 106 | ``` 107 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: HTTPX Limiter 2 | site_description: A lightweight package that provides rate-limited httpx transports. 3 | site_author: Moritz E. Beber 4 | site_url: https://midnighter.github.io/httpx-limiter/ 5 | 6 | markdown_extensions: 7 | - admonition 8 | - toc: 9 | permalink: true 10 | - pymdownx.highlight: 11 | anchor_linenums: true 12 | - pymdownx.inlinehilite 13 | - pymdownx.superfences: 14 | - pymdownx.emoji: 15 | emoji_index: !!python/name:material.extensions.emoji.twemoji 16 | emoji_generator: !!python/name:material.extensions.emoji.to_svg 17 | 18 | theme: 19 | features: 20 | - content.code.annotate 21 | - navigation.indexes 22 | - navigation.instant 23 | - navigation.tracking 24 | - navigation.top 25 | name: material 26 | palette: 27 | - media: "(prefers-color-scheme: light)" 28 | scheme: default 29 | primary: deep purple 30 | accent: teal 31 | toggle: 32 | icon: material/weather-night 33 | name: Switch to dark mode 34 | - media: "(prefers-color-scheme: dark)" 35 | scheme: slate 36 | primary: deep purple 37 | accent: teal 38 | toggle: 39 | icon: material/weather-sunny 40 | name: Switch to light mode 41 | 42 | plugins: 43 | - search 44 | - mkdocstrings: 45 | handlers: 46 | python: 47 | paths: [src] 48 | options: 49 | show_submodules: true 50 | show_if_no_docstring: true 51 | show_signature_annotations: true 52 | show_root_heading: true 53 | show_source: false 54 | default_handler: python 55 | - awesome-pages 56 | 57 | extra: 58 | version: 59 | provider: mike 60 | default: stable 61 | alias: true 62 | social: 63 | - icon: fontawesome/brands/github 64 | link: https://github.com/Midnighter 65 | name: Moritz E. Beber 66 | 67 | copyright: Copyright © 2024, 2025 Moritz E. Beber 68 | 69 | repo_url: https://github.com/Midnighter/httpx-limiter 70 | repo_name: Midnighter/httpx-limiter 71 | 72 | watch: 73 | - src/httpx_limiter 74 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Build Configuration 3 | ################################################################################ 4 | 5 | [build-system] 6 | build-backend = "hatchling.build" 7 | requires = ["hatchling", "hatch-vcs"] 8 | 9 | ################################################################################ 10 | # Project Configuration 11 | ################################################################################ 12 | 13 | [project] 14 | name = "httpx-limiter" 15 | dynamic = ["version"] 16 | description = "A lightweight package that provides rate-limited httpx transports." 17 | authors = [ 18 | { name = "Moritz E. Beber", email = "midnighter@posteo.net" }, 19 | ] 20 | license = "Apache-2.0" 21 | readme = {"file" = "README.md", "content-type" = "text/markdown"} 22 | requires-python = ">=3.10" 23 | # Please consult https://pypi.org/classifiers/ for a full list. 24 | classifiers = [ 25 | "Development Status :: 2 - Pre-Alpha", 26 | "Environment :: Web Environment", 27 | "Framework :: AsyncIO", 28 | "Intended Audience :: Developers", 29 | "License :: OSI Approved :: Apache Software License", 30 | "Natural Language :: English", 31 | "Operating System :: OS Independent", 32 | "Programming Language :: Python :: 3 :: Only", 33 | "Programming Language :: Python :: 3.10", 34 | "Programming Language :: Python :: 3.11", 35 | "Programming Language :: Python :: 3.12", 36 | "Programming Language :: Python :: 3.13", 37 | "Topic :: Internet :: WWW/HTTP", 38 | "Typing :: Typed", 39 | ] 40 | keywords = [ 41 | "httpx", 42 | "limiter", 43 | "rate-limit", 44 | "leaky bucket", 45 | ] 46 | dependencies = [ 47 | "httpx ~=0.25", 48 | "aiolimiter ~=1.2", 49 | ] 50 | 51 | [project.urls] 52 | Homepage = "https://github.com/Midnighter/httpx-limiter" 53 | Documentation = "https://httpx-limiter.readthedocs.io" 54 | "Source Code" = "https://github.com/Midnighter/httpx-limiter" 55 | "Bug Tracker" = "https://github.com/Midnighter/httpx-limiter/issues" 56 | Download = "https://pypi.org/project/httpx-limiter/#files" 57 | 58 | ################################################################################ 59 | # Tool Configuration 60 | ################################################################################ 61 | 62 | [tool.hatch.build] 63 | only-packages = true 64 | 65 | [tool.hatch.build.targets.wheel] 66 | packages = ["src/httpx_limiter"] 67 | 68 | [tool.hatch.build.hooks.vcs] 69 | version-file = "src/httpx_limiter/_version.py" 70 | 71 | [tool.hatch.version] 72 | source = "vcs" 73 | 74 | [tool.pytest.ini_options] 75 | testpaths = ["tests"] 76 | markers = ["raises"] 77 | 78 | [tool.coverage.paths] 79 | source = [ 80 | "src/httpx_limiter", 81 | "*/site-packages/httpx_limiter", 82 | ] 83 | 84 | [tool.coverage.run] 85 | branch = true 86 | parallel = true 87 | omit = [ 88 | "src/httpx_limiter/_version.py", 89 | ] 90 | 91 | [tool.coverage.report] 92 | exclude_lines = ["pragma: no cover"] 93 | precision = 2 94 | 95 | [tool.ruff] 96 | line-length = 88 97 | 98 | [tool.ruff.lint] 99 | select = ["ALL"] 100 | ignore = [ 101 | "D107", # 'Missing docstring in __init__' ignored because pydoclint wants us to document the class instead. 102 | "D203", # '1 blank line required before class docstring' ignored because we want no blank line. 103 | "D212", # 'Multi-line docstring summary should start at the first line' ignored because we want the summary to start on the second line. 104 | "D407", # 'Missing dashed underline after section' ignored because Google style docstrings don't underline. 105 | "ANN002", # 'Missing type annotation for {*args} in method'. 106 | "ANN003", # 'Missing type annotation for {*kwargs} in method'. 107 | ] 108 | exclude = [ 109 | "src/httpx_limiter/_version.py", 110 | ] 111 | 112 | [tool.ruff.lint.extend-per-file-ignores] 113 | "__init__.py" = [ 114 | "E401", # 'Multiple imports on one line' 115 | "E402", # 'Module level import not at top of file' 116 | "F401", # 'Imported but unused' 117 | "I001", # 'Import block is un-sorted or un-formatted' ignored because we may have to import in a particular, not-alphabetical order. 118 | ] 119 | "tests/**/*.py" = [ 120 | "S101", # 'Use of assert detected' ignored because we are using pytest. 121 | "INP001", # 'Insecure input' ignored because we are testing. 122 | "ANN201", # 'Missing type annotation for {return}' ignored because all tests return `None`. 123 | "PLR2004", # Magic numbers are okay in tests. 124 | ] 125 | 126 | [tool.ruff.lint.isort] 127 | case-sensitive = true 128 | known-first-party = ["src", "httpx_limiter"] 129 | lines-after-imports = 2 130 | 131 | [tool.pydoclint] 132 | style = "google" 133 | arg-type-hints-in-docstring = false 134 | check-return-types = false 135 | check-yield-types = false 136 | exclude = "_version.py" 137 | 138 | # TODO: Adjust mypy configuration. 139 | #[tool.mypy] 140 | #plugins = [ 141 | # "pydantic.mypy", 142 | #] 143 | 144 | # Stop mypy from complaining about missing types from imports. 145 | [[tool.mypy.overrides]] 146 | module = [ 147 | "limiter", 148 | ] 149 | ignore_missing_imports = true 150 | 151 | #[tool.pydantic-mypy] 152 | #init_forbid_extra = true 153 | #init_typed = true 154 | #warn_required_dynamic_aliases = true 155 | 156 | ################################################################################ 157 | # Hatch Environments 158 | ################################################################################ 159 | 160 | [tool.hatch.envs.default] 161 | installer = "uv" 162 | 163 | [tool.hatch.envs.dev] 164 | description = """Development environment.""" 165 | extra-dependencies = [ 166 | "anyio", 167 | "pytest", 168 | "pytest-httpx", 169 | "pytest-watcher", 170 | "ruff", 171 | ] 172 | 173 | [tool.hatch.envs.style] 174 | description = """Check the style of the codebase.""" 175 | dependencies = [ 176 | "pydoclint ~=0.6", 177 | "ruff", 178 | ] 179 | detached = true 180 | installer = "uv" 181 | 182 | [tool.hatch.envs.style.scripts] 183 | docstrings = ["pydoclint src", "pydoclint tests"] 184 | code = "ruff check {args}" 185 | format = "ruff format {args}" 186 | check = ["docstrings", "code"] 187 | 188 | [tool.hatch.envs.audit] 189 | description = """Check dependencies for security vulnerabilities.""" 190 | extra-dependencies = [ 191 | "pip-audit", 192 | ] 193 | 194 | [tool.hatch.envs.audit.scripts] 195 | check = ["pip-audit"] 196 | 197 | [tool.hatch.envs.types] 198 | description = """Check the static types of the codebase.""" 199 | dependencies = [ 200 | "mypy", 201 | ] 202 | 203 | [tool.hatch.envs.types.scripts] 204 | check = "mypy src/httpx_limiter" 205 | 206 | [tool.hatch.envs.docs] 207 | description = """Build or serve the documentation.""" 208 | extra-dependencies = [ 209 | "mkdocs-material ~=9.5", 210 | "mkdocstrings[python] ~=0.24", 211 | "mkdocs-awesome-pages-plugin ~=2.9", 212 | "mike ~=2.1", 213 | ] 214 | 215 | [tool.hatch.envs.docs.scripts] 216 | build = "mkdocs build {args:--clean --strict}" 217 | serve = "mkdocs serve {args}" 218 | deploy = "mike deploy {args}" 219 | 220 | #[tool.hatch.envs.install] 221 | #description = """Test the installation the package.""" 222 | #dependencies = [ 223 | # "pip", 224 | # "twine", 225 | #] 226 | #detached = true 227 | 228 | #[tool.hatch.envs.install.scripts] 229 | #check = [ 230 | # "pip check", 231 | # "hatch build {args:--clean}", 232 | # "twine check dist/*", 233 | #] 234 | 235 | [tool.hatch.envs.unit-tests] 236 | description = """Run the unit tests.""" 237 | extra-dependencies = [ 238 | "anyio", 239 | "pytest", 240 | "pytest-cov", 241 | "pytest-httpx", 242 | "pytest-raises", 243 | ] 244 | 245 | [[tool.hatch.envs.unit-tests.matrix]] 246 | python = ["3.10", "3.11", "3.12", "3.13"] 247 | 248 | [tool.hatch.envs.unit-tests.scripts] 249 | run = "pytest {args:--cov=httpx_limiter --cov-report=term-missing} {root}/tests/unit" 250 | 251 | [tool.hatch.envs.acceptance-tests] 252 | description = """Run the acceptance tests.""" 253 | extra-dependencies = [ 254 | "anyio", 255 | "pytest", 256 | "pytest-cov", 257 | "pytest-httpx", 258 | "pytest-raises", 259 | ] 260 | 261 | [[tool.hatch.envs.acceptance-tests.matrix]] 262 | python = ["3.10", "3.11", "3.12", "3.13"] 263 | 264 | [tool.hatch.envs.acceptance-tests.scripts] 265 | run = "pytest {args:--cov=httpx_limiter --cov-report=term-missing} {root}/tests/acceptance" 266 | -------------------------------------------------------------------------------- /src/httpx_limiter/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024 Moritz E. Beber 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | 16 | """Provide top level symbols.""" 17 | 18 | from .rate import Number, Rate 19 | from .async_rate_limited_transport import AsyncRateLimitedTransport 20 | from .abstract_rate_limiter_repository import AbstractRateLimiterRepository 21 | from .async_multi_rate_limited_transport import AsyncMultiRateLimitedTransport 22 | -------------------------------------------------------------------------------- /src/httpx_limiter/abstract_rate_limiter_repository.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2025 Moritz E. Beber 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | 16 | """Provide an abstract repository for rate limiters.""" 17 | 18 | from abc import ABC, abstractmethod 19 | 20 | import httpx 21 | from aiolimiter import AsyncLimiter 22 | 23 | from .rate import Rate 24 | 25 | 26 | class AbstractRateLimiterRepository(ABC): 27 | """Define the abstract repository for rate limiters.""" 28 | 29 | def __init__(self, **kwargs) -> None: 30 | super().__init__(**kwargs) 31 | self._limiters: dict[str, AsyncLimiter] = {} 32 | 33 | @abstractmethod 34 | def get_identifier(self, request: httpx.Request) -> str: 35 | """Return a request-specific identifier.""" 36 | 37 | @abstractmethod 38 | def get_rate(self, request: httpx.Request) -> Rate: 39 | """Return a request-specific rate.""" 40 | 41 | def get(self, request: httpx.Request) -> AsyncLimiter: 42 | """Return a request-specific rate limiter.""" 43 | identifier = self.get_identifier(request) 44 | 45 | if identifier not in self._limiters: 46 | rate = self.get_rate(request) 47 | self._limiters[identifier] = AsyncLimiter( 48 | max_rate=rate.magnitude, 49 | time_period=rate.in_seconds(), 50 | ) 51 | 52 | return self._limiters[identifier] 53 | 54 | -------------------------------------------------------------------------------- /src/httpx_limiter/async_multi_rate_limited_transport.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2025 Moritz E. Beber 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | 16 | """Provide an asynchronous multiple rate-limited transport.""" 17 | 18 | from __future__ import annotations 19 | 20 | from typing import TYPE_CHECKING 21 | 22 | import httpx 23 | 24 | 25 | if TYPE_CHECKING: # pragma: no cover 26 | from .abstract_rate_limiter_repository import AbstractRateLimiterRepository 27 | 28 | 29 | class AsyncMultiRateLimitedTransport(httpx.AsyncBaseTransport): 30 | """ 31 | Define the asynchronous multiple rate-limited transport. 32 | 33 | This transport consists of a composed transport for handling requests and a 34 | repository for rate limiters that are selected based on the request. 35 | 36 | """ 37 | 38 | def __init__( 39 | self, 40 | *, 41 | repository: AbstractRateLimiterRepository, 42 | transport: httpx.AsyncBaseTransport, 43 | **kwargs, 44 | ) -> None: 45 | super().__init__(**kwargs) 46 | self._repo = repository 47 | self._transport = transport 48 | 49 | @classmethod 50 | def create( 51 | cls, 52 | *, 53 | repository: AbstractRateLimiterRepository, 54 | **kwargs: dict, 55 | ) -> AsyncMultiRateLimitedTransport: 56 | """ 57 | Create an instance of an asynchronous multiple rate-limited transport. 58 | 59 | This factory method constructs the instance with an underlying 60 | `httpx.AsyncHTTPTransport`. 61 | That transport is passed any additional keyword arguments. 62 | 63 | Args: 64 | repository: The repository to use for rate limiters. 65 | **kwargs: Additional keyword arguments are used in the construction of an 66 | `httpx.AsyncHTTPTransport`. 67 | 68 | Returns: 69 | A default instance of the class created from the given arguments. 70 | 71 | """ 72 | return cls( 73 | repository=repository, 74 | transport=httpx.AsyncHTTPTransport(**kwargs), # type: ignore[arg-type] 75 | ) 76 | 77 | async def handle_async_request( 78 | self, 79 | request: httpx.Request, 80 | ) -> httpx.Response: 81 | """Handle an asynchronous request with rate limiting.""" 82 | limiter = self._repo.get(request) 83 | async with limiter: 84 | return await self._transport.handle_async_request(request) 85 | -------------------------------------------------------------------------------- /src/httpx_limiter/async_rate_limited_transport.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024 Moritz E. Beber 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | 16 | """Provide an asynchronous rate-limited transport.""" 17 | 18 | from __future__ import annotations 19 | 20 | from typing import TYPE_CHECKING 21 | 22 | import httpx 23 | from aiolimiter import AsyncLimiter 24 | 25 | 26 | if TYPE_CHECKING: # pragma: no cover 27 | from .rate import Rate 28 | 29 | 30 | class AsyncRateLimitedTransport(httpx.AsyncBaseTransport): 31 | """ 32 | Define the asynchronous rate-limited transport. 33 | 34 | This transport consists of a composed transport for handling requests and an 35 | implementation of a leaky bucket algorithm in order to rate-limit the number of 36 | requests. 37 | 38 | """ 39 | 40 | def __init__( 41 | self, 42 | *, 43 | limiter: AsyncLimiter, 44 | transport: httpx.AsyncBaseTransport, 45 | **kwargs, 46 | ) -> None: 47 | super().__init__(**kwargs) 48 | self._limiter = limiter 49 | self._transport = transport 50 | 51 | @classmethod 52 | def create( 53 | cls, 54 | *, 55 | rate: Rate, 56 | **kwargs: dict, 57 | ) -> AsyncRateLimitedTransport: 58 | """ 59 | Create an instance of asynchronous rate-limited transport. 60 | 61 | This factory method constructs the instance with an underlying 62 | `httpx.AsyncHTTPTransport`. 63 | That transport is passed any additional keyword arguments. 64 | 65 | Args: 66 | rate: The maximum rate per interval at which bucket capacity is restored. 67 | **kwargs: Additional keyword arguments are used in the construction of an 68 | `httpx.AsyncHTTPTransport`. 69 | 70 | Returns: 71 | A default instance of the class created from the given arguments. 72 | 73 | """ 74 | return cls( 75 | limiter=AsyncLimiter( 76 | max_rate=rate.magnitude, 77 | time_period=rate.in_seconds(), 78 | ), 79 | transport=httpx.AsyncHTTPTransport(**kwargs), # type: ignore[arg-type] 80 | ) 81 | 82 | async def handle_async_request( 83 | self, 84 | request: httpx.Request, 85 | ) -> httpx.Response: 86 | """Handle an asynchronous request with rate limiting.""" 87 | async with self._limiter: 88 | return await self._transport.handle_async_request(request) 89 | -------------------------------------------------------------------------------- /src/httpx_limiter/py.typed: -------------------------------------------------------------------------------- 1 | # PEP-561 Support File. 2 | # "Package maintainers who wish to support type checking of their code MUST add a marker file named py.typed to their package supporting typing". 3 | -------------------------------------------------------------------------------- /src/httpx_limiter/rate.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2025 Moritz E. Beber 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | 16 | """Provide a definition of a rate.""" 17 | 18 | from __future__ import annotations 19 | 20 | from datetime import timedelta 21 | from typing import NamedTuple 22 | 23 | 24 | Number = float | int 25 | 26 | 27 | class Rate(NamedTuple): 28 | """Define the rate.""" 29 | 30 | magnitude: float 31 | duration: timedelta 32 | 33 | @classmethod 34 | def create(cls, magnitude: Number = 1, duration: timedelta | Number = 1) -> Rate: 35 | """Create a rate.""" 36 | magnitude = float(magnitude) 37 | 38 | if not isinstance(duration, timedelta): 39 | duration = timedelta(seconds=float(duration)) 40 | 41 | return cls(magnitude=magnitude, duration=duration) 42 | 43 | def in_seconds(self) -> float: 44 | """Return the duration in unit seconds.""" 45 | return self.duration.total_seconds() 46 | -------------------------------------------------------------------------------- /tests/acceptance/test_async_multi_rate_limit.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2025 Moritz E. Beber 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | 16 | """ 17 | Test the expected behavior of the asynchronous rate-limited transport. 18 | 19 | These tests expect the local service stack defined in the `compose.yaml` file to be 20 | started. The service that we query lies behind a traefik proxy which implements a 21 | rate limiting middleware (see https://doc.traefik.io/traefik/middlewares/http/ratelimit/) 22 | that allows an average of twenty requests per second. 23 | 24 | """ 25 | 26 | from collections import Counter 27 | 28 | import anyio 29 | import httpx 30 | import pytest 31 | 32 | from httpx_limiter import ( 33 | AbstractRateLimiterRepository, 34 | AsyncMultiRateLimitedTransport, 35 | Rate, 36 | ) 37 | 38 | 39 | class HostRateLimiterRepository(AbstractRateLimiterRepository): 40 | """A concrete implementation of the abstract repository for testing.""" 41 | 42 | def get_identifier(self, request: httpx.Request) -> str: 43 | """Return a host-based identifier for testing.""" 44 | return request.url.host 45 | 46 | def get_rate(self, request: httpx.Request) -> Rate: 47 | """Return a host-dependent rate.""" 48 | match self.get_identifier(request): 49 | case "httpbin.localhost": 50 | return Rate.create(duration=1 / 20) 51 | case "fast.localhost": 52 | return Rate.create(duration=1 / 40) 53 | case _: 54 | return Rate.create(1) 55 | 56 | 57 | async def _record_response( 58 | client: httpx.AsyncClient, 59 | url: str, 60 | counter: Counter, 61 | ) -> None: 62 | response = await client.get(url) 63 | counter[response.status_code] += 1 64 | 65 | 66 | @pytest.mark.anyio 67 | async def test_limits(): 68 | """Test that an API's rate limit is maintained.""" 69 | slow_rate_codes = Counter() 70 | fast_rate_codes = Counter() 71 | 72 | async with ( 73 | httpx.AsyncClient( 74 | transport=AsyncMultiRateLimitedTransport.create( 75 | repository=HostRateLimiterRepository(), 76 | ), 77 | ) as client, 78 | anyio.create_task_group() as group, 79 | ): 80 | for _ in range(100): 81 | group.start_soon( 82 | _record_response, 83 | client, 84 | "http://httpbin.localhost/status/200", 85 | slow_rate_codes, 86 | ) 87 | group.start_soon( 88 | _record_response, 89 | client, 90 | "http://fast.localhost/status/200", 91 | fast_rate_codes, 92 | ) 93 | 94 | assert slow_rate_codes.total() == 100 95 | assert slow_rate_codes[httpx.codes.OK] in range(95, 101) 96 | 97 | 98 | assert fast_rate_codes.total() == 100 99 | assert fast_rate_codes[httpx.codes.OK] in range(95, 101) 100 | -------------------------------------------------------------------------------- /tests/acceptance/test_async_rate_limit.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024 Moritz E. Beber 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | 16 | """ 17 | Test the expected behavior of the asynchronous rate-limited transport. 18 | 19 | These tests expect the local service stack defined in the `compose.yaml` file to be 20 | started. The service that we query lies behind a traefik proxy which implements a 21 | rate limiting middleware (see https://doc.traefik.io/traefik/middlewares/http/ratelimit/) 22 | that allows an average of twenty requests per second. 23 | 24 | """ 25 | 26 | from collections import Counter 27 | 28 | import anyio 29 | import httpx 30 | import pytest 31 | 32 | from httpx_limiter import AsyncRateLimitedTransport, Rate 33 | 34 | 35 | async def _record_response(client: httpx.AsyncClient, counter: Counter) -> None: 36 | response = await client.get("http://httpbin.localhost/status/200") 37 | counter[response.status_code] += 1 38 | 39 | 40 | @pytest.mark.anyio 41 | async def test_limits(): 42 | """Test that an API's rate limit is maintained.""" 43 | response_codes = Counter() 44 | 45 | async with ( 46 | httpx.AsyncClient( 47 | transport=AsyncRateLimitedTransport.create( 48 | rate=Rate.create(duration=1 / 20), 49 | ), 50 | ) as client, 51 | anyio.create_task_group() as group, 52 | ): 53 | for _ in range(100): 54 | group.start_soon(_record_response, client, response_codes) 55 | 56 | assert response_codes.total() == 100 57 | assert response_codes[httpx.codes.OK] in range(95, 101) 58 | 59 | 60 | @pytest.mark.anyio 61 | async def test_exceed_limits(): 62 | """ 63 | Test that an API's rate limit is exceeded. 64 | 65 | We cannot predict exactly the alignment of the rate at which we make requests and 66 | the rate at which requests are accepted. Hence, we allow the response codes to be 67 | within a range of expected values. 68 | 69 | """ 70 | response_codes = Counter() 71 | 72 | async with ( 73 | httpx.AsyncClient( 74 | transport=AsyncRateLimitedTransport.create( 75 | rate=Rate.create(duration=1 / 25), 76 | ), 77 | ) as client, 78 | anyio.create_task_group() as group, 79 | ): 80 | for _ in range(125): 81 | group.start_soon(_record_response, client, response_codes) 82 | 83 | assert response_codes.total() == 125 84 | assert response_codes[httpx.codes.OK] in range(90, 101) 85 | assert response_codes[httpx.codes.TOO_MANY_REQUESTS] == ( 86 | 125 - response_codes[httpx.codes.OK] 87 | ) 88 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024 Moritz E. Beber 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | 16 | """Define test suite-level fixtures.""" 17 | 18 | import pytest 19 | 20 | 21 | @pytest.fixture( 22 | params=[ 23 | pytest.param(("asyncio", {"use_uvloop": False}), id="asyncio"), 24 | ], 25 | ) 26 | def anyio_backend(request: pytest.FixtureRequest) -> tuple[str, dict[str, bool]]: 27 | """Set the anyio event loop.""" 28 | return request.param 29 | -------------------------------------------------------------------------------- /tests/unit/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Midnighter/httpx-limiter/ceae3a826c6aefe7bba7247eecff8674944c68f7/tests/unit/.keep -------------------------------------------------------------------------------- /tests/unit/test_abstract_rate_limiter_repository.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2025 Moritz E. Beber 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | 16 | """Test the expected functionality of the rate limiter repository.""" 17 | 18 | import httpx 19 | from aiolimiter import AsyncLimiter 20 | 21 | from httpx_limiter import AbstractRateLimiterRepository, Rate 22 | 23 | 24 | class ConcreteRateLimiterRepository(AbstractRateLimiterRepository): 25 | """A concrete implementation of the abstract repository for testing.""" 26 | 27 | def __init__(self, identifier: str, rate: Rate) -> None: 28 | super().__init__() 29 | self._rate = rate 30 | self._identifier = identifier 31 | 32 | def get_identifier(self, _: httpx.Request) -> str: 33 | """Return a predefined identifier for testing.""" 34 | return self._identifier 35 | 36 | def get_rate(self, _: httpx.Request) -> Rate: 37 | """Return a predefined rate for testing.""" 38 | return self._rate 39 | 40 | 41 | class MethodRateLimiterRepository(AbstractRateLimiterRepository): 42 | """A concrete implementation of the abstract repository for testing.""" 43 | 44 | def get_identifier(self, request: httpx.Request) -> str: 45 | """Return a method-based identifier for testing.""" 46 | return request.method 47 | 48 | def get_rate(self, _: httpx.Request) -> Rate: 49 | """Return a constant rate.""" 50 | return Rate.create(1) 51 | 52 | 53 | def test_get_identifier(): 54 | """Test that get_identifier is called and returns the expected value.""" 55 | repo = ConcreteRateLimiterRepository(identifier="test_id", rate=Rate.create(1)) 56 | 57 | result = repo.get_identifier(httpx.Request(method="GET", url="http://test.com")) 58 | 59 | assert result == "test_id" 60 | 61 | 62 | def test_get_rate(): 63 | """Test that get_rate is called and returns the expected value.""" 64 | repo = ConcreteRateLimiterRepository(identifier="test_id", rate=Rate.create(9)) 65 | 66 | result = repo.get_rate(httpx.Request(method="GET", url="http://test.com")) 67 | 68 | assert result.magnitude == 9 69 | 70 | def test_get_new_limiter(): 71 | """Test that get creates and returns a new limiter when needed.""" 72 | repo = ConcreteRateLimiterRepository(identifier="test_id", rate=Rate.create(1)) 73 | 74 | result = repo.get(httpx.Request(method="GET", url="http://test.com")) 75 | 76 | assert isinstance(result, AsyncLimiter) 77 | 78 | def test_get_existing_limiter(): 79 | """Test that get returns an existing limiter if available.""" 80 | repo = ConcreteRateLimiterRepository(identifier="test_id", rate=Rate.create(1)) 81 | 82 | first_limiter = repo.get(httpx.Request(method="GET", url="http://test.com")) 83 | second_limiter = repo.get(httpx.Request(method="GET", url="http://test.com")) 84 | 85 | assert second_limiter is first_limiter 86 | 87 | def test_multiple_identifiers(): 88 | """Test that different identifiers create different limiters.""" 89 | repo = MethodRateLimiterRepository() 90 | 91 | first_limiter = repo.get(httpx.Request(method="GET", url="http://test.com")) 92 | second_limiter = repo.get(httpx.Request(method="POST", url="http://test.com")) 93 | 94 | assert second_limiter is not first_limiter 95 | -------------------------------------------------------------------------------- /tests/unit/test_async_rate_limited_transport.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024 Moritz E. Beber 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | 16 | """Test the expected functionality of the asynchronous rate-limited transport.""" 17 | 18 | import anyio 19 | import httpx 20 | import pytest 21 | from aiolimiter import AsyncLimiter 22 | from pytest_httpx import HTTPXMock 23 | 24 | from httpx_limiter import AsyncRateLimitedTransport, Rate 25 | 26 | 27 | def test_init(): 28 | """Test that an asynchronous rate-limited transport can be initialized.""" 29 | AsyncRateLimitedTransport( 30 | limiter=AsyncLimiter(10), 31 | transport=httpx.AsyncHTTPTransport(), 32 | ) 33 | 34 | 35 | def test_create(): 36 | """Test that an asynchronous rate-limited transport can be created.""" 37 | transport = AsyncRateLimitedTransport.create(rate=Rate.create()) 38 | assert isinstance(transport, AsyncRateLimitedTransport) 39 | 40 | 41 | @pytest.mark.anyio 42 | async def test_handle_async_request(httpx_mock: HTTPXMock): 43 | """Test that handled requests are rate-limited.""" 44 | counter = 0 45 | 46 | async def count_responses(request: httpx.Request) -> httpx.Response: # noqa: ARG001 47 | nonlocal counter 48 | 49 | counter += 1 50 | 51 | return httpx.Response(status_code=200) 52 | 53 | httpx_mock.add_callback(count_responses, is_reusable=True) 54 | 55 | # We configure the bucket with a rate of two to allow a burst of requests and a 56 | # refresh interval of 0.1 seconds. That means, when we create ten requests but then 57 | # cancel all outstanding requests after 0.06 seconds, a bit more than half the time 58 | # of the interval will have passed, and we expect a capacity of one to be returned. 59 | # Consequently, we expect three requests to succeed in total. 60 | async with ( 61 | httpx.AsyncClient( 62 | transport=AsyncRateLimitedTransport.create( 63 | rate=Rate.create(magnitude=2, duration=0.1), 64 | ), 65 | ) as client, 66 | anyio.create_task_group() as tg, 67 | ): 68 | with anyio.move_on_after(0.07) as scope: 69 | for _ in range(10): 70 | tg.start_soon(client.get, "http://example.com") 71 | await anyio.sleep(2) 72 | 73 | tg.cancel_scope.cancel() 74 | 75 | assert scope.cancelled_caught 76 | assert counter == 3 77 | -------------------------------------------------------------------------------- /tests/unit/test_rate.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2025 Moritz E. Beber 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | 16 | """Test the expected functionality of the rate value object.""" 17 | 18 | from datetime import timedelta 19 | 20 | import pytest 21 | 22 | from httpx_limiter import Number, Rate 23 | 24 | 25 | def test_init(): 26 | """Test that the rate is properly initialized.""" 27 | rate = Rate(magnitude=2.0, duration=timedelta(seconds=1)) 28 | 29 | assert isinstance(rate.magnitude, float) 30 | assert rate.magnitude == 2.0 31 | 32 | assert isinstance(rate.duration, timedelta) 33 | assert rate.duration == timedelta(seconds=1) 34 | 35 | 36 | @pytest.mark.parametrize( 37 | ("magnitude", "duration", "expected"), 38 | [ 39 | (2, 3, Rate(magnitude=2.0, duration=timedelta(seconds=3))), 40 | (4.1, 5.2, Rate(magnitude=4.1, duration=timedelta(seconds=5.2))), 41 | ( 42 | 6.1, 43 | timedelta(microseconds=230), 44 | Rate(magnitude=6.1, duration=timedelta(microseconds=230)), 45 | ), 46 | ], 47 | ) 48 | def test_create(magnitude: Number, duration: timedelta | Number, expected: Rate): 49 | """Test that rates are properly created.""" 50 | assert Rate.create(magnitude, duration) == expected 51 | 52 | 53 | @pytest.mark.parametrize( 54 | ("duration", "expected"), 55 | [ 56 | (timedelta(minutes=1.5), 90.0), 57 | (timedelta(milliseconds=500), 0.5), 58 | ], 59 | ) 60 | def test_in_seconds(duration: timedelta | Number, expected: float): 61 | """Test that the duration is correctly converted to seconds.""" 62 | assert Rate.create(duration=duration).in_seconds() == expected 63 | --------------------------------------------------------------------------------