├── .coveragerc ├── .editorconfig ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.yml │ ├── config.yml │ └── feature_request.yml └── workflows │ ├── ci.yml │ ├── documentation.yml │ └── publish.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── docs ├── advanced │ ├── async_client.md │ ├── authentication.md │ ├── client.md │ ├── hooks.md │ └── proxies.md ├── index.md ├── quickstart.md ├── static │ └── coingecko.png └── tls │ ├── configuration.md │ ├── index.md │ ├── install.md │ └── profiles.md ├── mkdocs.yml ├── pyproject.toml ├── requirements-dev.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── conftest.py ├── files │ ├── __init__.py │ └── coingecko.png ├── test_api.py ├── test_auth.py ├── test_cookies.py ├── test_encoders.py ├── test_headers.py ├── test_hooks.py ├── test_proxy.py ├── test_redirects.py ├── test_timeout.py └── test_urls.py ├── tls_requests ├── __init__.py ├── __version__.py ├── api.py ├── bin │ └── __init__.py ├── client.py ├── exceptions.py ├── models │ ├── __init__.py │ ├── auth.py │ ├── cookies.py │ ├── encoders.py │ ├── headers.py │ ├── libraries.py │ ├── request.py │ ├── response.py │ ├── status_codes.py │ ├── tls.py │ └── urls.py ├── settings.py ├── types.py └── utils.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = tests/* 3 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.{py,rst,ini}] 12 | indent_style = space 13 | indent_size = 4 14 | 15 | [*.{html,css,scss,json,yml,xml}] 16 | indent_style = space 17 | indent_size = 2 18 | 19 | [*.md] 20 | trim_trailing_whitespace = false 21 | 22 | [Makefile] 23 | indent_style = tab 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.yml: -------------------------------------------------------------------------------- 1 | name: Bug report 2 | description: File a bug report for the TLS Requests 3 | title: "[Bug]: " 4 | labels: ["bug"] 5 | body: 6 | 7 | - type: markdown 8 | attributes: 9 | value: | 10 | - Write an issue title above. 11 | - Search [open](https://github.com/thewebscraping/tls-requests/issues?q=is%3Aopen) and [closed](https://github.com/thewebscraping/tls-requests/issues?q=is%3Aclosed) issues to ensure it has not already been reported. 12 | 13 | - type: input 14 | attributes: 15 | label: TLS Requests version 16 | description: > 17 | Specify the TLS Requests version 18 | placeholder: v1.0.8 19 | validations: 20 | required: true 21 | 22 | - type: textarea 23 | attributes: 24 | label: Issue description 25 | description: | 26 | Describe your issue briefly. What doesn't work, and how do you expect it to work instead? 27 | validations: 28 | required: true 29 | 30 | - type: textarea 31 | attributes: 32 | label: Steps to reproduce / Code Sample 33 | description: | 34 | List of steps or sample code that reproduces the issue. 35 | validations: 36 | required: true 37 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.yml: -------------------------------------------------------------------------------- 1 | name: Feature Request 2 | description: Propose a feature for the TLS Requests 3 | title: "[Feature Request]: " 4 | labels: ["enhancement"] 5 | body: 6 | 7 | - type: markdown 8 | attributes: 9 | value: | 10 | - Write a proposal title above. 11 | - Search [open](https://github.com/thewebscraping/tls-requests/issues?q=is%3Aopen) and [closed](https://github.com/thewebscraping/tls-requests/issues?q=is%3Aclosed) proposals to ensure the feature has not already been suggested. 12 | 13 | - type: textarea 14 | attributes: 15 | label: Describe the feature / enhancement and how it would improve things 16 | description: | 17 | - Dont overcomplicate 18 | validations: 19 | required: true 20 | 21 | - type: textarea 22 | attributes: 23 | label: Describe how your proposal will work, with code and/or pseudo-code 24 | validations: 25 | required: true 26 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: ['master', 'main'] 6 | 7 | pull_request: 8 | branches: ['master', 'main', 'dev', 'develop'] 9 | 10 | jobs: 11 | build-on-ubuntu: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | fail-fast: false 15 | max-parallel: 3 16 | matrix: 17 | python-version: ['3.9', '3.10', '3.11'] 18 | steps: 19 | - uses: actions/checkout@v4 20 | - name: Set up Python ${{ matrix.python-version }} 21 | uses: actions/setup-python@v4 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | 25 | - name: Install Dependencies, Lint and Tests 26 | run: | 27 | make init-actions 28 | 29 | build-on-mac: 30 | runs-on: macos-latest 31 | strategy: 32 | fail-fast: false 33 | max-parallel: 1 34 | matrix: 35 | python-version: ['3.9'] 36 | steps: 37 | - uses: actions/checkout@v4 38 | - name: Set up Python ${{ matrix.python-version }} 39 | uses: actions/setup-python@v4 40 | with: 41 | python-version: ${{ matrix.python-version }} 42 | 43 | - name: Install Dependencies, Lint and Tests 44 | run: | 45 | make init-actions 46 | 47 | - name: Pytest 48 | run: | 49 | make pytest 50 | 51 | build-on-windows: 52 | runs-on: windows-latest 53 | strategy: 54 | fail-fast: false 55 | max-parallel: 1 56 | matrix: 57 | python-version: ['3.9'] 58 | steps: 59 | - uses: actions/checkout@v4 60 | - name: Set up Python ${{ matrix.python-version }} 61 | uses: actions/setup-python@v4 62 | with: 63 | python-version: ${{ matrix.python-version }} 64 | 65 | - name: Install Dependencies, Lint 66 | run: | 67 | make init-actions 68 | 69 | - name: Pytest 70 | run: | 71 | make pytest 72 | -------------------------------------------------------------------------------- /.github/workflows/documentation.yml: -------------------------------------------------------------------------------- 1 | name: Build Documentation 2 | on: 3 | release: 4 | types: [created] 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | max-parallel: 1 10 | matrix: 11 | python-version: ['3.9'] 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Set up Python ${{ matrix.python-version }} 15 | uses: actions/setup-python@v5 16 | with: 17 | python-version: ${{ matrix.python-version }} 18 | - name: Install Dependencies 19 | run: | 20 | python -m pip install --upgrade pip 21 | pip install -r requirements-dev.txt 22 | - name: Configure Git Credentials 23 | run: | 24 | git config user.name github-actions[bot] 25 | git config user.email 41898282+github-actions[bot]@users.noreply.github.com 26 | - uses: actions/cache@v4 27 | with: 28 | key: mkdocs-material-${{ env.cache_id }} 29 | path: .cache 30 | restore-keys: | 31 | mkdocs-material- 32 | - name: Publish Documentation 33 | run: | 34 | echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV 35 | mkdocs gh-deploy --force 36 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Upload Python Package 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | deploy: 9 | name: Publish PyPI 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@main 13 | - name: Set up Python 3.9 14 | uses: actions/setup-python@v4 15 | with: 16 | python-version: '3.9' 17 | - name: Build package 18 | run: | 19 | python -m pip install -U pip build setuptools twine 20 | - name: Publish 21 | env: 22 | TWINE_USERNAME: __token__ 23 | TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} 24 | run: | 25 | python setup.py sdist bdist_wheel 26 | twine upload --skip-existing dist/* 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 110 | .pdm.toml 111 | .pdm-python 112 | .pdm-build/ 113 | 114 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 115 | __pypackages__/ 116 | 117 | # Celery stuff 118 | celerybeat-schedule 119 | celerybeat.pid 120 | 121 | # SageMath parsed files 122 | *.sage.py 123 | 124 | # Environments 125 | .env 126 | .venv 127 | env/ 128 | venv/ 129 | ENV/ 130 | env.bak/ 131 | venv.bak/ 132 | 133 | # Spyder project settings 134 | .spyderproject 135 | .spyproject 136 | 137 | # Rope project settings 138 | .ropeproject 139 | 140 | # mkdocs documentation 141 | /site 142 | 143 | # mypy 144 | .mypy_cache/ 145 | .dmypy.json 146 | dmypy.json 147 | 148 | # Pyre type checker 149 | .pyre/ 150 | 151 | # pytype static type analyzer 152 | .pytype/ 153 | 154 | # Cython debug symbols 155 | cython_debug/ 156 | 157 | # PyCharm 158 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 159 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 160 | # and can be added to the global gitignore or merged into this file. For a more nuclear 161 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 162 | .idea/ 163 | tls_requests/bin/*xgo* 164 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: '^docs.sh/|scripts/' 2 | default_stages: [pre-commit] 3 | 4 | default_language_version: 5 | python: python3.10 6 | 7 | repos: 8 | - repo: https://github.com/pre-commit/pre-commit-hooks 9 | rev: v4.5.0 10 | hooks: 11 | - id: trailing-whitespace 12 | - id: end-of-file-fixer 13 | - id: check-json 14 | - id: check-toml 15 | - id: check-xml 16 | - id: check-yaml 17 | - id: debug-statements 18 | - id: check-builtin-literals 19 | - id: check-case-conflict 20 | - id: check-docstring-first 21 | - id: detect-private-key 22 | 23 | # run the isort. 24 | - repo: https://github.com/PyCQA/isort 25 | rev: 5.13.2 26 | hooks: 27 | - id: isort 28 | 29 | # run the flake8. 30 | - repo: https://github.com/PyCQA/flake8 31 | rev: 7.0.0 32 | hooks: 33 | - id: flake8 34 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Release History 2 | =============== 3 | 4 | 1.0.7 (2024-12-14) 5 | ------------------- 6 | **Bugfixes:** 7 | 8 | - Fix URL. 9 | - Fix Proxy. 10 | 11 | 1.0.6 (2024-12-12) 12 | ------------------- 13 | **Bugfixes:** 14 | 15 | - Fix request file (image file, etc). 16 | 17 | 1.0.5 (2024-12-11) 18 | ------------------- 19 | **Bugfixes:** 20 | 21 | - Fix mkdocs deploy. 22 | 23 | 1.0.4 (2024-12-11) 24 | ------------------- 25 | **Improvements:** 26 | 27 | - Add unit tests. 28 | - Improve document. 29 | 30 | **Bugfixes:** 31 | 32 | - Fix timeout. 33 | - Fix missing port redirection. 34 | 35 | 36 | 1.0.3 (2024-12-05) 37 | ------------------- 38 | **Improvements** 39 | 40 | - improve document. 41 | 42 | **Bugfixes** 43 | 44 | - Fix multipart encoders, cross share auth. 45 | 46 | 1.0.2 (2024-12-05) 47 | ------------------- 48 | **Improvements** 49 | - Download specific TLS library versions. 50 | - Add a document. 51 | 52 | 1.0.1 (2024-12-04) 53 | ------------------- 54 | - First release 55 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 thewebscraping 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.md 3 | include CHANGELOG.md 4 | recursive-include tls_requests docs Makefile *.md 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: docs 2 | init-actions: 3 | python -m pip install --upgrade pip 4 | python -m pip install -r requirements-dev.txt 5 | python -m black tls_requests 6 | python -m isort tls_requests 7 | python -m flake8 tls_requests 8 | 9 | test: 10 | tox -p 11 | rm -rf *.egg-info 12 | 13 | test-readme: 14 | python setup.py check --restructuredtext --strict && ([ $$? -eq 0 ] && echo "README.md and CHANGELOG.md ok") || echo "Invalid markup in README.md or CHANGELOG.md!" 15 | 16 | pytest: 17 | python -m pytest tests 18 | 19 | coverage: 20 | python -m pytest --cov-config .coveragerc --verbose --cov-report term --cov-report xml --cov=tls_requests tests 21 | 22 | docs: 23 | mkdocs serve 24 | 25 | publish-test-pypi: 26 | python -m pip install -r requirements-dev.txt 27 | python -m pip install 'twine>=6.0.1' 28 | python setup.py sdist bdist_wheel 29 | twine upload --repository testpypi --skip-existing dist/* 30 | rm -rf build dist .egg wrapper_tls_requests.egg-info 31 | 32 | publish-pypi: 33 | python -m pip install -r requirements-dev.txt 34 | python -m pip install 'twine>=6.0.1' 35 | python setup.py sdist bdist_wheel 36 | twine upload --skip-existing dist/* 37 | rm -rf build dist .egg wrapper_tls_requests.egg-info 38 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TLS Requests 2 | 3 | [![GitHub License](https://img.shields.io/github/license/thewebscraping/tls-requests)](https://github.com/thewebscraping/tls-requests/blob/main/LICENSE) 4 | [![CI](https://github.com/thewebscraping/tls-requests/actions/workflows/ci.yml/badge.svg)](https://github.com/thewebscraping/tls-requests/actions/workflows/ci.yml) 5 | [![PyPI - Version](https://img.shields.io/pypi/v/wrapper-tls-requests)](https://pypi.org/project/wrapper-tls-requests/) 6 | ![Python Version](https://img.shields.io/badge/Python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue?style=flat) 7 | ![Pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white) 8 | 9 | [![](https://img.shields.io/badge/Pytest-Linux%20%7C%20MacOS%20%7C%20Windows-blue?style=flat&logo=pytest&logoColor=white)](https://github.com/thewebscraping/tls-requests) 10 | [![Documentation](https://img.shields.io/badge/Mkdocs-Documentation-blue?style=flat&logo=MaterialForMkDocs&logoColor=white)](https://thewebscraping.github.io/tls-requests/) 11 | 12 | TLS Requests is a powerful Python library for secure HTTP requests, offering browser-like TLS client, fingerprinting, anti-bot page bypass, and high performance. 13 | 14 | * * * 15 | 16 | **Installation** 17 | ---------------- 18 | 19 | To install the library, you can choose between two methods: 20 | 21 | #### **1\. Install via PyPI:** 22 | 23 | ```shell 24 | pip install wrapper-tls-requests 25 | ``` 26 | 27 | #### **2\. Install via GitHub Repository:** 28 | 29 | ```shell 30 | pip install git+https://github.com/thewebscraping/tls-requests.git 31 | ``` 32 | 33 | **Quick Start** 34 | --------------- 35 | 36 | Start using TLS Requests with just a few lines of code: 37 | 38 | ```pycon 39 | >>> import tls_requests 40 | >>> r = tls_requests.get("https://httpbin.org/get") 41 | >>> r 42 | 43 | >>> r.status_code 44 | 200 45 | ``` 46 | 47 | **Introduction** 48 | ---------------- 49 | 50 | **TLS Requests** is a cutting-edge HTTP client for Python, offering a feature-rich, 51 | highly configurable alternative to the popular [`requests`](https://github.com/psf/requests) library. 52 | 53 | Built on top of [`tls-client`](https://github.com/bogdanfinn/tls-client), 54 | it combines ease of use with advanced functionality for secure networking. 55 | 56 | **Acknowledgment**: A big thank you to all contributors for their support! 57 | 58 | ### **Key Benefits** 59 | 60 | * **Bypass TLS Fingerprinting:** Mimic browser-like behaviors to navigate sophisticated anti-bot systems. 61 | * **Customizable TLS Client:** Select specific TLS fingerprints to meet your needs. 62 | * **Ideal for Developers:** Build scrapers, API clients, or other custom networking tools effortlessly. 63 | 64 | 65 | **Why Use TLS Requests?** 66 | ------------------------- 67 | 68 | Modern websites increasingly use **TLS Fingerprinting** and anti-bot tools like Cloudflare Bot Fight Mode to block web crawlers. 69 | 70 | **TLS Requests** bypass these obstacles by mimicking browser-like TLS behaviors, 71 | making it easy to scrape data or interact with websites that use sophisticated anti-bot measures. 72 | 73 | ### Unlocking Cloudflare Bot Fight Mode 74 | ![coingecko.png](https://raw.githubusercontent.com/thewebscraping/tls-requests/refs/heads/main/docs/static/coingecko.png) 75 | 76 | **Example Code:** 77 | 78 | ```pycon 79 | >>> import tls_requests 80 | >>> r = tls_requests.get('https://www.coingecko.com/') 81 | >>> r 82 | 83 | ``` 84 | 85 | **Key Features** 86 | ---------------- 87 | 88 | ### **Enhanced Capabilities** 89 | 90 | * **Browser-like TLS Fingerprinting**: Enables secure and reliable browser-mimicking connections. 91 | * **High-Performance Backend**: Built on a Go-based HTTP backend for speed and efficiency. 92 | * **Synchronous & Asynchronous Support**: Seamlessly switch between synchronous and asynchronous requests. 93 | * **Protocol Support**: Fully compatible with HTTP/1.1 and HTTP/2. 94 | * **Strict Timeouts**: Reliable timeout management for precise control over request durations. 95 | 96 | ### **Additional Features** 97 | 98 | * **Internationalized Domain & URL Support**: Handles non-ASCII URLs effortlessly. 99 | * **Cookie Management**: Ensures session-based cookie persistence. 100 | * **Authentication**: Native support for Basic and Function authentication. 101 | * **Content Decoding**: Automatic handling of gzip and brotli-encoded responses. 102 | * **Hooks**: Perfect for logging, monitoring, tracing, or pre/post-processing requests and responses. 103 | * **Unicode Support**: Effortlessly process Unicode response bodies. 104 | * **File Uploads**: Simplified multipart file upload support. 105 | * **Proxy Configuration**: Supports Socks5, HTTP, and HTTPS proxies for enhanced privacy. 106 | 107 | 108 | **Documentation** 109 | ----------------- 110 | 111 | Explore the full capabilities of TLS Requests in the documentation: 112 | 113 | * **[Quickstart Guide](https://thewebscraping.github.io/tls-requests/quickstart/)**: A beginner-friendly guide. 114 | * **[Advanced Topics](https://thewebscraping.github.io/tls-requests/advanced/client/)**: Learn to leverage specialized features. 115 | * **[Async Support](https://thewebscraping.github.io/tls-requests/advanced/async_client/)**: Handle high-concurrency scenarios. 116 | * **Custom TLS Configurations**: 117 | * **[Wrapper TLS Client](https://thewebscraping.github.io/tls-requests/tls/)** 118 | * **[TLS Client Profiles](https://thewebscraping.github.io/tls-requests/tls/profiles/)** 119 | * **[Custom TLS Configurations](https://thewebscraping.github.io/tls-requests/tls/configuration/)** 120 | 121 | 122 | Read the documentation: [**thewebscraping.github.io/tls-requests/**](https://thewebscraping.github.io/tls-requests/) 123 | 124 | **Report Issues** 125 | ----------------- 126 | 127 | Found a bug? Please [open an issue](https://github.com/thewebscraping/tls-requests/issues/). 128 | 129 | By reporting an issue you help improve the project. 130 | 131 | **Credits** 132 | ----------------- 133 | 134 | Special thanks to [bogdanfinn](https://github.com/bogdanfinn/) for creating the awesome [tls-client](https://github.com/bogdanfinn/tls-client). 135 | -------------------------------------------------------------------------------- /docs/advanced/async_client.md: -------------------------------------------------------------------------------- 1 | Async Support in TLS Requests 2 | ============================= 3 | 4 | TLS Requests provides support for asynchronous HTTP requests using the `AsyncClient`. This is especially useful when working in an asynchronous environment, such as with modern web frameworks, or when you need the performance benefits of asynchronous I/O. 5 | 6 | * * * 7 | 8 | Why Use Async? 9 | -------------- 10 | 11 | * **Improved Performance:** Async is more efficient than multi-threading for handling high concurrency workloads. 12 | * **Long-lived Connections:** Useful for protocols like WebSockets or long polling. 13 | * **Framework Compatibility:** Essential when integrating with async web frameworks (e.g., FastAPI, Starlette). 14 | 15 | Advanced usage with syntax similar to Client, refer to the [Client documentation](client). 16 | 17 | * * * 18 | 19 | Making Async Requests 20 | --------------------- 21 | 22 | To send asynchronous HTTP requests, use the `AsyncClient`: 23 | 24 | ```pycon 25 | >>> import asyncio 26 | >>> import random 27 | >>> import time 28 | >>> import tls_requests 29 | >>> async def fetch(idx, url): 30 | async with tls_requests.AsyncClient() as client: 31 | rand = random.uniform(0.1, 1.5) 32 | start_time = time.perf_counter() 33 | print("%s: Sleep for %.2f seconds." % (idx, rand)) 34 | await asyncio.sleep(rand) 35 | response = await client.get(url) 36 | end_time = time.perf_counter() 37 | print("%s: Took: %.2f" % (idx, (end_time - start_time))) 38 | return response 39 | >>> async def run(urls): 40 | tasks = [asyncio.create_task(fetch(idx, url)) for idx, url in enumerate(urls)] 41 | responses = await asyncio.gather(*tasks) 42 | return responses 43 | 44 | >>> start_urls = [ 45 | 'https://httpbin.org/absolute-redirect/1', 46 | 'https://httpbin.org/absolute-redirect/2', 47 | 'https://httpbin.org/absolute-redirect/3', 48 | 'https://httpbin.org/absolute-redirect/4', 49 | 'https://httpbin.org/absolute-redirect/5', 50 | ] 51 | 52 | 53 | >>> r = asyncio.run(run(start_urls)) 54 | >>> r 55 | [, , , , ] 56 | 57 | ``` 58 | 59 | !!! tip 60 | Use [IPython](https://ipython.readthedocs.io/en/stable/) or Python 3.8+ with `python -m asyncio` to try this code interactively, as they support executing `async`/`await` expressions in the console. 61 | 62 | * * * 63 | 64 | Key API Differences 65 | ------------------- 66 | 67 | When using `AsyncClient`, the API methods are asynchronous and must be awaited. 68 | 69 | ### Making Requests 70 | 71 | Use `await` for all request methods: 72 | 73 | * `await client.get(url, ...)` 74 | * `await client.post(url, ...)` 75 | * `await client.put(url, ...)` 76 | * `await client.patch(url, ...)` 77 | * `await client.delete(url, ...)` 78 | * `await client.options(url, ...)` 79 | * `await client.head(url, ...)` 80 | * `await client.request(method, url, ...)` 81 | * `await client.send(request, ...)` 82 | 83 | * * * 84 | 85 | ### Managing Client Lifecycle 86 | 87 | #### Context Manager 88 | 89 | For proper resource cleanup, use `async with`: 90 | 91 | ```python 92 | import asyncio 93 | 94 | async def fetch(url): 95 | async with tls_requests.AsyncClient() as client: 96 | response = await client.get(url) 97 | return response 98 | 99 | r = asyncio.run(fetch("https://httpbin.org/get")) 100 | print(r) # 101 | ``` 102 | 103 | #### Manual Closing 104 | 105 | Alternatively, explicitly close the client: 106 | 107 | ```python 108 | import asyncio 109 | 110 | async def fetch(url): 111 | client = tls_requests.AsyncClient() 112 | try: 113 | response = await client.get("https://httpbin.org/get") 114 | finally: 115 | await client.aclose() 116 | ``` 117 | 118 | * * * 119 | 120 | By using `AsyncClient`, you can unlock the full potential of asynchronous programming in Python while enjoying the simplicity and power of TLS Requests. 121 | -------------------------------------------------------------------------------- /docs/advanced/authentication.md: -------------------------------------------------------------------------------- 1 | # Authentication 2 | 3 | This section covers how to use authentication in your requests with `tls_requests`, offering both built-in options and the flexibility to define custom mechanisms. 4 | 5 | * * * 6 | 7 | Basic Authentication 8 | -------------------- 9 | 10 | ### Using a Tuple (Username and Password) 11 | 12 | For basic HTTP authentication, pass a tuple `(username, password)` when initializing a `Client`. 13 | This will automatically include the credentials in the `Authorization` header for all outgoing requests: 14 | 15 | 16 | ```pycon 17 | >>> client = tls_requests.Client(auth=("username", "secret")) 18 | >>> response = client.get("https://www.example.com/") 19 | ``` 20 | 21 | * * * 22 | 23 | ### Using a Custom Function 24 | 25 | To customize how authentication is handled, you can use a function that modifies the request directly: 26 | 27 | ```pycon 28 | >>> def custom_auth(request): 29 | request.headers["X-Authorization"] = "123456" 30 | return request 31 | 32 | >>> response = tls_requests.get("https://httpbin.org/headers", auth=custom_auth) 33 | >>> response 34 | 35 | >>> response.request.headers["X-Authorization"] 36 | '123456' 37 | >>> response.json()["headers"]["X-Authorization"] 38 | '123456' 39 | ``` 40 | 41 | * * * 42 | 43 | Custom Authentication 44 | --------------------- 45 | 46 | For advanced use cases, you can define custom authentication schemes by subclassing `tls_requests.Auth` and overriding the `build_auth` method. 47 | 48 | ### Bearer Token Authentication 49 | 50 | This example demonstrates how to implement Bearer token-based authentication by adding an `Authorization` header: 51 | 52 | 53 | ```python 54 | class BearerAuth(tls_requests.Auth): 55 | def __init__(self, token): 56 | self.token = token 57 | 58 | def build_auth(self, request: tls_requests.Request) -> tls_requests.Request | None: 59 | request.headers["Authorization"] = f"Bearer {self.token}" 60 | return request 61 | ``` 62 | 63 | * * * 64 | 65 | ### Usage Example 66 | 67 | To use your custom `BearerAuth` implementation: 68 | 69 | ```pycon 70 | >>> auth = BearerAuth(token="your_jwt_token") 71 | >>> response = tls_requests.get("https://httpbin.org/headers", auth=auth) 72 | >>> response 73 | 74 | >>> response.request.headers["Authorization"] 75 | 'Bearer your_jwt_token' 76 | >>> response.json()["headers"]["Authorization"] 77 | 'Bearer your_jwt_token' 78 | ``` 79 | 80 | With these approaches, you can integrate various authentication strategies into your `tls_requests` workflow, whether built-in or custom-designed for specific needs. 81 | -------------------------------------------------------------------------------- /docs/advanced/client.md: -------------------------------------------------------------------------------- 1 | Client Usage 2 | ================================ 3 | 4 | This guide details how to utilize the `tls_requests.Client` for efficient and advanced HTTP networking. 5 | If you're transitioning from the popular `requests` library, the `Client` in `tls_requests` provides a powerful alternative with enhanced capabilities. 6 | 7 | * * * 8 | 9 | Why Use a Client? 10 | ----------------- 11 | 12 | !!! hint 13 | If you’re familiar with `requests`, think of `tls_requests.Client()` as the equivalent of `requests.Session()`. 14 | 15 | ### TL;DR 16 | 17 | Use a `Client` instance if you're doing more than one-off scripts or prototypes. It optimizes network resource usage by reusing connections, which is critical for performance when making multiple requests. 18 | 19 | **Advantages:** 20 | 21 | * Efficient connection reuse. 22 | * Simplified configuration sharing across requests. 23 | * Advanced control over request behavior and customization. 24 | 25 | * * * 26 | 27 | Recommended Usage 28 | ----------------- 29 | 30 | ### Using a Context Manager 31 | 32 | The best practice is to use a `Client` as a context manager. This ensures connections are properly cleaned up: 33 | 34 | ```python 35 | with tls_requests.Client() as client: 36 | response = client.get("https://httpbin.org/get") 37 | print(response) # 38 | ``` 39 | 40 | ### Explicit Cleanup 41 | 42 | If not using a context manager, ensure to close the client explicitly: 43 | 44 | ```python 45 | client = tls_requests.Client() 46 | try: 47 | response = client.get("https://httpbin.org/get") 48 | print(response) # 49 | finally: 50 | client.close() 51 | ``` 52 | 53 | * * * 54 | 55 | Making Requests 56 | --------------- 57 | 58 | A `Client` can send requests using methods like `.get()`, `.post()`, etc.: 59 | 60 | ```python 61 | with tls_requests.Client() as client: 62 | response = client.get("https://httpbin.org/get") 63 | print(response) # 64 | ``` 65 | 66 | ### Custom Headers 67 | 68 | To include custom headers in a request: 69 | 70 | ```python 71 | headers = {'X-Custom': 'value'} 72 | with tls_requests.Client() as client: 73 | response = client.get("https://httpbin.org/get", headers=headers) 74 | print(response.request.headers['X-Custom']) # 'value' 75 | ``` 76 | 77 | * * * 78 | 79 | Sharing Configuration Across Requests 80 | ------------------------------------- 81 | 82 | You can apply default configurations, such as headers, for all requests made with the `Client`: 83 | 84 | ```python 85 | headers = {'user-agent': 'my-app/1.0'} 86 | with tls_requests.Client(headers=headers) as client: 87 | response = client.get("https://httpbin.org/headers") 88 | print(response.json()['headers']['User-Agent']) # 'my-app/1.0' 89 | ``` 90 | 91 | * * * 92 | 93 | Merging Configurations 94 | ---------------------- 95 | 96 | When client-level and request-level options overlap: 97 | 98 | * **Headers, query parameters, cookies:** Combined. Example: 99 | 100 | ```python 101 | client_headers = {'X-Auth': 'client'} 102 | request_headers = {'X-Custom': 'request'} 103 | with tls_requests.Client(headers=client_headers) as client: 104 | response = client.get("https://httpbin.org/get", headers=request_headers) 105 | print(response.request.headers['X-Auth']) # 'client' 106 | print(response.request.headers['X-Custom']) # 'request' 107 | ``` 108 | 109 | * **Other parameters:** Request-level options take precedence. 110 | 111 | ```python 112 | with tls_requests.Client(auth=('user', 'pass')) as client: 113 | response = client.get("https://httpbin.org/get", auth=('admin', 'adminpass')) 114 | print(response.request.headers['Authorization']) # Encoded 'admin:adminpass' 115 | 116 | ``` 117 | 118 | * * * 119 | 120 | Advanced Request Handling 121 | ------------------------- 122 | 123 | For more control, explicitly build and send `Request` instances: 124 | 125 | ```python 126 | request = tls_requests.Request("GET", "https://httpbin.org/get") 127 | with tls_requests.Client() as client: 128 | response = client.send(request) 129 | print(response) # 130 | ``` 131 | 132 | To combine client- and request-level configurations: 133 | 134 | ```python 135 | with tls_requests.Client(headers={"X-Client-ID": "ABC123"}) as client: 136 | request = client.build_request("GET", "https://httpbin.org/json") 137 | del request.headers["X-Client-ID"] # Modify as needed 138 | response = client.send(request) 139 | print(response) 140 | ``` 141 | 142 | * * * 143 | 144 | File Uploads 145 | ------------ 146 | 147 | Upload files with control over file name, content, and MIME type: 148 | 149 | ```python 150 | files = {'upload-file': (None, 'text content', 'text/plain')} 151 | response = tls_requests.post("https://httpbin.org/post", files=files) 152 | print(response.json()['form']['upload-file']) # 'text content' 153 | ``` 154 | 155 | For further details, refer to the library's documentation. 156 | -------------------------------------------------------------------------------- /docs/advanced/hooks.md: -------------------------------------------------------------------------------- 1 | Hooks 2 | =========================== 3 | 4 | TLS Requests supports hooks, enabling you to execute custom logic during specific events in the HTTP request/response lifecycle. 5 | These hooks are perfect for logging, monitoring, tracing, or pre/post-processing requests and responses. 6 | 7 | 8 | * * * 9 | 10 | Hook Types 11 | ---------- 12 | 13 | ### 1\. **Request Hook** 14 | 15 | Executed after the request is fully prepared but before being sent to the network. It receives the `request` object, enabling inspection or modification. 16 | 17 | ### 2\. **Response Hook** 18 | 19 | Triggered after the response is fetched from the network but before being returned to the caller. It receives the `response` object, allowing inspection or processing. 20 | 21 | 22 | * * * 23 | 24 | Setting Up Hooks 25 | ---------------- 26 | 27 | Hooks are registered by providing a dictionary with keys `'request'` and/or `'response'`, and their values are lists of callable functions. 28 | 29 | ### Example 1: Logging Requests and Responses 30 | 31 | ```python 32 | def log_request(request): 33 | print(f"Request event hook: {request.method} {request.url} - Waiting for response") 34 | 35 | def log_response(response): 36 | request = response.request 37 | print(f"Response event hook: {request.method} {request.url} - Status {response.status_code}") 38 | 39 | client = tls_requests.Client(hooks={'request': [log_request], 'response': [log_response]}) 40 | ``` 41 | 42 | * * * 43 | 44 | ### Example 2: Raising Errors on 4xx and 5xx Responses 45 | 46 | ```python 47 | def raise_on_4xx_5xx(response): 48 | response.raise_for_status() 49 | 50 | client = tls_requests.Client(hooks={'response': [raise_on_4xx_5xx]}) 51 | ``` 52 | 53 | ### Example 3: Adding a Timestamp Header to Requests 54 | 55 | ```python 56 | import datetime 57 | 58 | def add_timestamp(request): 59 | request.headers['x-request-timestamp'] = datetime.datetime.utcnow().isoformat() 60 | 61 | client = tls_requests.Client(hooks={'request': [add_timestamp]}) 62 | response = client.get('https://httpbin.org/get') 63 | print(response.text) 64 | ``` 65 | 66 | * * * 67 | 68 | Managing Hooks 69 | -------------- 70 | 71 | ### Setting Hooks During Client Initialization 72 | 73 | Provide a dictionary of hooks when creating the client: 74 | 75 | ```python 76 | client = tls_requests.Client(hooks={ 77 | 'request': [log_request], 78 | 'response': [log_response, raise_on_4xx_5xx], 79 | }) 80 | ``` 81 | 82 | ### Dynamically Updating Hooks 83 | 84 | Use the `.hooks` property to inspect or modify hooks after the client is created: 85 | 86 | ```python 87 | client = tls_requests.Client() 88 | 89 | # Add hooks 90 | client.hooks['request'] = [log_request] 91 | client.hooks['response'] = [log_response] 92 | 93 | # Replace hooks 94 | client.hooks = { 95 | 'request': [log_request], 96 | 'response': [log_response, raise_on_4xx_5xx], 97 | } 98 | ``` 99 | 100 | Best Practices 101 | -------------- 102 | 103 | 1. **Access Content**: Use `.read()` or `await .aread()` in asynchronous contexts to access `response.content` before returning it. 104 | 2. **Always Use Lists:** Hooks must be registered as **lists of callables**, even if you are adding only one function. 105 | 3. **Combine Hooks:** You can register multiple hooks for the same event type to handle various concerns, such as logging and error handling. 106 | 4. **Order Matters:** Hooks are executed in the order they are registered. 107 | 108 | With hooks, TLS Requests provides a flexible mechanism to seamlessly integrate monitoring, logging, or custom behaviors into your HTTP workflows. 109 | -------------------------------------------------------------------------------- /docs/advanced/proxies.md: -------------------------------------------------------------------------------- 1 | Using Proxies 2 | ================================ 3 | 4 | The `tls_requests` library supports HTTP and SOCKS proxies for routing traffic through an intermediary server. 5 | This guide explains how to configure proxies for your client or individual requests. 6 | 7 | * * * 8 | 9 | How Proxies Work 10 | ----------------- 11 | 12 | Proxies act as intermediaries between your client and the target server, handling requests and responses on your behalf. They can provide features like anonymity, filtering, or traffic logging. 13 | 14 | * * * 15 | 16 | ### HTTP Proxies 17 | 18 | To route traffic through an HTTP proxy, specify the proxy URL in the `proxy` parameter during client initialization: 19 | 20 | ```python 21 | with tls_requests.Client(proxy="http://localhost:8030") as client: 22 | response = client.get("https://httpbin.org/get") 23 | print(response) # 24 | 25 | ``` 26 | 27 | ### SOCKS Proxies 28 | 29 | For SOCKS proxies, use the `socks5` scheme in the proxy URL: 30 | 31 | ```python 32 | client = tls_requests.Client(proxy="socks5://user:pass@host:port") 33 | response = client.get("https://httpbin.org/get") 34 | print(response) # 35 | ``` 36 | 37 | ### Supported Protocols: 38 | * **HTTP**: Use the `http://` scheme. 39 | * **HTTPS**: Use the `https://` scheme. 40 | * **SOCKS5**: Use the `socks5://` scheme. 41 | 42 | * * * 43 | 44 | ### Proxy Authentication 45 | 46 | You can include proxy credentials in the `userinfo` section of the URL: 47 | 48 | ```python 49 | with tls_requests.Client(proxy="http://username:password@localhost:8030") as client: 50 | response = client.get("https://httpbin.org/get") 51 | print(response) # 52 | ``` 53 | 54 | Key Notes: 55 | ---------- 56 | 57 | * **HTTPS Support**: Both HTTP and SOCKS proxies work for HTTPS requests. 58 | * **Performance**: Using a proxy may slightly impact performance due to the additional routing layer. 59 | * **Security**: Ensure proxy credentials and configurations are handled securely to prevent data leaks. 60 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # TLS REQUESTS 2 | **A powerful and lightweight Python library for making secure and reliable HTTP/TLS fingerprint requests.** 3 | 4 | * * * 5 | 6 | **Installation** 7 | ---------------- 8 | 9 | To install the library, you can choose between two methods: 10 | 11 | #### **1\. Install via PyPI:** 12 | 13 | ```shell 14 | pip install wrapper-tls-requests 15 | ``` 16 | 17 | #### **2\. Install via GitHub Repository:** 18 | 19 | ```shell 20 | pip install git+https://github.com/thewebscraping/tls-requests.git 21 | ``` 22 | 23 | ### Quick Start 24 | 25 | Start using TLS Requests with just a few lines of code: 26 | 27 | ```pycon 28 | >>> import tls_requests 29 | >>> r = tls_requests.get("https://httpbin.org/get") 30 | >>> r 31 | 32 | >>> r.status_code 33 | 200 34 | ``` 35 | 36 | * * * 37 | 38 | **Introduction** 39 | ---------------- 40 | 41 | **TLS Requests** is a cutting-edge HTTP client for Python, offering a feature-rich, highly configurable alternative to the popular [`requests`](https://github.com/psf/requests) library. 42 | 43 | It is built on top of [`tls-client`](https://github.com/bogdanfinn/tls-client), combining ease of use with advanced functionality for secure networking. 44 | 45 | **Acknowledgment**: A big thank you to all contributors for their support! 46 | 47 | ### **Key Benefits** 48 | 49 | * **Bypass TLS Fingerprinting:** Mimic browser-like behaviors to navigate sophisticated anti-bot systems. 50 | * **Customizable TLS Clients:** Select specific TLS fingerprints to meet your needs. 51 | * **Ideal for Developers:** Build scrapers, API clients, or other custom networking tools effortlessly. 52 | 53 | * * * 54 | 55 | **Why Use TLS Requests?** 56 | ------------------------- 57 | 58 | Modern websites increasingly use **TLS Fingerprinting** and anti-bot tools like Cloudflare Bot Fight Mode to block web crawlers. 59 | 60 | **TLS Requests** bypasses these obstacles by mimicking browser-like TLS behaviors, making it easy to scrape data or interact with websites that use sophisticated anti-bot measures. 61 | 62 | ### Cloudflare Bot Fight Mode 63 | ![coingecko.png](static/coingecko.png) 64 | 65 | ### Unlock Content Behind Cloudflare Bot Fight Mode 66 | 67 | **Example Code:** 68 | 69 | ```pycon 70 | >>> import tls_requests 71 | >>> r = tls_requests.get('https://www.coingecko.com/') 72 | >>> r 73 | 74 | ``` 75 | * * * 76 | 77 | **Key Features** 78 | ---------------- 79 | 80 | ### **Enhanced Capabilities** 81 | 82 | * **Browser-like TLS Fingerprinting**: Enables secure and reliable browser-mimicking connections. 83 | * **High-Performance Backend**: Built on a Go-based HTTP backend for speed and efficiency. 84 | * **Synchronous & Asynchronous Support**: Seamlessly switch between synchronous and [asynchronous requests](advanced/async_client). 85 | * **Protocol Support**: Fully compatible with HTTP/1.1 and HTTP/2. 86 | * **Strict Timeouts**: Reliable timeout management for precise control over request durations. 87 | 88 | ### **Additional Features** 89 | 90 | * **Internationalized Domain & URL Support**: Handles non-ASCII URLs effortlessly. 91 | * **Cookie Management**: Ensures session-based cookie persistence. 92 | * **Authentication**: Native support for Basic and Function authentication. 93 | * **Content Decoding**: Automatic handling of gzip and brotli-encoded responses. 94 | * **Hooks**: Perfect for logging, monitoring, tracing, or pre/post-processing requests and responses. 95 | * **Unicode Support**: Effortlessly process Unicode response bodies. 96 | * **File Uploads**: Simplified multipart file upload support. 97 | * **Proxy Configuration**: Supports Socks5, HTTP, and HTTPS proxies for enhanced privacy. 98 | 99 | * * * 100 | 101 | **Documentation** 102 | ----------------- 103 | 104 | Explore the full capabilities of TLS Requests in the documentation: 105 | 106 | * **[Quickstart Guide](quickstart.md)**: A beginner-friendly guide. 107 | * **Advanced Topics**: Learn to leverage specialized features. 108 | * **[Async Support](advanced/async_client)**: Handle high-concurrency scenarios. 109 | * **Custom TLS Configurations**: 110 | * **[Wrapper TLS Client](tls/index)** 111 | * **[TLS Client Profiles](tls/profiles)** 112 | * **[Custom TLS Configurations](tls/configuration)** 113 | 114 | * * * 115 | -------------------------------------------------------------------------------- /docs/quickstart.md: -------------------------------------------------------------------------------- 1 | # Quickstart Guide for TLS Requests 2 | 3 | This guide provides a comprehensive overview of using the `tls_requests` Python library. Follow these examples to integrate the library efficiently into your projects. 4 | 5 | * * * 6 | 7 | Importing `tls_requests` 8 | ------------------------ 9 | 10 | Begin by importing the library: 11 | 12 | ```pycon 13 | >>> import tls_requests 14 | ``` 15 | 16 | Making HTTP Requests 17 | -------------------- 18 | 19 | ### GET Request 20 | 21 | Fetch a webpage using a GET request: 22 | 23 | ```pycon 24 | >>> r = tls_requests.get('https://httpbin.org/get') 25 | >>> r 26 | 27 | ``` 28 | 29 | ### POST Request 30 | 31 | Make a POST request with data: 32 | 33 | ```pycon 34 | >>> r = tls_requests.post('https://httpbin.org/post', data={'key': 'value'}) 35 | ``` 36 | 37 | ### Other HTTP Methods 38 | 39 | Use the same syntax for PUT, DELETE, HEAD, and OPTIONS: 40 | 41 | ```pycon 42 | >>> r = tls_requests.put('https://httpbin.org/put', data={'key': 'value'}) 43 | >>> r 44 | 45 | >>> r = tls_requests.delete('https://httpbin.org/delete') 46 | >>> r 47 | 48 | >>> r = tls_requests.head('https://httpbin.org/get') 49 | 50 | >>> r 51 | >>> r = tls_requests.options('https://httpbin.org/get') 52 | >>> r 53 | 54 | ``` 55 | 56 | * * * 57 | 58 | Using TLS Client Identifiers 59 | ---------------------------- 60 | 61 | Specify a TLS client profile using the [`tls_identifier`](tls/profiles#internal-profiles) parameter: 62 | 63 | ```pycon 64 | >>> r = tls_requests.get('https://httpbin.org/get', tls_identifier="chrome_120") 65 | ``` 66 | 67 | * * * 68 | 69 | HTTP/2 Support 70 | -------------- 71 | 72 | Enable HTTP/2 with the `http2` parameter: 73 | 74 | ```pycon 75 | >>> r = tls_requests.get('https://httpbin.org/get', http2=True, tls_identifier="chrome_120") # firefox_120 76 | ``` 77 | 78 | !!! tip 79 | - **`http2` parameter**: 80 | - `auto` or `None`: Automatically switch between HTTP/2 and HTTP/1, with HTTP/2 preferred. Used in cases of redirect requests. 81 | - `http1` or `False`: Force to HTTP/1. 82 | - `http2`, `True`: Force to HTTP/2. 83 | 84 | * * * 85 | 86 | URL Parameters 87 | -------------- 88 | 89 | Pass query parameters using the `params` keyword: 90 | 91 | ```pycon 92 | >>> import tls_requests 93 | >>> params = {'key1': 'value1', 'key2': 'value2'} 94 | >>> r = tls_requests.get('https://httpbin.org/get', params=params) 95 | >>> r.url 96 | '' 97 | >>> r.url.url 98 | 'https://httpbin.org/get' 99 | >>> r.url.params 100 | 101 | ``` 102 | 103 | Include lists or merge parameters with existing query strings: 104 | 105 | ```pycon 106 | >>> params = {'key1': 'value1', 'key2': ['value2', 'value3']} 107 | >>> r = tls_requests.get('https://httpbin.org/get?order_by=asc', params=params) 108 | >>> r.url 109 | '' 110 | ``` 111 | 112 | * * * 113 | 114 | Custom Headers 115 | -------------- 116 | 117 | Add custom headers to requests: 118 | 119 | ```pycon 120 | >>> url = 'https://httpbin.org/headers' 121 | >>> headers = {'user-agent': 'my-app/1.0.0'} 122 | >>> r = tls_requests.get(url, headers=headers) 123 | >>> r.json() 124 | { 125 | "headers": { 126 | ... 127 | "Host": "httpbin.org", 128 | "User-Agent": "my-app/1.0.0", 129 | ... 130 | } 131 | } 132 | ``` 133 | 134 | 135 | * * * 136 | 137 | Handling Response Content 138 | ------------------------- 139 | 140 | ### Text Content 141 | 142 | Decode response content automatically: 143 | 144 | ```pycon 145 | >>> r = tls_requests.get('https://httpbin.org/get') 146 | >>> print(r.text) 147 | { 148 | "args": {}, 149 | "headers": { 150 | "Accept": "*/*", 151 | "Accept-Encoding": "gzip, deflate, br", 152 | "Host": "httpbin.org", 153 | ... 154 | }, 155 | ... 156 | } 157 | >>> r.encoding 158 | 'UTF-8' 159 | ``` 160 | 161 | ### Binary Content 162 | 163 | Access non-text response content: 164 | 165 | ```pycon 166 | >>> r.content 167 | b'{\n "args": {}, \n "headers": {\n "Accept": "*/*", ...' 168 | ``` 169 | 170 | ### JSON Content 171 | 172 | Parse JSON responses directly: 173 | 174 | ```pycon 175 | >>> r.json() 176 | { 177 | "args": {}, 178 | "headers": { 179 | "Accept": "*/*", 180 | "Accept-Encoding": "gzip, deflate, br", 181 | "Host": "httpbin.org", 182 | ... 183 | }, 184 | ... 185 | } 186 | ``` 187 | 188 | ### Form-Encoded Data 189 | 190 | Include form data in POST requests: 191 | 192 | ```pycon 193 | >>> data = {'key1': 'value1', 'key2': 'value2'} 194 | >>> r = tls_requests.post("https://httpbin.org/post", data=data) 195 | >>> print(r.text) 196 | { 197 | "args": {}, 198 | "data": "key1=value1&key1=value2", 199 | "files": {}, 200 | "form": {}, 201 | ... 202 | } 203 | ``` 204 | 205 | Form encoded data can also include multiple values from a given key. 206 | 207 | ```pycon 208 | >>> data = {'key1': ['value1', 'value2']} 209 | >>> r = tls_requests.post("https://httpbin.org/post", data=data) 210 | >>> print(r.text) 211 | { 212 | ... 213 | "form": { 214 | "key1": [ 215 | "value1", 216 | "value2" 217 | ] 218 | }, 219 | ... 220 | } 221 | ``` 222 | 223 | ### Multipart File Uploads 224 | 225 | Upload files using `files`: 226 | 227 | ```pycon 228 | >>> files = {'image': open('docs.sh/static/load_library.png', 'rb')} 229 | >>> r = tls_requests.post("https://httpbin.org/post", files=files) 230 | >>> print(r.text) 231 | { 232 | "args": {}, 233 | "data": "", 234 | "files": { 235 | "image": "data:image/png;base64, ..." 236 | }, 237 | ... 238 | } 239 | ``` 240 | 241 | Add custom filenames or MIME types: 242 | 243 | ```pycon 244 | >>> files = {'image': ('image.png', open('docs.sh/static/load_library.png', 'rb'), 'image/*')} 245 | >>> r = tls_requests.post("https://httpbin.org/post", files=files) 246 | >>> print(r.text) 247 | { 248 | "args": {}, 249 | "data": "", 250 | "files": { 251 | "image": "data:image/png;base64, ..." 252 | }, 253 | ... 254 | } 255 | ``` 256 | 257 | If you need to include non-file data fields in the multipart form, use the `data=...` parameter: 258 | 259 | ```pycon 260 | >>> data = {'key1': ['value1', 'value2']} 261 | >>> files = {'image': open('docs.sh/static/load_library.png', 'rb')} 262 | >>> r = tls_requests.post("https://httpbin.org/post", data=data, files=files) 263 | >>> print(r.text) 264 | { 265 | "args": {}, 266 | "data": "", 267 | "files": { 268 | "image": "data:image/png;base64, ..." 269 | }, 270 | "form": { 271 | "key1": [ 272 | "value1", 273 | "value2" 274 | ] 275 | }, 276 | ... 277 | } 278 | ``` 279 | 280 | ### JSON Data 281 | 282 | Send complex JSON data structures: 283 | 284 | ```pycon 285 | >>> data = { 286 | 'integer': 1, 287 | 'boolean': True, 288 | 'list': ['1', '2', '3'], 289 | 'data': {'key': 'value'} 290 | } 291 | >>> r = tls_requests.post("https://httpbin.org/post", json=data) 292 | >>> print(r.text) 293 | { 294 | ... 295 | "json": { 296 | "boolean": true, 297 | "data": { 298 | "key": "value" 299 | }, 300 | "integer": 1, 301 | "list": [ 302 | "1", 303 | "2", 304 | "3" 305 | ] 306 | }, 307 | ... 308 | } 309 | ``` 310 | 311 | * * * 312 | 313 | Inspecting Responses 314 | -------------------- 315 | 316 | ### Status Codes 317 | 318 | Check the HTTP status code: 319 | 320 | ```pycon 321 | >>> r = tls_requests.get('https://httpbin.org/get') 322 | >>> r.status_code 323 | 200 324 | ``` 325 | 326 | Raise exceptions for non-2xx responses: 327 | 328 | ```pycon 329 | >>> not_found = tls_requests.get('https://httpbin.org/status/404') 330 | >>> not_found.status_code 331 | 404 332 | >>> not_found.raise_for_status() 333 | ``` 334 | ```text 335 | Traceback (most recent call last): 336 | *** 337 | File "", line 1, in 338 | File "***tls_requests/models/response.py", line 184, in raise_for_status 339 | raise HTTPError( 340 | tls_requests.exceptions.HTTPError: 404 Client Error: Not Found for url: https://httpbin.org/status/404 341 | ``` 342 | 343 | Any successful response codes will return the `Response` instance rather than raising an exception. 344 | 345 | ```pycon 346 | >>> r = tls_requests.get('https://httpbin.org/get') 347 | >>> raw = r.raise_for_status().text 348 | >>> print(raw) 349 | { 350 | "args": {}, 351 | "headers": { 352 | "Accept": "*/*", 353 | "Accept-Encoding": "gzip, deflate, br", 354 | "Host": "httpbin.org", 355 | ... 356 | }, 357 | ... 358 | } 359 | ``` 360 | 361 | ### Headers 362 | 363 | Access headers as a dictionary: 364 | 365 | ```pycon 366 | >>> r.headers 367 | 376 | ``` 377 | 378 | The `Headers` data type is case-insensitive, so you can use any capitalization. 379 | 380 | ```pycon 381 | >>> r.headers['Content-Type'] 382 | 'application/json' 383 | ``` 384 | 385 | ### Cookies 386 | 387 | Access cookies or include them in requests: 388 | 389 | ```pycon 390 | >>> url = 'https://httpbin.org/cookies/set?foo=bar' 391 | >>> r = tls_requests.get(url, follow_redirects=True) 392 | >>> r.cookies['foo'] 393 | 'bar' 394 | ``` 395 | 396 | * * * 397 | 398 | Redirection Handling 399 | -------------------- 400 | 401 | Control redirect behavior using the `follow_redirects` parameter: 402 | 403 | ```pycon 404 | >>> redirect_url = 'https://httpbin.org/absolute-redirect/3' 405 | >>> r = tls_requests.get(redirect_url, follow_redirects=False) 406 | >>> r 407 | 408 | >>> r.history 409 | [] 410 | >>> r.next 411 | 412 | ``` 413 | 414 | You can modify the default redirection handling with the `follow_redirects` parameter: 415 | 416 | ```pycon 417 | >>> redirect_url = 'https://httpbin.org/absolute-redirect/3' 418 | >>> r = tls_requests.get(redirect_url, follow_redirects=True) 419 | >>> r.status_code 420 | 200 421 | >>> r.history 422 | [, , ] 423 | ``` 424 | 425 | The `history` property of the response can be used to inspect any followed redirects. 426 | It contains a list of any redirect responses that were followed, in the order 427 | in which they were made. 428 | 429 | Timeouts 430 | -------- 431 | 432 | Set custom timeouts: 433 | 434 | ```pycon 435 | >>> tls_requests.get('https://github.com/', timeout=10) 436 | ``` 437 | 438 | * * * 439 | 440 | Authentication 441 | -------------- 442 | 443 | Perform Basic Authentication: 444 | 445 | ```pycon 446 | >>> r = tls_requests.get("https://httpbin.org/get", auth=("admin", "admin")) 447 | ``` 448 | 449 | * * * 450 | 451 | Exceptions 452 | ---------- 453 | 454 | Handle exceptions for network errors or invalid responses: 455 | 456 | ```python 457 | try: 458 | r = tls_requests.get('https://httpbin.org/status/404') 459 | r.raise_for_status() 460 | except tls_requests.exceptions.HTTPError as e: 461 | print(e) 462 | ``` 463 | -------------------------------------------------------------------------------- /docs/static/coingecko.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thewebscraping/tls-requests/79648285c47f65e101fcb8181b9512c71b3d2796/docs/static/coingecko.png -------------------------------------------------------------------------------- /docs/tls/configuration.md: -------------------------------------------------------------------------------- 1 | To use custom TLS Client configuration follow these instructions: 2 | 3 | 4 | ### Default TLS Config 5 | The `TLSConfig` class provides a structured and flexible way to configure TLS-specific settings for HTTP requests. 6 | It supports features like custom headers, cookie handling, proxy configuration, and advanced TLS options. 7 | 8 | Example: 9 | Initialize a `TLSConfig` object using predefined or default settings: 10 | 11 | ```pycon 12 | >>> import tls_requests 13 | >>> kwargs = { 14 | "catchPanics": false, 15 | "certificatePinningHosts": {}, 16 | "customTlsClient": {}, 17 | "followRedirects": false, 18 | "forceHttp1": false, 19 | "headerOrder": [ 20 | "accept", 21 | "user-agent", 22 | "accept-encoding", 23 | "accept-language" 24 | ], 25 | "headers": { 26 | "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", 27 | "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", 28 | "accept-encoding": "gzip, deflate, br", 29 | "accept-language": "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7" 30 | }, 31 | "insecureSkipVerify": false, 32 | "isByteRequest": false, 33 | "isRotatingProxy": false, 34 | "proxyUrl": "", 35 | "requestBody": "", 36 | "requestCookies": [ 37 | { 38 | "_name": "foo", 39 | "value": "bar", 40 | }, 41 | { 42 | "_name": "bar", 43 | "value": "foo", 44 | }, 45 | ], 46 | "requestMethod": "GET", 47 | "requestUrl": "https://microsoft.com", 48 | "sessionId": "2my-session-id", 49 | "timeoutSeconds": 30, 50 | "tlsClientIdentifier": "chrome_120", 51 | "withDebug": false, 52 | "withDefaultCookieJar": false, 53 | "withRandomTLSExtensionOrder": false, 54 | "withoutCookieJar": false 55 | } 56 | >>> obj = tls_requests.tls.TLSConfig.from_kwargs(**kwargs) 57 | >>> config_kwargs = obj.to_dict() 58 | >>> r = tls_requests.get("https://httpbin.org/get", **config_kwargs) 59 | >>> r 60 | 61 | ``` 62 | 63 | ### Custom TLS Client Configuration 64 | 65 | The `CustomTLSClientConfig` class defines advanced configuration options for customizing TLS client behavior. 66 | It includes support for ALPN, ALPS protocols, certificate compression, HTTP/2 settings, JA3 fingerprints, and 67 | other TLS-related settings. 68 | 69 | Example: 70 | Create a `CustomTLSClientConfig` instance with specific settings: 71 | 72 | ```pycon 73 | >>> import tls_requests 74 | >>> kwargs = { 75 | "alpnProtocols": [ 76 | "h2", 77 | "http/1.1" 78 | ], 79 | "alpsProtocols": [ 80 | "h2" 81 | ], 82 | "certCompressionAlgo": "brotli", 83 | "connectionFlow": 15663105, 84 | "h2Settings": { 85 | "HEADER_TABLE_SIZE": 65536, 86 | "MAX_CONCURRENT_STREAMS": 1000, 87 | "INITIAL_WINDOW_SIZE": 6291456, 88 | "MAX_HEADER_LIST_SIZE": 262144 89 | }, 90 | "h2SettingsOrder": [ 91 | "HEADER_TABLE_SIZE", 92 | "MAX_CONCURRENT_STREAMS", 93 | "INITIAL_WINDOW_SIZE", 94 | "MAX_HEADER_LIST_SIZE" 95 | ], 96 | "headerPriority": null, 97 | "ja3String": "771,2570-4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,2570-0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513-2570-21,2570-29-23-24,0", 98 | "keyShareCurves": [ 99 | "GREASE", 100 | "X25519" 101 | ], 102 | "priorityFrames": [], 103 | "pseudoHeaderOrder": [ 104 | ":method", 105 | ":authority", 106 | ":scheme", 107 | ":path" 108 | ], 109 | "supportedSignatureAlgorithms": [ 110 | "ECDSAWithP256AndSHA256", 111 | "PSSWithSHA256", 112 | "PKCS1WithSHA256", 113 | "ECDSAWithP384AndSHA384", 114 | "PSSWithSHA384", 115 | "PKCS1WithSHA384", 116 | "PSSWithSHA512", 117 | "PKCS1WithSHA512" 118 | ], 119 | "supportedVersions": [ 120 | "GREASE", 121 | "1.3", 122 | "1.2" 123 | ] 124 | } 125 | >>> custom_tls_client = tls_requests.tls.CustomTLSClientConfig.from_kwargs(**kwargs) 126 | >>> config_obj = tls_requests.tls.TLSConfig(customTlsClient=custom_tls_client, tlsClientIdentifier=None) 127 | >>> config_kwargs = config_obj.to_dict() 128 | >>> r = tls_requests.get("https://httpbin.org/get", **config_kwargs) 129 | >>> r 130 | 131 | ``` 132 | 133 | !!! note 134 | When using `CustomTLSClientConfig`, the `tlsClientIdentifier` parameter in TLSConfig is set to None. 135 | 136 | ### Passing Request Parameters Directly 137 | 138 | ```pycon 139 | >>> import tls_requests 140 | >>> r = tls_requests.get( 141 | url = "https://httpbin.org/get", 142 | proxy = "https://abc:123456@127.0.0.1:8080", 143 | http2 = True, 144 | timeout = 10.0, 145 | follow_redirects = True, 146 | verify = True, 147 | tls_identifier = "chrome_120", 148 | **config, 149 | ) 150 | >>> r 151 | 152 | ``` 153 | 154 | !!! note 155 | When using the `customTlsClient` parameter within `**config`, the `tls_identifier` parameter will not be set. 156 | Parameters such as `headers`, `cookies`, `proxy`, `timeout`, `verify`, and `tls_identifier` will override the existing configuration in TLSConfig. 157 | 158 | ### `Client` and `AsyncClient` Parameters 159 | ```pycon 160 | >>> import tls_requests 161 | >>> client = tls_requests.Client( 162 | proxy = "https://abc:123456@127.0.0.1:8080", 163 | http2 = True, 164 | timeout = 10.0, 165 | follow_redirects = True, 166 | verify = True, 167 | tls_identifier = "chrome_120", 168 | **config, 169 | ) 170 | >>> r = client.get(url = "https://httpbin.org/get",) 171 | >>> r 172 | 173 | 174 | ``` 175 | 176 | !!! note 177 | The `Client` and `AsyncClient` interfaces in `tls_requests` enable reusable and shared configurations for multiple requests, providing a more convenient and efficient approach for handling HTTP requests. 178 | -------------------------------------------------------------------------------- /docs/tls/index.md: -------------------------------------------------------------------------------- 1 | TLS-Client Documentation 2 | ======================== 3 | 4 | **Acknowledgment** 5 | 6 | Special thanks to [`bogdanfinn`](https://github.com/bogdanfinn/tls-client). For more details, visit the [GitHub repository](https://github.com/bogdanfinn/tls-client) or explore the [documentation](https://bogdanfinn.gitbook.io/open-source-oasis). 7 | 8 | ## Wrapper TLS Client 9 | 10 | The `TLSClient` class is a utility for managing and interacting with TLS sessions using a native library. It provides methods to handle cookies, sessions, and make HTTP requests with advanced TLS configurations. 11 | 12 | The TLSClient class is designed to be used as a singleton-like interface. Upon first instantiation, the class initializes the underlying native TLS library and sets up method bindings. 13 | 14 | ```pycon 15 | >>> from tls_requests import TLSClient 16 | >>> TLSClient.initialize() 17 | ``` 18 | 19 | !!! note 20 | The first time you initialize the TLSClient class, it will automatically find and load the appropriate library for your machine. 21 | 22 | ### Methods 23 | 24 | * * * 25 | #### `setup()` 26 | 27 | Initializes the native TLS library and binds its functions to the class methods. 28 | 29 | * **Purpose**: Sets up the library functions and their argument/return types for use in other methods. 30 | * **Usage**: This is automatically called when the class is first instantiated. 31 | 32 | ```pycon 33 | >>> from tls_requests import TLSClient 34 | >>> client = TLSClient.initialize() 35 | ``` 36 | 37 | * * * 38 | 39 | #### `get_cookies(session_id: TLSSessionId, url: str) -> dict` 40 | 41 | Retrieves cookies associated with a session for a specific URL. 42 | 43 | * **Parameters**: 44 | * `session_id` (_TLSSessionId_): The identifier for the TLS session. 45 | * `url` (_str_): The URL for which cookies are requested. 46 | * **Returns**: A dictionary of cookies. 47 | 48 | ```pycon 49 | >>> from tls_requests import TLSClient 50 | >>> TLSClient.initialize() 51 | >>> cookies = TLSClient.get_cookies(session_id="session123", url="https://httpbin.org/get") 52 | ``` 53 | 54 | * * * 55 | 56 | #### `add_cookies(session_id: TLSSessionId, payload: dict)` 57 | 58 | Adds cookies to a specific TLS session. 59 | 60 | * **Parameters**: 61 | * `session_id` (_TLSSessionId_): The identifier for the TLS session. 62 | * `payload` (_dict_): A dictionary containing cookies to be added. 63 | * **Returns**: The response object from the library. 64 | 65 | ```pycon 66 | >>> from tls_requests import TLSClient 67 | >>> TLSClient.initialize() 68 | >>> payload = { 69 | "cookies": [{ 70 | "_name": "foo2", 71 | "value": "bar2", 72 | },{ 73 | "_name": "bar2", 74 | "value": "baz2", 75 | }], 76 | "sessionId": "session123", 77 | "url": "https://httpbin.org/", 78 | } 79 | >>> TLSClient.add_cookies(session_id="session123", payload=payload) 80 | ``` 81 | 82 | * * * 83 | 84 | #### `destroy_all() -> bool` 85 | 86 | Destroys all active TLS sessions. 87 | 88 | * **Returns**: `True` if all sessions were successfully destroyed, otherwise `False`. 89 | 90 | ```pycon 91 | >>> from tls_requests import TLSClient 92 | >>> TLSClient.initialize() 93 | >>> success = TLSClient.destroy_all() 94 | ``` 95 | 96 | * * * 97 | #### `destroy_session(session_id: TLSSessionId) -> bool` 98 | 99 | Destroys a specific TLS session. 100 | 101 | * **Parameters**: 102 | * `session_id` (_TLSSessionId_): The identifier for the session to be destroyed. 103 | * **Returns**: `True` if the session was successfully destroyed, otherwise `False`. 104 | 105 | ```pycon 106 | >>> from tls_requests import TLSClient 107 | >>> TLSClient.initialize() 108 | success = TLSClient.destroy_session(session_id="session123") 109 | ``` 110 | 111 | 112 | * * * 113 | 114 | #### `free_memory(response_id: TLSSessionId)` 115 | 116 | Frees memory associated with a specific response. 117 | 118 | * **Parameters**: 119 | * `response_id` (_str_): The identifier for the response to be freed. 120 | * **Returns**: None. 121 | 122 | ```pycon 123 | >>> from tls_requests import TLSClient 124 | >>> TLSClient.initialize() 125 | >>> TLSClient.free_memory(response_id="response123") 126 | ``` 127 | 128 | * * * 129 | 130 | #### `request(payload: dict)` 131 | 132 | Sends a request using the TLS library. Using [TLSConfig](configuration) to generate payload. 133 | 134 | * **Parameters**: 135 | * `payload` (_dict_): A dictionary containing the request payload (e.g., method, headers, body, etc.). 136 | * **Returns**: The response object from the library. 137 | 138 | ```pycon 139 | >>> from tls_requests import TLSClient, TLSConfig 140 | >>> TLSClient.initialize() 141 | >>> config = TLSConfig(requestMethod="GET", requestUrl="https://httpbin.org/get") 142 | >>> response = TLSClient.request(config.to_dict()) 143 | ``` 144 | 145 | * * * 146 | 147 | #### `response(raw: bytes) -> TLSResponse` 148 | 149 | Parses a raw byte response and frees associated memory. 150 | 151 | * **Parameters**: 152 | * `raw` (_bytes_): The raw byte response from the TLS library. 153 | * **Returns**: A `TLSResponse` object. 154 | 155 | ```pycon 156 | >>> from tls_requests import TLSClient 157 | >>> TLSClient.initialize() 158 | >>> parsed_response = TLSClient.response(raw_bytes) 159 | ``` 160 | -------------------------------------------------------------------------------- /docs/tls/install.md: -------------------------------------------------------------------------------- 1 | ## Auto Download 2 | 3 | This approach simplifies usage as it automatically detects your OS and downloads the appropriate version of the library. To use it: 4 | 5 | ```pycon 6 | >>> import tls_requests 7 | >>> r = tls_requests.get('https://httpbin.org/get') 8 | ``` 9 | 10 | !!! note: 11 | The library takes care of downloading necessary files and stores them in the `tls_requests/bin` directory. 12 | 13 | ## Manual Download 14 | 15 | If you want more control, such as selecting a specific version of the library, you can use the manual method: 16 | 17 | ```pycon 18 | >>> from tls_requests.models.libraries import TLSLibrary 19 | >>> TLSLibrary.download('1.7.10') 20 | ``` 21 | 22 | This method is useful if you need to ensure compatibility with specific library versions. 23 | 24 | ### Notes 25 | 26 | 1. **Dependencies**: Ensure Python is installed and configured correctly in your environment. 27 | 2. **Custom Directory**: If needed, the library’s downloaded binaries can be relocated manually to suit specific project structures. 28 | 3. **Reference**: [TLS Client GitHub Releases](https://github.com/bogdanfinn/tls-client/releases/) provides details about available versions and updates. 29 | -------------------------------------------------------------------------------- /docs/tls/profiles.md: -------------------------------------------------------------------------------- 1 | Default TLS Configuration 2 | --------------------- 3 | 4 | When initializing a `Client` or `AsyncClient`, a `TLSClient` instance is created with the following default settings: 5 | 6 | * **Timeout:** 30 seconds. 7 | * **Profile:** Chrome 120. 8 | * **Random TLS Extension Order:** Enabled. 9 | * **Redirects:** Always `False`. 10 | * **Idle Connection Closure:** After 90 seconds. 11 | * **Session ID:** Auto generate V4 UUID string if set to None. 12 | * **Force HTTP/1.1:** Default `False`. 13 | 14 | All requests use [`Bogdanfinn's TLS-Client`](https://github.com/bogdanfinn/tls-client) to spoof the TLS client fingerprint. This process is automatic and transparent to the user. 15 | 16 | ```python 17 | import tls_requests 18 | r = tls_requests.get("https://httpbin.org/get", tls_identifier="chrome_120") 19 | print(r) # Output: 20 | 21 | ``` 22 | 23 | * * * 24 | 25 | Supported Client Profiles 26 | ------------------------- 27 | 28 | ### Internal Profiles 29 | 30 | #### Chrome 31 | 32 | * 103 (`chrome_103`) 33 | * 104 (`chrome_104`) 34 | * 105 (`chrome_105`) 35 | * 106 (`chrome_106`) 36 | * 107 (`chrome_107`) 37 | * 108 (`chrome_108`) 38 | * 109 (`chrome_109`) 39 | * 110 (`chrome_110`) 40 | * 111 (`chrome_111`) 41 | * 112 (`chrome_112`) 42 | * 116 with PSK (`chrome_116_PSK`) 43 | * 116 with PSK and PQ (`chrome_116_PSK_PQ`) 44 | * 117 (`chrome_117`) 45 | * 120 (`chrome_120`) 46 | * 124 (`chrome_124`) 47 | * 131 (`chrome_131`) 48 | * 131 with PSK (`chrome_131_PSK`) 49 | 50 | #### Safari 51 | 52 | * 15.6.1 (`safari_15_6_1`) 53 | * 16.0 (`safari_16_0`) 54 | 55 | #### iOS (Safari) 56 | 57 | * 15.5 (`safari_ios_15_5`) 58 | * 15.6 (`safari_ios_15_6`) 59 | * 16.0 (`safari_ios_16_0`) 60 | * 17.0 (`safari_ios_17_0`) 61 | 62 | #### iPadOS (Safari) 63 | 64 | * 15.6 (`safari_ios_15_6`) 65 | 66 | #### Firefox 67 | 68 | * 102 (`firefox_102`) 69 | * 104 (`firefox_104`) 70 | * 105 (`firefox_105`) 71 | * 106 (`firefox_106`) 72 | * 108 (`firefox_108`) 73 | * 110 (`firefox_110`) 74 | * 117 (`firefox_117`) 75 | * 120 (`firefox_120`) 76 | * 123 (`firefox_123`) 77 | * 132 (`firefox_132`) 78 | 79 | #### Opera 80 | 81 | * 89 (`opera_89`) 82 | * 90 (`opera_90`) 83 | * 91 (`opera_91`) 84 | 85 | * * * 86 | 87 | ### Custom Profiles 88 | 89 | * Zalando iOS Mobile (`zalando_ios_mobile`) 90 | * Nike iOS Mobile (`nike_ios_mobile`) 91 | * Cloudscraper 92 | * MMS iOS (`mms_ios` or `mms_ios_1`) 93 | * MMS iOS 2 (`mms_ios_2`) 94 | * MMS iOS 3 (`mms_ios_3`) 95 | * Mesh iOS (`mesh_ios` or `mesh_ios_1`) 96 | * Mesh Android (`mesh_android` or `mesh_android_1`) 97 | * Mesh Android 2 (`mesh_android_2`) 98 | * Confirmed iOS (`confirmed_ios`) 99 | * Zalando Android Mobile (`zalando_android_mobile`) 100 | * Confirmed Android (`confirmed_android`) 101 | * Confirmed Android 2 (`confirmed_android_2`) 102 | 103 | #### OkHttp4 104 | 105 | * Android 7 (`okhttp4_android_7`) 106 | * Android 8 (`okhttp4_android_8`) 107 | * Android 9 (`okhttp4_android_9`) 108 | * Android 10 (`okhttp4_android_10`) 109 | * Android 11 (`okhttp4_android_11`) 110 | * Android 12 (`okhttp4_android_12`) 111 | * Android 13 (`okhttp4_android_13`) 112 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: TLS Requests 2 | site_description: A powerful and lightweight Python library for making secure and reliable HTTP/TLS Fingerprint requests. 3 | site_url: https://github.com/thewebscraping/ 4 | 5 | theme: 6 | name: 'material' 7 | palette: 8 | - scheme: 'default' 9 | media: '(prefers-color-scheme: light)' 10 | toggle: 11 | icon: 'material/lightbulb' 12 | name: "Switch to dark mode" 13 | - scheme: 'slate' 14 | media: '(prefers-color-scheme: dark)' 15 | primary: 'blue' 16 | toggle: 17 | icon: 'material/lightbulb-outline' 18 | name: 'Switch to light mode' 19 | 20 | repo_name: thewebscraping/tls-requests 21 | repo_url: https://github.com/thewebscraping/tls-requests/ 22 | edit_uri: "" 23 | 24 | nav: 25 | - Introduction: 'index.md' 26 | - Quickstart Guide: 'quickstart.md' 27 | - Advanced Topics: 28 | - Client: 'advanced/client.md' 29 | - Async Client: 'advanced/async_client.md' 30 | - Authentication: 'advanced/authentication.md' 31 | - Hooks: 'advanced/hooks.md' 32 | - Proxies: 'advanced/proxies.md' 33 | - TLS Client: 34 | - Install: 'tls/install.md' 35 | - Wrapper TLS Client: 'tls/index.md' 36 | - TLS Client Profiles: 'tls/profiles.md' 37 | - Custom TLS Client Configs: 'tls/configuration.md' 38 | 39 | markdown_extensions: 40 | - admonition 41 | - codehilite: 42 | css_class: highlight 43 | - mkautodoc 44 | 45 | extra_css: 46 | - css/custom.css 47 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ['setuptools>=40.8.0'] 3 | build-backend = 'setuptools.build_meta' 4 | 5 | [tool.pytest.ini_options] 6 | asyncio_mode = "auto" 7 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | 3 | # Documentation 4 | mkdocs==1.6.1 5 | mkautodoc==0.2.0 6 | mkdocs-material==9.5.39 7 | 8 | # Packaging 9 | setuptools~=75.3.0 10 | twine~=6.0.1 11 | 12 | # Tests & Linting 13 | pre-commit 14 | pytest-cov 15 | Werkzeug 16 | black==24.3.0 17 | coverage[toml]==7.6.1 18 | isort==5.13.2 19 | flake8==7.1.1 20 | mypy==1.11.2 21 | pytest==8.3.3 22 | pytest-asyncio==0.24.0 23 | pytest_httpserver==1.1.0 24 | tox==4.23.2 25 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Base 2 | chardet~=5.2.0 3 | requests~=2.32.3 4 | tqdm~=4.67.1 5 | idna~=3.10 6 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | long_description = file: README.md 3 | long_description_content_type = text/markdown 4 | license = MIT 5 | license_file = LICENSE 6 | python_requires = >=3.8 7 | install_requires = 8 | chardet ~= 5.2.0 9 | requests ~= 2.32.3 10 | tqdm ~= 4.67.1 11 | idna ~= 3.10 12 | classifiers = 13 | Development Status :: 5 - Production/Stable 14 | Intended Audience :: Developers 15 | Environment :: Web Environment 16 | License :: OSI Approved :: MIT License 17 | Natural Language :: English 18 | Operating System :: OS Independent 19 | Programming Language :: Python 20 | Programming Language :: Python :: 3 21 | Programming Language :: Python :: 3.8 22 | Programming Language :: Python :: 3.9 23 | Programming Language :: Python :: 3.10 24 | Programming Language :: Python :: 3.11 25 | Programming Language :: Python :: 3.12 26 | Programming Language :: Python :: 3.13 27 | Programming Language :: Python :: 3 :: Only 28 | Programming Language :: Python :: Implementation :: CPython 29 | Programming Language :: Python :: Implementation :: PyPy 30 | Topic :: Internet :: WWW/HTTP 31 | Topic :: Software Development :: Libraries 32 | 33 | project_urls = 34 | Changelog = https://github.com/thewebscraping/tls-requests/blob/main/CHANGELOG.md 35 | Documentation = https://thewebscraping.github.io/tls-requests/ 36 | Source = https://github.com/thewebscraping/tls-requests 37 | Homepage = https://github.com/thewebscraping/tls-requests 38 | 39 | [options] 40 | zip_safe = False 41 | include_package_data = True 42 | packages = find: 43 | 44 | [options.packages.find] 45 | exclude = 46 | examples* 47 | tools* 48 | docs* 49 | 50 | [flake8] 51 | max-line-length = 120 52 | ignore = F821, E203, W503, E501, E231 53 | per-file-ignores = 54 | __init__.py: F405, F403, E402, F401 55 | 56 | [tool.black] 57 | line-length = 120 58 | include = '\.pyi?$' 59 | unstable = true 60 | 61 | 62 | [tool.isort] 63 | atomic = true 64 | profile = "black" 65 | line_length = 120 66 | skip_gitignore = true 67 | skip_glob = ["tests/data", "profiling"] 68 | known_first_party = ["black", "blib2to3", "blackd", "_black_version"] 69 | add_imports = "from __future__ import annotations" 70 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import re 5 | import sys 6 | from codecs import open 7 | 8 | from setuptools import setup 9 | 10 | BASE_DIR = os.path.dirname(__file__) 11 | CURRENT_PYTHON = sys.version_info[:2] 12 | REQUIRED_PYTHON = (3, 8) 13 | 14 | if CURRENT_PYTHON < REQUIRED_PYTHON: 15 | sys.stderr.write( 16 | """Python version not supported, you need to use Python version >= {}.{}""".format( 17 | *REQUIRED_PYTHON 18 | ) 19 | ) 20 | sys.exit(1) 21 | 22 | 23 | def normalize(name) -> str: 24 | name = re.sub(r"\s+", "-", name) 25 | return re.sub(r"[-_.]+", "-", name).lower() 26 | 27 | 28 | version = {} 29 | with open(os.path.join(BASE_DIR, "tls_requests", "__version__.py"), "r", "utf-8") as f: 30 | exec(f.read(), version) 31 | 32 | if __name__ == "__main__": 33 | setup( 34 | name=version["__title__"], 35 | version=version["__version__"], 36 | description=version["__description__"], 37 | long_description_content_type="text/markdown", 38 | author=version["__author__"], 39 | author_email=version["__author_email__"], 40 | url=version["__url__"], 41 | ) 42 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thewebscraping/tls-requests/79648285c47f65e101fcb8181b9512c71b3d2796/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import tls_requests 2 | 3 | pytest_plugins = ['pytest_httpserver', 'pytest_asyncio'] 4 | 5 | 6 | def pytest_configure(config): 7 | tls_requests.TLSLibrary.load() 8 | -------------------------------------------------------------------------------- /tests/files/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thewebscraping/tls-requests/79648285c47f65e101fcb8181b9512c71b3d2796/tests/files/__init__.py -------------------------------------------------------------------------------- /tests/files/coingecko.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thewebscraping/tls-requests/79648285c47f65e101fcb8181b9512c71b3d2796/tests/files/coingecko.png -------------------------------------------------------------------------------- /tests/test_api.py: -------------------------------------------------------------------------------- 1 | import tls_requests 2 | 3 | RESPONSE_BYTE = b"Hello World!" 4 | RESPONSE_TEXT = "Hello World!" 5 | 6 | 7 | def assert_response(response): 8 | assert response.status_code, 200 9 | assert response.reason, "OK" 10 | assert response.text, RESPONSE_TEXT 11 | assert response.content, RESPONSE_BYTE 12 | 13 | 14 | def make_request(request_fn, httpserver, is_assert_response: bool = True): 15 | httpserver.expect_request("/api").respond_with_data(RESPONSE_BYTE) 16 | response = request_fn(httpserver.url_for('/api')) 17 | if is_assert_response: 18 | assert_response(response) 19 | 20 | return response 21 | 22 | 23 | def test_get(httpserver): 24 | make_request(tls_requests.get, httpserver) 25 | 26 | 27 | def test_post(httpserver): 28 | make_request(tls_requests.post, httpserver) 29 | 30 | 31 | def test_put(httpserver): 32 | make_request(tls_requests.put, httpserver) 33 | 34 | 35 | def test_patch(httpserver): 36 | make_request(tls_requests.patch, httpserver) 37 | 38 | 39 | def test_delete(httpserver): 40 | make_request(tls_requests.delete, httpserver) 41 | 42 | 43 | def test_options(httpserver): 44 | make_request(tls_requests.options, httpserver) 45 | 46 | 47 | def test_head(httpserver): 48 | response = make_request(tls_requests.head, httpserver, False) 49 | assert response.status_code == 200 50 | assert response.reason == "OK" 51 | assert response.text == "" 52 | assert response.content == b"" 53 | -------------------------------------------------------------------------------- /tests/test_auth.py: -------------------------------------------------------------------------------- 1 | from base64 import b64encode 2 | 3 | import pytest 4 | 5 | import tls_requests 6 | 7 | auth = ("user", "pass") 8 | AUTH_TOKEN = "Basic %s" % b64encode(b":".join([s.encode() for s in auth])).decode() 9 | AUTH_HEADERS = {"authorization": AUTH_TOKEN} 10 | AUTH_FUNCTION_KEY = "x-authorization" 11 | AUTH_FUNCTION_VALUE = "123456" 12 | AUTH_FUNCTION_HEADERS = {AUTH_FUNCTION_KEY: AUTH_FUNCTION_VALUE} 13 | 14 | 15 | def auth_function(request): 16 | request.headers.update(AUTH_FUNCTION_HEADERS) 17 | 18 | 19 | @pytest.fixture 20 | def auth_url(httpserver): 21 | return httpserver.url_for('/auth') 22 | 23 | 24 | @pytest.fixture 25 | def http_auth_function(httpserver): 26 | httpserver.expect_request("/auth", headers=AUTH_FUNCTION_HEADERS).respond_with_data() 27 | return httpserver 28 | 29 | 30 | @pytest.fixture 31 | def http_auth(httpserver): 32 | httpserver.expect_request("/auth", headers=AUTH_HEADERS).respond_with_data() 33 | return httpserver 34 | 35 | 36 | def test_auth(http_auth, auth_url): 37 | response = tls_requests.get(auth_url, auth=auth) 38 | assert response.status_code == 200 39 | assert response.request.headers["Authorization"] == AUTH_TOKEN 40 | 41 | 42 | def test_auth_function(http_auth_function, auth_url): 43 | response = tls_requests.get(auth_url, auth=auth_function) 44 | assert response.status_code == 200 45 | assert response.request.headers[AUTH_FUNCTION_KEY] == AUTH_FUNCTION_VALUE 46 | 47 | 48 | def test_client_auth(http_auth, auth_url): 49 | with tls_requests.Client(auth=auth) as client: 50 | response = client.get(auth_url) 51 | 52 | assert response.status_code == 200 53 | assert bool(response.closed == client.closed) is True 54 | assert response.request.headers["Authorization"] == AUTH_TOKEN 55 | 56 | 57 | def test_client_auth_cross_sharing(http_auth, auth_url): 58 | with tls_requests.Client(auth=('1', '2')) as client: 59 | response = client.get(auth_url, auth=auth) 60 | 61 | assert response.status_code == 200 62 | assert bool(response.closed == client.closed) is True 63 | assert response.request.headers["Authorization"] == AUTH_TOKEN 64 | 65 | 66 | def test_client_auth_function_cross_sharing(http_auth_function, auth_url): 67 | with tls_requests.Client(auth=auth) as client: 68 | response = client.get(auth_url, auth=auth_function) 69 | 70 | assert response.status_code == 200 71 | assert bool(response.closed == client.closed) is True 72 | assert response.request.headers[AUTH_FUNCTION_KEY] == AUTH_FUNCTION_VALUE 73 | 74 | 75 | @pytest.mark.asyncio 76 | async def test_async_auth(http_auth, auth_url): 77 | async with tls_requests.AsyncClient(auth=auth) as client: 78 | response = await client.get(auth_url) 79 | 80 | assert response.status_code == 200 81 | assert bool(response.closed == client.closed) is True 82 | assert response.request.headers["Authorization"] == AUTH_TOKEN 83 | 84 | 85 | @pytest.mark.asyncio 86 | async def test_async_auth_function(http_auth_function, auth_url): 87 | async with tls_requests.AsyncClient(auth=auth_function) as client: 88 | response = await client.get(auth_url) 89 | 90 | assert response.status_code == 200 91 | assert bool(response.closed == client.closed) is True 92 | assert response.request.headers[AUTH_FUNCTION_KEY] == AUTH_FUNCTION_VALUE 93 | 94 | 95 | @pytest.mark.asyncio 96 | async def test_async_auth_function_cross_sharing(http_auth_function, auth_url): 97 | async with tls_requests.AsyncClient(auth=auth) as client: 98 | response = await client.get(auth_url, auth=auth_function) 99 | 100 | assert response.status_code == 200 101 | assert bool(response.closed == client.closed) is True 102 | assert response.request.headers[AUTH_FUNCTION_KEY] == AUTH_FUNCTION_VALUE 103 | -------------------------------------------------------------------------------- /tests/test_cookies.py: -------------------------------------------------------------------------------- 1 | from pytest_httpserver import HTTPServer 2 | from werkzeug import Request, Response 3 | 4 | import tls_requests 5 | 6 | 7 | def hook_request_cookies(_request: Request, response: Response) -> Response: 8 | for k, v in _request.cookies.items(): 9 | response.set_cookie(k, v) 10 | return response 11 | 12 | 13 | def hook_response_cookies(_request: Request, response: Response) -> Response: 14 | response.set_cookie("foo", "bar") 15 | return response 16 | 17 | 18 | def test_request_cookies(httpserver: HTTPServer): 19 | httpserver.expect_request("/cookies").with_post_hook(hook_request_cookies).respond_with_data(b"OK") 20 | response = tls_requests.get(httpserver.url_for("/cookies"), cookies={"foo": "bar"}) 21 | assert response.status_code == 200 22 | assert response.cookies.get("foo") == "bar" 23 | 24 | 25 | def test_response_cookies(httpserver: HTTPServer): 26 | httpserver.expect_request("/cookies").with_post_hook(hook_response_cookies).respond_with_data(b"OK") 27 | response = tls_requests.get(httpserver.url_for("/cookies")) 28 | assert response.status_code == 200 29 | assert response.cookies.get("foo") == "bar" 30 | -------------------------------------------------------------------------------- /tests/test_encoders.py: -------------------------------------------------------------------------------- 1 | from mimetypes import guess_type 2 | from pathlib import Path 3 | 4 | import pytest 5 | from pytest_httpserver import HTTPServer 6 | 7 | import tls_requests 8 | 9 | BASE_DIR = Path(__file__).resolve(strict=True).parent.parent 10 | 11 | CHUNK_SIZE = 65_536 12 | FILENAME = BASE_DIR / 'tests' / 'files' / 'coingecko.png' 13 | 14 | 15 | def get_image_bytes(filename: str = FILENAME): 16 | response_bytes = b"" 17 | with open(filename, 'rb') as f: 18 | while chunk := f.read(CHUNK_SIZE): 19 | response_bytes += chunk 20 | 21 | return response_bytes 22 | 23 | 24 | @pytest.fixture 25 | def mimetype(filename: str = FILENAME): 26 | return guess_type(filename)[0] 27 | 28 | 29 | @pytest.fixture 30 | def file_bytes(filename: str = FILENAME) -> bytes: 31 | return get_image_bytes() 32 | 33 | 34 | def hook_files(_request, response): 35 | image = _request.files['image'] 36 | image_bytes = b"".join(image) 37 | origin_bytes = get_image_bytes() 38 | response.headers['X-Image'] = 1 if image_bytes == origin_bytes else 0 39 | response.headers['X-Image-Content-Type'] = image.content_type 40 | return response 41 | 42 | 43 | def hook_multipart(_request, response): 44 | response.headers["X-Data-Values"] = ", ".join(_request.form.getlist('key1')) 45 | response.headers["X-Image-Content-Type"] = _request.files["image"].content_type 46 | return response 47 | 48 | 49 | def test_file(httpserver: HTTPServer): 50 | httpserver.expect_request("/files").with_post_hook(hook_files).respond_with_data(status=201) 51 | files = {'image': open(FILENAME, 'rb')} 52 | response = tls_requests.post(httpserver.url_for("/files"), files=files) 53 | assert response.status_code == 201 54 | assert response.headers.get('X-Image') == '1' 55 | 56 | 57 | def test_file_tuple_2(httpserver: HTTPServer): 58 | httpserver.expect_request("/files").with_post_hook(hook_files).respond_with_data(status=201) 59 | files = {'image': ('coingecko.png', open(FILENAME, 'rb'))} 60 | response = tls_requests.post(httpserver.url_for("/files"), files=files) 61 | assert response.status_code == 201 62 | assert response.headers.get('X-Image') == '1' 63 | 64 | 65 | def test_file_tuple_3(httpserver: HTTPServer): 66 | httpserver.expect_request("/files").with_post_hook(hook_files).respond_with_data(status=201) 67 | files = {'image': ('coingecko.png', open(FILENAME, 'rb'), 'image/png')} 68 | response = tls_requests.post(httpserver.url_for("/files"), files=files) 69 | assert response.status_code == 201 70 | assert response.headers.get('X-Image') == '1' 71 | assert response.headers.get('X-Image-Content-Type') == 'image/png' 72 | 73 | 74 | def test_multipart(httpserver: HTTPServer, file_bytes, mimetype): 75 | data = {'key1': ['value1', 'value2']} 76 | httpserver.expect_request("/multipart").with_post_hook(hook_multipart).respond_with_data(status=201) 77 | files = {'image': ('coingecko.png', open(FILENAME, 'rb'), 'image/png')} 78 | response = tls_requests.post(httpserver.url_for("/multipart"), data=data, files=files) 79 | assert response.status_code == 201 80 | assert response.headers["X-Image-Content-Type"] == "image/png" 81 | assert response.headers["X-Data-Values"] == ", ".join(data["key1"]) 82 | 83 | 84 | def test_json(httpserver: HTTPServer): 85 | data = { 86 | 'integer': 1, 87 | 'boolean': True, 88 | 'list': ['1', '2', '3'], 89 | 'data': {'key': 'value'} 90 | } 91 | httpserver.expect_request("/json", json=data).respond_with_data(b"OK", status=201) 92 | response = tls_requests.post(httpserver.url_for("/json"), json=data) 93 | assert response.status_code == 201 94 | assert response.content == b"OK" 95 | -------------------------------------------------------------------------------- /tests/test_headers.py: -------------------------------------------------------------------------------- 1 | from pytest_httpserver import HTTPServer 2 | from werkzeug import Request, Response 3 | 4 | import tls_requests 5 | 6 | 7 | def hook_request_headers(_request: Request, response: Response) -> Response: 8 | response.headers = _request.headers 9 | return response 10 | 11 | 12 | def hook_response_headers(_request: Request, response: Response) -> Response: 13 | response.headers["foo"] = "bar" 14 | return response 15 | 16 | 17 | def hook_response_case_insensitive_headers(_request: Request, response: Response) -> Response: 18 | response.headers["Foo"] = "bar" 19 | return response 20 | 21 | 22 | def test_request_headers(httpserver: HTTPServer): 23 | httpserver.expect_request("/headers").with_post_hook(hook_request_headers).respond_with_data(b"OK") 24 | response = tls_requests.get(httpserver.url_for("/headers"), headers={"foo": "bar"}) 25 | assert response.status_code == 200 26 | assert response.headers.get("foo") == "bar" 27 | 28 | 29 | def test_response_headers(httpserver: HTTPServer): 30 | httpserver.expect_request("/headers").with_post_hook(hook_response_headers).respond_with_data(b"OK") 31 | response = tls_requests.get(httpserver.url_for("/headers")) 32 | assert response.status_code, 200 33 | assert response.headers.get("foo") == "bar" 34 | 35 | 36 | def test_response_case_insensitive_headers(httpserver: HTTPServer): 37 | httpserver.expect_request("/headers").with_post_hook(hook_response_case_insensitive_headers).respond_with_data(b"OK") 38 | response = tls_requests.get(httpserver.url_for("/headers")) 39 | assert response.status_code, 200 40 | assert response.headers.get("foo") == "bar" 41 | -------------------------------------------------------------------------------- /tests/test_hooks.py: -------------------------------------------------------------------------------- 1 | from pytest_httpserver import HTTPServer 2 | 3 | import tls_requests 4 | 5 | 6 | def log_request_return(request): 7 | request.headers["X-Hook"] = '123456' 8 | return request 9 | 10 | 11 | def log_request_no_return(request): 12 | request.headers["X-Hook"] = '123456' 13 | 14 | 15 | def log_response_raise_on_4xx_5xx(response): 16 | response.raise_for_status() 17 | 18 | 19 | def test_request_hook(httpserver: HTTPServer): 20 | httpserver.expect_request("/hooks").respond_with_data(b"OK") 21 | response = tls_requests.get(httpserver.url_for("/hooks"), hooks={"request": [log_request_return]}) 22 | assert response.status_code == 200 23 | assert response.request.headers.get("X-Hook") == "123456" 24 | 25 | 26 | def test_request_hook_no_return(httpserver: HTTPServer): 27 | httpserver.expect_request("/hooks").respond_with_data(b"OK") 28 | response = tls_requests.get(httpserver.url_for("/hooks"), hooks={"request": [log_request_no_return]}) 29 | assert response.status_code == 200 30 | assert response.request.headers.get("X-Hook") == "123456" 31 | 32 | 33 | def test_response_hook(httpserver: HTTPServer): 34 | httpserver.expect_request("/hooks", ).respond_with_data(status=404) 35 | try: 36 | _ = tls_requests.get(httpserver.url_for("/hooks"), hooks={"response": [log_response_raise_on_4xx_5xx]}) 37 | except Exception as e: 38 | assert e, tls_requests.exceptions.HTTPError 39 | -------------------------------------------------------------------------------- /tests/test_proxy.py: -------------------------------------------------------------------------------- 1 | import tls_requests 2 | 3 | 4 | def test_http_proxy(): 5 | proxy = tls_requests.Proxy("http://localhost:8080") 6 | assert proxy.scheme == "http" 7 | assert proxy.host == "localhost" 8 | assert proxy.port == '8080' 9 | assert proxy.url == "http://localhost:8080" 10 | 11 | 12 | def test_https_proxy(): 13 | proxy = tls_requests.Proxy("https://localhost:8080") 14 | assert proxy.scheme == "https" 15 | assert proxy.host == "localhost" 16 | assert proxy.port == '8080' 17 | assert proxy.url == "https://localhost:8080" 18 | 19 | 20 | def test_socks5_proxy(): 21 | proxy = tls_requests.Proxy("socks5://localhost:8080") 22 | assert proxy.scheme == "socks5" 23 | assert proxy.host == "localhost" 24 | assert proxy.port == '8080' 25 | assert proxy.url == "socks5://localhost:8080" 26 | 27 | 28 | def test_proxy_with_params(): 29 | proxy = tls_requests.Proxy("http://localhost:8080?a=b", params={"foo": "bar"}) 30 | assert proxy.scheme == "http" 31 | assert proxy.host == "localhost" 32 | assert proxy.port == '8080' 33 | assert proxy.url == "http://localhost:8080" 34 | 35 | 36 | def test_auth_proxy(): 37 | proxy = tls_requests.Proxy("http://username:password@localhost:8080") 38 | assert proxy.scheme == "http" 39 | assert proxy.host == "localhost" 40 | assert proxy.port == '8080' 41 | assert proxy.auth == ("username", "password") 42 | assert proxy.url == "http://username:password@localhost:8080" 43 | 44 | 45 | def test_unsupported_proxy_scheme(): 46 | try: 47 | _ = tls_requests.Proxy("unknown://localhost:8080") 48 | except Exception as e: 49 | assert isinstance(e, tls_requests.exceptions.ProxyError) 50 | -------------------------------------------------------------------------------- /tests/test_redirects.py: -------------------------------------------------------------------------------- 1 | from pytest_httpserver import HTTPServer 2 | 3 | import tls_requests 4 | 5 | 6 | def test_missing_host_redirects(httpserver: HTTPServer): 7 | httpserver.expect_request("/redirects/3").respond_with_data(b"OK", status=302, headers={"Location": "/redirects/1"}) 8 | httpserver.expect_request("/redirects/1").respond_with_data(b"OK", status=302, headers={"Location": "/redirects/2"}) 9 | httpserver.expect_request("/redirects/2").respond_with_data(b"OK", status=302, headers={"Location": "/redirects/ok"}) 10 | httpserver.expect_request("/redirects/ok").respond_with_data(b"OK") 11 | response = tls_requests.get(httpserver.url_for("/redirects/3")) 12 | assert response.status_code == 200 13 | assert response.history[0].status_code == 302 14 | assert len(response.history) == 3 15 | 16 | 17 | def test_full_path_redirects(httpserver: HTTPServer): 18 | httpserver.expect_request("/redirects/3").respond_with_data(b"OK", status=302, headers={"Location": httpserver.url_for("/redirects/1")}) 19 | httpserver.expect_request("/redirects/1").respond_with_data(b"OK", status=302, headers={"Location": httpserver.url_for("/redirects/2")}) 20 | httpserver.expect_request("/redirects/2").respond_with_data(b"OK", status=302, headers={"Location": httpserver.url_for("/redirects/ok")}) 21 | httpserver.expect_request("/redirects/ok").respond_with_data(b"OK") 22 | response = tls_requests.get(httpserver.url_for("/redirects/3")) 23 | assert response.status_code == 200 24 | assert response.history[0].status_code == 302 25 | assert len(response.history) == 3 26 | 27 | 28 | def test_fragment_redirects(httpserver: HTTPServer): 29 | httpserver.expect_request("/redirects/3").respond_with_data(b"OK", status=302, headers={"Location": httpserver.url_for("/redirects/ok#fragment")}) 30 | httpserver.expect_request("/redirects/ok").respond_with_data(b"OK") 31 | response = tls_requests.get(httpserver.url_for("/redirects/3")) 32 | assert response.status_code == 200 33 | assert response.history[0].status_code == 302 34 | assert len(response.history) == 1 35 | assert response.request.url.fragment == "fragment" 36 | 37 | 38 | def test_too_many_redirects(httpserver: HTTPServer): 39 | httpserver.expect_request("/redirects/3").respond_with_data(b"OK", status=302, headers={"Location": "/redirects/1"}) 40 | httpserver.expect_request("/redirects/1").respond_with_data(b"OK", status=302, headers={"Location": "/redirects/2"}) 41 | httpserver.expect_request("/redirects/2").respond_with_data(b"OK", status=302, headers={"Location": "/redirects/3"}) 42 | httpserver.expect_request("/redirects/ok").respond_with_data(b"OK") 43 | try: 44 | _ = tls_requests.get(httpserver.url_for("/redirects/3")) 45 | except Exception as e: 46 | assert isinstance(e, tls_requests.exceptions.TooManyRedirects) 47 | -------------------------------------------------------------------------------- /tests/test_timeout.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from pytest_httpserver import HTTPServer 4 | 5 | import tls_requests 6 | 7 | 8 | def timeout_hook(_request, response): 9 | time.sleep(3) 10 | return response 11 | 12 | 13 | def test_timeout(httpserver: HTTPServer): 14 | httpserver.expect_request("/timeout").with_post_hook(timeout_hook).respond_with_data(b"OK") 15 | response = tls_requests.get(httpserver.url_for("/timeout"), timeout=1) 16 | assert response.status_code == 0 17 | -------------------------------------------------------------------------------- /tests/test_urls.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import unquote 2 | 3 | from pytest_httpserver import HTTPServer 4 | 5 | import tls_requests 6 | 7 | 8 | def request_hook(_request, response): 9 | response.headers['x-path'] = _request.full_path 10 | return response 11 | 12 | 13 | def test_request_params(httpserver: HTTPServer): 14 | params = {"a": "1", "b": "2"} 15 | httpserver.expect_request("/params").with_post_hook(request_hook).respond_with_data(b"OK") 16 | response = tls_requests.get(httpserver.url_for("/params"), params=params) 17 | assert response.status_code == 200 18 | assert unquote(str(response.url)).endswith(unquote(response.headers["x-path"])) 19 | 20 | 21 | def test_request_multi_params(httpserver: HTTPServer): 22 | params = {"a": ["1", "2", "3"]} 23 | httpserver.expect_request("/params").with_post_hook(request_hook).respond_with_data(b"OK") 24 | response = tls_requests.get(httpserver.url_for("/params"), params=params) 25 | assert response.status_code == 200 26 | assert unquote(str(response.url)).endswith(unquote(response.headers["x-path"])) 27 | -------------------------------------------------------------------------------- /tls_requests/__init__.py: -------------------------------------------------------------------------------- 1 | from .__version__ import __description__, __title__, __version__ 2 | from .api import * 3 | from .client import * 4 | from .exceptions import * 5 | from .models import * 6 | from .settings import * 7 | from .types import * 8 | 9 | __all__ = [ 10 | "__version__", 11 | "__author__", 12 | "__title__", 13 | "__description__", 14 | "AsyncClient", 15 | "Client", 16 | "Cookies", 17 | "CustomTLSClientConfig", 18 | "Headers", 19 | "Proxy", 20 | "Request", 21 | "Response", 22 | "StatusCodes", 23 | "TLSClient", 24 | "TLSConfig", 25 | "TLSLibrary", 26 | "TLSResponse", 27 | "URL", 28 | "URLParams", 29 | "delete", 30 | "get", 31 | "head", 32 | "options", 33 | "patch", 34 | "post", 35 | "put", 36 | "request", 37 | ] 38 | 39 | 40 | __locals = locals() 41 | for __name in __all__: 42 | if not __name.startswith("__"): 43 | setattr(__locals[__name], "__module__", "tls_requests") # noqa 44 | -------------------------------------------------------------------------------- /tls_requests/__version__.py: -------------------------------------------------------------------------------- 1 | __title__ = "wrapper-tls-requests" 2 | __description__ = "A powerful and lightweight Python library for making secure and reliable HTTP/TLS Fingerprint requests." 3 | __url__ = "https://github.com/thewebscraping/tls-requests" 4 | __author__ = "Tu Pham" 5 | __author_email__ = "thetwofarm@gmail.com" 6 | __version__ = "1.1.5" 7 | __license__ = "MIT" 8 | -------------------------------------------------------------------------------- /tls_requests/api.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import typing 4 | 5 | from .client import Client 6 | from .models import Response 7 | from .settings import (DEFAULT_FOLLOW_REDIRECTS, DEFAULT_TIMEOUT, 8 | DEFAULT_TLS_HTTP2, DEFAULT_TLS_IDENTIFIER) 9 | from .types import (AuthTypes, CookieTypes, HeaderTypes, MethodTypes, 10 | ProtocolTypes, ProxyTypes, RequestData, RequestFiles, 11 | TimeoutTypes, TLSIdentifierTypes, URLParamTypes, URLTypes) 12 | 13 | __all__ = [ 14 | "delete", 15 | "get", 16 | "head", 17 | "options", 18 | "patch", 19 | "post", 20 | "put", 21 | "request", 22 | ] 23 | 24 | 25 | def request( 26 | method: MethodTypes, 27 | url: URLTypes, 28 | *, 29 | params: URLParamTypes = None, 30 | data: RequestData = None, 31 | files: RequestFiles = None, 32 | json: typing.Any = None, 33 | headers: HeaderTypes = None, 34 | cookies: CookieTypes = None, 35 | auth: AuthTypes = None, 36 | proxy: ProxyTypes = None, 37 | http2: ProtocolTypes = DEFAULT_TLS_HTTP2, 38 | timeout: TimeoutTypes = DEFAULT_TIMEOUT, 39 | follow_redirects: bool = DEFAULT_FOLLOW_REDIRECTS, 40 | verify: bool = True, 41 | tls_identifier: TLSIdentifierTypes = DEFAULT_TLS_IDENTIFIER, 42 | **config, 43 | ) -> Response: 44 | """ 45 | Constructs and sends an HTTP request. 46 | 47 | This method builds a `Request` object based on the given parameters, sends 48 | it using the configured client, and returns the server's response. 49 | 50 | Parameters: 51 | - **method** (str): HTTP method to use (e.g., `"GET"`, `"POST"`). 52 | - **url** (URLTypes): The URL to send the request to. 53 | - **params** (optional): Query parameters to include in the request URL. 54 | - **data** (optional): Form data to include in the request body. 55 | - **json** (optional): A JSON serializable object to include in the request body. 56 | - **headers** (optional): Custom headers to include in the request. 57 | - **cookies** (optional): Cookies to include with the request. 58 | - **files** (optional): Files to upload in a multipart request. 59 | - **auth** (optional): Authentication credentials or handler. 60 | - **timeout** (optional): Timeout configuration for the request. 61 | - **follow_redirects** (optional): Whether to follow HTTP redirects. 62 | 63 | Returns: 64 | - **Response**: The client's response to the HTTP request. 65 | 66 | Usage: 67 | ```python 68 | 69 | import tls_requests 70 | >>> with tls_requests.Client() as sync_client: 71 | r = sync_client.request('GET', 'https://httpbin.org/get') 72 | >>> r 73 | 74 | ``` 75 | """ 76 | 77 | with Client( 78 | cookies=cookies, 79 | proxy=proxy, 80 | http2=http2, 81 | timeout=timeout, 82 | verify=verify, 83 | client_identifier=tls_identifier, 84 | **config, 85 | ) as client: 86 | return client.request( 87 | method=method, 88 | url=url, 89 | data=data, 90 | files=files, 91 | json=json, 92 | params=params, 93 | headers=headers, 94 | auth=auth, 95 | follow_redirects=follow_redirects, 96 | timeout=timeout, 97 | ) 98 | 99 | 100 | def get( 101 | url: URLTypes, 102 | *, 103 | params: URLParamTypes = None, 104 | headers: HeaderTypes = None, 105 | cookies: CookieTypes = None, 106 | auth: AuthTypes = None, 107 | proxy: ProxyTypes = None, 108 | http2: ProtocolTypes = DEFAULT_TLS_HTTP2, 109 | timeout: TimeoutTypes = DEFAULT_TIMEOUT, 110 | follow_redirects: bool = DEFAULT_FOLLOW_REDIRECTS, 111 | verify: bool = True, 112 | tls_identifier: TLSIdentifierTypes = DEFAULT_TLS_IDENTIFIER, 113 | **config, 114 | ) -> Response: 115 | """ 116 | Sends a `GET` request. 117 | 118 | **Parameters**: See `tls_requests.request`. 119 | 120 | Note that the `data`, `files`, `json` and `content` parameters are not available 121 | on this function, as `GET` requests should not include a request body. 122 | """ 123 | return request( 124 | "GET", 125 | url, 126 | params=params, 127 | headers=headers, 128 | cookies=cookies, 129 | auth=auth, 130 | proxy=proxy, 131 | http2=http2, 132 | follow_redirects=follow_redirects, 133 | timeout=timeout, 134 | verify=verify, 135 | tls_identifier=tls_identifier, 136 | **config, 137 | ) 138 | 139 | 140 | def options( 141 | url: URLTypes, 142 | *, 143 | params: URLParamTypes = None, 144 | headers: HeaderTypes = None, 145 | cookies: CookieTypes = None, 146 | auth: AuthTypes = None, 147 | proxy: ProxyTypes = None, 148 | http2: ProtocolTypes = DEFAULT_TLS_HTTP2, 149 | timeout: TimeoutTypes = DEFAULT_TIMEOUT, 150 | follow_redirects: bool = DEFAULT_FOLLOW_REDIRECTS, 151 | verify: bool = True, 152 | tls_identifier: TLSIdentifierTypes = DEFAULT_TLS_IDENTIFIER, 153 | **config, 154 | ) -> Response: 155 | """ 156 | Sends an `OPTIONS` request. 157 | 158 | **Parameters**: See `tls_requests.request`. 159 | 160 | Note that the `data`, `files`, `json` and `content` parameters are not available 161 | on this function, as `OPTIONS` requests should not include a request body. 162 | """ 163 | return request( 164 | "OPTIONS", 165 | url, 166 | params=params, 167 | headers=headers, 168 | cookies=cookies, 169 | auth=auth, 170 | proxy=proxy, 171 | http2=http2, 172 | follow_redirects=follow_redirects, 173 | timeout=timeout, 174 | verify=verify, 175 | tls_identifier=tls_identifier, 176 | **config, 177 | ) 178 | 179 | 180 | def head( 181 | url: URLTypes, 182 | *, 183 | params: URLParamTypes = None, 184 | headers: HeaderTypes = None, 185 | cookies: CookieTypes = None, 186 | auth: AuthTypes = None, 187 | proxy: ProxyTypes = None, 188 | http2: ProtocolTypes = DEFAULT_TLS_HTTP2, 189 | timeout: TimeoutTypes = DEFAULT_TIMEOUT, 190 | follow_redirects: bool = DEFAULT_FOLLOW_REDIRECTS, 191 | verify: bool = True, 192 | tls_identifier: TLSIdentifierTypes = DEFAULT_TLS_IDENTIFIER, 193 | **config, 194 | ) -> Response: 195 | """ 196 | Sends a `HEAD` request. 197 | 198 | **Parameters**: See `tls_requests.request`. 199 | 200 | Note that the `data`, `files`, `json` and `content` parameters are not available 201 | on this function, as `HEAD` requests should not include a request body. 202 | """ 203 | return request( 204 | "HEAD", 205 | url, 206 | params=params, 207 | headers=headers, 208 | cookies=cookies, 209 | auth=auth, 210 | proxy=proxy, 211 | http2=http2, 212 | timeout=timeout, 213 | follow_redirects=follow_redirects, 214 | verify=verify, 215 | tls_identifier=tls_identifier, 216 | **config, 217 | ) 218 | 219 | 220 | def post( 221 | url: URLTypes, 222 | *, 223 | data: RequestData = None, 224 | files: RequestFiles = None, 225 | json: typing.Any = None, 226 | params: URLParamTypes = None, 227 | headers: HeaderTypes = None, 228 | cookies: CookieTypes = None, 229 | auth: AuthTypes = None, 230 | proxy: ProxyTypes = None, 231 | http2: ProtocolTypes = DEFAULT_TLS_HTTP2, 232 | timeout: TimeoutTypes = DEFAULT_TIMEOUT, 233 | follow_redirects: bool = DEFAULT_FOLLOW_REDIRECTS, 234 | verify: bool = True, 235 | tls_identifier: TLSIdentifierTypes = DEFAULT_TLS_IDENTIFIER, 236 | **config, 237 | ) -> Response: 238 | """ 239 | Sends a `POST` request. 240 | 241 | **Parameters**: See `tls_requests.request`. 242 | """ 243 | return request( 244 | "POST", 245 | url, 246 | data=data, 247 | files=files, 248 | json=json, 249 | params=params, 250 | headers=headers, 251 | cookies=cookies, 252 | auth=auth, 253 | proxy=proxy, 254 | http2=http2, 255 | timeout=timeout, 256 | follow_redirects=follow_redirects, 257 | verify=verify, 258 | tls_identifier=tls_identifier, 259 | **config, 260 | ) 261 | 262 | 263 | def put( 264 | url: URLTypes, 265 | *, 266 | data: RequestData = None, 267 | files: RequestFiles = None, 268 | json: typing.Any = None, 269 | params: URLParamTypes = None, 270 | headers: HeaderTypes = None, 271 | cookies: CookieTypes = None, 272 | auth: AuthTypes = None, 273 | proxy: ProxyTypes = None, 274 | http2: ProtocolTypes = DEFAULT_TLS_HTTP2, 275 | timeout: TimeoutTypes = DEFAULT_TIMEOUT, 276 | follow_redirects: bool = DEFAULT_FOLLOW_REDIRECTS, 277 | verify: bool = True, 278 | tls_identifier: TLSIdentifierTypes = DEFAULT_TLS_IDENTIFIER, 279 | **config, 280 | ) -> Response: 281 | """ 282 | Sends a `PUT` request. 283 | 284 | **Parameters**: See `tls_requests.request`. 285 | """ 286 | return request( 287 | "PUT", 288 | url, 289 | data=data, 290 | files=files, 291 | json=json, 292 | params=params, 293 | headers=headers, 294 | cookies=cookies, 295 | auth=auth, 296 | proxy=proxy, 297 | http2=http2, 298 | timeout=timeout, 299 | follow_redirects=follow_redirects, 300 | verify=verify, 301 | tls_identifier=tls_identifier, 302 | **config, 303 | ) 304 | 305 | 306 | def patch( 307 | url: URLTypes, 308 | *, 309 | data: RequestData = None, 310 | files: RequestFiles = None, 311 | json: typing.Any = None, 312 | params: URLParamTypes = None, 313 | headers: HeaderTypes = None, 314 | cookies: CookieTypes = None, 315 | auth: AuthTypes = None, 316 | proxy: ProxyTypes = None, 317 | http2: ProtocolTypes = DEFAULT_TLS_HTTP2, 318 | timeout: TimeoutTypes = DEFAULT_TIMEOUT, 319 | follow_redirects: bool = DEFAULT_FOLLOW_REDIRECTS, 320 | verify: bool = True, 321 | tls_identifier: TLSIdentifierTypes = DEFAULT_TLS_IDENTIFIER, 322 | **config, 323 | ) -> Response: 324 | """ 325 | Sends a `PATCH` request. 326 | 327 | **Parameters**: See `tls_requests.request`. 328 | """ 329 | return request( 330 | "PATCH", 331 | url, 332 | data=data, 333 | files=files, 334 | json=json, 335 | params=params, 336 | headers=headers, 337 | cookies=cookies, 338 | auth=auth, 339 | proxy=proxy, 340 | http2=http2, 341 | timeout=timeout, 342 | follow_redirects=follow_redirects, 343 | verify=verify, 344 | tls_identifier=tls_identifier, 345 | **config, 346 | ) 347 | 348 | 349 | def delete( 350 | url: URLTypes, 351 | *, 352 | params: URLParamTypes | None = None, 353 | headers: HeaderTypes | None = None, 354 | cookies: CookieTypes | None = None, 355 | auth: AuthTypes | None = None, 356 | proxy: ProxyTypes = None, 357 | http2: ProtocolTypes = DEFAULT_TLS_HTTP2, 358 | timeout: TimeoutTypes = DEFAULT_TIMEOUT, 359 | follow_redirects: bool = DEFAULT_FOLLOW_REDIRECTS, 360 | verify: bool = True, 361 | tls_identifier: TLSIdentifierTypes = DEFAULT_TLS_IDENTIFIER, 362 | **config, 363 | ) -> Response: 364 | """ 365 | Sends a `DELETE` request. 366 | 367 | **Parameters**: See `tls_requests.request`. 368 | 369 | Note that the `data`, `files`, `json` and `content` parameters are not available 370 | on this function, as `DELETE` requests should not include a request body. 371 | """ 372 | return request( 373 | "DELETE", 374 | url, 375 | params=params, 376 | headers=headers, 377 | cookies=cookies, 378 | auth=auth, 379 | proxy=proxy, 380 | http2=http2, 381 | timeout=timeout, 382 | follow_redirects=follow_redirects, 383 | verify=verify, 384 | tls_identifier=tls_identifier, 385 | **config, 386 | ) 387 | -------------------------------------------------------------------------------- /tls_requests/bin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thewebscraping/tls-requests/79648285c47f65e101fcb8181b9512c71b3d2796/tls_requests/bin/__init__.py -------------------------------------------------------------------------------- /tls_requests/exceptions.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | if TYPE_CHECKING: 6 | pass 7 | 8 | __all__ = [ 9 | "CookieConflictError", 10 | "HTTPError", 11 | "URLError", 12 | "RemoteProtocolError", 13 | "ProtocolError", 14 | "StreamConsumed", 15 | "StreamError", 16 | "TooManyRedirects", 17 | "TLSError", 18 | ] 19 | 20 | 21 | class HTTPError(Exception): 22 | """HTTP Error""" 23 | 24 | def __init__(self, message: str, **kwargs) -> None: 25 | self.message = message 26 | response = kwargs.pop("response", None) 27 | self.response = response 28 | self.request = kwargs.pop("request", None) 29 | if response is not None and not self.request and hasattr(response, "request"): 30 | self.request = self.response.request 31 | super().__init__(message, **kwargs) 32 | 33 | 34 | class ProtocolError(HTTPError): 35 | """Protocol Error""" 36 | 37 | 38 | class RemoteProtocolError(HTTPError): 39 | """Remote Protocol Error""" 40 | 41 | 42 | class TooManyRedirects(HTTPError): 43 | """Too Many Redirects.""" 44 | 45 | 46 | class TLSError(HTTPError): 47 | """TLS Error""" 48 | 49 | 50 | class Base64DecodeError(HTTPError): 51 | """Base64 Decode Error""" 52 | 53 | 54 | class URLError(HTTPError): 55 | pass 56 | 57 | 58 | class ProxyError(HTTPError): 59 | pass 60 | 61 | 62 | class URLParamsError(URLError): 63 | pass 64 | 65 | 66 | class CookieError(HTTPError): 67 | pass 68 | 69 | 70 | class CookieConflictError(CookieError): 71 | pass 72 | 73 | 74 | class StreamError(HTTPError): 75 | pass 76 | 77 | 78 | class StreamConsumed(StreamError): 79 | pass 80 | 81 | 82 | class StreamClosed(StreamError): 83 | pass 84 | -------------------------------------------------------------------------------- /tls_requests/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .auth import Auth, BasicAuth 2 | from .cookies import Cookies 3 | from .encoders import (JsonEncoder, MultipartEncoder, StreamEncoder, 4 | UrlencodedEncoder) 5 | from .headers import Headers 6 | from .libraries import TLSLibrary 7 | from .request import Request 8 | from .response import Response 9 | from .status_codes import StatusCodes 10 | from .tls import CustomTLSClientConfig, TLSClient, TLSConfig, TLSResponse 11 | from .urls import URL, Proxy, URLParams 12 | -------------------------------------------------------------------------------- /tls_requests/models/auth.py: -------------------------------------------------------------------------------- 1 | from base64 import b64encode 2 | from typing import Any, Union 3 | 4 | from tls_requests.models.request import Request 5 | 6 | 7 | class Auth: 8 | """Base Auth""" 9 | 10 | def build_auth(self, request: Request) -> Union[Request, Any]: 11 | pass 12 | 13 | 14 | class BasicAuth(Auth): 15 | """Basic Authentication""" 16 | 17 | def __init__(self, username: Union[str, bytes], password: Union[str, bytes]): 18 | self.username = username 19 | self.password = password 20 | 21 | def build_auth(self, request: Request): 22 | return self._build_auth_headers(request) 23 | 24 | def _build_auth_headers(self, request: Request): 25 | auth_token = b64encode( 26 | b":".join([self._encode(self.username), self._encode(self.password)]) 27 | ).decode() 28 | request.headers["Authorization"] = "Basic %s" % auth_token 29 | 30 | def _encode(self, value: Union[str, bytes]) -> bytes: 31 | if isinstance(self.username, str): 32 | value = value.encode("latin1") 33 | 34 | if not isinstance(value, bytes): 35 | raise TypeError("`username` or `password` parameter must be str or byte.") 36 | 37 | return value 38 | -------------------------------------------------------------------------------- /tls_requests/models/encoders.py: -------------------------------------------------------------------------------- 1 | import binascii 2 | import os 3 | from io import BufferedReader, BytesIO, TextIOWrapper 4 | from mimetypes import guess_type 5 | from typing import Any, AsyncIterator, Dict, Iterator, Mapping, Tuple, TypeVar 6 | from urllib.parse import urlencode 7 | 8 | from tls_requests.types import (BufferTypes, ByteOrStr, RequestData, 9 | RequestFiles, RequestFileValue, RequestJson) 10 | from tls_requests.utils import to_bytes, to_str 11 | 12 | __all__ = [ 13 | "JsonEncoder", 14 | "UrlencodedEncoder", 15 | "MultipartEncoder", 16 | "StreamEncoder", 17 | ] 18 | 19 | T = TypeVar("T", bound="BaseEncoder") 20 | 21 | 22 | def guess_content_type(fp: str) -> str: 23 | content_type, _ = guess_type(fp) 24 | return content_type or "application/octet-stream" 25 | 26 | 27 | def format_header(name: str, value: ByteOrStr, encoding: str = "ascii") -> bytes: 28 | if isinstance(value, bytes): 29 | value = value.decode("utf-8") 30 | 31 | value = value.translate({10: "%0A", 13: "%0D", 34: "%22"}) 32 | return ('%s="%s"' % (name, value)).encode(encoding) 33 | 34 | 35 | def get_boundary(): 36 | return binascii.hexlify(os.urandom(16)) 37 | 38 | 39 | def iter_buffer(buffer: BufferTypes, chunk_size: int = 65_536): 40 | buffer.seek(0) 41 | while chunk := buffer.read(chunk_size): 42 | yield chunk 43 | 44 | 45 | class BaseField: 46 | def __init__(self, name: str, value: Any): 47 | self._name = name 48 | self._headers = {} 49 | 50 | @property 51 | def headers(self): 52 | return self.render_headers() 53 | 54 | def render_parts(self) -> bytes: 55 | parts = [ 56 | b"form-data", 57 | format_header("name", self._name), 58 | ] 59 | filename = getattr(self, "filename", None) 60 | if filename: 61 | parts.append(format_header("filename", filename)) 62 | 63 | return b"; ".join(parts) 64 | 65 | def render_headers(self) -> bytes: 66 | headers = self.get_headers() 67 | return ( 68 | b"\r\n".join(b"%s: %s" % (k, v) for k, v in headers.items()) + b"\r\n\r\n" 69 | ) 70 | 71 | def render_data(self, chunk_size: int = 65_536) -> Iterator[bytes]: 72 | yield b"" 73 | 74 | def render(self, chunk_size: int = 65_536) -> Iterator[bytes]: 75 | yield self.render_headers() 76 | yield from self.render_data(chunk_size) 77 | 78 | def get_headers(self) -> Dict[bytes, bytes]: 79 | self._headers[b"Content-Disposition"] = self.render_parts() 80 | content_type = getattr(self, "content_type", None) 81 | if content_type: 82 | self._headers[b"Content-Type"] = ( 83 | self.content_type.encode("ascii") 84 | if isinstance(content_type, str) 85 | else content_type 86 | ) 87 | return self._headers 88 | 89 | 90 | class DataField(BaseField): 91 | def __init__(self, name: str, value: Any) -> None: 92 | super(DataField, self).__init__(name, value) 93 | self.value = to_str(value) 94 | 95 | def render_data(self, chunk_size: int = 65_536) -> Iterator[bytes]: 96 | yield self.value.encode("utf-8") 97 | 98 | 99 | class FileField(BaseField): 100 | def __init__(self, name: str, value: RequestFileValue): 101 | super(FileField, self).__init__(name, value) 102 | self.filename, self._buffer, self.content_type = self.unpack(value) 103 | 104 | def unpack(self, value: RequestFileValue) -> Tuple[str, BufferTypes, str]: 105 | filename, content_type = None, None 106 | if isinstance(value, tuple): 107 | if len(value) > 1: 108 | filename, buffer, *args = value 109 | if args: 110 | content_type = args[0] 111 | else: 112 | buffer = value 113 | 114 | elif isinstance(value, str): 115 | buffer = value.encode("utf-8") 116 | else: 117 | buffer = value 118 | 119 | if isinstance(buffer, (TextIOWrapper, BufferedReader)): 120 | if not filename: 121 | _, filename = os.path.split(buffer.name) 122 | 123 | if not content_type: 124 | content_type = guess_content_type(buffer.name) 125 | 126 | if buffer.mode != "rb": 127 | buffer.close() 128 | buffer = open(buffer.name, "rb") 129 | 130 | elif not isinstance(buffer, bytes): 131 | raise ValueError 132 | else: 133 | buffer = BytesIO(buffer) 134 | 135 | return filename or "upload", buffer, content_type or "application/octet-stream" 136 | 137 | def render_data(self, chunk_size: int = 65_536) -> Iterator[bytes]: 138 | yield from iter_buffer(self._buffer, chunk_size) 139 | 140 | 141 | class BaseEncoder: 142 | _chunk_size = 65_536 143 | 144 | @property 145 | def headers(self) -> dict: 146 | return self.get_headers() 147 | 148 | @property 149 | def closed(self): 150 | return bool(getattr(self, "_is_closed", False)) 151 | 152 | def get_headers(self) -> dict: 153 | return {} 154 | 155 | def render(self) -> Iterator[bytes]: 156 | buffer = getattr(self, "_buffer", None) 157 | if buffer: 158 | yield from iter_buffer(buffer, self._chunk_size) 159 | yield b"" 160 | 161 | def close(self) -> None: 162 | if not self.closed: 163 | setattr(self, "_is_closed", True) 164 | buffer = getattr(self, "_buffer", None) 165 | if buffer: 166 | buffer.flush() 167 | buffer.close() 168 | 169 | def __iter__(self) -> Iterator[bytes]: 170 | for chunk in self.render(): 171 | yield chunk 172 | 173 | async def __aiter__(self) -> AsyncIterator[bytes]: 174 | for chunk in self.render(): 175 | yield chunk 176 | 177 | def __enter__(self) -> T: 178 | return self 179 | 180 | def __exit__(self, *args, **kwargs): 181 | self.close() 182 | 183 | 184 | class MultipartEncoder(BaseEncoder): 185 | def __init__( 186 | self, 187 | data: RequestData = None, 188 | files: RequestFiles = None, 189 | boundary: bytes = None, 190 | *, 191 | chunk_size: int = 65_536, 192 | **kwargs, 193 | ) -> None: 194 | self._chunk_size = chunk_size 195 | self._is_closed = False 196 | self.fields = self._prepare_fields(data, files) 197 | self.boundary = ( 198 | boundary if boundary and isinstance(boundary, bytes) else get_boundary() 199 | ) 200 | 201 | @property 202 | def headers(self) -> dict: 203 | if self.fields: 204 | return self.get_headers() 205 | return {} 206 | 207 | def get_headers(self): 208 | return {b"Content-Type": b"multipart/form-data; boundary=%s" % self.boundary} 209 | 210 | def render(self) -> Iterator[bytes]: 211 | if self.fields: 212 | for field in self.fields: 213 | yield b"--%s\r\n" % self.boundary 214 | yield b"".join(field.render(self._chunk_size)) 215 | yield b"\r\n" 216 | yield b"--%s--\r\n" % self.boundary 217 | yield b"" 218 | 219 | def _prepare_fields(self, data: RequestData, files: RequestFiles): 220 | fields = [] 221 | if isinstance(data, Mapping): 222 | for name, value in data.items(): 223 | if isinstance(value, (bytes, str, int, float, bool)): 224 | fields.append(DataField(name=name, value=value)) 225 | else: 226 | for item in value: 227 | fields.append(DataField(name=name, value=item)) 228 | 229 | if isinstance(files, Mapping): 230 | for name, value in files.items(): 231 | fields.append(FileField(name=name, value=value)) 232 | return fields 233 | 234 | 235 | class JsonEncoder(BaseEncoder): 236 | def __init__(self, data: RequestData, *, chunk_size: int = 65_536, **kwargs): 237 | self._buffer = self._prepare_fields(data) 238 | self._chunk_size = chunk_size 239 | self._is_closed = False 240 | 241 | def get_headers(self): 242 | return {b"Content-Type": b"application/json"} 243 | 244 | def _prepare_fields(self, data: RequestData): 245 | if isinstance(data, Mapping): 246 | return BytesIO(to_bytes(data)) 247 | 248 | 249 | class UrlencodedEncoder(BaseEncoder): 250 | def __init__(self, data: RequestData, *, chunk_size: int = 65_536, **kwargs): 251 | self._buffer = self._prepare_fields(data) 252 | self._chunk_size = chunk_size 253 | self._is_closed = False 254 | 255 | def get_headers(self): 256 | return {b"Content-Type": b"application/x-www-form-urlencoded"} 257 | 258 | def _prepare_fields(self, data: RequestData): 259 | fields = [] 260 | if isinstance(data, Mapping): 261 | for name, value in data.items(): 262 | if isinstance(value, (bytes, str, int, float, bool)): 263 | fields.append((name, to_str(value))) 264 | else: 265 | for item in value: 266 | fields.append((name, to_str(item))) 267 | 268 | return BytesIO(urlencode(fields, doseq=True).encode("utf-8")) 269 | 270 | 271 | class StreamEncoder(BaseEncoder): 272 | def __init__( 273 | self, 274 | data: RequestData = None, 275 | files: RequestFiles = None, 276 | json: RequestJson = None, 277 | *, 278 | chunk_size: int = 65_536, 279 | **kwargs, 280 | ): 281 | self._chunk_size = chunk_size if isinstance(chunk_size, int) else 65_536 282 | self._is_closed = False 283 | if files is not None: 284 | self._stream = MultipartEncoder(data, files) 285 | elif data is not None: 286 | self._stream = UrlencodedEncoder(data) 287 | elif json is not None: 288 | self._stream = JsonEncoder(json) 289 | else: 290 | self._stream = BaseEncoder() 291 | 292 | def render(self) -> Iterator[bytes]: 293 | yield from self._stream 294 | self._is_closed = True 295 | 296 | def get_headers(self) -> dict: 297 | return self._stream.get_headers() 298 | 299 | @classmethod 300 | def from_bytes(cls, raw: bytes, *, chunk_size: int = None) -> "StreamEncoder": 301 | ret = cls(chunk_size=chunk_size) 302 | ret._stream._buffer = BytesIO(raw) 303 | return ret 304 | 305 | def close(self): 306 | super().close() 307 | self._stream.close() 308 | 309 | def __exit__(self, *args, **kwargs): 310 | self.close() 311 | -------------------------------------------------------------------------------- /tls_requests/models/headers.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | from collections.abc import Mapping, MutableMapping 3 | from enum import Enum 4 | from typing import Any, ItemsView, KeysView, List, Literal, Tuple, ValuesView 5 | 6 | from tls_requests.types import ByteOrStr, HeaderTypes 7 | from tls_requests.utils import to_str 8 | 9 | __all__ = ["Headers"] 10 | 11 | HeaderAliasTypes = Literal["*", "lower", "capitalize"] 12 | 13 | 14 | class HeaderAlias(str, Enum): 15 | LOWER = "lower" 16 | CAPITALIZE = "capitalize" 17 | ALL = "*" 18 | 19 | def __contains__(self, key: str) -> bool: 20 | for item in self: 21 | if item == key: 22 | return True 23 | return False 24 | 25 | 26 | class Headers(MutableMapping, ABC): 27 | def __init__( 28 | self, 29 | headers: HeaderTypes = None, 30 | *, 31 | alias: HeaderAliasTypes = HeaderAlias.LOWER 32 | ): 33 | self.alias = ( 34 | alias if alias in HeaderAlias._value2member_map_ else HeaderAlias.LOWER 35 | ) 36 | self._items = self._prepare_items(headers) 37 | 38 | def get(self, key: str, default: Any = None) -> Any: 39 | key = self._normalize_key(key) 40 | for k, v in self._items: 41 | if k == key: 42 | return ",".join(v) 43 | return default 44 | 45 | def items(self) -> ItemsView: 46 | return {k: ",".join(v) for k, v in self._items}.items() 47 | 48 | def keys(self) -> KeysView: 49 | return {k: v for k, v in self.items()}.keys() 50 | 51 | def values(self) -> ValuesView: 52 | return {k: v for k, v in self.items()}.values() 53 | 54 | def update(self, headers: HeaderTypes) -> "Headers": # noqa 55 | headers = self.__class__(headers, alias=self.alias) # noqa 56 | for idx, (key, _) in enumerate(headers._items): 57 | if key in self: 58 | self.pop(key) 59 | 60 | self._items.extend(headers._items) 61 | return self 62 | 63 | def copy(self) -> "Headers": 64 | return self.__class__(self._items.copy(), alias=self.alias) # noqa 65 | 66 | def _prepare_items(self, headers: HeaderTypes) -> List[Tuple[str, Any]]: 67 | if headers is None: 68 | return [] 69 | if isinstance(headers, self.__class__): 70 | return [self._normalize(k, v) for k, v in headers._items] 71 | if isinstance(headers, Mapping): 72 | return [self._normalize(k, v) for k, v in headers.items()] 73 | if isinstance(headers, (list, tuple, set)): 74 | try: 75 | items = [self._normalize(k, args[0]) for k, *args in headers] 76 | return items 77 | except IndexError: 78 | pass 79 | raise TypeError 80 | 81 | def _normalize_key(self, key: ByteOrStr) -> str: 82 | key = to_str(key, encoding="ascii") 83 | if self.alias == HeaderAlias.ALL: 84 | return key 85 | 86 | if self.alias == HeaderAlias.CAPITALIZE: 87 | return "-".join([s.capitalize() for s in key.split("-")]) 88 | 89 | return key.lower() 90 | 91 | def _normalize_value(self, value) -> List[str]: 92 | if isinstance(value, dict): 93 | raise TypeError 94 | 95 | if isinstance(value, (list, tuple, set)): 96 | items = [] 97 | for item in value: 98 | if isinstance(item, dict): 99 | raise TypeError 100 | items.append(to_str(item)) 101 | return items 102 | 103 | return [to_str(value)] 104 | 105 | def _normalize(self, key, value) -> Tuple[str, List[str]]: 106 | return self._normalize_key(key), self._normalize_value(value) 107 | 108 | def __setitem__(self, key, value) -> None: 109 | found = False 110 | key, value = self._normalize(key, value) 111 | for idx, (k, _) in enumerate(self._items): 112 | if k == key: 113 | values = [v for v in value if v not in self._items[idx][1]] 114 | self._items[idx][1].extend(values) 115 | found = True 116 | break 117 | 118 | if not found: 119 | self._items.append((key, value)) 120 | 121 | def __getitem__(self, key): 122 | return self.get(key) 123 | 124 | def __delitem__(self, key): 125 | key = self._normalize_key(key) 126 | pop_idx = None 127 | for idx, (k, _) in enumerate(self._items): 128 | if key == k: 129 | pop_idx = idx 130 | break 131 | 132 | if pop_idx is not None: 133 | self._items.pop(pop_idx) 134 | 135 | def __contains__(self, key: Any) -> bool: 136 | key = self._normalize_key(key) 137 | for k, _ in self._items: 138 | if key == k: 139 | return True 140 | return False 141 | 142 | def __iter__(self): 143 | return (k for k, _ in self._items) 144 | 145 | def __len__(self): 146 | return len(self._headers) 147 | 148 | def __eq__(self, other: HeaderTypes): 149 | items = sorted(self._items) 150 | other = sorted(self._prepare_items(other)) 151 | return items == other 152 | 153 | def __repr__(self): 154 | SECURE = [ 155 | self._normalize_key(key) for key in ["Authorization", "Proxy-Authorization"] 156 | ] 157 | return "<%s: %s>" % ( 158 | self.__class__.__name__, 159 | {k: "[secure]" if k in SECURE else ",".join(v) for k, v in self._items}, 160 | ) 161 | -------------------------------------------------------------------------------- /tls_requests/models/libraries.py: -------------------------------------------------------------------------------- 1 | import ctypes 2 | import glob 3 | import os 4 | import platform 5 | import re 6 | import sys 7 | from dataclasses import dataclass, field, fields 8 | from pathlib import Path 9 | from platform import machine 10 | from typing import List, Optional 11 | 12 | import requests 13 | from tqdm import tqdm 14 | 15 | __all__ = ["TLSLibrary"] 16 | 17 | BIN_DIR = os.path.join(Path(__file__).resolve(strict=True).parent.parent / "bin") 18 | GITHUB_API_URL = "https://api.github.com/repos/bogdanfinn/tls-client/releases" 19 | PLATFORM = sys.platform 20 | IS_UBUNTU = False 21 | ARCH_MAPPING = { 22 | "amd64": "amd64", 23 | "x86_64": "amd64", 24 | "x86": "386", 25 | "i686": "386", 26 | "i386": "386", 27 | "arm64": "arm64", 28 | "aarch64": "arm64", 29 | "armv5l": "arm-5", 30 | "armv6l": "arm-6", 31 | "armv7l": "arm-7", 32 | "ppc64le": "ppc64le", 33 | "riscv64": "riscv64", 34 | "s390x": "s390x", 35 | } 36 | 37 | FILE_EXT = ".unk" 38 | MACHINE = ARCH_MAPPING.get(machine()) or machine() 39 | if PLATFORM == "linux": 40 | FILE_EXT = "so" 41 | try: 42 | platform_data = platform.freedesktop_os_release() 43 | if "ID" in platform_data: 44 | curr_system = platform_data["ID"] 45 | else: 46 | curr_system = platform_data.get("id") 47 | 48 | if "ubuntu" in str(curr_system).lower(): 49 | IS_UBUNTU = True 50 | 51 | except Exception as e: # noqa 52 | pass 53 | 54 | elif PLATFORM in ("win32", "cygwin"): 55 | PLATFORM = "windows" 56 | FILE_EXT = "dll" 57 | elif PLATFORM == "darwin": 58 | FILE_EXT = "dylib" 59 | 60 | PATTERN_RE = re.compile(r"%s-%s.*%s" % (PLATFORM, MACHINE, FILE_EXT), re.I) 61 | PATTERN_UBUNTU_RE = re.compile(r"%s-%s.*%s" % ("ubuntu", MACHINE, FILE_EXT), re.I) 62 | 63 | TLS_LIBRARY_PATH = os.getenv("TLS_LIBRARY_PATH") 64 | 65 | 66 | @dataclass 67 | class BaseRelease: 68 | 69 | @classmethod 70 | def model_fields_set(cls) -> set: 71 | return {model_field.name for model_field in fields(cls)} 72 | 73 | @classmethod 74 | def from_kwargs(cls, **kwargs): 75 | model_fields_set = cls.model_fields_set() 76 | return cls(**{k: v for k, v in kwargs.items() if k in model_fields_set}) 77 | 78 | 79 | @dataclass 80 | class ReleaseAsset(BaseRelease): 81 | browser_download_url: str 82 | name: Optional[str] = None 83 | 84 | 85 | @dataclass 86 | class Release(BaseRelease): 87 | name: Optional[str] = None 88 | tag_name: Optional[str] = None 89 | assets: List[ReleaseAsset] = field(default_factory=list) 90 | 91 | @classmethod 92 | def from_kwargs(cls, **kwargs): 93 | model_fields_set = cls.model_fields_set() 94 | assets = kwargs.pop("assets", []) or [] 95 | kwargs["assets"] = [ 96 | ReleaseAsset.from_kwargs(**asset_kwargs) for asset_kwargs in assets 97 | ] 98 | return cls(**{k: v for k, v in kwargs.items() if k in model_fields_set}) 99 | 100 | 101 | class TLSLibrary: 102 | """TLS Library 103 | 104 | A utility class for managing the TLS library, including discovery, validation, 105 | downloading, and loading. This class facilitates interaction with system-specific 106 | binaries, ensuring compatibility with the platform and machine architecture. 107 | 108 | Class Attributes: 109 | _PATH (str): The current path to the loaded TLS library. 110 | 111 | Methods: 112 | fetch_api(version: Optional[str] = None, retries: int = 3) -> Generator[str, None, None]: 113 | Fetches library download URLs from the GitHub API for the specified version. 114 | 115 | is_valid(fp: str) -> bool: 116 | Validates a file path against platform-specific patterns. 117 | 118 | find() -> str: 119 | Finds the first valid library binary in the binary directory. 120 | 121 | find_all() -> list[str]: 122 | Lists all library binaries in the binary directory. 123 | 124 | download(version: Optional[str] = None) -> str: 125 | Downloads the library binary for the specified version. 126 | 127 | set_path(fp: str): 128 | Sets the path to the currently loaded library. 129 | 130 | load() -> ctypes.CDLL: 131 | Loads the library, either from an existing path or by discovering and downloading it. 132 | """ 133 | 134 | _PATH: str = None 135 | _STATIC_API_DATA = { 136 | "name": "v1.7.10", 137 | "tag_name": "v1.7.10", 138 | "assets": [ 139 | { 140 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-darwin-amd64-1.7.10.dylib", 141 | "name": "tls-client-darwin-amd64-1.7.10.dylib", 142 | }, 143 | { 144 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-darwin-arm64-1.7.10.dylib", 145 | "name": "tls-client-darwin-arm64-1.7.10.dylib", 146 | }, 147 | { 148 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-linux-arm64-1.7.10.so", 149 | "name": "tls-client-linux-arm64-1.7.10.so", 150 | }, 151 | { 152 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-linux-armv7-1.7.10.so", 153 | "name": "tls-client-linux-armv7-1.7.10.so", 154 | }, 155 | { 156 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-linux-ubuntu-amd64-1.7.10.so", 157 | "name": "tls-client-linux-ubuntu-amd64-1.7.10.so", 158 | }, 159 | { 160 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-windows-32-1.7.10.dll", 161 | "name": "tls-client-windows-32-1.7.10.dll", 162 | }, 163 | { 164 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-windows-64-1.7.10.dll", 165 | "name": "tls-client-windows-64-1.7.10.dll", 166 | }, 167 | { 168 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-xgo-1.7.10-darwin-amd64.dylib", 169 | "name": "tls-client-xgo-1.7.10-darwin-amd64.dylib", 170 | }, 171 | { 172 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-xgo-1.7.10-darwin-arm64.dylib", 173 | "name": "tls-client-xgo-1.7.10-darwin-arm64.dylib", 174 | }, 175 | { 176 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-xgo-1.7.10-linux-386.so", 177 | "name": "tls-client-xgo-1.7.10-linux-386.so", 178 | }, 179 | { 180 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-xgo-1.7.10-linux-amd64.so", 181 | "name": "tls-client-xgo-1.7.10-linux-amd64.so", 182 | }, 183 | { 184 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-xgo-1.7.10-linux-arm-5.so", 185 | "name": "tls-client-xgo-1.7.10-linux-arm-5.so", 186 | }, 187 | { 188 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-xgo-1.7.10-linux-arm-6.so", 189 | "name": "tls-client-xgo-1.7.10-linux-arm-6.so", 190 | }, 191 | { 192 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-xgo-1.7.10-linux-arm-7.so", 193 | "name": "tls-client-xgo-1.7.10-linux-arm-7.so", 194 | }, 195 | { 196 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-xgo-1.7.10-linux-arm64.so", 197 | "name": "tls-client-xgo-1.7.10-linux-arm64.so", 198 | }, 199 | { 200 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-xgo-1.7.10-linux-ppc64le.so", 201 | "name": "tls-client-xgo-1.7.10-linux-ppc64le.so", 202 | }, 203 | { 204 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-xgo-1.7.10-linux-riscv64.so", 205 | "name": "tls-client-xgo-1.7.10-linux-riscv64.so", 206 | }, 207 | { 208 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-xgo-1.7.10-linux-s390x.so", 209 | "name": "tls-client-xgo-1.7.10-linux-s390x.so", 210 | }, 211 | { 212 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-xgo-1.7.10-windows-386.dll", 213 | "name": "tls-client-xgo-1.7.10-windows-386.dll", 214 | }, 215 | { 216 | "browser_download_url": "https://github.com/bogdanfinn/tls-client/releases/download/v1.7.10/tls-client-xgo-1.7.10-windows-amd64.dll", 217 | "name": "tls-client-xgo-1.7.10-windows-amd64.dll", 218 | }, 219 | ], 220 | } 221 | 222 | @classmethod 223 | def fetch_api(cls, version: str = None, retries: int = 3): 224 | def _find_release(data, version_: str = None): 225 | releases = [ 226 | Release.from_kwargs(**kwargs) for kwargs in data 227 | ] 228 | 229 | if version_ is not None: 230 | version_ = ( 231 | "v%s" % version_ 232 | if not str(version_).startswith("v") 233 | else str(version_) 234 | ) 235 | releases = [ 236 | release 237 | for release in releases 238 | if re.search(version_, release.name, re.I) 239 | ] 240 | 241 | for release in releases: 242 | for asset in release.assets: 243 | if IS_UBUNTU and PATTERN_UBUNTU_RE.search( 244 | asset.browser_download_url 245 | ): 246 | ubuntu_urls.append(asset.browser_download_url) 247 | if PATTERN_RE.search(asset.browser_download_url): 248 | asset_urls.append(asset.browser_download_url) 249 | 250 | asset_urls, ubuntu_urls = [], [] 251 | for _ in range(retries): 252 | try: 253 | response = requests.get(GITHUB_API_URL) 254 | if response.ok: 255 | _find_release(response.json()) 256 | break 257 | 258 | except Exception as ex: 259 | print("Unable to fetch GitHub API: %s" % ex) 260 | 261 | if not asset_urls and not ubuntu_urls: 262 | _find_release([cls._STATIC_API_DATA]) 263 | 264 | for url in ubuntu_urls: 265 | yield url 266 | 267 | for url in asset_urls: 268 | yield url 269 | 270 | @classmethod 271 | def find(cls) -> str: 272 | for fp in cls.find_all(): 273 | if PATTERN_RE.search(fp): 274 | return fp 275 | 276 | @classmethod 277 | def find_all(cls) -> List[str]: 278 | return [ 279 | src 280 | for src in glob.glob(os.path.join(BIN_DIR, r"*")) 281 | if src.lower().endswith(("so", "dll", "dylib")) 282 | ] 283 | 284 | @classmethod 285 | def download(cls, version: str = None) -> str: 286 | try: 287 | print( 288 | "System Info - Platform: %s, Machine: %s, File Ext : %s." 289 | % ( 290 | PLATFORM, 291 | "%s (Ubuntu)" % MACHINE if IS_UBUNTU else MACHINE, 292 | FILE_EXT, 293 | ) 294 | ) 295 | download_url = None 296 | for download_url in cls.fetch_api(version): 297 | if download_url: 298 | break 299 | 300 | print("Library Download URL: %s" % download_url) 301 | if download_url: 302 | destination = os.path.join(BIN_DIR, download_url.split("/")[-1]) 303 | with requests.get(download_url, stream=True) as response: 304 | response.raise_for_status() 305 | os.makedirs(BIN_DIR, exist_ok=True) 306 | total_size = int(response.headers.get("content-length", 0)) 307 | chunk_size = 1024 308 | with open( 309 | os.path.join(BIN_DIR, download_url.split("/")[-1]), "wb" 310 | ) as file, tqdm( 311 | desc=destination, 312 | total=total_size, 313 | unit="iB", 314 | unit_scale=True, 315 | unit_divisor=chunk_size, 316 | ) as progress_bar: 317 | for chunk in response.iter_content(chunk_size): 318 | size = file.write(chunk) 319 | progress_bar.update(size) 320 | 321 | return destination 322 | 323 | except requests.exceptions.HTTPError as ex: 324 | print("Unable to download file: %s" % ex) 325 | 326 | @classmethod 327 | def set_path(cls, fp: str): 328 | cls._PATH = fp 329 | 330 | @classmethod 331 | def load(cls): 332 | """Load libraries""" 333 | 334 | def _load_libraries(fp_): 335 | try: 336 | lib = ctypes.cdll.LoadLibrary(fp_) 337 | cls.set_path(fp_) 338 | return lib 339 | except Exception as ex: 340 | print("Unable to load TLS Library, details: %s" % ex) 341 | try: 342 | os.remove(fp_) 343 | except FileNotFoundError: 344 | pass 345 | 346 | if cls._PATH is not None: 347 | library = _load_libraries(cls._PATH) 348 | if library: 349 | return library 350 | 351 | if TLS_LIBRARY_PATH: 352 | library = _load_libraries(TLS_LIBRARY_PATH) 353 | if library: 354 | return library 355 | 356 | for fp in cls.find_all(): 357 | if IS_UBUNTU and PATTERN_UBUNTU_RE.search(fp): 358 | library = _load_libraries(fp) 359 | if library: 360 | return library 361 | if PATTERN_RE.search(fp): 362 | library = _load_libraries(fp) 363 | if library: 364 | return library 365 | 366 | download_fp = cls.download() 367 | if download_fp: 368 | library = _load_libraries(download_fp) 369 | if library: 370 | return library 371 | 372 | raise OSError("Your system does not support TLS Library.") 373 | -------------------------------------------------------------------------------- /tls_requests/models/request.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from tls_requests.models.cookies import Cookies 4 | from tls_requests.models.encoders import StreamEncoder 5 | from tls_requests.models.headers import Headers 6 | from tls_requests.models.urls import URL, Proxy 7 | from tls_requests.settings import DEFAULT_TIMEOUT 8 | from tls_requests.types import (CookieTypes, HeaderTypes, MethodTypes, 9 | ProxyTypes, RequestData, RequestFiles, 10 | TimeoutTypes, URLParamTypes, URLTypes) 11 | 12 | __all__ = ["Request"] 13 | 14 | 15 | class Request: 16 | def __init__( 17 | self, 18 | method: MethodTypes, 19 | url: URLTypes, 20 | *, 21 | data: RequestData = None, 22 | files: RequestFiles = None, 23 | json: Any = None, 24 | params: URLParamTypes = None, 25 | headers: HeaderTypes = None, 26 | cookies: CookieTypes = None, 27 | proxy: ProxyTypes = None, 28 | timeout: TimeoutTypes = None, 29 | ) -> None: 30 | self._content = None 31 | self._session_id = None 32 | self.url = URL(url, params=params) 33 | self.method = method.upper() 34 | self.cookies = Cookies(cookies) 35 | self.proxy = Proxy(proxy) if proxy else None 36 | self.timeout = timeout if isinstance(timeout, (float, int)) else DEFAULT_TIMEOUT 37 | self.stream = StreamEncoder(data, files, json) 38 | self.headers = self._prepare_headers(headers) 39 | 40 | def _prepare_headers(self, headers) -> Headers: 41 | headers = Headers(headers) 42 | headers.update(self.stream.headers) 43 | if self.url.host and "Host" not in headers: 44 | headers.setdefault(b"Host", self.url.host) 45 | 46 | return headers 47 | 48 | @property 49 | def id(self): 50 | return self._session_id 51 | 52 | @property 53 | def content(self) -> bytes: 54 | return self._content 55 | 56 | def read(self): 57 | return b"".join(self.stream.render()) 58 | 59 | async def aread(self): 60 | return b"".join(await self.stream.render()) 61 | 62 | def __repr__(self) -> str: 63 | return "<%s: (%s, %s)>" % (self.__class__.__name__, self.method, self.url) 64 | -------------------------------------------------------------------------------- /tls_requests/models/response.py: -------------------------------------------------------------------------------- 1 | import binascii 2 | import codecs 3 | import datetime 4 | from email.message import Message 5 | from typing import Any, Callable, Optional, TypeVar, Union 6 | 7 | from tls_requests.exceptions import Base64DecodeError, HTTPError 8 | from tls_requests.models.cookies import Cookies 9 | from tls_requests.models.encoders import StreamEncoder 10 | from tls_requests.models.headers import Headers 11 | from tls_requests.models.request import Request 12 | from tls_requests.models.status_codes import StatusCodes 13 | from tls_requests.models.tls import TLSResponse 14 | from tls_requests.settings import CHUNK_SIZE 15 | from tls_requests.types import CookieTypes, HeaderTypes, ResponseHistory 16 | from tls_requests.utils import b64decode, chardet, to_json 17 | 18 | __all__ = ["Response"] 19 | 20 | T = TypeVar("T", bound="Response") 21 | 22 | REDIRECT_STATUS = ( 23 | StatusCodes.MOVED_PERMANENTLY, # 301 24 | StatusCodes.FOUND, # 302 25 | StatusCodes.SEE_OTHER, # 303 26 | StatusCodes.TEMPORARY_REDIRECT, # 307 27 | StatusCodes.PERMANENT_REDIRECT, # 308 28 | ) 29 | 30 | 31 | class Response: 32 | def __init__( 33 | self, 34 | status_code: int, 35 | *, 36 | headers: HeaderTypes = None, 37 | cookies: CookieTypes = None, 38 | request: Union[Request] = None, 39 | history: ResponseHistory = None, 40 | body: bytes = None, 41 | stream: StreamEncoder = None, 42 | default_encoding: Union[str, Callable] = "utf-8", 43 | ) -> None: 44 | self._content = None 45 | self._elapsed = None 46 | self._encoding = None 47 | self._text = None 48 | self._response_id = None 49 | self._http_version = None 50 | self._request: Optional[Request] = request 51 | self._cookies = Cookies(cookies) 52 | self._is_stream_consumed = False 53 | self._is_closed = False 54 | self._next: Optional[Request] = None 55 | self.headers = Headers(headers) 56 | self.status_code = status_code 57 | self.history = history if isinstance(history, list) else [] 58 | self.default_encoding = default_encoding 59 | if isinstance(stream, StreamEncoder): 60 | self.stream = stream 61 | else: 62 | self.stream = StreamEncoder.from_bytes(body or b"", chunk_size=CHUNK_SIZE) 63 | 64 | @property 65 | def id(self) -> str: 66 | return self._response_id 67 | 68 | @property 69 | def elapsed(self) -> datetime.timedelta: 70 | return self._elapsed 71 | 72 | @elapsed.setter 73 | def elapsed(self, elapsed: datetime.timedelta) -> None: 74 | self._elapsed = elapsed 75 | 76 | @property 77 | def request(self) -> Request: 78 | if self._request is None: 79 | raise RuntimeError( 80 | "The request instance has not been set on this response." 81 | ) 82 | return self._request 83 | 84 | @request.setter 85 | def request(self, value: Request) -> None: 86 | self._request = value 87 | 88 | @property 89 | def next(self): 90 | return self._next 91 | 92 | @next.setter 93 | def next(self, value: Request) -> None: 94 | if isinstance(value, Request): 95 | self._next = value 96 | 97 | @property 98 | def http_version(self) -> str: 99 | return self._http_version or "HTTP/1.1" 100 | 101 | @property 102 | def cookies(self) -> Cookies: 103 | if self._cookies is None: 104 | self._cookies = Cookies() 105 | self._cookies.extract_cookies(self, self.request) 106 | return self._cookies 107 | 108 | @property 109 | def reason(self) -> str: 110 | if self.status_code == 0: 111 | return self.text or StatusCodes.get_reason(self.status_code) 112 | return StatusCodes.get_reason(self.status_code) 113 | 114 | @property 115 | def url(self): 116 | return self.request.url 117 | 118 | @property 119 | def content(self) -> bytes: 120 | return self._content 121 | 122 | @property 123 | def text(self) -> str: 124 | if self._text is None: 125 | self._text = "" 126 | if self.content: 127 | decoder = codecs.getincrementaldecoder(self.encoding)(errors="replace") 128 | self._text = decoder.decode(self.content) 129 | return self._text 130 | 131 | @property 132 | def charset(self) -> Optional[str]: 133 | if self.headers.get("Content-Type"): 134 | msg = Message() 135 | msg["content-type"] = self.headers["Content-Type"] 136 | return msg.get_content_charset(failobj=None) 137 | return None 138 | 139 | @property 140 | def encoding(self) -> str: 141 | if self._encoding is None: 142 | encoding = self.charset 143 | if encoding is None: 144 | if isinstance(self.default_encoding, str): 145 | try: 146 | codecs.lookup(self.default_encoding) 147 | encoding = self.default_encoding 148 | except LookupError: 149 | pass 150 | 151 | if not encoding and chardet and self.content: 152 | encoding = chardet.detect(self.content)["encoding"] 153 | 154 | self._encoding = encoding or "utf-8" 155 | return self._encoding 156 | 157 | @property 158 | def ok(self) -> bool: 159 | try: 160 | self.raise_for_status() 161 | except HTTPError: 162 | return False 163 | return True 164 | 165 | def __bool__(self) -> bool: 166 | return self.ok 167 | 168 | @property 169 | def is_redirect(self) -> bool: 170 | return "Location" in self.headers and self.status_code in REDIRECT_STATUS 171 | 172 | @property 173 | def is_permanent_redirect(self): 174 | return "Location" in self.headers and self.status_code in ( 175 | StatusCodes.MOVED_PERMANENTLY, 176 | StatusCodes.PERMANENT_REDIRECT, 177 | ) 178 | 179 | def raise_for_status(self) -> "Response": 180 | http_error_msg = "" 181 | if self.status_code < 100: 182 | http_error_msg = "{0} TLS Client Error: {1} for url: {2}" 183 | 184 | elif 400 <= self.status_code < 500: 185 | http_error_msg = "{0} Client Error: {1} for url: {2}" 186 | 187 | elif 500 <= self.status_code < 600: 188 | http_error_msg = "{0} Server Error: {1} for url: {2}" 189 | 190 | if http_error_msg: 191 | raise HTTPError( 192 | http_error_msg.format( 193 | self.status_code, 194 | ( 195 | self.reason 196 | if self.status_code < 100 197 | else StatusCodes.get_reason(self.status_code) 198 | ), 199 | self.url, 200 | ), 201 | response=self, 202 | ) 203 | 204 | return self 205 | 206 | def json(self, **kwargs: Any) -> Any: 207 | return to_json(self.text, **kwargs) 208 | 209 | def __repr__(self) -> str: 210 | return f"" 211 | 212 | def read(self) -> bytes: 213 | with self.stream as stream: 214 | self._content = b"".join(stream.render()) 215 | return self._content 216 | 217 | async def aread(self) -> bytes: 218 | with self.stream as stream: 219 | self._content = b"".join([chunk async for chunk in stream]) 220 | return self._content 221 | 222 | @property 223 | def closed(self): 224 | return self._is_closed 225 | 226 | def close(self) -> None: 227 | if not self._is_closed: 228 | self._is_closed = True 229 | self._is_stream_consumed = True 230 | self.stream.close() 231 | 232 | # Fix pickle dump 233 | # Ref: https://github.com/thewebscraping/tls-requests/issues/35 234 | self.stream = None 235 | 236 | async def aclose(self) -> None: 237 | return self.close() 238 | 239 | @classmethod 240 | def from_tls_response( 241 | cls, response: TLSResponse, is_byte_response: bool = False 242 | ) -> "Response": 243 | def _parse_response_body(value: Optional[str]) -> bytes: 244 | if value: 245 | if is_byte_response and response.status > 0: 246 | try: 247 | value = b64decode(value.split(",")[-1]) 248 | return value 249 | except (binascii.Error, AssertionError): 250 | raise Base64DecodeError( 251 | "Couldn't decode the base64 string into bytes." 252 | ) 253 | return value.encode("utf-8") 254 | return b"" 255 | 256 | ret = cls( 257 | status_code=response.status, 258 | body=_parse_response_body(response.body), 259 | headers=response.headers, 260 | cookies=response.cookies, 261 | ) 262 | ret._response_id = response.id 263 | ret._http_version = response.usedProtocol 264 | return ret 265 | -------------------------------------------------------------------------------- /tls_requests/models/status_codes.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | __all__ = ["StatusCodes"] 4 | 5 | 6 | class StatusCodes(int, Enum): 7 | def __new__(cls, value: int, reason: str = ""): 8 | obj = int.__new__(cls, value) 9 | obj._value_ = value 10 | obj.reason = reason 11 | return obj 12 | 13 | def __str__(self) -> str: 14 | return str(self.value) 15 | 16 | @classmethod 17 | def get_reason(cls, value: int): 18 | if value in cls._value2member_map_: 19 | return cls._value2member_map_[value].reason 20 | return "Unknown Error" 21 | 22 | CONTINUE = 100, "Continue" 23 | SWITCHING_PROTOCOLS = 101, "Switching Protocols" 24 | PROCESSING = 102, "Processing" 25 | CHECKPOINT = 103, "Checkpoint" 26 | URI_TOO_LONG = 122, "Uri Too Long" 27 | OK = 200, "OK" 28 | CREATED = 201, "Created" 29 | ACCEPTED = 202, "Accepted" 30 | NON_AUTHORITATIVE_INFO = 203, "Non Authoritative Info" 31 | NO_CONTENT = 204, "No Content" 32 | RESET_CONTENT = 205, "Reset Content" 33 | PARTIAL_CONTENT = 206, "Partial Content" 34 | MULTI_STATUS = 207, "Multi Status" 35 | ALREADY_REPORTED = 208, "Already Reported" 36 | IM_USED = 226, "Im Used" 37 | MULTIPLE_CHOICES = 300, "Multiple Choices" 38 | MOVED_PERMANENTLY = 301, "Moved Permanently" 39 | FOUND = 302, "Found" 40 | SEE_OTHER = 303, "See Other" 41 | NOT_MODIFIED = 304, "Not Modified" 42 | USE_PROXY = 305, "Use Proxy" 43 | SWITCH_PROXY = 306, "Switch Proxy" 44 | TEMPORARY_REDIRECT = 307, "Temporary Redirect" 45 | PERMANENT_REDIRECT = 308, "Permanent Redirect" 46 | BAD_REQUEST = 400, "Bad Request" 47 | UNAUTHORIZED = 401, "Unauthorized" 48 | PAYMENT_REQUIRED = 402, "Payment Required" 49 | FORBIDDEN = 403, "Forbidden" 50 | NOT_FOUND = 404, "Not Found" 51 | METHOD_NOT_ALLOWED = 405, "Method Not Allowed" 52 | NOT_ACCEPTABLE = 406, "Not Acceptable" 53 | PROXY_AUTHENTICATION_REQUIRED = 407, "Proxy Authentication Required" 54 | REQUEST_TIMEOUT = 408, "Request Timeout" 55 | CONFLICT = 409, "Conflict" 56 | GONE = 410, "Gone" 57 | LENGTH_REQUIRED = 411, "Length Required" 58 | PRECONDITION_FAILED = 412, "Precondition Failed" 59 | REQUEST_ENTITY_TOO_LARGE = 413, "Request Entity Too Large" 60 | REQUEST_URI_TOO_LARGE = 414, "Request Uri Too Large" 61 | UNSUPPORTED_MEDIA_TYPE = 415, "Unsupported Media Type" 62 | REQUESTED_RANGE_NOT_SATISFIABLE = 416, "Requested Range Not Satisfiable" 63 | EXPECTATION_FAILED = 417, "Expectation Failed" 64 | IM_A_TEAPOT = 418, "Im A Teapot" 65 | MISDIRECTED_REQUEST = 421, "Misdirected Request" 66 | UNPROCESSABLE_ENTITY = 422, "Unprocessable Entity" 67 | LOCKED = 423, "Locked" 68 | FAILED_DEPENDENCY = 424, "Failed Dependency" 69 | UNORDERED_COLLECTION = 425, "Unordered Collection" 70 | UPGRADE_REQUIRED = 426, "Upgrade Required" 71 | PRECONDITION_REQUIRED = 428, "Precondition Required" 72 | TOO_MANY_REQUESTS = 429, "Too Many Requests" 73 | HEADER_FIELDS_TOO_LARGE = 431, "Header Fields Too Large" 74 | NO_RESPONSE = 444, "No Response" 75 | RETRY_WITH = 449, "Retry With" 76 | BLOCKED_BY_WINDOWS_PARENTAL_CONTROLS = 450, "Blocked By Windows Parental Controls" 77 | UNAVAILABLE_FOR_LEGAL_REASONS = 451, "Unavailable For Legal Reasons" 78 | CLIENT_CLOSED_REQUEST = 499, "Client Closed Request" 79 | INTERNAL_SERVER_ERROR = 500, "Internal Server Error" 80 | NOT_IMPLEMENTED = 501, "Not Implemented" 81 | BAD_GATEWAY = 502, "Bad Gateway" 82 | SERVICE_UNAVAILABLE = 503, "Service Unavailable" 83 | GATEWAY_TIMEOUT = 504, "Gateway Timeout" 84 | HTTP_VERSION_NOT_SUPPORTED = 505, "Http Version Not Supported" 85 | VARIANT_ALSO_NEGOTIATES = 506, "Variant Also Negotiates" 86 | INSUFFICIENT_STORAGE = 507, "Insufficient Storage" 87 | BANDWIDTH_LIMIT_EXCEEDED = 509, "Bandwidth Limit Exceeded" 88 | NOT_EXTENDED = 510, "Not Extended" 89 | NETWORK_AUTHENTICATION_REQUIRED = 511, "Network Authentication Required" 90 | -------------------------------------------------------------------------------- /tls_requests/models/tls.py: -------------------------------------------------------------------------------- 1 | import ctypes 2 | import uuid 3 | from dataclasses import asdict, dataclass, field 4 | from dataclasses import fields as get_fields 5 | from typing import Any, List, Mapping, Optional, Set, TypeVar, Union 6 | 7 | from tls_requests.models.encoders import StreamEncoder 8 | from tls_requests.models.libraries import TLSLibrary 9 | from tls_requests.models.status_codes import StatusCodes 10 | from tls_requests.settings import (DEFAULT_HEADERS, DEFAULT_TIMEOUT, 11 | DEFAULT_TLS_DEBUG, DEFAULT_TLS_HTTP2, 12 | DEFAULT_TLS_IDENTIFIER) 13 | from tls_requests.types import (MethodTypes, TLSCookiesTypes, 14 | TLSIdentifierTypes, TLSSessionId, URLTypes) 15 | from tls_requests.utils import to_base64, to_bytes, to_json 16 | 17 | __all__ = [ 18 | "TLSClient", 19 | "TLSResponse", 20 | "TLSConfig", 21 | "CustomTLSClientConfig", 22 | ] 23 | 24 | T = TypeVar("T", bound="_BaseConfig") 25 | 26 | 27 | class TLSClient: 28 | """TLSClient 29 | 30 | The `TLSClient` class provides a high-level interface for performing secure TLS-based HTTP operations. It encapsulates 31 | interactions with a custom TLS library, offering functionality for managing sessions, cookies, and HTTP requests. 32 | This class is designed to be extensible and integrates seamlessly with the `TLSResponse` and `TLSConfig` classes 33 | for handling responses and configuring requests. 34 | 35 | Attributes: 36 | _library (Optional): Reference to the loaded TLS library. 37 | _getCookiesFromSession (Optional): Function for retrieving cookies from a session. 38 | _addCookiesToSession (Optional): Function for adding cookies to a session. 39 | _destroySession (Optional): Function for destroying a specific session. 40 | _destroyAll (Optional): Function for destroying all active sessions. 41 | _request (Optional): Function for performing a TLS-based HTTP request. 42 | _freeMemory (Optional): Function for freeing allocated memory for responses. 43 | 44 | Methods: 45 | setup(cls): 46 | Loads and sets up the TLS library and initializes function bindings. 47 | 48 | get_cookies(cls, session_id: TLSSessionId, url: str) -> TLSResponse: 49 | Retrieves cookies from a session for the given URL. 50 | 51 | add_cookies(cls, session_id: TLSSessionId, payload: dict): 52 | Adds cookies to a specific session. 53 | 54 | destroy_all(cls) -> bool: 55 | Destroys all active TLS sessions. Returns `True` if successful. 56 | 57 | destroy_session(cls, session_id: TLSSessionId) -> bool: 58 | Destroys a specific TLS session. Returns `True` if successful. 59 | 60 | request(cls, payload): 61 | Performs a TLS-based HTTP request with the provided payload. 62 | 63 | free_memory(cls, response_id: str) -> None: 64 | Frees the memory allocated for a specific response. 65 | 66 | response(cls, raw: bytes) -> TLSResponse: 67 | Processes a raw byte response and returns a `TLSResponse` object. 68 | 69 | _make_request(cls, fn: callable, payload: dict): 70 | Helper method to handle request processing and response generation. 71 | 72 | Example: 73 | Initialize the client and perform operations: 74 | 75 | >>> from tls_requests.tls import TLSClient 76 | >>> client = TLSClient.initialize() 77 | >>> session_id = "my-session-id" 78 | >>> url = "https://example.com" 79 | >>> response = client.get_cookies(session_id, url) 80 | >>> print(response) 81 | """ 82 | 83 | _library = None 84 | _getCookiesFromSession = None 85 | _addCookiesToSession = None 86 | _destroySession = None 87 | _destroyAll = None 88 | _request = None 89 | _freeMemory = None 90 | 91 | def __init__(self) -> None: 92 | if self._library is None: 93 | self.initialize() 94 | 95 | @classmethod 96 | def initialize(cls): 97 | cls._library = TLSLibrary.load() 98 | for name in [ 99 | "getCookiesFromSession", 100 | "addCookiesToSession", 101 | "destroySession", 102 | "freeMemory", 103 | "request", 104 | ]: 105 | fn_name = "_%s" % name 106 | setattr(cls, fn_name, getattr(cls._library, name, None)) 107 | fn = getattr(cls, fn_name, None) 108 | if fn and callable(fn): 109 | fn.argtypes = [ctypes.c_char_p] 110 | fn.restype = ctypes.c_char_p 111 | 112 | cls._destroyAll = cls._library.destroyAll 113 | cls._destroyAll.restype = ctypes.c_char_p 114 | return cls() 115 | 116 | @classmethod 117 | def get_cookies(cls, session_id: TLSSessionId, url: str) -> "TLSResponse": 118 | response = cls._send( 119 | cls._getCookiesFromSession, {"sessionId": session_id, "url": url} 120 | ) 121 | return response 122 | 123 | @classmethod 124 | def add_cookies(cls, session_id: TLSSessionId, payload: dict): 125 | payload["sessionId"] = session_id 126 | return cls._send( 127 | cls._addCookiesToSession, 128 | payload, 129 | ) 130 | 131 | @classmethod 132 | def destroy_all(cls) -> bool: 133 | response = TLSResponse.from_bytes(cls._destroyAll()) 134 | if response.success: 135 | return True 136 | return False 137 | 138 | @classmethod 139 | def destroy_session(cls, session_id: TLSSessionId) -> bool: 140 | response = cls._send(cls._destroySession, {"sessionId": session_id}) 141 | return response.success or False 142 | 143 | @classmethod 144 | def request(cls, payload): 145 | return cls._send(cls._request, payload) 146 | 147 | @classmethod 148 | def free_memory(cls, response_id: str) -> None: 149 | cls._freeMemory(to_bytes(response_id)) 150 | 151 | @classmethod 152 | def response(cls, raw: bytes) -> "TLSResponse": 153 | response = TLSResponse.from_bytes(raw) 154 | cls.free_memory(response.id) 155 | return response 156 | 157 | @classmethod 158 | async def aresponse(cls, raw: bytes): 159 | with StreamEncoder.from_bytes(raw) as stream: 160 | content = b"".join([chunk async for chunk in stream]) 161 | return TLSResponse.from_kwargs(**to_json(content)) 162 | 163 | @classmethod 164 | async def arequest(cls, payload): 165 | return await cls._aread(cls._request, payload) 166 | 167 | @classmethod 168 | def _send(cls, fn: callable, payload: dict): 169 | return cls.response(fn(to_bytes(payload))) 170 | 171 | @classmethod 172 | async def _aread(cls, fn: callable, payload: dict): 173 | return await cls.aresponse(fn(to_bytes(payload))) 174 | 175 | 176 | @dataclass 177 | class _BaseConfig: 178 | """Base configuration for TLSSession""" 179 | 180 | @classmethod 181 | def model_fields_set(cls) -> Set[str]: 182 | return { 183 | model_field.name 184 | for model_field in get_fields(cls) 185 | if not model_field.name.startswith("_") 186 | } 187 | 188 | @classmethod 189 | def from_kwargs(cls, **kwargs: Any) -> T: 190 | model_fields_set = cls.model_fields_set() 191 | return cls(**{k: v for k, v in kwargs.items() if k in model_fields_set and v}) 192 | 193 | def to_dict(self) -> dict: 194 | return {k: v for k, v in asdict(self).items() if not k.startswith("_")} 195 | 196 | def to_payload(self) -> dict: 197 | return self.to_dict() 198 | 199 | 200 | @dataclass 201 | class TLSResponse(_BaseConfig): 202 | """TLS Response 203 | 204 | Attributes: 205 | id (Optional[str]): A unique identifier for the response. Defaults to `None`. 206 | sessionId (Optional[str]): The session ID associated with the response. Defaults to `None`. 207 | status (Optional[int]): The HTTP status code of the response. Defaults to `0`. 208 | target (Optional[str]): The target URL or endpoint of the response. Defaults to `None`. 209 | body (Optional[str]): The body content of the response. Defaults to `None`. 210 | headers (Optional[dict]): A dictionary containing the headers of the response. Defaults to an empty dictionary. 211 | cookies (Optional[dict]): A dictionary containing the cookies of the response. Defaults to an empty dictionary. 212 | success (Optional[bool]): Indicates if the response was successful. Defaults to `False`. 213 | usedProtocol (Optional[str]): The protocol used in the response. Defaults to `"HTTP/1.1"`. 214 | 215 | Methods: 216 | from_bytes(cls, raw: bytes) -> TLSResponse: 217 | Parses a raw byte stream and constructs a `TLSResponse` object. 218 | 219 | reason_phrase -> str: 220 | A property that provides the reason phrase associated with the HTTP status code. 221 | If the status code is `0`, it returns `"Bad Request"`. 222 | """ 223 | 224 | id: Optional[str] = None 225 | sessionId: Optional[str] = None 226 | status: Optional[int] = 0 227 | target: Optional[str] = None 228 | body: Optional[str] = None 229 | headers: Optional[dict] = field(default_factory=dict) 230 | cookies: Optional[dict] = field(default_factory=dict) 231 | success: Optional[bool] = False 232 | usedProtocol: Optional[str] = "HTTP/1.1" 233 | 234 | @classmethod 235 | def from_bytes(cls, raw: bytes) -> "TLSResponse": 236 | with StreamEncoder.from_bytes(raw) as stream: 237 | return cls.from_kwargs(**to_json(b"".join(stream))) 238 | 239 | @property 240 | def reason(self) -> str: 241 | return StatusCodes.get_reason(self.status) 242 | 243 | def __repr__(self): 244 | return "" % self.status 245 | 246 | 247 | @dataclass 248 | class TLSRequestCookiesConfig(_BaseConfig): 249 | """ 250 | Request Cookies Configuration 251 | 252 | Represents a single request cookie with a _name and value. 253 | 254 | Attributes: 255 | name (str): The _name of the cookie. 256 | value (str): The value of the cookie. 257 | 258 | Example: 259 | Create a `TLSRequestCookiesConfig` object: 260 | 261 | >>> from tls_requests.tls import TLSRequestCookiesConfig 262 | >>> kwargs = { 263 | ... "_name": "foo2", 264 | ... "value": "bar2", 265 | ... } 266 | >>> obj = TLSRequestCookiesConfig(**kwargs) 267 | """ 268 | 269 | name: str 270 | value: str 271 | 272 | 273 | @dataclass 274 | class CustomTLSClientConfig(_BaseConfig): 275 | """ 276 | Custom TLS Client Configuration 277 | 278 | The `CustomTLSClientConfig` class defines advanced configuration options for customizing TLS client behavior. 279 | It includes support for ALPN, ALPS protocols, certificate compression, HTTP/2 settings, JA3 fingerprints, and 280 | other TLS-related settings. 281 | 282 | Attributes: 283 | alpnProtocols (list[str], optional): ALPN protocols. Defaults to `None`. 284 | alpsProtocols (list[str], optional): ALPS protocols. Defaults to `None`. 285 | certCompressionAlgo (str, optional): Certificate compression algorithm. Defaults to `None`. 286 | connectionFlow (int, optional): Connection flow. Defaults to `None`. 287 | h2Settings (list[str], optional): HTTP/2 settings. Defaults to `None`. 288 | h2SettingsOrder (list[str], optional): Order of HTTP/2 settings. Defaults to `None`. 289 | headerPriority (list[str], optional): Priority of headers. Defaults to `None`. 290 | ja3String (str, optional): JA3 string. Defaults to `None`. 291 | keyShareCurves (list[str], optional): Key share curves. Defaults to `None`. 292 | priorityFrames (list[str], optional): Priority of frames. Defaults to `None`. 293 | pseudoHeaderOrder (list[str], optional): Order of pseudo headers. Defaults to `None`. 294 | supportedSignatureAlgorithms (list[str], optional): Supported signature algorithms. Defaults to `None`. 295 | supportedVersions (list[str], optional): Supported versions. Defaults to `None`. 296 | 297 | Example: 298 | Create a `CustomTLSClientConfig` instance with specific settings: 299 | 300 | >>> from tls_requests.tls import CustomTLSClientConfig 301 | >>> kwargs = { 302 | ... "alpnProtocols": ["h2", "http/1.1"], 303 | ... "alpsProtocols": ["h2"], 304 | ... "certCompressionAlgo": "brotli", 305 | ... "connectionFlow": 15663105, 306 | ... "h2Settings": { 307 | ... "HEADER_TABLE_SIZE": 65536, 308 | ... "MAX_CONCURRENT_STREAMS": 1000, 309 | ... "INITIAL_WINDOW_SIZE": 6291456, 310 | ... "MAX_HEADER_LIST_SIZE": 262144 311 | ... }, 312 | ... "h2SettingsOrder": [ 313 | ... "HEADER_TABLE_SIZE", 314 | ... "MAX_CONCURRENT_STREAMS", 315 | ... "INITIAL_WINDOW_SIZE", 316 | ... "MAX_HEADER_LIST_SIZE" 317 | ... ], 318 | ... "headerPriority": None, 319 | ... "ja3String": "771,2570-4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,2570-0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513-2570-21,2570-29-23-24,0", 320 | ... "keyShareCurves": ["GREASE", "X25519"], 321 | ... "priorityFrames": [], 322 | ... "pseudoHeaderOrder": [ 323 | ... ":method", 324 | ... ":authority", 325 | ... ":scheme", 326 | ... ":path" 327 | ... ], 328 | ... "supportedSignatureAlgorithms": [ 329 | ... "ECDSAWithP256AndSHA256", 330 | ... "PSSWithSHA256", 331 | ... "PKCS1WithSHA256", 332 | ... "ECDSAWithP384AndSHA384", 333 | ... "PSSWithSHA384", 334 | ... "PKCS1WithSHA384", 335 | ... "PSSWithSHA512", 336 | ... "PKCS1WithSHA512" 337 | ... ], 338 | ... "supportedVersions": ["GREASE", "1.3", "1.2"] 339 | ... } 340 | >>> obj = CustomTLSClientConfig.from_kwargs(**kwargs) 341 | 342 | """ 343 | 344 | alpnProtocols: List[str] = None 345 | alpsProtocols: List[str] = None 346 | certCompressionAlgo: str = None 347 | connectionFlow: int = None 348 | h2Settings: List[str] = None 349 | h2SettingsOrder: List[str] = None 350 | headerPriority: List[str] = None 351 | ja3String: str = None 352 | keyShareCurves: List[str] = None 353 | priorityFrames: List[str] = None 354 | pseudoHeaderOrder: List[str] = None 355 | supportedSignatureAlgorithms: List[str] = None 356 | supportedVersions: List[str] = None 357 | 358 | 359 | @dataclass 360 | class TLSConfig(_BaseConfig): 361 | """TLS Configuration 362 | 363 | The `TLSConfig` class provides a structured and flexible way to configure TLS-specific settings for HTTP requests. 364 | It supports features like custom headers, cookie handling, proxy configuration, and advanced TLS options. 365 | 366 | Methods: 367 | to_dict(self) -> dict 368 | Converts the TLS configuration object into a dictionary. 369 | 370 | copy_with(self, **kwargs) -> "TLSConfig" 371 | Creates a new `TLSConfig` object with updated properties. 372 | 373 | from_kwargs(cls, **kwargs) -> "TLSConfig" 374 | Creates a `TLSConfig` instance from keyword arguments. 375 | 376 | Example: 377 | Initialize a `TLSConfig` object using predefined or custom settings: 378 | 379 | >>> from tls_requests.tls import TLSConfig 380 | >>> kwargs = { 381 | ... "catchPanics": false, 382 | ... "certificatePinningHosts": {}, 383 | ... "customTlsClient": {}, 384 | ... "followRedirects": false, 385 | ... "forceHttp1": false, 386 | ... "headerOrder": [ 387 | ... "accept", 388 | ... "user-agent", 389 | ... "accept-encoding", 390 | ... "accept-language" 391 | ... ], 392 | ... "headers": { 393 | ... "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", 394 | ... "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", 395 | ... "accept-encoding": "gzip, deflate, br", 396 | ... "accept-language": "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7" 397 | ... }, 398 | ... "insecureSkipVerify": false, 399 | ... "isByteRequest": false, 400 | ... "isRotatingProxy": false, 401 | ... "proxyUrl": "", 402 | ... "requestBody": "", 403 | ... "requestCookies": [ 404 | ... { 405 | ... "_name": "foo", 406 | ... "value": "bar", 407 | ... }, 408 | ... { 409 | ... "_name": "bar", 410 | ... "value": "foo", 411 | ... }, 412 | ... ], 413 | ... "requestMethod": "GET", 414 | ... "requestUrl": "https://microsoft.com", 415 | ... "sessionId": "2my-session-id", 416 | ... "timeoutSeconds": 30, 417 | ... "tlsClientIdentifier": "chrome_120", 418 | ... "withDebug": false, 419 | ... "withDefaultCookieJar": false, 420 | ... "withRandomTLSExtensionOrder": false, 421 | ... "withoutCookieJar": false 422 | ... } 423 | ... >>> obj = TLSConfig.from_kwargs(**kwargs) 424 | """ 425 | 426 | catchPanics: bool = False 427 | certificatePinningHosts: Mapping[str, str] = field(default_factory=dict) 428 | customTlsClient: Optional[CustomTLSClientConfig] = None 429 | followRedirects: bool = False 430 | forceHttp1: bool = False 431 | headerOrder: List[str] = field(default_factory=list) 432 | headers: Mapping[str, str] = field(default_factory=dict) 433 | insecureSkipVerify: bool = False 434 | isByteRequest: bool = False 435 | isByteResponse: bool = True 436 | isRotatingProxy: bool = False 437 | proxyUrl: str = "" 438 | requestBody: Union[str, bytes, bytearray, None] = None 439 | requestCookies: List[TLSRequestCookiesConfig] = field(default_factory=list) 440 | requestMethod: MethodTypes = None 441 | requestUrl: Optional[str] = None 442 | sessionId: str = field(default_factory=lambda: str(uuid.uuid4())) 443 | timeoutSeconds: int = 30 444 | tlsClientIdentifier: Optional[TLSIdentifierTypes] = DEFAULT_TLS_IDENTIFIER 445 | withDebug: bool = False 446 | withDefaultCookieJar: bool = False 447 | withRandomTLSExtensionOrder: bool = True 448 | withoutCookieJar: bool = False 449 | 450 | def to_dict(self) -> dict: 451 | """Converts the TLS configuration object into a dictionary.""" 452 | 453 | if self.customTlsClient: 454 | self.tlsClientIdentifier = None 455 | 456 | self.followRedirects = False 457 | if self.requestBody and isinstance(self.requestBody, (bytes, bytearray)): 458 | self.isByteRequest = True 459 | self.requestBody = to_base64(self.requestBody) 460 | else: 461 | self.isByteRequest = False 462 | self.requestBody = None 463 | 464 | self.timeoutSeconds = ( 465 | int(self.timeoutSeconds) 466 | if isinstance(self.timeoutSeconds, (float, int)) 467 | else DEFAULT_TIMEOUT 468 | ) 469 | return asdict(self) 470 | 471 | def copy_with( 472 | self, 473 | session_id: str = None, 474 | headers: Mapping[str, str] = None, 475 | cookies: TLSCookiesTypes = None, 476 | method: MethodTypes = None, 477 | url: URLTypes = None, 478 | body: Union[str, bytes, bytearray] = None, 479 | is_byte_request: bool = None, 480 | proxy: str = None, 481 | http2: bool = None, 482 | timeout: Union[float, int] = None, 483 | verify: bool = None, 484 | tls_identifier: Optional[TLSIdentifierTypes] = None, 485 | tls_debug: bool = None, 486 | **kwargs, 487 | ) -> "TLSConfig": 488 | """Creates a new `TLSConfig` object with updated properties.""" 489 | 490 | kwargs.update( 491 | dict( 492 | sessionId=session_id, 493 | headers=headers, 494 | requestCookies=cookies, 495 | requestMethod=method, 496 | requestUrl=url, 497 | requestBody=body, 498 | isByteRequest=is_byte_request, 499 | proxyUrl=proxy, 500 | forceHttp1=not http2, 501 | timeoutSeconds=timeout, 502 | insecureSkipVerify=not verify, 503 | tlsClientIdentifier=tls_identifier, 504 | withDebug=tls_debug, 505 | ) 506 | ) 507 | current_kwargs = asdict(self) 508 | for k, v in current_kwargs.items(): 509 | if kwargs.get(k) is not None: 510 | current_kwargs[k] = kwargs[k] 511 | 512 | return self.__class__(**current_kwargs) 513 | 514 | @classmethod 515 | def from_kwargs( 516 | cls, 517 | session_id: str = None, 518 | headers: Mapping[str, str] = None, 519 | cookies: TLSCookiesTypes = None, 520 | method: MethodTypes = None, 521 | url: URLTypes = None, 522 | body: Union[str, bytes, bytearray] = None, 523 | is_byte_request: bool = False, 524 | proxy: str = None, 525 | http2: bool = DEFAULT_TLS_HTTP2, 526 | timeout: Union[float, int] = DEFAULT_TIMEOUT, 527 | verify: bool = True, 528 | tls_identifier: Optional[TLSIdentifierTypes] = DEFAULT_TLS_IDENTIFIER, 529 | tls_debug: bool = DEFAULT_TLS_DEBUG, 530 | **kwargs: Any, 531 | ) -> "TLSConfig": 532 | """Creates a `TLSConfig` instance from keyword arguments.""" 533 | 534 | kwargs.update( 535 | dict( 536 | sessionId=session_id, 537 | headers=dict(headers) if headers else DEFAULT_HEADERS, 538 | requestCookies=cookies or [], 539 | requestMethod=method, 540 | requestUrl=url, 541 | requestBody=body, 542 | isByteRequest=is_byte_request, 543 | proxyUrl=proxy, 544 | forceHttp1=bool(not http2), 545 | timeoutSeconds=( 546 | int(timeout) 547 | if isinstance(timeout, (float, int)) 548 | else DEFAULT_TIMEOUT 549 | ), 550 | insecureSkipVerify=not verify, 551 | tlsClientIdentifier=tls_identifier, 552 | withDebug=tls_debug, 553 | ) 554 | ) 555 | return super().from_kwargs(**kwargs) 556 | -------------------------------------------------------------------------------- /tls_requests/models/urls.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from abc import ABC 4 | from collections.abc import Mapping 5 | from typing import Any, ItemsView, KeysView, Union, ValuesView 6 | from urllib.parse import ParseResult, quote, unquote, urlencode, urlparse 7 | 8 | import idna 9 | 10 | from tls_requests.exceptions import ProxyError, URLError, URLParamsError 11 | from tls_requests.types import (URL_ALLOWED_PARAMS, ProxyTypes, URLParamTypes, 12 | URLTypes) 13 | 14 | __all__ = ["URL", "URLParams", "Proxy"] 15 | 16 | 17 | class URLParams(Mapping, ABC): 18 | """URLParams 19 | 20 | Represents a mapping of URL parameters with utilities for normalization, encoding, and updating. 21 | This class provides a dictionary-like interface for managing URL parameters, ensuring that keys 22 | and values are properly validated and normalized. 23 | 24 | Attributes: 25 | - params (str): Returns the encoded URL parameters as a query string. 26 | 27 | Methods: 28 | - update(params: URLParamTypes = None, **kwargs): Updates the current parameters with new ones. 29 | - keys() -> KeysView: Returns a view of the parameter keys. 30 | - values() -> ValuesView: Returns a view of the parameter values. 31 | - items() -> ItemsView: Returns a view of the parameter key-value pairs. 32 | - copy() -> URLParams: Returns a copy of the current instance. 33 | - normalize(s: URL_ALLOWED_PARAMS): Normalizes a key or value to a string. 34 | 35 | Raises: 36 | - URLParamsError: Raised for invalid keys, values, or parameter types during initialization or updates. 37 | 38 | Example Usage: 39 | >>> params = URLParams({'key1': 'value1', 'key2': ['value2', 'value3']}) 40 | >>> print(str(params)) 41 | 'key1=value1&key2=value2&key2=value3' 42 | 43 | >>> params.update({'key3': 'value4'}) 44 | >>> print(params) 45 | 'key1=value1&key2=value2&key2=value3&key3=value4' 46 | 47 | >>> 'key1' in params 48 | True 49 | """ 50 | 51 | def __init__(self, params: URLParamTypes = None, **kwargs): 52 | self._data = self._prepare(params, **kwargs) 53 | 54 | @property 55 | def params(self) -> str: 56 | return str(self) 57 | 58 | def update(self, params: URLParamTypes = None, **kwargs): 59 | self._data.update(self._prepare(params, **kwargs)) 60 | return self 61 | 62 | def keys(self) -> KeysView: 63 | return self._data.keys() 64 | 65 | def values(self) -> ValuesView: 66 | return self._data.values() 67 | 68 | def items(self) -> ItemsView: 69 | return self._data.items() 70 | 71 | def copy(self) -> URLParams: 72 | return self.__class__(self._data.copy()) 73 | 74 | def __str__(self): 75 | return urlencode(self._data, doseq=True) 76 | 77 | def __repr__(self): 78 | return "<%s: %s>" % (self.__class__.__name__, self.items()) 79 | 80 | def __contains__(self, key: Any) -> bool: 81 | return key in self._data 82 | 83 | def __setitem__(self, key, value): 84 | self._data.update(self._prepare({key: value})) 85 | 86 | def __getitem__(self, key): 87 | return self._data[key] 88 | 89 | def __delitem__(self, key): 90 | del self._data[key] 91 | 92 | def __iter__(self): 93 | return (k for k in self.keys()) 94 | 95 | def __len__(self) -> int: 96 | return len(self._data) 97 | 98 | def __hash__(self) -> int: 99 | return hash(str(self)) 100 | 101 | def __eq__(self, other) -> bool: 102 | if not isinstance(other, self.__class__): 103 | if isinstance(other, Mapping): 104 | other = self.__class__(other) 105 | else: 106 | return False 107 | return bool(self.params == other.params) 108 | 109 | def _prepare(self, params: URLParamTypes = None, **kwargs) -> Mapping: 110 | params = params or {} 111 | if not isinstance(params, (dict, self.__class__)): 112 | raise URLParamsError("Invalid parameters.") 113 | 114 | params.update(kwargs) 115 | for k, v in params.items(): 116 | if not isinstance(k, (str, bytes)): 117 | raise URLParamsError("Invalid parameters key type.") 118 | 119 | if isinstance(v, (list, tuple, set)): 120 | v = [self.normalize(s) for s in v] 121 | else: 122 | v = self.normalize(v) 123 | 124 | params[self.normalize(k)] = v 125 | return params 126 | 127 | def normalize(self, s: URL_ALLOWED_PARAMS): 128 | if not isinstance(s, (str, bytes, int, float, bool)): 129 | raise URLParamsError("Invalid parameters value type.") 130 | 131 | if isinstance(s, bool): 132 | return str(s).lower() 133 | 134 | if isinstance(s, bytes): 135 | return s.decode("utf-8") 136 | 137 | return str(s) 138 | 139 | 140 | class URL: 141 | """URL 142 | 143 | A utility class for parsing, manipulating, and constructing URLs. It integrates with the 144 | `URLParams` class for managing query parameters and provides easy access to various components 145 | of a URL, such as scheme, host, port, and path. 146 | 147 | Attributes: 148 | - url (str): The raw or prepared URL string. 149 | - params (URLParams): An instance of URLParams to manage query parameters. 150 | - parsed (ParseResult): A `ParseResult` object containing the parsed components of the URL. 151 | - auth (tuple): A tuple of (username, password) extracted from the URL. 152 | - fragment (str): The fragment identifier of the URL. 153 | - host (str): The hostname (IDNA-encoded if applicable). 154 | - path (str): The path component of the URL. 155 | - netloc (str): The network location (host:port if port is present). 156 | - password (str): The password extracted from the URL. 157 | - port (str): The port number of the URL. 158 | - query (str): The query string, incorporating both existing and additional parameters. 159 | - scheme (str): The URL scheme (e.g., "http", "https"). 160 | - username (str): The username extracted from the URL. 161 | 162 | Methods: 163 | - _prepare(url: Union[U, str, bytes]) -> str: Prepares and validates a URL string or bytes to ParseResult. 164 | - _build(secure: bool = False) -> str: Constructs a URL string from its components. 165 | 166 | Raises: 167 | - URLError: Raised when an invalid URL or component is encountered. 168 | 169 | Example Usage: 170 | >>> url = URL("https://example.com/path?q=1#fragment", params={"key": "value"}) 171 | >>> print(url.scheme) 172 | 'https' 173 | >>> print(url.host) 174 | 'example.com' 175 | >>> print(url.query) 176 | 'q%3D1&key%3Dvalue' 177 | >>> print(url.params) 178 | 'key=value' 179 | >>> url.params.update({'key2': 'value2'}) 180 | >>> print(url.url) 181 | 'https://example.com/path?q%3D1&key%3Dvalue%26key2%3Dvalue2#fragment' 182 | >>> from urllib.parse import unquote 183 | >>> print(unquote(url.url)) 184 | 'https://example.com/path?q=1&key=value&key2=value2#fragment' 185 | >>> url.url = 'https://example.org/' 186 | >>> print(unquote(url.url)) 187 | 'https://example.org/?key=value&key2=value2' 188 | >>> url.url = 'https://httpbin.org/get' 189 | >>> print(unquote(url.url)) 190 | 'https://httpbin.org/get?key=value&key2=value2' 191 | """ 192 | 193 | __attrs__ = ( 194 | "auth", 195 | "scheme", 196 | "host", 197 | "port", 198 | "path", 199 | "fragment", 200 | "username", 201 | "password", 202 | ) 203 | 204 | def __init__(self, url: URLTypes, params: URLParamTypes = None, **kwargs): 205 | self._parsed = self._prepare(url) 206 | self._url = None 207 | self._params = URLParams(params) 208 | 209 | @property 210 | def url(self): 211 | if self._url is None: 212 | self._url = self._build(False) 213 | return self._url 214 | 215 | @url.setter 216 | def url(self, value): 217 | self._parsed = self._prepare(value) 218 | self._url = self._build(False) 219 | 220 | @property 221 | def params(self): 222 | return self._params 223 | 224 | @params.setter 225 | def params(self, value): 226 | self._url = None 227 | self._params = URLParams(value) 228 | 229 | @property 230 | def parsed(self) -> ParseResult: 231 | return self._parsed 232 | 233 | @property 234 | def netloc(self) -> str: 235 | return ":".join([self.host, self.port]) if self.port else self.host 236 | 237 | @property 238 | def query(self) -> str: 239 | query = "" 240 | if self.parsed.query and self.params.params: 241 | query = "&".join([quote(self.parsed.query), self.params.params]) 242 | elif self.params.params: 243 | query = self.params.params 244 | elif self.parsed.query: 245 | query = self.parsed.query 246 | return query 247 | 248 | def __str__(self): 249 | return self._build() 250 | 251 | def __repr__(self): 252 | return "<%s: %s>" % (self.__class__.__name__, unquote(self._build(True))) 253 | 254 | def _prepare(self, url: Union[T, str, bytes]) -> ParseResult: 255 | if isinstance(url, bytes): 256 | url = url.decode("utf-8") 257 | elif isinstance(url, self.__class__) or issubclass( 258 | self.__class__, url.__class__ 259 | ): 260 | url = str(url) 261 | 262 | if not isinstance(url, str): 263 | raise URLError("Invalid URL: %s" % url) 264 | 265 | for attr in self.__attrs__: 266 | setattr(self, attr, None) 267 | 268 | parsed = urlparse(url.lstrip()) 269 | 270 | self.auth = parsed.username, parsed.password 271 | self.scheme = parsed.scheme 272 | 273 | try: 274 | self.host = idna.encode(parsed.hostname.lower()).decode("ascii") 275 | except AttributeError: 276 | self.host = "" 277 | except idna.IDNAError: 278 | raise URLError("Invalid IDNA hostname.") 279 | 280 | self.port = "" 281 | try: 282 | if parsed.port: 283 | self.port = str(parsed.port) 284 | except ValueError as e: 285 | raise URLError("%s. port range must be 0 - 65535." % e.args[0]) 286 | 287 | self.path = parsed.path 288 | self.fragment = parsed.fragment 289 | self.username = parsed.username or "" 290 | self.password = parsed.password or "" 291 | return parsed 292 | 293 | def _build(self, secure: bool = False) -> str: 294 | urls = [self.scheme, "://"] 295 | authority = self.netloc 296 | if self.username or self.password: 297 | password = self.password or "" 298 | if secure: 299 | password = "[secure]" 300 | 301 | authority = "@".join( 302 | [ 303 | ":".join([self.username, password]), 304 | self.netloc, 305 | ] 306 | ) 307 | 308 | urls.append(authority) 309 | if self.query: 310 | urls.append("?".join([self.path, self.query])) 311 | else: 312 | urls.append(self.path) 313 | 314 | if self.fragment: 315 | urls.append("#" + self.fragment) 316 | 317 | return "".join(urls) 318 | 319 | 320 | class Proxy(URL): 321 | """Proxy 322 | 323 | A specialized subclass of `URL` designed to handle proxy URLs with specific schemes and additional 324 | validations. The class restricts allowed schemes to "http", "https", "socks5", and "socks5h". It 325 | also modifies the URL construction process to focus on proxy-specific requirements. 326 | 327 | Attributes: 328 | - ALLOWED_SCHEMES (tuple): A tuple of allowed schemes for the proxy ("http", "https", "socks5", "socks5h"). 329 | Raises: 330 | - ProxyError: Raised when an invalid proxy or unsupported protocol is encountered. 331 | 332 | Example Usage: 333 | >>> proxy = Proxy("http://user:pass@127.0.0.1:8080") 334 | >>> print(proxy.scheme) 335 | 'http' 336 | >>> print(proxy.netloc) 337 | '127.0.0.1:8080' 338 | >>> print(proxy) 339 | 'http://user:pass@127.0.0.1:8080' 340 | >>> print(proxy.__repr__()) 341 | '' 342 | 343 | >>> socks5 = Proxy("socks5://127.0.0.1:8080") 344 | >>> print(socks) 345 | 'socks5://127.0.0.1:8080' 346 | """ 347 | 348 | ALLOWED_SCHEMES = ("http", "https", "socks5", "socks5h") 349 | 350 | def _prepare(self, url: ProxyTypes) -> ParseResult: 351 | try: 352 | if isinstance(url, bytes): 353 | url = url.decode("utf-8") 354 | 355 | if isinstance(url, str): 356 | url = url.strip() 357 | 358 | parsed = super(Proxy, self)._prepare(url) 359 | if str(parsed.scheme).lower() not in self.ALLOWED_SCHEMES: 360 | raise ProxyError( 361 | "Invalid proxy scheme `%s`. The allowed schemes are ('http', 'https', 'socks5', 'socks5h')." 362 | % parsed.scheme 363 | ) 364 | 365 | return urlparse("%s://%s" % (parsed.scheme, parsed.netloc)) 366 | except URLError: 367 | raise ProxyError("Invalid proxy: %s" % url) 368 | 369 | def _build(self, secure: bool = False) -> str: 370 | urls = [self.scheme, "://"] 371 | authority = self.netloc 372 | if self.username or self.password: 373 | userinfo = ":".join([self.username, self.password]) 374 | if secure: 375 | userinfo = "[secure]" 376 | 377 | authority = "@".join( 378 | [ 379 | userinfo, 380 | self.netloc, 381 | ] 382 | ) 383 | 384 | urls.append(authority) 385 | return "".join(urls) 386 | -------------------------------------------------------------------------------- /tls_requests/settings.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from .__version__ import __version__ 4 | 5 | CHUNK_SIZE = 65_536 6 | DEFAULT_TIMEOUT = 30.0 7 | DEFAULT_MAX_REDIRECTS = 9 8 | DEFAULT_FOLLOW_REDIRECTS = True 9 | DEFAULT_TLS_DEBUG = False 10 | DEFAULT_TLS_INSECURE_SKIP_VERIFY = False 11 | DEFAULT_TLS_HTTP2 = "auto" 12 | DEFAULT_TLS_IDENTIFIER = "chrome_120" 13 | DEFAULT_HEADERS = { 14 | "accept": "*/*", 15 | "connection": "keep-alive", 16 | "user-agent": f"Python-TLS-Requests/{__version__}", 17 | "accept-encoding": "gzip, deflate, br, zstd", 18 | } 19 | -------------------------------------------------------------------------------- /tls_requests/types.py: -------------------------------------------------------------------------------- 1 | """ 2 | Type definitions for type checking purposes. 3 | """ 4 | 5 | from http.cookiejar import CookieJar 6 | from typing import (IO, TYPE_CHECKING, Any, BinaryIO, Callable, Dict, List, 7 | Literal, Mapping, Optional, Sequence, Set, Tuple, Union) 8 | from uuid import UUID 9 | 10 | if TYPE_CHECKING: # pragma: no cover 11 | from .models import Cookies, Headers, Request # noqa: F401 12 | 13 | AuthTypes = Optional[ 14 | Union[ 15 | Tuple[Union[str, bytes], Union[str, bytes]], 16 | Callable, 17 | "Auth", 18 | "BasicAuth", 19 | ] 20 | ] 21 | URLTypes = Union["URL", str, bytes] 22 | ProxyTypes = Union[str, bytes, "Proxy", "URL"] 23 | URL_ALLOWED_PARAMS = Union[str, bytes, int, float, bool] 24 | URLParamTypes = Optional[ 25 | Union[ 26 | "URLParams", 27 | Mapping[ 28 | Union[str, bytes], 29 | Union[ 30 | URL_ALLOWED_PARAMS, 31 | List[URL_ALLOWED_PARAMS], 32 | Tuple[URL_ALLOWED_PARAMS], 33 | Set[URL_ALLOWED_PARAMS], 34 | ], 35 | ], 36 | ] 37 | ] 38 | MethodTypes = Union[ 39 | "Method", Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"] 40 | ] 41 | ProtocolTypes = Optional[Union[Literal["auto", "http1", "http2"], bool]] 42 | HookTypes = Optional[Mapping[Literal["request", "response"], Sequence[Callable]]] 43 | TLSSession = Union["TLSSession", None] 44 | TLSSessionId = Union[str, UUID] 45 | TLSPayload = Union[dict, str, bytes, bytearray] 46 | TLSCookiesTypes = Optional[List[Dict[str, str]]] 47 | TLSIdentifierTypes = Literal[ 48 | "chrome_103", 49 | "chrome_104", 50 | "chrome_105", 51 | "chrome_106", 52 | "chrome_107", 53 | "chrome_108", 54 | "chrome_109", 55 | "chrome_110", 56 | "chrome_111", 57 | "chrome_112", 58 | "chrome_116_PSK", 59 | "chrome_116_PSK_PQ", 60 | "chrome_117", 61 | "chrome_120", 62 | "chrome_124", 63 | "safari_15_6_1", 64 | "safari_16_0", 65 | "safari_ios_15_5", 66 | "safari_ios_15_6", 67 | "safari_ios_16_0", 68 | "firefox_102", 69 | "firefox_104", 70 | "firefox_105", 71 | "firefox_106", 72 | "firefox_108", 73 | "firefox_110", 74 | "firefox_117", 75 | "firefox_120", 76 | "opera_89", 77 | "opera_90", 78 | "opera_91", 79 | "okhttp4_android_7", 80 | "okhttp4_android_8", 81 | "okhttp4_android_9", 82 | "okhttp4_android_10", 83 | "okhttp4_android_11", 84 | "okhttp4_android_12", 85 | "okhttp4_android_13", 86 | "zalando_ios_mobile", 87 | "zalando_android_mobile", 88 | "nike_ios_mobile", 89 | "nike_android_mobile", 90 | "mms_ios", 91 | "mms_ios_2", 92 | "mms_ios_3", 93 | "mesh_ios", 94 | "mesh_ios_2", 95 | "mesh_android", 96 | "mesh_android_2", 97 | "confirmed_ios", 98 | "confirmed_android", 99 | "confirmed_android_2", 100 | ] 101 | 102 | AnyList = List[ 103 | Union[ 104 | List[Union[str, Union[str, int, float]]], 105 | Tuple[Union[str, Union[str, int, float]]], 106 | Set[Union[str, Union[str, int, float]]], 107 | List[Union[str, bytes]], 108 | Tuple[Union[str, bytes]], 109 | Set[Union[str, bytes]], 110 | ] 111 | ] 112 | 113 | HeaderTypes = Optional[ 114 | Union[ 115 | "Headers", 116 | Mapping[str, Union[str, int, float]], 117 | Mapping[bytes, bytes], 118 | AnyList, 119 | ] 120 | ] 121 | 122 | CookieTypes = Optional[ 123 | Union[ 124 | "Cookies", 125 | CookieJar, 126 | Mapping[str, Union[str, int, float]], 127 | Mapping[bytes, bytes], 128 | AnyList, 129 | ] 130 | ] 131 | 132 | TimeoutTypes = Optional[Union[int, float]] 133 | ByteOrStr = Union[bytes, str] 134 | BufferTypes = Union[IO[bytes], "BytesIO", "BufferedReader"] 135 | FileContent = Union[ByteOrStr, BinaryIO] 136 | RequestFileValue = Union[ 137 | FileContent, # file (or file path, str and bytes) 138 | Tuple[ByteOrStr, FileContent], # filename, file (or file path, str and bytes)) 139 | Tuple[ 140 | ByteOrStr, FileContent, ByteOrStr 141 | ], # filename, file (or file path, str and bytes)), content type 142 | ] 143 | RequestData = Mapping[str, Any] 144 | RequestJson = Mapping[str, Any] 145 | RequestFiles = Mapping[ByteOrStr, RequestFileValue] 146 | ResponseHistory = List["Response"] 147 | -------------------------------------------------------------------------------- /tls_requests/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import base64 4 | import importlib 5 | import logging 6 | from typing import Any, AnyStr, Union 7 | 8 | FORMAT = "%(levelname)s:%(asctime)s:%(name)s:%(funcName)s:%(lineno)d >>> %(message)s" 9 | DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" 10 | 11 | 12 | def import_module(name: Union[str, list[str]]): 13 | modules = name if isinstance(name, list) else [name] 14 | for module in modules: 15 | if isinstance(module, str): 16 | try: 17 | module = importlib.import_module(module) 18 | return module 19 | except ImportError: 20 | pass 21 | 22 | 23 | chardet = import_module(["chardet", "charset_normalizer"]) 24 | orjson = import_module(["orjson"]) 25 | if orjson: 26 | jsonlib = orjson 27 | else: 28 | import json 29 | 30 | jsonlib = json 31 | 32 | 33 | def get_logger( 34 | name: str = "TLSRequests", level: int | str = logging.INFO 35 | ) -> logging.Logger: 36 | logging.basicConfig(format=FORMAT, datefmt=DATE_FORMAT, level=level) 37 | logger = logging.getLogger(name) 38 | logger.setLevel(level) 39 | return logger 40 | 41 | 42 | def to_bytes(value: Any, encoding: str = "utf-8", *, lower: bool = False) -> bytes: 43 | if isinstance(value, (bytes, bytearray)): 44 | return value 45 | return to_str(value, encoding).encode(encoding) 46 | 47 | 48 | def to_str( 49 | value: Any, 50 | encoding: str = "utf-8", 51 | *, 52 | lower: bool = False, 53 | ) -> str: 54 | if value is None: 55 | return "" 56 | 57 | value = value.decode(encoding) if isinstance(value, (bytes, bytearray)) else value 58 | if isinstance(value, (dict, list, tuple, set)): 59 | value = json_dumps( 60 | value if isinstance(value, dict) else list[value], 61 | **dict( 62 | ensure_ascii=True if str(encoding).lower() == "ascii" else False, 63 | default=str, 64 | ), 65 | ) 66 | 67 | if isinstance(value, bool): 68 | lower = True 69 | 70 | if lower: 71 | return str(value).lower() 72 | 73 | return str(value) 74 | 75 | 76 | def to_base64(value: Union[dict, str, bytes], encoding: str = "utf-8") -> AnyStr: 77 | return base64.b64encode(to_bytes(value, encoding)).decode(encoding) 78 | 79 | 80 | def b64decode(value: AnyStr) -> bytes: 81 | return base64.b64decode(value) 82 | 83 | 84 | def to_json(value: Union[str, bytes], encoding: str = "utf-8", **kwargs) -> dict: 85 | if isinstance(value, dict): 86 | return value 87 | try: 88 | json_data = jsonlib.loads(value, **kwargs) 89 | return json_data 90 | except jsonlib.JSONDecodeError: 91 | raise jsonlib.JSONDecodeError 92 | 93 | 94 | def json_dumps(value, **kwargs) -> str: 95 | try: 96 | if orjson: 97 | kwargs = {"default": kwargs.pop("default", None)} 98 | 99 | json_data = jsonlib.dumps(value, **kwargs) 100 | if isinstance(json_data, bytes): 101 | json_data = json_data.decode("utf-8") 102 | return json_data 103 | except jsonlib.JSONDecodeError: 104 | raise jsonlib.JSONDecodeError 105 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py{38,39,310,311,312,313}-default 3 | 4 | [testenv] 5 | deps = -r requirements-dev.txt 6 | commands = 7 | pytest {posargs:tests} 8 | 9 | [testenv:default] 10 | --------------------------------------------------------------------------------