├── requests_doh ├── connector │ ├── __init__.py │ ├── default.py │ └── proxies.py ├── __init__.py ├── exceptions.py ├── session.py ├── adapter.py ├── cachemanager.py └── resolver.py ├── requirements-docs.txt ├── .github ├── FUNDING.yml └── workflows │ └── python-publish.yml ├── requirements.txt ├── MANIFEST.in ├── .gitignore ├── docs ├── index.md ├── make.bat ├── Makefile ├── installation.md ├── api.rst ├── doh_providers.md ├── api_usage.md ├── conf.py └── changelog.md ├── .readthedocs.yml ├── LICENSE ├── README.md └── setup.py /requests_doh/connector/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements-docs.txt: -------------------------------------------------------------------------------- 1 | furo 2 | myst-parser[linkify] -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: ['mansuf'] 2 | ko_fi: rahmanyusuf -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests[socks]==2.32.3 2 | dnspython[doh]==2.6.1 3 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include LICENSE 3 | include requirements.txt 4 | include requirements-docs.txt -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | test.py 3 | *.egg-info 4 | # Vscode workspace 5 | *.code-workspace 6 | # Docs 7 | docs/_build -------------------------------------------------------------------------------- /requests_doh/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | DNS over HTTPS resolver for python requests 3 | """ 4 | 5 | __version__ = "1.0.0" 6 | __description__ = "DNS over HTTPS resolver for python requests" 7 | __author__ = "Rahman Yusuf" 8 | __author_email__ = "danipart4@gmail.com" 9 | __license__ = "MIT" 10 | __repository__ = "mansuf/requests-doh" 11 | 12 | from .session import * 13 | from .adapter import * 14 | from .resolver import * 15 | from .exceptions import * 16 | from .cachemanager import * -------------------------------------------------------------------------------- /requests_doh/exceptions.py: -------------------------------------------------------------------------------- 1 | class RequestsDOHException(Exception): 2 | """Base exception for requests_doh library""" 3 | 4 | class DNSQueryFailed(RequestsDOHException): 5 | """Failed to query DNS from given host""" 6 | pass 7 | 8 | class NoDoHProvider(RequestsDOHException): 9 | """There is no active DoH provider""" 10 | pass 11 | 12 | class DoHProviderNotExist(RequestsDOHException): 13 | """DoH provider is not exist in list of available DoH providers""" 14 | pass -------------------------------------------------------------------------------- /requests_doh/session.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from .adapter import DNSOverHTTPSAdapter 3 | 4 | __all__ = ('DNSOverHTTPSSession',) 5 | 6 | class DNSOverHTTPSSession(requests.Session): 7 | """A ready-to-use DoH (DNS-over-HTTPS) :class:`requests.Session` 8 | 9 | Parameters 10 | ----------- 11 | provider: :class:`str` 12 | A DoH provider 13 | cache_expire_time: :class:`float` 14 | Set DNS cache expire time 15 | """ 16 | def __init__(self, *args, **kwargs): 17 | super().__init__() 18 | 19 | doh = DNSOverHTTPSAdapter(*args, **kwargs) 20 | self.mount('https://', doh) 21 | self.mount('http://', doh) -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Welcome to requests-doh's documentation! 2 | 3 | DNS over HTTPS resolver for python requests using [dnspython](https://github.com/rthalley/dnspython) module 4 | 5 | ## Installation 6 | 7 | ```{toctree} 8 | :maxdepth: 2 9 | 10 | installation 11 | ``` 12 | 13 | ## API usage 14 | 15 | ```{toctree} 16 | :maxdepth: 2 17 | 18 | api_usage 19 | ``` 20 | 21 | ## API reference 22 | 23 | ```{toctree} 24 | :maxdepth: 2 25 | 26 | api 27 | ``` 28 | 29 | ## DoH (DNS-over-HTTPS) providers 30 | 31 | ```{toctree} 32 | :maxdepth: 2 33 | 34 | doh_providers 35 | ``` 36 | 37 | ```{toctree} 38 | :hidden: 39 | :caption: Development 40 | 41 | changelog 42 | Github repository 43 | ``` -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Build documentation in the docs/ directory with Sphinx 9 | sphinx: 10 | configuration: docs/conf.py 11 | builder: html 12 | 13 | # Optionally build your docs in additional formats such as PDF 14 | formats: 15 | - pdf 16 | 17 | # Set the OS, Python version and other tools you might need 18 | build: 19 | os: ubuntu-22.04 20 | tools: 21 | python: "3.8" 22 | 23 | # Optionally set the version of Python and requirements required to build your docs 24 | python: 25 | install: 26 | - method: pip 27 | path: . 28 | extra_requirements: 29 | - docs 30 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.https://www.sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | test-build: 16 | @$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) 17 | @$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) 18 | sudo cp -r "./_build/html" "/var/www" 19 | 20 | .PHONY: help test-build Makefile 21 | 22 | # Catch-all target: route all unknown targets to Sphinx using the new 23 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 24 | %: Makefile 25 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 26 | -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | ## Stable version 4 | 5 | ### With PyPI 6 | 7 | ```shell 8 | # For Windows 9 | py -3 -m pip install requests-doh 10 | 11 | # For Linux / Mac OS 12 | python3 -m pip install requests-doh 13 | ``` 14 | 15 | ## Development version 16 | 17 | ```{warning} 18 | This version is not stable and may crash during run. 19 | ``` 20 | 21 | ### With PyPI & Git 22 | 23 | **NOTE:** You must have git installed. If you don't have it, install it from here https://git-scm.com/. 24 | 25 | ```shell 26 | # For Windows 27 | py -3 -m pip install git+https://github.com/mansuf/requests-doh.git 28 | 29 | # For Linux / Mac OS 30 | python3 -m pip install git+https://github.com/mansuf/requests-doh.git 31 | ``` 32 | 33 | ### With Git only 34 | 35 | **NOTE:** You must have git installed. If you don't have it, install it from here https://git-scm.com/. 36 | 37 | ```shell 38 | git clone https://github.com/mansuf/requests-doh.git 39 | cd requests-doh 40 | python setup.py install 41 | ``` -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: requests_doh 2 | 3 | API Reference 4 | -------------- 5 | 6 | Session 7 | ======== 8 | 9 | .. autoclass:: DNSOverHTTPSSession 10 | 11 | Adapters 12 | ========== 13 | 14 | .. autoclass:: DNSOverHTTPSAdapter 15 | 16 | DNS resolver session 17 | ===================== 18 | 19 | .. autofunction:: set_resolver_session 20 | 21 | .. autofunction:: get_resolver_session 22 | 23 | DoH (DNS-over-HTTPS) Provider 24 | ============================== 25 | 26 | .. autofunction:: add_dns_provider 27 | 28 | .. autofunction:: remove_dns_provider 29 | 30 | .. autofunction:: set_dns_provider 31 | 32 | .. autofunction:: get_dns_provider 33 | 34 | .. autofunction:: get_all_dns_provider 35 | 36 | DNS Cache 37 | ========== 38 | 39 | .. autofunction:: set_dns_cache_expire_time 40 | 41 | .. autofunction:: purge_dns_cache 42 | 43 | Exceptions 44 | =========== 45 | 46 | .. autoexception:: RequestsDOHException 47 | 48 | .. autoexception:: DNSQueryFailed 49 | 50 | .. autoexception:: DoHProviderNotExist -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Rahman Yusuf 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Upload Python Package 10 | 11 | on: 12 | push: 13 | tags: 14 | - v* 15 | 16 | permissions: 17 | contents: read 18 | 19 | jobs: 20 | deploy: 21 | 22 | runs-on: ubuntu-latest 23 | 24 | steps: 25 | - uses: actions/checkout@v3 26 | - name: Set up Python 27 | uses: actions/setup-python@v3 28 | with: 29 | python-version: '3.x' 30 | - name: Install dependencies 31 | run: | 32 | python -m pip install --upgrade pip 33 | pip install build 34 | - name: Build package 35 | run: python -m build 36 | - name: Publish package 37 | uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 38 | with: 39 | user: __token__ 40 | password: ${{ secrets.PYPI_API_TOKEN }} 41 | -------------------------------------------------------------------------------- /docs/doh_providers.md: -------------------------------------------------------------------------------- 1 | # DoH (DNS-over-HTTPS) providers 2 | 3 | ```{option} google 4 | Basic google DNS (8.8.8.8 and 8.8.4.4) 5 | ``` 6 | 7 | ```{option} cloudflare 8 | Basic cloudflare DNS (1.1.1.1 and 1.0.0.1) 9 | ``` 10 | 11 | ```{option} cloudflare-security 12 | cloudflare DNS with malware protection (1.1.1.2 and 1.0.0.2) 13 | ``` 14 | 15 | ```{option} cloudflare-family 16 | cloudflare DNS with malware protection and blocking adult content (1.1.1.3 and 1.0.0.3) 17 | ``` 18 | 19 | ```{option} opendns 20 | Basic OpenDNS (208.67.222.222 and 208.67.220.220) 21 | ``` 22 | 23 | ```{option} opendns-family 24 | OpenDNS with adult content filter (208.67.222.123 and 208.67.220.123) 25 | ``` 26 | 27 | ```{option} adguard 28 | Default AdGuard DNS with ads, tracking and phising protection (94.140.14.14 and 94.140.15.15) 29 | ``` 30 | 31 | ```{option} adguard-family 32 | AdGuard DNS with default features + adult content filter + safe search (94.140.14.15 and 94.140.15.16) 33 | ``` 34 | 35 | ```{option} adguard-unfiltered 36 | AdGuard DNS with no default features and family protection (94.140.14.140 and 94.140.14.141) 37 | ``` 38 | 39 | ```{option} quad9 40 | Default Quad9 DNS with malware protection (9.9.9.9 and 149.112.112.112) 41 | ``` 42 | 43 | ```{option} quad9-unsecured 44 | Quad9 DNS with no malware protection (9.9.9.10 and 149.112.112.10) 45 | ``` 46 | -------------------------------------------------------------------------------- /docs/api_usage.md: -------------------------------------------------------------------------------- 1 | # API usage 2 | 3 | ## Easy usage 4 | 5 | ```python 6 | # for convenience 7 | from requests_doh import DNSOverHTTPSSession 8 | 9 | # By default, DoH provider will set to `cloudflare` 10 | session = DNSOverHTTPSSession(provider='google') 11 | r = session.get('https://google.com') 12 | print(r.status_code) 13 | ``` 14 | 15 | ## Basic usage with adapters 16 | 17 | ```python 18 | import requests 19 | from requests_doh import DNSOverHTTPSAdapter 20 | 21 | adapter = DNSOverHTTPSAdapter(provider='cloudflare-security') 22 | session = requests.Session() 23 | # For HTTPS 24 | session.mount('https://', adapter) 25 | # For HTTP 26 | session.mount('http://', adapter) 27 | 28 | r = session.get('https://google.com') 29 | print(r.status_code) 30 | ``` 31 | 32 | ## Add or remove custom DoH (DNS over HTTPS) provider 33 | 34 | ```python 35 | import requests 36 | from requests_doh import DNSOverHTTPSSession, add_dns_provider, remove_dns_provider 37 | 38 | # Adding a new DoH provider 39 | add_dns_provider("another-dns", "https://another-dns.example.com/dns-query") 40 | 41 | session = DNSOverHTTPSSession("another-dns") 42 | r = session.get("https://google.com/") 43 | print(r.status_code) 44 | ``` 45 | 46 | ```python 47 | import requests 48 | from requests_doh import DNSOverHTTPSSession, add_dns_provider, remove_dns_provider 49 | 50 | # Remove DoH provider 51 | remove_dns_provider("another-dns", fallback="cloudflare") 52 | ``` 53 | -------------------------------------------------------------------------------- /requests_doh/adapter.py: -------------------------------------------------------------------------------- 1 | from requests.adapters import HTTPAdapter 2 | from urllib3.connectionpool import HTTPSConnectionPool 3 | from urllib3.contrib.socks import ( 4 | SOCKSHTTPSConnectionPool, 5 | SOCKSHTTPConnectionPool, 6 | ) 7 | 8 | from .connector.default import ( 9 | DoHHTTPConnection, 10 | DoHHTTPSConnection, 11 | ) 12 | 13 | from .cachemanager import set_dns_cache_expire_time 14 | 15 | from .connector.proxies import ( 16 | SOCKSConnection, 17 | SOCKSHTTPSConnection 18 | ) 19 | 20 | from .resolver import set_dns_provider 21 | 22 | __all__ = ('DNSOverHTTPSAdapter',) 23 | 24 | class DNSOverHTTPSAdapter(HTTPAdapter): 25 | """An DoH (DNS over HTTPS) adapter for :class:`requests.Session` 26 | 27 | Parameters 28 | ----------- 29 | provider: :class:`str` 30 | A DoH provider 31 | cache_expire_time: :class:`float` 32 | Set DNS cache expire time 33 | **kwargs 34 | These parameters will be passed to :class:`requests.adapters.HTTPAdapter` 35 | """ 36 | def __init__(self, provider=None, cache_expire_time=None, **kwargs): 37 | if provider: 38 | set_dns_provider(provider) 39 | 40 | if cache_expire_time: 41 | set_dns_cache_expire_time(cache_expire_time) 42 | 43 | super().__init__(**kwargs) 44 | 45 | def get_connection_with_tls_context(self, *args, **kwargs): 46 | conn = super().get_connection_with_tls_context(*args, **kwargs) 47 | if isinstance(conn, SOCKSHTTPSConnectionPool): 48 | conn.ConnectionCls = SOCKSHTTPSConnection 49 | elif isinstance(conn, SOCKSHTTPConnectionPool): 50 | conn.ConnectionCls = SOCKSConnection 51 | elif isinstance(conn, HTTPSConnectionPool): 52 | conn.ConnectionCls = DoHHTTPSConnection 53 | else: 54 | # HTTP type 55 | conn.ConnectionCls = DoHHTTPConnection 56 | return conn -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![pypi-total-downloads](https://img.shields.io/pypi/dm/requests-doh?label=DOWNLOADS&style=for-the-badge)](https://pypi.org/project/requests-doh) 2 | [![python-ver](https://img.shields.io/pypi/pyversions/requests-doh?style=for-the-badge)](https://pypi.org/project/requests-doh) 3 | [![pypi-release-ver](https://img.shields.io/pypi/v/requests-doh?style=for-the-badge)](https://pypi.org/project/requests-doh) 4 | 5 | # requests-doh 6 | 7 | DNS over HTTPS resolver for python [requests](https://github.com/psf/requests) using [dnspython](https://github.com/rthalley/dnspython) module 8 | 9 | ## Key features 10 | 11 | - Resolve hosts using [public DNS servers](https://adguard-dns.io/kb/general/dns-providers) 12 | or custom DNS servers over HTTPS 13 | - DNS caching, making faster to resolve hosts 14 | - Easy to use 15 | 16 | ## Installation 17 | 18 | You must have Python 3.8.x or up with Pip installed. 19 | 20 | ### PyPI (stable version) 21 | 22 | ```shell 23 | # For Linux / Mac OS 24 | python3 -m pip install requests-doh 25 | 26 | # For Windows 27 | py -3 -m pip install requests-doh 28 | ``` 29 | 30 | ### Git (Development version) 31 | 32 | ```shell 33 | git clone https://github.com/mansuf/requests-doh.git 34 | cd requests-doh 35 | python setup.py install 36 | ``` 37 | 38 | For more information about installation, see [Installation](https://requests-doh.mansuf.link/en/stable/installation.html) 39 | 40 | ## Usage 41 | 42 | ```python 43 | # for convenience 44 | from requests_doh import DNSOverHTTPSSession 45 | 46 | # By default, DoH provider will set to `cloudflare` 47 | session = DNSOverHTTPSSession(provider='google') 48 | r = session.get('https://google.com') 49 | print(r.content) 50 | ``` 51 | 52 | For more information about usage, see [API usage](https://requests-doh.mansuf.link/en/stable/api_usage.html) 53 | 54 | ## Links 55 | 56 | - [PyPI](https://pypi.org/project/requests-doh/) 57 | - [Docs](https://requests-doh.mansuf.link/) 58 | 59 | ## License 60 | 61 | See [LICENSE](https://github.com/mansuf/requests-doh/blob/main/LICENSE) -------------------------------------------------------------------------------- /requests_doh/cachemanager.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | 3 | __all__ = ('set_dns_cache_expire_time', 'purge_dns_cache', 'cachemanager') 4 | 5 | class DNSCacheManager: 6 | def __init__(self): 7 | self._expire = timedelta(seconds=300) 8 | self._data = {} 9 | 10 | def set_expire_time(self, time): 11 | if isinstance(time, float) or isinstance(time, int): 12 | self._expire = timedelta(seconds=time) 13 | else: 14 | raise ValueError(f'{time.__class__.__name__} is not float type') 15 | 16 | def set_cache(self, host, answers): 17 | self._data[host] = { 18 | "expire": datetime.now() + self._expire, 19 | "data": answers 20 | } 21 | 22 | def get_cache(self, host): 23 | try: 24 | item = self._data[host] 25 | except KeyError: 26 | return None 27 | 28 | now = datetime.now() 29 | 30 | if item['expire'] < now: 31 | # DNS cache is expired 32 | self._data.pop(host) 33 | return None 34 | 35 | return item['data'] 36 | 37 | def purge(self, host): 38 | try: 39 | self._data.pop(host) 40 | except KeyError: 41 | raise ValueError(f"host '{host}' is not cached") 42 | 43 | def purge_all(self): 44 | self._data.clear() 45 | 46 | cachemanager = DNSCacheManager() 47 | 48 | def set_dns_cache_expire_time(time): 49 | """Set DNS cache expired time in seconds 50 | 51 | Parameters 52 | ----------- 53 | time: :class:`float` 54 | An expire time 55 | """ 56 | cachemanager.set_expire_time(time) 57 | 58 | def purge_dns_cache(host=None): 59 | """Purge DNS cache 60 | 61 | Parameters 62 | ----------- 63 | host: :class:`str` 64 | Cached DNS host want to be purged, if ``host`` is None, all DNS caches will be purged. 65 | """ 66 | if host: 67 | cachemanager.purge(host) 68 | else: 69 | cachemanager.purge_all() -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import re 15 | import sys 16 | sys.path.insert(0, os.path.abspath('..')) 17 | 18 | 19 | # -- Project information ----------------------------------------------------- 20 | 21 | project = 'requests-doh' 22 | copyright = '2022 - present, Rahman Yusuf' 23 | author = 'mansuf' 24 | 25 | # Find version without importing it 26 | regex_version = re.compile(r'[0-9]{1}.[0-9]{1,2}.[0-9]{1,3}') 27 | with open('../requests_doh/__init__.py', 'r') as r: 28 | _version = regex_version.search(r.read()) 29 | 30 | if _version is None: 31 | raise RuntimeError('version is not set') 32 | 33 | version = _version.group() 34 | 35 | # The full version, including alpha/beta/rc tags 36 | release = version 37 | 38 | 39 | # -- General configuration --------------------------------------------------- 40 | 41 | # Add any Sphinx extension module names here, as strings. They can be 42 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 43 | # ones. 44 | extensions = [ 45 | 'sphinx.ext.autodoc', 46 | 'sphinx.ext.extlinks', 47 | 'sphinx.ext.napoleon', 48 | 'sphinx.ext.intersphinx', 49 | 'myst_parser' 50 | ] 51 | 52 | myst_enable_extensions = [ 53 | 'dollarmath', 54 | 'linkify' 55 | ] 56 | 57 | myst_linkify_fuzzy_links=False 58 | 59 | myst_heading_anchors = 3 60 | 61 | source_suffix = { 62 | '.rst': 'restructuredtext', 63 | '.md': 'markdown', 64 | } 65 | 66 | # No typing in docs 67 | autodoc_member_order = 'bysource' 68 | autodoc_typehints = 'none' 69 | 70 | # Add any paths that contain templates here, relative to this directory. 71 | templates_path = ['_templates'] 72 | 73 | # List of patterns, relative to source directory, that match files and 74 | # directories to ignore when looking for source files. 75 | # This pattern also affects html_static_path and html_extra_path. 76 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 77 | 78 | # Intersphinx mapping 79 | intersphinx_mapping = { 80 | 'python': ('https://docs.python.org/3', None), 81 | } 82 | 83 | # -- Options for HTML output ------------------------------------------------- 84 | 85 | # The theme to use for HTML and HTML Help pages. See the documentation for 86 | # a list of builtin themes. 87 | # 88 | # Intersphinx mapping 89 | intersphinx_mapping = { 90 | 'python': ('https://docs.python.org/3', None), 91 | 'requests': ('https://requests.readthedocs.io/en/v2.8.1', None) 92 | } 93 | 94 | html_theme = 'furo' 95 | 96 | # Add any paths that contain custom static files (such as style sheets) here, 97 | # relative to this directory. They are copied after the builtin static files, 98 | # so a file named "default.css" will overwrite the builtin "default.css". 99 | html_static_path = ['_static'] 100 | 101 | html_title = project -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import re 3 | from setuptools import setup, find_packages 4 | 5 | # Root directory 6 | # (README.md, requests_doh/__init__.py) 7 | HERE = pathlib.Path(__file__).parent 8 | README = (HERE / "README.md").read_text() 9 | init_file = (HERE / "requests_doh/__init__.py").read_text() 10 | 11 | def get_version(): 12 | """Get version of the app""" 13 | re_version = r'__version__ = \"([0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3})\"' 14 | _version = re.search(re_version, init_file) 15 | 16 | if _version is None: 17 | raise RuntimeError("Version is not set") 18 | 19 | return _version.group(1) 20 | 21 | def get_value_var(var_name): 22 | """Get value of `__{var_name}__` from `requests_doh/__init__.py`""" 23 | var = f'__{var_name}__' 24 | regex = '%s = "(.{1,})"' % var 25 | 26 | found = re.search(regex, init_file) 27 | 28 | if found is None: 29 | raise RuntimeError(f'{var} is not set in "requests_doh/__init__.py"') 30 | 31 | return found.group(1) 32 | 33 | def get_requirements(): 34 | """Return tuple of library needed for app to run""" 35 | main = [] 36 | try: 37 | with open('./requirements.txt', 'r') as r: 38 | main = r.read().splitlines() 39 | except FileNotFoundError: 40 | raise RuntimeError("requirements.txt is needed to build requests-doh") 41 | 42 | if not main: 43 | raise RuntimeError("requirements.txt have no necessary libraries inside of it") 44 | 45 | docs = [] 46 | try: 47 | with open('./requirements-docs.txt', 'r') as r: 48 | docs = r.read().splitlines() 49 | except FileNotFoundError: 50 | # There is no docs requirements 51 | # Developers can ignore this error and continue to install without any problem. 52 | # However, this is needed if developers want to create documentation in readthedocs.org or local device. 53 | pass 54 | 55 | return main, { 56 | "docs": docs, 57 | } 58 | 59 | # Get requirements needed to build app 60 | requires_main, extras_require = get_requirements() 61 | 62 | # Get all modules 63 | packages = find_packages('.') 64 | 65 | # Get repository 66 | repo = get_value_var('repository') 67 | 68 | # Finally run main setup 69 | setup( 70 | name = 'requests-doh', 71 | packages = packages, 72 | version = get_version(), 73 | license=get_value_var('license'), 74 | description = get_value_var('description'), 75 | long_description= README, 76 | long_description_content_type= 'text/markdown', 77 | author = get_value_var('author'), 78 | author_email = get_value_var('author_email'), 79 | url = f'https://github.com/{repo}', 80 | keywords = [ 81 | 'requests', 82 | 'doh', 83 | 'dns', 84 | 'https', 85 | 'dns-over-https' 86 | ], 87 | install_requires=requires_main, 88 | extras_require=extras_require, 89 | classifiers=[ 90 | 'Development Status :: 5 - Production/Stable', 91 | 'Intended Audience :: Developers', 92 | 'Topic :: Internet :: Name Service (DNS)', 93 | 'Topic :: Internet :: WWW/HTTP', 94 | 'License :: OSI Approved :: MIT License', 95 | 'Programming Language :: Python :: 3 :: Only', 96 | 'Programming Language :: Python :: 3', 97 | 'Programming Language :: Python :: 3.8', 98 | 'Programming Language :: Python :: 3.9', 99 | 'Programming Language :: Python :: 3.10' 100 | ], 101 | python_requires='>=3.8' 102 | ) 103 | -------------------------------------------------------------------------------- /docs/changelog.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v1.0.0 4 | 5 | In summary, this update introduce some breaking changes to resolver session and update library dependencies. 6 | 7 | ### Breaking changes 8 | 9 | Now function `requests_doh.resolver.set_resolver_session` only accept `httpx.Client` rather than `requests.Client`. 10 | This is because [dnspython changes](https://dnspython.readthedocs.io/en/stable/whatsnew.html#id6) makes it no longer 11 | accepting `requests.Client` 12 | 13 | And `requests_doh.resolver.get_resolver_session` are now returning `httpx.Client` rather than `requests.Client` 14 | 15 | ### Dependencies 16 | 17 | - Bump requests from v2.31.0 to v2.32.3 due to [CVE-2024-35195](https://github.com/advisories/GHSA-9wx4-h78v-vm56) 18 | - Bump dnspython from v2.3.0 to v2.6.1 due to [CVE-2023-29483](https://github.com/advisories/GHSA-3rq5-2g8h-59hc) 19 | 20 | ## v0.3.3 21 | 22 | ### Fix bugs 23 | 24 | - Fixed missing dependencies (`ModuleNotFoundError: No module named socks`) 25 | 26 | ## v0.3.2 27 | 28 | ### Fix bugs 29 | 30 | - Fixed requests with socks proxy is not working #3 31 | 32 | ### Note: Potential breaking changes 33 | 34 | Functions `set_dns_cache_expire_time()` and `purge_dns_cache()` imported from module `requests_doh.connector` are no longer exists. Instead you can import it from `requests_doh.cachemanager` 35 | 36 | If you usually import those functions from `requests_doh` (root library), these changes doesn't affect you at all. 37 | 38 | For example: 39 | 40 | ```python 41 | # If you do this starting from v0.3.2, you will get `ImportError` 42 | from requests_doh.connector import set_dns_cache_expire_time, purge_dns_cache 43 | 44 | # Do this instead 45 | from requests_doh.cachemanager import set_dns_cache_expire_time, purge_dns_cache 46 | 47 | # Those changes doesn't affect you if you use this import method 48 | from requests_doh import set_dns_cache_expire_time, purge_dns_cache 49 | ``` 50 | 51 | ## v0.3.1 52 | 53 | This update fix `requests` dependencies because of [CVE-202-32681](https://github.com/psf/requests/security/advisories/GHSA-j8r2-6x86-q33q) 54 | 55 | ### Dependencies 56 | 57 | - Bump requests from v2.28.2 to v2.31.0 58 | 59 | ## v0.3.0 60 | 61 | ### New features 62 | 63 | - Added ability to add custom DNS over HTTPS provider [#1](https://github.com/mansuf/requests-doh/issues/1) 64 | - Added ability to remove DNS over HTTPS provider 65 | 66 | ### Improvements 67 | 68 | - Improved performance for querying DNS over HTTPS 69 | 70 | ## v0.2.4 71 | 72 | ### Dependecies 73 | 74 | - Updated requests from v2.28.1 to v2.28.2 75 | - Updated dnspython from v2.2.1 to v2.3.0 76 | 77 | ## v0.2.3 78 | 79 | ### Fix bugs 80 | 81 | - Fixed missing dependecies resulting error `dns.query.NoDOH: Neither httpx nor requests is available.` 82 | 83 | ## v0.2.2 84 | 85 | ### Improvements 86 | 87 | - Improved DoH resolving 88 | 89 | ## v0.2.1 90 | 91 | ### Fix bugs 92 | 93 | - Fixed unhandled exception if host doesn't contain AAAA type 94 | 95 | ## v0.2.0 96 | 97 | ```{warning} 98 | Broken, do not use this version. Instead use `v0.2.1` 99 | ``` 100 | 101 | ### New features 102 | 103 | - Added `get_all_dns_provider()`, returning all available DoH providers. 104 | - Added new DoH providers 105 | - cloudflare-security 106 | - cloudflare-family 107 | - opendns 108 | - opendns-family 109 | - adguard 110 | - adguard-family 111 | - adguard-unfiltered 112 | - quad9 113 | - quad9-unsecured 114 | - Added `DNSOverHTTPSSession` for ready-to-use DoH requests session 115 | 116 | ### Breaking changes 117 | 118 | - Starting from v0.2.0, requests-doh rely on [dnspython](https://github.com/rthalley/dnspython) module 119 | for extending it's library usage and query to many public and private DNS. 120 | 121 | ## v0.1.1 122 | 123 | ### Fix bugs 124 | 125 | - Fix ipv6 addresses is not handled properly 126 | 127 | ## v0.1.0 128 | 129 | ### New features 130 | 131 | - Added DoH local caching 132 | 133 | ### Fix bugs 134 | 135 | - Fix requests for http prefix (`http://`) is hanging up 136 | 137 | ## v0.0.1 138 | 139 | Initial release 140 | -------------------------------------------------------------------------------- /requests_doh/connector/default.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import ipaddress 3 | 4 | import socket 5 | import logging 6 | from urllib3.connection import HTTPSConnection, HTTPConnection 7 | from urllib3.util.connection import allowed_gai_family, _set_socket_options 8 | from urllib3.exceptions import ConnectTimeoutError, NewConnectionError, LocationParseError 9 | from socket import error as SocketError 10 | from socket import timeout as SocketTimeout 11 | 12 | try: 13 | from urllib3.packages.six import raise_from 14 | except ImportError: 15 | # Broken package mostly 16 | def raise_from(value, from_value): 17 | try: 18 | raise value from from_value 19 | finally: 20 | value = None 21 | 22 | from ..resolver import resolve_dns 23 | from ..cachemanager import cachemanager 24 | 25 | log = logging.getLogger(__name__) 26 | 27 | # This code is copied from urllib3/util/connection.py version 1.26.8 (from requests v2.28.1) 28 | def create_connection( 29 | address, 30 | timeout=socket._GLOBAL_DEFAULT_TIMEOUT, 31 | source_address=None, 32 | socket_options=None, 33 | proxy=None 34 | ): 35 | """Same as :meth:`urllib3.util.connection.create_connection()`, 36 | except it has DNS over HTTPS resovler inside of it. 37 | """ 38 | 39 | host, port = address 40 | if host.startswith("["): 41 | host = host.strip("[]") 42 | err = None 43 | 44 | # Using the value from allowed_gai_family() in the context of getaddrinfo lets 45 | # us select whether to work with IPv4 DNS records, IPv6 records, or both. 46 | # The original create_connection function always returns all records. 47 | family = allowed_gai_family() 48 | 49 | try: 50 | host.encode("idna") 51 | except UnicodeError: 52 | return raise_from( 53 | LocationParseError(u"'%s', label empty or too long" % host), None 54 | ) 55 | 56 | if not proxy: 57 | cached = cachemanager.get_cache(host) 58 | if not cached: 59 | # Uncached DNS 60 | answers = resolve_dns(host) 61 | 62 | cachemanager.set_cache(host, answers) 63 | else: 64 | answers = cached 65 | 66 | if proxy: 67 | # We must make sure that this isn't a DNS name 68 | try: 69 | socket.inet_aton(host) 70 | except OSError: 71 | # This is a DNS name 72 | # To be honest, this is messed up 73 | answers = [i[4][0] for i in socket.getaddrinfo(host, port)] 74 | else: 75 | # It's an ip address 76 | answers = [host] 77 | 78 | for answer in answers: 79 | try: 80 | ipaddress.ip_address(answer) 81 | except ValueError: 82 | # Most likely this is domain returned from DoH provider 83 | log.warning(f"Domain detected ({answer}) in DoH query result to {host}") 84 | continue 85 | 86 | for res in socket.getaddrinfo(answer, port, family, socket.SOCK_STREAM): 87 | af, socktype, proto, canonname, sa = res 88 | sock = None 89 | try: 90 | sock = socket.socket(af, socktype, proto) 91 | 92 | # If provided, set socket level options before connecting. 93 | _set_socket_options(sock, socket_options) 94 | 95 | if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: 96 | sock.settimeout(timeout) 97 | if source_address: 98 | sock.bind(source_address) 99 | sock.connect(sa) 100 | return sock 101 | 102 | except socket.error as e: 103 | err = e 104 | if sock is not None: 105 | sock.close() 106 | sock = None 107 | 108 | if err is not None: 109 | raise err 110 | 111 | class DoHHTTPConnection(HTTPConnection): 112 | # This code is copied from urllib3/connection.py version 1.26.8 (from requests v2.28.1) 113 | def _new_conn(self): 114 | """Establish a socket connection and set nodelay settings on it. 115 | 116 | :return: New socket connection. 117 | """ 118 | extra_kw = {} 119 | if self.source_address: 120 | extra_kw["source_address"] = self.source_address 121 | 122 | if self.socket_options: 123 | extra_kw["socket_options"] = self.socket_options 124 | 125 | extra_kw["proxy"] = self.proxy 126 | 127 | try: 128 | conn = create_connection( 129 | (self._dns_host, self.port), self.timeout, **extra_kw 130 | ) 131 | 132 | except SocketTimeout: 133 | raise ConnectTimeoutError( 134 | self, 135 | "Connection to %s timed out. (connect timeout=%s)" 136 | % (self.host, self.timeout), 137 | ) 138 | 139 | except SocketError as e: 140 | raise NewConnectionError( 141 | self, "Failed to establish a new connection: %s" % e 142 | ) 143 | 144 | return conn 145 | 146 | class DoHHTTPSConnection(DoHHTTPConnection, HTTPSConnection): 147 | pass 148 | -------------------------------------------------------------------------------- /requests_doh/resolver.py: -------------------------------------------------------------------------------- 1 | import httpx 2 | from dns.message import make_query 3 | from dns.rdatatype import RdataType 4 | from dns.query import https as query_https 5 | from dns.rcode import Rcode 6 | 7 | from .exceptions import ( 8 | DNSQueryFailed, 9 | DoHProviderNotExist, 10 | NoDoHProvider 11 | ) 12 | 13 | _resolver_session = None # type: httpx.Client 14 | _available_providers = { 15 | "cloudflare": "https://cloudflare-dns.com/dns-query", 16 | "cloudflare-security": "https://security.cloudflare-dns.com/dns-query", 17 | "cloudflare-family": "https://family.cloudflare-dns.com/dns-query", 18 | "opendns": "https://doh.opendns.com/dns-query", 19 | "opendns-family": "https://doh.familyshield.opendns.com/dns-query", 20 | "adguard": "https://dns.adguard.com/dns-query", 21 | "adguard-family": "https://dns-family.adguard.com/dns-query", 22 | "adguard-unfiltered": "https://unfiltered.adguard-dns.com/dns-query", 23 | "quad9": "https://dns.quad9.net/dns-query", 24 | "quad9-unsecured": "https://dns10.quad9.net/dns-query", 25 | "google": "https://dns.google/dns-query" 26 | } 27 | # Default provider 28 | _provider = _available_providers["cloudflare"] 29 | 30 | __all__ = ( 31 | 'set_resolver_session', 'get_resolver_session', 32 | 'set_dns_provider', 'get_dns_provider', 33 | 'add_dns_provider', 'remove_dns_provider', 34 | 'get_all_dns_provider', 'resolve_dns' 35 | ) 36 | 37 | def set_resolver_session(session): 38 | """Set http session to resolve DNS 39 | 40 | Parameters 41 | ----------- 42 | session: :class:`httpx.Client` 43 | An http session to resolve DNS 44 | 45 | Raises 46 | ------- 47 | ValueError 48 | ``session`` parameter is not :class:`httpx.Client` instance 49 | """ 50 | global _resolver_session 51 | 52 | if not isinstance(session, httpx.Client): 53 | raise ValueError(f"`session` must be `httpx.Client`, {session.__class__.__name__}") 54 | 55 | _resolver_session = session 56 | 57 | def get_resolver_session() -> httpx.Client: 58 | """ 59 | Return 60 | ------- 61 | httpx.Client 62 | Return an http session for DoH resolver 63 | """ 64 | return _resolver_session 65 | 66 | def set_dns_provider(provider): 67 | """Set a DoH provider, must be a valid DoH providers 68 | 69 | Parameters 70 | ----------- 71 | provider: :class:`str` 72 | An valid DoH provider, see :doc:`doh_providers` 73 | 74 | Raises 75 | ------- 76 | DoHProviderNotExist 77 | Invalid DoH provider 78 | """ 79 | global _provider 80 | 81 | if provider not in _available_providers.keys(): 82 | raise DoHProviderNotExist(f"invalid DoH provider, must be one of '{list(_available_providers.keys())}'") 83 | 84 | _provider = _available_providers[provider] 85 | 86 | def get_dns_provider(): 87 | """ 88 | Return 89 | ------- 90 | str 91 | Return current DoH provider 92 | """ 93 | return _provider 94 | 95 | def add_dns_provider(name, address, switch=False): 96 | """Add a DoH provider 97 | 98 | Parameters 99 | ----------- 100 | name: :class:`str` 101 | Name for DoH provider 102 | address: :class:`str` 103 | Full URL / endpoint for DoH provider 104 | switch: Optional[:class:`bool`] 105 | If ``True``, the DoH provider will automatically switch to 106 | newly created DoH provider 107 | """ 108 | _available_providers[name] = address 109 | 110 | if switch: 111 | set_dns_provider(name) 112 | 113 | def remove_dns_provider(name, fallback=None): 114 | """Remove a DoH provider 115 | 116 | If parameter ``name`` is an active DoH provider, 117 | :func:`get_dns_provider` will return ``None``. 118 | You must set ``fallback`` parameter to one of available DoH providers 119 | (``fallback`` and ``name`` parameters cannot be same value) 120 | or you can call :func:`set_dns_provider` after calling this function 121 | in order to get DoH working 122 | 123 | For example: 124 | 125 | .. code-block:: python3 126 | 127 | from requests_doh import DNSOverHTTPSSession, add_dns_provider, remove_dns_provider 128 | 129 | # Add a custom DNS and set it to active 130 | add_dns_provider("another-dns", "https://another-dns.example.com/dns-query", switch=True) 131 | 132 | # At this point, the session is still working 133 | session = DNSOverHTTPSSession("another-dns") 134 | r = session.get("https://example.com") 135 | print(r.status_code) 136 | 137 | # Let's try to remove the newly created DNS 138 | remove_dns_provider("another-dns", fallback="cloudflare") 139 | 140 | # Or we can call `set_dns_provider()` 141 | # if we didn't set `fallback` parameter 142 | # set_dns_provider("cloudflare") 143 | 144 | # At this point DoH provider "another-dns" is removed 145 | # and "cloudflare" is set to active DoH provider 146 | # the session is still working 147 | r = session.get("https://google.com") 148 | 149 | But what will happend if we didn't add ``fallback`` parameter or didn't call :func:`set_dns_provider()` ? 150 | Well error will occurred, take a look at this example: 151 | 152 | .. code-block:: python3 153 | 154 | from requests_doh import DNSOverHTTPSSession, add_dns_provider, remove_dns_provider 155 | 156 | # Add a custom DNS and set it to active 157 | add_dns_provider("another-dns", "https://another-dns.example.com/dns-query", switch=True) 158 | 159 | # At this point, the session is still working 160 | session = DNSOverHTTPSSession("another-dns") 161 | r = session.get("https://example.com") 162 | print(r.status_code) 163 | 164 | # Let's try to remove the newly created DNS 165 | remove_dns_provider("another-dns") 166 | 167 | # If we send request to same URL, it would still working 168 | r = session.get("https://example.com") 169 | print(r.status_code) 170 | 171 | # An error occurred when we send to another URL 172 | # Because we didn't set ``falback`` parameter in `remove_dns_provider()` 173 | # (or calling function `set_dns_provider()`) 174 | # `get_dns_provider()` will return ``None`` and thus resolving DNS will be failed 175 | # Because there is no valid endpoint where we wanna resolve DNS of the host 176 | r = session.get("https://google.com") 177 | 178 | Parameters 179 | ----------- 180 | name: :class:`str` 181 | DoH provider that want to remove 182 | fallback: :class:`str` 183 | Set a fallback DoH provider 184 | 185 | Raises 186 | ------- 187 | DoHProviderNotExist 188 | DoH provider is not exist in list of available DoH providers 189 | """ 190 | global _provider 191 | 192 | try: 193 | _available_providers.pop(name) 194 | except KeyError: 195 | raise DoHProviderNotExist( 196 | "DoH provider is not exist in list of available DoH providers" 197 | ) 198 | 199 | if fallback: 200 | set_dns_provider(fallback) 201 | else: 202 | _provider = None 203 | 204 | def get_all_dns_provider(): 205 | """ 206 | Return 207 | ------- 208 | tuple[str] 209 | Return all available DoH providers 210 | """ 211 | return tuple(_available_providers.keys()) 212 | 213 | def _resolve(session, doh_endpoint, host, rdatatype): 214 | req_message = make_query(host, rdatatype) 215 | res_message = query_https(req_message, doh_endpoint, session=session) 216 | rcode = Rcode(res_message.rcode()) 217 | if rcode != Rcode.NOERROR: 218 | raise DNSQueryFailed(f"Failed to query DNS {rdatatype.name} from host '{host}' (rcode = {rcode.name}") 219 | 220 | answers = res_message.resolve_chaining().answer 221 | if answers is None: 222 | return None 223 | 224 | return tuple(str(i) for i in answers) 225 | 226 | def resolve_dns(host): 227 | if _provider is None: 228 | raise NoDoHProvider("There is no active DoH provider") 229 | 230 | session = get_resolver_session() 231 | 232 | if session is None: 233 | session = httpx.Client() 234 | set_resolver_session(session) 235 | 236 | answers = set() 237 | 238 | # Reuse is good 239 | def query(rdatatype): 240 | return _resolve(session, _provider, host, rdatatype) 241 | 242 | # Query A type 243 | A_ANSWERS = query(RdataType.A) 244 | if A_ANSWERS is not None: 245 | answers.update(A_ANSWERS) 246 | 247 | # Query AAAA type 248 | AAAA_ANSWERS = query(RdataType.AAAA) 249 | if AAAA_ANSWERS is not None: 250 | answers.update(AAAA_ANSWERS) 251 | 252 | if not answers: 253 | raise DNSQueryFailed( 254 | f"DNS server {_provider} returned empty results from host '{host}'" 255 | ) 256 | 257 | return list(answers) -------------------------------------------------------------------------------- /requests_doh/connector/proxies.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module is intended to modify socks (socks5:// & socks4://) and http proxies to resolve DNS 3 | remotely to public DNS servers (such as google, cloudlflare, etc) or private DNS servers instead of locally. 4 | 5 | NOTE: This may be useless if users are using socks proxy with DNS remote resolver (socks5h:// & socks4a://) 6 | """ 7 | 8 | from base64 import b64encode 9 | import typing 10 | import socket 11 | import struct 12 | from urllib3.contrib.socks import socks, _TYPE_SOCKS_OPTIONS, SocketTimeout 13 | from urllib3.exceptions import ConnectTimeoutError, NewConnectionError 14 | from urllib3.connection import HTTPConnection, HTTPSConnection 15 | 16 | from ..resolver import resolve_dns 17 | from ..cachemanager import cachemanager 18 | 19 | class socksocketmod(socks.socksocket): 20 | """Modified socks socket to resolve DNS remotely to public or private DNS servers""" 21 | def _write_SOCKS5_address(self, addr, file): 22 | """ 23 | Return the host and port packed for the SOCKS5 protocol, 24 | and the resolved address as a tuple object. 25 | """ 26 | host, port = addr 27 | proxy_type, _, _, rdns, username, password = self.proxy 28 | family_to_byte = {socket.AF_INET: b"\x01", socket.AF_INET6: b"\x04"} 29 | 30 | # If the given destination address is an IP address, we'll 31 | # use the IP address request even if remote resolving was specified. 32 | # Detect whether the address is IPv4/6 directly. 33 | for family in (socket.AF_INET, socket.AF_INET6): 34 | try: 35 | addr_bytes = socket.inet_pton(family, host) 36 | file.write(family_to_byte[family] + addr_bytes) 37 | host = socket.inet_ntop(family, addr_bytes) 38 | file.write(struct.pack(">H", port)) 39 | return host, port 40 | except socket.error: 41 | continue 42 | 43 | # Well it's not an IP number, so it's probably a DNS name. 44 | if rdns: 45 | # Resolve remotely 46 | host_bytes = host.encode("idna") 47 | file.write(b"\x03" + chr(len(host_bytes)).encode() + host_bytes) 48 | else: 49 | # { MODIFIED CODE } 50 | # Resolve remotely 51 | cached = cachemanager.get_cache(host) 52 | if not cached: 53 | addresses = resolve_dns(host) 54 | 55 | cachemanager.set_cache(host, addresses) 56 | else: 57 | addresses = cached 58 | 59 | addresses = socket.getaddrinfo(addresses[0], port, socket.AF_UNSPEC, 60 | socket.SOCK_STREAM, 61 | socket.IPPROTO_TCP, 62 | socket.AI_ADDRCONFIG) 63 | 64 | # We can't really work out what IP is reachable, so just pick the 65 | # first. 66 | target_addr = addresses[0] 67 | family = target_addr[0] 68 | host = target_addr[4][0] 69 | 70 | addr_bytes = socket.inet_pton(family, host) 71 | file.write(family_to_byte[family] + addr_bytes) 72 | host = socket.inet_ntop(family, addr_bytes) 73 | file.write(struct.pack(">H", port)) 74 | return host, port 75 | 76 | # ============ 77 | # Negotiators 78 | # ============ 79 | 80 | def _negotiate_SOCKS5(self, *dest_addr): 81 | return super()._negotiate_SOCKS5(*dest_addr) 82 | 83 | def _negotiate_SOCKS4(self, dest_addr, dest_port): 84 | """Negotiates a connection through a SOCKS4 server.""" 85 | proxy_type, addr, port, rdns, username, password = self.proxy 86 | 87 | writer = self.makefile("wb") 88 | reader = self.makefile("rb", 0) # buffering=0 renamed in Python 3 89 | try: 90 | # Check if the destination address provided is an IP address 91 | remote_resolve = False 92 | try: 93 | addr_bytes = socket.inet_aton(dest_addr) 94 | except socket.error: 95 | # It's a DNS name. Check where it should be resolved. 96 | if rdns: 97 | addr_bytes = b"\x00\x00\x00\x01" 98 | remote_resolve = True 99 | else: 100 | # { MODIFIED CODE } 101 | cached = cachemanager.get_cache(dest_addr) 102 | if not cached: 103 | addresses = resolve_dns(dest_addr) 104 | 105 | cachemanager.set_cache(dest_addr, addresses) 106 | else: 107 | addresses = cached 108 | 109 | address = addresses[0] 110 | addr_bytes = socket.inet_aton(address) 111 | 112 | # Construct the request packet 113 | writer.write(struct.pack(">BBH", 0x04, 0x01, dest_port)) 114 | writer.write(addr_bytes) 115 | 116 | # The username parameter is considered userid for SOCKS4 117 | if username: 118 | writer.write(username) 119 | writer.write(b"\x00") 120 | 121 | # DNS name if remote resolving is required 122 | # NOTE: This is actually an extension to the SOCKS4 protocol 123 | # called SOCKS4A and may not be supported in all cases. 124 | if remote_resolve: 125 | writer.write(dest_addr.encode("idna") + b"\x00") 126 | writer.flush() 127 | 128 | # Get the response from the server 129 | resp = self._readall(reader, 8) 130 | if resp[0:1] != b"\x00": 131 | # Bad data 132 | raise socks.GeneralProxyError( 133 | "SOCKS4 proxy server sent invalid data") 134 | 135 | status = ord(resp[1:2]) 136 | if status != 0x5A: 137 | # Connection failed: server returned an error 138 | error = socks.SOCKS4_ERRORS.get(status, "Unknown error") 139 | raise socks.SOCKS4Error("{:#04x}: {}".format(status, error)) 140 | 141 | # Get the bound address/port 142 | self.proxy_sockname = (socket.inet_ntoa(resp[4:]), 143 | struct.unpack(">H", resp[2:4])[0]) 144 | if remote_resolve: 145 | self.proxy_peername = socket.inet_ntoa(addr_bytes), dest_port 146 | else: 147 | self.proxy_peername = dest_addr, dest_port 148 | finally: 149 | reader.close() 150 | writer.close() 151 | 152 | def _negotiate_HTTP(self, dest_addr, dest_port): 153 | """Negotiates a connection through an HTTP server. 154 | 155 | NOTE: This currently only supports HTTP CONNECT-style proxies.""" 156 | proxy_type, addr, port, rdns, username, password = self.proxy 157 | 158 | # { MODIFIED CODE } 159 | # If we need to resolve locally, we do this now (with caching) 160 | cached = cachemanager.get_cache(dest_addr) 161 | if not cached and not rdns: 162 | addresses = resolve_dns(dest_addr) 163 | 164 | cachemanager.set_cache(dest_addr, addresses) 165 | else: 166 | addresses = cached 167 | 168 | addr = dest_addr if rdns else addresses[0] 169 | 170 | http_headers = [ 171 | (b"CONNECT " + addr.encode("idna") + b":" 172 | + str(dest_port).encode() + b" HTTP/1.1"), 173 | b"Host: " + dest_addr.encode("idna") 174 | ] 175 | 176 | if username and password: 177 | http_headers.append(b"Proxy-Authorization: basic " 178 | + b64encode(username + b":" + password)) 179 | 180 | http_headers.append(b"\r\n") 181 | 182 | self.sendall(b"\r\n".join(http_headers)) 183 | 184 | # We just need the first line to check if the connection was successful 185 | fobj = self.makefile() 186 | status_line = fobj.readline() 187 | fobj.close() 188 | 189 | if not status_line: 190 | raise socks.GeneralProxyError("Connection closed unexpectedly") 191 | 192 | try: 193 | proto, status_code, status_msg = status_line.split(" ", 2) 194 | except ValueError: 195 | raise socks.GeneralProxyError("HTTP proxy server sent invalid response") 196 | 197 | if not proto.startswith("HTTP/"): 198 | raise socks.GeneralProxyError( 199 | "Proxy server does not appear to be an HTTP proxy") 200 | 201 | try: 202 | status_code = int(status_code) 203 | except ValueError: 204 | raise socks.HTTPError( 205 | "HTTP proxy server did not return a valid HTTP status") 206 | 207 | if status_code != 200: 208 | error = "{}: {}".format(status_code, status_msg) 209 | if status_code in (400, 403, 405): 210 | # It's likely that the HTTP proxy server does not support the 211 | # CONNECT tunneling method 212 | error += ("\n[*] Note: The HTTP proxy server may not be" 213 | " supported by PySocks (must be a CONNECT tunnel" 214 | " proxy)") 215 | raise socks.HTTPError(error) 216 | 217 | self.proxy_sockname = (b"0.0.0.0", 0) 218 | self.proxy_peername = addr, dest_port 219 | 220 | # Redeclare modified proxy negotiators 221 | _proxy_negotiators = { 222 | socks.SOCKS4: _negotiate_SOCKS4, 223 | socks.SOCKS5: _negotiate_SOCKS5, 224 | socks.HTTP: _negotiate_HTTP 225 | } 226 | 227 | # This code is copied from PySocks version 1.7.1 228 | def create_connection(dest_pair, 229 | timeout=None, source_address=None, 230 | proxy_type=None, proxy_addr=None, 231 | proxy_port=None, proxy_rdns=True, 232 | proxy_username=None, proxy_password=None, 233 | socket_options=None): 234 | """create_connection(dest_pair, *[, timeout], **proxy_args) -> socket object 235 | 236 | Like socket.create_connection(), but connects to proxy 237 | before returning the socket object. 238 | 239 | dest_pair - 2-tuple of (IP/hostname, port). 240 | **proxy_args - Same args passed to socksocket.set_proxy() if present. 241 | timeout - Optional socket timeout value, in seconds. 242 | source_address - tuple (host, port) for the socket to bind to as its source 243 | address before connecting (only for compatibility) 244 | """ 245 | # Remove IPv6 brackets on the remote address and proxy address. 246 | remote_host, remote_port = dest_pair 247 | if remote_host.startswith("["): 248 | remote_host = remote_host.strip("[]") 249 | if proxy_addr and proxy_addr.startswith("["): 250 | proxy_addr = proxy_addr.strip("[]") 251 | 252 | err = None 253 | 254 | # Allow the SOCKS proxy to be on IPv4 or IPv6 addresses. 255 | for r in socket.getaddrinfo(proxy_addr, proxy_port, 0, socket.SOCK_STREAM): 256 | family, socket_type, proto, canonname, sa = r 257 | sock = None 258 | try: 259 | sock = socksocketmod(family, socket_type, proto) 260 | 261 | if socket_options: 262 | for opt in socket_options: 263 | sock.setsockopt(*opt) 264 | 265 | if isinstance(timeout, (int, float)): 266 | sock.settimeout(timeout) 267 | 268 | if proxy_type: 269 | sock.set_proxy(proxy_type, proxy_addr, proxy_port, proxy_rdns, 270 | proxy_username, proxy_password) 271 | if source_address: 272 | sock.bind(source_address) 273 | 274 | sock.connect((remote_host, remote_port)) 275 | return sock 276 | 277 | except (socket.error, socks.ProxyError) as e: 278 | err = e 279 | if sock: 280 | sock.close() 281 | sock = None 282 | 283 | if err: 284 | raise err 285 | 286 | raise socket.error("gai returned empty list.") 287 | 288 | # This code is copied from urllib3/contrib/socks.py version 2.1.0 (from requests v2.31.0) 289 | class SOCKSConnection(HTTPConnection): 290 | """ 291 | A plain-text HTTP connection that connects via a SOCKS proxy. 292 | """ 293 | 294 | def __init__( 295 | self, 296 | _socks_options: _TYPE_SOCKS_OPTIONS, 297 | *args: typing.Any, 298 | **kwargs: typing.Any, 299 | ) -> None: 300 | self._socks_options = _socks_options 301 | super().__init__(*args, **kwargs) 302 | 303 | def _new_conn(self) -> socksocketmod: 304 | """ 305 | Establish a new connection via the SOCKS proxy. 306 | """ 307 | extra_kw: dict[str, typing.Any] = {} 308 | if self.source_address: 309 | extra_kw["source_address"] = self.source_address 310 | 311 | if self.socket_options: 312 | extra_kw["socket_options"] = self.socket_options 313 | 314 | try: 315 | conn = create_connection( 316 | (self.host, self.port), 317 | proxy_type=self._socks_options["socks_version"], 318 | proxy_addr=self._socks_options["proxy_host"], 319 | proxy_port=self._socks_options["proxy_port"], 320 | proxy_username=self._socks_options["username"], 321 | proxy_password=self._socks_options["password"], 322 | proxy_rdns=self._socks_options["rdns"], 323 | timeout=self.timeout, 324 | **extra_kw, 325 | ) 326 | 327 | except SocketTimeout as e: 328 | raise ConnectTimeoutError( 329 | self, 330 | f"Connection to {self.host} timed out. (connect timeout={self.timeout})", 331 | ) from e 332 | 333 | except socks.ProxyError as e: 334 | # This is fragile as hell, but it seems to be the only way to raise 335 | # useful errors here. 336 | if e.socket_err: 337 | error = e.socket_err 338 | if isinstance(error, SocketTimeout): 339 | raise ConnectTimeoutError( 340 | self, 341 | f"Connection to {self.host} timed out. (connect timeout={self.timeout})", 342 | ) from e 343 | else: 344 | # Adding `from e` messes with coverage somehow, so it's omitted. 345 | # See #2386. 346 | raise NewConnectionError( 347 | self, f"Failed to establish a new connection: {error}" 348 | ) 349 | else: 350 | raise NewConnectionError( 351 | self, f"Failed to establish a new connection: {e}" 352 | ) from e 353 | 354 | except OSError as e: # Defensive: PySocks should catch all these. 355 | raise NewConnectionError( 356 | self, f"Failed to establish a new connection: {e}" 357 | ) from e 358 | 359 | return conn 360 | 361 | class SOCKSHTTPSConnection(SOCKSConnection, HTTPSConnection): 362 | pass --------------------------------------------------------------------------------