├── .github ├── dependabot.yml └── workflows │ ├── build.yml │ └── release.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE.txt ├── Makefile ├── README.md ├── clients ├── __init__.py ├── aio.py ├── base.py └── py.typed ├── docs ├── index.md ├── reference.md └── requirements.in ├── mkdocs.yml ├── pyproject.toml └── tests ├── __init__.py ├── conftest.py ├── requirements.in ├── test_aio.py └── test_base.py /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | updates: 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "weekly" 8 | 9 | - package-ecosystem: "pip" 10 | directory: "/" 11 | schedule: 12 | interval: "weekly" 13 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: [main] 7 | pull_request: 8 | branches: [main] 9 | 10 | jobs: 11 | test: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | python-version: ['3.10', '3.11', '3.12', '3.13'] 16 | steps: 17 | - uses: actions/checkout@v4 18 | - uses: actions/setup-python@v5 19 | with: 20 | python-version: ${{ matrix.python-version }} 21 | - run: pip install -r tests/requirements.in 22 | - run: make check 23 | - run: coverage xml 24 | - uses: codecov/codecov-action@v5 25 | with: 26 | token: ${{ secrets.CODECOV_TOKEN }} 27 | 28 | lint: 29 | runs-on: ubuntu-latest 30 | steps: 31 | - uses: actions/checkout@v4 32 | - uses: actions/setup-python@v5 33 | with: 34 | python-version: 3.x 35 | - run: pip install ruff mypy 36 | - run: make lint 37 | 38 | docs: 39 | runs-on: ubuntu-latest 40 | steps: 41 | - uses: actions/checkout@v4 42 | - uses: actions/setup-python@v5 43 | with: 44 | python-version: 3.x 45 | - run: pip install -r docs/requirements.in 46 | - run: make html 47 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | publish: 10 | runs-on: ubuntu-latest 11 | permissions: write-all 12 | steps: 13 | - uses: actions/checkout@v4 14 | - uses: actions/setup-python@v5 15 | with: 16 | python-version: 3.x 17 | - run: pip install build -r docs/requirements.in 18 | - run: python -m build 19 | - run: python -m mkdocs gh-deploy --force 20 | - uses: pypa/gh-action-pypi-publish@release/v1 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | site/ 3 | .coverage 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). 5 | 6 | ## Unreleased 7 | ### Changed 8 | * Python >=3.10 required 9 | 10 | ## [1.5](https://pypi.org/project/clients/1.5/) - 2023-11-19 11 | ### Changed 12 | * Python >=3.8 required 13 | * httpx >=0.25 required 14 | 15 | ## [1.4](https://pypi.org/project/clients/1.4/) - 2022-11-19 16 | ### Changed 17 | * Python >=3.7 required 18 | * httpx >=0.23 required 19 | 20 | ### Removed 21 | * `requests` removed 22 | 23 | ## [1.3](https://pypi.org/project/clients/1.3/) - 2020-11-24 24 | * httpx >=0.15 required 25 | * requests deprecated 26 | 27 | ## [1.2](https://pypi.org/project/clients/1.2/) - 2020-01-09 28 | * Python 3 required 29 | * httpx >=0.11 required 30 | 31 | ## [1.1](https://pypi.org/project/clients/1.1/) - 2019-12-07 32 | * Async switched to httpx 33 | 34 | ## [1.0](https://pypi.org/project/clients/1.0/) - 2018-12-08 35 | * Allow missing content-type 36 | * Oauth access tokens supported in authorization header 37 | 38 | ## [0.5](https://pypi.org/project/clients/0.5/) - 2017-12-18 39 | * `AsyncClient` default params 40 | * `Remote` and `AsyncRemote` procedure calls 41 | * `Graph` and `AsyncGraph` execute GraphQL queries 42 | * `Proxy` and `AsyncProxy` clients 43 | 44 | ## [0.4](https://pypi.org/project/clients/0.4/) - 2017-06-11 45 | * Asynchronous clients and resources 46 | 47 | ## [0.3](https://pypi.org/project/clients/0.3/) - 2017-01-02 48 | * `singleton` decorator 49 | 50 | ## [0.2](https://pypi.org/project/clients/0.2/) - 2016-04-10 51 | * Resource attribute upcasts back to a `client` 52 | * `iter` and `download` implement GET requests with streamed content 53 | * `create` implements POST request and returns Location header 54 | * `update` implements PATCH request with json params 55 | * `__call__` implements GET request with params 56 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2022 Aric Coady 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | check: 2 | python -m pytest --cov 3 | 4 | lint: 5 | ruff check . 6 | ruff format --check . 7 | mypy -p clients 8 | 9 | html: 10 | python -m mkdocs build 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![image](https://img.shields.io/pypi/v/clients.svg)](https://pypi.org/project/clients/) 2 | ![image](https://img.shields.io/pypi/pyversions/clients.svg) 3 | [![image](https://pepy.tech/badge/clients)](https://pepy.tech/project/clients) 4 | ![image](https://img.shields.io/pypi/status/clients.svg) 5 | [![build](https://github.com/coady/clients/actions/workflows/build.yml/badge.svg)](https://github.com/coady/clients/actions/workflows/build.yml) 6 | [![image](https://codecov.io/gh/coady/clients/branch/main/graph/badge.svg)](https://codecov.io/gh/coady/clients/) 7 | [![CodeQL](https://github.com/coady/clients/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/coady/clients/actions/workflows/github-code-scanning/codeql) 8 | [![image](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) 9 | [![image](https://mypy-lang.org/static/mypy_badge.svg)](https://mypy-lang.org/) 10 | 11 | Clients originally provided [requests](https://python-requests.org) wrappers to encourage best practices, particularly always using Sessions to connect to the same host or api endpoint. The primary goals were: 12 | * provide a `Client` object with a convenient constructor 13 | * support a base url so that requests can provide a relative path 14 | * provide the same interface for asyncio 15 | 16 | Since then [httpx](https://www.encode.io/httpx) has emerged as the successor to `requests`, and supports the above features natively. So `clients.Client` can be replaced with `httpx.Client` for most use cases. The project will continue to be maintained for additional features, such as the `Resource` object. 17 | 18 | ## Usage 19 | Typical `requests` usage is redundant and inefficient, by not taking advantage of connection pooling. 20 | 21 | ```python 22 | r = requests.get('https://api.github.com/user', headers={'authorization': token}) 23 | r = requests.get('https://api.github.com/user/repos', headers={'authorization': token}) 24 | ``` 25 | 26 | Using sessions is the better approach, but more verbose and in practice requires manual url joining. 27 | 28 | ```python 29 | s = requests.Session() 30 | s.headers['authorization'] = token 31 | r = s.get('https://api.github.com/user') 32 | r = s.get('https://api.github.com/user/repos') 33 | ``` 34 | 35 | ### Client 36 | Clients make using sessions easier, with implicit url joining. 37 | 38 | ```python 39 | client = clients.Client('https://api.github.com/', headers={'authorization': token}) 40 | r = client.get('user') 41 | r = client.get('user/repos') 42 | ``` 43 | 44 | ### Resource 45 | Resources extend Clients to implicitly handle response content, with proper checking of status_code and content-type. 46 | 47 | ```python 48 | github = clients.Resource('https://api.github.com/', headers={'authorization': token}) 49 | for repo in github.get('user/repos', params={'visibility': 'public'}): 50 | ... 51 | ``` 52 | 53 | Resources also implement syntactic support for methods such as __getattr__ and __call__, providing most of the benefits of custom clients as is. 54 | 55 | ```python 56 | for repo in github.user.repos(visibility='public'): 57 | ... 58 | ``` 59 | 60 | Asynchronous variants of all client types are provided, e.g., `AsyncClient`. Additional clients for [RPC](https://en.wikipedia.org/wiki/Remote_procedure_call), [GraphQL](https://graphql.org), and proxies also provided. 61 | 62 | ## Installation 63 | ```console 64 | % pip install clients 65 | ``` 66 | 67 | ## Dependencies 68 | * httpx 69 | 70 | ## Tests 71 | 100% branch coverage. 72 | ```console 73 | % pytest [--cov] 74 | ``` 75 | -------------------------------------------------------------------------------- /clients/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import Client, Graph, Proxy, Remote, Resource # noqa 2 | from .aio import AsyncClient, AsyncGraph, AsyncProxy, AsyncRemote, AsyncResource # noqa 3 | 4 | 5 | def singleton(*args, **kwargs): 6 | """Return a decorator for singleton class instances.""" 7 | return lambda cls: cls(*args, **kwargs) 8 | -------------------------------------------------------------------------------- /clients/aio.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | import asyncio 3 | import contextlib 4 | from collections.abc import Callable, Mapping 5 | from urllib.parse import urljoin 6 | import httpx 7 | from .base import validate, BaseClient, Graph, Proxy, Remote, Resource 8 | 9 | 10 | class AsyncClient(BaseClient, httpx.AsyncClient): # type: ignore 11 | def run(self, name: str, *args, **kwargs): 12 | """Synchronously call method and run coroutine.""" 13 | return asyncio.new_event_loop().run_until_complete(getattr(self, name)(*args, **kwargs)) 14 | 15 | 16 | class AsyncResource(AsyncClient): 17 | """An `AsyncClient` which returns json content and has syntactic support for requests.""" 18 | 19 | client = property(AsyncClient.clone, doc="upcasted `AsyncClient`") 20 | __getattr__ = AsyncClient.__truediv__ 21 | __getitem__ = AsyncClient.get 22 | content_type = staticmethod(Resource.content_type) 23 | __call__ = Resource.__call__ 24 | 25 | async def request(self, method, path, **kwargs): 26 | """Send request with path and return processed content.""" 27 | response = (await super().request(method, path, **kwargs)).raise_for_status() 28 | match self.content_type(response): 29 | case 'json': 30 | return response.json() 31 | case 'text': 32 | return response.text 33 | return response.content 34 | 35 | async def updater(self, path='', **kwargs): 36 | response = (await super().request('GET', path, **kwargs)).raise_for_status() 37 | kwargs['headers'] = dict(kwargs.get('headers', {}), **validate(response)) 38 | yield await self.put(path, (yield response.json()), **kwargs) 39 | 40 | @contextlib.asynccontextmanager 41 | async def updating(self, path: str = '', **kwargs): 42 | """Context manager to GET and conditionally PUT json data.""" 43 | updater = self.updater(path, **kwargs) 44 | json = await updater.__anext__() 45 | yield json 46 | await updater.asend(json) 47 | 48 | async def update(self, path: str = '', callback: Callable | None = None, **json): 49 | """PATCH request with json params. 50 | 51 | Args: 52 | callback: optionally update with GET and validated PUT. 53 | `callback` is called on the json result with keyword params, i.e., 54 | `dict` correctly implements the simple update case. 55 | """ 56 | if callback is None: 57 | return await self.patch(path, json) 58 | updater = self.updater(path) 59 | return await updater.asend(callback(await updater.__anext__(), **json)) 60 | 61 | async def authorize(self, path: str = '', **kwargs) -> dict: 62 | """Acquire oauth access token and set `Authorization` header.""" 63 | method = 'GET' if {'json', 'data'}.isdisjoint(kwargs) else 'POST' 64 | result = await self.request(method, path, **kwargs) 65 | self.headers['authorization'] = f"{result['token_type']} {result['access_token']}" 66 | self._attrs['headers'] = self.headers 67 | return result 68 | 69 | 70 | class AsyncRemote(AsyncClient): 71 | """An `AsyncClient` which defaults to posts with json bodies, i.e., RPC. 72 | 73 | Args: 74 | url: base url for requests 75 | json: default json body for all calls 76 | **kwargs: same options as `AsyncClient` 77 | """ 78 | 79 | client = AsyncResource.client 80 | __getattr__ = AsyncResource.__getattr__ 81 | check = staticmethod(Remote.check) 82 | 83 | def __init__(self, url: str, json: Mapping = {}, **kwargs): 84 | super().__init__(url, **kwargs) 85 | self.json = dict(json) 86 | 87 | @classmethod 88 | def clone(cls, other, path=''): 89 | return AsyncClient.clone.__func__(cls, other, path, json=other.json) 90 | 91 | async def __call__(self, path='', **json): 92 | """POST request with json body and check result.""" 93 | response = (await self.post(path, json=dict(self.json, **json))).raise_for_status() 94 | return self.check(response.json()) 95 | 96 | 97 | class AsyncGraph(AsyncRemote): 98 | """An `AsyncRemote` client which executes GraphQL queries.""" 99 | 100 | Error = httpx.HTTPError 101 | execute = Graph.execute 102 | check = classmethod(Graph.check.__func__) # type: ignore 103 | 104 | 105 | class AsyncProxy(AsyncClient): 106 | """An extensible embedded proxy client to multiple hosts. 107 | 108 | The default implementation provides load balancing based on active connections. 109 | It does not provide error handling or retrying. 110 | 111 | Args: 112 | *urls: base urls for requests 113 | **kwargs: same options as `AsyncClient` 114 | """ 115 | 116 | Stats = Proxy.Stats 117 | priority = Proxy.priority 118 | choice = Proxy.choice 119 | 120 | def __init__(self, *urls: str, **kwargs): 121 | super().__init__('https://proxies', **kwargs) 122 | self.urls = {(url.rstrip('/') + '/'): self.Stats() for url in urls} 123 | 124 | @classmethod 125 | def clone(cls, other, path=''): 126 | urls = (urljoin(url, path) for url in other.urls) 127 | return cls(*urls, trailing=other.trailing, **other._attrs) 128 | 129 | async def request(self, method, path, **kwargs): 130 | """Send request with relative or absolute path and return response.""" 131 | url = self.choice(method) 132 | with self.urls[url] as stats: 133 | response = await super().request(method, urljoin(url, path), **kwargs) 134 | stats.add(failures=int(response.status_code >= 500)) 135 | return response 136 | -------------------------------------------------------------------------------- /clients/base.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | import collections 3 | import contextlib 4 | import functools 5 | import json 6 | import random 7 | import re 8 | import threading 9 | from collections.abc import Callable, Iterator, Mapping 10 | from urllib.parse import urljoin 11 | import httpx 12 | 13 | 14 | def content_type(response, **patterns): 15 | """Return name for response's content-type based on regular expression matches.""" 16 | ct = response.headers.get('content-type', '') 17 | matches = (name for name, pattern in patterns.items() if re.match(pattern, ct)) 18 | return next(matches, '') 19 | 20 | 21 | def validate(response): 22 | """Return validation headers from response translated for modification.""" 23 | headers = response.headers 24 | validators = {'etag': 'if-match', 'last-modified': 'if-unmodified-since'} 25 | return {validators[key]: headers[key] for key in validators if key in headers} 26 | 27 | 28 | class BaseClient: 29 | """Client mixin. 30 | 31 | Args: 32 | url: base url for requests 33 | trailing: trailing chars (e.g. /) appended to the url 34 | **attrs: additional Session attributes 35 | """ 36 | 37 | def __init__(self, url: str, *, trailing: str = '', **attrs): 38 | super().__init__(base_url=url.rstrip('/') + '/', **attrs) # type: ignore 39 | self._attrs = attrs 40 | self.trailing = trailing 41 | 42 | def __repr__(self): 43 | return f'{type(self).__name__}({self.url}... {self.trailing})' 44 | 45 | def __truediv__(self, path: str) -> Client: 46 | """Return a cloned client with appended path.""" 47 | return type(self).clone(self, path) 48 | 49 | @property 50 | def url(self): 51 | return str(self.base_url) 52 | 53 | @classmethod 54 | def clone(cls, other, path='', **kwargs): 55 | url = str(other.base_url.join(path)) 56 | return cls(url, trailing=other.trailing, **(other._attrs | kwargs)) 57 | 58 | def request(self, method, path, **kwargs): 59 | """Send request with relative or absolute path and return response.""" 60 | url = str(self.base_url.join(path)).rstrip('/') + self.trailing 61 | return super().request(method, url, **kwargs) 62 | 63 | def get(self, path='', **kwargs): 64 | """GET request with optional path.""" 65 | return self.request('GET', path, **kwargs) 66 | 67 | def options(self, path='', **kwargs): 68 | """OPTIONS request with optional path.""" 69 | return self.request('OPTIONS', path, **kwargs) 70 | 71 | def head(self, path='', **kwargs): 72 | """HEAD request with optional path.""" 73 | return self.request('HEAD', path, **kwargs) 74 | 75 | def post(self, path='', json=None, **kwargs): 76 | """POST request with optional path and json body.""" 77 | return self.request('POST', path, json=json, **kwargs) 78 | 79 | def put(self, path='', json=None, **kwargs): 80 | """PUT request with optional path and json body.""" 81 | return self.request('PUT', path, json=json, **kwargs) 82 | 83 | def patch(self, path='', json=None, **kwargs): 84 | """PATCH request with optional path and json body.""" 85 | return self.request('PATCH', path, json=json, **kwargs) 86 | 87 | def delete(self, path='', **kwargs): 88 | """DELETE request with optional path.""" 89 | return self.request('DELETE', path, **kwargs) 90 | 91 | 92 | class Client(BaseClient, httpx.Client): # type: ignore 93 | def stream(self, method, path, **kwargs): 94 | """Send request with relative or absolute path and stream response.""" 95 | url = str(self.base_url.join(path)).rstrip('/') + self.trailing 96 | return super().stream(method, url, **kwargs) 97 | 98 | 99 | class Resource(Client): 100 | """A `Client` which returns json content and has syntactic support for requests.""" 101 | 102 | client = property(Client.clone, doc="upcasted `Client`") 103 | __getitem__ = Client.get 104 | __setitem__ = Client.put 105 | __delitem__ = Client.delete 106 | __getattr__ = Client.__truediv__ 107 | content_type = staticmethod( 108 | functools.partial(content_type, text='text/', json=r'application/(\w|\.)*\+?json') 109 | ) 110 | 111 | def request(self, method, path, **kwargs): 112 | """Send request with path and return processed content.""" 113 | response = super().request(method, path, **kwargs).raise_for_status() 114 | match self.content_type(response): 115 | case 'json': 116 | return response.json() 117 | case 'text': 118 | return response.text 119 | return response.content 120 | 121 | def stream(self, method: str = 'GET', path: str = '', **kwargs) -> Iterator: # type: ignore 122 | """Iterate lines or chunks from streamed request.""" 123 | with super().stream(method, path, **kwargs) as response: 124 | match self.content_type(response.raise_for_status()): 125 | case 'json': 126 | yield from map(json.loads, response.iter_lines()) 127 | case 'text': 128 | yield from response.iter_lines() 129 | case _: 130 | yield from response.iter_bytes() 131 | 132 | __iter__ = stream 133 | 134 | def __contains__(self, path: str): 135 | """Return whether endpoint exists according to HEAD request.""" 136 | return not super().request('HEAD', path).is_error 137 | 138 | def __call__(self, path: str = '', **params): 139 | """GET request with params.""" 140 | return self.get(path, params=params) 141 | 142 | def updater(self, path='', **kwargs): 143 | response = super().request('GET', path, **kwargs).raise_for_status() 144 | kwargs['headers'] = dict(kwargs.get('headers', {}), **validate(response)) 145 | yield self.put(path, (yield response.json()), **kwargs) 146 | 147 | @contextlib.contextmanager 148 | def updating(self, path: str = '', **kwargs): 149 | """Context manager to GET and conditionally PUT json data.""" 150 | updater = self.updater(path, **kwargs) 151 | json = next(updater) 152 | yield json 153 | updater.send(json) 154 | 155 | def update(self, path: str = '', callback: Callable | None = None, **json): 156 | """PATCH request with json params. 157 | 158 | Args: 159 | callback: optionally update with GET and validated PUT. 160 | `callback` is called on the json result with keyword params, i.e., 161 | `dict` correctly implements the simple update case. 162 | """ 163 | if callback is None: 164 | return self.patch(path, json=json) 165 | updater = self.updater(path) 166 | return updater.send(callback(next(updater), **json)) 167 | 168 | def create(self, path: str = '', json=None, **kwargs) -> str: 169 | """POST request and return location.""" 170 | response = super().request('POST', path, json=json, **kwargs).raise_for_status() 171 | return response.headers.get('location') 172 | 173 | def download(self, file, path: str = '', **kwargs): 174 | """Output streamed GET request to file.""" 175 | for chunk in self.stream(path=path, **kwargs): 176 | file.write(chunk) 177 | return file 178 | 179 | def authorize(self, path: str = '', **kwargs) -> dict: 180 | """Acquire oauth access token and set `Authorization` header.""" 181 | method = 'GET' if {'json', 'data'}.isdisjoint(kwargs) else 'POST' 182 | result = self.request(method, path, **kwargs) 183 | self.headers['authorization'] = f"{result['token_type']} {result['access_token']}" 184 | return result 185 | 186 | 187 | class Remote(Client): 188 | """A `Client` which defaults to posts with json bodies, i.e., RPC. 189 | 190 | Args: 191 | url: base url for requests 192 | json: default json body for all calls 193 | **kwargs: same options as `Client` 194 | """ 195 | 196 | client = Resource.client 197 | __getattr__ = Resource.__getattr__ 198 | 199 | def __init__(self, url: str, json: Mapping = {}, **kwargs): 200 | super().__init__(url, **kwargs) 201 | self.json = dict(json) 202 | 203 | @classmethod 204 | def clone(cls, other, path=''): 205 | return Client.clone.__func__(cls, other, path, json=other.json) 206 | 207 | def __call__(self, path: str = '', **json): 208 | """POST request with json body and [check][clients.base.Remote.check] result.""" 209 | response = self.post(path, json=dict(self.json, **json)).raise_for_status() 210 | return self.check(response.json()) 211 | 212 | @staticmethod 213 | def check(result): 214 | """Override to return result or raise error, for APIs which don't use status codes.""" 215 | return result 216 | 217 | 218 | class Graph(Remote): 219 | """A `Remote` client which executes GraphQL queries.""" 220 | 221 | Error = httpx.HTTPError 222 | 223 | @classmethod 224 | def check(cls, result: dict): 225 | """Return `data` or raise `errors`.""" 226 | for error in result.get('errors', ()): 227 | raise cls.Error(error) 228 | return result.get('data') 229 | 230 | def execute(self, query: str, **variables): 231 | """Execute query over POST.""" 232 | return self(query=query, variables=variables) 233 | 234 | 235 | class Stats(collections.Counter): 236 | """Thread-safe Counter. 237 | 238 | Context manager tracks number of active connections and errors. 239 | """ 240 | 241 | def __init__(self): 242 | self.lock = threading.Lock() 243 | 244 | def add(self, **kwargs): 245 | """Atomically add data.""" 246 | with self.lock: 247 | self.update(kwargs) 248 | 249 | def __enter__(self): 250 | self.add(connections=1) 251 | return self 252 | 253 | def __exit__(self, *args): 254 | self.add(connections=-1, errors=int(any(args))) 255 | 256 | 257 | class Proxy(Client): 258 | """An extensible embedded proxy client to multiple hosts. 259 | 260 | The default implementation provides load balancing based on active connections. 261 | It does not provide error handling or retrying. 262 | 263 | Args: 264 | *urls: base urls for requests 265 | **kwargs: same options as `Client` 266 | """ 267 | 268 | Stats = Stats 269 | 270 | def __init__(self, *urls: str, **kwargs): 271 | super().__init__('https://proxies', **kwargs) 272 | self.urls = {(url.rstrip('/') + '/'): self.Stats() for url in urls} 273 | 274 | @classmethod 275 | def clone(cls, other, path=''): 276 | urls = (urljoin(url, path) for url in other.urls) 277 | return cls(*urls, trailing=other.trailing, **other._attrs) 278 | 279 | def priority(self, url: str): 280 | """Return comparable priority for url. 281 | 282 | Minimizes errors, failures (500s), and active connections. 283 | None may be used to eliminate from consideration. 284 | """ 285 | stats = self.urls[url] 286 | return tuple(stats[key] for key in ('errors', 'failures', 'connections')) 287 | 288 | def choice(self, method: str) -> str: 289 | """Return chosen url according to priority. 290 | 291 | Args: 292 | method: placeholder for extensions which distinguish read/write requests 293 | """ 294 | priorities = collections.defaultdict(list) 295 | for url in self.urls: 296 | priorities[self.priority(url)].append(url) 297 | priorities.pop(None, None) 298 | return random.choice(priorities[min(priorities)]) 299 | 300 | def request(self, method, path, **kwargs): 301 | """Send request with relative or absolute path and return response.""" 302 | url = self.choice(method) 303 | with self.urls[url] as stats: 304 | response = super().request(method, urljoin(url, path), **kwargs) 305 | stats.add(failures=int(response.is_server_error)) 306 | return response 307 | -------------------------------------------------------------------------------- /clients/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coady/clients/16d1ade01ffbea1a31481d376a8f257f0af39d57/clients/py.typed -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | ## Quickstart 2 | 3 | Typical [requests](https://python-requests.org) usage has falling into some anti-patterns. 4 | 5 | * Being url-based, realistically all code needs to deal with url joining. Which tends to be redundant and suffer from leading or trailing slash issues. 6 | * The module level methods don't take advantage of connection pooling, and require duplicate settings. Given the "100% automatic" documentation of connection reuse, it's unclear how widely known this is. 7 | * Using a `Session` requires assigning every setting individually, and still requires url joining. 8 | 9 | [Clients](reference.md#clients.Client) aim to be encourage best practices while still being convenient. Examples use the [httpbin](http://httpbin.org) client testing service. 10 | 11 | ```python 12 | client = clients.Client(url, auth=('user', 'pass'), headers={'x-test': 'true'}) 13 | r = client.get('headers', headers={'x-test2': 'true'}) 14 | assert {'x-test', 'x-test2'} <= set(r.request.headers) 15 | 16 | r = client.get('cookies') 17 | assert r.json() == {'cookies': {}} 18 | 19 | client.get('cookies/set', params={'sessioncookie': '123456789'}) 20 | r = client.get('cookies') 21 | assert r.json() == {'cookies': {'sessioncookie': '123456789'}} 22 | ``` 23 | 24 | Which reveals another anti-pattern regarding `Response` objects. Although the response object is sometimes required, naturally the most common use case is to access the content. But the onus is on the caller to check the `status_code` and `content-type`. 25 | 26 | [Resources](reference.md#clients.Resource) aim to making writing custom api clients or sdks easier. Their primary feature is to allow direct content access without silencing errors. Response content type is inferred from headers: `json`, `content`, or `text`. 27 | 28 | ```python 29 | resource = clients.Resource(url) 30 | assert resource.get('get')['url'] == url + '/get' 31 | with pytest.raises(IOError): 32 | resource.get('status/404') 33 | assert '' in resource.get('html') 34 | assert isinstance(resource.get('bytes/10'), bytes) 35 | ``` 36 | 37 | ## Advanced Usage 38 | 39 | `Clients` allow any base url, not just hosts, and consequently support path concatenation. Following the semantics of `urljoin` however, absolute paths and urls are treated as such. Hence there's no need to parse a url retrieved from an api. 40 | 41 | ```python 42 | client = clients.Client(url) 43 | cookies = client / 'cookies' 44 | assert isinstance(cookies, clients.Client) 45 | assert cookies.get().url == url + '/cookies' 46 | 47 | assert cookies.get('/').url == url + '/' 48 | assert cookies.get(url).url == url + '/' 49 | ``` 50 | 51 | Some api endpoints require trailing slashes; some forbid them. Set it and forget it. 52 | 53 | ```python 54 | client = clients.Client(url, trailing='/') 55 | assert client.get('ip').status_code == 404 56 | ``` 57 | 58 | Note `trailing` isn\'t limited to only being a slash. This can be useful for static paths below a parameter: `api/v1/{query}.json`. 59 | 60 | ## Asyncio 61 | 62 | [AsyncClients](reference.md#clients.AsyncClient) and [AsyncResources](reference.md#clients.AsyncResource) implement the same interface, except the request methods return asyncio [coroutines](https://docs.python.org/3/library/asyncio-task.html#coroutines). 63 | 64 | ## Avant-garde Usage 65 | 66 | `Resources` support operator overloaded syntax wherever sensible. These interfaces often obviate the need for writing custom clients specific to an API. 67 | 68 | * `__getattr__`: alternate path concatenation 69 | * `__getitem__`: GET content 70 | * `__setitem__`: PUT json 71 | * `__delitem__`: DELETE 72 | * `__contains__`: HEAD ok 73 | * `__iter__`: GET streamed lines or content 74 | * `__call__`: GET with params 75 | 76 | ```python 77 | resource = clients.Resource(url) 78 | assert set(resource['get']) == {'origin', 'headers', 'args', 'url'} 79 | resource['put'] = {} 80 | del resource['delete'] 81 | 82 | assert '200' in resource.status 83 | assert '404' not in resource.status 84 | assert [line['id'] for line in resource / 'stream/3'] == [0, 1, 2] 85 | assert next(iter(resource / 'html')) == '' 86 | assert resource('cookies/set', name='value') == {'cookies': {'name': 'value'}} 87 | ``` 88 | 89 | Higher-level methods for common requests. 90 | 91 | * `iter`: \_\_iter\_\_ with args 92 | * `update`: PATCH with json params, or GET with conditional PUT 93 | * `create`: POST and return location 94 | * `download`: GET streamed content to file 95 | * `authorize`: acquire oauth token 96 | 97 | ```python 98 | resource = clients.Resource(url) 99 | assert list(map(len, resource.iter('stream-bytes/256'))) == [128] * 2 100 | assert resource.update('patch', name='value')['json'] == {'name': 'value'} 101 | assert resource.create('post', {'name': 'value'}) is None 102 | file = resource.download(io.BytesIO(), 'image/png') 103 | assert file.tell() 104 | ``` 105 | 106 | A [singleton](reference.md#clients.singleton) decorator can be used on subclasses, conveniently creating a single custom instance. 107 | 108 | ```python 109 | @clients.singleton('http://localhost/') 110 | class custom_api(clients.Resource): 111 | pass # custom methods 112 | 113 | assert isinstance(custom_api, clients.Resource) 114 | assert custom_api.url == 'http://localhost/' 115 | ``` 116 | 117 | [Remote](reference.md#clients.Remote) and [AsyncRemote](reference.md#clients.AsyncRemote) clients default to POSTs with json bodies, for APIs which are more RPC than REST. 118 | 119 | [Graph](reference.md#clients.Graph) and [AsyncGraph](reference.md#clients.AsyncGraph) remote clients execute GraphQL queries. 120 | 121 | [Proxy](reference.md#clients.Proxy) and [AsyncProxy](reference.md#clients.AsyncProxy) clients provide load-balancing across multiple hosts, with an extensible interface for different algorithms. 122 | -------------------------------------------------------------------------------- /docs/reference.md: -------------------------------------------------------------------------------- 1 | ::: clients.base.BaseClient 2 | 3 | ::: clients.Client 4 | 5 | ::: clients.Resource 6 | 7 | ::: clients.Remote 8 | 9 | ::: clients.Graph 10 | 11 | ::: clients.Proxy 12 | 13 | ::: clients.AsyncClient 14 | 15 | ::: clients.AsyncResource 16 | 17 | ::: clients.AsyncRemote 18 | 19 | ::: clients.AsyncGraph 20 | 21 | ::: clients.AsyncProxy 22 | 23 | ::: clients.singleton 24 | -------------------------------------------------------------------------------- /docs/requirements.in: -------------------------------------------------------------------------------- 1 | httpx 2 | mkdocs-material 3 | mkdocstrings[python] 4 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: clients 2 | site_url: https://coady.github.io/clients/ 3 | site_description: High-level HTTP clients for Python. 4 | theme: material 5 | 6 | repo_name: coady/clients 7 | repo_url: https://github.com/coady/clients 8 | edit_uri: "" 9 | 10 | nav: 11 | - Introduction: index.md 12 | - Reference: reference.md 13 | 14 | plugins: 15 | - search 16 | - mkdocstrings: 17 | handlers: 18 | python: 19 | options: 20 | show_root_heading: true 21 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "clients" 3 | version = "1.5" 4 | description = "High-level HTTP clients for Python." 5 | readme = "README.md" 6 | requires-python = ">=3.10" 7 | license = {file = "LICENSE.txt"} 8 | authors = [{name = "Aric Coady", email = "aric.coady@gmail.com"}] 9 | keywords = ["requests", "sessions", "responses", "resources", "asyncio"] 10 | classifiers = [ 11 | "Development Status :: 6 - Mature", 12 | "Framework :: AsyncIO", 13 | "Intended Audience :: Developers", 14 | "License :: OSI Approved :: Apache Software License", 15 | "Operating System :: OS Independent", 16 | "Programming Language :: Python :: 3", 17 | "Programming Language :: Python :: 3.10", 18 | "Programming Language :: Python :: 3.11", 19 | "Programming Language :: Python :: 3.12", 20 | "Programming Language :: Python :: 3.13", 21 | "Topic :: Internet :: WWW/HTTP :: Session", 22 | "Topic :: Software Development :: Libraries :: Python Modules", 23 | "Typing :: Typed", 24 | ] 25 | dependencies = ["httpx>=0.25"] 26 | 27 | [project.urls] 28 | Homepage = "https://github.com/coady/clients" 29 | Documentation = "https://coady.github.io/clients" 30 | Changelog = "https://github.com/coady/clients/blob/main/CHANGELOG.md" 31 | Issues = "https://github.com/coady/clients/issues" 32 | 33 | [tool.ruff] 34 | line-length = 100 35 | 36 | [tool.ruff.format] 37 | quote-style = "preserve" 38 | 39 | [[tool.mypy.overrides]] 40 | module = ["httpx.*"] 41 | ignore_missing_imports = true 42 | 43 | [tool.pytest.ini_options] 44 | asyncio_mode = "auto" 45 | 46 | [tool.coverage.run] 47 | source = ["clients"] 48 | branch = true 49 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coady/clients/16d1ade01ffbea1a31481d376a8f257f0af39d57/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from importlib import metadata 2 | import pytest 3 | 4 | 5 | def pytest_report_header(config): 6 | return 'httpx: ' + metadata.version('httpx') 7 | 8 | 9 | @pytest.fixture 10 | def url(httpbin): 11 | return httpbin.url 12 | -------------------------------------------------------------------------------- /tests/requirements.in: -------------------------------------------------------------------------------- 1 | httpx>=0.25 2 | pytest-cov 3 | pytest-httpbin>=0.10.2 4 | pytest-asyncio>=0.17 5 | -------------------------------------------------------------------------------- /tests/test_aio.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import json 3 | import operator 4 | import httpx 5 | import pytest 6 | import clients 7 | 8 | 9 | async def test_client(url): 10 | client = clients.AsyncClient(url, params={'q': 0}) 11 | coros = ( 12 | client.head(), 13 | client.options(), 14 | client.post('post'), 15 | client.put('put'), 16 | client.patch('patch'), 17 | client.delete('delete'), 18 | ) 19 | for coro in coros: 20 | r = await coro 21 | assert r.status_code == 200 and r.url.query == b'q=0' 22 | r = await (client / 'ip').get(params={'q': 1}) 23 | assert set(r.json()) == {'origin'} and r.url.query == b'q=1' 24 | 25 | 26 | async def test_resource(url): 27 | params = {'etag': 'W/0', 'last-modified': 'now'} 28 | resource = clients.AsyncResource(url, params=params) 29 | assert isinstance(await resource['encoding/utf8'], str) 30 | assert isinstance(await resource('stream-bytes/1'), (str, bytes)) 31 | assert (await resource.update('patch', key='value'))['json'] == {'key': 'value'} 32 | with pytest.raises(httpx.HTTPError, match='404'): 33 | await resource.status('404') 34 | with pytest.raises(httpx.HTTPError): 35 | await resource.update('response-headers', callback=dict, key='value') 36 | with pytest.raises(httpx.HTTPError): 37 | async with resource.updating('response-headers') as data: 38 | assert data['etag'] == 'W/0' 39 | 40 | 41 | async def test_content(url): 42 | resource = clients.AsyncResource(url) 43 | resource.content_type = lambda response: 'json' 44 | coro = resource.get('robots.txt') 45 | assert not hasattr(coro, '__aenter__') 46 | with pytest.raises(ValueError): 47 | await coro 48 | 49 | 50 | def test_authorize(url, monkeypatch): 51 | resource = clients.AsyncResource(url) 52 | future = asyncio.Future(loop=asyncio.new_event_loop()) 53 | future.set_result({'access_token': 'abc123', 'token_type': 'Bearer', 'expires_in': 0}) 54 | monkeypatch.setattr(clients.AsyncResource, 'request', lambda *args, **kwargs: future) 55 | for key in ('params', 'data', 'json'): 56 | assert resource.run('authorize', **{key: {}}) == future.result() 57 | assert resource.headers['authorization'] == 'Bearer abc123' 58 | 59 | 60 | async def test_remote(url): 61 | remote = clients.AsyncRemote(url, json={'key': 'value'}) 62 | assert (await remote('post'))['json'] == {'key': 'value'} 63 | clients.AsyncRemote.check = operator.methodcaller('pop', 'json') 64 | assert await (remote / 'post')(name='value') == {'key': 'value', 'name': 'value'} 65 | 66 | 67 | async def test_graph(url): 68 | graph = clients.AsyncGraph(url).anything 69 | data = await graph.execute('{ viewer { login }}') 70 | assert json.loads(data) == {'query': '{ viewer { login }}', 'variables': {}} 71 | with pytest.raises(httpx.HTTPError, match='reason'): 72 | clients.AsyncGraph.check({'errors': ['reason']}) 73 | 74 | 75 | async def test_proxy(httpbin): 76 | proxy = clients.AsyncProxy(httpbin.url, f'http://localhost:{httpbin.port}') 77 | urls = {(await proxy.get('status/500')).url for _ in proxy.urls} 78 | assert len(urls) == len(proxy.urls) 79 | 80 | 81 | def test_clones(): 82 | client = clients.AsyncClient('http://localhost/', trailing='/') 83 | assert str(client) == 'AsyncClient(http://localhost/... /)' 84 | assert str(client / 'path') == 'AsyncClient(http://localhost/path/... /)' 85 | 86 | resource = clients.AsyncResource('http://localhost/').path 87 | assert str(resource) == 'AsyncResource(http://localhost/path/... )' 88 | assert type(resource.client) is clients.AsyncClient 89 | 90 | remote = clients.AsyncRemote('http://localhost/').path 91 | assert str(remote) == 'AsyncRemote(http://localhost/path/... )' 92 | assert type(remote.client) is clients.AsyncClient 93 | 94 | proxy = clients.AsyncProxy('http://localhost/', 'http://127.0.0.1') / 'path' 95 | assert str(proxy) == 'AsyncProxy(https://proxies/... )' 96 | -------------------------------------------------------------------------------- /tests/test_base.py: -------------------------------------------------------------------------------- 1 | import json 2 | import operator 3 | import io 4 | import httpx 5 | import pytest 6 | import clients 7 | 8 | 9 | def test_cookies(url): 10 | client = clients.Client(url, auth=('user', 'pass'), headers={'x-test': 'true'}) 11 | r = client.get('headers', headers={'x-test2': 'true'}) 12 | assert {'x-test', 'x-test2'} <= set(r.request.headers) 13 | 14 | r = client.get('cookies') 15 | assert r.json() == {'cookies': {}} 16 | 17 | client.get('cookies/set', params={'sessioncookie': '123456789'}) 18 | r = client.get('cookies') 19 | assert r.json() == {'cookies': {'sessioncookie': '123456789'}} 20 | 21 | 22 | def test_content(url): 23 | resource = clients.Resource(url) 24 | assert resource.get('get')['url'] == url + '/get' 25 | with pytest.raises(httpx.HTTPError): 26 | resource.get('status/404') 27 | assert '' in resource.get('html') 28 | assert isinstance(resource.get('stream-bytes/10'), bytes) 29 | 30 | 31 | def test_path(url): 32 | client = clients.Client(url) 33 | cookies = client / 'cookies' 34 | assert isinstance(cookies, clients.Client) 35 | assert str(cookies.get().url) == url + '/cookies' 36 | 37 | assert str(cookies.get('/').url) == url 38 | assert str(cookies.get(url).url) == url 39 | 40 | 41 | def test_trailing(url): 42 | client = clients.Client(url, trailing='/') 43 | assert client.get('ip').status_code == 404 44 | 45 | 46 | def test_syntax(url): 47 | resource = clients.Resource(url, follow_redirects=True) 48 | assert set(resource['get']) == {'origin', 'headers', 'args', 'url'} 49 | resource['put'] = {} 50 | del resource['delete'] 51 | 52 | assert '200' in resource.status 53 | assert '404' not in resource.status 54 | assert [line['id'] for line in resource / 'stream/3'] == [0, 1, 2] 55 | assert next(iter(resource / 'html')) == '' 56 | assert resource('cookies/set', name='value') == {'cookies': {'name': 'value'}} 57 | 58 | 59 | def test_methods(url): 60 | resource = clients.Resource(url) 61 | assert sum(map(len, resource.stream(path='stream-bytes/256'))) == 256 62 | assert resource.update('patch', name='value')['json'] == {'name': 'value'} 63 | assert resource.create('post', {'name': 'value'}) is None 64 | file = resource.download(io.BytesIO(), 'image/png') 65 | assert file.tell() 66 | 67 | 68 | def test_authorize(url, monkeypatch): 69 | resource = clients.Resource(url) 70 | result = {'access_token': 'abc123', 'token_type': 'Bearer', 'expires_in': 0} 71 | monkeypatch.setattr(clients.Resource, 'request', lambda *args, **kwargs: result) 72 | for key in ('params', 'data', 'json'): 73 | assert resource.authorize(**{key: {}}) == result 74 | assert resource.headers['authorization'] == 'Bearer abc123' 75 | 76 | 77 | def test_callback(url): 78 | resource = clients.Resource(url, params={'etag': 'W/0', 'last-modified': 'now'}) 79 | with pytest.raises(httpx.HTTPError, match='405') as exc: 80 | resource.update('response-headers', callback=dict, name='value') 81 | headers = exc.value.request.headers 82 | assert headers['if-match'] == 'W/0' and headers['if-unmodified-since'] == 'now' 83 | with pytest.raises(httpx.HTTPError, match='405') as exc: 84 | with resource.updating('response-headers') as data: 85 | data['name'] = 'value' 86 | assert b'"name":' in exc.value.request.read() 87 | 88 | 89 | def test_meta(url): 90 | client = clients.Client(url) 91 | response = client.options('get') 92 | assert not response.is_error and not response.content 93 | response = client.head('get') 94 | assert not response.is_error and not response.content 95 | del response.headers['content-type'] 96 | assert clients.Resource.content_type(response) == '' 97 | 98 | 99 | def test_remote(url): 100 | remote = clients.Remote(url, json={'key': 'value'}) 101 | assert remote('post')['json'] == {'key': 'value'} 102 | clients.Remote.check = operator.methodcaller('pop', 'json') 103 | assert (remote / 'post')(name='value') == {'key': 'value', 'name': 'value'} 104 | 105 | 106 | def test_graph(url): 107 | graph = clients.Graph(url).anything 108 | data = graph.execute('{ viewer { login }}') 109 | assert json.loads(data) == {'query': '{ viewer { login }}', 'variables': {}} 110 | with pytest.raises(httpx.HTTPError, match='reason'): 111 | clients.Graph.check({'errors': ['reason']}) 112 | 113 | 114 | def test_proxy(httpbin): 115 | proxy = clients.Proxy(httpbin.url, f'http://localhost:{httpbin.port}') 116 | urls = {proxy.get('status/500').url for _ in proxy.urls} 117 | assert len(urls) == len(proxy.urls) 118 | 119 | 120 | def test_clones(): 121 | client = clients.Client('http://localhost/', trailing='/') 122 | assert str(client) == 'Client(http://localhost/... /)' 123 | assert str(client / 'path') == 'Client(http://localhost/path/... /)' 124 | 125 | resource = clients.Resource('http://localhost/').path 126 | assert str(resource) == 'Resource(http://localhost/path/... )' 127 | assert type(resource.client) is clients.Client 128 | 129 | remote = clients.Remote('http://localhost/').path 130 | assert str(remote) == 'Remote(http://localhost/path/... )' 131 | assert type(remote.client) is clients.Client 132 | 133 | proxy = clients.Proxy('http://localhost/', 'http://127.0.0.1') / 'path' 134 | assert str(proxy) == 'Proxy(https://proxies/... )' 135 | 136 | 137 | def test_singleton(): 138 | @clients.singleton('http://localhost/') 139 | class custom_api(clients.Resource): 140 | pass # custom methods 141 | 142 | assert isinstance(custom_api, clients.Resource) 143 | assert custom_api.url == 'http://localhost/' 144 | --------------------------------------------------------------------------------