├── .git-blame-ignore-revs ├── .git_hooks_pre-commit ├── .github ├── FUNDING.yml └── workflows │ ├── main.yml │ └── stale.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── MANIFEST.in ├── README.rst ├── pyproject.toml ├── requests_futures ├── __init__.py └── sessions.py ├── requirements-dev.txt ├── requirements.txt ├── script ├── bootstrap ├── cibuild ├── cibuild-setup-py ├── coverage ├── format ├── lint ├── release ├── test └── update-requirements ├── setup.py └── tests └── test_requests_futures.py /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # Commit that added in black formatting support 2 | 7690e187ffb1a801c35e562c093cd1c2592fbd63 3 | -------------------------------------------------------------------------------- /.git_hooks_pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | HOOKS=$(dirname "$0") 6 | GIT=$(dirname "$HOOKS") 7 | ROOT=$(dirname "$GIT") 8 | 9 | . "$ROOT/env/bin/activate" 10 | "$ROOT/script/lint" 11 | "$ROOT/script/format" --check --quiet || (echo "Formatting check failed, run ./script/format" && exit 1) 12 | "$ROOT/script/coverage" 13 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [ross] 2 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: requests-futures 2 | on: [pull_request] 3 | 4 | jobs: 5 | ci: 6 | runs-on: ubuntu-latest 7 | strategy: 8 | fail-fast: false 9 | matrix: 10 | # Tested versions based on dates in https://devguide.python.org/versions/#versions 11 | python-version: ['3.9', '3.10', '3.11', '3.12', "3.13"] 12 | steps: 13 | - uses: actions/checkout@master 14 | - name: Setup python 15 | uses: actions/setup-python@v4 16 | with: 17 | python-version: ${{ matrix.python-version }} 18 | architecture: x64 19 | - name: Install dependencies 20 | run: | 21 | python -m pip install --upgrade pip 22 | pip install -r requirements.txt 23 | pip install virtualenv 24 | - name: CI Build 25 | run: | 26 | ./script/cibuild 27 | setup-py: 28 | runs-on: ubuntu-latest 29 | steps: 30 | - uses: actions/checkout@v4 31 | - name: Setup python 32 | uses: actions/setup-python@v4 33 | with: 34 | python-version: '3.13' 35 | architecture: x64 36 | - name: CI setup.py 37 | run: | 38 | ./script/cibuild-setup-py 39 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: 'Close stale issues and PRs' 2 | on: 3 | schedule: 4 | - cron: '42 4 * * *' 5 | jobs: 6 | stale: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/stale@v4 10 | with: 11 | stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 7 days.' 12 | days-before-stale: 90 13 | days-before-close: 7 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info/ 2 | *.pyc 3 | .coverage* 4 | /build/ 5 | build/ 6 | coverage.xml 7 | dist/ 8 | env/ 9 | htmlcov/ 10 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## v1.0.2 - 2024-11-15 - Helps if you have the address right 2 | 3 | * Correct setup.py email addr 4 | 5 | ## v1.0.1 - 2023-06-19 - The first one in the CHANGELOG 6 | 7 | * Add pytest.mark.network to test cases 8 | * pyproject.toml config for black, isort, and pytest 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2013 Ross McFarland 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst LICENSE test_requests_futures.py requirements.txt 2 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Asynchronous Python HTTP Requests for Humans 2 | ============================================ 3 | 4 | .. image:: https://travis-ci.org/ross/requests-futures.svg?branch=master 5 | :target: https://travis-ci.org/ross/requests-futures 6 | 7 | Small add-on for the python requests_ http library. Makes use of python 3.2's 8 | `concurrent.futures`_ or the backport_ for prior versions of python. 9 | 10 | The additional API and changes are minimal and strives to avoid surprises. 11 | 12 | The following synchronous code: 13 | 14 | .. code-block:: python 15 | 16 | from requests import Session 17 | 18 | session = Session() 19 | # first requests starts and blocks until finished 20 | response_one = session.get('http://httpbin.org/get') 21 | # second request starts once first is finished 22 | response_two = session.get('http://httpbin.org/get?foo=bar') 23 | # both requests are complete 24 | print('response one status: {0}'.format(response_one.status_code)) 25 | print(response_one.content) 26 | print('response two status: {0}'.format(response_two.status_code)) 27 | print(response_two.content) 28 | 29 | Can be translated to make use of futures, and thus be asynchronous by creating 30 | a FuturesSession and catching the returned Future in place of Response. The 31 | Response can be retrieved by calling the result method on the Future: 32 | 33 | .. code-block:: python 34 | 35 | from requests_futures.sessions import FuturesSession 36 | 37 | session = FuturesSession() 38 | # first request is started in background 39 | future_one = session.get('http://httpbin.org/get') 40 | # second requests is started immediately 41 | future_two = session.get('http://httpbin.org/get?foo=bar') 42 | # wait for the first request to complete, if it hasn't already 43 | response_one = future_one.result() 44 | print('response one status: {0}'.format(response_one.status_code)) 45 | print(response_one.content) 46 | # wait for the second request to complete, if it hasn't already 47 | response_two = future_two.result() 48 | print('response two status: {0}'.format(response_two.status_code)) 49 | print(response_two.content) 50 | 51 | By default a ThreadPoolExecutor is created with 8 workers. If you would like to 52 | adjust that value or share a executor across multiple sessions you can provide 53 | one to the FuturesSession constructor. 54 | 55 | .. code-block:: python 56 | 57 | from concurrent.futures import ThreadPoolExecutor 58 | from requests_futures.sessions import FuturesSession 59 | 60 | session = FuturesSession(executor=ThreadPoolExecutor(max_workers=10)) 61 | # ... 62 | 63 | As a shortcut in case of just increasing workers number you can pass 64 | `max_workers` straight to the `FuturesSession` constructor: 65 | 66 | .. code-block:: python 67 | 68 | from requests_futures.sessions import FuturesSession 69 | session = FuturesSession(max_workers=10) 70 | 71 | FutureSession will use an existing session object if supplied: 72 | 73 | .. code-block:: python 74 | 75 | from requests import session 76 | from requests_futures.sessions import FuturesSession 77 | my_session = session() 78 | future_session = FuturesSession(session=my_session) 79 | 80 | That's it. The api of requests.Session is preserved without any modifications 81 | beyond returning a Future rather than Response. As with all futures exceptions 82 | are shifted (thrown) to the future.result() call so try/except blocks should be 83 | moved there. 84 | 85 | 86 | Tying extra information to the request/response 87 | =============================================== 88 | 89 | The most common piece of information needed is the URL of the request. This can 90 | be accessed without any extra steps using the `request` property of the 91 | response object. 92 | 93 | .. code-block:: python 94 | 95 | from concurrent.futures import as_completed 96 | from pprint import pprint 97 | from requests_futures.sessions import FuturesSession 98 | 99 | session = FuturesSession() 100 | 101 | futures=[session.get(f'http://httpbin.org/get?{i}') for i in range(3)] 102 | 103 | for future in as_completed(futures): 104 | resp = future.result() 105 | pprint({ 106 | 'url': resp.request.url, 107 | 'content': resp.json(), 108 | }) 109 | 110 | There are situations in which you may want to tie additional information to a 111 | request/response. There are a number of ways to go about this, the simplest is 112 | to attach additional information to the future object itself. 113 | 114 | .. code-block:: python 115 | 116 | from concurrent.futures import as_completed 117 | from pprint import pprint 118 | from requests_futures.sessions import FuturesSession 119 | 120 | session = FuturesSession() 121 | 122 | futures=[] 123 | for i in range(3): 124 | future = session.get('http://httpbin.org/get') 125 | future.i = i 126 | futures.append(future) 127 | 128 | for future in as_completed(futures): 129 | resp = future.result() 130 | pprint({ 131 | 'i': future.i, 132 | 'content': resp.json(), 133 | }) 134 | 135 | Canceling queued requests (a.k.a cleaning up after yourself) 136 | ============================================================ 137 | 138 | If you know that you won't be needing any additional responses from futures that 139 | haven't yet resolved, it's a good idea to cancel those requests. You can do this 140 | by using the session as a context manager: 141 | 142 | .. code-block:: python 143 | 144 | from requests_futures.sessions import FuturesSession 145 | with FuturesSession(max_workers=1) as session: 146 | future = session.get('https://httpbin.org/get') 147 | future2 = session.get('https://httpbin.org/delay/10') 148 | future3 = session.get('https://httpbin.org/delay/10') 149 | response = future.result() 150 | 151 | In this example, the second or third request will be skipped, saving time and 152 | resources that would otherwise be wasted. 153 | 154 | Iterating over a list of requests responses 155 | =========================================== 156 | 157 | Without preserving the requests order: 158 | 159 | .. code-block:: python 160 | 161 | from concurrent.futures import as_completed 162 | from requests_futures.sessions import FuturesSession 163 | with FuturesSession() as session: 164 | futures = [session.get('https://httpbin.org/delay/{}'.format(i % 3)) for i in range(10)] 165 | for future in as_completed(futures): 166 | resp = future.result() 167 | print(resp.json()['url']) 168 | 169 | Working in the Background 170 | ========================= 171 | 172 | Additional processing can be done in the background using requests's hooks_ 173 | functionality. This can be useful for shifting work out of the foreground, for 174 | a simple example take json parsing. 175 | 176 | .. code-block:: python 177 | 178 | from pprint import pprint 179 | from requests_futures.sessions import FuturesSession 180 | 181 | session = FuturesSession() 182 | 183 | def response_hook(resp, *args, **kwargs): 184 | # parse the json storing the result on the response object 185 | resp.data = resp.json() 186 | 187 | future = session.get('http://httpbin.org/get', hooks={ 188 | 'response': response_hook, 189 | }) 190 | # do some other stuff, send some more requests while this one works 191 | response = future.result() 192 | print('response status {0}'.format(response.status_code)) 193 | # data will have been attached to the response object in the background 194 | pprint(response.data) 195 | 196 | Hooks can also be applied to the session. 197 | 198 | .. code-block:: python 199 | 200 | from pprint import pprint 201 | from requests_futures.sessions import FuturesSession 202 | 203 | def response_hook(resp, *args, **kwargs): 204 | # parse the json storing the result on the response object 205 | resp.data = resp.json() 206 | 207 | session = FuturesSession() 208 | session.hooks['response'] = response_hook 209 | 210 | future = session.get('http://httpbin.org/get') 211 | # do some other stuff, send some more requests while this one works 212 | response = future.result() 213 | print('response status {0}'.format(response.status_code)) 214 | # data will have been attached to the response object in the background 215 | pprint(response.data) pprint(response.data) 216 | 217 | A more advanced example that adds an `elapsed` property to all requests. 218 | 219 | .. code-block:: python 220 | 221 | from pprint import pprint 222 | from requests_futures.sessions import FuturesSession 223 | from time import time 224 | 225 | 226 | class ElapsedFuturesSession(FuturesSession): 227 | 228 | def request(self, method, url, hooks=None, *args, **kwargs): 229 | start = time() 230 | if hooks is None: 231 | hooks = {} 232 | 233 | def timing(r, *args, **kwargs): 234 | r.elapsed = time() - start 235 | 236 | try: 237 | if isinstance(hooks['response'], (list, tuple)): 238 | # needs to be first so we don't time other hooks execution 239 | hooks['response'].insert(0, timing) 240 | else: 241 | hooks['response'] = [timing, hooks['response']] 242 | except KeyError: 243 | hooks['response'] = timing 244 | 245 | return super(ElapsedFuturesSession, self) \ 246 | .request(method, url, hooks=hooks, *args, **kwargs) 247 | 248 | 249 | 250 | session = ElapsedFuturesSession() 251 | future = session.get('http://httpbin.org/get') 252 | # do some other stuff, send some more requests while this one works 253 | response = future.result() 254 | print('response status {0}'.format(response.status_code)) 255 | print('response elapsed {0}'.format(response.elapsed)) 256 | 257 | Using ProcessPoolExecutor 258 | ========================= 259 | 260 | Similarly to `ThreadPoolExecutor`, it is possible to use an instance of 261 | `ProcessPoolExecutor`. As the name suggest, the requests will be executed 262 | concurrently in separate processes rather than threads. 263 | 264 | .. code-block:: python 265 | 266 | from concurrent.futures import ProcessPoolExecutor 267 | from requests_futures.sessions import FuturesSession 268 | 269 | session = FuturesSession(executor=ProcessPoolExecutor(max_workers=10)) 270 | # ... use as before 271 | 272 | .. HINT:: 273 | Using the `ProcessPoolExecutor` is useful, in cases where memory 274 | usage per request is very high (large response) and cycling the interpreter 275 | is required to release memory back to OS. 276 | 277 | A base requirement of using `ProcessPoolExecutor` is that the `Session.request`, 278 | `FutureSession` all be pickle-able. 279 | 280 | This means that only Python 3.5 is fully supported, while Python versions 281 | 3.4 and above REQUIRE an existing `requests.Session` instance to be passed 282 | when initializing `FutureSession`. Python 2.X and < 3.4 are currently not 283 | supported. 284 | 285 | .. code-block:: python 286 | 287 | # Using python 3.4 288 | from concurrent.futures import ProcessPoolExecutor 289 | from requests import Session 290 | from requests_futures.sessions import FuturesSession 291 | 292 | session = FuturesSession(executor=ProcessPoolExecutor(max_workers=10), 293 | session=Session()) 294 | # ... use as before 295 | 296 | In case pickling fails, an exception is raised pointing to this documentation. 297 | 298 | .. code-block:: python 299 | 300 | # Using python 2.7 301 | from concurrent.futures import ProcessPoolExecutor 302 | from requests import Session 303 | from requests_futures.sessions import FuturesSession 304 | 305 | session = FuturesSession(executor=ProcessPoolExecutor(max_workers=10), 306 | session=Session()) 307 | Traceback (most recent call last): 308 | ... 309 | RuntimeError: Cannot pickle function. Refer to documentation: https://github.com/ross/requests-futures/#using-processpoolexecutor 310 | 311 | .. IMPORTANT:: 312 | * Python >= 3.4 required 313 | * A session instance is required when using Python < 3.5 314 | * If sub-classing `FuturesSession` it must be importable (module global) 315 | 316 | Installation 317 | ============ 318 | 319 | pip install requests-futures 320 | 321 | .. _`requests`: https://github.com/kennethreitz/requests 322 | .. _`concurrent.futures`: http://docs.python.org/dev/library/concurrent.futures.html 323 | .. _backport: https://pypi.python.org/pypi/futures 324 | .. _hooks: http://docs.python-requests.org/en/master/user/advanced/#event-hooks 325 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length=80 3 | skip-string-normalization=true 4 | skip-magic-trailing-comma=true 5 | 6 | [tool.isort] 7 | profile = "black" 8 | known_first_party="requests_futures" 9 | line_length=80 10 | 11 | [tool.pytest.ini_options] 12 | pythonpath = "." 13 | -------------------------------------------------------------------------------- /requests_futures/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Requests Futures 4 | 5 | """ 6 | async requests HTTP library 7 | ~~~~~~~~~~~~~~~~~~~~~ 8 | 9 | 10 | """ 11 | 12 | import logging 13 | 14 | __title__ = 'requests-futures' 15 | __version__ = '1.0.2' 16 | __build__ = 0x000000 17 | __author__ = 'Ross McFarland' 18 | __license__ = 'Apache 2.0' 19 | __copyright__ = 'Copyright 2013 Ross McFarland' 20 | 21 | # Set default logging handler to avoid "No handler found" warnings. 22 | try: # Python 2.7+ 23 | from logging import NullHandler 24 | except ImportError: 25 | 26 | class NullHandler(logging.Handler): 27 | def emit(self, record): 28 | pass 29 | 30 | 31 | logging.getLogger(__name__).addHandler(NullHandler()) 32 | -------------------------------------------------------------------------------- /requests_futures/sessions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | requests_futures 4 | ~~~~~~~~~~~~~~~~ 5 | 6 | This module provides a small add-on for the requests http library. It makes use 7 | of python 3.3's concurrent.futures or the futures backport for previous 8 | releases of python. 9 | 10 | from requests_futures.sessions import FuturesSession 11 | 12 | session = FuturesSession() 13 | # request is run in the background 14 | future = session.get('http://httpbin.org/get') 15 | # ... do other stuff ... 16 | # wait for the request to complete, if it hasn't already 17 | response = future.result() 18 | print('response status: {0}'.format(response.status_code)) 19 | print(response.content) 20 | 21 | """ 22 | from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor 23 | from functools import partial 24 | from logging import getLogger 25 | from pickle import PickleError, dumps 26 | 27 | from requests import Session 28 | from requests.adapters import DEFAULT_POOLSIZE, HTTPAdapter 29 | 30 | 31 | def wrap(self, sup, background_callback, *args_, **kwargs_): 32 | """A global top-level is required for ProcessPoolExecutor""" 33 | resp = sup(*args_, **kwargs_) 34 | return background_callback(self, resp) or resp 35 | 36 | 37 | PICKLE_ERROR = ( 38 | 'Cannot pickle function. Refer to documentation: https://' 39 | 'github.com/ross/requests-futures/#using-processpoolexecutor' 40 | ) 41 | 42 | 43 | class FuturesSession(Session): 44 | def __init__( 45 | self, 46 | executor=None, 47 | max_workers=8, 48 | session=None, 49 | adapter_kwargs=None, 50 | *args, 51 | **kwargs 52 | ): 53 | """Creates a FuturesSession 54 | 55 | Notes 56 | ~~~~~ 57 | * `ProcessPoolExecutor` may be used with Python > 3.4; 58 | see README for more information. 59 | 60 | * If you provide both `executor` and `max_workers`, the latter is 61 | ignored and provided executor is used as is. 62 | """ 63 | _adapter_kwargs = {} 64 | super(FuturesSession, self).__init__(*args, **kwargs) 65 | self._owned_executor = executor is None 66 | if executor is None: 67 | executor = ThreadPoolExecutor(max_workers=max_workers) 68 | # set connection pool size equal to max_workers if needed 69 | if max_workers > DEFAULT_POOLSIZE: 70 | _adapter_kwargs.update( 71 | { 72 | 'pool_connections': max_workers, 73 | 'pool_maxsize': max_workers, 74 | } 75 | ) 76 | 77 | _adapter_kwargs.update(adapter_kwargs or {}) 78 | 79 | if _adapter_kwargs: 80 | self.mount('https://', HTTPAdapter(**_adapter_kwargs)) 81 | self.mount('http://', HTTPAdapter(**_adapter_kwargs)) 82 | 83 | self.executor = executor 84 | self.session = session 85 | 86 | def request(self, *args, **kwargs): 87 | """Maintains the existing api for Session.request. 88 | 89 | Used by all of the higher level methods, e.g. Session.get. 90 | 91 | The background_callback param allows you to do some processing on the 92 | response in the background, e.g. call resp.json() so that json parsing 93 | happens in the background thread. 94 | 95 | :rtype : concurrent.futures.Future 96 | """ 97 | if self.session: 98 | func = self.session.request 99 | else: 100 | # avoid calling super to not break pickled method 101 | func = partial(Session.request, self) 102 | 103 | background_callback = kwargs.pop('background_callback', None) 104 | if background_callback: 105 | logger = getLogger(self.__class__.__name__) 106 | logger.warning( 107 | '`background_callback` is deprecated and will be ' 108 | 'removed in 1.0, use `hooks` instead' 109 | ) 110 | func = partial(wrap, self, func, background_callback) 111 | 112 | if isinstance(self.executor, ProcessPoolExecutor): 113 | # verify function can be pickled 114 | try: 115 | dumps(func) 116 | except (TypeError, PickleError): 117 | raise RuntimeError(PICKLE_ERROR) 118 | 119 | return self.executor.submit(func, *args, **kwargs) 120 | 121 | def close(self): 122 | super(FuturesSession, self).close() 123 | if self._owned_executor: 124 | self.executor.shutdown() 125 | 126 | def get(self, url, **kwargs): 127 | r""" 128 | Sends a GET request. Returns :class:`Future` object. 129 | 130 | :param url: URL for the new :class:`Request` object. 131 | :param \*\*kwargs: Optional arguments that ``request`` takes. 132 | :rtype : concurrent.futures.Future 133 | """ 134 | return super(FuturesSession, self).get(url, **kwargs) 135 | 136 | def options(self, url, **kwargs): 137 | r"""Sends a OPTIONS request. Returns :class:`Future` object. 138 | 139 | :param url: URL for the new :class:`Request` object. 140 | :param \*\*kwargs: Optional arguments that ``request`` takes. 141 | :rtype : concurrent.futures.Future 142 | """ 143 | return super(FuturesSession, self).options(url, **kwargs) 144 | 145 | def head(self, url, **kwargs): 146 | r"""Sends a HEAD request. Returns :class:`Future` object. 147 | 148 | :param url: URL for the new :class:`Request` object. 149 | :param \*\*kwargs: Optional arguments that ``request`` takes. 150 | :rtype : concurrent.futures.Future 151 | """ 152 | return super(FuturesSession, self).head(url, **kwargs) 153 | 154 | def post(self, url, data=None, json=None, **kwargs): 155 | r"""Sends a POST request. Returns :class:`Future` object. 156 | 157 | :param url: URL for the new :class:`Request` object. 158 | :param data: (optional) Dictionary, list of tuples, bytes, or file-like 159 | object to send in the body of the :class:`Request`. 160 | :param json: (optional) json to send in the body of the :class:`Request`. 161 | :param \*\*kwargs: Optional arguments that ``request`` takes. 162 | :rtype : concurrent.futures.Future 163 | """ 164 | return super(FuturesSession, self).post( 165 | url, data=data, json=json, **kwargs 166 | ) 167 | 168 | def put(self, url, data=None, **kwargs): 169 | r"""Sends a PUT request. Returns :class:`Future` object. 170 | 171 | :param url: URL for the new :class:`Request` object. 172 | :param data: (optional) Dictionary, list of tuples, bytes, or file-like 173 | object to send in the body of the :class:`Request`. 174 | :param \*\*kwargs: Optional arguments that ``request`` takes. 175 | :rtype : concurrent.futures.Future 176 | """ 177 | return super(FuturesSession, self).put(url, data=data, **kwargs) 178 | 179 | def patch(self, url, data=None, **kwargs): 180 | r"""Sends a PATCH request. Returns :class:`Future` object. 181 | 182 | :param url: URL for the new :class:`Request` object. 183 | :param data: (optional) Dictionary, list of tuples, bytes, or file-like 184 | object to send in the body of the :class:`Request`. 185 | :param \*\*kwargs: Optional arguments that ``request`` takes. 186 | :rtype : concurrent.futures.Future 187 | """ 188 | return super(FuturesSession, self).patch(url, data=data, **kwargs) 189 | 190 | def delete(self, url, **kwargs): 191 | r"""Sends a DELETE request. Returns :class:`Future` object. 192 | 193 | :param url: URL for the new :class:`Request` object. 194 | :param \*\*kwargs: Optional arguments that ``request`` takes. 195 | :rtype : concurrent.futures.Future 196 | """ 197 | return super(FuturesSession, self).delete(url, **kwargs) 198 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | Flask==3.1.0 2 | Jinja2==3.1.6 3 | MarkupSafe==3.0.2 4 | PyYAML==6.0.2 5 | Pygments==2.19.1 6 | Werkzeug==3.1.3 7 | attrs==25.3.0 8 | black==24.10.0 9 | blinker==1.9.0 10 | brotlicffi==1.1.0.0 11 | build==1.2.2.post1 12 | cffi==1.17.1 13 | click==8.1.8 14 | coverage==7.8.0 15 | decorator==5.2.1 16 | docutils==0.20.1 17 | flasgger==0.9.7.1 18 | greenlet==2.0.2; python_version<'3.12' 19 | greenlet==3.2.1; python_version>='3.12.0rc0' 20 | httpbin==0.10.2 21 | id==1.5.0 22 | iniconfig==2.1.0 23 | isort==6.0.1 24 | itsdangerous==2.2.0 25 | jaraco.classes==3.4.0 26 | jaraco.context==6.0.1 27 | jaraco.functools==4.1.0 28 | jsonschema-specifications==2025.4.1 29 | jsonschema==4.23.0 30 | keyring==25.6.0 31 | markdown-it-py==3.0.0 32 | mdurl==0.1.2 33 | mistune==3.1.3 34 | more-itertools==10.7.0 35 | mypy_extensions==1.1.0 36 | nh3==0.2.21 37 | packaging==25.0 38 | pathspec==0.12.1 39 | platformdirs==4.3.7 40 | pluggy==1.5.0 41 | pycparser==2.22 42 | pyflakes==3.3.2 43 | pyproject_hooks==1.2.0 44 | pytest-cov==6.1.1 45 | pytest-httpbin==2.1.0 46 | pytest==8.3.5 47 | readme_renderer==43.0 48 | referencing==0.36.2 49 | requests-toolbelt==1.0.0 50 | rfc3986==2.0.0 51 | rich==14.0.0 52 | rpds-py==0.24.0 53 | six==1.17.0 54 | twine==6.1.0 55 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | certifi==2025.4.26 2 | charset-normalizer==3.4.1 3 | idna==3.10 4 | requests==2.32.3 5 | urllib3==2.4.0 6 | -------------------------------------------------------------------------------- /script/bootstrap: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Usage: script/bootstrap 3 | # Ensures all dependencies are installed locally. 4 | 5 | set -e 6 | 7 | cd "$(dirname "$0")"/.. 8 | ROOT=$(pwd) 9 | 10 | if [ -z "$VENV_NAME" ]; then 11 | VENV_NAME="env" 12 | fi 13 | 14 | if [ ! -d "$VENV_NAME" ]; then 15 | if [ -z "$VENV_PYTHON" ]; then 16 | VENV_PYTHON=$(command -v python3) 17 | fi 18 | "$VENV_PYTHON" -m venv "$VENV_NAME" 19 | fi 20 | . "$VENV_NAME/bin/activate" 21 | 22 | # We're in the venv now, so use the first Python in $PATH. In particular, don't 23 | # use $VENV_PYTHON - that's the Python that *created* the venv, not the python 24 | # *inside* the venv 25 | python -m pip install -U 'pip>=10.0.1' 26 | python -m pip install -r requirements.txt 27 | 28 | if [ "$ENV" != "production" ]; then 29 | python -m pip install -r requirements-dev.txt 30 | fi 31 | 32 | if [ -d ".git" ]; then 33 | if [ -f ".git-blame-ignore-revs" ]; then 34 | echo "" 35 | echo "Setting blame.ignoreRevsFile to .git-blame-ingore-revs" 36 | git config --local blame.ignoreRevsFile .git-blame-ignore-revs 37 | fi 38 | if [ ! -L ".git/hooks/pre-commit" ]; then 39 | echo "" 40 | echo "Installing pre-commit hook" 41 | ln -s "$ROOT/.git_hooks_pre-commit" ".git/hooks/pre-commit" 42 | fi 43 | fi 44 | 45 | echo "" 46 | echo "Run source env/bin/activate to get your shell in to the virtualenv" 47 | echo "See README.md for more information." 48 | echo "" 49 | -------------------------------------------------------------------------------- /script/cibuild: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")/.." 6 | 7 | echo "## bootstrap ###################################################################" 8 | script/bootstrap 9 | 10 | if [ -z "$VENV_NAME" ]; then 11 | VENV_NAME="env" 12 | fi 13 | 14 | . "$VENV_NAME/bin/activate" 15 | 16 | echo "## environment & versions ######################################################" 17 | python --version 18 | pip --version 19 | echo "## modules: " 20 | pip freeze 21 | echo "## clean up ####################################################################" 22 | find requests_futures tests -name "*.pyc" -exec rm {} \; 23 | rm -f *.pyc 24 | echo "## begin #######################################################################" 25 | # For now it's just lint... 26 | echo "## lint ########################################################################" 27 | script/lint 28 | echo "## formatting ##################################################################" 29 | script/format --check || (echo "Formatting check failed, run ./script/format" && exit 1) 30 | echo "## tests/coverage ##############################################################" 31 | script/coverage 32 | echo "## complete ####################################################################" 33 | -------------------------------------------------------------------------------- /script/cibuild-setup-py: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | cd "$(dirname "$0")/.." 5 | 6 | echo "## create test venv ############################################################" 7 | TMP_DIR=$(mktemp -d -t ci-XXXXXXXXXX) 8 | python3 -m venv $TMP_DIR 9 | . "$TMP_DIR/bin/activate" 10 | pip install build setuptools 11 | echo "## environment & versions ######################################################" 12 | python --version 13 | pip --version 14 | echo "## validate setup.py build #####################################################" 15 | python -m build --sdist --wheel 16 | echo "## validate wheel install ###################################################" 17 | pip install dist/*$VERSION*.whl 18 | echo "## validate tests can run against installed code ###############################" 19 | pip install pytest pytest-httpbin Werkzeug 20 | pytest 21 | echo "## complete ####################################################################" 22 | -------------------------------------------------------------------------------- /script/coverage: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | cd "$(dirname "$0")/.." 5 | 6 | if [ -z "$VENV_NAME" ]; then 7 | VENV_NAME="env" 8 | fi 9 | 10 | ACTIVATE="$VENV_NAME/bin/activate" 11 | if [ ! -f "$ACTIVATE" ]; then 12 | echo "$ACTIVATE does not exist, run ./script/bootstrap" >&2 13 | exit 1 14 | fi 15 | . "$ACTIVATE" 16 | 17 | # Just to be sure/safe 18 | export AWS_ACCESS_KEY_ID= 19 | export AWS_SECRET_ACCESS_KEY= 20 | export CLOUDFLARE_EMAIL= 21 | export CLOUDFLARE_TOKEN= 22 | export DNSIMPLE_ACCOUNT= 23 | export DNSIMPLE_TOKEN= 24 | export DYN_CUSTOMER= 25 | export DYN_PASSWORD= 26 | export DYN_USERNAME= 27 | export GOOGLE_APPLICATION_CREDENTIALS= 28 | export ARM_CLIENT_ID= 29 | export ARM_CLIENT_SECRET= 30 | export ARM_TENANT_ID= 31 | export ARM_SUBSCRIPTION_ID= 32 | 33 | SOURCE_DIR="requests_futures/" 34 | 35 | # Don't allow disabling coverage 36 | grep -r -I --line-number "# pragma: +no.*cover" $SOURCE_DIR && { 37 | echo "Code coverage should not be disabled" 38 | exit 1 39 | } 40 | 41 | export PYTHONPATH=.:$PYTHONPATH 42 | 43 | # TODO: 44 | # --cov-fail-under=100 \ 45 | pytest \ 46 | --cov-reset \ 47 | --cov=$SOURCE_DIR \ 48 | --cov-report=html \ 49 | --cov-report=xml \ 50 | --cov-report=term \ 51 | --cov-branch \ 52 | "$@" 53 | -------------------------------------------------------------------------------- /script/format: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | SOURCES=$(find *.py requests_futures tests -name "*.py") 6 | 7 | . env/bin/activate 8 | 9 | isort "$@" $SOURCES 10 | black "$@" $SOURCES 11 | -------------------------------------------------------------------------------- /script/lint: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | cd "$(dirname "$0")/.." 5 | ROOT=$(pwd) 6 | 7 | if [ -z "$VENV_NAME" ]; then 8 | VENV_NAME="env" 9 | fi 10 | 11 | ACTIVATE="$VENV_NAME/bin/activate" 12 | if [ ! -f "$ACTIVATE" ]; then 13 | echo "$ACTIVATE does not exist, run ./script/bootstrap" >&2 14 | exit 1 15 | fi 16 | . "$ACTIVATE" 17 | 18 | SOURCES="*.py requests_futures/*.py tests/*.py" 19 | 20 | pyflakes $SOURCES 21 | -------------------------------------------------------------------------------- /script/release: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")"/.. 6 | ROOT=$(pwd) 7 | 8 | VERSION="$(grep __version__ "$ROOT/requests_futures/__init__.py" | sed -e "s/.* = '//" -e "s/'$//")" 9 | 10 | git tag -s "v$VERSION" -m "Release $VERSION" 11 | git push origin "v$VERSION" 12 | echo "Tagged and pushed v$VERSION" 13 | python -m build --sdist --wheel 14 | twine check dist/*$VERSION.tar.gz dist/*$VERSION*.whl 15 | twine upload dist/*$VERSION.tar.gz dist/*$VERSION*.whl 16 | echo "Uploaded $VERSION" 17 | -------------------------------------------------------------------------------- /script/test: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | cd "$(dirname "$0")/.." 5 | 6 | if [ -z "$VENV_NAME" ]; then 7 | VENV_NAME="env" 8 | fi 9 | 10 | ACTIVATE="$VENV_NAME/bin/activate" 11 | if [ ! -f "$ACTIVATE" ]; then 12 | echo "$ACTIVATE does not exist, run ./script/bootstrap" >&2 13 | exit 1 14 | fi 15 | . "$ACTIVATE" 16 | 17 | # Just to be sure/safe 18 | export AWS_ACCESS_KEY_ID= 19 | export AWS_SECRET_ACCESS_KEY= 20 | export CLOUDFLARE_EMAIL= 21 | export CLOUDFLARE_TOKEN= 22 | export DNSIMPLE_ACCOUNT= 23 | export DNSIMPLE_TOKEN= 24 | export DYN_CUSTOMER= 25 | export DYN_PASSWORD= 26 | export DYN_USERNAME= 27 | export GOOGLE_APPLICATION_CREDENTIALS= 28 | export ARM_CLIENT_ID= 29 | export ARM_CLIENT_SECRET= 30 | export ARM_TENANT_ID= 31 | export ARM_SUBSCRIPTION_ID= 32 | 33 | export PYTHONPATH=.:$PYTHONPATH 34 | 35 | pytest "$@" 36 | -------------------------------------------------------------------------------- /script/update-requirements: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import re 4 | from os.path import join 5 | from subprocess import check_call, check_output 6 | from tempfile import TemporaryDirectory 7 | 8 | 9 | def print_packages(packages, heading): 10 | print(f'{heading}:') 11 | print(' ', end='') 12 | print('\n '.join(packages)) 13 | 14 | 15 | # would be nice if there was a cleaner way to get this, but I've not found a 16 | # more reliable one. 17 | with open('setup.py') as fh: 18 | match = re.search(r"name='(?P[\w-]+)',", fh.read()) 19 | if not match: 20 | raise Exception('failed to determine our package name') 21 | our_package_name = match.group('pkg') 22 | print(f'our_package_name: {our_package_name}') 23 | 24 | with TemporaryDirectory() as tmpdir: 25 | check_call(['python3', '-m', 'venv', tmpdir]) 26 | 27 | # base needs 28 | check_call([join(tmpdir, 'bin', 'pip'), 'install', '.']) 29 | frozen = check_output([join(tmpdir, 'bin', 'pip'), 'freeze']) 30 | frozen = set(frozen.decode('utf-8').strip().split('\n')) 31 | 32 | # dev additions 33 | check_call([join(tmpdir, 'bin', 'pip'), 'install', '.[dev]']) 34 | dev_frozen = check_output([join(tmpdir, 'bin', 'pip'), 'freeze']) 35 | dev_frozen = set(dev_frozen.decode('utf-8').strip().split('\n')) - frozen 36 | 37 | # pip installs the module itself along with deps so we need to get that out of 38 | # our list by finding the thing that was file installed during dev 39 | frozen = sorted([p for p in frozen if not p.startswith(our_package_name)]) 40 | dev_frozen = sorted( 41 | [p for p in dev_frozen if not p.startswith(our_package_name)] 42 | ) 43 | 44 | # special handling for greenlet until python 3.11 is gone due to it dropping 45 | # support for active versions early 46 | i = [i for i, r in enumerate(dev_frozen) if r.startswith('greenlet==')][0] 47 | dev_frozen = ( 48 | dev_frozen[:i] 49 | + [ 50 | "greenlet==2.0.2; python_version<'3.12'", 51 | f"{dev_frozen[i]}; python_version>='3.12.0rc0'", 52 | ] 53 | + dev_frozen[i + 1 :] 54 | ) 55 | 56 | print_packages(frozen, 'frozen') 57 | print_packages(dev_frozen, 'dev_frozen') 58 | 59 | with open('requirements.txt', 'w') as fh: 60 | fh.write('\n'.join(frozen)) 61 | fh.write('\n') 62 | 63 | with open('requirements-dev.txt', 'w') as fh: 64 | fh.write('\n'.join(dev_frozen)) 65 | fh.write('\n') 66 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import sys 5 | 6 | import requests_futures 7 | 8 | try: 9 | from setuptools import setup 10 | except ImportError: 11 | from distutils.core import setup 12 | 13 | if sys.argv[-1] == 'publish': 14 | os.system('python setup.py sdist upload') 15 | sys.exit() 16 | 17 | packages = ['requests_futures'] 18 | 19 | requires = ['requests>=1.2.0'] 20 | 21 | tests_require = ( 22 | 'greenlet<=2.0.2; python_version<"3.12"', 23 | 'greenlet>=3.0.0; python_version>="3.12.0rc0"', 24 | 'pytest>=6.2.5', 25 | 'pytest-cov>=3.0.0', 26 | 'pytest-httpbin>=2.0.0', 27 | 'Werkzeug>=3.0.6', 28 | ) 29 | 30 | setup( 31 | name='requests-futures', 32 | version=requests_futures.__version__, 33 | description='Asynchronous Python HTTP for Humans.', 34 | extras_require={ 35 | 'dev': tests_require 36 | + ( 37 | 'black>=24.3.0,<25.0.0', 38 | 'build>=0.7.0', 39 | # docutils 0.21.x bumped to >=3.9 and 3.8 is still active. we'll 40 | # have to clamp it down until we remove 3.8 41 | 'docutils<=0.20.1', 42 | 'isort>=5.11.4', 43 | 'pyflakes>=2.2.0', 44 | 'readme_renderer[rst]>=26.0', 45 | 'twine>=3.4.2', 46 | ) 47 | }, 48 | long_description=open('README.rst').read(), 49 | long_description_content_type='text/x-rst', 50 | author='Ross McFarland', 51 | author_email='rwmcfa1@gmail.com', 52 | packages=packages, 53 | package_dir={'requests_futures': 'requests_futures'}, 54 | package_data={'requests_futures': ['LICENSE', 'README.rst']}, 55 | include_package_data=True, 56 | install_requires=requires, 57 | setup_requires=['setuptools>=38.6.1'], 58 | license='Apache License v2', 59 | tests_require=tests_require, 60 | url='https://github.com/ross/requests-futures', 61 | zip_safe=False, 62 | classifiers=[ 63 | 'Development Status :: 5 - Production/Stable', 64 | 'Intended Audience :: Developers', 65 | 'Natural Language :: English', 66 | 'License :: OSI Approved :: Apache Software License', 67 | 'Programming Language :: Python', 68 | 'Programming Language :: Python :: 2.7', 69 | 'Programming Language :: Python :: 3', 70 | 'Programming Language :: Python :: 3.6', 71 | 'Programming Language :: Python :: 3.7', 72 | 'Programming Language :: Python :: 3.8', 73 | ], 74 | options={'bdist_wheel': {'universal': True}}, 75 | ) 76 | -------------------------------------------------------------------------------- /tests/test_requests_futures.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Tests for Requests.""" 5 | 6 | from concurrent.futures import Future, ProcessPoolExecutor 7 | from os import environ 8 | from sys import version_info 9 | 10 | try: 11 | from sys import pypy_version_info 12 | except ImportError: 13 | pypy_version_info = None 14 | import logging 15 | from unittest import TestCase, main, skipIf 16 | 17 | import pytest 18 | from requests import Response, session 19 | from requests.adapters import DEFAULT_POOLSIZE 20 | 21 | from requests_futures.sessions import FuturesSession 22 | 23 | HTTPBIN = environ.get('HTTPBIN_URL', 'https://nghttp2.org/httpbin/') 24 | logging.basicConfig(level=logging.DEBUG) 25 | logging.getLogger('urllib3.connectionpool').level = logging.WARNING 26 | logging.getLogger('FuturesSession').level = logging.ERROR 27 | 28 | 29 | @pytest.fixture(scope="class", autouse=True) 30 | def httpbin_on_class(request, httpbin): 31 | request.cls.httpbin = httpbin 32 | 33 | 34 | class RequestsTestCase(TestCase): 35 | def test_futures_session(self): 36 | # basic futures get 37 | sess = FuturesSession() 38 | future = sess.get(self.httpbin.join('get')) 39 | self.assertIsInstance(future, Future) 40 | resp = future.result() 41 | self.assertIsInstance(resp, Response) 42 | self.assertEqual(200, resp.status_code) 43 | 44 | # non-200, 404 45 | future = sess.get(self.httpbin.join('status/404')) 46 | resp = future.result() 47 | self.assertEqual(404, resp.status_code) 48 | 49 | def cb(s, r): 50 | self.assertIsInstance(s, FuturesSession) 51 | self.assertIsInstance(r, Response) 52 | # add the parsed json data to the response 53 | r.data = r.json() 54 | 55 | future = sess.get(self.httpbin.join('get'), background_callback=cb) 56 | # this should block until complete 57 | resp = future.result() 58 | self.assertEqual(200, resp.status_code) 59 | # make sure the callback was invoked 60 | self.assertTrue(hasattr(resp, 'data')) 61 | 62 | def rasing_cb(s, r): 63 | raise Exception('boom') 64 | 65 | future = sess.get( 66 | self.httpbin.join('get'), background_callback=rasing_cb 67 | ) 68 | with self.assertRaises(Exception) as cm: 69 | resp = future.result() 70 | self.assertEqual('boom', cm.exception.args[0]) 71 | 72 | def test_supplied_session(self): 73 | """Tests the `session` keyword argument.""" 74 | requests_session = session() 75 | requests_session.headers['Foo'] = 'bar' 76 | sess = FuturesSession(session=requests_session) 77 | future = sess.get(self.httpbin.join('headers')) 78 | self.assertIsInstance(future, Future) 79 | resp = future.result() 80 | self.assertIsInstance(resp, Response) 81 | self.assertEqual(200, resp.status_code) 82 | self.assertEqual(resp.json()['headers']['Foo'], 'bar') 83 | 84 | def test_max_workers(self): 85 | """Tests the `max_workers` shortcut.""" 86 | from concurrent.futures import ThreadPoolExecutor 87 | 88 | session = FuturesSession() 89 | self.assertEqual(session.executor._max_workers, 8) 90 | session = FuturesSession(max_workers=5) 91 | self.assertEqual(session.executor._max_workers, 5) 92 | session = FuturesSession(executor=ThreadPoolExecutor(max_workers=10)) 93 | self.assertEqual(session.executor._max_workers, 10) 94 | session = FuturesSession( 95 | executor=ThreadPoolExecutor(max_workers=10), max_workers=5 96 | ) 97 | self.assertEqual(session.executor._max_workers, 10) 98 | 99 | def test_adapter_kwargs(self): 100 | """Tests the `adapter_kwargs` shortcut.""" 101 | from concurrent.futures import ThreadPoolExecutor 102 | 103 | session = FuturesSession() 104 | self.assertFalse(session.get_adapter('http://')._pool_block) 105 | session = FuturesSession( 106 | max_workers=DEFAULT_POOLSIZE + 1, 107 | adapter_kwargs={'pool_block': True}, 108 | ) 109 | adapter = session.get_adapter('http://') 110 | self.assertTrue(adapter._pool_block) 111 | self.assertEqual(adapter._pool_connections, DEFAULT_POOLSIZE + 1) 112 | self.assertEqual(adapter._pool_maxsize, DEFAULT_POOLSIZE + 1) 113 | session = FuturesSession( 114 | executor=ThreadPoolExecutor(max_workers=10), 115 | adapter_kwargs={'pool_connections': 20}, 116 | ) 117 | self.assertEqual(session.get_adapter('http://')._pool_connections, 20) 118 | 119 | def test_redirect(self): 120 | """Tests for the ability to cleanly handle redirects.""" 121 | sess = FuturesSession() 122 | future = sess.get(self.httpbin.join('redirect-to?url=get')) 123 | self.assertIsInstance(future, Future) 124 | resp = future.result() 125 | self.assertIsInstance(resp, Response) 126 | self.assertEqual(200, resp.status_code) 127 | 128 | future = sess.get(self.httpbin.join('redirect-to?url=status/404')) 129 | resp = future.result() 130 | self.assertEqual(404, resp.status_code) 131 | 132 | def test_context(self): 133 | class FuturesSessionTestHelper(FuturesSession): 134 | def __init__(self, *args, **kwargs): 135 | super(FuturesSessionTestHelper, self).__init__(*args, **kwargs) 136 | self._exit_called = False 137 | 138 | def __exit__(self, *args, **kwargs): 139 | self._exit_called = True 140 | return super(FuturesSessionTestHelper, self).__exit__( 141 | *args, **kwargs 142 | ) 143 | 144 | passout = None 145 | with FuturesSessionTestHelper() as sess: 146 | passout = sess 147 | future = sess.get(self.httpbin.join('get')) 148 | self.assertIsInstance(future, Future) 149 | resp = future.result() 150 | self.assertIsInstance(resp, Response) 151 | self.assertEqual(200, resp.status_code) 152 | 153 | self.assertTrue(passout._exit_called) 154 | 155 | 156 | # << test process pool executor >> 157 | # see discussion https://github.com/ross/requests-futures/issues/11 158 | def global_cb_modify_response(s, r): 159 | """add the parsed json data to the response""" 160 | assert s, FuturesSession 161 | assert r, Response 162 | r.data = r.json() 163 | r.__attrs__.append('data') # required for pickling new attribute 164 | 165 | 166 | def global_cb_return_result(s, r): 167 | """simply return parsed json data""" 168 | assert s, FuturesSession 169 | assert r, Response 170 | return r.json() 171 | 172 | 173 | def global_rasing_cb(s, r): 174 | raise Exception('boom') 175 | 176 | 177 | # pickling instance method supported only from here 178 | unsupported_platform = version_info < (3, 4) and not pypy_version_info 179 | session_required = version_info < (3, 5) and not pypy_version_info 180 | 181 | 182 | @skipIf(unsupported_platform, 'not supported in python < 3.4') 183 | class RequestsProcessPoolTestCase(TestCase): 184 | def setUp(self): 185 | self.proc_executor = ProcessPoolExecutor(max_workers=2) 186 | self.session = session() 187 | 188 | @skipIf(session_required, 'not supported in python < 3.5') 189 | def test_futures_session(self): 190 | self._assert_futures_session() 191 | 192 | @skipIf(not session_required, 'fully supported on python >= 3.5') 193 | def test_exception_raised(self): 194 | with self.assertRaises(RuntimeError): 195 | self._assert_futures_session() 196 | 197 | def test_futures_existing_session(self): 198 | self.session.headers['Foo'] = 'bar' 199 | self._assert_futures_session(session=self.session) 200 | 201 | def _assert_futures_session(self, session=None): 202 | # basic futures get 203 | if session: 204 | sess = FuturesSession(executor=self.proc_executor, session=session) 205 | else: 206 | sess = FuturesSession(executor=self.proc_executor) 207 | 208 | future = sess.get(self.httpbin.join('get')) 209 | self.assertIsInstance(future, Future) 210 | resp = future.result() 211 | self.assertIsInstance(resp, Response) 212 | self.assertEqual(200, resp.status_code) 213 | 214 | # non-200, 404 215 | future = sess.get(self.httpbin.join('status/404')) 216 | resp = future.result() 217 | self.assertEqual(404, resp.status_code) 218 | 219 | future = sess.get( 220 | self.httpbin.join('get'), 221 | background_callback=global_cb_modify_response, 222 | ) 223 | # this should block until complete 224 | resp = future.result() 225 | if session: 226 | self.assertEqual(resp.json()['headers']['Foo'], 'bar') 227 | self.assertEqual(200, resp.status_code) 228 | # make sure the callback was invoked 229 | self.assertTrue(hasattr(resp, 'data')) 230 | 231 | future = sess.get( 232 | self.httpbin.join('get'), 233 | background_callback=global_cb_return_result, 234 | ) 235 | # this should block until complete 236 | resp = future.result() 237 | # make sure the callback was invoked 238 | self.assertIsInstance(resp, dict) 239 | 240 | future = sess.get( 241 | self.httpbin.join('get'), background_callback=global_rasing_cb 242 | ) 243 | with self.assertRaises(Exception) as cm: 244 | resp = future.result() 245 | self.assertEqual('boom', cm.exception.args[0]) 246 | 247 | # Tests for the ability to cleanly handle redirects 248 | future = sess.get(self.httpbin.join('redirect-to?url=get')) 249 | self.assertIsInstance(future, Future) 250 | resp = future.result() 251 | self.assertIsInstance(resp, Response) 252 | self.assertEqual(200, resp.status_code) 253 | 254 | future = sess.get(self.httpbin.join('redirect-to?url=status/404')) 255 | resp = future.result() 256 | self.assertEqual(404, resp.status_code) 257 | 258 | @skipIf(session_required, 'not supported in python < 3.5') 259 | def test_context(self): 260 | self._assert_context() 261 | 262 | def test_context_with_session(self): 263 | self._assert_context(session=self.session) 264 | 265 | def _assert_context(self, session=None): 266 | if session: 267 | helper_instance = TopLevelContextHelper( 268 | executor=self.proc_executor, session=self.session 269 | ) 270 | else: 271 | helper_instance = TopLevelContextHelper(executor=self.proc_executor) 272 | passout = None 273 | with helper_instance as sess: 274 | passout = sess 275 | future = sess.get(self.httpbin.join('get')) 276 | self.assertIsInstance(future, Future) 277 | resp = future.result() 278 | self.assertIsInstance(resp, Response) 279 | self.assertEqual(200, resp.status_code) 280 | 281 | self.assertTrue(passout._exit_called) 282 | 283 | 284 | class TopLevelContextHelper(FuturesSession): 285 | def __init__(self, *args, **kwargs): 286 | super(TopLevelContextHelper, self).__init__(*args, **kwargs) 287 | self._exit_called = False 288 | 289 | def __exit__(self, *args, **kwargs): 290 | self._exit_called = True 291 | return super(TopLevelContextHelper, self).__exit__(*args, **kwargs) 292 | 293 | 294 | @skipIf(not unsupported_platform, 'Exception raised when unsupported') 295 | class ProcessPoolExceptionRaisedTestCase(TestCase): 296 | def test_exception_raised(self): 297 | executor = ProcessPoolExecutor(max_workers=2) 298 | sess = FuturesSession(executor=executor, session=session()) 299 | with self.assertRaises(RuntimeError): 300 | sess.get(self.httpbin.join('get')) 301 | 302 | 303 | if __name__ == '__main__': 304 | main() 305 | --------------------------------------------------------------------------------