├── README.rst ├── requests_threads.py └── setup.py /README.rst: -------------------------------------------------------------------------------- 1 | requests-threads 🎭 2 | =================== 3 | 4 | This repo contains a Requests session that returns the amazing `Twisted `_'s awaitable 5 | Deferreds instead of Response objects. 6 | 7 | It's awesome, basically — check it out: 8 | 9 | .. image:: https://farm5.staticflickr.com/4418/35904417594_c4933a2171_k_d.jpg 10 | 11 | 12 | Examples 13 | -------- 14 | 15 | Let's send 100 concurrent requests! \\o/ 16 | 17 | **Example Usage** using ``async``/``await`` — 18 | 19 | .. code:: python 20 | 21 | from requests_threads import AsyncSession 22 | 23 | session = AsyncSession(n=100) 24 | 25 | async def _main(): 26 | rs = [] 27 | for _ in range(100): 28 | rs.append(await session.get('http://httpbin.org/get')) 29 | print(rs) 30 | 31 | if __name__ == '__main__': 32 | session.run(_main) 33 | 34 | *This example works on Python 3 only.* You can also provide your own ``asyncio`` event loop! 35 | 36 | **Example Usage** using Twisted — 37 | 38 | .. code:: python 39 | 40 | 41 | from twisted.internet.defer import inlineCallbacks 42 | from twisted.internet.task import react 43 | from requests_threads import AsyncSession 44 | 45 | session = AsyncSession(n=100) 46 | 47 | @inlineCallbacks 48 | def main(reactor): 49 | responses = [] 50 | for i in range(100): 51 | responses.append(session.get('http://httpbin.org/get')) 52 | 53 | for response in responses: 54 | r = yield response 55 | print(r) 56 | 57 | if __name__ == '__main__': 58 | react(main) 59 | 60 | *This example works on both Python 2 and Python 3.* 61 | 62 | -------------------- 63 | 64 | Each request is sent via a new thread, automatically. This works fine for basic 65 | use cases. This automatically uses Twisted's ``asyncioreactor``, if you do not 66 | provide your own reactor (progress to be made there, help requested!). 67 | 68 | **This is a an experiment**, and a preview of the true asynchronous API we have planned for Requests 69 | that is currently *in the works*, but requires a lot of development time. If you'd like to help (p.s. **we need help**, `send me an email `_). 70 | 71 | This API is likely to change, over time, slightly. 72 | 73 | Installation 74 | ------------ 75 | 76 | :: 77 | 78 | $ pipenv install requests-threads 79 | ✨🍰✨ 80 | 81 | 82 | Inspiration 83 | ----------- 84 | 85 | This codebase was inspired by future work on Requests, as well as `requests-twisted `_. 86 | -------------------------------------------------------------------------------- /requests_threads.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | 3 | from twisted.internet import threads 4 | from twisted.internet.defer import ensureDeferred 5 | from twisted.internet.error import ReactorAlreadyInstalledError 6 | from twisted.internet import task 7 | 8 | from requests import Session 9 | 10 | 11 | class AsyncSession(Session): 12 | """An asynchronous Requests session. 13 | 14 | Provides cookie persistence, connection-pooling, and configuration. 15 | 16 | Basic Usage:: 17 | 18 | >>> import requests 19 | >>> s = requests.Session() 20 | >>> s.get('http://httpbin.org/get') 21 | 22 | 23 | Or as a context manager:: 24 | 25 | >>> with requests.Session() as s: 26 | >>> s.get('http://httpbin.org/get') 27 | 28 | """ 29 | 30 | def __init__(self, n=None, reactor=None, loop=None, *args, **kwargs): 31 | if reactor is None: 32 | try: 33 | import asyncio 34 | loop = loop or asyncio.get_event_loop() 35 | try: 36 | from twisted.internet import asyncioreactor 37 | asyncioreactor.install(loop) 38 | except (ReactorAlreadyInstalledError, ImportError): 39 | pass 40 | except ImportError: 41 | pass 42 | 43 | # Adjust the pool size, according to n. 44 | if n: 45 | from twisted.internet import reactor 46 | pool = reactor.getThreadPool() 47 | pool.adjustPoolsize(0, n) 48 | 49 | super(AsyncSession, self).__init__(*args, **kwargs) 50 | 51 | def request(self, *args, **kwargs): 52 | """Maintains the existing api for Session.request. 53 | Used by all of the higher level methods, e.g. Session.get. 54 | """ 55 | func = super(AsyncSession, self).request 56 | return threads.deferToThread(func, *args, **kwargs) 57 | 58 | def wrap(self, *args, **kwargs): 59 | return ensureDeferred(*args, **kwargs) 60 | 61 | def run(self, f): 62 | # Python 3 only. 63 | if hasattr(inspect, 'iscoroutinefunction'): 64 | # Is this a coroutine? 65 | if inspect.iscoroutinefunction(f): 66 | def w(reactor): 67 | return self.wrap(f()) 68 | # If so, convert coroutine to Deferred automatically. 69 | return task.react(w) 70 | else: 71 | # Otherwise, run the Deferred. 72 | return task.react(f) 73 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Note: To use the 'upload' functionality of this file, you must: 5 | # $ pip install twine 6 | 7 | import io 8 | import os 9 | import sys 10 | from shutil import rmtree 11 | 12 | from setuptools import setup, Command 13 | 14 | # Package meta-data. 15 | NAME = 'requests-threads' 16 | DESCRIPTION = 'A Requests session that returns awaitable Twisted Deferreds instead of response objects.' 17 | URL = 'https://github.com/requests/requests-threads' 18 | EMAIL = 'me@kennethreitz.org' 19 | AUTHOR = 'Kenneth Reitz' 20 | VERSION = '0.1.1' 21 | 22 | # What packages are required for this module to be executed? 23 | REQUIRED = [ 24 | 'requests', 25 | 'twisted' 26 | ] 27 | 28 | # The rest you shouldn't have to touch too much :) 29 | # ------------------------------------------------ 30 | # Except, perhaps the License and Trove Classifiers! 31 | 32 | here = os.path.abspath(os.path.dirname(__file__)) 33 | 34 | # Import the README and use it as the long-description. 35 | # Note: this will only work if 'README.rst' is present in your MANIFEST.in file! 36 | with io.open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: 37 | long_description = '\n' + f.read() 38 | 39 | 40 | class PublishCommand(Command): 41 | """Support setup.py publish.""" 42 | 43 | description = 'Build and publish the package.' 44 | user_options = [] 45 | 46 | @staticmethod 47 | def status(s): 48 | """Prints things in bold.""" 49 | print('\033[1m{0}\033[0m'.format(s)) 50 | 51 | def initialize_options(self): 52 | pass 53 | 54 | def finalize_options(self): 55 | pass 56 | 57 | def run(self): 58 | try: 59 | self.status('Removing previous builds…') 60 | rmtree(os.path.join(here, 'dist')) 61 | except FileNotFoundError: 62 | pass 63 | 64 | self.status('Building Source and Wheel (universal) distribution…') 65 | os.system('{0} setup.py sdist bdist_wheel --universal'.format(sys.executable)) 66 | 67 | self.status('Uploading the package to PyPi via Twine…') 68 | os.system('twine upload dist/*') 69 | 70 | sys.exit() 71 | 72 | 73 | # Where the magic happens: 74 | setup( 75 | name=NAME, 76 | version=VERSION, 77 | description=DESCRIPTION, 78 | long_description=long_description, 79 | author=AUTHOR, 80 | author_email=EMAIL, 81 | url=URL, 82 | # If your package is a single module, use this instead of 'packages': 83 | py_modules=['requests_threads'], 84 | install_requires=REQUIRED, 85 | include_package_data=True, 86 | license='ISC', 87 | classifiers=[ 88 | # Trove classifiers 89 | # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers 90 | 'License :: OSI Approved :: MIT License', 91 | 'Programming Language :: Python', 92 | 'Programming Language :: Python :: 2.6', 93 | 'Programming Language :: Python :: 2.7', 94 | 'Programming Language :: Python :: 3', 95 | 'Programming Language :: Python :: 3.3', 96 | 'Programming Language :: Python :: 3.4', 97 | 'Programming Language :: Python :: 3.5', 98 | 'Programming Language :: Python :: 3.6', 99 | 'Programming Language :: Python :: Implementation :: CPython', 100 | 'Programming Language :: Python :: Implementation :: PyPy' 101 | ], 102 | # $ setup.py publish support. 103 | cmdclass={ 104 | 'publish': PublishCommand, 105 | }, 106 | ) 107 | --------------------------------------------------------------------------------