├── requirements.txt ├── .coveragerc ├── .github └── workflows │ └── python-package.yml ├── setup.py ├── LICENSE ├── tox.ini ├── .gitignore ├── tests └── test_aiohttp_requests.py ├── README.rst ├── aiohttp_requests └── __init__.py └── docs └── CHANGELOG.rst /requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp[speedups] 2 | coworker==2.* 3 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | .git/* 4 | .tox/* 5 | docs/* 6 | setup.py 7 | test/* 8 | tests/* 9 | -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python 3 | 4 | name: Python package 5 | 6 | on: 7 | push: 8 | branches: [ "master" ] 9 | pull_request: 10 | branches: [ "master" ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] 20 | 21 | steps: 22 | - uses: actions/checkout@v3 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v3 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | - name: Install tox 28 | run: | 29 | python -m pip install --upgrade pip 30 | python -m pip install tox 31 | - name: Test with tox 32 | run: | 33 | tox 34 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | 4 | setuptools.setup( 5 | name='aiohttp-requests', 6 | version='0.2.5', 7 | 8 | author='Max Zheng', 9 | author_email='maxzheng.os @t gmail.com', 10 | 11 | description='A thin wrapper for aiohttp client with Requests simplicity', 12 | long_description=open('README.rst').read(), 13 | 14 | url='https://github.com/maxzheng/aiohttp-requests', 15 | 16 | install_requires=open('requirements.txt').read(), 17 | 18 | license='MIT', 19 | 20 | packages=setuptools.find_packages(), 21 | include_package_data=True, 22 | 23 | python_requires='>=3.6', 24 | setup_requires=['setuptools-git', 'wheel'], 25 | 26 | classifiers=[ 27 | 'Development Status :: 5 - Production/Stable', 28 | 29 | 'Intended Audience :: Developers', 30 | 'Topic :: Software Development :: Libraries :: Python Modules', 31 | 32 | 'License :: OSI Approved :: MIT License', 33 | 34 | 'Programming Language :: Python :: 3', 35 | ], 36 | 37 | keywords='aiohttp HTTP client async requests', 38 | ) 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Max Zheng 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = cover, style 3 | 4 | [testenv] 5 | # Consolidate all deps here instead of separately in test/style/cover so we 6 | # have a single env to work with, which makes debugging easier (like which env?). 7 | # Not as clean but easier to work with during development, which is better. 8 | deps = 9 | aioresponses 10 | flake8 11 | mock 12 | pytest 13 | pytest-aiohttp 14 | pytest-cov 15 | pytest-fixtures 16 | pytest-xdist 17 | sphinx 18 | install_command = 19 | pip install -U {packages} 20 | recreate = False 21 | skipsdist = True 22 | usedevelop = True 23 | setenv = 24 | PIP_PROCESS_DEPENDENCY_LINKS=1 25 | PIP_DEFAULT_TIMEOUT=60 26 | ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future 27 | basepython = python3 28 | 29 | [testenv:test] 30 | commands = 31 | pytest {env:PYTESTARGS:} 32 | 33 | [testenv:style] 34 | commands = 35 | flake8 --config tox.ini 36 | 37 | [testenv:cover] 38 | commands = 39 | pytest {env:PYTESTARGS:} --cov . --cov-report=xml --cov-report=html --cov-report=term --cov-fail-under=80 40 | env_dir = {work_dir}/aiohttp-requests 41 | 42 | [flake8] 43 | exclude = .git,.tox,.eggs,__pycache__,docs,build,dist 44 | ignore = E111,E121,W292,E123,E226 45 | max-line-length = 120 46 | 47 | [pytest] 48 | asyncio_mode = auto 49 | filterwarnings = ignore::DeprecationWarning:pkg_resources 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | textcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *.cover 48 | .hypothesis/ 49 | .pytest_cache/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | db.sqlite3 59 | 60 | # Flask stuff: 61 | instance/ 62 | .webassets-cache 63 | 64 | # Scrapy stuff: 65 | .scrapy 66 | 67 | # Sphinx documentation 68 | docs/_build/ 69 | 70 | # PyBuilder 71 | target/ 72 | 73 | # Jupyter Notebook 74 | .ipynb_checkpoints 75 | 76 | # pyenv 77 | .python-version 78 | 79 | # celery beat schedule file 80 | celerybeat-schedule 81 | 82 | # SageMath parsed files 83 | *.sage.py 84 | 85 | # Environments 86 | .env 87 | .venv 88 | env/ 89 | venv/ 90 | ENV/ 91 | env.bak/ 92 | venv.bak/ 93 | 94 | # Spyder project settings 95 | .spyderproject 96 | .spyproject 97 | 98 | # Rope project settings 99 | .ropeproject 100 | 101 | # mkdocs documentation 102 | /site 103 | 104 | # mypy 105 | .mypy_cache/ 106 | -------------------------------------------------------------------------------- /tests/test_aiohttp_requests.py: -------------------------------------------------------------------------------- 1 | from aiohttp_requests import requests 2 | from aioresponses import aioresponses 3 | 4 | 5 | async def test_aiohttp_requests(): 6 | test_url = 'http://dummy-url' 7 | test_payload = {'hello': 'world'} 8 | 9 | with aioresponses() as mocked: 10 | mocked.get(test_url, payload=test_payload) 11 | 12 | response = await requests.get(test_url) 13 | json = await response.json() 14 | 15 | assert test_payload == json 16 | 17 | requests.close() # Normally called on destroy 18 | 19 | 20 | async def test_aiohttp_requests_integration(): 21 | # One request 22 | response = await requests.get('https://www.google.com') 23 | content = await response.text() 24 | 25 | assert response.status == 200 26 | assert len(content) > 10000 27 | assert 'Search the world' in content 28 | 29 | 30 | async def test_aiohttp_requests_integration_multiple(): 31 | # Multiple requests 32 | responses = await requests.get(['https://www.google.com'] * 2) 33 | assert len(responses) == 2 34 | for response in responses: 35 | content = await response.text() 36 | 37 | assert response.status == 200 38 | assert len(content) > 10000 39 | assert 'Search the world' in content 40 | 41 | # Multiple requests as iterator 42 | responses = requests.get(['https://www.google.com'] * 2, as_iterator=True) 43 | for response in responses: 44 | response = await response 45 | content = await response.text() 46 | 47 | assert response.status == 200 48 | assert len(content) > 10000 49 | assert 'Search the world' in content 50 | 51 | 52 | async def test_aiohttp_requests_after_close(): 53 | # Closing ourself 54 | requests.close() 55 | 56 | await test_aiohttp_requests_integration() 57 | 58 | # Closing aiohttp session 59 | await requests.session.close() 60 | 61 | await test_aiohttp_requests_integration() 62 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | aiohttp-requests 2 | ============================================================ 3 | 4 | Behold, the power of aiohttp_ client with `Requests `_ simplicity: 5 | 6 | .. code-block:: python 7 | 8 | import asyncio 9 | 10 | import aiohttp 11 | from aiohttp_requests import requests 12 | 13 | async def main(): 14 | response = await requests.get('https://api.github.com', auth=aiohttp.BasicAuth('user', 'password')) 15 | text = await response.text() 16 | json = await response.json() 17 | return response, text, json 18 | 19 | r, text, json = asyncio.run(main()) 20 | 21 | >>> r 22 | 23 | >>> r.status 24 | 200 25 | >>> r.headers['Content-Type'] 26 | 'application/json; charset=utf-8' 27 | >>> r.get_encoding() 28 | 'utf-8' 29 | >>> text 30 | '{"current_user_url":"https://api.github.com/user",...' 31 | >>> json 32 | {'current_user_url': 'https://api.github.com/user', ... } 33 | 34 | Plus built-in concurrency control to do multiple requests safely: 35 | 36 | .. code-block:: python 37 | 38 | async def main(): 39 | # Pass in a list of urls instead of just one. Optionally pass in as_iterator=True to iterate the responses. 40 | responses = await requests.get(['https://api.github.com'] * 2, auth=aiohttp.BasicAuth('user', 'password')) 41 | print(responses) # [, , ] 42 | 43 | # It defaults to 10 concurrent requests maximum. If you can handle more, then set it higher: 44 | requests.max_concurrency = 100 45 | 46 | asyncio.run(main()) 47 | 48 | The `requests` object is just proxying `get` and other HTTP verb methods to `aiohttp.ClientSession`_, which returns `aiohttp.ClientResponse`_. To do anything else, read the aiohttp_ doc. 49 | 50 | .. _`aiohttp.ClientSession`: https://docs.aiohttp.org/en/stable/client_reference.html?#aiohttp.ClientSession 51 | .. _`aiohttp.ClientResponse`: https://docs.aiohttp.org/en/stable/client_reference.html?#aiohttp.ClientResponse 52 | .. _aiohttp: https://docs.aiohttp.org/en/stable/ 53 | 54 | Links & Contact Info 55 | ==================== 56 | 57 | | PyPI Package: https://pypi.python.org/pypi/aiohttp-requests 58 | | GitHub Source: https://github.com/maxzheng/aiohttp-requests 59 | | Report Issues/Bugs: https://github.com/maxzheng/aiohttp-requests/issues 60 | | 61 | | Connect: https://www.linkedin.com/in/maxzheng 62 | | Contact: maxzheng.os @t gmail.com 63 | -------------------------------------------------------------------------------- /aiohttp_requests/__init__.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | import functools 3 | 4 | from coworker import Coworker 5 | 6 | 7 | class _Corequest(Coworker): 8 | """ Worker for making concurrent requests """ 9 | async def do_task(self, task): 10 | request, verb, path, args, kwargs = task 11 | return await request(verb, path, *args, **kwargs) 12 | 13 | 14 | class Requests: 15 | """ Thin wrapper for aiohttp.ClientSession with Requests simplicity """ 16 | def __init__(self, *args, max_concurrency=10, **kwargs): 17 | self._session_args = (args, kwargs) 18 | self._session = None 19 | 20 | #: Worker for concurrent requests 21 | self._worker = _Corequest(max_concurrency=max_concurrency) 22 | 23 | @property 24 | def session(self): 25 | """ An instance of aiohttp.ClientSession """ 26 | if not self._session or self._session.closed or self._session._loop.is_closed(): 27 | self._session = aiohttp.ClientSession(*self._session_args[0], **self._session_args[1]) 28 | return self._session 29 | 30 | def __getattr__(self, attr): 31 | if attr.upper() in aiohttp.hdrs.METH_ALL: 32 | return functools.partial(self.request, attr.upper()) 33 | else: 34 | return super().__getattribute__(attr) 35 | 36 | def __setattr__(self, attr, value): 37 | if attr == 'max_concurrency': 38 | self._worker.max_concurrency = value 39 | else: 40 | super().__setattr__(attr, value) 41 | 42 | def _concurrent_request(self, request, verb, paths, args, kwargs, as_iterator=False): 43 | return self._worker.do([(request, verb, path, args, kwargs) for path in paths], as_iterator=as_iterator) 44 | 45 | def request(self, verb, path, *args, **kwargs): 46 | """ 47 | This ensures `self.session` is always called where it can check the session/loop state so can't use 48 | functools.partials as monkeypatch seems to do something weird where __getattr__ is only called once 49 | for each attribute after patch is undone 50 | 51 | :param verb: HTTP verb 52 | :param path: URL path 53 | :param args: Additional arguments for aiohttp.ClientSession.request 54 | :param kwargs: Additional keyword arguments for aiohttp.ClientSession.request 55 | """ 56 | if isinstance(path, (list, tuple)): 57 | return self._concurrent_request(self.session._request, verb.upper(), path, args, kwargs, 58 | as_iterator=kwargs.pop('as_iterator', False)) 59 | else: 60 | return self.session._request(verb.upper(), path, *args, **kwargs) 61 | 62 | def close(self): 63 | """ 64 | Close aiohttp.ClientSession. 65 | 66 | This is useful to be called manually in tests if each test when each test uses a new loop. After close, new 67 | requests will automatically create a new session. 68 | 69 | Note: We need a sync version for `__del__` and `aiohttp.ClientSession.close()` is async even though it doesn't 70 | have to be. 71 | """ 72 | if self._session: 73 | if not self._session.closed: 74 | # Older aiohttp does not have _connector_owner 75 | if not hasattr(self._session, '_connector_owner') or self._session._connector_owner: 76 | try: 77 | self._session._connector._close() # New version returns a coroutine in close() as warning 78 | except Exception: 79 | self._session._connector.close() 80 | self._session._connector = None 81 | self._session = None 82 | 83 | def __del__(self): 84 | self.close() 85 | 86 | 87 | requests = Requests() 88 | -------------------------------------------------------------------------------- /docs/CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | Version 0.2.4 2 | ================================================================================ 3 | 4 | * Add request() method 5 | * Update README.rst 6 | 7 | Version 0.2.3 8 | -------------------------------------------------------------------------------- 9 | 10 | * Support concurrent requests 11 | 12 | Version 0.2.2 13 | -------------------------------------------------------------------------------- 14 | 15 | * Update readme to use asyncio.run 16 | 17 | Version 0.2.1 18 | -------------------------------------------------------------------------------- 19 | 20 | * Fix links in README 21 | 22 | Version 0.2.0 23 | -------------------------------------------------------------------------------- 24 | 25 | * Remove patch and let garbage collector reap/close the responses (if not already closed by reading to eof) 26 | 27 | Version 0.1.6 28 | ================================================================================ 29 | 30 | * Check loop health for recreating session 31 | 32 | Version 0.1.5 33 | -------------------------------------------------------------------------------- 34 | 35 | * Wrap patched function 36 | 37 | Version 0.1.4 38 | -------------------------------------------------------------------------------- 39 | 40 | * Test against Python 3.7+ 41 | * Create python-package.yml 42 | * Fix tests and docs 43 | 44 | Version 0.1.4 45 | -------------------------------------------------------------------------------- 46 | 47 | * Test against Python 3.7+ 48 | * Create python-package.yml 49 | * Fix tests and docs 50 | 51 | Version 0.1.4 52 | -------------------------------------------------------------------------------- 53 | 54 | * Test against Python 3.7+ 55 | * Create python-package.yml 56 | * Fix tests and docs 57 | 58 | Version 0.1.4 59 | -------------------------------------------------------------------------------- 60 | 61 | * Test against Python 3.7+ 62 | * Create python-package.yml 63 | * Fix tests and docs 64 | 65 | Version 0.1.4 66 | -------------------------------------------------------------------------------- 67 | 68 | * Test against Python 3.7+ 69 | * Create python-package.yml 70 | * Fix tests and docs 71 | 72 | Version 0.1.3 73 | -------------------------------------------------------------------------------- 74 | 75 | * Remove version requirement for aiohttp as aioresponses suppports latest now 76 | 77 | Version 0.1.2 78 | -------------------------------------------------------------------------------- 79 | 80 | * Update setup 81 | 82 | Version 0.1.1 83 | -------------------------------------------------------------------------------- 84 | 85 | * Remove pip.req 86 | 87 | Version 0.1.0 88 | -------------------------------------------------------------------------------- 89 | 90 | * Recreate sessin if loop is closed 91 | 92 | Version 0.0.8 93 | ================================================================================ 94 | 95 | * Set min code coverage to 90 96 | * Improve session close and reopen automatically 97 | 98 | Version 0.0.7 99 | -------------------------------------------------------------------------------- 100 | 101 | * Fix code example 102 | 103 | Version 0.0.6 104 | -------------------------------------------------------------------------------- 105 | 106 | * Fix code example 107 | * Remove .pytest_cache 108 | * Add test 109 | 110 | Version 0.0.5 111 | -------------------------------------------------------------------------------- 112 | 113 | * Make close work for older aiohttp 114 | 115 | Version 0.0.4 116 | -------------------------------------------------------------------------------- 117 | 118 | * Drop min Python to 3.5 119 | 120 | Version 0.0.3 121 | -------------------------------------------------------------------------------- 122 | 123 | * Fix setup.py 124 | 125 | Version 0.0.2 126 | -------------------------------------------------------------------------------- 127 | 128 | * Fix readme 129 | * Add aiohttp wrapper 130 | * Initial commit 131 | --------------------------------------------------------------------------------