├── .coveragerc ├── .gitignore ├── .readthedocs.yml ├── .travis.yml ├── LICENSE ├── README.rst ├── asyncio_extras ├── __init__.py ├── asyncyield.py ├── contextmanager.py ├── file.py ├── generator.py └── threads.py ├── docs ├── conf.py ├── index.rst └── versionhistory.rst ├── pyproject.toml ├── setup.cfg ├── setup.py ├── tests ├── conftest.py ├── test_contextmanager.py ├── test_contextmanager_py36.py ├── test_file.py ├── test_generator.py ├── test_generator_py36.py └── test_threads.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = asyncio_extras 3 | 4 | [report] 5 | show_missing = true 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .project 2 | .pydevproject 3 | .idea 4 | .tox 5 | .coverage 6 | .cache 7 | .eggs/ 8 | *.egg-info/ 9 | *.pyc 10 | __pycache__/ 11 | docs/_build/ 12 | dist/ 13 | build/ 14 | virtualenv/ 15 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | type: sphinx 2 | python: 3 | version: "3.5" 4 | pip_install: true 5 | extra_requirements: [doc] 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | 4 | stages: 5 | - name: test 6 | - name: deploy to pypi 7 | if: type = push AND tag =~ ^\d+\.\d+\.\d+ 8 | 9 | jobs: 10 | fast_finish: true 11 | allow_failures: 12 | - python: "3.7-dev" 13 | include: 14 | - env: TOXENV=flake8 15 | 16 | - env: TOXENV=pypy3 17 | cache: pip 18 | python: pypy3 19 | 20 | - env: TOXENV=py35 21 | after_success: &after_success 22 | - pip install coveralls 23 | - coveralls 24 | 25 | - env: TOXENV=py36 26 | python: "3.6" 27 | after_success: *after_success 28 | 29 | - env: TOXENV=py37 30 | python: "3.7-dev" 31 | after_success: *after_success 32 | 33 | - stage: deploy to pypi 34 | install: pip install "setuptools >= 36.2.7" 35 | script: skip 36 | deploy: 37 | provider: pypi 38 | user: agronholm 39 | password: 40 | secure: NuwdObD6J8KbnOi7+WS5z1iR/v1NCbVkwlwni3Bz+cev9IvQ/zSy6T4vl3yoV2IiyKwxHCarUJi7aLXEn722FoVs19Rnwe5RsbXbiL7H0kkPLIKGdrXnq5zI1v4J2Zr1JLJSvwnq/9bc3uyIcCPWnSuEk76cHQkVv6W19HTeoDJN2T8zh4WltSUzDzJQK8vzKP8S2eZ/PSxZRMKG8uVkeLDcxStjD1nbSi1+iyIWOYV1D6S5y6KY1lonK5F/wvmBq18c0zfcJPpD0sNsYulme6YFc0ksCNDnGQUYbxpRVUBr0mILjMbEHiNxwW9qQpu8GmqVcxLjEzp7QLnHDA2d2rr33r627chggZ2EwgisZVLfsKI/FD0YtTC81nJJCPPG4qh482WCeSKR8efr3ehcasm8xaNyjhUXx6mCN/C9vscEIbQvBeVna38xj1FisywRjyJKDqpgVysJZ9DyxNJ1pSzf6+XGRczw48HcF8xYYOY8IqUGCVGH1m+bZQj6Z54wx6+pPaEkubyZt0GTLGsDCIQCNLAELP4YTkgeN9QcBnqVdQNWuO8nnY+vIImmMUMBelzJtVJcPZ1mlNQgo0raDsjiAj86OZ4+BPlbe2rIYZ4CHEgw5UbE5kTnf6dOIf774Zz2jbfDOxhjEoXUdu0r4A5NvTnE7JUWv6gz3pfmqxU= 41 | distributions: sdist bdist_wheel 42 | on: 43 | tags: true 44 | 45 | python: "3.5.2" 46 | 47 | install: 48 | - pip install "setuptools >= 36.2.7" 49 | - pip install tox 50 | 51 | script: tox 52 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is the MIT license: http://www.opensource.org/licenses/mit-license.php 2 | 3 | Copyright (c) Alex Grönholm 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this 6 | software and associated documentation files (the "Software"), to deal in the Software 7 | without restriction, including without limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons 9 | to whom the Software is furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all copies or 12 | substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 15 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR 16 | PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE 17 | FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | DEALINGS IN THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://travis-ci.org/agronholm/asyncio_extras.svg?branch=master 2 | :target: https://travis-ci.org/agronholm/asyncio_extras 3 | :alt: Build Status 4 | .. image:: https://coveralls.io/repos/agronholm/asyncio_extras/badge.svg?branch=master&service=github 5 | :target: https://coveralls.io/github/agronholm/asyncio_extras?branch=master 6 | :alt: Code Coverage 7 | .. image:: https://readthedocs.org/projects/asyncio-extras/badge/?version=latest 8 | :target: https://asyncio-extras.readthedocs.io/en/latest/?badge=latest 9 | :alt: Documentation Status 10 | 11 | This library provides several conveniences to users of asyncio_: 12 | 13 | * decorator for making asynchronous context managers (like ``contextlib.contextmanager``) 14 | * decorator and context manager for running a function or parts of a function in a thread pool 15 | * helpers for calling functions in the event loop from worker threads and vice versa 16 | * helpers for doing non-blocking file i/o 17 | 18 | .. _asyncio: https://docs.python.org/3/library/asyncio.html 19 | -------------------------------------------------------------------------------- /asyncio_extras/__init__.py: -------------------------------------------------------------------------------- 1 | from .asyncyield import * # noqa 2 | from .contextmanager import * # noqa 3 | from .file import * # noqa 4 | from .generator import * # noqa 5 | from .threads import * # noqa 6 | -------------------------------------------------------------------------------- /asyncio_extras/asyncyield.py: -------------------------------------------------------------------------------- 1 | from warnings import warn 2 | 3 | from async_generator import yield_ 4 | 5 | __all__ = ('yield_async',) 6 | 7 | 8 | def yield_async(value=None): 9 | """The equivalent of ``yield`` in an asynchronous context manager or asynchronous generator.""" 10 | warn('This function has been deprecated in favor of async_generator.yield_', 11 | DeprecationWarning) 12 | return yield_(value) 13 | -------------------------------------------------------------------------------- /asyncio_extras/contextmanager.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Coroutine 2 | from functools import wraps 3 | from inspect import iscoroutinefunction 4 | from typing import Callable, Union 5 | 6 | from async_generator import async_generator, isasyncgenfunction 7 | 8 | try: 9 | from types import AsyncGeneratorType 10 | 11 | generator_types = Union[Coroutine, AsyncGeneratorType] 12 | except ImportError: 13 | generator_types = Coroutine 14 | 15 | __all__ = ('async_contextmanager',) 16 | 17 | 18 | class _AsyncContextManager: 19 | __slots__ = 'generator' 20 | 21 | def __init__(self, generator) -> None: 22 | self.generator = generator 23 | 24 | def __aenter__(self): 25 | return self.generator.asend(None) 26 | 27 | async def __aexit__(self, exc_type, exc_val, exc_tb): 28 | if exc_val is not None: 29 | try: 30 | await self.generator.athrow(exc_val) 31 | except StopAsyncIteration: 32 | pass 33 | 34 | return True 35 | else: 36 | try: 37 | await self.generator.asend(None) 38 | except StopAsyncIteration: 39 | pass 40 | else: 41 | raise RuntimeError("async generator didn't stop") 42 | 43 | 44 | def async_contextmanager(func: Callable[..., generator_types]) -> Callable: 45 | """ 46 | Transform a coroutine function into something that works with ``async with``. 47 | 48 | This is an asynchronous counterpart to :func:`~contextlib.contextmanager`. 49 | The wrapped function can either be a native async generator function (``async def`` with 50 | ``yield``) or, if your code needs to be compatible with Python 3.5, you can use 51 | :func:`~async_generator.yield_` instead of the native ``yield`` statement. 52 | 53 | The generator must yield *exactly once*, just like with :func:`~contextlib.contextmanager`. 54 | 55 | Usage in Python 3.5 and earlier:: 56 | 57 | @async_contextmanager 58 | async def mycontextmanager(arg): 59 | context = await setup_remote_context(arg) 60 | await yield_(context) 61 | await context.teardown() 62 | 63 | async def frobnicate(arg): 64 | async with mycontextmanager(arg) as context: 65 | do_something_with(context) 66 | 67 | The same context manager function in Python 3.6+:: 68 | 69 | @async_contextmanager 70 | async def mycontextmanager(arg): 71 | context = await setup_remote_context(arg) 72 | yield context 73 | await context.teardown() 74 | 75 | :param func: an async generator function or a coroutine function using 76 | :func:`~async_generator.yield_` 77 | :return: a callable that can be used with ``async with`` 78 | 79 | """ 80 | if not isasyncgenfunction(func): 81 | if iscoroutinefunction(func): 82 | func = async_generator(func) 83 | else: 84 | '"func" must be an async generator function or a coroutine function' 85 | 86 | @wraps(func) 87 | def wrapper(*args, **kwargs): 88 | generator = func(*args, **kwargs) 89 | return _AsyncContextManager(generator) 90 | 91 | return wrapper 92 | -------------------------------------------------------------------------------- /asyncio_extras/file.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from concurrent.futures import Executor 3 | from io import IOBase # noqa 4 | from pathlib import Path 5 | from typing import Union, Optional 6 | 7 | from async_generator import async_generator, yield_ 8 | 9 | from asyncio_extras.threads import threadpool, call_in_executor 10 | 11 | __all__ = ('AsyncFileWrapper', 'open_async') 12 | 13 | 14 | class AsyncFileWrapper: 15 | """ 16 | Wraps certain file I/O operations so they're guaranteed to run in a thread pool. 17 | 18 | The wrapped methods work like coroutines when called in the event loop thread, but when called 19 | in any other thread, they work just like the methods of the ``file`` type. 20 | 21 | This class supports use as an asynchronous context manager. 22 | 23 | The wrapped methods are: 24 | 25 | * ``flush()`` 26 | * ``read()`` 27 | * ``readline()`` 28 | * ``readlines()`` 29 | * ``seek()`` 30 | * ``truncate()`` 31 | * ``write()`` 32 | * ``writelines()`` 33 | """ 34 | 35 | __slots__ = ('_open_args', '_open_kwargs', '_executor', '_raw_file', 'flush', 'read', 36 | 'readline', 'readlines', 'seek', 'truncate', 'write', 'writelines') 37 | 38 | def __init__(self, path: str, args: tuple, kwargs: dict, executor: Optional[Executor]) -> None: 39 | self._open_args = (path,) + args 40 | self._open_kwargs = kwargs 41 | self._executor = executor 42 | self._raw_file = None # type: IOBase 43 | 44 | def __getattr__(self, name): 45 | return getattr(self._raw_file, name) 46 | 47 | def __await__(self): 48 | if self._raw_file is None: 49 | self._raw_file = yield from call_in_executor( 50 | open, *self._open_args, executor=self._executor, **self._open_kwargs) 51 | self.flush = threadpool(self._executor)(self._raw_file.flush) 52 | self.read = threadpool(self._executor)(self._raw_file.read) 53 | self.readline = threadpool(self._executor)(self._raw_file.readline) 54 | self.readlines = threadpool(self._executor)(self._raw_file.readlines) 55 | self.seek = threadpool(self._executor)(self._raw_file.seek) 56 | self.truncate = threadpool(self._executor)(self._raw_file.truncate) 57 | self.write = threadpool(self._executor)(self._raw_file.write) 58 | self.writelines = threadpool(self._executor)(self._raw_file.writelines) 59 | 60 | return self 61 | 62 | def __aenter__(self): 63 | return self 64 | 65 | async def __aexit__(self, exc_type, exc_val, exc_tb): 66 | self._raw_file.close() 67 | 68 | if sys.version_info < (3, 5, 2): 69 | async def __aiter__(self): # pragma: no cover 70 | return self 71 | else: 72 | def __aiter__(self): 73 | return self 74 | 75 | async def __anext__(self): 76 | if self._raw_file is None: 77 | await self 78 | 79 | line = await self.readline() 80 | if line: 81 | return line 82 | else: 83 | raise StopAsyncIteration 84 | 85 | @async_generator 86 | async def async_readchunks(self, size: int): 87 | """ 88 | Read data from the file in chunks. 89 | 90 | :param size: the maximum number of bytes or characters to read at once 91 | :return: an asynchronous iterator yielding bytes or strings 92 | 93 | """ 94 | while True: 95 | data = await self.read(size) 96 | if data: 97 | await yield_(data) 98 | else: 99 | return 100 | 101 | 102 | def open_async(file: Union[str, Path], *args, executor: Executor = None, 103 | **kwargs) -> AsyncFileWrapper: 104 | """ 105 | Open a file and wrap it in an :class:`~AsyncFileWrapper`. 106 | 107 | Example:: 108 | 109 | async def read_file_contents(path: str) -> bytes: 110 | async with open_async(path, 'rb') as f: 111 | return await f.read() 112 | 113 | The file wrapper can also be asynchronously iterated line by line:: 114 | 115 | async def read_file_lines(path: str): 116 | async for line in open_async(path): 117 | print(line) 118 | 119 | :param file: the file path to open 120 | :param args: positional arguments to :func:`open` 121 | :param executor: the ``executor`` argument to :class:`~AsyncFileWrapper` 122 | :param kwargs: keyword arguments to :func:`open` 123 | :return: the wrapped file object 124 | 125 | """ 126 | return AsyncFileWrapper(str(file), args, kwargs, executor) 127 | -------------------------------------------------------------------------------- /asyncio_extras/generator.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Coroutine, AsyncIterator 2 | from typing import Callable 3 | from warnings import warn 4 | 5 | from async_generator import ( 6 | async_generator as async_generator_, isasyncgenfunction as isasyncgenfunction_) 7 | 8 | __all__ = ('async_generator', 'isasyncgenfunction', 'isasyncgeneratorfunction') 9 | 10 | 11 | def async_generator(func: Callable[..., Coroutine]) -> Callable[..., AsyncIterator]: 12 | warn('This function has been deprecated in favor of async_generator.async_generator', 13 | DeprecationWarning) 14 | return async_generator_(func) 15 | 16 | 17 | def isasyncgenfunction(obj) -> bool: 18 | warn('This function has been deprecated in favor of async_generator.isasyncgenfunction', 19 | DeprecationWarning) 20 | return isasyncgenfunction_(obj) 21 | 22 | 23 | isasyncgeneratorfunction = isasyncgenfunction 24 | -------------------------------------------------------------------------------- /asyncio_extras/threads.py: -------------------------------------------------------------------------------- 1 | import concurrent.futures 2 | import gc 3 | import inspect 4 | from asyncio import get_event_loop, Future, AbstractEventLoop 5 | from concurrent.futures import Executor 6 | from functools import wraps, partial 7 | from inspect import isawaitable 8 | from threading import Event 9 | from typing import Optional, Callable, Union 10 | 11 | try: 12 | from asyncio import _get_running_loop 13 | except ImportError: 14 | def _get_running_loop(): 15 | try: 16 | return get_event_loop() 17 | except RuntimeError: 18 | return None 19 | 20 | __all__ = ('threadpool', 'call_in_executor', 'call_async') 21 | 22 | 23 | class _ThreadSwitcher: 24 | __slots__ = 'executor', 'exited' 25 | 26 | def __init__(self, executor: Optional[Executor]) -> None: 27 | self.executor = executor 28 | self.exited = False 29 | 30 | def __aenter__(self): 31 | # This is run in the event loop thread 32 | return self 33 | 34 | def __await__(self): 35 | def exec_when_ready(): 36 | event.wait() 37 | coro.send(None) 38 | 39 | if not self.exited: 40 | raise RuntimeError('attempted to "await" in a worker thread') 41 | 42 | if self.exited: 43 | # This is run in the worker thread 44 | yield 45 | else: 46 | # This is run in the event loop thread 47 | previous_frame = inspect.currentframe().f_back 48 | coro = next(obj for obj in gc.get_referrers(previous_frame.f_code) 49 | if inspect.iscoroutine(obj) and obj.cr_frame is previous_frame) 50 | event = Event() 51 | loop = get_event_loop() 52 | future = loop.run_in_executor(self.executor, exec_when_ready) 53 | next(future.__await__()) # Make the future think it's being awaited on 54 | loop.call_soon(event.set) 55 | yield future 56 | 57 | def __aexit__(self, exc_type, exc_val, exc_tb): 58 | # This is run in the worker thread 59 | self.exited = True 60 | return self 61 | 62 | def __call__(self, func: Callable) -> Callable: 63 | @wraps(func) 64 | def wrapper(*args, **kwargs): 65 | try: 66 | loop = get_event_loop() 67 | except RuntimeError: 68 | # Event loop not available -- we're in a worker thread 69 | return func(*args, **kwargs) 70 | else: 71 | callback = partial(func, *args, **kwargs) 72 | return loop.run_in_executor(self.executor, callback) 73 | 74 | assert not inspect.iscoroutinefunction(func), \ 75 | 'Cannot wrap coroutine functions to be run in an executor' 76 | return wrapper 77 | 78 | 79 | def threadpool(arg: Union[Executor, Callable] = None): 80 | """ 81 | Return a decorator/asynchronous context manager that guarantees that the wrapped function or 82 | ``with`` block is run in the given executor. 83 | 84 | If no executor is given, the current event loop's default executor is used. 85 | Otherwise, the executor must be a PEP 3148 compliant thread pool executor. 86 | 87 | Callables wrapped with this must be used with ``await`` when called in the event loop thread. 88 | They can also be called in worker threads, just by omitting the ``await``. 89 | 90 | Example use as a decorator:: 91 | 92 | @threadpool 93 | def this_runs_in_threadpool(): 94 | return do_something_cpu_intensive() 95 | 96 | async def request_handler(): 97 | result = await this_runs_in_threadpool() 98 | 99 | Example use as an asynchronous context manager:: 100 | 101 | async def request_handler(in_url, out_url): 102 | page = await http_fetch(in_url) 103 | 104 | async with threadpool(): 105 | data = transform_page(page) 106 | 107 | await http_post(out_url, page) 108 | 109 | :param arg: either a callable (when used as a decorator) or an executor in which to run the 110 | wrapped callable or the ``with`` block (when used as a context manager) 111 | 112 | """ 113 | if callable(arg): 114 | # When used like @threadpool 115 | return _ThreadSwitcher(None)(arg) 116 | else: 117 | # When used like @threadpool(...) or async with threadpool(...) 118 | return _ThreadSwitcher(arg) 119 | 120 | 121 | def call_in_executor(func: Callable, *args, executor: Executor = None, **kwargs) -> Future: 122 | """ 123 | Call the given callable in an executor. 124 | 125 | This is a nicer version of the following:: 126 | 127 | get_event_loop().run_in_executor(executor, func, *args) 128 | 129 | If you need to pass keyword arguments named ``func`` or ``executor`` to the callable, use 130 | :func:`functools.partial` for that. 131 | 132 | :param func: a function 133 | :param args: positional arguments to call with 134 | :param executor: the executor to call the function in 135 | :param kwargs: keyword arguments to call with 136 | :return: a future that will resolve to the function call's return value 137 | 138 | """ 139 | callback = partial(func, *args, **kwargs) 140 | return get_event_loop().run_in_executor(executor, callback) 141 | 142 | 143 | def call_async(loop: AbstractEventLoop, func: Callable, *args, **kwargs): 144 | """ 145 | Call the given callable in the event loop thread. 146 | 147 | If the call returns an awaitable, it is resolved before returning to the caller. 148 | 149 | If you need to pass keyword arguments named ``loop`` or ``func`` to the callable, use 150 | :func:`functools.partial` for that. 151 | 152 | :param func: a regular function or a coroutine function 153 | :param args: positional arguments to call with 154 | :param loop: the event loop in which to call the function 155 | :param kwargs: keyword arguments to call with 156 | :return: the return value of the function call 157 | 158 | """ 159 | async def callback(): 160 | try: 161 | retval = func(*args, **kwargs) 162 | if isawaitable(retval): 163 | retval = await retval 164 | except BaseException as e: 165 | f.set_exception(e) 166 | else: 167 | f.set_result(retval) 168 | 169 | if _get_running_loop(): 170 | raise RuntimeError('call_async() must not be called from an event loop thread') 171 | 172 | f = concurrent.futures.Future() 173 | loop.call_soon_threadsafe(loop.create_task, callback()) 174 | return f.result() 175 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | import pkg_resources 2 | 3 | extensions = [ 4 | 'sphinx.ext.autodoc', 5 | 'sphinx.ext.intersphinx', 6 | 'sphinx_autodoc_typehints' 7 | ] 8 | 9 | templates_path = ['_templates'] 10 | source_suffix = '.rst' 11 | master_doc = 'index' 12 | 13 | # General information about the project. 14 | project = 'Asyncio Extras' 15 | author = 'Alex Grönholm' 16 | copyright = '2016, ' + author 17 | 18 | v = pkg_resources.get_distribution('asyncio_extras').parsed_version 19 | version = v.base_version 20 | release = v.public 21 | 22 | language = None 23 | 24 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 25 | pygments_style = 'sphinx' 26 | highlight_language = 'python3' 27 | todo_include_todos = False 28 | 29 | html_theme = 'classic' 30 | html_static_path = ['_static'] 31 | htmlhelp_basename = 'asyncio_extrasdoc' 32 | 33 | intersphinx_mapping = {'https://docs.python.org/3': None} 34 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Asyncio Extras 2 | ============== 3 | 4 | This library provides some "missing" features for the asyncio (:pep:`3156`) module: 5 | 6 | * decorator for making asynchronous context managers (like :func:`~contextlib.contextmanager`) 7 | * decorator and context manager for running a function or parts of a function in a thread pool 8 | * helpers for calling functions in the event loop from worker threads and vice versa 9 | * helpers for doing non-blocking file i/o 10 | 11 | :doc:`versionhistory` 12 | 13 | 14 | :mod:`asyncio_extras.contextmanager` 15 | ==================================== 16 | 17 | .. automodule:: asyncio_extras.contextmanager 18 | :members: 19 | 20 | :mod:`asyncio_extras.file` 21 | ========================== 22 | 23 | .. automodule:: asyncio_extras.file 24 | :members: 25 | 26 | :mod:`asyncio_extras.threads` 27 | ============================= 28 | 29 | .. automodule:: asyncio_extras.threads 30 | :members: 31 | -------------------------------------------------------------------------------- /docs/versionhistory.rst: -------------------------------------------------------------------------------- 1 | Version history 2 | =============== 3 | 4 | This library adheres to `Semantic Versioning `_. 5 | 6 | **1.3.2** (2018-06-04) 7 | 8 | - Fixed regression on older Python 3.5 releases caused by the introduction of 9 | ``_get_running_loop()`` calls 10 | 11 | **1.3.1** (2018-06-03) 12 | 13 | - Fixed ``StopAsyncIteration`` exceptions from leaking from async context managers if ``return`` 14 | is used after ``yield`` 15 | - Fixed exception being reraised from the context block even if it's handled inside the context 16 | manager function 17 | - Added safeguard to prevent ``call_async()`` from being called from the event loop thread 18 | 19 | **1.3.0** (2016-12-03) 20 | 21 | - Removed the asynchronous generator implementation in favor of Nathaniel J. Smith's 22 | async_generator library. The ``yield_async()``, ``isasyncgenfunction()`` and 23 | ``async_generator()`` functions are now deprecated and will be removed in the next major release. 24 | 25 | **1.2.0** (2016-09-23) 26 | 27 | - Renamed the ``isasyncgeneratorfunction`` function to ``isasyncgenfunction`` to match the new 28 | function in the ``inspect`` module in Python 3.6 (the old name still works though) 29 | - Updated ``isasyncgenfunction`` to recognize native asynchronous generator functions in Python 3.6 30 | - Updated ``async_contextmanager`` to work with native async generator functions in Python 3.6 31 | - Changed asynchronous generators to use the updated ``__aiter__`` protocol on Python 3.5.2 and 32 | above 33 | - Added the ability to asynchronously iterate through ``AsyncFileWrapper`` just like with a regular 34 | file object 35 | 36 | **1.1.3** (2016-09-05) 37 | 38 | - Fixed error when throwing an exception into an asynchronous generator when using asyncio's debug 39 | mode 40 | 41 | **1.1.2** (2016-08-14) 42 | 43 | - Fixed concurrency issue with ``async with threadpool()`` when more than one coroutine from the 44 | same coroutine function is being run 45 | 46 | **1.1.1** (2016-04-14) 47 | 48 | - Import ``call_async`` to the ``asyncio_extras`` package namespace (this was missing from the 49 | 1.1.0 release) 50 | 51 | **1.1.0** (2016-04-04) 52 | 53 | - Added the ``asyncio_extras.threads.call_async`` function 54 | 55 | **1.0.0** (2016-04-08) 56 | 57 | - Initial release 58 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools >= 36.2.7", "wheel", "setuptools_scm >= 1.7.0"] 3 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = asyncio_extras 3 | description = Asynchronous generators, context managers and more for asyncio 4 | long_description = file: README.rst 5 | author = Alex Grönholm 6 | author_email = alex.gronholm@nextday.fi 7 | url = https://github.com/agronholm/asyncio_extras 8 | project_urls = 9 | Documentation = http://asyncio-extras.readthedocs.io/ 10 | Bug Tracker = https://github.com/agronholm/asyncio_extras/issues 11 | license = MIT 12 | license_file = LICENSE 13 | classifiers = 14 | Development Status :: 5 - Production/Stable 15 | Intended Audience :: Developers 16 | License :: OSI Approved :: MIT License 17 | Programming Language :: Python 18 | Programming Language :: Python :: 3 19 | Programming Language :: Python :: 3.5 20 | Programming Language :: Python :: 3.6 21 | Programming Language :: Python :: 3.7 22 | 23 | [options.extras_require] 24 | test = 25 | pytest 26 | pytest-asyncio 27 | pytest-cov 28 | doc = 29 | sphinx-autodoc-typehints 30 | 31 | [options] 32 | packages = find: 33 | install_requires = async_generator >= 1.3 34 | 35 | [tool:pytest] 36 | addopts = -rsx --cov --tb=short 37 | testpaths = tests 38 | 39 | [flake8] 40 | max-line-length = 99 41 | exclude = .tox 42 | ignore = E251, F403 43 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | use_scm_version={ 5 | 'version_scheme': 'post-release', 6 | 'local_scheme': 'dirty-tag' 7 | }, 8 | setup_requires=[ 9 | 'setuptools >= 36.2.7', 10 | 'setuptools_scm >= 1.7.0' 11 | ] 12 | ) 13 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | def pytest_ignore_collect(path, config): 5 | return path.basename.endswith('_py36.py') and sys.version_info < (3, 6) 6 | -------------------------------------------------------------------------------- /tests/test_contextmanager.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import pytest 4 | from async_generator import yield_ 5 | 6 | from asyncio_extras import async_contextmanager 7 | 8 | 9 | @pytest.mark.asyncio 10 | async def test_async_contextmanager(): 11 | @async_contextmanager 12 | async def dummycontext(value): 13 | await yield_(value) 14 | 15 | async with dummycontext(2) as value: 16 | assert value == 2 17 | 18 | 19 | @pytest.mark.asyncio 20 | async def test_async_contextmanager_three_awaits(): 21 | @async_contextmanager 22 | async def dummycontext(value): 23 | await asyncio.sleep(0.1) 24 | await yield_(value) 25 | await asyncio.sleep(0.1) 26 | 27 | async with dummycontext(2) as value: 28 | assert value == 2 29 | 30 | 31 | @pytest.mark.asyncio 32 | async def test_async_contextmanager_exception(): 33 | @async_contextmanager 34 | async def dummycontext(value): 35 | nonlocal exception 36 | try: 37 | await yield_(value) 38 | except Exception as e: 39 | exception = e 40 | 41 | exception = None 42 | async with dummycontext(2): 43 | raise Exception('foo') 44 | 45 | assert str(exception) == 'foo' 46 | 47 | 48 | @pytest.mark.asyncio 49 | async def test_async_contextmanager_exception_passthrough(): 50 | @async_contextmanager 51 | async def dummycontext(): 52 | await yield_() 53 | 54 | with pytest.raises(Exception) as exception: 55 | async with dummycontext(): 56 | raise Exception('foo') 57 | 58 | assert exception.match('^foo$') 59 | 60 | 61 | @pytest.mark.asyncio 62 | async def test_async_contextmanager_no_yield(): 63 | @async_contextmanager 64 | async def dummycontext(): 65 | pass 66 | 67 | with pytest.raises(StopAsyncIteration): 68 | async with dummycontext(): 69 | pass 70 | 71 | 72 | @pytest.mark.asyncio 73 | async def test_async_contextmanager_extra_yield(): 74 | @async_contextmanager 75 | async def dummycontext(value): 76 | await yield_(value) 77 | await yield_(3) 78 | 79 | with pytest.raises(RuntimeError) as exc: 80 | async with dummycontext(2) as value: 81 | assert value == 2 82 | 83 | assert str(exc.value) == "async generator didn't stop" 84 | 85 | 86 | @pytest.mark.asyncio 87 | async def test_return_after_yield(): 88 | @async_contextmanager 89 | async def dummycontext(value): 90 | try: 91 | await yield_(value) 92 | except RuntimeError: 93 | return 94 | 95 | async with dummycontext(2) as value: 96 | assert value == 2 97 | raise RuntimeError 98 | -------------------------------------------------------------------------------- /tests/test_contextmanager_py36.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import pytest 4 | 5 | from asyncio_extras import async_contextmanager 6 | 7 | 8 | @pytest.mark.asyncio 9 | async def test_async_contextmanager(): 10 | @async_contextmanager 11 | async def dummycontext(value): 12 | yield value 13 | 14 | async with dummycontext(2) as value: 15 | assert value == 2 16 | 17 | 18 | @pytest.mark.asyncio 19 | async def test_async_contextmanager_two_awaits(): 20 | @async_contextmanager 21 | async def dummycontext(value): 22 | await asyncio.sleep(0.1) 23 | yield value 24 | await asyncio.sleep(0.1) 25 | 26 | async with dummycontext(2) as value: 27 | assert value == 2 28 | 29 | 30 | @pytest.mark.asyncio 31 | async def test_async_contextmanager_exception(): 32 | @async_contextmanager 33 | async def dummycontext(value): 34 | nonlocal exception 35 | try: 36 | yield value 37 | except Exception as e: 38 | exception = e 39 | 40 | exception = None 41 | async with dummycontext(2): 42 | raise Exception('foo') 43 | 44 | assert str(exception) == 'foo' 45 | 46 | 47 | @pytest.mark.asyncio 48 | async def test_async_contextmanager_exception_passthrough(): 49 | @async_contextmanager 50 | async def dummycontext(): 51 | yield 52 | 53 | with pytest.raises(Exception) as exception: 54 | async with dummycontext(): 55 | raise Exception('foo') 56 | 57 | assert exception.match('^foo$') 58 | 59 | 60 | @pytest.mark.asyncio 61 | async def test_async_contextmanager_extra_yield(): 62 | @async_contextmanager 63 | async def dummycontext(value): 64 | yield value 65 | yield 3 66 | 67 | with pytest.raises(RuntimeError) as exc: 68 | async with dummycontext(2) as value: 69 | assert value == 2 70 | 71 | assert str(exc.value) == "async generator didn't stop" 72 | 73 | 74 | @pytest.mark.asyncio 75 | async def test_return_after_yield(): 76 | @async_contextmanager 77 | async def dummycontext(value): 78 | try: 79 | yield value 80 | except RuntimeError: 81 | return 82 | 83 | async with dummycontext(2) as value: 84 | assert value == 2 85 | raise RuntimeError 86 | -------------------------------------------------------------------------------- /tests/test_file.py: -------------------------------------------------------------------------------- 1 | from contextlib import closing 2 | from pathlib import Path 3 | 4 | import pytest 5 | 6 | from asyncio_extras import open_async 7 | 8 | 9 | @pytest.fixture(scope='module') 10 | def testdata(): 11 | return b''.join(bytes([i] * 1000) for i in range(10)) 12 | 13 | 14 | @pytest.fixture 15 | def testdatafile(tmpdir_factory, testdata): 16 | file = tmpdir_factory.mktemp('file').join('testdata') 17 | file.write(testdata) 18 | return Path(str(file)) 19 | 20 | 21 | @pytest.mark.asyncio 22 | async def test_read(testdatafile, testdata): 23 | async with open_async(testdatafile, 'rb') as f: 24 | data = await f.read() 25 | 26 | assert f.closed 27 | assert data == testdata 28 | 29 | 30 | @pytest.mark.asyncio 31 | async def test_write(testdatafile, testdata): 32 | async with open_async(testdatafile, 'ab') as f: 33 | await f.write(b'f' * 1000) 34 | 35 | assert testdatafile.stat().st_size == len(testdata) + 1000 36 | 37 | 38 | @pytest.mark.asyncio 39 | async def test_async_readchunks(testdatafile): 40 | value = 0 41 | async with open_async(testdatafile, 'rb') as f: 42 | async for chunk in f.async_readchunks(1000): 43 | assert chunk == bytes([value] * 1000) 44 | value += 1 45 | 46 | 47 | @pytest.mark.asyncio 48 | async def test_no_contextmanager(testdatafile, testdata): 49 | """Test that open_async() can be used without an async context manager.""" 50 | with closing(await open_async(testdatafile, 'rb')) as f: 51 | data = await f.read() 52 | 53 | assert f.closed 54 | assert data == testdata 55 | 56 | 57 | @pytest.mark.asyncio 58 | async def test_iteration(testdatafile, testdata): 59 | lines = [] 60 | async for line in open_async(testdatafile, 'rb'): 61 | lines.append(line) 62 | 63 | data = b''.join(lines) 64 | assert data == testdata 65 | -------------------------------------------------------------------------------- /tests/test_generator.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from asyncio_extras.asyncyield import yield_async 4 | from asyncio_extras.generator import async_generator, isasyncgenfunction, isasyncgeneratorfunction 5 | 6 | 7 | @pytest.mark.asyncio 8 | async def test_yield(): 9 | """Test that values yielded by yield_async() end up in the consumer.""" 10 | @async_generator 11 | async def dummygenerator(start): 12 | await yield_async(start) 13 | await yield_async(start + 1) 14 | await yield_async(start + 2) 15 | 16 | values = [] 17 | async for value in dummygenerator(2): 18 | values.append(value) 19 | 20 | assert values == [2, 3, 4] 21 | 22 | 23 | @pytest.mark.asyncio 24 | async def test_exception(): 25 | """Test that an exception raised directly in the async generator is properly propagated.""" 26 | @async_generator 27 | async def dummygenerator(start): 28 | await yield_async(start) 29 | raise ValueError('foo') 30 | 31 | values = [] 32 | with pytest.raises(ValueError) as exc: 33 | async for value in dummygenerator(2): 34 | values.append(value) 35 | 36 | assert values == [2] 37 | assert str(exc.value) == 'foo' 38 | 39 | 40 | @pytest.mark.asyncio 41 | async def test_awaitable_exception(event_loop): 42 | """ 43 | Test that an exception raised in something awaited by the async generator is sent back to 44 | the generator. 45 | 46 | """ 47 | def raise_error(): 48 | raise ValueError('foo') 49 | 50 | @async_generator 51 | async def dummygenerator(): 52 | try: 53 | await event_loop.run_in_executor(None, raise_error) 54 | except ValueError as e: 55 | await yield_async(e) 56 | 57 | values = [] 58 | async for value in dummygenerator(): 59 | values.append(value) 60 | 61 | assert str(values[0]) == 'foo' 62 | 63 | 64 | def test_isasyncgenfunction(): 65 | async def normalfunc(): 66 | pass 67 | 68 | assert not isasyncgenfunction(normalfunc) 69 | assert isasyncgenfunction(async_generator(normalfunc)) 70 | 71 | 72 | def test_deprecated_isasyncgenfunction(): 73 | async def normalfunc(): 74 | pass 75 | 76 | with pytest.warns(DeprecationWarning): 77 | assert not isasyncgeneratorfunction(normalfunc) 78 | assert isasyncgeneratorfunction(async_generator(normalfunc)) 79 | -------------------------------------------------------------------------------- /tests/test_generator_py36.py: -------------------------------------------------------------------------------- 1 | from asyncio_extras import isasyncgenfunction 2 | 3 | 4 | def test_isasyncgenfunction(): 5 | async def asyncgenfunc(): 6 | yield 1 7 | 8 | assert isasyncgenfunction(asyncgenfunc) 9 | -------------------------------------------------------------------------------- /tests/test_threads.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import threading 3 | from asyncio.futures import Future 4 | from concurrent.futures.thread import ThreadPoolExecutor 5 | from threading import current_thread, main_thread 6 | 7 | import pytest 8 | import time 9 | 10 | from asyncio_extras import threadpool, call_in_executor 11 | from asyncio_extras.threads import call_async 12 | 13 | 14 | class TestThreadpool: 15 | @pytest.mark.parametrize('already_in_thread', [False, True]) 16 | @pytest.mark.asyncio 17 | async def test_threadpool_decorator_noargs(self, event_loop, already_in_thread): 18 | """Test that threadpool() without arguments works as a decorator.""" 19 | @threadpool 20 | def func(x, y): 21 | nonlocal func_thread 22 | func_thread = threading.current_thread() 23 | return x + y 24 | 25 | event_loop_thread = threading.current_thread() 26 | func_thread = None 27 | callback = (event_loop.run_in_executor(None, func, 1, 2) if already_in_thread else 28 | func(1, 2)) 29 | assert await callback == 3 30 | assert func_thread is not event_loop_thread 31 | 32 | @pytest.mark.parametrize('executor', [None, ThreadPoolExecutor(1)]) 33 | @pytest.mark.asyncio 34 | async def test_threadpool_decorator(self, executor): 35 | """Test that threadpool() with an argument works as a decorator.""" 36 | @threadpool(executor) 37 | def func(x, y): 38 | nonlocal func_thread 39 | func_thread = threading.current_thread() 40 | return x + y 41 | 42 | event_loop_thread = threading.current_thread() 43 | func_thread = None 44 | assert await func(1, 2) == 3 45 | assert func_thread is not event_loop_thread 46 | 47 | @pytest.mark.asyncio 48 | async def test_threadpool_contextmanager(self): 49 | """Test that threadpool() with an argument works as a context manager.""" 50 | event_loop_thread = threading.current_thread() 51 | 52 | async with threadpool(): 53 | func_thread = threading.current_thread() 54 | 55 | assert threading.current_thread() is event_loop_thread 56 | assert func_thread is not event_loop_thread 57 | 58 | @pytest.mark.asyncio 59 | async def test_threadpool_contextmanager_exception(self): 60 | """Test that an exception raised from a threadpool block is properly propagated.""" 61 | event_loop_thread = threading.current_thread() 62 | 63 | with pytest.raises(ValueError) as exc: 64 | async with threadpool(): 65 | raise ValueError('foo') 66 | 67 | assert threading.current_thread() is event_loop_thread 68 | assert str(exc.value) == 'foo' 69 | 70 | @pytest.mark.asyncio 71 | async def test_threadpool_await_in_thread(self): 72 | """Test that attempting to await in a thread results in a RuntimeError.""" 73 | future = Future() 74 | 75 | with pytest.raises(RuntimeError) as exc: 76 | async with threadpool(): 77 | await future 78 | 79 | assert str(exc.value) == 'attempted to "await" in a worker thread' 80 | 81 | @pytest.mark.asyncio 82 | async def test_threadpool_multiple_coroutine(self): 83 | """ 84 | Test that "async with threadpool()" works when there are multiple coroutine objects present 85 | for the same coroutine function. 86 | 87 | """ 88 | async def sleeper(): 89 | await asyncio.sleep(0.2) 90 | async with threadpool(): 91 | time.sleep(0.3) 92 | 93 | coros = [sleeper() for _ in range(10)] 94 | await asyncio.gather(*coros) 95 | 96 | 97 | @pytest.mark.parametrize('executor', [None, ThreadPoolExecutor(1)]) 98 | @pytest.mark.asyncio 99 | async def test_call_in_executor(executor): 100 | """Test that call_in_thread actually runs the target in a worker thread.""" 101 | assert not await call_in_executor(lambda: current_thread() is main_thread(), 102 | executor=executor) 103 | 104 | 105 | class TestCallAsync: 106 | @pytest.mark.asyncio 107 | async def test_call_async_plain(self, event_loop): 108 | def runs_in_event_loop(worker_thread, x, y): 109 | assert current_thread() is not worker_thread 110 | return x + y 111 | 112 | def runs_in_worker_thread(): 113 | worker_thread = current_thread() 114 | return call_async(event_loop, runs_in_event_loop, worker_thread, 1, y=2) 115 | 116 | assert await event_loop.run_in_executor(None, runs_in_worker_thread) == 3 117 | 118 | @pytest.mark.asyncio 119 | async def test_call_async_coroutine(self, event_loop): 120 | async def runs_in_event_loop(worker_thread, x, y): 121 | assert current_thread() is not worker_thread 122 | await asyncio.sleep(0.2) 123 | return x + y 124 | 125 | def runs_in_worker_thread(): 126 | worker_thread = current_thread() 127 | return call_async(event_loop, runs_in_event_loop, worker_thread, 1, y=2) 128 | 129 | assert await event_loop.run_in_executor(None, runs_in_worker_thread) == 3 130 | 131 | @pytest.mark.asyncio 132 | async def test_call_async_exception(self, event_loop): 133 | def runs_in_event_loop(): 134 | raise ValueError('foo') 135 | 136 | with pytest.raises(ValueError) as exc: 137 | await event_loop.run_in_executor(None, call_async, event_loop, runs_in_event_loop) 138 | 139 | assert str(exc.value) == 'foo' 140 | 141 | @pytest.mark.asyncio 142 | async def test_call_async_from_event_loop_thread(self, event_loop): 143 | exc = pytest.raises(RuntimeError, call_async, event_loop, lambda: None) 144 | exc.match(r'call_async\(\) must not be called from an event loop thread') 145 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py35, py36, py37, flake8 3 | skip_missing_interpreters = true 4 | 5 | [testenv] 6 | extras = test 7 | commands = python -m pytest {posargs} 8 | 9 | [testenv:docs] 10 | deps = -rdocs/requirements.txt 11 | commands = python setup.py build_sphinx {posargs} 12 | usedevelop = true 13 | 14 | [testenv:flake8] 15 | deps = flake8 16 | commands = flake8 asyncio_extras tests 17 | skip_install = true 18 | --------------------------------------------------------------------------------