├── .coverage └── .keep ├── .coveragerc ├── .gitignore ├── .gitlab-ci.yml ├── CHANGELOG.md ├── CONTRIBUTORS.md ├── DEVELOPMENT.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── TODO.md ├── async_reduce ├── __init__.py ├── async_reduceable.py ├── async_reducer.py ├── aux.py └── hooks │ ├── __init__.py │ ├── base.py │ ├── debug.py │ └── statistics.py ├── examples ├── example_async_reduce.py ├── example_async_reduceable.py └── example_hooks.py ├── pytest.ini ├── setup.cfg ├── setup.py ├── shell.nix ├── tests ├── __init__.py ├── test_async_reduceable.py ├── test_async_reducer.py ├── test_aux_get_coroutine_function_location.py └── test_hooks │ ├── __init__.py │ ├── test_base_multiplehooks.py │ ├── test_debug.py │ ├── test_statistics_detail.py │ └── test_statistics_overall.py ├── tox.ini └── versioning.py /.coverage/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sirkonst/async-reduce/4200212558e49ae7e3120396b151dea781851b15/.coverage/.keep -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = async_reduce 3 | branch = True 4 | data_file = .coverage/data 5 | 6 | [report] 7 | precision = 2 8 | fail_under = 100 9 | 10 | exclude_lines = 11 | pragma: no cover 12 | if __name__ == '__main__': 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /* 2 | !/.coverage/.keep 3 | !/.coveragerc 4 | !/.gitignore 5 | !/.gitlab-ci.yml 6 | !/async_reduce/ 7 | !/CHANGELOG.md 8 | !/CONTRIBUTORS.md 9 | !/DEVELOPMENT.md 10 | !/examples/ 11 | !/LICENSE 12 | !/Makefile 13 | !/MANIFEST.in 14 | !/pytest.ini 15 | !/README.md 16 | !/setup.cfg 17 | !/setup.py 18 | !/tests/ 19 | !/TODO.md 20 | !/tox.ini 21 | !/versioning.py 22 | !shell.nix 23 | *__pycache__* 24 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | stages: 2 | - test 3 | - test_post 4 | - release 5 | 6 | 7 | # -- Test --------------------------------------------------------------------- 8 | 9 | test: 10 | stage: test 11 | image: python:$PYTHON_VERSION 12 | before_script: 13 | - pip install tox 14 | script: 15 | - make test PYTHON=$PYTHON_VERSION 16 | artifacts: 17 | paths: 18 | - .coverage/data* 19 | parallel: 20 | matrix: 21 | - PYTHON_VERSION: 22 | - "3.7" 23 | - "3.8" 24 | - "3.9" 25 | - "3.10" 26 | - "3.11" 27 | - "3.12" 28 | - "3.13" 29 | 30 | # -- Test Post ---------------------------------------------------------------- 31 | 32 | coverage: 33 | stage: test_post 34 | image: python 35 | before_script: 36 | - pip install tox -U 37 | script: 38 | - tox -e coverage_report 39 | dependencies: 40 | - test 41 | coverage: '/TOTAL\s+\d+\s+\d+\s+\d+\s+\d+\s+(\d+[.]\d+?)\%/' 42 | artifacts: 43 | paths: 44 | - htmlcov/ 45 | 46 | 47 | # -- Release ------------------------------------------------------------------ 48 | 49 | upload_to_pypi: 50 | stage: release 51 | image: python 52 | before_script: 53 | - pip install setuptools wheel twine -U 54 | script: 55 | - make build_dist LOCALVERSION= 56 | - make upload 57 | when: manual 58 | only: 59 | - master 60 | - /^release/.+$/ 61 | - tags 62 | environment: 63 | name: PyPi 64 | url: https://pypi.org/project/async-reduce/ 65 | dependencies: [] # don't fetch artifacts 66 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | 1.4 5 | --- 6 | 7 | * Add support python 3.13 8 | 9 | 1.3 10 | --- 11 | 12 | * Add support python 3.12 13 | 14 | 1.2 15 | --- 16 | 17 | * Add support python 3.11 18 | * Update dependencies 19 | 20 | 1.1 21 | --- 22 | 23 | * Fix for InvalidStateError 24 | 25 | 1.0 26 | --- 27 | 28 | * Drop support python's 3.5 and 3.6 29 | * Migrate pylava -> pylama 30 | * Fix a running of tests 31 | 32 | 0.5 33 | --- 34 | 35 | * Add support python 3.10 36 | 37 | 0.4.1 38 | ----- 39 | 40 | * Just updated README.md and tox.ini 41 | 42 | 0.4 43 | --- 44 | 45 | * Add support python 3.9 46 | 47 | 0.3 48 | --- 49 | 50 | * Add support python 3.8 51 | 52 | 0.2.1 53 | ----- 54 | 55 | * Protect reduced coroutine from cancelling 56 | 57 | 0.2 58 | --- 59 | 60 | * Add hooks mechanism 61 | 62 | 0.1.2 63 | ----- 64 | 65 | * Change coroutine auto identification 66 | * Add caveat about return value 67 | 68 | 0.1.1 69 | ----- 70 | 71 | * Add CONTRIBUTORS.md 72 | * Fix typos 73 | * Improve imports 74 | 75 | 0.1 76 | --- 77 | 78 | * First public version 79 | -------------------------------------------------------------------------------- /CONTRIBUTORS.md: -------------------------------------------------------------------------------- 1 | CONTRIBUTORS 2 | ============ 3 | 4 | * Konstantin Enchant 5 | * Renat Nasyrov 6 | -------------------------------------------------------------------------------- /DEVELOPMENT.md: -------------------------------------------------------------------------------- 1 | Development 2 | =========== 3 | 4 | Requires: 5 | 6 | * python >= 3.7 7 | 8 | 9 | Prepare the environment 10 | ----------------------- 11 | 12 | - Setup and activate virtual environment like: 13 | 14 | ```bash 15 | $ python3 -m venv .venv 16 | 17 | # for bash 18 | $ source .venv/bin/activate 19 | 20 | # for fish 21 | $ . .venv/bin/activate.fish 22 | ``` 23 | 24 | - Update pre-install dependencies: 25 | 26 | ```bash 27 | $ pip install 'setuptools >= 30.4' -U 28 | $ pip install pip -U 29 | ``` 30 | 31 | - Install development version: 32 | 33 | ```bash 34 | $ make install_dev 35 | ``` 36 | 37 | 38 | Run tests 39 | --------- 40 | 41 | ```bash 42 | # Full-tests in a clean environment: 43 | $ make test 44 | 45 | # Re-run full-tests in exists environment (more quickly): 46 | $ tox 47 | 48 | # Run simple tests on local environment: 49 | $ pytest 50 | ``` 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2019 Konstantin Enchant 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include versioning.py 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # interpreter version (ex.: 3.7) or use current 2 | PYTHON := $(PYTHON) 3 | 4 | all: test 5 | 6 | 7 | .PHONY: install 8 | install: 9 | pip install . 10 | 11 | 12 | .PHONY: install_dev 13 | install_dev: 14 | pip install --editable .[develop,testing] 15 | 16 | 17 | .PHONY: test 18 | test: 19 | if [ -z "$(PYTHON)" ]; then \ 20 | tox --recreate --skip-missing-interpreters; \ 21 | else \ 22 | tox --recreate -e coverage_clean,py$(PYTHON); \ 23 | fi; 24 | 25 | 26 | .PHONY: clean_dist 27 | clean_dist: 28 | if [ -d dist/ ]; then rm -rv dist/; fi; 29 | 30 | 31 | .PHONY: build_dist 32 | build_dist: clean_dist 33 | python setup.py sdist bdist_wheel 34 | 35 | 36 | .PHONY: _check_dist 37 | _check_dist: 38 | test -d dist/ || ( \ 39 | echo -e "\n--> [!] run 'make build_dist' before!\n" && exit 1 \ 40 | ) 41 | 42 | 43 | .PHONY: upload 44 | upload: _check_dist 45 | twine upload --skip-existing dist/* 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Python versions](https://img.shields.io/badge/python-3.7%2C%203.8%2C%203.9%2C%203.10%2C%203.11%2C%203.12%2C%203.13-green.svg)]() 2 | [![PyPI version](https://badge.fury.io/py/async-reduce.svg)](https://pypi.org/project/async-reduce/) 3 | 4 | 5 | About Async-Reduce 6 | ================== 7 | 8 | ``async_reduce(coroutine)`` allows aggregate all *similar simultaneous* 9 | ready to run `coroutine`s and reduce to running **only one** `coroutine`. 10 | Other aggregated `coroutine`s will get result from single `coroutine`. 11 | 12 | It can boost application performance in highly competitive execution of the 13 | similar asynchronous operations and reduce load for inner systems. 14 | 15 | 16 | Quick example 17 | ------------- 18 | 19 | ```python 20 | from async_reduce import async_reduce 21 | 22 | 23 | async def fetch_user_data(user_id: int) -> dict: 24 | """" Get user data from inner service """ 25 | url = 'http://inner-service/user/{}'.format(user_id) 26 | 27 | return await http.get(url, timeout=10).json() 28 | 29 | 30 | @web_server.router('/users/(\d+)') 31 | async def handler_user_detail(request, user_id: int): 32 | """ Handler for get detail information about user """ 33 | 34 | # all simultaneous requests of fetching user data for `user_id` will 35 | # reduced to single request 36 | user_data = await async_reduce( 37 | fetch_user_data(user_id) 38 | ) 39 | 40 | # sometimes ``async_reduce`` cannot detect similar coroutines and 41 | # you should provide special argument `ident` for manually determination 42 | user_statistics = await async_reduce( 43 | DataBase.query('user_statistics').where(id=user_id).fetch_one(), 44 | ident='db_user_statistics:{}'.format(user_id) 45 | ) 46 | 47 | return Response(...) 48 | ``` 49 | 50 | In that example without using ``async_reduce`` if client performs **N** 51 | simultaneous requests like `GET http://web_server/users/42` *web_server* 52 | performs **N** requests to *inner-service* and **N** queries to *database*. 53 | In total: **N** simultaneous requests emits **2 * N** requests to inner systems. 54 | 55 | With ``async_reduce`` if client performs **N** simultaneous requests *web_server* 56 | performs **one** request to *inner-service* and **one** query to *database*. 57 | In total: **N** simultaneous requests emit only **2** requests to inner systems. 58 | 59 | See other real [examples](https://github.com/sirkonst/async-reduce/tree/master/examples). 60 | 61 | 62 | Similar coroutines determination 63 | -------------------------------- 64 | 65 | ``async_reduce(coroutine)`` tries to detect similar coroutines by hashing 66 | local variables bounded on call. It does not work correctly if: 67 | 68 | * one of the arguments is not hashable 69 | * coroutine function is a method of class with specific state (like ORM) 70 | * coroutine function has closure to unhashable variable 71 | 72 | You can disable auto-determination by setting custom key to argument ``ident``. 73 | 74 | 75 | Use as decorator 76 | ---------------- 77 | 78 | Also library provide special decorator ``@async_reduceable()``, example: 79 | 80 | ```python 81 | from async_reduce import async_reduceable 82 | 83 | 84 | @async_reduceable() 85 | async def fetch_user_data(user_id: int) -> dict: 86 | """" Get user data from inner service """ 87 | url = 'http://inner-servicce/user/{}'.format(user_id) 88 | 89 | return await http.get(url, timeout=10).json() 90 | 91 | 92 | @web_server.router('/users/(\d+)') 93 | async def handler_user_detail(request, user_id: int): 94 | """ Handler for get detail information about user """ 95 | return await fetch_user_data(user_id) 96 | ``` 97 | 98 | 99 | Hooks 100 | ----- 101 | 102 | Library supports hooks. Add-on hooks: 103 | 104 | * **DebugHooks** - print about all triggered hooks 105 | * **StatisticsOverallHooks** - general statistics on the use of `async_reduce` 106 | * **StatisticsDetailHooks** - like `StatisticsOverallHooks` but detail statistics 107 | about all `coroutine` processed by `async_reduce` 108 | 109 | Example: 110 | 111 | ```python 112 | from async_reduce import AsyncReducer 113 | from async_reduce.hooks import DebugHooks 114 | 115 | # define custom async_reduce with hooks 116 | async_reduce = AsyncReducer(hooks=DebugHooks()) 117 | 118 | 119 | async def handler_user_detail(request, user_id: int): 120 | user_data = await async_reduce(fetch_user_data(user_id)) 121 | ``` 122 | 123 | See more detail example in [examples/example_hooks.py](https://github.com/sirkonst/async-reduce/blob/master/examples/example_hooks.py). 124 | 125 | You can write custom hooks via inherit from [BaseHooks](https://github.com/sirkonst/async-reduce/blob/master/async_reduce/hooks/base.py). 126 | 127 | 128 | Caveats 129 | ------- 130 | 131 | * If single `coroutine` raises exceptions all aggregated `coroutine`s will get 132 | same exception too 133 | 134 | * If single `coroutine` is stuck all aggregated `coroutine`s will stuck too. 135 | Limit execution time for `coroutine` and add retries (optional) to avoid it. 136 | 137 | * Be careful when return mutable value from `coroutine` because single value 138 | will shared. Prefer to use non-mutable value as coroutine return. 139 | 140 | 141 | Development 142 | ----------- 143 | 144 | See [DEVELOPMENT.md](https://github.com/sirkonst/async-reduce/blob/master/DEVELOPMENT.md). 145 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | TODO 2 | ==== 3 | 4 | * strict mypy 5 | -------------------------------------------------------------------------------- /async_reduce/__init__.py: -------------------------------------------------------------------------------- 1 | from async_reduce.async_reducer import async_reduce, AsyncReducer 2 | from async_reduce.async_reduceable import async_reduceable 3 | 4 | __all__ = ( 5 | 'async_reduce', 6 | 'async_reduceable', 7 | 'AsyncReducer', 8 | ) 9 | -------------------------------------------------------------------------------- /async_reduce/async_reduceable.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from typing import Callable, TypeVar 3 | 4 | from async_reduce import async_reduce, AsyncReducer 5 | 6 | T_AsyncFunc = TypeVar('T_AsyncFunc') 7 | 8 | 9 | def async_reduceable( 10 | reducer: AsyncReducer = async_reduce, 11 | ) -> Callable[[T_AsyncFunc], T_AsyncFunc]: 12 | """ 13 | Decorator to apply ``async_reduce(...)`` automatically for each coroutine 14 | function call. 15 | 16 | Example: 17 | 18 | # simple usage 19 | @async_reduceable() 20 | async def foo(arg): 21 | pass 22 | 23 | # with custom reducer class 24 | @async_reduceable(MyAsyncReducer) 25 | async def bar(arg): 26 | pass 27 | """ 28 | 29 | def wrapper(fn): 30 | @wraps(fn) 31 | async def wrap(*args, **kwargs): 32 | return await reducer(fn(*args, **kwargs)) 33 | 34 | return wrap 35 | 36 | return wrapper 37 | 38 | 39 | # --- 40 | # 41 | # @async_reduceable() 42 | # async def foo(arg1: int) -> str: 43 | # return 'foo' 44 | # 45 | # reveal_type(foo) 46 | # 47 | # 48 | # async def amain() -> None: 49 | # f = foo(2) 50 | # reveal_type(f) 51 | # 52 | # a = await f 53 | # reveal_type(a) 54 | -------------------------------------------------------------------------------- /async_reduce/async_reducer.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import inspect 3 | from functools import partial 4 | from typing import Coroutine, Tuple, Any, TypeVar, Awaitable, Optional, Dict 5 | 6 | from async_reduce.aux import get_coroutine_function_location 7 | from async_reduce.hooks.base import BaseHooks 8 | 9 | T_Result = TypeVar('T_Result') 10 | 11 | 12 | class AsyncReducer: 13 | def __init__(self, hooks: Optional[BaseHooks] = None) -> None: 14 | self._running: Dict[str, asyncio.Future] = {} 15 | self._hooks = hooks 16 | 17 | def __call__( 18 | self, 19 | coro: Coroutine[Any, Any, T_Result], 20 | *, 21 | ident: Optional[str] = None 22 | ) -> Awaitable[T_Result]: 23 | # assert inspect.getcoroutinestate(coro) == inspect.CORO_CREATED 24 | 25 | if not ident: 26 | ident = self._auto_ident(coro) 27 | 28 | if self._hooks: 29 | self._hooks.on_apply_for(coro, ident) 30 | 31 | future, created = self._get_or_create_future(ident) 32 | 33 | if created: 34 | self._running[ident] = future 35 | coro_runner = self._runner(ident, coro, future) 36 | 37 | if self._hooks: 38 | self._hooks.on_executing_for(coro, ident) 39 | 40 | asyncio.create_task(coro_runner) 41 | else: 42 | if self._hooks: 43 | self._hooks.on_reducing_for(coro, ident) 44 | 45 | coro.close() 46 | del coro 47 | 48 | return self._waiter(future) 49 | 50 | @staticmethod 51 | def _auto_ident(coro: Coroutine[Any, Any, T_Result]) -> str: 52 | func_loc = get_coroutine_function_location(coro) 53 | 54 | try: 55 | hsh = hash(tuple(inspect.getcoroutinelocals(coro).items())) 56 | except TypeError: 57 | raise TypeError( 58 | 'Unable to auto calculate identity for coroutine because using' 59 | ' unhashable arguments, you should set `ident` manual like:' 60 | '\n\tawait async_reduce({}(...), ident="YOU-IDENT-FOR-THAT")' 61 | ''.format(getattr(coro, '__name__')) 62 | ) 63 | 64 | return '{}()'.format(func_loc, hsh) 65 | 66 | def _get_or_create_future(self, ident: str) -> Tuple[asyncio.Future, bool]: 67 | f = self._running.get(ident, None) 68 | if f is not None: 69 | return f, False 70 | else: 71 | f = asyncio.Future() 72 | self._running[ident] = f 73 | return f, True 74 | 75 | async def _runner( 76 | self, 77 | ident: str, 78 | coro: Coroutine[Any, Any, T_Result], 79 | future: asyncio.Future, 80 | ) -> None: 81 | try: 82 | result = await coro 83 | except (Exception, asyncio.CancelledError) as e: 84 | future.set_exception(e) 85 | 86 | if self._hooks: 87 | self._hooks.on_exception_for(coro, ident, e) 88 | else: 89 | future.set_result(result) 90 | 91 | if self._hooks: 92 | self._hooks.on_result_for(coro, ident, result) 93 | finally: 94 | del self._running[ident] 95 | 96 | @classmethod 97 | async def _waiter(cls, future: asyncio.Future) -> T_Result: 98 | wait_future: asyncio.Future = asyncio.Future() 99 | 100 | future.add_done_callback( 101 | partial(cls._set_wait_future_result, wait_future=wait_future) 102 | ) 103 | 104 | return await wait_future 105 | 106 | @staticmethod 107 | def _set_wait_future_result( 108 | result_future: asyncio.Future, wait_future: asyncio.Future 109 | ) -> None: 110 | if wait_future.cancelled(): 111 | return 112 | 113 | try: 114 | result = result_future.result() 115 | set_func = wait_future.set_result 116 | except (Exception, asyncio.CancelledError) as e: 117 | result = e 118 | set_func = wait_future.set_exception 119 | 120 | set_func(result) 121 | 122 | 123 | async_reduce = AsyncReducer() 124 | 125 | # --- 126 | # 127 | # reveal_type(async_reduce) 128 | # 129 | # async def fetch(url: str) -> str: 130 | # print(url) 131 | # return url 132 | # 133 | # 134 | # async def amain() -> None: 135 | # f = async_reduce(fetch('a')) 136 | # reveal_type(f) 137 | # 138 | # a = await f 139 | # reveal_type(a) 140 | -------------------------------------------------------------------------------- /async_reduce/aux.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from typing import Coroutine, Any 3 | 4 | _cached_sys_path = sorted( 5 | set(path for path in sys.path if path and path != '/') 6 | ) 7 | 8 | 9 | def get_coroutine_function_location(coro: Coroutine[Any, Any, Any]) -> str: 10 | """ 11 | Get relative location for coroutine function. 12 | """ 13 | code = getattr(coro, 'cr_code', None) 14 | if not code: # for generator base coroutine 15 | code = getattr(coro, 'gi_code') 16 | 17 | filename = code.co_filename 18 | file_path = next( 19 | ( 20 | filename[len(path):] 21 | for path in _cached_sys_path 22 | if filename.startswith(path) 23 | ), 24 | None, 25 | ) 26 | if file_path: 27 | module_path = file_path.lstrip('/').rstrip('.py').replace('/', '.') 28 | else: 29 | module_path = '' 30 | 31 | return '{}:{}'.format(module_path, getattr(coro, '__qualname__')) 32 | -------------------------------------------------------------------------------- /async_reduce/hooks/__init__.py: -------------------------------------------------------------------------------- 1 | from async_reduce.hooks.debug import DebugHooks 2 | from async_reduce.hooks.statistics import ( 3 | StatisticsOverallHooks, 4 | StatisticsDetailHooks, 5 | ) 6 | 7 | __all__ = ( 8 | 'DebugHooks', 9 | 'StatisticsOverallHooks', 10 | 'StatisticsDetailHooks', 11 | ) 12 | -------------------------------------------------------------------------------- /async_reduce/hooks/base.py: -------------------------------------------------------------------------------- 1 | from asyncio import CancelledError 2 | from typing import Any, Coroutine, Union 3 | 4 | 5 | class BaseHooks: 6 | """ 7 | Interface for implementation of hooks. 8 | """ 9 | 10 | def on_apply_for(self, coro: Coroutine[Any, Any, Any], ident: str) -> None: 11 | """ 12 | Calls when ``async_reduce`` apply to coroutine. 13 | """ 14 | 15 | def on_executing_for( 16 | self, coro: Coroutine[Any, Any, Any], ident: str 17 | ) -> None: 18 | """ 19 | Calls when coroutine executing as aggregated coroutine. 20 | """ 21 | 22 | def on_reducing_for( 23 | self, coro: Coroutine[Any, Any, Any], ident: str 24 | ) -> None: 25 | """ 26 | Calls when coroutine reduced to aggregated coroutine. 27 | """ 28 | 29 | def on_result_for( 30 | self, coro: Coroutine[Any, Any, Any], ident: str, result: Any 31 | ) -> None: 32 | """ 33 | Calls when aggregated coroutine returns value. 34 | """ 35 | 36 | def on_exception_for( 37 | self, 38 | coro: Coroutine[Any, Any, Any], 39 | ident: str, 40 | exception: Union[Exception, CancelledError], 41 | ) -> None: 42 | """ 43 | Calls when aggregated coroutine raises exception. 44 | """ 45 | 46 | def __and__(self, other: 'BaseHooks') -> 'MultipleHooks': 47 | if isinstance(other, MultipleHooks): 48 | return other & self 49 | 50 | return MultipleHooks(self, other) 51 | 52 | 53 | class MultipleHooks(BaseHooks): 54 | """ 55 | Internal class to gather multiple hooks (via operator `&`). 56 | 57 | Each hook will be called in the addition sequence. 58 | """ 59 | 60 | def __init__(self, *hooks: BaseHooks) -> None: 61 | self.hooks_list = [*hooks] 62 | 63 | def __and__(self, other: BaseHooks) -> 'MultipleHooks': 64 | if isinstance(other, MultipleHooks): 65 | self.hooks_list.extend(other.hooks_list) 66 | return self 67 | 68 | self.hooks_list.append(other) 69 | return self 70 | 71 | def __len__(self) -> int: 72 | """Count of gathered hooks""" 73 | return len(self.hooks_list) 74 | 75 | def on_apply_for(self, coro: Coroutine[Any, Any, Any], ident: str) -> None: 76 | for hooks in self.hooks_list: 77 | hooks.on_apply_for(coro, ident) 78 | 79 | def on_executing_for( 80 | self, coro: Coroutine[Any, Any, Any], ident: str 81 | ) -> None: 82 | for hooks in self.hooks_list: 83 | hooks.on_executing_for(coro, ident) 84 | 85 | def on_reducing_for( 86 | self, coro: Coroutine[Any, Any, Any], ident: str 87 | ) -> None: 88 | for hooks in self.hooks_list: 89 | hooks.on_reducing_for(coro, ident) 90 | 91 | def on_result_for( 92 | self, coro: Coroutine[Any, Any, Any], ident: str, result: Any 93 | ) -> None: 94 | for hooks in self.hooks_list: 95 | hooks.on_result_for(coro, ident, result) 96 | 97 | def on_exception_for( 98 | self, 99 | coro: Coroutine[Any, Any, Any], 100 | ident: str, 101 | exception: Union[Exception, CancelledError], 102 | ) -> None: 103 | for hooks in self.hooks_list: 104 | hooks.on_exception_for(coro, ident, exception) 105 | -------------------------------------------------------------------------------- /async_reduce/hooks/debug.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from asyncio import CancelledError 3 | from typing import IO, Any, Coroutine, Union 4 | 5 | from async_reduce.hooks.base import BaseHooks 6 | 7 | 8 | class DebugHooks(BaseHooks): 9 | """ 10 | Print about all triggered hooks. 11 | """ 12 | 13 | def __init__(self, stream: IO[str] = sys.stderr) -> None: 14 | self._steam = stream 15 | 16 | def on_apply_for(self, coro: Coroutine[Any, Any, Any], ident: str) -> None: 17 | print( 18 | '[{}] apply async_reduce() for {}'.format(ident, coro), 19 | file=self._steam, 20 | ) 21 | 22 | def on_executing_for( 23 | self, coro: Coroutine[Any, Any, Any], ident: str 24 | ) -> None: 25 | print('[{}] executing for {}'.format(ident, coro), file=self._steam) 26 | 27 | def on_reducing_for( 28 | self, coro: Coroutine[Any, Any, Any], ident: str 29 | ) -> None: 30 | print('[{}] reducing for {}'.format(ident, coro), file=self._steam) 31 | 32 | def on_result_for( 33 | self, coro: Coroutine[Any, Any, Any], ident: str, result: Any 34 | ) -> None: 35 | print( 36 | '[{}] result for {}: {}'.format(ident, coro, result), 37 | file=self._steam, 38 | ) 39 | 40 | def on_exception_for( 41 | self, 42 | coro: Coroutine[Any, Any, Any], 43 | ident: str, 44 | exception: Union[Exception, CancelledError], 45 | ) -> None: 46 | print( 47 | '[{}] get exception for {}: {}'.format(ident, coro, exception), 48 | file=self._steam, 49 | ) 50 | -------------------------------------------------------------------------------- /async_reduce/hooks/statistics.py: -------------------------------------------------------------------------------- 1 | from asyncio import CancelledError 2 | from typing import Any, Coroutine, Counter, Union 3 | 4 | from async_reduce.aux import get_coroutine_function_location 5 | from async_reduce.hooks.base import BaseHooks 6 | 7 | 8 | class StatisticsOverallHooks(BaseHooks): 9 | """ 10 | General statistics: 11 | 12 | * total - count of all ``async_reduce`` calls 13 | * executed - count of aggregated coroutines calls 14 | * reduced - count of reduced to aggregated coroutines 15 | * errors - count of raised errors from coroutines 16 | """ 17 | 18 | def __init__(self) -> None: 19 | self.total = 0 20 | self.executed = 0 21 | self.reduced = 0 22 | self.errors = 0 23 | 24 | def __str__(self) -> str: 25 | return 'Stats(total={}, executed={}, reduced={}, errors={})'.format( 26 | self.total, self.executed, self.reduced, self.errors 27 | ) 28 | 29 | def on_apply_for(self, coro: Coroutine[Any, Any, Any], ident: str) -> None: 30 | self.total += 1 31 | 32 | def on_result_for( 33 | self, coro: Coroutine[Any, Any, Any], ident: str, result: Any 34 | ) -> None: 35 | self.executed += 1 36 | 37 | def on_reducing_for( 38 | self, coro: Coroutine[Any, Any, Any], ident: str 39 | ) -> None: 40 | self.reduced += 1 41 | 42 | def on_exception_for( 43 | self, 44 | coro: Coroutine[Any, Any, Any], 45 | ident: str, 46 | exception: Union[Exception, CancelledError], 47 | ) -> None: 48 | self.errors += 1 49 | 50 | 51 | class StatisticsDetailHooks(BaseHooks): 52 | def __init__(self) -> None: 53 | self.total = Counter[str]() 54 | self.executed = Counter[str]() 55 | self.reduced = Counter[str]() 56 | self.errors = Counter[str]() 57 | 58 | def __str__(self) -> str: 59 | return ''.join( 60 | ( 61 | 'Top total:\n', 62 | ''.join( 63 | '\t{}: {}\n'.format(name, count) 64 | for name, count in self.total.most_common() 65 | ), 66 | 'Top executed:\n', 67 | ''.join( 68 | '\t{}: {}\n'.format(name, count) 69 | for name, count in self.executed.most_common() 70 | ), 71 | 'Top reduced:\n', 72 | ''.join( 73 | '\t{}: {}\n'.format(name, count) 74 | for name, count in self.reduced.most_common() 75 | ), 76 | 'Top errors:\n', 77 | ''.join( 78 | '\t{}: {}\n'.format(name, count) 79 | for name, count in self.errors.most_common() 80 | ), 81 | ) 82 | ) 83 | 84 | def on_apply_for(self, coro: Coroutine[Any, Any, Any], ident: str) -> None: 85 | self.total[get_coroutine_function_location(coro)] += 1 86 | 87 | def on_executing_for( 88 | self, coro: Coroutine[Any, Any, Any], ident: str 89 | ) -> None: 90 | self.executed[get_coroutine_function_location(coro)] += 1 91 | 92 | def on_reducing_for( 93 | self, coro: Coroutine[Any, Any, Any], ident: str 94 | ) -> None: 95 | self.reduced[get_coroutine_function_location(coro)] += 1 96 | 97 | def on_exception_for( 98 | self, 99 | coro: Coroutine[Any, Any, Any], 100 | ident: str, 101 | exception: Union[Exception, CancelledError], 102 | ) -> None: 103 | self.errors[get_coroutine_function_location(coro)] += 1 104 | -------------------------------------------------------------------------------- /examples/example_async_reduce.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import time 3 | 4 | from async_reduce import async_reduce 5 | 6 | 7 | async def fetch(url): 8 | print('- fetch page: ', url) 9 | await asyncio.sleep(1) 10 | return time.time() 11 | 12 | 13 | async def amain(): 14 | print('-- First simultaneous run') 15 | coros = [ 16 | async_reduce(fetch('/page')) for _ in range(10) 17 | ] 18 | results = await asyncio.gather(*coros) 19 | 20 | print('Results:') 21 | print('\n'.join(map(str, results))) 22 | 23 | print('-- Second simultaneous run') 24 | coros = [ 25 | async_reduce(fetch('/page')) for _ in range(10) 26 | ] 27 | results = await asyncio.gather(*coros) 28 | 29 | print('Results:') 30 | print('\n'.join(map(str, results))) 31 | 32 | print('-- Third simultaneous run with differences') 33 | coros = [ 34 | async_reduce(fetch('/page/{}'.format(i))) for i in range(10) 35 | ] 36 | results = await asyncio.gather(*coros) 37 | 38 | print('Results:') 39 | print('\n'.join(map(str, results))) 40 | 41 | 42 | def main(): 43 | asyncio.run(amain()) 44 | 45 | 46 | if __name__ == '__main__': 47 | main() 48 | -------------------------------------------------------------------------------- /examples/example_async_reduceable.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import time 3 | 4 | from async_reduce import async_reduceable 5 | 6 | 7 | @async_reduceable() 8 | async def fetch(url): 9 | print('- fetch page: ', url) 10 | await asyncio.sleep(1) 11 | return time.time() 12 | 13 | 14 | async def amain(): 15 | print('-- Simultaneous run') 16 | coros = [ 17 | fetch('/page') for _ in range(10) 18 | ] 19 | results = await asyncio.gather(*coros) 20 | 21 | print('Results:') 22 | print('\n'.join(map(str, results))) 23 | 24 | 25 | def main(): 26 | asyncio.run(amain()) 27 | 28 | 29 | if __name__ == '__main__': 30 | main() 31 | -------------------------------------------------------------------------------- /examples/example_hooks.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import random 3 | import time 4 | 5 | from async_reduce import AsyncReducer, async_reduceable 6 | from async_reduce.hooks import ( 7 | DebugHooks, 8 | StatisticsOverallHooks, 9 | StatisticsDetailHooks, 10 | ) 11 | 12 | stats_overall = StatisticsOverallHooks() 13 | stats_detail = StatisticsDetailHooks() 14 | async_reduce = AsyncReducer(hooks=DebugHooks() & stats_overall & stats_detail) 15 | 16 | 17 | @async_reduceable(async_reduce) 18 | async def foo(sec): 19 | await asyncio.sleep(sec / 100) 20 | return time.time() 21 | 22 | 23 | async def amain(): 24 | coros = [ 25 | foo(random.randint(1, 3)) for _ in range(5) 26 | ] 27 | await asyncio.gather(*coros) 28 | 29 | print('--- Overall stats ---') 30 | print(stats_overall) 31 | print('--- Detail stats ---') 32 | print(stats_detail) 33 | 34 | 35 | def main(): 36 | asyncio.run(amain()) 37 | 38 | 39 | if __name__ == '__main__': 40 | main() 41 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | asyncio_mode = strict 3 | 4 | # make all warnings as error by default 5 | filterwarnings = 6 | error 7 | ignore::DeprecationWarning:tests.test_aux_get_coroutine_function_location:24 8 | ignore: "@coroutine" decorator is deprecated since Python 3.8, use "async def" instead 9 | ignore: 'maxsplit' is passed as positional argument:DeprecationWarning:pylama 10 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = async_reduce 3 | # version = < see setup.py > 4 | description = Reducer for similar simultaneously coroutines 5 | long_description = file: README.md 6 | long_description_content_type = text/markdown 7 | url = https://github.com/sirkonst/async-reduce 8 | author = Konstantin Enchant 9 | author_email = sirkonst@gmail.com 10 | maintainer = Konstantin Enchant 11 | maintainer_email = sirkonst@gmail.com 12 | keywords = asyncio, reduce 13 | license = MIT 14 | classifiers = 15 | Development Status :: 5 - Production/Stable 16 | License :: OSI Approved :: MIT License 17 | Programming Language :: Python :: 3 :: Only 18 | Programming Language :: Python :: 3.7 19 | Programming Language :: Python :: 3.8 20 | Programming Language :: Python :: 3.9 21 | Programming Language :: Python :: 3.10 22 | Programming Language :: Python :: 3.11 23 | Programming Language :: Python :: 3.12 24 | Programming Language :: Python :: 3.13 25 | 26 | [options] 27 | python_requires = >=3.7 28 | packages = find: 29 | install_requires = 30 | 31 | [options.packages.find] 32 | exclude = 33 | tests 34 | 35 | [options.extras_require] 36 | develop = 37 | tox~=4.23 38 | wheel~=0.45 39 | twine~=6.0 40 | testing = 41 | setuptools~=68.2; python_version>="3.12" # needed for pylama 42 | pytest~=7.2 43 | pytest-asyncio~=0.20 44 | coverage~=7.0 45 | pylama~=8.4 46 | mypy~=0.991 47 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from setuptools import setup 5 | from versioning import version 6 | 7 | 8 | localversion = os.environ.get('LOCALVERSION', 'auto') or None 9 | 10 | setup( 11 | setup_requires=['setuptools>=30.4'], 12 | version=version(1, 4, localversion=localversion), 13 | ) 14 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | with import { }; 2 | 3 | let 4 | pythonPackages = python313Packages; 5 | pythonVenvDir = ".local/${pythonPackages.python.name}"; 6 | envPackages = [ 7 | gitMinimal 8 | ]; 9 | preInstallPypiPackages = [ 10 | "blue" 11 | "mypy" 12 | "pylama" 13 | "tox" 14 | "ipykernel" 15 | ]; 16 | in mkShell { 17 | name = "pythonProjectDevEnv"; 18 | venvDir = pythonVenvDir; 19 | 20 | buildInputs = with pythonPackages; [ 21 | python 22 | venvShellHook 23 | ] ++ envPackages; 24 | 25 | postVenvCreation = let 26 | toPypiInstall = lib.concatStringsSep " " preInstallPypiPackages; 27 | in '' 28 | unset SOURCE_DATE_EPOCH # allow pip to install wheels 29 | PIP_DISABLE_PIP_VERSION_CHECK=1 pip install ${toPypiInstall} 30 | ''; 31 | 32 | postShellHook = '' 33 | # fix for cython and ipython 34 | export LD_LIBRARY_PATH=${lib.makeLibraryPath [stdenv.cc.cc]} 35 | 36 | # allow pip to install wheels 37 | unset SOURCE_DATE_EPOCH 38 | 39 | export PIP_DISABLE_PIP_VERSION_CHECK=1 40 | 41 | # upgrade venv if python package was updated 42 | python -m venv --upgrade ${pythonVenvDir} 43 | ''; 44 | } 45 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sirkonst/async-reduce/4200212558e49ae7e3120396b151dea781851b15/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_async_reduceable.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import pytest 4 | 5 | from async_reduce import async_reduceable 6 | from async_reduce.async_reducer import AsyncReducer 7 | 8 | pytestmark = pytest.mark.asyncio 9 | 10 | 11 | async def test_decorator_default(): 12 | @async_reduceable() 13 | async def foo(arg, *, kw): 14 | foo.await_count += 1 15 | 16 | return 'result {} {}'.format(arg, kw) 17 | 18 | foo.await_count = 0 19 | 20 | coros = [foo('arg', kw='kw'), foo('arg', kw='kw')] 21 | results = await asyncio.gather(*coros) 22 | 23 | assert foo.await_count == 1 24 | assert all(res == 'result arg kw' for res in results) 25 | 26 | 27 | async def test_decorator_with_arg(): 28 | async def mock(): 29 | mock.await_count += 1 30 | 31 | mock.await_count = 0 32 | 33 | reducer = AsyncReducer() 34 | 35 | @async_reduceable(reducer) 36 | async def foo(arg, *, kw): 37 | """My foo doc""" 38 | await mock() 39 | 40 | return 'result {} {}'.format(arg, kw) 41 | 42 | assert foo.__name__ == 'foo' 43 | assert foo.__doc__ == 'My foo doc' 44 | 45 | coros = [foo('arg', kw='kw'), foo('arg', kw='kw')] 46 | results = await asyncio.gather(*coros) 47 | 48 | assert mock.await_count == 1 49 | assert all(r == 'result arg kw' for r in results) 50 | -------------------------------------------------------------------------------- /tests/test_async_reducer.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from contextlib import suppress 3 | 4 | import pytest 5 | 6 | from async_reduce import async_reduce 7 | 8 | pytestmark = pytest.mark.asyncio 9 | 10 | 11 | @pytest.mark.parametrize('count', [1, 2, 5, 10, 100, 1000]) 12 | async def test_simultaneity(count): 13 | result = object() 14 | 15 | async def mock(a, b, *, c): 16 | mock.await_count += 1 17 | return result 18 | 19 | mock.await_count = 0 20 | 21 | coros_1 = [async_reduce(mock(1, 2, c=3)) for _ in range(count)] 22 | 23 | results = await asyncio.gather(*coros_1) 24 | 25 | # mock.assert_awaited_once_with(*args, **kwargs) 26 | assert all(res == result for res in results) 27 | 28 | coros_2 = [async_reduce(mock(1, 2, c=3)) for _ in range(count)] 29 | 30 | results = await asyncio.gather(*coros_2) 31 | 32 | assert mock.await_count == 2 33 | assert all(res == result for res in results) 34 | 35 | 36 | class MyTestError(Exception): 37 | pass 38 | 39 | 40 | @pytest.mark.parametrize( 41 | 'count', 42 | [ 43 | 1, 44 | 2, 45 | 5, 46 | 10, 47 | 100, # 1000 48 | ], 49 | ) 50 | @pytest.mark.parametrize( 51 | 'error', 52 | [ 53 | MyTestError, 54 | Exception, 55 | asyncio.TimeoutError, 56 | asyncio.CancelledError, 57 | ], 58 | ) 59 | async def test_simultaneity_raise(count, error): 60 | async def mock(): 61 | mock.await_count += 1 62 | raise error('test error') 63 | 64 | mock.await_count = 0 65 | 66 | coros_1 = [async_reduce(mock()) for _ in range(count)] 67 | 68 | results = await asyncio.gather(*coros_1, return_exceptions=True) 69 | 70 | assert mock.await_count == 1 71 | for res in results: 72 | assert isinstance(res, error) 73 | if not isinstance(res, asyncio.CancelledError): 74 | assert str(res) == 'test error' 75 | 76 | coros_2 = [async_reduce(mock()) for _ in range(count)] 77 | 78 | results = await asyncio.gather(*coros_2, return_exceptions=True) 79 | 80 | assert mock.await_count == 2 81 | for res in results: 82 | assert isinstance(res, error) 83 | if not isinstance(res, asyncio.CancelledError): 84 | assert str(res) == 'test error' 85 | 86 | 87 | @pytest.mark.parametrize( 88 | 'value', [{}, {'a': 'b'}, object(), type('MyClass', (), {})] 89 | ) 90 | async def test_ident(value): 91 | async def foo(arg): 92 | return 'result' 93 | 94 | coro = foo({}) 95 | 96 | with pytest.raises(TypeError) as e: 97 | await async_reduce(coro) 98 | 99 | assert str(e.value) == ( 100 | 'Unable to auto calculate identity for coroutine because' 101 | ' using unhashable arguments, you should set `ident` manual like:' 102 | '\n\tawait async_reduce(foo(...), ident="YOU-IDENT-FOR-THAT")' 103 | ) 104 | 105 | result = await async_reduce(coro, ident='foo_{}'.format(id(value))) 106 | assert result == 'result' 107 | 108 | 109 | async def test_prevent_cancelling(): 110 | async def foo(): 111 | await asyncio.sleep(2) 112 | return 'result' 113 | 114 | coro_1 = async_reduce(foo()) 115 | coro_2 = async_reduce(foo()) 116 | 117 | with suppress(asyncio.TimeoutError): 118 | await asyncio.wait_for(coro_1, 1) 119 | 120 | result = await coro_2 121 | assert result == 'result' 122 | 123 | 124 | async def test_all_waiters_cancelled(caplog): 125 | caplog.set_level('ERROR', logger='asyncio') 126 | 127 | async def foo(): 128 | await asyncio.sleep(1) 129 | return 'result' 130 | 131 | coro_1 = async_reduce(foo()) 132 | coro_2 = async_reduce(foo()) 133 | 134 | gather = asyncio.gather(coro_1, coro_2) 135 | try: 136 | await asyncio.wait_for(gather, 1) 137 | except asyncio.TimeoutError: 138 | with suppress(asyncio.CancelledError): 139 | await gather 140 | 141 | # check for "asyncio.base_futures.InvalidStateError: invalid state" 142 | assert 'invalid state' not in caplog.text 143 | else: 144 | assert False 145 | -------------------------------------------------------------------------------- /tests/test_aux_get_coroutine_function_location.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import sys 3 | 4 | import pytest 5 | 6 | from async_reduce.aux import get_coroutine_function_location 7 | 8 | pytestmark = pytest.mark.asyncio 9 | 10 | 11 | def teardown_module(module): 12 | """ 13 | Reload ``async_reduce.aux`` for reset module caches after tests 14 | """ 15 | from async_reduce import aux 16 | 17 | importlib.reload(aux) 18 | 19 | 20 | async def coro_function(): 21 | pass 22 | 23 | 24 | async def test_coro(): 25 | coro = coro_function() 26 | 27 | result = get_coroutine_function_location(coro) 28 | assert result == ( 29 | 'tests.test_aux_get_coroutine_function_location:coro_function' 30 | ) 31 | 32 | coro.close() 33 | 34 | 35 | @pytest.mark.skipif( 36 | sys.version_info >= (3, 10), reason='asyncio.coroutine has been removed' 37 | ) 38 | async def test_gen(): 39 | import asyncio 40 | 41 | @asyncio.coroutine 42 | def gen_function(): 43 | pass 44 | 45 | coro = gen_function() 46 | 47 | result = get_coroutine_function_location(coro) 48 | assert result == ('asyncio.coroutines:test_gen..gen_function') 49 | 50 | coro.close() 51 | 52 | 53 | @pytest.mark.parametrize('value', [[], [''], ['/'], ['/other/']]) 54 | async def test_unmatched_with_sys_path(monkeypatch, value): 55 | monkeypatch.setattr('sys.path', value) 56 | 57 | from async_reduce import aux 58 | 59 | importlib.reload(aux) 60 | 61 | coro = coro_function() 62 | 63 | result = aux.get_coroutine_function_location(coro) 64 | assert result == ':coro_function' 65 | 66 | coro.close() 67 | -------------------------------------------------------------------------------- /tests/test_hooks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sirkonst/async-reduce/4200212558e49ae7e3120396b151dea781851b15/tests/test_hooks/__init__.py -------------------------------------------------------------------------------- /tests/test_hooks/test_base_multiplehooks.py: -------------------------------------------------------------------------------- 1 | from asyncio import CancelledError 2 | from collections import Counter 3 | from typing import Any, Coroutine, Union 4 | 5 | import pytest 6 | 7 | from async_reduce.hooks.base import BaseHooks, MultipleHooks 8 | 9 | 10 | def test_base_and_base(): 11 | hook_1 = BaseHooks() 12 | hook_2 = BaseHooks() 13 | hooks = hook_1 & hook_2 14 | 15 | assert isinstance(hooks, MultipleHooks) 16 | assert len(hooks) == 2 17 | assert hooks.hooks_list == [hook_1, hook_2] 18 | 19 | 20 | def test_multiple_and_base(): 21 | hook_1 = BaseHooks() 22 | hooks = MultipleHooks() & hook_1 23 | 24 | assert isinstance(hooks, MultipleHooks) 25 | assert len(hooks) == 1 26 | assert hooks.hooks_list == [hook_1] 27 | 28 | 29 | def test_base_and_multiple(): 30 | hook_1 = BaseHooks() 31 | hooks = hook_1 & MultipleHooks() 32 | 33 | assert isinstance(hooks, MultipleHooks) 34 | assert len(hooks) == 1 35 | assert hooks.hooks_list == [hook_1] 36 | 37 | 38 | def test_multiple_and_multiple(): 39 | m_1 = MultipleHooks() 40 | m_2 = MultipleHooks() 41 | hooks = m_1 & m_2 42 | 43 | assert isinstance(hooks, MultipleHooks) 44 | assert len(hooks) == 0 45 | assert hooks.hooks_list == [] 46 | assert hooks is m_1 47 | 48 | 49 | @pytest.mark.parametrize('count', [1, 2, 255]) 50 | def test_multiple_init(count): 51 | hooks_list = [] 52 | for _ in range(count): 53 | hooks_list.append(BaseHooks()) 54 | 55 | hooks = MultipleHooks(*hooks_list) 56 | 57 | assert len(hooks) == count 58 | assert hooks.hooks_list == hooks_list 59 | 60 | 61 | class CounterHooks(BaseHooks): 62 | def __init__(self): 63 | self.calls_counter = Counter() 64 | 65 | def on_apply_for(self, coro: Coroutine, ident: str) -> None: 66 | self.calls_counter['on_apply_for'] += 1 67 | 68 | def on_executing_for(self, coro: Coroutine, ident: str) -> None: 69 | self.calls_counter['on_executing_for'] += 1 70 | 71 | def on_reducing_for(self, coro: Coroutine, ident: str) -> None: 72 | self.calls_counter['on_reducing_for'] += 1 73 | 74 | def on_result_for(self, coro: Coroutine, ident: str, result: Any) -> None: 75 | self.calls_counter['on_result_for'] += 1 76 | 77 | def on_exception_for( 78 | self, 79 | coro: Coroutine, 80 | ident: str, 81 | exception: Union[Exception, CancelledError], 82 | ) -> None: 83 | self.calls_counter['on_exception_for'] += 1 84 | 85 | 86 | @pytest.mark.parametrize('count', [0, 1, 2, 255]) 87 | def test_multiple_hooks(count): 88 | hooks_list = [] 89 | for _ in range(count): 90 | hooks_list.append(CounterHooks()) 91 | 92 | hooks = MultipleHooks(*hooks_list) 93 | 94 | async def foo(): 95 | pass 96 | 97 | coro = foo() 98 | 99 | hooks.on_apply_for(coro, 'ident') 100 | hooks.on_executing_for(coro, 'ident') 101 | hooks.on_reducing_for(coro, 'ident') 102 | hooks.on_result_for(coro, 'ident', None) 103 | hooks.on_exception_for(coro, 'ident', RuntimeError('test')) 104 | 105 | coro.close() 106 | 107 | for hook in hooks_list: 108 | assert hook.calls_counter['on_apply_for'] == 1 109 | assert hook.calls_counter['on_executing_for'] == 1 110 | assert hook.calls_counter['on_reducing_for'] == 1 111 | assert hook.calls_counter['on_result_for'] == 1 112 | assert hook.calls_counter['on_exception_for'] == 1 113 | -------------------------------------------------------------------------------- /tests/test_hooks/test_debug.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import sys 3 | from io import StringIO 4 | import re 5 | 6 | import pytest 7 | 8 | from async_reduce import AsyncReducer 9 | from async_reduce.hooks import DebugHooks 10 | 11 | pytestmark = pytest.mark.asyncio 12 | 13 | 14 | async def test_debug_hooks(): 15 | stream = StringIO() 16 | 17 | async def foo(arg): 18 | return arg 19 | 20 | async def foo_error(): 21 | raise RuntimeError('test') 22 | 23 | async_reduce = AsyncReducer(hooks=DebugHooks(stream)) 24 | 25 | coros = [ 26 | async_reduce(foo(1)), 27 | async_reduce(foo(1)), 28 | async_reduce(foo(2)), 29 | async_reduce(foo_error()), 30 | ] 31 | 32 | await asyncio.gather(*coros, return_exceptions=True) 33 | 34 | lines = stream.getvalue().splitlines() 35 | assert len(lines) == 11 36 | 37 | module_loc = 'tests.test_hooks.test_debug' 38 | 39 | assert ( 40 | re.fullmatch( 41 | r'\[{module_loc}:{func_name}\(\)\]' 42 | r' apply async_reduce\(\) for' 43 | r' ' 44 | r''.format(module_loc=module_loc, func_name=foo.__qualname__), 45 | lines[0], 46 | ) 47 | is not None 48 | ), lines[0] 49 | assert ( 50 | re.fullmatch( 51 | r'\[{module_loc}:{func_name}\(\)\]' 52 | r' executing for' 53 | r' ' 54 | r''.format(module_loc=module_loc, func_name=foo.__qualname__), 55 | lines[1], 56 | ) 57 | is not None 58 | ), lines[1] 59 | assert ( 60 | re.fullmatch( 61 | r'\[{module_loc}:{func_name}\(\)\]' 62 | r' apply async_reduce\(\) for' 63 | r' ' 64 | r''.format(module_loc=module_loc, func_name=foo.__qualname__), 65 | lines[2], 66 | ) 67 | is not None 68 | ), lines[2] 69 | assert ( 70 | re.fullmatch( 71 | r'\[{module_loc}:{func_name}\(\)\]' 72 | r' reducing for' 73 | r' ' 74 | r''.format(module_loc=module_loc, func_name=foo.__qualname__), 75 | lines[3], 76 | ) 77 | is not None 78 | ), lines[3] 79 | assert ( 80 | re.fullmatch( 81 | r'\[{module_loc}:{func_name}\(\)\]' 82 | r' apply async_reduce\(\) for' 83 | r' ' 84 | r''.format(module_loc=module_loc, func_name=foo.__qualname__), 85 | lines[4], 86 | ) 87 | is not None 88 | ), lines[4] 89 | assert ( 90 | re.fullmatch( 91 | r'\[{module_loc}:{func_name}\(\)\]' 92 | r' executing for' 93 | r' ' 94 | r''.format(module_loc=module_loc, func_name=foo.__qualname__), 95 | lines[5], 96 | ) 97 | is not None 98 | ), lines[5] 99 | assert ( 100 | re.fullmatch( 101 | r'\[{module_loc}:{func_name}\(\)\]' 102 | r' apply async_reduce\(\) for' 103 | r' ' 104 | r''.format( 105 | module_loc=module_loc, func_name=foo_error.__qualname__ 106 | ), 107 | lines[6], 108 | ) 109 | is not None 110 | ), lines[6] 111 | assert ( 112 | re.fullmatch( 113 | r'\[{module_loc}:{func_name}\(\)\]' 114 | r' executing for' 115 | r' ' 116 | r''.format( 117 | module_loc=module_loc, func_name=foo_error.__qualname__ 118 | ), 119 | lines[7], 120 | ) 121 | is not None 122 | ), lines[7] 123 | assert ( 124 | re.fullmatch( 125 | r'\[{module_loc}:{func_name}\(\)\]' 126 | r' result for' 127 | r' : 1' 128 | r''.format(module_loc=module_loc, func_name=foo.__qualname__), 129 | lines[8], 130 | ) 131 | is not None 132 | ), lines[8] 133 | assert ( 134 | re.fullmatch( 135 | r'\[{module_loc}:{func_name}\(\)\]' 136 | r' result for' 137 | r' : 2' 138 | r''.format(module_loc=module_loc, func_name=foo.__qualname__), 139 | lines[9], 140 | ) 141 | is not None 142 | ), lines[9] 143 | assert ( 144 | re.fullmatch( 145 | r'\[{module_loc}:{func_name}\(\)\]' 146 | r' get exception for' 147 | r' : test' 148 | r''.format( 149 | module_loc=module_loc, func_name=foo_error.__qualname__ 150 | ), 151 | lines[10], 152 | ) 153 | is not None 154 | ), lines[10] 155 | 156 | 157 | async def test_debug_hooks_default(): 158 | async def foo(arg): 159 | return arg 160 | 161 | async_reduce = AsyncReducer(hooks=DebugHooks()) 162 | 163 | coros = [ 164 | async_reduce(foo(1)), 165 | async_reduce(foo(1)), 166 | async_reduce(foo(2)), 167 | ] 168 | 169 | await asyncio.gather(*coros) 170 | 171 | 172 | @pytest.mark.parametrize('stream', [sys.stderr, sys.stdout]) 173 | async def test_debug_hooks_stream(stream): 174 | async def foo(arg): 175 | return arg 176 | 177 | async_reduce = AsyncReducer(hooks=DebugHooks(stream)) 178 | 179 | coros = [ 180 | async_reduce(foo(1)), 181 | async_reduce(foo(1)), 182 | async_reduce(foo(2)), 183 | ] 184 | 185 | await asyncio.gather(*coros) 186 | -------------------------------------------------------------------------------- /tests/test_hooks/test_statistics_detail.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import pytest 4 | 5 | from async_reduce import AsyncReducer 6 | from async_reduce.hooks import StatisticsDetailHooks 7 | 8 | pytestmark = pytest.mark.asyncio 9 | 10 | 11 | async def test(): 12 | async def foo(arg): 13 | return arg 14 | 15 | async def foo_error(): 16 | raise RuntimeError('test') 17 | 18 | stats = StatisticsDetailHooks() 19 | async_reduce = AsyncReducer(hooks=stats) 20 | 21 | coros = [ 22 | async_reduce(foo(1)), 23 | async_reduce(foo(1)), 24 | async_reduce(foo(2)), 25 | async_reduce(foo_error()), 26 | ] 27 | 28 | await asyncio.gather(*coros, return_exceptions=True) 29 | 30 | assert stats.total == { 31 | 'tests.test_hooks.test_statistics_detail:test..foo': 3, 32 | 'tests.test_hooks.test_statistics_detail:test..foo_error': 1, 33 | } 34 | assert stats.executed == { 35 | 'tests.test_hooks.test_statistics_detail:test..foo': 2, 36 | 'tests.test_hooks.test_statistics_detail:test..foo_error': 1, 37 | } 38 | assert stats.reduced == { 39 | 'tests.test_hooks.test_statistics_detail:test..foo': 1 40 | } 41 | assert stats.errors == { 42 | 'tests.test_hooks.test_statistics_detail:test..foo_error': 1 43 | } 44 | 45 | assert str(stats) == ( 46 | """\ 47 | Top total: 48 | \ttests.test_hooks.test_statistics_detail:test..foo: 3 49 | \ttests.test_hooks.test_statistics_detail:test..foo_error: 1 50 | Top executed: 51 | \ttests.test_hooks.test_statistics_detail:test..foo: 2 52 | \ttests.test_hooks.test_statistics_detail:test..foo_error: 1 53 | Top reduced: 54 | \ttests.test_hooks.test_statistics_detail:test..foo: 1 55 | Top errors: 56 | \ttests.test_hooks.test_statistics_detail:test..foo_error: 1 57 | """ 58 | ) 59 | -------------------------------------------------------------------------------- /tests/test_hooks/test_statistics_overall.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import pytest 4 | 5 | from async_reduce import AsyncReducer 6 | from async_reduce.hooks import StatisticsOverallHooks 7 | 8 | pytestmark = pytest.mark.asyncio 9 | 10 | 11 | async def test(): 12 | async def foo(arg): 13 | return arg 14 | 15 | async def foo_error(): 16 | raise RuntimeError('test') 17 | 18 | stats = StatisticsOverallHooks() 19 | async_reduce = AsyncReducer(hooks=stats) 20 | 21 | coros = [ 22 | async_reduce(foo(1)), 23 | async_reduce(foo(1)), 24 | async_reduce(foo(2)), 25 | async_reduce(foo_error()), 26 | ] 27 | 28 | await asyncio.gather(*coros, return_exceptions=True) 29 | 30 | assert stats.total == 4 31 | assert stats.executed == 2 32 | assert stats.reduced == 1 33 | assert stats.errors == 1 34 | 35 | assert str(stats) == 'Stats(total=4, executed=2, reduced=1, errors=1)' 36 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | coverage_clean 4 | py{3.7,3.8,3.9,3.10,3.11,3.12,3.13} 5 | coverage_report 6 | 7 | [testenv] 8 | basepython = 9 | py3.7: python3.7 10 | py3.8: python3.8 11 | py3.9: python3.9 12 | py3.10: python3.10 13 | py3.11: python3.11 14 | py3.12: python3.12 15 | py3.13: python3.13 16 | 17 | extras = testing 18 | 19 | commands = 20 | mypy async_reduce 21 | coverage run --parallel-mode -m pytest --pylama 22 | 23 | [testenv:coverage_clean] 24 | basepython = python 25 | 26 | skip_install = True 27 | deps = coverage 28 | 29 | commands = 30 | coverage erase 31 | 32 | [testenv:coverage_report] 33 | basepython = python 34 | 35 | skip_install = True 36 | deps = coverage 37 | 38 | commands = 39 | coverage combine 40 | coverage report --show-missing 41 | coverage html 42 | -------------------------------------------------------------------------------- /versioning.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from subprocess import check_output, STDOUT, CalledProcessError 3 | 4 | 5 | def _get_git_version(): 6 | try: 7 | tag_describe = ( 8 | check_output('git describe --tags', shell=True, stderr=STDOUT) 9 | .strip() 10 | .decode('utf-8') 11 | ) 12 | except CalledProcessError as e: 13 | print('[!] Can not detect version in git repo: ', e, file=sys.stderr) 14 | return 0, 0, 0, None, None 15 | else: 16 | if '-' in tag_describe: 17 | tag, describe = tag_describe.split('-', 1) 18 | else: 19 | tag, describe = tag_describe, None 20 | 21 | segs = tag.split('.') 22 | assert 1 < len(segs) <= 3 23 | 24 | major = int(segs[0]) 25 | if len(segs) == 2: 26 | minor = int(segs[1]) 27 | patch = None 28 | elif len(segs) == 3: 29 | minor = int(segs[1]) 30 | patch = int(segs[2]) 31 | 32 | if describe: 33 | dev, localversion = describe.split('-', 1) 34 | dev = int(dev) 35 | else: 36 | dev = localversion = None 37 | 38 | return major, minor, patch, dev, localversion 39 | 40 | 41 | # pylama:ignore=C901 42 | def version(major=0, minor=None, patch=None, localversion='auto'): 43 | repo_version = _get_git_version() 44 | is_dev = False 45 | 46 | if major == repo_version[0]: 47 | v = str(major) 48 | elif major > repo_version[0]: 49 | is_dev = True 50 | v = str(major) 51 | else: 52 | assert False 53 | 54 | if minor: 55 | if minor == repo_version[1]: 56 | v = '{}.{}'.format(v, minor) 57 | elif minor > repo_version[1]: 58 | is_dev = True 59 | v = '{}.{}'.format(v, minor) 60 | else: 61 | assert False 62 | 63 | if patch: 64 | if patch == repo_version[2]: 65 | v = '{}.{}'.format(v, patch) 66 | elif patch > (repo_version[2] or 0): 67 | is_dev = True 68 | v = '{}.{}'.format(v, patch) 69 | else: 70 | assert False 71 | 72 | if is_dev: 73 | v = '{}.dev{}'.format(v, repo_version[3] or 0) 74 | elif repo_version[3]: 75 | is_dev = True 76 | v = '{}.post1.dev{}'.format(v, repo_version[3]) 77 | 78 | if localversion != 'auto' and localversion: 79 | v = '{}+{}'.format(v, localversion) 80 | 81 | if is_dev and repo_version[3] and localversion == 'auto': 82 | v = '{}+{}'.format(v, repo_version[4]) 83 | 84 | return v 85 | --------------------------------------------------------------------------------