├── .gitignore ├── dev_requirements.in ├── .coveragerc ├── .github ├── dependabot.yml └── workflows │ └── main.yml ├── test_concurrently.py ├── dev_requirements.txt ├── examples ├── sleepy_multiply.py └── downloading.py ├── LICENSE ├── README.md └── concurrently.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .coverage 3 | -------------------------------------------------------------------------------- /dev_requirements.in: -------------------------------------------------------------------------------- 1 | mypy 2 | pytest-cov 3 | ruff 4 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | include = 4 | concurrently.py 5 | test_concurrently.py 6 | 7 | [report] 8 | show_missing = True 9 | fail_under = 100 10 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | day: "monday" 8 | time: "09:00" 9 | - package-ecosystem: "pip" 10 | directory: "/" 11 | schedule: 12 | interval: "weekly" 13 | day: "monday" 14 | time: "09:00" 15 | ignore: 16 | - dependency-name: "mypy" 17 | - dependency-name: "ruff" 18 | -------------------------------------------------------------------------------- /test_concurrently.py: -------------------------------------------------------------------------------- 1 | import random 2 | import time 3 | 4 | from concurrently import concurrently 5 | 6 | 7 | def double(x: int) -> int: 8 | time.sleep(random.random() / 100) 9 | return x * 2 10 | 11 | 12 | def test_handles_iterator() -> None: 13 | result = set(concurrently(handler=double, inputs=range(10))) 14 | 15 | assert result == { 16 | (0, 0), 17 | (1, 2), 18 | (2, 4), 19 | (3, 6), 20 | (4, 8), 21 | (5, 10), 22 | (6, 12), 23 | (7, 14), 24 | (8, 16), 25 | (9, 18), 26 | } 27 | 28 | 29 | def test_handles_list() -> None: 30 | result = set(concurrently(handler=double, inputs=[1, 3, 5, 7, 9, 11, 13])) 31 | 32 | assert result == {(1, 2), (3, 6), (5, 10), (7, 14), (9, 18), (11, 22), (13, 26)} 33 | -------------------------------------------------------------------------------- /dev_requirements.txt: -------------------------------------------------------------------------------- 1 | # This file was autogenerated by uv via the following command: 2 | # uv pip compile dev_requirements.in --output-file dev_requirements.txt 3 | coverage[toml]==7.10.6 4 | # via pytest-cov 5 | exceptiongroup==1.2.2 6 | # via pytest 7 | iniconfig==2.1.0 8 | # via pytest 9 | mypy==1.18.2 10 | # via -r dev_requirements.in 11 | mypy-extensions==1.0.0 12 | # via mypy 13 | packaging==24.2 14 | # via pytest 15 | pathspec==0.12.1 16 | # via mypy 17 | pluggy==1.5.0 18 | # via 19 | # pytest 20 | # pytest-cov 21 | pytest==8.3.5 22 | # via pytest-cov 23 | pytest-cov==7.0.0 24 | # via -r dev_requirements.in 25 | ruff==0.13.2 26 | # via -r dev_requirements.in 27 | tomli==2.2.1 28 | # via 29 | # coverage 30 | # mypy 31 | # pytest 32 | typing-extensions==4.13.0 33 | # via mypy 34 | -------------------------------------------------------------------------------- /examples/sleepy_multiply.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | This example does a "sleepy multiply". 4 | 5 | It multiples two numbers together, but with a delay before returning. 6 | This mimics a computation that has to be fetched from a remote server. 7 | 8 | This example also shows how to handle functions that take multiple inputs. 9 | """ 10 | 11 | import time 12 | 13 | from concurrently import concurrently 14 | 15 | 16 | def sleepy_multiply(x, y): 17 | time.sleep(x / 10) 18 | return x * y 19 | 20 | 21 | if __name__ == "__main__": 22 | inputs = [ 23 | (1, 2), 24 | (2, 3), 25 | (7, 4), 26 | (3, 1), 27 | (5, 2), 28 | (4, 9), 29 | (7, 2), 30 | (6, 1), 31 | ] 32 | 33 | for (x, y), output in concurrently(lambda x: sleepy_multiply(*x), inputs=inputs): 34 | print(x, "*", y, "=", output) 35 | -------------------------------------------------------------------------------- /examples/downloading.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | This example downloads a collection of images from https://http.cat/ 4 | 5 | Rather than downloading the images one-by-one, it runs multiple instances 6 | of the download() function to complete the process faster. 7 | """ 8 | 9 | import tempfile 10 | import urllib.request 11 | 12 | from concurrently import concurrently 13 | 14 | 15 | def save_http_cat(status_code): 16 | """ 17 | Saves the JPEG from https://http.cat/ associated with this status code. 18 | 19 | Returns the path to the downloaded file. 20 | """ 21 | _, path = tempfile.mkstemp(suffix=f"_{status_code}.jpg") 22 | urllib.request.urlretrieve(f"https://http.cat/{status_code}", path) 23 | return path 24 | 25 | 26 | if __name__ == "__main__": 27 | codes = [200, 201, 202, 301, 302, 400, 405, 410, 418, 420, 451, 500] 28 | 29 | for input, output in concurrently(save_http_cat, inputs=codes): 30 | print(input, output) 31 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Run tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | pull_request: 9 | branches: 10 | - main 11 | 12 | jobs: 13 | test: 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - name: Check out repository code 18 | uses: actions/checkout@v6 19 | 20 | - name: Setup Python 21 | uses: actions/setup-python@v6 22 | with: 23 | python-version: 3.12 24 | cache: pip 25 | cache-dependency-path: dev_requirements.txt 26 | 27 | - name: Install dependencies 28 | run: python3 -m pip install -r dev_requirements.txt 29 | 30 | - name: Check formatting 31 | run: | 32 | ruff check . 33 | ruff format --check . 34 | 35 | - name: Check types 36 | run: mypy *.py --strict 37 | 38 | - name: Run test suite 39 | run: | 40 | coverage run -m pytest test_concurrently.py 41 | coverage report --skip-covered --fail-under=100 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Alex Chan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # concurrently 2 | 3 | I often use the following pattern in Python: 4 | 5 | ```python 6 | for task in get_tasks_to_do(): 7 | perform(task) 8 | ``` 9 | 10 | Tasks run one after the other: task 1 runs to completion, then task 2 runs to completion, then task 3 runs to completion, and so on until everything is done. 11 | 12 | This is fine for certain classes of task, but if `perform()` is heavily I/O bound, it's unnecessarily slow. 13 | If I could run multiple instances of `perform()` concurrently, the overall process would complete much faster. 14 | Task 1 could start, make a network request, then task 2 could start while task 1 is waiting. 15 | 16 | I wrote [my recipe for concurrent processing][blog] in a blog post in 2019, which explains how it works – but the code was a little cumbersome to use. 17 | This repo is a tidied up (and tested!) version of that code. 18 | 19 | It allows me to write code like: 20 | 21 | ```python 22 | from concurrently import concurrently 23 | 24 | for (input, output) in concurrently(handler=perform, inputs=get_tasks_to_do()): 25 | print(input, output) 26 | ``` 27 | 28 | It yields both the input and the output, because results may not come in the original order of inputs. 29 | 30 | I would recommend using this code instead of the code in the original blog post. 31 | 32 | [blog]: https://alexwlchan.net/2019/10/adventures-with-concurrent-futures/ 33 | 34 | 35 | 36 | ## Usage 37 | 38 | Copy and paste the file `concurrently.py` into your project. 39 | 40 | You can see examples in the [`examples` directory](examples). 41 | 42 | 43 | 44 | ## License 45 | 46 | MIT. 47 | -------------------------------------------------------------------------------- /concurrently.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Callable, Iterable 2 | import concurrent.futures 3 | import itertools 4 | import typing 5 | 6 | 7 | In = typing.TypeVar("In") 8 | Out = typing.TypeVar("Out") 9 | 10 | 11 | def concurrently( 12 | handler: Callable[[In], Out], inputs: Iterable[In], *, max_concurrency: int = 5 13 | ) -> Iterable[tuple[In, Out]]: 14 | """ 15 | Calls the function ``handler`` on the values ``inputs``. 16 | 17 | ``handler`` should be a function that takes a single input, which is the 18 | individual values in the iterable ``inputs``. 19 | 20 | Generates (input, output) tuples as the calls to ``handler`` complete. 21 | 22 | See https://alexwlchan.net/2019/10/adventures-with-concurrent-futures/ for an explanation 23 | of how this function works. 24 | 25 | """ 26 | # Make sure we get a consistent iterator throughout, rather than 27 | # getting the first element repeatedly. 28 | handler_inputs = iter(inputs) 29 | 30 | with concurrent.futures.ThreadPoolExecutor() as executor: 31 | futures = { 32 | executor.submit(handler, input): input 33 | for input in itertools.islice(handler_inputs, max_concurrency) 34 | } 35 | 36 | while futures: 37 | done, _ = concurrent.futures.wait( 38 | futures, return_when=concurrent.futures.FIRST_COMPLETED 39 | ) 40 | 41 | for fut in done: 42 | original_input = futures.pop(fut) 43 | yield original_input, fut.result() 44 | 45 | for input in itertools.islice(handler_inputs, len(done)): 46 | fut = executor.submit(handler, input) 47 | futures[fut] = input 48 | --------------------------------------------------------------------------------