├── .gitignore ├── requirements.txt ├── io-bound_sync.py ├── cpu-bound_sync.py ├── cpu-bound_parallel_1.py ├── io-bound_concurrent_1.py ├── io-bound_concurrent_2.py ├── io-bound_concurrent_3.py ├── cpu-bound_parallel_2.py ├── README.md ├── LICENSE └── tasks.py /.gitignore: -------------------------------------------------------------------------------- 1 | env 2 | __pycache__ 3 | *.pyc 4 | .DS_Store 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | httpx==0.23.0 2 | requests==2.28.1 3 | -------------------------------------------------------------------------------- /io-bound_sync.py: -------------------------------------------------------------------------------- 1 | # io-bound_sync.py 2 | 3 | import time 4 | 5 | from tasks import make_request 6 | 7 | 8 | def main(): 9 | for num in range(1, 101): 10 | make_request(num) 11 | 12 | 13 | if __name__ == "__main__": 14 | start_time = time.perf_counter() 15 | 16 | main() 17 | 18 | end_time = time.perf_counter() 19 | print(f"Elapsed run time: {end_time - start_time} seconds.") 20 | -------------------------------------------------------------------------------- /cpu-bound_sync.py: -------------------------------------------------------------------------------- 1 | # cpu-bound_sync.py 2 | 3 | import time 4 | 5 | from tasks import get_prime_numbers 6 | 7 | 8 | def main(): 9 | for num in range(1000, 16000): 10 | get_prime_numbers(num) 11 | 12 | 13 | if __name__ == "__main__": 14 | start_time = time.perf_counter() 15 | 16 | main() 17 | 18 | end_time = time.perf_counter() 19 | print(f"Elapsed run time: {end_time - start_time} seconds.") 20 | -------------------------------------------------------------------------------- /cpu-bound_parallel_1.py: -------------------------------------------------------------------------------- 1 | # cpu-bound_parallel_1.py 2 | 3 | import time 4 | from multiprocessing import Pool, cpu_count 5 | 6 | from tasks import get_prime_numbers 7 | 8 | 9 | def main(): 10 | with Pool(cpu_count() - 1) as p: 11 | p.starmap(get_prime_numbers, zip(range(1000, 16000))) 12 | p.close() 13 | p.join() 14 | 15 | 16 | if __name__ == "__main__": 17 | start_time = time.perf_counter() 18 | 19 | main() 20 | 21 | end_time = time.perf_counter() 22 | print(f"Elapsed run time: {end_time - start_time} seconds.") 23 | -------------------------------------------------------------------------------- /io-bound_concurrent_1.py: -------------------------------------------------------------------------------- 1 | # io-bound_concurrent_1.py 2 | 3 | import threading 4 | import time 5 | 6 | from tasks import make_request 7 | 8 | 9 | def main(): 10 | tasks = [] 11 | 12 | for num in range(1, 101): 13 | tasks.append(threading.Thread(target=make_request, args=(num,))) 14 | tasks[-1].start() 15 | 16 | for task in tasks: 17 | task.join() 18 | 19 | 20 | if __name__ == "__main__": 21 | start_time = time.perf_counter() 22 | 23 | main() 24 | 25 | end_time = time.perf_counter() 26 | print(f"Elapsed run time: {end_time - start_time} seconds.") 27 | -------------------------------------------------------------------------------- /io-bound_concurrent_2.py: -------------------------------------------------------------------------------- 1 | # io-bound_concurrent_2.py 2 | 3 | import time 4 | from concurrent.futures import ThreadPoolExecutor, wait 5 | 6 | from tasks import make_request 7 | 8 | 9 | def main(): 10 | futures = [] 11 | 12 | with ThreadPoolExecutor() as executor: 13 | for num in range(1, 101): 14 | futures.append(executor.submit(make_request, num)) 15 | 16 | wait(futures) 17 | 18 | 19 | if __name__ == "__main__": 20 | start_time = time.perf_counter() 21 | 22 | main() 23 | 24 | end_time = time.perf_counter() 25 | print(f"Elapsed run time: {end_time - start_time} seconds.") 26 | -------------------------------------------------------------------------------- /io-bound_concurrent_3.py: -------------------------------------------------------------------------------- 1 | # io-bound_concurrent_3.py 2 | 3 | import asyncio 4 | import time 5 | 6 | import httpx 7 | 8 | from tasks import make_request_async 9 | 10 | 11 | async def main(): 12 | async with httpx.AsyncClient() as client: 13 | return await asyncio.gather( 14 | *[make_request_async(num, client) for num in range(1, 101)] 15 | ) 16 | 17 | 18 | if __name__ == "__main__": 19 | start_time = time.perf_counter() 20 | 21 | loop = asyncio.get_event_loop() 22 | loop.run_until_complete(main()) 23 | 24 | end_time = time.perf_counter() 25 | elapsed_time = end_time - start_time 26 | print(f"Elapsed run time: {elapsed_time} seconds") 27 | -------------------------------------------------------------------------------- /cpu-bound_parallel_2.py: -------------------------------------------------------------------------------- 1 | # cpu-bound_parallel_2.py 2 | 3 | import time 4 | from concurrent.futures import ProcessPoolExecutor, wait 5 | from multiprocessing import cpu_count 6 | 7 | from tasks import get_prime_numbers 8 | 9 | 10 | def main(): 11 | futures = [] 12 | 13 | with ProcessPoolExecutor(cpu_count() - 1) as executor: 14 | for num in range(1000, 16000): 15 | futures.append(executor.submit(get_prime_numbers, num)) 16 | 17 | wait(futures) 18 | 19 | 20 | if __name__ == "__main__": 21 | start_time = time.perf_counter() 22 | 23 | main() 24 | 25 | end_time = time.perf_counter() 26 | print(f"Elapsed run time: {end_time - start_time} seconds.") 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Parallelism, Concurrency, and AsyncIO in Python - by example 2 | 3 | Speeding up CPU-bound and IO-bound operations with multiprocessing, threading, and AsyncIO 4 | 5 | > Blog post: [Parallelism, Concurrency, and AsyncIO in Python - by example](http://testdriven.io/blog/python-concurrency-parallelism/) 6 | 7 | ## Setup 8 | 9 | 1. Fork/Clone 10 | 1. Create and activate a virtual environment 11 | 1. Install the dependencies 12 | 13 | ## IO-bound Operation 14 | 15 | ```sh 16 | $ python io-bound_sync.py 17 | $ python io-bound_concurrent_1.py 18 | $ python io-bound_concurrent_2.py 19 | $ python io-bound_concurrent_3.py 20 | ``` 21 | 22 | ## CPU-bound Operation 23 | 24 | ```sh 25 | $ python cpu-bound_sync.py 26 | $ python cpu-bound_parallel_1.py 27 | $ python cpu-bound_parallel_2.py 28 | ``` 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Michael Herman 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tasks.py: -------------------------------------------------------------------------------- 1 | # tasks.py 2 | 3 | import os 4 | from multiprocessing import current_process 5 | from threading import current_thread 6 | 7 | import requests 8 | 9 | 10 | def make_request(num): 11 | # io-bound 12 | 13 | pid = os.getpid() 14 | thread_name = current_thread().name 15 | process_name = current_process().name 16 | print(f"{pid} - {process_name} - {thread_name}") 17 | 18 | requests.get("https://httpbin.org/ip") 19 | 20 | 21 | async def make_request_async(num, client): 22 | # io-bound 23 | 24 | pid = os.getpid() 25 | thread_name = current_thread().name 26 | process_name = current_process().name 27 | print(f"{pid} - {process_name} - {thread_name}") 28 | 29 | await client.get("https://httpbin.org/ip") 30 | 31 | 32 | def get_prime_numbers(num): 33 | # cpu-bound 34 | 35 | pid = os.getpid() 36 | thread_name = current_thread().name 37 | process_name = current_process().name 38 | print(f"{pid} - {process_name} - {thread_name}") 39 | 40 | numbers = [] 41 | 42 | prime = [True for i in range(num + 1)] 43 | p = 2 44 | 45 | while p * p <= num: 46 | if prime[p]: 47 | for i in range(p * 2, num + 1, p): 48 | prime[i] = False 49 | p += 1 50 | 51 | prime[0] = False 52 | prime[1] = False 53 | 54 | for p in range(num + 1): 55 | if prime[p]: 56 | numbers.append(p) 57 | 58 | return numbers 59 | --------------------------------------------------------------------------------