├── .gitignore ├── README.md ├── cover.png └── src ├── lesson01_pool.py ├── lesson01_process.py ├── lesson02_default.py ├── lesson02_initalizer.py ├── lesson03_apply.py ├── lesson03_map.py ├── lesson03_starmap.py ├── lesson04_apply_async.py ├── lesson04_map_async.py ├── lesson04_starmap_async.py ├── lesson05_imap.py ├── lesson05_imap_unordered.py ├── lesson06_asyncresult.py ├── lesson06_callback.py ├── lesson07_check_prime_parallel.py └── lesson07_check_prime_sequential.py /.gitignore: -------------------------------------------------------------------------------- 1 | # git ignore 2 | **/.DS_Store 3 | .DS_Store 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Python Multiprocessing Pool Jump-Start 2 | 3 | ![Python Multiprocessing Pool Jump-Start](cover.png) 4 | 5 | * 6 | 7 | This repository provides all source code for the book: 8 | 9 | * **Python Multiprocessing Pool Jump-Start**: _Run Your Python Functions In Parallel With Just A Few Lines Of Code_, Jason Brownlee, 2022. 10 | 11 | 12 | ## Source Code 13 | You can access all Python .py files directly here: 14 | 15 | * [src/](src/) 16 | 17 | ## Get the Book 18 | 19 | You can learn more about the book here: 20 | 21 | * [Gumroad](https://superfastpython.gumroad.com/l/pmpj) 22 | * [Amazon](https://www.amazon.com/dp/B0B6YSF8XQ) 23 | * [GooglePlay](https://play.google.com/store/books/details?id=in58EAAAQBAJ) 24 | * [GoogleBooks](http://books.google.com/books/about?id=in58EAAAQBAJ) 25 | * [Goodreads](https://www.goodreads.com/book/show/61606103-python-multiprocessing-pool-jump-start) 26 | 27 | ### Book Blurb 28 | 29 | > How much faster could your python code run (if it used all CPU cores)? 30 | > 31 | > The multiprocessing.Pool class provides easy-to-use process-based concurrency. 32 | > 33 | > This is not some random third-party library, this is a class provided in the Python standard library (already installed on your system). 34 | > 35 | > This is the class you need to use to make your code run faster. 36 | > 37 | > There's just one problem. No one knows about it (or how to use it well). 38 | > 39 | > Introducing: "Python Multiprocessing Pool Jump-Start". A new book designed to teach you multiprocessing pools in Python, super fast! 40 | > 41 | > You will get a fast-paced, 7-part course to get you started and make you awesome at using the multiprocessing pool. 42 | > 43 | > Each of the 7 lessons was carefully designed to teach one critical aspect of the multiprocessing pool, with explanations, code snippets and worked examples. 44 | > 45 | > Each lesson ends with an exercise for you to complete to confirm you understood the topic, a summary of what was learned, and links for further reading if you want to go deeper. 46 | > 47 | > Stop copy-pasting code from outdated StackOverflow answers. 48 | > 49 | > Learn Python concurrency correctly, step-by-step. 50 | -------------------------------------------------------------------------------- /cover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SuperFastPython/PythonMultiprocessingPoolJumpStart/aad5831b23a8ec561b6a9ebf00b2ede076d7f32f/cover.png -------------------------------------------------------------------------------- /src/lesson01_pool.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # example running a function in the multiprocessing pool 3 | from multiprocessing import Pool 4 | 5 | # custom function to be executed in a child process 6 | def task(): 7 | # report a message 8 | print('This is another process', flush=True) 9 | 10 | # protect the entry point 11 | if __name__ == '__main__': 12 | # create the multiprocessing pool 13 | with Pool() as pool: 14 | # issue the task 15 | async_result = pool.apply_async(task) 16 | # wait for the task to finish 17 | async_result.wait() 18 | # close the multiprocessing pool automatically 19 | -------------------------------------------------------------------------------- /src/lesson01_process.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # example of running a function in a new process 3 | from multiprocessing import Process 4 | 5 | # custom function to be executed in a child process 6 | def task(): 7 | # report a message 8 | print('This is another process', flush=True) 9 | 10 | # protect the entry point 11 | if __name__ == '__main__': 12 | # define a task to run in a new process 13 | process = Process(target=task) 14 | # start the task in a new process 15 | process.start() 16 | # wait for the child process to terminate 17 | process.join() 18 | -------------------------------------------------------------------------------- /src/lesson02_default.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # example reporting the details of a default pool 3 | from multiprocessing import Pool 4 | 5 | # protect the entry point 6 | if __name__ == '__main__': 7 | # create a multiprocessing pool 8 | pool = Pool() 9 | # report the status of the multiprocessing pool 10 | print(pool) 11 | # close the multiprocessing pool 12 | pool.close() 13 | -------------------------------------------------------------------------------- /src/lesson02_initalizer.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # example initializing worker processes in the pool 3 | from time import sleep 4 | from multiprocessing import Pool 5 | 6 | # custom function to be executed in a child process 7 | def task(): 8 | # report a message 9 | print('Worker executing task...', flush=True) 10 | # block for a moment 11 | sleep(1) 12 | 13 | # initialize a worker in the multiprocessing pool 14 | def initialize_worker(): 15 | # report a message 16 | print('Initializing worker...', flush=True) 17 | 18 | # protect the entry point 19 | if __name__ == '__main__': 20 | # create and configure the multiprocessing pool 21 | with Pool(2, initializer=initialize_worker) as pool: 22 | # issue tasks to the multiprocessing pool 23 | for _ in range(4): 24 | pool.apply_async(task) 25 | # close the multiprocessing pool 26 | pool.close() 27 | # wait for all tasks to complete 28 | pool.join() 29 | -------------------------------------------------------------------------------- /src/lesson03_apply.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # example of executing a one-off task and waiting 3 | from multiprocessing import Pool 4 | 5 | # custom function to be executed in a child process 6 | def task(): 7 | # report a message 8 | print('This is another process', flush=True) 9 | 10 | # protect the entry point 11 | if __name__ == '__main__': 12 | # create the multiprocessing pool 13 | with Pool() as pool: 14 | # issue a task and wait for it to complete 15 | pool.apply(task) 16 | -------------------------------------------------------------------------------- /src/lesson03_map.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # example of executing multiple tasks and waiting 3 | from multiprocessing import Pool 4 | 5 | # custom function to be executed in a child process 6 | def task(arg): 7 | # report a message 8 | print(f'From another process {arg}', flush=True) 9 | # return a value 10 | return arg * 2 11 | 12 | # protect the entry point 13 | if __name__ == '__main__': 14 | # create the multiprocessing pool 15 | with Pool() as pool: 16 | # issue multiple tasks and process return values 17 | for result in pool.map(task, range(10)): 18 | print(result) 19 | -------------------------------------------------------------------------------- /src/lesson03_starmap.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # example of multiple tasks with multiple arguments 3 | from multiprocessing import Pool 4 | 5 | # custom function to be executed in a child process 6 | def task(arg1, arg2, arg3): 7 | # report a message 8 | print(f'From another process {arg1}, {arg2}, {arg3}', 9 | flush=True) 10 | # return a value 11 | return arg1 + arg2 + arg3 12 | 13 | # protect the entry point 14 | if __name__ == '__main__': 15 | # create the multiprocessing pool 16 | with Pool() as pool: 17 | # prepare task arguments 18 | args = [(i, i*2, i*3) for i in range(10)] 19 | # issue multiple tasks and process return values 20 | for result in pool.starmap(task, args): 21 | print(result) 22 | -------------------------------------------------------------------------------- /src/lesson04_apply_async.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # example of executing a one-off task asynchronously 3 | from multiprocessing import Pool 4 | 5 | # custom function to be executed in a child process 6 | def task(): 7 | # report a message 8 | print('This is another process', flush=True) 9 | 10 | # protect the entry point 11 | if __name__ == '__main__': 12 | # create the multiprocessing pool 13 | with Pool() as pool: 14 | # issue a task asynchronously 15 | async_result = pool.apply_async(task) 16 | # wait for the task to complete 17 | async_result.wait() 18 | -------------------------------------------------------------------------------- /src/lesson04_map_async.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # example executing multiple tasks asynchronously 3 | from multiprocessing import Pool 4 | 5 | # custom function to be executed in a child process 6 | def task(arg): 7 | # report a message 8 | print(f'From another process {arg}', flush=True) 9 | # return a value 10 | return arg * 2 11 | 12 | # protect the entry point 13 | if __name__ == '__main__': 14 | # create the multiprocessing pool 15 | with Pool() as pool: 16 | # issue multiple tasks to the pool 17 | async_result = pool.map_async(task, range(10)) 18 | # process return values once all tasks completed 19 | for result in async_result.get(): 20 | print(result) 21 | -------------------------------------------------------------------------------- /src/lesson04_starmap_async.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # example many tasks multiple arguments asynchronously 3 | from multiprocessing import Pool 4 | 5 | # custom function to be executed in a child process 6 | def task(arg1, arg2, arg3): 7 | # report a message 8 | print(f'From another process {arg1}, {arg2}, {arg3}', 9 | flush=True) 10 | # return a value 11 | return arg1 + arg2 + arg3 12 | 13 | # protect the entry point 14 | if __name__ == '__main__': 15 | # create the multiprocessing pool 16 | with Pool() as pool: 17 | # prepare task arguments 18 | args = [(i, i*2, i*3) for i in range(10)] 19 | # issue multiple tasks to the pool 20 | async_result = pool.starmap_async(task, args) 21 | # process return values 22 | for result in async_result.get(): 23 | print(result) 24 | -------------------------------------------------------------------------------- /src/lesson05_imap.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # example of executing multiple tasks one-by-one 3 | from time import sleep 4 | from random import random 5 | from multiprocessing import Pool 6 | 7 | # custom function to be executed in a child process 8 | def task(arg): 9 | # block for a random fraction of a second 10 | sleep(random()) 11 | # report a message 12 | print(f'From another process {arg}', flush=True) 13 | # return a value 14 | return arg * 2 15 | 16 | # protect the entry point 17 | if __name__ == '__main__': 18 | # create the multiprocessing pool 19 | with Pool(4) as pool: 20 | # issue multiple tasks and process return values 21 | for result in pool.imap(task, range(10)): 22 | print(result) 23 | -------------------------------------------------------------------------------- /src/lesson05_imap_unordered.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # example getting many task results in completion order 3 | from time import sleep 4 | from random import random 5 | from multiprocessing import Pool 6 | 7 | # custom function to be executed in a child process 8 | def task(arg): 9 | # block for a random fraction of a second 10 | sleep(random()) 11 | # report a message 12 | print(f'From another process {arg}', flush=True) 13 | # return a value 14 | return arg * 2 15 | 16 | # protect the entry point 17 | if __name__ == '__main__': 18 | # create the multiprocessing pool 19 | with Pool(4) as pool: 20 | # issue multiple tasks and process return values 21 | for rs in pool.imap_unordered(task, range(10)): 22 | print(rs) 23 | -------------------------------------------------------------------------------- /src/lesson06_asyncresult.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # example check status and handle result of async task 3 | from time import sleep 4 | from random import random 5 | from multiprocessing import Pool 6 | 7 | # custom function to be executed in a child process 8 | def task(): 9 | # loop a few times to simulate a slow task 10 | for i in range(10): 11 | # generate a random value between 0 and 1 12 | value = random() 13 | # block for a fraction of a second 14 | sleep(value) 15 | # report a message 16 | print(f'>{i} got {value}', flush=True) 17 | 18 | # protect the entry point 19 | if __name__ == '__main__': 20 | # create the multiprocessing pool 21 | with Pool() as pool: 22 | # issue a task asynchronously 23 | async_result = pool.apply_async(task) 24 | # wait until the task is complete 25 | while not async_result.ready(): 26 | # report a message 27 | print('Main process waiting...') 28 | # block for a moment 29 | async_result.wait(timeout=1) 30 | # report if the task was successful 31 | if async_result.successful(): 32 | print('Task was successful.') 33 | -------------------------------------------------------------------------------- /src/lesson06_callback.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # example of callback function for a one-off async task 3 | from random import random 4 | from time import sleep 5 | from multiprocessing.pool import Pool 6 | 7 | # custom function to be executed in a child process 8 | def result_callback(return_value): 9 | # report a message 10 | print(f'Callback got: {return_value}', flush=True) 11 | 12 | # task executed in a worker process 13 | def task(identifier): 14 | # generate a value 15 | value = random() 16 | # report a message 17 | print(f'Task {identifier} executing with {value}', 18 | flush=True) 19 | # block for a moment 20 | sleep(value) 21 | # return the generated value 22 | return value 23 | 24 | # protect the entry point 25 | if __name__ == '__main__': 26 | # create and configure the multiprocessing pool 27 | with Pool() as pool: 28 | # issue tasks to the multiprocessing pool 29 | result = pool.apply_async(task, args=(0,), 30 | callback=result_callback) 31 | # close the multiprocessing pool 32 | pool.close() 33 | # wait for all tasks to complete 34 | pool.join() 35 | -------------------------------------------------------------------------------- /src/lesson07_check_prime_parallel.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # check if large numbers are prime concurrently 3 | from math import sqrt 4 | from math import floor 5 | from multiprocessing import Pool 6 | 7 | # returns True if prime, False otherwise 8 | def is_prime(number): 9 | # 1 is a special case of not prime 10 | if number <= 1: 11 | return False 12 | # 2 is a special case of a prime 13 | if number == 2: 14 | return True 15 | # check if the number divides by 2 with no remainder 16 | if number % 2 == 0: 17 | return False 18 | # limit divisors to sqrt(n)+1 so range will reach it 19 | limit = floor(sqrt(number)) + 1 20 | # check all odd numbers in range 21 | for i in range(3, limit, 2): 22 | # check if number is divisible with no remainder 23 | if number % i == 0: 24 | # number is divisible and is not a prime 25 | return False 26 | # number is probably prime 27 | return True 28 | 29 | # check if a series of numbers are prime or not 30 | def check_numbers_are_prime(numbers): 31 | # create multiprocessing pool 32 | with Pool() as pool: 33 | # issue the tasks 34 | results = pool.imap(is_prime, numbers) 35 | # report results as completed in order 36 | for number, isprime in zip(numbers, results): 37 | if isprime: 38 | print(f'{number} is prime') 39 | 40 | # protect the entry point 41 | if __name__ == '__main__': 42 | # define some numbers to check 43 | NUMS = [17977, 10619863, 106198, 6620830889, 44 | 80630964769, 228204732751, 1171432692373, 45 | 1398341745571, 10963707205259, 15285151248481, 46 | 99999199999, 304250263527209, 30425026352720, 47 | 10657331232548839, 10657331232548830, 48 | 44560482149, 1746860020068409] 49 | # check whether each number is a prime 50 | check_numbers_are_prime(NUMS) 51 | -------------------------------------------------------------------------------- /src/lesson07_check_prime_sequential.py: -------------------------------------------------------------------------------- 1 | # SuperFastPython.com 2 | # check if large numbers are prime 3 | from math import sqrt 4 | from math import floor 5 | 6 | # returns True if prime, False otherwise 7 | def is_prime(number): 8 | # 1 is a special case of not prime 9 | if number <= 1: 10 | return False 11 | # 2 is a special case of a prime 12 | if number == 2: 13 | return True 14 | # check if the number divides by 2 with no remainder 15 | if number % 2 == 0: 16 | return False 17 | # limit divisors to sqrt(n)+1 so range will reach it 18 | limit = floor(sqrt(number)) + 1 19 | # check all odd numbers in range 20 | for i in range(3, limit, 2): 21 | # check if number is divisible with no remainder 22 | if number % i == 0: 23 | # number is divisible and is not a prime 24 | return False 25 | # number is probably prime 26 | return True 27 | 28 | # check if a series of numbers are prime or not 29 | def check_numbers_are_prime(numbers): 30 | # check each number in turn 31 | for number in numbers: 32 | if is_prime(number): 33 | print(f'{number} is prime') 34 | 35 | # protect the entry point 36 | if __name__ == '__main__': 37 | # define some numbers to check 38 | NUMS = [17977, 10619863, 106198, 6620830889, 39 | 80630964769, 228204732751, 1171432692373, 40 | 1398341745571, 10963707205259, 15285151248481, 41 | 99999199999, 304250263527209, 30425026352720, 42 | 10657331232548839, 10657331232548830, 43 | 44560482149, 1746860020068409] 44 | # check whether each number is a prime 45 | check_numbers_are_prime(NUMS) 46 | --------------------------------------------------------------------------------