├── .gitattributes ├── 9781484244005.jpg ├── Contributing.md ├── LICENSE.txt ├── README.md ├── chapter_02 ├── __init__.py ├── sol_01_option_1.py ├── sol_01_option_2.py ├── sol_02.py ├── sol_03.py ├── sol_04_option_2.py ├── sol_04_unix.py ├── sol_05.py ├── sol_06.py ├── sol_07_option_1.py ├── sol_07_option_2.py ├── sol_07_option_3.py ├── sol_08_option_1.py ├── sol_08_option_2.py ├── sol_09_1_option_1.py ├── sol_09_1_option_2.py ├── sol_09_2.py ├── sol_10.py ├── sol_11_option_1.py ├── sol_11_option_2.py ├── sol_12.py ├── sol_13.py ├── sol_14.py └── sol_15.py ├── chapter_03 ├── __init__.py ├── sol_01.py ├── sol_02.py ├── sol_03.py ├── sol_04.py ├── sol_05.py ├── sol_06_option_1.py ├── sol_06_option_2.py ├── sol_07.py ├── sol_08.py ├── sol_09.py ├── sol_10.py ├── sol_11_option_1.py ├── sol_11_option_2.py ├── sol_11_option_3.py ├── sol_12.py └── sol_13.py ├── chapter_04 ├── __init__.py ├── sol_01.py ├── sol_02.py ├── sol_03.py ├── sol_04.py ├── sol_05_option_1.py ├── sol_05_option_2.py └── sol_06.py ├── chapter_05 ├── __init__.py ├── sol_01.py ├── sol_02.py ├── sol_03.py ├── sol_04.py ├── sol_05.py └── sol_06.py ├── chapter_06 ├── __init__.py ├── sol_01.py ├── sol_02.py ├── sol_03_unix.py ├── sol_03_win_unix.py ├── sol_04.py ├── sol_05_option_1.py └── sol_05_option_2.py ├── chapter_07 ├── __init__.py ├── sol_01.py ├── sol_02.py ├── sol_03.py ├── sol_04.py ├── sol_05.py ├── sol_06_option_1.py └── sol_06_option_2.py ├── chapter_08 ├── __init__.py ├── sol_01.py ├── sol_02.py ├── sol_03.py ├── sol_04_option_1.py ├── sol_04_option_2.py └── sol_06.py ├── chapter_09 ├── __init__.py ├── sol_01.py ├── sol_02.py ├── sol_03.py └── sol_04.py ├── chapter_10 ├── __init__.py ├── sol_01_option_1.py ├── sol_01_option_2.py ├── sol_02.py ├── sol_03.py ├── sol_04.py ├── sol_05.py ├── sol_06.py ├── sol_07.py └── sol_08.py └── errata.md /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /9781484244005.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/asyncio-recipes/f078242d4380407e483236695ebc49f3faf09e7f/9781484244005.jpg -------------------------------------------------------------------------------- /Contributing.md: -------------------------------------------------------------------------------- 1 | # Contributing to Apress Source Code 2 | 3 | Copyright for Apress source code belongs to the author(s). However, under fair use you are encouraged to fork and contribute minor corrections and updates for the benefit of the author(s) and other readers. 4 | 5 | ## How to Contribute 6 | 7 | 1. Make sure you have a GitHub account. 8 | 2. Fork the repository for the relevant book. 9 | 3. Create a new branch on which to make your change, e.g. 10 | `git checkout -b my_code_contribution` 11 | 4. Commit your change. Include a commit message describing the correction. Please note that if your commit message is not clear, the correction will not be accepted. 12 | 5. Submit a pull request. 13 | 14 | Thank you for your contribution! -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Freeware License, some rights reserved 2 | 3 | Copyright (c) 2019 Mohamed Mustapha Tahrioui 4 | 5 | Permission is hereby granted, free of charge, to anyone obtaining a copy 6 | of this software and associated documentation files (the "Software"), 7 | to work with the Software within the limits of freeware distribution and fair use. 8 | This includes the rights to use, copy, and modify the Software for personal use. 9 | Users are also allowed and encouraged to submit corrections and modifications 10 | to the Software for the benefit of other users. 11 | 12 | It is not allowed to reuse, modify, or redistribute the Software for 13 | commercial use in any way, or for a user’s educational materials such as books 14 | or blog articles without prior permission from the copyright holder. 15 | 16 | The above copyright notice and this permission notice need to be included 17 | in all copies or substantial portions of the software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 22 | AUTHORS OR COPYRIGHT HOLDERS OR APRESS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 25 | SOFTWARE. 26 | 27 | 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Apress Source Code 2 | 3 | This repository accompanies [*asyncio Recipes*](https://www.apress.com/9781484244005) by Mohamed Mustapha Tahrioui (Apress, 2019). 4 | 5 | [comment]: #cover 6 | ![Cover image](9781484244005.jpg) 7 | 8 | Download the files as a zip using the green button, or clone the repository to your machine using Git. 9 | 10 | ## Releases 11 | 12 | Release v1.0 corresponds to the code in the published book, without corrections or updates. 13 | 14 | ## Contributions 15 | 16 | See the file Contributing.md for more information on how you can contribute to this repository. -------------------------------------------------------------------------------- /chapter_02/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/asyncio-recipes/f078242d4380407e483236695ebc49f3faf09e7f/chapter_02/__init__.py -------------------------------------------------------------------------------- /chapter_02/sol_01_option_1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | loop = asyncio.get_event_loop() -------------------------------------------------------------------------------- /chapter_02/sol_01_option_2.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | try: 3 | loop = asyncio.get_running_loop() 4 | 5 | except RuntimeError: 6 | print("No loop running") 7 | -------------------------------------------------------------------------------- /chapter_02/sol_02.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import sys 3 | 4 | loop = asyncio.new_event_loop() 5 | 6 | print(loop) # Print the loop 7 | asyncio.set_event_loop(loop) 8 | 9 | if sys.platform != "win32": 10 | watcher = asyncio.get_child_watcher() 11 | watcher.attach_loop(loop) 12 | -------------------------------------------------------------------------------- /chapter_02/sol_03.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import threading 3 | 4 | 5 | def create_event_loop_thread(worker, *args, **kwargs): 6 | def _worker(*args, **kwargs): 7 | loop = asyncio.new_event_loop() 8 | asyncio.set_event_loop(loop) 9 | try: 10 | loop.run_until_complete(worker(*args, **kwargs)) 11 | finally: 12 | loop.close() 13 | 14 | return threading.Thread(target=_worker, args=args, kwargs=kwargs) 15 | 16 | 17 | async def print_coro(*args, **kwargs): 18 | print(f"Inside the print coro on {threading.get_ident()}:", (args, kwargs)) 19 | 20 | 21 | def start_threads(*threads): 22 | [t.start() for t in threads if isinstance(t, threading.Thread)] 23 | 24 | 25 | def join_threads(*threads): 26 | [t.join() for t in threads if isinstance(t, threading.Thread)] 27 | 28 | 29 | def main(): 30 | workers = [create_event_loop_thread(print_coro) for i in range(10)] 31 | start_threads(*workers) 32 | join_threads(*workers) 33 | 34 | 35 | if __name__ == '__main__': 36 | main() 37 | -------------------------------------------------------------------------------- /chapter_02/sol_04_option_2.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import os 3 | import random 4 | import typing 5 | from multiprocessing import Lock 6 | from multiprocessing import Process 7 | 8 | lock = Lock() 9 | pid_loops = {} 10 | processes = [] 11 | 12 | 13 | def cleanup(): 14 | global processes 15 | while processes: 16 | proc = processes.pop() 17 | proc.join() 18 | 19 | 20 | def get_event_loop(): 21 | global lock, pid_loops 22 | pid = os.getpid() 23 | with lock: 24 | if pid not in pid_loops: 25 | pid_loops[pid] = asyncio.new_event_loop() 26 | pid_loops[pid].pid = pid 27 | return pid_loops[pid] 28 | 29 | 30 | def asyncio_init(): 31 | with lock: 32 | pid = os.getpid() 33 | pid_loops[pid] = asyncio.new_event_loop() 34 | pid_loops[pid].pid = pid 35 | 36 | 37 | async def worker(*, loop: asyncio.AbstractEventLoop): 38 | print(await asyncio.sleep(random.randint(0, 3), result=f"Work {os.getpid()}")) 39 | 40 | 41 | def process_main(coro_worker: typing.Coroutine, num_of_coroutines: int): 42 | """ 43 | This is the main method of the process. 44 | We create the infrastructure for the coroutine worker here. 45 | We assert that we run one loop per process and use our own helper to get an event loop instance for that matter. 46 | :param coro_worker: 47 | :param url: 48 | :param num_of_coroutines: 49 | :return: 50 | """ 51 | asyncio_init() 52 | loop = get_event_loop() 53 | assert os.getpid() == loop.pid 54 | try: 55 | workers = [coro_worker(loop=loop) for i in range(num_of_coroutines)] 56 | loop.run_until_complete(asyncio.gather(*workers, loop=loop)) 57 | except KeyboardInterrupt: 58 | print(f"Stopping {os.getpid()}") 59 | loop.stop() 60 | finally: 61 | loop.close() 62 | 63 | 64 | def main(number_of_processes, number_of_coroutines, *, process_main): 65 | global processes 66 | for _ in range(number_of_processes): 67 | proc = Process(target=process_main, args=(worker, number_of_coroutines)) 68 | processes.append(proc) 69 | proc.start() 70 | cleanup() 71 | 72 | 73 | try: 74 | main(number_of_processes=10, number_of_coroutines=2, process_main=process_main) 75 | except KeyboardInterrupt: 76 | print("CTRL+C was pressed.. Stopping all subprocesses..") 77 | cleanup() 78 | print("Cleanup finished") 79 | -------------------------------------------------------------------------------- /chapter_02/sol_04_unix.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import os 3 | 4 | pid_loops = {} 5 | 6 | 7 | def get_event_loop(): 8 | pid = os.getpid() 9 | if pid not in pid_loops: 10 | pid_loops[pid] = asyncio.new_event_loop() 11 | return pid_loops[pid] 12 | 13 | 14 | def asyncio_init(): 15 | pid = os.getpid() 16 | pid_loops[pid] = asyncio.new_event_loop() 17 | pid_loops[pid].pid = pid 18 | 19 | 20 | os.register_at_fork(after_in_parent=asyncio_init, after_in_child=asyncio_init) 21 | 22 | if os.fork() == 0: 23 | # Child 24 | loop = get_event_loop() 25 | assert os.getpid() == loop.pid 26 | else: 27 | # Parent 28 | loop = get_event_loop() 29 | assert os.getpid() == loop.pid 30 | -------------------------------------------------------------------------------- /chapter_02/sol_05.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import sys 3 | 4 | loop = asyncio.new_event_loop() 5 | asyncio.set_event_loop(loop) 6 | 7 | if sys.platform != "win32": 8 | watcher = asyncio.get_child_watcher() 9 | watcher.attach_loop(loop) 10 | 11 | # Use asyncio.ensure_future to schedule your first coroutines 12 | # here or call loop.call_soon to schedule a synchronous callback 13 | 14 | try: 15 | loop.run_forever() 16 | finally: 17 | try: 18 | loop.run_until_complete(loop.shutdown_asyncgens()) 19 | finally: 20 | loop.close() 21 | -------------------------------------------------------------------------------- /chapter_02/sol_06.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def main(): 5 | pass 6 | 7 | 8 | asyncio.run(main()) 9 | -------------------------------------------------------------------------------- /chapter_02/sol_07_option_1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def main(): 5 | pass 6 | 7 | 8 | asyncio.run(main()) 9 | -------------------------------------------------------------------------------- /chapter_02/sol_07_option_2.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def main(): 5 | pass 6 | 7 | 8 | loop = asyncio.get_event_loop() 9 | 10 | try: 11 | loop.run_until_complete(main()) 12 | finally: 13 | try: 14 | loop.run_until_complete(loop.shutdown_asyncgens()) 15 | finally: 16 | loop.close() 17 | -------------------------------------------------------------------------------- /chapter_02/sol_07_option_3.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import sys 3 | 4 | 5 | async def main(): 6 | pass 7 | 8 | 9 | loop = asyncio.new_event_loop() 10 | asyncio.set_event_loop(loop) 11 | 12 | if sys.platform != "win32": 13 | watcher = asyncio.get_child_watcher() 14 | watcher.attach_loop(loop) 15 | 16 | try: 17 | loop.run_forever() 18 | finally: 19 | try: 20 | loop.run_until_complete(loop.shutdown_asyncgens()) 21 | finally: 22 | loop.close() 23 | -------------------------------------------------------------------------------- /chapter_02/sol_08_option_1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | loop = asyncio.get_event_loop() 4 | loop.call_soon(print, "I am scheduled on a loop!") 5 | loop.call_soon_threadsafe(print, "I am scheduled on a loop but threadsafely!") 6 | loop.call_later(1, print, "I am scheduled on a loop in one second") 7 | loop.call_at(loop.time() + 1, print, "I am scheduled on a loop in one second too") 8 | 9 | try: 10 | print("Stop the loop by hitting the CTRL+C keys…") 11 | # To see the callbacks running you need to start the running loop 12 | loop.run_forever() 13 | except KeyboardInterrupt: 14 | loop.stop() 15 | finally: 16 | loop.close() -------------------------------------------------------------------------------- /chapter_02/sol_08_option_2.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import random 3 | 4 | 5 | async def work(i): 6 | print(await asyncio.sleep(random.randint(0, i), result=f"Concurrent work {i}")) 7 | 8 | 9 | async def main(): 10 | tasks = [asyncio.ensure_future(work(i)) for i in range(10)] 11 | await asyncio.gather(*tasks) 12 | 13 | 14 | asyncio.run(main()) 15 | -------------------------------------------------------------------------------- /chapter_02/sol_09_1_option_1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import random 3 | 4 | 5 | async def work(i): 6 | print(await asyncio.sleep(random.randint(0, i), result=f"Concurrent work {i}")) 7 | 8 | 9 | async def main(): 10 | tasks = [asyncio.ensure_future(work(i)) for i in range(10)] 11 | await asyncio.gather(*tasks) 12 | 13 | 14 | asyncio.run(main()) 15 | -------------------------------------------------------------------------------- /chapter_02/sol_09_1_option_2.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import random 3 | 4 | 5 | async def work(i): 6 | print(await asyncio.sleep(random.randint(0, i), result=f"Concurrent work {i}")) 7 | 8 | 9 | loop = asyncio.get_event_loop() 10 | tasks = [asyncio.ensure_future(work(i)) for i in range(10)] 11 | loop.run_until_complete(asyncio.gather(*tasks)) 12 | -------------------------------------------------------------------------------- /chapter_02/sol_09_2.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def work(): 5 | print("Main was called.") 6 | 7 | 8 | class AsyncSchedulerLoop(asyncio.SelectorEventLoop): 9 | 10 | def __init__(self): 11 | super(AsyncSchedulerLoop, self).__init__() 12 | self.coros = asyncio.Queue(loop=self) 13 | 14 | def schedule(self, coro): 15 | task = self.create_task(coro) 16 | task.add_done_callback(lambda _: self.coros.task_done()) 17 | self.coros.put_nowait(task) 18 | 19 | async def wait_for_all(self): 20 | await self.coros.join() 21 | 22 | 23 | class AsyncSchedulerLoopPolicy(asyncio.DefaultEventLoopPolicy): 24 | def new_event_loop(self): 25 | return AsyncSchedulerLoop() 26 | 27 | 28 | asyncio.set_event_loop_policy(AsyncSchedulerLoopPolicy()) 29 | loop = asyncio.get_event_loop() 30 | 31 | for i in range(1000): 32 | loop.schedule(work()) 33 | 34 | loop.run_until_complete(loop.wait_for_all()) 35 | -------------------------------------------------------------------------------- /chapter_02/sol_10.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from concurrent.futures.thread import ThreadPoolExecutor 3 | 4 | import certifi 5 | import urllib3 6 | 7 | HTTP_POOL_MANAGER = urllib3.PoolManager(ca_certs=certifi.where()) 8 | EXECUTOR = ThreadPoolExecutor(10) 9 | URL = "https://apress.com" 10 | 11 | 12 | async def block_request(http, url, *, executor=None, loop: asyncio.AbstractEventLoop): 13 | return await loop.run_in_executor(executor, http.request, "GET", url) 14 | 15 | 16 | def multi_block_requests(http, url, n, *, executor=None, loop: asyncio.AbstractEventLoop): 17 | return (asyncio.ensure_future(block_request(http, url, executor=executor, loop=loop)) for _ in 18 | range(n)) 19 | 20 | 21 | async def consume_responses(*coro, loop): 22 | result = await asyncio.gather(*coro, loop=loop, return_exceptions=True) 23 | for res in result: 24 | if not isinstance(res, Exception): 25 | print(res.data) 26 | 27 | 28 | loop = asyncio.get_event_loop() 29 | loop.set_default_executor(EXECUTOR) 30 | loop.run_until_complete(consume_responses(block_request(HTTP_POOL_MANAGER, URL, loop=loop), loop=loop)) 31 | loop.run_until_complete( 32 | consume_responses(*multi_block_requests(HTTP_POOL_MANAGER, URL, 10, loop=loop), loop=loop)) 33 | -------------------------------------------------------------------------------- /chapter_02/sol_11_option_1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def main(loop): 5 | assert loop == asyncio.get_running_loop() 6 | 7 | 8 | loop = asyncio.get_event_loop() 9 | loop.run_until_complete(main(loop)) 10 | -------------------------------------------------------------------------------- /chapter_02/sol_11_option_2.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def main(): 5 | pass 6 | 7 | 8 | loop = asyncio.get_event_loop() 9 | task = loop.create_task(main()) 10 | task.add_done_callback(lambda fut: loop.stop()) 11 | # Or more generic if you don't have loop in scope: 12 | # task.add_done_callback(lambda fut: asyncio.get_running_loop().stop()) 13 | 14 | loop.run_forever() 15 | -------------------------------------------------------------------------------- /chapter_02/sol_12.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import functools 3 | 4 | 5 | async def main(loop): 6 | print("Print in main") 7 | 8 | 9 | def stop_loop(fut, *, loop): 10 | loop.call_soon_threadsafe(loop.stop) 11 | 12 | 13 | loop = asyncio.get_event_loop() 14 | tasks = [loop.create_task(main(loop)) for _ in range(10)] 15 | asyncio.gather(*tasks).add_done_callback(functools.partial(stop_loop, loop=loop)) 16 | try: 17 | loop.run_forever() 18 | finally: 19 | try: 20 | loop.run_until_complete(loop.shutdown_asyncgens()) 21 | finally: 22 | loop.close() # optional 23 | -------------------------------------------------------------------------------- /chapter_02/sol_13.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import functools 3 | import os 4 | import signal 5 | 6 | SIGNAL_NAMES = ('SIGINT', 'SIGTERM') 7 | SIGNAL_NAME_MESSAGE = " or ".join(SIGNAL_NAMES) 8 | 9 | 10 | def sigint_handler(signame, *, loop, ): 11 | print(f"Stopped loop because of {signame}") 12 | loop.stop() 13 | 14 | 15 | def sigterm_handler(signame, *, loop, ): 16 | print(f"Stopped loop because of {signame}") 17 | loop.stop() 18 | 19 | 20 | loop = asyncio.get_event_loop() 21 | 22 | for signame in SIGNAL_NAMES: 23 | loop.add_signal_handler(getattr(signal, signame), 24 | functools.partial(locals()[f"{signame.lower()}_handler"], signame, loop=loop)) 25 | 26 | print("Event loop running forever, press Ctrl+C to interrupt.") 27 | print(f"pid {os.getpid()}: send {SIGNAL_NAME_MESSAGE} to exit.") 28 | try: 29 | loop.run_forever() 30 | finally: 31 | loop.close() # optional 32 | -------------------------------------------------------------------------------- /chapter_02/sol_14.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import shutil 3 | import sys 4 | from typing import Tuple, Union 5 | 6 | 7 | async def invoke_command_async(*command, loop, encoding="UTF-8", decode=True) -> Tuple[ 8 | Union[str, bytes], Union[str, bytes], int]: 9 | """ 10 | Invoke a command asynchronously and return the stdout, stderr and the process return code. 11 | :param command: 12 | :param loop: 13 | :param encoding: 14 | :param decode: 15 | :return: 16 | """ 17 | if sys.platform != 'win32': 18 | asyncio.get_child_watcher().attach_loop(loop) 19 | process = await asyncio.create_subprocess_exec(*command, 20 | stdout=asyncio.subprocess.PIPE, 21 | stderr=asyncio.subprocess.PIPE, 22 | loop=loop) 23 | out, err = await process.communicate() 24 | 25 | ret_code = process.returncode 26 | 27 | if not decode: 28 | return out, err, ret_code 29 | 30 | output_decoded, err_decoded = out.decode(encoding) if out else None, \ 31 | err.decode(encoding) if err else None 32 | 33 | return output_decoded, err_decoded, ret_code 34 | 35 | 36 | async def main(loop): 37 | out, err, retcode = await invoke_command_async(shutil.which("ping"), "-c", "1", "8.8.8.8", loop=loop) 38 | print(out, err, retcode) 39 | 40 | 41 | if sys.platform == "win32": 42 | asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) 43 | 44 | loop = asyncio.get_event_loop() 45 | loop.run_until_complete(main(loop)) 46 | -------------------------------------------------------------------------------- /chapter_02/sol_15.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | # Quote from https://docs.python.org/3/library/asyncio-subprocess.html: 4 | # The child watcher must be instantiated in the main thread, before executing subprocesses from other threads. Call the get_child_watcher() function in the main thread to instantiate the child watcher. 5 | import functools 6 | import shutil 7 | import sys 8 | 9 | if sys.platform == "win32": 10 | asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) 11 | 12 | def stop_loop(*args, loop, **kwargs): 13 | loop.stop() 14 | 15 | 16 | async def is_windows_process_alive(process, delay=0.5): 17 | """ 18 | On windows the signal API is very sparse, meaning we don't have SIGCHILD. 19 | So we just check if we have a return code on our process object. 20 | :param process: 21 | :param delay: 22 | :return: 23 | """ 24 | while process.returncode == None: 25 | await asyncio.sleep(delay) 26 | 27 | 28 | async def main(process_coro, *, loop): 29 | process = await process_coro 30 | if sys.platform != "win32": 31 | child_watcher: asyncio.AbstractChildWatcher = asyncio.get_child_watcher() 32 | child_watcher.add_child_handler(process.pid, functools.partial(stop_loop, loop=loop)) 33 | else: 34 | await is_windows_process_alive(process) 35 | loop.stop() 36 | 37 | 38 | loop = asyncio.get_event_loop() 39 | 40 | process_coro = asyncio.create_subprocess_exec(shutil.which("ping"), "-c", "1", "127.0.0.1", 41 | stdout=asyncio.subprocess.DEVNULL, 42 | stderr=asyncio.subprocess.DEVNULL) 43 | 44 | loop.create_task(main(process_coro, loop=loop)) 45 | loop.run_forever() -------------------------------------------------------------------------------- /chapter_03/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/asyncio-recipes/f078242d4380407e483236695ebc49f3faf09e7f/chapter_03/__init__.py -------------------------------------------------------------------------------- /chapter_03/sol_01.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | @asyncio.coroutine 4 | def coro(): 5 | value = yield from inner() 6 | print(value) 7 | 8 | @asyncio.coroutine 9 | def inner(): 10 | return [1, 2, 3] 11 | 12 | asyncio.run(coro()) # will print [1, 2, 3] -------------------------------------------------------------------------------- /chapter_03/sol_02.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def coroutine(*args, **kwargs): 5 | pass 6 | 7 | 8 | assert asyncio.iscoroutine(coroutine()) 9 | assert asyncio.iscoroutinefunction(coroutine) 10 | -------------------------------------------------------------------------------- /chapter_03/sol_03.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def coroutine(*args, **kwargs): 5 | print("Waiting for the next coroutine...") 6 | await another_coroutine(*args, **kwargs) 7 | print("This will follow 'Done'") 8 | 9 | 10 | async def another_coroutine(*args, **kwargs): 11 | await asyncio.sleep(3) 12 | print("Done") 13 | -------------------------------------------------------------------------------- /chapter_03/sol_04.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def coroutine_to_run(): 5 | print(await asyncio.sleep(1, result="I have finished!")) 6 | 7 | 8 | async def main(): 9 | task = asyncio.create_task(coroutine_to_run()) 10 | await task 11 | 12 | 13 | asyncio.run(main()) 14 | -------------------------------------------------------------------------------- /chapter_03/sol_05.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def delayed_print(text, delay): 5 | print(await asyncio.sleep(delay, text)) 6 | 7 | 8 | async def main(): 9 | delay = 3 10 | 11 | on_time_coro = delayed_print(f"I will print after {delay} seconds", delay) 12 | await asyncio.wait_for(on_time_coro, delay + 1) 13 | 14 | try: 15 | delayed_coro = delayed_print(f"I will print after {delay+1} seconds", delay + 1) 16 | await asyncio.wait_for(delayed_coro, delay) 17 | except asyncio.TimeoutError: 18 | print(f"I timed out after {delay} seconds") 19 | 20 | 21 | asyncio.run(main()) -------------------------------------------------------------------------------- /chapter_03/sol_06_option_1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | async def cancellable(delay=10): 4 | loop = asyncio.get_running_loop() 5 | try: 6 | now = loop.time() 7 | print(f"Sleeping from {now} for {delay} seconds ...") 8 | await asyncio.sleep(delay, loop=loop) 9 | print(f"Slept {delay} seconds ...") 10 | except asyncio.CancelledError: 11 | print(f"Cancelled at {now} after {loop.time()-now} seconds") 12 | 13 | 14 | async def main(): 15 | coro = cancellable() 16 | task = asyncio.create_task(coro) 17 | await asyncio.sleep(3) 18 | task.cancel() 19 | 20 | 21 | asyncio.run(main()) 22 | -------------------------------------------------------------------------------- /chapter_03/sol_06_option_2.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def cancellable(delay=10): 5 | loop = asyncio.get_running_loop() 6 | try: 7 | now = loop.time() 8 | print(f"Sleeping from {now} for {delay} seconds ...") 9 | await asyncio.sleep(delay) 10 | print(f"Slept for {delay} seconds without disturbance...") 11 | except asyncio.CancelledError: 12 | print(f"Cancelled at {now} after {loop.time()-now} seconds") 13 | 14 | 15 | async def main(): 16 | coro = cancellable() 17 | task = asyncio.create_task(coro) 18 | await asyncio.sleep(3) 19 | 20 | def canceller(task, fut): 21 | task.cancel() 22 | fut.set_result(None) 23 | 24 | loop = asyncio.get_running_loop() 25 | fut = loop.create_future() 26 | loop.call_soon_threadsafe(canceller, task, fut) 27 | await fut 28 | 29 | 30 | asyncio.run(main()) 31 | -------------------------------------------------------------------------------- /chapter_03/sol_07.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | async def cancellable(delay=10, *, loop): 4 | try: 5 | now = loop.time() 6 | print(f"Sleeping from {now} for {delay} seconds ...") 7 | await asyncio.sleep(delay) 8 | print(f"Slept for {delay} seconds without disturbance...") 9 | except asyncio.CancelledError: 10 | print(f"Cancelled at {now} after {loop.time()-now} seconds") 11 | 12 | 13 | def canceller(task, fut): 14 | task.cancel() 15 | fut.set_result(None) 16 | 17 | 18 | async def cancel_threadsafe(gathered_tasks, loop): 19 | fut = loop.create_future() 20 | loop.call_soon_threadsafe(canceller, gathered_tasks, fut) 21 | await fut 22 | 23 | 24 | async def main(): 25 | loop = asyncio.get_running_loop() 26 | coros = [cancellable(i, loop=loop) for i in range(10)] 27 | gathered_tasks = asyncio.gather(*coros) 28 | # Add a delay here, so we can see that the first three coroutines run uninterrupted 29 | await asyncio.sleep(3) 30 | await cancel_threadsafe(gathered_tasks, loop) 31 | try: 32 | await gathered_tasks 33 | except asyncio.CancelledError: 34 | print("Was cancelled") 35 | 36 | 37 | asyncio.run(main()) 38 | -------------------------------------------------------------------------------- /chapter_03/sol_08.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def cancellable(delay=10): 5 | now = asyncio.get_running_loop().time() 6 | try: 7 | print(f"Sleeping from {now} for {delay} seconds ...") 8 | await asyncio.sleep(delay) 9 | print(f"Slept for {delay} seconds without disturbance...") 10 | except asyncio.CancelledError: 11 | print("I was disturbed in my sleep!") 12 | 13 | 14 | def canceller(task, fut): 15 | task.cancel() 16 | fut.set_result(None) 17 | 18 | 19 | async def cancel_threadsafe(task, *, delay=3, loop): 20 | await asyncio.sleep(delay) 21 | fut = loop.create_future() 22 | loop.call_soon_threadsafe(canceller, task, fut) 23 | await fut 24 | 25 | 26 | async def main(): 27 | complete_time = 10 28 | cancel_after_secs = 3 29 | loop = asyncio.get_running_loop() 30 | coro = cancellable(delay=complete_time) 31 | shielded_task = asyncio.shield(coro) 32 | asyncio.create_task( 33 | cancel_threadsafe(shielded_task, 34 | delay=cancel_after_secs, 35 | loop=loop) 36 | ) 37 | try: 38 | await shielded_task 39 | except asyncio.CancelledError: 40 | await asyncio.sleep(complete_time - cancel_after_secs) 41 | 42 | 43 | asyncio.run(main()) 44 | -------------------------------------------------------------------------------- /chapter_03/sol_09.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def print_delayed(delay, text): 5 | print(await asyncio.sleep(delay, text)) 6 | 7 | 8 | async def main(): 9 | await print_delayed(1, "Printing this after 1 second") 10 | await print_delayed(1, "Printing this after 2 seconds") 11 | await print_delayed(1, "Printing this after 3 seconds") 12 | 13 | 14 | asyncio.run(main()) 15 | -------------------------------------------------------------------------------- /chapter_03/sol_10.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def print_delayed(delay, text, result): 5 | print(await asyncio.sleep(delay, text)) 6 | return result 7 | 8 | 9 | async def main(): 10 | workload = [ 11 | print_delayed(1, "Printing this after 1 second", 1), 12 | print_delayed(1, "Printing this after 1 second", 2), 13 | print_delayed(1, "Printing this after 1 second", 3), 14 | ] 15 | 16 | results = await asyncio.gather(*workload) 17 | print(results) 18 | 19 | 20 | asyncio.run(main()) 21 | -------------------------------------------------------------------------------- /chapter_03/sol_11_option_1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def raiser(): 5 | raise Exception("An exception was raised!") 6 | 7 | 8 | async def main(): 9 | raiser_task = asyncio.create_task(raiser()) 10 | hello_world_task = asyncio.create_task(asyncio.sleep(1.0, "I have returned!")) 11 | tasks = {raiser_task, hello_world_task} 12 | finished, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) 13 | 14 | assert raiser_task in finished 15 | assert raiser_task not in pending 16 | assert hello_world_task in finished 17 | assert hello_world_task not in pending 18 | 19 | print(raiser_task.exception()) 20 | print(hello_world_task.result()) 21 | 22 | 23 | asyncio.run(main()) 24 | -------------------------------------------------------------------------------- /chapter_03/sol_11_option_2.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def raiser(): 5 | raise Exception("An exception was raised!") 6 | 7 | 8 | async def main(): 9 | raiser_task = asyncio.create_task(raiser()) 10 | hello_world_task = asyncio.create_task(asyncio.sleep(10.0, "I have returned!")) 11 | tasks = {raiser_task, hello_world_task} 12 | finished, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_EXCEPTION) 13 | assert raiser_task in finished 14 | assert raiser_task not in pending 15 | assert hello_world_task not in finished 16 | assert hello_world_task in pending 17 | print(raiser_task.exception()) 18 | err_was_thrown = None 19 | try: 20 | print(hello_world_task.result()) 21 | except asyncio.InvalidStateError as err: 22 | err_was_thrown = err 23 | 24 | assert err_was_thrown 25 | 26 | 27 | asyncio.run(main()) 28 | -------------------------------------------------------------------------------- /chapter_03/sol_11_option_3.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def raiser(): 5 | raise Exception("An exception was raised!") 6 | 7 | 8 | async def main(): 9 | raiser_task = asyncio.create_task(raiser()) 10 | hello_world_task = asyncio.create_task(asyncio.sleep(10.0, "I have returned!")) 11 | tasks = {raiser_task, hello_world_task} 12 | finished, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) 13 | assert raiser_task in finished 14 | assert raiser_task not in pending 15 | assert hello_world_task not in finished 16 | assert hello_world_task in pending 17 | print(raiser_task.exception()) 18 | err_was_thrown = None 19 | try: 20 | print(hello_world_task.result()) 21 | except asyncio.InvalidStateError as err: 22 | err_was_thrown = err 23 | 24 | assert err_was_thrown 25 | 26 | 27 | asyncio.run(main()) 28 | -------------------------------------------------------------------------------- /chapter_03/sol_12.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import sys 3 | 4 | 5 | async def print_delayed(delay, text, ): 6 | print(await asyncio.sleep(delay, text)) 7 | 8 | 9 | async def raise_delayed(delay, text, ): 10 | raise Exception(await asyncio.sleep(delay, text)) 11 | 12 | 13 | async def main(): 14 | workload = [ 15 | print_delayed(5, "Printing this after 5 seconds"), 16 | raise_delayed(5, "Raising this after 5 seconds"), 17 | print_delayed(5, "Printing this after 5 seconds"), 18 | ] 19 | 20 | res = None 21 | try: 22 | gathered = asyncio.gather(*workload, return_exceptions=True) 23 | res = await gathered 24 | except asyncio.CancelledError: 25 | print("The gathered task was cancelled", file=sys.stderr) 26 | finally: 27 | print("Result:", res) 28 | 29 | 30 | asyncio.run(main()) 31 | -------------------------------------------------------------------------------- /chapter_03/sol_13.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def execute_on(condition, coro, predicate): 5 | async with condition: 6 | await condition.wait_for(predicate) 7 | await coro 8 | 9 | 10 | async def print_coro(text): 11 | print(text) 12 | 13 | 14 | async def worker(numbers): 15 | while numbers: 16 | print("Numbers:", numbers) 17 | numbers.pop() 18 | await asyncio.sleep(0.25) 19 | 20 | 21 | async def main(): 22 | numbers = list(range(10)) 23 | condition = asyncio.Condition() 24 | is_empty = lambda: not numbers 25 | await worker(numbers) 26 | await execute_on(condition, print_coro("Finished!"), is_empty) 27 | 28 | 29 | asyncio.run(main()) 30 | -------------------------------------------------------------------------------- /chapter_04/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/asyncio-recipes/f078242d4380407e483236695ebc49f3faf09e7f/chapter_04/__init__.py -------------------------------------------------------------------------------- /chapter_04/sol_01.py: -------------------------------------------------------------------------------- 1 | 2 | import random 3 | import asyncio 4 | 5 | async def random_number_gen(delay,start,end): 6 | while True: 7 | yield random.randint(start,end) 8 | await asyncio.sleep(delay) 9 | -------------------------------------------------------------------------------- /chapter_04/sol_02.py: -------------------------------------------------------------------------------- 1 | import random 2 | import asyncio 3 | 4 | 5 | async def random_number_gen(delay, start, end): 6 | while True: 7 | yield random.randint(start, end) 8 | await asyncio.sleep(delay) 9 | 10 | 11 | async def main(): 12 | async for i in random_number_gen(1, 0, 100): 13 | print(i) 14 | 15 | 16 | try: 17 | print("Starting to print out random numbers...") 18 | print("Shut down the application with Ctrl+C") 19 | asyncio.run(main()) 20 | except KeyboardInterrupt: 21 | print("Closed the main loop..") 22 | -------------------------------------------------------------------------------- /chapter_04/sol_03.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import functools 3 | from concurrent.futures.thread import ThreadPoolExecutor 4 | 5 | import sys 6 | import certifi 7 | import urllib3 8 | 9 | 10 | async def request(poolmanager: urllib3.PoolManager, 11 | executor, 12 | *, 13 | method="GET", 14 | url, 15 | fields=None, 16 | headers=None, 17 | loop: asyncio.AbstractEventLoop = None, ): 18 | if not loop: 19 | loop = asyncio.get_running_loop() 20 | request = functools.partial(poolmanager.request, method, url, fields=fields, headers=headers) 21 | return loop.run_in_executor(executor, request) 22 | 23 | 24 | async def bulk_requests(poolmanager: urllib3.PoolManager, 25 | executor, 26 | *, 27 | method="GET", 28 | urls, 29 | fields=None, 30 | headers=None, 31 | loop: asyncio.AbstractEventLoop = None, ): 32 | for url in urls: 33 | yield await request(poolmanager, executor, url=url, fields=fields, headers=headers, loop=loop) 34 | 35 | 36 | def filter_unsuccesful_requests(responses_and_exceptions): 37 | return filter( 38 | lambda url_and_response: not isinstance(url_and_response[1], Exception), 39 | responses_and_exceptions.items() 40 | ) 41 | 42 | 43 | async def main(): 44 | poolmanager = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) 45 | executor = ThreadPoolExecutor(10) 46 | urls = [ 47 | "https://google.de", 48 | "https://apple.com", 49 | "https://apress.com", 50 | ] 51 | requests = [request async for request in bulk_requests(poolmanager, executor, urls=urls, )] 52 | responses_and_exceptions = dict(zip(urls, await asyncio.gather(*requests, return_exceptions=True))) 53 | responses = {url: resp.data for (url, resp) in filter_unsuccesful_requests(responses_and_exceptions)} 54 | 55 | for res in responses.items(): 56 | print(res) 57 | 58 | for url in urls: 59 | if url not in responses: 60 | print(f"No successful request could be made to {url}. Reason: {responses_and_exceptions[url]}",file=sys.stderr) 61 | 62 | 63 | asyncio.run(main()) -------------------------------------------------------------------------------- /chapter_04/sol_04.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import enum 3 | import logging 4 | import sys 5 | from dataclasses import dataclass 6 | 7 | 8 | class State(enum.Enum): 9 | IDLE = enum.auto() 10 | STARTED = enum.auto() 11 | PAUSED = enum.auto() 12 | 13 | 14 | @dataclass(frozen=True) 15 | class Event: 16 | name: str 17 | 18 | 19 | START = Event("Start") 20 | PAUSE = Event("Pause") 21 | STOP = Event("Stop") 22 | EXIT = Event("Exit") 23 | 24 | STATES = (START, PAUSE, STOP, EXIT) 25 | CHOICES = "\n".join([f"{i}: {state.name}" for i, state in enumerate(STATES)]) 26 | 27 | MENU = f""" 28 | Menu 29 | 30 | Enter your choice: 31 | 32 | {CHOICES} 33 | 34 | """ 35 | 36 | TRANSITIONS = { 37 | (State.IDLE, PAUSE): State.IDLE, 38 | (State.IDLE, START): State.STARTED, 39 | (State.IDLE, STOP): State.IDLE, 40 | 41 | (State.STARTED, START): State.STARTED, 42 | (State.STARTED, PAUSE): State.PAUSED, 43 | (State.STARTED, STOP): State.IDLE, 44 | 45 | (State.PAUSED, START): State.STARTED, 46 | (State.PAUSED, PAUSE): State.PAUSED, 47 | (State.PAUSED, STOP): State.IDLE, 48 | 49 | (State.IDLE, EXIT): State.IDLE, 50 | (State.STARTED, EXIT): State.IDLE, 51 | (State.PAUSED, EXIT): State.IDLE, 52 | } 53 | 54 | 55 | class StateMachineException(Exception): 56 | pass 57 | 58 | 59 | class StartStateMachineException(StateMachineException): 60 | pass 61 | 62 | 63 | class StopStateMachineException(StateMachineException): 64 | pass 65 | 66 | 67 | async def next_state(state_machine, event, *, exc=StateMachineException): 68 | try: 69 | if state_machine: 70 | await state_machine.asend(event) 71 | except StopAsyncIteration: 72 | if exc != StopStateMachineException: 73 | raise exc() 74 | except: 75 | raise exc() 76 | 77 | 78 | async def start_statemachine(state_machine, ): 79 | await next_state(state_machine, None, exc=StartStateMachineException) 80 | 81 | 82 | async def stop_statemachine(state_machine, ): 83 | await next_state(state_machine, EXIT, exc=StopStateMachineException) 84 | 85 | 86 | async def create_state_machine(transitions, *, logger=None, ): 87 | if not logger: 88 | logger = logging.getLogger(__name__) 89 | event, current_state = None, State.IDLE 90 | while event != EXIT: 91 | 92 | event = yield 93 | 94 | edge = (current_state, event) 95 | 96 | if edge not in transitions: 97 | logger.error("Cannot consume %s in state %s", event.name, current_state.name) 98 | continue 99 | 100 | next_state = transitions.get(edge) 101 | logger.debug("Transitioning from %s to %s", current_state.name, next_state.name) 102 | current_state = next_state 103 | 104 | 105 | def pick_next_event(logger): 106 | next_state = None 107 | 108 | while not next_state: 109 | try: 110 | next_state = STATES[int(input(MENU))] 111 | except (ValueError, IndexError): 112 | logger.error("Please enter a valid choice!") 113 | continue 114 | 115 | return next_state 116 | 117 | 118 | async def main(logger): 119 | state_machine = create_state_machine(TRANSITIONS, logger=logger) 120 | 121 | try: 122 | await start_statemachine(state_machine) 123 | 124 | while True: 125 | event = pick_next_event(logger) 126 | if event != EXIT: 127 | await next_state(state_machine, event) 128 | else: 129 | await stop_statemachine(state_machine) 130 | 131 | except StartStateMachineException: 132 | logger.error("Starting the statemachine was unsuccessful") 133 | except StopStateMachineException: 134 | logger.error("Stopping the statemachine was unsuccessful") 135 | except StateMachineException: 136 | logger.error("Transitioning the statemachine was unsuccessful") 137 | 138 | 139 | logger = logging.getLogger(__name__) 140 | logger.addHandler(logging.StreamHandler(sys.stdout)) 141 | logger.setLevel(logging.DEBUG) 142 | 143 | try: 144 | asyncio.get_event_loop().run_until_complete(main(logger)) 145 | except KeyboardInterrupt: 146 | logger.info("Closed loop..") -------------------------------------------------------------------------------- /chapter_04/sol_05_option_1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def async_gen_coro(): 5 | yield 1 6 | yield 2 7 | yield 3 8 | 9 | 10 | async def main(): 11 | async_generator = async_gen_coro() 12 | await async_generator.asend(None) 13 | await async_generator.asend(None) 14 | 15 | asyncio.run(main()) 16 | -------------------------------------------------------------------------------- /chapter_04/sol_05_option_2.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | async def endless_async_gen(): 4 | while True: 5 | yield 3 6 | await asyncio.sleep(1) 7 | 8 | async def main(): 9 | async for i in endless_async_gen(): 10 | print(i) 11 | 12 | loop = asyncio.new_event_loop() 13 | asyncio.set_event_loop(loop) 14 | 15 | 16 | try: 17 | loop.run_until_complete(main()) 18 | except KeyboardInterrupt: 19 | print("Caught Ctrl+C. Exiting now..") 20 | finally: 21 | try: 22 | loop.run_until_complete(loop.shutdown_asyncgens()) 23 | finally: 24 | loop.close() -------------------------------------------------------------------------------- /chapter_04/sol_06.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import re 4 | 5 | import typing 6 | 7 | from concurrent.futures import Executor, ThreadPoolExecutor 8 | 9 | from urllib.request import urlopen 10 | 11 | DEFAULT_EXECUTOR = ThreadPoolExecutor(4) 12 | ANCHOR_TAG_PATTERN = re.compile(b"", re.RegexFlag.MULTILINE | re.RegexFlag.IGNORECASE) 13 | 14 | async def wrap_async(generator: typing.Generator, 15 | executor: Executor = DEFAULT_EXECUTOR, 16 | sentinel=object(), 17 | *, 18 | loop: asyncio.AbstractEventLoop = None): 19 | """ 20 | We wrap a generator and return an asynchronous generator instead 21 | :param iterator: 22 | :param executor: 23 | :param sentinel: 24 | :param loop: 25 | :return: 26 | """ 27 | 28 | if not loop: 29 | loop = asyncio.get_running_loop() 30 | 31 | while True: 32 | result = await loop.run_in_executor(executor, next, generator, sentinel) 33 | if result == sentinel: 34 | break 35 | yield result 36 | 37 | 38 | def follow(*links): 39 | """ 40 | :param links: 41 | :return: 42 | """ 43 | 44 | return ((link, urlopen(link).read()) for link in links) 45 | 46 | 47 | def get_links(text: str): 48 | """ 49 | Get back an iterator that gets us all the links in a text iteratively and safely 50 | :param text: 51 | :return: 52 | """ 53 | 54 | # Always grab the last match, because that is how a smart http parser would interpret a malformed 55 | # anchor tag 56 | return (match.groups()[-1] 57 | for match in ANCHOR_TAG_PATTERN.finditer(text) 58 | # This portion is a safeguard against None matches and zero href matches 59 | if hasattr(match, "groups") and len(match.groups())) 60 | 61 | 62 | async def main(*links): 63 | async for current, body in wrap_async(follow(*links)): 64 | print("Current url:", current) 65 | print("Content:", body) 66 | async for link in wrap_async(get_links(body)): 67 | print(link) 68 | 69 | 70 | asyncio.run(main("http://apress.com")) -------------------------------------------------------------------------------- /chapter_05/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/asyncio-recipes/f078242d4380407e483236695ebc49f3faf09e7f/chapter_05/__init__.py -------------------------------------------------------------------------------- /chapter_05/sol_01.py: -------------------------------------------------------------------------------- 1 | from concurrent.futures.thread import ThreadPoolExecutor 2 | from contextlib import asynccontextmanager 3 | import asyncio 4 | 5 | 6 | class AsyncFile(object): 7 | 8 | def __init__(self, file, loop=None, executor=None): 9 | if not loop: 10 | loop = asyncio.get_running_loop() 11 | if not executor: 12 | executor = ThreadPoolExecutor(10) 13 | self.file = file 14 | self.loop = loop 15 | self.executor = executor 16 | self.pending = [] 17 | self.result = [] 18 | 19 | def write(self, string): 20 | self.pending.append( 21 | self.loop.run_in_executor(self.executor, self.file.write, string, ) 22 | ) 23 | 24 | def read(self, i): 25 | self.pending.append( 26 | self.loop.run_in_executor(self.executor, self.file.read, i, ) 27 | ) 28 | 29 | def readlines(self): 30 | self.pending.append( 31 | self.loop.run_in_executor(self.executor, self.file.readlines, ) 32 | ) 33 | 34 | 35 | @asynccontextmanager 36 | async def async_open(path, mode="w"): 37 | with open(path, mode=mode) as f: 38 | loop = asyncio.get_running_loop() 39 | file = AsyncFile(f, loop=loop) 40 | try: 41 | yield file 42 | finally: 43 | file.result = await asyncio.gather(*file.pending, loop=loop) -------------------------------------------------------------------------------- /chapter_05/sol_02.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | class Sync(): 5 | def __init__(self): 6 | self.pending = [] 7 | self.finished = None 8 | 9 | def schedule_coro(self, coro, shield=False): 10 | fut = asyncio.shield(coro) if shield else asyncio.ensure_future(coro) 11 | self.pending.append(fut) 12 | return fut 13 | 14 | async def __aenter__(self): 15 | return self 16 | 17 | async def __aexit__(self, exc_type, exc_val, exc_tb): 18 | self.finished = await asyncio.gather(*self.pending, return_exceptions=True) 19 | 20 | 21 | async def workload(): 22 | await asyncio.sleep(3) 23 | print("These coroutines will be executed simultaneously and return 42") 24 | return 42 25 | 26 | 27 | async def main(): 28 | async with Sync() as sync: 29 | sync.schedule_coro(workload()) 30 | sync.schedule_coro(workload()) 31 | sync.schedule_coro(workload()) 32 | print("All scheduled coroutines have retuned or thrown:", sync.finished) 33 | 34 | 35 | asyncio.run(main()) -------------------------------------------------------------------------------- /chapter_05/sol_03.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import tempfile 3 | import os 4 | 5 | from chapter_05.sol_01 import async_open 6 | 7 | 8 | async def main(): 9 | tempdir = tempfile.gettempdir() 10 | path = os.path.join(tempdir, "run.txt") 11 | print(f"Writing asynchronously to {path}") 12 | 13 | async with async_open(path, mode="w") as f: 14 | f.write("This\n") 15 | f.write("might\n") 16 | f.write("not\n") 17 | f.write("end\n") 18 | f.write("up\n") 19 | f.write("in\n") 20 | f.write("the\n") 21 | f.write("same\n") 22 | f.write("order!\n") 23 | 24 | 25 | asyncio.run(main()) -------------------------------------------------------------------------------- /chapter_05/sol_04.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import socket 3 | from contextlib import asynccontextmanager 4 | 5 | 6 | @asynccontextmanager 7 | async def tcp_client(host='google.de', port=80): 8 | address_info = (await asyncio.get_running_loop().getaddrinfo( 9 | host, port, 10 | proto=socket.IPPROTO_TCP, 11 | )).pop() 12 | 13 | if not address_info: 14 | raise ValueError(f"Could not resolve {host}:{port}") 15 | host,port =address_info[-1] 16 | reader, writer = await asyncio.open_connection(host, port) 17 | try: 18 | yield (reader, writer) 19 | finally: 20 | writer.close() 21 | await asyncio.shield(writer.wait_closed()) 22 | 23 | 24 | async def main(): 25 | async with tcp_client() as (reader, writer): 26 | writer.write(b"GET /us HTTP/1.1\r\nhost: apress.com\r\n\r\n") 27 | await writer.drain() 28 | content = await reader.read(1024**2) 29 | print(content) 30 | 31 | asyncio.run(main()) -------------------------------------------------------------------------------- /chapter_05/sol_05.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from contextlib import asynccontextmanager 3 | from functools import partial as func 4 | 5 | 6 | class SchedulerLoop(asyncio.SelectorEventLoop): 7 | 8 | def __init__(self): 9 | super(SchedulerLoop, self).__init__() 10 | self._scheduled_callback_futures = [] 11 | self.results = [] 12 | 13 | @staticmethod 14 | def unwrapper(fut: asyncio.Future, function): 15 | """ 16 | Function to get rid of the implicit fut parameter. 17 | :param fut: 18 | :type fut: 19 | :param function: 20 | :return: 21 | """ 22 | return function() 23 | 24 | def _future(self, done_hook): 25 | """ 26 | Create a future object that calls the done_hook when it is awaited 27 | :param loop: 28 | :param function: 29 | :return: 30 | """ 31 | fut = self.create_future() 32 | fut.add_done_callback(func(self.unwrapper, function=done_hook)) 33 | return fut 34 | 35 | def schedule_soon_threadsafe(self, callback, *args, context=None): 36 | fut = self._future(func(callback, *args)) 37 | self._scheduled_callback_futures.append(fut) 38 | self.call_soon_threadsafe(fut.set_result, None, context=context) 39 | 40 | def schedule_soon(self, callback, *args, context=None): 41 | fut = self._future(func(callback, *args)) 42 | self._scheduled_callback_futures.append(fut) 43 | self.call_soon(fut.set_result, None, context=context) 44 | 45 | def schedule_later(self, delay_in_seconds, callback, *args, context=None): 46 | fut = self._future(func(callback, *args)) 47 | self._scheduled_callback_futures.append(fut) 48 | self.call_later(delay_in_seconds, fut.set_result, None, context=context) 49 | 50 | def schedule_at(self, delay_in_seconds, callback, *args, context=None): 51 | fut = self._future(func(callback, *args)) 52 | self._scheduled_callback_futures.append(fut) 53 | self.call_at(delay_in_seconds, fut.set_result, None, context=context) 54 | 55 | async def await_callbacks(self): 56 | callback_futs = self._scheduled_callback_futures[:] 57 | self._scheduled_callback_futures[:] = [] 58 | return await asyncio.gather(*callback_futs, return_exceptions=True, loop=self) 59 | 60 | 61 | class SchedulerLoopPolicy(asyncio.DefaultEventLoopPolicy): 62 | def new_event_loop(self): 63 | return SchedulerLoop() 64 | 65 | 66 | @asynccontextmanager 67 | async def scheduler_loop(): 68 | loop = asyncio.get_running_loop() 69 | if not isinstance(loop, SchedulerLoop): 70 | raise ValueError("You can run the scheduler_loop async context manager only on a SchedulerLoop") 71 | 72 | try: 73 | yield loop 74 | finally: 75 | loop.results = await loop.await_callbacks() 76 | 77 | 78 | async def main(): 79 | async with scheduler_loop() as loop: 80 | loop.schedule_soon(print, "This") 81 | loop.schedule_soon(print, "works") 82 | loop.schedule_soon(print, "seamlessly") 83 | 84 | 85 | asyncio.set_event_loop_policy(SchedulerLoopPolicy()) 86 | asyncio.run(main()) -------------------------------------------------------------------------------- /chapter_05/sol_06.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from concurrent.futures.process import ProcessPoolExecutor 3 | from contextlib import asynccontextmanager 4 | from multiprocessing import get_context, freeze_support 5 | 6 | CONTEXT = get_context("spawn") 7 | 8 | 9 | class AsyncProcessPool: 10 | 11 | def __init__(self, executor, loop=None, ): 12 | self.executor = executor 13 | if not loop: 14 | loop = asyncio.get_running_loop() 15 | self.loop = loop 16 | self.pending = [] 17 | self.result = None 18 | 19 | def submit(self, fn, *args, **kwargs): 20 | fut = asyncio.wrap_future(self.executor.submit(fn, *args, **kwargs), loop=self.loop) 21 | self.pending.append(fut) 22 | return fut 23 | 24 | 25 | @asynccontextmanager 26 | async def pool(max_workers=None, mp_context=CONTEXT, 27 | initializer=None, initargs=(), loop=None, return_exceptions=True): 28 | with ProcessPoolExecutor(max_workers=max_workers, mp_context=mp_context, 29 | initializer=initializer, initargs=initargs) as executor: 30 | pool = AsyncProcessPool(executor, loop=loop) 31 | try: 32 | yield pool 33 | finally: 34 | pool.result = await asyncio.gather(*pool.pending, loop=pool.loop, return_exceptions=return_exceptions) 35 | 36 | 37 | async def main(): 38 | async with pool() as p: 39 | p.submit(print, "This works perfectly fine") 40 | result = await p.submit(sum, (1, 2)) 41 | print(result) 42 | print(p.result) 43 | 44 | if __name__ == '__main__': 45 | freeze_support() 46 | asyncio.run(main()) -------------------------------------------------------------------------------- /chapter_06/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/asyncio-recipes/f078242d4380407e483236695ebc49f3faf09e7f/chapter_06/__init__.py -------------------------------------------------------------------------------- /chapter_06/sol_01.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | from contextlib import asynccontextmanager 4 | 5 | 6 | class Interactor: 7 | def __init__(self, agen): 8 | self.agen = agen 9 | 10 | async def interact(self, *args, **kwargs, ): 11 | try: 12 | await self.agen.asend((args, kwargs)) 13 | except StopAsyncIteration: 14 | logging.exception("The async generator is already exhausted!") 15 | 16 | 17 | async def wrap_in_asyngen(handler): 18 | while True: 19 | args, kwargs = yield 20 | handler(*args, **kwargs) 21 | 22 | 23 | @asynccontextmanager 24 | async def start(agen): 25 | try: 26 | await agen.asend(None) 27 | yield Interactor(agen) 28 | finally: 29 | await agen.aclose() 30 | 31 | 32 | async def main(): 33 | async with start(wrap_in_asyngen(print)) as w: 34 | await w.interact("Put") 35 | await w.interact("the") 36 | await w.interact("worker") 37 | await w.interact("to") 38 | await w.interact("work!") 39 | 40 | 41 | asyncio.run(main()) -------------------------------------------------------------------------------- /chapter_06/sol_02.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | 4 | logging.basicConfig(level=logging.DEBUG) 5 | 6 | 7 | async def producer(iterable, queue: asyncio.Queue, shutdown_event: asyncio.Event): 8 | for i in iterable: 9 | if shutdown_event.is_set(): 10 | break 11 | try: 12 | queue.put_nowait(i) 13 | await asyncio.sleep(0) 14 | 15 | except asyncio.QueueFull as err: 16 | logging.warning("The queue is too full. Maybe the worker are too slow.") 17 | raise err 18 | 19 | shutdown_event.set() 20 | 21 | 22 | async def worker(name, handler, queue: asyncio.Queue, shutdown_event: asyncio.Event): 23 | while not shutdown_event.is_set() or not queue.empty(): 24 | try: 25 | work = queue.get_nowait() 26 | # Simulate work 27 | handler(await asyncio.sleep(1.0, work)) 28 | logging.debug(f"worker {name}: {work}") 29 | 30 | except asyncio.QueueEmpty: 31 | await asyncio.sleep(0) 32 | 33 | 34 | async def main(): 35 | n, handler, iterable = 10, lambda val: None, [i for i in range(500)] 36 | shutdown_event = asyncio.Event() 37 | queue = asyncio.Queue() 38 | worker_coros = [worker(f"worker_{i}", handler, queue, shutdown_event) for i in range(n)] 39 | producer_coro = producer(iterable, queue, shutdown_event) 40 | 41 | coro = asyncio.gather( 42 | producer_coro, 43 | *worker_coros, 44 | return_exceptions=True 45 | ) 46 | 47 | try: 48 | await coro 49 | except KeyboardInterrupt: 50 | shutdown_event.set() 51 | coro.cancel() 52 | 53 | 54 | try: 55 | asyncio.run(main()) 56 | except KeyboardInterrupt: 57 | # It bubbles up 58 | logging.info("Pressed ctrl+c...") -------------------------------------------------------------------------------- /chapter_06/sol_03_unix.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import asyncio 3 | import sys 4 | 5 | parser = argparse.ArgumentParser("streamserver") 6 | 7 | subparsers = parser.add_subparsers(dest="command") 8 | primary = subparsers.add_parser("primary") 9 | secondary = subparsers.add_parser("secondary") 10 | for subparser in (primary, secondary): 11 | subparser.add_argument("--path", default="/tmp/asyncio.socket") 12 | 13 | 14 | async def connection_handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): 15 | print("Handler started") 16 | writer.write(b"Hi there!") 17 | await writer.drain() 18 | message = await reader.read(1024) 19 | print(message) 20 | 21 | 22 | async def start_primary(path): 23 | await asyncio.create_subprocess_exec(sys.executable, __file__, "secondary", "--path", path) 24 | 25 | server = await asyncio.start_unix_server(connection_handler, path) 26 | async with server: 27 | await server.serve_forever() 28 | 29 | 30 | async def start_secondary(path): 31 | reader, writer = await asyncio.open_unix_connection(path) 32 | message = await reader.read(1024) 33 | print(message) 34 | writer.write(b"Hi yourself!") 35 | await writer.drain() 36 | writer.close() 37 | await writer.wait_closed() 38 | 39 | 40 | async def main(): 41 | args = parser.parse_args() 42 | 43 | if args.command == "primary": 44 | await start_primary(args.path) 45 | else: 46 | await start_secondary(args.path) 47 | try: 48 | import logging 49 | logging.basicConfig(level=logging.DEBUG) 50 | logging.debug("Press ctrl+c to stop") 51 | asyncio.run(main()) 52 | except KeyboardInterrupt: 53 | logging.debug("Stopped..") -------------------------------------------------------------------------------- /chapter_06/sol_03_win_unix.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import asyncio 3 | import sys 4 | 5 | parser = argparse.ArgumentParser("streamserver") 6 | 7 | subparsers = parser.add_subparsers(dest="command") 8 | primary = subparsers.add_parser("primary") 9 | secondary = subparsers.add_parser("secondary") 10 | for subparser in (primary, secondary): 11 | subparser.add_argument("--host", default="127.0.0.1") 12 | subparser.add_argument("--port", default=1234) 13 | 14 | 15 | async def connection_handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): 16 | print("Handler started") 17 | writer.write(b"Hi there!") 18 | await writer.drain() 19 | message = await reader.read(1024) 20 | print(message) 21 | 22 | 23 | async def start_primary(host, port): 24 | await asyncio.create_subprocess_exec(sys.executable, __file__, "secondary", "--host", host, "--port", str(port), ) 25 | 26 | server = await asyncio.start_server(connection_handler, host=host, port=port) 27 | async with server: 28 | await server.serve_forever() 29 | 30 | 31 | async def start_secondary(host, port): 32 | reader, writer = await asyncio.open_connection(host, port) 33 | message = await reader.read(1024) 34 | print(message) 35 | writer.write(b"Hi yourself!") 36 | await writer.drain() 37 | writer.close() 38 | await writer.wait_closed() 39 | 40 | 41 | async def main(): 42 | args = parser.parse_args() 43 | 44 | if args.command == "primary": 45 | await start_primary(args.host, args.port) 46 | else: 47 | await start_secondary(args.host, args.port) 48 | 49 | 50 | try: 51 | import logging 52 | 53 | logging.basicConfig(level=logging.DEBUG) 54 | logging.debug("Press ctrl+c to stop") 55 | if sys.platform == 'win32': 56 | asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) 57 | asyncio.run(main()) 58 | except KeyboardInterrupt: 59 | logging.debug("Stopped..") 60 | -------------------------------------------------------------------------------- /chapter_06/sol_04.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import asyncio 3 | import collections 4 | import inspect 5 | import logging 6 | import pickle 7 | import typing 8 | from contextlib import asynccontextmanager 9 | from pickle import PickleError 10 | from uuid import uuid4 11 | 12 | from hbmqtt.client import MQTTClient, ConnectException 13 | from hbmqtt.mqtt.constants import QOS_0 14 | 15 | GET_REMOTE_METHOD = "get_remote_method" 16 | GET_REMOTE_METHOD_RESPONSE = "get_remote_method/response" 17 | 18 | CALL_REMOTE_METHOD = "call_remote_method" 19 | CALL_REMOTE_METHOD_RESPONSE = "call_remote_method/response" 20 | 21 | REGISTER_REMOTE_METHOD = "register_remote_method" 22 | REGISTER_REMOTE_METHOD_RESPONSE = "register_remote_method/response" 23 | 24 | logging.basicConfig(level=logging.INFO) 25 | 26 | 27 | @asynccontextmanager 28 | async def connect(url): 29 | client = MQTTClient() 30 | try: 31 | await client.connect(url) 32 | yield client 33 | except ConnectException: 34 | logging.exception(f"Could not connect to {url}") 35 | finally: 36 | await client.disconnect() 37 | 38 | 39 | @asynccontextmanager 40 | async def pool(n, url): 41 | clients = [MQTTClient() for _ in range(n)] 42 | try: 43 | await asyncio.gather(*[client.connect(url) for client in clients]) 44 | yield clients 45 | except ConnectException: 46 | logging.exception(f"Could not connect to {url}") 47 | finally: 48 | await asyncio.gather(*[client.disconnect() for client in clients]) 49 | 50 | 51 | def set_future_result(fut, result): 52 | if not fut: 53 | pass 54 | if isinstance(result, Exception): 55 | fut.set_exception(result) 56 | else: 57 | fut.set_result(result) 58 | 59 | 60 | class RPCException(Exception): 61 | def __init__(self, message): 62 | self.message = message 63 | 64 | def __str__(self): 65 | return f"Error: {self.message}" 66 | 67 | 68 | class RegisterRemoteMethodException(RPCException): 69 | def __init__(self): 70 | super(RegisterRemoteMethodException, self).__init__(f"Could not respond to {REGISTER_REMOTE_METHOD} query") 71 | 72 | 73 | class GetRemoteMethodException(RPCException): 74 | def __init__(self): 75 | super(GetRemoteMethodException, self).__init__(f"Could not respond to {GET_REMOTE_METHOD} query") 76 | 77 | 78 | class CallRemoteMethodException(RPCException): 79 | def __init__(self): 80 | super(CallRemoteMethodException, self).__init__(f"Could not respond to {CALL_REMOTE_METHOD} query") 81 | 82 | 83 | class RCPBase: 84 | 85 | def __init__(self, client: MQTTClient, topics: typing.List[str], qos=QOS_0): 86 | self.client = client 87 | self.running_fut = None 88 | self.topics = topics 89 | self.qos = qos 90 | 91 | @abc.abstractmethod 92 | async def on_get_remote_method(self, uuid_, service_name, function_name): 93 | raise NotImplementedError("Not implemented on_get_remote_method!") 94 | 95 | @abc.abstractmethod 96 | async def on_register_remote_method(self, uuid_, service_name, function_name, signature): 97 | raise NotImplementedError("Not implemented on_register_remote_method!") 98 | 99 | @abc.abstractmethod 100 | async def on_call_remote_method(self, uuid_, service_name, function_name, args, kwargs): 101 | raise NotImplementedError("Not implemented on_call_remote_method!") 102 | 103 | @abc.abstractmethod 104 | async def on_get_remote_method_response(self, uuid_, service_name, function_name, signature_or_exception): 105 | raise NotImplementedError("Not implemented on_get_remote_method_response!") 106 | 107 | @abc.abstractmethod 108 | async def on_register_remote_method_response(self, uuid_, service_name, function_name, is_registered_or_exception): 109 | raise NotImplementedError("Not implemented on_register_remote_method_response!") 110 | 111 | @abc.abstractmethod 112 | async def on_call_remote_method_response(self, uuid_, service_name, function_name, result_or_exception): 113 | raise NotImplementedError("Not implemented on_call_remote_method_response!") 114 | 115 | async def next_message(self): 116 | message = await self.client.deliver_message() 117 | packet = message.publish_packet 118 | topic_name, payload = packet.variable_header.topic_name, packet.payload.data 119 | return topic_name, payload 120 | 121 | async def loop(self): 122 | while True: 123 | topic, payload = await self.next_message() 124 | try: 125 | yield topic, pickle.loads(payload) 126 | except (PickleError, AttributeError, EOFError, ImportError, IndexError): 127 | logging.exception("Could not deserialize payload: %s for topic: %s", payload, topic) 128 | 129 | async def __aenter__(self): 130 | self.running_fut = asyncio.ensure_future(self.start()) 131 | await self.client.subscribe([ 132 | (topic, self.qos) for topic in self.topics 133 | ]) 134 | return self 135 | 136 | async def __aexit__(self, exc_type, exc_val, exc_tb): 137 | await self.stop() 138 | await self.client.unsubscribe(self.topics) 139 | 140 | async def start(self): 141 | async for topic, payload in self.loop(): 142 | try: 143 | if topic == REGISTER_REMOTE_METHOD: 144 | await self.on_register_remote_method(*payload) 145 | elif topic == GET_REMOTE_METHOD: 146 | await self.on_get_remote_method(*payload) 147 | elif topic == CALL_REMOTE_METHOD: 148 | await self.on_call_remote_method(*payload) 149 | elif topic == REGISTER_REMOTE_METHOD_RESPONSE: 150 | await self.on_register_remote_method_response(*payload) 151 | elif topic == GET_REMOTE_METHOD_RESPONSE: 152 | await self.on_get_remote_method_response(*payload) 153 | elif topic == CALL_REMOTE_METHOD_RESPONSE: 154 | await self.on_call_remote_method_response(*payload) 155 | except TypeError: 156 | logging.exception(f"Could not call handler for topic: %s and payload: %s", topic, payload) 157 | except NotImplementedError: 158 | pass 159 | 160 | async def stop(self): 161 | if self.running_fut: 162 | self.running_fut.cancel() 163 | 164 | async def wait(self): 165 | if self.running_fut: 166 | await asyncio.shield(self.running_fut) 167 | 168 | 169 | class RemoteMethod: 170 | 171 | def __init__(self, rpc_client, signature, function_name, qos=QOS_0): 172 | self.rpc_client = rpc_client 173 | self.signature = signature 174 | self.function_name = function_name 175 | self.qos = qos 176 | 177 | async def __call__(self, *args, **kwargs, ): 178 | uuid_ = str(uuid4()) 179 | service_name = self.rpc_client.service_name 180 | payload = (uuid_, service_name, self.function_name, args, kwargs) 181 | fut = asyncio.Future() 182 | self.rpc_client.call_remote_method_requests.setdefault(service_name, {}).setdefault(self.function_name, {})[ 183 | uuid_] = fut 184 | await self.rpc_client.client.publish(CALL_REMOTE_METHOD, pickle.dumps(payload), qos=self.qos) 185 | return await fut 186 | 187 | 188 | class RPCClient(RCPBase): 189 | def __init__(self, client, service_name, topics=None, qos=QOS_0): 190 | if not topics: 191 | topics = [CALL_REMOTE_METHOD_RESPONSE, GET_REMOTE_METHOD_RESPONSE, ] 192 | super(RPCClient, self).__init__(client, topics, qos=qos) 193 | self.call_remote_method_requests = collections.defaultdict(dict) 194 | self.get_remote_method_requests = collections.defaultdict(dict) 195 | self.list_remote_methods_requests = collections.defaultdict(dict) 196 | self.responses = collections.defaultdict(dict) 197 | self.service_name = service_name 198 | self.remote_methods_cache = collections.defaultdict(dict) 199 | 200 | def __getattr__(self, item): 201 | return asyncio.ensure_future(self.get_remote_method(item)) 202 | 203 | async def get_remote_method(self, function_name): 204 | while True: 205 | uuid_ = str(uuid4()) 206 | payload = (uuid_, self.service_name, function_name) 207 | fut = asyncio.Future() 208 | self.get_remote_method_requests.setdefault(self.service_name, {}).setdefault(function_name, {})[uuid_] = fut 209 | await self.client.publish(GET_REMOTE_METHOD, pickle.dumps(payload), qos=QOS_0) 210 | # Might throw GetRemoteMethodException 211 | try: 212 | signature = await asyncio.shield(fut) 213 | return RemoteMethod(self, signature, function_name) 214 | except GetRemoteMethodException: 215 | await asyncio.sleep(0) 216 | 217 | async def on_call_remote_method_response(self, uuid_, service_name, function_name, result_or_exception): 218 | fut = self.call_remote_method_requests.get(service_name, {}).get(function_name, {}).pop(uuid_, None) 219 | set_future_result(fut, result_or_exception) 220 | 221 | async def on_get_remote_method_response(self, uuid_, service_name, function_name, signature_or_exception): 222 | fut = self.get_remote_method_requests.get(service_name, {}).get(function_name, {}).pop(uuid_, None) 223 | set_future_result(fut, signature_or_exception) 224 | 225 | 226 | class RPCService(RCPBase): 227 | def __init__(self, client: MQTTClient, name: str, topics: typing.List[str] = None, qos=QOS_0): 228 | if not topics: 229 | topics = [REGISTER_REMOTE_METHOD_RESPONSE, CALL_REMOTE_METHOD] 230 | super(RPCService, self).__init__(client, topics, qos=qos) 231 | self.name = name 232 | self.client = client 233 | self.qos = qos 234 | self.register_remote_method_requests = collections.defaultdict(dict) 235 | self.remote_methods = collections.defaultdict(dict) 236 | 237 | async def register_function(self, remote_function): 238 | function_name = remote_function.__name__ 239 | uuid_ = str(uuid4()) 240 | payload = pickle.dumps((uuid_, self.name, function_name, inspect.signature(remote_function))) 241 | fut = asyncio.Future() 242 | self.register_remote_method_requests.setdefault(self.name, {}).setdefault(function_name, {})[uuid_] = fut 243 | self.remote_methods[self.name][function_name] = remote_function 244 | await self.client.publish(REGISTER_REMOTE_METHOD, payload, qos=self.qos) 245 | return await asyncio.shield(fut) 246 | 247 | async def on_register_remote_method_response(self, uuid_, service_name, function_name, is_registered_or_exception): 248 | fut = self.register_remote_method_requests.get(service_name, {}).get(function_name, {}).get(uuid_, None) 249 | set_future_result(fut, is_registered_or_exception) 250 | 251 | async def on_call_remote_method(self, uuid_, service_name, function_name, args, kwargs): 252 | remote_method = self.remote_methods.get(service_name, {}).get(function_name, None) 253 | if not remote_method: 254 | payload = pickle.dumps((uuid_, service_name, function_name, CallRemoteMethodException())) 255 | return await self.client.publish(CALL_REMOTE_METHOD_RESPONSE, payload, qos=self.qos) 256 | try: 257 | result = await remote_method(*args, **kwargs) 258 | payload = pickle.dumps((uuid_, service_name, function_name, result)) 259 | return await self.client.publish(CALL_REMOTE_METHOD_RESPONSE, payload, qos=self.qos) 260 | except Exception as err: 261 | payload = pickle.dumps((uuid_, service_name, function_name, err)) 262 | return await self.client.publish(CALL_REMOTE_METHOD_RESPONSE, payload, qos=self.qos) 263 | 264 | 265 | class RemoteRegistrar(RCPBase): 266 | def __init__(self, client: MQTTClient, topics: typing.List[str] = None, qos=QOS_0): 267 | if not topics: 268 | topics = [REGISTER_REMOTE_METHOD, GET_REMOTE_METHOD] 269 | super(RemoteRegistrar, self).__init__(client, topics, qos=qos) 270 | self.registrar = collections.defaultdict(dict) 271 | 272 | async def on_register_remote_method(self, uuid_, service_name, function_name, signature): 273 | try: 274 | self.registrar.setdefault(service_name, {})[function_name] = signature 275 | payload = pickle.dumps((uuid_, service_name, function_name, True), ) 276 | await self.client.publish(REGISTER_REMOTE_METHOD_RESPONSE, payload) 277 | 278 | except Exception: 279 | # A broad exception clause like this is bad practice but we are only interested in the outcome 280 | # of saving the signature, so we convert it 281 | logging.exception(f"Failed to save signature: {signature}") 282 | payload = pickle.dumps((uuid_, service_name, function_name, RegisterRemoteMethodException())) 283 | await self.client.publish(REGISTER_REMOTE_METHOD_RESPONSE, payload, ) 284 | 285 | async def on_get_remote_method(self, uuid_, service_name, function_name): 286 | signature = self.registrar.get(service_name, {}).get(function_name, None) 287 | 288 | if signature: 289 | payload = pickle.dumps((uuid_, service_name, function_name, signature), ) 290 | await self.client.publish(GET_REMOTE_METHOD_RESPONSE, payload) 291 | else: 292 | payload = pickle.dumps((uuid_, service_name, function_name, GetRemoteMethodException()), ) 293 | await self.client.publish(GET_REMOTE_METHOD_RESPONSE, payload) 294 | 295 | 296 | async def remote_function(i: int, f: float, s: str): 297 | print("It worked") 298 | return f 299 | 300 | 301 | async def register_with_delay(rpc_service, remote_function, delay=3): 302 | await asyncio.sleep(delay) 303 | await rpc_service.register_function(remote_function) 304 | 305 | 306 | async def main(url="mqtt://localhost", service_name="TestService"): 307 | async with pool(3, url) as (client, client1, client2): 308 | async with RemoteRegistrar(client): 309 | async with RPCService(client1, service_name) as rpc_service: 310 | async with RPCClient(client2, service_name) as rpc_client: 311 | asyncio.ensure_future(register_with_delay(rpc_service, remote_function)) 312 | handler = await asyncio.wait_for(rpc_client.remote_function,timeout=10) 313 | res = await handler(1, 3.4, "") 314 | print(res) 315 | 316 | 317 | if __name__ == '__main__': 318 | asyncio.run(main()) -------------------------------------------------------------------------------- /chapter_06/sol_05_option_1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import contextvars 3 | from contextvars import ContextVar 4 | 5 | context = contextvars.copy_context() 6 | context_var = ContextVar('key', default=None) 7 | 8 | 9 | async def memory(context_var, value): 10 | old_value = context_var.get() 11 | context_var.set(value) 12 | print(old_value, value) 13 | 14 | 15 | async def main(): 16 | await asyncio.gather(*[memory(context_var, i) for i in range(10)]) 17 | 18 | 19 | asyncio.run(main()) 20 | -------------------------------------------------------------------------------- /chapter_06/sol_05_option_2.py: -------------------------------------------------------------------------------- 1 | import contextvars 2 | import functools 3 | from contextvars import ContextVar 4 | 5 | context = contextvars.copy_context() 6 | context_var = ContextVar('key', default=None) 7 | 8 | 9 | def resetter(context_var, token, invalid_values): 10 | value = context_var.get() 11 | if value in invalid_values: 12 | context_var.reset(token) 13 | 14 | 15 | def blacklist(context_var, value, resetter): 16 | old_value = context_var.get() 17 | token = context_var.set(value) 18 | resetter(context_var, token) 19 | print(old_value) 20 | 21 | 22 | for i in range(10): 23 | context.run(blacklist, context_var, i, functools.partial(resetter, invalid_values=[5, 6, 7, 8, 9])) 24 | -------------------------------------------------------------------------------- /chapter_07/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/asyncio-recipes/f078242d4380407e483236695ebc49f3faf09e7f/chapter_07/__init__.py -------------------------------------------------------------------------------- /chapter_07/sol_01.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | NON_ATOMIC_SUM_KEY = 'non_atomic_sum' 4 | ATOMIC_SUM_KEY = 'atomic_sum' 5 | DATABASE = {ATOMIC_SUM_KEY: 0, NON_ATOMIC_SUM_KEY: 0} 6 | 7 | 8 | async def add_with_delay(key, value, delay): 9 | old_value = DATABASE[key] 10 | await asyncio.sleep(delay) 11 | DATABASE[key] = old_value + value 12 | 13 | 14 | async def add_locked_with_delay(lock, key, value, delay): 15 | async with lock: 16 | old_value = DATABASE[key] 17 | await asyncio.sleep(delay) 18 | DATABASE[key] = old_value + value 19 | 20 | 21 | async def main(): 22 | # An asyncio lock can be used to guarantee exclusive access to a shared resource 23 | lock = asyncio.Lock() 24 | atomic_workers = [ 25 | add_locked_with_delay(lock, ATOMIC_SUM_KEY, 1, 3), 26 | add_locked_with_delay(lock, ATOMIC_SUM_KEY, 1, 2), 27 | ] 28 | non_atomic_workers = [ 29 | add_with_delay(NON_ATOMIC_SUM_KEY, 1, 3), 30 | add_with_delay(NON_ATOMIC_SUM_KEY, 1, 2), 31 | ] 32 | 33 | await asyncio.gather(*non_atomic_workers) 34 | await asyncio.gather(*atomic_workers) 35 | 36 | assert DATABASE.get(ATOMIC_SUM_KEY) == 2 37 | assert DATABASE.get(NON_ATOMIC_SUM_KEY) != 2 38 | 39 | if __name__ == '__main__': 40 | asyncio.run(main()) 41 | -------------------------------------------------------------------------------- /chapter_07/sol_02.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | import random 4 | 5 | logging.basicConfig(level=logging.INFO) 6 | 7 | 8 | async def busy_loop(interval, work, worker, shutdown_event): 9 | while not shutdown_event.is_set(): 10 | await worker(work) 11 | await asyncio.sleep(interval) 12 | logging.info("Shutdown event was set..") 13 | return work 14 | 15 | 16 | async def cleanup(mess, shutdown_event): 17 | await shutdown_event.wait() 18 | logging.info("Cleaning up the mess: %s...", mess) 19 | # Add cleanup logic here 20 | 21 | 22 | async def shutdown(delay, shutdown_event): 23 | await asyncio.sleep(delay) 24 | shutdown_event.set() 25 | 26 | 27 | async def add_mess(mess_pile, ): 28 | mess = random.randint(1, 10) 29 | logging.info("Adding the mess: %s...", mess) 30 | mess_pile.append(mess) 31 | 32 | 33 | async def main(): 34 | shutdown_event = asyncio.Event() 35 | shutdown_delay = 10 36 | work = [] 37 | await asyncio.gather(*[ 38 | shutdown(shutdown_delay, shutdown_event), 39 | cleanup(work, shutdown_event), 40 | busy_loop(1, work, add_mess, shutdown_event), 41 | ]) 42 | 43 | 44 | if __name__ == '__main__': 45 | asyncio.run(main()) 46 | -------------------------------------------------------------------------------- /chapter_07/sol_03.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import random 3 | 4 | STOCK_MARKET = { 5 | "DAX": 100, 6 | "SPR": 10, 7 | "AMAZON": 1000, 8 | } 9 | 10 | INITIAL_STOCK_MARKET = STOCK_MARKET.copy() 11 | 12 | 13 | class MarketException(BaseException): 14 | pass 15 | 16 | 17 | async def stock_watcher(on_alert, stock, price, cond): 18 | async with cond: 19 | print(f"Waiting for {stock} to be under {price}$") 20 | await cond.wait_for(lambda: STOCK_MARKET.get(stock) < price) 21 | await on_alert() 22 | 23 | 24 | def random_stock(): 25 | while True: 26 | yield random.choice(list(STOCK_MARKET.keys())) 27 | 28 | 29 | async def twitter_quotes(conds, threshold): 30 | for stock in random_stock(): 31 | STOCK_MARKET[stock] -= random.randint(1, 10) 32 | new_value = STOCK_MARKET[stock] 33 | print(f"New stock market value for {stock}: {new_value}") 34 | if new_value < threshold: 35 | cond = conds.get(stock) 36 | async with cond: 37 | cond.notify() 38 | await asyncio.sleep(.1) 39 | 40 | 41 | async def governmental_market_surveillance(): 42 | raise MarketException() 43 | 44 | 45 | async def main(): 46 | lock = asyncio.Lock() 47 | conditions = {stock: asyncio.Condition(lock) for stock in STOCK_MARKET} 48 | threshold = -50 49 | stock_watchers = [ 50 | stock_watcher( 51 | governmental_market_surveillance, 52 | stock, 53 | threshold, 54 | conditions.get(stock) 55 | ) for stock in STOCK_MARKET 56 | ] 57 | await asyncio.gather(*[twitter_quotes(conditions, threshold), *stock_watchers], return_exceptions=False) 58 | 59 | 60 | if __name__ == '__main__': 61 | try: 62 | asyncio.run(main()) 63 | except MarketException: 64 | print("Restoring the stock market..") 65 | STOCK_MARKET = INITIAL_STOCK_MARKET.copy() 66 | -------------------------------------------------------------------------------- /chapter_07/sol_04.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def run(i, semaphore): 5 | async with semaphore: 6 | print(f"{i} working..") 7 | return await asyncio.sleep(1) 8 | 9 | 10 | async def main(): 11 | semaphore = asyncio.Semaphore(10) 12 | await asyncio.gather(*[run(i, semaphore) for i in range(100)]) 13 | 14 | if __name__ == '__main__': 15 | asyncio.run(main()) 16 | -------------------------------------------------------------------------------- /chapter_07/sol_05.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def run(i, semaphore): 5 | async with semaphore: 6 | print(f"{i} working..") 7 | return await asyncio.sleep(1) 8 | 9 | 10 | async def main(): 11 | semaphore = asyncio.BoundedSemaphore(10) 12 | await asyncio.gather(*[run(i, semaphore) for i in range(100)]) 13 | 14 | 15 | if __name__ == '__main__': 16 | asyncio.run(main(), debug=True) 17 | -------------------------------------------------------------------------------- /chapter_07/sol_06_option_1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import typing 3 | 4 | 5 | async def delayed_add(delay, d: typing.Dict, key: typing.Hashable, value: float): 6 | last = d[key] # This is where the critical path starts, d is the shared resource and this is a read access 7 | await asyncio.sleep(delay) 8 | d[key] = last + value # This is where the critical path ends, d is the shared resource and this is a write access 9 | 10 | 11 | async def main(): 12 | d = {"value": 0} 13 | await asyncio.gather(delayed_add(2, d, "value", 1.0), delayed_add(1.0, d, "value", 1.0)) 14 | print(d) 15 | assert d["value"] != 2 16 | 17 | if __name__ == '__main__': 18 | asyncio.run(main()) 19 | -------------------------------------------------------------------------------- /chapter_07/sol_06_option_2.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import threading 3 | import time 4 | from concurrent.futures.thread import ThreadPoolExecutor 5 | 6 | 7 | def add_from_thread(delay, d, key, value): 8 | print(f"Thread {threading.get_ident()} started...") 9 | old = d[key] 10 | print(f"Thread {threading.get_ident()} read value {old}") 11 | time.sleep(delay) 12 | print(f"Thread {threading.get_ident()} slept for {delay} seconds") 13 | d[key] = old + value 14 | print(f"Thread {threading.get_ident()} wrote {d[key]}") 15 | 16 | 17 | async def main(): 18 | loop = asyncio.get_running_loop() 19 | d = {"value": 0} 20 | executor = ThreadPoolExecutor(10) 21 | futs = [loop.run_in_executor(executor, add_from_thread, 1, d, "value", 1), 22 | loop.run_in_executor(executor, add_from_thread, 3, d, "value", 1)] 23 | await asyncio.gather(*futs) 24 | 25 | assert d["value"] != 2 26 | 27 | if __name__ == '__main__': 28 | asyncio.run(main()) 29 | -------------------------------------------------------------------------------- /chapter_08/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/asyncio-recipes/f078242d4380407e483236695ebc49f3faf09e7f/chapter_08/__init__.py -------------------------------------------------------------------------------- /chapter_08/sol_01.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | import tracemalloc 4 | from functools import wraps 5 | 6 | logging.basicConfig(level=logging.DEBUG) 7 | 8 | 9 | class Profiler: 10 | def __init__(self): 11 | self.stats = {} 12 | self.logger = logging.getLogger(__name__) 13 | 14 | def profile_memory_usage(self, f): 15 | @wraps(f) 16 | async def wrapper(*args, **kwargs): 17 | snapshot = tracemalloc.take_snapshot() 18 | res = await f(*args, **kwargs) 19 | self.stats[f.__name__] = tracemalloc.take_snapshot().compare_to(snapshot, 'lineno') 20 | return res 21 | 22 | return wrapper 23 | 24 | def print(self): 25 | for name, stats in self.stats.items(): 26 | for entry in stats: 27 | self.logger.debug(entry) 28 | 29 | def __enter__(self): 30 | tracemalloc.start() 31 | return self 32 | 33 | def __exit__(self, exc_type, exc_val, exc_tb): 34 | tracemalloc.stop() 35 | self.print() 36 | 37 | 38 | profiler = Profiler() 39 | 40 | 41 | @profiler.profile_memory_usage 42 | async def main(): 43 | pass 44 | 45 | 46 | with profiler: 47 | asyncio.run(main()) 48 | -------------------------------------------------------------------------------- /chapter_08/sol_02.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import contextlib 3 | import inspect 4 | import json 5 | import logging 6 | import pickle 7 | import sys 8 | import tracemalloc 9 | from collections import defaultdict, namedtuple 10 | from contextlib import asynccontextmanager 11 | from functools import wraps 12 | from inspect import iscoroutinefunction 13 | from time import time 14 | from tracemalloc import Filter, take_snapshot, start as tracemalloc_start, \ 15 | stop as tracemalloc_stop 16 | 17 | logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) 18 | 19 | Timing = namedtuple("Timing", ["start", "end", "delta"]) 20 | 21 | 22 | class Profiler: 23 | 24 | def __init__(self, key_type="lineno", cumulative=False, debug=False, excluded_files=None): 25 | self.time_stats = defaultdict(list) 26 | self.memory_stats = defaultdict(dict) 27 | self.key_type = key_type 28 | self.cumulative = cumulative 29 | self.logger = logging.getLogger(__name__) 30 | self.debug = debug 31 | if not excluded_files: 32 | excluded_files = [tracemalloc.__file__, inspect.__file__, contextlib.__file__] 33 | self.excluded_files = excluded_files 34 | self.profile_memory_cache = False 35 | 36 | def time(self): 37 | try: 38 | return asyncio.get_running_loop().time() 39 | except RuntimeError: 40 | return time() 41 | 42 | def get_filter(self, include=False): 43 | return (Filter(include, filter_) for filter_ in self.excluded_files) 44 | 45 | def profile_memory(self, f): 46 | self.profile_memory_cache = True 47 | if iscoroutinefunction(f): 48 | @wraps(f) 49 | async def wrapper(*args, **kwargs): 50 | snapshot = take_snapshot().filter_traces(self.get_filter()) 51 | result = await f(*args, **kwargs) 52 | current_time = time() 53 | memory_delta = take_snapshot().filter_traces(self.get_filter()).compare_to(snapshot, self.key_type, 54 | self.cumulative) 55 | self.memory_stats[f.__name__][current_time] = memory_delta 56 | return result 57 | else: 58 | @wraps(f) 59 | def wrapper(*args, **kwargs): 60 | snapshot = take_snapshot().filter_traces(self.get_filter()) 61 | result = f(*args, **kwargs) 62 | current_time = time() 63 | memory_delta = take_snapshot().filter_traces(self.get_filter()).compare_to(snapshot, self.key_type, 64 | self.cumulative) 65 | self.memory_stats[f.__name__][current_time] = memory_delta 66 | return result 67 | return wrapper 68 | 69 | def profile_time(self, f): 70 | 71 | if iscoroutinefunction(f): 72 | @wraps(f) 73 | async def wrapper(*args, **kwargs): 74 | start = self.time() 75 | result = await f(*args, **kwargs) 76 | end = self.time() 77 | delta = end - start 78 | self.time_stats[f.__name__].append(Timing(start, end, delta)) 79 | return result 80 | else: 81 | @wraps(f) 82 | def wrapper(*args, **kwargs): 83 | start = self.time() 84 | result = f(*args, **kwargs) 85 | end = self.time() 86 | delta = end - start 87 | self.time_stats[f.__name__].append(Timing(start, end, delta)) 88 | return result 89 | return wrapper 90 | 91 | def __enter__(self): 92 | if self.profile_memory_cache: 93 | self.logger.debug("Starting tracemalloc..") 94 | tracemalloc_start() 95 | 96 | return self 97 | 98 | def __exit__(self, exc_type, exc_val, exc_tb): 99 | if self.profile_memory_cache: 100 | self.logger.debug("Stopping tracemalloc..") 101 | tracemalloc_stop() 102 | if self.debug: 103 | self.print_memory_stats() 104 | self.print_time_stats() 105 | 106 | def print_memory_stats(self): 107 | for name, stats in self.memory_stats.items(): 108 | for timestamp, entry in list(stats.items()): 109 | self.logger.debug("Memory measurements for call of %s at %s", name, timestamp) 110 | 111 | for stats_diff in entry: 112 | self.logger.debug("%s", stats_diff) 113 | 114 | def print_time_stats(self): 115 | for function_name, timings in self.time_stats.items(): 116 | for timing in timings: 117 | self.logger.debug("function %s was called at %s ms and took: %s ms", 118 | function_name, 119 | timing.start, 120 | timing.delta) 121 | 122 | 123 | async def read_message(reader, timeout=3): 124 | data = [] 125 | while True: 126 | try: 127 | chunk = await asyncio.wait_for(reader.read(1024), timeout=timeout) 128 | data += [chunk] 129 | except asyncio.TimeoutError: 130 | return b"".join(data) 131 | 132 | 133 | class ProfilerServer: 134 | def __init__(self, profiler, host, port): 135 | self.profiler = profiler 136 | self.host = host 137 | self.port = port 138 | 139 | async def on_connection(self, 140 | reader: asyncio.StreamReader, 141 | writer: asyncio.StreamWriter): 142 | message = await read_message(reader) 143 | logging.debug("Message %s:", message) 144 | 145 | try: 146 | event = json.loads(message, encoding="UTF-8") 147 | command = event["command"] 148 | if command not in ["memory_stats", "time_stats"]: 149 | raise ValueError(f"{command} is illegal!") 150 | 151 | handler = getattr(self.profiler, command, None) 152 | if not handler: 153 | raise ValueError(f"{message} is malformed") 154 | reply_message = handler 155 | 156 | writer.write(pickle.dumps(reply_message)) 157 | await writer.drain() 158 | except (UnicodeDecodeError, json.JSONDecodeError, TypeError,)as err: 159 | self.profiler.logger.error("Error occurred while transmission: %s", err) 160 | writer.write(pickle.dumps(err)) 161 | await writer.drain() 162 | finally: 163 | writer.close() 164 | 165 | 166 | class ProfilerClient: 167 | def __init__(self, host, port): 168 | self.host = host 169 | self.port = port 170 | 171 | async def send(self, **kwargs): 172 | message = json.dumps(kwargs) 173 | reader, writer = await asyncio.open_connection(self.host, self.port) 174 | writer.write(message.encode()) 175 | message = await reader.read() 176 | writer.close() 177 | try: 178 | return pickle.loads(message) 179 | except pickle.PickleError: 180 | return None 181 | 182 | async def get_memory_stats(self): 183 | return await self.send(command="memory_stats") 184 | 185 | async def get_time_stats(self): 186 | return await self.send(command="time_stats") 187 | 188 | 189 | @asynccontextmanager 190 | async def start_profiler_server(profiler, host, port): 191 | profiler_server = ProfilerServer(profiler, host, port) 192 | try: 193 | server = await asyncio.start_server(profiler_server.on_connection, host, port) 194 | async with server: 195 | yield 196 | await server.serve_forever() 197 | finally: 198 | pass 199 | 200 | 201 | profiler = Profiler(debug=True) 202 | 203 | 204 | @profiler.profile_time 205 | @profiler.profile_memory 206 | async def to_be_profiled(): 207 | await asyncio.sleep(3) 208 | list(i for i in range(10000)) 209 | 210 | 211 | async def main(profiler): 212 | host, port = "127.0.0.1", 1234 213 | client = ProfilerClient(host, port) 214 | async with start_profiler_server(profiler, host, port): 215 | await to_be_profiled() 216 | memory_stats = await client.get_memory_stats() 217 | logging.debug(memory_stats) 218 | 219 | 220 | if __name__ == '__main__': 221 | try: 222 | logging.debug("Press CTRL+C to close...") 223 | with profiler: 224 | asyncio.run(main(profiler)) 225 | except KeyboardInterrupt: 226 | logging.debug("Closed..") 227 | -------------------------------------------------------------------------------- /chapter_08/sol_03.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | import sys 4 | from functools import wraps 5 | 6 | THRESHOLD = 0.5 7 | sys.set_coroutine_origin_tracking_depth(10) 8 | 9 | 10 | def time_it_factory(handler): 11 | def time_it(f): 12 | @wraps(f) 13 | async def wrapper(*args, **kwargs): 14 | loop = asyncio.get_running_loop() 15 | start = loop.time() 16 | coro = f(*args, **kwargs) 17 | result = await coro 18 | delta = loop.time() - start 19 | handler(coro, delta) 20 | return result 21 | 22 | return wrapper 23 | 24 | return time_it 25 | 26 | 27 | @time_it_factory 28 | def log_time(coro, time_delta): 29 | if time_delta > THRESHOLD: 30 | logging.warning("The coroutine %s took more than %s ms", coro, time_delta) 31 | for frame in coro.cr_origin: 32 | logging.warning("file:%s line:%s function:%s", *frame) 33 | else: 34 | logging.warning("Coroutine has no origin !") 35 | 36 | 37 | @log_time 38 | async def main(): 39 | await asyncio.sleep(1) 40 | 41 | 42 | asyncio.run(main()) 43 | -------------------------------------------------------------------------------- /chapter_08/sol_04_option_1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import sys 3 | 4 | 5 | async def coro(): 6 | print("This works!") 7 | 8 | 9 | async def ensure_future_deprecated(): 10 | # Up to Python 3.6 11 | task = asyncio.ensure_future(coro()) 12 | 13 | # In Python 3.7+ 14 | task_2 = asyncio.create_task(coro()) 15 | 16 | 17 | async def main(): 18 | pass 19 | 20 | 21 | # Up to Python 3.6 22 | asyncio.get_event_loop().run_until_complete(main()) 23 | 24 | # Python 3.7+ 25 | asyncio.run(main()) 26 | 27 | 28 | async def wait_deprecated(): 29 | # Passing coroutines objects to wait() directly is deprecated: 30 | 31 | coros = [asyncio.sleep(10), asyncio.sleep(10)] 32 | done, pending = await asyncio.wait(coros) 33 | 34 | # Use asyncio.create_task 35 | 36 | futures = [asyncio.create_task(coro) for coro in (asyncio.sleep(10), asyncio.sleep(10))] 37 | done, pending = await asyncio.wait(futures) 38 | 39 | 40 | async def tasks_deprecated(loop): 41 | # Using Task class methods is deprecated: 42 | task = asyncio.Task.current_task(loop) 43 | tasks = asyncio.Task.all_tasks(loop) 44 | 45 | # Use the asyncio module level functions instead: 46 | task = asyncio.current_task(loop) 47 | tasks = asyncio.all_tasks(loop) 48 | 49 | 50 | async def coroutine_deprecated(): 51 | @asyncio.coroutine 52 | def gen_coro(): 53 | yield from asyncio.sleep(1) 54 | 55 | async def native_coroutine(): 56 | await asyncio.sleep(1) 57 | 58 | 59 | async def passing_loop_deprecated(): 60 | loop = asyncio.get_running_loop() 61 | # This is deprecated 62 | await asyncio.sleep(10, loop=loop) 63 | await asyncio.wait_for(asyncio.create_task(asyncio.sleep(10)), 11, loop=loop) 64 | futures = {asyncio.create_task(asyncio.sleep(10, loop=loop))} 65 | done, pending = await asyncio.wait(futures, loop=loop) 66 | 67 | await asyncio.sleep(10) 68 | await asyncio.wait_for(asyncio.create_task(asyncio.sleep(10)), 11, loop=loop) 69 | futures = {asyncio.create_task(asyncio.sleep(10))} 70 | done, pending = await asyncio.wait(futures) 71 | 72 | 73 | async def coroutine_wrapper_deprecated(): 74 | # set_coroutine_wrapper() and sys.get_coroutine_wrapper() will be removed in Python 3.8 75 | sys.set_coroutine_wrapper(sys.get_coroutine_wrapper()) 76 | # and are deprecated in favor of 77 | sys.set_coroutine_origin_tracking_depth(sys.get_coroutine_origin_tracking_depth()) 78 | # Of course passing sensible values! 79 | -------------------------------------------------------------------------------- /chapter_08/sol_04_option_2.py: -------------------------------------------------------------------------------- 1 | ## How to refactor "oldschool" asyncio code 2 | import argparse 3 | import ast 4 | import asyncio 5 | import functools 6 | import os 7 | from asyncio import coroutine 8 | 9 | parser = argparse.ArgumentParser("asyncompat") 10 | parser.add_argument("--path", default=__file__) 11 | 12 | 13 | ### TEST SECTION ### 14 | 15 | @coroutine 16 | def producer(): 17 | return 123 18 | 19 | 20 | @asyncio.coroutine 21 | def consumer(): 22 | value = yield from producer() 23 | return value 24 | 25 | 26 | def consumer2(): 27 | value = yield from producer() 28 | return value 29 | 30 | 31 | ### TEST SECTION END ### 32 | 33 | def is_coroutine_decorator(node): 34 | return (isinstance(node, ast.Attribute) and 35 | isinstance(node.value, ast.Name) and 36 | hasattr(node.value, "id") and 37 | node.value.id == "asyncio" and node.attr == "coroutine") 38 | 39 | 40 | def is_coroutine_decorator_from_module(node, *, imported_asyncio): 41 | return (isinstance(node, ast.Name) and 42 | node.id == "coroutine" and 43 | isinstance(node.ctx, ast.Load) and 44 | imported_asyncio) 45 | 46 | 47 | class FunctionDefVisitor(ast.NodeVisitor): 48 | def __init__(self): 49 | self.source = None 50 | self.first_run = True 51 | self.imported_asyncio = False 52 | 53 | def initiate_visit(self, source): 54 | self.source = source.splitlines() 55 | node = ast.parse(source) 56 | self.visit(node) 57 | self.first_run = False 58 | return self.visit(node) 59 | 60 | def visit_Import(self, node): 61 | for name in node.names: 62 | if name.name == "asyncio": 63 | self.imported_asyncio = True 64 | 65 | def visit_FunctionDef(self, node): 66 | if self.first_run: 67 | return 68 | 69 | decorators = list(filter(is_coroutine_decorator, node.decorator_list)) 70 | decorators_from_module = list( 71 | filter(functools.partial(is_coroutine_decorator_from_module, imported_asyncio=self.imported_asyncio), 72 | node.decorator_list)) 73 | if decorators: 74 | print(node.lineno, ":", self.source[node.lineno], "is an oldschool coroutine!") 75 | 76 | elif decorators_from_module: 77 | print(node.lineno, ":", self.source[node.lineno], "is an oldschool coroutine!") 78 | 79 | 80 | if __name__ == '__main__': 81 | v = FunctionDefVisitor() 82 | args = parser.parse_args() 83 | path = os.path.isfile(args.path) and os.path.abspath(args.path) 84 | if not path or not path.endswith(".py"): 85 | raise ValueError(f"{path} is not a valid path to a python file!") 86 | with open(path) as f: 87 | v.initiate_visit(f.read()) 88 | -------------------------------------------------------------------------------- /chapter_08/sol_06.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import random 3 | 4 | 5 | async def fetch(url, *, fut: asyncio.Future): 6 | await asyncio.sleep(random.randint(3, 5)) # Simulating work 7 | fut.set_result(random.getrandbits(1024 * 8)) 8 | 9 | 10 | async def checker(responses, url, *, fut: asyncio.Future): 11 | result = await fut 12 | responses[url] = result 13 | print(result) 14 | 15 | 16 | async def main(): 17 | loop = asyncio.get_running_loop() 18 | future = loop.create_future() 19 | responses = {} 20 | url = "https://apress.com" 21 | await asyncio.gather(fetch(url, fut=future), checker(responses, url, fut=future)) 22 | 23 | 24 | asyncio.run(main()) 25 | -------------------------------------------------------------------------------- /chapter_09/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/asyncio-recipes/f078242d4380407e483236695ebc49f3faf09e7f/chapter_09/__init__.py -------------------------------------------------------------------------------- /chapter_09/sol_01.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import functools 3 | import inspect 4 | import logging 5 | import sys 6 | from multiprocessing import freeze_support, get_context 7 | 8 | import cloudpickle as pickle 9 | 10 | logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) 11 | 12 | 13 | def on_error(exc, *, transport, peername): 14 | try: 15 | logging.exception("On error: Exception while handling a subprocess: %s ", exc) 16 | transport.write(pickle.dumps(exc)) 17 | 18 | finally: 19 | transport.close() 20 | logging.info("Disconnected %s", peername) 21 | 22 | 23 | def on_success(result, *, transport, peername, data): 24 | try: 25 | logging.debug("On success: Received payload from %s:%s and successfully executed:\n%s", *peername, data) 26 | transport.write(pickle.dumps(result)) 27 | finally: 28 | transport.close() 29 | logging.info("Disconnected %s", peername) 30 | 31 | 32 | def handle(data): 33 | f, args, kwargs = pickle.loads(data) 34 | if inspect.iscoroutinefunction(f): 35 | return asyncio.run(f(*args, *kwargs)) 36 | 37 | return f(*args, **kwargs) 38 | 39 | 40 | class CommandProtocol(asyncio.Protocol): 41 | 42 | def __init__(self, pool, loop, timeout=30): 43 | self.pool = pool 44 | self.loop = loop 45 | self.timeout = timeout 46 | self.transport = None 47 | 48 | def connection_made(self, transport): 49 | peername = transport.get_extra_info('peername') 50 | logging.info('%s connected', peername) 51 | self.transport = transport 52 | 53 | def data_received(self, data): 54 | peername = self.transport.get_extra_info('peername') 55 | on_error_ = functools.partial(on_error, transport=self.transport, peername=peername) 56 | on_success_ = functools.partial(on_success, transport=self.transport, peername=peername, data=data) 57 | result = self.pool.apply_async(handle, (data,), callback=on_success_, error_callback=on_error_) 58 | self.loop.call_soon(result.wait) 59 | self.loop.call_later(self.timeout, self.close, peername) 60 | 61 | def close(self, peername=None): 62 | try: 63 | if self.transport.is_closing(): 64 | return 65 | if not peername: 66 | peername = self.transport.get_extra_info('peername') 67 | finally: 68 | self.transport.close() 69 | logging.info("Disconnecting %s", peername) 70 | 71 | 72 | async def main(): 73 | loop = asyncio.get_running_loop() 74 | fork_context = get_context("fork") 75 | pool = fork_context.Pool() 76 | server = await loop.create_server(lambda: CommandProtocol(pool, loop), '127.0.0.1', 8888) 77 | try: 78 | async with server: 79 | await server.serve_forever() 80 | finally: 81 | pool.close() 82 | pool.join() 83 | 84 | 85 | if __name__ == '__main__': 86 | freeze_support() 87 | asyncio.run(main()) 88 | -------------------------------------------------------------------------------- /chapter_09/sol_02.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | import sys 4 | 5 | import cloudpickle as pickle 6 | 7 | logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) 8 | 9 | 10 | class CommandClientProtocol(asyncio.Protocol): 11 | def __init__(self, connection_lost): 12 | self._connection_lost = connection_lost 13 | self.transport = None 14 | 15 | def connection_made(self, transport): 16 | self.transport = transport 17 | 18 | def data_received(self, data): 19 | result = pickle.loads(data) 20 | if isinstance(result, Exception): 21 | raise result 22 | logging.info(result) 23 | 24 | def connection_lost(self, exc): 25 | logging.info('The server closed the connection') 26 | self._connection_lost.set_result(True) 27 | 28 | def execute_remotely(self, f, *args, **kwargs): 29 | self.transport.write(pickle.dumps((f, args, kwargs))) 30 | 31 | 32 | async def remote_function(msg): 33 | print(msg) # This will be printed out on the host 34 | return 42 35 | 36 | 37 | async def main(): 38 | loop = asyncio.get_running_loop() 39 | 40 | connection_lost = loop.create_future() 41 | 42 | transport, protocol = await loop.create_connection( 43 | lambda: CommandClientProtocol(connection_lost), 44 | '127.0.0.1', 8888) 45 | 46 | protocol.execute_remotely(remote_function, "This worked!") 47 | 48 | try: 49 | await connection_lost 50 | finally: 51 | transport.close() 52 | 53 | if __name__ == '__main__': 54 | asyncio.run(main()) 55 | -------------------------------------------------------------------------------- /chapter_09/sol_03.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from collections import defaultdict, OrderedDict 3 | from json import dumps 4 | from urllib.parse import urljoin 5 | from wsgiref.handlers import format_date_time 6 | 7 | from httptools import HttpRequestParser 8 | 9 | 10 | class HTTPProtocol(): 11 | 12 | def __init__(self, future=None): 13 | self.parser = HttpRequestParser(self) 14 | self.headers = {} 15 | self.body = b"" 16 | self.url = b"" 17 | self.future = future 18 | 19 | def on_url(self, url: bytes): 20 | self.url = url 21 | 22 | def on_header(self, name: bytes, value: bytes): 23 | self.headers[name] = value 24 | 25 | def on_body(self, body: bytes): 26 | self.body = body 27 | 28 | def on_message_complete(self): 29 | self.future.set_result(self) 30 | 31 | def feed_data(self, data): 32 | self.parser.feed_data(data) 33 | 34 | 35 | MAX_PAYLOAD_LEN = 65536 36 | DEFAULT_HTTP_VERSION = "HTTP/1.1" 37 | NOT_FOUND = """ 38 | 39 | 40 | 41 | 404 | Page not found 42 | 43 | 44 | 45 | 46 |

"Sorry ! the page you are looking for can't be found"

47 | 48 | """ 49 | 50 | REASONS = { 51 | 100: "Continue", 52 | 101: "Switching Protocols", 53 | 200: "OK", 54 | 201: "Created", 55 | 202: "Accepted", 56 | 203: "Non-Authoritative Information", 57 | 204: "No Content", 58 | 205: "Reset Content", 59 | 206: "Partial Content", 60 | 300: "Multiple Choices", 61 | 301: "Moved Permanently", 62 | 302: "Found", 63 | 303: "See Other", 64 | 304: "Not Modified", 65 | 305: "Use Proxy", 66 | 307: "Temporary Redirect", 67 | 400: "Bad Request", 68 | 401: "Unauthorized", 69 | 402: "Payment Required", 70 | 403: "Forbidden", 71 | 404: "Not Found", 72 | 405: "Method Not Allowed", 73 | 406: "Not Acceptable", 74 | 407: "Proxy Authentication Required", 75 | 408: "Request Time-out", 76 | 409: "Conflict", 77 | 410: "Gone", 78 | 411: "Length Required", 79 | 412: "Precondition Failed", 80 | 413: "Request Entity Too Large", 81 | 414: "Request-URI Too Large", 82 | 415: "Unsupported Media Type", 83 | 416: "Requested range not satisfiable", 84 | 417: "Expectation Failed", 85 | 500: "Internal Server Error", 86 | 501: "Not Implemented", 87 | 502: "Bad Gateway", 88 | 503: "Service Unavailable", 89 | 504: "Gateway Time-out", 90 | 505: "HTTP Version not supported" 91 | 92 | } 93 | 94 | 95 | class HTTPError(BaseException): 96 | def __init__(self, status_code): 97 | assert status_code >= 400 98 | self.status_code = status_code 99 | self.reason = REASONS.get(status_code, "") 100 | 101 | def __str__(self): 102 | return f"{self.status_code} - {self.reason}" 103 | 104 | 105 | class Response: 106 | def __init__(self, status_code, headers, http_version=DEFAULT_HTTP_VERSION, body=""): 107 | self.http_version = http_version 108 | self.status_code = status_code 109 | self.headers = headers 110 | self.reason = REASONS.get(status_code, "") 111 | self.body = body 112 | 113 | def __str__(self): 114 | status_line = f"{self.http_version} {self.status_code} {self.reason}\r\n" 115 | 116 | headers = "".join( 117 | (f'"{key}": {value}\r\n' for key, value in self.headers.items()) 118 | ) 119 | return f"{status_line}{headers}\r\n{self.body}" 120 | 121 | 122 | def get_default_headers(): 123 | return OrderedDict({ 124 | "Date": format_date_time(None).encode("ascii"), 125 | "Server": AsyncioHTTPHandler.banner 126 | }) 127 | 128 | 129 | def response(headers=None, status_code=200, content_type="text/html", http_version=DEFAULT_HTTP_VERSION, body=""): 130 | if not headers: 131 | headers = get_default_headers() 132 | headers.update({"Content-Type": content_type, 133 | "Content-Length": str(len(body))}) 134 | return Response(status_code, headers, http_version, body) 135 | 136 | 137 | def json(headers=None, status_code=200, content_type="application/json", http_version=DEFAULT_HTTP_VERSION, body=None): 138 | if not body: 139 | body = {} 140 | return response(headers, status_code, content_type, http_version, dumps(body)) 141 | 142 | 143 | class AsyncioHTTPHandler: 144 | allowed_methods = ["GET"] 145 | version = 1.0 146 | banner = f"AsyncioHTTPServer/{version}".encode("ascii") 147 | default_timeout = 30 148 | 149 | def __init__(self, host, timeout=default_timeout): 150 | self.host = host 151 | self.routes = defaultdict(dict) 152 | self.timeout = timeout 153 | 154 | def route(self, *args, method="GET", path=None): 155 | 156 | def register_me(f): 157 | nonlocal path, self 158 | 159 | if not path: 160 | path = f.__name__ 161 | http_method = method.upper() 162 | 163 | assert http_method in AsyncioHTTPHandler.allowed_methods 164 | 165 | if not path.startswith("/"): 166 | path = urljoin("/", path) 167 | self.routes[http_method][path] = f 168 | return f 169 | 170 | if args: 171 | f, = args 172 | return register_me(f) 173 | return register_me 174 | 175 | async def on_connection(self, reader, writer): 176 | try: 177 | request = await asyncio.wait_for(reader.read(MAX_PAYLOAD_LEN), self.timeout) 178 | await self.send(writer, await self.handle(request)) 179 | except HTTPError as err: 180 | if err.status_code == 404: 181 | await self.send(writer, response(status_code=err.status_code, body=NOT_FOUND)) 182 | elif err.status_code == 405: 183 | headers = get_default_headers() 184 | headers.update(Allow=", ".join(AsyncioHTTPHandler.allowed_methods)) 185 | await self.send(writer, json(headers, status_code=err.status_code)) 186 | else: 187 | await self.send(writer, json(status_code=err.status_code)) 188 | except TimeoutError: 189 | await self.send(writer, json(status_code=408)) 190 | finally: 191 | writer.close() 192 | 193 | async def handle(self, request): 194 | finish_parsing = asyncio.get_running_loop().create_future() 195 | proto = HTTPProtocol(future=finish_parsing) 196 | 197 | # Ignore upgrade requests for now 198 | proto.feed_data(request) 199 | await finish_parsing 200 | 201 | try: 202 | path = proto.url.decode("UTF-8") 203 | method = proto.parser.get_method().decode("UTF-8") 204 | except UnicodeDecodeError: 205 | raise HTTPError(500) 206 | 207 | if not method.upper() in AsyncioHTTPHandler.allowed_methods: 208 | raise HTTPError(405) 209 | 210 | handler = self.routes[method].get(path) 211 | if not handler: 212 | raise HTTPError(404) 213 | return await handler(self) 214 | 215 | async def send(self, writer, response): 216 | writer.write(str(response).encode("ascii")) 217 | await writer.drain() 218 | 219 | 220 | host = "127.0.0.1" 221 | port = 1234 222 | 223 | server = AsyncioHTTPHandler(host) 224 | 225 | 226 | @server.route() 227 | async def test_me(server): 228 | return json(body=dict(it_works=True)) 229 | 230 | 231 | async def main(): 232 | s = await asyncio.start_server(server.on_connection, host, port) 233 | async with s: 234 | await s.serve_forever() 235 | 236 | if __name__ == '__main__': 237 | try: 238 | asyncio.run(main()) 239 | except KeyboardInterrupt: 240 | print("Closed..") 241 | -------------------------------------------------------------------------------- /chapter_09/sol_04.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import getpass 3 | import inspect 4 | import itertools 5 | import logging 6 | import shutil 7 | import subprocess 8 | import sys 9 | from functools import wraps 10 | 11 | logging.basicConfig(level=logging.INFO) 12 | 13 | 14 | class NotFoundError(BaseException): 15 | pass 16 | 17 | 18 | class ProcessError(BaseException): 19 | def __init__(self, return_code, stderr): 20 | self.return_code = return_code 21 | self.stderr = stderr 22 | 23 | def __str__(self): 24 | return f"Process returned non 0 return code {self.return_code}.\n" \ 25 | f"{self.stderr.decode('utf-8')}" 26 | 27 | 28 | def get_ssh_client_path(): 29 | executable = shutil.which("ssh") 30 | if not executable: 31 | raise NotFoundError( 32 | "Could not find ssh client. You can install OpenSSH from https://www.OpenSSH.com/portable.html.\nOn Mac OSX we recommend using brew: brew install OpenSSH.\nOn Linux systems you should use the package manager of your choice, like so. apt-get install OpenSSH\nOn windows you can use Chocolatey: choco install OpenSSH.") 33 | return executable 34 | 35 | 36 | def get_ssh_client_path(): 37 | executable = shutil.which("ssh") 38 | if not executable: 39 | raise NotFoundError( 40 | "Could not find ssh client. You can install OpenSSH from https://www.OpenSSH.com/portable.html.\nOn Mac OSX we recommend using brew: brew install OpenSSH.\nOn Linux systems you should use the package manager of your choice, like so: apt-get install OpenSSH\nOn windows you can use Chocolatey: choco install OpenSSH.") 41 | return executable 42 | 43 | 44 | class Connection: 45 | def __init__(self, user=None, host="127.0.0.1", port=22, timeout=None, ssh_client=None): 46 | self.host = host 47 | self.port = port 48 | if not user: 49 | user = getpass.getuser() 50 | self.user = user 51 | self.timeout = timeout 52 | if not ssh_client: 53 | ssh_client = get_ssh_client_path() 54 | self.ssh_client = ssh_client 55 | 56 | async def run(self, *cmds, interactive=False): 57 | commands = [self.ssh_client, 58 | f"{self.user}@{self.host}", 59 | f"-p {self.port}", 60 | *cmds] 61 | logging.info(" ".join(commands)) 62 | proc = await asyncio.create_subprocess_exec(*commands, 63 | stdin=subprocess.PIPE, 64 | stdout=subprocess.PIPE, 65 | stderr=subprocess.PIPE, ) 66 | if not interactive: 67 | stdout, stderr = await asyncio.wait_for(proc.communicate(), self.timeout) 68 | 69 | if proc.returncode != 0: 70 | raise ProcessError(proc.returncode, stderr) 71 | 72 | return proc, stdout, stderr 73 | else: 74 | return proc, proc.stdout, proc.stderr 75 | 76 | 77 | def command(*args, interactive=False, **kwargs): 78 | def outer(f): 79 | cmd = f.__name__ 80 | for key, value in kwargs.items(): 81 | if sys.platform.startswith(key) and value: 82 | cmd = value 83 | 84 | if inspect.isasyncgenfunction(f): 85 | @wraps(f) 86 | async def wrapper(connection, *args): 87 | proc, stdout, stderr = await connection.run(shutil.which(cmd), *args, interactive=interactive) 88 | 89 | async for value in f(proc, stdout, stderr): 90 | yield value 91 | else: 92 | @wraps(f) 93 | async def wrapper(connection, *args): 94 | proc, stdout, stderr = await connection.run(shutil.which(cmd), *args, interactive=interactive) 95 | return await f(proc, stdout, stderr) 96 | 97 | return wrapper 98 | 99 | if not args: 100 | return outer 101 | else: 102 | return outer(*args) 103 | 104 | 105 | @command(win32="dir") 106 | async def ls(proc, stdout, stderr): 107 | for line in stdout.decode("utf-8").splitlines(): 108 | yield line 109 | 110 | 111 | @command(win32="tasklist", interactive=True) 112 | async def top(proc, stdout, stderr): 113 | c = itertools.count() 114 | 115 | async for value in stdout: 116 | if next(c) > 1000: 117 | break 118 | print(value) 119 | 120 | 121 | async def main(): 122 | con = Connection() 123 | try: 124 | async for line in ls(con): 125 | print(line) 126 | 127 | await top(con) 128 | 129 | except Exception as err: 130 | logging.error(err) 131 | 132 | 133 | if sys.platform == "win32": 134 | asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) 135 | 136 | asyncio.run(main()) 137 | -------------------------------------------------------------------------------- /chapter_10/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/asyncio-recipes/f078242d4380407e483236695ebc49f3faf09e7f/chapter_10/__init__.py -------------------------------------------------------------------------------- /chapter_10/sol_01_option_1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import sys 3 | 4 | 5 | class MockException(Exception): 6 | def __init__(self, message): 7 | self.message = message 8 | 9 | def __str__(self): 10 | return self.message 11 | 12 | 13 | async def raiser(text): 14 | raise MockException(text) 15 | 16 | 17 | async def main(): 18 | raise MockException("Caught mock exception outside the loop. The loop is not running anymore.") 19 | 20 | 21 | try: 22 | asyncio.run(main()) 23 | except MockException as err: 24 | print(err, file=sys.stderr) 25 | 26 | async def main(): 27 | await raiser("Caught inline mock exception outside the loop." 28 | "The loop is not running anymore.") 29 | 30 | 31 | try: 32 | asyncio.run(main(), debug=True) 33 | except MockException as err: 34 | print(err, file=sys.stderr) 35 | 36 | 37 | async def main(): 38 | try: 39 | await raiser("Caught mock exception raised in an awaited coroutine outside the loop." 40 | "The loop is still running.") 41 | except MockException as err: 42 | print(err, file=sys.stderr) 43 | 44 | if __name__ == '__main__': 45 | asyncio.run(main(), debug=True) 46 | -------------------------------------------------------------------------------- /chapter_10/sol_01_option_2.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import sys 3 | 4 | 5 | class MockException(Exception): 6 | def __init__(self, message): 7 | self.message = message 8 | 9 | def __str__(self): 10 | return self.message 11 | 12 | 13 | def raiser_sync(text): 14 | raise MockException(text) 15 | 16 | 17 | async def main(): 18 | loop = asyncio.get_running_loop() 19 | loop.call_soon(raiser_sync, "You cannot catch me like this!") 20 | await asyncio.sleep(3) 21 | 22 | 23 | if __name__ == '__main__': 24 | try: 25 | asyncio.run(main(), debug=True) 26 | except MockException as err: 27 | print(err, file=sys.stderr) 28 | 29 | 30 | async def main(): 31 | try: 32 | loop = asyncio.get_running_loop() 33 | loop.call_soon(raiser_sync, "You cannot catch me like this!") 34 | except MockException as err: 35 | print(err, file=sys.stderr) 36 | finally: 37 | await asyncio.sleep(3) 38 | 39 | 40 | if __name__ == '__main__': 41 | asyncio.run(main(), debug=True) 42 | 43 | 44 | def exception_handler(loop, context): 45 | exception = context.get("exception") 46 | if isinstance(exception, MockException): 47 | print(exception, file=sys.stderr) 48 | else: 49 | loop.default_exception_handler(context) 50 | 51 | 52 | async def main(): 53 | loop: asyncio.AbstractEventLoop = asyncio.get_running_loop() 54 | loop.set_exception_handler(exception_handler) 55 | loop.call_soon(raiser_sync, "Finally caught the loop.call_* mock exception!") 56 | 57 | 58 | if __name__ == '__main__': 59 | asyncio.run(main(), debug=True) 60 | -------------------------------------------------------------------------------- /chapter_10/sol_02.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | 4 | logging.basicConfig(level=logging.DEBUG) 5 | 6 | 7 | class MonitorTask(asyncio.Task): 8 | 9 | def __init__(self, coro, *, loop): 10 | super().__init__(coro, loop=loop) 11 | self.start = loop.time() 12 | self.loop = loop 13 | 14 | def __del__(self): 15 | super(MonitorTask, self).__del__() 16 | self.loop = None 17 | 18 | def __await__(self): 19 | it = super(MonitorTask, self).__await__() 20 | 21 | def awaited(self): 22 | try: 23 | for i in it: 24 | yield i 25 | except BaseException as err: 26 | raise err 27 | finally: 28 | try: 29 | logging.debug("%r took %s ms to run", self, self.loop.time() - self.start) 30 | except: 31 | logging.debug("Could not estimate endtime of %r") 32 | 33 | return awaited(self) 34 | 35 | @staticmethod 36 | def task_factory(loop, coro): 37 | task = MonitorTask(coro, loop=loop) 38 | # The traceback is truncated to hide internal calls in asyncio 39 | # show only the traceback from user code 40 | if task._source_traceback: 41 | del task._source_traceback[-1] 42 | return task 43 | 44 | 45 | async def work(): 46 | await asyncio.sleep(1) 47 | 48 | 49 | async def main(): 50 | loop = asyncio.get_running_loop() 51 | loop.set_task_factory(MonitorTask.task_factory) 52 | await asyncio.create_task(work()) 53 | 54 | 55 | if __name__ == '__main__': 56 | asyncio.run(main(), debug=True) 57 | -------------------------------------------------------------------------------- /chapter_10/sol_03.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import time 3 | 4 | 5 | def slow(): 6 | time.sleep(1.5) 7 | 8 | 9 | async def main(): 10 | loop = asyncio.get_running_loop() 11 | # This will print a debug message if the call takes more than 1 second 12 | loop.slow_callback_duration = 1 13 | loop.call_soon(slow) 14 | 15 | 16 | asyncio.run(main(), debug=True) 17 | -------------------------------------------------------------------------------- /chapter_10/sol_04.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import asyncio 3 | import inspect 4 | import os 5 | import pdb 6 | from functools import wraps 7 | 8 | 9 | def get_asyncio_debug_mode_parser(): 10 | parser = argparse.ArgumentParser() 11 | parser.add_argument("--asyncio-debug", action="store_true", dest="__asyncio_debug__", default=False) 12 | return parser 13 | 14 | 15 | def is_asyncio_debug_mode(parser=get_asyncio_debug_mode_parser()): 16 | return parser and parser.parse_args().__asyncio_debug__ or os.environ.get("CUSTOM_ASYNCIO_DEBUG") 17 | 18 | 19 | __asyncio_debug__ = is_asyncio_debug_mode() 20 | 21 | 22 | def post_mortem(f): 23 | if not __asyncio_debug__: 24 | return f 25 | 26 | if inspect.isasyncgenfunction(f): 27 | @wraps(f) 28 | async def wrapper(*args, **kwargs): 29 | try: 30 | async for payload in f(*args, **kwargs): 31 | yield payload 32 | except BaseException as err: 33 | pdb.post_mortem() 34 | raise err 35 | 36 | else: 37 | @wraps(f) 38 | async def wrapper(*args, **kwargs): 39 | try: 40 | return await f(*args, **kwargs) 41 | except BaseException as err: 42 | pdb.post_mortem() 43 | raise err 44 | 45 | return wrapper 46 | 47 | 48 | def pre_run(f): 49 | if not __asyncio_debug__: 50 | return f 51 | 52 | if inspect.isasyncgenfunction(f): 53 | @wraps(f) 54 | async def wrapper(*args, **kwargs): 55 | pdb.set_trace() 56 | async for payload in f(*args, **kwargs): 57 | yield payload 58 | 59 | else: 60 | @wraps(f) 61 | async def wrapper(*args, **kwargs): 62 | pdb.set_trace() 63 | return await f(*args, **kwargs) 64 | 65 | return wrapper 66 | 67 | 68 | def post_run(f): 69 | if not __asyncio_debug__: 70 | return f 71 | 72 | if inspect.isasyncgenfunction(f): 73 | @wraps(f) 74 | async def wrapper(*args, **kwargs): 75 | async for payload in f(*args, **kwargs): 76 | yield payload 77 | pdb.set_trace() 78 | 79 | else: 80 | @wraps(f) 81 | async def wrapper(*args, **kwargs): 82 | result = await f(*args, **kwargs) 83 | pdb.set_trace() 84 | return result 85 | return wrapper 86 | 87 | 88 | @post_mortem 89 | async def main(): 90 | raise Exception() 91 | 92 | 93 | if __name__ == '__main__': 94 | asyncio.run(main()) 95 | -------------------------------------------------------------------------------- /chapter_10/sol_05.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import functools 3 | from io import StringIO 4 | from unittest import TestCase, main as unittest_main 5 | from unittest.mock import patch 6 | 7 | 8 | def into_future(arg, *, loop=None): 9 | fut = (loop or asyncio.get_running_loop()).create_future() 10 | fut.set_exception(arg) if isinstance(arg, Exception) else fut.set_result(arg) 11 | return fut 12 | 13 | 14 | class AsyncTestCase(TestCase): 15 | def __getattribute__(self, name): 16 | attr = super().__getattribute__(name) 17 | if name.startswith('test') and asyncio.iscoroutinefunction(attr): 18 | return functools.partial(asyncio.run, attr()) 19 | else: 20 | return attr 21 | 22 | 23 | class AsyncTimer: 24 | async def execute_timely(self, delay, times, f, *args, **kwargs): 25 | for i in range(times): 26 | await asyncio.sleep(delay) 27 | (await f(*args, **kwargs)) if asyncio.iscoroutine(f) else f(*args, **kwargs) 28 | 29 | 30 | class AsyncTimerTest(AsyncTestCase): 31 | 32 | async def test_execute_timely(self): 33 | times = 3 34 | delay = 3 35 | 36 | with patch("asyncio.sleep", return_value=into_future(None)) as mock_sleep, \ 37 | patch('sys.stdout', new_callable=StringIO) as mock_stdout: 38 | async_timer = AsyncTimer() 39 | await async_timer.execute_timely(delay, times, print, "test_execute_timely") 40 | 41 | mock_sleep.assert_called_with(delay) 42 | assert mock_stdout.getvalue() == "test_execute_timely\ntest_execute_timely\ntest_execute_timely\n" 43 | 44 | 45 | if __name__ == '__main__': 46 | unittest_main() 47 | -------------------------------------------------------------------------------- /chapter_10/sol_06.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import sys 3 | from types import SimpleNamespace 4 | 5 | import pytest 6 | 7 | 8 | def check_pytest_asyncio_installed(): 9 | import os 10 | from importlib import util 11 | if not util.find_spec("pytest_asyncio"): 12 | print("You need to install pytest-asyncio first!", file=sys.stderr) 13 | sys.exit(os.EX_SOFTWARE) 14 | 15 | 16 | async def return_after_sleep(res): 17 | return await asyncio.sleep(2, result=res) 18 | 19 | 20 | async def setattr_async(loop, delay, ns, key, payload): 21 | loop.call_later(delay, setattr, ns, key, payload) 22 | 23 | 24 | @pytest.fixture() 25 | async def loop(): 26 | return asyncio.get_running_loop() 27 | 28 | 29 | @pytest.fixture() 30 | def namespace(): 31 | return SimpleNamespace() 32 | 33 | 34 | @pytest.mark.asyncio 35 | async def test_return_after_sleep(): 36 | expected_result = b'expected result' 37 | res = await return_after_sleep(expected_result) 38 | assert expected_result == res 39 | 40 | 41 | @pytest.mark.asyncio 42 | async def test_setattr_async(loop, namespace): 43 | key = "test" 44 | delay = 1.0 45 | expected_result = object() 46 | await setattr_async(loop, delay, namespace, key, expected_result) 47 | await asyncio.sleep(delay) 48 | assert getattr(namespace, key, None) is expected_result 49 | 50 | 51 | if __name__ == '__main__': 52 | check_pytest_asyncio_installed() 53 | pytest.main(sys.argv) 54 | -------------------------------------------------------------------------------- /chapter_10/sol_07.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from unittest.mock import patch 3 | 4 | import asynctest 5 | import pytest 6 | 7 | 8 | def check_pytest_asyncio_installed(): 9 | import os 10 | from importlib import util 11 | if not util.find_spec("pytest_asyncio"): 12 | print("You need to install pytest-asyncio first!", file=sys.stderr) 13 | sys.exit(os.EX_SOFTWARE) 14 | 15 | 16 | async def printer(*args, printfun, **kwargs): 17 | printfun(*args, kwargs) 18 | 19 | 20 | async def async_printer(*args, printcoro, printfun, **kwargs): 21 | await printcoro(*args, printfun=printfun, **kwargs) 22 | 23 | 24 | @pytest.mark.asyncio 25 | async def test_printer_with_print(): 26 | text = "Hello world!" 27 | dict_of_texts = dict(more_text="This is a nested text!") 28 | 29 | with patch('builtins.print') as mock_printfun: 30 | await printer(text, printfun=mock_printfun, **dict_of_texts) 31 | mock_printfun.assert_called_once_with(text, dict_of_texts) 32 | 33 | 34 | @pytest.mark.asyncio 35 | async def test_async_printer_with_print(): 36 | text = "Hello world!" 37 | dict_of_texts = dict(more_text="This is a nested text!") 38 | with patch('__main__.printer', new=asynctest.CoroutineMock()) as mock_printfun: 39 | await async_printer(text, printcoro=mock_printfun, printfun=print, **dict_of_texts) 40 | mock_printfun.assert_called_once_with(text, printfun=print, **dict_of_texts) 41 | 42 | 43 | if __name__ == '__main__': 44 | check_pytest_asyncio_installed() 45 | pytest.main(sys.argv) 46 | -------------------------------------------------------------------------------- /chapter_10/sol_08.py: -------------------------------------------------------------------------------- 1 | async def complicated(a, b, c): 2 | """ 3 | >>> import asyncio 4 | >>> asyncio.run(complicated(5,None,None)) 5 | True 6 | >>> asyncio.run(complicated(None,None,None)) 7 | Traceback (most recent call last): 8 | ... 9 | ValueError: This value: None is not an int or larger than 4 10 | >>> asyncio.run(complicated(None,"This","will be printed out")) 11 | This will be printed out 12 | 13 | :param a: This parameter controls the return value 14 | :param b: 15 | :param c: 16 | :return: 17 | """ 18 | if isinstance(a, int) and a > 4: 19 | return True 20 | elif b and c: 21 | print(b, c) 22 | else: 23 | raise ValueError(f"This value: {a} is not an int or larger than 4") 24 | 25 | 26 | if __name__ == "__main__": 27 | import doctest 28 | 29 | doctest.testmod() 30 | -------------------------------------------------------------------------------- /errata.md: -------------------------------------------------------------------------------- 1 | # Errata for *Book Title* 2 | 3 | On **page xx** [Summary of error]: 4 | 5 | Details of error here. Highlight key pieces in **bold**. 6 | 7 | *** 8 | 9 | On **page xx** [Summary of error]: 10 | 11 | Details of error here. Highlight key pieces in **bold**. 12 | 13 | *** --------------------------------------------------------------------------------