├── 01_recursive_coroutines └── recursive_coroutines.py ├── 02_fire_and_forget ├── 01_fire_and_forget.py ├── 02_fire_and_forget.py ├── 03_fire_and_forget.py └── 04_fire_and_forget.py ├── 03_periodic_coroutines ├── 01_periodic_coroutines.py ├── 02_periodic_coroutines.py ├── 03_periodic_coroutines.py ├── 04_periodic_coroutines.py └── 05_periodic_coroutines.py ├── 04_error_handling ├── 01_error_handling.py ├── 01b_error_handling.py ├── 01c_error_handling.py ├── 02_error_handling.py ├── 02b_error_handling.py ├── 02c_error_handling.py └── 03_error_handling.py ├── 05_cancelling_coroutines ├── 01_cancelling_coroutines.py ├── 02_cancelling_coroutines.py ├── 03_cancelling_coroutines.py └── 04_cancelling_coroutines.py ├── README.md └── requirements.txt /01_recursive_coroutines/recursive_coroutines.py: -------------------------------------------------------------------------------- 1 | """ 2 | A recursive function solves a problem by simplifying the input until 3 | we arrive at a base trivial case and then combining the results up the stack. 4 | 5 | Assume we want to calculate the number of comments of a particular post in 6 | Hacker News by recursively aggregating the number of descendents. 7 | 8 | """ 9 | 10 | import asyncio 11 | import argparse 12 | import logging 13 | from urllib.parse import urlparse, parse_qs 14 | from datetime import datetime 15 | 16 | import aiohttp 17 | import async_timeout 18 | 19 | 20 | LOGGER_FORMAT = '%(asctime)s %(message)s' 21 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 22 | FETCH_TIMEOUT = 10 23 | 24 | parser = argparse.ArgumentParser( 25 | description='Calculate the comments of a Hacker News post.') 26 | parser.add_argument('--id', type=int, default=8863, 27 | help='ID of the post in HN, defaults to 8863') 28 | parser.add_argument('--url', type=str, help='URL of a post in HN') 29 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 30 | 31 | 32 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 33 | log = logging.getLogger() 34 | log.setLevel(logging.INFO) 35 | 36 | fetch_counter = 0 37 | 38 | 39 | async def fetch(session, url): 40 | """Fetch a URL using aiohttp returning parsed JSON response. 41 | 42 | As suggested by the aiohttp docs we reuse the session. 43 | 44 | """ 45 | global fetch_counter 46 | with async_timeout.timeout(FETCH_TIMEOUT): 47 | fetch_counter += 1 48 | async with session.get(url) as response: 49 | return await response.json() 50 | 51 | 52 | async def post_number_of_comments(loop, session, post_id): 53 | """Retrieve data for current post and recursively for all comments. 54 | 55 | """ 56 | url = URL_TEMPLATE.format(post_id) 57 | now = datetime.now() 58 | response = await fetch(session, url) 59 | log.debug('{:^6} > Fetching of {} took {} seconds'.format( 60 | post_id, url, (datetime.now() - now).total_seconds())) 61 | 62 | if 'kids' not in response: # base case, there are no comments 63 | return 0 64 | 65 | # calculate this post's comments as number of comments 66 | number_of_comments = len(response['kids']) 67 | 68 | # create recursive tasks for all comments 69 | log.debug('{:^6} > Fetching {} child posts'.format( 70 | post_id, number_of_comments)) 71 | tasks = [post_number_of_comments( 72 | loop, session, kid_id) for kid_id in response['kids']] 73 | 74 | # schedule the tasks and retrieve results 75 | results = await asyncio.gather(*tasks) 76 | 77 | # reduce the descendents comments and add it to this post's 78 | number_of_comments += sum(results) 79 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 80 | 81 | return number_of_comments 82 | 83 | 84 | def id_from_HN_url(url): 85 | """Returns the value of the `id` query arg of a URL if present, or None. 86 | 87 | """ 88 | parse_result = urlparse(url) 89 | try: 90 | return parse_qs(parse_result.query)['id'][0] 91 | except (KeyError, IndexError): 92 | return None 93 | 94 | 95 | async def main(loop, post_id): 96 | """Async entry point coroutine. 97 | 98 | """ 99 | now = datetime.now() 100 | async with aiohttp.ClientSession(loop=loop) as session: 101 | now = datetime.now() 102 | comments = await post_number_of_comments(loop, session, post_id) 103 | log.info( 104 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 105 | (datetime.now() - now).total_seconds(), fetch_counter)) 106 | 107 | return comments 108 | 109 | 110 | if __name__ == '__main__': 111 | args = parser.parse_args() 112 | if args.verbose: 113 | log.setLevel(logging.DEBUG) 114 | 115 | post_id = id_from_HN_url(args.url) if args.url else args.id 116 | 117 | loop = asyncio.get_event_loop() 118 | comments = loop.run_until_complete(main(loop, post_id)) 119 | log.info("-- Post {} has {} comments".format(post_id, comments)) 120 | 121 | loop.close() 122 | -------------------------------------------------------------------------------- /02_fire_and_forget/01_fire_and_forget.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of the "fire and forget" pattern awaiting for lower priority 3 | tasks to be completed, noticably slower than normal since we are blocking the 4 | loop until all tasks complete. 5 | 6 | """ 7 | 8 | import asyncio 9 | import argparse 10 | import logging 11 | from random import random 12 | from datetime import datetime 13 | from urllib.parse import urlparse, parse_qs 14 | 15 | import aiohttp 16 | import async_timeout 17 | 18 | 19 | LOGGER_FORMAT = '%(asctime)s %(message)s' 20 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 21 | FETCH_TIMEOUT = 10 22 | MIN_COMMENTS = 5 23 | 24 | parser = argparse.ArgumentParser( 25 | description='Calculate the comments of a Hacker News post.') 26 | parser.add_argument('--id', type=int, default=8863, 27 | help='ID of the post in HN, defaults to 8863') 28 | parser.add_argument('--url', type=str, help='URL of a post in HN') 29 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 30 | 31 | 32 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 33 | log = logging.getLogger() 34 | log.setLevel(logging.INFO) 35 | 36 | fetch_counter = 0 37 | 38 | 39 | async def fetch(session, url): 40 | """Fetch a URL using aiohttp returning parsed JSON response. 41 | 42 | As suggested by the aiohttp docs we reuse the session. 43 | 44 | """ 45 | global fetch_counter 46 | with async_timeout.timeout(FETCH_TIMEOUT): 47 | fetch_counter += 1 48 | async with session.get(url) as response: 49 | return await response.json() 50 | 51 | 52 | async def post_number_of_comments(loop, session, post_id): 53 | """Retrieve data for current post and recursively for all comments. 54 | 55 | """ 56 | url = URL_TEMPLATE.format(post_id) 57 | response = await fetch(session, url) 58 | 59 | if 'kids' not in response: # base case, there are no comments 60 | return 0 61 | 62 | # calculate this post's comments as number of comments 63 | number_of_comments = len(response['kids']) 64 | 65 | # create recursive tasks for all comments 66 | tasks = [post_number_of_comments( 67 | loop, session, kid_id) for kid_id in response['kids']] 68 | 69 | # schedule the tasks and retrieve results 70 | results = await asyncio.gather(*tasks) 71 | 72 | # reduce the descendents comments and add it to this post's 73 | number_of_comments += sum(results) 74 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 75 | 76 | # Log if number of comments is over a threshold 77 | if number_of_comments > MIN_COMMENTS: 78 | await log_post(response) 79 | 80 | return number_of_comments 81 | 82 | 83 | async def log_post(post): 84 | """Simulate logging of a post. 85 | 86 | """ 87 | await asyncio.sleep(random() * 3) 88 | log.info("Post logged") 89 | 90 | 91 | def id_from_HN_url(url): 92 | """Returns the value of the `id` query arg of a URL if present, or None. 93 | 94 | """ 95 | parse_result = urlparse(url) 96 | try: 97 | return parse_qs(parse_result.query)['id'][0] 98 | except (KeyError, IndexError): 99 | return None 100 | 101 | 102 | async def main(loop, post_id): 103 | """Async entry point coroutine. 104 | 105 | """ 106 | now = datetime.now() 107 | async with aiohttp.ClientSession(loop=loop) as session: 108 | comments = await post_number_of_comments(loop, session, post_id) 109 | log.info( 110 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 111 | (datetime.now() - now).total_seconds(), fetch_counter)) 112 | 113 | return comments 114 | 115 | 116 | if __name__ == '__main__': 117 | args = parser.parse_args() 118 | if args.verbose: 119 | log.setLevel(logging.DEBUG) 120 | 121 | post_id = id_from_HN_url(args.url) if args.url else args.id 122 | 123 | loop = asyncio.get_event_loop() 124 | comments = loop.run_until_complete(main(loop, post_id)) 125 | log.info("-- Post {} has {} comments".format(post_id, comments)) 126 | 127 | loop.close() 128 | -------------------------------------------------------------------------------- /02_fire_and_forget/02_fire_and_forget.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of the "fire and forget" pattern awaiting using ensure_future 3 | which works but results in "Task was destroyed but it is pending!" messages. 4 | 5 | """ 6 | 7 | import asyncio 8 | import argparse 9 | import logging 10 | from random import random 11 | from datetime import datetime 12 | from urllib.parse import urlparse, parse_qs 13 | 14 | import aiohttp 15 | import async_timeout 16 | 17 | 18 | LOGGER_FORMAT = '%(asctime)s %(message)s' 19 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 20 | FETCH_TIMEOUT = 10 21 | MIN_COMMENTS = 5 22 | 23 | parser = argparse.ArgumentParser( 24 | description='Calculate the comments of a Hacker News post.') 25 | parser.add_argument('--id', type=int, default=8863, 26 | help='ID of the post in HN, defaults to 8863') 27 | parser.add_argument('--url', type=str, help='URL of a post in HN') 28 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 29 | 30 | 31 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 32 | log = logging.getLogger() 33 | log.setLevel(logging.INFO) 34 | 35 | fetch_counter = 0 36 | 37 | 38 | async def fetch(session, url): 39 | """Fetch a URL using aiohttp returning parsed JSON response. 40 | 41 | As suggested by the aiohttp docs we reuse the session. 42 | 43 | """ 44 | global fetch_counter 45 | with async_timeout.timeout(FETCH_TIMEOUT): 46 | fetch_counter += 1 47 | async with session.get(url) as response: 48 | return await response.json() 49 | 50 | 51 | async def post_number_of_comments(loop, session, post_id): 52 | """Retrieve data for current post and recursively for all comments. 53 | 54 | """ 55 | url = URL_TEMPLATE.format(post_id) 56 | response = await fetch(session, url) 57 | 58 | if 'kids' not in response: # base case, there are no comments 59 | return 0 60 | 61 | # calculate this post's comments as number of comments 62 | number_of_comments = len(response['kids']) 63 | 64 | # create recursive tasks for all comments 65 | tasks = [post_number_of_comments( 66 | loop, session, kid_id) for kid_id in response['kids']] 67 | 68 | # schedule the tasks and retrieve results 69 | results = await asyncio.gather(*tasks) 70 | 71 | # reduce the descendents comments and add it to this post's 72 | number_of_comments += sum(results) 73 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 74 | 75 | # Log if number of comments is over a threshold 76 | if number_of_comments > MIN_COMMENTS: 77 | asyncio.ensure_future(log_post(response)) 78 | 79 | return number_of_comments 80 | 81 | 82 | async def log_post(post): 83 | """Simulate logging of a post. 84 | 85 | """ 86 | await asyncio.sleep(random() * 3) 87 | log.info("Post logged") 88 | 89 | 90 | def id_from_HN_url(url): 91 | """Returns the value of the `id` query arg of a URL if present, or None. 92 | 93 | """ 94 | parse_result = urlparse(url) 95 | try: 96 | return parse_qs(parse_result.query)['id'][0] 97 | except (KeyError, IndexError): 98 | return None 99 | 100 | 101 | async def main(loop, post_id): 102 | """Async entry point coroutine. 103 | 104 | """ 105 | now = datetime.now() 106 | async with aiohttp.ClientSession(loop=loop) as session: 107 | comments = await post_number_of_comments(loop, session, post_id) 108 | log.info( 109 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 110 | (datetime.now() - now).total_seconds(), fetch_counter)) 111 | 112 | return comments 113 | 114 | 115 | if __name__ == '__main__': 116 | args = parser.parse_args() 117 | if args.verbose: 118 | log.setLevel(logging.DEBUG) 119 | 120 | post_id = id_from_HN_url(args.url) if args.url else args.id 121 | 122 | loop = asyncio.get_event_loop() 123 | comments = loop.run_until_complete(main(loop, post_id)) 124 | log.info("-- Post {} has {} comments".format(post_id, comments)) 125 | 126 | loop.close() 127 | -------------------------------------------------------------------------------- /02_fire_and_forget/03_fire_and_forget.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of the "fire and forget" pattern ensuring all pending tasks are 3 | given time to complete by use of `Task.all_tasks`. 4 | 5 | """ 6 | 7 | import asyncio 8 | import argparse 9 | import logging 10 | from random import random 11 | from datetime import datetime 12 | from urllib.parse import urlparse, parse_qs 13 | 14 | import aiohttp 15 | import async_timeout 16 | 17 | 18 | LOGGER_FORMAT = '%(asctime)s %(message)s' 19 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 20 | FETCH_TIMEOUT = 10 21 | MIN_COMMENTS = 5 22 | 23 | parser = argparse.ArgumentParser( 24 | description='Calculate the comments of a Hacker News post.') 25 | parser.add_argument('--id', type=int, default=8863, 26 | help='ID of the post in HN, defaults to 8863') 27 | parser.add_argument('--url', type=str, help='URL of a post in HN') 28 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 29 | 30 | 31 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 32 | log = logging.getLogger() 33 | log.setLevel(logging.INFO) 34 | 35 | fetch_counter = 0 36 | 37 | 38 | async def fetch(session, url): 39 | """Fetch a URL using aiohttp returning parsed JSON response. 40 | 41 | As suggested by the aiohttp docs we reuse the session. 42 | 43 | """ 44 | global fetch_counter 45 | with async_timeout.timeout(FETCH_TIMEOUT): 46 | fetch_counter += 1 47 | async with session.get(url) as response: 48 | return await response.json() 49 | 50 | 51 | async def post_number_of_comments(loop, session, post_id): 52 | """Retrieve data for current post and recursively for all comments. 53 | 54 | """ 55 | url = URL_TEMPLATE.format(post_id) 56 | response = await fetch(session, url) 57 | 58 | if 'kids' not in response: # base case, there are no comments 59 | return 0 60 | 61 | # calculate this post's comments as number of comments 62 | number_of_comments = len(response['kids']) 63 | 64 | # create recursive tasks for all comments 65 | tasks = [post_number_of_comments( 66 | loop, session, kid_id) for kid_id in response['kids']] 67 | 68 | # schedule the tasks and retrieve results 69 | results = await asyncio.gather(*tasks) 70 | 71 | # reduce the descendents comments and add it to this post's 72 | number_of_comments += sum(results) 73 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 74 | 75 | # Log if number of comments is over a threshold 76 | if number_of_comments > MIN_COMMENTS: 77 | asyncio.ensure_future(log_post(response)) 78 | 79 | return number_of_comments 80 | 81 | 82 | async def log_post(post): 83 | """Simulate logging of a post. 84 | 85 | """ 86 | await asyncio.sleep(random() * 3) 87 | log.info("Post logged") 88 | 89 | 90 | def id_from_HN_url(url): 91 | """Returns the value of the `id` query arg of a URL if present, or None. 92 | 93 | """ 94 | parse_result = urlparse(url) 95 | try: 96 | return parse_qs(parse_result.query)['id'][0] 97 | except (KeyError, IndexError): 98 | return None 99 | 100 | 101 | async def main(loop, post_id): 102 | """Async entry point coroutine. 103 | 104 | """ 105 | now = datetime.now() 106 | async with aiohttp.ClientSession(loop=loop) as session: 107 | now = datetime.now() 108 | comments = await post_number_of_comments(loop, session, post_id) 109 | log.info( 110 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 111 | (datetime.now() - now).total_seconds(), fetch_counter)) 112 | 113 | return comments 114 | 115 | 116 | if __name__ == '__main__': 117 | args = parser.parse_args() 118 | if args.verbose: 119 | log.setLevel(logging.DEBUG) 120 | 121 | post_id = id_from_HN_url(args.url) if args.url else args.id 122 | 123 | loop = asyncio.get_event_loop() 124 | comments = loop.run_until_complete(main(loop, post_id)) 125 | log.info("-- Post {} has {} comments".format(post_id, comments)) 126 | 127 | pending_tasks = [ 128 | task for task in asyncio.Task.all_tasks() if not task.done()] 129 | loop.run_until_complete(asyncio.gather(*pending_tasks)) 130 | 131 | loop.close() 132 | -------------------------------------------------------------------------------- /02_fire_and_forget/04_fire_and_forget.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of the "fire and forget" pattern registering low priority tasks. 3 | 4 | """ 5 | 6 | import asyncio 7 | import argparse 8 | import logging 9 | from random import random 10 | from datetime import datetime 11 | from urllib.parse import urlparse, parse_qs 12 | 13 | import aiohttp 14 | import async_timeout 15 | 16 | 17 | LOGGER_FORMAT = '%(asctime)s %(message)s' 18 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 19 | FETCH_TIMEOUT = 10 20 | MIN_COMMENTS = 5 21 | 22 | parser = argparse.ArgumentParser( 23 | description='Calculate the comments of a Hacker News post.') 24 | parser.add_argument('--id', type=int, default=8863, 25 | help='ID of the post in HN, defaults to 8863') 26 | parser.add_argument('--url', type=str, help='URL of a post in HN') 27 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 28 | 29 | 30 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 31 | log = logging.getLogger() 32 | log.setLevel(logging.INFO) 33 | 34 | fetch_counter = 0 35 | 36 | 37 | async def fetch(session, url): 38 | """Fetch a URL using aiohttp returning parsed JSON response. 39 | 40 | As suggested by the aiohttp docs we reuse the session. 41 | 42 | """ 43 | global fetch_counter 44 | with async_timeout.timeout(FETCH_TIMEOUT): 45 | fetch_counter += 1 46 | async with session.get(url) as response: 47 | return await response.json() 48 | 49 | 50 | async def post_number_of_comments(loop, session, post_id): 51 | """Retrieve data for current post and recursively for all comments. 52 | 53 | """ 54 | url = URL_TEMPLATE.format(post_id) 55 | response = await fetch(session, url) 56 | 57 | if 'kids' not in response: # base case, there are no comments 58 | return 0 59 | 60 | # calculate this post's comments as number of comments 61 | number_of_comments = len(response['kids']) 62 | 63 | # create recursive tasks for all comments 64 | tasks = [post_number_of_comments( 65 | loop, session, kid_id) for kid_id in response['kids']] 66 | 67 | # schedule the tasks and retrieve results 68 | results = await asyncio.gather(*tasks) 69 | 70 | # reduce the descendents comments and add it to this post's 71 | number_of_comments += sum(results) 72 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 73 | 74 | # Log if number of comments is over a threshold 75 | if number_of_comments > MIN_COMMENTS: 76 | # Add the future to the registry 77 | task_registry.append(asyncio.ensure_future(log_post(response))) 78 | 79 | return number_of_comments 80 | 81 | 82 | async def log_post(post): 83 | """Simulate logging of a post. 84 | 85 | """ 86 | await asyncio.sleep(random() * 3) 87 | log.info("Post logged") 88 | 89 | 90 | def id_from_HN_url(url): 91 | """Returns the value of the `id` query arg of a URL if present, or None. 92 | 93 | """ 94 | parse_result = urlparse(url) 95 | try: 96 | return parse_qs(parse_result.query)['id'][0] 97 | except (KeyError, IndexError): 98 | return None 99 | 100 | 101 | async def main(loop, post_id): 102 | """Async entry point coroutine. 103 | 104 | """ 105 | now = datetime.now() 106 | async with aiohttp.ClientSession(loop=loop) as session: 107 | now = datetime.now() 108 | comments = await post_number_of_comments(loop, session, post_id) 109 | log.info( 110 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 111 | (datetime.now() - now).total_seconds(), fetch_counter)) 112 | 113 | return comments 114 | 115 | 116 | if __name__ == '__main__': 117 | args = parser.parse_args() 118 | if args.verbose: 119 | log.setLevel(logging.DEBUG) 120 | 121 | post_id = id_from_HN_url(args.url) if args.url else args.id 122 | task_registry = [] # define our task registry 123 | 124 | loop = asyncio.get_event_loop() 125 | comments = loop.run_until_complete(main(loop, post_id)) 126 | log.info("-- Post {} has {} comments".format(post_id, comments)) 127 | 128 | pending_tasks = [ 129 | task for task in task_registry if not task.done()] 130 | loop.run_until_complete(asyncio.gather(*pending_tasks)) 131 | 132 | loop.close() 133 | -------------------------------------------------------------------------------- /03_periodic_coroutines/01_periodic_coroutines.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using an infinite loop of 3 | awaiting and sleeping. 4 | 5 | """ 6 | 7 | import asyncio 8 | import argparse 9 | import logging 10 | from datetime import datetime 11 | 12 | import aiohttp 13 | import async_timeout 14 | 15 | 16 | LOGGER_FORMAT = '%(asctime)s %(message)s' 17 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 18 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 19 | FETCH_TIMEOUT = 10 20 | 21 | parser = argparse.ArgumentParser( 22 | description='Calculate the number of comments of the top stories in HN.') 23 | parser.add_argument( 24 | '--period', type=int, default=5, help='Number of seconds between poll') 25 | parser.add_argument( 26 | '--limit', type=int, default=5, 27 | help='Number of new stories to calculate comments for') 28 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 29 | 30 | 31 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 32 | log = logging.getLogger() 33 | log.setLevel(logging.INFO) 34 | 35 | fetch_counter = 0 36 | 37 | 38 | async def fetch(session, url): 39 | """Fetch a URL using aiohttp returning parsed JSON response. 40 | 41 | As suggested by the aiohttp docs we reuse the session. 42 | 43 | """ 44 | global fetch_counter 45 | with async_timeout.timeout(FETCH_TIMEOUT): 46 | fetch_counter += 1 47 | async with session.get(url) as response: 48 | return await response.json() 49 | 50 | 51 | async def post_number_of_comments(loop, session, post_id): 52 | """Retrieve data for current post and recursively for all comments. 53 | 54 | """ 55 | url = URL_TEMPLATE.format(post_id) 56 | response = await fetch(session, url) 57 | 58 | if 'kids' not in response: # base case, there are no comments 59 | return 0 60 | 61 | # calculate this post's comments as number of comments 62 | number_of_comments = len(response['kids']) 63 | 64 | # create recursive tasks for all comments 65 | tasks = [post_number_of_comments( 66 | loop, session, kid_id) for kid_id in response['kids']] 67 | 68 | # schedule the tasks and retrieve results 69 | results = await asyncio.gather(*tasks) 70 | 71 | # reduce the descendents comments and add it to this post's 72 | number_of_comments += sum(results) 73 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 74 | 75 | return number_of_comments 76 | 77 | 78 | async def get_comments_of_top_stories(loop, session, limit, iteration): 79 | """Retrieve top stories in HN. 80 | 81 | """ 82 | response = await fetch(session, TOP_STORIES_URL) 83 | tasks = [post_number_of_comments( 84 | loop, session, post_id) for post_id in response[:limit]] 85 | results = await asyncio.gather(*tasks) 86 | for post_id, num_comments in zip(response[:limit], results): 87 | log.info("Post {} has {} comments ({})".format( 88 | post_id, num_comments, iteration)) 89 | 90 | 91 | async def poll_top_stories_for_comments(loop, session, period, limit): 92 | """Periodically poll for new stories and retrieve number of comments. 93 | 94 | """ 95 | global fetch_counter 96 | iteration = 1 97 | while True: 98 | now = datetime.now() 99 | log.info("Calculating comments for top {} stories. ({})".format( 100 | limit, iteration)) 101 | await get_comments_of_top_stories(loop, session, limit, iteration) 102 | 103 | log.info( 104 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 105 | (datetime.now() - now).total_seconds(), fetch_counter)) 106 | log.info("Waiting for {} seconds...".format(period)) 107 | iteration += 1 108 | fetch_counter = 0 109 | await asyncio.sleep(period) 110 | 111 | 112 | async def main(loop, period, limit): 113 | """Async entry point coroutine. 114 | 115 | """ 116 | async with aiohttp.ClientSession(loop=loop) as session: 117 | comments = await poll_top_stories_for_comments(loop, session, period, limit) 118 | 119 | return comments 120 | 121 | 122 | if __name__ == '__main__': 123 | args = parser.parse_args() 124 | if args.verbose: 125 | log.setLevel(logging.DEBUG) 126 | 127 | loop = asyncio.get_event_loop() 128 | loop.run_until_complete(main(loop, args.period, args.limit)) 129 | 130 | loop.close() 131 | -------------------------------------------------------------------------------- /03_periodic_coroutines/02_periodic_coroutines.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using an infinite loop of 3 | scheduling a task using ensure_future and sleeping. 4 | 5 | This will produce incorrect output since we've moved away from await 6 | which causes the messages to output as 0 seconds elapsed and 0 fetches. 7 | 8 | """ 9 | 10 | import asyncio 11 | import argparse 12 | import logging 13 | from datetime import datetime 14 | 15 | import aiohttp 16 | import async_timeout 17 | 18 | 19 | LOGGER_FORMAT = '%(asctime)s %(message)s' 20 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 21 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 22 | FETCH_TIMEOUT = 10 23 | 24 | parser = argparse.ArgumentParser( 25 | description='Calculate the number of comments of the top stories in HN.') 26 | parser.add_argument( 27 | '--period', type=int, default=5, help='Number of seconds between poll') 28 | parser.add_argument( 29 | '--limit', type=int, default=5, 30 | help='Number of new stories to calculate comments for') 31 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 32 | 33 | 34 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 35 | log = logging.getLogger() 36 | log.setLevel(logging.INFO) 37 | 38 | fetch_counter = 0 39 | 40 | 41 | async def fetch(session, url): 42 | """Fetch a URL using aiohttp returning parsed JSON response. 43 | 44 | As suggested by the aiohttp docs we reuse the session. 45 | 46 | """ 47 | global fetch_counter 48 | with async_timeout.timeout(FETCH_TIMEOUT): 49 | fetch_counter += 1 50 | async with session.get(url) as response: 51 | return await response.json() 52 | 53 | 54 | async def post_number_of_comments(loop, session, post_id): 55 | """Retrieve data for current post and recursively for all comments. 56 | 57 | """ 58 | url = URL_TEMPLATE.format(post_id) 59 | response = await fetch(session, url) 60 | 61 | # base case, there are no comments 62 | if response is None or 'kids' not in response: 63 | return 0 64 | 65 | # calculate this post's comments as number of comments 66 | number_of_comments = len(response['kids']) 67 | 68 | # create recursive tasks for all comments 69 | tasks = [post_number_of_comments( 70 | loop, session, kid_id) for kid_id in response['kids']] 71 | 72 | # schedule the tasks and retrieve results 73 | results = await asyncio.gather(*tasks) 74 | 75 | # reduce the descendents comments and add it to this post's 76 | number_of_comments += sum(results) 77 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 78 | 79 | return number_of_comments 80 | 81 | 82 | async def get_comments_of_top_stories(loop, session, limit, iteration): 83 | """Retrieve top stories in HN. 84 | 85 | """ 86 | response = await fetch(session, TOP_STORIES_URL) 87 | tasks = [post_number_of_comments( 88 | loop, session, post_id) for post_id in response[:limit]] 89 | results = await asyncio.gather(*tasks) 90 | for post_id, num_comments in zip(response[:limit], results): 91 | log.info("Post {} has {} comments ({})".format( 92 | post_id, num_comments, iteration)) 93 | 94 | 95 | async def poll_top_stories_for_comments(loop, session, period, limit): 96 | """Periodically poll for new stories and retrieve number of comments. 97 | 98 | """ 99 | global fetch_counter 100 | iteration = 1 101 | while True: 102 | now = datetime.now() 103 | log.info("Calculating comments for top {} stories. ({})".format( 104 | limit, iteration)) 105 | asyncio.ensure_future( 106 | get_comments_of_top_stories(loop, session, limit, iteration)) 107 | 108 | log.info( 109 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 110 | (datetime.now() - now).total_seconds(), fetch_counter)) 111 | log.info("Waiting for {} seconds...".format(period)) 112 | iteration += 1 113 | fetch_counter = 0 114 | await asyncio.sleep(period) 115 | 116 | 117 | async def main(loop, period, limit): 118 | """Async entry point coroutine. 119 | 120 | """ 121 | async with aiohttp.ClientSession(loop=loop) as session: 122 | comments = await poll_top_stories_for_comments(loop, session, period, limit) 123 | 124 | return comments 125 | 126 | 127 | if __name__ == '__main__': 128 | args = parser.parse_args() 129 | if args.verbose: 130 | log.setLevel(logging.DEBUG) 131 | 132 | loop = asyncio.get_event_loop() 133 | loop.run_until_complete(main(loop, args.period, args.limit)) 134 | 135 | loop.close() 136 | -------------------------------------------------------------------------------- /03_periodic_coroutines/03_periodic_coroutines.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using an infinite loop of 3 | scheduling a task using ensure_future and sleeping. 4 | 5 | Using a callback to the future returned by ensure_future we can now 6 | correctly output statistics on the elapsed time and fetches using the new 7 | URLFetcher class. But at a cost in readability. 8 | 9 | """ 10 | 11 | import asyncio 12 | import argparse 13 | import logging 14 | from datetime import datetime 15 | 16 | import aiohttp 17 | import async_timeout 18 | 19 | 20 | LOGGER_FORMAT = '%(asctime)s %(message)s' 21 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 22 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 23 | FETCH_TIMEOUT = 10 24 | 25 | parser = argparse.ArgumentParser( 26 | description='Calculate the number of comments of the top stories in HN.') 27 | parser.add_argument( 28 | '--period', type=int, default=5, help='Number of seconds between poll') 29 | parser.add_argument( 30 | '--limit', type=int, default=5, 31 | help='Number of new stories to calculate comments for') 32 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 33 | 34 | 35 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 36 | log = logging.getLogger() 37 | log.setLevel(logging.INFO) 38 | 39 | 40 | class URLFetcher(): 41 | """Provides counting of URL fetches for a particular task. 42 | 43 | """ 44 | 45 | def __init__(self): 46 | self.fetch_counter = 0 47 | 48 | async def fetch(self, session, url): 49 | """Fetch a URL using aiohttp returning parsed JSON response. 50 | 51 | As suggested by the aiohttp docs we reuse the session. 52 | 53 | """ 54 | with async_timeout.timeout(FETCH_TIMEOUT): 55 | self.fetch_counter += 1 56 | async with session.get(url) as response: 57 | return await response.json() 58 | 59 | 60 | async def post_number_of_comments(loop, session, fetcher, post_id): 61 | """Retrieve data for current post and recursively for all comments. 62 | 63 | """ 64 | url = URL_TEMPLATE.format(post_id) 65 | response = await fetcher.fetch(session, url) 66 | 67 | # base case, there are no comments 68 | if response is None or 'kids' not in response: 69 | return 0 70 | 71 | # calculate this post's comments as number of comments 72 | number_of_comments = len(response['kids']) 73 | 74 | # create recursive tasks for all comments 75 | tasks = [post_number_of_comments( 76 | loop, session, fetcher, kid_id) for kid_id in response['kids']] 77 | 78 | # schedule the tasks and retrieve results 79 | results = await asyncio.gather(*tasks) 80 | 81 | # reduce the descendents comments and add it to this post's 82 | number_of_comments += sum(results) 83 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 84 | 85 | return number_of_comments 86 | 87 | 88 | async def get_comments_of_top_stories(loop, session, limit, iteration): 89 | """Retrieve top stories in HN. 90 | 91 | """ 92 | fetcher = URLFetcher() # create a new fetcher for this task 93 | response = await fetcher.fetch(session, TOP_STORIES_URL) 94 | tasks = [post_number_of_comments( 95 | loop, session, fetcher, post_id) for post_id in response[:limit]] 96 | results = await asyncio.gather(*tasks) 97 | for post_id, num_comments in zip(response[:limit], results): 98 | log.info("Post {} has {} comments ({})".format( 99 | post_id, num_comments, iteration)) 100 | return fetcher.fetch_counter # return the fetch count 101 | 102 | 103 | async def poll_top_stories_for_comments(loop, session, period, limit): 104 | """Periodically poll for new stories and retrieve number of comments. 105 | 106 | """ 107 | iteration = 1 108 | while True: 109 | log.info("Calculating comments for top {} stories. ({})".format( 110 | limit, iteration)) 111 | 112 | future = asyncio.ensure_future( 113 | get_comments_of_top_stories(loop, session, limit, iteration)) 114 | 115 | now = datetime.now() 116 | 117 | def callback(fut): 118 | fetch_count = fut.result() 119 | log.info( 120 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 121 | (datetime.now() - now).total_seconds(), fetch_count)) 122 | 123 | future.add_done_callback(callback) 124 | 125 | log.info("Waiting for {} seconds...".format(period)) 126 | iteration += 1 127 | await asyncio.sleep(period) 128 | 129 | 130 | async def main(loop, period, limit): 131 | """Async entry point coroutine. 132 | 133 | """ 134 | async with aiohttp.ClientSession(loop=loop) as session: 135 | comments = await poll_top_stories_for_comments(loop, session, period, limit) 136 | 137 | return comments 138 | 139 | 140 | if __name__ == '__main__': 141 | args = parser.parse_args() 142 | if args.verbose: 143 | log.setLevel(logging.DEBUG) 144 | 145 | loop = asyncio.get_event_loop() 146 | loop.run_until_complete(main(loop, args.period, args.limit)) 147 | 148 | loop.close() 149 | -------------------------------------------------------------------------------- /03_periodic_coroutines/04_periodic_coroutines.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using a call_later to 3 | schedule the execution of a standard function that would schedule another task. 4 | 5 | """ 6 | 7 | import asyncio 8 | import argparse 9 | import logging 10 | from datetime import datetime 11 | from functools import partial 12 | 13 | import aiohttp 14 | import async_timeout 15 | 16 | 17 | LOGGER_FORMAT = '%(asctime)s %(message)s' 18 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 19 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 20 | FETCH_TIMEOUT = 10 21 | 22 | parser = argparse.ArgumentParser( 23 | description='Calculate the number of comments of the top stories in HN.') 24 | parser.add_argument( 25 | '--period', type=int, default=5, help='Number of seconds between poll') 26 | parser.add_argument( 27 | '--limit', type=int, default=5, 28 | help='Number of new stories to calculate comments for') 29 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 30 | 31 | 32 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 33 | log = logging.getLogger() 34 | log.setLevel(logging.INFO) 35 | 36 | 37 | class URLFetcher(): 38 | """Provides counting of URL fetches for a particular task. 39 | 40 | """ 41 | 42 | def __init__(self): 43 | self.fetch_counter = 0 44 | 45 | async def fetch(self, session, url): 46 | """Fetch a URL using aiohttp returning parsed JSON response. 47 | As suggested by the aiohttp docs we reuse the session. 48 | 49 | """ 50 | with async_timeout.timeout(FETCH_TIMEOUT): 51 | self.fetch_counter += 1 52 | async with session.get(url) as response: 53 | return await response.json() 54 | 55 | 56 | async def post_number_of_comments(loop, session, fetcher, post_id): 57 | """Retrieve data for current post and recursively for all comments. 58 | 59 | """ 60 | url = URL_TEMPLATE.format(post_id) 61 | response = await fetcher.fetch(session, url) 62 | 63 | # base case, there are no comments 64 | if response is None or 'kids' not in response: 65 | return 0 66 | 67 | # calculate this post's comments as number of comments 68 | number_of_comments = len(response['kids']) 69 | 70 | # create recursive tasks for all comments 71 | tasks = [post_number_of_comments( 72 | loop, session, fetcher, kid_id) for kid_id in response['kids']] 73 | 74 | # schedule the tasks and retrieve results 75 | results = await asyncio.gather(*tasks) 76 | 77 | # reduce the descendents comments and add it to this post's 78 | number_of_comments += sum(results) 79 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 80 | 81 | return number_of_comments 82 | 83 | 84 | async def get_comments_of_top_stories(loop, limit, iteration): 85 | """Retrieve top stories in HN. 86 | 87 | """ 88 | async with aiohttp.ClientSession(loop=loop) as session: 89 | fetcher = URLFetcher() # create a new fetcher for this task 90 | response = await fetcher.fetch(session, TOP_STORIES_URL) 91 | tasks = [post_number_of_comments( 92 | loop, session, fetcher, post_id) for post_id in response[:limit]] 93 | results = await asyncio.gather(*tasks) 94 | for post_id, num_comments in zip(response[:limit], results): 95 | log.info("Post {} has {} comments ({})".format( 96 | post_id, num_comments, iteration)) 97 | return fetcher.fetch_counter # return the fetch count 98 | 99 | 100 | def poll_top_stories_for_comments(loop, period, limit, iteration=0): 101 | """Periodic function that schedules get_comments_of_top_stories. 102 | 103 | """ 104 | log.info("Calculating comments for top {} stories ({})".format( 105 | limit, iteration)) 106 | 107 | future = asyncio.ensure_future( 108 | get_comments_of_top_stories(loop, limit, iteration)) 109 | 110 | now = datetime.now() 111 | 112 | def callback(fut): 113 | fetch_count = fut.result() 114 | log.info( 115 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 116 | (datetime.now() - now).total_seconds(), fetch_count)) 117 | 118 | future.add_done_callback(callback) 119 | 120 | log.info("Waiting for {} seconds...".format(period)) 121 | 122 | iteration += 1 123 | loop.call_later( 124 | period, 125 | partial( # or call_at(loop.time() + period) 126 | poll_top_stories_for_comments, 127 | loop, period, limit, iteration 128 | ) 129 | ) 130 | 131 | 132 | if __name__ == '__main__': 133 | args = parser.parse_args() 134 | if args.verbose: 135 | log.setLevel(logging.DEBUG) 136 | 137 | loop = asyncio.get_event_loop() 138 | 139 | # we don't `run_until_complete` anymore, we simply call the function 140 | poll_top_stories_for_comments(loop, args.period, args.limit) 141 | 142 | # and run the loop forever 143 | loop.run_forever() 144 | 145 | loop.close() 146 | -------------------------------------------------------------------------------- /03_periodic_coroutines/05_periodic_coroutines.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using an infinite loop of 3 | scheduling a task using ensure_future and sleeping. 4 | 5 | Introduced an artificial error to showcase asyncio's error messages. 6 | 7 | """ 8 | 9 | import asyncio 10 | import argparse 11 | import logging 12 | from datetime import datetime 13 | from functools import partial 14 | 15 | import aiohttp 16 | import async_timeout 17 | 18 | 19 | LOGGER_FORMAT = '%(asctime)s %(message)s' 20 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 21 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 22 | FETCH_TIMEOUT = 10 23 | MAXIMUM_FETCHES = 5 24 | 25 | parser = argparse.ArgumentParser( 26 | description='Calculate the number of comments of the top stories in HN.') 27 | parser.add_argument( 28 | '--period', type=int, default=5, help='Number of seconds between poll') 29 | parser.add_argument( 30 | '--limit', type=int, default=5, 31 | help='Number of new stories to calculate comments for') 32 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 33 | 34 | 35 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 36 | log = logging.getLogger() 37 | log.setLevel(logging.INFO) 38 | 39 | 40 | class URLFetcher(): 41 | """Provides counting of URL fetches for a particular task. 42 | 43 | """ 44 | 45 | def __init__(self): 46 | self.fetch_counter = 0 47 | 48 | async def fetch(self, session, url): 49 | """Fetch a URL using aiohttp returning parsed JSON response. 50 | 51 | As suggested by the aiohttp docs we reuse the session. 52 | 53 | """ 54 | with async_timeout.timeout(FETCH_TIMEOUT): 55 | self.fetch_counter += 1 56 | if self.fetch_counter > MAXIMUM_FETCHES: 57 | raise Exception('BOOM!') 58 | 59 | async with session.get(url) as response: 60 | return await response.json() 61 | 62 | 63 | async def post_number_of_comments(loop, session, fetcher, post_id): 64 | """Retrieve data for current post and recursively for all comments. 65 | 66 | """ 67 | url = URL_TEMPLATE.format(post_id) 68 | response = await fetcher.fetch(session, url) 69 | 70 | # base case, there are no comments 71 | if response is None or 'kids' not in response: 72 | return 0 73 | 74 | # calculate this post's comments as number of comments 75 | number_of_comments = len(response['kids']) 76 | 77 | # create recursive tasks for all comments 78 | tasks = [post_number_of_comments( 79 | loop, session, fetcher, kid_id) for kid_id in response['kids']] 80 | 81 | # schedule the tasks and retrieve results 82 | results = await asyncio.gather(*tasks) 83 | 84 | # reduce the descendents comments and add it to this post's 85 | number_of_comments += sum(results) 86 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 87 | 88 | return number_of_comments 89 | 90 | 91 | async def get_comments_of_top_stories(loop, limit, iteration): 92 | """Retrieve top stories in HN. 93 | 94 | """ 95 | async with aiohttp.ClientSession(loop=loop) as session: 96 | fetcher = URLFetcher() # create a new fetcher for this task 97 | response = await fetcher.fetch(session, TOP_STORIES_URL) 98 | tasks = [post_number_of_comments( 99 | loop, session, fetcher, post_id) for post_id in response[:limit]] 100 | results = await asyncio.gather(*tasks) 101 | for post_id, num_comments in zip(response[:limit], results): 102 | log.info("Post {} has {} comments ({})".format( 103 | post_id, num_comments, iteration)) 104 | return fetcher.fetch_counter # return the fetch count 105 | 106 | 107 | def poll_top_stories_for_comments(loop, period, limit, iteration=0): 108 | """Periodic function that schedules get_comments_of_top_stories. 109 | 110 | """ 111 | log.info("Calculating comments for top {} stories ({})".format( 112 | limit, iteration)) 113 | 114 | future = asyncio.ensure_future( 115 | get_comments_of_top_stories(loop, limit, iteration)) 116 | 117 | now = datetime.now() 118 | 119 | def callback(fut): 120 | fetch_count = fut.result() 121 | log.info( 122 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 123 | (datetime.now() - now).total_seconds(), fetch_count)) 124 | 125 | future.add_done_callback(callback) 126 | 127 | log.info("Waiting for {} seconds...".format(period)) 128 | 129 | iteration += 1 130 | loop.call_later( 131 | period, 132 | partial( # or call_at(loop.time() + period) 133 | poll_top_stories_for_comments, 134 | loop, period, limit, iteration 135 | ) 136 | ) 137 | 138 | 139 | if __name__ == '__main__': 140 | args = parser.parse_args() 141 | if args.verbose: 142 | log.setLevel(logging.DEBUG) 143 | 144 | loop = asyncio.get_event_loop() 145 | 146 | # we don't `run_until_complete` anymore, we simply call the function 147 | poll_top_stories_for_comments(loop, args.period, args.limit) 148 | 149 | # and run the loop forever 150 | loop.run_forever() 151 | 152 | loop.close() 153 | -------------------------------------------------------------------------------- /04_error_handling/01_error_handling.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using an infinite loop of 3 | scheduling a task using ensure_future and sleeping. 4 | 5 | Artificially produce an error on URLFetcher. 6 | 7 | """ 8 | 9 | import asyncio 10 | import argparse 11 | import logging 12 | from datetime import datetime 13 | 14 | import aiohttp 15 | import async_timeout 16 | 17 | 18 | LOGGER_FORMAT = '%(asctime)s %(message)s' 19 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 20 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 21 | FETCH_TIMEOUT = 10 22 | MAXIMUM_FETCHES = 5 23 | 24 | parser = argparse.ArgumentParser( 25 | description='Calculate the number of comments of the top stories in HN.') 26 | parser.add_argument( 27 | '--period', type=int, default=5, help='Number of seconds between poll') 28 | parser.add_argument( 29 | '--limit', type=int, default=5, 30 | help='Number of new stories to calculate comments for') 31 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 32 | 33 | 34 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 35 | log = logging.getLogger() 36 | log.setLevel(logging.INFO) 37 | 38 | 39 | class BoomException(Exception): 40 | pass 41 | 42 | 43 | class URLFetcher(): 44 | """Provides counting of URL fetches for a particular task. 45 | 46 | """ 47 | 48 | def __init__(self): 49 | self.fetch_counter = 0 50 | 51 | async def fetch(self, session, url): 52 | """Fetch a URL using aiohttp returning parsed JSON response. 53 | 54 | As suggested by the aiohttp docs we reuse the session. 55 | 56 | """ 57 | with async_timeout.timeout(FETCH_TIMEOUT): 58 | self.fetch_counter += 1 59 | if self.fetch_counter > MAXIMUM_FETCHES: 60 | raise BoomException('BOOM!') 61 | 62 | async with session.get(url) as response: 63 | return await response.json() 64 | 65 | 66 | async def post_number_of_comments(loop, session, fetcher, post_id): 67 | """Retrieve data for current post and recursively for all comments. 68 | 69 | """ 70 | url = URL_TEMPLATE.format(post_id) 71 | response = await fetcher.fetch(session, url) 72 | 73 | # base case, there are no comments 74 | if response is None or 'kids' not in response: 75 | return 0 76 | 77 | # calculate this post's comments as number of comments 78 | number_of_comments = len(response['kids']) 79 | 80 | # create recursive tasks for all comments 81 | tasks = [post_number_of_comments( 82 | loop, session, fetcher, kid_id) for kid_id in response['kids']] 83 | 84 | # schedule the tasks and retrieve results 85 | results = await asyncio.gather(*tasks) 86 | 87 | # reduce the descendents comments and add it to this post's 88 | number_of_comments += sum(results) 89 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 90 | 91 | return number_of_comments 92 | 93 | 94 | async def get_comments_of_top_stories(loop, session, limit, iteration): 95 | """Retrieve top stories in HN. 96 | 97 | """ 98 | fetcher = URLFetcher() # create a new fetcher for this task 99 | response = await fetcher.fetch(session, TOP_STORIES_URL) 100 | 101 | tasks = [post_number_of_comments( 102 | loop, session, fetcher, post_id) for post_id in response[:limit]] 103 | 104 | results = await asyncio.gather(*tasks) 105 | 106 | for post_id, num_comments in zip(response[:limit], results): 107 | log.info("Post {} has {} comments ({})".format( 108 | post_id, num_comments, iteration)) 109 | return fetcher.fetch_counter # return the fetch count 110 | 111 | 112 | async def poll_top_stories_for_comments(loop, session, period, limit): 113 | """Periodically poll for new stories and retrieve number of comments. 114 | 115 | """ 116 | iteration = 1 117 | while True: 118 | log.info("Calculating comments for top {} stories. ({})".format( 119 | limit, iteration)) 120 | 121 | future = asyncio.ensure_future( 122 | get_comments_of_top_stories(loop, session, limit, iteration)) 123 | 124 | now = datetime.now() 125 | 126 | def callback(fut): 127 | fetch_count = fut.result() 128 | log.info( 129 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 130 | (datetime.now() - now).total_seconds(), fetch_count)) 131 | 132 | future.add_done_callback(callback) 133 | 134 | log.info("Waiting for {} seconds...".format(period)) 135 | iteration += 1 136 | await asyncio.sleep(period) 137 | 138 | 139 | async def main(loop, period, limit): 140 | """Async entry point coroutine. 141 | 142 | """ 143 | async with aiohttp.ClientSession(loop=loop) as session: 144 | comments = await poll_top_stories_for_comments(loop, session, period, limit) 145 | 146 | return comments 147 | 148 | 149 | if __name__ == '__main__': 150 | args = parser.parse_args() 151 | if args.verbose: 152 | log.setLevel(logging.DEBUG) 153 | 154 | loop = asyncio.get_event_loop() 155 | loop.run_until_complete(main(loop, args.period, args.limit)) 156 | 157 | loop.close() 158 | -------------------------------------------------------------------------------- /04_error_handling/01b_error_handling.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using an infinite loop of 3 | scheduling a task using ensure_future and sleeping. 4 | 5 | Artificially produce an error on URLFetcher. 6 | 7 | """ 8 | 9 | import asyncio 10 | import argparse 11 | import logging 12 | from datetime import datetime 13 | 14 | import aiohttp 15 | import async_timeout 16 | 17 | 18 | LOGGER_FORMAT = '%(asctime)s %(message)s' 19 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 20 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 21 | FETCH_TIMEOUT = 10 22 | MAXIMUM_FETCHES = 5 23 | 24 | parser = argparse.ArgumentParser( 25 | description='Calculate the number of comments of the top stories in HN.') 26 | parser.add_argument( 27 | '--period', type=int, default=5, help='Number of seconds between poll') 28 | parser.add_argument( 29 | '--limit', type=int, default=5, 30 | help='Number of new stories to calculate comments for') 31 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 32 | 33 | 34 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 35 | log = logging.getLogger() 36 | log.setLevel(logging.INFO) 37 | 38 | 39 | class BoomException(Exception): 40 | pass 41 | 42 | 43 | class URLFetcher(): 44 | """Provides counting of URL fetches for a particular task. 45 | 46 | """ 47 | 48 | def __init__(self): 49 | self.fetch_counter = 0 50 | 51 | async def fetch(self, session, url): 52 | """Fetch a URL using aiohttp returning parsed JSON response. 53 | 54 | As suggested by the aiohttp docs we reuse the session. 55 | 56 | """ 57 | with async_timeout.timeout(FETCH_TIMEOUT): 58 | self.fetch_counter += 1 59 | if self.fetch_counter > MAXIMUM_FETCHES: 60 | raise BoomException('BOOM!') 61 | 62 | async with session.get(url) as response: 63 | return await response.json() 64 | 65 | 66 | async def post_number_of_comments(loop, session, fetcher, post_id): 67 | """Retrieve data for current post and recursively for all comments. 68 | 69 | """ 70 | url = URL_TEMPLATE.format(post_id) 71 | response = await fetcher.fetch(session, url) 72 | 73 | # base case, there are no comments 74 | if response is None or 'kids' not in response: 75 | return 0 76 | 77 | # calculate this post's comments as number of comments 78 | number_of_comments = len(response['kids']) 79 | 80 | # create recursive tasks for all comments 81 | tasks = [post_number_of_comments( 82 | loop, session, fetcher, kid_id) for kid_id in response['kids']] 83 | 84 | # schedule the tasks and retrieve results 85 | results = await asyncio.gather(*tasks) 86 | 87 | # reduce the descendents comments and add it to this post's 88 | number_of_comments += sum(results) 89 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 90 | 91 | return number_of_comments 92 | 93 | 94 | async def get_comments_of_top_stories(loop, session, limit, iteration): 95 | """Retrieve top stories in HN. 96 | 97 | """ 98 | fetcher = URLFetcher() # create a new fetcher for this task 99 | response = await fetcher.fetch(session, TOP_STORIES_URL) 100 | 101 | tasks = [post_number_of_comments( 102 | loop, session, fetcher, post_id) for post_id in response[:limit]] 103 | 104 | results = await asyncio.gather(*tasks) 105 | 106 | for post_id, num_comments in zip(response[:limit], results): 107 | log.info("Post {} has {} comments ({})".format( 108 | post_id, num_comments, iteration)) 109 | return fetcher.fetch_counter # return the fetch count 110 | 111 | 112 | async def poll_top_stories_for_comments(loop, session, period, limit): 113 | """Periodically poll for new stories and retrieve number of comments. 114 | 115 | """ 116 | iteration = 1 117 | while True: 118 | log.info("Calculating comments for top {} stories. ({})".format( 119 | limit, iteration)) 120 | 121 | future = asyncio.ensure_future( 122 | get_comments_of_top_stories(loop, session, limit, iteration)) 123 | 124 | now = datetime.now() 125 | 126 | def callback(fut): 127 | try: 128 | fetch_count = fut.result() 129 | except BoomException as e: 130 | log.exception("Something went BOOM") 131 | return 132 | 133 | log.info( 134 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 135 | (datetime.now() - now).total_seconds(), fetch_count)) 136 | 137 | future.add_done_callback(callback) 138 | 139 | log.info("Waiting for {} seconds...".format(period)) 140 | iteration += 1 141 | await asyncio.sleep(period) 142 | 143 | 144 | async def main(loop, period, limit): 145 | """Async entry point coroutine. 146 | 147 | """ 148 | async with aiohttp.ClientSession(loop=loop) as session: 149 | comments = await poll_top_stories_for_comments(loop, session, period, limit) 150 | 151 | return comments 152 | 153 | 154 | if __name__ == '__main__': 155 | args = parser.parse_args() 156 | if args.verbose: 157 | log.setLevel(logging.DEBUG) 158 | 159 | loop = asyncio.get_event_loop() 160 | loop.run_until_complete(main(loop, args.period, args.limit)) 161 | 162 | loop.close() 163 | -------------------------------------------------------------------------------- /04_error_handling/01c_error_handling.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using an infinite loop of 3 | scheduling a task using ensure_future and sleeping. 4 | 5 | Artificially produce an error on URLFetcher. 6 | 7 | """ 8 | 9 | import asyncio 10 | import argparse 11 | import logging 12 | from random import randint 13 | from datetime import datetime 14 | 15 | import aiohttp 16 | import async_timeout 17 | 18 | 19 | LOGGER_FORMAT = '%(asctime)s %(message)s' 20 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 21 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 22 | FETCH_TIMEOUT = 10 23 | MAXIMUM_FETCHES = 5 24 | 25 | parser = argparse.ArgumentParser( 26 | description='Calculate the number of comments of the top stories in HN.') 27 | parser.add_argument( 28 | '--period', type=int, default=5, help='Number of seconds between poll') 29 | parser.add_argument( 30 | '--limit', type=int, default=5, 31 | help='Number of new stories to calculate comments for') 32 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 33 | 34 | 35 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 36 | log = logging.getLogger() 37 | log.setLevel(logging.INFO) 38 | 39 | 40 | class BoomException(Exception): 41 | pass 42 | 43 | 44 | class URLFetcher(): 45 | """Provides counting of URL fetches for a particular task. 46 | 47 | """ 48 | 49 | def __init__(self): 50 | self.fetch_counter = 0 51 | 52 | async def fetch(self, session, url): 53 | """Fetch a URL using aiohttp returning parsed JSON response. 54 | 55 | As suggested by the aiohttp docs we reuse the session. 56 | 57 | """ 58 | with async_timeout.timeout(FETCH_TIMEOUT): 59 | self.fetch_counter += 1 60 | if self.fetch_counter > MAXIMUM_FETCHES: 61 | raise BoomException('BOOM!') 62 | elif randint(0, 3) == 0: 63 | raise Exception('Random generic exception') 64 | 65 | async with session.get(url) as response: 66 | return await response.json() 67 | 68 | 69 | async def post_number_of_comments(loop, session, fetcher, post_id): 70 | """Retrieve data for current post and recursively for all comments. 71 | 72 | """ 73 | url = URL_TEMPLATE.format(post_id) 74 | response = await fetcher.fetch(session, url) 75 | 76 | # base case, there are no comments 77 | if response is None or 'kids' not in response: 78 | return 0 79 | 80 | # calculate this post's comments as number of comments 81 | number_of_comments = len(response['kids']) 82 | 83 | # create recursive tasks for all comments 84 | tasks = [post_number_of_comments( 85 | loop, session, fetcher, kid_id) for kid_id in response['kids']] 86 | 87 | # schedule the tasks and retrieve results 88 | results = await asyncio.gather(*tasks) 89 | 90 | # reduce the descendents comments and add it to this post's 91 | number_of_comments += sum(results) 92 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 93 | 94 | return number_of_comments 95 | 96 | 97 | async def get_comments_of_top_stories(loop, session, limit, iteration): 98 | """Retrieve top stories in HN. 99 | 100 | """ 101 | fetcher = URLFetcher() # create a new fetcher for this task 102 | response = await fetcher.fetch(session, TOP_STORIES_URL) 103 | 104 | tasks = [post_number_of_comments( 105 | loop, session, fetcher, post_id) for post_id in response[:limit]] 106 | 107 | results = await asyncio.gather(*tasks) 108 | 109 | for post_id, num_comments in zip(response[:limit], results): 110 | log.info("Post {} has {} comments ({})".format( 111 | post_id, num_comments, iteration)) 112 | return fetcher.fetch_counter # return the fetch count 113 | 114 | 115 | async def poll_top_stories_for_comments(loop, session, period, limit): 116 | """Periodically poll for new stories and retrieve number of comments. 117 | 118 | """ 119 | iteration = 1 120 | while True: 121 | log.info("Calculating comments for top {} stories. ({})".format( 122 | limit, iteration)) 123 | 124 | future = asyncio.ensure_future( 125 | get_comments_of_top_stories(loop, session, limit, iteration)) 126 | 127 | now = datetime.now() 128 | 129 | def callback(fut): 130 | try: 131 | fetch_count = fut.result() 132 | except BoomException as e: 133 | log.exception("Something went BOOM") 134 | return 135 | 136 | log.info( 137 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 138 | (datetime.now() - now).total_seconds(), fetch_count)) 139 | 140 | future.add_done_callback(callback) 141 | 142 | log.info("Waiting for {} seconds...".format(period)) 143 | iteration += 1 144 | await asyncio.sleep(period) 145 | 146 | 147 | async def main(loop, period, limit): 148 | """Async entry point coroutine. 149 | 150 | """ 151 | async with aiohttp.ClientSession(loop=loop) as session: 152 | comments = await poll_top_stories_for_comments(loop, session, period, limit) 153 | 154 | return comments 155 | 156 | 157 | if __name__ == '__main__': 158 | args = parser.parse_args() 159 | if args.verbose: 160 | log.setLevel(logging.DEBUG) 161 | 162 | loop = asyncio.get_event_loop() 163 | loop.run_until_complete(main(loop, args.period, args.limit)) 164 | 165 | loop.close() 166 | -------------------------------------------------------------------------------- /04_error_handling/02_error_handling.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using an infinite loop of 3 | scheduling a task using ensure_future and sleeping. 4 | 5 | Artificially produce an error and use try..except clauses to catch them. 6 | 7 | """ 8 | 9 | import asyncio 10 | import argparse 11 | import logging 12 | from datetime import datetime 13 | from random import randint 14 | 15 | import aiohttp 16 | import async_timeout 17 | 18 | 19 | LOGGER_FORMAT = '%(asctime)s %(message)s' 20 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 21 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 22 | FETCH_TIMEOUT = 10 23 | MAXIMUM_FETCHES = 5 24 | 25 | parser = argparse.ArgumentParser( 26 | description='Calculate the number of comments of the top stories in HN.') 27 | parser.add_argument( 28 | '--period', type=int, default=5, help='Number of seconds between poll') 29 | parser.add_argument( 30 | '--limit', type=int, default=5, 31 | help='Number of new stories to calculate comments for') 32 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 33 | 34 | 35 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 36 | log = logging.getLogger() 37 | log.setLevel(logging.INFO) 38 | 39 | 40 | class BoomException(Exception): 41 | pass 42 | 43 | 44 | class URLFetcher(): 45 | """Provides counting of URL fetches for a particular task. 46 | 47 | """ 48 | 49 | def __init__(self): 50 | self.fetch_counter = 0 51 | 52 | async def fetch(self, session, url): 53 | """Fetch a URL using aiohttp returning parsed JSON response. 54 | 55 | As suggested by the aiohttp docs we reuse the session. 56 | 57 | """ 58 | with async_timeout.timeout(FETCH_TIMEOUT): 59 | self.fetch_counter += 1 60 | if self.fetch_counter > MAXIMUM_FETCHES: 61 | raise BoomException('BOOM!') 62 | elif randint(0, 3) == 0: 63 | raise Exception('Random generic exception') 64 | 65 | async with session.get(url) as response: 66 | return await response.json() 67 | 68 | 69 | async def post_number_of_comments(loop, session, fetcher, post_id): 70 | """Retrieve data for current post and recursively for all comments. 71 | 72 | """ 73 | url = URL_TEMPLATE.format(post_id) 74 | try: 75 | response = await fetcher.fetch(session, url) 76 | except BoomException as e: 77 | log.error("Error retrieving post {}: {}".format(post_id, e)) 78 | raise e 79 | 80 | # base case, there are no comments 81 | if response is None or 'kids' not in response: 82 | return 0 83 | 84 | # calculate this post's comments as number of comments 85 | number_of_comments = len(response['kids']) 86 | 87 | # create recursive tasks for all comments 88 | tasks = [post_number_of_comments( 89 | loop, session, fetcher, kid_id) for kid_id in response['kids']] 90 | 91 | # schedule the tasks and retrieve results 92 | try: 93 | results = await asyncio.gather(*tasks) 94 | except BoomException as e: 95 | log.error("Error retrieving comments for top stories: {}".format(e)) 96 | raise 97 | 98 | # reduce the descendents comments and add it to this post's 99 | number_of_comments += sum(results) 100 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 101 | 102 | return number_of_comments 103 | 104 | 105 | async def get_comments_of_top_stories(loop, session, limit, iteration): 106 | """Retrieve top stories in HN. 107 | 108 | """ 109 | fetcher = URLFetcher() # create a new fetcher for this task 110 | try: 111 | response = await fetcher.fetch(session, TOP_STORIES_URL) 112 | except Exception as e: 113 | log.error("Error retrieving top stories: {}".format(e)) 114 | raise 115 | 116 | tasks = [post_number_of_comments( 117 | loop, session, fetcher, post_id) for post_id in response[:limit]] 118 | 119 | try: 120 | results = await asyncio.gather(*tasks) 121 | except BoomException as e: 122 | log.error("Error retrieving comments for top stories: {}".format(e)) 123 | raise 124 | 125 | for post_id, num_comments in zip(response[:limit], results): 126 | log.info("Post {} has {} comments ({})".format( 127 | post_id, num_comments, iteration)) 128 | return fetcher.fetch_counter # return the fetch count 129 | 130 | 131 | async def poll_top_stories_for_comments(loop, session, period, limit): 132 | """Periodically poll for new stories and retrieve number of comments. 133 | 134 | """ 135 | iteration = 1 136 | while True: 137 | log.info("Calculating comments for top {} stories. ({})".format( 138 | limit, iteration)) 139 | 140 | future = asyncio.ensure_future( 141 | get_comments_of_top_stories(loop, session, limit, iteration)) 142 | 143 | now = datetime.now() 144 | 145 | def callback(fut): 146 | try: 147 | fetch_count = fut.result() 148 | except BoomException: 149 | pass # a handled exception 150 | except Exception as e: 151 | log.exception('Unexpected error') 152 | else: 153 | log.info( 154 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 155 | (datetime.now() - now).total_seconds(), fetch_count)) 156 | 157 | future.add_done_callback(callback) 158 | 159 | log.info("Waiting for {} seconds...".format(period)) 160 | iteration += 1 161 | await asyncio.sleep(period) 162 | 163 | 164 | async def main(loop, period, limit): 165 | """Async entry point coroutine. 166 | 167 | """ 168 | async with aiohttp.ClientSession(loop=loop) as session: 169 | comments = await poll_top_stories_for_comments(loop, session, period, limit) 170 | 171 | return comments 172 | 173 | 174 | if __name__ == '__main__': 175 | args = parser.parse_args() 176 | if args.verbose: 177 | log.setLevel(logging.DEBUG) 178 | 179 | loop = asyncio.get_event_loop() 180 | loop.run_until_complete(main(loop, args.period, args.limit)) 181 | 182 | loop.close() 183 | -------------------------------------------------------------------------------- /04_error_handling/02b_error_handling.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using an infinite loop of 3 | scheduling a task using ensure_future and sleeping. 4 | 5 | Artificially produce an error and use try..except clauses to catch them. 6 | 7 | """ 8 | 9 | import asyncio 10 | import argparse 11 | import logging 12 | from functools import partial 13 | from datetime import datetime 14 | from random import randint 15 | 16 | import aiohttp 17 | import async_timeout 18 | 19 | 20 | LOGGER_FORMAT = '%(asctime)s %(message)s' 21 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 22 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 23 | FETCH_TIMEOUT = 10 24 | MAXIMUM_FETCHES = 5 25 | 26 | parser = argparse.ArgumentParser( 27 | description='Calculate the number of comments of the top stories in HN.') 28 | parser.add_argument( 29 | '--period', type=int, default=5, help='Number of seconds between poll') 30 | parser.add_argument( 31 | '--limit', type=int, default=5, 32 | help='Number of new stories to calculate comments for') 33 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 34 | 35 | 36 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 37 | log = logging.getLogger() 38 | log.setLevel(logging.INFO) 39 | 40 | 41 | class BoomException(Exception): 42 | pass 43 | 44 | 45 | class URLFetcher(): 46 | """Provides counting of URL fetches for a particular task. 47 | 48 | """ 49 | 50 | def __init__(self): 51 | self.fetch_counter = 0 52 | 53 | async def fetch(self, session, url): 54 | """Fetch a URL using aiohttp returning parsed JSON response. 55 | 56 | As suggested by the aiohttp docs we reuse the session. 57 | 58 | """ 59 | with async_timeout.timeout(FETCH_TIMEOUT): 60 | self.fetch_counter += 1 61 | if self.fetch_counter > MAXIMUM_FETCHES: 62 | raise BoomException('BOOM!') 63 | elif randint(0, 3) == 0: 64 | raise Exception('Random generic exception') 65 | 66 | async with session.get(url) as response: 67 | return await response.json() 68 | 69 | 70 | async def post_number_of_comments(loop, session, fetcher, post_id): 71 | """Retrieve data for current post and recursively for all comments. 72 | 73 | """ 74 | url = URL_TEMPLATE.format(post_id) 75 | try: 76 | response = await fetcher.fetch(session, url) 77 | except BoomException as e: 78 | log.info("Error retrieving post {}: {}".format(post_id, e)) 79 | raise e 80 | 81 | # base case, there are no comments 82 | if response is None or 'kids' not in response: 83 | return 0 84 | 85 | # calculate this post's comments as number of comments 86 | number_of_comments = len(response['kids']) 87 | 88 | # create recursive tasks for all comments 89 | tasks = [post_number_of_comments( 90 | loop, session, fetcher, kid_id) for kid_id in response['kids']] 91 | 92 | # schedule the tasks and retrieve results 93 | try: 94 | results = await asyncio.gather(*tasks) 95 | except BoomException as e: 96 | log.info("Error retrieving comments for top stories: {}".format(e)) 97 | raise 98 | 99 | # reduce the descendents comments and add it to this post's 100 | number_of_comments += sum(results) 101 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 102 | 103 | return number_of_comments 104 | 105 | 106 | async def get_comments_of_top_stories(loop, session, limit, iteration): 107 | """Retrieve top stories in HN. 108 | 109 | """ 110 | fetcher = URLFetcher() # create a new fetcher for this task 111 | try: 112 | response = await fetcher.fetch(session, TOP_STORIES_URL) 113 | except Exception as e: 114 | log.info("Error retrieving top stories: {}".format(e)) 115 | raise 116 | 117 | tasks = [post_number_of_comments( 118 | loop, session, fetcher, post_id) for post_id in response[:limit]] 119 | 120 | try: 121 | results = await asyncio.gather(*tasks) 122 | except Exception as e: 123 | log.info("Error retrieving comments for top stories: {}".format(e)) 124 | raise 125 | 126 | for post_id, num_comments in zip(response[:limit], results): 127 | log.info("Post {} has {} comments ({})".format( 128 | post_id, num_comments, iteration)) 129 | return fetcher.fetch_counter # return the fetch count 130 | 131 | 132 | async def poll_top_stories_for_comments(loop, session, period, limit): 133 | """Periodically poll for new stories and retrieve number of comments. 134 | 135 | """ 136 | iteration = 1 137 | while True: 138 | log.info("Calculating comments for top {} stories. ({})".format( 139 | limit, iteration)) 140 | 141 | future = asyncio.ensure_future( 142 | get_comments_of_top_stories(loop, session, limit, iteration)) 143 | 144 | now = datetime.now() 145 | 146 | def callback(fut): 147 | try: 148 | fetch_count = fut.result() 149 | except BoomException: 150 | loop.stop() 151 | except Exception as e: 152 | log.exception('Unexpected error') 153 | loop.stop() 154 | else: 155 | log.info( 156 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 157 | (datetime.now() - now).total_seconds(), fetch_count)) 158 | 159 | future.add_done_callback(callback) 160 | 161 | log.info("Waiting for {} seconds...".format(period)) 162 | iteration += 1 163 | await asyncio.sleep(period) 164 | 165 | 166 | async def main(loop, period, limit): 167 | """Async entry point coroutine. 168 | 169 | """ 170 | async with aiohttp.ClientSession(loop=loop) as session: 171 | comments = await poll_top_stories_for_comments(loop, session, period, limit) 172 | 173 | return comments 174 | 175 | 176 | if __name__ == '__main__': 177 | args = parser.parse_args() 178 | if args.verbose: 179 | log.setLevel(logging.DEBUG) 180 | 181 | loop = asyncio.get_event_loop() 182 | loop.run_until_complete(main(loop, args.period, args.limit)) 183 | 184 | loop.close() 185 | -------------------------------------------------------------------------------- /04_error_handling/02c_error_handling.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using an infinite loop of 3 | scheduling a task using ensure_future and sleeping. 4 | 5 | Artificially produce an error and use try..except clauses to catch them. 6 | 7 | """ 8 | 9 | import asyncio 10 | import argparse 11 | import logging 12 | from functools import partial 13 | from datetime import datetime 14 | from random import randint 15 | 16 | import aiohttp 17 | import async_timeout 18 | 19 | 20 | LOGGER_FORMAT = '%(asctime)s %(message)s' 21 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 22 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 23 | FETCH_TIMEOUT = 10 24 | MAXIMUM_FETCHES = 5 25 | 26 | parser = argparse.ArgumentParser( 27 | description='Calculate the number of comments of the top stories in HN.') 28 | parser.add_argument( 29 | '--period', type=int, default=5, help='Number of seconds between poll') 30 | parser.add_argument( 31 | '--limit', type=int, default=5, 32 | help='Number of new stories to calculate comments for') 33 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 34 | 35 | 36 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 37 | log = logging.getLogger() 38 | log.setLevel(logging.INFO) 39 | 40 | 41 | class BoomException(Exception): 42 | pass 43 | 44 | 45 | class URLFetcher(): 46 | """Provides counting of URL fetches for a particular task. 47 | 48 | """ 49 | 50 | def __init__(self): 51 | self.fetch_counter = 0 52 | 53 | async def fetch(self, session, url): 54 | """Fetch a URL using aiohttp returning parsed JSON response. 55 | 56 | As suggested by the aiohttp docs we reuse the session. 57 | 58 | """ 59 | with async_timeout.timeout(FETCH_TIMEOUT): 60 | self.fetch_counter += 1 61 | if self.fetch_counter > MAXIMUM_FETCHES: 62 | raise BoomException('BOOM!') 63 | elif randint(0, 3) == 0: 64 | raise Exception('Random generic exception') 65 | 66 | async with session.get(url) as response: 67 | return await response.json() 68 | 69 | 70 | async def post_number_of_comments(loop, session, fetcher, post_id): 71 | """Retrieve data for current post and recursively for all comments. 72 | 73 | """ 74 | url = URL_TEMPLATE.format(post_id) 75 | try: 76 | response = await fetcher.fetch(session, url) 77 | except BoomException as e: 78 | log.error("Error retrieving post {}: {}".format(post_id, e)) 79 | raise e 80 | 81 | # base case, there are no comments 82 | if response is None or 'kids' not in response: 83 | return 0 84 | 85 | # calculate this post's comments as number of comments 86 | number_of_comments = len(response['kids']) 87 | 88 | # create recursive tasks for all comments 89 | tasks = [post_number_of_comments( 90 | loop, session, fetcher, kid_id) for kid_id in response['kids']] 91 | 92 | # schedule the tasks and retrieve results 93 | try: 94 | results = await asyncio.gather(*tasks) 95 | except BoomException as e: 96 | log.error("Error retrieving comments for top stories: {}".format(e)) 97 | raise 98 | 99 | # reduce the descendents comments and add it to this post's 100 | number_of_comments += sum(results) 101 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 102 | 103 | return number_of_comments 104 | 105 | 106 | async def get_comments_of_top_stories(loop, session, limit, iteration): 107 | """Retrieve top stories in HN. 108 | 109 | """ 110 | fetcher = URLFetcher() # create a new fetcher for this task 111 | try: 112 | response = await fetcher.fetch(session, TOP_STORIES_URL) 113 | except Exception as e: 114 | log.error("Error retrieving top stories: {}".format(e)) 115 | raise 116 | 117 | tasks = [post_number_of_comments( 118 | loop, session, fetcher, post_id) for post_id in response[:limit]] 119 | 120 | try: 121 | results = await asyncio.gather(*tasks) 122 | except Exception as e: 123 | log.error("Error retrieving comments for top stories: {}".format(e)) 124 | raise 125 | 126 | for post_id, num_comments in zip(response[:limit], results): 127 | log.info("Post {} has {} comments ({})".format( 128 | post_id, num_comments, iteration)) 129 | return fetcher.fetch_counter # return the fetch count 130 | 131 | 132 | async def poll_top_stories_for_comments(loop, session, period, limit): 133 | """Periodically poll for new stories and retrieve number of comments. 134 | 135 | """ 136 | iteration = 1 137 | errors = [] 138 | while True: 139 | if errors: 140 | log.info('Error detected, quitting') 141 | return 142 | 143 | log.info("Calculating comments for top {} stories. ({})".format( 144 | limit, iteration)) 145 | 146 | future = asyncio.ensure_future( 147 | get_comments_of_top_stories(loop, session, limit, iteration)) 148 | 149 | now = datetime.now() 150 | 151 | def callback(fut): 152 | try: 153 | fetch_count = fut.result() 154 | except BoomException as e: 155 | log.debug('Adding {} to errors'.format(e)) 156 | errors.append(e) 157 | except Exception as e: 158 | log.exception('Unexpected error') 159 | errors.append(e) 160 | else: 161 | log.info( 162 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 163 | (datetime.now() - now).total_seconds(), fetch_count)) 164 | 165 | future.add_done_callback(callback) 166 | 167 | log.info("Waiting for {} seconds...".format(period)) 168 | iteration += 1 169 | await asyncio.sleep(period) 170 | 171 | 172 | async def main(loop, period, limit): 173 | """Async entry point coroutine. 174 | 175 | """ 176 | async with aiohttp.ClientSession(loop=loop) as session: 177 | comments = await poll_top_stories_for_comments(loop, session, period, limit) 178 | 179 | return comments 180 | 181 | 182 | if __name__ == '__main__': 183 | args = parser.parse_args() 184 | if args.verbose: 185 | log.setLevel(logging.DEBUG) 186 | 187 | loop = asyncio.get_event_loop() 188 | loop.run_until_complete(main(loop, args.period, args.limit)) 189 | 190 | loop.close() 191 | -------------------------------------------------------------------------------- /04_error_handling/03_error_handling.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using an infinite loop of 3 | scheduling a task using ensure_future and sleeping. 4 | 5 | Artificially produce an error and use try..except clauses to catch them. 6 | 7 | Set return_exceptions to True to retrieve potentially complete tasks after 8 | an exception is raised. 9 | 10 | """ 11 | 12 | import asyncio 13 | import argparse 14 | import logging 15 | from functools import partial 16 | from datetime import datetime 17 | 18 | import aiohttp 19 | import async_timeout 20 | 21 | 22 | LOGGER_FORMAT = '%(asctime)s %(message)s' 23 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 24 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 25 | FETCH_TIMEOUT = 10 26 | MAXIMUM_FETCHES = 500 27 | 28 | parser = argparse.ArgumentParser( 29 | description='Calculate the number of comments of the top stories in HN.') 30 | parser.add_argument( 31 | '--period', type=int, default=5, help='Number of seconds between poll') 32 | parser.add_argument( 33 | '--limit', type=int, default=5, 34 | help='Number of new stories to calculate comments for') 35 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 36 | 37 | 38 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 39 | log = logging.getLogger() 40 | log.setLevel(logging.INFO) 41 | 42 | 43 | class BoomException(Exception): 44 | pass 45 | 46 | 47 | class URLFetcher(): 48 | """Provides counting of URL fetches for a particular task. 49 | 50 | """ 51 | 52 | def __init__(self): 53 | self.fetch_counter = 0 54 | 55 | async def fetch(self, session, url): 56 | """Fetch a URL using aiohttp returning parsed JSON response. 57 | 58 | As suggested by the aiohttp docs we reuse the session. 59 | 60 | """ 61 | with async_timeout.timeout(FETCH_TIMEOUT): 62 | self.fetch_counter += 1 63 | if self.fetch_counter > MAXIMUM_FETCHES: 64 | raise BoomException('BOOM!') 65 | 66 | async with session.get(url) as response: 67 | return await response.json() 68 | 69 | 70 | async def post_number_of_comments(loop, session, fetcher, post_id): 71 | """Retrieve data for current post and recursively for all comments. 72 | 73 | """ 74 | url = URL_TEMPLATE.format(post_id) 75 | 76 | response = await fetcher.fetch(session, url) 77 | 78 | # base case, there are no comments 79 | if response is None or 'kids' not in response: 80 | return 0 81 | 82 | # calculate this post's comments as number of comments 83 | number_of_comments = len(response['kids']) 84 | 85 | # create recursive tasks for all comments 86 | tasks = [post_number_of_comments( 87 | loop, session, fetcher, kid_id) for kid_id in response['kids']] 88 | 89 | # schedule the tasks and retrieve results 90 | try: 91 | results = await asyncio.gather(*tasks) 92 | except BoomException as e: 93 | # log.error("Error retrieving comments for top stories: {}".format(e)) 94 | raise 95 | 96 | # reduce the descendents comments and add it to this post's 97 | number_of_comments += sum(results) 98 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 99 | 100 | return number_of_comments 101 | 102 | 103 | async def get_comments_of_top_stories(loop, session, limit, iteration): 104 | """Retrieve top stories in HN. 105 | 106 | """ 107 | fetcher = URLFetcher() # create a new fetcher for this task 108 | try: 109 | response = await fetcher.fetch(session, TOP_STORIES_URL) 110 | except BoomException as e: 111 | log.error("Error retrieving top stories: {}".format(e)) 112 | # return instead of re-raising as it will go unnoticed 113 | return 114 | except Exception as e: # catch generic exceptions 115 | log.error("Unexpected exception: {}".format(e)) 116 | return 117 | 118 | tasks = [post_number_of_comments( 119 | loop, session, fetcher, post_id) for post_id in response[:limit]] 120 | 121 | # we're not using a try..except anymore as passing `return_exceptions` 122 | # as True will simply return the Exception object produced 123 | results = await asyncio.gather(*tasks, return_exceptions=True) 124 | 125 | # we can safely iterate the results 126 | for post_id, result in zip(response[:limit], results): 127 | # and manually check the types 128 | if isinstance(result, BoomException): 129 | log.error("Error retrieving comments for top stories: {}".format( 130 | result)) 131 | elif isinstance(result, Exception): 132 | log.error("Unexpected exception: {}".format(result)) 133 | else: 134 | log.info("Post {} has {} comments ({})".format( 135 | post_id, result, iteration)) 136 | 137 | return fetcher.fetch_counter 138 | 139 | 140 | async def poll_top_stories_for_comments(loop, session, period, limit): 141 | """Periodically poll for new stories and retrieve number of comments. 142 | 143 | """ 144 | iteration = 1 145 | errors = [] 146 | while True: 147 | if errors: 148 | log.info('Error detected, quitting') 149 | return 150 | 151 | log.info("Calculating comments for top {} stories. ({})".format( 152 | limit, iteration)) 153 | 154 | future = asyncio.ensure_future( 155 | get_comments_of_top_stories(loop, session, limit, iteration)) 156 | 157 | now = datetime.now() 158 | 159 | def callback(fut, errors): 160 | try: 161 | fetch_count = fut.result() 162 | except BoomException as e: 163 | log.debug('Adding {} to errors'.format(e)) 164 | errors.append(e) 165 | except Exception as e: 166 | log.exception('Unexpected error') 167 | errors.append(e) 168 | else: 169 | log.info( 170 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 171 | (datetime.now() - now).total_seconds(), fetch_count)) 172 | 173 | future.add_done_callback(partial(callback, errors=errors)) 174 | 175 | log.info("Waiting for {} seconds...".format(period)) 176 | iteration += 1 177 | await asyncio.sleep(period) 178 | 179 | 180 | async def main(loop, period, limit): 181 | """Async entry point coroutine. 182 | 183 | """ 184 | async with aiohttp.ClientSession(loop=loop) as session: 185 | comments = await poll_top_stories_for_comments(loop, session, period, limit) 186 | 187 | return comments 188 | 189 | 190 | if __name__ == '__main__': 191 | args = parser.parse_args() 192 | if args.verbose: 193 | log.setLevel(logging.DEBUG) 194 | 195 | loop = asyncio.get_event_loop() 196 | loop.run_until_complete(main(loop, args.period, args.limit)) 197 | 198 | loop.close() 199 | -------------------------------------------------------------------------------- /05_cancelling_coroutines/01_cancelling_coroutines.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using an infinite loop of 3 | scheduling a task using ensure_future and sleeping. 4 | 5 | Artificially produce an error and use try..except clauses to catch them. 6 | 7 | Use `wait` to cancel pending coroutines in case if an exception. 8 | 9 | """ 10 | 11 | import asyncio 12 | import argparse 13 | import logging 14 | from functools import partial 15 | from datetime import datetime 16 | from concurrent.futures import FIRST_EXCEPTION 17 | 18 | import aiohttp 19 | import async_timeout 20 | 21 | 22 | LOGGER_FORMAT = '%(asctime)s %(message)s' 23 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 24 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 25 | FETCH_TIMEOUT = 10 26 | MAXIMUM_FETCHES = 500 27 | 28 | parser = argparse.ArgumentParser( 29 | description='Calculate the number of comments of the top stories in HN.') 30 | parser.add_argument( 31 | '--period', type=int, default=5, help='Number of seconds between poll') 32 | parser.add_argument( 33 | '--limit', type=int, default=5, 34 | help='Number of new stories to calculate comments for') 35 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 36 | 37 | 38 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 39 | log = logging.getLogger() 40 | log.setLevel(logging.INFO) 41 | 42 | 43 | class BoomException(Exception): 44 | pass 45 | 46 | 47 | class URLFetcher(): 48 | """Provides counting of URL fetches for a particular task. 49 | 50 | """ 51 | 52 | def __init__(self): 53 | self.fetch_counter = 0 54 | 55 | async def fetch(self, session, url): 56 | """Fetch a URL using aiohttp returning parsed JSON response. 57 | 58 | As suggested by the aiohttp docs we reuse the session. 59 | 60 | """ 61 | with async_timeout.timeout(FETCH_TIMEOUT): 62 | self.fetch_counter += 1 63 | if self.fetch_counter > MAXIMUM_FETCHES: 64 | raise BoomException('BOOM!') 65 | 66 | async with session.get(url) as response: 67 | return await response.json() 68 | 69 | 70 | async def post_number_of_comments(loop, session, fetcher, post_id): 71 | """Retrieve data for current post and recursively for all comments. 72 | 73 | """ 74 | url = URL_TEMPLATE.format(post_id) 75 | try: 76 | response = await fetcher.fetch(session, url) 77 | except BoomException as e: 78 | log.debug("Error retrieving post {}: {}".format(post_id, e)) 79 | raise e 80 | 81 | # base case, there are no comments 82 | if response is None or 'kids' not in response: 83 | return 0 84 | 85 | # calculate this post's comments as number of comments 86 | number_of_comments = len(response['kids']) 87 | 88 | # create recursive tasks for all comments 89 | tasks = [post_number_of_comments( 90 | loop, session, fetcher, kid_id) for kid_id in response['kids']] 91 | 92 | # schedule the tasks and retrieve results 93 | try: 94 | results = await asyncio.gather(*tasks) 95 | except BoomException as e: 96 | log.debug("Error retrieving comments for top stories: {}".format(e)) 97 | raise 98 | 99 | # reduce the descendents comments and add it to this post's 100 | number_of_comments += sum(results) 101 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 102 | 103 | return number_of_comments 104 | 105 | 106 | async def get_comments_of_top_stories(loop, session, limit, iteration): 107 | """Retrieve top stories in HN. 108 | 109 | """ 110 | fetcher = URLFetcher() # create a new fetcher for this task 111 | try: 112 | response = await fetcher.fetch(session, TOP_STORIES_URL) 113 | except BoomException as e: 114 | log.error("Error retrieving top stories: {}".format(e)) 115 | # return instead of re-raising as it will go unnoticed 116 | return 117 | except Exception as e: # catch generic exceptions 118 | log.error("Unexpected exception: {}".format(e)) 119 | return 120 | 121 | tasks = [post_number_of_comments( 122 | loop, session, fetcher, post_id) for post_id in response[:limit]] 123 | 124 | # return on first exception to cancel any pending tasks 125 | done, pending = await asyncio.wait(tasks, return_when=FIRST_EXCEPTION) 126 | 127 | # cancel any pending tasks, the tuple could be empty so it's safe 128 | for pending_task in pending: 129 | pending_task.cancel() 130 | 131 | # process the done tasks 132 | for done_task in done: 133 | # one of the Tasks could raise an exception 134 | try: 135 | print("Post ??? has {} comments ({})".format( 136 | done_task.result(), iteration)) 137 | except BoomException as e: 138 | print("Error retrieving comments for top stories: {}".format(e)) 139 | 140 | return fetcher.fetch_counter 141 | 142 | 143 | async def poll_top_stories_for_comments(loop, session, period, limit): 144 | """Periodically poll for new stories and retrieve number of comments. 145 | 146 | """ 147 | iteration = 1 148 | errors = [] 149 | while True: 150 | if errors: 151 | log.info('Error detected, quitting') 152 | return 153 | 154 | log.info("Calculating comments for top {} stories. ({})".format( 155 | limit, iteration)) 156 | 157 | future = asyncio.ensure_future( 158 | get_comments_of_top_stories(loop, session, limit, iteration)) 159 | 160 | now = datetime.now() 161 | 162 | def callback(fut, errors): 163 | try: 164 | fetch_count = fut.result() 165 | except BoomException as e: 166 | log.debug('Adding {} to errors'.format(e)) 167 | errors.append(e) 168 | except Exception as e: 169 | log.exception('Unexpected error') 170 | errors.append(e) 171 | else: 172 | log.info( 173 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 174 | (datetime.now() - now).total_seconds(), fetch_count)) 175 | 176 | future.add_done_callback(partial(callback, errors=errors)) 177 | 178 | log.info("Waiting for {} seconds...".format(period)) 179 | iteration += 1 180 | await asyncio.sleep(period) 181 | 182 | 183 | async def main(loop, period, limit): 184 | """Async entry point coroutine. 185 | 186 | """ 187 | async with aiohttp.ClientSession(loop=loop) as session: 188 | comments = await poll_top_stories_for_comments(loop, session, period, limit) 189 | 190 | return comments 191 | 192 | 193 | if __name__ == '__main__': 194 | args = parser.parse_args() 195 | if args.verbose: 196 | log.setLevel(logging.DEBUG) 197 | 198 | loop = asyncio.get_event_loop() 199 | loop.run_until_complete(main(loop, args.period, args.limit)) 200 | 201 | loop.close() 202 | -------------------------------------------------------------------------------- /05_cancelling_coroutines/02_cancelling_coroutines.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using an infinite loop of 3 | scheduling a task using ensure_future and sleeping. 4 | 5 | Artificially produce an error and use try..except clauses to catch them. 6 | 7 | Use `wait` to cancel pending coroutines in case if an exception. 8 | 9 | """ 10 | 11 | import asyncio 12 | import argparse 13 | import logging 14 | from functools import partial 15 | from datetime import datetime 16 | from concurrent.futures import FIRST_EXCEPTION 17 | 18 | import aiohttp 19 | import async_timeout 20 | 21 | 22 | LOGGER_FORMAT = '%(asctime)s %(message)s' 23 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 24 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 25 | FETCH_TIMEOUT = 10 26 | MAXIMUM_FETCHES = 520 27 | 28 | parser = argparse.ArgumentParser( 29 | description='Calculate the number of comments of the top stories in HN.') 30 | parser.add_argument( 31 | '--period', type=int, default=5, help='Number of seconds between poll') 32 | parser.add_argument( 33 | '--limit', type=int, default=5, 34 | help='Number of new stories to calculate comments for') 35 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 36 | 37 | 38 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 39 | log = logging.getLogger() 40 | log.setLevel(logging.INFO) 41 | 42 | 43 | class BoomException(Exception): 44 | pass 45 | 46 | 47 | class URLFetcher(): 48 | """Provides counting of URL fetches for a particular task. 49 | 50 | """ 51 | 52 | def __init__(self): 53 | self.fetch_counter = 0 54 | 55 | async def fetch(self, session, url): 56 | """Fetch a URL using aiohttp returning parsed JSON response. 57 | 58 | As suggested by the aiohttp docs we reuse the session. 59 | 60 | """ 61 | with async_timeout.timeout(FETCH_TIMEOUT): 62 | self.fetch_counter += 1 63 | if self.fetch_counter > MAXIMUM_FETCHES: 64 | raise BoomException('BOOM!') 65 | 66 | async with session.get(url) as response: 67 | return await response.json() 68 | 69 | 70 | async def post_number_of_comments(loop, session, fetcher, post_id): 71 | """Retrieve data for current post and recursively for all comments. 72 | 73 | """ 74 | url = URL_TEMPLATE.format(post_id) 75 | try: 76 | response = await fetcher.fetch(session, url) 77 | except BoomException as e: 78 | log.debug("Error retrieving post {}: {}".format(post_id, e)) 79 | raise e 80 | 81 | # base case, there are no comments 82 | if response is None or 'kids' not in response: 83 | return 0 84 | 85 | # calculate this post's comments as number of comments 86 | number_of_comments = len(response['kids']) 87 | 88 | # create recursive tasks for all comments 89 | tasks = [post_number_of_comments( 90 | loop, session, fetcher, kid_id) for kid_id in response['kids']] 91 | 92 | # schedule the tasks and retrieve results 93 | try: 94 | results = await asyncio.gather(*tasks) 95 | except BoomException as e: 96 | log.debug("Error retrieving comments for top stories: {}".format(e)) 97 | raise 98 | 99 | # reduce the descendents comments and add it to this post's 100 | number_of_comments += sum(results) 101 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 102 | 103 | return number_of_comments 104 | 105 | 106 | async def get_comments_of_top_stories(loop, session, limit, iteration): 107 | """Retrieve top stories in HN. 108 | 109 | """ 110 | fetcher = URLFetcher() # create a new fetcher for this task 111 | try: 112 | response = await fetcher.fetch(session, TOP_STORIES_URL) 113 | except BoomException as e: 114 | log.error("Error retrieving top stories: {}".format(e)) 115 | # return instead of re-raising as it will go unnoticed 116 | return 117 | except Exception as e: # catch generic exceptions 118 | log.error("Unexpected exception: {}".format(e)) 119 | return 120 | 121 | tasks = { 122 | asyncio.ensure_future( 123 | post_number_of_comments(loop, session, fetcher, post_id) 124 | ): post_id for post_id in response[:limit]} 125 | 126 | # return on first exception to cancel any pending tasks 127 | done, pending = await asyncio.wait( 128 | tasks.keys(), return_when=FIRST_EXCEPTION) 129 | 130 | # if there are pending tasks is because there was an exception 131 | # cancel any pending tasks 132 | for pending_task in pending: 133 | pending_task.cancel() 134 | 135 | # process the done tasks 136 | for done_task in done: 137 | # if an exception is raised one of the Tasks will raise 138 | try: 139 | print("Post {} has {} comments ({})".format( 140 | tasks[done_task], done_task.result(), iteration)) 141 | except BoomException as e: 142 | print("Error retrieving comments for top stories: {}".format(e)) 143 | 144 | return fetcher.fetch_counter 145 | 146 | 147 | async def poll_top_stories_for_comments(loop, session, period, limit): 148 | """Periodically poll for new stories and retrieve number of comments. 149 | 150 | """ 151 | iteration = 1 152 | errors = [] 153 | while True: 154 | if errors: 155 | log.info('Error detected, quitting') 156 | return 157 | 158 | log.info("Calculating comments for top {} stories. ({})".format( 159 | limit, iteration)) 160 | 161 | future = asyncio.ensure_future( 162 | get_comments_of_top_stories(loop, session, limit, iteration)) 163 | 164 | now = datetime.now() 165 | 166 | def callback(fut, errors): 167 | try: 168 | fetch_count = fut.result() 169 | except BoomException as e: 170 | log.debug('Adding {} to errors'.format(e)) 171 | errors.append(e) 172 | except Exception as e: 173 | log.exception('Unexpected error') 174 | errors.append(e) 175 | else: 176 | log.info( 177 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 178 | (datetime.now() - now).total_seconds(), fetch_count)) 179 | 180 | future.add_done_callback(partial(callback, errors=errors)) 181 | 182 | log.info("Waiting for {} seconds...".format(period)) 183 | iteration += 1 184 | await asyncio.sleep(period) 185 | 186 | 187 | async def main(loop, period, limit): 188 | """Async entry point coroutine. 189 | 190 | """ 191 | async with aiohttp.ClientSession(loop=loop) as session: 192 | comments = await poll_top_stories_for_comments(loop, session, period, limit) 193 | 194 | return comments 195 | 196 | 197 | if __name__ == '__main__': 198 | args = parser.parse_args() 199 | if args.verbose: 200 | log.setLevel(logging.DEBUG) 201 | 202 | loop = asyncio.get_event_loop() 203 | loop.run_until_complete(main(loop, args.period, args.limit)) 204 | 205 | loop.close() 206 | -------------------------------------------------------------------------------- /05_cancelling_coroutines/03_cancelling_coroutines.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using an infinite loop of 3 | scheduling a task using ensure_future and sleeping. 4 | 5 | Artificially produce an error and use try..except clauses to catch them. 6 | 7 | Use `wait` to cancel pending coroutines in case if an exception. 8 | 9 | """ 10 | 11 | import asyncio 12 | import argparse 13 | import logging 14 | from functools import partial 15 | from datetime import datetime 16 | from concurrent.futures import FIRST_EXCEPTION 17 | 18 | import aiohttp 19 | import async_timeout 20 | 21 | 22 | LOGGER_FORMAT = '%(asctime)s %(message)s' 23 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 24 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 25 | FETCH_TIMEOUT = 10 26 | MAXIMUM_FETCHES = 550 27 | 28 | parser = argparse.ArgumentParser( 29 | description='Calculate the number of comments of the top stories in HN.') 30 | parser.add_argument( 31 | '--period', type=int, default=5, help='Number of seconds between poll') 32 | parser.add_argument( 33 | '--limit', type=int, default=5, 34 | help='Number of new stories to calculate comments for') 35 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 36 | 37 | 38 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 39 | log = logging.getLogger() 40 | log.setLevel(logging.INFO) 41 | 42 | 43 | class BoomException(Exception): 44 | pass 45 | 46 | 47 | class URLFetcher(): 48 | """Provides counting of URL fetches for a particular task. 49 | 50 | """ 51 | 52 | def __init__(self): 53 | self.fetch_counter = 0 54 | 55 | async def fetch(self, session, url): 56 | """Fetch a URL using aiohttp returning parsed JSON response. 57 | 58 | As suggested by the aiohttp docs we reuse the session. 59 | 60 | """ 61 | with async_timeout.timeout(FETCH_TIMEOUT): 62 | self.fetch_counter += 1 63 | if self.fetch_counter > MAXIMUM_FETCHES: 64 | raise BoomException('BOOM!') 65 | 66 | async with session.get(url) as response: 67 | return await response.json() 68 | 69 | 70 | async def post_number_of_comments(loop, session, fetcher, post_id): 71 | """Retrieve data for current post and recursively for all comments. 72 | 73 | """ 74 | url = URL_TEMPLATE.format(post_id) 75 | try: 76 | response = await fetcher.fetch(session, url) 77 | except BoomException as e: 78 | log.debug("Error retrieving post {}: {}".format(post_id, e)) 79 | raise e 80 | 81 | # base case, there are no comments 82 | if response is None or 'kids' not in response: 83 | return 0 84 | 85 | # calculate this post's comments as number of comments 86 | number_of_comments = len(response['kids']) 87 | 88 | try: 89 | # create recursive tasks for all comments 90 | tasks = [asyncio.ensure_future(post_number_of_comments( 91 | loop, session, fetcher, kid_id)) for kid_id in response['kids']] 92 | 93 | # schedule the tasks and retrieve results 94 | try: 95 | results = await asyncio.gather(*tasks) 96 | except BoomException as e: 97 | log.debug("Error retrieving comments for top stories: {}".format(e)) 98 | raise 99 | 100 | # reduce the descendents comments and add it to this post's 101 | number_of_comments += sum(results) 102 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 103 | 104 | return number_of_comments 105 | except asyncio.CancelledError: 106 | if tasks: 107 | log.info("Comments for post {} cancelled, cancelling {} child tasks".format( 108 | post_id, len(tasks))) 109 | for task in tasks: 110 | task.cancel() 111 | else: 112 | log.info("Comments for post {} cancelled".format(post_id)) 113 | raise 114 | 115 | 116 | async def get_comments_of_top_stories(loop, session, limit, iteration): 117 | """Retrieve top stories in HN. 118 | 119 | """ 120 | fetcher = URLFetcher() # create a new fetcher for this task 121 | try: 122 | response = await fetcher.fetch(session, TOP_STORIES_URL) 123 | except BoomException as e: 124 | log.error("Error retrieving top stories: {}".format(e)) 125 | # return instead of re-raising as it will go unnoticed 126 | return 127 | except Exception as e: # catch generic exceptions 128 | log.error("Unexpected exception: {}".format(e)) 129 | return 130 | 131 | tasks = { 132 | asyncio.ensure_future( 133 | post_number_of_comments(loop, session, fetcher, post_id) 134 | ): post_id for post_id in response[:limit]} 135 | 136 | # return on first exception to cancel any pending tasks 137 | done, pending = await asyncio.wait( 138 | tasks.keys(), return_when=FIRST_EXCEPTION) 139 | 140 | # if there are pending tasks is because there was an exception 141 | # cancel any pending tasks 142 | for pending_task in pending: 143 | pending_task.cancel() 144 | 145 | # process the done tasks 146 | for done_task in done: 147 | # if an exception is raised one of the Tasks will raise 148 | try: 149 | print("Post {} has {} comments ({})".format( 150 | tasks[done_task], done_task.result(), iteration)) 151 | except BoomException as e: 152 | print("Error retrieving comments for top stories: {}".format(e)) 153 | 154 | return fetcher.fetch_counter 155 | 156 | 157 | async def poll_top_stories_for_comments(loop, session, period, limit): 158 | """Periodically poll for new stories and retrieve number of comments. 159 | 160 | """ 161 | iteration = 1 162 | errors = [] 163 | while True: 164 | if errors: 165 | log.info('Error detected, quitting') 166 | return 167 | 168 | log.info("Calculating comments for top {} stories. ({})".format( 169 | limit, iteration)) 170 | 171 | future = asyncio.ensure_future( 172 | get_comments_of_top_stories(loop, session, limit, iteration)) 173 | 174 | now = datetime.now() 175 | 176 | def callback(fut, errors): 177 | try: 178 | fetch_count = fut.result() 179 | except BoomException as e: 180 | log.debug('Adding {} to errors'.format(e)) 181 | errors.append(e) 182 | except Exception as e: 183 | log.exception('Unexpected error') 184 | errors.append(e) 185 | else: 186 | log.info( 187 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 188 | (datetime.now() - now).total_seconds(), fetch_count)) 189 | 190 | future.add_done_callback(partial(callback, errors=errors)) 191 | 192 | log.info("Waiting for {} seconds...".format(period)) 193 | iteration += 1 194 | await asyncio.sleep(period) 195 | 196 | 197 | async def main(loop, period, limit): 198 | """Async entry point coroutine. 199 | 200 | """ 201 | async with aiohttp.ClientSession(loop=loop) as session: 202 | comments = await poll_top_stories_for_comments(loop, session, period, limit) 203 | 204 | return comments 205 | 206 | 207 | if __name__ == '__main__': 208 | args = parser.parse_args() 209 | if args.verbose: 210 | log.setLevel(logging.DEBUG) 211 | 212 | loop = asyncio.get_event_loop() 213 | loop.run_until_complete(main(loop, args.period, args.limit)) 214 | 215 | loop.close() 216 | -------------------------------------------------------------------------------- /05_cancelling_coroutines/04_cancelling_coroutines.py: -------------------------------------------------------------------------------- 1 | """ 2 | An example of periodically scheduling coroutines using an infinite loop of 3 | scheduling a task using ensure_future and sleeping. 4 | 5 | Artificially produce an error and use try..except clauses to catch them. 6 | 7 | Use `wait` to cancel pending coroutines in case if an exception. 8 | 9 | """ 10 | 11 | import asyncio 12 | import argparse 13 | import logging 14 | from functools import partial 15 | from datetime import datetime 16 | from concurrent.futures import FIRST_EXCEPTION 17 | 18 | import aiohttp 19 | import async_timeout 20 | 21 | 22 | LOGGER_FORMAT = '%(asctime)s %(message)s' 23 | URL_TEMPLATE = "https://hacker-news.firebaseio.com/v0/item/{}.json" 24 | TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json" 25 | FETCH_TIMEOUT = 10 26 | MAXIMUM_FETCHES = 250 27 | 28 | parser = argparse.ArgumentParser( 29 | description='Calculate the number of comments of the top stories in HN.') 30 | parser.add_argument( 31 | '--period', type=int, default=5, help='Number of seconds between poll') 32 | parser.add_argument( 33 | '--limit', type=int, default=5, 34 | help='Number of new stories to calculate comments for') 35 | parser.add_argument('--verbose', action='store_true', help='Detailed output') 36 | 37 | 38 | logging.basicConfig(format=LOGGER_FORMAT, datefmt='[%H:%M:%S]') 39 | log = logging.getLogger() 40 | log.setLevel(logging.INFO) 41 | 42 | 43 | class BoomException(Exception): 44 | pass 45 | 46 | 47 | class URLFetcher(): 48 | """Provides counting of URL fetches for a particular task. 49 | 50 | """ 51 | 52 | def __init__(self): 53 | self.fetch_counter = 0 54 | 55 | async def fetch(self, session, url): 56 | """Fetch a URL using aiohttp returning parsed JSON response. 57 | 58 | As suggested by the aiohttp docs we reuse the session. 59 | 60 | """ 61 | with async_timeout.timeout(FETCH_TIMEOUT): 62 | self.fetch_counter += 1 63 | if self.fetch_counter > MAXIMUM_FETCHES: 64 | raise BoomException('BOOM!') 65 | 66 | async with session.get(url) as response: 67 | return await response.json() 68 | 69 | 70 | async def post_number_of_comments(loop, session, fetcher, post_id): 71 | """Retrieve data for current post and recursively for all comments. 72 | 73 | """ 74 | url = URL_TEMPLATE.format(post_id) 75 | try: 76 | response = await fetcher.fetch(session, url) 77 | except BoomException as e: 78 | log.debug("Error retrieving post {}: {}".format(post_id, e)) 79 | raise e 80 | 81 | # base case, there are no comments 82 | if response is None or 'kids' not in response: 83 | return 0 84 | 85 | # calculate this post's comments as number of comments 86 | number_of_comments = len(response['kids']) 87 | 88 | try: 89 | # create recursive tasks for all comments 90 | tasks = [asyncio.ensure_future(post_number_of_comments( 91 | loop, session, fetcher, kid_id)) for kid_id in response['kids']] 92 | 93 | # schedule the tasks and retrieve results 94 | try: 95 | results = await asyncio.gather(*tasks) 96 | except BoomException as e: 97 | log.debug("Error retrieving comments for top stories: {}".format(e)) 98 | raise 99 | 100 | # reduce the descendents comments and add it to this post's 101 | number_of_comments += sum(results) 102 | log.debug('{:^6} > {} comments'.format(post_id, number_of_comments)) 103 | 104 | return number_of_comments 105 | except asyncio.CancelledError: 106 | if tasks: 107 | log.info("Comments for post {} cancelled, cancelling {} child tasks".format( 108 | post_id, len(tasks))) 109 | for task in tasks: 110 | task.cancel() 111 | else: 112 | log.info("Comments for post {} cancelled".format(post_id)) 113 | raise 114 | 115 | 116 | async def get_comments_of_top_stories(loop, session, limit, iteration): 117 | """Retrieve top stories in HN. 118 | 119 | """ 120 | fetcher = URLFetcher() # create a new fetcher for this task 121 | try: 122 | response = await fetcher.fetch(session, TOP_STORIES_URL) 123 | except BoomException as e: 124 | log.error("Error retrieving top stories: {}".format(e)) 125 | # return instead of re-raising as it will go unnoticed 126 | return 127 | except Exception as e: # catch generic exceptions 128 | log.error("Unexpected exception: {}".format(e)) 129 | return 130 | 131 | tasks = { 132 | asyncio.ensure_future( 133 | post_number_of_comments(loop, session, fetcher, post_id) 134 | ): post_id for post_id in response[:limit]} 135 | 136 | # return on first exception to cancel any pending tasks 137 | done, pending = await asyncio.shield(asyncio.wait( 138 | tasks.keys(), return_when=FIRST_EXCEPTION)) 139 | 140 | # if there are pending tasks is because there was an exception 141 | # cancel any pending tasks 142 | for pending_task in pending: 143 | pending_task.cancel() 144 | 145 | # process the done tasks 146 | for done_task in done: 147 | # if an exception is raised one of the Tasks will raise 148 | try: 149 | print("Post {} has {} comments ({})".format( 150 | tasks[done_task], done_task.result(), iteration)) 151 | except BoomException as e: 152 | print("Error retrieving comments for top stories: {}".format(e)) 153 | 154 | return fetcher.fetch_counter 155 | 156 | 157 | async def poll_top_stories_for_comments(loop, session, period, limit): 158 | """Periodically poll for new stories and retrieve number of comments. 159 | 160 | """ 161 | iteration = 1 162 | errors = [] 163 | while True: 164 | if errors: 165 | log.info('Error detected, quitting') 166 | return 167 | 168 | log.info("Calculating comments for top {} stories. ({})".format( 169 | limit, iteration)) 170 | 171 | future = asyncio.ensure_future( 172 | get_comments_of_top_stories(loop, session, limit, iteration)) 173 | 174 | now = datetime.now() 175 | 176 | def callback(fut, errors): 177 | try: 178 | fetch_count = fut.result() 179 | except BoomException as e: 180 | log.debug('Adding {} to errors'.format(e)) 181 | errors.append(e) 182 | except Exception as e: 183 | log.exception('Unexpected error') 184 | errors.append(e) 185 | else: 186 | log.info( 187 | '> Calculating comments took {:.2f} seconds and {} fetches'.format( 188 | (datetime.now() - now).total_seconds(), fetch_count)) 189 | 190 | future.add_done_callback(partial(callback, errors=errors)) 191 | 192 | log.info("Waiting for {} seconds...".format(period)) 193 | iteration += 1 194 | await asyncio.sleep(period) 195 | 196 | 197 | if __name__ == '__main__': 198 | args = parser.parse_args() 199 | if args.verbose: 200 | log.setLevel(logging.DEBUG) 201 | 202 | loop = asyncio.get_event_loop() 203 | with aiohttp.ClientSession(loop=loop) as session: 204 | loop.run_until_complete( 205 | poll_top_stories_for_comments( 206 | loop, session, args.period, args.limit)) 207 | 208 | loop.close() 209 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AsyncIO Coroutine Patterns 2 | 3 | Full code examples for the two articles on Medium, [*Asyncio Coroutine Patterns: Beyond await*](https://medium.com/@yeraydiazdiaz/asyncio-coroutine-patterns-beyond-await-a6121486656f) and [*Asyncio Coroutine Patterns: Errors and cancellation*](https://medium.com/@yeraydiazdiaz/asyncio-coroutine-patterns-errors-and-cancellation-3bb422e961ff). 4 | 5 | These examples require Python 3.5 or above and [`aiohttp`](http://aiohttp.readthedocs.io/en/stable/). 6 | 7 | ## Installation 8 | 9 | 1. Make sure your version of Python is 3.5 or above: `python3 --version` 10 | 2. Clone this repo 11 | 3. Create a virtualenv 12 | 4. `pip install -r requirements.txt` 13 | 5. Run any file you'd like, i.e. `python 01_recursive_coroutines/recursive_coroutines.py` 14 | 15 | 16 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.3.2 2 | aiodns==1.1.1 --------------------------------------------------------------------------------