├── .gitignore
├── .idea
    ├── dictionaries
    │   └── mkennedy.xml
    ├── inspectionProfiles
    │   └── Project_Default.xml
    └── vcs.xml
├── code
    ├── generator
    │   └── simple_gen.py
    ├── producer_consumer
    │   ├── prod_async
    │   │   ├── async_program.py
    │   │   └── requirements.txt
    │   └── prod_sync
    │   │   ├── requirements.txt
    │   │   └── sync_program.py
    ├── the_unsync
    │   ├── nosync.py
    │   ├── presync.py
    │   ├── requirements.txt
    │   └── thesync.py
    └── web_scraping
    │   ├── async_scrape
    │       ├── program.py
    │       └── requirements.txt
    │   └── sync_scrape
    │       ├── program.py
    │       └── requirements.txt
├── readme.md
├── readme_resources
    └── webinar.png
└── slides
    └── async_jetbrains_feb2018.pdf


/.gitignore:
--------------------------------------------------------------------------------
  1 | # Created by .ignore support plugin (hsz.mobi)
  2 | ### Python template
  3 | # Byte-compiled / optimized / DLL files
  4 | __pycache__/
  5 | *.py[cod]
  6 | *$py.class
  7 | 
  8 | # C extensions
  9 | *.so
 10 | 
 11 | # Distribution / packaging
 12 | .Python
 13 | build/
 14 | develop-eggs/
 15 | dist/
 16 | downloads/
 17 | eggs/
 18 | .eggs/
 19 | lib/
 20 | lib64/
 21 | parts/
 22 | sdist/
 23 | var/
 24 | wheels/
 25 | *.egg-info/
 26 | .installed.cfg
 27 | *.egg
 28 | MANIFEST
 29 | 
 30 | # PyInstaller
 31 | #  Usually these files are written by a python script from a template
 32 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 33 | *.manifest
 34 | *.spec
 35 | 
 36 | # Installer logs
 37 | pip-log.txt
 38 | pip-delete-this-directory.txt
 39 | 
 40 | # Unit test / coverage reports
 41 | htmlcov/
 42 | .tox/
 43 | .coverage
 44 | .coverage.*
 45 | .cache
 46 | nosetests.xml
 47 | coverage.xml
 48 | *.cover
 49 | .hypothesis/
 50 | .pytest_cache/
 51 | 
 52 | # Translations
 53 | *.mo
 54 | *.pot
 55 | 
 56 | # Django stuff:
 57 | *.log
 58 | local_settings.py
 59 | db.sqlite3
 60 | 
 61 | # Flask stuff:
 62 | instance/
 63 | .webassets-cache
 64 | 
 65 | # Scrapy stuff:
 66 | .scrapy
 67 | 
 68 | # Sphinx documentation
 69 | docs/_build/
 70 | 
 71 | # PyBuilder
 72 | target/
 73 | 
 74 | # Jupyter Notebook
 75 | .ipynb_checkpoints
 76 | 
 77 | # pyenv
 78 | .python-version
 79 | 
 80 | # celery beat schedule file
 81 | celerybeat-schedule
 82 | 
 83 | # SageMath parsed files
 84 | *.sage.py
 85 | 
 86 | # Environments
 87 | .env
 88 | .venv
 89 | env/
 90 | venv/
 91 | ENV/
 92 | env.bak/
 93 | venv.bak/
 94 | 
 95 | # Spyder project settings
 96 | .spyderproject
 97 | .spyproject
 98 | 
 99 | # Rope project settings
100 | .ropeproject
101 | 
102 | # mkdocs documentation
103 | /site
104 | 
105 | # mypy
106 | .mypy_cache/
107 | 
108 | .idea/encodings.xml
109 | .idea/misc.xml
110 | .idea/modules.xml
111 | .idea/webcast.iml
112 | .idea/workspace.xml
113 | 


--------------------------------------------------------------------------------
/.idea/dictionaries/mkennedy.xml:
--------------------------------------------------------------------------------
 1 | <component name="ProjectDictionaryState">
 2 |   <dictionary name="mkennedy">
 3 |     <words>
 4 |       <w>aiodns</w>
 5 |       <w>aiohttp</w>
 6 |       <w>asyncio</w>
 7 |       <w>cchardet</w>
 8 |       <w>colorama</w>
 9 |       <w>unsync</w>
10 |     </words>
11 |   </dictionary>
12 | </component>


--------------------------------------------------------------------------------
/.idea/inspectionProfiles/Project_Default.xml:
--------------------------------------------------------------------------------
 1 | <component name="InspectionProjectProfileManager">
 2 |   <profile version="1.0">
 3 |     <option name="myName" value="Project Default" />
 4 |     <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
 5 |       <option name="ignoredPackages">
 6 |         <value>
 7 |           <list size="1">
 8 |             <item index="0" class="java.lang.String" itemvalue="bson" />
 9 |           </list>
10 |         </value>
11 |       </option>
12 |     </inspection_tool>
13 |     <inspection_tool class="UnterminatedStatementJS" enabled="false" level="WARNING" enabled_by_default="false">
14 |       <option name="ignoreSemicolonAtEndOfBlock" value="true" />
15 |     </inspection_tool>
16 |   </profile>
17 | </component>


--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="VcsDirectoryMappings">
4 |     <mapping directory="$PROJECT_DIR$" vcs="Git" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/code/generator/simple_gen.py:
--------------------------------------------------------------------------------
 1 | from typing import List
 2 | 
 3 | 
 4 | # def fib(n: int) -> List[int]:
 5 | #     numbers = []
 6 | #     current, nxt = 0, 1
 7 | #     while len(numbers) < n:
 8 | #         current, nxt = nxt, current + nxt
 9 | #         numbers.append(current)
10 | #
11 | #     return numbers
12 | 
13 | 
14 | def fib():
15 |     current, nxt = 0, 1
16 |     while True:
17 |         current, nxt = nxt, current + nxt
18 |         yield current
19 | 
20 | 
21 | result = fib()
22 | 
23 | for n in result:
24 |     print(n, end=', ')
25 |     if n > 10000:
26 |         break
27 | 
28 | print()
29 | print("Done")
30 | 


--------------------------------------------------------------------------------
/code/producer_consumer/prod_async/async_program.py:
--------------------------------------------------------------------------------
 1 | import asyncio
 2 | import datetime
 3 | from asyncio import AbstractEventLoop
 4 | 
 5 | import colorama
 6 | import random
 7 | import time
 8 | 
 9 | 
10 | def main():
11 |     # Create the asyncio loop
12 |     loop: AbstractEventLoop = asyncio.get_event_loop()
13 | 
14 |     t0 = datetime.datetime.now()
15 |     print(colorama.Fore.WHITE + "App started.", flush=True)
16 | 
17 |     data = asyncio.Queue()  # maybe a better data structure?
18 | 
19 |     # Run these with asyncio.gather()
20 | 
21 |     task = asyncio.gather(
22 |         generate_data(10, data),
23 |         generate_data(10, data),
24 |         process_data(5, data),
25 |         process_data(5, data),
26 |         process_data(5, data),
27 |         process_data(5, data)
28 |     )
29 | 
30 |     loop.run_until_complete(task)
31 | 
32 |     dt = datetime.datetime.now() - t0
33 |     print(colorama.Fore.WHITE + "App exiting, total time: {:,.2f} sec.".format(dt.total_seconds()), flush=True)
34 | 
35 | 
36 | async def generate_data(num: int, data: asyncio.Queue):
37 |     for idx in range(1, num + 1):
38 |         item = idx * idx
39 |         # Use queue
40 |         work = (item, datetime.datetime.now())
41 |         # data.append(work)
42 |         await data.put(work)
43 | 
44 |         print(colorama.Fore.YELLOW + " -- generated item {}".format(idx), flush=True)
45 |         # Sleep better
46 |         # time.sleep(random.random() + .5)
47 |         await asyncio.sleep(random.random() + .5)
48 | 
49 | 
50 | async def process_data(num: int, data: asyncio.Queue):
51 |     processed = 0
52 |     while processed < num:
53 |         # Use queue
54 |         # item = data.pop(0)
55 |         # if not item:
56 |         #     time.sleep(.01)
57 |         #     continue
58 |         item = await data.get()
59 |         # item is a tuple
60 | 
61 |         processed += 1
62 |         value = item[0]
63 |         t = item[1]
64 |         dt = datetime.datetime.now() - t
65 | 
66 |         print(colorama.Fore.CYAN +
67 |               " +++ Processed value {} after {:,.2f} sec.".format(value, dt.total_seconds()), flush=True)
68 |         # Sleep better
69 |         # time.sleep(.5)
70 |         await asyncio.sleep(.5)
71 | 
72 | 
73 | if __name__ == '__main__':
74 |     main()
75 | 


--------------------------------------------------------------------------------
/code/producer_consumer/prod_async/requirements.txt:
--------------------------------------------------------------------------------
1 | colorama
2 | 


--------------------------------------------------------------------------------
/code/producer_consumer/prod_sync/requirements.txt:
--------------------------------------------------------------------------------
1 | colorama


--------------------------------------------------------------------------------
/code/producer_consumer/prod_sync/sync_program.py:
--------------------------------------------------------------------------------
 1 | import datetime
 2 | import colorama
 3 | import random
 4 | import time
 5 | 
 6 | 
 7 | def main():
 8 |     t0 = datetime.datetime.now()
 9 |     print(colorama.Fore.WHITE + "App started.", flush=True)
10 |     data = []
11 |     generate_data(10, data)
12 |     generate_data(10, data)
13 |     process_data(20, data)
14 | 
15 |     dt = datetime.datetime.now() - t0
16 |     print(colorama.Fore.WHITE + "App exiting, total time: {:,.2f} sec.".format(dt.total_seconds()), flush=True)
17 | 
18 | 
19 | def generate_data(num: int, data: list):
20 |     for idx in range(1, num + 1):
21 |         item = idx*idx
22 |         data.append((item, datetime.datetime.now()))
23 | 
24 |         print(colorama.Fore.YELLOW + " -- generated item {}".format(idx), flush=True)
25 |         time.sleep(random.random() + .5)
26 | 
27 | 
28 | def process_data(num: int, data: list):
29 |     processed = 0
30 |     while processed < num:
31 |         item = data.pop(0)
32 |         if not item:
33 |             time.sleep(.01)
34 |             continue
35 | 
36 |         processed += 1
37 |         value = item[0]
38 |         t = item[1]
39 |         dt = datetime.datetime.now() - t
40 | 
41 |         print(colorama.Fore.CYAN +
42 |               " +++ Processed value {} after {:,.2f} sec.".format(value, dt.total_seconds()), flush=True)
43 |         time.sleep(.5)
44 | 
45 | 
46 | if __name__ == '__main__':
47 |     main()
48 | 


--------------------------------------------------------------------------------
/code/the_unsync/nosync.py:
--------------------------------------------------------------------------------
 1 | import datetime
 2 | import math
 3 | import time
 4 | import requests
 5 | 
 6 | 
 7 | def main():
 8 |     t0 = datetime.datetime.now()
 9 | 
10 |     compute_some()
11 |     compute_some()
12 |     compute_some()
13 |     download_some()
14 |     download_some()
15 |     download_some_more()
16 |     download_some_more()
17 |     wait_some()
18 |     wait_some()
19 |     wait_some()
20 |     wait_some()
21 | 
22 |     dt = datetime.datetime.now() - t0
23 |     print("Synchronous version done in {:,.2f} seconds.".format(dt.total_seconds()))
24 | 
25 | 
26 | def compute_some():
27 |     print("Computing...")
28 |     for _ in range(1, 10_000_000):
29 |         math.sqrt(25 ** 25 + .01)
30 | 
31 | 
32 | def download_some():
33 |     print("Downloading...")
34 |     url = 'https://talkpython.fm/episodes/show/174/coming-into-python-from-another-industry-part-2'
35 |     resp = requests.get(url)
36 |     resp.raise_for_status()
37 | 
38 |     text = resp.text
39 | 
40 |     print("Downloaded (more) {:,} characters.".format(len(text)))
41 | 
42 | 
43 | def download_some_more():
44 |     print("Downloading more ...")
45 |     url = 'https://pythonbytes.fm/episodes/show/92/will-your-python-be-compiled'
46 |     resp = requests.get(url)
47 |     resp.raise_for_status()
48 | 
49 |     text = resp.text
50 | 
51 |     print("Downloaded {:,} characters.".format(len(text)))
52 | 
53 | 
54 | def wait_some():
55 |     print("Waiting...")
56 |     for _ in range(1, 1000):
57 |         time.sleep(.001)
58 | 
59 | 
60 | if __name__ == '__main__':
61 |     main()
62 | 


--------------------------------------------------------------------------------
/code/the_unsync/presync.py:
--------------------------------------------------------------------------------
 1 | import asyncio
 2 | import datetime
 3 | import math
 4 | 
 5 | import aiohttp
 6 | import requests
 7 | 
 8 | 
 9 | def main():
10 |     t0 = datetime.datetime.now()
11 | 
12 |     loop = asyncio.get_event_loop()
13 | 
14 |     tasks = [
15 |         loop.create_task(compute_some()),
16 |         loop.create_task(compute_some()),
17 |         loop.create_task(compute_some()),
18 |         loop.create_task(download_some()),
19 |         loop.create_task(download_some()),
20 |         loop.create_task(download_some_more()),
21 |         loop.create_task(download_some_more()),
22 |         loop.create_task(wait_some()),
23 |         loop.create_task(wait_some()),
24 |         loop.create_task(wait_some()),
25 |         loop.create_task(wait_some()),
26 |     ]
27 | 
28 |     loop.run_until_complete(asyncio.gather(*tasks))
29 | 
30 |     dt = datetime.datetime.now() - t0
31 |     print("Synchronous version done in {:,.2f} seconds.".format(dt.total_seconds()))
32 | 
33 | 
34 | async def compute_some():
35 |     print("Computing...")
36 |     for _ in range(1, 10_000_000):
37 |         math.sqrt(25 ** 25 + .01)
38 | 
39 | 
40 | async def download_some():
41 |     print("Downloading...")
42 |     url = 'https://talkpython.fm/episodes/show/174/coming-into-python-from-another-industry-part-2'
43 |     async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session:
44 |         async with session.get(url) as resp:
45 |             resp.raise_for_status()
46 | 
47 |             text = await resp.text()
48 | 
49 |     print("Downloaded (more) {:,} characters.".format(len(text)))
50 | 
51 | 
52 | async def download_some_more():
53 |     print("Downloading more ...")
54 |     url = 'https://pythonbytes.fm/episodes/show/92/will-your-python-be-compiled'
55 |     resp = requests.get(url)
56 |     resp.raise_for_status()
57 | 
58 |     text = resp.text
59 | 
60 |     print("Downloaded {:,} characters.".format(len(text)))
61 | 
62 | 
63 | async def wait_some():
64 |     print("Waiting...")
65 |     for _ in range(1, 1000):
66 |         await asyncio.sleep(.001)
67 | 
68 | 
69 | if __name__ == '__main__':
70 |     main()
71 | 


--------------------------------------------------------------------------------
/code/the_unsync/requirements.txt:
--------------------------------------------------------------------------------
 1 | unsync
 2 | aiohttp
 3 | cchardet
 4 | requests
 5 | bs4
 6 | colorama
 7 | 
 8 | # This one can give you trouble on Windows
 9 | # Feel free to uncomment it but it also works without it
10 | # Just a little less well.
11 | # aiodns
12 | 


--------------------------------------------------------------------------------
/code/the_unsync/thesync.py:
--------------------------------------------------------------------------------
 1 | import asyncio
 2 | import datetime
 3 | import math
 4 | from unsync import unsync
 5 | import aiohttp
 6 | import requests
 7 | 
 8 | 
 9 | def main():
10 |     t0 = datetime.datetime.now()
11 | 
12 |     tasks = [
13 |         compute_some(),
14 |         compute_some(),
15 |         compute_some(),
16 |         download_some(),
17 |         download_some(),
18 |         download_some_more(),
19 |         download_some_more(),
20 |         wait_some(),
21 |         wait_some(),
22 |         wait_some(),
23 |         wait_some()
24 |     ]
25 | 
26 |     # wait on them.
27 |     [t.result() for t in tasks]
28 | 
29 |     dt = datetime.datetime.now() - t0
30 |     print("Synchronous version done in {:,.2f} seconds.".format(dt.total_seconds()))
31 | 
32 | 
33 | @unsync(cpu_bound=True)
34 | def compute_some():
35 |     print("Computing...")
36 |     for _ in range(1, 10_000_000):
37 |         math.sqrt(25 ** 25 + .01)
38 |     print("Compute done")
39 | 
40 | 
41 | @unsync
42 | async def download_some():
43 |     print("Downloading...")
44 |     url = 'https://talkpython.fm/episodes/show/174/coming-into-python-from-another-industry-part-2'
45 |     async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session:
46 |         async with session.get(url) as resp:
47 |             resp.raise_for_status()
48 | 
49 |             text = await resp.text()
50 | 
51 |     print("Downloaded (more) {:,} characters.".format(len(text)))
52 | 
53 | 
54 | @unsync
55 | def download_some_more():
56 |     print("Downloading more ...")
57 |     url = 'https://pythonbytes.fm/episodes/show/92/will-your-python-be-compiled'
58 |     resp = requests.get(url)
59 |     resp.raise_for_status()
60 | 
61 |     text = resp.text
62 | 
63 |     print("Downloaded {:,} characters.".format(len(text)))
64 | 
65 | 
66 | @unsync
67 | async def wait_some():
68 |     print("Waiting...")
69 |     for _ in range(1, 1000):
70 |         await asyncio.sleep(.001)
71 | 
72 | 
73 | if __name__ == '__main__':
74 |     main()
75 | 


--------------------------------------------------------------------------------
/code/web_scraping/async_scrape/program.py:
--------------------------------------------------------------------------------
 1 | import asyncio
 2 | from asyncio import AbstractEventLoop
 3 | 
 4 | import aiohttp
 5 | import requests
 6 | import bs4
 7 | from colorama import Fore
 8 | 
 9 | 
10 | def main():
11 |     # Create loop
12 |     loop = asyncio.get_event_loop()
13 |     loop.run_until_complete(get_title_range(loop))
14 |     print("Done.")
15 | 
16 | 
17 | async def get_html(episode_number: int) -> str:
18 |     print(Fore.YELLOW + f"Getting HTML for episode {episode_number}", flush=True)
19 | 
20 |     # Make this async with aiohttp's ClientSession
21 |     url = f'https://talkpython.fm/{episode_number}'
22 |     # resp = await requests.get(url)
23 |     # resp.raise_for_status()
24 | 
25 |     async with aiohttp.ClientSession() as session:
26 |         async with session.get(url) as resp:
27 |             resp.raise_for_status()
28 | 
29 |             html = await resp.text()
30 |             return html
31 | 
32 | 
33 | def get_title(html: str, episode_number: int) -> str:
34 |     print(Fore.CYAN + f"Getting TITLE for episode {episode_number}", flush=True)
35 |     soup = bs4.BeautifulSoup(html, 'html.parser')
36 |     header = soup.select_one('h1')
37 |     if not header:
38 |         return "MISSING"
39 | 
40 |     return header.text.strip()
41 | 
42 | 
43 | async def get_title_range(loop: AbstractEventLoop):
44 |     # Please keep this range pretty small to not DDoS my site. ;)
45 |     tasks = []
46 |     for n in range(190, 200):
47 |         tasks.append((loop.create_task(get_html(n)), n))
48 | 
49 |     for task, n in tasks:
50 |         html = await task
51 |         title = get_title(html, n)
52 |         print(Fore.WHITE + f"Title found: {title}", flush=True)
53 | 
54 | 
55 | if __name__ == '__main__':
56 |     main()
57 | 


--------------------------------------------------------------------------------
/code/web_scraping/async_scrape/requirements.txt:
--------------------------------------------------------------------------------
 1 | bs4
 2 | colorama
 3 | 
 4 | aiohttp
 5 | cchardet
 6 | 
 7 | # This one can give you trouble on Windows
 8 | # Feel free to uncomment it but it also works without it
 9 | # Just a little less well.
10 | # aiodns
11 | 


--------------------------------------------------------------------------------
/code/web_scraping/sync_scrape/program.py:
--------------------------------------------------------------------------------
 1 | import requests
 2 | import bs4
 3 | from colorama import Fore
 4 | 
 5 | 
 6 | def main():
 7 |     get_title_range()
 8 |     print("Done.")
 9 | 
10 | 
11 | def get_html(episode_number: int) -> str:
12 |     print(Fore.YELLOW + f"Getting HTML for episode {episode_number}", flush=True)
13 | 
14 |     url = f'https://talkpython.fm/{episode_number}'
15 |     resp = requests.get(url)
16 |     resp.raise_for_status()
17 | 
18 |     return resp.text
19 | 
20 | 
21 | def get_title(html: str, episode_number: int) -> str:
22 |     print(Fore.CYAN + f"Getting TITLE for episode {episode_number}", flush=True)
23 |     soup = bs4.BeautifulSoup(html, 'html.parser')
24 |     header = soup.select_one('h1')
25 |     if not header:
26 |         return "MISSING"
27 | 
28 |     return header.text.strip()
29 | 
30 | 
31 | def get_title_range():
32 |     # Please keep this range pretty small to not DDoS my site. ;)
33 |     for n in range(185, 200):
34 |         html = get_html(n)
35 |         title = get_title(html, n)
36 |         print(Fore.WHITE + f"Title found: {title}", flush=True)
37 | 
38 | 
39 | if __name__ == '__main__':
40 |     main()
41 | 


--------------------------------------------------------------------------------
/code/web_scraping/sync_scrape/requirements.txt:
--------------------------------------------------------------------------------
1 | requests
2 | bs4
3 | colorama
4 | 
5 | 


--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
1 | # Webinar: "Demystifying Python’s async and await Keywords" with Michael Kennedy
2 | 
3 | [![](./readme_resources/webinar.png)](https://blog.jetbrains.com/pycharm/2019/02/webinar-demystifying-pythons-async-and-await-keywords-with-michael-kennedy/)
4 | 
5 | 


--------------------------------------------------------------------------------
/readme_resources/webinar.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeckennedy/async-await-jetbrains-webcast/94e791b7b75d9f54eec90fca1a9117037c6dff72/readme_resources/webinar.png


--------------------------------------------------------------------------------
/slides/async_jetbrains_feb2018.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeckennedy/async-await-jetbrains-webcast/94e791b7b75d9f54eec90fca1a9117037c6dff72/slides/async_jetbrains_feb2018.pdf


--------------------------------------------------------------------------------