├── .gitignore ├── 10.py ├── 20.py ├── 30.py ├── 40.py ├── 50.py ├── README ├── requirements.txt └── server.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | -------------------------------------------------------------------------------- /10.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import time 3 | 4 | def get(path): 5 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 6 | s.connect(('localhost', 5000)) 7 | s.send(('GET %s HTTP/1.0\r\n\r\n' % path).encode()) 8 | 9 | buf = [] 10 | while True: 11 | chunk = s.recv(1000) 12 | if not chunk: 13 | break 14 | buf.append(chunk) 15 | 16 | s.close() 17 | print((b''.join(buf)).decode().split('\n')[0]) 18 | 19 | start = time.time() 20 | get('/foo') 21 | get('/bar') 22 | print('took %.2f seconds' % (time.time() - start)) 23 | -------------------------------------------------------------------------------- /20.py: -------------------------------------------------------------------------------- 1 | from selectors import DefaultSelector, EVENT_WRITE 2 | import socket 3 | import time 4 | 5 | selector = DefaultSelector() 6 | n_jobs = 0 7 | 8 | def get(path): 9 | global n_jobs 10 | n_jobs += 1 11 | s = socket.socket() 12 | s.setblocking(False) 13 | try: 14 | s.connect(('localhost', 5000)) 15 | except BlockingIOError: 16 | pass 17 | selector.register(s.fileno(), EVENT_WRITE, lambda: connected(s, path)) 18 | 19 | def connected(s, path): 20 | global n_jobs 21 | s.send(('GET %s HTTP/1.0\r\n\r\n' % path).encode()) 22 | 23 | buf = [] 24 | while True: 25 | try: 26 | chunk = s.recv(1000) 27 | if not chunk: 28 | break 29 | buf.append(chunk) 30 | except OSError: 31 | pass 32 | 33 | s.close() 34 | print((b''.join(buf)).decode().split('\n')[0]) 35 | n_jobs -= 1 36 | 37 | start = time.time() 38 | get('/foo') 39 | get('/bar') 40 | 41 | while n_jobs: 42 | events = selector.select() 43 | for key, mask in events: 44 | callback = key.data 45 | callback() 46 | 47 | print('took %.2f seconds' % (time.time() - start)) 48 | -------------------------------------------------------------------------------- /30.py: -------------------------------------------------------------------------------- 1 | from selectors import DefaultSelector, EVENT_WRITE, EVENT_READ 2 | import socket 3 | import time 4 | 5 | selector = DefaultSelector() 6 | n_jobs = 0 7 | 8 | def get(path): 9 | global n_jobs 10 | n_jobs += 1 11 | s = socket.socket() 12 | s.setblocking(False) 13 | try: 14 | s.connect(('localhost', 5000)) 15 | except BlockingIOError: 16 | pass 17 | selector.register(s.fileno(), EVENT_WRITE, lambda: connected(s, path)) 18 | 19 | def connected(s, path): 20 | selector.unregister(s.fileno()) 21 | s.send(('GET %s HTTP/1.0\r\n\r\n' % path).encode()) 22 | buf = [] 23 | selector.register(s.fileno(), EVENT_READ, lambda: readable(s, buf)) 24 | 25 | def readable(s, buf): 26 | global n_jobs 27 | chunk = s.recv(1000) 28 | if chunk: 29 | buf.append(chunk) 30 | else: 31 | # Finished. 32 | selector.unregister(s.fileno()) 33 | s.close() 34 | print((b''.join(buf)).decode().split('\n')[0]) 35 | n_jobs -= 1 36 | 37 | start = time.time() 38 | get('/foo') 39 | get('/bar') 40 | 41 | while n_jobs: 42 | events = selector.select() 43 | for key, mask in events: 44 | callback = key.data 45 | callback() 46 | 47 | print('took %.2f seconds' % (time.time() - start)) 48 | -------------------------------------------------------------------------------- /40.py: -------------------------------------------------------------------------------- 1 | from selectors import DefaultSelector, EVENT_WRITE, EVENT_READ 2 | import socket 3 | import time 4 | 5 | selector = DefaultSelector() 6 | n_jobs = 0 7 | 8 | class Future: 9 | def __init__(self): 10 | self.callback = None 11 | 12 | def resolve(self): 13 | self.callback() 14 | 15 | def get(path): 16 | global n_jobs 17 | n_jobs += 1 18 | s = socket.socket() 19 | s.setblocking(False) 20 | try: 21 | s.connect(('localhost', 5000)) 22 | except BlockingIOError: 23 | pass 24 | 25 | f = Future() 26 | f.callback = lambda: connected(s, path) 27 | selector.register(s.fileno(), EVENT_WRITE, f) 28 | 29 | def connected(s, path): 30 | selector.unregister(s.fileno()) 31 | s.send(('GET %s HTTP/1.0\r\n\r\n' % path).encode()) 32 | buf = [] 33 | f = Future() 34 | f.callback = lambda: readable(s, buf) 35 | selector.register(s.fileno(), EVENT_READ, f) 36 | 37 | def readable(s, buf): 38 | global n_jobs 39 | selector.unregister(s.fileno()) 40 | chunk = s.recv(1000) 41 | if chunk: 42 | buf.append(chunk) 43 | f = Future() 44 | f.callback = lambda: readable(s, buf) 45 | selector.register(s.fileno(), EVENT_READ, f) 46 | else: 47 | # Finished. 48 | s.close() 49 | print((b''.join(buf)).decode().split('\n')[0]) 50 | n_jobs -= 1 51 | 52 | start = time.time() 53 | get('/foo') 54 | get('/bar') 55 | 56 | while n_jobs: 57 | events = selector.select() 58 | for key, mask in events: 59 | future = key.data 60 | future.resolve() 61 | 62 | print('took %.2f seconds' % (time.time() - start)) 63 | -------------------------------------------------------------------------------- /50.py: -------------------------------------------------------------------------------- 1 | from selectors import DefaultSelector, EVENT_WRITE, EVENT_READ 2 | import socket 3 | import time 4 | 5 | selector = DefaultSelector() 6 | n_jobs = 0 7 | 8 | class Future: 9 | def __init__(self): 10 | self.callback = None 11 | 12 | def resolve(self): 13 | self.callback() 14 | 15 | def __await__(self): 16 | yield self 17 | 18 | class Task: 19 | def __init__(self, coro): 20 | self.coro = coro 21 | self.step() 22 | 23 | def step(self): 24 | try: 25 | f = self.coro.send(None) 26 | except StopIteration: 27 | return 28 | 29 | f.callback = self.step 30 | 31 | async def get(path): 32 | global n_jobs 33 | n_jobs += 1 34 | s = socket.socket() 35 | s.setblocking(False) 36 | try: 37 | s.connect(('localhost', 5000)) 38 | except BlockingIOError: 39 | pass 40 | 41 | f = Future() 42 | selector.register(s.fileno(), EVENT_WRITE, f) 43 | await f 44 | selector.unregister(s.fileno()) 45 | 46 | s.send(('GET %s HTTP/1.0\r\n\r\n' % path).encode()) 47 | buf = [] 48 | 49 | while True: 50 | f = Future() 51 | selector.register(s.fileno(), EVENT_READ, f) 52 | await f 53 | selector.unregister(s.fileno()) 54 | chunk = s.recv(1000) 55 | if chunk: 56 | buf.append(chunk) 57 | else: 58 | break 59 | 60 | # Finished. 61 | print((b''.join(buf)).decode().split('\n')[0]) 62 | n_jobs -= 1 63 | 64 | start = time.time() 65 | Task(get('/foo')) 66 | Task(get('/bar')) 67 | 68 | while n_jobs: 69 | events = selector.select() 70 | for key, mask in events: 71 | future = key.data 72 | future.resolve() 73 | 74 | print('took %.2f seconds' % (time.time() - start)) 75 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | How Do Python Coroutines Work? 2 | ============================== 3 | 4 | Script for live-coding an implementation of Python async coroutines 5 | analogous to asyncio.coroutine in the Python 3.4 standard library: 6 | 7 | https://pygotham.org/2015/talks/162/how-do-python-coroutines/ 8 | 9 | tornado.gen.coroutine is an earlier inspiration for the same idea. 10 | 11 | The example program is a "web crawler" that fetches only two URLs. 12 | Files are named in order. 13 | 14 | 10. The first version does a blocking fetch of the two URLs, one at a time. 15 | 20. Then it puts its sockets in non-blocking mode and uses an event loop and 16 | callbacks to connect asynchronously. Still fetches serially. 17 | 30. Add more callbacks to fetch asynchronously. 18 | 40. Add a "Future" to abstract callbacks. 19 | 50. Introduce "Task" and replace callbacks with generators. The combination 20 | of Task, Future, and generators is a coroutine implementation. 21 | 22 | To run the examples, first start server.py in a separate terminal session. 23 | (It requires Flask.) server.py is designed to be slow - each URL takes about 24 | a second to download. This provides a good example of async's strength suit. 25 | 26 | The material for this demo is adapted from a chapter I wrote with Guido van 27 | Rossum for an upcoming book in the Architecture of Open Source Applications 28 | series: 29 | 30 | https://github.com/aosabook/500lines/blob/master/crawler/crawler.markdown 31 | 32 | The chapter presents a far more sophisticated code example than is demo'ed 33 | here, and covers the relevant ideas in much greater depth and detail. 34 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | flask 2 | -------------------------------------------------------------------------------- /server.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | 3 | from flask import Flask, Response 4 | 5 | 6 | app = Flask(__name__) 7 | 8 | 9 | message = b'Hello PyTennessee! ' * 100 + b'\n' 10 | CHUNK_LEN = 100 11 | N_CHUNKS = len(message) / CHUNK_LEN 12 | 13 | @app.route("/foo") 14 | @app.route("/bar") 15 | def hello(): 16 | def generate(): 17 | i = 0 18 | while True: 19 | chunk = message[i:i + CHUNK_LEN] 20 | if chunk: 21 | yield chunk 22 | sleep(0.877 / N_CHUNKS) 23 | i += CHUNK_LEN 24 | else: 25 | break 26 | 27 | return Response(generate()) 28 | 29 | if __name__ == "__main__": 30 | app.run(threaded=True) 31 | --------------------------------------------------------------------------------