├── tests ├── __init__.py ├── test_gather.py ├── test_sleep.py ├── test_dot.py └── test_wait.py ├── .pre-commit-config.yaml ├── awaitwhat ├── wait.py ├── helpers.py ├── .clang-format ├── utils.py ├── wait_for.py ├── sleep.py ├── dot.py ├── gather.py ├── blocker.py ├── __init__.py ├── stack.py ├── what.c └── graph.py ├── .gitignore ├── examples ├── test_signal.py ├── test_wait_for.py ├── test_stack.py ├── test_same_future.py ├── test_shield.py └── test_future.py ├── license.txt ├── pyproject.toml ├── readme.md └── doc └── test_future.svg /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: 21.6b0 4 | hooks: 5 | - id: black 6 | python_version: python3.9 7 | -------------------------------------------------------------------------------- /awaitwhat/wait.py: -------------------------------------------------------------------------------- 1 | import asyncio.tasks 2 | 3 | 4 | def mine(frame): 5 | return asyncio.tasks._wait.__code__ == frame.f_code 6 | 7 | 8 | def decode(frame): 9 | pass 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | /build/ 3 | /dist/ 4 | __pycache__/ 5 | /*.egg-info/ 6 | # poetry install will add the compiled module here 😡 7 | /awaitwhat/*.so 8 | graph.dot 9 | graph.svg 10 | .vscode/ 11 | -------------------------------------------------------------------------------- /awaitwhat/helpers.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import signal 3 | 4 | from awaitwhat import dot 5 | 6 | 7 | def signal_handler(signal_number, frame): 8 | tt = asyncio.all_tasks() 9 | print(dot.dumps(tt)) 10 | 11 | 12 | def register_signal(signal_number=signal.SIGWINCH): 13 | signal.signal(signal_number, signal_handler) 14 | -------------------------------------------------------------------------------- /examples/test_signal.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import awaitwhat 3 | import signal 4 | 5 | 6 | async def main(): 7 | await job() 8 | 9 | 10 | async def job(): 11 | await foo() 12 | 13 | 14 | async def foo(): 15 | signal.alarm(1) 16 | await asyncio.sleep(1) 17 | 18 | 19 | if __name__ == "__main__": 20 | awaitwhat.helpers.register_signal(signal.SIGALRM) 21 | asyncio.run(main()) 22 | -------------------------------------------------------------------------------- /awaitwhat/.clang-format: -------------------------------------------------------------------------------- 1 | # PEP 7 2 | BasedOnStyle: Google 3 | AllowShortLoopsOnASingleLine: false 4 | AllowShortIfStatementsOnASingleLine: false 5 | AlwaysBreakAfterReturnType: All 6 | AlignAfterOpenBracket: Align 7 | BreakBeforeBraces: Stroustrup 8 | ColumnLimit: 79 9 | DerivePointerAlignment: false 10 | IndentWidth: 4 11 | Language: Cpp 12 | PointerAlignment: Right 13 | ReflowComments: true 14 | SpaceBeforeParens: ControlStatements 15 | SpacesInParentheses: false 16 | TabWidth: 4 17 | UseTab: Never 18 | -------------------------------------------------------------------------------- /awaitwhat/utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | 4 | def concise_stack_trace(trace): 5 | def clean(line): 6 | if line.startswith("Stack for "): 7 | return 8 | if '""' in line: 9 | return 10 | if re.search('File ".*/site-packages/.*"', line): 11 | line = re.sub('[^"]*/site-packages/', "", line) 12 | if re.search('File ".*/lib/python[0-9][.][0-9]/.*"', line): 13 | line = re.sub('[^"]*/lib/python[0-9][.][0-9]/', "", line) 14 | return line 15 | 16 | return "\n".join(filter(None, (clean(line) for line in trace.split("\n")))) 17 | -------------------------------------------------------------------------------- /awaitwhat/wait_for.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | def mine(frame): 5 | return asyncio.wait_for.__code__ == frame.f_code 6 | 7 | 8 | def decode(frame): 9 | try: 10 | timeout = frame.f_locals["timeout"] 11 | except Exception: 12 | timeout = "?" 13 | try: 14 | timeout_handle_deadline = frame.f_locals["timeout_handle"].when() 15 | now = frame.f_locals["waiter"].get_loop().time() 16 | timeout_remaining = timeout_handle_deadline - now 17 | except Exception: 18 | timeout_remaining = "?" 19 | try: 20 | awaitable = frame.f_locals["fut"] 21 | except Exception: 22 | awaitable = None 23 | return [ 24 | f"asyncio.wait_for: timeout {timeout} remaining {timeout_remaining}", 25 | awaitable, 26 | ] 27 | -------------------------------------------------------------------------------- /awaitwhat/sleep.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | def mine(frame): 5 | return asyncio.sleep.__code__ == frame.f_code 6 | 7 | 8 | def decode(frame): 9 | try: 10 | delay = frame.f_locals["delay"] 11 | except Exception: 12 | delay = "?" 13 | 14 | try: 15 | deadline = frame.f_locals["h"].when() 16 | now = frame.f_locals["future"].get_loop().time() 17 | remaining = deadline - now 18 | except Exception: 19 | remaining = "?" 20 | 21 | try: 22 | scheduled = frame.f_locals["h"]._scheduled 23 | cancelled = frame.f_locals["h"]._cancelled 24 | state = ( 25 | "cancelled" if cancelled else "scheduled" if scheduled else "not started" 26 | ) 27 | except Exception: 28 | state = "?" 29 | 30 | return f"asyncio.sleep: state {state} delay {delay} remaining {remaining}" 31 | -------------------------------------------------------------------------------- /awaitwhat/dot.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from . import graph 4 | 5 | 6 | def label(vertex): 7 | """Graph vertex label in dot format""" 8 | label = f"{vertex.name} {vertex.state or ''}\n{vertex.traceback or ''}" 9 | if not label.endswith("\n"): 10 | label += "\n" 11 | label = json.dumps(label).replace("\\n", r"\l") 12 | return f"[label={label}]" 13 | 14 | 15 | def dumps(tasks): 16 | """ 17 | Renders Task dependency graph in graphviz format. 18 | Returns a string. 19 | """ 20 | 21 | prefix = "\n " 22 | 23 | vertices, edges = graph.new(tasks) 24 | vertices = prefix.join(f"{id(vertex.task)} {label(vertex)}" for vertex in vertices) 25 | edges = prefix.join(f"{id(edge.src.task)} -> {id(edge.dst.task)}" for edge in edges) 26 | 27 | return f""" 28 | digraph {{ 29 | node [shape="note", fontname="Courier New"]; 30 | {vertices} 31 | {edges} 32 | }} 33 | """.strip() 34 | -------------------------------------------------------------------------------- /awaitwhat/gather.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | 3 | # work in progress: 4 | 5 | # when a task is ultimately waiting on gather(), _done_callback is visible. 6 | 7 | 8 | def decipher_done_callback(done_callback): 9 | closure = inspect.getclosurevars(done_callback).nonlocals 10 | state = f"progress: {closure['nfinished']}/{closure['nfuts']}" 11 | future = closure["outer"] 12 | print(future, state) 13 | children = closure["children"] 14 | for child in children: 15 | if child.done(): 16 | # resolved, exception, cancelled 17 | if child.cancelled(): 18 | print(child, "cancelled") 19 | elif child.exception(): 20 | print(child, child.exception()) 21 | else: 22 | print(child, child.result()) 23 | else: 24 | print(child, "pending") 25 | 26 | 27 | def task_callback(t): 28 | # FIXME: there may be several; py 3.8 vs py 3.7 29 | return t._callbacks[0][0] 30 | -------------------------------------------------------------------------------- /examples/test_wait_for.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from examples.test_shield import frob_a_branch 3 | import awaitwhat 4 | 5 | 6 | FUTURES = [] 7 | 8 | 9 | async def main(): 10 | t = asyncio.create_task(test()) 11 | await do_work() 12 | await t 13 | 14 | 15 | async def do_work(): 16 | loop = asyncio.get_running_loop() 17 | f = loop.create_future() 18 | FUTURES.append(f) 19 | branch = frob_a_branch(f) 20 | await asyncio.wait_for(branch, 15) 21 | 22 | 23 | async def frob_a_branch(f): 24 | await f 25 | 26 | 27 | def name(t): 28 | try: 29 | # Python3.8 30 | return t.get_name() 31 | except AttributeError: 32 | return f"Task-{id(t)}" 33 | 34 | 35 | async def test(): 36 | import sys 37 | 38 | await asyncio.sleep(1) 39 | try: 40 | tt = asyncio.all_tasks() 41 | print(awaitwhat.dot.dumps(tt)) 42 | finally: 43 | for f in FUTURES: 44 | f.set_result(None) 45 | 46 | 47 | if __name__ == "__main__": 48 | asyncio.run(main()) 49 | -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Dima Tisnek 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tests/test_gather.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import awaitwhat.blocker 3 | from awaitwhat.stack import task_get_stack 4 | 5 | 6 | async def a(): 7 | await asyncio.gather(asyncio.sleep(0.01), asyncio.sleep(999 + 1)) 8 | 9 | 10 | def fixme_dont_test_sleep(): 11 | async def test(): 12 | t = asyncio.create_task(a()) 13 | try: 14 | await asyncio.sleep(0.11) 15 | 16 | stack = task_get_stack(t, None) 17 | # assert awaitwhat.sleep.mine(stack[-2]) 18 | # text = awaitwhat.sleep.decode(stack[-2]) 19 | # assert "asyncio.sleep" in text 20 | # assert "scheduled" in text 21 | # assert "delay 42" in text 22 | # assert "remaining 41.8" in text 23 | finally: 24 | t.cancel() 25 | 26 | asyncio.run(test()) 27 | 28 | 29 | def test_blockers(): 30 | async def test(): 31 | t = asyncio.create_task(a()) 32 | try: 33 | await asyncio.sleep(0.11) 34 | 35 | text = str(awaitwhat.blocker.blockers(t)) 36 | assert "Task finished" in text 37 | assert "Task pending" in text 38 | finally: 39 | t.cancel() 40 | 41 | asyncio.run(test()) 42 | -------------------------------------------------------------------------------- /awaitwhat/blocker.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import random 3 | from . import sleep 4 | from .stack import task_get_stack 5 | from . import wait_for 6 | 7 | 8 | def blockers(task): 9 | """What does this task wait for?""" 10 | waiter = task._fut_waiter 11 | if not waiter: 12 | return [f""] 13 | 14 | stack = task_get_stack(task, None) 15 | 16 | if len(stack) > 2: 17 | if sleep.mine(stack[-2]): 18 | return [sleep.decode(stack[-2])] 19 | elif wait_for.mine(stack[-2]): 20 | status, awaitable = wait_for.decode(stack[-2]) 21 | return [status, awaitable] 22 | 23 | # asyncio.gather() 24 | try: 25 | # ideally check `w` type 26 | return waiter._children 27 | except AttributeError: 28 | pass 29 | 30 | # FIXME shield should be shown as a frame on the top of the stack 31 | # asyncio.shield() 32 | try: 33 | # ideally check if it's an `_outer_done_callback` 34 | callback, _ctx = waiter._callbacks[0] 35 | return [inspect.getclosurevars(callback).nonlocals["inner"]] 36 | except (AttributeError, TypeError, IndexError): 37 | pass 38 | 39 | # FIXME other awaitables 40 | return [waiter] 41 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "awaitwhat" 3 | version = "24.11" 4 | description = "async/await introspection" 5 | authors = [{name="Dima Tisnek", email="dimaqq@gmail.com"}] 6 | license = { file = "LICENSE" } 7 | readme = "readme.md" 8 | keywords = ["asyncio"] 9 | classifiers = [ 10 | "Development Status :: 3 - Alpha", 11 | "Programming Language :: Python :: Implementation :: CPython", 12 | "Framework :: AsyncIO", 13 | "Topic :: Software Development :: Debuggers", 14 | "Programming Language :: Python :: 3.6", 15 | "Programming Language :: Python :: 3.7", 16 | "Programming Language :: Python :: 3.8", 17 | "Programming Language :: Python :: 3.9", 18 | "Operating System :: POSIX :: Linux", 19 | "Operating System :: MacOS :: MacOS X", 20 | "Operating System :: Microsoft :: Windows", 21 | ] 22 | 23 | [project.urls] 24 | homepage = "https://github.com/dimaqq/awaitwhat" 25 | repository = "https://github.com/dimaqq/awaitwhat" 26 | documentation = "https://github.com/dimaqq/awaitwhat" 27 | 28 | [build-system] 29 | requires = ["setuptools>=42", "wheel"] 30 | build-backend = "setuptools.build_meta" 31 | 32 | [tool.setuptools] 33 | packages = ["awaitwhat"] 34 | ext-modules = [ 35 | { name = "awaitwhat._what", sources = ["awaitwhat/what.c"] } 36 | ] 37 | -------------------------------------------------------------------------------- /examples/test_stack.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import awaitwhat 3 | import sys 4 | 5 | 6 | def trace_all_tasks(): 7 | print("### Python native task stack traces") 8 | for t in asyncio.Task.all_tasks(): 9 | asyncio.base_tasks._task_print_stack(t, None, sys.stdout) 10 | print() 11 | 12 | print("### Extended task stack traces") 13 | for t in asyncio.Task.all_tasks(): 14 | awaitwhat.stack.task_print_stack(t, None, sys.stdout) 15 | print() 16 | 17 | tt = asyncio.Task.all_tasks() 18 | t = list(tt)[0] 19 | print(t) 20 | cb = awaitwhat.gather.task_callback(t) 21 | try: 22 | awaitwhat.gather.decipher_done_callback(cb) 23 | except Exception as e: 24 | print("failed to decipher", e) 25 | # __import__("pdb").set_trace() 26 | 27 | 28 | async def tester(): 29 | await asyncio.sleep(0.1) 30 | trace_all_tasks() 31 | 32 | 33 | async def job(): 34 | await foo() 35 | 36 | 37 | async def foo(): 38 | await bar() 39 | 40 | 41 | async def bar(): 42 | await baz() 43 | 44 | 45 | async def baz(): 46 | await leaf() 47 | 48 | 49 | async def leaf(): 50 | await asyncio.sleep(1) 51 | 52 | 53 | async def work(): 54 | await asyncio.gather(tester(), job()) 55 | 56 | 57 | if __name__ == "__main__": 58 | asyncio.run(work()) 59 | -------------------------------------------------------------------------------- /tests/test_sleep.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import awaitwhat.sleep 3 | import awaitwhat.blocker 4 | from awaitwhat.stack import task_get_stack 5 | 6 | 7 | async def a(): 8 | await b() 9 | 10 | 11 | async def b(): 12 | await asyncio.sleep(42) 13 | 14 | 15 | def test_sleep(): 16 | async def test(): 17 | t = asyncio.create_task(a()) 18 | try: 19 | await asyncio.sleep(0.11) 20 | 21 | stack = task_get_stack(t, None) 22 | assert awaitwhat.sleep.mine(stack[-2]) 23 | text = awaitwhat.sleep.decode(stack[-2]) 24 | assert "asyncio.sleep" in text 25 | assert "scheduled" in text 26 | assert "delay 42" in text 27 | assert "remaining 41.8" in text 28 | finally: 29 | t.cancel() 30 | 31 | asyncio.run(test()) 32 | 33 | 34 | def test_blockers(): 35 | async def test(): 36 | t = asyncio.create_task(a()) 37 | try: 38 | await asyncio.sleep(0.11) 39 | 40 | (text,) = awaitwhat.blocker.blockers(t) 41 | assert "asyncio.sleep" in text 42 | assert "scheduled" in text 43 | assert "delay 42" in text 44 | assert "remaining 41.8" in text 45 | finally: 46 | t.cancel() 47 | 48 | asyncio.run(test()) 49 | -------------------------------------------------------------------------------- /examples/test_same_future.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import awaitwhat 3 | 4 | 5 | global_futures = list() # Structure to hold a global future 6 | 7 | 8 | async def main(): 9 | # generate a global future 10 | global_futures.append(asyncio.Future()) 11 | t = asyncio.create_task(test()) 12 | await do_work() 13 | await t 14 | 15 | 16 | async def do_work(): 17 | futs = [frob_a_tree() for i in range(3)] 18 | await asyncio.gather(*futs) 19 | 20 | 21 | async def frob_a_tree(): 22 | await b_tree() 23 | 24 | 25 | async def b_tree(): 26 | # Create a task with a global future 27 | f = global_futures[-1] 28 | t = asyncio.create_task(frob_a_branch(f)) 29 | await asyncio.gather(t) 30 | 31 | 32 | async def frob_a_branch(f): 33 | await b_branch(f) 34 | 35 | 36 | async def b_branch(f): 37 | await f 38 | 39 | 40 | def name(t): 41 | try: 42 | # Python3.8 43 | return t.get_name() 44 | except AttributeError: 45 | return f"Task-{id(t)}" 46 | 47 | 48 | async def test(): 49 | import sys 50 | 51 | await asyncio.sleep(0.1) 52 | 53 | try: 54 | tt = asyncio.all_tasks() 55 | print(awaitwhat.dot.dumps(tt)) 56 | finally: 57 | global_futures[-1].set_result(None) 58 | 59 | 60 | if __name__ == "__main__": 61 | asyncio.run(main()) 62 | -------------------------------------------------------------------------------- /examples/test_shield.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import awaitwhat 3 | 4 | 5 | FUTURES = [] 6 | 7 | 8 | async def main(): 9 | t = asyncio.create_task(test()) 10 | await do_work() 11 | await t 12 | 13 | 14 | async def do_work(): 15 | futs = [frob_a_tree() for i in range(1)] 16 | await asyncio.gather(*futs) 17 | 18 | 19 | async def frob_a_tree(): 20 | await b_tree() 21 | 22 | 23 | async def b_tree(): 24 | f = asyncio.Future() 25 | FUTURES.append(f) 26 | await asyncio.shield(frob_a_branch(f)) 27 | 28 | 29 | async def frob_a_branch(f): 30 | await b_branch(f) 31 | 32 | 33 | async def b_branch(f): 34 | await f 35 | 36 | 37 | def name(t): 38 | try: 39 | # Python3.8 40 | return t.get_name() 41 | except AttributeError: 42 | return f"Task-{id(t)}" 43 | 44 | 45 | async def test(): 46 | import sys 47 | 48 | await asyncio.sleep(0.1) 49 | # print("### Python native") 50 | # for t in asyncio.all_tasks(): 51 | # print(name(t)) 52 | # asyncio.base_tasks._task_print_stack(t, None, sys.stdout) 53 | # print() 54 | 55 | # Extended stack 56 | # import awaitwhat 57 | 58 | try: 59 | tt = asyncio.all_tasks() 60 | print(awaitwhat.dot.dumps(tt)) 61 | finally: 62 | for f in FUTURES: 63 | f.set_result(None) 64 | 65 | 66 | if __name__ == "__main__": 67 | asyncio.run(main()) 68 | -------------------------------------------------------------------------------- /examples/test_future.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import awaitwhat 3 | 4 | 5 | FUTURES = [] 6 | 7 | 8 | async def main(): 9 | t = asyncio.create_task(test()) 10 | await do_work() 11 | await t 12 | 13 | 14 | async def do_work(): 15 | futs = [frob_a_tree() for i in range(3)] 16 | await asyncio.gather(*futs) 17 | 18 | 19 | async def frob_a_tree(): 20 | await b_tree() 21 | 22 | 23 | async def b_tree(): 24 | f = asyncio.Future() 25 | FUTURES.append(f) 26 | t = asyncio.create_task(frob_a_branch(f)) 27 | await asyncio.gather(t) 28 | 29 | 30 | async def frob_a_branch(f): 31 | await b_branch(f) 32 | 33 | 34 | async def b_branch(f): 35 | await f 36 | 37 | 38 | def name(t): 39 | try: 40 | # Python3.8 41 | return t.get_name() 42 | except AttributeError: 43 | return f"Task-{id(t)}" 44 | 45 | 46 | async def test(): 47 | import sys 48 | 49 | await asyncio.sleep(0.1) 50 | # print("### Python native") 51 | # for t in asyncio.all_tasks(): 52 | # print(name(t)) 53 | # asyncio.base_tasks._task_print_stack(t, None, sys.stdout) 54 | # print() 55 | 56 | # Extended stack 57 | # import awaitwhat 58 | 59 | try: 60 | tt = asyncio.all_tasks() 61 | print(awaitwhat.dot.dumps(tt)) 62 | finally: 63 | for f in FUTURES: 64 | f.set_result(None) 65 | 66 | 67 | if __name__ == "__main__": 68 | asyncio.run(main()) 69 | -------------------------------------------------------------------------------- /tests/test_dot.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import awaitwhat 3 | import sys 4 | 5 | 6 | def test_stack(): 7 | FUTURES = [] 8 | 9 | async def main(): 10 | t = asyncio.create_task(test()) 11 | await do_work() 12 | await t 13 | 14 | async def do_work(): 15 | futs = [frob_a_tree() for i in range(1)] 16 | await asyncio.gather(*futs) 17 | 18 | async def frob_a_tree(): 19 | await b_tree() 20 | 21 | async def b_tree(): 22 | f = asyncio.Future() 23 | FUTURES.append(f) 24 | await asyncio.shield(frob_a_branch(f)) 25 | 26 | async def frob_a_branch(f): 27 | await b_branch(f) 28 | 29 | async def b_branch(f): 30 | await f 31 | 32 | def name(t): 33 | try: 34 | # Python3.8 35 | return t.get_name() 36 | except AttributeError: 37 | return f"Task-{id(t)}" 38 | 39 | async def test(): 40 | import sys 41 | 42 | await asyncio.sleep(0.1) 43 | # print("### Python native") 44 | # for t in asyncio.all_tasks(): 45 | # print(name(t)) 46 | # asyncio.base_tasks._task_print_stack(t, None, sys.stdout) 47 | # print() 48 | 49 | # Extended stack 50 | # import awaitwhat 51 | 52 | try: 53 | tt = asyncio.all_tasks() 54 | print(awaitwhat.dot.dumps(tt)) 55 | finally: 56 | for f in FUTURES: 57 | f.set_result(None) 58 | 59 | asyncio.run(main()) 60 | -------------------------------------------------------------------------------- /tests/test_wait.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import pytest 3 | import awaitwhat.wait 4 | import awaitwhat.blocker 5 | from awaitwhat.stack import task_get_stack 6 | 7 | 8 | async def a(): 9 | await asyncio.wait(b(), timeout=123) 10 | 11 | 12 | async def b(): 13 | await asyncio.sleep(42) 14 | 15 | 16 | async def debug(): 17 | t = asyncio.create_task(a()) 18 | await asyncio.sleep(0.11) 19 | stack = task_get_stack(t, None) 20 | return stack 21 | 22 | 23 | @pytest.mark.xfail(reason="asyncio.wait support incomplete #6") 24 | def test_wait(): 25 | async def test(): 26 | t = asyncio.create_task(a()) 27 | try: 28 | await asyncio.sleep(0.11) 29 | 30 | stack = task_get_stack(t, None) 31 | # FIXME this is broken now 32 | assert awaitwhat.wait.mine(stack[-2]) 33 | text = awaitwhat.wait.decode(stack[-2]) 34 | assert "asyncio.wait" in text 35 | assert "scheduled" in text 36 | assert "delay 42" in text 37 | assert "remaining 41.8" in text 38 | finally: 39 | t.cancel() 40 | 41 | return asyncio.run(test()) 42 | 43 | 44 | @pytest.mark.xfail(reason="asyncio.wait support incomplete #6") 45 | def test_blockers(): 46 | async def test(): 47 | t = asyncio.create_task(a()) 48 | try: 49 | await asyncio.sleep(0.11) 50 | 51 | text = str(awaitwhat.blocker.blockers(t)) 52 | assert "asyncio.wait" in text 53 | assert "scheduled" in text 54 | assert "delay 42" in text 55 | assert "remaining 41.8" in text 56 | finally: 57 | t.cancel() 58 | 59 | asyncio.run(test()) 60 | -------------------------------------------------------------------------------- /awaitwhat/__init__.py: -------------------------------------------------------------------------------- 1 | from asyncio.base_tasks import _task_print_stack, _task_get_stack 2 | import types 3 | from . import _what 4 | from . import gather 5 | from . import stack 6 | from . import dot 7 | from . import blocker 8 | from . import helpers 9 | 10 | # FIXME: shorter to do 11 | # 12 | # - mock up a run - 3xTask - coro - 3xTask - naked Future scenario 13 | # 14 | # - report Task id's (everywhere) 15 | # - bridge _GatheringFuture to children (Tasks) 16 | # 17 | # - unwrap TaskWakeupMethWrapper (work forwards) 18 | # or 19 | # - ungather leaf gather calls (work backwards) 20 | 21 | # FIXME: what can be awaited: 22 | # 23 | # - coro (done) 24 | # - Task (enumerated, but needs a bridge) 25 | # - Future (naked, unclear who ought to resolve it) 26 | # - Event (e.wait() is a coro, but needs a bridge?) 27 | # - gather (via callback) 28 | # - shield (via callback) 29 | # - asyncio.run (via callback) 30 | # 31 | # - TaskWakeupMethWrapper (via callback) 32 | # - bridge FutureIter to Future (or just rely on Task wait_for=<...> 33 | 34 | # FIXME: callbacks: 35 | # 36 | # top-level event loop thing: 37 | # cb=[_run_until_complete_cb() at /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/base_events.py:158 38 | # 39 | # gather: 40 | # cb=[gather.._done_callback() at /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/tasks.py:664 41 | # 42 | # shield: 43 | # ... 44 | 45 | 46 | # TODO 47 | # 48 | # When a coro is blocked on gather(), the thing on the stack is: 49 | # <_asyncio.FutureIter object at 0x1086b1040> 50 | # 51 | # This is because instruction preceding YIELD_FROM is GET_AWAITABLE 52 | # Which converts Future into an iterator: 53 | # 54 | # https://github.com/python/cpython/blob/51aac15f6d525595e200e3580409c4b8656e8a96/Modules/_asynciomodule.c#L1633 55 | -------------------------------------------------------------------------------- /awaitwhat/stack.py: -------------------------------------------------------------------------------- 1 | """ Shows what a coroutine waits for """ 2 | import asyncio 3 | import types 4 | from . import _what 5 | 6 | 7 | class FakeCode: 8 | co_filename = "" 9 | co_name = None 10 | 11 | def __init__(self, name): 12 | self.co_name = name 13 | 14 | 15 | class FakeFrame: 16 | f_lineno = 0 17 | f_globals = None 18 | 19 | def __init__(self, name): 20 | self.f_code = FakeCode(name) 21 | 22 | 23 | def extend_stack(s, limit=None): 24 | stack = s[:] 25 | while stack and isinstance(stack[-1], types.FrameType): 26 | if limit is not None and limit >= len(stack): 27 | break 28 | try: 29 | n = _what.next(stack[-1]) 30 | except Exception as e: 31 | n = str(e) 32 | try: 33 | f = n.cr_frame 34 | except Exception as e: 35 | f = FakeFrame(f"{n}: {e}") 36 | stack.append(f) 37 | return stack 38 | 39 | 40 | def task_get_stack(task, limit): 41 | stack = asyncio.base_tasks._task_get_stack(task, limit) 42 | if limit is None or len(stack) < limit: 43 | # FIXME should the stack be extended if there's an exception? 44 | stack = extend_stack(stack, limit) 45 | return stack 46 | 47 | 48 | class Wrapper: 49 | def __init__(self, task): 50 | self.task = task 51 | 52 | def __str__(self): 53 | return str(self.task) 54 | 55 | def __repr__(self): 56 | return repr(self.task) 57 | 58 | @property 59 | def _exception(self): 60 | return self.task._exception 61 | 62 | def get_stack(self, limit=None): 63 | # FIXME, should the stack be extended if there's an exception? 64 | return task_get_stack(self.task, limit) 65 | 66 | 67 | def task_print_stack(task, limit, file): 68 | return asyncio.base_tasks._task_print_stack(Wrapper(task), limit, file) 69 | -------------------------------------------------------------------------------- /awaitwhat/what.c: -------------------------------------------------------------------------------- 1 | #define PY_SSIZE_T_CLEAN 2 | #include 3 | #include "frameobject.h" 4 | 5 | static PyObject * 6 | what_next(PyObject *self, PyObject *arg) 7 | { 8 | if (!PyFrame_Check(arg)) { 9 | PyErr_SetString(PyExc_TypeError, "Argument must be a frame object."); 10 | return NULL; 11 | } 12 | 13 | PyFrameObject *frame = (PyFrameObject *)arg; 14 | 15 | // FIXME validate that current instruction is YIELD_FROM 16 | // Due to optimisations, it may be the preceding GET_AWAITABLE 17 | // int opcode = _Py_OPCODE(*next_instr); 18 | 19 | // printf("val %p top %p delta value to top: %ld\n", 20 | // frame->f_valuestack, 21 | // frame->f_stacktop, 22 | // (long)(frame->f_stacktop - frame->f_valuestack)); 23 | 24 | PyObject **stack_pointer = frame->f_stacktop; 25 | 26 | if (frame->f_stacktop == frame->f_valuestack) { 27 | PyErr_SetString(PyExc_ValueError, "stack is empty"); 28 | return NULL; 29 | } 30 | 31 | if (!stack_pointer) { 32 | PyErr_SetString(PyExc_ValueError, "stack pointer is null?"); 33 | return NULL; 34 | } 35 | 36 | // Top-most on the value stack ought to be the "receiver". 37 | PyObject *next = stack_pointer[-1]; 38 | 39 | if (!next) { 40 | PyErr_SetString(PyExc_SystemError, "thing on stack is null?"); 41 | return NULL; 42 | } 43 | 44 | Py_INCREF(next); 45 | return next; 46 | } 47 | 48 | static PyMethodDef what_functions[] = { 49 | {"next", what_next, METH_O, 50 | "Next frame for the given frame."}, /* FIXME: function docstring */ 51 | {NULL, NULL, 0, NULL}}; 52 | 53 | static struct PyModuleDef whatmodule = { 54 | PyModuleDef_HEAD_INIT, "what", /* name of module */ 55 | NULL, /* FIXME: module docstring, module documentation, may be NULL */ 56 | 0, /* size of per-interpreter state of the module */ 57 | what_functions}; 58 | 59 | PyMODINIT_FUNC 60 | PyInit__what(void) 61 | { 62 | PyObject *m; 63 | 64 | m = PyModule_Create(&whatmodule); 65 | if (m == NULL) 66 | return NULL; 67 | 68 | PyModule_AddFunctions(m, what_functions); 69 | return m; 70 | } 71 | -------------------------------------------------------------------------------- /awaitwhat/graph.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | import asyncio 3 | import io 4 | from dataclasses import dataclass 5 | from .stack import task_print_stack 6 | from .utils import concise_stack_trace 7 | from .blocker import blockers 8 | 9 | 10 | def new(tasks) -> Tuple[Set[Vertex], List[Edge]]: 11 | try: 12 | current = asyncio.current_task() 13 | except RuntimeError: 14 | current = None 15 | 16 | vertices = set() 17 | edges = list() 18 | 19 | for who in tasks: 20 | children = blockers(who) 21 | src = Vertex.new( 22 | who, current, extra_text=[c for c in children if isinstance(c, str)] 23 | ) 24 | vertices.add(src) 25 | for what in children: 26 | dst = Vertex.new(what, current) 27 | vertices.add(dst) 28 | edges.append(Edge(src, dst)) 29 | 30 | return vertices, edges 31 | 32 | 33 | @dataclass(frozen=True) 34 | class Vertex: 35 | name: str 36 | state: str 37 | traceback: str 38 | task: str = None 39 | 40 | def __hash__(self): 41 | return hash(self.task) 42 | 43 | def __eq__(self, other): 44 | return isinstance(other, Vertex) and self.task == other.task 45 | 46 | @classmethod 47 | def new(cls, task, current, *, extra_text=()): 48 | extra = "\n".join(extra_text) 49 | if isinstance(task, asyncio.Task): 50 | buf = io.StringIO() 51 | task_print_stack(task, None, buf) 52 | try: 53 | name = task.get_name() 54 | except AttributeError: 55 | name = "Task" 56 | if task.done(): 57 | state = "done" 58 | else: 59 | state = "current" if task is current else "pending" 60 | traceback = "\n".join( 61 | filter(None, (concise_stack_trace(buf.getvalue()), extra)) 62 | ) 63 | return cls(name, state, traceback, task) 64 | elif isinstance(task, asyncio.Future): 65 | return cls("Future", None, extra, task) 66 | else: 67 | return cls(str(task), None, extra, task) 68 | 69 | 70 | @dataclass(frozen=True) 71 | class Edge: 72 | src: Vertex 73 | dst: Vertex 74 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Await, What? 2 | 3 | Tells you what waits for what in an `async/await` program. 4 | 5 | ### Build 6 | 7 | `uv build` 8 | 9 | ### ⚠️ Python 3.10.0a1 10 | 11 | It seems the API was changed in 3.10 and the C extension doesn't compile. 12 | I'll investigate... 13 | 14 | ### Alpine 15 | 16 | You'll need `apk add build-base openssl-dev libffi-dev` 17 | 18 | ## 2019 Sprint Setup 19 | 20 | This is out of date, kept for historical reasons. 21 | 22 | Comms: https://gitter.im/awaitwhat/community (no longer in use) 23 | 24 | * Python 3.9, Python 3.8 (preferred) or Python 3.7 25 | * Your platform dev tools (compiler, etc). 26 | * Ensure that `python` is 3.9 or 3.8 or 3.7 27 | * Install `poetry` 28 | * Install `graphviz` 29 | * Clone this repository 30 | * Look at [tests](https://github.com/dimaqq/awaitwhat/tree/master/test) 31 | * Look at [issues](https://github.com/dimaqq/awaitwhat/issues) 32 | 33 | ```console 34 | > python --version 35 | Python 3.9.0b4 #🧡 36 | Python 3.8.4 #👌 37 | 38 | > dot -V 39 | dot - graphviz version 2.40.1 40 | 41 | > curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python 42 | # add ~/.poetry/bin to your PATH 43 | 44 | > git clone git@github.com:dimaqq/awaitwhat.git 45 | > cd awaitwhat 46 | ~/awaitwhat (dev|✔) > poetry shell # creates a venv and drops you in it 47 | 48 | (awaitwhat-x-py3.9) ~/awaitwhat (dev|✔) > poetry install # installs projects dependencies in a venv 49 | (awaitwhat-x-py3.9) ~/awaitwhat (dev|✔) > poetry build # builds a C extension in this project 50 | 51 | (awaitwhat-x-py3.9) ~/awaitwhat (dev|✔) > env PYTHONPATH=. python examples/test_shield.py | tee graph.dot 52 | (awaitwhat-x-py3.9) ~/awaitwhat (dev|✔) > dot -Tsvg graph.dot -o graph.svg 53 | (awaitwhat-x-py3.9) ~/awaitwhat (dev|✔) > open graph.svg # or load it in a browser 54 | ``` 55 | 56 | ### TL;DR 57 | 58 | Say you have this code: 59 | ```py 60 | 61 | async def job(): 62 | await foo() 63 | 64 | 65 | async def foo(): 66 | await bar() 67 | 68 | 69 | async def bar(): 70 | await baz() 71 | 72 | 73 | async def baz(): 74 | await leaf() 75 | 76 | 77 | async def leaf(): 78 | await asyncio.sleep(1) # imagine you don't know this 79 | 80 | 81 | async def work(): 82 | await asyncio.gather(..., job()) 83 | ``` 84 | 85 | Now that code is stuck and and you want to know why. 86 | 87 | #### Python built-in 88 | ```py 89 | Stack for wait_for=()]> cb=[…]> (most recent call last): 90 | File "test/test_stack.py", line 34, in job 91 | await foo() 92 | ``` 93 | 94 | #### This library 95 | ```py 96 | Stack for wait_for=()]> cb=[…]> (most recent call last): 97 | File "test/test_stack.py", line 34, in job 98 | await foo() 99 | File "test/test_stack.py", line 38, in foo 100 | await bar() 101 | File "test/test_stack.py", line 42, in bar 102 | await baz() 103 | File "test/test_stack.py", line 46, in baz 104 | await leaf() 105 | File "test/test_stack.py", line 50, in leaf 106 | await asyncio.sleep(1) 107 | File "/…/asyncio/tasks.py", line 568, in sleep 108 | return await future 109 | File "", line 0, in <_asyncio.FutureIter object at 0x7fb6981690d8>: … 110 | ``` 111 | 112 | ### Dependency Graph 113 | 114 | 115 | 116 | ### References 117 | 118 | https://mail.python.org/archives/list/async-sig@python.org/thread/6E2LRVLKYSMGEAZ7OYOYR3PMZUUYSS3K/ 119 | 120 | > Hi group, 121 | > 122 | > I'm recently debugging a long-running asyncio program that appears to get stuck about once a week. 123 | > 124 | > The tools I've discovered so far are: 125 | > * high level: `asyncio.all_tasks()` + `asyncio.Task.get_stack()` 126 | > * low level: `loop._selector._fd_to_key` 127 | > 128 | > What's missing is the middle level, i.e. stack-like linkage of what is waiting for what. For a practical example, consider: 129 | > 130 | > ```py 131 | > async def leaf(): await somesocket.recv() 132 | > async def baz(): await leaf() 133 | > async def bar(): await baz() 134 | > async def foo(): await bar() 135 | > async def job(): await foo() 136 | > async def work(): await asyncio.gather(..., job()) 137 | > async def main(): asyncio.run(work()) 138 | > ``` 139 | > 140 | > The task stack will contain: 141 | > * main and body of work with line number 142 | > * job task with line number pointing to foo 143 | > 144 | > The file descriptor mapping, socket fd, `loop._recv()` and a `Future`. 145 | > 146 | > What's missing are connections `foo->bar->baz->leaf`. 147 | > That is, I can't tell which task is waiting for what terminal `Future`. 148 | > 149 | > Is this problem solved in some way that I'm not aware of? 150 | > Is there a library or external tool for this already? 151 | > 152 | > Perhaps, if I could get a list of all pending coroutines, I could figure out what's wrong. 153 | > 154 | > If no such API exists, I'm thinking of the following: 155 | > 156 | > ```py 157 | > async def foo(): 158 | > await bar() 159 | > 160 | > In [37]: dis.dis(foo) 161 | > 1 0 LOAD_GLOBAL 0 (bar) 162 | > 2 CALL_FUNCTION 0 163 | > 4 GET_AWAITABLE 164 | > 6 LOAD_CONST 0 (None) 165 | > 8 YIELD_FROM 166 | > 10 POP_TOP 167 | > 12 LOAD_CONST 0 (None) 168 | > 14 RETURN_VALUE 169 | > ``` 170 | > 171 | > Starting from a pending task, I'd get it's coroutine and: 172 | > 173 | > Get the coroutine frame, and if current instruction is `YIELD_FROM`, then the reference to the awaitable should be on the top of the stack. 174 | > If that reference points to a pending coroutine, I'd add that to the "forward trace" and repeat. 175 | > 176 | > At some point I'd reach an awaitable that's not a pending coroutine, which may be: another `Task` (I already got those), a low-level `Future` (can be looked up in event loop), an `Event` (tough luck, shoulda logged all `Event`'s on creation) or a dozen other corner cases. 177 | > 178 | > What do y'all think of this approach? 179 | > 180 | > Thanks, 181 | > D. 182 | 183 | ### Build for Alpine 184 | 185 | This is out of date... 186 | 187 | ```sh 188 | cd /src 189 | apk update 190 | apk add build-base openssl-dev libffi-dev curl 191 | curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python 192 | cp -a ~/.poetry/lib/poetry/_vendor/py3.8 ~/.poetry/lib/poetry/_vendor/py3.9 193 | source $HOME/.poetry/env 194 | poetry install 195 | poetry run pytest 196 | env _PYTHON_HOST_PLATFORM=alpine_x86_64 poetry build 197 | ``` 198 | 199 | ### Manylinux2014 200 | 201 | This is out of date... 202 | 203 | 🚧 Work in progress, doesn't tag the wheels correctly 🚧 204 | 205 | `docker run -v (pwd):/src -it quay.io/pypa/manylinux2014_x86_64 sh` 206 | 207 | ```sh 208 | curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | /opt/python/cp38-cp38/bin/python 209 | source $HOME/.poetry/env 210 | cd /src 211 | poetry env use /opt/python/cp38-cp38/bin/python 212 | poetry install 213 | poetry run pytest 214 | poetry build 215 | poetry env use /opt/python/cp39-cp39/bin/python 216 | poetry install 217 | poetry run pytest 218 | poetry build 219 | ``` 220 | -------------------------------------------------------------------------------- /doc/test_future.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | %3 11 | 12 | 13 | 14 | 140644342210056 15 | 16 | 17 | 18 | Task 19 |  File "test/test_future.py", line 20, in frob_a_tree 20 |    await b_tree() 21 |  File "test/test_future.py", line 27, in b_tree 22 |    await asyncio.gather(t) 23 | 24 | 25 | 26 | 140644342209416 27 | 28 | 29 | 30 | Task 31 |  File "test/test_future.py", line 31, in frob_a_branch 32 |    await b_branch(f) 33 |  File "test/test_future.py", line 35, in b_branch 34 |    await f 35 | 36 | 37 | 38 | 140644342210056->140644342209416 39 | 40 | 41 | 42 | 43 | 44 | 140644342209576 45 | 46 | 47 | 48 | Task 49 |  File "test/test_future.py", line 10, in main 50 |    await do_work() 51 |  File "test/test_future.py", line 16, in do_work 52 |    await asyncio.gather(*futs) 53 | 54 | 55 | 56 | 140644342209576->140644342210056 57 | 58 | 59 | 60 | 61 | 62 | 140644342210216 63 | 64 | 65 | 66 | Task 67 |  File "test/test_future.py", line 20, in frob_a_tree 68 |    await b_tree() 69 |  File "test/test_future.py", line 27, in b_tree 70 |    await asyncio.gather(t) 71 | 72 | 73 | 74 | 140644342209576->140644342210216 75 | 76 | 77 | 78 | 79 | 80 | 140644342209896 81 | 82 | 83 | 84 | Task 85 |  File "test/test_future.py", line 20, in frob_a_tree 86 |    await b_tree() 87 |  File "test/test_future.py", line 27, in b_tree 88 |    await asyncio.gather(t) 89 | 90 | 91 | 92 | 140644342209576->140644342209896 93 | 94 | 95 | 96 | 97 | 98 | 140644342496400 99 | 100 | 101 | 102 | <Not blocked 0.41964167998853763> 103 | 104 | 105 | 106 | 140644342209096 107 | 108 | 109 | 110 | Task 111 |  File "test/test_future.py", line 31, in frob_a_branch 112 |    await b_branch(f) 113 |  File "test/test_future.py", line 35, in b_branch 114 |    await f 115 | 116 | 117 | 118 | 140644342622280 119 | 120 | 121 | 122 | Future 123 | 124 | 125 | 126 | 140644342209096->140644342622280 127 | 128 | 129 | 130 | 131 | 132 | 140644342621256 133 | 134 | 135 | 136 | Future 137 | 138 | 139 | 140 | 140644342621768 141 | 142 | 143 | 144 | Future 145 | 146 | 147 | 148 | 140644342210216->140644342209096 149 | 150 | 151 | 152 | 153 | 154 | 140644342209736 155 | 156 | 157 | 158 | Task 159 |  File "test/test_future.py", line 68, in <module> 160 |    asyncio.run(main()) 161 |  File "asyncio/runners.py", line 43, in run 162 |    return loop.run_until_complete(main) 163 |  File "asyncio/base_events.py", line 571, in run_until_complete 164 |    self.run_forever() 165 |  File "asyncio/base_events.py", line 539, in run_forever 166 |    self._run_once() 167 |  File "asyncio/base_events.py", line 1775, in _run_once 168 |    handle._run() 169 |  File "asyncio/events.py", line 88, in _run 170 |    self._context.run(self._callback, *self._args) 171 |  File "test/test_future.py", line 61, in test 172 |    print(awaitwhat.dot.dumps(tt)) 173 | 174 | 175 | 176 | 140644342209736->140644342496400 177 | 178 | 179 | 180 | 181 | 182 | 140644342210376 183 | 184 | 185 | 186 | Task 187 |  File "test/test_future.py", line 31, in frob_a_branch 188 |    await b_branch(f) 189 |  File "test/test_future.py", line 35, in b_branch 190 |    await f 191 | 192 | 193 | 194 | 140644342210376->140644342621256 195 | 196 | 197 | 198 | 199 | 200 | 140644342209896->140644342210376 201 | 202 | 203 | 204 | 205 | 206 | 140644342209416->140644342621768 207 | 208 | 209 | 210 | 211 | 212 | --------------------------------------------------------------------------------