├── .gitignore ├── Makefile ├── README.md ├── azure-pipelines.yml ├── cyberbrain ├── __init__.py ├── api.py ├── backtrace.py ├── basis.py ├── callsite.py ├── callsite_test.py ├── computation.py ├── flow.py ├── flow_test.py ├── format.py ├── testing.py ├── utils.py └── vars.py ├── images ├── cb_concept.jpg ├── cb_output.jpg └── cb_output.png ├── poetry.lock ├── pyproject.toml ├── test ├── doc_example │ ├── computation.golden.json │ ├── doc_example.py │ └── flow.golden.json ├── exclude_events │ ├── call_libs.py │ ├── computation.golden.json │ └── flow.golden.json ├── function │ ├── computation.golden.json │ ├── flow.golden.json │ └── simple_func.py ├── gen_golden.py ├── hello_world │ ├── computation.golden.json │ ├── flow.golden.json │ └── hello.py ├── list_comp │ ├── computation.golden.json │ ├── flow.golden.json │ └── list_comp.py ├── loop │ ├── computation.golden.json │ ├── flow.golden.json │ └── loop.py ├── modules │ ├── bar.py │ ├── computation.golden.json │ ├── flow.golden.json │ ├── foo.py │ └── main.py ├── multicall │ ├── computation.golden.json │ ├── flow.golden.json │ └── multicall.py ├── multiline │ ├── computation.golden.json │ ├── flow.golden.json │ └── multiline_statement.py └── test_executor.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # project specific 107 | legacy/ 108 | 109 | # Graphviz output 110 | *.svg 111 | *.pdf 112 | 113 | .pytype/ 114 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: gen_test_data override_test_data 2 | 3 | gen_golden: 4 | tox -e py37 --run-command "python test/gen_golden.py" 5 | 6 | override_golden: 7 | tox -e py37 --run-command "python test/gen_golden.py --override" 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cyberbrain: helps you remember everything 2 | 3 | # [Deprecation]: This project is deprecated and archived. I rewrote it from scratch, see https://github.com/laike9m/Cyberbrain. 4 | 5 | [![Build Status](https://dev.azure.com/laike9m/laike9m/_apis/build/status/laike9m.Cyberbrain?branchName=master)](https://dev.azure.com/laike9m/laike9m/_build/latest?definitionId=1&branchName=master) 6 | 7 | 8 | 9 | 10 | NOTE: This is a WIP, **DON'T** use it in production. 11 | 12 | 📢 I'm looking for collaborators who's interested in **visualization**. Please don't hesitate to contact me. 13 | 14 | I gave a talk at PyCon China 2019 about Cyberbrain, slide can be found [here](https://yanshuo.io/assets/player/?deck=5d6c9136d37616007449891e#/). 15 | 16 | ## How to use 17 | 1. Install [Graphviz](https://www.graphviz.org/download/) 18 | 2. `pip install cyberbrain` 19 | 3. In your program, first call `cyberbrain.init()`, then call `cyberbrain.register(target_variable)`. 20 | 21 | Here's an example. 22 | 23 | ```python 24 | def func_f(bar): 25 | x = len(bar) 26 | return x 27 | 28 | def func_c(baa): 29 | baa.append(None) 30 | baa.append('?') 31 | 32 | def func_a(foo): 33 | for i in range(2): pass 34 | ba = [foo] 35 | func_c(ba) 36 | foo = func_f(ba) # foo is our target 37 | cyberbrain.register(foo) 38 | 39 | import cyberbrain 40 | cyberbrain.init() 41 | fo = 1 42 | func_a(fo) 43 | ``` 44 | 45 | Run it, a pdf like this will be generated and automatically opened. 46 | 47 | 48 | 49 | # Developement 50 | First install [`Poetry`](https://github.com/sdispater/poetry), then run `poetry install`. 51 | -------------------------------------------------------------------------------- /azure-pipelines.yml: -------------------------------------------------------------------------------- 1 | # Using https://github.com/tox-dev/azure-pipelines-template 2 | # and learned from https://github.com/wasp/waspy/blob/master/azure-pipelines.yml 3 | 4 | name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) 5 | resources: 6 | repositories: 7 | - repository: tox 8 | type: github 9 | endpoint: github 10 | name: tox-dev/azure-pipelines-template 11 | 12 | trigger: 13 | batch: true 14 | branches: 15 | include: 16 | - master 17 | 18 | pr: 19 | branches: 20 | include: 21 | - master 22 | 23 | variables: 24 | PYTEST_ADDOPTS: "-v -v -ra --showlocals" 25 | PYTEST_XDIST_PROC_NR: 'auto' 26 | CI_RUN: 'yes' 27 | 28 | jobs: 29 | - template: run-tox-env.yml@tox 30 | parameters: 31 | jobs: 32 | py36: 33 | image: [linux, windows, macOs] 34 | py37: 35 | image: [linux, windows, macOs] 36 | py38: 37 | image: [linux, windows, macOs] 38 | -------------------------------------------------------------------------------- /cyberbrain/__init__.py: -------------------------------------------------------------------------------- 1 | from .api import init, register 2 | -------------------------------------------------------------------------------- /cyberbrain/api.py: -------------------------------------------------------------------------------- 1 | """Cyberbrain public API and tracer setup.""" 2 | 3 | import os 4 | import sys 5 | 6 | from absl import flags 7 | 8 | from . import backtrace, flow, format, testing, utils 9 | from .basis import _dummy 10 | from .computation import computation_manager, Computation 11 | 12 | FLAGS = flags.FLAGS 13 | 14 | # run: Invoked by users. 15 | # test: Verifies cyberbrain's internal state is the same as golden. 16 | # golden: Generates golden file. 17 | # debug: Prints cyberbrain's internal state, but don't do assertion or dumps files. 18 | flags.DEFINE_enum( 19 | "mode", 20 | "run", 21 | ["run", "test", "golden", "debug"], 22 | "The mode which Cyberbrain runs in.", 23 | ) 24 | flags.DEFINE_string("test_dir", None, "Directory to save test output to.") 25 | flags.DEFINE_string( 26 | "backtrace_stop_at", 27 | None, 28 | "Line to enable breakpoint for. Backtrace will stop when node.code_str matches this flag.", 29 | ) 30 | 31 | 32 | def global_tracer(frame, event_type, arg): 33 | """Global trace function.""" 34 | if utils.should_exclude(frame.f_code.co_filename): 35 | return 36 | # print("\nglobal: ", frame, event_type, frame.f_code.co_filename, frame.f_lineno) 37 | 38 | assert event_type == "call" 39 | succeeded = computation_manager.add_computation(event_type, frame, arg) 40 | 41 | # https://docs.python.org/3/library/inspect.html#the-interpreter-stack 42 | del frame 43 | if succeeded: 44 | return local_tracer 45 | 46 | 47 | def local_tracer(frame, event_type, arg): 48 | """Local trace function.""" 49 | if utils.should_exclude(frame.f_code.co_filename): 50 | return 51 | # print("\nlocal: ", frame, event_type, frame.f_code.co_filename, frame.f_lineno) 52 | 53 | if event_type in {"line", "return"}: 54 | computation_manager.add_computation(event_type, frame, arg) 55 | 56 | del frame 57 | 58 | 59 | global_frame = None 60 | 61 | 62 | def init(): 63 | """Initializes tracing.""" 64 | global global_frame 65 | 66 | global_frame = sys._getframe(1) 67 | sys.settrace(global_tracer) 68 | global_frame.f_trace = local_tracer 69 | 70 | 71 | def register(target=_dummy): 72 | """Receives target variable and stops recording computation. 73 | 74 | If target is not given, it is only called to terminate tracing and dump data. 75 | """ 76 | FLAGS(sys.argv) # See https://github.com/chris-chris/pysc2-examples/issues/5. 77 | sys.settrace(None) 78 | global_frame.f_trace = None 79 | if target is _dummy: 80 | return 81 | 82 | should_dump_output = FLAGS.mode in {"test", "golden", "debug"} 83 | if should_dump_output: 84 | # Even if build flow fails, comp will still be dumped. 85 | testing.dump_computation(computation_manager) 86 | execution_flow = flow.build_flow(computation_manager) 87 | testing.print_if_debug(testing.get_dumpable_flow(execution_flow)) 88 | 89 | backtrace.trace_flow(execution_flow) 90 | graph_name = os.path.basename(FLAGS.test_dir) if FLAGS.test_dir else "output" 91 | if FLAGS.mode in {"run", "debug"}: 92 | format.generate_output(execution_flow, graph_name) 93 | if should_dump_output: 94 | testing.dump_flow(execution_flow) 95 | 96 | if FLAGS.mode == "debug": 97 | # TODO: Profile time. 98 | print(f"Vars total size: {Computation.vars_total_size}") 99 | -------------------------------------------------------------------------------- /cyberbrain/backtrace.py: -------------------------------------------------------------------------------- 1 | """Backtraces var change from target to program start.""" 2 | 3 | from . import utils 4 | from .flow import Flow, Node, VarSwitch 5 | 6 | 7 | def trace_flow(flow: Flow): 8 | """Traces a flow and adds information regarding var changes to its nodes. 9 | 10 | This function is the *core* of Cyberbrain. 11 | """ 12 | current: Node 13 | next: Node 14 | current, next = flow.target.prev, flow.target 15 | 16 | while current is not flow.ROOT: 17 | # Case 1: non-callsite 18 | if not current.is_callsite: 19 | current.sync_tracking_with(next) 20 | if any(current.get_and_update_var_changes(next)): 21 | # If any change happened, track all ids appeared in this node. 22 | current.add_tracking(*utils.find_names(current.code_ast)) 23 | next, current = current, current.prev 24 | continue 25 | 26 | # Case 2: current is a callsite, next is the first line within the called 27 | # function. 28 | if current.step_into is next: 29 | for identifier in next.tracking: 30 | for arg_id in current.param_to_arg[identifier]: 31 | current.add_tracking(arg_id) 32 | current.add_var_switches( 33 | VarSwitch( 34 | arg_id=arg_id, 35 | param_id=identifier, 36 | value=current.vars[arg_id], 37 | ) 38 | ) 39 | next, current = current, current.prev 40 | continue 41 | 42 | # Case 3: current is a callsite, next is the line after the call. 43 | current.sync_tracking_with(next) 44 | args = current.get_args() 45 | # ids on the left, like 'a', 'b' in 'a, b = f()' 46 | ids_assigned_to = utils.find_names(current.code_ast) - args 47 | returned_from = current.returned_from 48 | for var_change in current.get_and_update_var_changes(next): 49 | if var_change.id in args: 50 | returned_from.add_tracking(current.arg_to_param[var_change.id]) 51 | elif var_change.id in ids_assigned_to: 52 | # The return statement contributes to relevant changes. 53 | returned_from.is_relevant_return = True 54 | returned_from.add_tracking(*utils.find_names(returned_from.code_ast)) 55 | returned_from.update_var_changes_before_return() 56 | 57 | next, current = returned_from, returned_from.prev 58 | -------------------------------------------------------------------------------- /cyberbrain/basis.py: -------------------------------------------------------------------------------- 1 | """Some basic data structures used throughout the project.""" 2 | 3 | from collections import defaultdict 4 | from enum import Enum 5 | from typing import Dict, NamedTuple, Tuple, Union 6 | 7 | # "surrounding" is a 2-element tuple (start_lineno, end_lineno), representing a 8 | # logical line. Line number is frame-wise. 9 | # 10 | # For single-line statement, start_lineno = end_lineno, and is the line number of the 11 | # physical line returned by get_lineno_from_lnotab. 12 | # 13 | # For multiline statement, start_lineno is the line number of the first physical line, 14 | # end_lineno is the last. Lines from start_lineno to end_lineno -1 should end with 15 | # token.NL(or tokenize.NL before 3.7), line end_lineno should end with token.NEWLINE. 16 | # 17 | # Example: 18 | # 0 a = true 19 | # 1 a = true 20 | # 2 b = { 21 | # 3 'foo': 'bar' 22 | # 4 } 23 | # 5 c = false 24 | # 25 | # For the assignment of b, start_lineno = 2, end_lineno = 4 26 | Surrounding = NamedTuple("Surrounding", [("start_lineno", int), ("end_lineno", int)]) 27 | 28 | SourceLocation = NamedTuple( 29 | "SourceLocation", [("filepath", str), ("surrounding", Surrounding)] 30 | ) 31 | 32 | _dummy = object() 33 | 34 | 35 | class NodeType(Enum): 36 | """Just node types.""" 37 | 38 | LINE = 1 39 | CALL = 2 40 | 41 | 42 | class FrameID: 43 | """Class that represents a frame. 44 | 45 | Basically, a frame id is just a tuple, where each element represents the frame index 46 | within the same parent frame. For example, consider this snippet: 47 | 48 | def f(): g() 49 | 50 | def g(): pass 51 | 52 | f() 53 | f() 54 | 55 | Assuming the frame id for global frame is (0,). We called f two times with two 56 | frames (0, 0) and (0, 1). f calls g, which also generates two frames (0, 0, 0) and 57 | (0, 1, 0). By comparing prefixes, it's easy to know whether one frame is the parent 58 | frame of the other. 59 | 60 | We also maintain the frame id of current code location. New frame ids are generated 61 | based on event type and current frame id. 62 | 63 | TODO: record function name. 64 | """ 65 | 66 | current_ = (0,) 67 | 68 | # Mapping from parent frame id to max child frame index. 69 | child_index: Dict[Tuple, int] = defaultdict(int) 70 | 71 | def __init__(self, frame_id_tuple: Tuple[int, ...], co_name: str = ""): 72 | self._frame_id_tuple = frame_id_tuple 73 | self.co_name = co_name 74 | 75 | def __eq__(self, other: Union["FrameID", Tuple[int, ...]]): 76 | if isinstance(other, FrameID): 77 | return self._frame_id_tuple == other._frame_id_tuple 78 | return isinstance(other, Tuple) and self._frame_id_tuple == other 79 | 80 | def __hash__(self): 81 | return hash(self._frame_id_tuple) 82 | 83 | def __add__(self, other: Tuple): 84 | return FrameID(self._frame_id_tuple + other) 85 | 86 | @property 87 | def tuple(self): 88 | return self._frame_id_tuple 89 | 90 | @classmethod 91 | def current(cls): 92 | return FrameID(cls.current_) 93 | 94 | @property 95 | def parent(self): 96 | return FrameID(self._frame_id_tuple[:-1]) 97 | 98 | def is_child_of(self, other): 99 | return other == self._frame_id_tuple 100 | 101 | def is_parent_of(self, other): 102 | return self == other._frame_id_tuple 103 | 104 | @classmethod 105 | def create(cls, event: str): 106 | assert event in {"line", "call", "return"} 107 | if event == "line": 108 | return cls.current() 109 | if event == "call": 110 | frame_id = cls.current() 111 | cls.current_ = cls.current_ + (cls.child_index[cls.current_],) 112 | return frame_id # callsite is in caller frame. 113 | if event == "return": 114 | call_frame = cls.current() 115 | cls.current_ = cls.current_[:-1] 116 | # After exiting call frame, increments call frame's child index. 117 | cls.child_index[cls.current_] += 1 118 | return call_frame 119 | 120 | def __str__(self): 121 | """Prints the tuple representation.""" 122 | return f"{str(self._frame_id_tuple)} {self.co_name}" 123 | 124 | 125 | class ID(str): 126 | """A class that represents an identifier. 127 | 128 | There's no need to save frame info, because at a ceratain time, a computation or 129 | node only sees one value for one identifier, and we can omit others. 130 | """ 131 | -------------------------------------------------------------------------------- /cyberbrain/callsite.py: -------------------------------------------------------------------------------- 1 | """Utilities to get call site.""" 2 | 3 | import ast 4 | import inspect 5 | import itertools 6 | from collections import namedtuple 7 | from typing import Dict, Iterable, Set, Tuple 8 | 9 | from . import utils 10 | from .basis import ID 11 | 12 | Args = namedtuple("Args", ["args", "kwargs"]) 13 | 14 | 15 | def get_param_arg_pairs( 16 | callsite_ast: ast.Call, arg_info: inspect.ArgInfo 17 | ) -> Iterable[Tuple[ast.AST, str]]: 18 | """Generates parameter, argument pairs. 19 | 20 | Example: 21 | 22 | def def f(foo, bar, baz=1, *args, **kwargs): 23 | pass 24 | f(a,b,c,d,qux=e) 25 | 26 | Generates: 27 | 28 | Name(id='a', ctx=Load()), foo 29 | Name(id='b', ctx=Load()), bar 30 | Name(id='c', ctx=Load()), baz 31 | Name(id='d', ctx=Load()), args 32 | keyword(arg='qux', value=Name(id='e', ctx=Load())), kwargs 33 | """ 34 | _ARGS = arg_info.varargs # extra arguments' name, could be anything. 35 | _KWARGS = arg_info.keywords # extra kw-arguments' name, could be anything. 36 | 37 | pos_args = callsite_ast.args 38 | kw_args = callsite_ast.keywords 39 | # Builds a parameter list that expands *args and **kwargs to their length, so that 40 | # we can emit a 1-to-1 pair of (arg, param). 41 | # Excludes self since it's not explicitly passed from caller. 42 | parameters = [arg for arg in arg_info.args if arg != "self"] 43 | 44 | # There could be no *args or *kwargs in signature. 45 | if _ARGS is not None: 46 | parameters += [_ARGS] * len(arg_info.locals[_ARGS]) 47 | if _KWARGS is not None: 48 | parameters += [_KWARGS] * len(arg_info.locals[_KWARGS]) 49 | 50 | for arg, param in zip(itertools.chain(pos_args, kw_args), parameters): 51 | yield arg, param 52 | 53 | 54 | def get_param_to_arg( 55 | callsite_ast: ast.Call, arg_info: inspect.ArgInfo 56 | ) -> Dict[ID, Set[ID]]: 57 | """Maps argument identifiers to parameter identifiers. 58 | 59 | For now we'll flatten parameter identifiers as long as they contribute to the same 60 | argument, for example: 61 | 62 | def f(x, **kwargs): 63 | pass 64 | f(x = {a: 1, b: 2}, y=1, z=2) 65 | 66 | Generates: 67 | { 68 | ID('x'): {ID('a'), ID('b')}, 69 | ID('kwargs'): {ID('y'), ID('z')} 70 | } 71 | 72 | In the future, we *might* record fine grained info. 73 | """ 74 | return { 75 | ID(param): utils.find_names(arg) 76 | for arg, param in get_param_arg_pairs(callsite_ast, arg_info) 77 | } 78 | -------------------------------------------------------------------------------- /cyberbrain/callsite_test.py: -------------------------------------------------------------------------------- 1 | """Unittests for callsite.""" 2 | 3 | import ast 4 | import inspect 5 | 6 | from . import callsite 7 | from .basis import ID 8 | 9 | 10 | def _get_call(module_ast: ast.Module) -> ast.Call: 11 | assert isinstance( 12 | module_ast.body[0], ast.Expr 13 | ), "Passed in code is not a call expression." 14 | 15 | return module_ast.body[0].value 16 | 17 | 18 | def test_get_param_to_arg(): 19 | def f(foo, bar, baz=1, *args, **kwargs): 20 | return inspect.getargvalues(inspect.currentframe()) 21 | 22 | # Tests passing values directly. 23 | assert callsite.get_param_to_arg(_get_call(ast.parse("f(1,2)")), f(1, 2)) == { 24 | ID("foo"): set(), 25 | ID("bar"): set(), 26 | } 27 | 28 | # Tests passing variables. 29 | a, b, c = 1, 2, 3 30 | assert callsite.get_param_to_arg( 31 | _get_call(ast.parse("f(a,b,c)")), f(a, b, z=c) 32 | ) == {ID("foo"): {ID("a")}, ID("bar"): {ID("b")}, ID("baz"): {ID("c")}} 33 | 34 | # Tests catching extra args. 35 | d, e = 4, 5 36 | assert callsite.get_param_to_arg( 37 | _get_call(ast.parse("f(a,b,c,d,qux=e)")), f(a, b, c, d, qux=e) 38 | ) == { 39 | ID("foo"): {ID("a")}, 40 | ID("bar"): {ID("b")}, 41 | ID("baz"): {ID("c")}, 42 | ID("args"): {ID("d")}, 43 | ID("kwargs"): {ID("e")}, 44 | } 45 | 46 | # Tests binding multiple params to one argument. 47 | assert callsite.get_param_to_arg( 48 | _get_call(ast.parse("f(a,(b,c),c,qux=(d, e))")), f(a, (b, c), c, qux=(d, e)) 49 | ) == { 50 | ID("foo"): {ID("a")}, 51 | ID("bar"): {ID("b"), ID("c")}, 52 | ID("baz"): {ID("c")}, 53 | ID("kwargs"): {ID("d"), ID("e")}, 54 | } 55 | 56 | # Tests using custom names for args and kwargs. 57 | def g(*foo, **bar): 58 | return inspect.getargvalues(inspect.currentframe()) 59 | 60 | assert callsite.get_param_to_arg( 61 | _get_call(ast.parse("g(d,qux=e)")), g(d, qux=e) 62 | ) == {ID("foo"): {ID("d")}, ID("bar"): {ID("e")}} 63 | 64 | # Tests signature without args or kwargs. 65 | def h(x): 66 | return inspect.getargvalues(inspect.currentframe()) 67 | 68 | assert callsite.get_param_to_arg(_get_call(ast.parse("h(a)")), h(a)) == { 69 | ID("x"): {ID("a")} 70 | } 71 | 72 | # TODO: tests nested call. 73 | 74 | 75 | def test_get_param_to_arg_from_method(): 76 | class MyClass: 77 | """Class under test.""" 78 | 79 | def __init__(self, x: int, y: int): 80 | self.x = x 81 | self.y = y 82 | self.arg_values = inspect.getargvalues(inspect.currentframe()) 83 | 84 | def __eq__(self, other): 85 | return self.x == other.x and self.y == other.y 86 | 87 | def increment(self): 88 | self.x += 1 89 | self.y += 1 90 | 91 | # Tests implicit self is ignored. 92 | arg_values = MyClass(1, y=2).arg_values 93 | assert callsite.get_param_to_arg( 94 | _get_call(ast.parse("MyClass(1, y=2)")), arg_values 95 | ) == {ID("x"): set(), ID("y"): set()} 96 | -------------------------------------------------------------------------------- /cyberbrain/computation.py: -------------------------------------------------------------------------------- 1 | """Vars structures for recording program execution.""" 2 | 3 | import abc 4 | import ast 5 | import inspect 6 | from collections import defaultdict 7 | from pathlib import PurePath 8 | from typing import Dict, List, Union 9 | 10 | import black 11 | import executing 12 | from pympler import asizeof 13 | 14 | from . import utils 15 | from .basis import FrameID, SourceLocation, Surrounding 16 | from .vars import Vars 17 | 18 | 19 | class Computation(metaclass=abc.ABCMeta): 20 | """Base class to represent a computation unit of the program.""" 21 | 22 | code_str: str 23 | event_type: str 24 | source_location: SourceLocation 25 | VARS_ATTR_NAME = "vars" 26 | vars_total_size = 0 27 | 28 | def __new__(cls, **kwargs): 29 | """Automatically sums up the size of vars.""" 30 | assert cls.VARS_ATTR_NAME in kwargs 31 | Computation.vars_total_size += asizeof.asizeof(kwargs[cls.VARS_ATTR_NAME]) 32 | return super().__new__(cls) 33 | 34 | @abc.abstractmethod 35 | def to_dict(self): 36 | """Serializes attrs to dict. Subclasses must override this method.""" 37 | surrounding = self.source_location.surrounding 38 | if surrounding.start_lineno == surrounding.end_lineno: 39 | lineno_str = surrounding.start_lineno 40 | else: 41 | lineno_str = f"{surrounding.start_lineno} ~ {surrounding.end_lineno}" 42 | return { 43 | "event": self.event_type, 44 | "filepath": PurePath(self.source_location.filepath).name, 45 | "lineno": lineno_str, 46 | "code_str": self.code_str, 47 | } 48 | 49 | def __repr__(self): 50 | return self.code_str 51 | 52 | def __str__(self): 53 | return str(self.to_dict()) 54 | 55 | 56 | class Line(Computation): 57 | """Class that represents a logical line without entering into a new call.""" 58 | 59 | def __init__( 60 | self, 61 | *, 62 | code_str: str, 63 | source_location: SourceLocation, 64 | vars: Vars, 65 | frame_id: FrameID, 66 | event_type: str, 67 | surrounding: Surrounding, 68 | ): 69 | self.code_str = code_str 70 | try: 71 | self.code_str = black.format_str( 72 | self.code_str, mode=black.FileMode() 73 | ).strip() 74 | except black.InvalidInput: 75 | pass 76 | self.source_location = source_location 77 | self.vars = vars 78 | self.event_type = event_type 79 | self.frame_id = frame_id 80 | self.surrounding = surrounding 81 | self.vars_before_return = None 82 | 83 | def to_dict(self): 84 | return {**super().to_dict(), "frame_id": str(self.frame_id)} 85 | 86 | 87 | class Call(Computation): 88 | """Class that represents a call site.""" 89 | 90 | def __init__( 91 | self, 92 | *, 93 | callsite_ast: ast.AST, 94 | source_location: SourceLocation, 95 | arg_values: inspect.ArgInfo, 96 | func_name: str, 97 | vars: Vars, 98 | event_type: str, 99 | frame_id: FrameID, 100 | callee_frame_id: FrameID, 101 | surrounding: Surrounding, 102 | ): 103 | self.callsite_ast = callsite_ast 104 | self.source_location = source_location 105 | self.arg_values = arg_values 106 | self.func_name = func_name 107 | self.vars = vars 108 | self.event_type = event_type 109 | self.frame_id = frame_id 110 | self.callee_frame_id = callee_frame_id 111 | self.code_str = utils.ast_to_str(self.callsite_ast) 112 | self.vars_before_return = None 113 | self.surrounding = surrounding 114 | 115 | def to_dict(self): 116 | return { 117 | **super().to_dict(), 118 | "caller_frame_id": str(self.frame_id), 119 | "callee_frame_id": str(self.callee_frame_id), 120 | } 121 | 122 | @staticmethod 123 | def create(frame): 124 | caller_frame = frame.f_back 125 | _, surrounding = utils.get_code_str_and_surrounding(caller_frame) 126 | callsite_ast = executing.Source.executing(caller_frame).node 127 | # If it's not ast.Call, like ast.ListComp, ignore for now. 128 | if not isinstance(callsite_ast, ast.Call): 129 | return None 130 | frame_id = FrameID.create("call") 131 | frame_id.co_name = caller_frame.f_code.co_name 132 | return Call( 133 | callsite_ast=callsite_ast, 134 | source_location=SourceLocation( 135 | filepath=caller_frame.f_code.co_filename, surrounding=surrounding 136 | ), 137 | arg_values=inspect.getargvalues(frame), 138 | func_name=frame.f_code.co_name, 139 | vars=Vars(caller_frame), 140 | event_type="call", 141 | frame_id=frame_id, 142 | callee_frame_id=FrameID.current(), 143 | surrounding=surrounding, 144 | ) 145 | 146 | 147 | class ComputationManager: 148 | """Class that stores and manages all computations.""" 149 | 150 | REGISTER_CALL = "cyberbrain.register" 151 | 152 | def __init__(self): 153 | self.frame_groups: Dict[FrameID, List[Union[Line, Call]]] = defaultdict(list) 154 | self.target = None 155 | 156 | def add_computation(self, event_type, frame, arg) -> bool: 157 | """Adds a computation to manager. 158 | 159 | Returns Whether a new computation has been created and added. 160 | """ 161 | assert event_type in {"line", "call", "return"} 162 | if event_type == "line": 163 | code_str, surrounding = utils.get_code_str_and_surrounding(frame) 164 | frame_id = FrameID.create(event_type) 165 | frame_id.co_name = frame.f_code.co_name 166 | # Skips if the same logical line has been added. 167 | if ( 168 | self.frame_groups[frame_id] 169 | and self.frame_groups[frame_id][-1].surrounding == surrounding 170 | ): 171 | return False 172 | comp = Line( 173 | code_str=code_str.rsplit("#", 1)[0].strip(), # Removes comment. 174 | source_location=SourceLocation( 175 | filepath=frame.f_code.co_filename, surrounding=surrounding 176 | ), 177 | vars=Vars(frame), 178 | event_type=event_type, 179 | frame_id=frame_id, 180 | surrounding=surrounding, 181 | ) 182 | if comp.code_str.startswith(self.REGISTER_CALL): 183 | self.target = comp 184 | self.frame_groups[frame_id].append(comp) 185 | return True 186 | 187 | if event_type == "call": 188 | # In Python 3.8, for multiline statement, after the events triggered by each 189 | # line, there will be an extra line event triggered by the first line. This 190 | # will cause the lineno for call comp to be different in different Python 191 | # verions. 192 | computation = Call.create(frame) 193 | # Don't trace cyberbrain.register. 194 | if not computation or computation.code_str.startswith(self.REGISTER_CALL): 195 | return False 196 | frame_id = computation.frame_id 197 | # When entering a new call, replaces previous line(aka caller) with a 198 | # call computation. 199 | if ( 200 | self.frame_groups[frame_id] 201 | and self.frame_groups[frame_id][-1].event_type == "line" 202 | ): 203 | # Always keeps Line computation at the end. 204 | self.frame_groups[frame_id].insert( 205 | len(self.frame_groups[frame_id]) - 1, computation 206 | ) 207 | else: 208 | self.frame_groups[frame_id].append(computation) 209 | 210 | return True 211 | 212 | # event is "return". 213 | frame_id = FrameID.create(event_type) 214 | assert self.frame_groups[frame_id][-1].event_type == "line" 215 | self.frame_groups[frame_id][-1].return_value = arg 216 | self.frame_groups[frame_id][-1].vars_before_return = Vars(frame) 217 | return True 218 | 219 | 220 | computation_manager = ComputationManager() 221 | -------------------------------------------------------------------------------- /cyberbrain/flow.py: -------------------------------------------------------------------------------- 1 | """Execution flow that represents a program's execution.""" 2 | 3 | import ast 4 | import inspect 5 | import itertools 6 | import re 7 | from collections import defaultdict, namedtuple 8 | from dataclasses import dataclass 9 | from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union 10 | 11 | import astor 12 | 13 | from . import callsite, utils 14 | from .basis import ID, FrameID, NodeType, SourceLocation, _dummy 15 | from .computation import ComputationManager 16 | 17 | 18 | @dataclass() 19 | class VarAppearance: 20 | """Variable appears in current frame.""" 21 | 22 | id: ID 23 | value: Any 24 | 25 | 26 | @dataclass() 27 | class VarModification: 28 | """Variable value modified in current frame.""" 29 | 30 | id: ID 31 | old_value: Any 32 | new_value: Any 33 | 34 | 35 | @dataclass() 36 | class VarSwitch: 37 | """Variable switches at callsite.""" 38 | 39 | arg_id: ID 40 | param_id: ID 41 | value: Any 42 | 43 | 44 | class TrackingMetadata: 45 | """Class that stores metadata during tracing.""" 46 | 47 | def __init__( 48 | self, 49 | vars: Dict[ID, Any], 50 | code_str: str = None, 51 | vars_before_return=None, 52 | source_location: SourceLocation = None, 53 | ): 54 | self.vars = vars 55 | self.source_location = source_location 56 | self.code_str = code_str 57 | self.code_ast = utils.parse_code_str(code_str) 58 | 59 | # It seems that tracking and data should all be flattened, aka they should sdss 60 | # simply be a mapping of ID -> value. When backtracing, we don't really care 61 | # about where an identifer is defined in, we only care about whether its value 62 | # has changed during execution. 63 | self.tracking: Set[ID] = set() 64 | 65 | self.var_appearances: Set[VarAppearance] = [] 66 | self.var_modifications: Set[VarModification] = [] 67 | 68 | # var_switches are set on call node. When some id is switched, it is not counted 69 | # again in var_appearances. 70 | self.var_switches: Set[VarSwitch] = [] 71 | self.vars_before_return = vars_before_return 72 | self.return_value = _dummy 73 | self.is_relevant_return = False 74 | 75 | def set_param_arg_mapping(self, arg_values: inspect.ArgInfo): 76 | param_to_arg = callsite.get_param_to_arg( 77 | self.code_ast.body[0].value, arg_values 78 | ) 79 | self.param_to_arg = param_to_arg 80 | self.arg_to_param = {} 81 | for param, args in param_to_arg.items(): 82 | for arg in args: 83 | self.arg_to_param[arg] = param 84 | 85 | def get_args(self) -> Set[ID]: 86 | # pytype: disable=bad-return-type 87 | return set(itertools.chain.from_iterable(self.param_to_arg.values())) 88 | # pytype: enable=bad-return-type 89 | 90 | def add_var_appearances(self, *var_appearances: VarAppearance): 91 | self.var_appearances.extend(var_appearances) 92 | 93 | def add_var_modifications(self, *var_modifications: VarModification): 94 | self.var_modifications.extend(var_modifications) 95 | 96 | def add_var_switches(self, *var_switches: VarSwitch): 97 | self.var_switches.extend(var_switches) 98 | 99 | def sync_tracking_with(self, other: "Node"): 100 | self.add_tracking(*other.tracking) 101 | 102 | def add_tracking(self, *new_ids: ID): 103 | """Updates identifiers being tracked. 104 | 105 | Identifiers being tracked must exist in data because we can't track something 106 | that don't exist in previous nodes. 107 | """ 108 | for new_id in new_ids: 109 | if new_id == "self": 110 | breakpoint() 111 | if new_id in self.vars: 112 | self.tracking.add(new_id) 113 | 114 | 115 | class Node: 116 | """Basic unit of an execution flow.""" 117 | 118 | __slots__ = frozenset( 119 | [ 120 | "type", 121 | "frame_id", 122 | "prev", 123 | "next", 124 | "step_into", 125 | "returned_from", 126 | "metadata", 127 | "is_target", 128 | ] 129 | ) 130 | 131 | def __init__( 132 | self, 133 | frame_id: Union[FrameID, Tuple[int, ...]], 134 | type: Optional[NodeType] = None, 135 | **kwargs, 136 | ): 137 | self.type = type 138 | if isinstance(frame_id, FrameID): 139 | self.frame_id = frame_id 140 | elif isinstance(frame_id, tuple): 141 | self.frame_id = FrameID(frame_id) 142 | self.prev: Optional[Node] = None 143 | self.next: Optional[Node] = None 144 | self.step_into: Optional[Node] = None 145 | self.returned_from: Optional[Node] = None 146 | self.metadata = TrackingMetadata(**kwargs) 147 | self.is_target = False 148 | 149 | def __repr__(self): 150 | return f"" 151 | 152 | def __getattr__(self, name): 153 | return getattr(self.metadata, name) 154 | 155 | def __setattr__(self, name, value): 156 | if name in self.__slots__: 157 | super().__setattr__(name, value) 158 | else: 159 | setattr(self.metadata, name, value) 160 | 161 | @property 162 | def shown_in_output(self) -> bool: 163 | """Whether this node should be shown in output.""" 164 | # TODO: Only inserts call node if there are var changes inside the call. 165 | return ( 166 | self.metadata.is_relevant_return 167 | or self.metadata.var_appearances 168 | or self.metadata.var_modifications 169 | or self.is_target 170 | or self.is_callsite 171 | ) 172 | 173 | @property 174 | def is_callsite(self): 175 | return self.step_into is not None 176 | 177 | def build_relation(self, **relation_dict: Dict[str, "Node"]): 178 | """A convenient function to add relations at once. 179 | 180 | Usage: 181 | node.build_relation(prev=node_x, next=node_y) 182 | """ 183 | for relation_name, node in relation_dict.items(): 184 | if relation_name not in {"prev", "next", "step_into", "returned_from"}: 185 | raise Exception("wrong relation_name: " + relation_name) 186 | setattr(self, relation_name, node) 187 | 188 | def get_and_update_var_changes( 189 | self, other: "Node" 190 | ) -> Iterable[Union[VarModification, VarAppearance]]: 191 | """Gets variable changes and stores them to current node. 192 | 193 | Current and next must live in the same frame. 194 | """ 195 | assert self.frame_id == other.frame_id 196 | for var_id in other.tracking: 197 | old_value = self.vars.get(var_id, _dummy) 198 | new_value = other.vars[var_id] 199 | if old_value is _dummy: 200 | var_appearance = VarAppearance(id=var_id, value=new_value) 201 | self.add_var_appearances(var_appearance) 202 | yield var_appearance 203 | elif utils.has_diff(new_value, old_value): 204 | var_modification = VarModification(var_id, old_value, new_value) 205 | self.add_var_modifications(var_modification) 206 | yield var_modification 207 | 208 | def update_var_changes_before_return(self): 209 | """Compares data with vars_before_return, records changes.""" 210 | if self.vars_before_return is None: 211 | pass 212 | for var_id in self.tracking: 213 | old_value = self.vars.get(var_id, _dummy) 214 | new_value = self.vars_before_return[var_id] 215 | if old_value is _dummy: 216 | var_appearance = VarAppearance(id=var_id, value=new_value) 217 | self.add_var_appearances(var_appearance) 218 | elif utils.has_diff(new_value, old_value): 219 | var_modification = VarModification(var_id, old_value, new_value) 220 | self.add_var_modifications(var_modification) 221 | 222 | 223 | class Flow: 224 | """Class that represents program's execution. 225 | 226 | A flow consists of multiple Calls and Nodes. 227 | """ 228 | 229 | ROOT = object() 230 | 231 | def __init__(self, start: Node, target: Node): 232 | self.start = start 233 | self.start.prev = self.ROOT 234 | self.target = target 235 | self.target.is_target = True 236 | self._update_target_id() 237 | 238 | def _update_target_id(self): 239 | """Gets ID('x') out of cyberbrain.register(x).""" 240 | register_call_ast = ast.parse(self.target.code_str.strip()) 241 | assert register_call_ast.body[0].value.func.value.id == "cyberbrain" 242 | 243 | # Finds the target identifier by checking argument passed to register(). 244 | # Assuming argument is a single identifier. 245 | self.target.add_tracking(ID(register_call_ast.body[0].value.args[0].id)) 246 | 247 | def __iter__(self): 248 | yield from self._trace_frame(self.start) 249 | 250 | def _trace_frame(self, current: Node): 251 | """Iterates and yields node in the frame where node is at.""" 252 | while current is not None: 253 | yield current 254 | if current.is_callsite: 255 | yield from self._trace_frame(current.step_into) 256 | current = current.next 257 | 258 | 259 | NodeInfo = namedtuple("NodeInfo", ["node", "surrounding", "arg_values"]) 260 | 261 | 262 | def build_flow(cm: ComputationManager) -> Flow: 263 | """Builds flow from computations. 264 | 265 | 1. Traverse through computations, create node, group nodes by frame id. 266 | 2. For each frame group, flatten nested calls, computes param_to_arg. 267 | 3. Add step_into and returned_from edges. 268 | 269 | call node should pass full code str to node, callsite_ast is only needed to 270 | generate param_to_arg 271 | """ 272 | start_node: Node 273 | target_node: Node 274 | frame_groups: Dict[FrameID, List[NodeInfo]] = defaultdict(list) 275 | 276 | for frame_id, comps in cm.frame_groups.items(): 277 | for comp in comps: 278 | if comp.event_type == "line": 279 | node = Node( 280 | type=NodeType.LINE, 281 | frame_id=frame_id, 282 | vars=comp.vars, 283 | code_str=comp.code_str, 284 | vars_before_return=comp.vars_before_return, 285 | source_location=comp.source_location, 286 | ) 287 | if hasattr(comp, "return_value"): 288 | node.return_value = comp.return_value 289 | elif comp.event_type == "call": 290 | node = Node( 291 | type=NodeType.CALL, 292 | frame_id=frame_id, 293 | vars=comp.vars, 294 | code_str=comp.code_str, 295 | source_location=comp.source_location, 296 | ) 297 | if frame_groups[frame_id]: 298 | frame_groups[frame_id][-1].node.next = node 299 | node.prev = frame_groups[frame_id][-1].node 300 | frame_groups[frame_id].append( 301 | NodeInfo(node, comp.surrounding, getattr(comp, "arg_values", None)) 302 | ) 303 | if comp is cm.target: 304 | target_node = node 305 | 306 | replace_calls(frame_groups) 307 | 308 | # Assuming init is called at program start. This may change in the future. 309 | start_node = frame_groups[(0,)][0].node 310 | 311 | return Flow(start_node, target_node) 312 | 313 | 314 | def replace_calls(frame_groups: Dict[FrameID, List[NodeInfo]]): 315 | """Replaces call exprs with intermediate variables.""" 316 | for _, frame in frame_groups.items(): 317 | i = 0 # call index in this frame. 318 | for _, group in itertools.groupby(frame, lambda x: x.surrounding): 319 | ast_to_intermediate: Dict[str, str] = {} 320 | intermediate_vars = {} # Mapping of intermediate vars and their values. 321 | for node, _, arg_values in group: 322 | # ri_ appeared before should be captured in node.vars. 323 | node.vars.update(intermediate_vars) 324 | # Replaces nested calls with intermediate vars. 325 | for inner_call, intermediate in ast_to_intermediate.items(): 326 | node.code_str = node.code_str.replace(inner_call, intermediate, 1) 327 | if node.type is NodeType.CALL: 328 | ast_to_intermediate[node.code_str] = f"r{i}_" 329 | node.code_str = f"r{i}_ = " + node.code_str 330 | node.step_into = frame_groups[node.frame_id + (i,)][0].node 331 | node.step_into.prev = node 332 | node.returned_from = frame_groups[node.frame_id + (i,)][-1].node 333 | intermediate_vars[f"r{i}_"] = node.returned_from.return_value 334 | i += 1 335 | node.code_ast = utils.parse_code_str(node.code_str) 336 | if node.type is NodeType.CALL: 337 | assert arg_values, "call node should have arg_values." 338 | node.set_param_arg_mapping(arg_values) 339 | 340 | # Deals with some special cases. 341 | assert len(node.code_ast.body) == 1 342 | stmt = node.code_ast.body[0] 343 | 344 | # Checks if LHS is ri_ and ri_ only, e.g. r1_ = f(1, 2) 345 | lhs_is_ri = lambda stmt: ( 346 | isinstance(stmt.value, ast.Name) and re.match(r"r[\d]+_", stmt.value.id) 347 | ) 348 | 349 | if isinstance(stmt, ast.Expr) and lhs_is_ri(stmt): 350 | # Current node is "r0_", previous node is "r0_ = f()". 351 | # This happens when the whole line is just "f()". 352 | # Solution: removes current node, restores previous node to "f()". 353 | assert node.type is NodeType.LINE 354 | prev = node.prev 355 | assert ( 356 | prev 357 | and prev.type is NodeType.CALL 358 | and prev.code_str.startswith(f"{stmt.value.id}") 359 | ) 360 | prev.next = node.next 361 | if node.next: 362 | node.next.prev = prev 363 | prev.code_str = prev.code_str.split("=", 1)[1].lstrip() 364 | prev.code_ast = utils.parse_code_str(node.code_str) 365 | elif isinstance(stmt, ast.Assign) and lhs_is_ri(stmt): 366 | # Current node represents "a = r0_", previous node is "r0_ = f()" 367 | # Solution: changes previous to 'a = f()', discards current node. 368 | # We don't need to modify frame_groups, it's not used in tracing. 369 | value = stmt.value 370 | prev = node.prev 371 | assert ( 372 | prev 373 | and prev.type is NodeType.CALL 374 | and prev.code_str.startswith(f"{value.id} =") 375 | ) 376 | prev.next = node.next 377 | if node.next: 378 | node.next.prev = prev 379 | prev.code_ast.body[0].targets = stmt.targets 380 | prev.code_str = astor.to_source(prev.code_ast).strip() 381 | -------------------------------------------------------------------------------- /cyberbrain/flow_test.py: -------------------------------------------------------------------------------- 1 | """Unit tests for flow.""" 2 | 3 | import inspect 4 | 5 | from . import backtrace 6 | from .flow import Node, Flow 7 | from .basis import ID 8 | 9 | 10 | def create_flow(): 11 | """Creates an execution flow. 12 | 13 | start 14 | | b 15 | a+---+ 16 | | | 17 | | | d 18 | | c+-------+ 19 | | | g | 20 | f+----+ | 21 | | | | 22 | target h e 23 | 24 | start { next: a, prev: None} 25 | a { next: None, prev: start, step_into: b, returned_from: None} 26 | b { next: c, prev: a} 27 | c { next: f, prev: b, step_into: d, returned_from: e} 28 | d { next: e, prev: c} 29 | e { prev: d} 30 | f { next: target, prev: b, step_into: g, returned_from: h} 31 | g { next: h, prev: f} 32 | h { prev: g} 33 | target { next: None, prev: f} 34 | 35 | Assuming code live in a single module, like this: 36 | 37 | def func_f(bar): 38 | x = len(bar) # g 39 | return x # h 40 | 41 | def func_c(baa): 42 | baa.append(None) # d 43 | baa.append('?') # e 44 | 45 | def func_a(foo): 46 | ba = [foo] # b 47 | func_c(ba) # c 48 | foo = func_f(ba) # f 49 | cyberbrain.register(foo) # target 50 | 51 | cyberbrain.init() 52 | fo = 1 # start 53 | func_a(fo) # a 54 | """ 55 | GLOBAL_FRAME = (0,) 56 | FUNC_A_FRAME = (0, 0) 57 | FUNC_C_FRAME = (0, 0, 0) 58 | FUNC_F_FRAME = (0, 0, 1) 59 | 60 | # Common data 61 | functions = { 62 | ID("func_f"): "", 63 | ID("func_c"): "", 64 | ID("func_a"): "", 65 | } 66 | 67 | # Creates nodes. 68 | node_start = Node(GLOBAL_FRAME, code_str="fo = 1", vars={**functions}) 69 | node_a = Node(GLOBAL_FRAME, code_str="func_a(fo)", vars={ID("fo"): 1, **functions}) 70 | node_a.set_param_arg_mapping( 71 | inspect.ArgInfo(args=["foo"], varargs=None, keywords=None, locals={"foo": 1}) 72 | ) 73 | node_b = Node(FUNC_A_FRAME, code_str="ba = [foo]", vars={ID("foo"): 1, **functions}) 74 | node_c = Node( 75 | FUNC_A_FRAME, 76 | code_str="func_c(ba)", 77 | vars={ID("foo"): 1, ID("ba"): [1], **functions}, 78 | ) 79 | node_c.set_param_arg_mapping( 80 | inspect.ArgInfo(args=["baa"], varargs=None, keywords=None, locals={"baa": [1]}) 81 | ) 82 | node_d = Node( 83 | FUNC_C_FRAME, code_str="baa.append(None)", vars={ID("baa"): [1], **functions} 84 | ) 85 | node_e = Node( 86 | FUNC_C_FRAME, 87 | code_str="baa.append('?')", 88 | vars={ID("baa"): [1, None], **functions}, 89 | vars_before_return={ID("baa"): [1, None, "?"], **functions}, 90 | ) 91 | node_f = Node( 92 | FUNC_A_FRAME, 93 | code_str="foo = func_f(ba)", 94 | vars={ID("foo"): 1, ID("ba"): [1, None, "?"], **functions}, 95 | ) 96 | node_f.set_param_arg_mapping( 97 | inspect.ArgInfo( 98 | args=["bar"], varargs=None, keywords=None, locals={"bar": [1, None, "?"]} 99 | ) 100 | ) 101 | node_g = Node( 102 | FUNC_F_FRAME, 103 | code_str="x = len(bar)", 104 | vars={ID("bar"): [1, None, "?"], **functions}, 105 | ) 106 | node_h = Node( 107 | FUNC_F_FRAME, 108 | code_str="return x", 109 | vars={ID("bar"): [1, None, "?"], ID("x"): 3, **functions}, 110 | vars_before_return={ID("bar"): [1, None, "?"], ID("x"): 3, **functions}, 111 | ) 112 | node_target = Node( 113 | FUNC_A_FRAME, 114 | code_str="cyberbrain.register(foo)", 115 | vars={ID("foo"): 3, ID("ba"): [1, None, "?"], **functions}, 116 | ) 117 | 118 | # Builds relation. 119 | node_start.next = node_a 120 | node_a.build_relation(prev=node_start, step_into=node_b) 121 | node_b.build_relation(next=node_c, prev=node_a) 122 | node_c.build_relation( 123 | next=node_f, prev=node_b, step_into=node_d, returned_from=node_e 124 | ) 125 | node_d.build_relation(next=node_e, prev=node_c) 126 | node_e.build_relation(prev=node_d) 127 | node_f.build_relation( 128 | next=node_target, prev=node_c, step_into=node_g, returned_from=node_h 129 | ) 130 | node_g.build_relation(next=node_h, prev=node_f) 131 | node_h.build_relation(prev=node_g) 132 | node_target.build_relation(prev=node_f) 133 | 134 | return Flow(start=node_start, target=node_target) 135 | 136 | 137 | def test_traverse_flow(): 138 | flow = create_flow() 139 | backtrace.trace_flow(flow) 140 | assert [str(node) for node in flow] == [ 141 | "", 142 | "", 143 | "", 144 | "", 145 | "", 146 | "", 147 | "", 148 | "", 149 | "", 150 | "", 151 | ] 152 | -------------------------------------------------------------------------------- /cyberbrain/format.py: -------------------------------------------------------------------------------- 1 | """Formats trace result to user friendly output.""" 2 | 3 | import html 4 | import itertools 5 | from os.path import abspath, basename, expanduser, join 6 | from typing import List 7 | 8 | from graphviz import Digraph 9 | 10 | from . import utils 11 | from .flow import Flow, Node 12 | 13 | DESKTOP = abspath(join(expanduser("~"), "Desktop")) 14 | 15 | # g = Digraph( 16 | # name="Cyberbrain Output", graph_attr=[("forcelabels", "true")], format="canon" 17 | # ) 18 | g = Digraph(name="Cyberbrain Output") 19 | 20 | # Color comes from https://paletton.com/#uid=35d0u0kkuDpafTcfVLxoCwpucs+. 21 | g.attr("edge", color="#E975B0", penwidth="2") 22 | 23 | 24 | class NodeView: 25 | """Class that wraps a node and deal with visualization.""" 26 | 27 | _portname_cache = {} # Maps node address to their node to avoid duplicates. 28 | _incrementor = itertools.count() # Generates 1, 2, 3, ... 29 | 30 | def __init__(self, node: Node): 31 | self._node = node 32 | self._portname = self._generate_portname() 33 | 34 | def __getattr__(self, name): 35 | """Redirects attribute access to stored node.""" 36 | return getattr(self._node, name) 37 | 38 | def _generate_portname(self): 39 | node_addr = id(self._node) 40 | if node_addr not in self._portname_cache: 41 | self._portname_cache[node_addr] = str(next(self._incrementor)) 42 | return self._portname_cache[node_addr] 43 | 44 | @property 45 | def portname(self): 46 | return self._portname 47 | 48 | @property 49 | def tracking(self): 50 | """Tracking does not necessarily need to be displayed.""" 51 | return str(self._node.tracking) 52 | 53 | @property 54 | def var_changes(self): 55 | """Formats var changes.""" 56 | output = "" 57 | for ap in self.var_appearances: 58 | output += f"{ap.id} = {ap.value}\n" 59 | 60 | for mod in self.var_modifications: 61 | output += f"{mod.id} {mod.old_value} → {mod.new_value}\n" 62 | 63 | if self.is_target: 64 | output += "🐸🐸🐸🐸🐸🐸" 65 | 66 | if self.is_relevant_return: 67 | output += f"return {self.return_value}" 68 | 69 | return output 70 | 71 | @property 72 | def next(self): 73 | return NodeView(self._node.next) if self._node.next else None 74 | 75 | @property 76 | def step_into(self): 77 | return NodeView(self._node.step_into) if self._node.step_into else None 78 | 79 | @property 80 | def returned_from(self): 81 | return NodeView(self._node.returned_from) if self._node.returned_from else None 82 | 83 | 84 | def generate_subgraph(frame_start: NodeView): 85 | current = frame_start 86 | name = str(frame_start.frame_id) + "_code" 87 | lines: List[str] = [] 88 | while current is not None: 89 | if not current.shown_in_output: 90 | current = current.next 91 | continue 92 | # Syntax hilight is very hard in Graphviz, because modern highlighters don't use 93 | # the deprecated way but class and css, which is not supported 94 | # in Graphviz. 95 | lines.append( 96 | utils.dedent( 97 | f""" 98 | 99 | {html.escape(current.code_str)} 100 | 101 | " 102 | 103 | ◤ {html.escape(current.var_changes)} 104 | 105 | 106 | """ 107 | ) 108 | ) 109 | if current.is_callsite: 110 | g.edge( 111 | f"{name}:{current.portname}", 112 | generate_subgraph(current.step_into), 113 | # TODO: Ideally this should be param → argument expression, 114 | # e.g. x → [1, 2, 3], so that users can see the passed value. 115 | label="\n".join( 116 | f" {','.join(args)} → {param}, {current.step_into.vars[param]}" 117 | for param, args in current.param_to_arg.items() 118 | ), 119 | ) 120 | current = current.next 121 | rows = ( 122 | [ 123 | utils.dedent( 124 | f""" 125 | 126 | {html.escape( 127 | basename(frame_start.source_location.filepath))} 128 | : {html.escape(frame_start.frame_id.co_name)} 129 | 130 | 131 | 132 | """ 133 | ) 134 | ] 135 | + lines 136 | ) 137 | g.node( 138 | name, 139 | label="<%s
>" % "".join(rows), 140 | shape="plaintext", 141 | ) 142 | return name 143 | 144 | 145 | def generate_output(flow: Flow, filename=None): 146 | generate_subgraph(NodeView(flow.start)) 147 | # print(g.pipe().decode("utf-8")) 148 | g.render(join(DESKTOP, filename or "output"), view=True) 149 | -------------------------------------------------------------------------------- /cyberbrain/testing.py: -------------------------------------------------------------------------------- 1 | """Debugging utilities.""" 2 | 3 | import json 4 | import os 5 | from collections import defaultdict 6 | from os.path import basename 7 | from pprint import pprint 8 | from typing import Dict, List 9 | 10 | from absl import flags 11 | from crayons import yellow # pylint: disable=E0611 12 | 13 | from .computation import ComputationManager 14 | from .flow import Flow, Node 15 | 16 | FLAGS = flags.FLAGS 17 | 18 | COMPUTATION_TEST_OUTPUT = "computation.json" 19 | COMPUTATION_GOLDEN = "computation.golden.json" 20 | FLOW_TEST_OUTPUT = "flow.json" 21 | FLOW_GOLDEN = "flow.golden.json" 22 | 23 | 24 | class _SetEncoder(json.JSONEncoder): 25 | """Custom encoder to dump set as list.""" 26 | 27 | def default(self, o): # pylint: disable=E0202 28 | if isinstance(o, set): 29 | return sorted(list(o)) 30 | return json.JSONEncoder.default(self, o) 31 | 32 | 33 | def print_if_debug(content): 34 | """Prints content if in debug mode.""" 35 | if FLAGS.mode == "debug": 36 | pprint(content) 37 | 38 | 39 | def dump_computation(cm: ComputationManager): 40 | """Converts computations to a JSON and writes to stdout. 41 | 42 | Caller should receive and handle output. 43 | """ 44 | if FLAGS.mode == "test": 45 | filepath = os.path.join(FLAGS.test_dir, COMPUTATION_TEST_OUTPUT) 46 | elif FLAGS.mode == "golden": 47 | filepath = os.path.join(FLAGS.test_dir, COMPUTATION_GOLDEN) 48 | print(yellow("Generating test data: " + filepath)) 49 | 50 | output = { 51 | str(fid): [c.to_dict() for c in comps] for fid, comps in cm.frame_groups.items() 52 | } 53 | if FLAGS.mode == "debug": 54 | pprint(f"Computation is:\n{json.dumps(obj=output, indent=2, cls=_SetEncoder)}") 55 | return 56 | 57 | with open(filepath, "w") as f: 58 | json.dump(obj=output, fp=f, indent=2, cls=_SetEncoder) 59 | 60 | 61 | def _dump_node(node: Node) -> Dict: 62 | result = defaultdict(list) 63 | result.update( 64 | { 65 | "code": node.code_str, 66 | "next": getattr(node.next, "code_str", ""), 67 | "prev": getattr(node.prev, "code_str", ""), 68 | "step_into": getattr(node.step_into, "code_str", ""), 69 | "returned_from": getattr(node.returned_from, "code_str", ""), 70 | } 71 | ) 72 | for ap in node.var_appearances: 73 | result["var_changes"].append(f"appear {ap.id}={ap.value}\n") 74 | for mod in node.var_modifications: 75 | result["var_changes"].append( 76 | f"modify {mod.id} {mod.old_value} -> {mod.new_value}\n" 77 | ) 78 | result[ 79 | "location" 80 | ] = f"{basename(node.source_location.filepath)}: {node.frame_id.co_name}" 81 | if node.tracking: 82 | result["tracking"] = node.tracking 83 | if hasattr(node, "param_to_arg"): 84 | result["param_to_arg"] = node.param_to_arg 85 | return result 86 | 87 | 88 | def get_dumpable_flow(flow: Flow) -> List[Dict]: 89 | """Transforms flow to a dumpable representation.""" 90 | return [_dump_node(node) for node in flow] 91 | 92 | 93 | def dump_flow(flow: Flow): 94 | output = get_dumpable_flow(flow) 95 | 96 | if FLAGS.mode == "test": 97 | filepath = os.path.join(FLAGS.test_dir, FLOW_TEST_OUTPUT) 98 | elif FLAGS.mode == "golden": 99 | filepath = os.path.join(FLAGS.test_dir, FLOW_GOLDEN) 100 | print(yellow("Generating test data: " + filepath)) 101 | 102 | if FLAGS.mode == "debug": 103 | pprint(f"Flow is:\n{json.dumps(obj=output, indent=2, cls=_SetEncoder)}") 104 | return 105 | 106 | with open(filepath, "w") as f: 107 | json.dump(obj=output, fp=f, indent=2, cls=_SetEncoder) 108 | -------------------------------------------------------------------------------- /cyberbrain/utils.py: -------------------------------------------------------------------------------- 1 | """Utility functions.""" 2 | 3 | import ast 4 | import dis 5 | import inspect 6 | import io 7 | import itertools 8 | import sysconfig 9 | import token 10 | import tokenize 11 | import typing 12 | from functools import lru_cache 13 | from typing import List, Tuple 14 | 15 | import astor 16 | import black 17 | import deepdiff 18 | 19 | from .basis import ID, Surrounding 20 | 21 | try: 22 | from token import NL as token_NL 23 | except ImportError: 24 | from tokenize import NL as token_NL 25 | 26 | _INSTALLATION_PATHS = list(sysconfig.get_paths().values()) 27 | 28 | 29 | @lru_cache() 30 | def should_exclude(filename): 31 | """Determines whether we should log events from file. 32 | 33 | As of now, we exclude files from installation path, which usually means: 34 | .../3.7.1/lib/python3.7 35 | .../3.7.1/include/python3.7m 36 | .../lib/python3.7/site-packages 37 | 38 | Also we exclude frozen modules, as well as some weird cases. 39 | """ 40 | if any(filename.startswith(path) for path in _INSTALLATION_PATHS) or any( 41 | name in filename 42 | for name in ( 43 | "importlib._boostrap", 44 | "importlib._bootstrap_external", 45 | "zipimport", 46 | "", # Dynamically generated frames, like 47 | ) 48 | ): 49 | return True 50 | 51 | return False 52 | 53 | 54 | def grouped(iterable, n): 55 | """Copies from https://stackoverflow.com/a/5389547/2142577. 56 | 57 | s -> (s0,s1,s2,...sn-1), (sn,sn+1,sn+2,...s2n-1), (s2n,s2n+1,s2n+2,...s3n-1), ... 58 | """ 59 | return zip(*[iter(iterable)] * n) 60 | 61 | 62 | def _get_lineno(frame) -> int: 63 | """Gets line number given current state of the frame. 64 | 65 | Line number is the absolute line number in the module where current execution is at. 66 | In the past we used to calculate lineno based on lnotab, but since 67 | `dis.findlinestarts` already does that, there's no need to do it ourselves. 68 | """ 69 | f_lasti = frame.f_lasti 70 | lineno_at_lasti = 0 71 | for offset, lineno in dis.findlinestarts(frame.f_code): 72 | if offset > f_lasti: 73 | break 74 | lineno_at_lasti = lineno 75 | # Returns here if this is last line in that frame. 76 | return lineno_at_lasti 77 | 78 | 79 | _tokens_cache = {} 80 | 81 | 82 | def _get_module_token_groups(frame) -> List[List[tokenize.TokenInfo]]: 83 | """Tokenizes module's source code that the frame belongs to, yields tokens. 84 | 85 | Return value is a list, with every element being a list of tokens that belong to the 86 | same logical line. 87 | """ 88 | module, filename = inspect.getmodule(frame), inspect.getsourcefile(frame) 89 | if filename in _tokens_cache: 90 | return _tokens_cache[filename] 91 | 92 | source_code = inspect.getsource(module) 93 | it = tokenize.tokenize(io.BytesIO(source_code.encode("utf-8")).readline) 94 | next(it, None) # Skips the first element which is always token.ENCODING 95 | # Groups tokens by logical lines. 96 | # [tok1, tok2, NEWLINE, tok3, NEWLINE, tok4, NEWLINE] will result in 97 | # [[tok1, tok2], [tok3], [tok4]] 98 | groups = [ 99 | list(group) 100 | for is_newline, group in itertools.groupby( 101 | it, lambda tok: tok.type == token.NEWLINE 102 | ) 103 | if not is_newline 104 | ] 105 | _tokens_cache[filename] = groups 106 | return groups 107 | 108 | 109 | def get_code_str_and_surrounding(frame) -> Tuple[str, Surrounding]: 110 | """Gets code string and surrounding information for line event. 111 | 112 | The reason to record both code_str and surrounding is because code_str is not 113 | guaranteed to be unique, for example "a = true" appeared twice. While 114 | (frame_id, surrounding) is distinct, therefore we can detect duplicate computations 115 | by checking their (frame_id, surrounding). 116 | 117 | Both lineno and surrounding are 1-based, aka the smallest lineno is 1. 118 | """ 119 | lineno = _get_lineno(frame) 120 | groups: List[List[tokenize.TokenInfo]] = _get_module_token_groups(frame) 121 | 122 | # Given a lineno, locates the logical line that contains this line. 123 | if len(groups) == 1: 124 | return ( 125 | inspect.getsource(frame), 126 | Surrounding(start_lineno=lineno, end_lineno=lineno), 127 | ) 128 | 129 | for group, next_group in zip(groups[:-1], groups[1:]): 130 | start_lineno, end_lineno = group[0].start[0], group[-1].end[0] 131 | if start_lineno <= lineno <= end_lineno: 132 | break 133 | else: 134 | # Reachs end of groups 135 | group = next_group 136 | 137 | # Removes leading NL and DEDENT as they cause untokenize to fail. 138 | while group[0].type in {token_NL, token.DEDENT, token.INDENT}: 139 | group.pop(0) 140 | # When untokenizing, Python adds \\\n for absent lines(because lineno in 141 | # group doesn't start from 1), removes them. 142 | # Note that since we've removed the leading ENCODING token, untokenize will return 143 | # a str instead of encoded bytes. 144 | return ( 145 | tokenize.untokenize(group).lstrip("\\\n"), 146 | Surrounding(start_lineno=group[0].start[0], end_lineno=group[-1].end[0]), 147 | ) 148 | 149 | 150 | class _NameVisitor(ast.NodeVisitor): 151 | def __init__(self): 152 | self.names: typing.Set[ID] = set() 153 | super().__init__() 154 | 155 | def visit_Name(self, node): 156 | self.names.add(node.id) 157 | self.generic_visit(node) 158 | 159 | 160 | def find_names(code_ast: ast.AST) -> typing.Set[ID]: 161 | """Finds idenditifiers in given ast node.""" 162 | visitor = _NameVisitor() 163 | visitor.visit(code_ast) 164 | return {ID(name) for name in visitor.names} 165 | 166 | 167 | def has_diff(x, y): 168 | return deepdiff.DeepDiff(x, y) != {} 169 | 170 | 171 | def parse_code_str(code_str) -> ast.AST: 172 | """Parses code string in a computation, which can be incomplete. 173 | 174 | Once we found something that leads to error while parsing, we should handle it here. 175 | """ 176 | if code_str.endswith(":"): 177 | code_str += "pass" 178 | try: 179 | return ast.parse(code_str) 180 | except IndentationError: 181 | return ast.parse(code_str.strip()) 182 | 183 | 184 | def ast_to_str(code_ast: ast.AST) -> str: 185 | # Makes sure code is always in the same format. 186 | return black.format_str(astor.to_source(code_ast), mode=black.FileMode()).strip() 187 | 188 | 189 | def dedent(text: str): 190 | return "\n".join([line.strip() for line in text.splitlines()]) 191 | -------------------------------------------------------------------------------- /cyberbrain/vars.py: -------------------------------------------------------------------------------- 1 | """Variable managing utilities.""" 2 | 3 | import copy 4 | import inspect 5 | import itertools 6 | from collections import UserDict 7 | 8 | 9 | class Vars(UserDict): # pylint: disable=too-many-ancestors 10 | """A class that holds variable values in a trace event.""" 11 | 12 | def __init__(self, frame): 13 | super().__init__() 14 | self._scan_namespaces(frame) 15 | del frame 16 | 17 | def __getitem__(self, name): 18 | return self.data[name] 19 | 20 | def _scan_namespaces(self, frame): 21 | """Records variables from bottom to top.""" 22 | for name, value in itertools.chain.from_iterable( 23 | [frame.f_locals.items(), frame.f_globals.items()] 24 | ): 25 | # TODO: exclude other stuff we don't need. 26 | if inspect.ismodule(value) or inspect.isbuiltin(value): 27 | continue 28 | # Tries copy as deep as possible so that changes won't affect stored value. 29 | try: 30 | self.data[name] = copy.deepcopy(value) 31 | except TypeError: 32 | try: 33 | self.data[name] = copy.copy(value) 34 | except TypeError: 35 | self.data[name] = value 36 | -------------------------------------------------------------------------------- /images/cb_concept.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/laike9m/Cyberbrain-Deprecated/e79102a15499daa3d93d471616f453e96ececfdb/images/cb_concept.jpg -------------------------------------------------------------------------------- /images/cb_output.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/laike9m/Cyberbrain-Deprecated/e79102a15499daa3d93d471616f453e96ececfdb/images/cb_output.jpg -------------------------------------------------------------------------------- /images/cb_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/laike9m/Cyberbrain-Deprecated/e79102a15499daa3d93d471616f453e96ececfdb/images/cb_output.png -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- 1 | [[package]] 2 | category = "main" 3 | description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." 4 | name = "absl-py" 5 | optional = false 6 | python-versions = "*" 7 | version = "0.7.1" 8 | 9 | [package.dependencies] 10 | six = "*" 11 | 12 | [[package]] 13 | category = "dev" 14 | description = "apipkg: namespace control and lazy-import mechanism" 15 | name = "apipkg" 16 | optional = false 17 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 18 | version = "1.5" 19 | 20 | [[package]] 21 | category = "main" 22 | description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." 23 | name = "appdirs" 24 | optional = false 25 | python-versions = "*" 26 | version = "1.4.3" 27 | 28 | [[package]] 29 | category = "main" 30 | description = "Read/rewrite/write Python ASTs" 31 | name = "astor" 32 | optional = false 33 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 34 | version = "0.8.0" 35 | 36 | [[package]] 37 | category = "dev" 38 | description = "Pretty print the output of python stdlib `ast.parse`." 39 | name = "astpretty" 40 | optional = false 41 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 42 | version = "1.6.0" 43 | 44 | [[package]] 45 | category = "main" 46 | description = "An abstract syntax tree for Python with inference support." 47 | name = "astroid" 48 | optional = false 49 | python-versions = ">=3.5.*" 50 | version = "2.3.2" 51 | 52 | [package.dependencies] 53 | lazy-object-proxy = ">=1.4.0,<1.5.0" 54 | six = "1.12" 55 | wrapt = ">=1.11.0,<1.12.0" 56 | 57 | [package.dependencies.typed-ast] 58 | python = "<3.8" 59 | version = ">=1.4.0,<1.5" 60 | 61 | [[package]] 62 | category = "dev" 63 | description = "Atomic file writes." 64 | name = "atomicwrites" 65 | optional = false 66 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 67 | version = "1.3.0" 68 | 69 | [[package]] 70 | category = "main" 71 | description = "Classes Without Boilerplate" 72 | name = "attrs" 73 | optional = false 74 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 75 | version = "19.3.0" 76 | 77 | [[package]] 78 | category = "main" 79 | description = "The uncompromising code formatter." 80 | name = "black" 81 | optional = false 82 | python-versions = ">=3.6" 83 | version = "19.10b0" 84 | 85 | [package.dependencies] 86 | appdirs = "*" 87 | attrs = ">=18.1.0" 88 | click = ">=6.5" 89 | pathspec = ">=0.6,<1" 90 | regex = "*" 91 | toml = ">=0.9.4" 92 | typed-ast = ">=1.4.0" 93 | 94 | [[package]] 95 | category = "main" 96 | description = "Composable command line interface toolkit" 97 | name = "click" 98 | optional = false 99 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 100 | version = "7.0" 101 | 102 | [[package]] 103 | category = "main" 104 | description = "Cross-platform colored terminal text." 105 | name = "colorama" 106 | optional = false 107 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 108 | version = "0.4.1" 109 | 110 | [[package]] 111 | category = "main" 112 | description = "TextUI colors for Python." 113 | name = "crayons" 114 | optional = false 115 | python-versions = "*" 116 | version = "0.2.0" 117 | 118 | [package.dependencies] 119 | colorama = "*" 120 | 121 | [[package]] 122 | category = "main" 123 | description = "A backport of the dataclasses module for Python 3.6" 124 | name = "dataclasses" 125 | optional = false 126 | python-versions = "*" 127 | version = "0.6" 128 | 129 | [[package]] 130 | category = "dev" 131 | description = "Decorators for Humans" 132 | name = "decorator" 133 | optional = false 134 | python-versions = ">=2.6, !=3.0.*, !=3.1.*" 135 | version = "4.4.1" 136 | 137 | [[package]] 138 | category = "main" 139 | description = "Deep Difference and Search of any Python object/data." 140 | name = "deepdiff" 141 | optional = false 142 | python-versions = ">=3.4" 143 | version = "4.0.8" 144 | 145 | [package.dependencies] 146 | jsonpickle = ">=1.0" 147 | ordered-set = ">=3.1.1" 148 | 149 | [[package]] 150 | category = "dev" 151 | description = "execnet: rapid multi-Python deployment" 152 | name = "execnet" 153 | optional = false 154 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 155 | version = "1.7.1" 156 | 157 | [package.dependencies] 158 | apipkg = ">=1.4" 159 | 160 | [[package]] 161 | category = "main" 162 | description = "Get the currently executing AST node of a frame, and other information" 163 | name = "executing" 164 | optional = false 165 | python-versions = "*" 166 | version = "0.3.2" 167 | 168 | [[package]] 169 | category = "dev" 170 | description = "A platform independent file lock." 171 | name = "filelock" 172 | optional = false 173 | python-versions = "*" 174 | version = "3.0.12" 175 | 176 | [[package]] 177 | category = "main" 178 | description = "Simple Python interface for Graphviz" 179 | name = "graphviz" 180 | optional = false 181 | python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" 182 | version = "0.11.1" 183 | 184 | [[package]] 185 | category = "dev" 186 | description = "A library to calculate python dependency graphs." 187 | name = "importlab" 188 | optional = false 189 | python-versions = ">=2.7.0" 190 | version = "0.5.1" 191 | 192 | [package.dependencies] 193 | networkx = "*" 194 | six = "*" 195 | 196 | [[package]] 197 | category = "dev" 198 | description = "Read metadata from Python packages" 199 | marker = "python_version < \"3.8\"" 200 | name = "importlib-metadata" 201 | optional = false 202 | python-versions = ">=2.7,!=3.0,!=3.1,!=3.2,!=3.3" 203 | version = "0.23" 204 | 205 | [package.dependencies] 206 | zipp = ">=0.5" 207 | 208 | [[package]] 209 | category = "main" 210 | description = "A Python utility / library to sort Python imports." 211 | name = "isort" 212 | optional = false 213 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 214 | version = "4.3.21" 215 | 216 | [[package]] 217 | category = "main" 218 | description = "Python library for serializing any arbitrary object graph into JSON" 219 | name = "jsonpickle" 220 | optional = false 221 | python-versions = "*" 222 | version = "1.2" 223 | 224 | [[package]] 225 | category = "main" 226 | description = "A fast and thorough lazy object proxy." 227 | name = "lazy-object-proxy" 228 | optional = false 229 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 230 | version = "1.4.3" 231 | 232 | [[package]] 233 | category = "main" 234 | description = "McCabe checker, plugin for flake8" 235 | name = "mccabe" 236 | optional = false 237 | python-versions = "*" 238 | version = "0.6.1" 239 | 240 | [[package]] 241 | category = "dev" 242 | description = "More routines for operating on iterables, beyond itertools" 243 | marker = "python_version < \"3.8\" or python_version > \"2.7\"" 244 | name = "more-itertools" 245 | optional = false 246 | python-versions = ">=3.4" 247 | version = "7.2.0" 248 | 249 | [[package]] 250 | category = "dev" 251 | description = "Python package for creating and manipulating graphs and networks" 252 | name = "networkx" 253 | optional = false 254 | python-versions = ">=3.5" 255 | version = "2.4" 256 | 257 | [package.dependencies] 258 | decorator = ">=4.3.0" 259 | 260 | [[package]] 261 | category = "dev" 262 | description = "Ninja is a small build system with a focus on speed" 263 | name = "ninja" 264 | optional = false 265 | python-versions = "*" 266 | version = "1.9.0.post1" 267 | 268 | [[package]] 269 | category = "main" 270 | description = "A MutableSet that remembers its order, so that every entry has an index." 271 | name = "ordered-set" 272 | optional = false 273 | python-versions = ">=2.7" 274 | version = "3.1.1" 275 | 276 | [[package]] 277 | category = "dev" 278 | description = "Core utilities for Python packages" 279 | name = "packaging" 280 | optional = false 281 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 282 | version = "19.2" 283 | 284 | [package.dependencies] 285 | pyparsing = ">=2.0.2" 286 | six = "*" 287 | 288 | [[package]] 289 | category = "main" 290 | description = "Utility library for gitignore style pattern matching of file paths." 291 | name = "pathspec" 292 | optional = false 293 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 294 | version = "0.6.0" 295 | 296 | [[package]] 297 | category = "dev" 298 | description = "plugin and hook calling mechanisms for python" 299 | name = "pluggy" 300 | optional = false 301 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 302 | version = "0.13.0" 303 | 304 | [package.dependencies] 305 | [package.dependencies.importlib-metadata] 306 | python = "<3.8" 307 | version = ">=0.12" 308 | 309 | [[package]] 310 | category = "dev" 311 | description = "library with cross-python path, ini-parsing, io, code, log facilities" 312 | name = "py" 313 | optional = false 314 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 315 | version = "1.8.0" 316 | 317 | [[package]] 318 | category = "dev" 319 | description = "Hamcrest framework for matcher objects" 320 | name = "pyhamcrest" 321 | optional = false 322 | python-versions = "*" 323 | version = "1.9.0" 324 | 325 | [package.dependencies] 326 | setuptools = "*" 327 | six = "*" 328 | 329 | [[package]] 330 | category = "main" 331 | description = "python code static checker" 332 | name = "pylint" 333 | optional = false 334 | python-versions = ">=3.5.*" 335 | version = "2.5.0.dev1" 336 | 337 | [package.dependencies] 338 | astroid = ">=2.3.0,<2.4" 339 | colorama = "*" 340 | isort = ">=4.2.5,<5" 341 | mccabe = ">=0.6,<0.7" 342 | toml = ">=0.7.1" 343 | 344 | [package.source] 345 | reference = "96b3892506de6033ecb4f6e288bd4f5b211d894c" 346 | type = "git" 347 | url = "https://github.com/PyCQA/pylint.git" 348 | 349 | [[package]] 350 | category = "main" 351 | description = "A development tool to measure, monitor and analyze the memory behavior of Python objects." 352 | name = "pympler" 353 | optional = false 354 | python-versions = "*" 355 | version = "0.8" 356 | 357 | [[package]] 358 | category = "dev" 359 | description = "Python parsing module" 360 | name = "pyparsing" 361 | optional = false 362 | python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" 363 | version = "2.4.2" 364 | 365 | [[package]] 366 | category = "dev" 367 | description = "pytest: simple powerful testing with Python" 368 | name = "pytest" 369 | optional = false 370 | python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" 371 | version = "4.6.6" 372 | 373 | [package.dependencies] 374 | atomicwrites = ">=1.0" 375 | attrs = ">=17.4.0" 376 | colorama = "*" 377 | packaging = "*" 378 | pluggy = ">=0.12,<1.0" 379 | py = ">=1.5.0" 380 | six = ">=1.10.0" 381 | wcwidth = "*" 382 | 383 | [package.dependencies.importlib-metadata] 384 | python = "<3.8" 385 | version = ">=0.12" 386 | 387 | [package.dependencies.more-itertools] 388 | python = ">=2.8" 389 | version = ">=4.0.0" 390 | 391 | [[package]] 392 | category = "dev" 393 | description = "run tests in isolated forked subprocesses" 394 | name = "pytest-forked" 395 | optional = false 396 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 397 | version = "1.1.3" 398 | 399 | [package.dependencies] 400 | pytest = ">=3.1.0" 401 | 402 | [[package]] 403 | category = "dev" 404 | description = "pytest xdist plugin for distributed testing and loop-on-failing modes" 405 | name = "pytest-xdist" 406 | optional = false 407 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 408 | version = "1.30.0" 409 | 410 | [package.dependencies] 411 | execnet = ">=1.1" 412 | pytest = ">=4.4.0" 413 | pytest-forked = "*" 414 | six = "*" 415 | 416 | [[package]] 417 | category = "dev" 418 | description = "Python type inferencer" 419 | name = "pytype" 420 | optional = false 421 | python-versions = "*" 422 | version = "2019.10.17" 423 | 424 | [package.dependencies] 425 | attrs = "*" 426 | importlab = ">=0.5.1" 427 | ninja = "*" 428 | pyyaml = ">=3.11" 429 | six = "*" 430 | typed_ast = "*" 431 | 432 | [[package]] 433 | category = "dev" 434 | description = "YAML parser and emitter for Python" 435 | name = "pyyaml" 436 | optional = false 437 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 438 | version = "5.1.2" 439 | 440 | [[package]] 441 | category = "main" 442 | description = "Alternative regular expression module, to replace re." 443 | name = "regex" 444 | optional = false 445 | python-versions = "*" 446 | version = "2019.08.19" 447 | 448 | [[package]] 449 | category = "main" 450 | description = "Python 2 and 3 compatibility utilities" 451 | name = "six" 452 | optional = false 453 | python-versions = ">=2.6, !=3.0.*, !=3.1.*" 454 | version = "1.12.0" 455 | 456 | [[package]] 457 | category = "main" 458 | description = "Python Library for Tom's Obvious, Minimal Language" 459 | name = "toml" 460 | optional = false 461 | python-versions = "*" 462 | version = "0.10.0" 463 | 464 | [[package]] 465 | category = "dev" 466 | description = "tox is a generic virtualenv management and test command line tool" 467 | name = "tox" 468 | optional = false 469 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 470 | version = "3.14.0" 471 | 472 | [package.dependencies] 473 | filelock = ">=3.0.0,<4" 474 | packaging = ">=14" 475 | pluggy = ">=0.12.0,<1" 476 | py = ">=1.4.17,<2" 477 | six = ">=1.0.0,<2" 478 | toml = ">=0.9.4" 479 | virtualenv = ">=14.0.0" 480 | 481 | [package.dependencies.importlib-metadata] 482 | python = "<3.8" 483 | version = ">=0.12,<1" 484 | 485 | [[package]] 486 | category = "dev" 487 | description = "tox plugin that makes tox use `pyenv which` to find python executables" 488 | name = "tox-pyenv" 489 | optional = false 490 | python-versions = "*" 491 | version = "1.1.0" 492 | 493 | [package.dependencies] 494 | tox = ">=2.0" 495 | 496 | [[package]] 497 | category = "dev" 498 | description = "tox plugin to run arbitrary commands in a virtualenv" 499 | name = "tox-run-command" 500 | optional = false 501 | python-versions = "*" 502 | version = "0.4" 503 | 504 | [package.dependencies] 505 | tox = ">=2.0" 506 | 507 | [[package]] 508 | category = "main" 509 | description = "a fork of Python 2 and 3 ast modules with type comment support" 510 | name = "typed-ast" 511 | optional = false 512 | python-versions = "*" 513 | version = "1.4.0" 514 | 515 | [[package]] 516 | category = "dev" 517 | description = "Virtual Python Environment builder" 518 | name = "virtualenv" 519 | optional = false 520 | python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" 521 | version = "16.7.7" 522 | 523 | [[package]] 524 | category = "dev" 525 | description = "Measures number of Terminal column cells of wide-character codes" 526 | name = "wcwidth" 527 | optional = false 528 | python-versions = "*" 529 | version = "0.1.7" 530 | 531 | [[package]] 532 | category = "main" 533 | description = "Module for decorators, wrappers and monkey patching." 534 | name = "wrapt" 535 | optional = false 536 | python-versions = "*" 537 | version = "1.11.2" 538 | 539 | [[package]] 540 | category = "dev" 541 | description = "Backport of pathlib-compatible object wrapper for zip files" 542 | marker = "python_version < \"3.8\"" 543 | name = "zipp" 544 | optional = false 545 | python-versions = ">=2.7" 546 | version = "0.6.0" 547 | 548 | [package.dependencies] 549 | more-itertools = "*" 550 | 551 | [metadata] 552 | content-hash = "9190f3173eb81ad280e821ff121ada679cac981858a23d80f01af302cd11dfd3" 553 | python-versions = ">= 3.6.7" 554 | 555 | [metadata.hashes] 556 | absl-py = ["b943d1c567743ed0455878fcd60bc28ac9fae38d129d1ccfad58079da00b8951"] 557 | apipkg = ["37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6", "58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c"] 558 | appdirs = ["9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", "d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e"] 559 | astor = ["0e41295809baf43ae8303350e031aff81ae52189b6f881f36d623fa8b2f1960e", "37a6eed8b371f1228db08234ed7f6cfdc7817a3ed3824797e20cbb11dc2a7862"] 560 | astpretty = ["d45c06d07cbd4387aaa696cbb955d809fd6977fea8d00c9c409c1524d358cd54", "f9769a73381a9095581435d9477811b2cdd9cd4c721e2650db564624183191a1"] 561 | astroid = ["09a3fba616519311f1af8a461f804b68f0370e100c9264a035aa7846d7852e33", "5a79c9b4bd6c4be777424593f957c996e20beb5f74e0bc332f47713c6f675efe"] 562 | atomicwrites = ["03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", "75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"] 563 | attrs = ["08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", "f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"] 564 | black = ["1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b", "c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"] 565 | click = ["2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", "5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"] 566 | colorama = ["05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", "f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48"] 567 | crayons = ["41f0843815a8e3ac6fb445b7970d8b9c766e6f164092d84e7ea809b4c91418ec", "8edcadb7f197e25f2cc094aec5bf7f1b6001d3f76c82d56f8d46f6fb1405554f"] 568 | dataclasses = ["454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f", "6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84"] 569 | decorator = ["54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce", "5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d"] 570 | deepdiff = ["51a1228346c91c8dbb586caefb9df6dddfed3a0abff834e72f547a5e405e0fc6", "62ca21020e9c01383b5c45b9d89f418bdb78b4bb53f1d2374cd09db549e6959a"] 571 | execnet = ["cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50", "d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547"] 572 | executing = ["d55c219ec7df0913472bb20e5a96242d855f95f19fde4862b435b6a4ea6fa151"] 573 | filelock = ["18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", "929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836"] 574 | graphviz = ["6d0f69c107cfdc9bd1df3763fad99569bbcba29d0c52ffcbc6f266621d8bf709", "914b8b124942d82e3e1dcef499c9fe77c10acd3d18a1cfeeb2b9de05f6d24805"] 575 | importlab = ["d855350d19dc10a17aabd2fe6f4b428ff1a936071f692fbf686a73694d26a51c"] 576 | importlib-metadata = ["aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", "d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af"] 577 | isort = ["54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", "6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"] 578 | jsonpickle = ["d0c5a4e6cb4e58f6d5406bdded44365c2bcf9c836c4f52910cc9ba7245a59dc2", "d3e922d781b1d0096df2dad89a2e1f47177d7969b596aea806a9d91b4626b29b"] 579 | lazy-object-proxy = ["0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", "194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", "1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", "4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", "48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", "5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", "59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", "8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", "9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", "9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", "97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", "9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", "a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", "a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", "ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", "cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", "d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", "d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", "eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", "efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", "f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0"] 580 | mccabe = ["ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", "dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"] 581 | more-itertools = ["409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", "92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4"] 582 | networkx = ["cdfbf698749a5014bf2ed9db4a07a5295df1d3a53bf80bf3cbd61edf9df05fa1", "f8f4ff0b6f96e4f9b16af6b84622597b5334bf9cae8cf9b2e42e7985d5c95c64"] 583 | ninja = ["0184e69a70bb055621935b935f967b3dc4e189c8f1494d9ea0b90ed15d0308c4", "0d700c1471f9771978415cab503dc6b55e6267dc21428865c9a8f1a906f3a06d", "35d3c2fd77e9271bbfb01beb2c8b733ca647356369da41bb095e14b0369ea3cf", "6ef795816ef3cd3a2def4c4b8e5f1fb7e470bb913c0bae7bb38afe498d0075aa", "75ebbbaeb1b3298bf001cc7866555d88ba33bbdab4cb99eae1e2a59efe23f47b", "9090b6695d86643354cbd394ef835f40c0179cc24969d09446eae7931b702f12", "965bf62d59f3794306b40dc08e31a9286650cff0b11b44acd0a61e61f6030553", "a8503e5fc4f742520e5b3389324e9710eecbc9fa60956b7034adaf1f0650ba3f", "a998d98ffd7262e03be4655e742fa918af93fb19ac36e9140afc0fe8190920a6", "db31cef1eb979e4fe4539046cf04311e00f271f8687bde7dfb64d85f4e4d2b1e", "fd72664f0e2506f2c8002f2ee67ddd50b87604fe8c1bd04d2108dfeacc82420d"] 584 | ordered-set = ["a7bfa858748c73b096e43db14eb23e2bc714a503f990c89fac8fab9b0ee79724"] 585 | packaging = ["28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", "d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108"] 586 | pathspec = ["e285ccc8b0785beadd4c18e5708b12bb8fcf529a1e61215b3feff1d1e559ea5c"] 587 | pluggy = ["0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6", "fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34"] 588 | py = ["64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", "dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"] 589 | pyhamcrest = ["6b672c02fdf7470df9674ab82263841ce8333fb143f32f021f6cb26f0e512420", "7a4bdade0ed98c699d728191a058a60a44d2f9c213c51e2dd1e6fb42f2c6128a", "8ffaa0a53da57e89de14ced7185ac746227a8894dbd5a3c718bf05ddbd1d56cd", "bac0bea7358666ce52e3c6c85139632ed89f115e9af52d44b3c36e0bf8cf16a9", "f30e9a310bcc1808de817a92e95169ffd16b60cbc5a016a49c8d0e8ababfae79"] 590 | pylint = [] 591 | pympler = ["f74cd2982c5cd92ded55561191945616f2bb904a0ae5cdacdb566c6696bdb922"] 592 | pyparsing = ["6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", "d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4"] 593 | pytest = ["5d0d20a9a66e39b5845ab14f8989f3463a7aa973700e6cdf02db69da9821e738", "692d9351353ef709c1126266579edd4fd469dcf6b5f4f583050f72161d6f3592"] 594 | pytest-forked = ["1805699ed9c9e60cb7a8179b8d4fa2b8898098e82d229b0825d8095f0f261100", "1ae25dba8ee2e56fb47311c9638f9e58552691da87e82d25b0ce0e4bf52b7d87"] 595 | pytest-xdist = ["5d1b1d4461518a6023d56dab62fb63670d6f7537f23e2708459a557329accf48", "a8569b027db70112b290911ce2ed732121876632fb3f40b1d39cd2f72f58b147"] 596 | pytype = ["ff43dd0abc144ddf4d895ea52918b5d4bbd0f96b690af205e477cb1074b6ef4e"] 597 | pyyaml = ["0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", "01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", "5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", "5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", "7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", "7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", "87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", "9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", "a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", "b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", "b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", "bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", "f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8"] 598 | regex = ["1e9f9bc44ca195baf0040b1938e6801d2f3409661c15fe57f8164c678cfc663f", "587b62d48ca359d2d4f02d486f1f0aa9a20fbaf23a9d4198c4bed72ab2f6c849", "835ccdcdc612821edf132c20aef3eaaecfb884c9454fdc480d5887562594ac61", "93f6c9da57e704e128d90736430c5c59dd733327882b371b0cae8833106c2a21", "a46f27d267665016acb3ec8c6046ec5eae8cf80befe85ba47f43c6f5ec636dcd", "c5c8999b3a341b21ac2c6ec704cfcccbc50f1fedd61b6a8ee915ca7fd4b0a557", "d4d1829cf97632673aa49f378b0a2c3925acd795148c5ace8ef854217abbee89", "d96479257e8e4d1d7800adb26bf9c5ca5bab1648a1eddcac84d107b73dc68327", "f20f4912daf443220436759858f96fefbfc6c6ba9e67835fd6e4e9b73582791a", "f2b37b5b2c2a9d56d9e88efef200ec09c36c7f323f9d58d0b985a90923df386d", "fe765b809a1f7ce642c2edeee351e7ebd84391640031ba4b60af8d91a9045890"] 599 | six = ["3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", "d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"] 600 | toml = ["229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", "235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e", "f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3"] 601 | tox = ["0bc216b6a2e6afe764476b4a07edf2c1dab99ed82bb146a1130b2e828f5bff5e", "c4f6b319c20ba4913dbfe71ebfd14ff95d1853c4231493608182f66e566ecfe1"] 602 | tox-pyenv = ["916c2213577aec0b3b5452c5bfb32fd077f3a3196f50a81ad57d7ef3fc2599e4", "e470c18af115fe52eeff95e7e3cdd0793613eca19709966fc2724b79d55246cb"] 603 | tox-run-command = ["5c5fe892f732d20276c8b2d29ce079bbbcadc153d696b31cdd8fb3edf5c29ec6"] 604 | typed-ast = ["18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", "262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", "2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", "354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", "4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", "630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", "66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", "71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", "95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", "bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", "cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", "d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", "d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", "d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", "ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12"] 605 | virtualenv = ["11cb4608930d5fd3afb545ecf8db83fa50e1f96fc4fca80c94b07d2c83146589", "d257bb3773e48cac60e475a19b608996c73f4d333b3ba2e4e57d5ac6134e0136"] 606 | wcwidth = ["3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", "f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"] 607 | wrapt = ["565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"] 608 | zipp = ["3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", "f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"] 609 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "cyberbrain" 3 | version = "0.0.0.2" 4 | description = "A Cyberbrain that helps you remember everything." 5 | authors = ["laike9m "] 6 | license = "MIT" 7 | readme="README.md" 8 | homepage = "https://github.com/laike9m/Cyberbrain" 9 | repository = "https://github.com/laike9m/Cyberbrain" 10 | keywords = ["debug", "debugging", "debugger"] 11 | include = [ 12 | "LICENSE", 13 | ] 14 | classifiers = [ 15 | "Development Status :: 1 - Planning", 16 | "Topic :: Software Development :: Debuggers", 17 | "Topic :: Software Development :: Libraries :: Python Modules", 18 | "Programming Language :: Python :: 3.6", 19 | "Programming Language :: Python :: 3.7" 20 | ] 21 | 22 | [tool.poetry.dependencies] 23 | python = ">= 3.6.7" 24 | crayons = "^0.2.0" 25 | astor = "^0.8.0" 26 | deepdiff = "^4.0" 27 | dataclasses = "^0.6.0" 28 | graphviz = "^0.11.1" 29 | absl-py = "^0.7.1" 30 | black = {version = "*", allows-prereleases = true} 31 | executing = "^0.3.2" 32 | Pympler = "^0.8.0" 33 | 34 | [tool.poetry.dev-dependencies] 35 | pylint = { git = "https://github.com/PyCQA/pylint.git", branch = "master" } 36 | pytest = "^4.6" 37 | tox = "^3.13" 38 | tox-run-command = "^0.4.0" 39 | tox-pyenv = "^1.1" 40 | pyhamcrest = "^1.9" 41 | astpretty = "^1.6" 42 | pytype = "*" 43 | pytest-xdist = "^1.30" 44 | 45 | [tool.pylint.messages_control] 46 | disable = """ 47 | bad-continuation,attribute-defined-outside-init,W0511,R0902,R0913,W0622,C0116,C0103, 48 | R0903,W0631,W0212,W0603,R1710,R0201 49 | """ 50 | 51 | [build-system] 52 | requires = ["poetry>=0.12"] 53 | build-backend = "poetry.masonry.api" 54 | -------------------------------------------------------------------------------- /test/doc_example/computation.golden.json: -------------------------------------------------------------------------------- 1 | { 2 | "(0,) ": [ 3 | { 4 | "event": "line", 5 | "filepath": "doc_example.py", 6 | "lineno": 26, 7 | "code_str": "fo = 1", 8 | "frame_id": "(0,) " 9 | }, 10 | { 11 | "event": "call", 12 | "filepath": "doc_example.py", 13 | "lineno": 27, 14 | "code_str": "func_a(fo)", 15 | "caller_frame_id": "(0,) ", 16 | "callee_frame_id": "(0, 0) " 17 | }, 18 | { 19 | "event": "line", 20 | "filepath": "doc_example.py", 21 | "lineno": 27, 22 | "code_str": "func_a(fo)", 23 | "frame_id": "(0,) " 24 | } 25 | ], 26 | "(0, 0) func_a": [ 27 | { 28 | "event": "line", 29 | "filepath": "doc_example.py", 30 | "lineno": 17, 31 | "code_str": "for i in range(2):", 32 | "frame_id": "(0, 0) func_a" 33 | }, 34 | { 35 | "event": "line", 36 | "filepath": "doc_example.py", 37 | "lineno": 18, 38 | "code_str": "pass", 39 | "frame_id": "(0, 0) func_a" 40 | }, 41 | { 42 | "event": "line", 43 | "filepath": "doc_example.py", 44 | "lineno": 17, 45 | "code_str": "for i in range(2):", 46 | "frame_id": "(0, 0) func_a" 47 | }, 48 | { 49 | "event": "line", 50 | "filepath": "doc_example.py", 51 | "lineno": 18, 52 | "code_str": "pass", 53 | "frame_id": "(0, 0) func_a" 54 | }, 55 | { 56 | "event": "line", 57 | "filepath": "doc_example.py", 58 | "lineno": 17, 59 | "code_str": "for i in range(2):", 60 | "frame_id": "(0, 0) func_a" 61 | }, 62 | { 63 | "event": "line", 64 | "filepath": "doc_example.py", 65 | "lineno": 19, 66 | "code_str": "ba = [foo]", 67 | "frame_id": "(0, 0) func_a" 68 | }, 69 | { 70 | "event": "call", 71 | "filepath": "doc_example.py", 72 | "lineno": 20, 73 | "code_str": "func_c(ba)", 74 | "caller_frame_id": "(0, 0) func_a", 75 | "callee_frame_id": "(0, 0, 0) " 76 | }, 77 | { 78 | "event": "line", 79 | "filepath": "doc_example.py", 80 | "lineno": 20, 81 | "code_str": "func_c(ba)", 82 | "frame_id": "(0, 0) func_a" 83 | }, 84 | { 85 | "event": "call", 86 | "filepath": "doc_example.py", 87 | "lineno": 21, 88 | "code_str": "func_f(ba)", 89 | "caller_frame_id": "(0, 0) func_a", 90 | "callee_frame_id": "(0, 0, 1) " 91 | }, 92 | { 93 | "event": "line", 94 | "filepath": "doc_example.py", 95 | "lineno": 21, 96 | "code_str": "foo = func_f(ba)", 97 | "frame_id": "(0, 0) func_a" 98 | }, 99 | { 100 | "event": "line", 101 | "filepath": "doc_example.py", 102 | "lineno": 22, 103 | "code_str": "cyberbrain.register(foo)", 104 | "frame_id": "(0, 0) func_a" 105 | } 106 | ], 107 | "(0, 0, 0) func_c": [ 108 | { 109 | "event": "line", 110 | "filepath": "doc_example.py", 111 | "lineno": 12, 112 | "code_str": "baa.append(None)", 113 | "frame_id": "(0, 0, 0) func_c" 114 | }, 115 | { 116 | "event": "line", 117 | "filepath": "doc_example.py", 118 | "lineno": 13, 119 | "code_str": "baa.append(\"?\")", 120 | "frame_id": "(0, 0, 0) func_c" 121 | } 122 | ], 123 | "(0, 0, 1) func_f": [ 124 | { 125 | "event": "line", 126 | "filepath": "doc_example.py", 127 | "lineno": 7, 128 | "code_str": "x = len(bar)", 129 | "frame_id": "(0, 0, 1) func_f" 130 | }, 131 | { 132 | "event": "line", 133 | "filepath": "doc_example.py", 134 | "lineno": 8, 135 | "code_str": "return x", 136 | "frame_id": "(0, 0, 1) func_f" 137 | } 138 | ] 139 | } -------------------------------------------------------------------------------- /test/doc_example/doc_example.py: -------------------------------------------------------------------------------- 1 | """Example program in design doc.""" 2 | 3 | import cyberbrain 4 | 5 | 6 | def func_f(bar): 7 | x = len(bar) # g 8 | return x # h 9 | 10 | 11 | def func_c(baa): 12 | baa.append(None) # d 13 | baa.append("?") # e 14 | 15 | 16 | def func_a(foo): 17 | for i in range(2): 18 | pass 19 | ba = [foo] # b 20 | func_c(ba) # c 21 | foo = func_f(ba) # f 22 | cyberbrain.register(foo) # target 23 | 24 | 25 | cyberbrain.init() 26 | fo = 1 # start 27 | func_a(fo) # a 28 | -------------------------------------------------------------------------------- /test/doc_example/flow.golden.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "code": "fo = 1", 4 | "next": "func_a(fo)", 5 | "prev": "", 6 | "step_into": "", 7 | "returned_from": "", 8 | "var_changes": [ 9 | "appear fo=1\n" 10 | ], 11 | "location": "doc_example.py: " 12 | }, 13 | { 14 | "code": "func_a(fo)", 15 | "next": "", 16 | "prev": "fo = 1", 17 | "step_into": "for i in range(2):", 18 | "returned_from": "cyberbrain.register(foo)", 19 | "location": "doc_example.py: ", 20 | "tracking": [ 21 | "fo" 22 | ], 23 | "param_to_arg": { 24 | "foo": [ 25 | "fo" 26 | ] 27 | } 28 | }, 29 | { 30 | "code": "for i in range(2):", 31 | "next": "pass", 32 | "prev": "func_a(fo)", 33 | "step_into": "", 34 | "returned_from": "", 35 | "location": "doc_example.py: func_a", 36 | "tracking": [ 37 | "foo" 38 | ] 39 | }, 40 | { 41 | "code": "pass", 42 | "next": "for i in range(2):", 43 | "prev": "for i in range(2):", 44 | "step_into": "", 45 | "returned_from": "", 46 | "location": "doc_example.py: func_a", 47 | "tracking": [ 48 | "foo" 49 | ] 50 | }, 51 | { 52 | "code": "for i in range(2):", 53 | "next": "pass", 54 | "prev": "pass", 55 | "step_into": "", 56 | "returned_from": "", 57 | "location": "doc_example.py: func_a", 58 | "tracking": [ 59 | "foo" 60 | ] 61 | }, 62 | { 63 | "code": "pass", 64 | "next": "for i in range(2):", 65 | "prev": "for i in range(2):", 66 | "step_into": "", 67 | "returned_from": "", 68 | "location": "doc_example.py: func_a", 69 | "tracking": [ 70 | "foo" 71 | ] 72 | }, 73 | { 74 | "code": "for i in range(2):", 75 | "next": "ba = [foo]", 76 | "prev": "pass", 77 | "step_into": "", 78 | "returned_from": "", 79 | "location": "doc_example.py: func_a", 80 | "tracking": [ 81 | "foo" 82 | ] 83 | }, 84 | { 85 | "code": "ba = [foo]", 86 | "next": "func_c(ba)", 87 | "prev": "for i in range(2):", 88 | "step_into": "", 89 | "returned_from": "", 90 | "var_changes": [ 91 | "appear ba=[1]\n" 92 | ], 93 | "location": "doc_example.py: func_a", 94 | "tracking": [ 95 | "foo" 96 | ] 97 | }, 98 | { 99 | "code": "func_c(ba)", 100 | "next": "foo = func_f(ba)", 101 | "prev": "ba = [foo]", 102 | "step_into": "baa.append(None)", 103 | "returned_from": "baa.append(\"?\")", 104 | "var_changes": [ 105 | "modify ba [1] -> [1, None, '?']\n" 106 | ], 107 | "location": "doc_example.py: func_a", 108 | "tracking": [ 109 | "ba", 110 | "foo" 111 | ], 112 | "param_to_arg": { 113 | "baa": [ 114 | "ba" 115 | ] 116 | } 117 | }, 118 | { 119 | "code": "baa.append(None)", 120 | "next": "baa.append(\"?\")", 121 | "prev": "func_c(ba)", 122 | "step_into": "", 123 | "returned_from": "", 124 | "var_changes": [ 125 | "modify baa [1] -> [1, None]\n" 126 | ], 127 | "location": "doc_example.py: func_c", 128 | "tracking": [ 129 | "baa" 130 | ] 131 | }, 132 | { 133 | "code": "baa.append(\"?\")", 134 | "next": "", 135 | "prev": "baa.append(None)", 136 | "step_into": "", 137 | "returned_from": "", 138 | "var_changes": [ 139 | "modify baa [1, None] -> [1, None, '?']\n" 140 | ], 141 | "location": "doc_example.py: func_c", 142 | "tracking": [ 143 | "baa" 144 | ] 145 | }, 146 | { 147 | "code": "foo = func_f(ba)", 148 | "next": "cyberbrain.register(foo)", 149 | "prev": "func_c(ba)", 150 | "step_into": "x = len(bar)", 151 | "returned_from": "return x", 152 | "var_changes": [ 153 | "modify foo 1 -> 3\n" 154 | ], 155 | "location": "doc_example.py: func_a", 156 | "tracking": [ 157 | "ba", 158 | "foo" 159 | ], 160 | "param_to_arg": { 161 | "bar": [ 162 | "ba" 163 | ] 164 | } 165 | }, 166 | { 167 | "code": "x = len(bar)", 168 | "next": "return x", 169 | "prev": "foo = func_f(ba)", 170 | "step_into": "", 171 | "returned_from": "", 172 | "var_changes": [ 173 | "appear x=3\n" 174 | ], 175 | "location": "doc_example.py: func_f", 176 | "tracking": [ 177 | "bar" 178 | ] 179 | }, 180 | { 181 | "code": "return x", 182 | "next": "", 183 | "prev": "x = len(bar)", 184 | "step_into": "", 185 | "returned_from": "", 186 | "location": "doc_example.py: func_f", 187 | "tracking": [ 188 | "x" 189 | ] 190 | }, 191 | { 192 | "code": "cyberbrain.register(foo)", 193 | "next": "", 194 | "prev": "foo = func_f(ba)", 195 | "step_into": "", 196 | "returned_from": "", 197 | "location": "doc_example.py: func_a", 198 | "tracking": [ 199 | "foo" 200 | ] 201 | } 202 | ] -------------------------------------------------------------------------------- /test/exclude_events/call_libs.py: -------------------------------------------------------------------------------- 1 | """A program that calls functions from Python stdlib and 3rd-party libs. 2 | 3 | Since we've excluded events from files in installation path, we shouldn't receive any 4 | events from stdlib or 3rd-party libs. Neither will C calls since they are not catched by 5 | settrace (https://stackoverflow.com/q/16115027/2142577). 6 | """ 7 | 8 | from collections import Counter 9 | 10 | import crayons 11 | 12 | import cyberbrain 13 | 14 | cyberbrain.init() 15 | c = Counter() 16 | c["red"] += 1 17 | c["blue"] += 1 18 | c["red"] += 1 19 | c.most_common(10) 20 | crayons.blue("blue") 21 | cyberbrain.register(c) 22 | -------------------------------------------------------------------------------- /test/exclude_events/computation.golden.json: -------------------------------------------------------------------------------- 1 | { 2 | "(0,) ": [ 3 | { 4 | "event": "line", 5 | "filepath": "call_libs.py", 6 | "lineno": 15, 7 | "code_str": "c = Counter()", 8 | "frame_id": "(0,) " 9 | }, 10 | { 11 | "event": "line", 12 | "filepath": "call_libs.py", 13 | "lineno": 16, 14 | "code_str": "c[\"red\"] += 1", 15 | "frame_id": "(0,) " 16 | }, 17 | { 18 | "event": "line", 19 | "filepath": "call_libs.py", 20 | "lineno": 17, 21 | "code_str": "c[\"blue\"] += 1", 22 | "frame_id": "(0,) " 23 | }, 24 | { 25 | "event": "line", 26 | "filepath": "call_libs.py", 27 | "lineno": 18, 28 | "code_str": "c[\"red\"] += 1", 29 | "frame_id": "(0,) " 30 | }, 31 | { 32 | "event": "line", 33 | "filepath": "call_libs.py", 34 | "lineno": 19, 35 | "code_str": "c.most_common(10)", 36 | "frame_id": "(0,) " 37 | }, 38 | { 39 | "event": "line", 40 | "filepath": "call_libs.py", 41 | "lineno": 20, 42 | "code_str": "crayons.blue(\"blue\")", 43 | "frame_id": "(0,) " 44 | }, 45 | { 46 | "event": "line", 47 | "filepath": "call_libs.py", 48 | "lineno": 21, 49 | "code_str": "cyberbrain.register(c)", 50 | "frame_id": "(0,) " 51 | } 52 | ] 53 | } -------------------------------------------------------------------------------- /test/exclude_events/flow.golden.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "code": "c = Counter()", 4 | "next": "c[\"red\"] += 1", 5 | "prev": "", 6 | "step_into": "", 7 | "returned_from": "", 8 | "var_changes": [ 9 | "appear c=Counter()\n" 10 | ], 11 | "location": "call_libs.py: ", 12 | "tracking": [ 13 | "Counter" 14 | ] 15 | }, 16 | { 17 | "code": "c[\"red\"] += 1", 18 | "next": "c[\"blue\"] += 1", 19 | "prev": "c = Counter()", 20 | "step_into": "", 21 | "returned_from": "", 22 | "var_changes": [ 23 | "modify c Counter() -> Counter({'red': 1})\n" 24 | ], 25 | "location": "call_libs.py: ", 26 | "tracking": [ 27 | "c" 28 | ] 29 | }, 30 | { 31 | "code": "c[\"blue\"] += 1", 32 | "next": "c[\"red\"] += 1", 33 | "prev": "c[\"red\"] += 1", 34 | "step_into": "", 35 | "returned_from": "", 36 | "var_changes": [ 37 | "modify c Counter({'red': 1}) -> Counter({'red': 1, 'blue': 1})\n" 38 | ], 39 | "location": "call_libs.py: ", 40 | "tracking": [ 41 | "c" 42 | ] 43 | }, 44 | { 45 | "code": "c[\"red\"] += 1", 46 | "next": "c.most_common(10)", 47 | "prev": "c[\"blue\"] += 1", 48 | "step_into": "", 49 | "returned_from": "", 50 | "var_changes": [ 51 | "modify c Counter({'red': 1, 'blue': 1}) -> Counter({'red': 2, 'blue': 1})\n" 52 | ], 53 | "location": "call_libs.py: ", 54 | "tracking": [ 55 | "c" 56 | ] 57 | }, 58 | { 59 | "code": "c.most_common(10)", 60 | "next": "crayons.blue(\"blue\")", 61 | "prev": "c[\"red\"] += 1", 62 | "step_into": "", 63 | "returned_from": "", 64 | "location": "call_libs.py: ", 65 | "tracking": [ 66 | "c" 67 | ] 68 | }, 69 | { 70 | "code": "crayons.blue(\"blue\")", 71 | "next": "cyberbrain.register(c)", 72 | "prev": "c.most_common(10)", 73 | "step_into": "", 74 | "returned_from": "", 75 | "location": "call_libs.py: ", 76 | "tracking": [ 77 | "c" 78 | ] 79 | }, 80 | { 81 | "code": "cyberbrain.register(c)", 82 | "next": "", 83 | "prev": "crayons.blue(\"blue\")", 84 | "step_into": "", 85 | "returned_from": "", 86 | "location": "call_libs.py: ", 87 | "tracking": [ 88 | "c" 89 | ] 90 | } 91 | ] -------------------------------------------------------------------------------- /test/function/computation.golden.json: -------------------------------------------------------------------------------- 1 | { 2 | "(0,) ": [ 3 | { 4 | "event": "call", 5 | "filepath": "simple_func.py", 6 | "lineno": 17, 7 | "code_str": "main()", 8 | "caller_frame_id": "(0,) ", 9 | "callee_frame_id": "(0, 0) " 10 | }, 11 | { 12 | "event": "line", 13 | "filepath": "simple_func.py", 14 | "lineno": 17, 15 | "code_str": "main()", 16 | "frame_id": "(0,) " 17 | } 18 | ], 19 | "(0, 0) main": [ 20 | { 21 | "event": "line", 22 | "filepath": "simple_func.py", 23 | "lineno": 7, 24 | "code_str": "def f(x, y):", 25 | "frame_id": "(0, 0) main" 26 | }, 27 | { 28 | "event": "line", 29 | "filepath": "simple_func.py", 30 | "lineno": 10, 31 | "code_str": "x = 1", 32 | "frame_id": "(0, 0) main" 33 | }, 34 | { 35 | "event": "call", 36 | "filepath": "simple_func.py", 37 | "lineno": 11, 38 | "code_str": "f(1, 1)", 39 | "caller_frame_id": "(0, 0) main", 40 | "callee_frame_id": "(0, 0, 0) " 41 | }, 42 | { 43 | "event": "call", 44 | "filepath": "simple_func.py", 45 | "lineno": 11, 46 | "code_str": "f(x, f(1, 1))", 47 | "caller_frame_id": "(0, 0) main", 48 | "callee_frame_id": "(0, 0, 1) " 49 | }, 50 | { 51 | "event": "line", 52 | "filepath": "simple_func.py", 53 | "lineno": 11, 54 | "code_str": "y = f(x, f(1, 1))", 55 | "frame_id": "(0, 0) main" 56 | }, 57 | { 58 | "event": "line", 59 | "filepath": "simple_func.py", 60 | "lineno": 12, 61 | "code_str": "cyberbrain.register(y)", 62 | "frame_id": "(0, 0) main" 63 | } 64 | ], 65 | "(0, 0, 0) f": [ 66 | { 67 | "event": "line", 68 | "filepath": "simple_func.py", 69 | "lineno": 8, 70 | "code_str": "return x + y", 71 | "frame_id": "(0, 0, 0) f" 72 | } 73 | ], 74 | "(0, 0, 1) f": [ 75 | { 76 | "event": "line", 77 | "filepath": "simple_func.py", 78 | "lineno": 8, 79 | "code_str": "return x + y", 80 | "frame_id": "(0, 0, 1) f" 81 | } 82 | ] 83 | } -------------------------------------------------------------------------------- /test/function/flow.golden.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "code": "main()", 4 | "next": "", 5 | "prev": "", 6 | "step_into": "def f(x, y):", 7 | "returned_from": "cyberbrain.register(y)", 8 | "location": "simple_func.py: ", 9 | "param_to_arg": {} 10 | }, 11 | { 12 | "code": "def f(x, y):", 13 | "next": "x = 1", 14 | "prev": "main()", 15 | "step_into": "", 16 | "returned_from": "", 17 | "location": "simple_func.py: main" 18 | }, 19 | { 20 | "code": "x = 1", 21 | "next": "r0_ = f(1, 1)", 22 | "prev": "def f(x, y):", 23 | "step_into": "", 24 | "returned_from": "", 25 | "var_changes": [ 26 | "appear x=1\n" 27 | ], 28 | "location": "simple_func.py: main" 29 | }, 30 | { 31 | "code": "r0_ = f(1, 1)", 32 | "next": "y = f(x, r0_)", 33 | "prev": "x = 1", 34 | "step_into": "return x + y", 35 | "returned_from": "return x + y", 36 | "var_changes": [ 37 | "appear r0_=2\n" 38 | ], 39 | "location": "simple_func.py: main", 40 | "tracking": [ 41 | "x" 42 | ], 43 | "param_to_arg": { 44 | "x": [], 45 | "y": [] 46 | } 47 | }, 48 | { 49 | "code": "return x + y", 50 | "next": "", 51 | "prev": "r0_ = f(1, 1)", 52 | "step_into": "", 53 | "returned_from": "", 54 | "location": "simple_func.py: f", 55 | "tracking": [ 56 | "x", 57 | "y" 58 | ] 59 | }, 60 | { 61 | "code": "y = f(x, r0_)", 62 | "next": "cyberbrain.register(y)", 63 | "prev": "r0_ = f(1, 1)", 64 | "step_into": "return x + y", 65 | "returned_from": "return x + y", 66 | "var_changes": [ 67 | "appear y=3\n" 68 | ], 69 | "location": "simple_func.py: main", 70 | "tracking": [ 71 | "r0_", 72 | "x" 73 | ], 74 | "param_to_arg": { 75 | "x": [ 76 | "x" 77 | ], 78 | "y": [ 79 | "r0_" 80 | ] 81 | } 82 | }, 83 | { 84 | "code": "return x + y", 85 | "next": "", 86 | "prev": "y = f(x, r0_)", 87 | "step_into": "", 88 | "returned_from": "", 89 | "location": "simple_func.py: f", 90 | "tracking": [ 91 | "x", 92 | "y" 93 | ] 94 | }, 95 | { 96 | "code": "cyberbrain.register(y)", 97 | "next": "", 98 | "prev": "y = f(x, r0_)", 99 | "step_into": "", 100 | "returned_from": "", 101 | "location": "simple_func.py: main", 102 | "tracking": [ 103 | "y" 104 | ] 105 | } 106 | ] -------------------------------------------------------------------------------- /test/function/simple_func.py: -------------------------------------------------------------------------------- 1 | """Program with (nested) function calls.""" 2 | 3 | import cyberbrain 4 | 5 | 6 | def main(): 7 | def f(x, y): 8 | return x + y 9 | 10 | x = 1 11 | y = f(x, f(1, 1)) 12 | cyberbrain.register(y) 13 | 14 | 15 | if __name__ == "__main__": 16 | cyberbrain.init() 17 | main() 18 | -------------------------------------------------------------------------------- /test/gen_golden.py: -------------------------------------------------------------------------------- 1 | """Run all test scripts and generates golden data.""" 2 | 3 | import argparse 4 | import glob 5 | import os 6 | from subprocess import Popen 7 | 8 | from crayons import cyan, green 9 | from cyberbrain.testing import COMPUTATION_GOLDEN, FLOW_GOLDEN 10 | 11 | parser = argparse.ArgumentParser(description="Process some integers.") 12 | parser.add_argument( 13 | "--override", action="store_true", help="Whether to override test data." 14 | ) 15 | 16 | 17 | def generate_test_data(test_dir, filename): 18 | override = parser.parse_args().override 19 | 20 | if ( 21 | os.path.exists(os.path.join(test_dir, COMPUTATION_GOLDEN)) 22 | and os.path.exists(os.path.join(test_dir, FLOW_GOLDEN)) 23 | and not override 24 | ): 25 | print(green("Test data already exists, skips " + test_dir)) 26 | return 27 | 28 | test_filepath = os.path.join(test_dir, filename) 29 | print(cyan("Running test: " + test_filepath)) 30 | Popen(["python", test_filepath, "--mode=golden", f"--test_dir={test_dir}"]).wait() 31 | 32 | 33 | def collect_and_run_test_files(): 34 | """Collects all test scripts under test/, run them and generate test data. 35 | 36 | If there are multiple files or sub-folders in a folder under test, main.py is always 37 | the entry point. All test files should respect this convention. 38 | """ 39 | test_dirs = glob.glob("./test/*/") 40 | for test_dir in test_dirs: 41 | if test_dir.startswith("__"): 42 | # Excludes __pycache__ 43 | continue 44 | py_files = [f for f in glob.glob(test_dir + "*") if f.endswith(".py")] 45 | if not py_files: 46 | continue 47 | target_file = ( 48 | py_files[0] 49 | if len(py_files) == 1 50 | else next(filter(lambda f: f.endswith("main.py"), py_files)) 51 | ) 52 | generate_test_data(os.path.dirname(target_file), os.path.basename(target_file)) 53 | 54 | 55 | if __name__ == "__main__": 56 | collect_and_run_test_files() 57 | -------------------------------------------------------------------------------- /test/hello_world/computation.golden.json: -------------------------------------------------------------------------------- 1 | { 2 | "(0,) ": [ 3 | { 4 | "event": "line", 5 | "filepath": "hello.py", 6 | "lineno": 6, 7 | "code_str": "x = \"hello world\"", 8 | "frame_id": "(0,) " 9 | }, 10 | { 11 | "event": "line", 12 | "filepath": "hello.py", 13 | "lineno": 7, 14 | "code_str": "y = x", 15 | "frame_id": "(0,) " 16 | }, 17 | { 18 | "event": "line", 19 | "filepath": "hello.py", 20 | "lineno": 8, 21 | "code_str": "cyberbrain.register(y)", 22 | "frame_id": "(0,) " 23 | } 24 | ] 25 | } -------------------------------------------------------------------------------- /test/hello_world/flow.golden.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "code": "x = \"hello world\"", 4 | "next": "y = x", 5 | "prev": "", 6 | "step_into": "", 7 | "returned_from": "", 8 | "var_changes": [ 9 | "appear x=hello world\n" 10 | ], 11 | "location": "hello.py: " 12 | }, 13 | { 14 | "code": "y = x", 15 | "next": "cyberbrain.register(y)", 16 | "prev": "x = \"hello world\"", 17 | "step_into": "", 18 | "returned_from": "", 19 | "var_changes": [ 20 | "appear y=hello world\n" 21 | ], 22 | "location": "hello.py: ", 23 | "tracking": [ 24 | "x" 25 | ] 26 | }, 27 | { 28 | "code": "cyberbrain.register(y)", 29 | "next": "", 30 | "prev": "y = x", 31 | "step_into": "", 32 | "returned_from": "", 33 | "location": "hello.py: ", 34 | "tracking": [ 35 | "y" 36 | ] 37 | } 38 | ] -------------------------------------------------------------------------------- /test/hello_world/hello.py: -------------------------------------------------------------------------------- 1 | """Just a hello world.""" 2 | 3 | import cyberbrain 4 | 5 | cyberbrain.init() 6 | x = "hello world" 7 | y = x 8 | cyberbrain.register(y) 9 | -------------------------------------------------------------------------------- /test/list_comp/computation.golden.json: -------------------------------------------------------------------------------- 1 | { 2 | "(0,) ": [ 3 | { 4 | "event": "line", 5 | "filepath": "list_comp.py", 6 | "lineno": 18, 7 | "code_str": "x = [1 for i in range(3)]", 8 | "frame_id": "(0,) " 9 | }, 10 | { 11 | "event": "line", 12 | "filepath": "list_comp.py", 13 | "lineno": 20, 14 | "code_str": "cyberbrain.register(x)", 15 | "frame_id": "(0,) " 16 | } 17 | ] 18 | } -------------------------------------------------------------------------------- /test/list_comp/flow.golden.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "code": "x = [1 for i in range(3)]", 4 | "next": "cyberbrain.register(x)", 5 | "prev": "", 6 | "step_into": "", 7 | "returned_from": "", 8 | "var_changes": [ 9 | "appear x=[1, 1, 1]\n" 10 | ], 11 | "location": "list_comp.py: " 12 | }, 13 | { 14 | "code": "cyberbrain.register(x)", 15 | "next": "", 16 | "prev": "x = [1 for i in range(3)]", 17 | "step_into": "", 18 | "returned_from": "", 19 | "location": "list_comp.py: ", 20 | "tracking": [ 21 | "x" 22 | ] 23 | } 24 | ] -------------------------------------------------------------------------------- /test/list_comp/list_comp.py: -------------------------------------------------------------------------------- 1 | """Program that contains list comprehension. 2 | 3 | For now, list comprehensions should not be treated as a call, but a line, though it 4 | triggers a "call" event. 5 | 6 | Main reasons: 7 | 1. Hard to visualize and trace, since list comprehensions is essentially packing 8 | a loop into one line. 9 | 2. Usually the logic is simple. Function calls can exist in list comprehension, 10 | like [f(i) for i in range(3)], but it's fine to assume f and i affects the generated 11 | list. 12 | """ 13 | 14 | import cyberbrain 15 | 16 | cyberbrain.init() 17 | 18 | x = [1 for i in range(3)] 19 | 20 | cyberbrain.register(x) 21 | -------------------------------------------------------------------------------- /test/loop/computation.golden.json: -------------------------------------------------------------------------------- 1 | { 2 | "(0,) ": [ 3 | { 4 | "event": "line", 5 | "filepath": "loop.py", 6 | "lineno": 7, 7 | "code_str": "for i in range(3):", 8 | "frame_id": "(0,) " 9 | }, 10 | { 11 | "event": "line", 12 | "filepath": "loop.py", 13 | "lineno": 8, 14 | "code_str": "print(i)", 15 | "frame_id": "(0,) " 16 | }, 17 | { 18 | "event": "line", 19 | "filepath": "loop.py", 20 | "lineno": 7, 21 | "code_str": "for i in range(3):", 22 | "frame_id": "(0,) " 23 | }, 24 | { 25 | "event": "line", 26 | "filepath": "loop.py", 27 | "lineno": 8, 28 | "code_str": "print(i)", 29 | "frame_id": "(0,) " 30 | }, 31 | { 32 | "event": "line", 33 | "filepath": "loop.py", 34 | "lineno": 7, 35 | "code_str": "for i in range(3):", 36 | "frame_id": "(0,) " 37 | }, 38 | { 39 | "event": "line", 40 | "filepath": "loop.py", 41 | "lineno": 8, 42 | "code_str": "print(i)", 43 | "frame_id": "(0,) " 44 | }, 45 | { 46 | "event": "line", 47 | "filepath": "loop.py", 48 | "lineno": 7, 49 | "code_str": "for i in range(3):", 50 | "frame_id": "(0,) " 51 | }, 52 | { 53 | "event": "line", 54 | "filepath": "loop.py", 55 | "lineno": 10, 56 | "code_str": "print(\"in else\")", 57 | "frame_id": "(0,) " 58 | }, 59 | { 60 | "event": "line", 61 | "filepath": "loop.py", 62 | "lineno": 12, 63 | "code_str": "while i > 0:", 64 | "frame_id": "(0,) " 65 | }, 66 | { 67 | "event": "line", 68 | "filepath": "loop.py", 69 | "lineno": 13, 70 | "code_str": "i -= 1", 71 | "frame_id": "(0,) " 72 | }, 73 | { 74 | "event": "line", 75 | "filepath": "loop.py", 76 | "lineno": 14, 77 | "code_str": "if i == 1:", 78 | "frame_id": "(0,) " 79 | }, 80 | { 81 | "event": "line", 82 | "filepath": "loop.py", 83 | "lineno": 15, 84 | "code_str": "break", 85 | "frame_id": "(0,) " 86 | }, 87 | { 88 | "event": "line", 89 | "filepath": "loop.py", 90 | "lineno": 17, 91 | "code_str": "cyberbrain.register(i)", 92 | "frame_id": "(0,) " 93 | } 94 | ] 95 | } -------------------------------------------------------------------------------- /test/loop/flow.golden.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "code": "for i in range(3):", 4 | "next": "print(i)", 5 | "prev": "", 6 | "step_into": "", 7 | "returned_from": "", 8 | "var_changes": [ 9 | "appear i=0\n" 10 | ], 11 | "location": "loop.py: " 12 | }, 13 | { 14 | "code": "print(i)", 15 | "next": "for i in range(3):", 16 | "prev": "for i in range(3):", 17 | "step_into": "", 18 | "returned_from": "", 19 | "location": "loop.py: ", 20 | "tracking": [ 21 | "i" 22 | ] 23 | }, 24 | { 25 | "code": "for i in range(3):", 26 | "next": "print(i)", 27 | "prev": "print(i)", 28 | "step_into": "", 29 | "returned_from": "", 30 | "var_changes": [ 31 | "modify i 0 -> 1\n" 32 | ], 33 | "location": "loop.py: ", 34 | "tracking": [ 35 | "i" 36 | ] 37 | }, 38 | { 39 | "code": "print(i)", 40 | "next": "for i in range(3):", 41 | "prev": "for i in range(3):", 42 | "step_into": "", 43 | "returned_from": "", 44 | "location": "loop.py: ", 45 | "tracking": [ 46 | "i" 47 | ] 48 | }, 49 | { 50 | "code": "for i in range(3):", 51 | "next": "print(i)", 52 | "prev": "print(i)", 53 | "step_into": "", 54 | "returned_from": "", 55 | "var_changes": [ 56 | "modify i 1 -> 2\n" 57 | ], 58 | "location": "loop.py: ", 59 | "tracking": [ 60 | "i" 61 | ] 62 | }, 63 | { 64 | "code": "print(i)", 65 | "next": "for i in range(3):", 66 | "prev": "for i in range(3):", 67 | "step_into": "", 68 | "returned_from": "", 69 | "location": "loop.py: ", 70 | "tracking": [ 71 | "i" 72 | ] 73 | }, 74 | { 75 | "code": "for i in range(3):", 76 | "next": "print(\"in else\")", 77 | "prev": "print(i)", 78 | "step_into": "", 79 | "returned_from": "", 80 | "location": "loop.py: ", 81 | "tracking": [ 82 | "i" 83 | ] 84 | }, 85 | { 86 | "code": "print(\"in else\")", 87 | "next": "while i > 0:", 88 | "prev": "for i in range(3):", 89 | "step_into": "", 90 | "returned_from": "", 91 | "location": "loop.py: ", 92 | "tracking": [ 93 | "i" 94 | ] 95 | }, 96 | { 97 | "code": "while i > 0:", 98 | "next": "i -= 1", 99 | "prev": "print(\"in else\")", 100 | "step_into": "", 101 | "returned_from": "", 102 | "location": "loop.py: ", 103 | "tracking": [ 104 | "i" 105 | ] 106 | }, 107 | { 108 | "code": "i -= 1", 109 | "next": "if i == 1:", 110 | "prev": "while i > 0:", 111 | "step_into": "", 112 | "returned_from": "", 113 | "var_changes": [ 114 | "modify i 2 -> 1\n" 115 | ], 116 | "location": "loop.py: ", 117 | "tracking": [ 118 | "i" 119 | ] 120 | }, 121 | { 122 | "code": "if i == 1:", 123 | "next": "break", 124 | "prev": "i -= 1", 125 | "step_into": "", 126 | "returned_from": "", 127 | "location": "loop.py: ", 128 | "tracking": [ 129 | "i" 130 | ] 131 | }, 132 | { 133 | "code": "break", 134 | "next": "cyberbrain.register(i)", 135 | "prev": "if i == 1:", 136 | "step_into": "", 137 | "returned_from": "", 138 | "location": "loop.py: ", 139 | "tracking": [ 140 | "i" 141 | ] 142 | }, 143 | { 144 | "code": "cyberbrain.register(i)", 145 | "next": "", 146 | "prev": "break", 147 | "step_into": "", 148 | "returned_from": "", 149 | "location": "loop.py: ", 150 | "tracking": [ 151 | "i" 152 | ] 153 | } 154 | ] -------------------------------------------------------------------------------- /test/loop/loop.py: -------------------------------------------------------------------------------- 1 | # Program that contains loops 2 | 3 | import cyberbrain 4 | 5 | cyberbrain.init() 6 | 7 | for i in range(3): 8 | print(i) 9 | else: 10 | print("in else") 11 | 12 | while i > 0: 13 | i -= 1 14 | if i == 1: 15 | break 16 | 17 | cyberbrain.register(i) 18 | -------------------------------------------------------------------------------- /test/modules/bar.py: -------------------------------------------------------------------------------- 1 | def func_in_bar(): 2 | return 1 3 | -------------------------------------------------------------------------------- /test/modules/computation.golden.json: -------------------------------------------------------------------------------- 1 | { 2 | "(0,) ": [ 3 | { 4 | "event": "call", 5 | "filepath": "main.py", 6 | "lineno": 7, 7 | "code_str": "foo.func_in_foo()", 8 | "caller_frame_id": "(0,) ", 9 | "callee_frame_id": "(0, 0) " 10 | }, 11 | { 12 | "event": "line", 13 | "filepath": "main.py", 14 | "lineno": 7, 15 | "code_str": "x = foo.func_in_foo()", 16 | "frame_id": "(0,) " 17 | }, 18 | { 19 | "event": "line", 20 | "filepath": "main.py", 21 | "lineno": 8, 22 | "code_str": "cyberbrain.register(x)", 23 | "frame_id": "(0,) " 24 | } 25 | ], 26 | "(0, 0) func_in_foo": [ 27 | { 28 | "event": "call", 29 | "filepath": "foo.py", 30 | "lineno": 5, 31 | "code_str": "bar.func_in_bar()", 32 | "caller_frame_id": "(0, 0) func_in_foo", 33 | "callee_frame_id": "(0, 0, 0) " 34 | }, 35 | { 36 | "event": "line", 37 | "filepath": "foo.py", 38 | "lineno": 5, 39 | "code_str": "a = bar.func_in_bar()", 40 | "frame_id": "(0, 0) func_in_foo" 41 | }, 42 | { 43 | "event": "line", 44 | "filepath": "foo.py", 45 | "lineno": 6, 46 | "code_str": "return a", 47 | "frame_id": "(0, 0) func_in_foo" 48 | } 49 | ], 50 | "(0, 0, 0) func_in_bar": [ 51 | { 52 | "event": "line", 53 | "filepath": "bar.py", 54 | "lineno": 2, 55 | "code_str": "return 1", 56 | "frame_id": "(0, 0, 0) func_in_bar" 57 | } 58 | ] 59 | } -------------------------------------------------------------------------------- /test/modules/flow.golden.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "code": "x = foo.func_in_foo()", 4 | "next": "cyberbrain.register(x)", 5 | "prev": "", 6 | "step_into": "a = bar.func_in_bar()", 7 | "returned_from": "return a", 8 | "var_changes": [ 9 | "appear x=1\n" 10 | ], 11 | "location": "main.py: ", 12 | "param_to_arg": {} 13 | }, 14 | { 15 | "code": "a = bar.func_in_bar()", 16 | "next": "return a", 17 | "prev": "x = foo.func_in_foo()", 18 | "step_into": "return 1", 19 | "returned_from": "return 1", 20 | "var_changes": [ 21 | "appear a=1\n" 22 | ], 23 | "location": "foo.py: func_in_foo", 24 | "param_to_arg": {} 25 | }, 26 | { 27 | "code": "return 1", 28 | "next": "", 29 | "prev": "a = bar.func_in_bar()", 30 | "step_into": "", 31 | "returned_from": "", 32 | "location": "bar.py: func_in_bar" 33 | }, 34 | { 35 | "code": "return a", 36 | "next": "", 37 | "prev": "a = bar.func_in_bar()", 38 | "step_into": "", 39 | "returned_from": "", 40 | "location": "foo.py: func_in_foo", 41 | "tracking": [ 42 | "a" 43 | ] 44 | }, 45 | { 46 | "code": "cyberbrain.register(x)", 47 | "next": "", 48 | "prev": "x = foo.func_in_foo()", 49 | "step_into": "", 50 | "returned_from": "", 51 | "location": "main.py: ", 52 | "tracking": [ 53 | "x" 54 | ] 55 | } 56 | ] -------------------------------------------------------------------------------- /test/modules/foo.py: -------------------------------------------------------------------------------- 1 | import bar 2 | 3 | 4 | def func_in_foo(): 5 | a = bar.func_in_bar() 6 | return a 7 | -------------------------------------------------------------------------------- /test/modules/main.py: -------------------------------------------------------------------------------- 1 | """Program that invokes functions from other modules.""" 2 | 3 | import cyberbrain 4 | import foo 5 | 6 | cyberbrain.init() 7 | x = foo.func_in_foo() 8 | cyberbrain.register(x) 9 | -------------------------------------------------------------------------------- /test/multicall/computation.golden.json: -------------------------------------------------------------------------------- 1 | { 2 | "(0,) ": [ 3 | { 4 | "event": "line", 5 | "filepath": "multicall.py", 6 | "lineno": 8, 7 | "code_str": "def f(*args, **kwargs):", 8 | "frame_id": "(0,) " 9 | }, 10 | { 11 | "event": "call", 12 | "filepath": "multicall.py", 13 | "lineno": 12, 14 | "code_str": "f(x=1)", 15 | "caller_frame_id": "(0,) ", 16 | "callee_frame_id": "(0, 0) " 17 | }, 18 | { 19 | "event": "call", 20 | "filepath": "multicall.py", 21 | "lineno": 12, 22 | "code_str": "f(y=2)", 23 | "caller_frame_id": "(0,) ", 24 | "callee_frame_id": "(0, 1) " 25 | }, 26 | { 27 | "event": "line", 28 | "filepath": "multicall.py", 29 | "lineno": 12, 30 | "code_str": "x = {f(x=1), f(y=2)}", 31 | "frame_id": "(0,) " 32 | }, 33 | { 34 | "event": "line", 35 | "filepath": "multicall.py", 36 | "lineno": 15, 37 | "code_str": "cyberbrain.register(x)", 38 | "frame_id": "(0,) " 39 | } 40 | ], 41 | "(0, 0) f": [ 42 | { 43 | "event": "line", 44 | "filepath": "multicall.py", 45 | "lineno": 9, 46 | "code_str": "pass", 47 | "frame_id": "(0, 0) f" 48 | } 49 | ], 50 | "(0, 1) f": [ 51 | { 52 | "event": "line", 53 | "filepath": "multicall.py", 54 | "lineno": 9, 55 | "code_str": "pass", 56 | "frame_id": "(0, 1) f" 57 | } 58 | ] 59 | } -------------------------------------------------------------------------------- /test/multicall/flow.golden.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "code": "def f(*args, **kwargs):", 4 | "next": "r0_ = f(x=1)", 5 | "prev": "", 6 | "step_into": "", 7 | "returned_from": "", 8 | "location": "multicall.py: " 9 | }, 10 | { 11 | "code": "r0_ = f(x=1)", 12 | "next": "r1_ = f(y=2)", 13 | "prev": "def f(*args, **kwargs):", 14 | "step_into": "pass", 15 | "returned_from": "pass", 16 | "var_changes": [ 17 | "appear r0_=None\n" 18 | ], 19 | "location": "multicall.py: ", 20 | "param_to_arg": { 21 | "kwargs": [] 22 | } 23 | }, 24 | { 25 | "code": "pass", 26 | "next": "", 27 | "prev": "r0_ = f(x=1)", 28 | "step_into": "", 29 | "returned_from": "", 30 | "location": "multicall.py: f" 31 | }, 32 | { 33 | "code": "r1_ = f(y=2)", 34 | "next": "x = {r0_, r1_}", 35 | "prev": "r0_ = f(x=1)", 36 | "step_into": "pass", 37 | "returned_from": "pass", 38 | "var_changes": [ 39 | "appear r1_=None\n" 40 | ], 41 | "location": "multicall.py: ", 42 | "tracking": [ 43 | "r0_" 44 | ], 45 | "param_to_arg": { 46 | "kwargs": [] 47 | } 48 | }, 49 | { 50 | "code": "pass", 51 | "next": "", 52 | "prev": "r1_ = f(y=2)", 53 | "step_into": "", 54 | "returned_from": "", 55 | "location": "multicall.py: f" 56 | }, 57 | { 58 | "code": "x = {r0_, r1_}", 59 | "next": "cyberbrain.register(x)", 60 | "prev": "r1_ = f(y=2)", 61 | "step_into": "", 62 | "returned_from": "", 63 | "var_changes": [ 64 | "appear x={None}\n" 65 | ], 66 | "location": "multicall.py: ", 67 | "tracking": [ 68 | "r0_", 69 | "r1_" 70 | ] 71 | }, 72 | { 73 | "code": "cyberbrain.register(x)", 74 | "next": "", 75 | "prev": "x = {r0_, r1_}", 76 | "step_into": "", 77 | "returned_from": "", 78 | "location": "multicall.py: ", 79 | "tracking": [ 80 | "x" 81 | ] 82 | } 83 | ] -------------------------------------------------------------------------------- /test/multicall/multicall.py: -------------------------------------------------------------------------------- 1 | """Multiple calls in one logical line.""" 2 | 3 | import cyberbrain 4 | 5 | cyberbrain.init() 6 | 7 | 8 | def f(*args, **kwargs): 9 | pass 10 | 11 | 12 | x = {f(x=1), f(y=2)} 13 | 14 | 15 | cyberbrain.register(x) 16 | -------------------------------------------------------------------------------- /test/multiline/computation.golden.json: -------------------------------------------------------------------------------- 1 | { 2 | "(0,) ": [ 3 | { 4 | "event": "line", 5 | "filepath": "multiline_statement.py", 6 | "lineno": "9 ~ 13", 7 | "code_str": "y = {\n \"longlonglonglonglonglonglonglong\": 1,\n \"longlonglonglonglonglonglonglonglong\": 2,\n \"longlonglonglonglonglonglonglonglonglong\": 3,\n}", 8 | "frame_id": "(0,) " 9 | }, 10 | { 11 | "event": "line", 12 | "filepath": "multiline_statement.py", 13 | "lineno": 16, 14 | "code_str": "def f(**kwargs):", 15 | "frame_id": "(0,) " 16 | }, 17 | { 18 | "event": "call", 19 | "filepath": "multiline_statement.py", 20 | "lineno": "20 ~ 24", 21 | "code_str": "f(\n longlonglonglonglonglonglonglong=1,\n longlonglonglonglonglonglonglonglong=2,\n longlonglonglonglonglonglonglonglonglong=3,\n)", 22 | "caller_frame_id": "(0,) ", 23 | "callee_frame_id": "(0, 0) " 24 | }, 25 | { 26 | "event": "line", 27 | "filepath": "multiline_statement.py", 28 | "lineno": "20 ~ 24", 29 | "code_str": "x = f(\n longlonglonglonglonglonglonglong=1,\n longlonglonglonglonglonglonglonglong=2,\n longlonglonglonglonglonglonglonglonglong=3,\n)", 30 | "frame_id": "(0,) " 31 | }, 32 | { 33 | "event": "line", 34 | "filepath": "multiline_statement.py", 35 | "lineno": 26, 36 | "code_str": "cyberbrain.register(y)", 37 | "frame_id": "(0,) " 38 | } 39 | ], 40 | "(0, 0) f": [ 41 | { 42 | "event": "line", 43 | "filepath": "multiline_statement.py", 44 | "lineno": 17, 45 | "code_str": "pass", 46 | "frame_id": "(0, 0) f" 47 | } 48 | ] 49 | } -------------------------------------------------------------------------------- /test/multiline/flow.golden.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "code": "y = {\n \"longlonglonglonglonglonglonglong\": 1,\n \"longlonglonglonglonglonglonglonglong\": 2,\n \"longlonglonglonglonglonglonglonglonglong\": 3,\n}", 4 | "next": "def f(**kwargs):", 5 | "prev": "", 6 | "step_into": "", 7 | "returned_from": "", 8 | "var_changes": [ 9 | "appear y={'longlonglonglonglonglonglonglong': 1, 'longlonglonglonglonglonglonglonglong': 2, 'longlonglonglonglonglonglonglonglonglong': 3}\n" 10 | ], 11 | "location": "multiline_statement.py: " 12 | }, 13 | { 14 | "code": "def f(**kwargs):", 15 | "next": "x = f(longlonglonglonglonglonglonglong=1,\n longlonglonglonglonglonglonglonglong=2,\n longlonglonglonglonglonglonglonglonglong=3)", 16 | "prev": "y = {\n \"longlonglonglonglonglonglonglong\": 1,\n \"longlonglonglonglonglonglonglonglong\": 2,\n \"longlonglonglonglonglonglonglonglonglong\": 3,\n}", 17 | "step_into": "", 18 | "returned_from": "", 19 | "location": "multiline_statement.py: ", 20 | "tracking": [ 21 | "y" 22 | ] 23 | }, 24 | { 25 | "code": "x = f(longlonglonglonglonglonglonglong=1,\n longlonglonglonglonglonglonglonglong=2,\n longlonglonglonglonglonglonglonglonglong=3)", 26 | "next": "cyberbrain.register(y)", 27 | "prev": "def f(**kwargs):", 28 | "step_into": "pass", 29 | "returned_from": "pass", 30 | "location": "multiline_statement.py: ", 31 | "tracking": [ 32 | "y" 33 | ], 34 | "param_to_arg": { 35 | "kwargs": [] 36 | } 37 | }, 38 | { 39 | "code": "pass", 40 | "next": "", 41 | "prev": "x = f(longlonglonglonglonglonglonglong=1,\n longlonglonglonglonglonglonglonglong=2,\n longlonglonglonglonglonglonglonglonglong=3)", 42 | "step_into": "", 43 | "returned_from": "", 44 | "location": "multiline_statement.py: f" 45 | }, 46 | { 47 | "code": "cyberbrain.register(y)", 48 | "next": "", 49 | "prev": "x = f(longlonglonglonglonglonglonglong=1,\n longlonglonglonglonglonglonglonglong=2,\n longlonglonglonglonglonglonglonglonglong=3)", 50 | "step_into": "", 51 | "returned_from": "", 52 | "location": "multiline_statement.py: ", 53 | "tracking": [ 54 | "y" 55 | ] 56 | } 57 | ] -------------------------------------------------------------------------------- /test/multiline/multiline_statement.py: -------------------------------------------------------------------------------- 1 | """Program that has multiline statements.""" 2 | 3 | 4 | import cyberbrain 5 | 6 | cyberbrain.init() # Can we use import hook to achieve this? 7 | 8 | 9 | y = { 10 | "longlonglonglonglonglonglonglong": 1, 11 | "longlonglonglonglonglonglonglonglong": 2, 12 | "longlonglonglonglonglonglonglonglonglong": 3, 13 | } 14 | 15 | 16 | def f(**kwargs): 17 | pass 18 | 19 | 20 | x = f( 21 | longlonglonglonglonglonglonglong=1, 22 | longlonglonglonglonglonglonglonglong=2, 23 | longlonglonglonglonglonglonglonglonglong=3, 24 | ) 25 | 26 | cyberbrain.register(y) 27 | -------------------------------------------------------------------------------- /test/test_executor.py: -------------------------------------------------------------------------------- 1 | """Collects and run all test scripts.""" 2 | 3 | import os 4 | from subprocess import Popen 5 | 6 | import pytest 7 | from cyberbrain.testing import ( 8 | COMPUTATION_GOLDEN, 9 | COMPUTATION_TEST_OUTPUT, 10 | FLOW_GOLDEN, 11 | FLOW_TEST_OUTPUT, 12 | ) 13 | 14 | 15 | @pytest.fixture 16 | def run_scripts_and_compare(): 17 | def runner(directory, filename): 18 | test_dir = os.path.abspath(os.path.join("test", directory)) 19 | 20 | Popen( 21 | [ 22 | "python", 23 | os.path.join(test_dir, filename), 24 | "--mode=test", 25 | f"--test_dir={test_dir}", 26 | ] 27 | ).wait() 28 | 29 | # Checks computation is equal. 30 | output_path = os.path.join(test_dir, COMPUTATION_TEST_OUTPUT) 31 | with open(output_path, "r") as test_out, open( 32 | os.path.join(test_dir, COMPUTATION_GOLDEN), "r" 33 | ) as golden: 34 | assert test_out.read().replace("\r\n", "\n").replace( 35 | r"\r\n", r"\n" 36 | ) == golden.read().strip("\r\n") 37 | 38 | os.remove(output_path) 39 | 40 | # Checks flow is equal. 41 | output_path = os.path.join(test_dir, FLOW_TEST_OUTPUT) 42 | with open(output_path, "r") as test_out, open( 43 | os.path.join(test_dir, FLOW_GOLDEN), "r" 44 | ) as golden: 45 | assert test_out.read().replace("\r\n", "\n").replace( 46 | r"\r\n", r"\n" 47 | ) == golden.read().strip("\r\n") 48 | 49 | os.remove(output_path) 50 | 51 | return runner 52 | 53 | 54 | def test_hello_world(run_scripts_and_compare): 55 | run_scripts_and_compare("hello_world", "hello.py") 56 | 57 | 58 | def test_function(run_scripts_and_compare): 59 | run_scripts_and_compare("function", "simple_func.py") 60 | 61 | 62 | def test_multiline(run_scripts_and_compare): 63 | run_scripts_and_compare("multiline", "multiline_statement.py") 64 | 65 | 66 | def test_exclude_events(run_scripts_and_compare): 67 | run_scripts_and_compare("exclude_events", "call_libs.py") 68 | 69 | 70 | def test_modules(run_scripts_and_compare): 71 | run_scripts_and_compare("modules", "main.py") 72 | 73 | 74 | def test_loop(run_scripts_and_compare): 75 | run_scripts_and_compare("loop", "loop.py") 76 | 77 | 78 | def test_list_comp(run_scripts_and_compare): 79 | run_scripts_and_compare("list_comp", "list_comp.py") 80 | 81 | 82 | def test_multicall(run_scripts_and_compare): 83 | run_scripts_and_compare("multicall", "multicall.py") 84 | 85 | 86 | def test_doc_example(run_scripts_and_compare): 87 | run_scripts_and_compare("doc_example", "doc_example.py") 88 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | isolated_build = true 3 | envlist = py36, py37, py38 4 | 5 | [testenv] 6 | commands = 7 | pip install poetry 8 | poetry config settings.virtualenvs.create false 9 | poetry install --no-dev 10 | pip install pytest pytest-xdist 11 | py.test -n auto -s -vv {posargs} {toxinidir}/test/test_executor.py 12 | --------------------------------------------------------------------------------