├── .gitignore ├── pycrunch_trace ├── __init__.py ├── client │ ├── __init__.py │ ├── api │ │ ├── version.py │ │ ├── __init__.py │ │ ├── trace_decorator.py │ │ ├── network_client.py │ │ └── trace.py │ ├── networking │ │ ├── strategies │ │ │ ├── __init__.py │ │ │ ├── abstract_strategy.py │ │ │ ├── native_write_strategy.pyx │ │ │ └── network_strategy.py │ │ ├── __init__.py │ │ ├── commands.py │ │ ├── client_trace_introspection.py │ │ └── ClientOutgoingQueueThread.py │ └── command_buffer.py ├── demo │ ├── __init__.py │ ├── test_simulated_real │ │ ├── __init__.py │ │ └── test_tree_call_stack_cursors.py │ ├── demo_with_dict.py │ ├── demo_golden.py │ ├── factorial_demo.py │ ├── demo_stepping_nested_calls.py │ ├── multiple_execution_stack.py │ ├── dict_client_demo.py │ ├── timings_demo.py │ ├── requests_demo.py │ ├── interactive_demo_03.py │ ├── demo_no_functon_calls.py │ ├── demo_tree_v1.py │ ├── interactive_demo_04.py │ ├── demo_million_of_calls.py │ ├── interactive_demo_02.py │ └── interactive_demo_01.py ├── events │ ├── __init__.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_stack_frame.py │ │ └── test_size_prediction.py │ ├── base_event.py │ ├── file_contents_in_protobuf.pyx │ ├── event_buffer_in_protobuf.py │ ├── method_enter.py │ ├── size_prediction.py │ └── native_event_buffer_in_protobuf.pyx ├── native │ ├── __init__.py │ ├── native_call_stack.pyx │ ├── native_models.pyx │ ├── native_models.pxd │ └── native_tracer.pyx ├── proto │ ├── __init__.py │ ├── readme.md │ └── message.proto ├── server │ ├── __init__.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_perf.py │ │ └── test_chunks_ordering.py │ ├── shared.py │ ├── state.py │ ├── trace_in_progress.py │ ├── events.py │ ├── incoming_traces.py │ ├── perf.py │ ├── chunks_ordering.py │ ├── trace_persistance.py │ └── recording_server_websocket.py ├── tests │ ├── __init__.py │ ├── test_stack.py │ ├── test_exclusions.py │ ├── test_trace_file.py │ ├── test_buffer.py │ └── test_file_chunks.py ├── file_system │ ├── __init__.py │ ├── tags.py │ ├── human_readable_size.py │ ├── test_trace_session.py │ ├── trace_session.py │ ├── session_store.py │ ├── chunked_trace.py │ ├── persisted_session.py │ └── trace_file.py ├── profiles │ ├── __init__.py │ └── test_profile_list.py ├── samples │ ├── __init__.py │ ├── module_a.py │ ├── invalid_picker_with_exception.py │ ├── module_c.py │ └── module_b.py ├── sandbox │ ├── __init__.py │ └── test_playground.py ├── session │ ├── __init__.py │ ├── snapshots │ │ └── a.pycrunch-trace │ ├── snapshot.py │ └── active_clients.py ├── tracing │ ├── __init__.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_inline_profiler.py │ │ └── test_file_map.py │ ├── simulation │ │ ├── __init__.py │ │ ├── models.py │ │ └── test_sim.py │ ├── perf.py │ ├── file_map.py │ ├── readme.md │ ├── inline_profiler.py │ ├── call_stack.py │ ├── simulator_sink.py │ └── simple_tracer.py ├── filters │ ├── tests │ │ ├── __init__.py │ │ ├── test_file_filter.py │ │ └── test_types_filter.py │ ├── __init__.py │ ├── types_filter.py │ ├── AbstractFileFilter.py │ ├── DefaultFileFilter.py │ └── CustomFileFilter.py ├── reference_code │ ├── __init__.py │ ├── readme.txt │ ├── sys_settrace_call.py │ ├── sys_settrace_line.py │ ├── google_bdb.py │ └── pytracer_cov.py ├── serialization │ ├── __init__.py │ └── shared.py ├── oop │ ├── __init__.py │ ├── writable_file.py │ ├── clock.py │ ├── safe_filename.py │ ├── file.py │ └── directory.py ├── logging-configuration.yaml ├── config.py ├── demo.py ├── pycrunch-profiles │ └── default.profile.yaml └── main.py ├── .pycrunch-config.yaml ├── setup.cfg ├── requirements.txt ├── pypi_readme.md ├── MANIFEST.in ├── LICENSE ├── setup.py └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ -------------------------------------------------------------------------------- /pycrunch_trace/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/client/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/demo/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/events/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/native/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/proto/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/server/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/file_system/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/profiles/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/samples/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/sandbox/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/session/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/tracing/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/events/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/filters/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/native/native_call_stack.pyx: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/native/native_models.pyx: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/reference_code/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/sandbox/test_playground.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/server/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/tracing/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.pycrunch-config.yaml: -------------------------------------------------------------------------------- 1 | engine: 2 | runtime: pytest -------------------------------------------------------------------------------- /pycrunch_trace/demo/test_simulated_real/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md -------------------------------------------------------------------------------- /pycrunch_trace/client/api/version.py: -------------------------------------------------------------------------------- 1 | version = '0.01' 2 | -------------------------------------------------------------------------------- /pycrunch_trace/client/networking/strategies/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pycrunch_trace/serialization/__init__.py: -------------------------------------------------------------------------------- 1 | from .shared import to_string -------------------------------------------------------------------------------- /pycrunch_trace/tracing/simulation/__init__.py: -------------------------------------------------------------------------------- 1 | from .models import EventKeys -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | wheel 2 | PyYAML 3 | protobuf>=3.11.3,<4.0.0 4 | Cython 5 | jsonpickle 6 | -------------------------------------------------------------------------------- /pycrunch_trace/client/networking/__init__.py: -------------------------------------------------------------------------------- 1 | from .ClientOutgoingQueueThread import event_queue 2 | -------------------------------------------------------------------------------- /pycrunch_trace/client/api/__init__.py: -------------------------------------------------------------------------------- 1 | from .trace_decorator import trace 2 | from .trace import Trace 3 | -------------------------------------------------------------------------------- /pycrunch_trace/reference_code/readme.txt: -------------------------------------------------------------------------------- 1 | examples are from 2 | https://pymotw.com/3/sys/tracing.html#tracing-a-program-as-it-runs -------------------------------------------------------------------------------- /pycrunch_trace/server/shared.py: -------------------------------------------------------------------------------- 1 | import socketio 2 | 3 | tracer_socket_server = socketio.AsyncServer(cors_allowed_origins='*') 4 | -------------------------------------------------------------------------------- /pycrunch_trace/server/state.py: -------------------------------------------------------------------------------- 1 | from pycrunch_trace.session import active_clients 2 | 3 | connections = active_clients.ActiveConnections() 4 | -------------------------------------------------------------------------------- /pycrunch_trace/session/snapshots/a.pycrunch-trace: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gleb-sevruk/pycrunch-trace/HEAD/pycrunch_trace/session/snapshots/a.pycrunch-trace -------------------------------------------------------------------------------- /pycrunch_trace/events/base_event.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class Event(ABC): 5 | # timestamp of event 6 | event_name: str 7 | ts: float 8 | -------------------------------------------------------------------------------- /pypi_readme.md: -------------------------------------------------------------------------------- 1 | https://medium.com/@joel.barmettler/how-to-upload-your-python-package-to-pypi-65edc5fe9c56 2 | 3 | 4 | 5 | `python setup.py sdist` 6 | 7 | `twine upload dist/*` -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include pycrunch_trace/logging-configuration.yaml 2 | include pycrunch_trace/pycrunch-profiles/default.profile.yaml 3 | 4 | global-include *.pyx 5 | global-include *.pxd 6 | -------------------------------------------------------------------------------- /pycrunch_trace/samples/module_a.py: -------------------------------------------------------------------------------- 1 | 2 | def some_method(some_number): 3 | a = 2 4 | b = 3 5 | result = a + b + some_number 6 | return result 7 | 8 | # x = some_method(1) 9 | # print(x) 10 | -------------------------------------------------------------------------------- /pycrunch_trace/filters/__init__.py: -------------------------------------------------------------------------------- 1 | from .AbstractFileFilter import AbstractFileFilter 2 | from .DefaultFileFilter import DefaultFileFilter 3 | from .CustomFileFilter import CustomFileFilter 4 | from .types_filter import can_trace_type 5 | 6 | -------------------------------------------------------------------------------- /pycrunch_trace/oop/__init__.py: -------------------------------------------------------------------------------- 1 | from .safe_filename import SafeFilename 2 | from .directory import Directory 3 | from .file import AbstractFile 4 | from .file import File 5 | from .writable_file import WriteableFile 6 | from .clock import Clock 7 | -------------------------------------------------------------------------------- /pycrunch_trace/serialization/shared.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import jsonpickle 4 | 5 | 6 | def to_string(command_stack): 7 | dumps = json.dumps(json.loads(jsonpickle.encode(command_stack, unpicklable=True,keys=True)), indent=2) 8 | return dumps -------------------------------------------------------------------------------- /pycrunch_trace/samples/invalid_picker_with_exception.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import sys 3 | 4 | def op(): 5 | try: 6 | x = pickle.dumps(dict(a=1,b='11')) 7 | x = pickle.dumps(sys) 8 | except Exception as e: 9 | sss = str(e) 10 | pass 11 | -------------------------------------------------------------------------------- /pycrunch_trace/proto/readme.md: -------------------------------------------------------------------------------- 1 | protoc --proto_path=pycrunch_tracer/proto --js_out=import_style=commonjs,binary:pycrunch_tracer/proto pycrunch_tracer/proto/message.proto 2 | 3 | 4 | protoc --proto_path=pycrunch_tracer/proto --python_out=pycrunch_tracer/proto pycrunch_tracer/proto/message.proto 5 | -------------------------------------------------------------------------------- /pycrunch_trace/file_system/tags.py: -------------------------------------------------------------------------------- 1 | 2 | # TLV format: 3 | # TAG | LENGTH | VALUE 4 | # Header contains nested TLV triplets 5 | 6 | TRACE_TAG_HEADER = 1 7 | 8 | TRACE_TAG_EVENTS = 2 9 | TRACE_TAG_FILES = 3 10 | TRACE_TAG_METADATA = 4 11 | 12 | HEADER_TAG_VERSION = 1 13 | HEADER_TAG_FILES = 2 14 | HEADER_TAG_METADATA = 3 15 | -------------------------------------------------------------------------------- /pycrunch_trace/oop/writable_file.py: -------------------------------------------------------------------------------- 1 | import io 2 | 3 | 4 | class WriteableFile: 5 | def __init__(self, file_name: str, buffer: bytes): 6 | self.buffer = buffer 7 | self.file_name = file_name 8 | 9 | def save(self): 10 | with io.FileIO(self.file_name, 'w') as file: 11 | file.write(self.buffer) 12 | -------------------------------------------------------------------------------- /pycrunch_trace/demo/demo_with_dict.py: -------------------------------------------------------------------------------- 1 | from pycrunch_trace.client.api import trace 2 | from pycrunch_trace.tracing.file_map import FileMap 3 | 4 | 5 | 6 | 7 | @trace 8 | def run(): 9 | loc = dict() 10 | fff = FileMap() 11 | id__ = fff.file_id('test') 12 | loc['fff'] = fff 13 | loc['id__'] = id__ 14 | loc['file'] = 'test' 15 | 16 | 17 | 18 | run() -------------------------------------------------------------------------------- /pycrunch_trace/samples/module_c.py: -------------------------------------------------------------------------------- 1 | def find(symbol_to_find, in_str: str): 2 | accum = 0 3 | for s in in_str: 4 | if s == symbol_to_find: 5 | return accum 6 | accum += 1 7 | return -1 8 | 9 | def kwar_testing(**kwargs): 10 | x = kwargs.get('x') 11 | y = kwargs.get('b') 12 | if not y: 13 | x = 2 + x 14 | something = 42 -------------------------------------------------------------------------------- /pycrunch_trace/filters/tests/test_file_filter.py: -------------------------------------------------------------------------------- 1 | from pycrunch_trace import config 2 | from pycrunch_trace.filters import DefaultFileFilter 3 | 4 | 5 | def test_included(): 6 | sut = DefaultFileFilter() 7 | assert sut.should_trace(config.absolute_path) 8 | 9 | def test_not_included(): 10 | sut = DefaultFileFilter() 11 | assert not sut.should_trace('/crap/module/x.py') 12 | -------------------------------------------------------------------------------- /pycrunch_trace/filters/types_filter.py: -------------------------------------------------------------------------------- 1 | allowed_types = [int, str, float, dict, type(None), bool] 2 | # allowed_types = [int, str, float, type(None), bool] 3 | 4 | 5 | def can_trace_type(variable): 6 | # return False 7 | current_type = type(variable) 8 | if current_type in allowed_types: 9 | return True 10 | 11 | # print(current_type) 12 | 13 | return False -------------------------------------------------------------------------------- /pycrunch_trace/demo/demo_golden.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | from pycrunch_trace.client.api import trace 4 | 5 | 6 | def some_code(): 7 | for x in range(1): 8 | req = requests.get('https://google.com') 9 | code = req.status_code 10 | print(str(x)) 11 | print(code) 12 | 13 | @trace 14 | def run_youtube_code(): 15 | some_code() 16 | 17 | 18 | run_youtube_code() -------------------------------------------------------------------------------- /pycrunch_trace/filters/AbstractFileFilter.py: -------------------------------------------------------------------------------- 1 | class AbstractFileFilter: 2 | def should_trace(self, filename: str) -> bool: 3 | self._abstract_exception() 4 | 5 | def should_record_variables(self) -> bool: 6 | self._abstract_exception() 7 | 8 | def _abstract_exception(self): 9 | raise Exception('AbstractFileFilter: called too abstract should_trace function') 10 | -------------------------------------------------------------------------------- /pycrunch_trace/demo/factorial_demo.py: -------------------------------------------------------------------------------- 1 | from pycrunch_trace.client.api.trace import Trace 2 | 3 | 4 | 5 | 6 | def my_factorial(num: int): 7 | if num == 1: 8 | return num 9 | else: 10 | return num * my_factorial(num - 1) 11 | 12 | yoba = Trace() 13 | yoba.start(session_name='factorial') 14 | 15 | my_factorial(10) 16 | 17 | yoba.stop() 18 | # print(x._tracer.simulation.simulated_code()) -------------------------------------------------------------------------------- /pycrunch_trace/oop/clock.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | class Clock: 5 | """ 6 | Counts time since creating of the object 7 | """ 8 | def __init__(self): 9 | self.started_at = self._system_clock() 10 | pass 11 | 12 | def now(self): 13 | return (time.perf_counter() - self.started_at) * 1000 14 | 15 | def _system_clock(self): 16 | return time.perf_counter() 17 | -------------------------------------------------------------------------------- /pycrunch_trace/demo/demo_stepping_nested_calls.py: -------------------------------------------------------------------------------- 1 | from pycrunch_trace.client.api import trace 2 | 3 | 4 | def nested_two(x): 5 | return x * 2 6 | 7 | # down arrow (step over) should not enter this function 8 | def nested_one(x): 9 | return x + 1 10 | 11 | 12 | @trace 13 | def demo_stepping_nested_calls(): 14 | x = 1 15 | y = nested_one(nested_two(x)) 16 | print(y) 17 | 18 | 19 | demo_stepping_nested_calls() 20 | -------------------------------------------------------------------------------- /pycrunch_trace/demo/multiple_execution_stack.py: -------------------------------------------------------------------------------- 1 | from pycrunch_trace.client.api.trace import Trace 2 | 3 | 4 | def c(number): 5 | print('in c') 6 | response = number * 2 7 | return response 8 | 9 | 10 | def b(y): 11 | print('opa') 12 | return y * 2 13 | 14 | 15 | def a(x): 16 | y = x + 1 17 | res = b(y) 18 | y = res + 1 19 | res = c(res + 1) 20 | return res 21 | 22 | yoba = Trace() 23 | yoba.start('multiple_stack') 24 | 25 | a(2) 26 | 27 | yoba.stop() -------------------------------------------------------------------------------- /pycrunch_trace/demo/dict_client_demo.py: -------------------------------------------------------------------------------- 1 | from pycrunch_trace.client.api.trace import Trace 2 | 3 | 4 | def xxx(x: int): 5 | y = x * 9 6 | xy = x * 2 7 | data_container = dict(x=1, y='some_data') 8 | for i in range(xy): 9 | xy += i 10 | data_container[str(i)] = i * 8 11 | return xy 12 | 13 | 14 | yoba = Trace() 15 | yoba.start(session_name='network_demo3') 16 | 17 | 18 | def run(param): 19 | param(4) 20 | 21 | 22 | run(xxx) 23 | 24 | 25 | yoba.stop() 26 | 27 | print('stoped') 28 | -------------------------------------------------------------------------------- /pycrunch_trace/tracing/tests/test_inline_profiler.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | 3 | from pycrunch_trace.tracing.inline_profiler import ProfilingScope, inline_profiler_instance 4 | 5 | 6 | def call_some_funtion(i): 7 | with ProfilingScope('call_some_funtion'): 8 | sleep(0.1) 9 | pass 10 | 11 | 12 | def test_nesting(): 13 | with ProfilingScope('root'): 14 | for i in range(10): 15 | with ProfilingScope('loop'): 16 | call_some_funtion(i) 17 | inline_profiler_instance.print_timings() -------------------------------------------------------------------------------- /pycrunch_trace/samples/module_b.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | 4 | def some_method(some_number): 5 | a = 2 6 | b = 3 7 | b = 3*4 8 | b = 3*4 9 | b = 1*4 10 | result = a + b + some_number 11 | return result 12 | 13 | 14 | def another_m(some_new_var): 15 | x = 2 16 | for i in range(some_new_var): 17 | if i % 2 == 0: 18 | x = 3 + i 19 | else: 20 | x = 2 * i 21 | 22 | def string_m(): 23 | return str(datetime.now()) 24 | # x = some_method(1) 25 | # print(x) 26 | -------------------------------------------------------------------------------- /pycrunch_trace/server/trace_in_progress.py: -------------------------------------------------------------------------------- 1 | 2 | from datetime import datetime, timezone 3 | 4 | def utc_now(): 5 | return datetime.utcnow().replace(tzinfo=timezone.utc) 6 | 7 | class TraceInProgress: 8 | session_id: str 9 | total_events: int 10 | started_at: datetime 11 | 12 | def __init__(self, session_id: str): 13 | self.session_id = session_id 14 | self.total_events = 0 15 | self.started_at = utc_now() 16 | 17 | def add_events(self, events_count): 18 | self.total_events += events_count 19 | -------------------------------------------------------------------------------- /pycrunch_trace/tests/test_stack.py: -------------------------------------------------------------------------------- 1 | import struct 2 | 3 | from pycrunch_trace.events.method_enter import ExecutionCursor 4 | from pycrunch_trace.tracing.simple_tracer import CallStack 5 | 6 | 7 | def test_simple(): 8 | x = struct.calcsize("i") 9 | print('size='+str(x)) 10 | sut = CallStack() 11 | sut.enter_frame(ExecutionCursor('test', 1)) 12 | sut.enter_frame(ExecutionCursor('test', 2)) 13 | print(sut.top_level_frame_as_clone()) 14 | sut.enter_frame(ExecutionCursor('test', 3)) 15 | print(sut.top_level_frame_as_clone()) 16 | -------------------------------------------------------------------------------- /pycrunch_trace/oop/safe_filename.py: -------------------------------------------------------------------------------- 1 | class SafeFilename: 2 | possibly_unsafe_filename: str 3 | 4 | def __init__(self, possibly_unsafe_filename: str): 5 | self.possibly_unsafe_filename = possibly_unsafe_filename 6 | 7 | def __str__(self): 8 | return self._make_safe_filename() 9 | 10 | def _make_safe_filename(self): 11 | def safe_char(c): 12 | if c.isalnum(): 13 | return c 14 | else: 15 | return "_" 16 | 17 | return "".join(safe_char(c) for c in self.possibly_unsafe_filename).rstrip("_") 18 | 19 | 20 | -------------------------------------------------------------------------------- /pycrunch_trace/demo/timings_demo.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from time import sleep 3 | 4 | from pycrunch_trace.client.api import trace 5 | 6 | 7 | @trace('timings_precision') 8 | def run(x): 9 | y = x 10 | sleep(0.3) 11 | b() 12 | info = str(sys.float_info) 13 | print(info) 14 | sleep(1) 15 | 16 | 17 | def x(): 18 | sleep(0.1) 19 | 20 | 21 | def b(): 22 | y = 'sleeped 01' 23 | sleep(0.2) 24 | for i in range(10): 25 | x() 26 | c() 27 | 28 | 29 | def c(): 30 | y = 'sleeped 01' 31 | sleep(0.7) 32 | y = 'sleeped 1 sec total' 33 | 34 | 35 | run(1) 36 | 37 | -------------------------------------------------------------------------------- /pycrunch_trace/file_system/human_readable_size.py: -------------------------------------------------------------------------------- 1 | class HumanReadableByteSize: 2 | def __init__(self, total_bytes: int): 3 | self.total_bytes = total_bytes 4 | 5 | def __str__(self): 6 | return self.human_readable_size(self.total_bytes) 7 | 8 | @staticmethod 9 | def human_readable_size(size, decimal_places=1): 10 | if size < 1024: 11 | return f"{round(size)} bytes" 12 | for unit in ['B', 'Kb', 'MB', 'GB', 'TB']: 13 | if size < 1024.0: 14 | break 15 | size /= 1024.0 16 | return f"{size:.{decimal_places}f} {unit}" 17 | -------------------------------------------------------------------------------- /pycrunch_trace/tracing/simulation/models.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | 4 | class EventKeys: 5 | call = 'call' 6 | line = 'line' 7 | event_return = 'return' 8 | exception = 'exception' 9 | 10 | 11 | class Code: 12 | co_filename: str 13 | 14 | """ function name """ 15 | co_name: str 16 | 17 | """ Number of arguments (excluding keyword, *, and **) """ 18 | co_argcount: int 19 | 20 | """ machine bytecode """ 21 | co_code: bytearray 22 | 23 | class Frame: 24 | f_locals: dict 25 | f_back: Any 26 | f_builtins: Any 27 | f_code: Code 28 | f_lineno: int 29 | -------------------------------------------------------------------------------- /pycrunch_trace/events/tests/test_stack_frame.py: -------------------------------------------------------------------------------- 1 | from pycrunch.insights import trace 2 | 3 | from pycrunch_trace.events.method_enter import ExecutionCursor 4 | from pycrunch_trace.tracing.simple_tracer import CallStack 5 | 6 | 7 | def test_1(): 8 | sut = CallStack() 9 | sut.enter_frame(ExecutionCursor('a', 1)) 10 | sut.enter_frame(ExecutionCursor('b', 2)) 11 | sut.enter_frame(ExecutionCursor('c', 3)) 12 | sut.new_cursor_in_current_frame(ExecutionCursor('b', 3)) 13 | sut.exit_frame() 14 | 15 | x = sut.current_frame() 16 | ex = sut.top_level_frame_as_clone() 17 | # trace(yyy=str(x)) 18 | trace(ex=str(ex)) 19 | -------------------------------------------------------------------------------- /pycrunch_trace/tests/test_exclusions.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from pycrunch_trace.filters import CustomFileFilter 4 | from pycrunch_trace.oop import File 5 | 6 | 7 | def test_additional_exclusions_considered(): 8 | package_directory = Path(__file__).parent.parent 9 | 10 | sut = CustomFileFilter(File(package_directory.joinpath('pycrunch-profiles', 'default.profile.yaml'))) 11 | sut._ensure_loaded() 12 | sut.add_additional_exclusions(['/this/is/excluded']) 13 | 14 | actual = sut.should_trace('/this/is/excluded/some.py') 15 | assert actual is False 16 | 17 | assert sut.should_trace('/this/is/included/main.py') 18 | -------------------------------------------------------------------------------- /pycrunch_trace/client/networking/strategies/abstract_strategy.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from pycrunch_trace.client.networking.commands import EventsSlice, FileContentSlice 4 | 5 | 6 | class AbstractRecordingStrategy: 7 | def prepare(self): 8 | pass 9 | 10 | def recording_start(self, session_id: str): 11 | pass 12 | 13 | def recording_stop(self, session_id: str, files_included: List[str], files_excluded: List[str]): 14 | pass 15 | 16 | def recording_slice(self, x: EventsSlice): 17 | pass 18 | 19 | def files_slice(self, x: FileContentSlice): 20 | pass 21 | 22 | def clean(self): 23 | pass -------------------------------------------------------------------------------- /pycrunch_trace/filters/tests/test_types_filter.py: -------------------------------------------------------------------------------- 1 | from pycrunch_trace import config 2 | from pycrunch_trace.filters import can_trace_type 3 | 4 | 5 | def test_can_trace_int(): 6 | assert can_trace_type(1) 7 | 8 | 9 | def test_can_trace_float(): 10 | assert can_trace_type(1.5) 11 | 12 | def test_can_trace_bool(): 13 | assert can_trace_type(True) 14 | 15 | def test_can_trace_none(): 16 | assert can_trace_type(None) 17 | 18 | def test_can_trace_string(): 19 | assert can_trace_type('some_str') 20 | 21 | def test_can_trace_dict(): 22 | assert can_trace_type(dict(kwargs=dict(a=1))) 23 | 24 | def test_no_trace_module(): 25 | assert not can_trace_type(config) 26 | -------------------------------------------------------------------------------- /pycrunch_trace/native/native_models.pxd: -------------------------------------------------------------------------------- 1 | cdef class NativeExecutionCursor: 2 | cdef int file 3 | cdef int line 4 | cdef str function_name 5 | 6 | cdef class NativeVariables: 7 | cdef list variables 8 | 9 | cdef class NativeStackFrame: 10 | cdef NativeStackFrame parent 11 | cdef NativeExecutionCursor cursor 12 | cdef int id 13 | 14 | cdef class NativeCodeEvent: 15 | cdef str event_name 16 | cdef NativeExecutionCursor cursor 17 | cdef NativeStackFrame stack 18 | cdef double ts 19 | cdef NativeVariables locals 20 | cdef NativeVariables input_variables 21 | cdef NativeVariables return_variables 22 | 23 | cdef class NativeVariable: 24 | cdef str name 25 | cdef str value 26 | 27 | 28 | -------------------------------------------------------------------------------- /pycrunch_trace/tracing/perf.py: -------------------------------------------------------------------------------- 1 | class TracerPerf: 2 | def __init__(self): 3 | self.total_samples = 1 4 | self.total_time = 0.00 5 | pass 6 | 7 | def did_execute_line(self, ts_diff: float): 8 | self.total_samples += 1 9 | self.total_time += ts_diff 10 | 11 | def print_avg_time(self): 12 | each = 1 13 | should_print = self.total_samples % each == 0 14 | should_print = True 15 | if should_print: 16 | time_per_sample = self.total_time / self.total_samples 17 | print(f'total_samples - {self.total_samples}') 18 | print(f'total overhead time - {self.total_time}') 19 | print(f'{time_per_sample:.5f} ms avg call time overhead') -------------------------------------------------------------------------------- /pycrunch_trace/logging-configuration.yaml: -------------------------------------------------------------------------------- 1 | version: 1 2 | formatters: 3 | simple: 4 | format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' 5 | basic: 6 | format: '%(name)s - %(message)s' 7 | handlers: 8 | console: 9 | class: logging.StreamHandler 10 | level: DEBUG 11 | formatter: simple 12 | stream: ext://sys.stdout 13 | loggers: 14 | pycrunch: 15 | level: DEBUG 16 | sampleLogger: 17 | level: DEBUG 18 | handlers: [console] 19 | propagate: no 20 | werkzeug: 21 | level: ERROR 22 | engineio: 23 | level: WARN 24 | socketio: 25 | level: WARN 26 | 27 | root: 28 | level: DEBUG 29 | handlers: [console] 30 | # handlers: [] 31 | 32 | #xx x 33 | #xx x 34 | #xx x 35 | #xx x 36 | -------------------------------------------------------------------------------- /pycrunch_trace/demo/requests_demo.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | 3 | import requests 4 | 5 | from pycrunch_trace.client.api.trace import Trace 6 | 7 | 8 | def some_method(): 9 | accum_counter = 1 10 | while accum_counter < 2: 11 | accum_counter += 1 12 | # try: 13 | r = requests.get('http://dev.iws.briteapps.space/') 14 | # r = requests.get('https://google.com/') 15 | # print(r.status_code) 16 | # print(r.content) 17 | x = accum_counter 18 | # except Exception as e: 19 | # print("!!!!! FAIL", e) 20 | 21 | 22 | yoba = Trace() 23 | yoba.start('request_exce2',profile_name='default.profile.yaml') 24 | 25 | some_method() 26 | 27 | yoba.stop() 28 | sleep(10) 29 | # print(yoba.command_buffer) 30 | # x = 1 -------------------------------------------------------------------------------- /pycrunch_trace/demo/interactive_demo_03.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | 3 | 4 | def function_call(**kwargs): 5 | print(kwargs) 6 | sleep(0.1) 7 | sum = kwargs['a'] + kwargs['b'] 8 | return sum 9 | 10 | 11 | def show_me_how_to_navigate_using_graph(): 12 | print('Lets say we have a function call') 13 | function_call(a=1, b=2) 14 | sleep(0.12) 15 | 16 | print('Clicking on function in a graph panel') 17 | print(' brings you to the beginning of the method') 18 | 19 | print(' Holding Shift and click will navigate you to the END') 20 | print(' So you can overview return values and state of locals') 21 | print(' Right before exiting from the method') 22 | 23 | print('You can step out of any function by Shift + Down Arrow') 24 | return 42 25 | 26 | -------------------------------------------------------------------------------- /pycrunch_trace/demo/demo_no_functon_calls.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | from pycrunch_trace.client.api.trace import Trace 4 | 5 | yoba = Trace() 6 | yoba.start(session_name='timing_datetime') 7 | 8 | def f(): 9 | import time 10 | from datetime import datetime 11 | from time import sleep 12 | diff = 0 13 | for _ in range(10000): 14 | start_time = datetime.now() 15 | # sleep(0.01) 16 | end_time = datetime.now() 17 | diff += (end_time - start_time).total_seconds() 18 | print(diff) 19 | 20 | diff = 0 21 | for _ in range(10000): 22 | start_time = time.perf_counter() 23 | # sleep(0.01) 24 | end_time = time.perf_counter() 25 | diff += (end_time - start_time) 26 | print(diff) 27 | 28 | f() 29 | yoba.stop() 30 | # print(yoba._tracer.simulation.simulated_code()) -------------------------------------------------------------------------------- /pycrunch_trace/demo/demo_tree_v1.py: -------------------------------------------------------------------------------- 1 | from pycrunch_trace.client.api.trace import Trace 2 | 3 | def d___(): 4 | a = 1 5 | b = 2 6 | c = 3 7 | 8 | def c___(): 9 | a = 1 10 | b = 2 11 | d___() 12 | c = 3 13 | 14 | 15 | def b___(): 16 | a = 1 17 | c___() 18 | b = 2 19 | c = 3 20 | 21 | 22 | 23 | def a___(): 24 | a = 1 25 | b = 2 26 | b___() 27 | c = 3 28 | 29 | 30 | 31 | 32 | tracer = Trace() 33 | tracer.start('recording_name') 34 | 35 | a___() 36 | 37 | tracer.stop() 38 | # print(y._tracer.simulation.simulated_code()) 39 | # import pydevd_pycharm 40 | # pydevd_pycharm.settrace('localhost', port=44441, stdoutToServer=True, stderrToServer=True) 41 | # for (x, cmd) in enumerate(y.command_buffer): 42 | # print(str(x+1) + ' - line:' + str(cmd.cursor.line) + ':' + str(cmd)) -------------------------------------------------------------------------------- /pycrunch_trace/demo/interactive_demo_04.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | 3 | from pycrunch_trace.client.api import trace 4 | 5 | 6 | def alternative_ways_to_trace(): 7 | sleep(0.25) 8 | print('You can use Trace object to manually start and stop tracing') 9 | print(' Or by applying @trace decorator to the method') 10 | print(' See examples bellow') 11 | 12 | def example_without_decorators(): 13 | from pycrunch_trace.client.api import Trace 14 | 15 | tracer = Trace() 16 | tracer.start('recording_name') 17 | 18 | code_under_trace() 19 | another_code_to_trace() 20 | 21 | tracer.stop() 22 | 23 | 24 | @trace 25 | def example_with_decorator(): 26 | # Recording will be named the same as the method name 27 | pass 28 | 29 | 30 | @trace('this_is_custom_name') 31 | def example_with_custom_name(): 32 | pass -------------------------------------------------------------------------------- /pycrunch_trace/demo/demo_million_of_calls.py: -------------------------------------------------------------------------------- 1 | from pycrunch_trace.client.api import trace, Trace 2 | from pycrunch_trace.oop import Clock 3 | 4 | 5 | def nested_function(input_a2): 6 | input_a2 += 2 7 | return input_a2 8 | 9 | def another_function(input_argument): 10 | x = nested_function(input_argument) 11 | y = nested_function(input_argument + 1) 12 | return x + y 13 | 14 | def million_of_calls(): 15 | # 20000 = 300008 16 | max_events = 10 17 | max_events = 100000 18 | for x in range(max_events): 19 | another_function(x) 20 | 21 | clock = Clock() 22 | my_yoba = Trace() 23 | my_yoba.start('million_of_calls') 24 | start = clock.now() 25 | 26 | print('before', start) 27 | million_of_calls() 28 | end = clock.now() 29 | print('after', end) 30 | print('diff', end - start) 31 | 32 | my_yoba.stop() 33 | # print(my_yoba._tracer.simulation.simulated_code()) 34 | -------------------------------------------------------------------------------- /pycrunch_trace/config.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | 4 | class TracerConfig: 5 | absolute_path = '/pycrunch_tracer/samples/module_b.py' 6 | def __init__(self): 7 | self.engine_directory = None 8 | self.working_directory = Path('.') 9 | self.recording_directory = self.get_default_recording_directory() 10 | self.package_directory = None 11 | 12 | def get_default_recording_directory(self): 13 | # return Path('/Volumes/WD_BLACK/pycrunch-trace') 14 | # return Path('/Volumes/Kingston/pycrunch-trace') 15 | return Path.joinpath(self.working_directory, 'pycrunch-recordings') 16 | 17 | def set_engine_directory(self, engine_directory: str): 18 | self.engine_directory = engine_directory 19 | 20 | def set_package_directory(self, package_directory: str): 21 | self.package_directory = package_directory 22 | 23 | config = TracerConfig() 24 | 25 | -------------------------------------------------------------------------------- /pycrunch_trace/server/events.py: -------------------------------------------------------------------------------- 1 | class TracingServerEvent: 2 | event_name: str 3 | 4 | 5 | class RecordingStartEvent(TracingServerEvent): 6 | session_id: str 7 | 8 | def __init__(self, session_id: str): 9 | self.session_id = session_id 10 | self.event_name = 'recording_start' 11 | 12 | 13 | class RecordingCompleteEvent(TracingServerEvent): 14 | session_id: str 15 | 16 | def __init__(self, session_id: str): 17 | self.session_id = session_id 18 | self.event_name = 'recording_complete' 19 | 20 | 21 | class PartialFileChunkEvent(TracingServerEvent): 22 | def __init__(self, event_number, session_id, bytes_to_write, events_in_payload): 23 | self.events_in_payload = events_in_payload 24 | self.bytes_to_write = bytes_to_write 25 | self.session_id = session_id 26 | self.event_number = event_number 27 | self.event_name = 'recording_chunk' 28 | -------------------------------------------------------------------------------- /pycrunch_trace/tracing/file_map.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Any 2 | 3 | 4 | class FileMap: 5 | # this is for size reduction 6 | # filename.py -> id 7 | files: Dict[str, int] 8 | 9 | def __init__(self): 10 | self.files = dict() 11 | pass 12 | 13 | def file_id(self, filename: str) -> int: 14 | file_id = self.files.get(filename) 15 | if not file_id: 16 | file_id = len(self.files) + 1 17 | self.files[filename] = file_id 18 | 19 | return file_id 20 | 21 | def filename_by_id(self, search_for_file_id : int): 22 | # slow 23 | for (name, file_id) in self.files.items(): 24 | if file_id == search_for_file_id: 25 | return name 26 | return f'file {search_for_file_id} not found' 27 | 28 | @classmethod 29 | def from_reverse(cls, files: Dict[str, int]): 30 | x = FileMap() 31 | x.files = files 32 | return x 33 | -------------------------------------------------------------------------------- /pycrunch_trace/tracing/readme.md: -------------------------------------------------------------------------------- 1 | https://opensource.com/article/19/8/debug-python 2 | 3 | settrace registers a trace function for the interpreter, which may be called in any of the following cases: 4 | 5 | - Function call 6 | - Line execution 7 | - Function return 8 | - Exception raised 9 | 10 | 11 | or: 12 | 13 | - 'call' 14 | - 'line' 15 | - 'return' 16 | - 'exception' 17 | 18 | When looking at this function, the first things that come to mind are its arguments and return values. The trace function arguments are: 19 | 20 | frame object, which is the full state of the interpreter at the point of the function's execution 21 | event string, which can be call, line, return, or exception 22 | arg object, which is optional and depends on the event type 23 | 24 | 25 | The local trace function should return a reference to itself (or to another function for further tracing in that scope), or None to turn off tracing in that scope. 26 | 27 | https://docs.python.org/3/library/inspect.html 28 | 29 | -------------------------------------------------------------------------------- /pycrunch_trace/file_system/test_trace_session.py: -------------------------------------------------------------------------------- 1 | from pycrunch_trace.file_system.session_store import SessionStore 2 | from pycrunch_trace.file_system.trace_session import TraceSession 3 | from pycrunch_trace.tests.test_buffer import build_testing_events 4 | 5 | def a(): 6 | pass 7 | 8 | def test_1(): 9 | x = TraceSession() 10 | x.save('a', build_testing_events()) 11 | 12 | def test_session_store(): 13 | x = SessionStore() 14 | session = x.load_session('a') 15 | session.load_metadata() 16 | print(vars(session.metadata)) 17 | def test_load_sessions(): 18 | x = SessionStore() 19 | print(x.all_sessions()) 20 | 21 | def test_multiple_files(): 22 | x = TraceSession('aaa') 23 | x.did_enter_traceable_file('a') 24 | x.did_enter_traceable_file('a2') 25 | x.did_enter_traceable_file('b') 26 | print('aaa') 27 | x.will_skip_file('c') 28 | x.will_skip_file('c') 29 | x.will_skip_file('d') 30 | x.buffer_became_available(build_testing_events()) 31 | x.save() 32 | -------------------------------------------------------------------------------- /pycrunch_trace/reference_code/sys_settrace_call.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | def trace_calls(frame, event, arg): 5 | if event != 'call': 6 | return 7 | co = frame.f_code 8 | func_name = co.co_name 9 | if func_name == 'write': 10 | # Ignore write() calls from printing 11 | return 12 | func_line_no = frame.f_lineno 13 | func_filename = co.co_filename 14 | if not func_filename.endswith('sys_settrace_call.py'): 15 | # Ignore calls not in this module 16 | return 17 | caller = frame.f_back 18 | caller_line_no = caller.f_lineno 19 | caller_filename = caller.f_code.co_filename 20 | print('* Call to', func_name) 21 | print('* on line {} of {}'.format( 22 | func_line_no, func_filename)) 23 | print('* from line {} of {}'.format( 24 | caller_line_no, caller_filename)) 25 | return 26 | 27 | 28 | def b(): 29 | print('inside b()\n') 30 | 31 | 32 | def a(): 33 | print('inside a()\n') 34 | b() 35 | 36 | 37 | sys.settrace(trace_calls) 38 | a() -------------------------------------------------------------------------------- /pycrunch_trace/client/command_buffer.py: -------------------------------------------------------------------------------- 1 | from collections import deque 2 | from typing import List 3 | 4 | from pycrunch_trace.events.base_event import Event 5 | 6 | 7 | class ArrayCommandBuffer: 8 | def __init__(self): 9 | # todo swap implementation to linked list? 10 | self._command_buffer = [] 11 | 12 | def add_event(self, evt): 13 | self._command_buffer.append(evt) 14 | 15 | def finish_chunk(self) -> List[Event]: 16 | old_buffer = self._command_buffer 17 | self._command_buffer = [] 18 | return old_buffer 19 | 20 | def how_many_events(self): 21 | return len(self._command_buffer) 22 | 23 | class DequeCommandBuffer: 24 | def __init__(self): 25 | self._command_buffer = deque() 26 | 27 | def add_event(self, evt): 28 | self._command_buffer.append(evt) 29 | 30 | def finish_chunk(self) -> List[Event]: 31 | old_buffer = self._command_buffer.copy() 32 | self._command_buffer.clear() 33 | return list(old_buffer) 34 | 35 | def how_many_events(self): 36 | return len(self._command_buffer) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021-2022 Gleb Sevruk and other contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /pycrunch_trace/proto/message.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | message TraceSession { 3 | repeated TraceEvent events = 1; 4 | repeated StackFrame stack_frames = 2; 5 | repeated File files = 3; 6 | } 7 | 8 | message FilesInSession { 9 | repeated FileContent files = 1; 10 | } 11 | 12 | message FileContent { 13 | int32 id = 1; 14 | bytes content = 2; 15 | } 16 | 17 | message TraceEvent { 18 | string event_name = 1; 19 | ExecutionCursor cursor = 2; 20 | int32 stack_id = 3; 21 | Variables input_variables = 4; 22 | Variables locals = 5; 23 | Variables return_variables = 6; 24 | // timestamp of the event 25 | // todo will I lose precision on long events? 26 | float ts = 7; 27 | } 28 | message Variables { 29 | repeated Variable variables = 1; 30 | } 31 | 32 | message Variable { 33 | string name = 1; 34 | string value = 2; 35 | } 36 | message ExecutionCursor { 37 | int32 file = 1; 38 | int32 line = 2; 39 | string function_name = 3; 40 | 41 | } 42 | message File { 43 | int32 id = 1; 44 | string file = 2; 45 | } 46 | 47 | message StackFrame { 48 | int32 id = 1; 49 | int32 file = 2; 50 | int32 line = 3; 51 | int32 parent_id = 4; 52 | string function_name = 5; 53 | 54 | } -------------------------------------------------------------------------------- /pycrunch_trace/oop/file.py: -------------------------------------------------------------------------------- 1 | import io 2 | from pathlib import Path 3 | from typing import Union 4 | 5 | class AbstractFile: 6 | def as_bytes(self): 7 | pass 8 | 9 | def short_name(self): 10 | pass 11 | 12 | class File(AbstractFile): 13 | class Mock(AbstractFile): 14 | def __init__(self, buffer: bytes, filename: str = None): 15 | if filename: 16 | self.filename = filename 17 | else: 18 | self.filename = 'mock' 19 | 20 | self.buffer = buffer 21 | 22 | def as_bytes(self): 23 | return self.buffer 24 | 25 | def short_name(self): 26 | return self.buffer 27 | filename: str 28 | 29 | def __init__(self, filename: Union[str, Path]): 30 | if type(filename) == str: 31 | self.filename = filename 32 | elif isinstance(filename, Path): 33 | self.filename = str(filename) 34 | 35 | def as_bytes(self): 36 | with io.FileIO(self.filename, mode='r') as current_file: 37 | return current_file.readall() 38 | 39 | def short_name(self) -> str: 40 | return Path(self.filename).name 41 | 42 | def __str__(self): 43 | return self.filename 44 | 45 | -------------------------------------------------------------------------------- /pycrunch_trace/oop/directory.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import Union 3 | 4 | from .file import File 5 | 6 | 7 | class Directory: 8 | path: Path 9 | 10 | def __init__(self, path: Union[str, Path]): 11 | if type(path) == str: 12 | self.path = Path(path) 13 | elif isinstance(path, Path): 14 | self.path = path 15 | else: 16 | raise Exception('Not supported path: ' + str(path)) 17 | 18 | def folders(self): 19 | result = [] 20 | self._ensure_created() 21 | for maybe_be_folder in self.path.glob('*'): 22 | if maybe_be_folder.is_dir(): 23 | result.append(maybe_be_folder.name) 24 | return result 25 | 26 | def files(self, extension: str): 27 | if not self._exists(): 28 | return [] 29 | 30 | result = [] 31 | for file in self.path.glob(f'*.{extension}'): 32 | result.append(File(file)) 33 | return result 34 | 35 | def _ensure_created(self): 36 | if not self._exists(): 37 | self.path.mkdir(exist_ok=True) 38 | 39 | def _exists(self): 40 | return self.path.exists() 41 | 42 | def __str__(self): 43 | return str(self.path) 44 | -------------------------------------------------------------------------------- /pycrunch_trace/file_system/trace_session.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from pathlib import Path 3 | from typing import List, Set, Any 4 | 5 | from pycrunch_trace.events.base_event import Event 6 | from pycrunch_trace.file_system.session_store import SessionStore 7 | 8 | 9 | 10 | 11 | class TraceSession: 12 | recording_directory : Path 13 | files_in_session: Set[str] 14 | excluded_files : Set[str] 15 | environment_during_run: dict 16 | event_buffer: List[Any] 17 | 18 | def __init__(self): 19 | self.files_in_session = set() 20 | self.excluded_files = set() 21 | self.environment_during_run = dict() 22 | self.session_store = SessionStore() 23 | 24 | def buffer_became_available(self, event_buffer): 25 | self.event_buffer = event_buffer 26 | 27 | def did_enter_traceable_file(self, filename: str): 28 | self.files_in_session.add(filename) 29 | 30 | def will_skip_file(self, filename: str): 31 | self.excluded_files.add(filename) 32 | 33 | def save(self): 34 | persistent_session = self.session_store.new_session(self.session_name) 35 | persistent_session.save_with_metadata(self.event_buffer, self.files_in_session, self.excluded_files) 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | from distutils.core import setup 3 | 4 | setup(name='pycrunch-trace', 5 | version='0.3', 6 | description='PyCrunch Time Travel Debugger', 7 | url='https://pytrace.com/', 8 | author='Gleb Sevruk', 9 | author_email='gleb@pycrunch.com', 10 | license='MIT', 11 | keywords=[ 12 | 'tracing', 13 | 'debugging', 14 | 'time-travel debugging', 15 | 'record-and-replay debugging', 16 | 'live coding', 17 | ], 18 | packages=setuptools.find_packages(), 19 | setup_requires=['wheel', 'Cython'], 20 | install_requires=[ 21 | 'Cython', 22 | 'jsonpickle', 23 | 'PyYAML', 24 | 'protobuf>=3.11.3,<4', 25 | 'jsonpickle', 26 | ], 27 | classifiers=[ 28 | 'Development Status :: 3 - Alpha', 29 | 'License :: OSI Approved :: MIT License', 30 | 'Programming Language :: Python :: 3', 31 | 'Topic :: Software Development :: Debuggers', 32 | ], 33 | project_urls={ 34 | 'Source': 'https://github.com/gleb-sevruk/pycrunch-trace/', 35 | 'Funding': 'https://pycrunch.com/donate', 36 | }, 37 | include_package_data=True, 38 | zip_safe=False) 39 | -------------------------------------------------------------------------------- /pycrunch_trace/events/file_contents_in_protobuf.pyx: -------------------------------------------------------------------------------- 1 | from io import FileIO 2 | from typing import Union, Dict 3 | 4 | from pycrunch_trace.events.base_event import Event 5 | 6 | from pycrunch_trace.native.native_models cimport NativeCodeEvent, NativeVariable, NativeStackFrame 7 | 8 | from pycrunch_trace.proto import message_pb2 9 | from pycrunch_trace.tracing.file_map import FileMap 10 | 11 | cdef class FileContentsInProtobuf: 12 | cdef object files 13 | 14 | def __init__(self, files: Dict[str, int]): 15 | self.files = files 16 | 17 | def as_bytes(self): 18 | session = message_pb2.FilesInSession() 19 | for (filename, file_id) in self.files.items(): 20 | current_file_content = message_pb2.FileContent() 21 | current_file_content.id = file_id 22 | try: 23 | with FileIO(filename, 'r') as f: 24 | current_file_content.content = f.readall() 25 | except Exception as e: 26 | print('error! at store_file_contents: ' + filename) 27 | print(e) 28 | current_file_content.content = b'error! at store_file_contents: ' + filename.encode('utf-8') 29 | session.files.append(current_file_content) 30 | 31 | return session.SerializeToString() 32 | -------------------------------------------------------------------------------- /pycrunch_trace/server/tests/test_perf.py: -------------------------------------------------------------------------------- 1 | from pycrunch_trace.server.perf import PerformanceInsights 2 | 3 | 4 | def test_average_package_size_one_chunk(): 5 | sut = PerformanceInsights() 6 | 7 | sut.sample(session_id='session_id', number_of_events=10, buffer_size=2000) 8 | 9 | actual = sut.average_event_size(session_id='session_id') 10 | assert actual == 200 11 | 12 | def test_average_package_size_three_chunks(): 13 | sut = PerformanceInsights() 14 | 15 | sut.sample(session_id='session_id', number_of_events=1, buffer_size=200) 16 | sut.sample(session_id='session_id', number_of_events=1, buffer_size=250) 17 | sut.sample(session_id='session_id', number_of_events=1, buffer_size=300) 18 | 19 | actual = sut.average_event_size(session_id='session_id') 20 | assert actual == 250 21 | 22 | def test_average_package_size_three_sessions(): 23 | sut = PerformanceInsights() 24 | 25 | sut.sample(session_id='1', number_of_events=1, buffer_size=200) 26 | sut.sample(session_id='2', number_of_events=1, buffer_size=250) 27 | sut.sample(session_id='3', number_of_events=2, buffer_size=700) 28 | 29 | assert 200 == sut.average_event_size(session_id='1') 30 | assert 250 == sut.average_event_size(session_id='2') 31 | assert 350 == sut.average_event_size(session_id='3') 32 | -------------------------------------------------------------------------------- /pycrunch_trace/server/incoming_traces.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Any 2 | 3 | import logging 4 | 5 | from pycrunch_trace.server.trace_in_progress import TraceInProgress 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | 11 | 12 | class IncomingTraces: 13 | sessions_by_id: Dict[str, TraceInProgress] 14 | 15 | def __init__(self): 16 | self.sessions_by_id = dict() 17 | 18 | def trace_will_start(self, session_id): 19 | self.delete_possible_old_session(session_id) 20 | self._create_trace_in_progress(session_id) 21 | 22 | def _create_trace_in_progress(self, session_id): 23 | self.sessions_by_id[session_id] = TraceInProgress(session_id) 24 | 25 | def did_receive_more_events(self, session_id, events_count): 26 | current = self.get_session_with_id(session_id) 27 | current.add_events(events_count) 28 | 29 | def get_session_with_id(self, session_id) -> TraceInProgress: 30 | return self.sessions_by_id[session_id] 31 | 32 | def delete_possible_old_session(self, session_id): 33 | if session_id in self.sessions_by_id: 34 | self.log('Deleting stale/old session') 35 | del self.sessions_by_id[session_id] 36 | 37 | def log(self, msg): 38 | logger.info(msg) 39 | 40 | incoming_traces: IncomingTraces = IncomingTraces() 41 | -------------------------------------------------------------------------------- /pycrunch_trace/client/networking/commands.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, List 2 | 3 | 4 | class AbstractNetworkCommand: 5 | command_name: str 6 | 7 | 8 | class StartCommand(AbstractNetworkCommand): 9 | def __init__(self, session_id: str): 10 | self.session_id = session_id 11 | self.command_name = 'StartCommand' 12 | 13 | class EventsSlice(AbstractNetworkCommand): 14 | files: Dict[str, int] 15 | 16 | def __init__(self, session_id: str, chunk_number: int, events: list, files: Dict[str, int]): 17 | self.files = files 18 | self.chunk_number = chunk_number 19 | self.events = events 20 | self.command_name = 'EventsSlice' 21 | self.session_id = session_id 22 | 23 | class FileContentSlice(AbstractNetworkCommand): 24 | files: Dict[str, int] 25 | 26 | def __init__(self, session_id: str, files: Dict[str, int]): 27 | self.command_name = 'FileContentSlice' 28 | self.files = files 29 | self.session_id = session_id 30 | 31 | 32 | class StopCommand(AbstractNetworkCommand): 33 | def __init__(self, session_id: str, files_included: List[str], files_excluded: List[str]): 34 | self.command_name = 'StopCommand' 35 | self.session_id = session_id 36 | self.files_included = files_included 37 | self.files_excluded = files_excluded 38 | 39 | -------------------------------------------------------------------------------- /pycrunch_trace/demo.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from ppretty import ppretty 3 | 4 | def unnamed_method(x: str): 5 | print('unnamed_method') 6 | 7 | def sum(a: int, b: int) -> int: 8 | print('!a') 9 | print('!b') 10 | unnamed_method('a') 11 | return a + b 12 | 13 | def trace_calls(frame, event, arg): 14 | print(f'event: {event}') 15 | print('f_locals '+ ppretty(frame.f_locals)) 16 | # print(ppretty(frame)) 17 | if event != 'call': 18 | return 19 | frame__f_code = frame.f_code 20 | func_name = frame__f_code.co_name 21 | print(f'func_name: {func_name}') 22 | if func_name == 'write': 23 | # Ignore write() calls from printing 24 | return 25 | func_line_no = frame.f_lineno 26 | func_filename = frame__f_code.co_filename 27 | if not func_filename.endswith('demo.py'): 28 | # Ignore calls not in this module 29 | return 30 | caller = frame.f_back 31 | caller_line_no = caller.f_lineno 32 | caller_filename = caller.f_code.co_filename 33 | print('* Call to', func_name) 34 | print(f'* on line {func_line_no} of {func_filename}') 35 | print(f'* from line {caller_line_no} of {caller_filename}') 36 | return 37 | 38 | print('--- begin') 39 | 40 | sys.settrace(trace_calls) 41 | 42 | result = sum(1, 2) 43 | print(result) 44 | 45 | print('--- end') 46 | 47 | -------------------------------------------------------------------------------- /pycrunch_trace/filters/DefaultFileFilter.py: -------------------------------------------------------------------------------- 1 | from random import Random 2 | 3 | from . import AbstractFileFilter 4 | 5 | 6 | class DefaultFileFilter(AbstractFileFilter): 7 | def should_trace(self, filename: str) -> bool: 8 | # start or end with 9 | exclusions = ( 10 | '/Users/gleb/code/pycrunch_tracing/', 11 | '/Users/gleb/code/bc/briteapps-admin/', 12 | 'module_a.py', 13 | 'module_b.py', 14 | 'module_c.py', 15 | 'invalid_picker_with_exception.py', 16 | '/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6' 17 | '/Users/gleb/venv/PyCrunch/lib/python3.6/site-packages/', 18 | # 'copyreg.py', 19 | '/Users/gleb/venv/PyCrunch/lib/python3.6/site-packages/py/', 20 | 'api/tracing.py' 21 | ) 22 | 23 | # r= Random() 24 | # if r.choice(range(0, 10)) == 1: 25 | # return True 26 | # return True 27 | if filename.endswith(exclusions) or filename.startswith(exclusions): 28 | return False 29 | 30 | return True 31 | 32 | if filename.endswith(end_patterns): 33 | return True 34 | if filename.startswith(start_patterns): 35 | return True 36 | 37 | print('should_trace - false', filename) 38 | return False -------------------------------------------------------------------------------- /pycrunch_trace/profiles/test_profile_list.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import List, Any 3 | 4 | import yaml 5 | 6 | from ..filters import CustomFileFilter 7 | from ..oop import Directory, AbstractFile 8 | from ..oop import File 9 | 10 | 11 | class Profiles: 12 | def __init__(self, directory: Directory): 13 | self.directory = directory 14 | 15 | 16 | def test_profiles(): 17 | directory = Directory(Path.joinpath(Path(__file__).parent.parent, 'pycrunch-profiles')) 18 | print(directory) 19 | all = directory.folders() 20 | all = directory.files('yaml') 21 | print(str(all[0].short_name())) 22 | 23 | 24 | 25 | 26 | def test_profile_should_have_excluded(): 27 | buffer = b""" 28 | exclusions: 29 | - no_trace.py 30 | - /Library/ 31 | """ 32 | sut = CustomFileFilter(File.Mock(buffer)) 33 | 34 | assert not sut.should_trace('no_trace.py') 35 | assert not sut.should_trace('/Library/a.py') 36 | assert sut.should_trace('b.py') 37 | assert sut.should_trace('/code/b.py') 38 | 39 | 40 | def test_profile_trace_vars_false(): 41 | buffer = b""" 42 | trace_variables: false 43 | """ 44 | sut = CustomFileFilter(File.Mock(buffer)) 45 | 46 | assert not sut.should_record_variables() 47 | 48 | 49 | def profiles_dir(): 50 | return Directory(Path.joinpath(Path(__file__).parent, 'pycrunch-profiles')) -------------------------------------------------------------------------------- /pycrunch_trace/client/networking/client_trace_introspection.py: -------------------------------------------------------------------------------- 1 | import collections 2 | from collections import defaultdict 3 | from typing import List, Dict 4 | 5 | from pycrunch_trace.tracing.file_map import FileMap 6 | 7 | 8 | class ClientTraceIntrospection: 9 | total_events: int 10 | 11 | def __init__(self): 12 | self.total_events = 0 13 | self.stats = defaultdict(int) 14 | # file id -> hit count 15 | self.top_files = defaultdict(int) 16 | 17 | def save_events(self, events: List): 18 | self.total_events += len(events) 19 | for e in events: 20 | self.stats[e.event_name] += 1 21 | self.top_files[e.cursor.file] += 1 22 | 23 | def print_to_console(self, files: Dict[str, int]): 24 | print('TraceIntrospection:') 25 | print(' stats:') 26 | for (each, hit_count) in self.stats.items(): 27 | print(f' - {each}:{hit_count}') 28 | 29 | print(' files:') 30 | filemap = FileMap.from_reverse(files) 31 | sorted_x = sorted(self.top_files.items(), reverse=True, key=lambda kv: kv[1]) 32 | sortir = collections.OrderedDict(sorted_x) 33 | 34 | 35 | for (each, hit_count) in sortir.items(): 36 | print(f' - {hit_count} hits in {filemap.filename_by_id(each)}') 37 | 38 | 39 | client_introspection = ClientTraceIntrospection() 40 | -------------------------------------------------------------------------------- /pycrunch_trace/demo/interactive_demo_02.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | 3 | 4 | def method_in_another_file(some_number: int): 5 | print('File contents are stored as they were at the moment of the recording.') 6 | 7 | print('For easy navigation, press G to open graph panel') 8 | print('Graph will give overview of method calls and their approximate duration.') 9 | 10 | print('Check this:') 11 | sleep(0.15) 12 | 13 | print('You can see on the graph that we are at 150ms mark of the recording.') 14 | print('Current time (since recording start) is also displayed in inspector') 15 | print(' And on graph panel as a needle') 16 | 17 | print('You can skip next method call by using Down Arrow [↓]') 18 | print('Or enter into by Right Arrow [→]') 19 | dummy_method() 20 | 21 | print('Note __return pseudo-variable as we exit from this method') 22 | return some_number * 3 + 1 23 | 24 | 25 | def dummy_method(): 26 | print('See how `index` variable is changed as you step through the loop') 27 | for index in range(5): 28 | print('If you got trapped in a loop like this, use Shift+Down Arrow to step out of this method') 29 | print('You can also try Shift+Up Arrow to get to the place where this method was called') 30 | sleep(0.03) 31 | 32 | # if you are not able to read all text, you can toggle inspector 33 | # by pressing [i] 34 | -------------------------------------------------------------------------------- /pycrunch_trace/pycrunch-profiles/default.profile.yaml: -------------------------------------------------------------------------------- 1 | exclusions: 2 | - pycrunch_trace/client/api/trace_decorator.py 3 | - pycrunch_trace/client/api/trace.py 4 | ##- /Library/Frameworks/Python.framework/Versions/3.6/ 5 | #- Workbook.py 6 | #- /Users/gleb/venv/ba-admin/lib/python3.6/site-packages/setuptools-39.1.0 7 | #- /Users/gleb/venv/ba-admin/lib/python3.6/site-packages/raven/ 8 | #- /Users/gleb/venv/ba-admin/lib/python3.6/site-packages/django 9 | ##- /Users/gleb/venv/ba-admin/lib/python3.6/site-packages/setuptools-39.1.0-py3.6.egg/pkg_resources/ 10 | ##- /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/posixpath.py 11 | ##- /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/parse.py 12 | ##- /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ 13 | ##- /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/genericpath.py 14 | ##- /Users/gleb/venv/ba-admin/lib/python3.6/site-packages/django/template/base.py 15 | - < 16 | ##- /Users/gleb/venv/PyCrunch/lib/python3.6/site-packages/ 17 | #- /Users/gleb/venv/pycrunch_tracing2/lib/python3.6/site-packages/pluggy 18 | #- bool: 30 | possible_tracer = self.find_tracer_with_id(reference) 31 | if not possible_tracer: 32 | return False 33 | 34 | self.tracers_online.remove(possible_tracer) 35 | return True 36 | 37 | def find_tracer_with_id(self, reference) -> Optional[LiveSession]: 38 | for tracer in self.tracers_online: 39 | if tracer.sid == reference: 40 | return tracer 41 | 42 | return None 43 | 44 | def tracer_did_connect(self, sid, version): 45 | self.tracers_online.append(LiveSession(sid, version)) 46 | pass 47 | 48 | def tracer_did_disconnect(self, sid): 49 | return self.remove_if_any_tracer(sid) 50 | 51 | -------------------------------------------------------------------------------- /pycrunch_trace/tests/test_trace_file.py: -------------------------------------------------------------------------------- 1 | import io 2 | import os 3 | import struct 4 | from pathlib import Path 5 | from tempfile import NamedTemporaryFile 6 | 7 | from pycrunch_trace.client.api import trace 8 | from pycrunch_trace.file_system.trace_file import TraceFile 9 | 10 | 11 | def test_trace_file_header(): 12 | f = NamedTemporaryFile(prefix='pytracer', delete=False) 13 | name = f.name 14 | f.close() 15 | print(name) 16 | try: 17 | sut = TraceFile('dummy', Path(f.name)) 18 | 19 | x = sut.write_header_placeholder() 20 | sut.update_file_header_files_section(123) 21 | with io.FileIO(name, 'r') as actual: 22 | # SIGNATURE 23 | buffer = actual.read(struct.calcsize('>i')) 24 | sig = struct.unpack('>i', buffer)[0] 25 | assert sig == 15051991 26 | 27 | buffer = actual.read(struct.calcsize('>i')) 28 | tag = struct.unpack('>i', buffer)[0] 29 | assert tag == 1 30 | 31 | buffer = actual.read(struct.calcsize('>i')) 32 | header_size = struct.unpack('>i', buffer)[0] 33 | assert header_size == 16 * 1024 34 | 35 | 36 | new_pos = actual.seek(header_size + 8 + 4, io.SEEK_SET) 37 | print(new_pos) 38 | # SIGNATURE 39 | buffer = actual.read(struct.calcsize('>i')) 40 | sig = struct.unpack('>i', buffer)[0] 41 | assert sig == 15051991 42 | 43 | finally: 44 | os.remove(f.name) 45 | pass 46 | pass -------------------------------------------------------------------------------- /pycrunch_trace/tests/test_buffer.py: -------------------------------------------------------------------------------- 1 | import pycrunch_trace.events.method_enter as e 2 | from pycrunch_trace.config import config 3 | from pycrunch_trace.serialization import to_string 4 | 5 | 6 | def test_simple(): 7 | command_stack = build_testing_events() 8 | # print(command_stack) 9 | print(to_string(command_stack)) 10 | 11 | 12 | 13 | 14 | 15 | def build_testing_events(): 16 | command_stack = [] 17 | stack = ['a:11', 'b:12'] 18 | method_enter_event = e.MethodEnterEvent(e.ExecutionCursor(config.absolute_path, 2), stack) 19 | method_enter_event.input_variables.push_variable('some_number', 1) 20 | command_stack.append(method_enter_event) 21 | line_1 = e.LineExecutionEvent(e.ExecutionCursor(config.absolute_path, 3), stack) 22 | line_1.locals.push_variable('a', 2) 23 | command_stack.append(line_1) 24 | line_2 = e.LineExecutionEvent(e.ExecutionCursor(config.absolute_path, 4), stack) 25 | line_2.locals.push_variable('a', 2) 26 | line_2.locals.push_variable('b', 3) 27 | command_stack.append(line_2) 28 | line_3 = e.LineExecutionEvent(e.ExecutionCursor(config.absolute_path, 5), stack) 29 | line_3.locals.push_variable('a', 2) 30 | line_3.locals.push_variable('b', 3) 31 | line_3.locals.push_variable('result', 6) 32 | command_stack.append(line_3) 33 | line_return = e.MethodExitEvent(e.ExecutionCursor(config.absolute_path, 6), stack) 34 | line_return.return_variables.push_variable('return_value', 6) 35 | command_stack.append(line_return) 36 | return command_stack 37 | -------------------------------------------------------------------------------- /pycrunch_trace/tracing/tests/test_file_map.py: -------------------------------------------------------------------------------- 1 | from pycrunch_trace.events.method_enter import MethodEnterEvent, ExecutionCursor, StackFrame, MethodExitEvent 2 | from pycrunch_trace.tracing.file_map import FileMap 3 | 4 | 5 | def test_file_id(): 6 | sut = FileMap() 7 | assert 1 == sut.file_id('a.py') 8 | 9 | 10 | def test_same_file_should_return_same_id(): 11 | sut = FileMap() 12 | assert 1 == sut.file_id('a.py') 13 | assert 1 == sut.file_id('a.py') 14 | assert 1 == sut.file_id('a.py') 15 | 16 | def test_multiple_files(): 17 | sut = FileMap() 18 | assert 1 == sut.file_id('a.py') 19 | assert 2 == sut.file_id('b.py') 20 | assert 3 == sut.file_id('c.py') 21 | assert 1 == sut.file_id('a.py') 22 | assert 2 == sut.file_id('b.py') 23 | assert 3 == sut.file_id('c.py') 24 | 25 | def test_a(): 26 | from pycrunch_trace.client.networking.client_trace_introspection import ClientTraceIntrospection 27 | sut = ClientTraceIntrospection() 28 | files = dict() 29 | files['c'] = 7 30 | files['a'] = 1 31 | files['b'] = 2 32 | sut.save_events([ 33 | MethodEnterEvent( 34 | ExecutionCursor(7,1,'1'), 35 | StackFrame(-1, 1,1,'a'), 36 | 1), 37 | MethodExitEvent( 38 | ExecutionCursor(1, 1, '1'), 39 | StackFrame(-1, 1, 1, 'a'), 40 | 2), 41 | MethodExitEvent( 42 | ExecutionCursor(1, 1, '1'), 43 | StackFrame(-1, 1, 1, 'a'), 44 | 2), 45 | ]) 46 | sut.print_to_console(files) -------------------------------------------------------------------------------- /pycrunch_trace/reference_code/sys_settrace_line.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import sys 3 | 4 | from ppretty import ppretty 5 | 6 | 7 | def trace_lines(frame, event, arg): 8 | if event == 'return': 9 | print(f' -> return') 10 | print('f_locals ' + ppretty(frame)) 11 | 12 | return 13 | 14 | if event != 'line': 15 | return 16 | co = frame.f_code 17 | func_name = co.co_name 18 | line_no = frame.f_lineno 19 | print(f'* {func_name} line {line_no}') 20 | print(f'* locals: {frame.f_locals}') 21 | 22 | 23 | def trace_calls(frame, event, arg, to_be_traced): 24 | if event != 'call': 25 | return 26 | co = frame.f_code 27 | func_name = co.co_name 28 | if func_name == 'write': 29 | # Ignore write() calls from printing 30 | return 31 | line_no = frame.f_lineno 32 | filename = co.co_filename 33 | if not filename.endswith('sys_settrace_line.py'): 34 | # Ignore calls not in this module 35 | return 36 | print('* Call to {} on line {} of {}'.format( 37 | func_name, line_no, filename)) 38 | if func_name in to_be_traced: 39 | # Trace into this function 40 | return trace_lines 41 | return 42 | 43 | 44 | def c(input): 45 | print('input =', input) 46 | print('Leaving c()') 47 | 48 | 49 | def b(arg): 50 | val = arg * 5 51 | val = val * 2 52 | c(val) 53 | print('Leaving b()') 54 | return 42 55 | 56 | 57 | def a(): 58 | b(2) 59 | print('Leaving a()') 60 | 61 | 62 | tracer = functools.partial(trace_calls, to_be_traced=['b']) 63 | sys.settrace(tracer) 64 | a() -------------------------------------------------------------------------------- /pycrunch_trace/client/api/network_client.py: -------------------------------------------------------------------------------- 1 | import socketio 2 | 3 | from . import version 4 | 5 | 6 | class TracingClient: 7 | 8 | def __init__(self, host_url: str): 9 | self.sio = socketio.Client() 10 | connection_headers = dict( 11 | version=version.version, 12 | product='pycrunch-tracing-node', 13 | ) 14 | self.sio.connect(url=host_url, headers=connection_headers, transports=['websocket']) 15 | 16 | @self.sio.event 17 | def message(data): 18 | print('CLIENT: I received a message!') 19 | 20 | @self.sio.on('my message') 21 | def on_message(data): 22 | print('CLIENT: I received a message!') 23 | 24 | @self.sio.event 25 | def connect(): 26 | print("CLIENT: I'm connected!") 27 | 28 | @self.sio.event 29 | def connect_error(): 30 | print("CLIENT: The connection failed!") 31 | 32 | @self.sio.event 33 | def disconnect(): 34 | print("CLIENT: I'm disconnected!") 35 | 36 | def push_message(self, entire_tracing_sesssion): 37 | # dumps = pickle.dumps(entire_tracing_sesssion) 38 | # print(f'dumped {len(dumps)} bytes') 39 | try: 40 | print(f' ...sending bytes') 41 | 42 | self.sio.emit('event', dict( 43 | action='new_recording', 44 | # buffer=dumps, 45 | )) 46 | print(f' ...sent') 47 | except Exception as e: 48 | print(' -- !fail to send') 49 | print(str(e)) 50 | 51 | def disconnect(self): 52 | 53 | self.sio.disconnect() 54 | -------------------------------------------------------------------------------- /pycrunch_trace/tracing/inline_profiler.py: -------------------------------------------------------------------------------- 1 | from collections import deque, OrderedDict 2 | from typing import Dict, Any 3 | 4 | from pycrunch_trace.oop import Clock 5 | 6 | 7 | class InlineProfiler: 8 | timings: Dict[str, float] 9 | 10 | def __init__(self): 11 | # method scope-> total time 12 | self.timings = OrderedDict() 13 | self.execution_stack = deque() 14 | 15 | def enter_scope(self, scope): 16 | self.execution_stack.append(scope) 17 | 18 | def get_full_stack(self): 19 | return '.'.join(self.execution_stack) 20 | 21 | def exit_scope(self): 22 | self.execution_stack.pop() 23 | 24 | def append_timing(self, scope: str, time_spent: float): 25 | stack___key = self.get_full_stack() + '.' + scope 26 | if not self.timings.get(stack___key): 27 | self.timings[stack___key] = time_spent 28 | else: 29 | self.timings[stack___key] += time_spent 30 | 31 | def print_timings(self): 32 | print(f'----print_timings----') 33 | for (key, val) in self.timings.items(): 34 | print(f'{key}\t{val}') 35 | 36 | 37 | inline_profiler_instance: InlineProfiler = InlineProfiler() 38 | clock = Clock() 39 | 40 | class ProfilingScope(): 41 | def __init__(self, scope): 42 | self.scope = scope 43 | self.ts_begin = None 44 | self.ts_end = None 45 | 46 | def __enter__(self): 47 | self.ts_begin = clock.now() 48 | inline_profiler_instance.enter_scope(self.scope) 49 | return None 50 | 51 | def __exit__(self, exc_type, exc_value, exc_traceback): 52 | self.ts_end = clock.now() 53 | inline_profiler_instance.exit_scope() 54 | 55 | inline_profiler_instance.append_timing(self.scope, self.ts_end - self.ts_begin) -------------------------------------------------------------------------------- /pycrunch_trace/file_system/session_store.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import List 3 | 4 | from pycrunch_trace.config import config 5 | from pycrunch_trace.file_system.persisted_session import PersistedSession, LazyLoadedSession 6 | 7 | 8 | class SessionStore: 9 | recording_directory: Path 10 | 11 | def __init__(self): 12 | self.recording_directory = config.recording_directory 13 | pass 14 | 15 | def all_sessions(self) -> List[str]: 16 | result = [] 17 | self.ensure_recording_directory_created() 18 | for maybe_be_folder in self.recording_directory.glob('*'): 19 | if maybe_be_folder.is_dir(): 20 | result.append(maybe_be_folder.name) 21 | return result 22 | 23 | def load_session(self, session_name: str) -> LazyLoadedSession: 24 | self.ensure_recording_directory_created() 25 | load_from = self.recording_directory.joinpath(session_name) 26 | session = PersistedSession.load_from_directory(load_from) 27 | return session 28 | 29 | def new_session(self, session_name) -> PersistedSession: 30 | self.ensure_recording_directory_created() 31 | session_directory = self.create_session_directory(session_name) 32 | return PersistedSession(session_directory) 33 | 34 | def create_session_directory(self, session_name): 35 | session_directory = self.recording_directory.joinpath(session_name) 36 | self.ensure_directory_created(session_directory) 37 | return session_directory 38 | 39 | def ensure_recording_directory_created(self): 40 | self.ensure_directory_created(self.recording_directory) 41 | 42 | def ensure_directory_created(self, directory: Path): 43 | if not directory.exists(): 44 | directory.mkdir(exist_ok=True) 45 | 46 | -------------------------------------------------------------------------------- /pycrunch_trace/tracing/call_stack.py: -------------------------------------------------------------------------------- 1 | import pycrunch_trace.events.method_enter as events 2 | import collections 3 | 4 | 5 | class CallStack: 6 | def __init__(self): 7 | self.stack = collections.deque() 8 | 9 | def enter_frame(self, execution_cursor: events.ExecutionCursor): 10 | parent_frame = self.get_parent_frame() 11 | # print(f"{execution_cursor.file}:{execution_cursor.line} -> {parent_frame} ") 12 | frame = events.StackFrame.new(parent_frame, execution_cursor) 13 | self.stack.append(frame) 14 | 15 | def get_parent_frame(self): 16 | if len(self.stack) > 0: 17 | return self.stack[-1] 18 | return None 19 | 20 | def new_cursor_in_current_frame(self, new_cursor: events.ExecutionCursor): 21 | stack_frame: events.StackFrame = self.top_level_frame_as_clone() 22 | stack_frame.line = new_cursor.line 23 | stack_frame.file = new_cursor.file 24 | stack_frame.function_name = new_cursor.function_name 25 | # todo this is probably dirty hack? 26 | # or just replacing last-known stack frame 27 | # todo what about performance ? 28 | if len(self.stack) > 0: 29 | self.stack.pop() 30 | self.stack.append(stack_frame) 31 | # self.stack[-1] = stack_frame 32 | else: 33 | # session just begin, not yet in any stack 34 | self.stack.append(stack_frame) 35 | 36 | 37 | def exit_frame(self): 38 | self.stack.pop() 39 | 40 | def top_level_frame_as_clone(self): 41 | current: events.StackFrame = self.current_frame() 42 | # print('top_level_frame_as_clone ->' + str(current)) 43 | return events.StackFrame.clone(current) 44 | # ??? 45 | # return current 46 | 47 | def current_frame(self): 48 | frame = self.get_parent_frame() 49 | return frame 50 | -------------------------------------------------------------------------------- /pycrunch_trace/server/perf.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Any 2 | 3 | from pycrunch_trace.file_system.human_readable_size import HumanReadableByteSize 4 | 5 | 6 | class SessionStats: 7 | total_bytes: int 8 | total_events: int 9 | session_id: str 10 | total_samples: int 11 | 12 | 13 | def __init__(self, session_id: str): 14 | self.session_id = session_id 15 | self.total_events = 0 16 | self.total_bytes = 0 17 | self.total_samples = 0 18 | 19 | def sample(self, events, buffer_size): 20 | size_per_event = buffer_size / events 21 | print(f'{self.session_id}:{self.total_samples} avg bytes: {HumanReadableByteSize(size_per_event)} ') 22 | self.total_events += events 23 | self.total_bytes += buffer_size 24 | self.total_samples += 1 25 | 26 | def average_event_size(self): 27 | if self.total_events == 0: 28 | return 0 29 | return self.total_bytes / self.total_events 30 | 31 | def __str__(self): 32 | return f'SessionStats({self.session_id}) average: {self.average_event_size()}; total: {self.total_events}; bytes:{self.total_bytes}' 33 | 34 | class PerformanceInsights: 35 | 36 | sessions: Dict[str, SessionStats] 37 | 38 | def __init__(self): 39 | self.sessions = dict() 40 | pass 41 | 42 | def sample(self, session_id, number_of_events, buffer_size): 43 | stats = self.get_or_create_session(session_id) 44 | 45 | stats.sample(number_of_events, buffer_size) 46 | 47 | def get_or_create_session(self, session_id): 48 | if session_id not in self.sessions: 49 | stats = SessionStats(session_id) 50 | self.sessions[session_id] = stats 51 | else: 52 | stats = self.sessions[session_id] 53 | return stats 54 | 55 | def average_event_size(self, session_id): 56 | stats = self.get_or_create_session(session_id) 57 | return stats.average_event_size() 58 | 59 | 60 | performance_insights = PerformanceInsights() 61 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # UI Overview 2 | 3 | ![PyTrace UI](https://hsto.org/webt/vp/im/xd/vpimxdvufmcmirahmktwpii79vw.png) 4 | 5 | 6 | # Quick start 7 | 8 | [Interactive Demo](https://app.pytrace.com/?open=v0.1-interactive-demo) 9 | 10 | [Documentation](https://pytrace.com) 11 | 12 | 13 | `pip install pycrunch-trace` 14 | 15 | Then, Add attribute `@trace` to the method you want to record 16 | 17 | ```python 18 | from pycrunch_trace.client.api import trace 19 | 20 | @trace 21 | def run(): 22 | some_code() 23 | ``` 24 | 25 | Or, alternatively, without decorator: 26 | 27 | ```python 28 | from pycrunch_trace.client.api import Trace 29 | 30 | tracer = Trace() 31 | tracer.start('recording_name') 32 | 33 | some_code() 34 | 35 | tracer.stop() 36 | ``` 37 | 38 | 39 | Optional session_name can be also passed to decorator: 40 | ```python 41 | @trace('my_custom_recording_name') 42 | ``` 43 | 44 | ### Specifying custom folders/files to exclude 45 | this will greatly speed-up profiler, however calls to the ignored directories will be ignored. 46 | 47 | Exclusion will be considered if absolute file path either `starts_with` or `ends_with` with given stop-list. 48 | 49 | ```python 50 | from pycrunch_trace.client.api import Trace 51 | 52 | t = Trace() 53 | t.start(additional_excludes=[ 54 | '/Users/gleb/.venvs/pycrunch_trace' 55 | '/Users/gleb/.pyenv/versions/3.6.15/', 56 | 'unwanted_file.py', 57 | ]) 58 | 59 | some_code() 60 | 61 | t.stop() 62 | 63 | ``` 64 | 65 | This is also possible via decorator: 66 | 67 | ```python 68 | from pycrunch_trace.client.api import trace 69 | 70 | @trace(additional_excludes=['/Users/gleb/.venvs/pycrunch_trace']) 71 | def run(): 72 | some_code() 73 | ``` 74 | 75 | 76 | 77 | 78 | Use web app for replaying recording: 79 | 80 | http://app.pytrace.com/ 81 | 82 | In case if you want to run UI locally, instead of using hosted version: 83 | [Link for web app source code](https://github.com/gleb-sevruk/pycrunch-tracing-webui) 84 | 85 | (Replays are not sent anywhere and processed entirely in-memory) 86 | -------------------------------------------------------------------------------- /pycrunch_trace/client/networking/strategies/native_write_strategy.pyx: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from pycrunch_trace.client.networking.commands import EventsSlice, FileContentSlice 4 | from pycrunch_trace.client.networking.strategies.abstract_strategy import AbstractRecordingStrategy 5 | 6 | import pyximport 7 | 8 | from pycrunch_trace.events.file_contents_in_protobuf import FileContentsInProtobuf 9 | from pycrunch_trace.file_system import tags 10 | 11 | pyximport.install() 12 | from pycrunch_trace.events.native_event_buffer_in_protobuf import NativeEventBufferInProtobuf 13 | 14 | from pycrunch_trace.server.incoming_traces import incoming_traces 15 | from pycrunch_trace.server.trace_persistance import TracePersistence 16 | 17 | 18 | class NativeLocalRecordingStrategy(AbstractRecordingStrategy): 19 | def __init__(self): 20 | self.persistence = TracePersistence() 21 | pass 22 | 23 | def prepare(self): 24 | # nothing to prepare here 25 | pass 26 | 27 | def recording_start(self, session_id: str): 28 | incoming_traces.trace_will_start(session_id) 29 | self.persistence.initialize_file(session_id) 30 | 31 | def recording_stop(self, session_id: str, files_included: List[str], files_excluded: List[str]): 32 | # write json metadata 33 | 34 | 35 | self.persistence.recording_complete(session_id, files_included, files_excluded) 36 | 37 | def recording_slice(self, x: EventsSlice): 38 | incoming_traces.did_receive_more_events(x.session_id, len(x.events)) 39 | bytes_to_disk = NativeEventBufferInProtobuf(x.events, x.files).as_bytes() 40 | 41 | self.persistence.flush_chunk(x.session_id, tags.TRACE_TAG_EVENTS, bytes_to_disk) 42 | 43 | def files_slice(self, x: FileContentSlice): 44 | bytes_to_disk = FileContentsInProtobuf(x.files).as_bytes() 45 | 46 | self.persistence.update_file_header_files_section(x.session_id, len(bytes_to_disk)) 47 | self.persistence.flush_chunk(x.session_id, tags.TRACE_TAG_FILES, bytes_to_disk) 48 | 49 | 50 | 51 | 52 | 53 | def clean(self): 54 | # nothing to clean 55 | pass -------------------------------------------------------------------------------- /pycrunch_trace/tracing/simulation/test_sim.py: -------------------------------------------------------------------------------- 1 | from pycrunch.insights import trace 2 | 3 | import pycrunch_trace.tracing.simulation.models as sim 4 | from pycrunch_trace.serialization import to_string 5 | from pycrunch_trace.session.snapshot import snapshot 6 | from pycrunch_trace.tracing.simple_tracer import SimpleTracer 7 | 8 | absolute_path = '/pycrunch_tracer/samples/module_a.py' 9 | 10 | def test_sim1(): 11 | def code_method() -> sim.Code: 12 | code = sim.Code() 13 | code.co_filename = absolute_path 14 | code.co_name = 'some_method' 15 | code.co_argcount = 1 16 | code.co_argcount = 1 17 | return code 18 | 19 | method = code_method() 20 | 21 | def get_frame(line: int): 22 | frame = sim.Frame() 23 | frame.f_lineno = line 24 | frame.f_code = method 25 | frame.f_locals = dict(some_number=1) 26 | return frame 27 | event_buffer = [] 28 | sut = SimpleTracer(event_buffer, 'testing',, 29 | sut.simple_tracer(get_frame(2), sim.EventKeys.call, None) 30 | sut.simple_tracer(get_frame(3), sim.EventKeys.line, None) 31 | sut.simple_tracer(get_frame(4), sim.EventKeys.line, None) 32 | 33 | sut.simple_tracer(get_frame(2), sim.EventKeys.call, None) 34 | sut.simple_tracer(get_frame(3), sim.EventKeys.line, None) 35 | sut.simple_tracer(get_frame(2), sim.EventKeys.call, None) 36 | sut.simple_tracer(get_frame(3), sim.EventKeys.line, None) 37 | sut.simple_tracer(get_frame(4), sim.EventKeys.line, None) 38 | sut.simple_tracer(get_frame(4), sim.EventKeys.event_return, None) 39 | sut.simple_tracer(get_frame(4), sim.EventKeys.line, None) 40 | sut.simple_tracer(get_frame(4), sim.EventKeys.event_return, None) 41 | 42 | sut.simple_tracer(get_frame(5), sim.EventKeys.line, None) 43 | sut.simple_tracer(get_frame(6), sim.EventKeys.line, None) 44 | sut.simple_tracer(get_frame(6), sim.EventKeys.event_return, 6) 45 | 46 | coe = sut.simulation.simulated_code() 47 | trace(coe=coe) 48 | trace(sixt=to_string(event_buffer[6])) 49 | trace(buffer=to_string(event_buffer)) 50 | snapshot.save('a', event_buffer) 51 | # trace(x=x) 52 | pass 53 | 54 | 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /pycrunch_trace/file_system/chunked_trace.py: -------------------------------------------------------------------------------- 1 | import io 2 | import struct 3 | from pathlib import Path 4 | 5 | from pycrunch_trace.proto import message_pb2 6 | 7 | 8 | class ChunkedTrace: 9 | # size in bytes; this return 4 on my machine 10 | header_size = struct.calcsize("i") 11 | 12 | def __init__(self, filename: Path): 13 | self.filename = filename 14 | 15 | 16 | def events(self) -> list: 17 | file_map = dict() 18 | entire_session = message_pb2.TraceSession() 19 | events_so_far = 0 20 | with io.FileIO(self.filename, 'r') as file_to_read: 21 | while True: 22 | header_bytes = file_to_read.read(ChunkedTrace.header_size) 23 | if len(header_bytes) <= 0: 24 | print(f' -- Read to end') 25 | 26 | break 27 | next_chunk_length = struct.unpack('i', header_bytes)[0] 28 | print(f'next_chunk_length {next_chunk_length}') 29 | 30 | read_bytes = file_to_read.read(next_chunk_length) 31 | interrim = message_pb2.TraceSession() 32 | try: 33 | interrim.ParseFromString(read_bytes) 34 | except Exception as eeeeeee: 35 | print(eeeeeee) 36 | raise 37 | 38 | for stack_frame in interrim.stack_frames: 39 | entire_session.stack_frames.append(stack_frame) 40 | print(f'total stack_frames {len(entire_session.stack_frames)}') 41 | events_so_far += len(interrim.events) 42 | print(f'total events {events_so_far}') 43 | for event in interrim.events: 44 | entire_session.events.append(event) 45 | 46 | for f in interrim.files: 47 | file_map[f.file] = f.id 48 | 49 | if events_so_far > 10000000: 50 | break 51 | for (file, file_id) in file_map.items(): 52 | result_file = message_pb2.File() 53 | result_file.id = file_id 54 | result_file.file = file 55 | entire_session.files.append(result_file) 56 | 57 | return entire_session 58 | -------------------------------------------------------------------------------- /pycrunch_trace/tests/test_file_chunks.py: -------------------------------------------------------------------------------- 1 | import io 2 | import os 3 | import struct 4 | from pathlib import Path 5 | 6 | from pycrunch_trace.file_system.persisted_session import PersistedSession 7 | from pycrunch_trace.file_system.session_store import SessionStore 8 | 9 | 10 | def test_write(): 11 | x = SessionStore() 12 | x.ensure_recording_directory_created() 13 | directory__joinpath = Path(x.recording_directory).joinpath('session_fake') 14 | x.ensure_directory_created(directory__joinpath) 15 | 16 | target_file = directory__joinpath.joinpath(PersistedSession.chunked_recording_filename) 17 | os.remove(target_file) 18 | 19 | write_once(target_file, 1) 20 | write_once(target_file, 2) 21 | write_once(target_file, 3) 22 | 23 | def test_read(): 24 | x = SessionStore() 25 | x.ensure_recording_directory_created() 26 | directory__joinpath = Path(x.recording_directory).joinpath('session_fake') 27 | x.ensure_directory_created(directory__joinpath) 28 | 29 | header_size = struct.calcsize("i") 30 | target_file = directory__joinpath.joinpath(PersistedSession.chunked_recording_filename) 31 | with io.FileIO(target_file, 'r') as file_to_read: 32 | while True: 33 | header_bytes = file_to_read.read(header_size) 34 | if len(header_bytes) <= 0: 35 | print(f' -- Read to end') 36 | 37 | break 38 | next_chunk_length = struct.unpack('i', header_bytes)[0] 39 | print(f'next_chunk_length {next_chunk_length}') 40 | read_bytes = file_to_read.read(next_chunk_length) 41 | result = read_bytes.decode('utf-8') 42 | print(f'result {result}') 43 | 44 | # 45 | # write_once(target_file, 1) 46 | # write_once(target_file, 2) 47 | # write_once(target_file, 3) 48 | 49 | 50 | def write_once(target_file, index): 51 | target_mode = 'a' 52 | if not target_file.exists(): 53 | target_mode = 'w' 54 | with io.FileIO(target_file, target_mode) as file_to_write: 55 | bytes_to_write = b'testing_text' + str(index).encode('utf-8') 56 | 57 | length_of_message = len(bytes_to_write) 58 | header_bytes = struct.pack("i", length_of_message) 59 | 60 | file_to_write.write(header_bytes + bytes_to_write) 61 | -------------------------------------------------------------------------------- /pycrunch_trace/filters/CustomFileFilter.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import yaml 4 | 5 | from . import AbstractFileFilter 6 | from ..oop import AbstractFile 7 | 8 | 9 | class CustomFileFilter(AbstractFileFilter): 10 | exclusions: tuple 11 | _trace_variables: bool 12 | profile_file: AbstractFile 13 | _loaded: bool 14 | 15 | 16 | def __init__(self, profile_file: AbstractFile): 17 | self.profile_file = profile_file 18 | self._loaded = False 19 | self._trace_variables = True 20 | self.exclusions = () 21 | 22 | def all_exclusions(self): 23 | self._ensure_loaded() 24 | return list(self.exclusions) 25 | 26 | def should_trace(self, filename: str): 27 | self._ensure_loaded() 28 | if filename.startswith(self.exclusions) or filename.endswith(self.exclusions): 29 | # print('should_trace: false - filename= '+filename) 30 | return False 31 | return True 32 | 33 | def _ensure_loaded(self): 34 | if self._loaded: 35 | return 36 | 37 | self._load() 38 | self._loaded = True 39 | 40 | def _load(self): 41 | all = yaml.load(self.profile_file.as_bytes(), Loader=yaml.FullLoader) 42 | self.load_exclusions(all) 43 | self.load_variables(all) 44 | 45 | def load_exclusions(self, all): 46 | tmp = set() 47 | exclusions = all.get('exclusions') 48 | if exclusions: 49 | for e in exclusions: 50 | tmp.add(e) 51 | self.exclusions = tuple(tmp) 52 | 53 | def add_additional_exclusions(self, additional_excludes: List[str]): 54 | if len(additional_excludes) <= 0: 55 | return 56 | exclusions_copy = set(self.exclusions) 57 | for exclude in additional_excludes: 58 | exclusions_copy.add(exclude) 59 | 60 | self.exclusions = tuple(exclusions_copy) 61 | 62 | def load_variables(self, all): 63 | trace_vars = all.get('trace_variables') 64 | if isinstance(trace_vars, bool): 65 | self._trace_variables = trace_vars 66 | if not self._trace_variables: 67 | print('!! PyCrunch - Variables will not be recorded in session') 68 | 69 | def should_record_variables(self) -> bool: 70 | self._ensure_loaded() 71 | return self._trace_variables 72 | 73 | -------------------------------------------------------------------------------- /pycrunch_trace/server/chunks_ordering.py: -------------------------------------------------------------------------------- 1 | from queue import Queue 2 | from typing import Dict, Any 3 | 4 | 5 | class PyCrunchTraceException(Exception): 6 | pass 7 | 8 | 9 | class PyCrunchTraceServerException(PyCrunchTraceException): 10 | pass 11 | 12 | 13 | class ReceivedChunks: 14 | def __init__(self, session_id: str): 15 | self.session_id = session_id 16 | self.received_chunks = set() 17 | 18 | def did_receive_chunk(self, chunk_number): 19 | self.throw_if_chunk_out_of_order(chunk_number) 20 | self.received_chunks.add(chunk_number) 21 | 22 | def throw_if_chunk_out_of_order(self, chunk_number): 23 | self.throw_if_chunk_already_received(chunk_number) 24 | if len(self.received_chunks) == 0: 25 | self.throw_if_first_chunk_lost(chunk_number) 26 | else: 27 | self.throw_if_previous_chunk_lost(chunk_number) 28 | 29 | def throw_if_previous_chunk_lost(self, chunk_number): 30 | current_max = max(self.received_chunks) 31 | if chunk_number - 1 > current_max: 32 | raise PyCrunchTraceServerException(f'{self.session_id}: received {chunk_number} but {current_max} is expected for') 33 | 34 | def throw_if_first_chunk_lost(self, chunk_number): 35 | if chunk_number != 1: 36 | raise PyCrunchTraceServerException(f'{self.session_id}: received {chunk_number} but expected chunk #1') 37 | 38 | def throw_if_chunk_already_received(self, chunk_number): 39 | if chunk_number in self.received_chunks: 40 | raise PyCrunchTraceServerException(f'already received {chunk_number} for {self.session_id}') 41 | 42 | 43 | class ChunksOrdering: 44 | order_by_session: Dict[str, ReceivedChunks] 45 | 46 | def __init__(self): 47 | self.order_by_session = dict() 48 | self.history = Queue(maxsize=200) 49 | 50 | def session_will_start(self, session_id): 51 | self.order_by_session[session_id] = ReceivedChunks(session_id) 52 | 53 | def did_receive_chunk(self, session_id, chunk_number): 54 | if session_id not in self.order_by_session: 55 | raise PyCrunchTraceServerException(f'{session_id}: not found') 56 | 57 | self.order_by_session[session_id].did_receive_chunk(chunk_number) 58 | 59 | def session_will_finish(self, session_id): 60 | self.history.put_nowait(self.order_by_session[session_id]) 61 | del self.order_by_session[session_id] 62 | 63 | 64 | chunks_ordering: ChunksOrdering = ChunksOrdering() 65 | -------------------------------------------------------------------------------- /pycrunch_trace/server/tests/test_chunks_ordering.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from pycrunch_trace.server.chunks_ordering import ChunksOrdering, PyCrunchTraceServerException 4 | 5 | 6 | def test_add_chunk_works(): 7 | sut = ChunksOrdering() 8 | sut.session_will_start(session_id='s1') 9 | sut.did_receive_chunk(session_id='s1', chunk_number=1) 10 | 11 | assert 1 in sut.order_by_session['s1'].received_chunks 12 | 13 | def test_duplicate_chunk_raises(): 14 | with pytest.raises(PyCrunchTraceServerException): 15 | sut = ChunksOrdering() 16 | sut.session_will_start(session_id='s1') 17 | sut.did_receive_chunk(session_id='s1', chunk_number=1) 18 | sut.did_receive_chunk(session_id='s1', chunk_number=2) 19 | sut.did_receive_chunk(session_id='s1', chunk_number=2) 20 | 21 | def test_lost_chunk_raises(): 22 | with pytest.raises(PyCrunchTraceServerException): 23 | sut = ChunksOrdering() 24 | sut.session_will_start(session_id='s1') 25 | sut.did_receive_chunk(session_id='s1', chunk_number=1) 26 | sut.did_receive_chunk(session_id='s1', chunk_number=2) 27 | # lost 6 frames 28 | sut.did_receive_chunk(session_id='s1', chunk_number=8) 29 | 30 | def test_first_chunk_lost(): 31 | with pytest.raises(PyCrunchTraceServerException): 32 | sut = ChunksOrdering() 33 | sut.session_will_start(session_id='s1') 34 | sut.did_receive_chunk(session_id='s1', chunk_number=2) 35 | 36 | def test_happy_case_no_errors(): 37 | sut = ChunksOrdering() 38 | sut.session_will_start(session_id='s1') 39 | sut.did_receive_chunk(session_id='s1', chunk_number=1) 40 | sut.did_receive_chunk(session_id='s1', chunk_number=2) 41 | sut.did_receive_chunk(session_id='s1', chunk_number=3) 42 | 43 | 44 | def test_multiple_sessions(): 45 | sut = ChunksOrdering() 46 | sut.session_will_start(session_id='s1') 47 | sut.did_receive_chunk(session_id='s1', chunk_number=1) 48 | sut.session_will_start(session_id='s2') 49 | sut.did_receive_chunk(session_id='s2', chunk_number=1) 50 | sut.did_receive_chunk(session_id='s2', chunk_number=2) 51 | sut.session_will_start(session_id='s3') 52 | sut.did_receive_chunk(session_id='s3', chunk_number=1) 53 | sut.did_receive_chunk(session_id='s3', chunk_number=2) 54 | 55 | 56 | def test_chunk_after_session_finished(): 57 | with pytest.raises(PyCrunchTraceServerException): 58 | sut = ChunksOrdering() 59 | sut.session_will_start('s1') 60 | sut.did_receive_chunk('s1', 1) 61 | sut.session_will_finish('s1') 62 | sut.did_receive_chunk('s1', 1) 63 | -------------------------------------------------------------------------------- /pycrunch_trace/main.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import io 3 | 4 | # import aiohttp_debugtoolbar 5 | import logging.config 6 | 7 | from pathlib import Path 8 | 9 | import yaml 10 | from aiohttp import web 11 | import socketio 12 | from pycrunch.api.shared import sio 13 | from pycrunch_trace.config import config 14 | 15 | import cgitb 16 | cgitb.enable(format='text') 17 | 18 | 19 | 20 | 21 | 22 | package_directory = Path(__file__).parent 23 | print(package_directory) 24 | engine_directory = package_directory.parent 25 | config.set_package_directory(package_directory) 26 | config.set_engine_directory(engine_directory) 27 | configuration_yaml_ = package_directory.joinpath('logging-configuration.yaml') 28 | print(configuration_yaml_) 29 | with open(configuration_yaml_, 'r') as f: 30 | logging.config.dictConfig(yaml.safe_load(f.read())) 31 | 32 | 33 | import logging 34 | 35 | logger = logging.getLogger(__name__) 36 | 37 | 38 | 39 | import sys 40 | 41 | if sys.platform == 'win32': 42 | policy = asyncio.get_event_loop_policy() 43 | policy._loop_factory = asyncio.ProactorEventLoop 44 | 45 | async def shutdown(loop, signal=None): 46 | print('PTrace Server: shutdown') 47 | """Cleanup tasks tied to the service's shutdown.""" 48 | if signal: 49 | logging.info(f"Received exit signal {signal.name}...") 50 | logging.info("Closing database connections") 51 | 52 | def handle_exception(loop, context): 53 | # context["message"] will always be there; but context["exception"] may not 54 | print('PTrace Server: !!! Handling Exception') 55 | msg = context.get("exception", context["message"]) 56 | print('error was : ' + str(msg)) 57 | 58 | logging.error(f"Caught exception: {msg}") 59 | logging.info("Shutting down...") 60 | # asyncio.create_task(shutdown(loop)) 61 | 62 | def run(): 63 | loop = asyncio.get_event_loop() 64 | loop.set_debug(True) 65 | loop.set_exception_handler(handle_exception) 66 | 67 | app = web.Application() 68 | # aiohttp_debugtoolbar.setup(app) 69 | 70 | 71 | # keep import outside main 72 | from pycrunch_trace.server.shared import tracer_socket_server 73 | 74 | # attach sio events 75 | # noinspection PyUnresolvedReferences 76 | import pycrunch_trace.server.recording_server_websocket 77 | 78 | tracer_socket_server.attach(app) 79 | 80 | 81 | async def index(request): 82 | """Serve the client-side application.""" 83 | with io.open(config.absolute_path, 'r') as f: 84 | lines = f.read() 85 | return web.Response(text=lines, content_type='application/json') 86 | 87 | 88 | # app.router.add_static('/static', 'static') 89 | app.router.add_get('/', index) 90 | web.run_app(app) 91 | 92 | if __name__ == '__main__': 93 | run() -------------------------------------------------------------------------------- /pycrunch_trace/tracing/simulator_sink.py: -------------------------------------------------------------------------------- 1 | from typing import List, Any, Optional 2 | 3 | from pycrunch_trace.tracing.simulation import models 4 | from pycrunch_trace.tracing.simulation.models import Frame 5 | 6 | 7 | class SimulationEvent: 8 | frame: Frame 9 | event: str 10 | arg: Optional[Any] 11 | 12 | def __init__(self, frame: models.Frame, event: str, arg: Optional[Any]): 13 | self.arg = arg 14 | self.event = event 15 | self.frame = frame 16 | 17 | 18 | class SimulationEventAsCode: 19 | origin: SimulationEvent 20 | 21 | def __init__(self, origin: SimulationEvent): 22 | self.origin = origin 23 | 24 | def as_python_code(self, index: int): 25 | x = f""" 26 | def create_event_{index}(): 27 | code_clone = models.Code() 28 | code_clone.co_name = '{self.origin.frame.f_code.co_name}' 29 | code_clone.co_filename = '{self.origin.frame.f_code.co_filename}' 30 | code_clone.co_argcount = {self.origin.frame.f_code.co_argcount} 31 | 32 | sim_frame = models.Frame() 33 | sim_frame.f_lineno = {self.origin.frame.f_lineno} 34 | sim_frame.f_locals = {self.origin.frame.f_locals} 35 | sim_frame.f_code = code_clone 36 | evt = SimulationEvent(sim_frame, '{self.origin.event}', {self.origin.arg}) 37 | 38 | return evt 39 | 40 | """ 41 | return x 42 | 43 | class DisabledSimulatorSink(object): 44 | def save_for_simulator(self, frame: models.Frame, event: str, arg): 45 | pass 46 | 47 | class SimulatorSink: 48 | current_index: int 49 | simulation: List[str] 50 | 51 | def __init__(self): 52 | self.simulation = [] 53 | self.current_index = 1 54 | 55 | def simulated_code(self): 56 | imports_code = """ 57 | from pycrunch_trace.simulation import models 58 | from pycrunch_trace.tracing.simulator_sink import SimulationEvent 59 | 60 | """ 61 | 62 | events = ''.join(self.simulation) 63 | 64 | test_code = """ 65 | def test_simulated(): 66 | events = []""" 67 | for x in range(1, self.current_index): 68 | test_code += f'\n events.append(create_event_{x}())' 69 | return imports_code + events + test_code 70 | 71 | def save_for_simulator(self, frame: models.Frame, event: str, arg): 72 | code = frame.f_code 73 | 74 | code_clone = models.Code() 75 | code_clone.co_name = code.co_name 76 | code_clone.co_filename = code.co_filename 77 | code_clone.co_argcount = code.co_argcount 78 | 79 | clone = models.Frame() 80 | clone.f_lineno = frame.f_lineno 81 | clone.f_locals = frame.f_locals 82 | clone.f_code = code_clone 83 | self.simulation.append(SimulationEventAsCode(SimulationEvent(clone, event, arg)).as_python_code(self.current_index)) 84 | self.current_index += 1 85 | -------------------------------------------------------------------------------- /pycrunch_trace/events/event_buffer_in_protobuf.py: -------------------------------------------------------------------------------- 1 | from typing import Union, Dict 2 | 3 | from pycrunch_trace.events.base_event import Event 4 | from pycrunch_trace.proto import message_pb2 5 | 6 | 7 | class EventBufferInProtobuf: 8 | files: Dict[str, int] 9 | 10 | def __init__(self, event_buffer, files: Dict[str, int]): 11 | self.files = files 12 | self.event_buffer = event_buffer 13 | 14 | def as_bytes(self): 15 | session = message_pb2.TraceSession() 16 | for e in self.event_buffer: 17 | evt = self.pb_event_from_py(e) 18 | 19 | session.events.append(evt) 20 | 21 | frame = message_pb2.StackFrame() 22 | if e.stack: 23 | frame.id = e.stack.id 24 | 25 | 26 | if not e.stack.line: 27 | frame.line = -1 28 | frame.file = '' 29 | frame.function_name = '??' 30 | print('cannot find line for stack') 31 | print(e.stack) 32 | else: 33 | frame.line = e.stack.line 34 | frame.file = e.stack.file 35 | frame.function_name = e.stack.function_name 36 | 37 | if e.stack and e.stack.parent: 38 | frame.parent_id = e.stack.parent.id 39 | else: 40 | frame.parent_id = -1 41 | else: 42 | frame.id = 0 43 | 44 | session.stack_frames.append(frame) 45 | 46 | self.add_files_to_pb_envelope(session) 47 | 48 | return session.SerializeToString() 49 | 50 | def add_files_to_pb_envelope(self, session): 51 | for (filename, file_id) in self.files.items(): 52 | current_file = message_pb2.File() 53 | # print(f'file_id: {file_id}') 54 | # print(f'filename: {filename}') 55 | current_file.id = file_id 56 | current_file.file = filename 57 | session.files.append(current_file) 58 | 59 | def pb_event_from_py(self, e: Event): 60 | evt = message_pb2.TraceEvent() 61 | evt.event_name = e.event_name 62 | evt.ts = e.ts 63 | if e.event_name == 'line' or e.event_name == 'method_exit': 64 | for key, value in e.locals.variables.items(): 65 | pb_var = message_pb2.Variable() 66 | pb_var.name = key 67 | pb_var.value = str(value) 68 | evt.locals.variables.append(pb_var) 69 | 70 | if e.event_name == 'method_enter': 71 | for key, value in e.input_variables.variables.items(): 72 | pb_var = message_pb2.Variable() 73 | pb_var.name = key 74 | pb_var.value = str(value) 75 | evt.input_variables.variables.append(pb_var) 76 | 77 | if e.event_name == 'method_exit': 78 | for key, value in e.return_variables.variables.items(): 79 | pb_var = message_pb2.Variable() 80 | pb_var.name = key 81 | pb_var.value = str(value) 82 | evt.return_variables.variables.append(pb_var) 83 | 84 | evt.cursor.file = e.cursor.file 85 | evt.cursor.line = e.cursor.line 86 | evt.cursor.function_name = e.cursor.function_name 87 | # ptask.task.MergeFrom(task) 88 | if e.stack: 89 | evt.stack_id = e.stack.id 90 | return evt 91 | -------------------------------------------------------------------------------- /pycrunch_trace/events/method_enter.py: -------------------------------------------------------------------------------- 1 | from typing import TypeVar 2 | 3 | from .base_event import Event 4 | from ..filters import can_trace_type 5 | 6 | 7 | class ExecutionCursor: 8 | function_name: str 9 | file: int 10 | line: int 11 | 12 | def __init__(self, file: int, line: int, function_name: str): 13 | self.function_name = function_name 14 | self.file = file 15 | self.line = line 16 | 17 | class StackFrameIds: 18 | last_id: int 19 | def __init__(self): 20 | self.last_id = 1 21 | 22 | def new_id(self): 23 | ret = self.last_id 24 | self.last_id += 1 25 | return ret 26 | 27 | stack_ids = StackFrameIds() 28 | 29 | class StackFrame: 30 | # parent: StackFrame 31 | def __init__(self, parent, file: id, line: int, function_name: str): 32 | self.parent = parent 33 | self.file = file 34 | self.line = line 35 | self.id = stack_ids.new_id() 36 | self.function_name = function_name 37 | 38 | def as_id(self): 39 | return self.id 40 | 41 | 42 | @classmethod 43 | def new(cls, parent, execution_cursor: ExecutionCursor): 44 | return StackFrame(parent, execution_cursor.file, execution_cursor.line, execution_cursor.function_name) 45 | 46 | @classmethod 47 | def clone(cls, origin): 48 | if not origin: 49 | return StackFrame.empty() 50 | return StackFrame(origin.parent, origin.file, origin.line, origin.function_name) 51 | 52 | @classmethod 53 | def empty(cls): 54 | return StackFrame(None, None, None, None) 55 | 56 | def __str__(self): 57 | return f'{self.file}:{self.function_name}:{self.line} -> \n\t {self.parent}' 58 | 59 | 60 | 61 | 62 | class Variables: 63 | variables: dict 64 | 65 | def __init__(self): 66 | # self.variables = None 67 | # return 68 | self.variables = dict() 69 | 70 | def push_variable(self, name, value): 71 | self.variables[name] = self.ensure_safe_for_serialization(value) 72 | 73 | def ensure_safe_for_serialization(self, value): 74 | # return 'a' 75 | # todo is this slowdown? 76 | if not can_trace_type(value): 77 | value = str(type(value)) 78 | if type(value) == dict: 79 | return value.copy() 80 | return value 81 | 82 | 83 | class MethodEnterEvent(Event): 84 | cursor: ExecutionCursor 85 | input_variables: Variables 86 | stack: StackFrame 87 | 88 | def __init__(self, cursor: ExecutionCursor, stack: StackFrame, ts: float): 89 | self.cursor = cursor 90 | self.input_variables = Variables() 91 | self.stack = stack 92 | self.event_name = 'method_enter' 93 | self.ts = ts 94 | 95 | 96 | class LineExecutionEvent(Event): 97 | cursor: ExecutionCursor 98 | locals: Variables 99 | stack: StackFrame 100 | 101 | def __init__(self, cursor, stack: StackFrame, ts: float): 102 | self.cursor = cursor 103 | self.locals = Variables() 104 | self.event_name = 'line' 105 | self.stack = stack 106 | self.ts = ts 107 | 108 | 109 | 110 | 111 | 112 | class MethodExitEvent(Event): 113 | cursor: ExecutionCursor 114 | return_variables: Variables 115 | locals: Variables 116 | stack: StackFrame 117 | 118 | def __init__(self, cursor: ExecutionCursor, stack: StackFrame, ts: float): 119 | self.cursor = cursor 120 | self.return_variables = Variables() 121 | self.locals = Variables() 122 | self.event_name = 'method_exit' 123 | self.stack = stack 124 | self.ts = ts 125 | 126 | -------------------------------------------------------------------------------- /pycrunch_trace/events/size_prediction.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from . import base_event 4 | from .method_enter import MethodEnterEvent, MethodExitEvent, LineExecutionEvent 5 | from ..file_system.human_readable_size import HumanReadableByteSize 6 | from ..file_system.session_store import SessionStore 7 | 8 | import pickle 9 | 10 | 11 | def count_every_element(self, cleanup_function= None): 12 | accum = 0 13 | for e in self.buffer: 14 | if cleanup_function: 15 | cleanup_function(e) 16 | bytes_ = pickle.dumps(e) 17 | accum += len(bytes_) 18 | return accum 19 | 20 | class SizeWithoutStack: 21 | def __init__(self, buffer): 22 | self.buffer = buffer 23 | 24 | def size(self): 25 | return count_every_element(self, self.clean_up_stack) 26 | 27 | def clean_up_stack(self, e): 28 | e.stack = None 29 | 30 | 31 | class SizeOriginal: 32 | def __init__(self, buffer): 33 | self.buffer = buffer 34 | 35 | def size(self): 36 | return count_every_element(self) 37 | 38 | def clean_up_stack(self, e): 39 | e.stack = None 40 | 41 | 42 | class SizeWithoutVariables: 43 | def __init__(self, buffer): 44 | self.buffer = buffer 45 | 46 | def size(self): 47 | return count_every_element(self, self.clean_up_vars) 48 | 49 | def clean_up_vars(self, e): 50 | if isinstance(e, MethodEnterEvent): 51 | e.input_variables = None 52 | if isinstance(e, MethodExitEvent): 53 | e.return_variables = None 54 | e.locals = None 55 | if isinstance(e, LineExecutionEvent): 56 | e.locals = None 57 | 58 | 59 | class SizeWithoutCursor: 60 | def __init__(self, buffer): 61 | self.buffer = buffer 62 | 63 | def size(self): 64 | return count_every_element(self, self.clean_up_cursor) 65 | 66 | def clean_up_cursor(self, e): 67 | e.cursor = None 68 | 69 | class SizeBreakdown: 70 | event_buffer: List[base_event.Event] 71 | 72 | @staticmethod 73 | def load_from_session(): 74 | sess = SessionStore().load_session('request_exce') 75 | sess.load_metadata() 76 | print(f'metadata thinks size is: {sess.metadata.file_size_on_disk}') 77 | print() 78 | 79 | orig = SizeOriginal(sess.load_buffer()) 80 | real_size = orig.size() 81 | SizeBreakdown.print_size('real size', real_size) 82 | 83 | total_bytes_so_far = SizeWithoutStack(sess.load_buffer()) 84 | without_stack = total_bytes_so_far.size() 85 | SizeBreakdown.print_size('without stack', without_stack) 86 | 87 | without_variables = SizeWithoutVariables(sess.load_buffer()) 88 | without_variables_size = without_variables.size() 89 | SizeBreakdown.print_size('without variables', without_variables_size) 90 | 91 | without_cursor = SizeWithoutCursor(sess.load_buffer()) 92 | without_cursor_size = without_cursor.size() 93 | SizeBreakdown.print_size('without cursor', without_cursor_size) 94 | 95 | cursor = SizeWithoutCursor(sess.load_buffer()) 96 | cursor.size() 97 | without_cursor_and_vars = SizeWithoutVariables(cursor.buffer) 98 | without_cursor_and_vars_size = without_cursor_and_vars.size() 99 | SizeBreakdown.print_size('without_cursor and vars', without_cursor_and_vars_size) 100 | 101 | print('matan:') 102 | 103 | 104 | # for i in range(100): 105 | # print(bugger[i].event_name) 106 | 107 | @staticmethod 108 | def print_size(prefix, real_size): 109 | print(f'{prefix}: {HumanReadableByteSize(real_size)} ({real_size})') -------------------------------------------------------------------------------- /pycrunch_trace/events/tests/test_size_prediction.py: -------------------------------------------------------------------------------- 1 | # from pycrunch_trace.events.size_prediction import SizeBreakdown 2 | 3 | 4 | # @pytest.mark.skip() 5 | # def test_1(): 6 | # x = SizeBreakdown.load_from_session() 7 | import random 8 | import string 9 | 10 | from pycrunch.insights import trace 11 | 12 | from pycrunch_trace.file_system.human_readable_size import HumanReadableByteSize 13 | 14 | 15 | def randomString(stringLength=10): 16 | """Generate a random string of fixed length """ 17 | letters = string.ascii_lowercase 18 | return ''.join(random.choice(letters) for i in range(stringLength)) 19 | 20 | 21 | class Repeat(object): 22 | def __init__(self, times): 23 | self.times = times 24 | 25 | def __str__(self): 26 | res = '' 27 | for i in range(self.times): 28 | res += '/' + randomString(20) 29 | return res 30 | 31 | 32 | def random_filename(): 33 | return 'users/' + randomString(5) + '/projects/' + randomString(20) + '/' + randomString(20) + Repeat(random.randint(1, 6)).__str__() + '.py' 34 | 35 | 36 | class NewEvent: 37 | event_names = ['method_enter', 'line', 'method_exit'] 38 | 39 | def __init__(self, filename): 40 | self.name = random.choice(OldEvent.event_names) 41 | self.line = random.randint(1, 1000) 42 | self.filename = filename 43 | self.function_name = randomString(random.randint(1,20)) 44 | 45 | def size(self): 46 | function_name_size = len(self.function_name) 47 | 48 | call_stack_length = 8 + 8 + 8 + function_name_size + 8 49 | method_name_size = len(self.name) 50 | method_name_size = 0 51 | return call_stack_length + method_name_size + 8 + 8 + function_name_size 52 | 53 | 54 | class OldEvent: 55 | event_names = ['method_enter', 'line', 'method_exit'] 56 | 57 | def __init__(self, filename): 58 | self.name = random.choice(OldEvent.event_names) 59 | self.line = random.randint(1, 1000) 60 | self.filename = filename 61 | self.function_name = randomString(random.randint(1,20)) 62 | 63 | def size(self): 64 | call_stack_length = 8 + 8 + len(self.name) + len(self.filename) 65 | return call_stack_length + len(self.name) + 8 + len(self.filename) + len(self.function_name) 66 | 67 | def test_old_size(): 68 | total_files = 255 69 | files_in_session = [] 70 | for x in range(total_files): 71 | files_in_session.append(random_filename()) 72 | # trace(files_in_session=files_in_session) 73 | total = 0 74 | for x in range(10000): 75 | total += size_of_random_event(files_in_session) 76 | # 10000 ='3.2 MB' 77 | # 100000 = 31 MB 78 | # 1 000 000 = 310 MB 79 | str__ = HumanReadableByteSize(total).__str__() 80 | print(str__) 81 | trace(total=str__) 82 | 83 | def test_new_size(): 84 | total_files = 255 85 | files_in_session = [] 86 | size_files = 0 87 | for x in range(total_files): 88 | 89 | filename = random_filename() 90 | size_files += len(files_in_session) 91 | files_in_session.append(filename) 92 | 93 | trace(size_files=size_files) 94 | 95 | # trace(files_in_session=files_in_session) 96 | total = 0 97 | for x in range(10000): 98 | total += size_of_new_random_event(files_in_session) 99 | # 10000 ='3.2 MB' 100 | # 100000 = 31 MB 101 | # 1 000 000 = 310 MB 102 | str__ = HumanReadableByteSize(total).__str__() 103 | print(str__) 104 | trace(total=str__) 105 | 106 | def size_of_random_event(files_in_session): 107 | return OldEvent(random.choice(files_in_session)).size() 108 | 109 | def size_of_new_random_event(files_in_session): 110 | return NewEvent(random.choice(files_in_session)).size() 111 | 112 | def test_232(): 113 | print('s') -------------------------------------------------------------------------------- /pycrunch_trace/events/native_event_buffer_in_protobuf.pyx: -------------------------------------------------------------------------------- 1 | from io import FileIO 2 | from typing import Union, Dict 3 | 4 | from pycrunch_trace.events.base_event import Event 5 | 6 | from pycrunch_trace.native.native_models cimport NativeCodeEvent, NativeVariable, NativeStackFrame 7 | 8 | from pycrunch_trace.proto import message_pb2 9 | from pycrunch_trace.tracing.file_map import FileMap 10 | 11 | cdef class NativeEventBufferInProtobuf: 12 | cdef object files 13 | cdef object event_buffer 14 | cdef object store_file_contents 15 | 16 | def __init__(self, event_buffer, files: Dict[str, int]): 17 | self.files = files 18 | self.event_buffer = event_buffer 19 | 20 | def as_bytes(self): 21 | cdef NativeCodeEvent e 22 | cdef object evt 23 | session = message_pb2.TraceSession() 24 | for e in self.event_buffer: 25 | evt = self.pb_event_from_py(e) 26 | 27 | session.events.append(evt) 28 | 29 | frame = message_pb2.StackFrame() 30 | if e.stack: 31 | 32 | frame.id = e.stack.id 33 | 34 | if not e.stack.cursor.line: 35 | frame.line = -1 36 | frame.file = '' 37 | frame.function_name = '??' 38 | else: 39 | frame.line = e.stack.cursor.line 40 | frame.file = e.stack.cursor.file 41 | frame.function_name = e.stack.cursor.function_name 42 | 43 | if e.stack and e.stack.parent: 44 | frame.parent_id = e.stack.parent.id 45 | else: 46 | frame.parent_id = -1 47 | else: 48 | frame.id = 0 49 | 50 | session.stack_frames.append(frame) 51 | 52 | self.add_files_to_pb_envelope(session) 53 | 54 | return session.SerializeToString() 55 | 56 | def add_files_to_pb_envelope(self, session): 57 | for (filename, file_id) in self.files.items(): 58 | current_file = message_pb2.File() 59 | # print(f'file_id: {file_id}') 60 | # print(f'filename: {filename}') 61 | current_file.id = file_id 62 | current_file.file = filename 63 | session.files.append(current_file) 64 | 65 | 66 | cdef pb_event_from_py(self, NativeCodeEvent e): 67 | cdef NativeVariable v 68 | evt = message_pb2.TraceEvent() 69 | evt.event_name = e.event_name 70 | evt.ts = e.ts 71 | if e.event_name == 'line': 72 | if e.locals is not None: 73 | for v in e.locals.variables: 74 | pb_var = message_pb2.Variable() 75 | pb_var.name = v.name 76 | pb_var.value = v.value 77 | evt.locals.variables.append(pb_var) 78 | 79 | if e.event_name == 'method_enter': 80 | if e.input_variables is not None: 81 | for v in e.input_variables.variables: 82 | pb_var = message_pb2.Variable() 83 | pb_var.name = v.name 84 | pb_var.value = v.value 85 | evt.input_variables.variables.append(pb_var) 86 | 87 | if e.event_name == 'method_exit': 88 | if e.return_variables is not None: 89 | for v in e.return_variables.variables: 90 | pb_var = message_pb2.Variable() 91 | pb_var.name = v.name 92 | pb_var.value = v.value 93 | evt.return_variables.variables.append(pb_var) 94 | 95 | evt.cursor.file = e.cursor.file 96 | evt.cursor.line = e.cursor.line 97 | evt.cursor.function_name = e.cursor.function_name 98 | # ptask.task.MergeFrom(task) 99 | if e.stack: 100 | evt.stack_id = e.stack.id 101 | else: 102 | evt.stack_id = -1 103 | return evt 104 | -------------------------------------------------------------------------------- /pycrunch_trace/client/api/trace.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import sys 3 | import uuid 4 | from pathlib import Path 5 | from typing import List 6 | 7 | from pycrunch_trace.client.command_buffer import ArrayCommandBuffer 8 | from pycrunch_trace.client.networking import event_queue 9 | from pycrunch_trace.filters import CustomFileFilter 10 | from pycrunch_trace.oop import File, Clock, SafeFilename 11 | from pycrunch_trace.tracing.inline_profiler import inline_profiler_instance 12 | 13 | import pyximport 14 | pyximport.install() 15 | from pycrunch_trace.native.native_tracer import NativeTracer 16 | 17 | from pycrunch_trace.tracing.simple_tracer import SimpleTracer 18 | 19 | 20 | class Trace: 21 | clock: Clock 22 | _tracer: SimpleTracer 23 | 24 | def __init__(self): 25 | self.default_host = 'http://0.0.0.0:8080' 26 | self.command_buffer = ArrayCommandBuffer() 27 | self.is_tracing = False 28 | self.session_name = None 29 | self._tracer = None 30 | # self._client: network_client.TracingClient = None 31 | self.clock = None 32 | self.host = None 33 | self.outgoingQueue = None 34 | 35 | def generate_session_name(self) -> str: 36 | iso_time = datetime.datetime.now().replace(microsecond=0).isoformat() 37 | return f'{iso_time}_{str(uuid.uuid4())[:5]}' 38 | 39 | def start(self, session_name: str = None, host: str = None, profile_name: str = None, additional_excludes: List[str] = None): 40 | 41 | if self.is_tracing: 42 | raise Exception('PyCrunch tracer ERROR: tracing already started') 43 | 44 | self.prepare_state(host, session_name) 45 | self.warn_if_another_tracing_set() 46 | 47 | if not profile_name: 48 | profile_name = 'default.profile.yaml' 49 | package_directory = Path(__file__).parent.parent.parent 50 | file_filter = CustomFileFilter(File(package_directory.joinpath('pycrunch-profiles', profile_name))) 51 | file_filter._ensure_loaded() 52 | 53 | if additional_excludes is not None: 54 | file_filter.add_additional_exclusions(additional_excludes) 55 | 56 | self.start_queue() 57 | 58 | self.clock = Clock() 59 | # todo maybe move command buffer to tracer? 60 | # self._tracer = SimpleTracer(self.command_buffer, self.session_name, f_filter, self.clock, self.outgoingQueue) 61 | # TODO windows test 62 | self._tracer = NativeTracer(self.session_name, self.outgoingQueue, file_filter) 63 | self.outgoingQueue.start() 64 | 65 | self.outgoingQueue.tracing_will_start(self.session_name) 66 | 67 | # also trace parent function 68 | sys._getframe().f_back.f_trace = self._tracer.simple_tracer 69 | 70 | sys.settrace(self._tracer.simple_tracer) 71 | 72 | self.is_tracing = True 73 | 74 | def start_queue(self): 75 | self.outgoingQueue = event_queue 76 | self.outgoingQueue.start() 77 | 78 | def warn_if_another_tracing_set(self): 79 | if sys.gettrace(): 80 | # there is already trace 81 | print('PyCrunch tracer WARNING:') 82 | print(' -- there is already trace function set. ') 83 | print(' -- continuing might result in errors ') 84 | 85 | def prepare_state(self, host, session_name): 86 | if not session_name: 87 | self.session_name = SafeFilename(self.generate_session_name()).__str__() 88 | else: 89 | self.session_name = SafeFilename(session_name).__str__() 90 | if host: 91 | self.host = host 92 | else: 93 | self.host = self.default_host 94 | 95 | def stop(self): 96 | sys.settrace(None) 97 | 98 | inline_profiler_instance.print_timings() 99 | # import pydevd_pycharm 100 | # pydevd_pycharm.settrace('localhost', port=44441, stdoutToServer=True, stderrToServer=True) 101 | print('tracing complete, saving results') 102 | self.is_tracing = False 103 | # snapshot.save('a', self.command_buffer) 104 | local = False 105 | # local = True 106 | if local: 107 | self._tracer.session.buffer_became_available(self.command_buffer) 108 | self._tracer.session.save() 109 | 110 | self._tracer.flush_outstanding_events() 111 | self._tracer.finalize() 112 | 113 | # self._tracer.session.save() 114 | 115 | 116 | # self._client.push_message(self._tracer.session) 117 | # self._client.disconnect() 118 | -------------------------------------------------------------------------------- /pycrunch_trace/demo/interactive_demo_01.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from time import sleep 3 | 4 | from pycrunch_trace.client.api import trace 5 | 6 | from pycrunch_trace.demo.interactive_demo_02 import method_in_another_file 7 | from pycrunch_trace.demo.interactive_demo_03 import show_me_how_to_navigate_using_graph 8 | from pycrunch_trace.demo.interactive_demo_04 import alternative_ways_to_trace 9 | 10 | 11 | @trace('interactive_demo') 12 | def method_you_want_to_trace(some_number: int, some_string: str): 13 | # Take a look at the inspector panel. 14 | # Variable you passed, are recorded there. 15 | print('Press down arrow on keyboard to step into next line') 16 | print('You can also use toolbar button [↓] near slider') 17 | 18 | print('Lets now change value of some_number.') 19 | some_number -= 1 20 | print('Notice that now, inspector recorded value to be 2.') 21 | print('Any time you can step to previous line using Up Arrow or [↑] button.') 22 | 23 | print('Now, to the function calls.') 24 | print('Press Right Arrow on keyboard or [→] button to step into the next method') 25 | # You can always go back using Left Arrow 26 | from_external_method = method_in_another_file(some_number) 27 | 28 | print("Press [G], if you haven't open graph yet, ") 29 | print("All toolbars can be accessed using settings button in the top right corner") 30 | 31 | print("Use [→] to `step into` next method") 32 | sleep(0.1) 33 | we_need_to_go_deeper() 34 | alternative_ways_to_trace() 35 | 36 | print('You can start this tutorial again by dragging slider to the beginning') 37 | print(' This is similar to scrolling through song in a music player or watching game replay') 38 | 39 | print("That's it. Start tracing now by installing:") 40 | print(" pip install pycrunch-trace") 41 | 42 | one_last_thing('keyboard shortcuts') 43 | 44 | return str(datetime.utcnow()) 45 | 46 | 47 | def we_need_to_go_deeper(): 48 | print("Sometimes insignificant call time makes it hard to navigate the graph") 49 | 50 | print("Lets take factorial for instance") 51 | sleep(0.1) 52 | 53 | print("Click at `we_need_to_go_deeper` method in the graph, holding [Command] key") 54 | # or (Control key on Windows) 55 | factorial = recursive_factorial(6) 56 | 57 | print("You can scope even deeper in call tree.") 58 | print(" By clicking with the mouse while holding [Command]") 59 | 60 | print("Entire call stack is drawn under the method you scoped to") 61 | 62 | print("You can exit from the scope same way:") 63 | print(" By holding [Command] and clicking on outer method") 64 | 65 | # Right arrow [→] to continue 66 | show_me_how_to_navigate_using_graph() 67 | 68 | 69 | def recursive_factorial(num: int): 70 | sleep(0.005) 71 | if num == 1: 72 | return num 73 | else: 74 | return num * recursive_factorial(num - 1) 75 | 76 | 77 | def one_last_thing(dummy_variable): 78 | sleep(0.1) 79 | print('Most useful keyboard shortcuts:') 80 | shortcuts = dict() 81 | shortcuts['[G]'] = 'Toggle Graph View' 82 | shortcuts['[i]'] = 'Toggle Inspector' 83 | 84 | # Moving forward in debugger: 85 | regular_debug = dict() 86 | regular_debug['[→]'] = 'Next Event' 87 | regular_debug['[↓]'] = 'Step Over; Ignoring method calls' 88 | regular_debug['[Shift]+[↓]'] = 'Step Out of method' 89 | 90 | # Moving back in debugger: 91 | time_travel_debug = dict() 92 | time_travel_debug['[←]'] = 'Previous Event' 93 | time_travel_debug['[↑]'] = 'Step Over, in backward direction;' 94 | time_travel_debug['[Shift]+[↑]'] = 'Step Out of method, backward direction, before call' 95 | 96 | mutable_variable = 1 97 | mutable_variable = example_of_step_out_back(mutable_variable + 1) 98 | 99 | # Just use this: 100 | # 101 | # [↑] 102 | # [←][↓][→] 103 | # 104 | # instead of remembering all the F5...F10 105 | return mutable_variable 106 | 107 | 108 | def example_of_step_out_back(mutable_variable: int): 109 | print(mutable_variable) 110 | mutable_variable += 1 111 | print('Now try this: [Shift]+[↑]') 112 | print('Step Out Back will bring you back, right before current function call.') 113 | print(' When the `mutable_variable` was equal to 1') 114 | print(' It is really useful if you need to re-enter method') 115 | print(' Or check state before function call') 116 | return mutable_variable + 1 117 | 118 | 119 | if __name__ == "__main__": 120 | method_you_want_to_trace(3, 'Welcome To PyTrace') 121 | 122 | -------------------------------------------------------------------------------- /pycrunch_trace/client/networking/strategies/network_strategy.py: -------------------------------------------------------------------------------- 1 | import threading 2 | from time import sleep 3 | 4 | import socketio 5 | from socketio import Client 6 | 7 | import logging 8 | 9 | from pycrunch_trace.client.networking.commands import EventsSlice 10 | from pycrunch_trace.client.networking.strategies.abstract_strategy import AbstractRecordingStrategy 11 | from pycrunch_trace.events.event_buffer_in_protobuf import EventBufferInProtobuf 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class OverWireRecordingStrategy(AbstractRecordingStrategy): 17 | sio : Client 18 | 19 | def __init__(self): 20 | self.sio = socketio.Client() 21 | self.host = 'http://0.0.0.0:8080' 22 | self.sio = None 23 | self.manual_reset_event = threading.Event() 24 | 25 | pass 26 | def clean(self): 27 | self.sio.disconnect() 28 | 29 | def callback(self): 30 | print(f'#callback : delivered') 31 | 32 | logger.info(f'#callback : delivered') 33 | print(f'#callback : event set') 34 | self.manual_reset_event.set() 35 | 36 | # logger.info(args) 37 | 38 | def callback_for_disconnection(self): 39 | print('Disconnection Callback') 40 | # self.sio.disconnect() 41 | 42 | def recording_start(self, session_id: str): 43 | self.sio.emit('tracing_node_event', 44 | dict( 45 | action='events_stream_will_start', 46 | session_id=session_id), 47 | ) 48 | def recording_stop(self, session_id: str): 49 | sleep(1) 50 | self.sio.emit('tracing_node_event', dict(action='events_stream_did_complete', session_id=session_id), callback=self.callback_for_disconnection) 51 | 52 | def recording_slice(self, x: EventsSlice): 53 | x: EventsSlice = x 54 | # client_introspection.save_events(x.events) 55 | # client_introspection.print_to_console(x.files) 56 | events_in_payload = len(x.events) 57 | bytes_to_send = EventBufferInProtobuf(x.events, x.files).as_bytes() 58 | print(f'manual_reset_event.waiting...') 59 | self.manual_reset_event.wait() 60 | print(f' -- done waiting for manual_reset_event, sending chunk') 61 | payload_size = len(bytes_to_send) 62 | print(f' -- sending chunk {x.chunk_number} of {x.session_id} with size {payload_size}') 63 | self.sio.emit('tracing_node_event', 64 | dict( 65 | action='events_stream', 66 | session_id=x.session_id, 67 | event_number=x.chunk_number, 68 | bytes=bytes_to_send, 69 | events_in_payload=events_in_payload, 70 | payload_size=payload_size), 71 | callback=self.callback) 72 | sleep(0.01) 73 | 74 | def prepare(self): 75 | transports = ['websocket'] 76 | # transports = ['polling'] 77 | print('socketio connect') 78 | 79 | self.sio = socketio.Client() 80 | 81 | # self.sio.connect(url=self.host, headers=self.connection_headers() ) 82 | self.sio.connect(url=self.host, transports=transports, headers=self.connection_headers()) 83 | 84 | @self.sio.event 85 | def message(data): 86 | logger.info('CLIENT: I received a message!') 87 | 88 | @self.sio.on('my message') 89 | def on_message(data): 90 | print('on_message') 91 | logger.info('CLIENT: I received a message!') 92 | 93 | @self.sio.event 94 | def connect(): 95 | print('connect') 96 | self.is_connected = True 97 | self.manual_reset_event.set() 98 | logger.info("CLIENT: I'm connected!") 99 | 100 | @self.sio.event 101 | def connect_error(): 102 | print('connect_error') 103 | logger.info("CLIENT: The connection failed!") 104 | 105 | @self.sio.event 106 | def disconnect(): 107 | print('disconnect') 108 | # put everything in garbage? 109 | self.is_connected = False 110 | logger.info("Clearing event... until connection established back") 111 | self.manual_reset_event.clear() 112 | 113 | logger.info("CLIENT: I'm disconnected!") 114 | 115 | 116 | def connection_headers(self): 117 | from pycrunch_trace.client.api.version import version 118 | connection_headers = dict( 119 | version=version, 120 | product='pycrunch-tracing-node', 121 | ) 122 | return connection_headers 123 | 124 | -------------------------------------------------------------------------------- /pycrunch_trace/server/trace_persistance.py: -------------------------------------------------------------------------------- 1 | import io 2 | import shutil 3 | import struct 4 | from datetime import datetime 5 | from pathlib import Path 6 | from typing import Dict, List 7 | 8 | import jsonpickle 9 | 10 | from pycrunch_trace.file_system import tags 11 | from pycrunch_trace.file_system.human_readable_size import HumanReadableByteSize 12 | from pycrunch_trace.file_system.persisted_session import PersistedSession, TraceSessionMetadata 13 | from pycrunch_trace.file_system.session_store import SessionStore 14 | from pycrunch_trace.file_system.trace_file import TraceFile 15 | from pycrunch_trace.server.chunks_ordering import PyCrunchTraceException 16 | from pycrunch_trace.server.incoming_traces import incoming_traces 17 | 18 | 19 | class TracePersistence: 20 | trace_files: Dict[str, TraceFile] 21 | 22 | def __init__(self): 23 | x = SessionStore() 24 | x.ensure_recording_directory_created() 25 | self.rec_dir = x.recording_directory 26 | self.trace_files = dict() 27 | pass 28 | 29 | def initialize_file(self, session_id: str): 30 | dir_path = Path(self.rec_dir) 31 | session_id = session_id 32 | rec_dir = dir_path.joinpath(session_id) 33 | if rec_dir.exists(): 34 | shutil.rmtree(rec_dir) 35 | 36 | tf = TraceFile(session_id, self.get_chunked_trace_output_file(session_id)) 37 | self.trace_files[session_id] = tf 38 | tf.write_header_placeholder() 39 | 40 | def recording_complete(self, session_id, files_included: List[str], files_excluded: List[str]): 41 | metadata_bytes, metadata_file_path = self.get_metadata_bytes(session_id, files_included, files_excluded) 42 | self.update_file_header_metadata_section(session_id, len(metadata_bytes)) 43 | self.flush_chunk(session_id, tags.TRACE_TAG_METADATA, metadata_bytes) 44 | 45 | with io.FileIO(metadata_file_path, mode='w') as file: 46 | bytes_written = file.write(metadata_bytes) 47 | print(f'metadata saved to {metadata_file_path.absolute()}') 48 | 49 | 50 | def get_metadata_bytes(self, session_id, files_included: List[str], files_excluded: List[str]): 51 | dir_path = Path(self.rec_dir) 52 | rec_dir = dir_path.joinpath(session_id) 53 | x = SessionStore() 54 | target_chunk_file = rec_dir.joinpath(PersistedSession.chunked_recording_filename) 55 | bytes_written = target_chunk_file.stat().st_size 56 | metadata_file_path = rec_dir.joinpath(PersistedSession.metadata_filename) 57 | meta = TraceSessionMetadata() 58 | meta.files_in_session = files_included 59 | meta.excluded_files = files_excluded 60 | meta.file_size_in_bytes = bytes_written 61 | meta.file_size_on_disk = str(HumanReadableByteSize(bytes_written)) 62 | trace_in_progress = incoming_traces.get_session_with_id(session_id) 63 | meta.events_in_session = trace_in_progress.total_events 64 | meta.start_time = trace_in_progress.started_at 65 | meta.end_time = datetime.utcnow() 66 | meta.name = str(session_id) 67 | result = jsonpickle.dumps(meta, unpicklable=False) 68 | metadata_bytes = result.encode('utf-8') 69 | return metadata_bytes, metadata_file_path 70 | 71 | def flush_chunk(self, session_id, tag_id: int, bytes_to_write): 72 | trace_file = self._get_trace_file_or_throw(session_id) 73 | trace_file.flush_chunk(tag_id, bytes_to_write) 74 | 75 | def get_chunked_trace_output_file(self, session_id): 76 | dir_path = Path(self.rec_dir) 77 | rec_dir = dir_path.joinpath(session_id) 78 | x = SessionStore() 79 | x.ensure_directory_created(rec_dir) 80 | target_file = rec_dir.joinpath(PersistedSession.chunked_recording_filename) 81 | return target_file 82 | 83 | def get_write_mode_if_file_exist(self, target_file): 84 | target_mode = 'a' 85 | if not target_file.exists(): 86 | target_mode = 'w' 87 | return target_mode 88 | 89 | def update_file_header_files_section(self, session_id, total_bytes): 90 | trace_file = self._get_trace_file_or_throw(session_id) 91 | trace_file.update_file_header_files_section(total_bytes) 92 | 93 | 94 | def _get_trace_file_or_throw(self, session_id) -> TraceFile: 95 | trace_file = self.trace_files.get(session_id) 96 | if not trace_file: 97 | raise PyCrunchTraceException(f'Cannot find trace file with session id {session_id}') 98 | return trace_file 99 | 100 | def update_file_header_metadata_section(self, session_id, metadata_bytes_len): 101 | trace_file = self._get_trace_file_or_throw(session_id) 102 | trace_file.update_file_header_metadata_section(metadata_bytes_len) 103 | 104 | -------------------------------------------------------------------------------- /pycrunch_trace/file_system/persisted_session.py: -------------------------------------------------------------------------------- 1 | import io 2 | import json 3 | import pickle 4 | from datetime import datetime 5 | from pathlib import Path 6 | from typing import List 7 | 8 | import jsonpickle 9 | 10 | from pycrunch_trace.events.event_buffer_in_protobuf import EventBufferInProtobuf 11 | from pycrunch_trace.file_system.chunked_trace import ChunkedTrace 12 | from pycrunch_trace.file_system.human_readable_size import HumanReadableByteSize 13 | from pycrunch_trace.proto import message_pb2 14 | import logging 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | class TraceSessionMetadata: 20 | # time in UTC 21 | start_time: datetime 22 | end_time: datetime 23 | # size in bytes 24 | file_size_in_bytes: int 25 | file_size_on_disk: str 26 | files_in_session: List[str] 27 | 28 | events_in_session: int 29 | 30 | working_directory: str 31 | 32 | name: str 33 | 34 | 35 | class LazyLoadedSession: 36 | metadata: TraceSessionMetadata 37 | trace_filename: Path 38 | metadata_file: Path 39 | raw_metadata: dict 40 | 41 | def __init__(self, buffer_file: Path, metadata_file: Path, chunked: bool): 42 | self.chunked = chunked 43 | self.trace_filename = buffer_file 44 | self.metadata_file = metadata_file 45 | self.raw_metadata = None 46 | 47 | def load_buffer(self): 48 | if not self.chunked: 49 | return self.load_unchunked() 50 | else: 51 | return ChunkedTrace(self.trace_filename).events() 52 | 53 | def load_unchunked(self): 54 | with io.FileIO(self.trace_filename, mode='r') as file: 55 | buffer = file.readall() 56 | # try: 57 | # result = json.loads(buffer) 58 | # except: 59 | 60 | # result = pickle.loads(buffer) 61 | 62 | result = message_pb2.TraceSession() 63 | result.ParseFromString(buffer) 64 | return result 65 | 66 | def load_metadata(self): 67 | with io.FileIO(self.metadata_file, 'r') as file: 68 | file_bytes = file.readall() 69 | json_representation = file_bytes.decode('utf-8') 70 | json_dict = json.loads(json_representation) 71 | self.raw_metadata = json_dict 72 | meta = TraceSessionMetadata() 73 | meta.files_in_session = json_dict.get('files_in_session') 74 | meta.excluded_files = json_dict.get('events_in_session') 75 | meta.events_in_session = json_dict.get('events_in_session') 76 | meta.file_size_in_bytes = json_dict.get('file_size_in_bytes') 77 | meta.file_size_on_disk = json_dict.get('file_size_on_disk') 78 | meta.name = json_dict.get('name') 79 | self.metadata = meta 80 | 81 | 82 | class PersistedSession: 83 | def __init__(self, session_directory: Path): 84 | self.session_directory = session_directory 85 | 86 | metadata_filename = 'pycrunch-trace.meta.json' 87 | recording_filename = 'session.pycrunch-trace' 88 | chunked_recording_filename = 'session.chunked.pycrunch-trace' 89 | 90 | pass 91 | 92 | def save_with_metadata(self, event_buffer, files_in_session, excluded_files): 93 | file_to_save = self.session_directory.joinpath(self.recording_filename) 94 | bytes_written = -42 95 | with io.FileIO(file_to_save, mode='w') as file: 96 | try: 97 | result = self.serialize_to_bytes(event_buffer) 98 | bytes_written = file.write(result) 99 | except Exception as ex: 100 | logger.exception('failed to save session', exc_info=ex) 101 | 102 | meta = TraceSessionMetadata() 103 | meta.files_in_session = list(files_in_session) 104 | meta.excluded_files = list(excluded_files) 105 | meta.file_size_in_bytes = bytes_written 106 | meta.file_size_on_disk = str(HumanReadableByteSize(bytes_written)) 107 | meta.events_in_session = len(event_buffer) 108 | meta.name = str(self.session_directory) 109 | print(f'tracing --- protobuf binary array results saved to file {file_to_save}') 110 | 111 | self.save_metadata(self.session_directory, meta) 112 | 113 | def serialize_to_bytes(self, event_buffer): 114 | return EventBufferInProtobuf(event_buffer, x.file_map).as_bytes() 115 | # todo add multiple serialization plugin/options 116 | # return pickle.dumps(event_buffer) 117 | 118 | def save_metadata(self, session_directory: Path, meta: TraceSessionMetadata): 119 | metadata_file_path = session_directory.joinpath(self.metadata_filename) 120 | 121 | with io.FileIO(metadata_file_path, mode='w') as file: 122 | result = self.serialize_to_json(meta) 123 | bytes_written = file.write(result.encode('utf-8')) 124 | 125 | def serialize_to_json(self, meta) -> str: 126 | return jsonpickle.dumps(meta, unpicklable=False) 127 | 128 | @classmethod 129 | def load_from_directory(cls, load_from_directory: Path) -> LazyLoadedSession: 130 | chunked = False 131 | joinpath = load_from_directory.joinpath(PersistedSession.recording_filename) 132 | if not joinpath.exists(): 133 | joinpath = load_from_directory.joinpath(PersistedSession.chunked_recording_filename) 134 | chunked = True 135 | print(joinpath) 136 | return LazyLoadedSession(joinpath, load_from_directory.joinpath(PersistedSession.metadata_filename), chunked) 137 | -------------------------------------------------------------------------------- /pycrunch_trace/file_system/trace_file.py: -------------------------------------------------------------------------------- 1 | import io 2 | import struct 3 | from pathlib import Path 4 | 5 | from . import tags 6 | 7 | 8 | class TLV: 9 | length: int 10 | offset: int 11 | tag: int 12 | 13 | def __init__(self): 14 | self.length = None 15 | self.offset = None 16 | self.tag = None 17 | 18 | def data_offset(self): 19 | int_size = 4 20 | # TAG 4b | LEN 4b | <----- 21 | return self.offset + int_size * 2 22 | 23 | class TraceFile: 24 | header: TLV 25 | file_section: TLV 26 | chunked_recording_filename = 'session.chunked.pycrunch-trace' 27 | 28 | def __init__(self, session_id: str, target_file: Path): 29 | self.target_file = target_file 30 | self.session_id = session_id 31 | # Header size may change in future 32 | self.header_size = 16 * 1024 33 | self.version_major = 0 34 | self.version_minor = 1 35 | 36 | self.header = None 37 | self.file_section = None 38 | 39 | 40 | def update_file_header_metadata_section(self, metadata_bytes_len): 41 | int_size = struct.calcsize(">i") 42 | uint64_size = struct.calcsize(">Q") 43 | 44 | bytes_written = self.target_file.stat().st_size 45 | 46 | with io.FileIO(self.target_file, 'r+') as file_to_write: 47 | header_metadata_begins_at = int_size * 2 + (uint64_size * 2) 48 | file_to_write.seek(self.header.data_offset()) 49 | self.skip_to_free_header_chunk(file_to_write) 50 | 51 | # Q - unsigned = 8 bytes 52 | 53 | file_to_write.write(Int32(tags.HEADER_TAG_METADATA).bytes()) 54 | file_to_write.write(Int32(uint64_size*2).bytes()) 55 | 56 | file_to_write.write(Int64(bytes_written).bytes()) 57 | file_to_write.write(Int64(metadata_bytes_len).bytes()) 58 | 59 | def update_file_header_files_section(self, total_bytes): 60 | bytes_written = self.target_file.stat().st_size 61 | 62 | with io.FileIO(self.target_file, 'r+') as file_to_write: 63 | # Magic_Number | HEADER_SIZE | FILES_BEGINS | FILES_SIZE 64 | 65 | header_begins_at = self.header.data_offset() 66 | file_to_write.seek(header_begins_at) 67 | self.skip_to_free_header_chunk(file_to_write) 68 | 69 | # Q - unsigned = 8 bytes 70 | file_to_write.write(Int32(tags.HEADER_TAG_FILES).bytes()) 71 | file_to_write.write(Int32(16).bytes()) 72 | file_to_write.write(Int64(bytes_written).bytes()) 73 | file_to_write.write(Int64(total_bytes).bytes()) 74 | 75 | def skip_to_free_header_chunk(self, file_to_write): 76 | print('skip_to_free_header_chunk') 77 | print(f'pos = {file_to_write.tell()}') 78 | int_size = struct.calcsize(">i") 79 | has_data = True 80 | while has_data: 81 | buffer = file_to_write.read(int_size) 82 | tag = struct.unpack('>i', buffer)[0] 83 | if tag == 0: 84 | has_data = False 85 | # return cursor to free space 86 | file_to_write.seek(-int_size, io.SEEK_CUR) 87 | break 88 | buffer = file_to_write.read(int_size) 89 | next_payload_length = struct.unpack('>i', buffer)[0] 90 | # skip to next record 91 | file_to_write.seek(next_payload_length, io.SEEK_CUR) 92 | print(f'after pos = {file_to_write.tell()}') 93 | 94 | def write_header_placeholder(self): 95 | target_mode = 'w' 96 | with io.FileIO(self.target_file, target_mode) as file_to_write: 97 | self.write_signature(file_to_write) 98 | self.write_header(file_to_write) 99 | 100 | def write_signature(self, file_to_write): 101 | signature = Int32(15051991).bytes() 102 | file_to_write.write(signature) 103 | 104 | def write_header(self, file_to_write): 105 | self.header = TLV() 106 | self.header.offset = file_to_write.tell() 107 | 108 | # SIG | [ Tag | Len | Val ] 109 | 110 | self.header.length = self.header_size 111 | print(f'offse {self.header.offset}') 112 | print(f'l {self.header.length}') 113 | file_to_write.write(Int32(tags.TRACE_TAG_HEADER).bytes()) 114 | file_to_write.write(Int32(self.header_size).bytes()) 115 | 116 | file_to_write.write(Int32(tags.HEADER_TAG_VERSION).bytes()) 117 | file_to_write.write(Int32(8).bytes()) 118 | file_to_write.write(Int32(self.version_major).bytes()) 119 | file_to_write.write(Int32(self.version_minor).bytes()) 120 | 121 | ppp = file_to_write.seek( 122 | self.header.offset + 0 + 4 + 4 + self.header.length, io.SEEK_SET 123 | ) 124 | print(f'hs={self.header_size}') 125 | print(f'ppp={ppp}') 126 | # this is to make sure file will have 16kb allocated at the beginning 127 | self.write_signature(file_to_write) 128 | 129 | def flush_chunk(self, tag_id: int, bytes_to_write): 130 | length_of_message = len(bytes_to_write) 131 | with io.FileIO(self.target_file, 'a') as file_to_write: 132 | # TLV triplet again 133 | file_to_write.write(Int32(tag_id).bytes()) 134 | file_to_write.write(Int32(length_of_message).bytes()) 135 | file_to_write.write(bytes_to_write) 136 | 137 | class Int32: 138 | def __init__(self, value: int): 139 | self._value = value 140 | 141 | def bytes(self): 142 | return struct.pack(">i", self._value) 143 | 144 | class Int64: 145 | def __init__(self, value: int): 146 | self._value = value 147 | 148 | def bytes(self): 149 | return struct.pack(">Q", self._value) 150 | -------------------------------------------------------------------------------- /pycrunch_trace/reference_code/google_bdb.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import sys 4 | import types 5 | 6 | import six 7 | from functools import partial 8 | 9 | from .exts.cloud_debug_python.module_explorer import GetCodeObjectAtLine, _GetModuleCodeObjects, _GetLineNumbers 10 | 11 | from rook.logger import logger 12 | 13 | from ..exceptions import RookBdbCodeNotFound, RookBdbSetBreakpointFailed, RookInvalidPositionException 14 | 15 | try: 16 | from . import cdbg_native 17 | cdbg_native.InitializeModule(None) 18 | except ImportError: 19 | # Special handling for Google AppEngine (Python 2.7) 20 | from google.devtools.cdbg.debuglets.python import cdbg_native 21 | 22 | 23 | class BPStatus(object): 24 | __slots__ = ["disabled"] 25 | 26 | def __init__(self): 27 | self.disabled = False 28 | 29 | class Bdb(object): 30 | def __init__(self): 31 | self.fncache = {} 32 | self._cookies = {} 33 | self._bp_status = {} 34 | self.user_line = None 35 | 36 | def set_trace(self): 37 | # Not needed 38 | pass 39 | 40 | def canonic(self, filename): 41 | if filename[0] == "<" and filename[-1] == ">": 42 | return filename 43 | canonic = self.fncache.get(filename) 44 | if not canonic: 45 | canonic = os.path.abspath(filename) 46 | canonic = os.path.normpath(canonic) 47 | self.fncache[filename] = canonic 48 | return canonic 49 | 50 | def ignore_current_thread(self): 51 | # Not needed 52 | pass 53 | 54 | def set_break(self, item, filename, lineno, aug_id): 55 | filename = self.canonic(filename) 56 | 57 | if isinstance(item, types.ModuleType): 58 | self._set_break_module(item, filename, lineno, aug_id) 59 | elif isinstance(item, types.CodeType): 60 | # the caller doesn't know if the code object has this line, so verify here 61 | self._set_break_code_object(item, filename, lineno, aug_id) 62 | else: 63 | raise KeyError(type(item)) 64 | 65 | def _set_break_module(self, module, filename, lineno, aug_id): 66 | status, code_object = GetCodeObjectAtLine(module, lineno) 67 | if not status: 68 | if hasattr(module, '__file__'): 69 | logger.debug("CodeNotFound module filename %s", module.__file__) 70 | 71 | for cobj in _GetModuleCodeObjects(module): 72 | logger.debug("Name: %s", cobj.co_name) 73 | for cline in _GetLineNumbers(cobj): 74 | logger.debug("Name: %s, Line %d", cobj.co_name, cline) 75 | 76 | if code_object == (None, None): 77 | raise RookBdbCodeNotFound(filename=filename) 78 | else: 79 | raise RookInvalidPositionException(filename=filename, line=lineno, alternatives=code_object) 80 | 81 | self._set_break_code_object(code_object, filename, lineno, aug_id) 82 | 83 | def _set_break_code_object(self, code_object, filename, lineno, aug_id): 84 | # Install the breakpoint 85 | bp_status = BPStatus() 86 | cookie = cdbg_native.SetConditionalBreakpoint(code_object, lineno, None, partial(_callback, lineno=lineno, 87 | user_line=self.user_line, 88 | callback_object_id=id(code_object), 89 | bp_status=bp_status, 90 | filename=filename, 91 | pid=os.getpid(), 92 | aug_id=aug_id)) 93 | 94 | if cookie < 0: 95 | raise RookBdbSetBreakpointFailed("%s on line %d" % (code_object.co_name, lineno)) 96 | 97 | self._cookies[aug_id] = cookie 98 | self._bp_status[aug_id] = bp_status 99 | 100 | def clear_break(self, aug_id): 101 | try: 102 | cookie = self._cookies[aug_id] 103 | status = self._bp_status[aug_id] 104 | except KeyError: 105 | return 106 | 107 | cdbg_native.ClearConditionalBreakpoint(cookie) 108 | status.disabled = True 109 | del self._cookies[aug_id] 110 | del self._bp_status[aug_id] 111 | 112 | def clear_all_breaks(self): 113 | for cookie in six.itervalues(self._cookies): 114 | cdbg_native.ClearConditionalBreakpoint(cookie) 115 | 116 | for status in six.itervalues(self._bp_status): 117 | status.disabled = True 118 | 119 | self._cookies = {} 120 | self._bp_status = {} 121 | 122 | def close(self): 123 | pass 124 | 125 | 126 | # This function has been moved outside of the class so that it can be pickled 127 | # safely by cloudpickle (which will pickle any objects referred to by its closure) 128 | # When changing it, take care to avoid using references to anything not imported within the function 129 | def _callback(lineno, user_line, callback_object_id, bp_status, filename, pid, aug_id): 130 | try: 131 | if bp_status.disabled is True: 132 | return 133 | 134 | import inspect 135 | frame = inspect.currentframe().f_back 136 | 137 | callback_was_pickled = callback_object_id != id(frame.f_code) or pid != os.getpid() 138 | 139 | if not callback_was_pickled and frame and user_line: 140 | user_line(frame, filename, lineno=lineno, aug_id=aug_id) 141 | except: 142 | pass -------------------------------------------------------------------------------- /pycrunch_trace/client/networking/ClientOutgoingQueueThread.py: -------------------------------------------------------------------------------- 1 | import threading 2 | from queue import Queue, Empty 3 | 4 | from pycrunch_trace.client.networking.commands import EventsSlice, StopCommand, AbstractNetworkCommand, StartCommand, FileContentSlice 5 | 6 | import sys 7 | import pyximport 8 | 9 | from pycrunch_trace.client.networking.strategies.abstract_strategy import AbstractRecordingStrategy 10 | from pycrunch_trace.file_system.trace_session import TraceSession 11 | 12 | pyximport.install() 13 | from pycrunch_trace.client.networking.strategies.native_write_strategy import NativeLocalRecordingStrategy 14 | 15 | 16 | import logging 17 | 18 | 19 | logger = logging.getLogger(__name__) 20 | 21 | # root = logging.getLogger() 22 | # root.setLevel(logging.DEBUG) 23 | # 24 | # handler = logging.StreamHandler(sys.stdout) 25 | # handler.setLevel(logging.DEBUG) 26 | # formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 27 | # handler.setFormatter(formatter) 28 | # root.addHandler(handler) 29 | # root.getChild('engineio.client').disabled = True 30 | # root.getChild('socketio.client').disabled = True 31 | # logger = logging.getLogger(__name__) 32 | 33 | 34 | import os 35 | 36 | class ClientQueueThread: 37 | available_recording_strategies = ['network', 'local'] 38 | 39 | is_thread_running: bool 40 | _counter: int 41 | _strategy: AbstractRecordingStrategy 42 | 43 | def __init__(self): 44 | print(f'PID: {os.getpid()} ClientQueueThread init') 45 | self._counter = 0 46 | self.so_far = 0 47 | self.is_connected = False 48 | self.outgoingQueue = Queue() 49 | self.is_thread_running = False 50 | current_strategy = 'native_local' 51 | if current_strategy == 'native_local': 52 | self._strategy = NativeLocalRecordingStrategy() 53 | 54 | def tracing_will_start(self, session_id: str): 55 | self.ensure_thread_started() 56 | try: 57 | self.outgoingQueue.put_nowait(StartCommand(session_id)) 58 | except Exception as e: 59 | print('EXCEPTION') 60 | print(e) 61 | 62 | 63 | def put_events(self, events: EventsSlice): 64 | self.so_far += len(events.events) 65 | print(f'{events.session_id} - put_events: so far: {self.so_far}') 66 | self.ensure_thread_started() 67 | try: 68 | self.outgoingQueue.put_nowait(events) 69 | except Exception as e: 70 | print('EXCEPTION while put_events') 71 | print(e) 72 | 73 | def put_file_slice(self, events: FileContentSlice): 74 | print('put_file_slice') 75 | self.ensure_thread_started() 76 | try: 77 | self.outgoingQueue.put_nowait(events) 78 | except Exception as e: 79 | print('EXCEPTION while put_file_slice') 80 | print(e) 81 | 82 | def tracing_did_complete(self, session_id, session: TraceSession): 83 | print('tracing_did_complete') 84 | self.ensure_thread_started() 85 | self.outgoingQueue.put_nowait( 86 | StopCommand( 87 | session_id, 88 | list(session.files_in_session.copy()), 89 | list(session.excluded_files.copy()), 90 | ) 91 | ) 92 | 93 | def start(self): 94 | print(f'start thread dispather queue with pid: {os.getpid()}') 95 | if self.is_thread_running: 96 | return 97 | 98 | print('socketio init') 99 | x = threading.Thread(target=self.thread_proc, args=(42,)) 100 | # x.setDaemon(True) 101 | x.setDaemon(False) 102 | x.start() 103 | # todo lock? 104 | self.is_thread_running = True 105 | 106 | 107 | def thread_proc(self, obj): 108 | logging.info("Thread ClientQueueThread::thread_proc: starting") 109 | 110 | self._strategy.prepare() 111 | 112 | 113 | while True: 114 | logger.info('outgoingQueue.get: Waiting for message...') 115 | try: 116 | x: AbstractNetworkCommand = self.outgoingQueue.get(True, 3) 117 | print(f'queue length {len(self.outgoingQueue.queue)}') 118 | 119 | if x is not None: 120 | self.process_single_message(x) 121 | except Empty: 122 | print('Timeout while waiting for new msg... Thread will stop for now') 123 | break 124 | pass 125 | 126 | except Exception as ex: 127 | logger.info('Ex while getting message from queue') 128 | print('===!!! Ex while getting message from queue') 129 | print(str(ex)) 130 | exc_type, exc_obj, exc_tb = sys.exc_info() 131 | fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] 132 | print(exc_type, fname, exc_tb.tb_lineno) 133 | continue 134 | # end while 135 | print('Thread stopped') 136 | self._strategy.clean() 137 | self.is_thread_running = False 138 | 139 | def process_single_message(self, x: AbstractNetworkCommand): 140 | print(f'got evt {x.command_name}') 141 | if x.command_name == 'StartCommand': 142 | self._strategy.recording_start(x.session_id) 143 | if x.command_name == 'StopCommand': 144 | logger.info('got ' + x.command_name) 145 | self._strategy.recording_stop(x.session_id, x.files_included, x.files_excluded) 146 | if x.command_name == 'FileContentSlice': 147 | logger.info('got ' + x.command_name) 148 | self._strategy.files_slice(x) 149 | logger.info('Sending... ' + x.command_name) 150 | if x.command_name == 'EventsSlice': 151 | self._strategy.recording_slice(x) 152 | logger.info('Sent... ' + x.command_name) 153 | 154 | def ensure_thread_started(self): 155 | if not self.is_thread_running: 156 | self.start() 157 | 158 | 159 | event_queue: ClientQueueThread = ClientQueueThread() 160 | -------------------------------------------------------------------------------- /pycrunch_trace/tracing/simple_tracer.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import pycrunch_trace.tracing.simulation.models as models 4 | import pycrunch_trace.events.method_enter as events 5 | from pycrunch_trace.client.command_buffer import ArrayCommandBuffer 6 | from pycrunch_trace.client.networking.commands import EventsSlice 7 | from pycrunch_trace.file_system.trace_session import TraceSession 8 | from pycrunch_trace.filters import AbstractFileFilter 9 | from pycrunch_trace.oop import Clock 10 | from pycrunch_trace.tracing.call_stack import CallStack 11 | from pycrunch_trace.tracing.file_map import FileMap 12 | from pycrunch_trace.tracing.perf import TracerPerf 13 | from pycrunch_trace.tracing.simulation import EventKeys 14 | from pycrunch_trace.tracing.simulator_sink import SimulatorSink, DisabledSimulatorSink 15 | 16 | 17 | class SimpleTracer: 18 | file_map: FileMap 19 | event_number: int 20 | file_filter: AbstractFileFilter 21 | event_buffer: ArrayCommandBuffer 22 | call_stack: CallStack 23 | session: TraceSession 24 | simulation: SimulatorSink 25 | events_so_far: int 26 | 27 | def __init__(self, event_buffer, session_name, file_filter: AbstractFileFilter, clock: Clock, queue): 28 | self.event_number = 1 29 | self.events_so_far = 0 30 | self.clock = clock 31 | self.event_buffer = event_buffer 32 | self.file_filter = file_filter 33 | self.should_record_variables = file_filter.should_record_variables() 34 | self.call_stack = CallStack() 35 | self.session = TraceSession() 36 | self.simulation = DisabledSimulatorSink() 37 | # self.simulation = SimulatorSink() 38 | self.file_map = FileMap() 39 | self.perf = TracerPerf() 40 | self.queue = queue 41 | self.max_events_before_send = 1000 42 | self.threshold_before_switching_to_sampling = 1000000 43 | self.skip = False 44 | # interval = sys.getscheckinterval() 45 | # print(f'interval {interval}') 46 | # sys.setcheckinterval(100000000) 47 | 48 | # interval = sys.getswitchinterval() 49 | # 50 | # print(f'getswitchinterval {interval}') 51 | # sys.setswitchinterval(3) 52 | # interval = sys.getswitchinterval() 53 | # print(f'getswitchinterval {interval}') 54 | 55 | def simple_tracer(self, frame: models.Frame, event: str, arg): 56 | entered_at = self.clock.now() 57 | self.process_events(entered_at, event, frame, arg) 58 | # self.simulation.save_for_simulator(frame, event, arg) 59 | 60 | # print(f"[{co.co_argcount}]{event}: {func_name} {line_no} -> {arg}") 61 | # print(f" {frame.f_locals}") 62 | 63 | end_at = self.clock.now() 64 | diff = end_at - entered_at 65 | self.perf.did_execute_line(diff) 66 | return self.simple_tracer 67 | 68 | def process_events(self, now: float, event: str, frame: models.Frame, arg): 69 | file_path_under_cursor = frame.f_code.co_filename 70 | if not self.file_filter.should_trace(file_path_under_cursor): 71 | will_record_current_event = False 72 | self.session.will_skip_file(file_path_under_cursor) 73 | else: 74 | will_record_current_event = True 75 | self.events_so_far += 1 76 | self.session.did_enter_traceable_file(file_path_under_cursor) 77 | 78 | if event == EventKeys.line: 79 | over_threshold = self.events_so_far > self.threshold_before_switching_to_sampling 80 | if will_record_current_event and not over_threshold: 81 | cursor = self.create_cursor(file_path_under_cursor, frame) 82 | self.call_stack.new_cursor_in_current_frame(cursor) 83 | 84 | stack = self.get_execution_stack() 85 | current = events.LineExecutionEvent(cursor, stack, now) 86 | if self.should_record_variables: 87 | self.push_traceable_variables(frame, current.locals) 88 | self.add_to_event_buffer(current) 89 | 90 | if event == EventKeys.call: 91 | cursor = self.create_cursor(file_path_under_cursor, frame) 92 | self.call_stack.enter_frame(cursor) 93 | # lets try to add methods 94 | # [if sampling mode] 95 | if will_record_current_event: 96 | current = events.MethodEnterEvent(cursor, self.get_execution_stack(), now) 97 | if self.should_record_variables: 98 | variables = current.input_variables 99 | self.push_traceable_variables(frame, variables) 100 | self.add_to_event_buffer(current) 101 | 102 | if event == EventKeys.event_return: 103 | self.call_stack.exit_frame() 104 | # [? is sampling] 105 | if will_record_current_event: 106 | cursor = self.create_cursor(file_path_under_cursor, frame) 107 | current = events.MethodExitEvent(cursor, self.get_execution_stack(), now) 108 | if self.should_record_variables: 109 | current.return_variables.push_variable('__return', arg) 110 | self.push_traceable_variables(frame, current.locals) 111 | self.add_to_event_buffer(current) 112 | 113 | self.flush_queue_if_full() 114 | 115 | def add_to_event_buffer(self, current): 116 | # todo: is this caused because of array dynamic size/doubling? 117 | # if True: 118 | # return 119 | self.event_buffer.add_event(current) 120 | 121 | def create_cursor(self, file_path_under_cursor, frame): 122 | file_id = self.file_map.file_id(file_path_under_cursor) 123 | cursor = events.ExecutionCursor(file_id, frame.f_lineno, frame.f_code.co_name) 124 | return cursor 125 | 126 | def get_execution_stack(self): 127 | return self.call_stack.current_frame() 128 | 129 | def push_traceable_variables(self, frame, variables): 130 | for (name, value) in frame.f_locals.items(): 131 | # todo use variable values diffing 132 | variables.push_variable(name, value) 133 | 134 | def flush_outstanding_events(self): 135 | old_buffer = self.event_buffer.finish_chunk() 136 | self.perf.print_avg_time() 137 | 138 | self.queue.put_events(EventsSlice(self.session.session_name, self.event_number, old_buffer, self.file_map.files.copy())) 139 | self.event_number += 1 140 | 141 | def flush_queue_if_full(self): 142 | if self.event_buffer.how_many_events() >= self.max_events_before_send: 143 | self.flush_outstanding_events() 144 | -------------------------------------------------------------------------------- /pycrunch_trace/server/recording_server_websocket.py: -------------------------------------------------------------------------------- 1 | import io 2 | 3 | import yaml 4 | 5 | from . import shared 6 | 7 | from pycrunch_trace.events.event_buffer_in_protobuf import EventBufferInProtobuf 8 | from pycrunch_trace.file_system.session_store import SessionStore 9 | from pycrunch_trace.file_system.trace_session import TraceSession 10 | from pycrunch_trace.filters import CustomFileFilter 11 | from pycrunch_trace.oop import Directory, WriteableFile 12 | from pycrunch_trace.oop.file import File 13 | from pycrunch_trace.session import active_clients 14 | from pycrunch_trace.session.snapshot import snapshot 15 | import pickle 16 | 17 | 18 | import logging 19 | 20 | from .incoming_traces import incoming_traces 21 | from .state import connections 22 | from ..config import config 23 | from ..file_system.human_readable_size import HumanReadableByteSize 24 | from ..file_system.persisted_session import TraceSessionMetadata 25 | 26 | logger = logging.getLogger(__name__) 27 | 28 | 29 | @shared.tracer_socket_server.event 30 | async def connect(sid, environ): 31 | print("connect -", sid) 32 | # print("connect - environ", environ) 33 | product_name = environ.get('HTTP_PRODUCT') 34 | if product_name: 35 | if product_name == 'pycrunch-tracing-node': 36 | version = environ.get('HTTP_VERSION') 37 | 38 | connections.tracer_did_connect(sid, version) 39 | await shared.tracer_socket_server.emit('front', dict( 40 | event_name='new_tracker', 41 | sid=sid, 42 | version=version, 43 | )) 44 | 45 | 46 | async def new_recording(req, sid): 47 | logger.info('Started saving new recording') 48 | event_buffer_bytes = req.get('buffer') 49 | # todo this is double loading 50 | if (event_buffer_bytes): 51 | x: TraceSession = pickle.loads(event_buffer_bytes) 52 | x.save() 53 | 54 | logger.info('Recording saved successfully') 55 | await load_sessions(None) 56 | # await sio.emit('reply', event_buffer) 57 | 58 | total_bytes = 0 59 | 60 | 61 | 62 | @shared.tracer_socket_server.event 63 | async def event(sid, req): 64 | # print(req) 65 | action: str = req.get('action') 66 | logger.info(f'WebSocket event: {action}') 67 | 68 | if action == 'load_buffer': 69 | pass 70 | elif action == 'load_file': 71 | await load_file_event(req, sid) 72 | elif action == 'load_profiles': 73 | await load_profiles_event(req, sid) 74 | elif action == 'load_sessions': 75 | await load_sessions(sid) 76 | elif action == 'load_profile_details': 77 | await load_profile_details(req, sid) 78 | elif action == 'load_single_session': 79 | await load_single_session(req, sid) 80 | elif action == 'save_profile_details': 81 | await save_profile_details(req, sid) 82 | elif action == 'new_recording': 83 | print('new_recording') 84 | await new_recording(req, sid) 85 | else: 86 | await shared.tracer_socket_server.emit('reply_unknown', room=sid) 87 | 88 | 89 | async def load_sessions(sid): 90 | logging.debug('Loading sessions') 91 | store = SessionStore() 92 | all_names = store.all_sessions() 93 | result = [] 94 | for name in all_names: 95 | try: 96 | lazy_loaded = store.load_session(name) 97 | lazy_loaded.load_metadata() 98 | metadata = lazy_loaded.raw_metadata 99 | metadata['short_name'] = name 100 | except: 101 | metadata = dict() 102 | metadata['short_name'] = name 103 | metadata['events_in_session'] = 'missing meta' 104 | 105 | result.append(metadata) 106 | 107 | logging.debug(f'Sessions loaded, sending back to client {sid}') 108 | await shared.tracer_socket_server.emit('session_list_loaded', result, room=sid) 109 | pass 110 | 111 | 112 | async def load_single_session(req, sid): 113 | logger.info('begin: load_single_session...') 114 | store = SessionStore() 115 | session_name = req.get('session_name') 116 | logging.info(f'Loading session {session_name}') 117 | ses = store.load_session(session_name) 118 | 119 | # await sio.emit('reply', to_string(buffer), room=sid) 120 | try: 121 | 122 | logger.info('sending reply...') 123 | 124 | file_as_bytes = ses.load_buffer().SerializeToString() 125 | logger.info('bytes loaded...') 126 | # todo maybe send back in chunks? 127 | await shared.tracer_socket_server.emit('reply', data=file_as_bytes 128 | , room=sid 129 | ) 130 | logger.info('Event sent') 131 | 132 | except Exception as ex: 133 | logger.exception('Failed to load session ' + session_name, exc_info=ex) 134 | 135 | 136 | async def save_profile_details(req, sid): 137 | logger.debug(f'save_profile_details: `{req}`') 138 | profile = req.get('profile') 139 | profile_name = profile.get('profile_name') 140 | xxx = yaml.dump(profile) 141 | logger.debug(xxx) 142 | profiles__joinpath = config.package_directory.joinpath('pycrunch-profiles').joinpath(profile_name) 143 | 144 | WriteableFile(profiles__joinpath, xxx.encode('utf-8')).save() 145 | 146 | 147 | async def load_file_event(req, sid): 148 | file_to_load = req.get('file_to_load') 149 | logger.debug(f'file_to_load: `{file_to_load}`') 150 | with io.open(file_to_load, 'r', encoding='utf-8') as f: 151 | lines = f.read() 152 | await shared.tracer_socket_server.emit('file_did_load', dict(filename=file_to_load, contents=lines), room=sid) 153 | 154 | 155 | async def load_profile_details(req, sid): 156 | d = Directory(config.package_directory.joinpath('pycrunch-profiles')) 157 | profile_name = req.get('profile_name') 158 | joinpath = config.package_directory.joinpath('pycrunch-profiles').joinpath(profile_name) 159 | print(joinpath) 160 | fff = CustomFileFilter(File(joinpath)) 161 | 162 | raw = fff.all_exclusions() 163 | 164 | await shared.tracer_socket_server.emit('profile_details_loaded', dict( 165 | exclusions=raw, 166 | trace_variables=fff.should_record_variables(), 167 | profile_name=profile_name), room=sid) 168 | 169 | 170 | async def load_profiles_event(req, sid): 171 | d = Directory(config.package_directory.joinpath('pycrunch-profiles')) 172 | res = d.files('yaml') 173 | print(res) 174 | raw = [] 175 | for f in res: 176 | raw.append(f.short_name()) 177 | await shared.tracer_socket_server.emit('profiles_loaded', dict(profiles=raw), room=sid) 178 | 179 | 180 | @shared.tracer_socket_server.event 181 | async def disconnect(sid): 182 | logging.info(f'disconnect {sid}') 183 | if connections.tracer_did_disconnect(sid): 184 | logging.debug(f' -- sending notification about disconnected tracker {sid}') 185 | await shared.tracer_socket_server.emit('front', dict( 186 | event_name='tracker_did_disconnect', 187 | sid=sid, 188 | )) 189 | -------------------------------------------------------------------------------- /pycrunch_trace/reference_code/pytracer_cov.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 2 | # For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt 3 | 4 | """Raw data collector for coverage.py.""" 5 | 6 | import atexit 7 | import dis 8 | import sys 9 | 10 | 11 | # We need the YIELD_VALUE opcode below, in a comparison-friendly form. 12 | YIELD_VALUE = dis.opmap['YIELD_VALUE'] 13 | 14 | 15 | class PyTracer(object): 16 | """Python implementation of the raw data tracer.""" 17 | 18 | # Because of poor implementations of trace-function-manipulating tools, 19 | # the Python trace function must be kept very simple. In particular, there 20 | # must be only one function ever set as the trace function, both through 21 | # sys.settrace, and as the return value from the trace function. Put 22 | # another way, the trace function must always return itself. It cannot 23 | # swap in other functions, or return None to avoid tracing a particular 24 | # frame. 25 | # 26 | # The trace manipulator that introduced this restriction is DecoratorTools, 27 | # which sets a trace function, and then later restores the pre-existing one 28 | # by calling sys.settrace with a function it found in the current frame. 29 | # 30 | # Systems that use DecoratorTools (or similar trace manipulations) must use 31 | # PyTracer to get accurate results. The command-line --timid argument is 32 | # used to force the use of this tracer. 33 | 34 | def __init__(self): 35 | # Attributes set from the collector: 36 | self.data = dict() 37 | self.trace_arcs = False 38 | self.should_trace = None 39 | self.should_trace_cache = dict() 40 | self.warn = None 41 | # The threading module to use, if any. 42 | self.threading = None 43 | 44 | self.cur_file_dict = None 45 | self.last_line = 0 # int, but uninitialized. 46 | self.cur_file_name = None 47 | 48 | self.data_stack = [] 49 | self.last_exc_back = None 50 | self.last_exc_firstlineno = 0 51 | self.thread = None 52 | self.stopped = False 53 | self._activity = False 54 | 55 | self.in_atexit = False 56 | # On exit, self.in_atexit = True 57 | atexit.register(setattr, self, 'in_atexit', True) 58 | 59 | def __repr__(self): 60 | return "".format( 61 | id(self), 62 | sum(len(v) for v in self.data.values()), 63 | len(self.data), 64 | ) 65 | 66 | def log(self, marker, *args): 67 | """For hard-core pc_logging of what this tracer is doing.""" 68 | with open("/tmp/debug_trace.txt", "a") as f: 69 | f.write("{} {:x}.{:x}[{}] {:x} {}\n".format( 70 | marker, 71 | id(self), 72 | self.thread.ident, 73 | len(self.data_stack), 74 | self.threading.currentThread().ident, 75 | " ".join(map(str, args)) 76 | )) 77 | 78 | def _trace(self, frame, event, arg_unused): 79 | """The trace function passed to sys.settrace.""" 80 | 81 | #self.log(":", frame.f_code.co_filename, frame.f_lineno, event) 82 | 83 | if (self.stopped and sys.gettrace() == self._trace): 84 | # The PyTrace.stop() method has been called, possibly by another 85 | # thread, let's deactivate ourselves now. 86 | #self.log("X", frame.f_code.co_filename, frame.f_lineno) 87 | sys.settrace(None) 88 | return None 89 | 90 | if self.last_exc_back: 91 | if frame == self.last_exc_back: 92 | # Someone forgot a return event. 93 | if self.trace_arcs and self.cur_file_dict: 94 | pair = (self.last_line, -self.last_exc_firstlineno) 95 | self.cur_file_dict[pair] = None 96 | self.cur_file_dict, self.cur_file_name, self.last_line = self.data_stack.pop() 97 | self.last_exc_back = None 98 | 99 | if event == 'call': 100 | # Entering a new function context. Decide if we should trace 101 | # in this file. 102 | self._activity = True 103 | self.data_stack.append((self.cur_file_dict, self.cur_file_name, self.last_line)) 104 | filename = frame.f_code.co_filename 105 | self.cur_file_name = filename 106 | disp = self.should_trace_cache.get(filename) 107 | if disp is None: 108 | disp = True 109 | self.should_trace_cache[filename] = disp 110 | 111 | self.cur_file_dict = None 112 | if disp: 113 | tracename = 'xxxx' 114 | if tracename not in self.data: 115 | self.data[tracename] = {} 116 | self.cur_file_dict = self.data[tracename] 117 | # The call event is really a "start frame" event, and happens for 118 | # function calls and re-entering generators. The f_lasti field is 119 | # -1 for calls, and a real offset for generators. Use <0 as the 120 | # line number for calls, and the real line number for generators. 121 | if getattr(frame, 'f_lasti', -1) < 0: 122 | self.last_line = -frame.f_code.co_firstlineno 123 | else: 124 | self.last_line = frame.f_lineno 125 | elif event == 'line': 126 | # Record an executed line. 127 | if self.cur_file_dict is not None: 128 | lineno = frame.f_lineno 129 | #if frame.f_code.co_filename != self.cur_file_name: 130 | # self.log("*", frame.f_code.co_filename, self.cur_file_name, lineno) 131 | if self.trace_arcs: 132 | self.cur_file_dict[(self.last_line, lineno)] = None 133 | else: 134 | self.cur_file_dict[lineno] = None 135 | self.last_line = lineno 136 | elif event == 'return': 137 | if self.trace_arcs and self.cur_file_dict: 138 | # Record an arc leaving the function, but beware that a 139 | # "return" event might just mean yielding from a generator. 140 | # Jython seems to have an empty co_code, so just assume return. 141 | code = frame.f_code.co_code 142 | if (not code) or code[frame.f_lasti] != YIELD_VALUE: 143 | first = frame.f_code.co_firstlineno 144 | self.cur_file_dict[(self.last_line, -first)] = None 145 | # Leaving this function, pop the filename stack. 146 | self.cur_file_dict, self.cur_file_name, self.last_line = self.data_stack.pop() 147 | elif event == 'exception': 148 | self.last_exc_back = frame.f_back 149 | self.last_exc_firstlineno = frame.f_code.co_firstlineno 150 | return self._trace 151 | 152 | def start(self): 153 | """Start this Tracer. 154 | 155 | Return a Python function suitable for use with sys.settrace(). 156 | 157 | """ 158 | self.stopped = False 159 | if self.threading: 160 | if self.thread is None: 161 | self.thread = self.threading.currentThread() 162 | else: 163 | if self.thread.ident != self.threading.currentThread().ident: 164 | # Re-starting from a different thread!? Don't set the trace 165 | # function, but we are marked as running again, so maybe it 166 | # will be ok? 167 | #self.log("~", "starting on different threads") 168 | return self._trace 169 | 170 | sys.settrace(self._trace) 171 | return self._trace 172 | 173 | def stop(self): 174 | """Stop this Tracer.""" 175 | # Get the activate tracer callback before setting the stop flag to be 176 | # able to detect if the tracer was changed prior to stopping it. 177 | tf = sys.gettrace() 178 | 179 | # Set the stop flag. The actual call to sys.settrace(None) will happen 180 | # in the self._trace callback itself to make sure to call it from the 181 | # right thread. 182 | self.stopped = True 183 | 184 | if self.threading and self.thread.ident != self.threading.currentThread().ident: 185 | # Called on a different thread than started us: we can't unhook 186 | # ourselves, but we've set the flag that we should stop, so we 187 | # won't do any more tracing. 188 | #self.log("~", "stopping on different threads") 189 | return 190 | 191 | if self.warn: 192 | # PyPy clears the trace function before running atexit functions, 193 | # so don't warn if we are in atexit on PyPy and the trace function 194 | # has changed to None. 195 | dont_warn = False 196 | if (not dont_warn) and tf != self._trace: 197 | self.warn( 198 | "Trace function changed, measurement is likely wrong: %r" % (tf,), 199 | slug="trace-changed", 200 | ) 201 | 202 | def activity(self): 203 | """Has there been any activity?""" 204 | return self._activity 205 | 206 | def reset_activity(self): 207 | """Reset the activity() flag.""" 208 | self._activity = False 209 | 210 | def get_stats(self): 211 | """Return a dictionary of statistics, or None.""" 212 | return None 213 | -------------------------------------------------------------------------------- /pycrunch_trace/native/native_tracer.pyx: -------------------------------------------------------------------------------- 1 | import time 2 | import collections 3 | 4 | from pycrunch_trace.client.networking.commands import EventsSlice, FileContentSlice, StopCommand 5 | from pycrunch_trace.file_system.trace_session import TraceSession 6 | from pycrunch_trace.tracing.file_map import FileMap 7 | from pycrunch_trace.native.native_models cimport NativeCodeEvent, NativeExecutionCursor, NativeVariables, NativeVariable, NativeStackFrame 8 | 9 | allowed_types = [int, str, float, dict, type(None), bool] 10 | # allowed_types = [int, str, float, type(None), bool] 11 | 12 | 13 | cdef class NativeCallStack: 14 | cdef object stack 15 | cdef int last_id 16 | def __init__(self): 17 | self.stack = collections.deque() 18 | self.last_id = 0 19 | 20 | cdef NativeStackFrame enter_frame(self, NativeExecutionCursor execution_cursor): 21 | cdef NativeStackFrame new_candidate 22 | parent_frame = self.current_frame() 23 | # print(f"{execution_cursor.file}:{execution_cursor.line} -> {parent_frame} ") 24 | new_candidate = NativeStackFrame() 25 | new_candidate.cursor = execution_cursor 26 | new_candidate.parent = parent_frame 27 | new_candidate.id = self.next_id() 28 | 29 | self.stack.append(new_candidate) 30 | return new_candidate 31 | 32 | cdef int next_id(self): 33 | self.last_id += 1 34 | return self.last_id 35 | 36 | cdef NativeStackFrame new_cursor_in_current_frame(self, NativeExecutionCursor new_cursor): 37 | cdef NativeStackFrame cloned 38 | cdef NativeStackFrame parent 39 | parent = self.current_frame() 40 | 41 | cloned = NativeStackFrame() 42 | cloned.cursor = new_cursor 43 | if parent is not None: 44 | cloned.parent = parent.parent 45 | else: 46 | cloned.parent = parent 47 | 48 | cloned.id = self.next_id() 49 | 50 | if len(self.stack) > 0: 51 | self.stack.pop() 52 | self.stack.append(cloned) 53 | # self.stack[-1] = stack_frame 54 | else: 55 | # session just begin, not yet in any stack 56 | self.stack.append(cloned) 57 | return cloned 58 | 59 | cdef exit_frame(self): 60 | self.stack.pop() 61 | 62 | cdef NativeStackFrame top_level_frame_as_clone(self): 63 | cdef NativeStackFrame current 64 | cdef NativeStackFrame new_result 65 | current = self.current_frame() 66 | if current: 67 | new_result = NativeStackFrame() 68 | new_result.id = self.next_id() 69 | new_result.parent = current.parent 70 | new_result.cursor = current.cursor 71 | return new_result 72 | else: 73 | return None 74 | # ??? 75 | # return current 76 | 77 | cdef NativeStackFrame current_frame(self): 78 | if len(self.stack) > 0: 79 | return self.stack[-1] 80 | return None 81 | 82 | cdef class NativeBuffer: 83 | cdef list _buffer 84 | def __init__(self): 85 | self._buffer = [] 86 | 87 | cdef void append(self, NativeCodeEvent evt): 88 | self._buffer.append(evt) 89 | 90 | cdef int count(self): 91 | return self._buffer.__len__() 92 | 93 | cdef finish_chunk(self): 94 | clone = self._buffer 95 | self._buffer = [] 96 | return clone 97 | 98 | cdef class NativeTracerPerf: 99 | cdef int total_samples 100 | cdef double total_time 101 | 102 | def __init__(self): 103 | self.total_samples = 1 104 | self.total_time = 0.00 105 | pass 106 | 107 | cdef void did_execute_line(self, double ts_diff): 108 | self.total_samples += 1 109 | self.total_time += ts_diff 110 | 111 | def print_avg_time(self): 112 | each = 1 113 | should_print = self.total_samples % each == 0 114 | should_print = True 115 | if should_print: 116 | time_per_sample = self.total_time / self.total_samples 117 | print(f'total_samples - {self.total_samples}') 118 | print(f'total overhead time - {round(self.total_time)} ms') 119 | print(f' {self.total_time}') 120 | print(f'{time_per_sample:.5f} ms avg call time overhead') 121 | 122 | cdef class NativeClock: 123 | cdef double started_at 124 | 125 | def __init__(self): 126 | self.started_at = self._system_clock() 127 | pass 128 | 129 | cdef double now(self): 130 | cdef double now_without_offset 131 | now_without_offset = self._system_clock() 132 | return (now_without_offset - self.started_at) * 1000 133 | 134 | cdef double _system_clock(self): 135 | return time.perf_counter() 136 | pass 137 | 138 | cdef class NativeTracer: 139 | cdef int event_number 140 | cdef int events_so_far 141 | cdef bint should_trace_variables 142 | cdef str session_name 143 | cdef NativeClock clock 144 | cdef NativeTracerPerf perf 145 | cdef NativeBuffer event_buffer 146 | cdef int max_events_before_send 147 | cdef object file_map 148 | cdef object file_filter 149 | cdef object queue 150 | cdef object session 151 | cdef NativeCallStack call_stack 152 | 153 | # cpdef ClientQueueThread queue 154 | 155 | def __init__(self, session_name, queue, file_filter): 156 | self.events_so_far = 0 157 | self.clock = NativeClock() 158 | self.perf = NativeTracerPerf() 159 | self.event_buffer = NativeBuffer() 160 | self.session_name = session_name 161 | self.max_events_before_send = 500 162 | self.file_map = FileMap() 163 | self.file_filter = file_filter 164 | self.should_trace_variables = file_filter.should_record_variables() 165 | self.session = TraceSession() 166 | 167 | self.queue = queue 168 | self.call_stack = NativeCallStack() 169 | 170 | def simple_tracer(self, frame, str event, arg): 171 | self.internal_c_call(frame, event, arg) 172 | return self.simple_tracer 173 | 174 | cdef internal_c_call(self, frame, str evt, arg): 175 | cdef str func_name 176 | cdef double entered_at 177 | cdef double end_at 178 | cdef double diff 179 | entered_at = self.clock.now() 180 | self.wip(entered_at, frame, evt, arg) 181 | self.events_so_far += 1 182 | # self.process_events(entered_at, event, frame, arg) 183 | # self.simulation.save_for_simulator(frame, event, arg) 184 | # print(f"[{co.co_argcount}]{event}: {func_name} {line_no} -> {arg}") 185 | # print(f" {frame.f_locals}") 186 | end_at = self.clock.now() 187 | diff = end_at - entered_at 188 | self.perf.did_execute_line(diff) 189 | 190 | cdef wip(self, double entered_at, frame, str event, arg): 191 | # cdef boolean will_record_current_event 192 | cdef NativeCodeEvent current 193 | cdef NativeVariables input_variables 194 | cdef NativeVariables return_variables 195 | cdef NativeVariables locals_v 196 | cdef NativeExecutionCursor current_cursor 197 | cdef NativeVariable return_var 198 | cdef NativeStackFrame stack 199 | cdef int line_no 200 | cdef bint will_record_current_event 201 | cdef str function_name 202 | file_path_under_cursor = frame.f_code.co_filename 203 | 204 | if not self.file_filter.should_trace(file_path_under_cursor): 205 | will_record_current_event = False 206 | self.session.will_skip_file(file_path_under_cursor) 207 | else: 208 | will_record_current_event = True 209 | self.events_so_far += 1 210 | self.session.did_enter_traceable_file(file_path_under_cursor) 211 | 212 | line_no = frame.f_lineno 213 | function_name = frame.f_code.co_name 214 | current = NativeCodeEvent() 215 | 216 | if event == 'call' or event == 'line' or event == 'return': 217 | file_id = self.file_map.file_id(file_path_under_cursor) 218 | current_cursor = NativeExecutionCursor() 219 | current_cursor.file = file_id 220 | current_cursor.line = line_no 221 | current_cursor.function_name = function_name 222 | 223 | if event == 'line': 224 | current.event_name = 'line' 225 | stack = self.call_stack.new_cursor_in_current_frame(current_cursor) 226 | 227 | if event == 'call': 228 | current.event_name = 'method_enter' 229 | stack = self.call_stack.enter_frame(current_cursor) 230 | if event == 'return': 231 | current.event_name = 'method_exit' 232 | self.call_stack.exit_frame() 233 | stack = self.call_stack.current_frame() 234 | if not will_record_current_event: 235 | # do not go any further 236 | return 237 | 238 | current.cursor = current_cursor 239 | current.stack = stack 240 | current.ts = entered_at 241 | if self.should_trace_variables: 242 | if event == 'line': 243 | locals_v = NativeVariables() 244 | locals_v.variables = [] 245 | self.push_traceable_variables(frame, locals_v) 246 | current.locals = locals_v 247 | 248 | if event == 'call': 249 | input_variables = NativeVariables() 250 | input_variables.variables = [] 251 | self.push_traceable_variables(frame, input_variables) 252 | current.input_variables = input_variables 253 | if event == 'return': 254 | return_variables = NativeVariables() 255 | return_variables.variables = [] 256 | return_var = NativeVariable() 257 | return_var.name = '__return' 258 | return_var.value = self.ensure_safe_for_serialization(arg) 259 | return_variables.variables.append(return_var) 260 | current.return_variables = return_variables 261 | 262 | self.add_to_event_buffer(current) 263 | 264 | self.flush_queue_if_full() 265 | 266 | cdef str ensure_safe_for_serialization(self, value): 267 | current_type = type(value) 268 | if current_type not in allowed_types: 269 | return str(current_type) 270 | # return 'a' 271 | # todo is this slowdown? 272 | return str(value) 273 | 274 | cdef push_traceable_variables(self, frame, NativeVariables locals): 275 | cdef NativeVariable current 276 | for (name, value) in frame.f_locals.items(): 277 | current = NativeVariable() 278 | current.name = name 279 | current.value = self.ensure_safe_for_serialization(value) 280 | locals.variables.append(current) 281 | 282 | def add_to_event_buffer(self, current): 283 | # todo: is this caused because of array dynamic size/doubling? 284 | # if True: 285 | # return 286 | self.event_buffer.append(current) 287 | 288 | # def create_cursor(self, file_path_under_cursor, frame): 289 | # file_id = self.file_map.file_id(file_path_under_cursor) 290 | # cursor = events.ExecutionCursor(file_id, frame.f_lineno, frame.f_code.co_name) 291 | # return cursor 292 | 293 | # def get_execution_stack(self): 294 | # return self.call_stack.current_frame() 295 | # 296 | # def push_traceable_variables(self, frame, variables): 297 | # for (name, value) in frame.f_locals.items(): 298 | # # todo use variable values diffing 299 | # variables.push_variable(name, value) 300 | 301 | def flush_outstanding_events(self): 302 | self.perf.print_avg_time() 303 | print(f'total events C: {self.event_buffer.count()}') 304 | 305 | old_buffer = self.event_buffer.finish_chunk() 306 | # self.perf.print_avg_time() 307 | # 308 | self.queue.put_events(EventsSlice(self.session_name, self.event_number, old_buffer, self.file_map.files.copy())) 309 | 310 | def finalize(self): 311 | print('finalizing native tracer') 312 | self.queue.put_file_slice(FileContentSlice(self.session_name, self.file_map.files.copy())) 313 | 314 | self.queue.tracing_did_complete( 315 | self.session_name, 316 | self.session, 317 | ) 318 | 319 | cdef flush_queue_if_full(self): 320 | if self.event_buffer.count() >= self.max_events_before_send: 321 | self.flush_outstanding_events() 322 | -------------------------------------------------------------------------------- /pycrunch_trace/demo/test_simulated_real/test_tree_call_stack_cursors.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from pycrunch_trace.filters import CustomFileFilter 4 | from pycrunch_trace.oop import Clock, File 5 | from pycrunch_trace.tracing.simulation import models 6 | from pycrunch_trace.tracing.simple_tracer import SimpleTracer 7 | from pycrunch_trace.tracing.simulator_sink import SimulationEvent 8 | 9 | 10 | def create_event_1(): 11 | code_clone = models.Code() 12 | code_clone.co_name = 'a___' 13 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 14 | code_clone.co_argcount = 0 15 | 16 | sim_frame = models.Frame() 17 | sim_frame.f_lineno = 23 18 | sim_frame.f_locals = {} 19 | sim_frame.f_code = code_clone 20 | evt = SimulationEvent(sim_frame, 'call', None) 21 | 22 | return evt 23 | 24 | 25 | def create_event_2(): 26 | code_clone = models.Code() 27 | code_clone.co_name = 'a___' 28 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 29 | code_clone.co_argcount = 0 30 | 31 | sim_frame = models.Frame() 32 | sim_frame.f_lineno = 24 33 | sim_frame.f_locals = {} 34 | sim_frame.f_code = code_clone 35 | evt = SimulationEvent(sim_frame, 'line', None) 36 | 37 | return evt 38 | 39 | 40 | def create_event_3(): 41 | code_clone = models.Code() 42 | code_clone.co_name = 'a___' 43 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 44 | code_clone.co_argcount = 0 45 | 46 | sim_frame = models.Frame() 47 | sim_frame.f_lineno = 25 48 | sim_frame.f_locals = { 49 | 'a': 1} 50 | sim_frame.f_code = code_clone 51 | evt = SimulationEvent(sim_frame, 'line', None) 52 | 53 | return evt 54 | 55 | 56 | def create_event_4(): 57 | code_clone = models.Code() 58 | code_clone.co_name = 'a___' 59 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 60 | code_clone.co_argcount = 0 61 | 62 | sim_frame = models.Frame() 63 | sim_frame.f_lineno = 26 64 | sim_frame.f_locals = { 65 | 'a': 1, 66 | 'b': 2} 67 | sim_frame.f_code = code_clone 68 | evt = SimulationEvent(sim_frame, 'line', None) 69 | 70 | return evt 71 | 72 | 73 | def create_event_5(): 74 | code_clone = models.Code() 75 | code_clone.co_name = 'b___' 76 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 77 | code_clone.co_argcount = 0 78 | 79 | sim_frame = models.Frame() 80 | sim_frame.f_lineno = 15 81 | sim_frame.f_locals = {} 82 | sim_frame.f_code = code_clone 83 | evt = SimulationEvent(sim_frame, 'call', None) 84 | 85 | return evt 86 | 87 | 88 | def create_event_6(): 89 | code_clone = models.Code() 90 | code_clone.co_name = 'b___' 91 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 92 | code_clone.co_argcount = 0 93 | 94 | sim_frame = models.Frame() 95 | sim_frame.f_lineno = 16 96 | sim_frame.f_locals = {} 97 | sim_frame.f_code = code_clone 98 | evt = SimulationEvent(sim_frame, 'line', None) 99 | 100 | return evt 101 | 102 | 103 | def create_event_7(): 104 | code_clone = models.Code() 105 | code_clone.co_name = 'b___' 106 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 107 | code_clone.co_argcount = 0 108 | 109 | sim_frame = models.Frame() 110 | sim_frame.f_lineno = 17 111 | sim_frame.f_locals = { 112 | 'a': 1} 113 | sim_frame.f_code = code_clone 114 | evt = SimulationEvent(sim_frame, 'line', None) 115 | 116 | return evt 117 | 118 | 119 | def create_event_8(): 120 | code_clone = models.Code() 121 | code_clone.co_name = 'c___' 122 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 123 | code_clone.co_argcount = 0 124 | 125 | sim_frame = models.Frame() 126 | sim_frame.f_lineno = 8 127 | sim_frame.f_locals = {} 128 | sim_frame.f_code = code_clone 129 | evt = SimulationEvent(sim_frame, 'call', None) 130 | 131 | return evt 132 | 133 | 134 | def create_event_9(): 135 | code_clone = models.Code() 136 | code_clone.co_name = 'c___' 137 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 138 | code_clone.co_argcount = 0 139 | 140 | sim_frame = models.Frame() 141 | sim_frame.f_lineno = 9 142 | sim_frame.f_locals = {} 143 | sim_frame.f_code = code_clone 144 | evt = SimulationEvent(sim_frame, 'line', None) 145 | 146 | return evt 147 | 148 | 149 | def create_event_10(): 150 | code_clone = models.Code() 151 | code_clone.co_name = 'c___' 152 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 153 | code_clone.co_argcount = 0 154 | 155 | sim_frame = models.Frame() 156 | sim_frame.f_lineno = 10 157 | sim_frame.f_locals = { 158 | 'a': 1} 159 | sim_frame.f_code = code_clone 160 | evt = SimulationEvent(sim_frame, 'line', None) 161 | 162 | return evt 163 | 164 | 165 | def create_event_11(): 166 | code_clone = models.Code() 167 | code_clone.co_name = 'c___' 168 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 169 | code_clone.co_argcount = 0 170 | 171 | sim_frame = models.Frame() 172 | sim_frame.f_lineno = 11 173 | sim_frame.f_locals = { 174 | 'a': 1, 175 | 'b': 2} 176 | sim_frame.f_code = code_clone 177 | evt = SimulationEvent(sim_frame, 'line', None) 178 | 179 | return evt 180 | 181 | 182 | def create_event_12(): 183 | code_clone = models.Code() 184 | code_clone.co_name = 'd___' 185 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 186 | code_clone.co_argcount = 0 187 | 188 | sim_frame = models.Frame() 189 | sim_frame.f_lineno = 3 190 | sim_frame.f_locals = {} 191 | sim_frame.f_code = code_clone 192 | evt = SimulationEvent(sim_frame, 'call', None) 193 | 194 | return evt 195 | 196 | 197 | def create_event_13(): 198 | code_clone = models.Code() 199 | code_clone.co_name = 'd___' 200 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 201 | code_clone.co_argcount = 0 202 | 203 | sim_frame = models.Frame() 204 | sim_frame.f_lineno = 4 205 | sim_frame.f_locals = {} 206 | sim_frame.f_code = code_clone 207 | evt = SimulationEvent(sim_frame, 'line', None) 208 | 209 | return evt 210 | 211 | 212 | def create_event_14(): 213 | code_clone = models.Code() 214 | code_clone.co_name = 'd___' 215 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 216 | code_clone.co_argcount = 0 217 | 218 | sim_frame = models.Frame() 219 | sim_frame.f_lineno = 5 220 | sim_frame.f_locals = { 221 | 'a': 1} 222 | sim_frame.f_code = code_clone 223 | evt = SimulationEvent(sim_frame, 'line', None) 224 | 225 | return evt 226 | 227 | 228 | def create_event_15(): 229 | code_clone = models.Code() 230 | code_clone.co_name = 'd___' 231 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 232 | code_clone.co_argcount = 0 233 | 234 | sim_frame = models.Frame() 235 | sim_frame.f_lineno = 6 236 | sim_frame.f_locals = { 237 | 'a': 1, 238 | 'b': 2} 239 | sim_frame.f_code = code_clone 240 | evt = SimulationEvent(sim_frame, 'line', None) 241 | 242 | return evt 243 | 244 | 245 | def create_event_16(): 246 | code_clone = models.Code() 247 | code_clone.co_name = 'd___' 248 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 249 | code_clone.co_argcount = 0 250 | 251 | sim_frame = models.Frame() 252 | sim_frame.f_lineno = 6 253 | sim_frame.f_locals = { 254 | 'a': 1, 255 | 'b': 2, 256 | 'c': 3} 257 | sim_frame.f_code = code_clone 258 | evt = SimulationEvent(sim_frame, 'return', None) 259 | 260 | return evt 261 | 262 | 263 | def create_event_17(): 264 | code_clone = models.Code() 265 | code_clone.co_name = 'c___' 266 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 267 | code_clone.co_argcount = 0 268 | 269 | sim_frame = models.Frame() 270 | sim_frame.f_lineno = 12 271 | sim_frame.f_locals = { 272 | 'a': 1, 273 | 'b': 2} 274 | sim_frame.f_code = code_clone 275 | evt = SimulationEvent(sim_frame, 'line', None) 276 | 277 | return evt 278 | 279 | 280 | def create_event_18(): 281 | code_clone = models.Code() 282 | code_clone.co_name = 'c___' 283 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 284 | code_clone.co_argcount = 0 285 | 286 | sim_frame = models.Frame() 287 | sim_frame.f_lineno = 12 288 | sim_frame.f_locals = { 289 | 'a': 1, 290 | 'b': 2, 291 | 'c': 3} 292 | sim_frame.f_code = code_clone 293 | evt = SimulationEvent(sim_frame, 'return', None) 294 | 295 | return evt 296 | 297 | 298 | def create_event_19(): 299 | code_clone = models.Code() 300 | code_clone.co_name = 'b___' 301 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 302 | code_clone.co_argcount = 0 303 | 304 | sim_frame = models.Frame() 305 | sim_frame.f_lineno = 18 306 | sim_frame.f_locals = { 307 | 'a': 1} 308 | sim_frame.f_code = code_clone 309 | evt = SimulationEvent(sim_frame, 'line', None) 310 | 311 | return evt 312 | 313 | 314 | def create_event_20(): 315 | code_clone = models.Code() 316 | code_clone.co_name = 'b___' 317 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 318 | code_clone.co_argcount = 0 319 | 320 | sim_frame = models.Frame() 321 | sim_frame.f_lineno = 19 322 | sim_frame.f_locals = { 323 | 'a': 1, 324 | 'b': 2} 325 | sim_frame.f_code = code_clone 326 | evt = SimulationEvent(sim_frame, 'line', None) 327 | 328 | return evt 329 | 330 | 331 | def create_event_21(): 332 | code_clone = models.Code() 333 | code_clone.co_name = 'b___' 334 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 335 | code_clone.co_argcount = 0 336 | 337 | sim_frame = models.Frame() 338 | sim_frame.f_lineno = 19 339 | sim_frame.f_locals = { 340 | 'a': 1, 341 | 'b': 2, 342 | 'c': 3} 343 | sim_frame.f_code = code_clone 344 | evt = SimulationEvent(sim_frame, 'return', None) 345 | 346 | return evt 347 | 348 | 349 | def create_event_22(): 350 | code_clone = models.Code() 351 | code_clone.co_name = 'a___' 352 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 353 | code_clone.co_argcount = 0 354 | 355 | sim_frame = models.Frame() 356 | sim_frame.f_lineno = 27 357 | sim_frame.f_locals = { 358 | 'a': 1, 359 | 'b': 2} 360 | sim_frame.f_code = code_clone 361 | evt = SimulationEvent(sim_frame, 'line', None) 362 | 363 | return evt 364 | 365 | 366 | def create_event_23(): 367 | code_clone = models.Code() 368 | code_clone.co_name = 'a___' 369 | code_clone.co_filename = '/pycrunch_tracer/demo/demo_tree_v1.py' 370 | code_clone.co_argcount = 0 371 | 372 | sim_frame = models.Frame() 373 | sim_frame.f_lineno = 27 374 | sim_frame.f_locals = { 375 | 'a': 1, 376 | 'b': 2, 377 | 'c': 3} 378 | sim_frame.f_code = code_clone 379 | evt = SimulationEvent(sim_frame, 'return', None) 380 | 381 | return evt 382 | 383 | 384 | def create_event_24(): 385 | code_clone = models.Code() 386 | code_clone.co_name = 'stop' 387 | code_clone.co_filename = '/pycrunch_tracer/client/api/tracing.py' 388 | code_clone.co_argcount = 1 389 | 390 | sim_frame = models.Frame() 391 | sim_frame.f_lineno = 70 392 | sim_frame.f_locals = { 393 | 'self': '< pycrunch_tracer.api.tracing'} 394 | 395 | sim_frame.f_code = code_clone 396 | evt = SimulationEvent(sim_frame, 'call', None) 397 | 398 | return evt 399 | 400 | 401 | def create_event_25(): 402 | code_clone = models.Code() 403 | code_clone.co_name = 'stop' 404 | code_clone.co_filename = '/pycrunch_tracer/client/api/tracing.py' 405 | code_clone.co_argcount = 1 406 | 407 | sim_frame = models.Frame() 408 | sim_frame.f_lineno = 71 409 | sim_frame.f_locals = { 'self': '< pycrunch_tracer.api.tracing'} 410 | sim_frame.f_code = code_clone 411 | evt = SimulationEvent(sim_frame, 'line', None) 412 | 413 | return evt 414 | 415 | 416 | def test_simulated(): 417 | events = [] 418 | events.append(create_event_1()) 419 | events.append(create_event_2()) 420 | events.append(create_event_3()) 421 | events.append(create_event_4()) 422 | events.append(create_event_5()) 423 | events.append(create_event_6()) 424 | events.append(create_event_7()) 425 | events.append(create_event_8()) 426 | events.append(create_event_9()) 427 | events.append(create_event_10()) 428 | events.append(create_event_11()) 429 | events.append(create_event_12()) 430 | events.append(create_event_13()) 431 | events.append(create_event_14()) 432 | events.append(create_event_15()) 433 | events.append(create_event_16()) 434 | events.append(create_event_17()) 435 | events.append(create_event_18()) 436 | events.append(create_event_19()) 437 | events.append(create_event_20()) 438 | events.append(create_event_21()) 439 | events.append(create_event_22()) 440 | events.append(create_event_23()) 441 | events.append(create_event_24()) 442 | events.append(create_event_25()) 443 | package_directory = Path(__file__).parent.parent.parent 444 | profile_name = 'default.profile.yaml' 445 | 446 | f_filter = CustomFileFilter(File(package_directory.joinpath('pycrunch-profiles', profile_name))) 447 | 448 | sut = SimpleTracer(events, 'sim_round2', f_filter, Clock(), self.outgoingQueue) 449 | for (i, x) in enumerate(events): 450 | if i > 24: 451 | break 452 | print(i) 453 | sut.simple_tracer(x.frame, x.event, x.arg) 454 | 455 | --------------------------------------------------------------------------------