├── .gitignore ├── LICENSE ├── README.txt ├── run_tests ├── setup.py ├── trace_event.py └── trace_event_impl ├── __init__.py ├── decorators.py ├── decorators_test.py ├── log.py ├── log_io_test.py ├── log_multiprocess_io_test.py ├── log_single_thread_test.py ├── multiprocessing_shim.py ├── multiprocessing_shim_test.py ├── parsed_trace_events.py └── trace_test.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | \#*# 3 | .#* 4 | *.swp 5 | third_party/chrome 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The Chromium Authors. All rights reserved. 2 | // 3 | // Redistribution and use in source and binary forms, with or without 4 | // modification, are permitted provided that the following conditions are 5 | // met: 6 | // 7 | // * Redistributions of source code must retain the above copyright 8 | // notice, this list of conditions and the following disclaimer. 9 | // * Redistributions in binary form must reproduce the above 10 | // copyright notice, this list of conditions and the following disclaimer 11 | // in the documentation and/or other materials provided with the 12 | // distribution. 13 | // * Neither the name of Google Inc. nor the names of its 14 | // contributors may be used to endorse or promote products derived from 15 | // this software without specific prior written permission. 16 | // 17 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | py_trace_event allows low-overhead instrumentation of a multi-threaded, 2 | multi-process application in order to study its global performance 3 | characteristics. It uses the trace event format used in Chromium/Chrome's 4 | about:tracing system. 5 | 6 | Trace files generated by py_trace_event can be viewed and manipulated by 7 | trace_event_viewer. 8 | -------------------------------------------------------------------------------- /run_tests: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2011 The Chromium Authors. All rights reserved. 3 | # Use of this source code is governed by a BSD-style license that can be 4 | # found in the LICENSE file. 5 | import logging 6 | import optparse 7 | import os 8 | import platform 9 | import re 10 | import sys 11 | import types 12 | import traceback 13 | import unittest 14 | 15 | def discover(dir, filters): 16 | if hasattr(unittest.TestLoader, 'discover'): 17 | return unittest.TestLoader().discover(dir, '*') 18 | 19 | # poor mans unittest.discover 20 | loader = unittest.TestLoader() 21 | subsuites = [] 22 | 23 | for (dirpath, dirnames, filenames) in os.walk(dir): 24 | for filename in [x for x in filenames if re.match('.*_test\.py$', x)]: 25 | if filename.startswith('.') or filename.startswith('_'): 26 | continue 27 | fqn = dirpath.replace('/', '.') + '.' + re.match('(.+)\.py$', filename).group(1) 28 | 29 | # load the test 30 | try: 31 | module = __import__(fqn,fromlist=[True]) 32 | except: 33 | print "While importing [%s]\n" % fqn 34 | traceback.print_exc() 35 | continue 36 | 37 | def test_is_selected(name): 38 | for f in filters: 39 | if re.search(f,name): 40 | return True 41 | return False 42 | 43 | if hasattr(module, 'suite'): 44 | base_suite = module.suite() 45 | else: 46 | base_suite = loader.loadTestsFromModule(module) 47 | new_suite = unittest.TestSuite() 48 | for t in base_suite: 49 | if isinstance(t, unittest.TestSuite): 50 | for i in t: 51 | if test_is_selected(i.id()): 52 | new_suite.addTest(i) 53 | elif isinstance(t, unittest.TestCase): 54 | if test_is_selected(t.id()): 55 | new_suite.addTest(t) 56 | else: 57 | raise Exception("Wtf, expected TestSuite or TestCase, got %s" % t) 58 | 59 | if new_suite.countTestCases(): 60 | subsuites.append(new_suite) 61 | 62 | return unittest.TestSuite(subsuites) 63 | 64 | def main(): 65 | parser = optparse.OptionParser() 66 | parser.add_option( 67 | '-v', '--verbose', action='count', default=0, 68 | help='Increase verbosity level (repeat as needed)') 69 | parser.add_option('--debug', dest='debug', action='store_true', default=False, help='Break into pdb when an assertion fails') 70 | parser.add_option('--incremental', dest='incremental', action='store_true', default=False, help='Run tests one at a time.') 71 | parser.add_option('--stop', dest='stop_on_error', action='store_true', default=False, help='Stop running tests on error.') 72 | (options, args) = parser.parse_args() 73 | 74 | if options.verbose >= 2: 75 | logging.basicConfig(level=logging.DEBUG) 76 | elif options.verbose: 77 | logging.basicConfig(level=logging.INFO) 78 | else: 79 | logging.basicConfig(level=logging.WARNING) 80 | 81 | # install hook on set_trace if --debug 82 | if options.debug: 83 | import exceptions 84 | class DebuggingAssertionError(exceptions.AssertionError): 85 | def __init__(self, *args): 86 | exceptions.AssertionError.__init__(self, *args) 87 | print "Assertion failed, entering PDB..." 88 | import pdb 89 | if hasattr(sys, '_getframe'): 90 | pdb.Pdb().set_trace(sys._getframe().f_back.f_back) 91 | else: 92 | pdb.set_trace() 93 | unittest.TestCase.failureException = DebuggingAssertionError 94 | 95 | def hook(*args): 96 | import traceback, pdb 97 | traceback.print_exception(*args) 98 | pdb.pm() 99 | sys.excepthook = hook 100 | 101 | import browser 102 | browser.debug_mode = True 103 | 104 | else: 105 | def hook(exc, value, tb): 106 | import traceback 107 | if not str(value).startswith("_noprint"): 108 | traceback.print_exception(exc, value, tb) 109 | import src.message_loop 110 | if src.message_loop.is_main_loop_running(): 111 | if not str(value).startswith("_noprint"): 112 | print "Untrapped exception! Exiting message loop with exception." 113 | src.message_loop.quit_main_loop(quit_with_exception=True) 114 | 115 | sys.excepthook = hook 116 | 117 | # make sure cwd is the base directory! 118 | os.chdir(os.path.dirname(__file__)) 119 | 120 | if len(args) > 0: 121 | suites = discover('trace_event_impl', args) 122 | else: 123 | suites = discover('trace_event_impl', ['.*']) 124 | 125 | r = unittest.TextTestRunner() 126 | if not options.incremental: 127 | res = r.run(suites) 128 | if res.wasSuccessful(): 129 | return 0 130 | return 255 131 | else: 132 | ok = True 133 | for s in suites: 134 | if isinstance(s, unittest.TestSuite): 135 | for t in s: 136 | print '----------------------------------------------------------------------' 137 | print 'Running %s' % str(t) 138 | res = r.run(t) 139 | if not res.wasSuccessful(): 140 | ok = False 141 | if options.stop_on_error: 142 | break 143 | if ok == False and options.stop_on_error: 144 | break 145 | else: 146 | res = r.run(s) 147 | if not res.wasSuccessful(): 148 | ok = False 149 | if options.stop_on_error: 150 | break 151 | if ok: 152 | return 0 153 | return 255 154 | 155 | if __name__ == "__main__": 156 | sys.exit(main()) 157 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2011 The Chromium Authors. All rights reserved. 3 | # Use of this source code is governed by a BSD-style license that can be 4 | # found in the LICENSE file. 5 | from distutils.core import setup 6 | setup( 7 | name = 'py_trace_event', 8 | packages = ['trace_event_impl'], 9 | version = '0.1.0', 10 | description = 'Performance tracing for python', 11 | author='Nat Duca' 12 | ) 13 | -------------------------------------------------------------------------------- /trace_event.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Chromium Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be 3 | # found in the LICENSE file. 4 | r"""Instrumentation-based profiling for Python. 5 | 6 | trace_event allows you to hand-instrument your code with areas of interest. 7 | When enabled, trace_event logs the start and stop times of these events to a 8 | logfile. These resulting logfiles can be viewed with either Chrome's about:tracing 9 | UI or with the standalone trace_event_viewer available at 10 | http://www.github.com/natduca/trace_event_viewer/ 11 | 12 | To use trace event, simply call trace_event_enable and start instrumenting your code: 13 | from trace_event import * 14 | 15 | if "--trace" in sys.argv: 16 | trace_enable("myfile.trace") 17 | 18 | @traced 19 | def foo(): 20 | ... 21 | 22 | class MyFoo(object): 23 | @traced 24 | def bar(self): 25 | ... 26 | 27 | trace_event records trace events to an in-memory buffer. If your application is 28 | long running and you want to see the results of a trace before it exits, you can call 29 | trace_flush to write any in-memory events to disk. 30 | 31 | To help intregrating trace_event into existing codebases that dont want to add 32 | trace_event as a dependancy, trace_event is split into an import shim 33 | (trace_event.py) and an implementaiton (trace_event_impl/*). You can copy the 34 | shim, trace_event.py, directly into your including codebase. If the 35 | trace_event_impl is not found, the shim will simply noop. 36 | 37 | trace_event is safe with regard to Python threads. Simply trace as you normally would and each 38 | thread's timing will show up in the trace file. 39 | 40 | Multiple processes can safely output into a single trace_event logfile. If you 41 | fork after enabling tracing, the child process will continue outputting to the 42 | logfile. Use of the multiprocessing module will work as well. In both cases, 43 | however, note that disabling tracing in the parent process will not stop tracing 44 | in the child processes. 45 | """ 46 | 47 | try: 48 | import trace_event_impl 49 | except ImportError: 50 | trace_event_impl = None 51 | 52 | def trace_can_enable(): 53 | """ 54 | Returns True if a trace_event_impl was found. If false, 55 | trace_enable will fail. Regular tracing methods, including 56 | trace_begin and trace_end, will simply be no-ops. 57 | """ 58 | return trace_event_impl != None 59 | 60 | if trace_event_impl: 61 | import time 62 | 63 | def trace_is_enabled(): 64 | return trace_event_impl.trace_is_enabled() 65 | 66 | def trace_enable(logfile): 67 | return trace_event_impl.trace_enable(logfile) 68 | 69 | def trace_disable(): 70 | return trace_event_impl.trace_disable() 71 | 72 | def trace_flush(): 73 | trace_event_impl.trace_flush() 74 | 75 | def trace_begin(name, **kwargs): 76 | args_to_log = {key: repr(value) for key, value in kwargs.iteritems()} 77 | trace_event_impl.add_trace_event("B", time.time(), "python", name, args_to_log) 78 | 79 | def trace_end(name): 80 | trace_event_impl.add_trace_event("E", time.time(), "python", name) 81 | 82 | def trace(name, **kwargs): 83 | return trace_event_impl.trace(name, **kwargs) 84 | 85 | def traced(fn): 86 | return trace_event_impl.traced(fn) 87 | 88 | else: 89 | import contextlib 90 | 91 | def trace_enable(): 92 | raise TraceException("Cannot enable trace_event. No trace_event_impl module found.") 93 | 94 | def trace_disable(): 95 | pass 96 | 97 | def trace_is_enabled(): 98 | return False 99 | 100 | def trace_flush(): 101 | pass 102 | 103 | def trace_begin(self, name, **kwargs): 104 | pass 105 | 106 | def trace_end(self, name): 107 | pass 108 | 109 | @contextlib.contextmanager 110 | def trace(name, **kwargs): 111 | yield 112 | 113 | def traced(fn): 114 | return fn 115 | 116 | 117 | trace_enable.__doc__ = """Enables tracing. 118 | 119 | Once enabled, the enabled bit propagates to forked processes and 120 | multiprocessing subprocesses. Regular child processes, e.g. those created via 121 | os.system/popen, or subprocess.Popen instances, will not get traced. You can, 122 | however, enable tracing on those subprocess manually. 123 | 124 | Trace files are multiprocess safe, so you can have multiple processes 125 | outputting to the same tracelog at once. 126 | 127 | log_file can be one of three things: 128 | 129 | None: a logfile is opened based on sys[argv], namely 130 | "./" + sys.argv[0] + ".json" 131 | 132 | string: a logfile of the given name is opened. 133 | 134 | file-like object: the fileno() is is used. The underlying file descriptor 135 | must support fcntl.lockf() operations. 136 | """ 137 | 138 | trace_disable.__doc__ = """Disables tracing, if enabled. 139 | 140 | Will not disable tracing on any existing child proceses that were forked 141 | from this process. You must disable them yourself. 142 | """ 143 | 144 | trace_flush.__doc__ = """Flushes any currently-recorded trace data to disk. 145 | 146 | trace_event records traces into an in-memory buffer for efficiency. Flushing 147 | is only done at process exit or when this method is called. 148 | """ 149 | 150 | trace_is_enabled.__doc__ = """Returns whether tracing is enabled. 151 | """ 152 | 153 | trace_begin.__doc__ = """Records the beginning of an event of the given name. 154 | 155 | The building block for performance tracing. A typical example is: 156 | from trace_event import * 157 | def something_heavy(): 158 | trace_begin("something_heavy") 159 | 160 | trace_begin("read") 161 | try: 162 | lines = open().readlines() 163 | finally: 164 | trace_end("read") 165 | 166 | trace_begin("parse") 167 | try: 168 | parse(lines) 169 | finally: 170 | trace_end("parse") 171 | 172 | trace_end("something_heavy") 173 | 174 | Note that a trace_end call must be issued for every trace_begin call. When 175 | tracing around blocks that might throw exceptions, you should use the trace function, 176 | or a try-finally pattern to ensure that the trace_end method is called. 177 | 178 | See the documentation for the @traced decorator for a simpler way to instrument 179 | functions and methods. 180 | """ 181 | 182 | trace_end.__doc__ = """Records the end of an event of the given name. 183 | 184 | See the documentation for trace_begin for more information. 185 | 186 | Make sure to issue a trace_end for every trace_begin issued. Failure to pair 187 | these calls will lead to bizarrely tall looking traces in the 188 | trace_event_viewer UI. 189 | """ 190 | 191 | trace.__doc__ = """Traces a block of code using a with statement. 192 | 193 | Example usage: 194 | from trace_event import * 195 | def something_heavy(lines): 196 | with trace("parse_lines", lines=lines): 197 | parse(lines) 198 | 199 | If tracing an entire function call, prefer the @traced decorator. 200 | """ 201 | 202 | traced.__doc__ = """ 203 | Traces the provided function, using the function name for the actual generated event. 204 | 205 | Prefer this decorator over the explicit trace_begin and trace_end functions 206 | whenever you are tracing the start and stop of a function. It automatically 207 | issues trace_begin/end events, even when the wrapped function throws. 208 | 209 | You can also pass the function's argument names to traced, and the argument 210 | values will be added to the trace. Example usage: 211 | from trace_event import * 212 | @traced("url") 213 | def send_request(url): 214 | urllib2.urlopen(url).read() 215 | """ 216 | -------------------------------------------------------------------------------- /trace_event_impl/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Chromium Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be 3 | # found in the LICENSE file. 4 | from log import * 5 | from decorators import * 6 | import multiprocessing_shim 7 | -------------------------------------------------------------------------------- /trace_event_impl/decorators.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Chromium Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be 3 | # found in the LICENSE file. 4 | import contextlib 5 | import inspect 6 | import time 7 | import functools 8 | 9 | import log 10 | 11 | @contextlib.contextmanager 12 | def trace(name, **kwargs): 13 | category = "python" 14 | start = time.time() 15 | args_to_log = {key: repr(value) for key, value in kwargs.iteritems()} 16 | log.add_trace_event("B", start, category, name, args_to_log) 17 | try: 18 | yield 19 | finally: 20 | end = time.time() 21 | log.add_trace_event("E", end, category, name) 22 | 23 | def traced(*args): 24 | def get_wrapper(func): 25 | if inspect.isgeneratorfunction(func): 26 | raise Exception("Can not trace generators.") 27 | 28 | category = "python" 29 | 30 | arg_spec = inspect.getargspec(func) 31 | is_method = arg_spec.args and arg_spec.args[0] == "self" 32 | 33 | def arg_spec_tuple(name): 34 | arg_index = arg_spec.args.index(name) 35 | defaults_length = len(arg_spec.defaults) if arg_spec.defaults else 0 36 | default_index = arg_index + defaults_length - len(arg_spec.args) 37 | if default_index >= 0: 38 | default = arg_spec.defaults[default_index] 39 | else: 40 | default = None 41 | return (name, arg_index, default) 42 | 43 | args_to_log = map(arg_spec_tuple, arg_names) 44 | 45 | @functools.wraps(func) 46 | def traced_function(*args, **kwargs): 47 | # Everything outside traced_function is done at decoration-time. 48 | # Everything inside traced_function is done at run-time and must be fast. 49 | if not log._enabled: # This check must be at run-time. 50 | return func(*args, **kwargs) 51 | 52 | def get_arg_value(name, index, default): 53 | if name in kwargs: 54 | return kwargs[name] 55 | elif index < len(args): 56 | return args[index] 57 | else: 58 | return default 59 | 60 | if is_method: 61 | name = "%s.%s" % (args[0].__class__.__name__, func.__name__) 62 | else: 63 | name = "%s.%s" % (func.__module__, func.__name__) 64 | 65 | # Be sure to repr before calling func, because the argument values may change. 66 | arg_values = { 67 | name: repr(get_arg_value(name, index, default)) 68 | for name, index, default in args_to_log} 69 | 70 | start = time.time() 71 | log.add_trace_event("B", start, category, name, arg_values) 72 | try: 73 | return func(*args, **kwargs) 74 | finally: 75 | end = time.time() 76 | log.add_trace_event("E", end, category, name) 77 | return traced_function 78 | 79 | no_decorator_arguments = len(args) == 1 and callable(args[0]) 80 | if no_decorator_arguments: 81 | arg_names = () 82 | return get_wrapper(args[0]) 83 | else: 84 | arg_names = args 85 | return get_wrapper 86 | -------------------------------------------------------------------------------- /trace_event_impl/decorators_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Chromium Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be 3 | # found in the LICENSE file. 4 | import decorators 5 | import unittest 6 | from .trace_test import TraceTest 7 | 8 | def generator(): 9 | yield 1 10 | yield 2 11 | 12 | class DecoratorTests(unittest.TestCase): 13 | def test_tracing_object_fails(self): 14 | self.assertRaises(Exception, lambda: decorators.trace(1)) 15 | self.assertRaises(Exception, lambda: decorators.trace("")) 16 | self.assertRaises(Exception, lambda: decorators.trace([])) 17 | 18 | def test_tracing_generators_fail(self): 19 | self.assertRaises(Exception, lambda: decorators.trace(generator)) 20 | 21 | class ClassToTest(object): 22 | @decorators.traced 23 | def method1(self): 24 | return 1 25 | 26 | @decorators.traced 27 | def method2(self): 28 | return 1 29 | 30 | @decorators.traced 31 | def traced_func(): 32 | return 1 33 | 34 | class DecoratorTests(TraceTest): 35 | def _get_decorated_method_name(self, f): 36 | res = self.go(f) 37 | events = res.findEventsOnThread(res.findThreadIds()[0]) 38 | 39 | # Sanity checks. 40 | self.assertEquals(2, len(events)) 41 | self.assertEquals(events[0]["name"], events[1]["name"]) 42 | return events[1]["name"] 43 | 44 | 45 | def test_func_names_work(self): 46 | self.assertEquals('traced_func', self._get_decorated_method_name(traced_func)) 47 | 48 | def test_method_names_work(self): 49 | ctt = ClassToTest() 50 | self.assertEquals('method1', self._get_decorated_method_name(ctt.method1)) 51 | self.assertEquals('ClassToTest.method2', self._get_decorated_method_name(ctt.method2)) 52 | -------------------------------------------------------------------------------- /trace_event_impl/log.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Chromium Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be 3 | # found in the LICENSE file. 4 | import atexit 5 | import fcntl 6 | import json 7 | import os 8 | import sys 9 | import time 10 | import threading 11 | 12 | _lock = threading.Lock() 13 | 14 | _enabled = False 15 | _log_file = None 16 | 17 | _cur_events = [] # events that have yet to be buffered 18 | 19 | _tls = threading.local() # tls used to detect forking/etc 20 | _atexit_regsitered_for_pid = None 21 | 22 | _control_allowed = True 23 | 24 | class TraceException(Exception): 25 | pass 26 | 27 | def _note(msg, *args): 28 | pass 29 | # print "%i: %s" % (os.getpid(), msg) 30 | 31 | 32 | def _locked(fn): 33 | def locked_fn(*args,**kwargs): 34 | _lock.acquire() 35 | try: 36 | ret = fn(*args,**kwargs) 37 | finally: 38 | _lock.release() 39 | return ret 40 | return locked_fn 41 | 42 | def _disallow_tracing_control(): 43 | global _control_allowed 44 | _control_allowed = False 45 | 46 | def trace_enable(log_file=None): 47 | _trace_enable(log_file) 48 | 49 | @_locked 50 | def _trace_enable(log_file=None): 51 | global _enabled 52 | if _enabled: 53 | raise TraceException("Already enabled") 54 | if not _control_allowed: 55 | raise TraceException("Tracing control not allowed in child processes.") 56 | _enabled = True 57 | global _log_file 58 | if log_file == None: 59 | if sys.argv[0] == '': 60 | n = 'trace_event' 61 | else: 62 | n = sys.argv[0] 63 | log_file = open("%s.json" % n, "ab", False) 64 | _note("trace_event: tracelog name is %s.json" % n) 65 | elif isinstance(log_file, basestring): 66 | _note("trace_event: tracelog name is %s" % log_file) 67 | log_file = open("%s" % log_file, "ab", False) 68 | elif not hasattr(log_file, 'fileno'): 69 | raise TraceException("Log file must be None, a string, or a file-like object with a fileno()") 70 | 71 | _log_file = log_file 72 | fcntl.lockf(_log_file.fileno(), fcntl.LOCK_EX) 73 | _log_file.seek(0, os.SEEK_END) 74 | 75 | lastpos = _log_file.tell() 76 | creator = lastpos == 0 77 | if creator: 78 | _note("trace_event: Opened new tracelog, lastpos=%i", lastpos) 79 | _log_file.write('[') 80 | 81 | tid = threading.current_thread().ident 82 | if not tid: 83 | tid = os.getpid() 84 | x = {"ph": "M", "category": "process_argv", 85 | "pid": os.getpid(), "tid": threading.current_thread().ident, 86 | "ts": time.time(), 87 | "name": "process_argv", "args": {"argv": sys.argv}} 88 | _log_file.write("%s\n" % json.dumps(x)) 89 | else: 90 | _note("trace_event: Opened existing tracelog") 91 | _log_file.flush() 92 | fcntl.lockf(_log_file.fileno(), fcntl.LOCK_UN) 93 | 94 | @_locked 95 | def trace_flush(): 96 | if _enabled: 97 | _flush() 98 | 99 | @_locked 100 | def trace_disable(): 101 | global _enabled 102 | if not _control_allowed: 103 | raise TraceException("Tracing control not allowed in child processes.") 104 | if not _enabled: 105 | return 106 | _enabled = False 107 | _flush(close=True) 108 | 109 | def _flush(close=False): 110 | global _log_file 111 | fcntl.lockf(_log_file.fileno(), fcntl.LOCK_EX) 112 | _log_file.seek(0, os.SEEK_END) 113 | if len(_cur_events): 114 | _log_file.write(",\n") 115 | _log_file.write(",\n".join([json.dumps(e) for e in _cur_events])) 116 | del _cur_events[:] 117 | 118 | if close: 119 | # We might not be the only process writing to this logfile. So, 120 | # we will simply close the file rather than writign the trailing ] that 121 | # it technically requires. The trace viewer understands that this may happen 122 | # and will insert a trailing ] during loading. 123 | pass 124 | _log_file.flush() 125 | fcntl.lockf(_log_file.fileno(), fcntl.LOCK_UN) 126 | 127 | if close: 128 | _note("trace_event: Closed") 129 | _log_file.close() 130 | _log_file = None 131 | else: 132 | _note("trace_event: Flushed") 133 | 134 | @_locked 135 | def trace_is_enabled(): 136 | return _enabled 137 | 138 | @_locked 139 | def add_trace_event(ph, ts, category, name, args=None): 140 | global _enabled 141 | if not _enabled: 142 | return 143 | if not hasattr(_tls, 'pid') or _tls.pid != os.getpid(): 144 | _tls.pid = os.getpid() 145 | global _atexit_regsitered_for_pid 146 | if _tls.pid != _atexit_regsitered_for_pid: 147 | _atexit_regsitered_for_pid = _tls.pid 148 | atexit.register(_trace_disable_atexit) 149 | _tls.pid = os.getpid() 150 | del _cur_events[:] # we forked, clear the event buffer! 151 | tid = threading.current_thread().ident 152 | if not tid: 153 | tid = os.getpid() 154 | _tls.tid = tid 155 | 156 | if ts: 157 | ts = 1000000 * ts 158 | _cur_events.append({"ph": ph, "category": category, 159 | "pid": _tls.pid, "tid": _tls.tid, 160 | "ts": ts, 161 | "name": name, "args": args or {}}); 162 | 163 | def trace_begin(name, args=None): 164 | add_trace_event("B", time.time(), "python", name, args) 165 | 166 | def trace_end(name, args=None): 167 | add_trace_event("E", time.time(), "python", name, args) 168 | 169 | def _trace_disable_atexit(): 170 | trace_disable() 171 | -------------------------------------------------------------------------------- /trace_event_impl/log_io_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Chromium Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be 3 | # found in the LICENSE file. 4 | import os 5 | import sys 6 | import tempfile 7 | import unittest 8 | 9 | 10 | from .log import * 11 | from .parsed_trace_events import * 12 | 13 | class LogIOTest(unittest.TestCase): 14 | def test_enable_with_file(self): 15 | file = tempfile.NamedTemporaryFile() 16 | trace_enable(open(file.name, 'w+')) 17 | trace_disable() 18 | e = ParsedTraceEvents(trace_filename = file.name) 19 | file.close() 20 | self.assertTrue(len(e) > 0) 21 | 22 | def test_enable_with_filename(self): 23 | file = tempfile.NamedTemporaryFile() 24 | trace_enable(file.name) 25 | trace_disable() 26 | e = ParsedTraceEvents(trace_filename = file.name) 27 | file.close() 28 | self.assertTrue(len(e) > 0) 29 | 30 | def test_enable_with_implicit_filename(self): 31 | expected_filename = "%s.json" % sys.argv[0] 32 | def do_work(): 33 | file = tempfile.NamedTemporaryFile() 34 | trace_enable() 35 | trace_disable() 36 | e = ParsedTraceEvents(trace_filename = expected_filename) 37 | file.close() 38 | self.assertTrue(len(e) > 0) 39 | try: 40 | do_work() 41 | finally: 42 | if os.path.exists(expected_filename): 43 | os.unlink(expected_filename) 44 | -------------------------------------------------------------------------------- /trace_event_impl/log_multiprocess_io_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Chromium Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be 3 | # found in the LICENSE file. 4 | import multiprocessing 5 | import tempfile 6 | import time 7 | import subprocess 8 | import sys 9 | import unittest 10 | from .log import * 11 | from .trace_test import * 12 | 13 | import os 14 | 15 | class LogMultipleProcessIOTest(TraceTest): 16 | def setUp(self): 17 | self.proc = None 18 | 19 | def tearDown(self): 20 | if self.proc != None: 21 | self.proc.kill() 22 | 23 | # Test that starting a subprocess to record into an existing tracefile works. 24 | def test_one_subprocess(self): 25 | def test(): 26 | trace_begin("parent") 27 | proc = subprocess.Popen([sys.executable, "-m", __name__, self.trace_filename, "_test_one_subprocess_child"]) 28 | proc.wait() 29 | trace_end("parent") 30 | res = self.go(test) 31 | parent_events = res.findByName('parent') 32 | child_events = res.findByName('child') 33 | self.assertEquals(2, len(parent_events)) 34 | self.assertEquals(2, len(child_events)) 35 | 36 | def _test_one_subprocess_child(self): 37 | trace_begin("child") 38 | time.sleep(0.2) 39 | trace_end("child") 40 | 41 | if __name__ == "__main__": 42 | if len(sys.argv) != 3: 43 | raise Exception("Expected: method name") 44 | trace_enable(sys.argv[1]) 45 | t = LogMultipleProcessIOTest(sys.argv[2]) 46 | t.run() 47 | trace_disable() 48 | -------------------------------------------------------------------------------- /trace_event_impl/log_single_thread_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Chromium Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be 3 | # found in the LICENSE file. 4 | import json 5 | import math 6 | import tempfile 7 | import time 8 | import unittest 9 | from .log import * 10 | from .trace_test import * 11 | 12 | class SingleThreadTest(TraceTest): 13 | def test_one_func(self): 14 | actual_diff = [] 15 | def func1(): 16 | trace_begin("func1") 17 | start = time.time() 18 | time.sleep(0.25) 19 | end = time.time() 20 | actual_diff.append(end-start) # Pass via array because of Python scoping 21 | trace_end("func1") 22 | 23 | res = self.go(func1) 24 | tids = res.findThreadIds() 25 | self.assertEquals(1, len(tids)) 26 | events = res.findEventsOnThread(tids[0]) 27 | self.assertEquals(2, len(events)) 28 | self.assertEquals('B', events[0]["ph"]) 29 | self.assertEquals('E', events[1]["ph"]) 30 | measured_diff = events[1]["ts"] - events[0]["ts"] 31 | actual_diff = 1000000 * actual_diff[0] 32 | self.assertTrue(math.fabs(actual_diff - measured_diff) < 1000) 33 | 34 | def test_redundant_flush(self): 35 | def func1(): 36 | trace_begin("func1") 37 | time.sleep(0.25) 38 | trace_flush() 39 | trace_flush() 40 | trace_end("func1") 41 | 42 | res = self.go(func1) 43 | events = res.findEventsOnThread(res.findThreadIds()[0]) 44 | self.assertEquals(2, len(events)) 45 | self.assertEquals('B', events[0]["ph"]) 46 | self.assertEquals('E', events[1]["ph"]) 47 | 48 | def test_nested_func(self): 49 | def func1(): 50 | trace_begin("func1") 51 | time.sleep(0.25) 52 | func2() 53 | trace_end("func1") 54 | 55 | def func2(): 56 | trace_begin("func2") 57 | time.sleep(0.05) 58 | trace_end("func2") 59 | 60 | res = self.go(func1) 61 | self.assertEquals(1, len(res.findThreadIds())) 62 | 63 | tids = res.findThreadIds() 64 | self.assertEquals(1, len(tids)) 65 | events = res.findEventsOnThread(tids[0]) 66 | efmt = ["%s %s" % (e["ph"], e["name"]) for e in events] 67 | self.assertEquals( 68 | ["B func1", 69 | "B func2", 70 | "E func2", 71 | "E func1"], 72 | efmt); 73 | 74 | -------------------------------------------------------------------------------- /trace_event_impl/multiprocessing_shim.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Chromium Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be 3 | # found in the LICENSE file. 4 | import multiprocessing 5 | import log 6 | import time 7 | 8 | _RealProcess = multiprocessing.Process 9 | 10 | __all__ = [] 11 | 12 | class ProcessSubclass(_RealProcess): 13 | def __init__(self, shim, *args, **kwards): 14 | _RealProcess.__init__(self, *args, **kwards) 15 | self._shim = shim 16 | 17 | def run(self,*args,**kwargs): 18 | log._disallow_tracing_control() 19 | try: 20 | r = _RealProcess.run(self, *args, **kwargs) 21 | finally: 22 | if log.trace_is_enabled(): 23 | log.trace_flush() # todo, reduce need for this... 24 | return r 25 | 26 | class ProcessShim(): 27 | def __init__(self, group=None, target=None, name=None, args=(), kwargs={}): 28 | self._proc = ProcessSubclass(self, group, target, name, args, kwargs) 29 | # hint to testing code that the shimming worked 30 | self._shimmed_by_trace_event = True 31 | 32 | def run(self): 33 | self._proc.run() 34 | 35 | def start(self): 36 | self._proc.start() 37 | 38 | def terminate(self): 39 | if log.trace_is_enabled(): 40 | time.sleep(0.25) # give the flush a chance to finish --> ergh, find some other way 41 | self._proc.terminate() 42 | 43 | def join(self, timeout=None): 44 | self._proc.join( timeout) 45 | 46 | def is_alive(self): 47 | return self._proc.is_alive() 48 | 49 | @property 50 | def name(self): 51 | return self._proc.name 52 | 53 | @name.setter 54 | def name(self, name): 55 | self._proc.name = name 56 | 57 | @property 58 | def daemon(self): 59 | return self._proc.daemon 60 | 61 | @daemon.setter 62 | def daemon(self, daemonic): 63 | self._proc.daemon = daemonic 64 | 65 | @property 66 | def authkey(self): 67 | return self._proc._authkey 68 | 69 | @authkey.setter 70 | def authkey(self, authkey): 71 | self._proc.authkey = AuthenticationString(authkey) 72 | 73 | @property 74 | def exitcode(self): 75 | return self._proc.exitcode 76 | 77 | @property 78 | def ident(self): 79 | return self._proc.ident 80 | 81 | @property 82 | def pid(self): 83 | return self._proc.pid 84 | 85 | def __repr__(self): 86 | return self._proc.__repr__() 87 | 88 | # Monkeypatch in our process replacement. 89 | if multiprocessing.Process != ProcessShim: 90 | multiprocessing.Process = ProcessShim 91 | -------------------------------------------------------------------------------- /trace_event_impl/multiprocessing_shim_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Chromium Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be 3 | # found in the LICENSE file. 4 | import multiprocessing 5 | import tempfile 6 | import time 7 | import unittest 8 | from .log import * 9 | from .trace_test import * 10 | 11 | import os 12 | 13 | def DoWork(): 14 | """ 15 | Sadly, you can't @trace toplevel functions, as it prevents them 16 | from being called in the child process. :( 17 | 18 | So, we wrap the function of interest. 19 | """ 20 | def do_work(): 21 | trace_begin("do_work") 22 | time.sleep(0.25) 23 | trace_end("do_work") 24 | do_work() 25 | 26 | def AssertTracingEnabled(): 27 | assert trace_is_enabled() 28 | 29 | def AssertTracingDisabled(): 30 | assert not trace_is_enabled() 31 | 32 | def TryToDisableTracing(): 33 | trace_disable(); 34 | 35 | class MultiprocessingShimTest(TraceTest): 36 | def test_shimmed(self): 37 | p = multiprocessing.Process() 38 | self.assertTrue(hasattr(p, "_shimmed_by_trace_event")) 39 | 40 | def test_trace_enable_throws_in_child(self): 41 | def work(): 42 | trace_begin("work") 43 | p = multiprocessing.Pool(1) 44 | self.assertRaises(Exception, lambda: p.apply(TryToDisableTracing, ())) 45 | p.close() 46 | p.terminate() 47 | p.join() 48 | trace_end("work") 49 | res = self.go(work) 50 | 51 | def test_trace_enabled_in_child(self): 52 | def work(): 53 | trace_begin("work") 54 | p = multiprocessing.Pool(1) 55 | p.apply(AssertTracingEnabled, ()) 56 | p.close() 57 | p.terminate() 58 | p.join() 59 | trace_end("work") 60 | res = self.go(work) 61 | 62 | def test_one_func(self): 63 | def work(): 64 | trace_begin("work") 65 | p = multiprocessing.Pool(1) 66 | p.apply(DoWork, ()) 67 | p.close() 68 | p.terminate() 69 | p.join() 70 | trace_end("work") 71 | res = self.go(work) 72 | work_events = res.findByName('work') 73 | do_work_events = res.findByName('do_work') 74 | self.assertEquals(2, len(work_events)) 75 | self.assertEquals(2, len(do_work_events)) 76 | 77 | -------------------------------------------------------------------------------- /trace_event_impl/parsed_trace_events.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Chromium Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be 3 | # found in the LICENSE file. 4 | import math 5 | import json 6 | 7 | class ParsedTraceEvents(object): 8 | def __init__(self, events = None, trace_filename = None): 9 | """ 10 | Utility class for filtering and manipulating trace data. 11 | 12 | events -- An iterable object containing trace events 13 | trace_filename -- A file object that contains a complete trace. 14 | 15 | """ 16 | if trace_filename and events: 17 | raise Exception("Provide either a trace file or event list") 18 | if not trace_filename and events == None: 19 | raise Exception("Provide either a trace file or event list") 20 | 21 | if trace_filename: 22 | f = open(trace_filename, 'r') 23 | t = f.read() 24 | f.close() 25 | 26 | # If the event data begins with a [, then we know it should end with a ]. 27 | # The reason we check for this is because some tracing implementations 28 | # cannot guarantee that a ']' gets written to the trace file. So, we are 29 | # forgiving and if this is obviously the case, we fix it up before 30 | # throwing the string at JSON.parse. 31 | if t[0] == '[': 32 | n = len(t); 33 | if t[n - 1] != ']' and t[n - 1] != '\n': 34 | t = t + ']' 35 | elif t[n - 2] != ']' and t[n - 1] == '\n': 36 | t = t + ']' 37 | elif t[n - 3] != ']' and t[n - 2] == '\r' and t[n - 1] == '\n': 38 | t = t + ']' 39 | 40 | try: 41 | events = json.loads(t) 42 | except ValueError: 43 | raise Exception("Corrupt trace, did not parse. Value: %s" % t) 44 | 45 | if 'traceEvents' in events: 46 | events = events['traceEvents'] 47 | 48 | if not hasattr(events, '__iter__'): 49 | raise Exception, 'events must be iteraable.' 50 | self.events = events 51 | self.pids = None 52 | self.tids = None 53 | 54 | def __len__(self): 55 | return len(self.events) 56 | 57 | def __getitem__(self, i): 58 | return self.events[i] 59 | 60 | def __setitem__(self, i, v): 61 | self.events[i] = v 62 | 63 | def __repr__(self): 64 | return "[%s]" % ",\n ".join([repr(e) for e in self.events]) 65 | 66 | def findProcessIds(self): 67 | if self.pids: 68 | return self.pids 69 | pids = set() 70 | for e in self.events: 71 | if "pid" in e and e["pid"]: 72 | pids.add(e["pid"]) 73 | self.pids = list(pids) 74 | return self.pids 75 | 76 | def findThreadIds(self): 77 | if self.tids: 78 | return self.tids 79 | tids = set() 80 | for e in self.events: 81 | if "tid" in e and e["tid"]: 82 | tids.add(e["tid"]) 83 | self.tids = list(tids) 84 | return self.tids 85 | 86 | def findEventsOnProcess(self, pid): 87 | return ParsedTraceEvents([e for e in self.events if e["pid"] == pid]) 88 | 89 | def findEventsOnThread(self, tid): 90 | return ParsedTraceEvents([e for e in self.events if e["ph"] != "M" and e["tid"] == tid]) 91 | 92 | def findByPhase(self, ph): 93 | return ParsedTraceEvents([e for e in self.events if e["ph"] == ph]) 94 | 95 | def findByName(self, n): 96 | return ParsedTraceEvents([e for e in self.events if e["name"] == n]) 97 | -------------------------------------------------------------------------------- /trace_event_impl/trace_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 The Chromium Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be 3 | # found in the LICENSE file. 4 | import tempfile 5 | import unittest 6 | 7 | from .log import * 8 | from .parsed_trace_events import * 9 | 10 | 11 | class TraceTest(unittest.TestCase): 12 | def __init__(self, *args): 13 | """ 14 | Infrastructure for running tests of the tracing system. 15 | 16 | Does not actually run any tests. Look at subclasses for those. 17 | """ 18 | unittest.TestCase.__init__(self, *args) 19 | self._file = None 20 | 21 | def go(self, cb): 22 | """ 23 | Enables tracing, runs the provided callback, and if successful, returns a 24 | TraceEvents object with the results. 25 | """ 26 | self._file = tempfile.NamedTemporaryFile() 27 | trace_enable(open(self._file.name, 'a+')) 28 | 29 | try: 30 | cb() 31 | finally: 32 | trace_disable() 33 | e = ParsedTraceEvents(trace_filename = self._file.name) 34 | self._file.close() 35 | self._file = None 36 | return e 37 | 38 | @property 39 | def trace_filename(self): 40 | return self._file.name 41 | 42 | def tearDown(self): 43 | if trace_is_enabled(): 44 | trace_disable() 45 | if self._file: 46 | self._file.close() 47 | --------------------------------------------------------------------------------