├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── profiling ├── __about__.py ├── __init__.py ├── __main__.py ├── profiler.py ├── remote │ ├── __init__.py │ ├── asyncio.py │ ├── background.py │ ├── client.py │ ├── gevent.py │ └── select.py ├── sampling │ ├── __init__.py │ └── samplers.py ├── sortkeys.py ├── speedup.c ├── stats.py ├── tracing │ ├── __init__.py │ └── timers.py ├── utils.py └── viewer.py ├── requirements.txt ├── screenshots ├── sampling.png └── tracing.png ├── setup.cfg ├── setup.py ├── test ├── _utils.py ├── fit_requirements.py ├── requirements.txt ├── test_cli.py ├── test_profiler.py ├── test_sampling.py ├── test_stats.py ├── test_timers.py ├── test_tracing.py ├── test_utils.py └── test_viewer.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | .coverage 26 | .tox 27 | nosetests.xml 28 | 29 | # Translations 30 | *.mo 31 | 32 | # Mr Developer 33 | .mr.developer.cfg 34 | .project 35 | .pydevproject 36 | 37 | # Vim 38 | .*.sw[pqonm] 39 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | python: 4 | - 2.7 5 | - 3.3 6 | - 3.4 7 | - 3.5 8 | - 3.6 9 | - pypy 10 | install: 11 | - pip install -e . 12 | - pip install -U `python test/fit_requirements.py test/requirements.txt` 13 | - pip install flake8 flake8-import-order pytest-cov coveralls 14 | script: 15 | - | # flake8 16 | if python -c 'import sys; sys.version_info < (3, 3) or sys.exit(1)' 17 | then 18 | EXCLUDE=profiling/remote/asyncio.py 19 | fi 20 | flake8 profiling test setup.py -v --show-source --exclude=$EXCLUDE 21 | - | # pytest 22 | py.test -v --cov=profiling --cov-report=term-missing 23 | after_success: 24 | - coveralls 25 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | 0.1.1 5 | ----- 6 | 7 | Released on Christmas 2015. 8 | 9 | The first public release. Finally you can install `profiling` via PyPI: 10 | 11 | ```sh 12 | $ pip install profiling 13 | ``` 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014-2017, What! Studio 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | Redistributions in binary form must reproduce the above copyright notice, this 11 | list of conditions and the following disclaimer in the documentation and/or 12 | other materials provided with the distribution. 13 | 14 | Neither the name of the copyright holder nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE test/test_*.py requirements.txt test/requirements.txt 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ☠ This project is not maintained anymore. We highly recommend switching to 2 | [py-spy](https://github.com/benfred/py-spy) which provides better performance 3 | and usability. 4 | 5 | --- 6 | 7 | Profiling 8 | ========= 9 | 10 | The profiling package is an interactive continuous Python profiler. It is 11 | inspired from [Unity 3D] profiler. This package provides these features: 12 | 13 | - Profiling statistics keep the frame stack. 14 | - An interactive TUI profiling statistics viewer. 15 | - Provides both of statistical and deterministic profiling. 16 | - Utilities for remote profiling. 17 | - Thread or greenlet aware CPU timer. 18 | - Supports Python 2.7, 3.3, 3.4 and 3.5. 19 | - Currently supports only Linux. 20 | 21 | [![Build Status](https://img.shields.io/travis/what-studio/profiling.svg)](https://travis-ci.org/what-studio/profiling) 22 | [![Coverage Status](https://img.shields.io/coveralls/what-studio/profiling.svg)](https://coveralls.io/r/what-studio/profiling) 23 | 24 | [Unity 3D]: http://unity3d.com/ 25 | 26 | Installation 27 | ------------ 28 | 29 | Install the latest release via PyPI: 30 | 31 | ```sh 32 | $ pip install profiling 33 | ``` 34 | 35 | Profiling 36 | --------- 37 | 38 | To profile a single program, simply run the `profiling` command: 39 | 40 | ```sh 41 | $ profiling your-program.py 42 | ``` 43 | 44 | Then an interactive viewer will be executed: 45 | 46 | ![](screenshots/tracing.png) 47 | 48 | If your program uses greenlets, choose `greenlet` timer: 49 | 50 | ```sh 51 | $ profiling --timer=greenlet your-program.py 52 | ``` 53 | 54 | With `--dump` option, it saves the profiling result to a file. You can 55 | browse the saved result by using the `view` subcommand: 56 | 57 | ```sh 58 | $ profiling --dump=your-program.prf your-program.py 59 | $ profiling view your-program.prf 60 | ``` 61 | 62 | If your script reads ``sys.argv``, append your arguments after ``--``. 63 | It isolates your arguments from the ``profiling`` command: 64 | 65 | ```sh 66 | $ profiling your-program.py -- --your-flag --your-param=42 67 | ``` 68 | 69 | Live Profiling 70 | -------------- 71 | 72 | If your program has a long life time like a web server, a profiling result 73 | at the end of program is not helpful enough. Probably you need a continuous 74 | profiler. It can be achived by the `live-profile` subcommand: 75 | 76 | ```sh 77 | $ profiling live-profile webserver.py 78 | ``` 79 | 80 | See a demo: 81 | 82 | [![asciicast](https://asciinema.org/a/25394.png)](https://asciinema.org/a/25394) 83 | 84 | There's a live-profiling server also. The server doesn't profile the 85 | program at ordinary times. But when a client connects to the server, it 86 | starts to profile and reports the results to the all connected clients. 87 | 88 | Start a profling server by the `remote-profile` subcommand: 89 | 90 | ```sh 91 | $ profiling remote-profile webserver.py --bind 127.0.0.1:8912 92 | ``` 93 | 94 | And also run a client for the server by the `view` subcommand: 95 | 96 | ```sh 97 | $ profiling view 127.0.0.1:8912 98 | ``` 99 | 100 | Statistical Profiling 101 | --------------------- 102 | 103 | `TracingProfiler`, the default profiler, implements a deterministic profiler 104 | for deep call graph. Of course, it has heavy overhead. The overhead can 105 | pollute your profiling result or can make your application to be slow. 106 | 107 | In contrast, `SamplingProfiler` implements a statistical profiler. Like other 108 | statistical profilers, it also has only very cheap overhead. When you profile 109 | you can choose it by just `--sampling` (shortly `-S`) option: 110 | 111 | ```sh 112 | $ profiling live-profile -S webserver.py 113 | ^^ 114 | ``` 115 | 116 | ![](screenshots/sampling.png) 117 | 118 | Timeit then Profiling 119 | --------------------- 120 | 121 | Do you use `timeit` to check the performance of your code? 122 | 123 | ```sh 124 | $ python -m timeit -s 'from trueskill import *' 'rate_1vs1(Rating(), Rating())' 125 | 1000 loops, best of 3: 722 usec per loop 126 | ``` 127 | 128 | If you want to profile the checked code, simply use the `timeit` subcommand: 129 | 130 | ```sh 131 | $ profiling timeit -s 'from trueskill import *' 'rate_1vs1(Rating(), Rating())' 132 | ^^^^^^^^^ 133 | ``` 134 | 135 | Profiling from Code 136 | ------------------- 137 | 138 | You can also profile your program by ``profiling.tracing.TracingProfiler`` or 139 | ``profiling.sampling.SamplingProfiler`` directly: 140 | 141 | ```python 142 | from profiling.tracing import TracingProfiler 143 | 144 | # profile your program. 145 | profiler = TracingProfiler() 146 | profiler.start() 147 | ... # run your program. 148 | profiler.stop() 149 | 150 | # or using context manager. 151 | with profiler: 152 | ... # run your program. 153 | 154 | # view and interact with the result. 155 | profiler.run_viewer() 156 | # or save profile data to file 157 | profiler.dump('path/to/file') 158 | ``` 159 | 160 | Viewer Key Bindings 161 | ------------------- 162 | 163 | - q - Quit. 164 | - space - Pause/Resume. 165 | - \\ - Toggle layout between NESTED and FLAT. 166 | - ↑ and ↓ - Navigate frames. 167 | - → - Expand the frame. 168 | - ← - Fold the frame. 169 | - > - Go to the hotspot. 170 | - esc - Defocus. 171 | - [ and ] - Change sorting column. 172 | 173 | Columns 174 | ------- 175 | 176 | ### Common 177 | 178 | - `FUNCTION` 179 | 1. The function name with the code location. 180 | (e.g. `my_func (my_code.py:42)`, `my_func (my_module:42)`) 181 | 1. Only the location without line number. (e.g. `my_code.py`, `my_module`) 182 | 183 | ### Tracing Profiler 184 | 185 | - `CALLS` - Total call count of the function. 186 | - `OWN` (Exclusive Time) - Total spent time in the function excluding sub 187 | calls. 188 | - `/CALL` after `OWN` - Exclusive time per call. 189 | - `%` after `OWN` - Exclusive time per total spent time. 190 | - `DEEP` (Inclusive Time) - Total spent time in the function. 191 | - `/CALL` after `DEEP` - Inclusive time per call. 192 | - `%` after `DEEP` - Inclusive time per total spent time. 193 | 194 | ### Sampling Profiler 195 | 196 | - `OWN` (Exclusive Samples) - Number of samples which are collected during the 197 | direct execution of the function. 198 | - `%` after `OWN` - Exclusive samples per number of the total samples. 199 | - `DEEP` (Inclusive Samples) - Number of samples which are collected during the 200 | excution of the function. 201 | - `%` after `DEEP` - Inclusive samples per number of the total samples. 202 | 203 | Testing 204 | ------- 205 | 206 | There are some additional requirements to run the test code, which can be 207 | installed by running the following command. 208 | 209 | ```sh 210 | $ pip install $(python test/fit_requirements.py test/requirements.txt) 211 | ``` 212 | 213 | Then you should be able to run `pytest`. 214 | 215 | ```sh 216 | $ pytest -v 217 | ``` 218 | 219 | Thanks to 220 | --------- 221 | 222 | - [Seungmyeong Yang](https://github.com/sequoiayang) 223 | who suggested this project. 224 | - [Pavel](https://github.com/htch) 225 | who inspired to implement ``-m`` option. 226 | 227 | Licensing 228 | --------- 229 | 230 | Written by [Heungsub Lee] at [What! Studio] in [Nexon], and 231 | distributed under the [BSD 3-Clause] license. 232 | 233 | [Heungsub Lee]: http://subl.ee/ 234 | [What! Studio]: https://github.com/what-studio 235 | [Nexon]: http://nexon.com/ 236 | [BSD 3-Clause]: http://opensource.org/licenses/BSD-3-Clause 237 | -------------------------------------------------------------------------------- /profiling/__about__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.__about__ 4 | ~~~~~~~~~~~~~~~~~~~ 5 | 6 | :copyright: (c) 2014-2017, What! Studio 7 | :license: BSD, see LICENSE for more details. 8 | 9 | """ 10 | __version__ = '0.1.3' 11 | __license__ = 'BSD' 12 | __author__ = 'What! Studio' 13 | __maintainer__ = 'Heungsub Lee' 14 | __maintainer_email__ = 'sub@nexon.co.kr' 15 | __url__ = 'https://github.com/what-studio/profiling' 16 | __description__ = 'An interactive continuous Python profiler.' 17 | -------------------------------------------------------------------------------- /profiling/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling 4 | ~~~~~~~~~ 5 | 6 | :copyright: (c) 2014-2017, What! Studio 7 | :license: BSD, see LICENSE for more details. 8 | 9 | """ 10 | from __future__ import absolute_import 11 | 12 | from profiling.__about__ import __version__ # noqa 13 | from profiling.profiler import Profiler 14 | 15 | 16 | __all__ = ['Profiler'] 17 | -------------------------------------------------------------------------------- /profiling/__main__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.__main__ 4 | ~~~~~~~~~~~~~~~~~~ 5 | 6 | The command-line interface to profile a script or view profiling results. 7 | 8 | .. sourcecode:: console 9 | 10 | $ profiling --help 11 | 12 | :copyright: (c) 2014-2017, What! Studio 13 | :license: BSD, see LICENSE for more details. 14 | 15 | """ 16 | from __future__ import absolute_import 17 | 18 | from datetime import datetime 19 | from functools import partial, wraps 20 | import importlib 21 | import os 22 | try: 23 | import cPickle as pickle 24 | except ImportError: 25 | import pickle 26 | import runpy 27 | import signal 28 | import socket 29 | from stat import S_ISREG, S_ISSOCK 30 | import sys 31 | import threading 32 | import time 33 | import traceback 34 | 35 | import click 36 | from click_default_group import DefaultGroup 37 | from six import exec_ 38 | from six.moves import builtins 39 | from six.moves.configparser import ConfigParser, NoOptionError, NoSectionError 40 | 41 | from profiling import remote, sampling, tracing 42 | from profiling.__about__ import __version__ 43 | from profiling.profiler import Profiler 44 | from profiling.remote.background import BackgroundProfiler 45 | from profiling.remote.client import FailoverProfilingClient, ProfilingClient 46 | from profiling.remote.select import SelectProfilingServer 47 | from profiling.sampling import samplers, SamplingProfiler 48 | from profiling.tracing import timers, TracingProfiler 49 | from profiling.viewer import bind_game_keys, bind_vim_keys, StatisticsViewer 50 | 51 | 52 | __all__ = ['cli', 'profile', 'view'] 53 | 54 | 55 | DEFAULT_ENDPOINT = '127.0.0.1:8912' 56 | 57 | 58 | class ProfilingCLI(DefaultGroup): 59 | 60 | def __init__(self, *args, **kwargs): 61 | super(ProfilingCLI, self).__init__(*args, **kwargs) 62 | self.command_name_aliases = {} 63 | 64 | def command(self, *args, **kwargs): 65 | """Usage:: 66 | 67 | @cli.command(aliases=['ci']) 68 | def commit(): 69 | ... 70 | 71 | """ 72 | aliases = kwargs.pop('aliases', None) 73 | decorator = super(ProfilingCLI, self).command(*args, **kwargs) 74 | if aliases is None: 75 | return decorator 76 | def _decorator(f): 77 | cmd = decorator(f) 78 | for alias in aliases: 79 | self.command_name_aliases[alias] = cmd.name 80 | return cmd 81 | return _decorator 82 | 83 | def get_command(self, ctx, cmd_name): 84 | # Resolve alias. 85 | try: 86 | cmd_name = self.command_name_aliases[cmd_name] 87 | except KeyError: 88 | pass 89 | return super(ProfilingCLI, self).get_command(ctx, cmd_name) 90 | 91 | 92 | @click.command('profiling', cls=ProfilingCLI, default='profile') 93 | @click.version_option(__version__) 94 | def cli(): 95 | sys.path.insert(0, os.curdir) 96 | bind_vim_keys() 97 | bind_game_keys() 98 | 99 | 100 | class read_config(object): 101 | """Reads a config once in a Click context.""" 102 | 103 | filenames = ['setup.cfg', '.profiling'] 104 | ctx_and_config = (None, None) 105 | 106 | def __new__(cls): 107 | ctx, config = cls.ctx_and_config 108 | current_ctx = click.get_current_context(silent=True) 109 | if current_ctx != ctx: 110 | config = ConfigParser() 111 | config.read(cls.filenames) 112 | cls.ctx_and_config = (current_ctx, config) 113 | return config 114 | 115 | 116 | def option_getter(type): 117 | """Gets an unbound method to get a configuration option as the given type. 118 | """ 119 | option_getters = {None: ConfigParser.get, 120 | int: ConfigParser.getint, 121 | float: ConfigParser.getfloat, 122 | bool: ConfigParser.getboolean} 123 | return option_getters.get(type, option_getters[None]) 124 | 125 | 126 | def config_default(option, default=None, type=None, section=cli.name): 127 | """Guesses a default value of a CLI option from the configuration. 128 | 129 | :: 130 | 131 | @click.option('--locale', default=config_default('locale')) 132 | 133 | """ 134 | def f(option=option, default=default, type=type, section=section): 135 | config = read_config() 136 | if type is None and default is not None: 137 | # detect type from default. 138 | type = builtins.type(default) 139 | get_option = option_getter(type) 140 | try: 141 | return get_option(config, section, option) 142 | except (NoOptionError, NoSectionError): 143 | return default 144 | return f 145 | 146 | 147 | def config_flag(option, value, default=False, section=cli.name): 148 | """Guesses whether a CLI flag should be turned on or off from the 149 | configuration. If the configuration option value is same with the given 150 | value, it returns ``True``. 151 | 152 | :: 153 | 154 | @click.option('--ko-kr', 'locale', is_flag=True, 155 | default=config_flag('locale', 'ko_KR')) 156 | 157 | """ 158 | class x(object): 159 | def __bool__(self, option=option, value=value, 160 | default=default, section=section): 161 | config = read_config() 162 | type = builtins.type(value) 163 | get_option = option_getter(type) 164 | try: 165 | return get_option(config, section, option) == value 166 | except (NoOptionError, NoSectionError): 167 | return default 168 | __nonzero__ = __bool__ 169 | return x() 170 | 171 | 172 | def get_title(src_name, src_type=None): 173 | """Normalizes a source name as a string to be used for viewer's title.""" 174 | if src_type == 'tcp': 175 | return '{0}:{1}'.format(*src_name) 176 | return os.path.basename(src_name) 177 | 178 | 179 | def make_viewer(mono=False, *loop_args, **loop_kwargs): 180 | """Makes a :class:`profiling.viewer.StatisticsViewer` with common options. 181 | """ 182 | viewer = StatisticsViewer() 183 | loop = viewer.loop(*loop_args, **loop_kwargs) 184 | if mono: 185 | loop.screen.set_terminal_properties(1) 186 | return (viewer, loop) 187 | 188 | 189 | def spawn_thread(func, *args, **kwargs): 190 | """Spawns a daemon thread.""" 191 | thread = threading.Thread(target=func, args=args, kwargs=kwargs) 192 | thread.daemon = True 193 | thread.start() 194 | return thread 195 | 196 | 197 | def spawn(mode, func, *args, **kwargs): 198 | """Spawns a thread-like object which runs the given function concurrently. 199 | 200 | Available modes: 201 | 202 | - `threading` 203 | - `greenlet` 204 | - `eventlet` 205 | 206 | """ 207 | if mode is None: 208 | # 'threading' is the default mode. 209 | mode = 'threading' 210 | elif mode not in spawn.modes: 211 | # validate the given mode. 212 | raise ValueError('Invalid spawn mode: %s' % mode) 213 | if mode == 'threading': 214 | return spawn_thread(func, *args, **kwargs) 215 | elif mode == 'gevent': 216 | import gevent 217 | import gevent.monkey 218 | gevent.monkey.patch_select() 219 | gevent.monkey.patch_socket() 220 | return gevent.spawn(func, *args, **kwargs) 221 | elif mode == 'eventlet': 222 | import eventlet 223 | eventlet.patcher.monkey_patch(select=True, socket=True) 224 | return eventlet.spawn(func, *args, **kwargs) 225 | assert False 226 | 227 | 228 | spawn.modes = ['threading', 'gevent', 'eventlet'] 229 | 230 | 231 | #: Just returns the first argument. 232 | noop = lambda x: x 233 | 234 | 235 | def import_(module_name, name): 236 | """Imports an object by a relative module path:: 237 | 238 | Profiler = import_('profiling.profiler', 'Profiler') 239 | 240 | """ 241 | module = importlib.import_module(module_name, __package__) 242 | return getattr(module, name) 243 | 244 | 245 | #: Makes a function which import an object by :func:`import_` lazily. 246 | importer = lambda module_name, name: partial(import_, module_name, name) 247 | 248 | 249 | # custom parameter types 250 | 251 | 252 | class Class(click.ParamType): 253 | 254 | def __init__(self, modules, base, base_name=None, postfix=True): 255 | self.modules = modules 256 | self.base = base 257 | self.base_name = base_name 258 | self.postfix = postfix 259 | 260 | def convert(self, value, param, ctx): 261 | if value == self.base_name: 262 | return self.base 263 | name = value.title() 264 | if self.postfix: 265 | name += self.base.__name__.title() 266 | for mod in self.modules: 267 | try: 268 | cls = getattr(mod, name) 269 | except AttributeError: 270 | continue 271 | if not isinstance(cls, type): 272 | continue 273 | elif not issubclass(cls, self.base): 274 | continue 275 | return cls 276 | self.fail('%s not found' % name) 277 | 278 | def get_metavar(self, param): 279 | return self.base.__name__.upper() 280 | 281 | 282 | class Script(click.File): 283 | """A parameter type for Python script.""" 284 | 285 | def __init__(self): 286 | super(Script, self).__init__('rb') 287 | 288 | def convert(self, value, param, ctx): 289 | with super(Script, self).convert(value, param, ctx) as f: 290 | filename = f.name 291 | code = compile(f.read(), filename, 'exec') 292 | globals_ = {'__file__': filename, '__name__': '__main__', 293 | '__package__': None, '__doc__': None} 294 | return (filename, code, globals_) 295 | 296 | def get_metavar(self, param): 297 | return 'PYTHON' 298 | 299 | 300 | class Module(click.ParamType): 301 | 302 | def convert(self, value, param, ctx): 303 | # inspired by @htch's fork. 304 | # https://github.com/htch/profiling/commit/4a4eb6e 305 | try: 306 | detail = runpy._get_module_details(value) 307 | except ImportError as exc: 308 | ctx.fail(str(exc)) 309 | try: 310 | # since Python 3.4. 311 | mod_name, mod_spec, code = detail 312 | except ValueError: 313 | mod_name, loader, code, filename = detail 314 | else: 315 | loader = mod_spec.loader 316 | filename = mod_spec.origin 317 | # follow runpy's behavior. 318 | pkg_name = mod_name.rpartition('.')[0] 319 | globals_ = sys.modules['__main__'].__dict__.copy() 320 | globals_.update(__name__='__main__', __file__=filename, 321 | __loader__=loader, __package__=pkg_name) 322 | return (filename, code, globals_) 323 | 324 | def get_metavar(self, param): 325 | return 'PYTHON-MODULE' 326 | 327 | 328 | class Command(click.ParamType): 329 | 330 | def convert(self, value, param, ctx): 331 | filename = '' 332 | code = compile(value, filename, 'exec') 333 | globals_ = {'__name__': '__main__', 334 | '__package__': None, '__doc__': None} 335 | return (filename, code, globals_) 336 | 337 | def get_metavar(self, param): 338 | return 'PYTHON-COMMAND' 339 | 340 | 341 | class Endpoint(click.ParamType): 342 | """A parameter type for IP endpoint.""" 343 | 344 | def convert(self, value, param, ctx): 345 | host, port = value.split(':') 346 | port = int(port) 347 | return (host, port) 348 | 349 | def get_metavar(self, param): 350 | return 'HOST:PORT' 351 | 352 | 353 | class ViewerSource(click.ParamType): 354 | """A parameter type for :class:`profiling.viewer.StatisticsViewer` source. 355 | """ 356 | 357 | def convert(self, value, param, ctx): 358 | src_type = False 359 | try: 360 | mode = os.stat(value).st_mode 361 | except OSError: 362 | try: 363 | src_name = Endpoint().convert(value, param, ctx) 364 | except ValueError: 365 | pass 366 | else: 367 | src_type = 'tcp' 368 | else: 369 | src_name = value 370 | if S_ISSOCK(mode): 371 | src_type = 'sock' 372 | elif S_ISREG(mode): 373 | src_type = 'dump' 374 | if not src_type: 375 | raise ValueError('Dump file or socket address required.') 376 | return (src_type, src_name) 377 | 378 | def get_metavar(self, param): 379 | return 'SOURCE' 380 | 381 | 382 | class SignalNumber(click.ParamType): 383 | """A parameter type for signal number.""" 384 | 385 | @staticmethod 386 | def name_of(signum): 387 | for name, value in signal.__dict__.items(): 388 | if signum == value: 389 | if name.startswith('SIG') and not name.startswith('SIG_'): 390 | return name 391 | return str(signum) 392 | 393 | def convert(self, value, param, ctx): 394 | if isinstance(value, int): 395 | return value 396 | elif value.isdigit(): 397 | return int(value) 398 | signame = value.upper() 399 | if not signame.startswith('SIG'): 400 | signame = 'SIG' + signame 401 | if signame.startswith('SIG_'): 402 | self.fail('Invalid signal %s' % signame) 403 | try: 404 | signum = getattr(signal, signame) 405 | except AttributeError: 406 | self.fail('Unknown signal %s' % signame) 407 | return signum 408 | 409 | def get_metavar(self, param): 410 | return 'SIGNUM' 411 | 412 | 413 | # common parameters 414 | 415 | 416 | class Params(object): 417 | 418 | def __init__(self, params): 419 | self.params = params 420 | 421 | def __call__(self, f): 422 | for param in self.params[::-1]: 423 | f = param(f) 424 | return f 425 | 426 | def __add__(self, params): 427 | return self.__class__(self.params + params) 428 | 429 | 430 | def profiler_options(f): 431 | # tracing profiler options 432 | @click.option( 433 | '-T', '--tracing', 'import_profiler_class', 434 | flag_value=importer('profiling.tracing', 'TracingProfiler'), 435 | default=config_flag('profiler', 'tracing', True), 436 | help='Use tracing profiler. (default)') 437 | @click.option( 438 | '--timer', 'timer_class', 439 | type=Class([timers], timers.Timer, 'basic'), 440 | default=config_default('timer', 'basic'), 441 | help='Choose CPU timer for tracing profiler. (basic|thread|greenlet)') 442 | # sampling profiler options 443 | @click.option( 444 | '-S', '--sampling', 'import_profiler_class', 445 | flag_value=importer('profiling.sampling', 'SamplingProfiler'), 446 | default=config_flag('profiler', 'sampling', False), 447 | help='Use sampling profiler.') 448 | @click.option( 449 | '--sampler', 'sampler_class', 450 | type=Class([samplers], samplers.Sampler), 451 | default=config_default('sampler', 'itimer'), 452 | help='Choose frames sampler for sampling profiler. (itimer|tracing)') 453 | @click.option( 454 | '--sampling-interval', type=float, 455 | default=config_default('sampling-interval', samplers.INTERVAL), 456 | help='How often sample. (default: %.3f cpu sec)' % samplers.INTERVAL) 457 | # etc 458 | @click.option( 459 | '--pickle-protocol', type=int, 460 | default=config_default('pickle-protocol', remote.PICKLE_PROTOCOL), 461 | help='Pickle protocol to dump result.') 462 | @wraps(f) 463 | def wrapped(import_profiler_class, timer_class, sampler_class, 464 | sampling_interval, **kwargs): 465 | profiler_class = import_profiler_class() 466 | assert issubclass(profiler_class, Profiler) 467 | if issubclass(profiler_class, TracingProfiler): 468 | # profiler requires timer. 469 | timer_class = timer_class or tracing.TIMER_CLASS 470 | timer = timer_class() 471 | profiler_kwargs = {'timer': timer} 472 | elif issubclass(profiler_class, SamplingProfiler): 473 | sampler_class = sampler_class or sampling.SAMPLER_CLASS 474 | sampler = sampler_class(sampling_interval) 475 | profiler_kwargs = {'sampler': sampler} 476 | else: 477 | profiler_kwargs = {} 478 | profiler_factory = partial(profiler_class, **profiler_kwargs) 479 | return f(profiler_factory=profiler_factory, **kwargs) 480 | return wrapped 481 | 482 | 483 | def profiler_arguments(f): 484 | @click.argument('argv', nargs=-1) 485 | @click.option('-m', 'module', type=Module(), 486 | help='Run library module as a script.') 487 | @click.option('-c', 'command', type=Command(), 488 | help='Program passed in as string.') 489 | @wraps(f) 490 | def wrapped(argv, module, command, **kwargs): 491 | if module is not None and command is not None: 492 | raise click.UsageError('Option -m and -c are exclusive') 493 | script = module or command 494 | if script is None: 495 | # -m and -c not passed. 496 | try: 497 | script_filename, argv = argv[0], argv[1:] 498 | except IndexError: 499 | raise click.UsageError('Script not specified') 500 | script = Script().convert(script_filename, None, None) 501 | kwargs.update(script=script, argv=argv) 502 | return f(**kwargs) 503 | return wrapped 504 | 505 | 506 | viewer_options = Params([ 507 | click.option('--mono', is_flag=True, help='Disable coloring.'), 508 | ]) 509 | onetime_profiler_options = Params([ 510 | click.option( 511 | '-d', '--dump', 'dump_filename', type=click.Path(writable=True), 512 | help='Profiling result dump filename.'), 513 | ]) 514 | live_profiler_options = Params([ 515 | click.option( 516 | '-i', '--interval', type=float, 517 | default=config_default('interval', remote.INTERVAL), 518 | help='How often update result. (default: %.0f sec)' % remote.INTERVAL), 519 | click.option( 520 | '--spawn', type=click.Choice(spawn.modes), 521 | default=config_default('spawn'), 522 | callback=lambda c, p, v: partial(spawn, v), 523 | help='How to spawn profiler server in background.'), 524 | click.option( 525 | '--signum', type=SignalNumber(), 526 | default=config_default('signum', BackgroundProfiler.signum), 527 | help=( 528 | 'For communication between server and application. (default: %s)' % 529 | SignalNumber.name_of(BackgroundProfiler.signum) 530 | )) 531 | ]) 532 | 533 | 534 | # sub-commands 535 | 536 | 537 | def __profile__(filename, code, globals_, profiler_factory, 538 | pickle_protocol=remote.PICKLE_PROTOCOL, dump_filename=None, 539 | mono=False): 540 | frame = sys._getframe() 541 | profiler = profiler_factory(base_frame=frame, base_code=code) 542 | profiler.start() 543 | try: 544 | exec_(code, globals_) 545 | except BaseException: 546 | # don't profile print_exc(). 547 | profiler.stop() 548 | traceback.print_exc() 549 | else: 550 | profiler.stop() 551 | # discard this __profile__ function from the result. 552 | profiler.stats.discard_child(frame.f_code) 553 | if dump_filename is None: 554 | try: 555 | profiler.run_viewer(get_title(filename), mono=mono) 556 | except KeyboardInterrupt: 557 | pass 558 | else: 559 | profiler.dump(dump_filename, pickle_protocol) 560 | 561 | click.echo('To view statistics:') 562 | click.echo(' $ profiling view ', nl=False) 563 | click.secho(dump_filename, underline=True) 564 | 565 | 566 | class ProfilingCommand(click.Command): 567 | 568 | def collect_usage_pieces(self, ctx): 569 | """Prepend "[--]" before "[ARGV]...".""" 570 | pieces = super(ProfilingCommand, self).collect_usage_pieces(ctx) 571 | assert pieces[-1] == '[ARGV]...' 572 | pieces.insert(-1, 'SCRIPT') 573 | pieces.insert(-1, '[--]') 574 | return pieces 575 | 576 | 577 | @cli.command(cls=ProfilingCommand) 578 | @profiler_arguments 579 | @profiler_options 580 | @onetime_profiler_options 581 | @viewer_options 582 | def profile(script, argv, profiler_factory, 583 | pickle_protocol, dump_filename, mono): 584 | """Profile a Python script.""" 585 | filename, code, globals_ = script 586 | sys.argv[:] = [filename] + list(argv) 587 | __profile__(filename, code, globals_, profiler_factory, 588 | pickle_protocol=pickle_protocol, dump_filename=dump_filename, 589 | mono=mono) 590 | 591 | 592 | @cli.command('live-profile', aliases=['live'], cls=ProfilingCommand) 593 | @profiler_arguments 594 | @profiler_options 595 | @live_profiler_options 596 | @viewer_options 597 | def live_profile(script, argv, profiler_factory, interval, spawn, signum, 598 | pickle_protocol, mono): 599 | """Profile a Python script continuously.""" 600 | filename, code, globals_ = script 601 | sys.argv[:] = [filename] + list(argv) 602 | parent_sock, child_sock = socket.socketpair() 603 | stderr_r_fd, stderr_w_fd = os.pipe() 604 | pid = os.fork() 605 | if pid: 606 | # parent 607 | os.close(stderr_w_fd) 608 | viewer, loop = make_viewer(mono) 609 | # loop.screen._term_output_file = open(os.devnull, 'w') 610 | title = get_title(filename) 611 | client = ProfilingClient(viewer, loop.event_loop, parent_sock, title) 612 | client.start() 613 | try: 614 | loop.run() 615 | except KeyboardInterrupt: 616 | os.kill(pid, signal.SIGINT) 617 | except BaseException: 618 | # unexpected profiler error. 619 | os.kill(pid, signal.SIGTERM) 620 | raise 621 | finally: 622 | parent_sock.close() 623 | # get exit code of child. 624 | w_pid, status = os.waitpid(pid, os.WNOHANG) 625 | if w_pid == 0: 626 | os.kill(pid, signal.SIGTERM) 627 | exit_code = os.WEXITSTATUS(status) 628 | # print stderr of child. 629 | with os.fdopen(stderr_r_fd, 'r') as f: 630 | child_stderr = f.read() 631 | if child_stderr: 632 | sys.stdout.flush() 633 | sys.stderr.write(child_stderr) 634 | # exit with exit code of child. 635 | sys.exit(exit_code) 636 | else: 637 | # child 638 | os.close(stderr_r_fd) 639 | # mute stdin, stdout. 640 | devnull = os.open(os.devnull, os.O_RDWR) 641 | for f in [sys.stdin, sys.stdout]: 642 | os.dup2(devnull, f.fileno()) 643 | # redirect stderr to parent. 644 | os.dup2(stderr_w_fd, sys.stderr.fileno()) 645 | frame = sys._getframe() 646 | profiler = profiler_factory(base_frame=frame, base_code=code) 647 | profiler_trigger = BackgroundProfiler(profiler, signum) 648 | profiler_trigger.prepare() 649 | server_args = (interval, noop, pickle_protocol) 650 | server = SelectProfilingServer(None, profiler_trigger, *server_args) 651 | server.clients.add(child_sock) 652 | spawn(server.connected, child_sock) 653 | try: 654 | exec_(code, globals_) 655 | finally: 656 | os.close(stderr_w_fd) 657 | child_sock.shutdown(socket.SHUT_WR) 658 | 659 | 660 | @cli.command('remote-profile', aliases=['remote'], cls=ProfilingCommand) 661 | @profiler_arguments 662 | @profiler_options 663 | @live_profiler_options 664 | @click.option('-b', '--bind', 'endpoint', type=Endpoint(), 665 | default=config_default('endpoint', DEFAULT_ENDPOINT), 666 | help='IP endpoint to serve profiling results.') 667 | @click.option('-v', '--verbose', is_flag=True, 668 | help='Print profiling server logs.') 669 | def remote_profile(script, argv, profiler_factory, interval, spawn, signum, 670 | pickle_protocol, endpoint, verbose): 671 | """Launch a server to profile continuously. The default endpoint is 672 | 127.0.0.1:8912. 673 | """ 674 | filename, code, globals_ = script 675 | sys.argv[:] = [filename] + list(argv) 676 | # create listener. 677 | listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 678 | listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 679 | listener.bind(endpoint) 680 | listener.listen(1) 681 | # be verbose or quiet. 682 | if verbose: 683 | log = lambda x: click.echo(click.style('> ', fg='cyan') + x) 684 | bound_addr = listener.getsockname() 685 | log('Listening on {0}:{1} for profiling...'.format(*bound_addr)) 686 | else: 687 | log = noop 688 | # start profiling server. 689 | frame = sys._getframe() 690 | profiler = profiler_factory(base_frame=frame, base_code=code) 691 | profiler_trigger = BackgroundProfiler(profiler, signum) 692 | profiler_trigger.prepare() 693 | server_args = (interval, log, pickle_protocol) 694 | server = SelectProfilingServer(listener, profiler_trigger, *server_args) 695 | spawn(server.serve_forever) 696 | # exec the script. 697 | try: 698 | exec_(code, globals_) 699 | except KeyboardInterrupt: 700 | pass 701 | 702 | 703 | @cli.command() 704 | @click.argument('src', type=ViewerSource(), 705 | default=config_default('endpoint', DEFAULT_ENDPOINT)) 706 | @viewer_options 707 | def view(src, mono): 708 | """Inspect statistics by TUI view.""" 709 | src_type, src_name = src 710 | title = get_title(src_name, src_type) 711 | viewer, loop = make_viewer(mono) 712 | if src_type == 'dump': 713 | time = datetime.fromtimestamp(os.path.getmtime(src_name)) 714 | with open(src_name, 'rb') as f: 715 | profiler_class, (stats, cpu_time, wall_time) = pickle.load(f) 716 | viewer.set_profiler_class(profiler_class) 717 | viewer.set_result(stats, cpu_time, wall_time, title=title, at=time) 718 | viewer.activate() 719 | elif src_type in ('tcp', 'sock'): 720 | family = {'tcp': socket.AF_INET, 'sock': socket.AF_UNIX}[src_type] 721 | client = FailoverProfilingClient(viewer, loop.event_loop, 722 | src_name, family, title=title) 723 | client.start() 724 | try: 725 | loop.run() 726 | except KeyboardInterrupt: 727 | pass 728 | 729 | 730 | @cli.command('timeit-profile', aliases=['timeit']) 731 | @click.argument('stmt', metavar='STATEMENT', default='pass') 732 | @click.option('-n', '--number', type=int, 733 | help='How many times to execute the statement.') 734 | @click.option('-r', '--repeat', type=int, default=3, 735 | help='How many times to repeat the timer.') 736 | @click.option('-s', '--setup', default='pass', 737 | help='Statement to be executed once initially.') 738 | @click.option('-t', '--time', help='Ignored.') 739 | @click.option('-c', '--clock', help='Ignored.') 740 | @click.option('-v', '--verbose', help='Ignored.') 741 | @profiler_options 742 | @onetime_profiler_options 743 | @viewer_options 744 | def timeit_profile(stmt, number, repeat, setup, 745 | profiler_factory, pickle_protocol, dump_filename, mono, 746 | **_ignored): 747 | """Profile a Python statement like timeit.""" 748 | del _ignored 749 | globals_ = {} 750 | exec_(setup, globals_) 751 | if number is None: 752 | # determine number so that 0.2 <= total time < 2.0 like timeit. 753 | dummy_profiler = profiler_factory() 754 | dummy_profiler.start() 755 | for x in range(1, 10): 756 | number = 10 ** x 757 | t = time.time() 758 | for y in range(number): 759 | exec_(stmt, globals_) 760 | if time.time() - t >= 0.2: 761 | break 762 | dummy_profiler.stop() 763 | del dummy_profiler 764 | code = compile('for _ in range(%d): %s' % (number, stmt), 765 | 'STATEMENT', 'exec') 766 | __profile__(stmt, code, globals_, profiler_factory, 767 | pickle_protocol=pickle_protocol, dump_filename=dump_filename, 768 | mono=mono) 769 | 770 | 771 | # Deprecated. 772 | main = cli 773 | 774 | 775 | if __name__ == '__main__': 776 | cli(prog_name='python -m profiling') 777 | -------------------------------------------------------------------------------- /profiling/profiler.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.profiler 4 | ~~~~~~~~~~~~~~~~~~ 5 | 6 | :copyright: (c) 2014-2017, What! Studio 7 | :license: BSD, see LICENSE for more details. 8 | 9 | """ 10 | from __future__ import absolute_import 11 | 12 | try: 13 | import cPickle as pickle 14 | except ImportError: 15 | import pickle 16 | import time 17 | 18 | from profiling.stats import RecordingStatistics 19 | from profiling.utils import frame_stack, Runnable 20 | from profiling.viewer import StatisticsTable, StatisticsViewer 21 | 22 | 23 | __all__ = ['Profiler', 'ProfilerWrapper'] 24 | 25 | 26 | class Profiler(Runnable): 27 | """The base class for profiler.""" 28 | 29 | #: A widget class which extends :class:`profiling.viewer.StatisticsTable`. 30 | table_class = StatisticsTable 31 | 32 | #: The root recording statistics. 33 | stats = None 34 | 35 | base_frame = None 36 | base_code = None 37 | ignored_frames = () 38 | ignored_codes = () 39 | 40 | def __init__(self, base_frame=None, base_code=None, 41 | ignored_frames=(), ignored_codes=()): 42 | self.base_frame = base_frame 43 | self.base_code = base_code 44 | self.ignored_frames = ignored_frames 45 | self.ignored_codes = ignored_codes 46 | self.stats = RecordingStatistics() 47 | 48 | def start(self): 49 | self._cpu_time_started = time.clock() 50 | self._wall_time_started = time.time() 51 | self.stats.clear() 52 | return super(Profiler, self).start() 53 | 54 | def frame_stack(self, frame): 55 | return frame_stack(frame, self.base_frame, self.base_code, 56 | self.ignored_frames, self.ignored_codes) 57 | 58 | def exclude_code(self, code): 59 | """Excludes statistics of the given code.""" 60 | try: 61 | self.stats.remove_child(code) 62 | except KeyError: 63 | pass 64 | 65 | def result(self): 66 | """Gets the frozen statistics to serialize by Pickle.""" 67 | try: 68 | cpu_time = max(0, time.clock() - self._cpu_time_started) 69 | wall_time = max(0, time.time() - self._wall_time_started) 70 | except AttributeError: 71 | cpu_time = wall_time = 0.0 72 | return self.stats, cpu_time, wall_time 73 | 74 | def dump(self, dump_filename, pickle_protocol=pickle.HIGHEST_PROTOCOL): 75 | """Saves the profiling result to a file 76 | 77 | :param dump_filename: path to a file 78 | :type dump_filename: str 79 | 80 | :param pickle_protocol: version of pickle protocol 81 | :type pickle_protocol: int 82 | """ 83 | result = self.result() 84 | 85 | with open(dump_filename, 'wb') as f: 86 | pickle.dump((self.__class__, result), f, pickle_protocol) 87 | 88 | def make_viewer(self, title=None, at=None): 89 | """Makes a statistics viewer from the profiling result. 90 | """ 91 | viewer = StatisticsViewer() 92 | viewer.set_profiler_class(self.__class__) 93 | stats, cpu_time, wall_time = self.result() 94 | viewer.set_result(stats, cpu_time, wall_time, title=title, at=at) 95 | viewer.activate() 96 | return viewer 97 | 98 | def run_viewer(self, title=None, at=None, mono=False, 99 | *loop_args, **loop_kwargs): 100 | """A shorter form of: 101 | 102 | :: 103 | 104 | viewer = profiler.make_viewer() 105 | loop = viewer.loop() 106 | loop.run() 107 | 108 | """ 109 | viewer = self.make_viewer(title, at=at) 110 | loop = viewer.loop(*loop_args, **loop_kwargs) 111 | if mono: 112 | loop.screen.set_terminal_properties(1) 113 | loop.run() 114 | 115 | 116 | class ProfilerWrapper(Profiler): 117 | 118 | for attr in ['table_class', 'stats', 'top_frame', 'top_code', 'result', 119 | 'is_running']: 120 | f = lambda self, attr=attr: getattr(self.profiler, attr) 121 | locals()[attr] = property(f) 122 | del f 123 | 124 | def __init__(self, profiler): 125 | self.profiler = profiler 126 | -------------------------------------------------------------------------------- /profiling/remote/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.remote 4 | ~~~~~~~~~~~~~~~~ 5 | 6 | Utilities for remote profiling. They help you to implement profiling 7 | server and client. 8 | 9 | :copyright: (c) 2014-2017, What! Studio 10 | :license: BSD, see LICENSE for more details. 11 | 12 | """ 13 | from __future__ import absolute_import 14 | 15 | from errno import EBADF, ECONNRESET, EPIPE 16 | import functools 17 | import io 18 | from logging import getLogger as get_logger 19 | try: 20 | import cPickle as pickle 21 | except ImportError: 22 | import pickle 23 | import socket 24 | import struct 25 | 26 | from profiling.__about__ import __version__ 27 | 28 | 29 | __all__ = ['LOGGER', 'LOG', 'INTERVAL', 'PICKLE_PROTOCOL', 30 | 'SIZE_STRUCT_FORMAT', 'pack_result', 'recv_msg', 'fmt_connected', 31 | 'fmt_disconnected', 'fmt_profiler_started', 'fmt_profiler_stopped', 32 | 'ProfilingServer'] 33 | 34 | 35 | #: The standard logger. 36 | LOGGER = get_logger('Profiling') 37 | 38 | #: The standard log function. 39 | LOG = LOGGER.debug 40 | 41 | #: The default profiling interval. 42 | INTERVAL = 1 43 | 44 | #: The default Pickle protocol. 45 | PICKLE_PROTOCOL = getattr(pickle, 'DEFAULT_PROTOCOL', pickle.HIGHEST_PROTOCOL) 46 | 47 | #: The struct format to pack message size. (uint32) 48 | SIZE_STRUCT_FORMAT = '!I' 49 | 50 | #: The struct format to pack method. (uint8) 51 | METHOD_STRUCT_FORMAT = '!B' 52 | 53 | 54 | # methods 55 | WELCOME = 0x10 56 | PROFILER = 0x11 57 | RESULT = 0x12 58 | 59 | 60 | def pack_msg(method, msg, pickle_protocol=PICKLE_PROTOCOL): 61 | """Packs a method and message.""" 62 | dump = io.BytesIO() 63 | pickle.dump(msg, dump, pickle_protocol) 64 | size = dump.tell() 65 | return (struct.pack(METHOD_STRUCT_FORMAT, method) + 66 | struct.pack(SIZE_STRUCT_FORMAT, size) + dump.getvalue()) 67 | 68 | 69 | def recv(sock, size): 70 | """Receives exactly `size` bytes. This function blocks the thread.""" 71 | data = sock.recv(size, socket.MSG_WAITALL) 72 | if len(data) < size: 73 | raise socket.error(ECONNRESET, 'Connection closed') 74 | return data 75 | 76 | 77 | def recv_msg(sock): 78 | """Receives a method and message from the socket. This function blocks the 79 | current thread. 80 | """ 81 | data = recv(sock, struct.calcsize(METHOD_STRUCT_FORMAT)) 82 | method, = struct.unpack(METHOD_STRUCT_FORMAT, data) 83 | data = recv(sock, struct.calcsize(SIZE_STRUCT_FORMAT)) 84 | size, = struct.unpack(SIZE_STRUCT_FORMAT, data) 85 | data = recv(sock, size) 86 | msg = pickle.loads(data) 87 | return method, msg 88 | 89 | 90 | def fmt_connected(addr, num_clients): 91 | if addr: 92 | fmt = 'Connected from {0[0]}:{0[1]} (total: {1})' 93 | else: 94 | fmt = 'A client connected (total: {1})' 95 | return fmt.format(addr, num_clients) 96 | 97 | 98 | def fmt_disconnected(addr, num_clients): 99 | if addr: 100 | fmt = 'Disconnected from {0[0]}:{0[1]} (total: {1})' 101 | else: 102 | fmt = 'A client disconnected (total: {1})' 103 | return fmt.format(addr, num_clients) 104 | 105 | 106 | def fmt_profiler_started(interval): 107 | return 'Profiling every {0} seconds...'.format(interval) 108 | 109 | 110 | def fmt_profiler_stopped(): 111 | return 'Profiler stopped' 112 | 113 | 114 | def abstract(message): 115 | def decorator(f): 116 | @functools.wraps(f) 117 | def wrapped(*args, **kwargs): 118 | raise NotImplementedError(message) 119 | return wrapped 120 | return decorator 121 | 122 | 123 | class ProfilingServer(object): 124 | """The base class for profiling server implementations. Implement abstract 125 | methods and call :meth:`connected` when a client connected. 126 | """ 127 | 128 | _latest_result_data = None 129 | 130 | def __init__(self, profiler, interval=INTERVAL, 131 | log=LOG, pickle_protocol=PICKLE_PROTOCOL): 132 | self.profiler = profiler 133 | self.interval = interval 134 | self.log = log 135 | self.pickle_protocol = pickle_protocol 136 | self.clients = set() 137 | 138 | @abstract('Implement serve_forever() to run a server synchronously.') 139 | def serve_forever(self): 140 | pass 141 | 142 | @abstract('Implement _send() to send data to the client.') 143 | def _send(self, client, data): 144 | pass 145 | 146 | @abstract('Implement _close() to close the client.') 147 | def _close(self, client): 148 | pass 149 | 150 | @abstract('Implement _addr() to get the address from the client.') 151 | def _addr(self, client): 152 | pass 153 | 154 | @abstract('Implement _start_profiling() to start a profiling loop.') 155 | def _start_profiling(self): 156 | pass 157 | 158 | @abstract('Implement _start_watching() to add a disconnection callback to ' 159 | 'the client') 160 | def _start_watching(self, client): 161 | pass 162 | 163 | def profiling(self): 164 | """A generator which profiles then broadcasts the result. Implement 165 | sleeping loop using this:: 166 | 167 | def profile_periodically(self): 168 | for __ in self.profiling(): 169 | time.sleep(self.interval) 170 | 171 | """ 172 | self._log_profiler_started() 173 | while self.clients: 174 | try: 175 | self.profiler.start() 176 | except RuntimeError: 177 | pass 178 | # should sleep. 179 | yield 180 | self.profiler.stop() 181 | result = self.profiler.result() 182 | data = pack_msg(RESULT, result, 183 | pickle_protocol=self.pickle_protocol) 184 | self._latest_result_data = data 185 | # broadcast. 186 | closed_clients = [] 187 | for client in self.clients: 188 | try: 189 | self._send(client, data) 190 | except socket.error as exc: 191 | if exc.errno == EPIPE: 192 | closed_clients.append(client) 193 | del data 194 | # handle disconnections. 195 | for client in closed_clients: 196 | self.disconnected(client) 197 | self._log_profiler_stopped() 198 | 199 | def send_msg(self, client, method, msg, pickle_protocol=None): 200 | if pickle_protocol is None: 201 | pickle_protocol = self.pickle_protocol 202 | data = pack_msg(method, msg, pickle_protocol=pickle_protocol) 203 | self._send(client, data) 204 | 205 | def connected(self, client): 206 | """Call this method when a client connected.""" 207 | self.clients.add(client) 208 | self._log_connected(client) 209 | self._start_watching(client) 210 | self.send_msg(client, WELCOME, (self.pickle_protocol, __version__), 211 | pickle_protocol=0) 212 | profiler = self.profiler 213 | while True: 214 | try: 215 | profiler = profiler.profiler 216 | except AttributeError: 217 | break 218 | self.send_msg(client, PROFILER, type(profiler)) 219 | if self._latest_result_data is not None: 220 | try: 221 | self._send(client, self._latest_result_data) 222 | except socket.error as exc: 223 | if exc.errno in (EBADF, EPIPE): 224 | self.disconnected(client) 225 | return 226 | raise 227 | if len(self.clients) == 1: 228 | self._start_profiling() 229 | 230 | def disconnected(self, client): 231 | """Call this method when a client disconnected.""" 232 | if client not in self.clients: 233 | # already disconnected. 234 | return 235 | self.clients.remove(client) 236 | self._log_disconnected(client) 237 | self._close(client) 238 | 239 | def _log_connected(self, client): 240 | addr = self._addr(client) 241 | addr = addr if isinstance(addr, tuple) else None 242 | self.log(fmt_connected(addr, len(self.clients))) 243 | 244 | def _log_disconnected(self, client): 245 | addr = self._addr(client) 246 | addr = addr if isinstance(addr, tuple) else None 247 | self.log(fmt_disconnected(addr, len(self.clients))) 248 | 249 | def _log_profiler_started(self): 250 | self.log(fmt_profiler_started(self.interval)) 251 | 252 | def _log_profiler_stopped(self): 253 | self.log(fmt_profiler_stopped()) 254 | -------------------------------------------------------------------------------- /profiling/remote/asyncio.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.remote.async 4 | ~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Implements a profiling server based on `asyncio`_. Only for Python 3.4 or 7 | later. 8 | 9 | .. _asyncio: https://docs.python.org/3/library/asyncio.html 10 | 11 | :copyright: (c) 2014-2017, What! Studio 12 | :license: BSD, see LICENSE for more details. 13 | 14 | """ 15 | from __future__ import absolute_import 16 | 17 | import asyncio 18 | 19 | from profiling.remote import ProfilingServer 20 | 21 | 22 | __all__ = ['AsyncIOProfilingServer'] 23 | 24 | 25 | class AsyncIOProfilingServer(ProfilingServer): 26 | """A profiling server implementation based on `asyncio`_. Launch a server 27 | by ``asyncio.start_server`` or ``asyncio.start_unix_server``:: 28 | 29 | server = AsyncIOProfilingServer(interval=10) 30 | ready = asyncio.start_server(server) 31 | loop = asyncio.get_event_loop() 32 | loop.run_until_complete(ready) 33 | loop.run_forever() 34 | 35 | .. _asyncio: https://docs.python.org/3/library/asyncio.html 36 | 37 | """ 38 | 39 | def __init__(self, *args, **kwargs): 40 | super(AsyncIOProfilingServer, self).__init__(*args, **kwargs) 41 | self.clients = set() 42 | 43 | def serve_forever(self, addr): 44 | host, port = addr 45 | loop = asyncio.get_event_loop() 46 | ready_to_serve = asyncio.start_server(self, host, port) 47 | loop.run_until_complete(ready_to_serve) 48 | loop.run_forever() 49 | 50 | def _send(self, client, data): 51 | reader, writer = client 52 | writer.write(data) 53 | 54 | def _close(self, client): 55 | reader, writer = client 56 | writer.close() 57 | 58 | def _addr(self, client): 59 | reader, writer = client 60 | return writer.get_extra_info('peername') 61 | 62 | def _start_profiling(self): 63 | asyncio.async(self.profile_periodically()) 64 | 65 | def _start_watching(self, client): 66 | reader, writer = client 67 | disconnected = lambda x: self.disconnected(reader, writer) 68 | asyncio.async(reader.read()).add_done_callback(disconnected) 69 | 70 | @asyncio.coroutine 71 | def profile_periodically(self): 72 | for __ in self.profiling(): 73 | yield from asyncio.sleep(self.interval) 74 | 75 | def __call__(self, reader, writer): 76 | client = (reader, writer) 77 | self.connected(client) 78 | -------------------------------------------------------------------------------- /profiling/remote/background.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.remote.background 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Utilities to run a profiler in a background thread. 7 | 8 | :copyright: (c) 2014-2017, What! Studio 9 | :license: BSD, see LICENSE for more details. 10 | 11 | """ 12 | from __future__ import absolute_import 13 | 14 | import os 15 | import signal 16 | import threading 17 | 18 | from profiling.profiler import ProfilerWrapper 19 | 20 | 21 | __all__ = ['BackgroundProfiler'] 22 | 23 | 24 | class BackgroundProfiler(ProfilerWrapper): 25 | 26 | signum = signal.SIGUSR2 27 | 28 | def __init__(self, profiler, signum=None): 29 | super(BackgroundProfiler, self).__init__(profiler) 30 | if signum is not None: 31 | self.signum = signum 32 | self.event = threading.Event() 33 | 34 | def prepare(self): 35 | """Registers :meth:`_signal_handler` as a signal handler to start 36 | and/or stop the profiler from the background thread. So this function 37 | must be called at the main thread. 38 | """ 39 | return signal.signal(self.signum, self._signal_handler) 40 | 41 | def run(self): 42 | self._send_signal() 43 | yield 44 | self._send_signal() 45 | 46 | def _send_signal(self): 47 | self.event.clear() 48 | os.kill(os.getpid(), self.signum) 49 | self.event.wait() 50 | 51 | def _signal_handler(self, signum, frame): 52 | if self.profiler.is_running(): 53 | self.profiler.stop() 54 | else: 55 | self.profiler.start() 56 | self.event.set() 57 | -------------------------------------------------------------------------------- /profiling/remote/client.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.remote.client 4 | ~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | :copyright: (c) 2014-2017, What! Studio 7 | :license: BSD, see LICENSE for more details. 8 | 9 | """ 10 | from __future__ import absolute_import 11 | 12 | from datetime import datetime 13 | from errno import ECONNREFUSED, EINPROGRESS, ENOENT 14 | import socket 15 | 16 | from valuedispatch import valuedispatch 17 | 18 | from profiling.remote import PROFILER, recv_msg, RESULT, WELCOME 19 | 20 | 21 | __all__ = ['ProfilingClient', 'FailoverProfilingClient'] 22 | 23 | 24 | @valuedispatch 25 | def protocol(method, msg, client): 26 | pass 27 | 28 | 29 | @protocol.register(WELCOME) 30 | def handle_welcome(_, __, client): 31 | client.viewer.activate() 32 | 33 | 34 | @protocol.register(PROFILER) 35 | def handle_profiler(_, profiler_class, client): 36 | client.viewer.set_profiler_class(profiler_class) 37 | 38 | 39 | @protocol.register(RESULT) 40 | def handle_result(_, result, client): 41 | stats, cpu_time, wall_time = result 42 | client.viewer.set_result(stats, cpu_time, wall_time, 43 | client.title, datetime.now()) 44 | 45 | 46 | class ProfilingClient(object): 47 | """A client of profiling server which is running behind the `Urwid`_ event 48 | loop. 49 | 50 | .. _Urwid: http://urwid.org/ 51 | 52 | """ 53 | 54 | def __init__(self, viewer, event_loop, sock, 55 | title=None, protocol=protocol): 56 | self.viewer = viewer 57 | self.event_loop = event_loop 58 | self.sock = sock 59 | self.title = title 60 | self.protocol = protocol 61 | 62 | def start(self): 63 | self.event_loop.watch_file(self.sock.fileno(), self.handle) 64 | 65 | def handle(self): 66 | try: 67 | method, msg = recv_msg(self.sock) 68 | except socket.error as exc: 69 | self.erred(exc.errno) 70 | return 71 | self.protocol(method, msg, self) 72 | 73 | def erred(self, errno): 74 | self.event_loop.remove_watch_file(self.sock.fileno()) 75 | self.viewer.inactivate() 76 | 77 | 78 | class FailoverProfilingClient(ProfilingClient): 79 | """A profiling client but it tries to reconnect constantly.""" 80 | 81 | failover_interval = 1 82 | 83 | def __init__(self, viewer, event_loop, addr=None, family=socket.AF_INET, 84 | title=None, protocol=protocol): 85 | self.addr = addr 86 | self.family = family 87 | base = super(FailoverProfilingClient, self) 88 | base.__init__(viewer, event_loop, None, title, protocol) 89 | 90 | def connect(self): 91 | while True: 92 | errno = self.sock.connect_ex(self.addr) 93 | if not errno: 94 | # connected immediately. 95 | break 96 | elif errno == EINPROGRESS: 97 | # will be connected. 98 | break 99 | elif errno == ENOENT: 100 | # no such socket file. 101 | self.create_connection(self.failover_interval) 102 | return 103 | else: 104 | raise ValueError('Unexpected socket errno: %d' % errno) 105 | self.event_loop.watch_file(self.sock.fileno(), self.handle) 106 | 107 | def disconnect(self, errno): 108 | self.sock.close() 109 | # try to reconnect. 110 | delay = self.failover_interval if errno == ECONNREFUSED else 0 111 | self.create_connection(delay) 112 | 113 | def create_connection(self, delay=0): 114 | self.sock = socket.socket(self.family) 115 | self.sock.setblocking(0) 116 | self.event_loop.alarm(delay, self.connect) 117 | 118 | def start(self): 119 | self.create_connection() 120 | 121 | def erred(self, errno): 122 | super(FailoverProfilingClient, self).erred(errno) 123 | self.disconnect(errno) 124 | -------------------------------------------------------------------------------- /profiling/remote/gevent.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.remote.gevent 4 | ~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Implements a profiling server based on `gevent`_. 7 | 8 | .. _gevent: http://gevent.org/ 9 | 10 | :copyright: (c) 2014-2017, What! Studio 11 | :license: BSD, see LICENSE for more details. 12 | 13 | """ 14 | from __future__ import absolute_import 15 | 16 | import socket 17 | 18 | import gevent 19 | from gevent.lock import Semaphore 20 | from gevent.server import StreamServer 21 | from gevent.util import wrap_errors 22 | 23 | from profiling.remote import INTERVAL, LOG, PICKLE_PROTOCOL, ProfilingServer 24 | 25 | 26 | __all__ = ['GeventProfilingServer'] 27 | 28 | 29 | class GeventProfilingServer(StreamServer, ProfilingServer): 30 | """A profiling server implementation based on `gevent`_. When you choose 31 | it, you should set a :class:`profiling.timers.greenlet.GreenletTimer` for 32 | the profiler's timer:: 33 | 34 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 35 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 36 | sock.bind(('', 0)) 37 | sock.listen(1) 38 | 39 | profiler = Profiler(GreenletTimer()) 40 | server = GeventProfilingServer(sock, profiler) 41 | server.serve_forever() 42 | 43 | .. _gevent: http://gevent.org/ 44 | 45 | """ 46 | 47 | def __init__(self, listener, profiler=None, interval=INTERVAL, 48 | log=LOG, pickle_protocol=PICKLE_PROTOCOL, **server_kwargs): 49 | StreamServer.__init__(self, listener, **server_kwargs) 50 | ProfilingServer.__init__(self, profiler, interval, 51 | log, pickle_protocol) 52 | self.lock = Semaphore() 53 | self.profiling_greenlet = None 54 | 55 | def _send(self, sock, data): 56 | sock.sendall(data) 57 | 58 | def _close(self, sock): 59 | sock.close() 60 | 61 | def _addr(self, sock): 62 | return sock.getsockname() 63 | 64 | def _start_profiling(self): 65 | self.profiling_greenlet = gevent.spawn(self.profile_periodically) 66 | 67 | def _start_watching(self, sock): 68 | disconnected = lambda x: self.disconnected(sock) 69 | recv = wrap_errors(socket.error, sock.recv) 70 | gevent.spawn(recv, 1).link(disconnected) 71 | 72 | def profile_periodically(self): 73 | with self.lock: 74 | for __ in self.profiling(): 75 | gevent.sleep(self.interval) 76 | 77 | def handle(self, sock, addr=None): 78 | self.connected(sock) 79 | 80 | # Disconnect once profiling has stopped. 81 | if self.profiling_greenlet: 82 | self.profiling_greenlet.join() 83 | -------------------------------------------------------------------------------- /profiling/remote/select.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.remote.select 4 | ~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Implements a profiling server based on `select`_. 7 | 8 | .. _select: https://docs.python.org/library/select.html 9 | 10 | :copyright: (c) 2014-2017, What! Studio 11 | :license: BSD, see LICENSE for more details. 12 | 13 | """ 14 | from __future__ import absolute_import 15 | 16 | from errno import ECONNRESET, EINTR, ENOTCONN 17 | import select 18 | import socket 19 | import time 20 | 21 | from profiling.remote import ProfilingServer 22 | 23 | 24 | __all__ = ['SelectProfilingServer'] 25 | 26 | 27 | class SelectProfilingServer(ProfilingServer): 28 | 29 | def __init__(self, listener, *args, **kwargs): 30 | super(SelectProfilingServer, self).__init__(*args, **kwargs) 31 | self.listener = listener 32 | 33 | def serve_forever(self): 34 | while True: 35 | self.dispatch_sockets() 36 | 37 | def _send(self, sock, data): 38 | sock.sendall(data, socket.MSG_DONTWAIT) 39 | 40 | def _close(self, sock): 41 | sock.close() 42 | 43 | def _addr(self, sock): 44 | try: 45 | return sock.getpeername() 46 | except socket.error as exc: 47 | if exc.errno == ENOTCONN: 48 | return None 49 | else: 50 | raise 51 | 52 | def _start_profiling(self): 53 | self.profile_periodically() 54 | 55 | def profile_periodically(self): 56 | for __ in self.profiling(): 57 | self.dispatch_sockets(self.interval) 58 | 59 | def _start_watching(self, sock): 60 | pass 61 | 62 | def sockets(self): 63 | """Returns the set of the sockets.""" 64 | if self.listener is None: 65 | return self.clients 66 | else: 67 | return self.clients.union([self.listener]) 68 | 69 | def select_sockets(self, timeout=None): 70 | """EINTR safe version of `select`. It focuses on just incoming 71 | sockets. 72 | """ 73 | if timeout is not None: 74 | t = time.time() 75 | while True: 76 | try: 77 | ready, __, __ = select.select(self.sockets(), (), (), timeout) 78 | except ValueError: 79 | # there's fd=0 socket. 80 | pass 81 | except select.error as exc: 82 | # ignore an interrupted system call. 83 | if exc.args[0] != EINTR: 84 | raise 85 | else: 86 | # succeeded. 87 | return ready 88 | # retry. 89 | if timeout is None: 90 | continue 91 | # decrease timeout. 92 | t2 = time.time() 93 | timeout -= t2 - t 94 | t = t2 95 | if timeout <= 0: 96 | # timed out. 97 | return [] 98 | 99 | def dispatch_sockets(self, timeout=None): 100 | """Dispatches incoming sockets.""" 101 | for sock in self.select_sockets(timeout=timeout): 102 | if sock is self.listener: 103 | listener = sock 104 | sock, addr = listener.accept() 105 | self.connected(sock) 106 | else: 107 | try: 108 | sock.recv(1) 109 | except socket.error as exc: 110 | if exc.errno != ECONNRESET: 111 | raise 112 | self.disconnected(sock) 113 | -------------------------------------------------------------------------------- /profiling/sampling/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.sampling 4 | ~~~~~~~~~~~~~~~~~~ 5 | 6 | Statistical profiling. 7 | 8 | :copyright: (c) 2014-2017, What! Studio 9 | :license: BSD, see LICENSE for more details. 10 | 11 | """ 12 | from __future__ import absolute_import 13 | 14 | from profiling import sortkeys 15 | from profiling.profiler import Profiler 16 | from profiling.sampling.samplers import ItimerSampler, Sampler 17 | from profiling.stats import ( 18 | RecordingStatistics, VoidRecordingStatistics as void) 19 | from profiling.viewer import fmt, StatisticsTable 20 | 21 | 22 | __all__ = ['SamplingProfiler', 'SamplingStatisticsTable'] 23 | 24 | 25 | SAMPLER_CLASS = ItimerSampler 26 | 27 | 28 | class SamplingStatisticsTable(StatisticsTable): 29 | 30 | columns = [ 31 | ('FUNCTION', 'left', ('weight', 1), sortkeys.by_function), 32 | ('OWN', 'right', (6,), sortkeys.by_own_hits), 33 | ('%', 'left', (4,), None), 34 | ('DEEP', 'right', (6,), sortkeys.by_deep_hits), 35 | ('%', 'left', (4,), None), 36 | ] 37 | order = sortkeys.by_deep_hits 38 | 39 | def make_cells(self, node, stats): 40 | root_stats = node.get_root().get_value() 41 | yield fmt.make_stat_text(stats) 42 | yield fmt.make_int_or_na_text(stats.own_hits) 43 | yield fmt.make_percent_text(stats.own_hits, root_stats.deep_hits) 44 | yield fmt.make_int_or_na_text(stats.deep_hits) 45 | yield fmt.make_percent_text(stats.deep_hits, root_stats.deep_hits) 46 | 47 | 48 | class SamplingProfiler(Profiler): 49 | 50 | table_class = SamplingStatisticsTable 51 | 52 | #: The frames sampler. Usually it is an instance of :class:`profiling. 53 | #: sampling.samplers.Sampler`. 54 | sampler = None 55 | 56 | def __init__(self, base_frame=None, base_code=None, 57 | ignored_frames=(), ignored_codes=(), sampler=None): 58 | sampler = sampler or SAMPLER_CLASS() 59 | if not isinstance(sampler, Sampler): 60 | raise TypeError('Not a sampler instance') 61 | base = super(SamplingProfiler, self) 62 | base.__init__(base_frame, base_code, ignored_frames, ignored_codes) 63 | self.sampler = sampler 64 | 65 | def sample(self, frame): 66 | """Samples the given frame.""" 67 | frames = self.frame_stack(frame) 68 | if frames: 69 | frames.pop() 70 | parent_stats = self.stats 71 | for f in frames: 72 | parent_stats = parent_stats.ensure_child(f.f_code, void) 73 | stats = parent_stats.ensure_child(frame.f_code, RecordingStatistics) 74 | stats.own_hits += 1 75 | 76 | def run(self): 77 | self.sampler.start(self) 78 | yield 79 | self.sampler.stop() 80 | -------------------------------------------------------------------------------- /profiling/sampling/samplers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.sampling.samplers 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | :copyright: (c) 2014-2017, What! Studio 7 | :license: BSD, see LICENSE for more details. 8 | 9 | """ 10 | from __future__ import absolute_import 11 | 12 | import functools 13 | import signal 14 | import sys 15 | import threading 16 | import weakref 17 | 18 | import six.moves._thread as _thread 19 | 20 | from profiling.utils import deferral, Runnable, thread_clock 21 | 22 | 23 | __all__ = ['Sampler', 'ItimerSampler', 'TracingSampler'] 24 | 25 | 26 | INTERVAL = 1e-3 # 1ms 27 | 28 | 29 | class Sampler(Runnable): 30 | """The base class for samplers.""" 31 | 32 | #: Sampling interval. 33 | interval = INTERVAL 34 | 35 | def __init__(self, interval=INTERVAL): 36 | self.interval = interval 37 | 38 | 39 | class ItimerSampler(Sampler): 40 | """Uses ``signal.ITIMER_PROF`` to sample running frames. 41 | 42 | .. note:: 43 | 44 | ``signal.SIGPROF`` is triggeres by only the main thread. If you need 45 | sample multiple threads, use :class:`TracingSampler` instead. 46 | 47 | """ 48 | 49 | def handle_signal(self, profiler, signum, frame): 50 | profiler.sample(frame) 51 | 52 | def run(self, profiler): 53 | weak_profiler = weakref.proxy(profiler) 54 | handle = functools.partial(self.handle_signal, weak_profiler) 55 | t = self.interval 56 | with deferral() as defer: 57 | prev_handle = signal.signal(signal.SIGPROF, handle) 58 | if prev_handle == signal.SIG_DFL: 59 | # sometimes the process receives SIGPROF although the sampler 60 | # unsets the itimer. If the previous handler was SIG_DFL, the 61 | # process will crash when received SIGPROF. To prevent this 62 | # risk, it makes the process to ignore SIGPROF when it isn't 63 | # running if the previous handler was SIG_DFL. 64 | prev_handle = signal.SIG_IGN 65 | defer(signal.signal, signal.SIGPROF, prev_handle) 66 | prev_itimer = signal.setitimer(signal.ITIMER_PROF, t, t) 67 | defer(signal.setitimer, signal.ITIMER_PROF, *prev_itimer) 68 | yield 69 | 70 | 71 | class TracingSampler(Sampler): 72 | """Uses :func:`sys.setprofile` and :func:`threading.setprofile` to sample 73 | running frames per thread. It can be used at systems which do not support 74 | profiling signals. 75 | 76 | Just like :class:`profiling.tracing.timers.ThreadTimer`, `Yappi`_ is 77 | required for earlier than Python 3.3. 78 | 79 | .. _Yappi: https://code.google.com/p/yappi/ 80 | 81 | """ 82 | 83 | def __init__(self, *args, **kwargs): 84 | super(TracingSampler, self).__init__(*args, **kwargs) 85 | self.sampled_times = {} 86 | self.counter = 0 87 | 88 | def _profile(self, profiler, frame, event, arg): 89 | t = thread_clock() 90 | thread_id = _thread.get_ident() 91 | sampled_at = self.sampled_times.get(thread_id, 0) 92 | if t - sampled_at < self.interval: 93 | return 94 | self.sampled_times[thread_id] = t 95 | profiler.sample(frame) 96 | self.counter += 1 97 | if self.counter % 10000 == 0: 98 | self._clear_for_dead_threads() 99 | 100 | def _clear_for_dead_threads(self): 101 | for thread_id in sys._current_frames().keys(): 102 | self.sampled_times.pop(thread_id, None) 103 | 104 | def run(self, profiler): 105 | profile = functools.partial(self._profile, profiler) 106 | with deferral() as defer: 107 | sys.setprofile(profile) 108 | defer(sys.setprofile, None) 109 | threading.setprofile(profile) 110 | defer(threading.setprofile, None) 111 | yield 112 | -------------------------------------------------------------------------------- /profiling/sortkeys.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.sortkeys 4 | ~~~~~~~~~~~~~~~~~~ 5 | 6 | :copyright: (c) 2014-2017, What! Studio 7 | :license: BSD, see LICENSE for more details. 8 | 9 | """ 10 | from __future__ import absolute_import 11 | 12 | 13 | __all__ = ['by_name', 'by_module', 'by_deep_hits', 'by_own_hits', 14 | 'by_deep_time', 'by_own_time', 'by_deep_time_per_call', 15 | 'by_own_time_per_call'] 16 | 17 | 18 | class SortKey(object): 19 | 20 | def __init__(self, func): 21 | super(SortKey, self).__init__() 22 | self.func = func 23 | 24 | def __call__(self, stat): 25 | return self.func(stat) 26 | 27 | def __invert__(self): 28 | cls = type(self) 29 | return cls(lambda stat: -self.func(stat)) 30 | 31 | 32 | #: Sorting by name in ascending order. 33 | by_name = SortKey(lambda stat: stat.name) 34 | 35 | #: Sorting by module in ascending order. 36 | by_module = SortKey(lambda stat: stat.module) 37 | 38 | #: Sorting by module and name in ascending order. 39 | by_function = SortKey(lambda stat: (stat.module, stat.name)) 40 | 41 | #: Sorting by number of inclusive hits in descending order. 42 | by_deep_hits = SortKey(lambda stat: -stat.deep_hits) 43 | 44 | #: Sorting by number of exclusive hits in descending order. 45 | by_own_hits = SortKey(lambda stat: -stat.own_hits) 46 | 47 | #: Sorting by inclusive elapsed time in descending order. 48 | by_deep_time = SortKey(lambda stat: -stat.deep_time) 49 | 50 | #: Sorting by exclusive elapsed time in descending order. 51 | by_own_time = SortKey(lambda stat: (-stat.own_time, -stat.deep_time)) 52 | 53 | 54 | @SortKey 55 | def by_deep_time_per_call(stat): 56 | """Sorting by inclusive elapsed time per call in descending order.""" 57 | return -stat.deep_time_per_call if stat.deep_hits else -stat.deep_time 58 | 59 | 60 | @SortKey 61 | def by_own_time_per_call(stat): 62 | """Sorting by exclusive elapsed time per call in descending order.""" 63 | return (-stat.own_time_per_call if stat.own_hits else -stat.own_time, 64 | by_deep_time_per_call(stat)) 65 | -------------------------------------------------------------------------------- /profiling/speedup.c: -------------------------------------------------------------------------------- 1 | #include "Python.h" 2 | #include "frameobject.h" 3 | 4 | #define PYOBJ PyObject* 5 | 6 | static PyObject * 7 | frame_stack(PyObject *self, PyObject *args) 8 | { 9 | // frame_stack(frame, base_frame, base_code, ignored_frames, ignored_codes) 10 | // returns a list of frames. 11 | PyFrameObject* frame; 12 | const PyFrameObject* base_frame; 13 | const PyCodeObject* base_code; 14 | const PySetObject* ignored_frames; 15 | const PySetObject* ignored_codes; 16 | if (!PyArg_ParseTuple(args, "OOOOO", &frame, &base_frame, &base_code, 17 | &ignored_frames, &ignored_codes)) 18 | { 19 | return NULL; 20 | } 21 | PyObject* frame_stack = PyList_New(0); 22 | if (frame_stack == NULL) 23 | { 24 | return NULL; 25 | } 26 | while (frame != NULL) 27 | { 28 | if (frame == base_frame || frame->f_code == base_code) 29 | { 30 | break; 31 | } 32 | if (PySet_Contains((PYOBJ)ignored_frames, (PYOBJ)frame) == 0 && 33 | PySet_Contains((PYOBJ)ignored_codes, (PYOBJ)frame->f_code) == 0) 34 | { 35 | // Not ignored. 36 | if (PyList_Append(frame_stack, (PyObject*)frame) == -1) 37 | { 38 | return NULL; 39 | } 40 | } 41 | frame = frame->f_back; 42 | } 43 | PyList_Reverse(frame_stack); 44 | return frame_stack; 45 | } 46 | 47 | static PyMethodDef SpeedupMethods[] = { 48 | {"frame_stack", frame_stack, METH_VARARGS, ""}, 49 | {NULL, NULL, 0, NULL} 50 | }; 51 | 52 | // https://docs.python.org/3/howto/cporting.html 53 | #if PY_MAJOR_VERSION >= 3 54 | static struct PyModuleDef SpeedupModule = { 55 | PyModuleDef_HEAD_INIT, 56 | "speedup", 57 | NULL, 58 | -1, 59 | SpeedupMethods 60 | }; 61 | PyMODINIT_FUNC 62 | PyInit_speedup(void) 63 | { 64 | return PyModule_Create(&SpeedupModule); 65 | } 66 | #else 67 | PyMODINIT_FUNC 68 | initspeedup(void) 69 | { 70 | (void) Py_InitModule("speedup", SpeedupMethods); 71 | } 72 | #endif 73 | -------------------------------------------------------------------------------- /profiling/stats.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.stats 4 | ~~~~~~~~~~~~~~~ 5 | 6 | Statistics classes. 7 | 8 | :copyright: (c) 2014-2017, What! Studio 9 | :license: BSD, see LICENSE for more details. 10 | 11 | """ 12 | from __future__ import absolute_import, division 13 | 14 | from collections import deque 15 | import inspect 16 | import itertools 17 | 18 | from six import itervalues, with_metaclass 19 | from six.moves import zip 20 | 21 | from profiling.sortkeys import by_deep_time 22 | from profiling.utils import noop 23 | 24 | 25 | __all__ = ['Statistics', 'RecordingStatistics', 'VoidRecordingStatistics', 26 | 'FrozenStatistics', 'FlatFrozenStatistics'] 27 | 28 | 29 | class spread_t(object): 30 | __slots__ = ('flag',) 31 | __bool__ = __nonzero__ = lambda x: x.flag 32 | def clear(self): 33 | self.flag = False 34 | def __call__(self): 35 | self.flag = True 36 | 37 | 38 | def spread_stats(stats, spreader=False): 39 | """Iterates all descendant statistics under the given root statistics. 40 | 41 | When ``spreader=True``, each iteration yields a descendant statistics and 42 | `spread()` function together. You should call `spread()` if you want to 43 | spread the yielded statistics also. 44 | 45 | """ 46 | spread = spread_t() if spreader else True 47 | descendants = deque(stats) 48 | while descendants: 49 | _stats = descendants.popleft() 50 | if spreader: 51 | spread.clear() 52 | yield _stats, spread 53 | else: 54 | yield _stats 55 | if spread: 56 | descendants.extend(_stats) 57 | 58 | 59 | class default(object): 60 | 61 | __slots__ = ('value',) 62 | 63 | def __init__(self, value): 64 | self.value = value 65 | 66 | 67 | class StatisticsMeta(type): 68 | 69 | def __new__(meta, name, bases, attrs): 70 | slots = attrs.get('__slots__', ()) 71 | defaults = {} 72 | for attr in slots: 73 | if attr not in attrs: 74 | continue 75 | elif isinstance(attrs[attr], default): 76 | defaults[attr] = attrs.pop(attr).value 77 | cls = super(StatisticsMeta, meta).__new__(meta, name, bases, attrs) 78 | try: 79 | base_defaults = cls.__defaults__ 80 | except AttributeError: 81 | pass 82 | else: 83 | # inherit defaults from the base classes. 84 | for attr in slots: 85 | if attr not in defaults and attr in base_defaults: 86 | defaults[attr] = base_defaults[attr] 87 | cls.__defaults__ = defaults 88 | return cls 89 | 90 | def __call__(cls, *args, **kwargs): 91 | obj = super(StatisticsMeta, cls).__call__(*args, **kwargs) 92 | for attr, value in cls.__defaults__.items(): 93 | if not hasattr(obj, attr): 94 | setattr(obj, attr, value) 95 | return obj 96 | 97 | 98 | class Statistics(with_metaclass(StatisticsMeta)): 99 | """Statistics of a function.""" 100 | 101 | __slots__ = ('name', 'filename', 'lineno', 'module', 102 | 'own_hits', 'deep_time') 103 | 104 | name = default(None) 105 | filename = default(None) 106 | lineno = default(None) 107 | module = default(None) 108 | #: The inclusive calling/sampling number. 109 | own_hits = default(0) 110 | #: The exclusive execution time. 111 | deep_time = default(0.0) 112 | 113 | def __init__(self, *args, **kwargs): 114 | for attr, value in zip(self.__slots__, args): 115 | setattr(self, attr, value) 116 | for attr, value in kwargs.items(): 117 | setattr(self, attr, value) 118 | 119 | @property 120 | def regular_name(self): 121 | name, module = self.name, self.module 122 | if name and module: 123 | return ':'.join([module, name]) 124 | return name or module 125 | 126 | @property 127 | def deep_hits(self): 128 | """The inclusive calling/sampling number. 129 | 130 | Calculates as sum of the own hits and deep hits of the children. 131 | """ 132 | hits = [self.own_hits] 133 | hits.extend(stats.own_hits for stats in spread_stats(self)) 134 | return sum(hits) 135 | 136 | @property 137 | def own_time(self): 138 | """The exclusive execution time.""" 139 | sub_time = sum(stats.deep_time for stats in self) 140 | return max(0., self.deep_time - sub_time) 141 | 142 | @property 143 | def deep_time_per_call(self): 144 | try: 145 | return self.deep_time / self.own_hits 146 | except ZeroDivisionError: 147 | return 0.0 148 | 149 | @property 150 | def own_time_per_call(self): 151 | try: 152 | return self.own_time / self.own_hits 153 | except ZeroDivisionError: 154 | return 0.0 155 | 156 | def sorted(self, order=by_deep_time): 157 | return sorted(self, key=order) 158 | 159 | def __iter__(self): 160 | """Override it to walk statistics children.""" 161 | return iter(()) 162 | 163 | def __len__(self): 164 | """Override it to count statistics children.""" 165 | return 0 166 | 167 | def __hash__(self): 168 | """Statistics can be a key.""" 169 | return hash((self.name, self.filename, self.lineno)) 170 | 171 | def __reduce__(self): 172 | """Freezes this statistics to safen to pack/unpack in Pickle.""" 173 | tree = make_frozen_stats_tree(self) 174 | return (frozen_stats_from_tree, (tree,)) 175 | 176 | def __repr__(self): 177 | # format name 178 | regular_name = self.regular_name 179 | name_string = "'{0}' ".format(regular_name) if regular_name else '' 180 | # format hits 181 | deep_hits = self.deep_hits 182 | if self.own_hits == deep_hits: 183 | hits_string = str(self.own_hits) 184 | else: 185 | hits_string = '{0}/{1}'.format(self.own_hits, deep_hits) 186 | # format time 187 | own_time = self.own_time 188 | if own_time == self.deep_time: 189 | time_string = '{0:.6f}'.format(self.deep_time) 190 | else: 191 | time_string = '{0:.6f}/{1:.6f}'.format(own_time, self.deep_time) 192 | # join all 193 | class_name = type(self).__name__ 194 | return ('<{0} {1}hits={2} time={3}>' 195 | ''.format(class_name, name_string, hits_string, time_string)) 196 | 197 | 198 | class RecordingStatistics(Statistics): 199 | """Recordig statistics measures execution time of a code.""" 200 | 201 | __slots__ = ('own_hits', 'deep_time', 'code', '_children') 202 | 203 | own_hits = default(0) 204 | deep_time = default(0.0) 205 | 206 | def __init__(self, code=None): 207 | self.code = code 208 | self._children = {} 209 | 210 | @property 211 | def name(self): 212 | if self.code is None: 213 | return 214 | name = self.code.co_name 215 | if name == '': 216 | return 217 | return name 218 | 219 | @property 220 | def filename(self): 221 | return self.code and self.code.co_filename 222 | 223 | @property 224 | def lineno(self): 225 | return self.code and self.code.co_firstlineno 226 | 227 | @property 228 | def module(self): 229 | if self.code is None: 230 | return 231 | module = inspect.getmodule(self.code) 232 | if not module: 233 | return 234 | return module.__name__ 235 | 236 | @property 237 | def children(self): 238 | return list(itervalues(self._children)) 239 | 240 | def get_child(self, code): 241 | return self._children[code] 242 | 243 | def add_child(self, code, stats): 244 | self._children[code] = stats 245 | 246 | def remove_child(self, code): 247 | del self._children[code] 248 | 249 | def discard_child(self, code): 250 | self._children.pop(code, None) 251 | 252 | def ensure_child(self, code, adding_stat_class=None): 253 | stats = self._children.get(code) 254 | if stats is None: 255 | stat_class = adding_stat_class or type(self) 256 | stats = stat_class(code) 257 | self.add_child(code, stats) 258 | return stats 259 | 260 | def clear(self): 261 | self._children.clear() 262 | for attr, value in self.__defaults__.items(): 263 | setattr(self, attr, value) 264 | 265 | def __iter__(self): 266 | return itervalues(self._children) 267 | 268 | def __len__(self): 269 | return len(self._children) 270 | 271 | def __contains__(self, code): 272 | return code in self._children 273 | 274 | 275 | class VoidRecordingStatistics(RecordingStatistics): 276 | """Statistics for an absent frame.""" 277 | 278 | __slots__ = ('code', '_children') 279 | 280 | own_hits = property(lambda x: 0, noop) 281 | 282 | def deep_time(self): 283 | times = [] 284 | for stats, spread in spread_stats(self, spreader=True): 285 | if isinstance(stats, VoidRecordingStatistics): 286 | spread() 287 | else: 288 | times.append(stats.deep_time) 289 | return sum(times) 290 | 291 | deep_time = property(deep_time, noop) 292 | 293 | 294 | class FrozenStatistics(Statistics): 295 | """Frozen :class:`Statistics` to serialize by Pickle.""" 296 | 297 | __slots__ = ('name', 'filename', 'lineno', 'module', 298 | 'own_hits', 'deep_time', 'children') 299 | 300 | def __init__(self, *args, **kwargs): 301 | super(FrozenStatistics, self).__init__(*args, **kwargs) 302 | if not hasattr(self, 'children'): 303 | self.children = [] 304 | 305 | def __iter__(self): 306 | return iter(self.children) 307 | 308 | def __len__(self): 309 | return len(self.children) 310 | 311 | 312 | def make_frozen_stats_tree(stats): 313 | """Makes a flat members tree of the given statistics. The statistics can 314 | be restored by :func:`frozen_stats_from_tree`. 315 | """ 316 | tree, stats_tree = [], [(None, stats)] 317 | for x in itertools.count(): 318 | try: 319 | parent_offset, _stats = stats_tree[x] 320 | except IndexError: 321 | break 322 | stats_tree.extend((x, s) for s in _stats) 323 | members = (_stats.name, _stats.filename, _stats.lineno, 324 | _stats.module, _stats.own_hits, _stats.deep_time) 325 | tree.append((parent_offset, members)) 326 | return tree 327 | 328 | 329 | def frozen_stats_from_tree(tree): 330 | """Restores a statistics from the given flat members tree. 331 | :func:`make_frozen_stats_tree` makes a tree for this function. 332 | """ 333 | if not tree: 334 | raise ValueError('Empty tree') 335 | stats_index = [] 336 | for parent_offset, members in tree: 337 | stats = FrozenStatistics(*members) 338 | stats_index.append(stats) 339 | if parent_offset is not None: 340 | stats_index[parent_offset].children.append(stats) 341 | return stats_index[0] 342 | 343 | 344 | class FlatFrozenStatistics(FrozenStatistics): 345 | 346 | __slots__ = ('name', 'filename', 'lineno', 'module', 347 | 'own_hits', 'deep_hits', 'own_time', 'deep_time', 348 | 'children') 349 | 350 | own_hits = default(0) 351 | deep_hits = default(0) 352 | own_time = default(0.0) 353 | deep_time = default(0.0) 354 | children = default(()) 355 | 356 | @classmethod 357 | def flatten(cls, stats): 358 | """Makes a flat statistics from the given statistics.""" 359 | flat_children = {} 360 | for _stats in spread_stats(stats): 361 | key = (_stats.name, _stats.filename, _stats.lineno, _stats.module) 362 | try: 363 | flat_stats = flat_children[key] 364 | except KeyError: 365 | flat_stats = flat_children[key] = cls(*key) 366 | flat_stats.own_hits += _stats.own_hits 367 | flat_stats.deep_hits += _stats.deep_hits 368 | flat_stats.own_time += _stats.own_time 369 | flat_stats.deep_time += _stats.deep_time 370 | children = list(itervalues(flat_children)) 371 | return cls(stats.name, stats.filename, stats.lineno, stats.module, 372 | stats.own_hits, stats.deep_hits, stats.own_time, 373 | stats.deep_time, children) 374 | -------------------------------------------------------------------------------- /profiling/tracing/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.tracing 4 | ~~~~~~~~~~~~~~~~~ 5 | 6 | Profiles deterministically by :func:`sys.setprofile`. 7 | 8 | :copyright: (c) 2014-2017, What! Studio 9 | :license: BSD, see LICENSE for more details. 10 | 11 | """ 12 | from __future__ import absolute_import 13 | 14 | import sys 15 | import threading 16 | 17 | from profiling import sortkeys 18 | from profiling.profiler import Profiler 19 | from profiling.stats import ( 20 | RecordingStatistics, VoidRecordingStatistics as void) 21 | from profiling.tracing.timers import Timer 22 | from profiling.utils import deferral 23 | from profiling.viewer import fmt, StatisticsTable 24 | 25 | 26 | __all__ = ['TracingProfiler', 'TracingStatisticsTable'] 27 | 28 | 29 | TIMER_CLASS = Timer 30 | 31 | 32 | class TracingStatisticsTable(StatisticsTable): 33 | 34 | columns = [ 35 | ('FUNCTION', 'left', ('weight', 1), sortkeys.by_function), 36 | ('CALLS', 'right', (6,), sortkeys.by_own_hits), 37 | ('OWN', 'right', (6,), sortkeys.by_own_time), 38 | ('/CALL', 'right', (6,), sortkeys.by_own_time_per_call), 39 | ('%', 'left', (4,), None), 40 | ('DEEP', 'right', (6,), sortkeys.by_deep_time), 41 | ('/CALL', 'right', (6,), sortkeys.by_deep_time_per_call), 42 | ('%', 'left', (4,), None), 43 | ] 44 | order = sortkeys.by_deep_time 45 | 46 | def make_cells(self, node, stats): 47 | yield fmt.make_stat_text(stats) 48 | yield fmt.make_int_or_na_text(stats.own_hits) 49 | yield fmt.make_time_text(stats.own_time) 50 | yield fmt.make_time_text(stats.own_time_per_call) 51 | yield fmt.make_percent_text(stats.own_time, self.cpu_time) 52 | yield fmt.make_time_text(stats.deep_time) 53 | yield fmt.make_time_text(stats.deep_time_per_call) 54 | yield fmt.make_percent_text(stats.deep_time, self.cpu_time) 55 | 56 | 57 | class TracingProfiler(Profiler): 58 | """The tracing profiler.""" 59 | 60 | table_class = TracingStatisticsTable 61 | 62 | #: The CPU timer. Usually it is an instance of :class:`profiling.tracing. 63 | #: timers.Timer`. 64 | timer = None 65 | 66 | #: The CPU time of profiling overhead. It's the time spent in 67 | #: :meth:`_profile`. 68 | overhead = 0.0 69 | 70 | def __init__(self, base_frame=None, base_code=None, 71 | ignored_frames=(), ignored_codes=(), timer=None): 72 | timer = timer or TIMER_CLASS() 73 | if not isinstance(timer, Timer): 74 | raise TypeError('Not a timer instance') 75 | base = super(TracingProfiler, self) 76 | base.__init__(base_frame, base_code, ignored_frames, ignored_codes) 77 | self.timer = timer 78 | self._times_entered = {} 79 | 80 | def _profile(self, frame, event, arg): 81 | """The callback function to register by :func:`sys.setprofile`.""" 82 | # c = event.startswith('c_') 83 | if event.startswith('c_'): 84 | return 85 | time1 = self.timer() 86 | frames = self.frame_stack(frame) 87 | if frames: 88 | frames.pop() 89 | parent_stats = self.stats 90 | for f in frames: 91 | parent_stats = parent_stats.ensure_child(f.f_code, void) 92 | code = frame.f_code 93 | frame_key = id(frame) 94 | # if c: 95 | # event = event[2:] 96 | # code = mock_code(arg.__name__) 97 | # frame_key = id(arg) 98 | # record 99 | time2 = self.timer() 100 | self.overhead += time2 - time1 101 | if event == 'call': 102 | time = time2 - self.overhead 103 | self.record_entering(time, code, frame_key, parent_stats) 104 | elif event == 'return': 105 | time = time1 - self.overhead 106 | self.record_leaving(time, code, frame_key, parent_stats) 107 | time3 = self.timer() 108 | self.overhead += time3 - time2 109 | 110 | def record_entering(self, time, code, frame_key, parent_stats): 111 | """Entered to a function call.""" 112 | stats = parent_stats.ensure_child(code, RecordingStatistics) 113 | self._times_entered[(code, frame_key)] = time 114 | stats.own_hits += 1 115 | 116 | def record_leaving(self, time, code, frame_key, parent_stats): 117 | """Left from a function call.""" 118 | try: 119 | stats = parent_stats.get_child(code) 120 | time_entered = self._times_entered.pop((code, frame_key)) 121 | except KeyError: 122 | return 123 | time_elapsed = time - time_entered 124 | stats.deep_time += max(0, time_elapsed) 125 | 126 | def result(self): 127 | base = super(TracingProfiler, self) 128 | frozen_stats, cpu_time, wall_time = base.result() 129 | return (frozen_stats, cpu_time - self.overhead, wall_time) 130 | 131 | def run(self): 132 | if sys.getprofile() is not None: 133 | # NOTE: There's no threading.getprofile(). 134 | # The profiling function will be stored at threading._profile_hook 135 | # but it's not documented. 136 | raise RuntimeError('Another profiler already registered') 137 | with deferral() as defer: 138 | self._times_entered.clear() 139 | self.overhead = 0.0 140 | sys.setprofile(self._profile) 141 | defer(sys.setprofile, None) 142 | threading.setprofile(self._profile) 143 | defer(threading.setprofile, None) 144 | self.timer.start(self) 145 | defer(self.timer.stop) 146 | yield 147 | -------------------------------------------------------------------------------- /profiling/tracing/timers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.tracing.timers 4 | ~~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | :copyright: (c) 2014-2017, What! Studio 7 | :license: BSD, see LICENSE for more details. 8 | 9 | """ 10 | from __future__ import absolute_import 11 | 12 | import time 13 | 14 | from profiling.utils import lazy_import, Runnable, thread_clock 15 | 16 | 17 | __all__ = ['Timer', 'ContextualTimer', 'ThreadTimer', 'GreenletTimer'] 18 | 19 | 20 | class Timer(Runnable): 21 | """The basic timer.""" 22 | 23 | #: The raw function to get the CPU time. 24 | clock = time.clock 25 | 26 | def __call__(self): 27 | return self.clock() 28 | 29 | def run(self, profiler): 30 | yield 31 | 32 | 33 | class ContextualTimer(Timer): 34 | 35 | def __new__(cls, *args, **kwargs): 36 | timer = super(ContextualTimer, cls).__new__(cls, *args, **kwargs) 37 | timer._contextual_times = {} 38 | return timer 39 | 40 | def __call__(self, context=None): 41 | if context is None: 42 | context = self.detect_context() 43 | paused_at, resumed_at = self._contextual_times.get(context, (0, 0)) 44 | if resumed_at is None: # paused 45 | return paused_at 46 | return paused_at + self.clock() - resumed_at 47 | 48 | def pause(self, context=None): 49 | if context is None: 50 | context = self.detect_context() 51 | self._contextual_times[context] = (self(context), None) 52 | 53 | def resume(self, context=None): 54 | if context is None: 55 | context = self.detect_context() 56 | paused_at, __ = self._contextual_times.get(context, (0, 0)) 57 | self._contextual_times[context] = (paused_at, self.clock()) 58 | 59 | def detect_context(self): 60 | raise NotImplementedError('detect_context() should be implemented') 61 | 62 | 63 | class ThreadTimer(Timer): 64 | """A timer to get CPU time per thread. Python 3.3 or later uses the 65 | built-in :mod:`time` module. Earlier Python versions requires `Yappi`_ to 66 | be installed. 67 | 68 | .. _Yappi: https://code.google.com/p/yappi/ 69 | 70 | """ 71 | 72 | def __call__(self): 73 | return thread_clock() 74 | 75 | 76 | class GreenletTimer(ContextualTimer): 77 | """A timer to get CPU time per greenlet.""" 78 | 79 | greenlet = lazy_import('greenlet') 80 | 81 | def detect_context(self): 82 | if self.greenlet: 83 | return id(self.greenlet.getcurrent()) 84 | 85 | def _trace(self, event, args): 86 | origin, target = args 87 | self.pause(id(origin)) 88 | self.resume(id(target)) 89 | 90 | def run(self, profiler): 91 | self.greenlet.settrace(self._trace) 92 | yield 93 | self.greenlet.settrace(None) 94 | -------------------------------------------------------------------------------- /profiling/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.utils 4 | ~~~~~~~~~~~~~~~ 5 | 6 | :copyright: (c) 2014-2017, What! Studio 7 | :license: BSD, see LICENSE for more details. 8 | 9 | """ 10 | from __future__ import absolute_import 11 | 12 | from collections import deque 13 | from contextlib import contextmanager 14 | import sys 15 | 16 | try: 17 | from profiling import speedup 18 | except ImportError: 19 | speedup = False 20 | 21 | 22 | __all__ = ['Runnable', 'frame_stack', 'repr_frame', 'lazy_import', 'deferral', 23 | 'thread_clock', 'noop'] 24 | 25 | 26 | class Runnable(object): 27 | """The base class for runnable classes such as :class:`profiling.profiler. 28 | Profiler`. 29 | """ 30 | 31 | #: The generator :meth:`run` returns. It will be set by :meth:`start`. 32 | _running = None 33 | 34 | def is_running(self): 35 | """Whether the instance is running.""" 36 | return self._running is not None 37 | 38 | def start(self, *args, **kwargs): 39 | """Starts the instance. 40 | 41 | :raises RuntimeError: has been already started. 42 | :raises TypeError: :meth:`run` is not canonical. 43 | 44 | """ 45 | if self.is_running(): 46 | raise RuntimeError('Already started') 47 | self._running = self.run(*args, **kwargs) 48 | try: 49 | yielded = next(self._running) 50 | except StopIteration: 51 | raise TypeError('run() must yield just one time') 52 | if yielded is not None: 53 | raise TypeError('run() must yield without value') 54 | 55 | def stop(self): 56 | """Stops the instance. 57 | 58 | :raises RuntimeError: has not been started. 59 | :raises TypeError: :meth:`run` is not canonical. 60 | 61 | """ 62 | if not self.is_running(): 63 | raise RuntimeError('Not started') 64 | running, self._running = self._running, None 65 | try: 66 | next(running) 67 | except StopIteration: 68 | # expected. 69 | pass 70 | else: 71 | raise TypeError('run() must yield just one time') 72 | 73 | def run(self, *args, **kwargs): 74 | """Override it to implement the starting and stopping behavior. 75 | 76 | An overriding method must be a generator function which yields just one 77 | time without any value. :meth:`start` creates and iterates once the 78 | generator it returns. Then :meth:`stop` will iterates again. 79 | 80 | :raises NotImplementedError: :meth:`run` is not overridden. 81 | 82 | """ 83 | raise NotImplementedError('Implement run()') 84 | yield 85 | 86 | def __enter__(self): 87 | self.start() 88 | return self 89 | 90 | def __exit__(self, *exc_info): 91 | self.stop() 92 | 93 | 94 | if speedup: 95 | def frame_stack(frame, base_frame=None, base_code=None, 96 | ignored_frames=(), ignored_codes=()): 97 | return speedup.frame_stack(frame, base_frame, base_code, 98 | set(ignored_frames), set(ignored_codes)) 99 | else: 100 | def frame_stack(frame, base_frame=None, base_code=None, 101 | ignored_frames=(), ignored_codes=()): 102 | """Returns a deque of frame stack.""" 103 | frames = deque() 104 | while frame is not None: 105 | if frame is base_frame or frame.f_code is base_code: 106 | break 107 | if frame in ignored_frames or frame.f_code in ignored_codes: 108 | pass 109 | else: 110 | frames.appendleft(frame) 111 | frame = frame.f_back 112 | return frames 113 | 114 | 115 | def repr_frame(frame): 116 | return '%s:%d' % (frame.f_code.co_filename, frame.f_lineno) 117 | 118 | 119 | class LazyImport(object): 120 | 121 | def __init__(self, module_name): 122 | self.module_name = module_name 123 | self.module = None 124 | 125 | def __get__(self, obj, cls): 126 | if self.module is None: 127 | self.module = __import__(self.module_name) 128 | return self.module 129 | 130 | 131 | lazy_import = LazyImport 132 | 133 | 134 | @contextmanager 135 | def deferral(): 136 | """Defers a function call when it is being required like Go. 137 | 138 | :: 139 | 140 | with deferral() as defer: 141 | sys.setprofile(f) 142 | defer(sys.setprofile, None) 143 | # do something. 144 | 145 | """ 146 | deferred = [] 147 | defer = lambda f, *a, **k: deferred.append((f, a, k)) 148 | try: 149 | yield defer 150 | finally: 151 | while deferred: 152 | f, a, k = deferred.pop() 153 | f(*a, **k) 154 | 155 | 156 | if sys.version_info < (3, 3): 157 | class _yappi_holder_type(object): 158 | yappi = lazy_import('yappi') 159 | _yappi_holder = _yappi_holder_type() 160 | def thread_clock(): 161 | return _yappi_holder.yappi.get_clock_time() 162 | else: 163 | import time 164 | def thread_clock(): 165 | return time.clock_gettime(time.CLOCK_THREAD_CPUTIME_ID) 166 | 167 | 168 | #: Does nothing. It allows any arguments. 169 | noop = lambda x, *a, **k: None 170 | -------------------------------------------------------------------------------- /profiling/viewer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling.viewer 4 | ~~~~~~~~~~~~~~~~ 5 | 6 | A text user interface application which inspects statistics. To run it 7 | easily do: 8 | 9 | .. sourcecode:: console 10 | 11 | $ profiling view SOURCE 12 | 13 | :: 14 | 15 | viewer = StatisticsViewer() 16 | loop = viewer.loop() 17 | loop.run() 18 | 19 | :copyright: (c) 2014-2017, What! Studio 20 | :license: BSD, see LICENSE for more details. 21 | 22 | """ 23 | from __future__ import absolute_import 24 | 25 | from collections import deque 26 | 27 | import urwid 28 | from urwid import connect_signal as on 29 | 30 | from profiling import sortkeys 31 | from profiling.stats import FlatFrozenStatistics 32 | 33 | 34 | __all__ = ['StatisticsTable', 'StatisticsViewer', 'fmt', 35 | 'bind_vim_keys', 'bind_game_keys'] 36 | 37 | 38 | NESTED = 0 39 | FLAT = 1 40 | 41 | 42 | def get_func(f): 43 | if isinstance(f, staticmethod): 44 | return f.__func__ 45 | return f 46 | 47 | 48 | class Formatter(object): 49 | 50 | def _markup(get_string, get_attr=None): 51 | get_string = get_func(get_string) 52 | get_attr = get_func(get_attr) 53 | @staticmethod 54 | def markup(*args, **kwargs): 55 | string = get_string(*args, **kwargs) 56 | if get_attr is None: 57 | return string 58 | attr = get_attr(*args, **kwargs) 59 | return (attr, string) 60 | return markup 61 | 62 | _numeric = {'align': 'right', 'wrap': 'clip'} 63 | 64 | def _make_text(get_markup, **text_kwargs): 65 | get_markup = get_func(get_markup) 66 | @staticmethod 67 | def make_text(*args, **kwargs): 68 | markup = get_markup(*args, **kwargs) 69 | return urwid.Text(markup, **text_kwargs) 70 | return make_text 71 | 72 | # percent 73 | 74 | @staticmethod 75 | def format_percent(ratio, denom=1, unit=False): 76 | # width: 4~5 (with unit) 77 | # examples: 78 | # 0.01: 1.00% 79 | # 0.1: 10.0% 80 | # 1: 100% 81 | try: 82 | ratio /= float(denom) 83 | except ZeroDivisionError: 84 | ratio = 0 85 | if round(ratio, 2) >= 1: 86 | precision = 0 87 | elif round(ratio, 2) >= 0.1: 88 | precision = 1 89 | else: 90 | precision = 2 91 | string = ('{:.' + str(precision) + 'f}').format(ratio * 100) 92 | if unit: 93 | return string + '%' 94 | else: 95 | return string 96 | 97 | @staticmethod 98 | def attr_ratio(ratio, denom=1, unit=False): 99 | try: 100 | ratio /= float(denom) 101 | except ZeroDivisionError: 102 | ratio = 0 103 | if ratio > 0.9: 104 | return 'danger' 105 | elif ratio > 0.7: 106 | return 'caution' 107 | elif ratio > 0.3: 108 | return 'warning' 109 | elif ratio > 0.1: 110 | return 'notice' 111 | elif ratio <= 0: 112 | return 'zero' 113 | 114 | markup_percent = _markup(format_percent, attr_ratio) 115 | make_percent_text = _make_text(markup_percent, **_numeric) 116 | 117 | # int 118 | 119 | @staticmethod 120 | def format_int(num, units='kMGTPEZY'): 121 | # width: 1~6 122 | # examples: 123 | # 0: 0 124 | # 1: 1 125 | # 10: 10 126 | # 100: 100 127 | # 1000: 1.0K 128 | # 10000: 10.0K 129 | # 100000: 100.0K 130 | # 1000000: 1.0M 131 | # -10: -11 132 | unit = None 133 | unit_iter = iter(units) 134 | while abs(round(num, 1)) >= 1e3: 135 | num /= 1e3 136 | try: 137 | unit = next(unit_iter) 138 | except StopIteration: 139 | # overflow or underflow. 140 | return 'o/f' if num > 0 else 'u/f' 141 | if unit is None: 142 | return '{:.0f}'.format(num) 143 | else: 144 | return '{:.1f}{}'.format(num, unit) 145 | 146 | @staticmethod 147 | def attr_int(num): 148 | return None if num else 'zero' 149 | 150 | markup_int = _markup(format_int, attr_int) 151 | make_int_text = _make_text(markup_int, **_numeric) 152 | 153 | # int or n/a 154 | 155 | @staticmethod 156 | def format_int_or_na(num): 157 | # width: 1~6 158 | # examples: 159 | # 0: n/a 160 | # 1: 1 161 | # 10: 10 162 | # 100: 100 163 | # 1000: 1.0K 164 | # 10000: 10.0K 165 | # 100000: 100.0K 166 | # 1000000: 1.0M 167 | # -10: -11 168 | if num == 0: 169 | return 'n/a' 170 | else: 171 | return Formatter.format_int(num) 172 | 173 | markup_int_or_na = _markup(format_int_or_na, attr_int) 174 | make_int_or_na_text = _make_text(markup_int_or_na, **_numeric) 175 | 176 | # time 177 | 178 | @staticmethod 179 | def format_time(sec): 180 | # width: 1~6 (most cases) 181 | # examples: 182 | # 0: 0 183 | # 0.000001: 1us 184 | # 0.000123: 123us 185 | # 0.012345: 12ms 186 | # 0.123456: 123ms 187 | # 1.234567: 1.2sec 188 | # 12.34567: 12.3sec 189 | # 123.4567: 2min3s 190 | # 6120: 102min 191 | if sec == 0: 192 | return '0' 193 | elif sec < 1e-3: 194 | # 1us ~ 999us 195 | return '{:.0f}us'.format(sec * 1e6) 196 | elif sec < 1: 197 | # 1ms ~ 999ms 198 | return '{:.0f}ms'.format(sec * 1e3) 199 | elif sec < 60: 200 | # 1.0sec ~ 59.9sec 201 | return '{:.1f}sec'.format(sec) 202 | elif sec < 600: 203 | # 1min0s ~ 9min59s 204 | return '{:.0f}m{:.0f}s'.format(sec // 60, sec % 60) 205 | else: 206 | return '{:.0f}m'.format(sec // 60) 207 | 208 | @staticmethod 209 | def attr_time(sec): 210 | if sec == 0: 211 | return 'zero' 212 | elif sec < 1e-3: 213 | return 'usec' 214 | elif sec < 1: 215 | return 'msec' 216 | elif sec < 60: 217 | return 'sec' 218 | else: 219 | return 'min' 220 | 221 | markup_time = _markup(format_time, attr_time) 222 | make_time_text = _make_text(markup_time, **_numeric) 223 | 224 | # stats 225 | 226 | @staticmethod 227 | def markup_stats(stats): 228 | if stats.name: 229 | loc = ('({0}:{1})' 230 | ''.format(stats.module or stats.filename, stats.lineno)) 231 | return [('name', stats.name), ' ', ('loc', loc)] 232 | else: 233 | return ('loc', stats.module or stats.filename) 234 | 235 | make_stat_text = _make_text(markup_stats, wrap='clip') 236 | 237 | del _markup 238 | del _make_text 239 | 240 | 241 | fmt = Formatter 242 | 243 | 244 | class StatisticsWidget(urwid.TreeWidget): 245 | 246 | signals = ['expanded', 'collapsed'] 247 | icon_chars = ('+', '-', ' ') # collapsed, expanded, leaf 248 | 249 | def __init__(self, node): 250 | super(StatisticsWidget, self).__init__(node) 251 | self._w = urwid.AttrWrap(self._w, None, StatisticsViewer.focus_map) 252 | 253 | def selectable(self): 254 | return True 255 | 256 | @property 257 | def expanded(self): 258 | return self._expanded 259 | 260 | @expanded.setter 261 | def expanded(self, expanded): 262 | in_init = not hasattr(self, 'expanded') 263 | self._expanded = expanded 264 | if in_init: 265 | return 266 | if expanded: 267 | urwid.emit_signal(self, 'expanded') 268 | else: 269 | urwid.emit_signal(self, 'collapsed') 270 | 271 | def get_mark(self): 272 | """Gets an expanded, collapsed, or leaf icon.""" 273 | if self.is_leaf: 274 | char = self.icon_chars[2] 275 | else: 276 | char = self.icon_chars[int(self.expanded)] 277 | return urwid.SelectableIcon(('mark', char), 0) 278 | 279 | def load_inner_widget(self): 280 | node = self.get_node() 281 | return node.table.make_row(node) 282 | 283 | def get_indented_widget(self): 284 | icon = self.get_mark() 285 | widget = self.get_inner_widget() 286 | node = self.get_node() 287 | widget = urwid.Columns([('fixed', 1, icon), widget], 1) 288 | indent = (node.get_depth() - 1) 289 | widget = urwid.Padding(widget, left=indent) 290 | return widget 291 | 292 | def update_mark(self): 293 | widget = self._w.base_widget 294 | try: 295 | widget.widget_list[0] = self.get_mark() 296 | except (TypeError, AttributeError): 297 | return 298 | 299 | def update_expanded_icon(self): 300 | self.update_mark() 301 | 302 | def expand(self): 303 | self.expanded = True 304 | self.update_mark() 305 | 306 | def collapse(self): 307 | self.expanded = False 308 | self.update_mark() 309 | 310 | def keypress(self, size, key): 311 | command = self._command_map[key] 312 | if command == urwid.ACTIVATE: 313 | key = '-' if self.expanded else '+' 314 | elif command == urwid.CURSOR_RIGHT: 315 | key = '+' 316 | elif self.expanded and command == urwid.CURSOR_LEFT: 317 | key = '-' 318 | return super(StatisticsWidget, self).keypress(size, key) 319 | 320 | 321 | class EmptyWidget(urwid.Widget): 322 | """A widget which doesn't render anything.""" 323 | 324 | def __init__(self, rows=0): 325 | super(EmptyWidget, self).__init__() 326 | self._rows = rows 327 | 328 | def rows(self, size, focus=False): 329 | return self._rows 330 | 331 | def render(self, size, focus=False): 332 | return urwid.SolidCanvas(' ', size[0], self.rows(size, focus)) 333 | 334 | 335 | class RootStatisticsWidget(StatisticsWidget): 336 | 337 | def load_inner_widget(self): 338 | return EmptyWidget() 339 | 340 | def get_indented_widget(self): 341 | return self.get_inner_widget() 342 | 343 | def get_mark(self): 344 | raise TypeError('Statistics widget has no mark') 345 | 346 | def update(self): 347 | pass 348 | 349 | def unexpand(self): 350 | pass 351 | 352 | 353 | class StatisticsNodeBase(urwid.TreeNode): 354 | 355 | def __init__(self, stats=None, parent=None, key=None, depth=None, 356 | table=None): 357 | super(StatisticsNodeBase, self).__init__(stats, parent, key, depth) 358 | self.table = table 359 | 360 | def get_focus(self): 361 | widget, focus = super(StatisticsNodeBase, self).get_focus() 362 | if self.table is not None: 363 | self.table.walker.set_focus(self) 364 | return widget, focus 365 | 366 | def get_widget(self, reload=False): 367 | if self._widget is None or reload: 368 | self._widget = self.load_widget() 369 | self.setup_widget(self._widget) 370 | return self._widget 371 | 372 | def load_widget(self): 373 | return self._widget_class(self) 374 | 375 | def setup_widget(self, widget): 376 | if self.table is None: 377 | return 378 | stats = self.get_value() 379 | if hash(stats) in self.table._expanded_stat_hashes: 380 | widget.expand() 381 | 382 | 383 | class NullStatisticsWidget(StatisticsWidget): 384 | 385 | def __init__(self, node): 386 | urwid.TreeWidget.__init__(self, node) 387 | 388 | def get_inner_widget(self): 389 | widget = urwid.Text(('weak', '- Not Available -'), align='center') 390 | widget = urwid.Filler(widget) 391 | widget = urwid.BoxAdapter(widget, 3) 392 | return widget 393 | 394 | 395 | class NullStatisticsNode(StatisticsNodeBase): 396 | 397 | _widget_class = NullStatisticsWidget 398 | 399 | 400 | class LeafStatisticsNode(StatisticsNodeBase): 401 | 402 | _widget_class = StatisticsWidget 403 | 404 | 405 | class StatisticsNode(StatisticsNodeBase, urwid.ParentNode): 406 | 407 | def deep_usage(self): 408 | stats = self.get_value() 409 | table = self.get_root() 410 | try: 411 | return stats.deep_time / table.cpu_time 412 | except AttributeError: 413 | return 0.0 414 | 415 | def load_widget(self): 416 | if self.is_root(): 417 | widget_class = RootStatisticsWidget 418 | else: 419 | widget_class = StatisticsWidget 420 | widget = widget_class(self) 421 | widget.collapse() 422 | return widget 423 | 424 | def setup_widget(self, widget): 425 | super(StatisticsNode, self).setup_widget(widget) 426 | if self.get_depth() == 0: 427 | # Just expand the root node. 428 | widget.expand() 429 | return 430 | table = self.table 431 | if table is None: 432 | return 433 | on(widget, 'expanded', table._widget_expanded, widget) 434 | on(widget, 'collapsed', table._widget_collapsed, widget) 435 | 436 | def load_child_keys(self): 437 | stats = self.get_value() 438 | if stats is None: 439 | return () 440 | return stats.sorted(self.table.order) 441 | 442 | def load_child_node(self, stats): 443 | depth = self.get_depth() + 1 444 | node_class = StatisticsNode if len(stats) else LeafStatisticsNode 445 | return node_class(stats, self, stats, depth, self.table) 446 | 447 | 448 | class StatisticsListBox(urwid.TreeListBox): 449 | 450 | signals = ['focus_changed'] 451 | 452 | def change_focus(self, *args, **kwargs): 453 | super(StatisticsListBox, self).change_focus(*args, **kwargs) 454 | focus = self.get_focus() 455 | urwid.emit_signal(self, 'focus_changed', focus) 456 | 457 | 458 | class StatisticsWalker(urwid.TreeWalker): 459 | 460 | signals = ['focus_changed'] 461 | 462 | def set_focus(self, focus): 463 | super(StatisticsWalker, self).set_focus(focus) 464 | urwid.emit_signal(self, 'focus_changed', focus) 465 | 466 | 467 | class StatisticsTable(urwid.WidgetWrap): 468 | 469 | #: The column declarations. Define it with a list of (name, align, width, 470 | #: order) tuples. 471 | columns = [('FUNCTION', 'left', ('weight', 1), sortkeys.by_function)] 472 | 473 | #: The initial order. 474 | order = sortkeys.by_function 475 | 476 | #: The children statistics layout. One of `NESTED` or `FLAT`. 477 | layout = NESTED 478 | 479 | title = None 480 | stats = None 481 | time = None 482 | 483 | def __init__(self, viewer): 484 | self._expanded_stat_hashes = set() 485 | self.walker = StatisticsWalker(NullStatisticsNode()) 486 | on(self.walker, 'focus_changed', self._walker_focus_changed) 487 | tbody = StatisticsListBox(self.walker) 488 | thead = urwid.AttrMap(self.make_columns([ 489 | urwid.Text(name, align, 'clip') 490 | for name, align, __, __ in self.columns 491 | ]), None) 492 | header = urwid.Columns([]) 493 | widget = urwid.Frame(tbody, urwid.Pile([header, thead])) 494 | super(StatisticsTable, self).__init__(widget) 495 | self.viewer = viewer 496 | self.update_frame() 497 | 498 | def make_row(self, node): 499 | stats = node.get_value() 500 | return self.make_columns(self.make_cells(node, stats)) 501 | 502 | def make_cells(self, node, stats): 503 | yield fmt.make_stat_text(stats) 504 | 505 | @classmethod 506 | def make_columns(cls, column_widgets): 507 | widget_list = [] 508 | widths = (width for __, __, width, __ in cls.columns) 509 | for width, widget in zip(widths, column_widgets): 510 | widget_list.append(width + (widget,)) 511 | return urwid.Columns(widget_list, 1) 512 | 513 | @property 514 | def tbody(self): 515 | return self._w.body 516 | 517 | @tbody.setter 518 | def tbody(self, body): 519 | self._w.body = body 520 | 521 | @property 522 | def thead(self): 523 | return self._w.header.contents[1][0] 524 | 525 | @thead.setter 526 | def thead(self, thead): 527 | self._w.header.contents[1] = (thead, ('pack', None)) 528 | 529 | @property 530 | def header(self): 531 | return self._w.header.contents[0][0] 532 | 533 | @header.setter 534 | def header(self, header): 535 | self._w.header.contents[0] = (header, ('pack', None)) 536 | 537 | @property 538 | def footer(self): 539 | return self._w.footer 540 | 541 | @footer.setter 542 | def footer(self, footer): 543 | self._w.footer = footer 544 | 545 | def get_focus(self): 546 | return self.tbody.get_focus() 547 | 548 | def set_focus(self, focus): 549 | self.tbody.set_focus(focus) 550 | 551 | def get_path(self): 552 | """Gets the path to the focused statistics. Each step is a hash of 553 | statistics object. 554 | """ 555 | path = deque() 556 | __, node = self.get_focus() 557 | while not node.is_root(): 558 | stats = node.get_value() 559 | path.appendleft(hash(stats)) 560 | node = node.get_parent() 561 | return path 562 | 563 | def find_node(self, node, path): 564 | """Finds a node by the given path from the given node.""" 565 | for hash_value in path: 566 | if isinstance(node, LeafStatisticsNode): 567 | break 568 | for stats in node.get_child_keys(): 569 | if hash(stats) == hash_value: 570 | node = node.get_child_node(stats) 571 | break 572 | else: 573 | break 574 | return node 575 | 576 | def get_stats(self): 577 | return self.stats 578 | 579 | def set_result(self, stats, cpu_time=0.0, wall_time=0.0, 580 | title=None, at=None): 581 | self.stats = stats 582 | self.cpu_time = cpu_time 583 | self.wall_time = wall_time 584 | self.title = title 585 | self.at = at 586 | self.refresh() 587 | 588 | def set_layout(self, layout): 589 | if layout == self.layout: 590 | return # Ignore. 591 | self.layout = layout 592 | self.refresh() 593 | 594 | def sort_stats(self, order=sortkeys.by_deep_time): 595 | assert callable(order) 596 | if order == self.order: 597 | return # Ignore. 598 | self.order = order 599 | self.refresh() 600 | 601 | def shift_order(self, delta): 602 | orders = [order for __, __, __, order in self.columns if order] 603 | x = orders.index(self.order) 604 | order = orders[(x + delta) % len(orders)] 605 | self.sort_stats(order) 606 | 607 | def refresh(self): 608 | stats = self.get_stats() 609 | if stats is None: 610 | return 611 | if self.layout == FLAT: 612 | stats = FlatFrozenStatistics.flatten(stats) 613 | node = StatisticsNode(stats, table=self) 614 | path = self.get_path() 615 | node = self.find_node(node, path) 616 | self.set_focus(node) 617 | 618 | def update_frame(self, focus=None): 619 | # Set thead attr. 620 | if self.viewer.paused: 621 | thead_attr = 'thead.paused' 622 | elif not self.viewer.active: 623 | thead_attr = 'thead.inactive' 624 | else: 625 | thead_attr = 'thead' 626 | self.thead.set_attr_map({None: thead_attr}) 627 | # Set sorting column in thead attr. 628 | for x, (__, __, __, order) in enumerate(self.columns): 629 | attr = thead_attr + '.sorted' if order is self.order else None 630 | widget = self.thead.base_widget.contents[x][0] 631 | text, __ = widget.get_text() 632 | widget.set_text((attr, text)) 633 | if self.viewer.paused: 634 | return 635 | # Update header. 636 | stats = self.get_stats() 637 | if stats is None: 638 | return 639 | title = self.title 640 | time = self.time 641 | if title or time: 642 | if time is not None: 643 | time_string = '{:%H:%M:%S}'.format(time) 644 | if title and time: 645 | markup = [('weak', title), ' ', time_string] 646 | elif title: 647 | markup = title 648 | else: 649 | markup = time_string 650 | meta_info = urwid.Text(markup, align='right') 651 | else: 652 | meta_info = None 653 | fraction_string = '({0}/{1})'.format( 654 | fmt.format_time(self.cpu_time), 655 | fmt.format_time(self.wall_time)) 656 | try: 657 | cpu_usage = self.cpu_time / self.wall_time 658 | except ZeroDivisionError: 659 | cpu_usage = 0.0 660 | cpu_info = urwid.Text([ 661 | 'CPU ', fmt.markup_percent(cpu_usage, unit=True), 662 | ' ', ('weak', fraction_string)]) 663 | # Set header columns. 664 | col_opts = ('weight', 1, False) 665 | self.header.contents = \ 666 | [(w, col_opts) for w in [cpu_info, meta_info] if w] 667 | 668 | def focus_hotspot(self, size): 669 | widget, __ = self.tbody.get_focus() 670 | while widget: 671 | node = widget.get_node() 672 | widget.expand() 673 | widget = widget.first_child() 674 | self.tbody.change_focus(size, node) 675 | 676 | def defocus(self): 677 | __, node = self.get_focus() 678 | self.set_focus(node.get_root()) 679 | 680 | def keypress(self, size, key): 681 | command = self._command_map[key] 682 | if key == ']': 683 | self.shift_order(+1) 684 | return True 685 | elif key == '[': 686 | self.shift_order(-1) 687 | return True 688 | elif key == '>': 689 | self.focus_hotspot(size) 690 | return True 691 | elif key == '\\': 692 | layout = {FLAT: NESTED, NESTED: FLAT}[self.layout] 693 | self.set_layout(layout) 694 | return True 695 | command = self._command_map[key] 696 | if command == 'menu': 697 | # key: ESC. 698 | self.defocus() 699 | return True 700 | elif command == urwid.CURSOR_RIGHT: 701 | if self.layout == FLAT: 702 | return True # Ignore. 703 | widget, node = self.tbody.get_focus() 704 | if widget.expanded: 705 | heavy_widget = widget.first_child() 706 | if heavy_widget is not None: 707 | heavy_node = heavy_widget.get_node() 708 | self.tbody.change_focus(size, heavy_node) 709 | return True 710 | elif command == urwid.CURSOR_LEFT: 711 | if self.layout == FLAT: 712 | return True # Ignore. 713 | widget, node = self.tbody.get_focus() 714 | if not widget.expanded: 715 | parent_node = node.get_parent() 716 | if parent_node is not None and not parent_node.is_root(): 717 | self.tbody.change_focus(size, parent_node) 718 | return True 719 | elif command == urwid.ACTIVATE: 720 | # key: Enter or Space. 721 | if self.viewer.paused: 722 | self.viewer.resume() 723 | else: 724 | self.viewer.pause() 725 | return True 726 | return super(StatisticsTable, self).keypress(size, key) 727 | 728 | # Signal handlers. 729 | 730 | def _walker_focus_changed(self, focus): 731 | self.update_frame(focus) 732 | 733 | def _widget_expanded(self, widget): 734 | stats = widget.get_node().get_value() 735 | self._expanded_stat_hashes.add(hash(stats)) 736 | 737 | def _widget_collapsed(self, widget): 738 | stats = widget.get_node().get_value() 739 | self._expanded_stat_hashes.discard(hash(stats)) 740 | 741 | 742 | class StatisticsViewer(object): 743 | 744 | weak_color = 'light green' 745 | palette = [ 746 | ('weak', weak_color, ''), 747 | ('focus', 'standout', '', 'standout'), 748 | # ui 749 | ('thead', 'dark cyan, standout', '', 'standout'), 750 | ('thead.paused', 'dark red, standout', '', 'standout'), 751 | ('thead.inactive', 'brown, standout', '', 'standout'), 752 | ('mark', 'dark magenta', ''), 753 | # risk 754 | ('danger', 'dark red', '', 'blink'), 755 | ('caution', 'light red', '', 'blink'), 756 | ('warning', 'brown', '', 'blink'), 757 | ('notice', 'dark green', '', 'blink'), 758 | # clock 759 | ('min', 'dark red', ''), 760 | ('sec', 'brown', ''), 761 | ('msec', '', ''), 762 | ('usec', weak_color, ''), 763 | # etc 764 | ('zero', weak_color, ''), 765 | ('name', 'bold', ''), 766 | ('loc', 'dark blue', ''), 767 | ] 768 | # add thead.*.sorted palette entries 769 | for entry in palette[:]: 770 | attr = entry[0] 771 | if attr is not None and attr.startswith('thead'): 772 | fg, bg, mono = entry[1:4] 773 | palette.append((attr + '.sorted', fg + ', underline', 774 | bg, mono + ', underline')) 775 | 776 | focus_map = {None: 'focus'} 777 | focus_map.update((x[0], 'focus') for x in palette) 778 | 779 | #: Whether the viewer is active. 780 | active = False 781 | 782 | #: Whether the viewer is paused. 783 | paused = False 784 | 785 | def unhandled_input(self, key): 786 | if key in ('q', 'Q'): 787 | raise urwid.ExitMainLoop() 788 | 789 | def __init__(self): 790 | self.table = StatisticsTable(self) 791 | self.widget = urwid.Padding(self.table, right=1) 792 | 793 | def loop(self, *args, **kwargs): 794 | kwargs.setdefault('unhandled_input', self.unhandled_input) 795 | loop = urwid.MainLoop(self.widget, self.palette, *args, **kwargs) 796 | return loop 797 | 798 | def set_profiler_class(self, profiler_class): 799 | table_class = profiler_class.table_class 800 | # NOTE: Don't use isinstance() at the below line. 801 | if type(self.table) is table_class: 802 | return 803 | self.table = table_class(self) 804 | self.widget.original_widget = self.table 805 | 806 | def set_result(self, stats, cpu_time=0.0, wall_time=0.0, 807 | title=None, at=None): 808 | self._final_result = (stats, cpu_time, wall_time, title, at) 809 | if not self.paused: 810 | self.update_result() 811 | 812 | def update_result(self): 813 | """Updates the result on the table.""" 814 | try: 815 | if self.paused: 816 | result = self._paused_result 817 | else: 818 | result = self._final_result 819 | except AttributeError: 820 | self.table.update_frame() 821 | return 822 | stats, cpu_time, wall_time, title, at = result 823 | self.table.set_result(stats, cpu_time, wall_time, title, at) 824 | 825 | def activate(self): 826 | self.active = True 827 | self.table.update_frame() 828 | 829 | def inactivate(self): 830 | self.active = False 831 | self.table.update_frame() 832 | 833 | def pause(self): 834 | self.paused = True 835 | try: 836 | self._paused_result = self._final_result 837 | except AttributeError: 838 | pass 839 | self.table.update_frame() 840 | 841 | def resume(self): 842 | self.paused = False 843 | try: 844 | del self._paused_result 845 | except AttributeError: 846 | pass 847 | self.update_result() 848 | 849 | 850 | def bind_vim_keys(urwid=urwid): 851 | urwid.command_map['h'] = urwid.command_map['left'] 852 | urwid.command_map['j'] = urwid.command_map['down'] 853 | urwid.command_map['k'] = urwid.command_map['up'] 854 | urwid.command_map['l'] = urwid.command_map['right'] 855 | 856 | 857 | def bind_game_keys(urwid=urwid): 858 | urwid.command_map['a'] = urwid.command_map['left'] 859 | urwid.command_map['s'] = urwid.command_map['down'] 860 | urwid.command_map['w'] = urwid.command_map['up'] 861 | urwid.command_map['d'] = urwid.command_map['right'] 862 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | valuedispatch>=0.0.1 2 | click>=4.1 3 | click-default-group>=1.2 4 | six>=1.8.0 5 | urwid>=1.2.1 6 | -------------------------------------------------------------------------------- /screenshots/sampling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/what-studio/profiling/824ae554b35909ecbb8aead8853429573e70aeee/screenshots/sampling.png -------------------------------------------------------------------------------- /screenshots/tracing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/what-studio/profiling/824ae554b35909ecbb8aead8853429573e70aeee/screenshots/tracing.png -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E301, E306, E731 3 | import-order-style = google 4 | application-import-names = profiling, _utils 5 | 6 | [tool:pytest] 7 | python_files = test/test_*.py 8 | norecursedirs = profiling .* *.egg* *env* 9 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | profiling 4 | ~~~~~~~~~ 5 | 6 | .. image:: https://img.shields.io/travis/what-studio/profiling.svg 7 | :target: https://travis-ci.org/what-studio/profiling 8 | 9 | .. image:: https://img.shields.io/coveralls/what-studio/profiling.svg 10 | :target: https://coveralls.io/r/what-studio/profiling 11 | 12 | The profiling package is an interactive continuous Python profiler. It is 13 | inspired from `Unity 3D `_ profiler. This package 14 | provides these features: 15 | 16 | - Profiling statistics keep the frame stack. 17 | - An interactive TUI profiling statistics viewer. 18 | - Provides both of statistical and deterministic profiling. 19 | - Utilities for remote profiling. 20 | - Thread or greenlet aware CPU timer. 21 | - Supports Python 2.7, 3.3, 3.4 and 3.5. 22 | - Currently supports only Linux. 23 | 24 | Links 25 | ''''' 26 | 27 | GitHub: 28 | https://github.com/what-studio/profiling 29 | Demo: 30 | https://asciinema.org/a/25394 31 | 32 | """ 33 | from __future__ import with_statement 34 | 35 | import os 36 | import sys 37 | from textwrap import dedent 38 | 39 | from setuptools import find_packages, setup 40 | from setuptools.command.install import install 41 | from setuptools.command.test import test 42 | from setuptools.extension import Extension 43 | 44 | try: 45 | import __pypy__ 46 | except ImportError: 47 | __pypy__ = False 48 | 49 | 50 | # include __about__.py. 51 | __dir__ = os.path.dirname(__file__) 52 | about = {} 53 | with open(os.path.join(__dir__, 'profiling', '__about__.py')) as f: 54 | exec(f.read(), about) 55 | 56 | 57 | # these files require specific python version or later. they will be replaced 58 | # with a placeholder which raises a runtime error on installation. 59 | PYTHON_VERSION_REQUIREMENTS = { 60 | 'profiling/remote/asyncio.py': (3, 4), 61 | } 62 | INCOMPATIBLE_PYTHON_VERSION_PLACEHOLDER = dedent(''' 63 | # -*- coding: utf-8 -*- 64 | raise RuntimeError('Python {version} or later required.') 65 | ''').strip() 66 | 67 | 68 | def requirements(filename): 69 | """Reads requirements from a file.""" 70 | with open(filename) as f: 71 | return [x.strip() for x in f.readlines() if x.strip()] 72 | 73 | 74 | # use pytest instead. 75 | def run_tests(self): 76 | raise SystemExit(__import__('pytest').main(['-v'])) 77 | 78 | 79 | # replace files which are incompatible with the current python version. 80 | def replace_incompatible_files(): 81 | for filename, version_info in PYTHON_VERSION_REQUIREMENTS.items(): 82 | if sys.version_info >= version_info: 83 | continue 84 | version = '.'.join(str(v) for v in version_info) 85 | code = INCOMPATIBLE_PYTHON_VERSION_PLACEHOLDER.format(version=version) 86 | with open(filename, 'w') as f: 87 | f.write(code) 88 | 89 | 90 | test.run_tests = run_tests 91 | run_install = install.run 92 | install.run = lambda x: (replace_incompatible_files(), run_install(x)) 93 | 94 | 95 | # build profiling.speedup on cpython. 96 | if __pypy__: 97 | ext_modules = [] 98 | else: 99 | ext_modules = [Extension('profiling.speedup', ['profiling/speedup.c'])] 100 | 101 | 102 | setup( 103 | name='profiling', 104 | version=about['__version__'], 105 | license=about['__license__'], 106 | author=about['__author__'], 107 | maintainer=about['__maintainer__'], 108 | maintainer_email=about['__maintainer_email__'], 109 | url=about['__url__'], 110 | description=about['__description__'], 111 | long_description=dedent(__doc__), 112 | platforms='linux', 113 | packages=find_packages(), 114 | ext_modules=ext_modules, 115 | entry_points={ 116 | 'console_scripts': ['profiling = profiling.__main__:cli'] 117 | }, 118 | classifiers=[ 119 | 'Development Status :: 4 - Beta', 120 | 'Environment :: Console', 121 | 'Intended Audience :: Developers', 122 | 'License :: OSI Approved :: BSD License', 123 | 'Operating System :: POSIX :: Linux', 124 | 'Programming Language :: Python', 125 | 'Programming Language :: Python :: 2', 126 | 'Programming Language :: Python :: 2.7', 127 | 'Programming Language :: Python :: 3', 128 | 'Programming Language :: Python :: 3.3', 129 | 'Programming Language :: Python :: 3.4', 130 | 'Programming Language :: Python :: 3.5', 131 | 'Programming Language :: Python :: 3.6', 132 | 'Programming Language :: Python :: Implementation :: CPython', 133 | 'Programming Language :: Python :: Implementation :: PyPy', 134 | 'Topic :: Software Development :: Debuggers', 135 | ], 136 | install_requires=requirements('requirements.txt'), 137 | tests_require=requirements('test/requirements.txt'), 138 | test_suite='...', 139 | ) 140 | -------------------------------------------------------------------------------- /test/_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from textwrap import dedent 3 | import time 4 | 5 | 6 | __all__ = ['find_multiple_stats', 'find_stats', 'factorial', 'spin', 7 | 'mock_code_names'] 8 | 9 | 10 | def find_multiple_stats(stats, name, _found=None, _on_found=None): 11 | if _found is None: 12 | _found = [] 13 | for child_stats in stats: 14 | if child_stats.name == name: 15 | _found.append(child_stats) 16 | if callable(_on_found): 17 | _on_found(_found) 18 | find_multiple_stats(child_stats, name, _found) 19 | return _found 20 | 21 | 22 | def find_stats(stats, name): 23 | def _on_found(found): 24 | raise StopIteration 25 | return find_multiple_stats(stats, name)[0] 26 | 27 | 28 | def factorial(n): 29 | f = 1 30 | while n: 31 | f *= n 32 | n -= 1 33 | return f 34 | 35 | 36 | def spin(seconds): 37 | t = time.time() 38 | x = 0 39 | while time.time() - t < seconds: 40 | x += 1 41 | return x 42 | 43 | 44 | foo = None # placeheolder 45 | mock_code_names = ['foo', 'bar', 'baz'] 46 | source = '' 47 | for name, next_name in zip(mock_code_names[:-1], mock_code_names[1:]): 48 | source += dedent(''' 49 | def {0}(): 50 | return {1}() 51 | ''').format(name, next_name) 52 | source += ''' 53 | def {0}(): 54 | return __import__('sys')._getframe() 55 | '''.format(mock_code_names[-1]) 56 | exec(source) 57 | __all__.extend(mock_code_names) 58 | -------------------------------------------------------------------------------- /test/fit_requirements.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import print_function 3 | 4 | import re 5 | import sys 6 | 7 | import pkg_resources 8 | 9 | 10 | #: Matches with comment string starts with `#`. 11 | comment_re = re.compile(r'#\s*(.+)\s*$') 12 | 13 | 14 | # Detect the Python and PyPy versions. 15 | PYTHON_VERSION = '{0}.{1}.{2}'.format(*sys.version_info) 16 | try: 17 | PYPY_VERSION = '{0}.{1}.{2}'.format(*sys.pypy_version_info) 18 | except AttributeError: 19 | PYPY_VERSION = None 20 | 21 | 22 | def installs(sysreq_string, 23 | python_version=PYTHON_VERSION, pypy_version=PYPY_VERSION): 24 | for sysreq in sysreq_string.split(): 25 | if sysreq == '!pypy' and pypy_version is not None: 26 | return False 27 | if sysreq.startswith('python'): 28 | if python_version not in pkg_resources.Requirement.parse(sysreq): 29 | return False 30 | elif sysreq.startswith('pypy'): 31 | if pypy_version is None: 32 | return False 33 | elif pypy_version not in pkg_resources.Requirement.parse(sysreq): 34 | return False 35 | return True 36 | 37 | 38 | def fit_requirements(requirements, 39 | python_version=PYTHON_VERSION, pypy_version=PYPY_VERSION): 40 | """Yields requirement lines only compatible with the current system. 41 | 42 | It parses comments of the given requirement lines. A comment string can 43 | include python version requirement and `!pypy` flag to skip to install on 44 | incompatible system: 45 | 46 | .. sourcecode:: 47 | 48 | # requirements.txt 49 | pytest>=2.6.1 50 | eventlet>=0.15 # python>=2.6,<3 51 | gevent>=1 # python>=2.5,<3 !pypy 52 | gevent==1.1rc1 # pypy<2.6.1 53 | greenlet>=0.4.4 # python>=2.4 54 | yappi>=0.92 # python>=2.6,!=3.0 !pypy 55 | 56 | .. sourcecode:: console 57 | 58 | $ pypy3.4 fit_requirements.py requirements.txt 59 | pytest>=2.6.1 60 | greenlet>=0.4.4 61 | 62 | """ 63 | for line in requirements: 64 | match = comment_re.search(line) 65 | if match is None: 66 | yield line 67 | continue 68 | comment = match.group(1) 69 | if installs(comment, python_version, pypy_version): 70 | yield line[:match.start()].rstrip() + '\n' 71 | 72 | 73 | if __name__ == '__main__': 74 | filename = sys.argv[1] 75 | with open(filename) as f: 76 | requirements = f.readlines() 77 | requirements = fit_requirements(requirements) 78 | print(''.join(requirements)) 79 | -------------------------------------------------------------------------------- /test/requirements.txt: -------------------------------------------------------------------------------- 1 | pytest<4.0.0 2 | pytest-benchmark>=3.0.0 3 | pytest-rerunfailures>=0.05 4 | eventlet>=0.15 # python>=2.6,<3 5 | gevent>=1 # python>=2.5,<3 !pypy 6 | gevent>=1.1a1 # python>=3 !pypy 7 | gevent==1.1rc1 # pypy<2.6.1 8 | gevent>=1.1rc2 # pypy>=2.6.1 9 | greenlet>=0.4.4 # python>=2.4 10 | yappi>=0.92 # python>=2.6,!=3.0 !pypy 11 | -------------------------------------------------------------------------------- /test/test_cli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import io 3 | import textwrap 4 | 5 | import click 6 | from click.testing import CliRunner 7 | from six.moves import builtins 8 | from valuedispatch import valuedispatch 9 | 10 | from profiling.__about__ import __version__ 11 | from profiling.__main__ import cli, Module, profiler_options, ProfilingCLI 12 | from profiling.sampling import SamplingProfiler 13 | from profiling.sampling.samplers import TracingSampler 14 | from profiling.tracing import TracingProfiler 15 | 16 | 17 | class MockFileIO(io.StringIO): 18 | 19 | def close(self): 20 | self.seek(0) 21 | 22 | 23 | def mock_file(indented_content): 24 | return MockFileIO(textwrap.dedent(indented_content)) 25 | 26 | 27 | cli_runner = CliRunner() 28 | 29 | 30 | def test_module_param_type(): 31 | t = Module() 32 | # timeit 33 | filename, code, globals_ = t.convert('timeit', None, None) 34 | assert filename.endswith('timeit.py') 35 | assert code.co_filename.endswith('timeit.py') 36 | assert globals_['__name__'] == '__main__' 37 | assert globals_['__file__'].endswith('timeit.py') 38 | assert globals_['__package__'] == '' 39 | # profiling.__main__ 40 | filename, code, globals_ = t.convert('profiling', None, None) 41 | assert filename.endswith('profiling/__main__.py') 42 | assert code.co_filename.endswith('profiling/__main__.py') 43 | assert globals_['__name__'] == '__main__' 44 | assert globals_['__file__'].endswith('profiling/__main__.py') 45 | assert globals_['__package__'] == 'profiling' 46 | 47 | 48 | def test_customized_cli(): 49 | cli = ProfilingCLI(default='bar') 50 | @cli.command(aliases=['fooo', 'foooo']) 51 | def foo(): 52 | pass 53 | @cli.command() 54 | @click.argument('l', default='answer') 55 | @click.option('-n', type=int, default=0) 56 | def bar(l, n=0): 57 | click.echo('%s: %d' % (l, n)) 58 | assert len(cli.commands) == 2 59 | ctx = click.Context(cli) 60 | assert cli.get_command(ctx, 'foo').name == 'foo' 61 | assert cli.get_command(ctx, 'fooo').name == 'foo' 62 | assert cli.get_command(ctx, 'foooo').name == 'foo' 63 | assert cli.get_command(ctx, 'bar').name == 'bar' 64 | assert cli.get_command(ctx, 'hello.txt').name == 'bar' 65 | assert 'Usage:' in cli_runner.invoke(cli, []).output 66 | assert cli_runner.invoke(cli, ['zero']).output == 'zero: 0\n' 67 | assert cli_runner.invoke(cli, ['one', '-n', '1']).output == 'one: 1\n' 68 | assert cli_runner.invoke(cli, ['-n', '42']).output == 'answer: 42\n' 69 | assert 'no such option' in cli_runner.invoke(cli, ['-x']).output 70 | 71 | 72 | def test_profiling_command_usage(): 73 | for cmd in ['profile', 'live-profile', 'remote-profile']: 74 | r = cli_runner.invoke(cli, [cmd, '--help']) 75 | assert 'SCRIPT [--] [ARGV]...' in r.output 76 | 77 | 78 | def test_version(): 79 | r = cli_runner.invoke(cli, ['--version']) 80 | assert r.output.strip() == 'profiling, version %s' % __version__ 81 | 82 | 83 | def test_config(monkeypatch): 84 | @click.command() 85 | @profiler_options 86 | def f(profiler_factory, **kwargs): 87 | profiler = profiler_factory() 88 | return profiler, kwargs 89 | # no config. 90 | def io_error(*args, **kwargs): 91 | raise IOError 92 | monkeypatch.setattr(builtins, 'open', io_error) 93 | profiler, kwargs = f([], standalone_mode=False) 94 | assert isinstance(profiler, TracingProfiler) 95 | # config to use SamplingProfiler. 96 | monkeypatch.setattr(builtins, 'open', lambda *a, **k: mock_file(u''' 97 | [profiling] 98 | profiler = sampling 99 | sampler = tracing 100 | ''')) 101 | profiler, kwargs = f([], standalone_mode=False) 102 | assert isinstance(profiler, SamplingProfiler) 103 | assert isinstance(profiler.sampler, TracingSampler) 104 | # set both of setup.cfg and .profiling. 105 | @valuedispatch 106 | def mock_open(path, *args, **kwargs): 107 | raise IOError 108 | @mock_open.register('setup.cfg') 109 | def open_setup_cfg(*_, **__): 110 | return mock_file(u''' 111 | [profiling] 112 | profiler = sampling 113 | pickle-protocol = 3 114 | ''') 115 | @mock_open.register('.profiling') 116 | def open_profiling(*_, **__): 117 | return mock_file(u''' 118 | [profiling] 119 | pickle-protocol = 0 120 | ''') 121 | monkeypatch.setattr(builtins, 'open', mock_open) 122 | profiler, kwargs = f([], standalone_mode=False) 123 | assert isinstance(profiler, SamplingProfiler) # from setup.cfg 124 | assert kwargs['pickle_protocol'] == 0 # from .profiling 125 | -------------------------------------------------------------------------------- /test/test_profiler.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | try: 3 | import cPickle as pickle 4 | except ImportError: 5 | import pickle 6 | from os import path 7 | import tempfile 8 | 9 | import pytest 10 | 11 | from _utils import foo, spin 12 | from profiling.profiler import Profiler, ProfilerWrapper 13 | 14 | 15 | class NullProfiler(Profiler): 16 | 17 | def run(self): 18 | yield 19 | 20 | 21 | class NullProfilerWrapper(ProfilerWrapper): 22 | 23 | def run(self): 24 | with self.profiler: 25 | yield 26 | 27 | 28 | @pytest.fixture 29 | def profiler(): 30 | return NullProfiler() 31 | 32 | 33 | def test_exclude_code(profiler): 34 | foo_code = foo().f_code 35 | with profiler: 36 | assert foo_code not in profiler.stats 37 | profiler.stats.ensure_child(foo_code) 38 | assert foo_code in profiler.stats 39 | profiler.exclude_code(foo_code) 40 | assert foo_code not in profiler.stats 41 | profiler.exclude_code(foo_code) 42 | assert foo_code not in profiler.stats 43 | 44 | 45 | def test_result(profiler): 46 | __, cpu_time, wall_time = profiler.result() 47 | assert cpu_time == wall_time == 0.0 48 | with profiler: 49 | spin(0.1) 50 | __, cpu_time, wall_time = profiler.result() 51 | assert cpu_time > 0.0 52 | assert wall_time >= 0.1 53 | 54 | 55 | def test_dump(profiler): 56 | temp_dir = tempfile.mkdtemp() 57 | temp_file = path.join(temp_dir, "file.prf") 58 | 59 | profiler.dump(temp_file) 60 | 61 | assert path.getsize(temp_file) > 0 62 | 63 | with open(temp_file, 'rb') as f: 64 | profiler_class, (stats, cpu_time, wall_time) = pickle.load(f) 65 | 66 | assert profiler.__class__ == profiler_class 67 | assert cpu_time == wall_time == 0.0 68 | 69 | 70 | def test_wrapper(profiler): 71 | wrapper = NullProfilerWrapper(profiler) 72 | assert isinstance(wrapper, Profiler) 73 | assert wrapper.table_class is profiler.table_class 74 | assert wrapper.stats is profiler.stats 75 | __, cpu_time, wall_time = wrapper.result() 76 | assert cpu_time == wall_time == 0.0 77 | with wrapper: 78 | assert wrapper.is_running() 79 | assert profiler.is_running() 80 | assert not wrapper.is_running() 81 | assert not profiler.is_running() 82 | -------------------------------------------------------------------------------- /test/test_sampling.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division 3 | 4 | import os 5 | import signal 6 | import sys 7 | 8 | import pytest 9 | 10 | from _utils import find_stats, spin 11 | from profiling.sampling import SamplingProfiler 12 | from profiling.sampling.samplers import ItimerSampler, TracingSampler 13 | 14 | 15 | def spin_100ms(): 16 | spin(0.1) 17 | 18 | 19 | def spin_500ms(): 20 | spin(0.5) 21 | 22 | 23 | def _test_sampling_profiler(sampler): 24 | profiler = SamplingProfiler(base_frame=sys._getframe(), sampler=sampler) 25 | with profiler: 26 | spin_100ms() 27 | spin_500ms() 28 | stat1 = find_stats(profiler.stats, 'spin_100ms') 29 | stat2 = find_stats(profiler.stats, 'spin_500ms') 30 | ratio = stat1.deep_hits / stat2.deep_hits 31 | # 1:5 expected, but tolerate (0.8~1.2):5 32 | assert 0.8 <= ratio * 5 <= 1.2 33 | 34 | 35 | @pytest.mark.flaky(reruns=10) 36 | def test_itimer_sampler(): 37 | assert signal.getsignal(signal.SIGPROF) == signal.SIG_DFL 38 | try: 39 | _test_sampling_profiler(ItimerSampler(0.0001)) 40 | # no crash caused by SIGPROF. 41 | assert signal.getsignal(signal.SIGPROF) == signal.SIG_IGN 42 | for x in range(10): 43 | os.kill(os.getpid(), signal.SIGPROF) 44 | # respect custom handler. 45 | handler = lambda *x: x 46 | signal.signal(signal.SIGPROF, handler) 47 | _test_sampling_profiler(ItimerSampler(0.0001)) 48 | assert signal.getsignal(signal.SIGPROF) == handler 49 | finally: 50 | signal.signal(signal.SIGPROF, signal.SIG_DFL) 51 | 52 | 53 | @pytest.mark.flaky(reruns=10) 54 | def test_tracing_sampler(): 55 | pytest.importorskip('yappi') 56 | _test_sampling_profiler(TracingSampler(0.0001)) 57 | 58 | 59 | @pytest.mark.flaky(reruns=10) 60 | def test_tracing_sampler_does_not_sample_too_often(): 61 | pytest.importorskip('yappi') 62 | # pytest-cov cannot detect a callback function registered by 63 | # :func:`sys.setprofile`. 64 | class fake_profiler(object): 65 | samples = [] 66 | @classmethod 67 | def sample(cls, frame): 68 | cls.samples.append(frame) 69 | @classmethod 70 | def count_and_clear_samples(cls): 71 | count = len(cls.samples) 72 | del cls.samples[:] 73 | return count 74 | sampler = TracingSampler(0.1) 75 | sampler._profile(fake_profiler, None, None, None) 76 | assert fake_profiler.count_and_clear_samples() == 1 77 | sampler._profile(fake_profiler, None, None, None) 78 | assert fake_profiler.count_and_clear_samples() == 0 79 | spin(0.5) 80 | sampler._profile(fake_profiler, None, None, None) 81 | assert fake_profiler.count_and_clear_samples() == 1 82 | 83 | 84 | def test_not_sampler(): 85 | with pytest.raises(TypeError): 86 | SamplingProfiler(sampler=123) 87 | 88 | 89 | def test_sample_1_depth(): 90 | frame = sys._getframe() 91 | while frame.f_back is not None: 92 | frame = frame.f_back 93 | assert frame.f_back is None 94 | profiler = SamplingProfiler() 95 | profiler.sample(frame) 96 | -------------------------------------------------------------------------------- /test/test_stats.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | try: 3 | import cPickle as pickle 4 | except ImportError: 5 | import pickle 6 | import sys 7 | from textwrap import dedent 8 | from types import CodeType 9 | 10 | import pytest 11 | from six import exec_, PY3 12 | 13 | from _utils import spin 14 | import profiling 15 | from profiling.sortkeys import \ 16 | by_deep_time_per_call, by_name, by_own_hits, by_own_time_per_call 17 | from profiling.stats import \ 18 | FlatFrozenStatistics, FrozenStatistics, RecordingStatistics, \ 19 | spread_stats, Statistics 20 | from profiling.tracing import TracingProfiler 21 | 22 | 23 | def mock_code(name): 24 | """Makes a fake code object by name for built-in functions.""" 25 | left_args = (0, 0, 0, 0, b'', (), (), (), '') 26 | if PY3: 27 | left_args = (0,) + left_args 28 | args = left_args + (name, 0, b'') 29 | return CodeType(*args) 30 | 31 | 32 | def test_stats(): 33 | stats = Statistics(name='foo', filename='bar', lineno=42) 34 | assert stats.regular_name == 'foo' 35 | stats.module = 'baz' 36 | assert stats.regular_name == 'baz:foo' 37 | assert stats.deep_time_per_call == 0 38 | assert stats.own_time_per_call == 0 39 | stats.deep_time = 128 40 | stats.own_hits = 4 41 | assert stats.deep_time_per_call == 32 42 | assert stats.own_time_per_call == 32 43 | assert len(stats) == 0 44 | assert not list(stats) 45 | 46 | 47 | def test_repr(): 48 | stats = Statistics(name='foo', own_hits=4, deep_time=128) 49 | assert repr(stats) == "" 50 | frozen = FrozenStatistics(name='foo', own_hits=4, deep_time=128, 51 | children=[]) 52 | frozen.children.append(Statistics(name='baz', own_hits=1, deep_time=120)) 53 | assert \ 54 | repr(frozen) == \ 55 | "" 56 | 57 | 58 | def test_hash(): 59 | stats1 = Statistics(name='foo', filename='bar', lineno=42) 60 | stats2 = Statistics(name='baz', filename='bar', lineno=89) 61 | stats_dics = {stats1: 1, stats2: 2} 62 | assert stats_dics[stats1] == 1 63 | assert stats_dics[stats2] == 2 64 | 65 | 66 | def test_recording(): 67 | # profiling is a module not code. 68 | # but inspect.getmodule() works like with code of a module. 69 | assert RecordingStatistics(profiling).module == 'profiling' 70 | # use discard_child. 71 | stats = RecordingStatistics() 72 | assert None not in stats 73 | stats.ensure_child(None) 74 | assert None in stats 75 | stats.discard_child(None) 76 | assert None not in stats 77 | stats.discard_child(None) 78 | assert None not in stats 79 | 80 | 81 | # def test_recording(): 82 | # stats.wall = lambda: 10 83 | # stats.record_starting(0) 84 | # code = mock_code('foo') 85 | # stats = RecordingStatistics(code) 86 | # assert stats.name == 'foo' 87 | # assert stats.own_hits == 0 88 | # assert stats.deep_time == 0 89 | # stats.record_entering(100) 90 | # stats.record_leaving(200) 91 | # assert stats.own_hits == 1 92 | # assert stats.deep_time == 100 93 | # stats.record_entering(200) 94 | # stats.record_leaving(400) 95 | # assert stats.own_hits == 2 96 | # assert stats.deep_time == 300 97 | # code2 = mock_code('bar') 98 | # stats2 = RecordingStatistics(code2) 99 | # assert code2 not in stats 100 | # stats.add_child(code2, stats2) 101 | # assert code2 in stats 102 | # assert stats.get_child(code2) is stats2 103 | # assert len(stats) == 1 104 | # assert list(stats) == [stats2] 105 | # assert stats.deep_time == 300 106 | # assert stats.own_time == 300 107 | # stats2.record_entering(1000) 108 | # stats2.record_leaving(1004) 109 | # assert stats2.deep_time == 4 110 | # assert stats2.own_time == 4 111 | # assert stats.deep_time == 300 112 | # assert stats.own_time == 296 113 | # stats.clear() 114 | # assert len(stats) == 0 115 | # with pytest.raises(TypeError): 116 | # pickle.dumps(stats) 117 | # stats3 = stats.ensure_child(mock_code('baz'), VoidRecordingStatistics) 118 | # assert isinstance(stats3, VoidRecordingStatistics) 119 | # stats.wall = lambda: 2000 120 | # stats.record_stopping(400) 121 | # assert stats.cpu_time == 400 122 | # assert stats.wall_time == 1990 123 | # assert stats.cpu_usage == 400 / 1990. 124 | 125 | 126 | def test_pickle(): 127 | stats = Statistics(name='ok') 128 | for protocol in range(pickle.HIGHEST_PROTOCOL + 1): 129 | assert pickle.loads(pickle.dumps(stats, protocol)).name == 'ok' 130 | 131 | 132 | def test_frozen(): 133 | code = mock_code('foo') 134 | stats = RecordingStatistics(code) 135 | stats.deep_time = 10 136 | stats.ensure_child(None) 137 | # RecordingStatistics are frozen at pickling. 138 | frozen_stats = pickle.loads(pickle.dumps(stats)) 139 | assert frozen_stats.name == 'foo' 140 | assert frozen_stats.deep_time == 10 141 | assert len(frozen_stats) == 1 142 | restored_frozen_stats = pickle.loads(pickle.dumps(frozen_stats)) 143 | assert restored_frozen_stats.name == 'foo' 144 | assert restored_frozen_stats.deep_time == 10 145 | assert len(restored_frozen_stats) == 1 146 | 147 | 148 | def test_flatten(): 149 | stats = FrozenStatistics(children=[ 150 | FrozenStatistics('foo', own_hits=10, children=[ 151 | FrozenStatistics('foo', own_hits=20, children=[]), 152 | FrozenStatistics('bar', own_hits=30, children=[]), 153 | ]), 154 | FrozenStatistics('bar', own_hits=40, children=[]), 155 | FrozenStatistics('baz', own_hits=50, children=[]), 156 | ]) 157 | flat_stats = FlatFrozenStatistics.flatten(stats) 158 | children = {stats.name: stats for stats in flat_stats} 159 | assert len(children) == 3 160 | assert children['foo'].own_hits == 30 161 | assert children['bar'].own_hits == 70 162 | assert children['baz'].own_hits == 50 163 | 164 | 165 | def test_spread_stats(): 166 | stats = FrozenStatistics(children=[ 167 | FrozenStatistics('foo', own_hits=10, children=[ 168 | FrozenStatistics('foo', own_hits=20, children=[]), 169 | FrozenStatistics('bar', own_hits=30, children=[]), 170 | ]), 171 | FrozenStatistics('bar', own_hits=40, children=[]), 172 | FrozenStatistics('baz', own_hits=50, children=[]), 173 | ]) 174 | descendants = list(spread_stats(stats)) 175 | assert len(descendants) == 5 176 | assert descendants[0].name == 'foo' 177 | assert descendants[1].name == 'bar' 178 | assert descendants[2].name == 'baz' 179 | assert descendants[3].name == 'foo' 180 | assert descendants[4].name == 'bar' 181 | 182 | 183 | def test_sorting(): 184 | stats = RecordingStatistics(mock_code('foo')) 185 | stats1 = RecordingStatistics(mock_code('bar')) 186 | stats2 = RecordingStatistics(mock_code('baz')) 187 | stats3 = RecordingStatistics(mock_code('qux')) 188 | stats.add_child(stats1.code, stats1) 189 | stats.add_child(stats2.code, stats2) 190 | stats.add_child(stats3.code, stats3) 191 | stats.deep_time = 100 192 | stats1.deep_time = 20 193 | stats1.own_hits = 3 194 | stats2.deep_time = 30 195 | stats2.own_hits = 2 196 | stats3.deep_time = 40 197 | stats3.own_hits = 4 198 | assert stats.sorted() == [stats3, stats2, stats1] 199 | assert stats.sorted(by_own_hits) == [stats3, stats1, stats2] 200 | assert stats.sorted(~by_own_hits) == [stats2, stats1, stats3] 201 | assert stats.sorted(by_deep_time_per_call) == [stats2, stats3, stats1] 202 | assert stats.sorted(by_own_time_per_call) == [stats2, stats3, stats1] 203 | assert stats.sorted(by_name) == [stats1, stats2, stats3] 204 | 205 | 206 | @pytest.fixture 207 | def deep_stats(depth=sys.getrecursionlimit(), skip_if_no_recursion_error=True): 208 | # Define a function with deep recursion. 209 | def x0(frames): 210 | frames.append(sys._getframe()) 211 | locals_ = locals() 212 | for x in range(1, depth): 213 | code = dedent(''' 214 | import sys 215 | def x%d(frames): 216 | frames.append(sys._getframe()) 217 | x%d(frames) 218 | ''' % (x, x - 1)) 219 | exec_(code, locals_) 220 | f = locals_['x%d' % (depth - 1)] 221 | frames = [] 222 | try: 223 | f(frames) 224 | except RuntimeError: 225 | # Expected. 226 | pass 227 | else: 228 | # Maybe PyPy. 229 | if skip_if_no_recursion_error: 230 | pytest.skip('Recursion limit not exceeded') 231 | # Profile the deepest frame. 232 | profiler = TracingProfiler() 233 | profiler._profile(frames[-1], 'call', None) 234 | spin(0.5) 235 | profiler._profile(frames[-1], 'return', None) 236 | # Test with the result. 237 | stats, __, __ = profiler.result() 238 | return stats 239 | 240 | 241 | def test_recursion_limit(deep_stats): 242 | deepest_stats = list(spread_stats(deep_stats))[-1] 243 | assert deepest_stats.deep_time > 0 244 | # It exceeded the recursion limit until 6fe1b48. 245 | assert deep_stats.children[0].deep_time == deepest_stats.deep_time 246 | # Pickling. 247 | assert isinstance(deep_stats, RecordingStatistics) 248 | data = pickle.dumps(deep_stats) 249 | frozen_stats = pickle.loads(data) 250 | assert isinstance(frozen_stats, FrozenStatistics) 251 | deepest_frozen_stats = list(spread_stats(frozen_stats))[-1] 252 | assert deepest_stats.deep_time == deepest_frozen_stats.deep_time 253 | 254 | 255 | def test_deep_stats_dump_performance(benchmark): 256 | stats = deep_stats(100, skip_if_no_recursion_error=False) 257 | benchmark(lambda: pickle.dumps(stats)) 258 | 259 | 260 | def test_deep_stats_load_performance(benchmark): 261 | stats = deep_stats(100, skip_if_no_recursion_error=False) 262 | data = pickle.dumps(stats) 263 | benchmark(lambda: pickle.loads(data)) 264 | 265 | 266 | def test_shallow_stats_dump_performance(benchmark): 267 | stats = deep_stats(5, skip_if_no_recursion_error=False) 268 | benchmark(lambda: pickle.dumps(stats)) 269 | 270 | 271 | def test_shallow_stats_load_performance(benchmark): 272 | stats = deep_stats(5, skip_if_no_recursion_error=False) 273 | data = pickle.dumps(stats) 274 | benchmark(lambda: pickle.loads(data)) 275 | -------------------------------------------------------------------------------- /test/test_timers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import re 3 | import sys 4 | import time 5 | 6 | import pytest 7 | 8 | from _utils import factorial, find_stats 9 | from profiling.__main__ import spawn_thread 10 | from profiling.tracing import TracingProfiler 11 | from profiling.tracing.timers import GreenletTimer, ThreadTimer 12 | 13 | 14 | # is it running on pypy? 15 | try: 16 | import __pypy__ 17 | except ImportError: 18 | PYPY = False 19 | else: 20 | PYPY = True 21 | del __pypy__ 22 | 23 | 24 | def _test_timer_with_threads(timer, sleep, spawn, join=lambda x: x.join()): 25 | def light(): 26 | factorial(10) 27 | sleep(0.1) 28 | factorial(10) 29 | def heavy(): 30 | factorial(10000) 31 | def profile(profiler): 32 | with profiler: 33 | c1 = spawn(light) 34 | c2 = spawn(heavy) 35 | for c in [c1, c2]: 36 | join(c) 37 | stat1 = find_stats(profiler.stats, 'light') 38 | stat2 = find_stats(profiler.stats, 'heavy') 39 | return (stat1, stat2) 40 | # using the default timer. 41 | # light() ends later than heavy(). its total time includes heavy's also. 42 | normal_profiler = TracingProfiler(base_frame=sys._getframe()) 43 | stat1, stat2 = profile(normal_profiler) 44 | assert stat1.deep_time >= stat2.deep_time 45 | # using the given timer. 46 | # light() ends later than heavy() like the above case. but the total time 47 | # doesn't include heavy's. each contexts should have isolated cpu time. 48 | contextual_profiler = TracingProfiler(base_frame=sys._getframe(), 49 | timer=timer) 50 | stat1, stat2 = profile(contextual_profiler) 51 | assert stat1.deep_time < stat2.deep_time 52 | 53 | 54 | @pytest.mark.flaky(reruns=10) 55 | def test_thread_timer(): 56 | if sys.version_info < (3, 3): 57 | pytest.importorskip('yappi') 58 | _test_timer_with_threads(ThreadTimer(), time.sleep, spawn_thread) 59 | 60 | 61 | @pytest.mark.xfail(PYPY, reason='greenlet.settrace() not available on PyPy.') 62 | def test_greenlet_timer_with_gevent(): 63 | try: 64 | gevent = pytest.importorskip('gevent', '1') 65 | except ValueError: 66 | # gevent Alpha or Beta versions doesn't respect Semantic Versioning. 67 | # e.g. 1.1a1, 1.1b5, 1.1rc1 68 | gevent = pytest.importorskip('gevent') 69 | assert re.match(r'^1\.1([ab]|rc\d+)', gevent.__version__) 70 | _test_timer_with_threads(GreenletTimer(), gevent.sleep, gevent.spawn) 71 | 72 | 73 | @pytest.mark.xfail(PYPY, reason='greenlet.settrace() not available on PyPy.') 74 | def test_greenlet_timer_with_eventlet(): 75 | eventlet = pytest.importorskip('eventlet', '0.15') 76 | _test_timer_with_threads(GreenletTimer(), eventlet.sleep, eventlet.spawn, 77 | eventlet.greenthread.GreenThread.wait) 78 | -------------------------------------------------------------------------------- /test/test_tracing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import sys 3 | 4 | import pytest 5 | 6 | from _utils import factorial, find_stats, foo 7 | from profiling.stats import RecordingStatistics 8 | from profiling.tracing import TracingProfiler 9 | 10 | 11 | def test_setprofile(): 12 | profiler = TracingProfiler() 13 | assert sys.getprofile() is None 14 | with profiler: 15 | assert sys.getprofile() == profiler._profile 16 | assert sys.getprofile() is None 17 | sys.setprofile(lambda *x: x) 18 | with pytest.raises(RuntimeError): 19 | profiler.start() 20 | sys.setprofile(None) 21 | 22 | 23 | def test_profile(): 24 | profiler = TracingProfiler() 25 | frame = foo() 26 | profiler._profile(frame, 'call', None) 27 | profiler._profile(frame, 'return', None) 28 | assert len(profiler.stats) == 1 29 | stats1 = find_stats(profiler.stats, 'foo') 30 | stats2 = find_stats(profiler.stats, 'bar') 31 | stats3 = find_stats(profiler.stats, 'baz') 32 | assert stats1.own_hits == 0 33 | assert stats2.own_hits == 0 34 | assert stats3.own_hits == 1 35 | assert stats1.deep_hits == 1 36 | assert stats2.deep_hits == 1 37 | assert stats3.deep_hits == 1 38 | 39 | 40 | def test_profiler(): 41 | profiler = TracingProfiler(base_frame=sys._getframe()) 42 | assert isinstance(profiler.stats, RecordingStatistics) 43 | stats, cpu_time, wall_time = profiler.result() 44 | assert len(stats) == 0 45 | with profiler: 46 | factorial(1000) 47 | factorial(10000) 48 | stats1 = find_stats(profiler.stats, 'factorial') 49 | stats2 = find_stats(profiler.stats, '__enter__') 50 | stats3 = find_stats(profiler.stats, '__exit__') 51 | assert stats1.deep_time != 0 52 | assert stats1.deep_time == stats1.own_time 53 | assert stats1.own_time > stats2.own_time 54 | assert stats1.own_time > stats3.own_time 55 | assert stats1.own_hits == 2 56 | assert stats2.own_hits == 0 # entering to __enter__() wasn't profiled. 57 | assert stats3.own_hits == 1 58 | -------------------------------------------------------------------------------- /test/test_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from collections import deque 3 | 4 | import pytest 5 | import six 6 | 7 | from _utils import baz, foo, mock_code_names 8 | from profiling.utils import frame_stack, lazy_import, repr_frame, Runnable 9 | 10 | 11 | def test_runnable(): 12 | # not implemented. 13 | runnable = Runnable() 14 | with pytest.raises(NotImplementedError): 15 | runnable.start() 16 | # implemented well. 17 | class Implementation(Runnable): 18 | step = 1 19 | def run(self): 20 | self.step = 2 21 | yield 22 | self.step = 3 23 | runnable = Implementation() 24 | assert runnable.step == 1 25 | assert not runnable.is_running() 26 | runnable.start() 27 | assert runnable.step == 2 28 | assert runnable.is_running() 29 | with pytest.raises(RuntimeError): 30 | runnable.start() 31 | runnable.stop() 32 | assert runnable.step == 3 33 | assert not runnable.is_running() 34 | with pytest.raises(RuntimeError): 35 | runnable.stop() 36 | # implemented not well. 37 | class NotYield(Runnable): 38 | def run(self): 39 | if False: 40 | yield 41 | runnable = NotYield() 42 | with pytest.raises(TypeError): 43 | runnable.start() 44 | class YieldSomething(Runnable): 45 | def run(self): 46 | yield 123 47 | runnable = YieldSomething() 48 | with pytest.raises(TypeError): 49 | runnable.start() 50 | class YieldTwice(Runnable): 51 | def run(self): 52 | yield 53 | yield 54 | runnable = YieldTwice() 55 | runnable.start() 56 | with pytest.raises(TypeError): 57 | runnable.stop() 58 | 59 | 60 | def test_frame_stack(): 61 | def to_code_names(frames): 62 | code_names = deque() 63 | for frame in reversed(frames): 64 | code_name = frame.f_code.co_name 65 | if code_name not in mock_code_names: 66 | break 67 | code_names.appendleft(code_name) 68 | return list(code_names) 69 | baz_frame = foo() 70 | foo_frame = baz_frame.f_back.f_back 71 | frames = frame_stack(baz_frame) 72 | assert to_code_names(frames) == ['foo', 'bar', 'baz'] 73 | # base frame. 74 | frames = frame_stack(baz_frame, base_frame=foo_frame) 75 | assert to_code_names(frames) == ['bar', 'baz'] 76 | # ignored codes. 77 | frames = frame_stack(baz_frame, ignored_codes=[ 78 | six.get_function_code(foo), 79 | six.get_function_code(baz), 80 | ]) 81 | assert to_code_names(frames) == ['bar'] 82 | 83 | 84 | def test_lazy_import(): 85 | class MathHolder(object): 86 | math = lazy_import('math') 87 | assert MathHolder.math is __import__('math') 88 | 89 | 90 | def test_repr_frame(): 91 | frame = foo() 92 | assert repr_frame(frame) == ':9' 93 | assert repr_frame(frame.f_back) == ':6' 94 | -------------------------------------------------------------------------------- /test/test_viewer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from profiling.viewer import fmt 3 | 4 | 5 | def test_fmt(): 6 | assert fmt.markup_percent(1.00) == ('danger', '100') 7 | assert fmt.markup_percent(0.80) == ('caution', '80.0') 8 | assert fmt.markup_percent(0.50) == ('warning', '50.0') 9 | assert fmt.markup_percent(0.20) == ('notice', '20.0') 10 | assert fmt.markup_percent(0.05) == (None, '5.00') 11 | assert fmt.markup_percent(0.00) == ('zero', '0.00') 12 | assert fmt.markup_percent(0.00, unit=True) == ('zero', '0.00%') 13 | assert fmt.markup_int(1.234) == (None, '1') 14 | assert fmt.markup_int(4.567) == (None, '5') 15 | assert fmt.markup_int(0) == ('zero', '0') 16 | assert fmt.markup_int_or_na(1.234) == (None, '1') 17 | assert fmt.markup_int_or_na(0) == ('zero', 'n/a') 18 | assert fmt.markup_time(0) == ('zero', '0') 19 | assert fmt.markup_time(0.123456) == ('msec', '123ms') 20 | assert fmt.markup_time(12.34567) == ('sec', '12.3sec') 21 | 22 | 23 | def test_format_int(): 24 | assert fmt.format_int(0) == '0' 25 | assert fmt.format_int(123) == '123' 26 | assert fmt.format_int(12345) == '12.3k' 27 | assert fmt.format_int(-12345) == '-12.3k' 28 | assert fmt.format_int(99999999) == '100.0M' 29 | assert fmt.format_int(-99999999) == '-100.0M' 30 | assert fmt.format_int(999999999) == '1.0G' 31 | assert fmt.format_int(-999999999) == '-1.0G' 32 | assert fmt.format_int(1e255) == 'o/f' 33 | assert fmt.format_int(-1e255) == 'u/f' 34 | 35 | 36 | def test_format_int_or_na(): 37 | assert fmt.format_int_or_na(0) == 'n/a' 38 | assert fmt.format_int_or_na(12345) == '12.3k' 39 | 40 | 41 | def test_format_time(): 42 | assert fmt.format_time(0) == '0' 43 | assert fmt.format_time(0.000001) == '1us' 44 | assert fmt.format_time(0.000123) == '123us' 45 | assert fmt.format_time(0.012345) == '12ms' 46 | assert fmt.format_time(0.123456) == '123ms' 47 | assert fmt.format_time(1.234567) == '1.2sec' 48 | assert fmt.format_time(12.34567) == '12.3sec' 49 | assert fmt.format_time(123.4567) == '2m3s' 50 | assert fmt.format_time(6120.000) == '102m' 51 | 52 | 53 | def test_format_percent(): 54 | assert fmt.format_percent(1) == '100' 55 | assert fmt.format_percent(0.999999) == '100' 56 | assert fmt.format_percent(0.9999) == '100' 57 | assert fmt.format_percent(0.988) == '98.8' 58 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = pypy, py27, py32, py33, py34, flake8 3 | minversion = 1.6.0 4 | 5 | [testenv:flake8] 6 | basepython = python3.4 7 | deps = flake8 8 | commands = flake8 profiling test setup.py -v --show-source --ignore=E301 9 | 10 | [testenv] 11 | deps = pytest 12 | # conditional settings require tox-1.8 or later. 13 | # py26: gevent 14 | # py27: gevent 15 | commands = py.test {posargs:-v} 16 | --------------------------------------------------------------------------------