├── tests ├── .gitignore ├── .gitattributes ├── pstats │ ├── cProfile.pstats │ └── cProfile.orig.dot └── test_main.py ├── sample.png ├── .gitignore ├── .travis.yml ├── tox.ini ├── setup.py ├── README.markdown └── gprof2dot.py /tests/.gitignore: -------------------------------------------------------------------------------- 1 | val3 2 | -------------------------------------------------------------------------------- /tests/.gitattributes: -------------------------------------------------------------------------------- 1 | *.xperf -crlf 2 | -------------------------------------------------------------------------------- /sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/gprof2dot/HEAD/sample.png -------------------------------------------------------------------------------- /tests/pstats/cProfile.pstats: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yelp/gprof2dot/HEAD/tests/pstats/cProfile.pstats -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .*.sw? 2 | *.dot 3 | *.egg-info 4 | *.orig 5 | *.png 6 | *.pyc 7 | build 8 | dist 9 | venv 10 | !sample.png 11 | !tests/pstats/cProfile.orig.dot 12 | 13 | .cache/ 14 | .tox/ 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | sudo: false 4 | 5 | addons: 6 | apt: 7 | packages: 8 | - graphviz 9 | 10 | python: 11 | - "2.7" 12 | - "3.4" 13 | - "3.5" 14 | 15 | script: 16 | - python tests/test.py 17 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | # TODO: add py3 tests 3 | envlist = py27 4 | 5 | [testenv] 6 | setenv = 7 | PYTHONHASHSEED = 100 8 | deps = 9 | ipdb 10 | pytest 11 | commands = 12 | python -m pytest -s {posargs:tests} 13 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # The purpose of this script is to enable uploading gprof2dot.py to the Python 4 | # Package Index, which can be easily done by doing: 5 | # 6 | # python setup.py register 7 | # python setup.py sdist upload 8 | # 9 | # See also: 10 | # - https://code.google.com/p/jrfonseca/issues/detail?id=19 11 | # - http://docs.python.org/2/distutils/packageindex.html 12 | # 13 | # Original version was written by Jose Fonseca; see 14 | # https://github.com/jrfonseca/gprof2dot 15 | 16 | import sys 17 | from setuptools import setup 18 | 19 | setup( 20 | name='yelp-gprof2dot', 21 | version='1.2.0', 22 | author='Yelp Performance Team', 23 | url='https://github.com/Yelp/gprof2dot', 24 | description="Generate a dot graph from the output of python profilers.", 25 | long_description=""" 26 | gprof2dot.py is a Python script to convert the output from python 27 | profilers (anything in pstats format) into a dot graph. It is a 28 | fork of the original gprof2dot script to extend python features. 29 | """, 30 | license="LGPL", 31 | 32 | py_modules=['gprof2dot'], 33 | entry_points=dict(console_scripts=['gprof2dot=gprof2dot:main']), 34 | ) 35 | -------------------------------------------------------------------------------- /tests/test_main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2013 Jose Fonseca 4 | # 5 | # This program is free software: you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published 7 | # by the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU Lesser General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | # 18 | import io 19 | import difflib 20 | import os.path 21 | import subprocess 22 | 23 | 24 | def run(cmd): 25 | p = subprocess.Popen(cmd) 26 | try: 27 | return p.wait() 28 | except KeyboardInterrupt: 29 | p.terminate() 30 | raise 31 | 32 | 33 | def assert_no_diff(a, b): 34 | # Asserts no difference between two files 35 | a_lines = io.open(a, 'rt', encoding='UTF-8').readlines() 36 | b_lines = io.open(b, 'rt', encoding='UTF-8').readlines() 37 | diff_lines = difflib.unified_diff(a_lines, b_lines, fromfile=a, tofile=b) 38 | diff_lines = ''.join(diff_lines) 39 | assert diff_lines == '' 40 | 41 | 42 | def test_main(): 43 | # A fairly brittle end-to-end tests for the gprof2dot executable. 44 | # Runs gprof2dot in a subprocess and compares results against a 45 | # pre-created file. (Note that python's hash seed matters, since a lot 46 | # of the graph creation is iterating through dictionaries. Check tox.ini) 47 | test_subdir = os.path.join(os.getcwd(), 'tests/pstats') 48 | pstats_files = [f for f in os.listdir(test_subdir) if f.endswith('.pstats')] 49 | for filename in pstats_files: 50 | name, _ = os.path.splitext(filename) 51 | profile = os.path.join(test_subdir, filename) 52 | dot = os.path.join(test_subdir, name + '.dot') 53 | png = os.path.join(test_subdir, name + '.png') 54 | 55 | ref_dot = os.path.join(test_subdir, name + '.orig.dot') 56 | ref_png = os.path.join(test_subdir, name + '.orig.png') 57 | 58 | if run(['python', 'gprof2dot.py', '-o', dot, profile]) != 0: 59 | continue 60 | 61 | if run(['dot', '-Tpng', '-o', png, dot]) != 0: 62 | continue 63 | 64 | assert_no_diff(ref_dot, dot) 65 | -------------------------------------------------------------------------------- /tests/pstats/cProfile.orig.dot: -------------------------------------------------------------------------------- 1 | digraph { 2 | graph [fontname=Arial, nodesep=0.125, ranksep=0.25]; 3 | node [fontcolor=white, fontname=Arial, height=0, shape=box, style=filled, width=0]; 4 | edge [fontname=Arial]; 5 | 1 [color="#ff0100", fontcolor="#ffffff", fontsize="10.00", label="pystone:33:\n99.89% (3865ms)\n0.02% (0.6ms)\n1×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 6 | 1 -> 18 [arrowsize="1.00", color="#ff0100", fontcolor="#ff0100", fontsize="10.00", label="99.87%\n1×", labeldistance="3.99", penwidth="3.99"]; 7 | 2 [color="#ff0000", fontcolor="#ffffff", fontsize="10.00", label="~:0:\n100.00% (3869ms)\n0.11% (4.3ms)\n1×", tooltip="~"]; 8 | 2 -> 1 [arrowsize="1.00", color="#ff0100", fontcolor="#ff0100", fontsize="10.00", label="99.89%\n1×", labeldistance="4.00", penwidth="4.00"]; 9 | 3 [color="#ff0000", fontcolor="#ffffff", fontsize="10.00", label=":1:\n100.00% (3869ms)\n0.00% (0.0ms)\n1×", tooltip=""]; 10 | 3 -> 2 [arrowsize="1.00", color="#ff0000", fontcolor="#ff0000", fontsize="10.00", label="100.00%\n1×", labeldistance="4.00", penwidth="4.00"]; 11 | 4 [color="#0d397f", fontcolor="#ffffff", fontsize="10.00", label="pystone:208:Proc8\n9.68% (374ms)\n7.57% (292.9ms)\n50000×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 12 | 4 -> 14 [arrowsize="0.35", color="#0d1675", fontcolor="#0d1675", fontsize="10.00", label="2.11%\n50000×", labeldistance="0.50", penwidth="0.50"]; 13 | 5 [color="#ff0100", fontcolor="#ffffff", fontsize="10.00", label="pystone:79:Proc0\n99.86% (3864ms)\n29.86% (1155.3ms)\n1×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 14 | 5 -> 4 [arrowsize="0.35", color="#0d397f", fontcolor="#0d397f", fontsize="10.00", label="9.68%\n50000×", labeldistance="0.50", penwidth="0.50"]; 15 | 5 -> 6 [arrowsize="0.59", color="#0ba067", fontcolor="#0ba067", fontsize="10.00", label="34.56%\n50000×", labeldistance="1.38", penwidth="1.38"]; 16 | 5 -> 7 [arrowsize="0.35", color="#0d1976", fontcolor="#0d1976", fontsize="10.00", label="2.80%\n100000×", labeldistance="0.50", penwidth="0.50"]; 17 | 5 -> 10 [arrowsize="0.35", color="#0d1976", fontcolor="#0d1976", fontsize="10.00", label="2.81%\n50000×", labeldistance="0.50", penwidth="0.50"]; 18 | 5 -> 12 [arrowsize="0.35", color="#0d1b77", fontcolor="#0d1b77", fontsize="10.00", label="3.26%\n100000×", labeldistance="0.50", penwidth="0.50"]; 19 | 5 -> 13 [arrowsize="0.35", color="#0d1575", fontcolor="#0d1575", fontsize="10.00", label="2.03%\n50000×", labeldistance="0.50", penwidth="0.50"]; 20 | 5 -> 20 [arrowsize="0.35", color="#0d1575", fontcolor="#0d1575", fontsize="10.00", label="1.89%\n50000×", labeldistance="0.50", penwidth="0.50"]; 21 | 5 -> 22 [arrowsize="0.35", color="#0d1475", fontcolor="#0d1475", fontsize="10.00", label="1.79%\n50000×", labeldistance="0.50", penwidth="0.50"]; 22 | 5 -> 24 [arrowsize="0.35", color="#0d2e7c", fontcolor="#0d2e7c", fontsize="10.00", label="7.40%\n50000×", labeldistance="0.50", penwidth="0.50"]; 23 | 5 -> 25 [arrowsize="0.35", color="#0d1d78", fontcolor="#0d1d78", fontsize="10.00", label="3.73%\n100000×", labeldistance="0.50", penwidth="0.50"]; 24 | 6 [color="#0ba067", fontcolor="#ffffff", fontsize="10.00", label="pystone:133:Proc1\n34.56% (1337ms)\n13.10% (507.0ms)\n50000×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 25 | 6 -> 9 [arrowsize="0.35", color="#0d277a", fontcolor="#0d277a", fontsize="10.00", label="5.95%\n50000×", labeldistance="0.50", penwidth="0.50"]; 26 | 6 -> 16 [arrowsize="0.35", color="#0d2f7d", fontcolor="#0d2f7d", fontsize="10.00", label="7.70%\n50000×", labeldistance="0.50", penwidth="0.50"]; 27 | 6 -> 17 [arrowsize="0.35", color="#0d287b", fontcolor="#0d287b", fontsize="10.00", label="6.12%\n50000×", labeldistance="0.50", penwidth="0.50"]; 28 | 6 -> 22 [arrowsize="0.35", color="#0d1475", fontcolor="#0d1475", fontsize="10.00", label="1.68%\n50000×", labeldistance="0.50", penwidth="0.50"]; 29 | 7 [color="#0d1976", fontcolor="#ffffff", fontsize="10.00", label="~:0:\n2.80% (108ms)\n2.80% (108.4ms)\n100000×", tooltip="~"]; 30 | 8 [color="#0d1475", fontcolor="#ffffff", fontsize="10.00", label="pystone:246:Func3\n1.74% (68ms)\n1.74% (67.5ms)\n50000×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 31 | 9 [color="#0d277a", fontcolor="#ffffff", fontsize="10.00", label="pystone:184:Proc6\n5.95% (230ms)\n4.21% (162.8ms)\n50000×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 32 | 9 -> 8 [arrowsize="0.35", color="#0d1475", fontcolor="#0d1475", fontsize="10.00", label="1.74%\n50000×", labeldistance="0.50", penwidth="0.50"]; 33 | 10 [color="#0d1976", fontcolor="#ffffff", fontsize="10.00", label="pystone:149:Proc2\n2.81% (109ms)\n2.81% (108.8ms)\n50000×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 34 | 12 [color="#0d1b77", fontcolor="#ffffff", fontsize="10.00", label="~:0:\n3.26% (126ms)\n3.26% (126.0ms)\n100000×", tooltip="~"]; 35 | 13 [color="#0d1575", fontcolor="#ffffff", fontsize="10.00", label="pystone:170:Proc4\n2.03% (79ms)\n2.03% (78.7ms)\n50000×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 36 | 14 [color="#0d1676", fontcolor="#ffffff", fontsize="10.00", label="~:0:\n2.17% (84ms)\n2.17% (83.9ms)\n50003×", tooltip="~"]; 37 | 15 [color="#0d1876", fontcolor="#ffffff", fontsize="10.00", label="pystone:45:__init__\n2.63% (102ms)\n2.63% (101.8ms)\n50002×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 38 | 16 [color="#0d2f7d", fontcolor="#ffffff", fontsize="10.00", label="pystone:53:copy\n7.70% (298ms)\n5.07% (196.3ms)\n50000×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 39 | 16 -> 15 [arrowsize="0.35", color="#0d1876", fontcolor="#0d1876", fontsize="10.00", label="2.63%\n50000×", labeldistance="0.50", penwidth="0.50"]; 40 | 17 [color="#0d287b", fontcolor="#ffffff", fontsize="10.00", label="pystone:160:Proc3\n6.12% (237ms)\n4.24% (164.0ms)\n50000×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 41 | 17 -> 22 [arrowsize="0.35", color="#0d1575", fontcolor="#0d1575", fontsize="10.00", label="1.88%\n50000×", labeldistance="0.50", penwidth="0.50"]; 42 | 18 [color="#ff0100", fontcolor="#ffffff", fontsize="10.00", label="pystone:60:main\n99.87% (3864ms)\n0.01% (0.3ms)\n1×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 43 | 18 -> 19 [arrowsize="1.00", color="#ff0100", fontcolor="#ff0100", fontsize="10.00", label="99.86%\n1×", labeldistance="3.99", penwidth="3.99"]; 44 | 19 [color="#ff0100", fontcolor="#ffffff", fontsize="10.00", label="pystone:67:pystones\n99.86% (3864ms)\n0.00% (0.0ms)\n1×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 45 | 19 -> 5 [arrowsize="1.00", color="#ff0100", fontcolor="#ff0100", fontsize="10.00", label="99.86%\n1×", labeldistance="3.99", penwidth="3.99"]; 46 | 20 [color="#0d1575", fontcolor="#ffffff", fontsize="10.00", label="pystone:177:Proc5\n1.89% (73ms)\n1.89% (73.1ms)\n50000×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 47 | 22 [color="#0d247a", fontcolor="#ffffff", fontsize="10.00", label="pystone:203:Proc7\n5.35% (207ms)\n5.35% (207.1ms)\n150000×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 48 | 24 [color="#0d2e7c", fontcolor="#ffffff", fontsize="10.00", label="pystone:229:Func2\n7.40% (286ms)\n5.56% (215.0ms)\n50000×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 49 | 24 -> 25 [arrowsize="0.35", color="#0d1475", fontcolor="#0d1475", fontsize="10.00", label="1.84%\n50000×", labeldistance="0.50", penwidth="0.50"]; 50 | 25 [color="#0d257a", fontcolor="#ffffff", fontsize="10.00", label="pystone:221:Func1\n5.57% (215ms)\n5.57% (215.4ms)\n150000×", tooltip="/usr/lib/python2.5/test/pystone.py"]; 51 | } 52 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | # About _gprof2dot_ 2 | 3 | This is a Python script to convert the output from python's profiles into a [dot graph](http://www.graphviz.org/doc/info/lang.html). It is forked from the original version of the library to address Yelp's specific needs for finer-grained control over display of python profiles. 4 | 5 | It has the following features: 6 | 7 | * reads output from: 8 | * [python profilers](http://docs.python.org/2/library/profile.html#profile-stats) 9 | * prunes nodes and edges below a certain threshold; 10 | * uses an heuristic to propagate time inside mutually recursive functions; 11 | * uses color efficiently to draw attention to hot-spots; 12 | * works on any platform where Python and graphviz is available, i.e, virtually anywhere. 13 | 14 | **If you want an interactive viewer for gprof2dot output graphs, check [xdot.py](https://github.com/jrfonseca/xdot.py).** 15 | 16 | ## Example 17 | 18 | This is the result from the [example data](http://linuxgazette.net/100/misc/vinayak/overall-profile.txt) in the [Linux Gazette article](http://linuxgazette.net/100/vinayak.html) with the default settings: 19 | 20 | 21 | ![Sample](sample.png) 22 | 23 | # Requirements 24 | 25 | * [Python](http://www.python.org/download/): known to work with version 2.7 and 3.3; it will most likely _not_ work with earlier releases. 26 | * [Graphviz](http://www.graphviz.org/Download.php): tested with version 2.26.3, but should work fine with other versions. 27 | 28 | ## Windows users 29 | 30 | * Download and install [Python for Windows](http://www.python.org/download/) 31 | * Download and install [Graphviz for Windows](http://www.graphviz.org/Download_windows.php) 32 | 33 | ## Debian/Ubuntu users 34 | 35 | * Run: 36 | 37 | apt-get install python graphviz 38 | 39 | 40 | # Download 41 | 42 | * [PyPI](https://pypi.python.org/pypi/gprof2dot/) 43 | 44 | pip install gprof2dot 45 | 46 | * [Standalone script](https://raw.githubusercontent.com/jrfonseca/gprof2dot/master/gprof2dot.py) 47 | 48 | * [Git repository](https://github.com/jrfonseca/gprof2dot) 49 | 50 | 51 | # Documentation 52 | 53 | ## Usage 54 | 55 | Usage: 56 | gprof2dot.py [options] [file] ... 57 | 58 | Options: 59 | -h, --help show this help message and exit 60 | -o FILE, --output=FILE 61 | output filename [stdout] 62 | -n PERCENTAGE, --node-thres=PERCENTAGE 63 | eliminate nodes below this threshold [default: 0.5] 64 | -e PERCENTAGE, --edge-thres=PERCENTAGE 65 | eliminate edges below this threshold [default: 0.1] 66 | --total=TOTALMETHOD preferred method of calculating total time: callratios 67 | or callstacks (currently affects only perf format) 68 | [default: callratios] 69 | -c THEME, --colormap=THEME 70 | color map: color, pink, gray, bw, or print [default: 71 | color] 72 | -s, --strip strip function parameters, template parameters, and 73 | const modifiers from demangled C++ function names 74 | -w, --wrap wrap function names 75 | --show-samples show function samples 76 | -z ROOT, --root=ROOT prune call graph to show only descendants of specified 77 | root function 78 | -l LEAF, --leaf=LEAF prune call graph to show only ancestors of specified 79 | leaf function 80 | --skew=THEME_SKEW skew the colorization curve. Values < 1.0 give more 81 | variety to lower percentages. Values > 1.0 give less 82 | variety to lower percentages 83 | 84 | ## Examples 85 | 86 | ### python profile 87 | 88 | python -m profile -o output.pstats path/to/your/script arg1 arg2 89 | gprof2dot.py -f pstats output.pstats | dot -Tpng -o output.png 90 | 91 | ### python cProfile (formerly known as lsprof) 92 | 93 | python -m cProfile -o output.pstats path/to/your/script arg1 arg2 94 | gprof2dot.py -f pstats output.pstats | dot -Tpng -o output.png 95 | 96 | ### python hotshot profiler 97 | 98 | The hotshot profiler does not include a main function. Use the [hotshotmain.py](hotshotmain.py) script instead. 99 | 100 | hotshotmain.py -o output.pstats path/to/your/script arg1 arg2 101 | gprof2dot.py -f pstats output.pstats | dot -Tpng -o output.png 102 | 103 | 104 | ## Output 105 | 106 | A node in the output graph represents a function and has the following layout: 107 | 108 | +------------------------------+ 109 | | function name | 110 | | total time % (ms) | 111 | | self time % (ms) | 112 | | total calls | 113 | +------------------------------+ 114 | 115 | where: 116 | 117 | * _total time %_ is the percentage of the running time spent in this function and all its children; 118 | * _self time %_ is the percentage of the running time spent in this function alone; 119 | * (ms) are the actual milliseconds spent in the function 120 | * _total calls_ is the total number of times this function was called (including recursive calls). 121 | 122 | An edge represents the calls between two functions and has the following layout: 123 | 124 | total time % 125 | calls 126 | parent --------------------> children 127 | 128 | Where: 129 | 130 | * _total time %_ is the percentage of the running time transfered from the children to this parent (if available); 131 | * _calls_ is the number of calls the parent function called the children. 132 | 133 | Note that in recursive cycles, the _total time %_ in the node is the same for the whole functions in the cycle, and there is no _total time %_ figure in the edges inside the cycle, since such figure would make no sense. 134 | 135 | The color of the nodes and edges varies according to the _total time %_ value. In the default _temperature-like_ color-map, functions where most time is spent (hot-spots) are marked as saturated red, and functions where little time is spent are marked as dark blue. Note that functions where negligible or no time is spent do not appear in the graph by default. 136 | 137 | ## Frequently Asked Questions 138 | 139 | ### How can I generate a complete call graph? 140 | 141 | By default `gprof2dot.py` generates a _partial_ call graph, excluding nodes and edges with little or no impact in the total computation time. If you want the full call graph then set a zero threshold for nodes and edges via the `-n` / `--node-thres` and `-e` / `--edge-thres` options, as: 142 | 143 | gprof2dot.py -n0 -e0 144 | 145 | ### The node labels are too wide. How can I narrow them? 146 | 147 | The node labels can get very wide when profiling C++ code, due to inclusion of scope, function arguments, and template arguments in demangled C++ function names. 148 | 149 | If you do not need function and template arguments information, then pass the `-s` / `--strip` option to strip them. 150 | 151 | If you want to keep all that information, or if the labels are still too wide, then you can pass the `-w` / `--wrap`, to wrap the labels. Note that because `dot` does not wrap labels automatically the label margins will not be perfectly aligned. 152 | 153 | ### Why there is no output, or it is all in the same color? 154 | 155 | Likely, the total execution time is too short, so there is not enough precision in the profile to determine where time is being spent. 156 | 157 | You can still force displaying the whole graph by setting a zero threshold for nodes and edges via the `-n` / `--node-thres` and `-e` / `--edge-thres` options, as: 158 | 159 | gprof2dot.py -n0 -e0 160 | 161 | But to get meaningful results you will need to find a way to run the program for a longer time period (aggregate results from multiple runs). 162 | 163 | ### Why don't the percentages add up? 164 | 165 | You likely have an execution time too short, causing the round-off errors to be large. 166 | 167 | See question above for ways to increase execution time. 168 | -------------------------------------------------------------------------------- /gprof2dot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2008-2014 Jose Fonseca 4 | # 5 | # This program is free software: you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published 7 | # by the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU Lesser General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | # 18 | 19 | """Generate a dot graph from the output of several profilers.""" 20 | 21 | __author__ = "Jose Fonseca et al" 22 | 23 | 24 | import base64 25 | import sys 26 | import math 27 | import os.path 28 | import re 29 | import textwrap 30 | import optparse 31 | import xml.parsers.expat 32 | import collections 33 | import locale 34 | import json 35 | 36 | 37 | # Python 2.x/3.x compatibility 38 | if sys.version_info[0] >= 3: 39 | PYTHON_3 = True 40 | def compat_iteritems(x): return x.items() # No iteritems() in Python 3 41 | def compat_itervalues(x): return x.values() # No itervalues() in Python 3 42 | def compat_keys(x): return list(x.keys()) # keys() is a generator in Python 3 43 | basestring = str # No class basestring in Python 3 44 | unichr = chr # No unichr in Python 3 45 | xrange = range # No xrange in Python 3 46 | else: 47 | PYTHON_3 = False 48 | def compat_iteritems(x): return x.iteritems() 49 | def compat_itervalues(x): return x.itervalues() 50 | def compat_keys(x): return x.keys() 51 | 52 | 53 | try: 54 | # Debugging helper module 55 | import debug 56 | except ImportError: 57 | pass 58 | 59 | 60 | 61 | ######################################################################## 62 | # Model 63 | 64 | 65 | MULTIPLICATION_SIGN = unichr(0xd7) 66 | 67 | 68 | def times(x): 69 | return "%u%s" % (x, MULTIPLICATION_SIGN) 70 | 71 | def percentage(p): 72 | return "%.02f%%" % (p*100.0,) 73 | 74 | def add(a, b): 75 | return a + b 76 | 77 | def fail(a, b): 78 | assert False 79 | 80 | 81 | tol = 2 ** -23 82 | 83 | def ratio(numerator, denominator): 84 | try: 85 | ratio = float(numerator)/float(denominator) 86 | except ZeroDivisionError: 87 | # 0/0 is undefined, but 1.0 yields more useful results 88 | return 1.0 89 | if ratio < 0.0: 90 | if ratio < -tol: 91 | sys.stderr.write('warning: negative ratio (%s/%s)\n' % (numerator, denominator)) 92 | return 0.0 93 | if ratio > 1.0: 94 | if ratio > 1.0 + tol: 95 | sys.stderr.write('warning: ratio greater than one (%s/%s)\n' % (numerator, denominator)) 96 | return 1.0 97 | return ratio 98 | 99 | 100 | class UndefinedEvent(Exception): 101 | """Raised when attempting to get an event which is undefined.""" 102 | 103 | def __init__(self, event): 104 | Exception.__init__(self) 105 | self.event = event 106 | 107 | def __str__(self): 108 | return 'unspecified event %s' % self.event.name 109 | 110 | 111 | class Event(object): 112 | """Describe a kind of event, and its basic operations.""" 113 | 114 | def __init__(self, name, null, aggregator, formatter=str): 115 | self.name = name 116 | self._null = null 117 | self._aggregator = aggregator 118 | self._formatter = formatter 119 | 120 | def __eq__(self, other): 121 | return self is other 122 | 123 | def __hash__(self): 124 | return id(self) 125 | 126 | def null(self): 127 | return self._null 128 | 129 | def aggregate(self, val1, val2): 130 | """Aggregate two event values.""" 131 | assert val1 is not None 132 | assert val2 is not None 133 | return self._aggregator(val1, val2) 134 | 135 | def format(self, val): 136 | """Format an event value.""" 137 | assert val is not None 138 | return self._formatter(val) 139 | 140 | 141 | CALLS = Event("Calls", 0, add, times) 142 | 143 | TIME = Event("Time", 0.0, add, lambda x: '(' + str(x) + ')') 144 | TIME_RATIO = Event("Time ratio", 0.0, add, lambda x: '(' + percentage(x) + ')') 145 | TOTAL_TIME = Event("Total time", 0.0, fail) 146 | TOTAL_TIME_RATIO = Event("Total time ratio", 0.0, fail, percentage) 147 | 148 | 149 | class Object(object): 150 | """Base class for all objects in profile which can store events.""" 151 | 152 | def __init__(self, events=None): 153 | if events is None: 154 | self.events = {} 155 | else: 156 | self.events = events 157 | 158 | def __hash__(self): 159 | return id(self) 160 | 161 | def __eq__(self, other): 162 | return self is other 163 | 164 | def __contains__(self, event): 165 | return event in self.events 166 | 167 | def __getitem__(self, event): 168 | try: 169 | return self.events[event] 170 | except KeyError: 171 | raise UndefinedEvent(event) 172 | 173 | def __setitem__(self, event, value): 174 | if value is None: 175 | if event in self.events: 176 | del self.events[event] 177 | else: 178 | self.events[event] = value 179 | 180 | 181 | class Call(Object): 182 | """A call between functions. 183 | 184 | There should be at most one call object for every pair of functions. 185 | """ 186 | 187 | def __init__(self, callee_id): 188 | Object.__init__(self) 189 | self.callee_id = callee_id 190 | self.ratio = None 191 | self.weight = None 192 | 193 | 194 | class Function(Object): 195 | """A function.""" 196 | 197 | def __init__(self, id, name): 198 | Object.__init__(self) 199 | self.id = id 200 | self.name = name 201 | self.module = None 202 | self.process = None 203 | self.calls = {} 204 | self.called = None 205 | self.weight = None 206 | self.cycle = None 207 | self.filename = None 208 | 209 | def add_call(self, call): 210 | if call.callee_id in self.calls: 211 | sys.stderr.write('warning: overwriting call from function %s to %s\n' % (str(self.id), str(call.callee_id))) 212 | self.calls[call.callee_id] = call 213 | 214 | def get_call(self, callee_id): 215 | if not callee_id in self.calls: 216 | call = Call(callee_id) 217 | call[CALLS] = 0 218 | self.calls[callee_id] = call 219 | return self.calls[callee_id] 220 | 221 | _parenthesis_re = re.compile(r'\([^()]*\)') 222 | _angles_re = re.compile(r'<[^<>]*>') 223 | _const_re = re.compile(r'\s+const$') 224 | 225 | def stripped_name(self): 226 | """Remove extraneous information from C++ demangled function names.""" 227 | 228 | name = self.name 229 | 230 | # Strip function parameters from name by recursively removing paired parenthesis 231 | while True: 232 | name, n = self._parenthesis_re.subn('', name) 233 | if not n: 234 | break 235 | 236 | # Strip const qualifier 237 | name = self._const_re.sub('', name) 238 | 239 | # Strip template parameters from name by recursively removing paired angles 240 | while True: 241 | name, n = self._angles_re.subn('', name) 242 | if not n: 243 | break 244 | 245 | return name 246 | 247 | # TODO: write utility functions 248 | 249 | def __repr__(self): 250 | return self.name 251 | 252 | 253 | class Cycle(Object): 254 | """A cycle made from recursive function calls.""" 255 | 256 | def __init__(self): 257 | Object.__init__(self) 258 | self.functions = set() 259 | 260 | def add_function(self, function): 261 | assert function not in self.functions 262 | self.functions.add(function) 263 | if function.cycle is not None: 264 | for other in function.cycle.functions: 265 | if function not in self.functions: 266 | self.add_function(other) 267 | function.cycle = self 268 | 269 | 270 | class Profile(Object): 271 | """The whole profile.""" 272 | 273 | def __init__(self): 274 | Object.__init__(self) 275 | self.functions = {} 276 | self.cycles = [] 277 | 278 | def add_function(self, function): 279 | if function.id in self.functions: 280 | sys.stderr.write('warning: overwriting function %s (id %s)\n' % (function.name, str(function.id))) 281 | self.functions[function.id] = function 282 | 283 | def add_cycle(self, cycle): 284 | self.cycles.append(cycle) 285 | 286 | def validate(self): 287 | """Validate the edges.""" 288 | 289 | for function in compat_itervalues(self.functions): 290 | for callee_id in compat_keys(function.calls): 291 | assert function.calls[callee_id].callee_id == callee_id 292 | if callee_id not in self.functions: 293 | sys.stderr.write('warning: call to undefined function %s from function %s\n' % (str(callee_id), function.name)) 294 | del function.calls[callee_id] 295 | 296 | def find_cycles(self): 297 | """Find cycles using Tarjan's strongly connected components algorithm.""" 298 | 299 | # Apply the Tarjan's algorithm successively until all functions are visited 300 | visited = set() 301 | for function in compat_itervalues(self.functions): 302 | if function not in visited: 303 | self._tarjan(function, 0, [], {}, {}, visited) 304 | cycles = [] 305 | for function in compat_itervalues(self.functions): 306 | if function.cycle is not None and function.cycle not in cycles: 307 | cycles.append(function.cycle) 308 | self.cycles = cycles 309 | if 0: 310 | for cycle in cycles: 311 | sys.stderr.write("Cycle:\n") 312 | for member in cycle.functions: 313 | sys.stderr.write("\tFunction %s\n" % member.name) 314 | 315 | def prune_root(self, root): 316 | visited = set() 317 | frontier = set([root]) 318 | while len(frontier) > 0: 319 | node = frontier.pop() 320 | visited.add(node) 321 | f = self.functions[node] 322 | newNodes = f.calls.keys() 323 | frontier = frontier.union(set(newNodes) - visited) 324 | subtreeFunctions = {} 325 | for n in visited: 326 | subtreeFunctions[n] = self.functions[n] 327 | self.functions = subtreeFunctions 328 | 329 | def prune_leaf(self, leaf): 330 | edgesUp = collections.defaultdict(set) 331 | for f in self.functions.keys(): 332 | for n in self.functions[f].calls.keys(): 333 | edgesUp[n].add(f) 334 | # build the tree up 335 | visited = set() 336 | frontier = set([leaf]) 337 | while len(frontier) > 0: 338 | node = frontier.pop() 339 | visited.add(node) 340 | frontier = frontier.union(edgesUp[node] - visited) 341 | downTree = set(self.functions.keys()) 342 | upTree = visited 343 | path = downTree.intersection(upTree) 344 | pathFunctions = {} 345 | for n in path: 346 | f = self.functions[n] 347 | newCalls = {} 348 | for c in f.calls.keys(): 349 | if c in path: 350 | newCalls[c] = f.calls[c] 351 | f.calls = newCalls 352 | pathFunctions[n] = f 353 | self.functions = pathFunctions 354 | 355 | 356 | def getFunctionId(self, funcName): 357 | for f in self.functions: 358 | if self.functions[f].name == funcName: 359 | return f 360 | return False 361 | 362 | def _tarjan(self, function, order, stack, orders, lowlinks, visited): 363 | """Tarjan's strongly connected components algorithm. 364 | 365 | See also: 366 | - http://en.wikipedia.org/wiki/Tarjan's_strongly_connected_components_algorithm 367 | """ 368 | 369 | visited.add(function) 370 | orders[function] = order 371 | lowlinks[function] = order 372 | order += 1 373 | pos = len(stack) 374 | stack.append(function) 375 | for call in compat_itervalues(function.calls): 376 | callee = self.functions[call.callee_id] 377 | # TODO: use a set to optimize lookup 378 | if callee not in orders: 379 | order = self._tarjan(callee, order, stack, orders, lowlinks, visited) 380 | lowlinks[function] = min(lowlinks[function], lowlinks[callee]) 381 | elif callee in stack: 382 | lowlinks[function] = min(lowlinks[function], orders[callee]) 383 | if lowlinks[function] == orders[function]: 384 | # Strongly connected component found 385 | members = stack[pos:] 386 | del stack[pos:] 387 | if len(members) > 1: 388 | cycle = Cycle() 389 | for member in members: 390 | cycle.add_function(member) 391 | return order 392 | 393 | def call_ratios(self, event): 394 | # Aggregate for incoming calls 395 | cycle_totals = {} 396 | for cycle in self.cycles: 397 | cycle_totals[cycle] = 0.0 398 | function_totals = {} 399 | for function in compat_itervalues(self.functions): 400 | function_totals[function] = 0.0 401 | 402 | # Pass 1: function_total gets the sum of call[event] for all 403 | # incoming arrows. Same for cycle_total for all arrows 404 | # that are coming into the *cycle* but are not part of it. 405 | for function in compat_itervalues(self.functions): 406 | for call in compat_itervalues(function.calls): 407 | if call.callee_id != function.id: 408 | callee = self.functions[call.callee_id] 409 | if event in call.events: 410 | function_totals[callee] += call[event] 411 | if callee.cycle is not None and callee.cycle is not function.cycle: 412 | cycle_totals[callee.cycle] += call[event] 413 | else: 414 | sys.stderr.write("call_ratios: No data for " + function.name + " call to " + callee.name + "\n") 415 | 416 | # Pass 2: Compute the ratios. Each call[event] is scaled by the 417 | # function_total of the callee. Calls into cycles use the 418 | # cycle_total, but not calls within cycles. 419 | for function in compat_itervalues(self.functions): 420 | for call in compat_itervalues(function.calls): 421 | assert call.ratio is None 422 | if call.callee_id != function.id: 423 | callee = self.functions[call.callee_id] 424 | if event in call.events: 425 | if callee.cycle is not None and callee.cycle is not function.cycle: 426 | total = cycle_totals[callee.cycle] 427 | else: 428 | total = function_totals[callee] 429 | call.ratio = ratio(call[event], total) 430 | else: 431 | # Warnings here would only repeat those issued above. 432 | call.ratio = 0.0 433 | 434 | def integrate(self, outevent, inevent): 435 | """Propagate function time ratio along the function calls. 436 | 437 | Must be called after finding the cycles. 438 | 439 | See also: 440 | - http://citeseer.ist.psu.edu/graham82gprof.html 441 | """ 442 | 443 | # Sanity checking 444 | assert outevent not in self 445 | for function in compat_itervalues(self.functions): 446 | assert outevent not in function 447 | assert inevent in function 448 | for call in compat_itervalues(function.calls): 449 | assert outevent not in call 450 | if call.callee_id != function.id: 451 | assert call.ratio is not None 452 | 453 | # Aggregate the input for each cycle 454 | for cycle in self.cycles: 455 | total = inevent.null() 456 | for function in compat_itervalues(self.functions): 457 | total = inevent.aggregate(total, function[inevent]) 458 | self[inevent] = total 459 | 460 | # Integrate along the edges 461 | total = inevent.null() 462 | for function in compat_itervalues(self.functions): 463 | total = inevent.aggregate(total, function[inevent]) 464 | self._integrate_function(function, outevent, inevent) 465 | self[outevent] = total 466 | 467 | def _integrate_function(self, function, outevent, inevent): 468 | if function.cycle is not None: 469 | return self._integrate_cycle(function.cycle, outevent, inevent) 470 | else: 471 | if outevent not in function: 472 | total = function[inevent] 473 | for call in compat_itervalues(function.calls): 474 | if call.callee_id != function.id: 475 | total += self._integrate_call(call, outevent, inevent) 476 | function[outevent] = total 477 | return function[outevent] 478 | 479 | def _integrate_call(self, call, outevent, inevent): 480 | assert outevent not in call 481 | assert call.ratio is not None 482 | callee = self.functions[call.callee_id] 483 | subtotal = call.ratio *self._integrate_function(callee, outevent, inevent) 484 | call[outevent] = subtotal 485 | return subtotal 486 | 487 | def _integrate_cycle(self, cycle, outevent, inevent): 488 | if outevent not in cycle: 489 | 490 | # Compute the outevent for the whole cycle 491 | total = inevent.null() 492 | for member in cycle.functions: 493 | subtotal = member[inevent] 494 | for call in compat_itervalues(member.calls): 495 | callee = self.functions[call.callee_id] 496 | if callee.cycle is not cycle: 497 | subtotal += self._integrate_call(call, outevent, inevent) 498 | total += subtotal 499 | cycle[outevent] = total 500 | 501 | # Compute the time propagated to callers of this cycle 502 | callees = {} 503 | for function in compat_itervalues(self.functions): 504 | if function.cycle is not cycle: 505 | for call in compat_itervalues(function.calls): 506 | callee = self.functions[call.callee_id] 507 | if callee.cycle is cycle: 508 | try: 509 | callees[callee] += call.ratio 510 | except KeyError: 511 | callees[callee] = call.ratio 512 | 513 | for member in cycle.functions: 514 | member[outevent] = outevent.null() 515 | 516 | for callee, call_ratio in compat_iteritems(callees): 517 | ranks = {} 518 | call_ratios = {} 519 | partials = {} 520 | self._rank_cycle_function(cycle, callee, 0, ranks) 521 | self._call_ratios_cycle(cycle, callee, ranks, call_ratios, set()) 522 | partial = self._integrate_cycle_function(cycle, callee, call_ratio, partials, ranks, call_ratios, outevent, inevent) 523 | assert partial == max(partials.values()) 524 | assert abs(call_ratio*total - partial) <= 0.001*call_ratio*total 525 | 526 | return cycle[outevent] 527 | 528 | def _rank_cycle_function(self, cycle, function, rank, ranks): 529 | if function not in ranks or ranks[function] > rank: 530 | ranks[function] = rank 531 | for call in compat_itervalues(function.calls): 532 | if call.callee_id != function.id: 533 | callee = self.functions[call.callee_id] 534 | if callee.cycle is cycle: 535 | self._rank_cycle_function(cycle, callee, rank + 1, ranks) 536 | 537 | def _call_ratios_cycle(self, cycle, function, ranks, call_ratios, visited): 538 | if function not in visited: 539 | visited.add(function) 540 | for call in compat_itervalues(function.calls): 541 | if call.callee_id != function.id: 542 | callee = self.functions[call.callee_id] 543 | if callee.cycle is cycle: 544 | if ranks[callee] > ranks[function]: 545 | call_ratios[callee] = call_ratios.get(callee, 0.0) + call.ratio 546 | self._call_ratios_cycle(cycle, callee, ranks, call_ratios, visited) 547 | 548 | def _integrate_cycle_function(self, cycle, function, partial_ratio, partials, ranks, call_ratios, outevent, inevent): 549 | if function not in partials: 550 | partial = partial_ratio*function[inevent] 551 | for call in compat_itervalues(function.calls): 552 | if call.callee_id != function.id: 553 | callee = self.functions[call.callee_id] 554 | if callee.cycle is not cycle: 555 | assert outevent in call 556 | partial += partial_ratio*call[outevent] 557 | else: 558 | if ranks[callee] > ranks[function]: 559 | callee_partial = self._integrate_cycle_function(cycle, callee, partial_ratio, partials, ranks, call_ratios, outevent, inevent) 560 | call_ratio = ratio(call.ratio, call_ratios[callee]) 561 | call_partial = call_ratio*callee_partial 562 | try: 563 | call[outevent] += call_partial 564 | except UndefinedEvent: 565 | call[outevent] = call_partial 566 | partial += call_partial 567 | partials[function] = partial 568 | try: 569 | function[outevent] += partial 570 | except UndefinedEvent: 571 | function[outevent] = partial 572 | return partials[function] 573 | 574 | def aggregate(self, event): 575 | """Aggregate an event for the whole profile.""" 576 | 577 | total = event.null() 578 | for function in compat_itervalues(self.functions): 579 | try: 580 | total = event.aggregate(total, function[event]) 581 | except UndefinedEvent: 582 | return 583 | self[event] = total 584 | 585 | def ratio(self, outevent, inevent): 586 | assert outevent not in self 587 | assert inevent in self 588 | for function in compat_itervalues(self.functions): 589 | assert outevent not in function 590 | assert inevent in function 591 | function[outevent] = ratio(function[inevent], self[inevent]) 592 | for call in compat_itervalues(function.calls): 593 | assert outevent not in call 594 | if inevent in call: 595 | call[outevent] = ratio(call[inevent], self[inevent]) 596 | self[outevent] = 1.0 597 | 598 | def prune(self, node_thres, edge_thres, colour_nodes_by_selftime): 599 | """Prune the profile""" 600 | 601 | # compute the prune ratios 602 | for function in compat_itervalues(self.functions): 603 | try: 604 | function.weight = function[TOTAL_TIME_RATIO] 605 | except UndefinedEvent: 606 | pass 607 | 608 | for call in compat_itervalues(function.calls): 609 | callee = self.functions[call.callee_id] 610 | 611 | if TOTAL_TIME_RATIO in call: 612 | # handle exact cases first 613 | call.weight = call[TOTAL_TIME_RATIO] 614 | else: 615 | try: 616 | # make a safe estimate 617 | call.weight = min(function[TOTAL_TIME_RATIO], callee[TOTAL_TIME_RATIO]) 618 | except UndefinedEvent: 619 | pass 620 | 621 | # prune the nodes 622 | for function_id in compat_keys(self.functions): 623 | function = self.functions[function_id] 624 | if function.weight is not None: 625 | if function.weight < node_thres: 626 | del self.functions[function_id] 627 | 628 | # prune the edges 629 | for function in compat_itervalues(self.functions): 630 | for callee_id in compat_keys(function.calls): 631 | call = function.calls[callee_id] 632 | if callee_id not in self.functions or call.weight is not None and call.weight < edge_thres: 633 | del function.calls[callee_id] 634 | 635 | # compute the weights for coloring 636 | if colour_nodes_by_selftime: 637 | weights = [] 638 | for function in compat_itervalues(self.functions): 639 | try: 640 | weights.append(function[TIME_RATIO]) 641 | except UndefinedEvent: 642 | pass 643 | max_ratio = max(weights or [1]) 644 | 645 | # apply rescaled weights for coloring 646 | for function in compat_itervalues(self.functions): 647 | try: 648 | function.weight = function[TIME_RATIO] / max_ratio 649 | except (ZeroDivisionError, UndefinedEvent): 650 | pass 651 | 652 | def dump(self): 653 | for function in compat_itervalues(self.functions): 654 | sys.stderr.write('Function %s:\n' % (function.name,)) 655 | self._dump_events(function.events) 656 | for call in compat_itervalues(function.calls): 657 | callee = self.functions[call.callee_id] 658 | sys.stderr.write(' Call %s:\n' % (callee.name,)) 659 | self._dump_events(call.events) 660 | for cycle in self.cycles: 661 | sys.stderr.write('Cycle:\n') 662 | self._dump_events(cycle.events) 663 | for function in cycle.functions: 664 | sys.stderr.write(' Function %s\n' % (function.name,)) 665 | 666 | def _dump_events(self, events): 667 | for event, value in compat_iteritems(events): 668 | sys.stderr.write(' %s: %s\n' % (event.name, event.format(value))) 669 | 670 | 671 | 672 | ######################################################################## 673 | # Parsers 674 | 675 | 676 | class Struct: 677 | """Masquerade a dictionary with a structure-like behavior.""" 678 | 679 | def __init__(self, attrs = None): 680 | if attrs is None: 681 | attrs = {} 682 | self.__dict__['_attrs'] = attrs 683 | 684 | def __getattr__(self, name): 685 | try: 686 | return self._attrs[name] 687 | except KeyError: 688 | raise AttributeError(name) 689 | 690 | def __setattr__(self, name, value): 691 | self._attrs[name] = value 692 | 693 | def __str__(self): 694 | return str(self._attrs) 695 | 696 | def __repr__(self): 697 | return repr(self._attrs) 698 | 699 | 700 | class ParseError(Exception): 701 | """Raised when parsing to signal mismatches.""" 702 | 703 | def __init__(self, msg, line): 704 | Exception.__init__(self) 705 | self.msg = msg 706 | # TODO: store more source line information 707 | self.line = line 708 | 709 | def __str__(self): 710 | return '%s: %r' % (self.msg, self.line) 711 | 712 | 713 | class Parser: 714 | """Parser interface.""" 715 | 716 | stdinInput = True 717 | multipleInput = False 718 | 719 | def __init__(self): 720 | pass 721 | 722 | def parse(self): 723 | raise NotImplementedError 724 | 725 | 726 | class PstatsParser: 727 | """Parser python profiling statistics saved with te pstats module.""" 728 | 729 | stdinInput = False 730 | multipleInput = True 731 | 732 | def __init__(self, *filename): 733 | import pstats 734 | try: 735 | self.stats = pstats.Stats(*filename) 736 | except ValueError: 737 | if PYTHON_3: 738 | sys.stderr.write('error: failed to load %s\n' % ', '.join(filename)) 739 | sys.exit(1) 740 | import hotshot.stats 741 | self.stats = hotshot.stats.load(filename[0]) 742 | self.profile = Profile() 743 | self.function_ids = {} 744 | 745 | def get_function_name(self, key): 746 | filename, line, name = key 747 | module = os.path.splitext(filename)[0] 748 | module = os.path.basename(module) 749 | return "%s:%d:%s" % (module, line, name) 750 | 751 | def get_function(self, key): 752 | try: 753 | id = self.function_ids[key] 754 | except KeyError: 755 | id = len(self.function_ids) 756 | name = self.get_function_name(key) 757 | function = Function(id, name) 758 | function.filename = key[0] 759 | self.profile.functions[id] = function 760 | self.function_ids[key] = id 761 | else: 762 | function = self.profile.functions[id] 763 | return function 764 | 765 | def parse(self): 766 | self.profile[TIME] = 0.0 767 | self.profile[TOTAL_TIME] = self.stats.total_tt 768 | for fn, (cc, nc, tt, ct, callers) in compat_iteritems(self.stats.stats): 769 | callee = self.get_function(fn) 770 | callee.called = nc 771 | callee[TOTAL_TIME] = ct 772 | callee[TIME] = tt 773 | self.profile[TIME] += tt 774 | self.profile[TOTAL_TIME] = max(self.profile[TOTAL_TIME], ct) 775 | for fn, value in compat_iteritems(callers): 776 | caller = self.get_function(fn) 777 | call = Call(callee.id) 778 | if isinstance(value, tuple): 779 | for i in xrange(0, len(value), 4): 780 | nc, cc, tt, ct = value[i:i+4] 781 | if CALLS in call: 782 | call[CALLS] += cc 783 | else: 784 | call[CALLS] = cc 785 | 786 | if TOTAL_TIME in call: 787 | call[TOTAL_TIME] += ct 788 | else: 789 | call[TOTAL_TIME] = ct 790 | 791 | else: 792 | call[CALLS] = value 793 | call[TOTAL_TIME] = ratio(value, nc)*ct 794 | 795 | caller.add_call(call) 796 | 797 | # Compute derived events 798 | self.profile.validate() 799 | self.profile.ratio(TIME_RATIO, TIME) 800 | self.profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME) 801 | 802 | return self.profile 803 | 804 | 805 | ######################################################################## 806 | # Output 807 | 808 | 809 | class Theme: 810 | 811 | def __init__(self, 812 | bgcolor = (0.0, 0.0, 1.0), 813 | mincolor = (0.0, 0.0, 0.0), 814 | maxcolor = (0.0, 0.0, 1.0), 815 | fontname = "Arial", 816 | fontcolor = "white", 817 | nodestyle = "filled", 818 | minfontsize = 10.0, 819 | maxfontsize = 10.0, 820 | minpenwidth = 0.5, 821 | maxpenwidth = 4.0, 822 | gamma = 2.2, 823 | skew = 1.0): 824 | self.bgcolor = bgcolor 825 | self.mincolor = mincolor 826 | self.maxcolor = maxcolor 827 | self.fontname = fontname 828 | self.fontcolor = fontcolor 829 | self.nodestyle = nodestyle 830 | self.minfontsize = minfontsize 831 | self.maxfontsize = maxfontsize 832 | self.minpenwidth = minpenwidth 833 | self.maxpenwidth = maxpenwidth 834 | self.gamma = gamma 835 | self.skew = skew 836 | 837 | def graph_bgcolor(self): 838 | return self.hsl_to_rgb(*self.bgcolor) 839 | 840 | def graph_fontname(self): 841 | return self.fontname 842 | 843 | def graph_fontcolor(self): 844 | return self.fontcolor 845 | 846 | def graph_fontsize(self): 847 | return self.minfontsize 848 | 849 | def node_bgcolor(self, weight): 850 | return self.color(weight) 851 | 852 | def node_fgcolor(self, weight): 853 | if self.nodestyle == "filled": 854 | return self.graph_bgcolor() 855 | else: 856 | return self.color(weight) 857 | 858 | def node_fontsize(self, weight): 859 | return self.fontsize(weight) 860 | 861 | def node_style(self): 862 | return self.nodestyle 863 | 864 | def edge_color(self, weight): 865 | return self.color(weight) 866 | 867 | def edge_fontsize(self, weight): 868 | return self.fontsize(weight) 869 | 870 | def edge_penwidth(self, weight): 871 | return max(weight*self.maxpenwidth, self.minpenwidth) 872 | 873 | def edge_arrowsize(self, weight): 874 | return 0.5 * math.sqrt(self.edge_penwidth(weight)) 875 | 876 | def fontsize(self, weight): 877 | return max(weight**2 * self.maxfontsize, self.minfontsize) 878 | 879 | def color(self, weight): 880 | weight = min(max(weight, 0.0), 1.0) 881 | 882 | hmin, smin, lmin = self.mincolor 883 | hmax, smax, lmax = self.maxcolor 884 | 885 | if self.skew < 0: 886 | raise ValueError("Skew must be greater than 0") 887 | elif self.skew == 1.0: 888 | h = hmin + weight*(hmax - hmin) 889 | s = smin + weight*(smax - smin) 890 | l = lmin + weight*(lmax - lmin) 891 | else: 892 | base = self.skew 893 | h = hmin + ((hmax-hmin)*(-1.0 + (base ** weight)) / (base - 1.0)) 894 | s = smin + ((smax-smin)*(-1.0 + (base ** weight)) / (base - 1.0)) 895 | l = lmin + ((lmax-lmin)*(-1.0 + (base ** weight)) / (base - 1.0)) 896 | 897 | return self.hsl_to_rgb(h, s, l) 898 | 899 | def hsl_to_rgb(self, h, s, l): 900 | """Convert a color from HSL color-model to RGB. 901 | 902 | See also: 903 | - http://www.w3.org/TR/css3-color/#hsl-color 904 | """ 905 | 906 | h = h % 1.0 907 | s = min(max(s, 0.0), 1.0) 908 | l = min(max(l, 0.0), 1.0) 909 | 910 | if l <= 0.5: 911 | m2 = l*(s + 1.0) 912 | else: 913 | m2 = l + s - l*s 914 | m1 = l*2.0 - m2 915 | r = self._hue_to_rgb(m1, m2, h + 1.0/3.0) 916 | g = self._hue_to_rgb(m1, m2, h) 917 | b = self._hue_to_rgb(m1, m2, h - 1.0/3.0) 918 | 919 | # Apply gamma correction 920 | r **= self.gamma 921 | g **= self.gamma 922 | b **= self.gamma 923 | 924 | return (r, g, b) 925 | 926 | def _hue_to_rgb(self, m1, m2, h): 927 | if h < 0.0: 928 | h += 1.0 929 | elif h > 1.0: 930 | h -= 1.0 931 | if h*6 < 1.0: 932 | return m1 + (m2 - m1)*h*6.0 933 | elif h*2 < 1.0: 934 | return m2 935 | elif h*3 < 2.0: 936 | return m1 + (m2 - m1)*(2.0/3.0 - h)*6.0 937 | else: 938 | return m1 939 | 940 | 941 | TEMPERATURE_COLORMAP = Theme( 942 | mincolor = (2.0/3.0, 0.80, 0.25), # dark blue 943 | maxcolor = (0.0, 1.0, 0.5), # satured red 944 | gamma = 1.0 945 | ) 946 | 947 | PINK_COLORMAP = Theme( 948 | mincolor = (0.0, 1.0, 0.90), # pink 949 | maxcolor = (0.0, 1.0, 0.5), # satured red 950 | ) 951 | 952 | GRAY_COLORMAP = Theme( 953 | mincolor = (0.0, 0.0, 0.85), # light gray 954 | maxcolor = (0.0, 0.0, 0.0), # black 955 | ) 956 | 957 | BW_COLORMAP = Theme( 958 | minfontsize = 8.0, 959 | maxfontsize = 24.0, 960 | mincolor = (0.0, 0.0, 0.0), # black 961 | maxcolor = (0.0, 0.0, 0.0), # black 962 | minpenwidth = 0.1, 963 | maxpenwidth = 8.0, 964 | ) 965 | 966 | PRINT_COLORMAP = Theme( 967 | minfontsize = 18.0, 968 | maxfontsize = 30.0, 969 | fontcolor = "black", 970 | nodestyle = "solid", 971 | mincolor = (0.0, 0.0, 0.0), # black 972 | maxcolor = (0.0, 0.0, 0.0), # black 973 | minpenwidth = 0.1, 974 | maxpenwidth = 8.0, 975 | ) 976 | 977 | 978 | themes = { 979 | "color": TEMPERATURE_COLORMAP, 980 | "pink": PINK_COLORMAP, 981 | "gray": GRAY_COLORMAP, 982 | "bw": BW_COLORMAP, 983 | "print": PRINT_COLORMAP, 984 | } 985 | 986 | 987 | def sorted_iteritems(d): 988 | # Used mostly for result reproducibility (while testing.) 989 | keys = compat_keys(d) 990 | keys.sort() 991 | for key in keys: 992 | value = d[key] 993 | yield key, value 994 | 995 | 996 | class DotWriter: 997 | """Writer for the DOT language. 998 | 999 | See also: 1000 | - "The DOT Language" specification 1001 | http://www.graphviz.org/doc/info/lang.html 1002 | """ 1003 | 1004 | strip = False 1005 | wrap = False 1006 | 1007 | def __init__(self, fp): 1008 | self.fp = fp 1009 | 1010 | def wrap_function_name(self, name): 1011 | """Split the function name on multiple lines.""" 1012 | 1013 | if len(name) > 32: 1014 | ratio = 2.0/3.0 1015 | height = max(int(len(name)/(1.0 - ratio) + 0.5), 1) 1016 | width = max(len(name)/height, 32) 1017 | # TODO: break lines in symbols 1018 | name = textwrap.fill(name, width, break_long_words=False) 1019 | 1020 | # Take away spaces 1021 | name = name.replace(", ", ",") 1022 | name = name.replace("> >", ">>") 1023 | name = name.replace("> >", ">>") # catch consecutive 1024 | 1025 | return name 1026 | 1027 | def _name_label(self, function): 1028 | if self.strip: 1029 | function_name = function.stripped_name() 1030 | else: 1031 | function_name = function.name 1032 | if self.wrap: 1033 | function_name = self.wrap_function_name(function_name) 1034 | return function_name 1035 | 1036 | def _total_time_ratio_label(self, function, total_time): 1037 | ratio = function[TOTAL_TIME_RATIO] 1038 | return "{ratio:.2%} ({time:.0f}ms)".format( 1039 | ratio=ratio, 1040 | time=ratio*total_time*1000, 1041 | ) 1042 | 1043 | def _time_ratio_label(self, function, total_time): 1044 | ratio = function[TIME_RATIO] 1045 | return "{ratio:.2%} ({time:.1f}ms)".format( 1046 | ratio=ratio, 1047 | time=ratio*total_time*1000, 1048 | ) 1049 | 1050 | def _called_label(self, function): 1051 | return u"{called}{sign}".format(called=function.called, sign=MULTIPLICATION_SIGN) 1052 | 1053 | def graph(self, profile, theme): 1054 | self.begin_graph() 1055 | 1056 | fontname = theme.graph_fontname() 1057 | fontcolor = theme.graph_fontcolor() 1058 | nodestyle = theme.node_style() 1059 | 1060 | b64style = base64.b64encode(b'text { font-size: 10px; }').decode() 1061 | self.attr('graph', fontname=fontname, ranksep=0.25, nodesep=0.125, stylesheet='data:text/css;charset=utf-8;base64,{0}'.format(b64style)) 1062 | self.attr('node', fontname=fontname, shape="box", style=nodestyle, fontcolor=fontcolor, width=0, height=0) 1063 | self.attr('edge', fontname=fontname) 1064 | 1065 | total_time = profile.events[TIME] 1066 | 1067 | for _, function in sorted_iteritems(profile.functions): 1068 | labels = [] 1069 | 1070 | labels.append(self._name_label(function)) 1071 | labels.append(self._total_time_ratio_label(function, total_time)) 1072 | labels.append(self._time_ratio_label(function, total_time)) 1073 | labels.append(self._called_label(function)) 1074 | 1075 | if function.weight is not None: 1076 | weight = function.weight 1077 | else: 1078 | weight = 0.0 1079 | 1080 | label = '\n'.join(labels) 1081 | self.node(function.id, 1082 | label = label, 1083 | color = self.color(theme.node_bgcolor(weight)), 1084 | fontcolor = self.color(theme.node_fgcolor(weight)), 1085 | fontsize = "%.2f" % theme.node_fontsize(weight), 1086 | tooltip = function.filename, 1087 | ) 1088 | 1089 | for _, call in sorted_iteritems(function.calls): 1090 | callee = profile.functions[call.callee_id] 1091 | 1092 | labels = [] 1093 | for event in [TOTAL_TIME_RATIO, CALLS]: 1094 | if event in call.events: 1095 | label = event.format(call[event]) 1096 | labels.append(label) 1097 | 1098 | if call.weight is not None: 1099 | weight = call.weight 1100 | elif callee.weight is not None: 1101 | weight = callee.weight 1102 | else: 1103 | weight = 0.0 1104 | 1105 | label = '\n'.join(labels) 1106 | 1107 | self.edge(function.id, call.callee_id, 1108 | label = label, 1109 | color = self.color(theme.edge_color(weight)), 1110 | fontcolor = self.color(theme.edge_color(weight)), 1111 | fontsize = "%.2f" % theme.edge_fontsize(weight), 1112 | penwidth = "%.2f" % theme.edge_penwidth(weight), 1113 | labeldistance = "%.2f" % theme.edge_penwidth(weight), 1114 | arrowsize = "%.2f" % theme.edge_arrowsize(weight), 1115 | ) 1116 | 1117 | self.end_graph() 1118 | 1119 | def begin_graph(self): 1120 | self.write('digraph {\n') 1121 | 1122 | def end_graph(self): 1123 | self.write('}\n') 1124 | 1125 | def attr(self, what, **attrs): 1126 | self.write("\t") 1127 | self.write(what) 1128 | self.attr_list(attrs) 1129 | self.write(";\n") 1130 | 1131 | def node(self, node, **attrs): 1132 | self.write("\t") 1133 | self.id(node) 1134 | self.attr_list(attrs) 1135 | self.write(";\n") 1136 | 1137 | def edge(self, src, dst, **attrs): 1138 | self.write("\t") 1139 | self.id(src) 1140 | self.write(" -> ") 1141 | self.id(dst) 1142 | self.attr_list(attrs) 1143 | self.write(";\n") 1144 | 1145 | def attr_list(self, attrs): 1146 | if not attrs: 1147 | return 1148 | self.write(' [') 1149 | first = True 1150 | for name, value in sorted_iteritems(attrs): 1151 | if value is None: 1152 | continue 1153 | if first: 1154 | first = False 1155 | else: 1156 | self.write(", ") 1157 | self.id(name) 1158 | self.write('=') 1159 | self.id(value) 1160 | self.write(']') 1161 | 1162 | def id(self, id): 1163 | if isinstance(id, (int, float)): 1164 | s = str(id) 1165 | elif isinstance(id, basestring): 1166 | if id.isalnum() and not id.startswith('0x'): 1167 | s = id 1168 | else: 1169 | s = self.escape(id) 1170 | else: 1171 | raise TypeError 1172 | self.write(s) 1173 | 1174 | def color(self, rgb): 1175 | r, g, b = rgb 1176 | 1177 | def float2int(f): 1178 | if f <= 0.0: 1179 | return 0 1180 | if f >= 1.0: 1181 | return 255 1182 | return int(255.0*f + 0.5) 1183 | 1184 | return "#" + "".join(["%02x" % float2int(c) for c in (r, g, b)]) 1185 | 1186 | def escape(self, s): 1187 | if not PYTHON_3: 1188 | s = s.encode('utf-8') 1189 | s = s.replace('\\', r'\\') 1190 | s = s.replace('\n', r'\n') 1191 | s = s.replace('\t', r'\t') 1192 | s = s.replace('"', r'\"') 1193 | return '"' + s + '"' 1194 | 1195 | def write(self, s): 1196 | self.fp.write(s) 1197 | 1198 | 1199 | ######################################################################## 1200 | # Main program 1201 | 1202 | 1203 | def naturalJoin(values): 1204 | if len(values) >= 2: 1205 | return ', '.join(values[:-1]) + ' or ' + values[-1] 1206 | 1207 | else: 1208 | return ''.join(values) 1209 | 1210 | 1211 | def parse_options(): 1212 | optparser = optparse.OptionParser( 1213 | usage="\n\t%prog [options] [file] ...") 1214 | optparser.add_option( 1215 | '-o', '--output', metavar='FILE', 1216 | type="string", dest="output", 1217 | help="output filename [stdout]") 1218 | optparser.add_option( 1219 | '-n', '--node-thres', metavar='PERCENTAGE', 1220 | type="float", dest="node_thres", default=0.5, 1221 | help="eliminate nodes below this threshold [default: %default]") 1222 | optparser.add_option( 1223 | '-e', '--edge-thres', metavar='PERCENTAGE', 1224 | type="float", dest="edge_thres", default=0.1, 1225 | help="eliminate edges below this threshold [default: %default]") 1226 | optparser.add_option( 1227 | '-c', '--colormap', 1228 | type="choice", choices=('color', 'pink', 'gray', 'bw', 'print'), 1229 | dest="theme", default="color", 1230 | help="color map: color, pink, gray, bw, or print [default: %default]") 1231 | optparser.add_option( 1232 | '-s', '--strip', 1233 | action="store_true", 1234 | dest="strip", default=False, 1235 | help="strip function parameters, template parameters, and const modifiers from demangled C++ function names") 1236 | optparser.add_option( 1237 | '--colour-nodes-by-selftime', 1238 | action="store_true", 1239 | dest="colour_nodes_by_selftime", default=False, 1240 | help="colour nodes by self time, rather than by total time (sum of self and descendants)") 1241 | optparser.add_option( 1242 | '-w', '--wrap', 1243 | action="store_true", 1244 | dest="wrap", default=False, 1245 | help="wrap function names") 1246 | # add option to create subtree or show paths 1247 | optparser.add_option( 1248 | '-z', '--root', 1249 | type="string", 1250 | dest="root", default="", 1251 | help="prune call graph to show only descendants of specified root function") 1252 | optparser.add_option( 1253 | '-l', '--leaf', 1254 | type="string", 1255 | dest="leaf", default="", 1256 | help="prune call graph to show only ancestors of specified leaf function") 1257 | # add a new option to control skew of the colorization curve 1258 | optparser.add_option( 1259 | '--skew', 1260 | type="float", dest="theme_skew", default=1.0, 1261 | help="skew the colorization curve. Values < 1.0 give more variety to lower percentages. Values > 1.0 give less variety to lower percentages") 1262 | (options, args) = optparser.parse_args(sys.argv[1:]) 1263 | return options, args 1264 | 1265 | 1266 | def main(): 1267 | """Main program.""" 1268 | options, args = parse_options() 1269 | 1270 | try: 1271 | theme = themes[options.theme] 1272 | except KeyError: 1273 | optparser.error('invalid colormap \'%s\'' % options.theme) 1274 | 1275 | # set skew on the theme now that it has been picked. 1276 | if options.theme_skew: 1277 | theme.skew = options.theme_skew 1278 | 1279 | if not args: 1280 | optparser.error('at least a file must be specified for pstats format') 1281 | parser = PstatsParser(*args) 1282 | 1283 | profile = parser.parse() 1284 | 1285 | if options.output is None: 1286 | output = sys.stdout 1287 | else: 1288 | if PYTHON_3: 1289 | output = open(options.output, 'wt', encoding='UTF-8') 1290 | else: 1291 | output = open(options.output, 'wt') 1292 | 1293 | dot = DotWriter(output) 1294 | dot.strip = options.strip 1295 | dot.wrap = options.wrap 1296 | 1297 | profile = profile 1298 | profile.prune(options.node_thres/100.0, options.edge_thres/100.0, options.colour_nodes_by_selftime) 1299 | 1300 | if options.root: 1301 | rootId = profile.getFunctionId(options.root) 1302 | if not rootId: 1303 | sys.stderr.write('root node ' + options.root + ' not found (might already be pruned : try -e0 -n0 flags)\n') 1304 | sys.exit(1) 1305 | profile.prune_root(rootId) 1306 | if options.leaf: 1307 | leafId = profile.getFunctionId(options.leaf) 1308 | if not leafId: 1309 | sys.stderr.write('leaf node ' + options.leaf + ' not found (maybe already pruned : try -e0 -n0 flags)\n') 1310 | sys.exit(1) 1311 | profile.prune_leaf(leafId) 1312 | 1313 | dot.graph(profile, theme) 1314 | 1315 | 1316 | if __name__ == '__main__': 1317 | main() 1318 | --------------------------------------------------------------------------------