├── UNLICENSE ├── flamegraph ├── __init__.py ├── __main__.py └── flamegraph.py ├── docs ├── ycanta-pdf.png ├── attic-create.png └── ycanta-full.png ├── example.py ├── setup.py ├── .gitignore ├── LICENSE └── README.rst /UNLICENSE: -------------------------------------------------------------------------------- 1 | LICENSE -------------------------------------------------------------------------------- /flamegraph/__init__.py: -------------------------------------------------------------------------------- 1 | from .flamegraph import start_profile_thread, ProfileThread 2 | -------------------------------------------------------------------------------- /docs/ycanta-pdf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evanhempel/python-flamegraph/HEAD/docs/ycanta-pdf.png -------------------------------------------------------------------------------- /docs/attic-create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evanhempel/python-flamegraph/HEAD/docs/attic-create.png -------------------------------------------------------------------------------- /docs/ycanta-full.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evanhempel/python-flamegraph/HEAD/docs/ycanta-full.png -------------------------------------------------------------------------------- /flamegraph/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from flamegraph import flamegraph 3 | if __name__ == '__main__': 4 | flamegraph.main() 5 | -------------------------------------------------------------------------------- /example.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example usage of flamegraph. 3 | 4 | To view a flamegraph run these commands: 5 | $ python example.py 6 | $ flamegraph.pl perf.log > perf.svg 7 | $ inkview perf.svg 8 | """ 9 | 10 | import time 11 | import sys 12 | import flamegraph 13 | 14 | def foo(): 15 | time.sleep(.1) 16 | bar() 17 | 18 | def bar(): 19 | time.sleep(.05) 20 | 21 | if __name__ == "__main__": 22 | flamegraph.start_profile_thread(fd=open("./perf.log", "w")) 23 | 24 | N = 10 25 | for x in xrange(N): 26 | print "{}/{}".format(x, N) 27 | foo() 28 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | setup( 3 | name = "flamegraph", 4 | version = "0.1", 5 | packages = ['flamegraph'], 6 | 7 | author = 'Evan Hempel', 8 | author_email = 'evanhempel@evanhempel.com', 9 | description = 'Statistical profiler which outputs in format suitable for FlameGraph', 10 | long_description = open('README.rst').read(), 11 | license = 'UNLICENSE', 12 | keywords = 'profiler flamegraph', 13 | url = 'https://github.com/evanhempel/python-flamegraph', 14 | classifiers=[ 15 | 'Development Status :: 3 - Alpha', 16 | 'Environment :: Console', 17 | 'Intended Audience :: Developers', 18 | 'License :: Public Domain', 19 | 'Programming Language :: Python', 20 | 'Topic :: Software Development :: Debuggers', 21 | ] 22 | ) 23 | 24 | 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # PyInstaller 26 | # Usually these files are written by a python script from a template 27 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 28 | *.manifest 29 | *.spec 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | 43 | # Translations 44 | *.mo 45 | *.pot 46 | 47 | # Django stuff: 48 | *.log 49 | 50 | # Sphinx documentation 51 | docs/_build/ 52 | 53 | # PyBuilder 54 | target/ 55 | 56 | # Vim files 57 | *~ 58 | 59 | # IDEA & PyCharm 60 | .idea 61 | 62 | perf.log 63 | perf.svg -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | 26 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | A simple statistical profiler which outputs in format suitable for FlameGraph_. 2 | 3 | INSTALL: 4 | -------- 5 | 6 | Simply run:: 7 | 8 | pip install git+https://github.com/evanhempel/python-flamegraph.git 9 | 10 | USAGE: 11 | ------ 12 | 13 | Run your script under the profiler:: 14 | 15 | python -m flamegraph -o perf.log myscript.py --your-script args here 16 | 17 | Or, run the profiler from your script:: 18 | 19 | flamegraph.start_profile_thread(fd=open("./perf.log", "w")) 20 | 21 | Run Brendan Gregg's FlameGraph_ tool against the output:: 22 | 23 | flamegraph.pl --title "MyScript CPU" perf.log > perf.svg 24 | 25 | Enjoy the output: 26 | 27 | .. image:: docs/attic-create.png 28 | :alt: Attic create flame graph 29 | :width: 679 30 | :height: 781 31 | :align: center 32 | 33 | **Filtering** 34 | 35 | Sometimes you may want to exclude a method 36 | (for example in a server the method that waits for a new request) 37 | or you may want to profile only a subset of your code 38 | (a particular method and its children which are performance critical). 39 | 40 | Filtering can be done by passing a python regular expression to the 41 | ``-f`` or ``--filter`` command line option 42 | which will restrict output to only those lines which match. 43 | Filtering is done against the entire line so you can filter by 44 | function name, thread name, both, or even by 45 | more complex filters such as function ABC calls DEF (``ABC.*DEF``). 46 | 47 | Alternatively since the output is stackframes each on a line by itself, 48 | this can simply be done with a simple grep filter.:: 49 | 50 | Exclude: 51 | 52 | grep -v waiting_method perf.log > removed_waiting.log 53 | 54 | Include: 55 | 56 | grep function_name perf.log > filtered.log 57 | 58 | Then run the flamegraph.pl script against the filtered file. 59 | 60 | .. figure:: docs/ycanta-full.png 61 | :alt: yCanta webapp full profile of PDF export 62 | :align: center 63 | 64 | Full profile output of yCanta_ webapp PDF export. Most time is 65 | spent in wait state and graph is not very helpful. 66 | 67 | .. figure:: docs/ycanta-pdf.png 68 | :alt: yCanta webapp filtered for PDF export format function. 69 | :align: center 70 | 71 | Filtered profile output of yCanta_ webapp PDF export. Filtering was on the 72 | pdf format function so time spent in wait state has been excluded and the 73 | graph is now helpful. 74 | 75 | .. _FlameGraph: http://www.brendangregg.com/flamegraphs.html 76 | 77 | .. _yCanta: https://github.com/yCanta/yCanta 78 | -------------------------------------------------------------------------------- /flamegraph/flamegraph.py: -------------------------------------------------------------------------------- 1 | import re 2 | import sys 3 | import time 4 | import os.path 5 | import argparse 6 | import threading 7 | import traceback 8 | import collections 9 | import atexit 10 | import functools 11 | 12 | def get_thread_name(ident): 13 | for th in threading.enumerate(): 14 | if th.ident == ident: 15 | return th.getName() 16 | return str(ident) # couldn't find, return something useful anyways 17 | 18 | def default_format_entry(threadname, fname, line, fun, fmt='%(threadname)s`%(fun)s'): 19 | return fmt % locals() 20 | 21 | def create_flamegraph_entry(thread_id, frame, format_entry, collapse_recursion=False): 22 | threadname = get_thread_name(thread_id) 23 | 24 | # [1:] to skip first frame which is in this program 25 | if collapse_recursion: 26 | ret = [] 27 | last = None 28 | for fn, ln, fun, text in traceback.extract_stack(frame)[1:]: 29 | if last != fun: 30 | ret.append(format_entry(threadname, fn, ln, fun)) 31 | last = fun 32 | return ';'.join(ret) 33 | 34 | return ';'.join(format_entry(threadname, fn, ln, fun) 35 | for fn, ln, fun, text in traceback.extract_stack(frame)[1:]) 36 | 37 | class ProfileThread(threading.Thread): 38 | def __init__(self, fd, interval, filter, format_entry, collapse_recursion=False): 39 | threading.Thread.__init__(self, name="FlameGraph Thread") 40 | self.daemon = True 41 | 42 | self._lock = threading.Lock() 43 | self._fd = fd 44 | self._written = False 45 | self._interval = interval 46 | self._format_entry = format_entry 47 | self._collapse_recursion = collapse_recursion 48 | if filter is not None: 49 | self._filter = re.compile(filter) 50 | else: 51 | self._filter = None 52 | 53 | 54 | self._stats = collections.defaultdict(int) 55 | 56 | self._keeprunning = True 57 | self._stopevent = threading.Event() 58 | 59 | atexit.register(self.stop) 60 | 61 | def run(self): 62 | my_thread = threading.current_thread().ident 63 | while self._keeprunning: 64 | for thread_id, frame in sys._current_frames().items(): 65 | if thread_id == my_thread: 66 | continue 67 | 68 | entry = create_flamegraph_entry(thread_id, frame, self._format_entry, self._collapse_recursion) 69 | if self._filter is None or self._filter.search(entry): 70 | with self._lock: 71 | self._stats[entry] += 1 72 | 73 | self._stopevent.wait(self._interval) # basically a sleep for x seconds unless someone asked to stop 74 | 75 | self._write_results() 76 | 77 | def _write_results(self): 78 | with self._lock: 79 | if self._written: 80 | return 81 | self._written = True 82 | for key in sorted(self._stats.keys()): 83 | self._fd.write('%s %d\n' % (key, self._stats[key])) 84 | self._fd.close() 85 | 86 | def num_frames(self, unique=False): 87 | if unique: 88 | return len(self._stats) 89 | else: 90 | return sum(self._stats.values()) 91 | 92 | def stop(self): 93 | self._keeprunning = False 94 | self._stopevent.set() 95 | self._write_results() 96 | # Wait for the thread to actually stop. 97 | # Using atexit without this line can result in the interpreter shutting 98 | # down while the thread is alive, raising an exception. 99 | self.join() 100 | 101 | def start_profile_thread(fd, interval=0.001, filter=None, format_entry=default_format_entry, collapse_recursion=False): 102 | """Start a profiler thread.""" 103 | profile_thread = ProfileThread( 104 | fd=fd, 105 | interval=interval, 106 | filter=filter, 107 | format_entry=format_entry, 108 | collapse_recursion=collapse_recursion) 109 | profile_thread.start() 110 | return profile_thread 111 | 112 | def main(): 113 | parser = argparse.ArgumentParser(prog='python -m flamegraph', description="Sample python stack frames for use with FlameGraph") 114 | parser.add_argument('script_file', metavar='script.py', type=str, 115 | help='Script to profile') 116 | parser.add_argument('script_args', metavar='[arguments...]', type=str, nargs=argparse.REMAINDER, 117 | help='Arguments for script') 118 | parser.add_argument('-o', '--output', nargs='?', type=argparse.FileType('w'), default=sys.stderr, 119 | help='Save stats to file. If not specified default is to stderr') 120 | parser.add_argument('-i', '--interval', type=float, nargs='?', default=0.001, 121 | help='Interval in seconds for collection of stackframes (default: %(default)ss)') 122 | parser.add_argument('-c', '--collapse-recursion', action='store_true', 123 | help='Collapse simple recursion (function calls itself) into one stack frame in output') 124 | parser.add_argument('-f', '--filter', type=str, nargs='?', default=None, 125 | help='Regular expression to filter which stack frames are profiled. The ' 126 | 'regular expression is run against each entire line of output so you can ' 127 | 'filter by function or thread or both.') 128 | parser.add_argument('-F', '--format', type=str, nargs='?', default='%(threadname)s`%(fun)s', 129 | help='Format-string (old-style) for encoding each stack frame into text.' 130 | ' May include: "threadname", "fname", "fun" and "line"') 131 | 132 | args = parser.parse_args() 133 | print(args) 134 | 135 | format_entry = functools.partial(default_format_entry, fmt=args.format) 136 | thread = ProfileThread(args.output, args.interval, args.filter, format_entry, args.collapse_recursion) 137 | 138 | if not os.path.isfile(args.script_file): 139 | parser.error('Script file does not exist: ' + args.script_file) 140 | 141 | sys.argv = [args.script_file] + args.script_args 142 | sys.path.insert(0, os.path.dirname(args.script_file)) 143 | script_compiled = compile(open(args.script_file, 'rb').read(), args.script_file, 'exec') 144 | script_globals = {'__name__': '__main__', '__file__': args.script_file, '__package__': None} 145 | 146 | start_time = time.clock() 147 | thread.start() 148 | 149 | try: 150 | # exec docs say globals and locals should be same dictionary else treated as class context 151 | exec(script_compiled, script_globals, script_globals) 152 | finally: 153 | thread.stop() 154 | thread.join() 155 | print('Elapsed Time: %2.2f seconds. Collected %d stack frames (%d unique)' 156 | % (time.clock() - start_time, thread.num_frames(), thread.num_frames(unique=True))) 157 | 158 | if __name__ == '__main__': 159 | main() 160 | --------------------------------------------------------------------------------