├── .clang-format ├── .github ├── CONTRIBUTING.md └── ISSUE_TEMPLATE.md ├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile.am ├── NEWS.org ├── README.md ├── autogen.sh ├── configure.ac ├── docs ├── conf.py ├── contributing.rst ├── faq.rst ├── generate-man.sh ├── index.rst ├── installation.rst ├── man.md ├── pyflame.man └── usage.rst ├── m4 ├── ax_append_compile_flags.m4 ├── ax_append_flag.m4 ├── ax_check_compile_flag.m4 └── ax_require_defined.m4 ├── runtests.sh ├── src ├── Makefile.am ├── aslr.cc ├── aslr.h ├── exc.h ├── frame.cc ├── frame.h ├── frob.cc ├── frob26.cc ├── frob34.cc ├── frob36.cc ├── namespace.cc ├── namespace.h ├── posix.cc ├── posix.h ├── prober.cc ├── prober.h ├── ptrace.cc ├── ptrace.h ├── pyflame.cc ├── pyfrob.cc ├── pyfrob.h ├── setns.h ├── symbol.cc ├── symbol.h ├── thread.cc └── thread.h ├── tests ├── dijkstra.py ├── exit_early.py ├── forker.py ├── sleep.sh ├── sleeper.py ├── sleeper_ユニコード.py ├── test_end_to_end.py ├── threaded_busy.py └── threaded_sleeper.py └── utils └── flame-chart-json /.clang-format: -------------------------------------------------------------------------------- 1 | BasedOnStyle: Google 2 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How To Contribute To Pyflame 2 | 3 | We welcome patches for Pyflame---some of the most interesting features have come 4 | from users of Pyflame. There are a few guidelines you should follow when 5 | submitting a pull request. 6 | 7 | For all pull requests, please make sure the test suite passes before submitting 8 | a pull request. You can run the test suite with `make check`. 9 | 10 | For C++ changes: 11 | 12 | * We ask you to stick to the [Google C++ Style 13 | Guide](http://google.github.io/styleguide/cppguide.html). 14 | * Run `clang-format` to reformat your code. This tool will automatically format 15 | the source files to put them into the correct style for Pyflame. 16 | 17 | For Python (i.e. test suite) changes: 18 | 19 | * Conform to [PEP-8](https://www.python.org/dev/peps/pep-0008/). 20 | * Format your code using [yapf](https://github.com/google/yapf). 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Make sure that these boxes are checked before submitting your issue: 2 | 3 | - [ ] Include the output of `pyflame -v` 4 | - [ ] Include the exact version of the Python interpreter you are profiling. 5 | - [ ] Include the exact text of any error messages. 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.a 2 | *.la 3 | *.lo 4 | *.o 5 | *.svg 6 | *.in 7 | *.log 8 | *.status 9 | *.m4 10 | *.swp 11 | *.deps/ 12 | *.cache/ 13 | Makefile 14 | __pycache__/ 15 | libtool 16 | src/pyflame 17 | build-aux/ 18 | configure 19 | config.h 20 | \.pytest_cache/ 21 | stamp-h1 22 | \.test_env/ 23 | libtool.m4 24 | lt*.m4 25 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | language: cpp 4 | compiler: gcc 5 | 6 | env: 7 | global: 8 | - MAKEOPTS=-j3 9 | matrix: 10 | - PYVERSION=python2.6 11 | - PYVERSION=python2.7 12 | - PYVERSION=python3.4 13 | - PYVERSION=python3.5 14 | - PYVERSION=python3.6 15 | 16 | addons: 17 | apt: 18 | sources: 19 | - sourceline: 'ppa:fkrull/deadsnakes' 20 | - autotools-dev 21 | - libtool 22 | - pkg-config 23 | 24 | install: 25 | - sudo sysctl kernel.yama.ptrace_scope=0 26 | - travis_retry sudo apt-get update 27 | - travis_retry sudo apt-get install --allow-unauthenticated "${PYVERSION}"{,-minimal,-dev} 28 | 29 | # Travis puts some other Python versions in /opt, so it's very important that we 30 | # use an explicit /usr/bin path when running the tests. 31 | script: 32 | - ./autogen.sh 33 | - ./configure --disable-silent-rules --enable-werror 34 | - make $MAKEOPTS 35 | - file ./src/pyflame 36 | - ./runtests.sh -v /usr/bin/"${PYVERSION}" 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = src 2 | EXTRA_DIST = autogen.sh LICENSE README.md 3 | ACLOCAL_AMFLAGS = -I m4 4 | 5 | man1_MANS = docs/pyflame.man 6 | 7 | .PHONY: clean-local 8 | clean-local: 9 | rm -f core.* pyflame 10 | 11 | .PHONY: check 12 | check: 13 | ./runtests.sh -v 14 | -------------------------------------------------------------------------------- /NEWS.org: -------------------------------------------------------------------------------- 1 | * NEWS (user visible changes) 2 | 3 | ** 1.6.6 (2018-05-02) 4 | 5 | - Add new -n/--no-line-numbers option (Igor Nikolaev) 6 | 7 | ** 1.6.5 (2018-04-23) 8 | 9 | - No user-visible changes, version bump to fix RPM packaging 10 | 11 | ** 1.6.4 (2018-04-18) 12 | 13 | - Change default sample rate from 1ms to 10ms (Evan Klitzke) 14 | - Improved handling of ptrace errors (Steven Karas) 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pyflame: A Ptracing Profiler For Python 2 | 3 | [![Build Status](https://api.travis-ci.org/uber/pyflame.svg?branch=master)](https://travis-ci.org/uber/pyflame) [![Docs Status](https://readthedocs.org/projects/pyflame/badge/?version=latest)](http://pyflame.readthedocs.io/en/latest/?badge=latest) [![COPR Status](https://copr.fedorainfracloud.org/coprs/eklitzke/pyflame/package/pyflame/status_image/last_build.png)](https://copr.fedorainfracloud.org/coprs/eklitzke/pyflame/) 4 | 5 | (This project is deprecated and not maintained.) 6 | 7 | Pyflame is a high performance profiling tool that 8 | generates [flame graphs](http://www.brendangregg.com/flamegraphs.html) for 9 | Python. Pyflame is implemented in C++, and uses the 10 | Linux [ptrace(2)](http://man7.org/linux/man-pages/man2/ptrace.2.html) system 11 | call to collect profiling information. It can take snapshots of the Python call 12 | stack without explicit instrumentation, meaning you can profile a program 13 | without modifying its source code. Pyflame is capable of profiling embedded 14 | Python interpreters like [uWSGI](https://uwsgi-docs.readthedocs.io/en/latest/). 15 | It fully supports profiling multi-threaded Python programs. 16 | 17 | Pyflame usually introduces significantly less overhead than the builtin 18 | `profile` (or `cProfile`) modules, and emits richer profiling data. The 19 | profiling overhead is low enough that you can use it to profile live processes 20 | in production. 21 | 22 | **Full Documentation:** https://pyflame.readthedocs.io 23 | 24 | ![pyflame](https://cloud.githubusercontent.com/assets/2734/17949703/8ef7d08c-6a0b-11e6-8bbd-41f82086d862.png) 25 | 26 | ## Quickstart 27 | 28 | ### Building And Installing 29 | 30 | For Debian/Ubuntu, install the following: 31 | 32 | ```bash 33 | # Install build dependencies on Debian or Ubuntu. 34 | sudo apt-get install autoconf automake autotools-dev g++ pkg-config python-dev python3-dev libtool make 35 | ``` 36 | 37 | Once you have the build dependencies installed: 38 | 39 | ```bash 40 | ./autogen.sh 41 | ./configure 42 | make 43 | ``` 44 | 45 | The `make` command will produce an executable at `src/pyflame` that you can run 46 | and use. 47 | 48 | Optionally, if you have `virtualenv` installed, you can test the executable you 49 | produced using `make check`. 50 | 51 | ### Using Pyflame 52 | 53 | The full documentation for using Pyflame 54 | is [here](https://pyflame.readthedocs.io/en/latest/usage.html). But 55 | here's a quick guide: 56 | 57 | ```bash 58 | # Attach to PID 12345 and profile it for 1 second 59 | pyflame -p 12345 60 | 61 | # Attach to PID 768 and profile it for 5 seconds, sampling every 0.01 seconds 62 | pyflame -s 5 -r 0.01 -p 768 63 | 64 | # Run py.test against tests/, emitting sample data to prof.txt 65 | pyflame -o prof.txt -t py.test tests/ 66 | ``` 67 | 68 | In all of these cases you will get flame graph data on stdout (or to a file if 69 | you used `-o`). This data is in the format expected by `flamegraph.pl`, which 70 | you can find [here](https://github.com/brendangregg/FlameGraph). 71 | 72 | ## FAQ 73 | 74 | The full FAQ is [here](https://pyflame.readthedocs.io/en/latest/faq.html). 75 | 76 | ### What's The Deal With (idle) Time? 77 | 78 | Full 79 | answer 80 | [here](https://pyflame.readthedocs.io/en/latest/faq.html#what-is-idle-time). 81 | tl;dr: use the `-x` flag to suppress (idle) output. 82 | 83 | ### What About These Ptrace Errors? 84 | 85 | See [here](https://pyflame.readthedocs.io/en/latest/faq.html#what-are-these-ptrace-permissions-errors). 86 | 87 | ### How Do I Profile Threaded Applications? 88 | 89 | Use the `--threads` option. 90 | 91 | ### Is There A Way To Just Dump Stack Traces? 92 | 93 | Yes, use the `-d` option. 94 | -------------------------------------------------------------------------------- /autogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | autoreconf --install 3 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | AC_PREREQ([2.68]) 2 | AC_INIT([pyflame], [1.6.6], [evan@eklitzke.org]) 3 | AC_CONFIG_AUX_DIR([build-aux]) 4 | AC_CONFIG_HEADERS([src/config.h]) 5 | AC_CONFIG_MACRO_DIR([m4]) 6 | AC_CONFIG_SRCDIR([src/pyflame.cc]) 7 | 8 | # Detect errors with pkg-tools macro expansion 9 | m4_pattern_forbid([^PKG_]) 10 | 11 | # Fail early if the user tries to build for BSD/OS X 12 | AC_CANONICAL_HOST 13 | AS_CASE([$host_os], 14 | [linux-gnu*], [], 15 | [AC_MSG_ERROR([Pyflame can only be built for Linux hosts])]) 16 | 17 | enable_threads=no 18 | AS_IF([test x"$host_cpu" = xx86_64], 19 | [AC_MSG_NOTICE([x86-64 system, threads will be supported]) 20 | AC_DEFINE([ENABLE_THREADS], [1], [Threads are enabled.]) 21 | AC_DEFINE([USE_ELF64], [1], [Expect 64-bit ELF symbols.]) 22 | enable_threads=yes; 23 | ], 24 | [AC_MSG_NOTICE([Threading support will be disabled (only works on x86-64)]) 25 | AC_DEFINE([ENABLE_THREADS], [0], [Threads are enabled.]) 26 | AC_DEFINE([USE_ELF64], [0], [Expect 64-bit ELF symbols.]) 27 | ]) 28 | 29 | AC_DEFINE_UNQUOTED([HOST_CPU], ["$host_cpu"], [CPU of target architecture.]) 30 | 31 | AM_INIT_AUTOMAKE([dist-bzip2 foreign subdir-objects]) 32 | 33 | dnl make the compilation flags quiet unless V=1 is used 34 | m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) 35 | 36 | # Checks for programs. 37 | AC_PROG_CXX 38 | AC_PROG_AWK 39 | AC_PROG_CC 40 | AC_PROG_CPP 41 | AC_PROG_INSTALL 42 | AC_PROG_LN_S 43 | AC_PROG_MAKE_SET 44 | 45 | AM_PROG_AR 46 | 47 | LT_INIT 48 | 49 | # Ensure Linux kernel headers are present 50 | AC_CHECK_HEADERS([linux/ptrace.h], [], [AC_MSG_ERROR([Linux kernel headers missing])]) 51 | 52 | AC_LANG_PUSH([C++]) 53 | 54 | # Minimum C++ version is C++11 55 | AX_CHECK_COMPILE_FLAG(["-std=c++11"], 56 | [AX_APPEND_FLAG(["-std=c++11"], [CXXFLAGS])], 57 | [AC_MSG_ERROR([failed to detect C++11 support])]) 58 | 59 | # Yes please. 60 | AX_APPEND_COMPILE_FLAGS([-Wall], [CXXFLAGS]) 61 | 62 | # Turn warnings into errors 63 | AC_ARG_ENABLE([werror], 64 | [AS_HELP_STRING([--enable-werror], 65 | [Treat certain compiler warnings as errors (default is no)])], 66 | [enable_werror=$enableval], 67 | [enable_werror=no]) 68 | 69 | if test "x$enable_werror" = "xyes"; then 70 | AX_APPEND_COMPILE_FLAGS([-Werror], [CXXFLAGS]) 71 | fi 72 | 73 | # Checks for libraries. 74 | 75 | # Checks for header files. 76 | AC_CHECK_HEADERS([fcntl.h limits.h sys/time.h unistd.h]) 77 | 78 | # Checks for typedefs, structures, and compiler characteristics. 79 | AC_CHECK_HEADER_STDBOOL 80 | AC_C_INLINE 81 | AC_TYPE_PID_T 82 | AC_TYPE_SIZE_T 83 | AC_TYPE_SSIZE_T 84 | AC_TYPE_UINT16_T 85 | AC_TYPE_UINT8_T 86 | 87 | # Checks for library functions. 88 | AC_FUNC_FORK 89 | AC_FUNC_LSTAT_FOLLOWS_SLASHED_SYMLINK 90 | AC_FUNC_MMAP 91 | AC_CHECK_FUNCS([getpagesize memmove munmap strerror strtol strtoul]) 92 | 93 | AC_DEFINE_UNQUOTED( 94 | [PYFLAME_VERSION_STR], 95 | ["pyflame $PACKAGE_VERSION $host_os $host_cpu"], 96 | [A string containing build information.]) 97 | 98 | enable_py26=no 99 | PKG_CHECK_MODULES([PY26], [python2], [enable_py26="yes"], [AC_MSG_WARN([Building without Python 2.6/2.7 support])]) 100 | AM_CONDITIONAL([ENABLE_PY26], [test x"$enable_py26" = xyes]) 101 | AM_COND_IF([ENABLE_PY26], [AC_DEFINE([ENABLE_PY26], [1], [Python 2.6/2.7 will be enabled])]) 102 | 103 | enable_py34=no 104 | PKG_CHECK_MODULES([PY34], [python-3.4], [enable_py34="yes"], 105 | [PKG_CHECK_MODULES([PY34], [python-3.5], [enable_py34="yes"], [AC_MSG_WARN([Building without Python 3.4/3.5 support])])]) 106 | AM_CONDITIONAL([ENABLE_PY34], [test x"$enable_py34" = xyes]) 107 | AM_COND_IF([ENABLE_PY34], [AC_DEFINE([ENABLE_PY34], [1], [Python 3.4/3.5 will be enabled])]) 108 | 109 | enable_py36=no 110 | PKG_CHECK_MODULES([PY36], [python-3.6], [enable_py36="yes"], [AC_MSG_WARN([Building without Python 3.6 support])]) 111 | AM_CONDITIONAL([ENABLE_PY36], [test x"$enable_py36" = xyes]) 112 | AM_COND_IF([ENABLE_PY36], [AC_DEFINE([ENABLE_PY36], [1], [Python 3.6 will be enabled])]) 113 | 114 | AS_IF([test x"$enable_py26" = xyes -o x"$enable_py34" = xyes -o x"$enable_py36" = xyes], 115 | [AC_MSG_NOTICE([Found at least one copy of Python.h])], 116 | [AC_MSG_ERROR([Failed to find a supported Python.h])] 117 | ) 118 | 119 | AC_LANG_POP 120 | 121 | AC_CONFIG_FILES([Makefile src/Makefile]) 122 | AC_OUTPUT 123 | 124 | echo 125 | echo "Options used to compile and link:" 126 | echo 127 | echo " with threads = $enable_threads" 128 | echo " with Python 2.6/7 = $enable_py26" 129 | echo " with Python 3.4/5 = $enable_py34" 130 | echo " with Python 3.6+ = $enable_py36" 131 | echo 132 | echo " CXX = $CXX" 133 | echo " CXXFLAGS = $CXXFLAGS" 134 | echo 135 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Pyflame documentation build configuration file, created by 5 | # sphinx-quickstart on Fri Jul 14 14:13:03 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | # import os 21 | # import sys 22 | # sys.path.insert(0, os.path.abspath('.')) 23 | 24 | # -- General configuration ------------------------------------------------ 25 | 26 | # If your documentation needs a minimal Sphinx version, state it here. 27 | # 28 | # needs_sphinx = '1.0' 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = [] 34 | 35 | # Add any paths that contain templates here, relative to this directory. 36 | templates_path = ['_templates'] 37 | 38 | # The suffix(es) of source filenames. 39 | # You can specify multiple suffix as a list of string: 40 | # 41 | # source_suffix = ['.rst', '.md'] 42 | source_suffix = '.rst' 43 | 44 | # The master toctree document. 45 | master_doc = 'index' 46 | 47 | # General information about the project. 48 | project = 'Pyflame' 49 | copyright = '2017, Uber Technologies, Inc.' 50 | author = 'Evan Klitzke' 51 | 52 | # The version info for the project you're documenting, acts as replacement for 53 | # |version| and |release|, also used in various other places throughout the 54 | # built documents. 55 | # 56 | # The short X.Y version. 57 | version = '1.4' 58 | # The full version, including alpha/beta/rc tags. 59 | release = '1.4.0' 60 | 61 | # The language for content autogenerated by Sphinx. Refer to documentation 62 | # for a list of supported languages. 63 | # 64 | # This is also used if you do content translation via gettext catalogs. 65 | # Usually you set "language" from the command line for these cases. 66 | language = None 67 | 68 | # List of patterns, relative to source directory, that match files and 69 | # directories to ignore when looking for source files. 70 | # This patterns also effect to html_static_path and html_extra_path 71 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 72 | 73 | # The name of the Pygments (syntax highlighting) style to use. 74 | pygments_style = 'sphinx' 75 | 76 | # If true, `todo` and `todoList` produce output, else they produce nothing. 77 | todo_include_todos = False 78 | 79 | # -- Options for HTML output ---------------------------------------------- 80 | 81 | # The theme to use for HTML and HTML Help pages. See the documentation for 82 | # a list of builtin themes. 83 | # 84 | html_theme = 'default' 85 | 86 | # Theme options are theme-specific and customize the look and feel of a theme 87 | # further. For a list of options available for each theme, see the 88 | # documentation. 89 | # 90 | # html_theme_options = {} 91 | 92 | # Add any paths that contain custom static files (such as style sheets) here, 93 | # relative to this directory. They are copied after the builtin static files, 94 | # so a file named "default.css" will overwrite the builtin "default.css". 95 | html_static_path = ['_static'] 96 | 97 | # Custom sidebar templates, must be a dictionary that maps document names 98 | # to template names. 99 | # 100 | # This is required for the alabaster theme 101 | # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars 102 | html_sidebars = { 103 | '**': [ 104 | 'about.html', 105 | 'navigation.html', 106 | 'relations.html', # needs 'show_related': True theme option to display 107 | 'searchbox.html', 108 | 'donate.html', 109 | ] 110 | } 111 | 112 | # -- Options for HTMLHelp output ------------------------------------------ 113 | 114 | # Output file base name for HTML help builder. 115 | htmlhelp_basename = 'Pyflamedoc' 116 | 117 | # -- Options for LaTeX output --------------------------------------------- 118 | 119 | latex_elements = { 120 | # The paper size ('letterpaper' or 'a4paper'). 121 | # 122 | # 'papersize': 'letterpaper', 123 | 124 | # The font size ('10pt', '11pt' or '12pt'). 125 | # 126 | # 'pointsize': '10pt', 127 | 128 | # Additional stuff for the LaTeX preamble. 129 | # 130 | # 'preamble': '', 131 | 132 | # Latex figure (float) alignment 133 | # 134 | # 'figure_align': 'htbp', 135 | } 136 | 137 | # Grouping the document tree into LaTeX files. List of tuples 138 | # (source start file, target name, title, 139 | # author, documentclass [howto, manual, or own class]). 140 | latex_documents = [ 141 | (master_doc, 'Pyflame.tex', 'Pyflame Documentation', 'Evan Klitzke', 142 | 'manual'), 143 | ] 144 | 145 | # -- Options for manual page output --------------------------------------- 146 | 147 | # One entry per manual page. List of tuples 148 | # (source start file, name, description, authors, manual section). 149 | man_pages = [(master_doc, 'pyflame', 'Pyflame Documentation', [author], 1)] 150 | 151 | # -- Options for Texinfo output ------------------------------------------- 152 | 153 | # Grouping the document tree into Texinfo files. List of tuples 154 | # (source start file, target name, title, author, 155 | # dir menu entry, description, category) 156 | texinfo_documents = [ 157 | (master_doc, 'Pyflame', 'Pyflame Documentation', author, 'Pyflame', 158 | 'One line description of project.', 'Miscellaneous'), 159 | ] 160 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | We love getting pull requests and bug reports! This section outlines some ways 5 | you can contribute to Pyflame. 6 | 7 | Hacking 8 | ------- 9 | 10 | This section will explain the Pyflame code for people who are interested in 11 | contributing source code patches. 12 | 13 | A good way to start understanding the code is to read the two blog posts (linked 14 | on the main docs page) written by Evan Klitzke. They cover the basics about how 15 | Pyflame works, and have some helpful information about how the code is 16 | organized. 17 | 18 | The code style in Pyflame (mostly) conforms to the `Google C++ Style Guide 19 | `__. Additionally, all of the 20 | source code is formatted with `clang-format 21 | `__. There's a ``.clang-format`` 22 | file checked into the root of this repository which will make ``clang-format`` 23 | do the right thing. Different clang releases may format the source code slightly 24 | differently, as the formatting rules are updated within clang itself. Therefore 25 | you should eyeball the changes made when formatting, especially if you have an 26 | older version of clang. 27 | 28 | If you are changing any of the low-level C++ bits, and end up with a broken 29 | build, you may want to try by getting the following command working before 30 | testing with the full test suite: 31 | 32 | .. code:: bash 33 | 34 | # Sanity check Pyflame. 35 | pyflame -t python -c 'print(sum(i for i in range(100000)))' 36 | 37 | To run the full test suite locally: 38 | 39 | .. code:: bash 40 | 41 | # Run the Pyflame test suite. 42 | make check 43 | 44 | If you change any of the Python files in the ``tests/`` directory, please run 45 | your changes through `YAPF `__ before submitting 46 | a pull request. 47 | 48 | How Else Can I Help? 49 | -------------------- 50 | 51 | Patches are not the only way to contribute to Pyflame! Bug reports are very 52 | useful as well. If you file a bug, make sure you tell us the exact version of 53 | Python you're using, and how to reproduce the issue. 54 | -------------------------------------------------------------------------------- /docs/faq.rst: -------------------------------------------------------------------------------- 1 | FAQ 2 | === 3 | 4 | What Python Versions Are Supported? 5 | ----------------------------------- 6 | 7 | Python 2 is tested with Python 2.6 and 2.7. Earlier versions of Python 2 are 8 | likely to work as well, but have not been tested. 9 | 10 | Python 3 is tested with Python 3.4, 3.5, and 3.6. Python 3.6 introduces a new 11 | ABI for the ``PyCodeObject`` type, so Pyflame only supports the Python 3 12 | versions that header files were available for when Pyflame was compiled. 13 | 14 | It's possible for Pyflame to get confused about what Python version the target 15 | process is when profiling an embedded Python build, such as uWSGI. If you run 16 | into this issue, use the ``--abi`` option to force a particular Python ABI. 17 | 18 | What Is "(idle)" Time? 19 | ---------------------- 20 | 21 | In Python, only one thread can execute Python code at any one time, due to the 22 | Global Interpreter Lock, or GIL. The exception to this rule is that threads can 23 | execute non-Python code (such as IO, or some native libraries such as NumPy) 24 | without the GIL. 25 | 26 | By default Pyflame will only profile code that holds the Global Interpreter 27 | Lock. Since this is the only thread that can run Python code, in some sense this 28 | is a more accurate representation of the profile of an application, even when it 29 | is multithreaded. If nothing holds the GIL (so no Python code is executing) 30 | Pyflame will report the time as "idle". 31 | 32 | If you don't want to include this time you can use the invocation ``pyflame 33 | -x``. 34 | 35 | If instead you invoke Pyflame with the ``--threads`` option, Pyflame will take a 36 | snapshot of each thread's stack each time it samples the target process. At the 37 | end of the invocation, the profiling data for each thread will be printed to 38 | stdout sequentially. This gives you a more accurate profile in the sense that 39 | you will see what each thread was trying to do, even if it wasn't actually 40 | scheduled to run. 41 | 42 | **Pyflame may "freeze" the target process if you use this option with older 43 | versions of the Linux kernel.** In particular, for this option to work you need 44 | a kernel built with `waitid() ptrace support 45 | `__. This change was landed for Linux kernel 46 | 4.7. Most Linux distros also backported this change to older kernels, e.g. this 47 | change was backported to the 3.16 kernel series in 3.16.37 (which is in Debian 48 | Jessie's kernel patches). For more extensive discussion, see `issue #55 49 | `__. 50 | 51 | One interesting use of this feature is to get a point-in-time snapshot of what 52 | each thread is doing, like so: 53 | 54 | .. code:: bash 55 | 56 | # Get a point-in-time snapshot of what each thread is currently running. 57 | pyflame -s 0 --threads -p PID 58 | 59 | Are BSD / OS X / macOS Supported? 60 | --------------------------------- 61 | 62 | Pyflame uses a few Linux-specific interfaces, so unfortunately it is the only 63 | platform supported right now. Pull requests to add support for other platforms 64 | are very much wanted. 65 | 66 | Someone who is proficient with low-level C systems programming can probably get 67 | BSD to work without *too much* difficulty. The necessary work to adapt the code 68 | is described in `Issue #3 `__. 69 | 70 | By comparison, it is probably *much more* work to get Pyflame working on macOS. 71 | The current code assumes that the host uses `ELF 72 | `__ 73 | object/executable files. Apple uses a different object file format, called 74 | `Mach-O `__, so porting Pyflame to macOS 75 | would entail doing all of the work to port Pyflame to BSD, *plus* additional 76 | work to parse Mach-O object files. That said, the Mach-O format is documented 77 | online (e.g. `here `__), so a 78 | sufficiently motivated person could get macOS support working. 79 | 80 | What Are These Ptrace Permissions Errors? 81 | ----------------------------------------- 82 | 83 | Because it's so powerful, the ``ptrace(2)`` system call is often disabled or 84 | severely restricted. In order to use ptrace, these conditions must be met: 85 | 86 | - You must have the ``SYS_PTRACE`` 87 | `capability `__ 88 | (which is denied by default within Docker images). 89 | - The kernel must not have ``kernel.yama.ptrace_scope`` set to a value 90 | that is too restrictive. 91 | 92 | In both scenarios you'll also find that ``strace`` and ``gdb`` do not work as 93 | expected. 94 | 95 | Ptrace Errors Within Docker Containers 96 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 97 | 98 | By default Docker images do not have the ``SYS_PTRACE`` capability. If you want 99 | it enabled, invoke ``docker run`` using the ``--cap-add SYS_PTRACE`` option: 100 | 101 | .. code:: bash 102 | 103 | # Allows processes within the Docker container to use ptrace. 104 | docker run --cap-add SYS_PTRACE ... 105 | 106 | You can also use `capsh(1) 107 | `__ to list your current 108 | capabilities: 109 | 110 | .. code:: bash 111 | 112 | # You should see cap_sys_ptrace in the "Bounding set". 113 | capsh --print 114 | 115 | You do not need to run Pyflame from within a Docker container. If you have 116 | sufficient permissions (i.e. you are root, or the same UID as the Docker 117 | process) Pyflame can be run from outside a container to inspect a process inside 118 | a container. This is better for security, since you can keep ptrace disabled in 119 | the container. 120 | 121 | Ptrace Errors Outside Docker Containers Or When Not Using Docker 122 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 123 | 124 | If you're not in a Docker container, or you're not using Docker at all, ptrace 125 | permissions errors are likely related to you having too restrictive a value set 126 | for the ``kernel.yama.ptrace_scope`` sysfs knob. 127 | 128 | Debian Jessie ships with ``ptrace_scope`` set to 1 by default, which will 129 | prevent unprivileged users from attaching to already running processes. 130 | 131 | To see the current value of this setting: 132 | 133 | .. code:: bash 134 | 135 | # Prints the current value for the ptrace_scope setting. 136 | sysctl kernel.yama.ptrace_scope 137 | 138 | If you see a value other than 0 you may want to change it. Note that by doing 139 | this you'll affect the security of your system. Please read `the relevant kernel 140 | documentation `__ 141 | for a comprehensive discussion of the possible settings and what you're 142 | changing. If you want to completely disable the ptrace settings and get 143 | "classic" permissions (i.e. root can ptrace anything, unprivileged users can 144 | ptrace processes with the same user id) then use: 145 | 146 | .. code:: bash 147 | 148 | # Use this if you want "classic" ptrace permissions. 149 | sudo sysctl kernel.yama.ptrace_scope=0 150 | 151 | Ptrace With SELinux 152 | ~~~~~~~~~~~~~~~~~~~ 153 | 154 | If you're using SELinux, `you may have problems with ptrace 155 | `__. To check if 156 | ptrace is disabled: 157 | 158 | .. code:: bash 159 | 160 | # Check if SELinux is denying ptrace. 161 | getsebool deny_ptrace 162 | 163 | If you'd like to enable it: 164 | 165 | .. code:: bash 166 | 167 | # Enable ptrace under SELinux. 168 | setsebool -P deny_ptrace 0 169 | -------------------------------------------------------------------------------- /docs/generate-man.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Generate a manual page from man.md, using pandoc(1). The generated man page is 4 | # checked in, so that users don't need to install pandoc to build/package 5 | # Pyflame. 6 | 7 | set -eu 8 | 9 | SILENT=0 10 | BASEDIR="$(dirname "${BASH_SOURCE[0]}")" 11 | OUTPUT="${BASEDIR}/pyflame.man" 12 | 13 | while getopts ":ho:s" opt; do 14 | case $opt in 15 | h) 16 | echo "Usage: $0 [-h] [-s] [-o OUTPUT]" 17 | exit 18 | ;; 19 | o) 20 | OUTPUT="$OPTARG" 21 | ;; 22 | s) 23 | SILENT=1 24 | ;; 25 | \?) 26 | echo "Invalid option: -$OPTARG" >&2 27 | ;; 28 | esac 29 | done 30 | 31 | pandoc "${BASEDIR}/man.md" -M date="$(date '+%B %Y')" -s -t man -o "$OUTPUT" 32 | 33 | if [ "$SILENT" -eq 0 ]; then 34 | man -l "$OUTPUT" 35 | fi 36 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Pyflame documentation master file, created by 2 | sphinx-quickstart on Fri Jul 14 14:13:03 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Pyflame: A Ptracing Profiler For Python 7 | ======================================= 8 | 9 | Pyflame is a unique profiling tool that generates `flame graphs 10 | `__ for Python. Pyflame is the 11 | only Python profiler based on the Linux `ptrace(2) 12 | `__ system call. This allows 13 | it to take snapshots of the Python call stack without explicit instrumentation, 14 | meaning you can profile a program without modifying its source code! Pyflame is 15 | capable of profiling embedded Python interpreters like `uWSGI 16 | `__. It fully supports profiling 17 | multi-threaded Python programs. 18 | 19 | Pyflame is written in C++, with attention to speed and performance. Pyflame 20 | usually introduces less overhead than the builtin ``profile`` (or ``cProfile``) 21 | modules, and also emits richer profiling data. The profiling overhead is low 22 | enough that you can use it to profile live processes in production. 23 | 24 | .. toctree:: 25 | :maxdepth: 2 26 | :caption: Contents: 27 | 28 | installation 29 | versions 30 | usage 31 | faq 32 | contributing 33 | 34 | Websites 35 | -------- 36 | 37 | - `Project homepage 38 | `_ (this documentation) 39 | - `Source code at Github 40 | `_ 41 | 42 | 43 | Blog Posts 44 | ---------- 45 | 46 | Some existing articles and blog posts on Pyflame include: 47 | 48 | - `Pyflame: Uber Engineering's Ptracing Profiler For 49 | Python `__ by Evan Klitzke (2016-09) 50 | - `Pyflame Dual Interpreter 51 | Mode `__ by Evan 52 | Klitzke (2016-10) 53 | - `Using Uber's Pyflame and Logs to Tackle Scaling 54 | Issues `__ 55 | by Benoit Bernard (2017-02) 56 | - `Building Pyflame on Centos 57 | 6 `__ 58 | (Chinese) by Faicker Mo (2017-04) 59 | 60 | If you write a new post about Pyflame, please let us know and we'll add it here! 61 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | Installing Pyflame 2 | ================== 3 | 4 | You have two options for installing Pyflame: you can try a pre-built package, or 5 | you can install from source. To build from source, you will need a C++ compiler 6 | with basic C++11 support. Pyflame is known to compile on versions of GCC as old 7 | as GCC 4.6. 8 | 9 | Build Dependencies 10 | ------------------ 11 | 12 | Generally you'll need autotools, automake, libtool, pkg-config, and the Python 13 | headers. If you have headers for both Python 2 and Python 3 installed you'll get 14 | a Pyflame build that can target either version of Python. 15 | 16 | Debian/Ubuntu 17 | ~~~~~~~~~~~~~ 18 | 19 | Install the following packages if you are building for Debian or Ubuntu. 20 | Note that you technically only need one of ``python-dev`` or 21 | ``python3-dev``, but if you have both installed then you can use Pyflame 22 | to profile both Python 2 and Python 3 processes. 23 | 24 | .. code:: bash 25 | 26 | # Install build dependencies on Debian or Ubuntu. 27 | sudo apt-get install autoconf automake autotools-dev g++ pkg-config python-dev python3-dev libtool make 28 | 29 | Fedora/CentOS 30 | ~~~~~~~~~~~~~~~ 31 | 32 | Again, you technically only need one of ``python-devel`` and 33 | ``python3-devel``, although installing both is recommended. 34 | 35 | .. code:: bash 36 | 37 | # Install build dependencies on Fedora. 38 | sudo dnf install autoconf automake gcc-c++ python-devel python3-devel libtool 39 | 40 | Compiling 41 | --------- 42 | 43 | Once you've installed the appropriate build dependencies, you can compile 44 | Pyflame like so: 45 | 46 | .. code:: bash 47 | 48 | ./autogen.sh 49 | ./configure # Plus any options like --prefix. 50 | make 51 | make check # Optional, test the build! Should take < 1 minute. 52 | make install # Optional, install into the configure prefix. 53 | 54 | The Pyflame executable produced by the ``make`` command will be located at 55 | ``src/pyflame``. Note that the ``make check`` command requires that you have the 56 | ``virtualenv`` command installed. You can also sanity check your build with a 57 | command like: 58 | 59 | .. code:: bash 60 | 61 | # Or use -t python3, as appropriate. 62 | pyflame -t python -c 'print(sum(i for i in range(100000)))' 63 | 64 | Creating A Debian Package 65 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 66 | 67 | If you'd like to build a Debian package, run the following from the root 68 | of your Pyflame git checkout: 69 | 70 | .. code:: bash 71 | 72 | # Install additional dependencies required for packaging. 73 | sudo apt-get install debhelper dh-autoreconf dpkg-dev 74 | 75 | # This create a file named something like ../pyflame_1.3.1_amd64.deb 76 | dpkg-buildpackage -uc -us 77 | 78 | Pre-Built Packages 79 | ------------------ 80 | 81 | Several Pyflame users have created unofficial pre-built packages for different 82 | distros. Uploads of these packages tend to lag the official Pyflame releases, so 83 | you are **strongly encouraged to check the pre-built version** to ensure that it 84 | is not too old. If you want the newest version of Pyflame, build from source. 85 | 86 | Fedora/CentOS 87 | ~~~~~~~~~~~~~~~ 88 | 89 | `Evan Klitzke `__ maintains a COPR for 90 | Pyflame: 91 | 92 | .. code:: bash 93 | 94 | dnf copr enable eklitzke/pyflame 95 | dnf install pyflame 96 | 97 | Conda 98 | ~~~~~ 99 | 100 | `Evan Klitzke `__ maintains a Conda package of 101 | Pyflame: 102 | 103 | .. code:: bash 104 | 105 | conda install -c eklitzke pyflame 106 | 107 | Ubuntu PPA 108 | ~~~~~~~~~~ 109 | 110 | `Trevor Joynson `__ has set up an unofficial 111 | PPA for all current Ubuntu releases: `ppa:trevorjay/pyflame 112 | `__. 113 | 114 | .. code:: bash 115 | 116 | sudo apt-add-repository ppa:trevorjay/pyflame 117 | sudo apt-get update 118 | sudo apt-get install pyflame 119 | 120 | Note also that you can build your own Debian package easily, using the one 121 | provided in the ``debian/`` directory of this project. 122 | 123 | Arch Linux 124 | ~~~~~~~~~~ 125 | 126 | `Oleg Senin `__ has added an Arch Linux package 127 | to `AUR `__. 128 | -------------------------------------------------------------------------------- /docs/man.md: -------------------------------------------------------------------------------- 1 | % PYFLAME(1) 2 | % Evan Klitzke 3 | 4 | # NAME 5 | 6 | pyflame - A Ptracing Python Profiler 7 | 8 | # SYNOPSIS 9 | 10 | **pyflame** [**options**] [**-p**|**--pid**] *PID* 11 | 12 | **pyflame** [**options**] [**-t**|**--trace**] *command* [*args*...] 13 | 14 | # DESCRIPTION 15 | 16 | **pyflame** is a Python profiler that created flame graphs. It uses 17 | **ptrace**(2) to extract stack information. The output of **pyflame** is 18 | intended to be used with Brendan Gregg's *flamegraph.pl* script, which can be 19 | found on GitHub at . 20 | 21 | # GENERAL OPTIONS 22 | 23 | There are two invocation forms. When **-p** *PID* is used, pyflame will attach 24 | to the running process specified by *PID* to collect profiling data. The meaning 25 | of this option is analogous to its meaning in commands like **strace**(1) or 26 | **gdb**(1). 27 | 28 | When **-t** is given, pyflame will instead go into "trace mode". In this mode, 29 | it interprets the rest of the command line as a command to run, and traces the 30 | command to completion. This is analogous to how **strace**(1) works when a PID 31 | is not specified. 32 | 33 | **-d**, **--dump** 34 | : Dump stacks from all threads (implies **--threads**). 35 | 36 | **-h**, **--help** 37 | : Display a friendly help message. 38 | 39 | **-o**, **--output**=*FILENAME* 40 | : Write profiling output to *FILENAME* (otherwise stdout is used). 41 | 42 | **-p**, **--pid**=*PID* 43 | : Specify which *PID* to trace. 44 | 45 | Older versions of pyflame received *PID* as a positional argument, where 46 | *PID* was interpreted as the last argument. This usage mode still works, but 47 | is considered deprecated. You should use **-p** or **--pid** when specifying 48 | *PID*. 49 | 50 | **-s**, **--seconds**=*SECONDS* 51 | : Profile the process for duration *SECONDS* before detaching. The default is 52 | to profile for 1 second. This option is not compatible with trace mode. 53 | 54 | **-r**, **--rate**=*RATE* 55 | : Sample the process at this frequency. The argument *RATE* is interpreted as 56 | a fractional value, measured in seconds. For example, **-r 0.1** would mean 57 | to sample the process every 0.1 seconds (i.e. every 100 milliseconds). The 58 | default value for *RATE* is 0.01, which samples every ten milliseconds. 59 | 60 | Note that setting a low value for rate will increase the accuracy of 61 | profiles, but it also increases the overhead introduced by pyflame. The 62 | default frequency used by pyflame is relatively aggressive; a less 63 | aggressive value like **-r 0.01** may be more appropriate if you are 64 | profiling processes in production. 65 | 66 | **-t**, **--trace** *command* [*args*...] 67 | : Run pyflame in trace mode, which traces the child process until completion. 68 | If used, this must be the final argument (the rest of the arguments will be 69 | interpreted as a command plus arguments to the command). This is analogous 70 | to **strace**(1) in its default mode. 71 | 72 | **-v**, **--version** 73 | : Print the version. 74 | 75 | **-x**, **--exclude-idle** 76 | : Exclude "idle" time from output. 77 | 78 | **--threads** 79 | : Enable profiling multi-threaded Python apps. 80 | 81 | ## ADVANCED OPTIONS 82 | 83 | The following options are less commonly used. 84 | 85 | **--abi**=*VERSION* 86 | : Force a particular Python ABI. This option should only be needed in edge 87 | cases when profiling embedded Python builds (e.g. uWSGI), and only if 88 | pyflame doesn't automatically detect the correct ABI. *VERSION* should be a 89 | two digit integer consisting of the Python major and minor version, e.g. 27 90 | for Python 2.7 or 36 for Python 3.6. 91 | 92 | **--flamechart** 93 | : Print the timestamp for each stack. This is useful for generating "flame 94 | chart" profiles. Generally regular flame graphs are encouraged, since the 95 | timestamp flame charts are harder to use. 96 | 97 | # ONLINE DOCUMENTATION 98 | 99 | You can find the complete documentation online 100 | at: . The online documentation is more 101 | comprehensive than this man page, and includes usage examples. 102 | 103 | # REPORTING BUGS 104 | 105 | If you find any bugs, please create a new issue on 106 | GitHub: 107 | -------------------------------------------------------------------------------- /docs/pyflame.man: -------------------------------------------------------------------------------- 1 | .\" Automatically generated by Pandoc 1.19.1 2 | .\" 3 | .TH "PYFLAME" "1" "March 2018" "" "" 4 | .hy 5 | .SH NAME 6 | .PP 7 | pyflame \- A Ptracing Python Profiler 8 | .SH SYNOPSIS 9 | .PP 10 | \f[B]pyflame\f[] [\f[B]options\f[]] [\f[B]\-p\f[]|\f[B]\-\-pid\f[]] 11 | \f[I]PID\f[] 12 | .PP 13 | \f[B]pyflame\f[] [\f[B]options\f[]] [\f[B]\-t\f[]|\f[B]\-\-trace\f[]] 14 | \f[I]command\f[] [\f[I]args\f[]...] 15 | .SH DESCRIPTION 16 | .PP 17 | \f[B]pyflame\f[] is a Python profiler that created flame graphs. 18 | It uses \f[B]ptrace\f[](2) to extract stack information. 19 | The output of \f[B]pyflame\f[] is intended to be used with Brendan 20 | Gregg\[aq]s \f[I]flamegraph.pl\f[] script, which can be found on GitHub 21 | at . 22 | .SH GENERAL OPTIONS 23 | .PP 24 | There are two invocation forms. 25 | When \f[B]\-p\f[] \f[I]PID\f[] is used, pyflame will attach to the 26 | running process specified by \f[I]PID\f[] to collect profiling data. 27 | The meaning of this option is analogous to its meaning in commands like 28 | \f[B]strace\f[](1) or \f[B]gdb\f[](1). 29 | .PP 30 | When \f[B]\-t\f[] is given, pyflame will instead go into "trace mode". 31 | In this mode, it interprets the rest of the command line as a command to 32 | run, and traces the command to completion. 33 | This is analogous to how \f[B]strace\f[](1) works when a PID is not 34 | specified. 35 | .TP 36 | .B \f[B]\-d\f[], \f[B]\-\-dump\f[] 37 | Dump stacks from all threads (implies \f[B]\-\-threads\f[]). 38 | .RS 39 | .RE 40 | .TP 41 | .B \f[B]\-h\f[], \f[B]\-\-help\f[] 42 | Display a friendly help message. 43 | .RS 44 | .RE 45 | .TP 46 | .B \f[B]\-o\f[], \f[B]\-\-output\f[]=\f[I]FILENAME\f[] 47 | Write profiling output to \f[I]FILENAME\f[] (otherwise stdout is used). 48 | .RS 49 | .RE 50 | .TP 51 | .B \f[B]\-p\f[], \f[B]\-\-pid\f[]=\f[I]PID\f[] 52 | Specify which \f[I]PID\f[] to trace. 53 | .RS 54 | .PP 55 | Older versions of pyflame received \f[I]PID\f[] as a positional 56 | argument, where \f[I]PID\f[] was interpreted as the last argument. 57 | This usage mode still works, but is considered deprecated. 58 | You should use \f[B]\-p\f[] or \f[B]\-\-pid\f[] when specifying 59 | \f[I]PID\f[]. 60 | .RE 61 | .TP 62 | .B \f[B]\-s\f[], \f[B]\-\-seconds\f[]=\f[I]SECONDS\f[] 63 | Profile the process for duration \f[I]SECONDS\f[] before detaching. 64 | The default is to profile for 1 second. 65 | This option is not compatible with trace mode. 66 | .RS 67 | .RE 68 | .TP 69 | .B \f[B]\-r\f[], \f[B]\-\-rate\f[]=\f[I]RATE\f[] 70 | Sample the process at this frequency. 71 | The argument \f[I]RATE\f[] is interpreted as a fractional value, 72 | measured in seconds. 73 | For example, \f[B]\-r 0.1\f[] would mean to sample the process every 0.1 74 | seconds (i.e. 75 | every 100 milliseconds). 76 | The default value for \f[I]RATE\f[] is 0.01, which samples every ten 77 | milliseconds. 78 | .RS 79 | .PP 80 | Note that setting a low value for rate will increase the accuracy of 81 | profiles, but it also increases the overhead introduced by pyflame. 82 | The default frequency used by pyflame is relatively aggressive; a less 83 | aggressive value like \f[B]\-r 0.01\f[] may be more appropriate if you 84 | are profiling processes in production. 85 | .RE 86 | .TP 87 | .B \f[B]\-t\f[], \f[B]\-\-trace\f[] \f[I]command\f[] [\f[I]args\f[]...] 88 | Run pyflame in trace mode, which traces the child process until 89 | completion. 90 | If used, this must be the final argument (the rest of the arguments will 91 | be interpreted as a command plus arguments to the command). 92 | This is analogous to \f[B]strace\f[](1) in its default mode. 93 | .RS 94 | .RE 95 | .TP 96 | .B \f[B]\-v\f[], \f[B]\-\-version\f[] 97 | Print the version. 98 | .RS 99 | .RE 100 | .TP 101 | .B \f[B]\-x\f[], \f[B]\-\-exclude\-idle\f[] 102 | Exclude "idle" time from output. 103 | .RS 104 | .RE 105 | .TP 106 | .B \f[B]\-\-threads\f[] 107 | Enable profiling multi\-threaded Python apps. 108 | .RS 109 | .RE 110 | .SS ADVANCED OPTIONS 111 | .PP 112 | The following options are less commonly used. 113 | .TP 114 | .B \f[B]\-\-abi\f[]=\f[I]VERSION\f[] 115 | Force a particular Python ABI. 116 | This option should only be needed in edge cases when profiling embedded 117 | Python builds (e.g. 118 | uWSGI), and only if pyflame doesn\[aq]t automatically detect the correct 119 | ABI. 120 | \f[I]VERSION\f[] should be a two digit integer consisting of the Python 121 | major and minor version, e.g. 122 | 27 for Python 2.7 or 36 for Python 3.6. 123 | .RS 124 | .RE 125 | .TP 126 | .B \f[B]\-\-flamechart\f[] 127 | Print the timestamp for each stack. 128 | This is useful for generating "flame chart" profiles. 129 | Generally regular flame graphs are encouraged, since the timestamp flame 130 | charts are harder to use. 131 | .RS 132 | .RE 133 | .SH ONLINE DOCUMENTATION 134 | .PP 135 | You can find the complete documentation online at: 136 | . 137 | The online documentation is more comprehensive than this man page, and 138 | includes usage examples. 139 | .SH REPORTING BUGS 140 | .PP 141 | If you find any bugs, please create a new issue on GitHub: 142 | 143 | .SH AUTHORS 144 | Evan Klitzke . 145 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | Using Pyflame 2 | ============= 3 | 4 | Pyflame has two distinct modes: you can attach to a running process, or you can 5 | trace a command from start to finish. 6 | 7 | Attaching To A Running Python Process 8 | ------------------------------------- 9 | 10 | The default behavior of Pyflame is to attach to an existing Python process. The 11 | target process is specified via its PID: 12 | 13 | .. code:: bash 14 | 15 | # Profile PID for 1s, sampling every 1ms. 16 | pyflame -p PID 17 | 18 | This will print data to stdout in a format that is suitable for usage with 19 | Brendan Gregg's ``flamegraph.pl`` tool (which you can get `here 20 | `__). A typical command pipeline 21 | might be like this: 22 | 23 | .. code:: bash 24 | 25 | # Generate flame graph for pid 12345; assumes flamegraph.pl is in your $PATH. 26 | pyflame -p 12345 | flamegraph.pl > myprofile.svg 27 | 28 | You can also change the sample time with ``-s``, and the sampling frequency with 29 | ``-r``. Both units are measured in seconds. 30 | 31 | .. code:: bash 32 | 33 | # Profile PID for 60 seconds, sampling every 10ms. 34 | pyflame -s 60 -r 0.01 -p PID 35 | 36 | The default behavior is to sample for 1 second (equivalent to ``-s 1``), taking 37 | a snapshot every ten milliseconds (equivalent to ``-r 0.01``). 38 | 39 | Attaching To Docker/Containerized Processes 40 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 41 | 42 | Pyflame knows how to do something interesting: it can attach to containerized 43 | processes from **outside the container**. It does this by directly using the 44 | `setns(2) `__ system call 45 | (which is how Docker works under the hood). 46 | 47 | If you choose to profile a process from outside the container, use the true PID, 48 | as reported by ``ps`` on the host (i.e. outside of the container). 49 | 50 | You can also run Pyflame from inside containers, although this is a bit more 51 | annoying, since normally ptrace is disabled inside containers for security 52 | reasons. If you attach to a process this way, you will need to use the 53 | inside-the-container PID. You can find this by running ``ps`` inside of the 54 | container itself. 55 | 56 | We recommend running Pyflame from outside containers, since it means you can 57 | keep ptrace disabled inside containers. If you want to run Pyflame inside 58 | containers, and have problems, please make sure to read the Docker notes in the 59 | `FAQ <#faq>`__. 60 | 61 | Tracing Python Commands 62 | ----------------------- 63 | 64 | Sometimes you want to trace a command from start to finish. An example would be 65 | tracing the run of a test suite or batch job. Pass ``-t`` as the **last** 66 | Pyflame flag to run in trace mode. Anything after the ``-t`` flag is interpreted 67 | literally as part of the command to run: 68 | 69 | .. code:: bash 70 | 71 | # Trace a given command until completion. 72 | pyflame [regular pyflame options] -t command arg1 arg2... 73 | 74 | Often ``command`` will be ``python`` or ``python3``, but it could be something 75 | else, like ``uwsgi`` or ``py.test``. For instance, here's how Pyflame can be 76 | used to trace its own test suite: 77 | 78 | .. code:: bash 79 | 80 | # Trace the Pyflame test suite, a.k.a. pyflameception! 81 | pyflame -t py.test tests/ 82 | 83 | As described in the docs for attach mode, you can use ``-r`` to control the 84 | sampling frequency. 85 | 86 | Tracing Programs That Print To Stdout 87 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 88 | 89 | By default, Pyflame will send flame graph data to stdout. If the profiled 90 | program is also sending data to stdout, then ``flamegraph.pl`` will see the 91 | output from both programs, and will get confused. To solve this, use the ``-o`` 92 | option: 93 | 94 | .. code:: bash 95 | 96 | # Trace a process, sending profiling information to profile.txt 97 | pyflame -o profile.txt -t python -c 'for x in range(1000): print(x)' 98 | 99 | # Convert profile.txt to a flame graph named profile.svg 100 | flamegraph.pl profile.svg 101 | 102 | Timestamp ("Flame Chart") Mode 103 | ------------------------------ 104 | 105 | Generally we recommend using regular flame graphs, generated by 106 | ``flamegraph.pl``. However, Pyflame can also generate data with a special time 107 | stamp output format, useful for generating `"flame charts" 108 | `__ (somewhat like an 109 | inverted flame graph) that are viewable in Chrome. In some cases, the flame 110 | chart format is easier to understand. 111 | 112 | To generate a flame chart, use ``pyflame --flamechart``, and then pass the 113 | output to ``utils/flame-chart-json`` to convert the output into the JSON format 114 | required by the Chrome CPU profiler: 115 | 116 | .. code:: bash 117 | 118 | # Generate flame chart data viewable in Chrome. 119 | pyflame --flamechart [other pyflame options] | flame-chart-json > foo.cpuprofile 120 | 121 | Read the following `Chrome DevTools article 122 | `__ 123 | for instructions on loading a ``.cpuprofile`` file in Chrome 58+. 124 | -------------------------------------------------------------------------------- /m4/ax_append_compile_flags.m4: -------------------------------------------------------------------------------- 1 | # ============================================================================ 2 | # https://www.gnu.org/software/autoconf-archive/ax_append_compile_flags.html 3 | # ============================================================================ 4 | # 5 | # SYNOPSIS 6 | # 7 | # AX_APPEND_COMPILE_FLAGS([FLAG1 FLAG2 ...], [FLAGS-VARIABLE], [EXTRA-FLAGS], [INPUT]) 8 | # 9 | # DESCRIPTION 10 | # 11 | # For every FLAG1, FLAG2 it is checked whether the compiler works with the 12 | # flag. If it does, the flag is added FLAGS-VARIABLE 13 | # 14 | # If FLAGS-VARIABLE is not specified, the current language's flags (e.g. 15 | # CFLAGS) is used. During the check the flag is always added to the 16 | # current language's flags. 17 | # 18 | # If EXTRA-FLAGS is defined, it is added to the current language's default 19 | # flags (e.g. CFLAGS) when the check is done. The check is thus made with 20 | # the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to 21 | # force the compiler to issue an error when a bad flag is given. 22 | # 23 | # INPUT gives an alternative input source to AC_COMPILE_IFELSE. 24 | # 25 | # NOTE: This macro depends on the AX_APPEND_FLAG and 26 | # AX_CHECK_COMPILE_FLAG. Please keep this macro in sync with 27 | # AX_APPEND_LINK_FLAGS. 28 | # 29 | # LICENSE 30 | # 31 | # Copyright (c) 2011 Maarten Bosmans 32 | # 33 | # This program is free software: you can redistribute it and/or modify it 34 | # under the terms of the GNU General Public License as published by the 35 | # Free Software Foundation, either version 3 of the License, or (at your 36 | # option) any later version. 37 | # 38 | # This program is distributed in the hope that it will be useful, but 39 | # WITHOUT ANY WARRANTY; without even the implied warranty of 40 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 41 | # Public License for more details. 42 | # 43 | # You should have received a copy of the GNU General Public License along 44 | # with this program. If not, see . 45 | # 46 | # As a special exception, the respective Autoconf Macro's copyright owner 47 | # gives unlimited permission to copy, distribute and modify the configure 48 | # scripts that are the output of Autoconf when processing the Macro. You 49 | # need not follow the terms of the GNU General Public License when using 50 | # or distributing such scripts, even though portions of the text of the 51 | # Macro appear in them. The GNU General Public License (GPL) does govern 52 | # all other use of the material that constitutes the Autoconf Macro. 53 | # 54 | # This special exception to the GPL applies to versions of the Autoconf 55 | # Macro released by the Autoconf Archive. When you make and distribute a 56 | # modified version of the Autoconf Macro, you may extend this special 57 | # exception to the GPL to apply to your modified version as well. 58 | 59 | #serial 6 60 | 61 | AC_DEFUN([AX_APPEND_COMPILE_FLAGS], 62 | [AX_REQUIRE_DEFINED([AX_CHECK_COMPILE_FLAG]) 63 | AX_REQUIRE_DEFINED([AX_APPEND_FLAG]) 64 | for flag in $1; do 65 | AX_CHECK_COMPILE_FLAG([$flag], [AX_APPEND_FLAG([$flag], [$2])], [], [$3], [$4]) 66 | done 67 | ])dnl AX_APPEND_COMPILE_FLAGS 68 | -------------------------------------------------------------------------------- /m4/ax_append_flag.m4: -------------------------------------------------------------------------------- 1 | # =========================================================================== 2 | # http://www.gnu.org/software/autoconf-archive/ax_append_flag.html 3 | # =========================================================================== 4 | # 5 | # SYNOPSIS 6 | # 7 | # AX_APPEND_FLAG(FLAG, [FLAGS-VARIABLE]) 8 | # 9 | # DESCRIPTION 10 | # 11 | # FLAG is appended to the FLAGS-VARIABLE shell variable, with a space 12 | # added in between. 13 | # 14 | # If FLAGS-VARIABLE is not specified, the current language's flags (e.g. 15 | # CFLAGS) is used. FLAGS-VARIABLE is not changed if it already contains 16 | # FLAG. If FLAGS-VARIABLE is unset in the shell, it is set to exactly 17 | # FLAG. 18 | # 19 | # NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. 20 | # 21 | # LICENSE 22 | # 23 | # Copyright (c) 2008 Guido U. Draheim 24 | # Copyright (c) 2011 Maarten Bosmans 25 | # 26 | # This program is free software: you can redistribute it and/or modify it 27 | # under the terms of the GNU General Public License as published by the 28 | # Free Software Foundation, either version 3 of the License, or (at your 29 | # option) any later version. 30 | # 31 | # This program is distributed in the hope that it will be useful, but 32 | # WITHOUT ANY WARRANTY; without even the implied warranty of 33 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 34 | # Public License for more details. 35 | # 36 | # You should have received a copy of the GNU General Public License along 37 | # with this program. If not, see . 38 | # 39 | # As a special exception, the respective Autoconf Macro's copyright owner 40 | # gives unlimited permission to copy, distribute and modify the configure 41 | # scripts that are the output of Autoconf when processing the Macro. You 42 | # need not follow the terms of the GNU General Public License when using 43 | # or distributing such scripts, even though portions of the text of the 44 | # Macro appear in them. The GNU General Public License (GPL) does govern 45 | # all other use of the material that constitutes the Autoconf Macro. 46 | # 47 | # This special exception to the GPL applies to versions of the Autoconf 48 | # Macro released by the Autoconf Archive. When you make and distribute a 49 | # modified version of the Autoconf Macro, you may extend this special 50 | # exception to the GPL to apply to your modified version as well. 51 | 52 | #serial 6 53 | 54 | AC_DEFUN([AX_APPEND_FLAG], 55 | [dnl 56 | AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_SET_IF 57 | AS_VAR_PUSHDEF([FLAGS], [m4_default($2,_AC_LANG_PREFIX[FLAGS])]) 58 | AS_VAR_SET_IF(FLAGS,[ 59 | AS_CASE([" AS_VAR_GET(FLAGS) "], 60 | [*" $1 "*], [AC_RUN_LOG([: FLAGS already contains $1])], 61 | [ 62 | AS_VAR_APPEND(FLAGS,[" $1"]) 63 | AC_RUN_LOG([: FLAGS="$FLAGS"]) 64 | ]) 65 | ], 66 | [ 67 | AS_VAR_SET(FLAGS,[$1]) 68 | AC_RUN_LOG([: FLAGS="$FLAGS"]) 69 | ]) 70 | AS_VAR_POPDEF([FLAGS])dnl 71 | ])dnl AX_APPEND_FLAG 72 | -------------------------------------------------------------------------------- /m4/ax_check_compile_flag.m4: -------------------------------------------------------------------------------- 1 | # =========================================================================== 2 | # http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html 3 | # =========================================================================== 4 | # 5 | # SYNOPSIS 6 | # 7 | # AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT]) 8 | # 9 | # DESCRIPTION 10 | # 11 | # Check whether the given FLAG works with the current language's compiler 12 | # or gives an error. (Warnings, however, are ignored) 13 | # 14 | # ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on 15 | # success/failure. 16 | # 17 | # If EXTRA-FLAGS is defined, it is added to the current language's default 18 | # flags (e.g. CFLAGS) when the check is done. The check is thus made with 19 | # the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to 20 | # force the compiler to issue an error when a bad flag is given. 21 | # 22 | # INPUT gives an alternative input source to AC_COMPILE_IFELSE. 23 | # 24 | # NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this 25 | # macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG. 26 | # 27 | # LICENSE 28 | # 29 | # Copyright (c) 2008 Guido U. Draheim 30 | # Copyright (c) 2011 Maarten Bosmans 31 | # 32 | # This program is free software: you can redistribute it and/or modify it 33 | # under the terms of the GNU General Public License as published by the 34 | # Free Software Foundation, either version 3 of the License, or (at your 35 | # option) any later version. 36 | # 37 | # This program is distributed in the hope that it will be useful, but 38 | # WITHOUT ANY WARRANTY; without even the implied warranty of 39 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 40 | # Public License for more details. 41 | # 42 | # You should have received a copy of the GNU General Public License along 43 | # with this program. If not, see . 44 | # 45 | # As a special exception, the respective Autoconf Macro's copyright owner 46 | # gives unlimited permission to copy, distribute and modify the configure 47 | # scripts that are the output of Autoconf when processing the Macro. You 48 | # need not follow the terms of the GNU General Public License when using 49 | # or distributing such scripts, even though portions of the text of the 50 | # Macro appear in them. The GNU General Public License (GPL) does govern 51 | # all other use of the material that constitutes the Autoconf Macro. 52 | # 53 | # This special exception to the GPL applies to versions of the Autoconf 54 | # Macro released by the Autoconf Archive. When you make and distribute a 55 | # modified version of the Autoconf Macro, you may extend this special 56 | # exception to the GPL to apply to your modified version as well. 57 | 58 | #serial 4 59 | 60 | AC_DEFUN([AX_CHECK_COMPILE_FLAG], 61 | [AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF 62 | AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl 63 | AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ 64 | ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS 65 | _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" 66 | AC_COMPILE_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])], 67 | [AS_VAR_SET(CACHEVAR,[yes])], 68 | [AS_VAR_SET(CACHEVAR,[no])]) 69 | _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) 70 | AS_VAR_IF(CACHEVAR,yes, 71 | [m4_default([$2], :)], 72 | [m4_default([$3], :)]) 73 | AS_VAR_POPDEF([CACHEVAR])dnl 74 | ])dnl AX_CHECK_COMPILE_FLAGS 75 | -------------------------------------------------------------------------------- /m4/ax_require_defined.m4: -------------------------------------------------------------------------------- 1 | # =========================================================================== 2 | # https://www.gnu.org/software/autoconf-archive/ax_require_defined.html 3 | # =========================================================================== 4 | # 5 | # SYNOPSIS 6 | # 7 | # AX_REQUIRE_DEFINED(MACRO) 8 | # 9 | # DESCRIPTION 10 | # 11 | # AX_REQUIRE_DEFINED is a simple helper for making sure other macros have 12 | # been defined and thus are available for use. This avoids random issues 13 | # where a macro isn't expanded. Instead the configure script emits a 14 | # non-fatal: 15 | # 16 | # ./configure: line 1673: AX_CFLAGS_WARN_ALL: command not found 17 | # 18 | # It's like AC_REQUIRE except it doesn't expand the required macro. 19 | # 20 | # Here's an example: 21 | # 22 | # AX_REQUIRE_DEFINED([AX_CHECK_LINK_FLAG]) 23 | # 24 | # LICENSE 25 | # 26 | # Copyright (c) 2014 Mike Frysinger 27 | # 28 | # Copying and distribution of this file, with or without modification, are 29 | # permitted in any medium without royalty provided the copyright notice 30 | # and this notice are preserved. This file is offered as-is, without any 31 | # warranty. 32 | 33 | #serial 2 34 | 35 | AC_DEFUN([AX_REQUIRE_DEFINED], [dnl 36 | m4_ifndef([$1], [m4_fatal([macro ]$1[ is not defined; is a m4 file missing?])]) 37 | ])dnl AX_REQUIRE_DEFINED 38 | -------------------------------------------------------------------------------- /runtests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Run the Pyflame test suite. 4 | # 5 | # If invoked without arguments, this will make a best effort to run the test 6 | # suite against python2 and python3. If you would like to force the test suite 7 | # to run against a specific version of python, invoke with the python 8 | # interpreter names as arguments. These should be strings suitable for passing 9 | # to virtualenv -p. 10 | 11 | set -e 12 | 13 | ENVDIR="./.test_env" 14 | trap 'rm -rf ${ENVDIR}' EXIT 15 | 16 | VERBOSE=0 17 | export PYMAJORVERSION 18 | 19 | while getopts ":hvx" opt; do 20 | case $opt in 21 | h) 22 | echo "Usage: $0 [-h] [-x] python..." 23 | exit 1 24 | ;; 25 | v) 26 | VERBOSE=1 27 | ;; 28 | x) 29 | set -x 30 | ;; 31 | \?) 32 | echo "Invalid option: -$OPTARG" >&2 33 | ;; 34 | esac 35 | done 36 | 37 | shift "$((OPTIND-1))" 38 | 39 | exists() { 40 | command -v "$1" &>/dev/null 41 | } 42 | 43 | pytest() { 44 | if [ "$VERBOSE" -eq 0 ]; then 45 | py.test -q "$@" 46 | else 47 | py.test -v "$@" 48 | fi 49 | } 50 | 51 | mkvenv() { 52 | if exists virtualenv-3; then 53 | virtualenv-3 "$@" 54 | elif exists virtualenv; then 55 | virtualenv "$@" 56 | else 57 | echo "failed to find virtualenv command" 58 | exit 1 59 | fi 60 | } 61 | 62 | # Run tests using pip; $1 = python version 63 | run_pip_tests() { 64 | local activated 65 | if [ -z "${VIRTUAL_ENV}" ]; then 66 | rm -rf "${ENVDIR}" 67 | mkvenv -q -p "$1" "${ENVDIR}" &>/dev/null 68 | 69 | # shellcheck source=/dev/null 70 | . "${ENVDIR}/bin/activate" 71 | activated=1 72 | else 73 | echo "Warning: reusing virtualenv" 74 | fi 75 | 76 | PYMAJORVERSION=$(python -c 'import sys; print(sys.version_info[0])') 77 | echo "Running test suite against interpreter $("$1" --version 2>&1)" 78 | 79 | find tests/ -name '*.pyc' -delete 80 | pip install -q pytest 81 | pytest -v tests/ 82 | if [ "$activated" -eq 1 ]; then 83 | deactivate 84 | fi 85 | } 86 | 87 | # Make a best effort to run the tests against some Python version. 88 | try_pip_tests() { 89 | if command -v "$1" &>/dev/null; then 90 | run_pip_tests "$1" 91 | else 92 | echo "skipping $1 tests (no such command)" 93 | fi 94 | } 95 | 96 | # Tests run when building RPMs are not allowed to use virtualenv. 97 | run_rpm_tests() { 98 | for pytest in py.test-2 py.test-2.7; do 99 | if exists "$pytest"; then 100 | PYMAJORVERSION=2 "$pytest" -v tests/ 101 | break 102 | fi 103 | done 104 | 105 | for pytest in py.test-3 py.test-3.4; do 106 | if exists "$pytest"; then 107 | PYMAJORVERSION=3 "$pytest" -v tests/ 108 | break 109 | fi 110 | done 111 | } 112 | 113 | if [ $# -eq 0 ]; then 114 | PYMAJORVERSION=2 try_pip_tests python2 115 | PYMAJORVERSION=3 try_pip_tests python3 116 | elif [ "$1" = "rpm" ]; then 117 | run_rpm_tests 118 | else 119 | for py in "$@"; do 120 | run_pip_tests "$py" 121 | done 122 | fi 123 | -------------------------------------------------------------------------------- /src/Makefile.am: -------------------------------------------------------------------------------- 1 | # Here be dragons. 2 | # 3 | # The way this code is structured is the code that know about Python interpreter 4 | # internals is in frob.cc. This file isn't compiled directly, instead there are 5 | # files frob{26,34,36}.cc that define preprocessor macros to compile frob.cc for 6 | # different Python ABIs. 7 | # 8 | # The libtool magic here makes it so that frob26.cc is compiled with python2 9 | # flags, and frob3{4,6}.cc are compiled with python3.4/3.6 flags. 10 | 11 | bin_PROGRAMS = pyflame 12 | pyflame_SOURCES = aslr.cc frame.cc thread.cc namespace.cc posix.cc prober.cc ptrace.cc pyflame.cc pyfrob.cc symbol.cc 13 | pyflame_LDADD = 14 | 15 | noinst_LTLIBRARIES = 16 | 17 | if ENABLE_PY26 18 | libfrob26_la_SOURCES = frob26.cc 19 | libfrob26_la_CXXFLAGS = $(PY26_CFLAGS) 20 | noinst_LTLIBRARIES += libfrob26.la 21 | pyflame_LDADD += libfrob26.la 22 | endif 23 | 24 | if ENABLE_PY34 25 | libfrob34_la_SOURCES = frob34.cc 26 | libfrob34_la_CXXFLAGS = $(PY34_CFLAGS) 27 | noinst_LTLIBRARIES += libfrob34.la 28 | pyflame_LDADD += libfrob34.la 29 | endif 30 | 31 | if ENABLE_PY36 32 | libfrob36_la_SOURCES = frob36.cc 33 | libfrob36_la_CXXFLAGS = $(PY36_CFLAGS) 34 | noinst_LTLIBRARIES += libfrob36.la 35 | pyflame_LDADD += libfrob36.la 36 | endif 37 | -------------------------------------------------------------------------------- /src/aslr.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "./aslr.h" 16 | #include "./exc.h" 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | namespace pyflame { 23 | // Find libpython2.7.so and its offset for an ASLR process 24 | size_t LocateLibPython(pid_t pid, const std::string &hint, std::string *path) { 25 | std::ostringstream ss; 26 | ss << "/proc/" << pid << "/maps"; 27 | std::ifstream fp(ss.str()); 28 | std::string line; 29 | std::string elf_path; 30 | while (std::getline(fp, line)) { 31 | if (line.find(hint) != std::string::npos && 32 | line.find(" r-xp ") != std::string::npos) { 33 | size_t pos = line.find('/'); 34 | if (pos == std::string::npos) { 35 | throw FatalException("Did not find libpython absolute path"); 36 | } 37 | *path = line.substr(pos); 38 | pos = line.find('-'); 39 | if (pos == std::string::npos) { 40 | throw FatalException("Did not find libpython virtual memory address"); 41 | } 42 | return std::strtoul(line.substr(0, pos).c_str(), nullptr, 16); 43 | } 44 | } 45 | return 0; 46 | } 47 | } // namespace pyflame 48 | -------------------------------------------------------------------------------- /src/aslr.h: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | #include 20 | 21 | namespace pyflame { 22 | // Find libpython2.7.so and its offset for an ASLR process. 23 | size_t LocateLibPython(pid_t pid, const std::string &hint, std::string *path); 24 | } // namespace pyflame 25 | -------------------------------------------------------------------------------- /src/exc.h: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | #include 19 | 20 | namespace pyflame { 21 | // An expected exception, indicating that Pyflame can exit with status 0. 22 | class TerminateException : public std::runtime_error { 23 | public: 24 | explicit TerminateException(const std::string &what_arg) 25 | : std::runtime_error(what_arg) {} 26 | }; 27 | 28 | // An unexpected exception, indicating that Pyflame should exit with non-zero 29 | // status. 30 | class FatalException : public std::runtime_error { 31 | public: 32 | explicit FatalException(const std::string &what_arg) 33 | : std::runtime_error(what_arg) {} 34 | }; 35 | 36 | class SymbolException : public FatalException { 37 | public: 38 | explicit SymbolException(const std::string &what_arg) 39 | : FatalException(what_arg) {} 40 | }; 41 | 42 | class PtraceException : public std::runtime_error { 43 | public: 44 | explicit PtraceException(const std::string &what_arg) 45 | : std::runtime_error(what_arg) {} 46 | }; 47 | } // namespace pyflame 48 | -------------------------------------------------------------------------------- /src/frame.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "./frame.h" 16 | 17 | namespace pyflame { 18 | std::ostream &operator<<(std::ostream &os, const Frame &frame) { 19 | os << frame.file() << ':' << frame.name() << ':' << frame.line(); 20 | return os; 21 | } 22 | 23 | void print_frame(std::ostream &os, const Frame &frame) { 24 | os << frame.file() << ':' << frame.name() << ':' << frame.line(); 25 | } 26 | 27 | void print_frame_without_line_number(std::ostream &os, const Frame &frame) { 28 | os << frame.file() << ':' << frame.name(); 29 | } 30 | } // namespace pyflame 31 | -------------------------------------------------------------------------------- /src/frame.h: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "./namespace.h" 26 | 27 | namespace pyflame { 28 | 29 | class Frame { 30 | public: 31 | Frame() = delete; 32 | Frame(const Frame &other) 33 | : file_(other.file_), name_(other.name_), line_(other.line_) {} 34 | Frame(const std::string &file, const std::string &name, size_t line) 35 | : file_(file), name_(name), line_(line) {} 36 | 37 | inline const std::string &file() const { return file_; } 38 | inline const std::string &name() const { return name_; } 39 | inline size_t line() const { return line_; } 40 | 41 | inline bool operator==(const Frame &other) const { 42 | return file_ == other.file_ && line_ == other.line_; 43 | } 44 | 45 | private: 46 | std::string file_; 47 | std::string name_; 48 | size_t line_; 49 | }; 50 | 51 | std::ostream &operator<<(std::ostream &os, const Frame &frame); 52 | void print_frame(std::ostream &os, const Frame &frame); 53 | void print_frame_without_line_number(std::ostream &os, const Frame &frame); 54 | 55 | typedef void (*print_frame_t) (std::ostream &, const Frame &); 56 | typedef std::vector frames_t; 57 | 58 | struct FrameHash { 59 | size_t operator()(const frames_t &frames) const { 60 | size_t hash = 0; 61 | for (size_t i = 0; i < frames.size(); i++) { 62 | hash ^= std::hash()(i); 63 | hash ^= std::hash()(frames[i].file()); 64 | } 65 | return hash; 66 | } 67 | }; 68 | 69 | struct FrameTS { 70 | std::chrono::system_clock::time_point ts; 71 | frames_t frames; 72 | }; 73 | } // namespace pyflame 74 | -------------------------------------------------------------------------------- /src/frob.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // XXX: This file isn't compiled directly. It's included by frob2.cc or 16 | // frob3.cc, which define PYFLAME_PY_VERSION. Since Makefile.am for more 17 | // information. 18 | 19 | #include 20 | #include 21 | 22 | #if PYFLAME_PY_VERSION >= 34 23 | #include 24 | #endif 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | #include "./config.h" 36 | #include "./exc.h" 37 | #include "./ptrace.h" 38 | #include "./pyfrob.h" 39 | #include "./symbol.h" 40 | 41 | // why would this not be true idk 42 | static_assert(sizeof(long) == sizeof(void *), "wat platform r u on"); 43 | 44 | namespace pyflame { 45 | 46 | #if PYFLAME_PY_VERSION == 26 47 | namespace py26 { 48 | unsigned long StringSize(unsigned long addr) { 49 | return addr + offsetof(PyStringObject, ob_size); 50 | } 51 | 52 | unsigned long ByteData(unsigned long addr) { 53 | return addr + offsetof(PyStringObject, ob_sval); 54 | } 55 | 56 | std::string StringData(pid_t pid, unsigned long addr) { 57 | return PtracePeekString(pid, ByteData(addr)); 58 | } 59 | 60 | #elif PYFLAME_PY_VERSION == 34 61 | namespace py34 { 62 | std::string StringDataPython3(pid_t pid, unsigned long addr); 63 | 64 | unsigned long StringSize(unsigned long addr) { 65 | return addr + offsetof(PyVarObject, ob_size); 66 | } 67 | 68 | std::string StringData(pid_t pid, unsigned long addr) { 69 | return StringDataPython3(pid, addr); 70 | } 71 | 72 | unsigned long ByteData(unsigned long addr) { 73 | return addr + offsetof(PyBytesObject, ob_sval); 74 | } 75 | 76 | #elif PYFLAME_PY_VERSION == 36 77 | namespace py36 { 78 | std::string StringDataPython3(pid_t pid, unsigned long addr); 79 | 80 | unsigned long StringSize(unsigned long addr) { 81 | return addr + offsetof(PyVarObject, ob_size); 82 | } 83 | 84 | std::string StringData(pid_t pid, unsigned long addr) { 85 | return StringDataPython3(pid, addr); 86 | } 87 | 88 | unsigned long ByteData(unsigned long addr) { 89 | return addr + offsetof(PyBytesObject, ob_sval); 90 | } 91 | 92 | #else 93 | static_assert(false, "uh oh, bad PYFLAME_PY_VERSION"); 94 | #endif 95 | 96 | #if PYFLAME_PY_VERSION >= 34 97 | std::string StringDataPython3(pid_t pid, unsigned long addr) { 98 | // TODO: This function only works for Python >= 3.3. Is it also possible to 99 | // support older versions of Python 3? 100 | 101 | // TODO: Can we guarantee that the same padding is used for the bitfield? 102 | const std::unique_ptr unicode_bytes = 103 | PtracePeekBytes(pid, addr, sizeof(PyASCIIObject)); 104 | PyASCIIObject *unicode = 105 | reinterpret_cast(unicode_bytes.get()); 106 | 107 | // Because both the filename and function name string objects are made by the 108 | // Python interpreter itself, we can probably assume they are compact. This 109 | // means that the data immediately follows the object, and is of type {ASCII, 110 | // Latin-1, UCS-2, UCS-4}. 111 | assert(unicode->state.compact); 112 | 113 | const long str_offset = unicode->state.ascii ? sizeof(PyASCIIObject) 114 | : sizeof(PyCompactUnicodeObject); 115 | 116 | // NOTE: From CPython commit c47adb04 onwards the kind matches directly to 117 | // character size. This is different from the unicode format specification 118 | // outlined in PEP 393, which still had only two bits allocated to the kind 119 | // field. 120 | const unsigned int ch_size = unicode->state.kind; 121 | const ssize_t str_length = ch_size * unicode->length; 122 | const std::unique_ptr bytes = 123 | PtracePeekBytes(pid, addr + str_offset, str_length); 124 | 125 | std::ostringstream dump; 126 | 127 | for (int i = 0; i < str_length; i += ch_size) { 128 | Py_UCS4 ch = 0; 129 | 130 | switch (unicode->state.kind) { 131 | case PyUnicode_1BYTE_KIND: 132 | ch = bytes[i]; 133 | break; 134 | case PyUnicode_2BYTE_KIND: { 135 | Py_UCS2 *data_2 = reinterpret_cast(&bytes.get()[i]); 136 | ch = *data_2; 137 | break; 138 | } 139 | case PyUnicode_4BYTE_KIND: { 140 | Py_UCS4 *data_4 = reinterpret_cast(&bytes.get()[i]); 141 | ch = *data_4; 142 | break; 143 | } 144 | default: 145 | // We are not supposed to come here, as the WCHAR kind is not supported 146 | // when the object is compact. 147 | assert(false); 148 | break; 149 | } 150 | 151 | // TODO: Is it alright to assume a lack of surrogates. They might be present 152 | // in the UCS-2 representation if the UTF-16 approach is used. We currently 153 | // assume that CPython will instead use UCS-4 for such characters, instead 154 | // of using surrogates. 155 | 156 | // Below section is taken from CPython's STRINGLIB(utf8_encoder) routine. 157 | // The differences are that (1) we use a string builder instead of a char 158 | // buffer, and (2) that we skip the surrogate handling entirely. 159 | if (ch < 0x80) { 160 | /* Encode ASCII */ 161 | dump << (char)ch; 162 | } else if (ch < 0x0800) { 163 | /* Encode Latin-1 */ 164 | dump << (char)(0xc0 | (ch >> 6)); 165 | dump << (char)(0x80 | (ch & 0x3f)); 166 | } else if (ch < 0x10000) { 167 | dump << (char)(0xe0 | (ch >> 12)); 168 | dump << (char)(0x80 | ((ch >> 6) & 0x3f)); 169 | dump << (char)(0x80 | (ch & 0x3f)); 170 | } else { 171 | /* ch >= 0x10000 */ 172 | assert(ch <= 0x10ffff); // Maximum code point of Unicode 6.0 173 | 174 | /* Encode UCS4 Unicode ordinals */ 175 | dump << (char)(0xf0 | (ch >> 18)); 176 | dump << (char)(0x80 | ((ch >> 12) & 0x3f)); 177 | dump << (char)(0x80 | ((ch >> 6) & 0x3f)); 178 | dump << (char)(0x80 | (ch & 0x3f)); 179 | } 180 | } 181 | 182 | return dump.str(); 183 | } 184 | #endif 185 | 186 | // Extract the line number from the code object. Python uses a compressed table 187 | // data structure to store line numbers. See: 188 | // 189 | // https://svn.python.org/projects/python/trunk/Objects/lnotab_notes.txt 190 | // 191 | // This is essentially an implementation of PyFrame_GetLineNumber / 192 | // PyCode_Addr2Line. 193 | size_t GetLine(pid_t pid, unsigned long frame, unsigned long f_code) { 194 | const long f_trace = PtracePeek(pid, frame + offsetof(_frame, f_trace)); 195 | if (f_trace) { 196 | return static_cast( 197 | PtracePeek(pid, frame + offsetof(_frame, f_lineno)) & 198 | std::numeric_limits::max()); 199 | } 200 | 201 | const int f_lasti = PtracePeek(pid, frame + offsetof(_frame, f_lasti)) & 202 | std::numeric_limits::max(); 203 | const long co_lnotab = 204 | PtracePeek(pid, f_code + offsetof(PyCodeObject, co_lnotab)); 205 | 206 | int size = 207 | PtracePeek(pid, StringSize(co_lnotab)) & std::numeric_limits::max(); 208 | int line = PtracePeek(pid, f_code + offsetof(PyCodeObject, co_firstlineno)) & 209 | std::numeric_limits::max(); 210 | const std::unique_ptr tbl = 211 | PtracePeekBytes(pid, ByteData(co_lnotab), size); 212 | size /= 2; // since we increment twice in each loop iteration 213 | const uint8_t *p = tbl.get(); 214 | int addr = 0; 215 | while (--size >= 0) { 216 | addr += *p++; 217 | if (addr > f_lasti) { 218 | break; 219 | } 220 | line += *p++; 221 | } 222 | return static_cast(line); 223 | } 224 | 225 | // This method will fill the stack trace. Normally in the C API there are some 226 | // methods that you can use to extract the filename and line number from a frame 227 | // object. We implement the same logic here just using PTRACE_PEEKDATA. In 228 | // principle we could also execute code in the context of the process, but this 229 | // approach is harder to mess up. 230 | void FollowFrame(pid_t pid, unsigned long frame, std::vector *stack) { 231 | const long f_code = PtracePeek(pid, frame + offsetof(_frame, f_code)); 232 | const long co_filename = 233 | PtracePeek(pid, f_code + offsetof(PyCodeObject, co_filename)); 234 | const std::string filename = StringData(pid, co_filename); 235 | const long co_name = 236 | PtracePeek(pid, f_code + offsetof(PyCodeObject, co_name)); 237 | const std::string name = StringData(pid, co_name); 238 | 239 | stack->push_back({filename, name, GetLine(pid, frame, f_code)}); 240 | 241 | const long f_back = PtracePeek(pid, frame + offsetof(_frame, f_back)); 242 | if (f_back != 0) { 243 | FollowFrame(pid, f_back, stack); 244 | } 245 | } 246 | 247 | // N.B. To better understand how this method works, read the implementation of 248 | // pystate.c in the CPython source code. 249 | std::vector GetThreads(pid_t pid, PyAddresses addrs, 250 | bool enable_threads) { 251 | // Pointer to the current interpreter state. Python has a very rarely used 252 | // feature called "sub-interpreters", Pyflame only supports profiling a single 253 | // sub-interpreter. 254 | unsigned long istate = 0; 255 | 256 | // First try to get interpreter state via dereferencing 257 | // _PyThreadState_Current. This won't work if the main thread doesn't hold 258 | // the GIL (_Current will be null). 259 | unsigned long tstate = PtracePeek(pid, addrs.tstate_addr); 260 | unsigned long current_tstate = tstate; 261 | if (enable_threads) { 262 | if (tstate != 0) { 263 | istate = static_cast( 264 | PtracePeek(pid, tstate + offsetof(PyThreadState, interp))); 265 | // Secondly try to get it via the static interp_head symbol, if we managed 266 | // to find it: 267 | // - interp_head is not strictly speaking part of the public API so it 268 | // might get removed! 269 | // - interp_head is not part of the dynamic symbol table, so e.g. strip 270 | // will drop it 271 | } else if (addrs.interp_head_addr != 0) { 272 | istate = 273 | static_cast(PtracePeek(pid, addrs.interp_head_addr)); 274 | } else if (addrs.interp_head_hint != 0) { 275 | // Finally. check if we have already put a hint into interp_head_hint - 276 | // currently this can only happen if we called PyInterpreterState_Head. 277 | istate = addrs.interp_head_hint; 278 | } 279 | if (istate != 0) { 280 | tstate = static_cast( 281 | PtracePeek(pid, istate + offsetof(PyInterpreterState, tstate_head))); 282 | } 283 | } 284 | 285 | // Walk the thread list. 286 | std::vector threads; 287 | while (tstate != 0) { 288 | const unsigned long id = 289 | PtracePeek(pid, tstate + offsetof(PyThreadState, thread_id)); 290 | const bool is_current = tstate == current_tstate; 291 | 292 | // Dereference the thread's current frame. 293 | const unsigned long frame_addr = static_cast( 294 | PtracePeek(pid, tstate + offsetof(PyThreadState, frame))); 295 | 296 | std::vector stack; 297 | if (frame_addr != 0) { 298 | FollowFrame(pid, frame_addr, &stack); 299 | threads.push_back(Thread(id, is_current, stack)); 300 | } 301 | 302 | if (enable_threads) { 303 | tstate = PtracePeek(pid, tstate + offsetof(PyThreadState, next)); 304 | } else { 305 | tstate = 0; 306 | } 307 | }; 308 | 309 | return threads; 310 | } 311 | } // namespace py* 312 | } // namespace pyflame 313 | -------------------------------------------------------------------------------- /src/frob26.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // ABI for Python 2.6/2.7 16 | #define PYFLAME_PY_VERSION 26 17 | 18 | #include "./frob.cc" 19 | -------------------------------------------------------------------------------- /src/frob34.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // ABI for Python 3.4/3.5 16 | #define PYFLAME_PY_VERSION 34 17 | 18 | #include "./frob.cc" 19 | -------------------------------------------------------------------------------- /src/frob36.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // ABI for Python 3.6 16 | #define PYFLAME_PY_VERSION 36 17 | 18 | #include "./frob.cc" 19 | -------------------------------------------------------------------------------- /src/namespace.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "./namespace.h" 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | #include "./exc.h" 28 | #include "./posix.h" 29 | 30 | namespace { 31 | const char kOurMnt[] = "/proc/self/ns/mnt"; 32 | } 33 | 34 | namespace pyflame { 35 | Namespace::Namespace(pid_t pid) : out_(-1), in_(-1) { 36 | struct stat in_st; 37 | std::ostringstream os; 38 | os << "/proc/" << pid << "/ns/mnt"; 39 | const std::string their_mnt = os.str(); 40 | 41 | struct stat out_st; 42 | 43 | // In the case of no namespace support (ie ancient boxen), still make an 44 | // attempt to work 45 | if (lstat(kOurMnt, &out_st) < 0) { 46 | std::cerr << "Failed to lstat path " << kOurMnt << ": " << strerror(errno); 47 | out_ = in_ = -1; 48 | return; 49 | } 50 | 51 | // Since Linux 3.8 symbolic links are used. 52 | if (S_ISLNK(out_st.st_mode)) { 53 | char our_name[PATH_MAX]; 54 | ssize_t ourlen = readlink(kOurMnt, our_name, sizeof(our_name)); 55 | if (ourlen < 0) { 56 | std::ostringstream ss; 57 | ss << "Failed to readlink " << kOurMnt << ": " << strerror(errno); 58 | throw FatalException(ss.str()); 59 | } 60 | our_name[ourlen] = '\0'; 61 | 62 | char their_name[PATH_MAX]; 63 | ssize_t theirlen = 64 | readlink(their_mnt.c_str(), their_name, sizeof(their_name)); 65 | if (theirlen < 0) { 66 | std::ostringstream ss; 67 | ss << "Failed to readlink " << their_mnt.c_str() << ": " 68 | << strerror(errno); 69 | throw FatalException(ss.str()); 70 | } 71 | their_name[theirlen] = '\0'; 72 | 73 | if (strcmp(our_name, their_name) != 0) { 74 | out_ = OpenRdonly(kOurMnt); 75 | in_ = OpenRdonly(their_mnt.c_str()); 76 | } 77 | } else { 78 | // Before Linux 3.8 these are hard links. 79 | out_ = OpenRdonly(kOurMnt); 80 | Fstat(out_, &out_st); 81 | 82 | in_ = OpenRdonly(os.str().c_str()); 83 | Fstat(in_, &in_st); 84 | if (out_st.st_ino == in_st.st_ino) { 85 | Close(out_); 86 | Close(in_); 87 | out_ = in_ = -1; 88 | } 89 | } 90 | } 91 | 92 | int Namespace::Open(const char *path) { 93 | if (in_ != -1) { 94 | SetNs(in_); 95 | int fd = open(path, O_RDONLY); 96 | SetNs(out_); 97 | return fd; 98 | } else { 99 | return open(path, O_RDONLY); 100 | } 101 | } 102 | 103 | Namespace::~Namespace() { 104 | if (out_) { 105 | Close(out_); 106 | Close(in_); 107 | } 108 | } 109 | } // namespace pyflame 110 | -------------------------------------------------------------------------------- /src/namespace.h: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | namespace pyflame { 20 | // Implementation of a Linux filesystem namespace 21 | class Namespace { 22 | public: 23 | Namespace() = delete; 24 | explicit Namespace(pid_t pid); 25 | ~Namespace(); 26 | 27 | // Get a file descriptor in the namespace 28 | int Open(const char *path); 29 | 30 | private: 31 | int out_; // file descriptor that lets us return to our original namespace 32 | int in_; // file descriptor that lets us enter the target namespace 33 | }; 34 | } // namespace pyflame 35 | -------------------------------------------------------------------------------- /src/posix.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "./posix.h" 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | #include 25 | 26 | #include "./exc.h" 27 | #include "./setns.h" 28 | 29 | namespace pyflame { 30 | int OpenRdonly(const char *path) { 31 | int fd = open(path, O_RDONLY); 32 | if (fd < 0) { 33 | std::ostringstream ss; 34 | ss << "Failed to open " << path << ": " << strerror(errno); 35 | throw FatalException(ss.str()); 36 | } 37 | return fd; 38 | } 39 | 40 | void Close(int fd) { 41 | if (fd < 0) { 42 | return; 43 | } 44 | while (close(fd) == -1) 45 | ; 46 | } 47 | 48 | void Fstat(int fd, struct stat *buf) { 49 | if (fstat(fd, buf) < 0) { 50 | std::ostringstream ss; 51 | ss << "Failed to fstat file descriptor " << fd << ": " << strerror(errno); 52 | throw FatalException(ss.str()); 53 | } 54 | } 55 | 56 | void Lstat(const char *path, struct stat *buf) { 57 | if (lstat(path, buf) < 0) { 58 | std::ostringstream ss; 59 | ss << "Failed to lstat path " << path << ": " << strerror(errno); 60 | throw FatalException(ss.str()); 61 | } 62 | } 63 | 64 | void SetNs(int fd) { 65 | if (setns(fd, 0)) { 66 | std::ostringstream ss; 67 | ss << "Failed to setns " << fd << ": " << strerror(errno); 68 | throw FatalException(ss.str()); 69 | } 70 | } 71 | 72 | std::string ReadLink(const char *path) { 73 | char buf[PATH_MAX]; 74 | ssize_t nbytes = readlink(path, buf, sizeof(buf)); 75 | if (nbytes < 0) { 76 | std::ostringstream ss; 77 | ss << "Failed to read symlink " << path << ": " << strerror(errno); 78 | throw FatalException(ss.str()); 79 | } 80 | buf[nbytes] = '\0'; 81 | return {buf, static_cast(nbytes)}; 82 | } 83 | } // namespace pyflame 84 | -------------------------------------------------------------------------------- /src/posix.h: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | #include 19 | 20 | #include 21 | 22 | namespace pyflame { 23 | int OpenRdonly(const char *path); 24 | void Close(int fd); 25 | 26 | void Fstat(int fd, struct stat *buf); 27 | void Lstat(const char *path, struct stat *buf); 28 | 29 | void SetNs(int fd); 30 | 31 | std::string ReadLink(const char *path); 32 | } // namespace pyflame 33 | -------------------------------------------------------------------------------- /src/prober.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Evan Klitzke 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "./prober.h" 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | 36 | #include "./config.h" 37 | #include "./exc.h" 38 | #include "./ptrace.h" 39 | #include "./pyfrob.h" 40 | #include "./symbol.h" 41 | #include "./thread.h" 42 | 43 | // Microseconds in a second. 44 | static const char usage_str[] = 45 | ("Usage: pyflame [options] [-p] PID\n" 46 | " pyflame [options] -t command arg1 arg2...\n" 47 | "\n" 48 | "Common Options:\n" 49 | #ifdef ENABLE_THREADS 50 | " --threads Enable multi-threading support\n" 51 | " -d, --dump Dump stacks from all threads (implies --threads)\n" 52 | #else 53 | " -d, --dump Dump the current interpreter stack\n" 54 | #endif 55 | " -h, --help Show help\n" 56 | " -n, --no-line-numbers Do not append line numbers to function names\n" 57 | " -o, --output=PATH Output to file path\n" 58 | " -p, --pid=PID The PID to trace\n" 59 | " -r, --rate=RATE Sample rate, as a fractional value of seconds " 60 | "(default 0.01)\n" 61 | " -s, --seconds=SECS How many seconds to run for (default 1)\n" 62 | " -t, --trace Trace a child process\n" 63 | " -v, --version Show the version\n" 64 | " -x, --exclude-idle Exclude idle time from statistics\n" 65 | "\n" 66 | "Advanced Options:\n" 67 | " --abi Force a particular Python ABI (26, 34, 36)\n" 68 | " --flamechart Include timestamps for generating Chrome " 69 | "\"flamecharts\"\n"); 70 | 71 | // The ABIs supported in this Pyflame build. 72 | static const int build_abis[] = { 73 | #ifdef ENABLE_PY26 74 | 26, 75 | #endif 76 | #ifdef ENABLE_PY34 77 | 34, 78 | #endif 79 | #ifdef ENABLE_PY36 80 | 36, 81 | #endif 82 | }; 83 | 84 | static_assert(sizeof(build_abis) > 0, "No Python ABIs detected!"); 85 | 86 | static inline void ShowVersion(std::ostream &out) { 87 | const size_t sz = sizeof(build_abis) / sizeof(int); 88 | out << PYFLAME_VERSION_STR << " (ABI list: "; 89 | for (size_t i = 0; i < sz - 1; i++) { 90 | out << build_abis[i] << " "; 91 | } 92 | out << build_abis[sz - 1] << ")\n"; 93 | } 94 | 95 | static inline std::chrono::microseconds ToMicroseconds(double val) { 96 | return std::chrono::microseconds{static_cast(val * 1000000)}; 97 | } 98 | 99 | static inline bool EndsWith(std::string const &value, 100 | std::string const &ending) { 101 | if (ending.size() > value.size()) { 102 | return false; 103 | } 104 | return std::equal(ending.rbegin(), ending.rend(), value.rbegin()); 105 | } 106 | 107 | namespace pyflame { 108 | 109 | typedef std::unordered_map buckets_t; 110 | 111 | // Prints all stack traces 112 | static void PrintFrames(std::ostream &out, 113 | const std::vector &call_stacks, 114 | size_t idle_count, size_t failed_count, bool include_line_number) { 115 | // Choose function to print frame 116 | print_frame_t print_frame_ = include_line_number ? print_frame : print_frame_without_line_number; 117 | 118 | if (idle_count) { 119 | out << "(idle) " << idle_count << "\n"; 120 | } 121 | if (failed_count) { 122 | out << "(failed) " << failed_count << "\n"; 123 | } 124 | // Put the call stacks into buckets 125 | buckets_t buckets; 126 | for (const auto &call_stack : call_stacks) { 127 | auto bucket = buckets.find(call_stack.frames); 128 | if (bucket == buckets.end()) { 129 | buckets.insert(bucket, {call_stack.frames, 1}); 130 | } else { 131 | bucket->second++; 132 | } 133 | } 134 | // Process the frames 135 | for (const auto &kv : buckets) { 136 | if (kv.first.empty()) { 137 | std::cerr << "fatal error\n"; 138 | return; 139 | } 140 | auto last = kv.first.rend(); 141 | last--; 142 | for (auto it = kv.first.rbegin(); it != last; ++it) { 143 | print_frame_(out, *it); 144 | out << ";"; 145 | } 146 | print_frame_(out, *last); 147 | out << " " << kv.second << "\n"; 148 | } 149 | } 150 | 151 | // Prints all stack traces with timestamps 152 | static void PrintFramesTS(std::ostream &out, 153 | const std::vector &call_stacks, bool include_line_number) { 154 | // Choose function to print frame 155 | print_frame_t print_frame_ = include_line_number ? print_frame : print_frame_without_line_number; 156 | 157 | for (const auto &call_stack : call_stacks) { 158 | out << std::chrono::duration_cast( 159 | call_stack.ts.time_since_epoch()) 160 | .count() 161 | << "\n"; 162 | // Handle idle 163 | if (call_stack.frames.empty()) { 164 | out << "(idle)\n"; 165 | continue; 166 | } 167 | if (call_stack.frames.size() == 1 && 168 | call_stack.frames.front().file() == "(failed)") { 169 | out << "(failed)\n"; 170 | continue; 171 | } 172 | // Print the call stack 173 | for (auto it = call_stack.frames.rbegin(); it != call_stack.frames.rend(); 174 | ++it) { 175 | print_frame_(out, *it); 176 | out << ";"; 177 | } 178 | out << "\n"; 179 | } 180 | } 181 | 182 | int Prober::ParseOpts(int argc, char **argv) { 183 | static const char short_opts[] = "dhno:p:r:s:tvx"; 184 | static struct option long_opts[] = { 185 | {"abi", required_argument, 0, 'a'}, 186 | {"dump", no_argument, 0, 'd'}, 187 | {"help", no_argument, 0, 'h'}, 188 | {"rate", required_argument, 0, 'r'}, 189 | {"seconds", required_argument, 0, 's'}, 190 | #if ENABLE_THREADS 191 | {"threads", no_argument, 0, 'L'}, 192 | #endif 193 | {"no-line-numbers", no_argument, 0, 'n'}, 194 | {"output", required_argument, 0, 'o'}, 195 | {"pid", required_argument, 0, 'p'}, 196 | {"trace", no_argument, 0, 't'}, 197 | {"flamechart", no_argument, 0, 'T'}, 198 | {"version", no_argument, 0, 'v'}, 199 | {"exclude-idle", no_argument, 0, 'x'}, 200 | {0, 0, 0, 0} 201 | }; 202 | 203 | long abi_version; 204 | for (;;) { 205 | int c = getopt_long(argc, argv, short_opts, long_opts, nullptr); 206 | if (c == -1) { 207 | break; 208 | } 209 | switch (c) { 210 | case 'a': 211 | abi_version = std::strtol(optarg, nullptr, 10); 212 | switch (abi_version) { 213 | case 26: 214 | case 27: 215 | abi_ = PyABI::Py26; 216 | break; 217 | case 34: 218 | case 35: 219 | abi_ = PyABI::Py34; 220 | break; 221 | case 36: 222 | abi_ = PyABI::Py36; 223 | break; 224 | default: 225 | std::cerr << "Unknown or unsupported ABI version: " << abi_version 226 | << "\n"; 227 | return 1; 228 | break; 229 | } 230 | break; 231 | case 'd': 232 | dump_ = true; 233 | #if ENABLE_THREADS 234 | enable_threads_ = true; 235 | #endif 236 | break; 237 | case 'h': 238 | std::cout << PYFLAME_VERSION_STR << "\n\n" << usage_str; 239 | return 0; 240 | break; 241 | #ifdef ENABLE_THREADS 242 | case 'L': 243 | enable_threads_ = true; 244 | break; 245 | #endif 246 | case 'p': 247 | if ((pid_ = ParsePid(optarg)) == -1) { 248 | return 1; 249 | } 250 | break; 251 | case 'r': 252 | sample_rate_ = std::stod(optarg); 253 | break; 254 | case 's': 255 | seconds_ = std::stod(optarg); 256 | break; 257 | case 't': 258 | trace_ = true; 259 | seconds_ = -1; 260 | goto finish_arg_parse; 261 | break; 262 | case 'T': 263 | include_ts_ = true; 264 | break; 265 | case 'v': 266 | ShowVersion(std::cout); 267 | return 0; 268 | break; 269 | case 'x': 270 | include_idle_ = false; 271 | break; 272 | case 'o': 273 | output_file_ = optarg; 274 | break; 275 | case 'n': 276 | include_line_number_ = false; 277 | break; 278 | case '?': 279 | // getopt_long should already have printed an error message 280 | break; 281 | default: 282 | std::cerr << "unrecognized command line flag: " << optarg << "\n"; 283 | abort(); 284 | } 285 | } 286 | finish_arg_parse: 287 | if (trace_) { 288 | if (dump_) { 289 | std::cerr << "Options -t and -d are not mutually compatible.\n"; 290 | return 1; 291 | } else if (pid_ != -1) { 292 | std::cerr << "Options -t and -p are not mutually compatible.\n"; 293 | return 1; 294 | } else if (optind == argc) { 295 | std::cerr << "Option -t requires a command to run.\n\n"; 296 | std::cerr << usage_str; 297 | return 1; 298 | } 299 | trace_target_ = argv[optind]; 300 | } else if (pid_ == -1) { 301 | // Users should use -p to supply the PID to trace. However, older versions 302 | // of Pyflame used a convention where the PID to trace was the final 303 | // argument to the pyflame command. This code path handles this legacy use 304 | // case, to preserve backward compatibility. 305 | if (optind != argc - 1 || (pid_ = ParsePid(argv[optind])) == -1) { 306 | std::cerr << usage_str; 307 | return 1; 308 | } 309 | std::cerr << "WARNING: Specifying a PID to trace without -p is deprecated; " 310 | "see Pyflame issue #99 for details.\n"; 311 | } 312 | interval_ = ToMicroseconds(sample_rate_); 313 | return -1; 314 | } 315 | 316 | int Prober::InitiatePtrace(char **argv) { 317 | if (trace_) { 318 | if (EndsWith(trace_target_, "pyflame")) { 319 | std::cerr << "You tried to pyflame a pyflame, naughty!\n"; 320 | return 1; 321 | } 322 | // In trace mode, all of the remaining arguments are a command to run. We 323 | // fork and have the child run the command; the parent traces. 324 | pid_ = fork(); 325 | if (pid_ == -1) { 326 | perror("fork()"); 327 | return 1; 328 | } else if (pid_ == 0) { 329 | // Child: request to be traced. 330 | PtraceTraceme(); 331 | if (execvp(trace_target_.c_str(), argv + optind)) { 332 | std::cerr << "execvp() failed for: " << trace_target_ 333 | << ", err = " << strerror(errno) << "\n"; 334 | return 1; 335 | } 336 | } else { 337 | // Parent: we trace the child until it's exec'ed the new process before 338 | // proceeding. For a dynamically linked Python build, there's still a race 339 | // condition between when the exec() happens and when symbols are 340 | // available. But there's no point in polling the child until it's at 341 | // least had a chance to run exec. 342 | pid_t child = waitpid(0, nullptr, 0); 343 | assert(child == pid_); 344 | PtraceSetOptions(pid_, PTRACE_O_TRACEEXEC); 345 | PtraceCont(pid_); 346 | int status = 0; 347 | while (!SawEventExec(status)) { 348 | pid_t p = waitpid(-1, &status, 0); 349 | if (p == -1) { 350 | perror("waitpid()"); 351 | return 1; 352 | } 353 | if (WIFEXITED(status)) { 354 | std::cerr << "Child process exited with status: " 355 | << WEXITSTATUS(status) << "\n"; 356 | return 1; 357 | } 358 | } 359 | // We can only use PtraceInterrupt, used later in the main loop, if the 360 | // process was seized. So we reattach and seize. 361 | PtraceDetach(pid_); 362 | PtraceSeize(pid_); 363 | } 364 | } else { 365 | try { 366 | PtraceSeize(pid_); 367 | } catch (const PtraceException &err) { 368 | std::cerr << "Failed to seize PID " << pid_ << "\n"; 369 | return 1; 370 | } 371 | } 372 | PtraceInterrupt(pid_); 373 | return 0; 374 | } 375 | 376 | int Prober::Run(const PyFrob &frobber) { 377 | std::unique_ptr file_ptr; 378 | std::ostream *output; 379 | if (output_file_.empty()) { 380 | output = &std::cout; 381 | } else { 382 | file_ptr.reset(new std::ofstream); 383 | file_ptr->open(output_file_, std::ios::out | std::ios::trunc); 384 | if (file_ptr->is_open()) { 385 | output = file_ptr.get(); 386 | } else { 387 | std::cerr << "cannot open file \"" << output_file_ << "\" as output\n"; 388 | return 1; 389 | } 390 | } 391 | return dump_ ? DumpStacks(frobber, output) : ProbeLoop(frobber, output); 392 | } 393 | 394 | // Main loop to probe the Python process. 395 | int Prober::ProbeLoop(const PyFrob &frobber, std::ostream *out) { 396 | std::vector call_stacks; 397 | int return_code = 0; 398 | size_t idle_count = 0; 399 | size_t failed_count = 0; 400 | bool check_end = seconds_ >= 0; 401 | auto end = std::chrono::system_clock::now() + ToMicroseconds(seconds_); 402 | for (;;) { 403 | auto now = std::chrono::system_clock::now(); 404 | try { 405 | std::vector threads = frobber.GetThreads(); 406 | 407 | // Only true for non-GIL stacks that we couldn't find a way to profile 408 | // Currently this means stripped builds on non-AMD64 archs 409 | if (threads.empty() && include_idle_) { 410 | idle_count++; 411 | // Timestamp empty call stacks only if required. Since lots of time the 412 | // process will be idle, this is a good optimization to have. 413 | if (include_ts_) { 414 | call_stacks.push_back({now, {}}); 415 | } 416 | } 417 | 418 | for (const auto &thread : threads) { 419 | call_stacks.push_back({now, thread.frames()}); 420 | } 421 | 422 | if (check_end && (now + interval_ >= end)) { 423 | break; 424 | } 425 | PtraceCont(pid_); 426 | std::this_thread::sleep_for(interval_); 427 | PtraceInterrupt(pid_); 428 | } catch (const TerminateException &exc) { 429 | // If the process terminates early then we just print the stack traces up 430 | // until that point in time. 431 | goto finish; 432 | } catch (const PtraceException &exc) { 433 | failed_count++; 434 | if (include_ts_) { 435 | // include the exact failures in the call stacks 436 | call_stacks.push_back({now, {{"(failed)", exc.what(), 0}}}); 437 | } 438 | std::cerr << "Unexpected ptrace(2) exception: " << exc.what() << "\n"; 439 | } catch (const std::exception &exc) { 440 | std::cerr << "Unexpected generic exception: " << exc.what() << "\n"; 441 | return_code = 1; 442 | goto finish; 443 | } 444 | } 445 | finish: 446 | if (!call_stacks.empty() || idle_count || failed_count) { 447 | if (!include_ts_) { 448 | PrintFrames(*out, call_stacks, idle_count, failed_count, include_line_number_); 449 | } else { 450 | PrintFramesTS(*out, call_stacks, include_line_number_); 451 | } 452 | } 453 | return return_code; 454 | } 455 | 456 | int Prober::DumpStacks(const PyFrob &frobber, std::ostream *out) { 457 | std::vector threads = frobber.GetThreads(); 458 | for (size_t i = 0; i < threads.size(); i++) { 459 | *out << threads[i]; 460 | if (i < threads.size() - 1) { 461 | *out << "\n"; 462 | } 463 | } 464 | return 0; 465 | } 466 | 467 | int Prober::FindSymbols(PyFrob *frobber) { 468 | // When tracing a dynamically linked Python build, it may take a while for 469 | // ld.so to actually load symbols into the process. Therefore we retry probing 470 | // in a loop, until the symbols are loaded. A more reliable way of doing this 471 | // would be to break at entry to a known static function (e.g. Py_Main), but 472 | // this isn't reliable in all cases. For instance, /usr/bin/python{,3} will 473 | // start at Py_Main, but uWSGI will not. 474 | try { 475 | for (size_t i = 0;;) { 476 | if (frobber->DetectABI(abi_)) { 477 | if (++i >= MaxRetries()) { 478 | std::cerr << "Failed to locate libpython within timeout period.\n"; 479 | return 1; 480 | } 481 | PtraceCont(pid_); 482 | std::this_thread::sleep_for(interval_); 483 | PtraceInterrupt(pid_); 484 | continue; 485 | } 486 | break; 487 | } 488 | } catch (const FatalException &exc) { 489 | std::cerr << exc.what() << "\n"; 490 | return 1; 491 | } 492 | return 0; 493 | } 494 | 495 | pid_t Prober::ParsePid(const char *pid_str) { 496 | long pid = std::strtol(pid_str, nullptr, 10); 497 | if (pid <= 0 || pid > std::numeric_limits::max()) { 498 | std::cerr << "Error: failed to parse \"" << pid_str << "\" as a PID.\n\n"; 499 | return -1; 500 | } 501 | return static_cast(pid); 502 | } 503 | } // namespace pyflame 504 | -------------------------------------------------------------------------------- /src/prober.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Evan Klitzke 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | #include 19 | 20 | #include "./pyfrob.h" 21 | #include "./symbol.h" 22 | 23 | // Maximum number of times to retry checking for Python symbols when -p is used. 24 | #define MAX_ATTACH_RETRIES 1 25 | 26 | // Maximum number of times to retry checking for Python symbols when -t is used. 27 | #define MAX_TRACE_RETRIES 50 28 | 29 | namespace pyflame { 30 | 31 | class Prober { 32 | public: 33 | Prober() 34 | : abi_(PyABI::Unknown), 35 | pid_(-1), 36 | dump_(false), 37 | trace_(false), 38 | include_idle_(true), 39 | include_ts_(false), 40 | include_line_number_(true), 41 | enable_threads_(false), 42 | seconds_(1), 43 | sample_rate_(0.01) {} 44 | Prober(const Prober &other) = delete; 45 | 46 | int ParseOpts(int argc, char **argv); 47 | 48 | int InitiatePtrace(char **argv); 49 | 50 | int FindSymbols(PyFrob *frobber); 51 | 52 | int Run(const PyFrob &frobber); 53 | 54 | inline bool enable_threads() const { return enable_threads_; } 55 | inline pid_t pid() const { return pid_; } 56 | 57 | private: 58 | PyABI abi_; 59 | pid_t pid_; 60 | bool dump_; 61 | bool trace_; 62 | bool include_idle_; 63 | bool include_ts_; 64 | bool include_line_number_; 65 | bool enable_threads_; 66 | double seconds_; 67 | double sample_rate_; 68 | std::chrono::microseconds interval_; 69 | std::string output_file_; 70 | std::string trace_target_; 71 | 72 | pid_t ParsePid(const char *pid_str); 73 | 74 | int ProbeLoop(const PyFrob &frobber, std::ostream *out); 75 | 76 | int DumpStacks(const PyFrob &frobber, std::ostream *out); 77 | 78 | inline size_t MaxRetries() const { 79 | return trace_ ? MAX_TRACE_RETRIES : MAX_ATTACH_RETRIES; 80 | } 81 | }; 82 | } // namespace pyflame 83 | -------------------------------------------------------------------------------- /src/ptrace.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "./ptrace.h" 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | #include "./exc.h" 35 | 36 | namespace pyflame { 37 | int DoWait(pid_t pid, int options) { 38 | int status; 39 | std::ostringstream ss; 40 | for (;;) { 41 | pid_t progeny = waitpid(pid, &status, options); 42 | if (progeny == -1) { 43 | ss << "Failed to waitpid(): " << strerror(errno); 44 | throw PtraceException(ss.str()); 45 | } 46 | assert(progeny == pid); 47 | if (WIFSTOPPED(status)) { 48 | int signum = WSTOPSIG(status); 49 | if (signum == SIGTRAP) { 50 | break; 51 | } else if (signum == SIGCHLD) { 52 | PtraceCont(pid); // see issue #122 53 | continue; 54 | } 55 | ss << "waitpid() indicated a WIFSTOPPED process, but got unexpected " 56 | "signal " 57 | << signum; 58 | throw PtraceException(ss.str()); 59 | } else if (WIFEXITED(status)) { 60 | ss << "Child process " << pid << " exited with status " 61 | << WEXITSTATUS(status); 62 | throw TerminateException(ss.str()); 63 | } else { 64 | ss << "Child process " << pid 65 | << " returned an unexpected waitpid() code: " << status; 66 | throw PtraceException(ss.str()); 67 | } 68 | } 69 | return status; 70 | } 71 | 72 | bool SawEventExec(int status) { 73 | return status >> 8 == (SIGTRAP | (PTRACE_EVENT_EXEC << 8)); 74 | } 75 | 76 | void PtraceTraceme() { 77 | if (ptrace(PTRACE_TRACEME, getpid(), 0, 0)) { 78 | throw PtraceException("Failed to PTRACE_TRACEME"); 79 | } 80 | raise(SIGSTOP); 81 | } 82 | 83 | void PtraceAttach(pid_t pid) { 84 | if (ptrace(PTRACE_ATTACH, pid, 0, 0)) { 85 | std::ostringstream ss; 86 | ss << "Failed to attach to PID " << pid << ": " << strerror(errno); 87 | throw PtraceException(ss.str()); 88 | } 89 | int status; 90 | if (waitpid(pid, &status, __WALL) != pid || !WIFSTOPPED(status)) { 91 | std::ostringstream ss; 92 | ss << "Failed to wait on PID " << pid << ": " << strerror(errno); 93 | throw PtraceException(ss.str()); 94 | } 95 | } 96 | 97 | void PtraceSeize(pid_t pid) { 98 | if (ptrace(PTRACE_SEIZE, pid, 0, 0)) { 99 | std::ostringstream ss; 100 | ss << "Failed to attach to PID " << pid << ": " << strerror(errno); 101 | throw PtraceException(ss.str()); 102 | } 103 | } 104 | 105 | void PtraceDetach(pid_t pid) { 106 | if (ptrace(PTRACE_DETACH, pid, 0, 0)) { 107 | std::ostringstream ss; 108 | ss << "Failed to detach PID " << pid << ": " << strerror(errno); 109 | throw PtraceException(ss.str()); 110 | } 111 | } 112 | 113 | // Like PtraceDetach(), but ignore errors. 114 | static inline void SafeDetach(pid_t pid) noexcept { 115 | ptrace(PTRACE_DETACH, pid, 0, 0); 116 | } 117 | 118 | void PtraceInterrupt(pid_t pid) { 119 | if (ptrace(PTRACE_INTERRUPT, pid, 0, 0)) { 120 | throw PtraceException("Failed to PTRACE_INTERRUPT"); 121 | } 122 | DoWait(pid); 123 | } 124 | 125 | user_regs_struct PtraceGetRegs(pid_t pid) { 126 | user_regs_struct regs; 127 | if (ptrace(PTRACE_GETREGS, pid, 0, ®s)) { 128 | std::ostringstream ss; 129 | ss << "Failed to PTRACE_GETREGS: " << strerror(errno); 130 | throw PtraceException(ss.str()); 131 | } 132 | return regs; 133 | } 134 | 135 | void PtraceSetRegs(pid_t pid, user_regs_struct regs) { 136 | if (ptrace(PTRACE_SETREGS, pid, 0, ®s)) { 137 | std::ostringstream ss; 138 | ss << "Failed to PTRACE_SETREGS: " << strerror(errno); 139 | throw PtraceException(ss.str()); 140 | } 141 | } 142 | 143 | void PtracePoke(pid_t pid, unsigned long addr, long data) { 144 | if (ptrace(PTRACE_POKEDATA, pid, addr, (void *)data)) { 145 | std::ostringstream ss; 146 | ss << "Failed to PTRACE_POKEDATA at " << reinterpret_cast(addr) 147 | << ": " << strerror(errno); 148 | throw PtraceException(ss.str()); 149 | } 150 | } 151 | 152 | long PtracePeek(pid_t pid, unsigned long addr) { 153 | errno = 0; 154 | const long data = ptrace(PTRACE_PEEKDATA, pid, addr, 0); 155 | if (data == -1 && errno != 0) { 156 | std::ostringstream ss; 157 | ss << "Failed to PTRACE_PEEKDATA (pid " << pid << ", addr " 158 | << reinterpret_cast(addr) << "): " << strerror(errno); 159 | throw PtraceException(ss.str()); 160 | } 161 | return data; 162 | } 163 | 164 | void PtraceSetOptions(pid_t pid, long options) { 165 | if (ptrace(PTRACE_SETOPTIONS, pid, 0, options)) { 166 | throw PtraceException("Failed to PTRACE_SETOPTIONS"); 167 | } 168 | } 169 | 170 | void PtraceCont(pid_t pid) { 171 | if (ptrace(PTRACE_CONT, pid, 0, 0) == -1) { 172 | std::ostringstream ss; 173 | ss << "Failed to PTRACE_CONT: " << strerror(errno); 174 | throw PtraceException(ss.str()); 175 | } 176 | } 177 | 178 | void PtraceSingleStep(pid_t pid) { 179 | if (ptrace(PTRACE_SINGLESTEP, pid, 0, 0) == -1) { 180 | std::ostringstream ss; 181 | ss << "Failed to PTRACE_SINGLESTEP: " << strerror(errno); 182 | throw PtraceException(ss.str()); 183 | } 184 | DoWait(pid); 185 | } 186 | 187 | std::string PtracePeekString(pid_t pid, unsigned long addr) { 188 | std::ostringstream dump; 189 | unsigned long off = 0; 190 | while (true) { 191 | const long val = PtracePeek(pid, addr + off); 192 | 193 | // XXX: this can be micro-optimized, c.f. 194 | // https://graphics.stanford.edu/~seander/bithacks.html#ZeroInWord 195 | const std::string chunk(reinterpret_cast(&val), sizeof(val)); 196 | dump << chunk.c_str(); 197 | if (chunk.find_first_of('\0') != std::string::npos) { 198 | break; 199 | } 200 | off += sizeof(val); 201 | } 202 | return dump.str(); 203 | } 204 | 205 | std::unique_ptr PtracePeekBytes(pid_t pid, unsigned long addr, 206 | size_t nbytes) { 207 | // align the buffer to a word size 208 | if (nbytes % sizeof(long)) { 209 | nbytes = (nbytes / sizeof(long) + 1) * sizeof(long); 210 | } 211 | std::unique_ptr bytes(new uint8_t[nbytes]); 212 | 213 | size_t off = 0; 214 | while (off < nbytes) { 215 | const long val = PtracePeek(pid, addr + off); 216 | memmove(bytes.get() + off, &val, sizeof(val)); 217 | off += sizeof(val); 218 | } 219 | return bytes; 220 | } 221 | 222 | #if defined(__amd64__) && ENABLE_THREADS 223 | static const long syscall_x86 = 0x050f; // x86 code for SYSCALL 224 | 225 | static unsigned long probe_ = 0; 226 | 227 | static unsigned long AllocPage(pid_t pid) { 228 | user_regs_struct oldregs = PtraceGetRegs(pid); 229 | long orig_code = PtracePeek(pid, oldregs.rip); 230 | PtracePoke(pid, oldregs.rip, syscall_x86); 231 | 232 | user_regs_struct newregs = oldregs; 233 | newregs.rax = SYS_mmap; 234 | newregs.rdi = 0; // addr 235 | newregs.rsi = getpagesize(); // len 236 | newregs.rdx = PROT_READ | PROT_WRITE | PROT_EXEC; // prot 237 | newregs.r10 = MAP_PRIVATE | MAP_ANONYMOUS; // flags 238 | newregs.r8 = -1; // fd 239 | newregs.r9 = 0; // offset 240 | PtraceSetRegs(pid, newregs); 241 | PtraceSingleStep(pid); 242 | unsigned long result = PtraceGetRegs(pid).rax; 243 | 244 | PtraceSetRegs(pid, oldregs); 245 | PtracePoke(pid, oldregs.rip, orig_code); 246 | 247 | return result; 248 | } 249 | 250 | static std::vector ListThreads(pid_t pid) { 251 | std::vector result; 252 | std::ostringstream dirname; 253 | dirname << "/proc/" << pid << "/task"; 254 | DIR *dir = opendir(dirname.str().c_str()); 255 | if (dir == nullptr) { 256 | throw PtraceException("Failed to list threads"); 257 | } 258 | dirent *entry; 259 | while ((entry = readdir(dir)) != nullptr) { 260 | std::string name = entry->d_name; 261 | if (name != "." && name != "..") { 262 | result.push_back(static_cast(std::stoi(name))); 263 | } 264 | } 265 | return result; 266 | } 267 | 268 | static void PauseChildThreads(pid_t pid) { 269 | for (auto tid : ListThreads(pid)) { 270 | if (tid != pid) PtraceAttach(tid); 271 | } 272 | } 273 | 274 | static void ResumeChildThreads(pid_t pid) { 275 | for (auto tid : ListThreads(pid)) { 276 | if (tid != pid) PtraceDetach(tid); 277 | } 278 | } 279 | 280 | long PtraceCallFunction(pid_t pid, unsigned long addr) { 281 | if (probe_ == 0) { 282 | PauseChildThreads(pid); 283 | probe_ = AllocPage(pid); 284 | ResumeChildThreads(pid); 285 | if (probe_ == (unsigned long)MAP_FAILED) { 286 | return -1; 287 | } 288 | 289 | long code = 0; 290 | uint8_t *new_code_bytes = (uint8_t *)&code; 291 | new_code_bytes[0] = 0xff; // CALL 292 | new_code_bytes[1] = 0xd0; // rax 293 | new_code_bytes[2] = 0xcc; // TRAP 294 | PtracePoke(pid, probe_, code); 295 | } 296 | 297 | user_regs_struct oldregs = PtraceGetRegs(pid); 298 | user_regs_struct newregs = oldregs; 299 | newregs.rax = addr; 300 | newregs.rip = probe_; 301 | 302 | PtraceSetRegs(pid, newregs); 303 | PtraceCont(pid); 304 | DoWait(pid); 305 | 306 | newregs = PtraceGetRegs(pid); 307 | PtraceSetRegs(pid, oldregs); 308 | return newregs.rax; 309 | }; 310 | 311 | void PtraceCleanup(pid_t pid) noexcept { 312 | // Clean up the memory area allocated by AllocPage(). 313 | if (probe_ != 0) { 314 | try { 315 | const user_regs_struct oldregs = PtraceGetRegs(pid); 316 | const long orig_code = PtracePeek(pid, oldregs.rip); 317 | 318 | user_regs_struct newregs = oldregs; 319 | newregs.rax = SYS_munmap; 320 | newregs.rdi = probe_; // addr 321 | newregs.rsi = getpagesize(); // len 322 | 323 | // Prepare to run munmap(2) syscall. 324 | PauseChildThreads(pid); 325 | PtracePoke(pid, oldregs.rip, syscall_x86); 326 | do_munmap: 327 | PtraceSetRegs(pid, newregs); 328 | 329 | // Actually call munmap(2), and check the return value. 330 | PtraceSingleStep(pid); 331 | const long rax = PtraceGetRegs(pid).rax; 332 | switch (rax) { 333 | case 0: 334 | probe_ = 0; 335 | break; 336 | case EAGAIN: 337 | goto do_munmap; 338 | break; 339 | default: 340 | std::cerr << "Warning: failed to munmap(2) trampoline page, %rax = " 341 | << rax << "\n"; 342 | break; 343 | } 344 | 345 | // Clean up and resume the child threads. 346 | PtracePoke(pid, oldregs.rip, orig_code); 347 | PtraceSetRegs(pid, oldregs); 348 | ResumeChildThreads(pid); 349 | } catch (...) { 350 | // If the process has already exited, then we'll get a ptrace error, which 351 | // can be safely ignored. This *should* happen at the initial 352 | // PtraceGetRegs() call, but we wrap the entire block to be safe. 353 | } 354 | } 355 | 356 | SafeDetach(pid); 357 | } 358 | #else 359 | void PtraceCleanup(pid_t pid) noexcept { SafeDetach(pid); } 360 | #endif 361 | 362 | } // namespace pyflame 363 | -------------------------------------------------------------------------------- /src/ptrace.h: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | #include 23 | 24 | #include "./config.h" 25 | 26 | #if defined(__arm__) 27 | typedef struct user_regs user_regs_struct; 28 | #else 29 | typedef struct user_regs_struct user_regs_struct; 30 | #endif 31 | 32 | namespace pyflame { 33 | 34 | int DoWait(pid_t pid, int options = 0); 35 | 36 | bool SawEventExec(int status); 37 | 38 | void PtraceTraceme(); 39 | 40 | // detach a process 41 | void PtraceAttach(pid_t pid); 42 | void PtraceDetach(pid_t pid); 43 | 44 | void PtraceSeize(pid_t pid); 45 | void PtraceInterrupt(pid_t pid); 46 | 47 | // get regs from a process 48 | user_regs_struct PtraceGetRegs(pid_t pid); 49 | 50 | // set regs in a process 51 | void PtraceSetRegs(pid_t pid, user_regs_struct regs); 52 | 53 | // poke a long word into an address 54 | void PtracePoke(pid_t pid, unsigned long addr, long data); 55 | 56 | // read the long word at an address 57 | long PtracePeek(pid_t pid, unsigned long addr); 58 | 59 | void PtraceSetOptions(pid_t pid, long options); 60 | 61 | // peek a null-terminated string 62 | std::string PtracePeekString(pid_t pid, unsigned long addr); 63 | 64 | // peek some number of bytes 65 | std::unique_ptr PtracePeekBytes(pid_t pid, unsigned long addr, 66 | size_t nbytes); 67 | 68 | // Continue a traced process 69 | void PtraceCont(pid_t pid); 70 | 71 | // Execute a single instruction in a traced process 72 | void PtraceSingleStep(pid_t pid); 73 | 74 | #if ENABLE_THREADS 75 | // Call a function pointer. 76 | long PtraceCallFunction(pid_t pid, unsigned long addr); 77 | #endif 78 | 79 | // Detach, and maybe dealloc the page allocated in PtraceCallFunction(); 80 | void PtraceCleanup(pid_t pid) noexcept; 81 | } // namespace pyflame 82 | -------------------------------------------------------------------------------- /src/pyflame.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "./prober.h" 19 | 20 | using namespace pyflame; 21 | 22 | int main(int argc, char **argv) { 23 | Prober prober; 24 | int ret = prober.ParseOpts(argc, argv); 25 | if (ret != -1) { 26 | return ret; 27 | } 28 | if (prober.InitiatePtrace(argv)) { 29 | return 1; 30 | } 31 | PyFrob frobber(prober.pid(), prober.enable_threads()); 32 | if (prober.FindSymbols(&frobber)) { 33 | return 1; 34 | } 35 | 36 | // Probe in a loop. 37 | return prober.Run(frobber); 38 | } 39 | -------------------------------------------------------------------------------- /src/pyfrob.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "./pyfrob.h" 16 | 17 | #include 18 | #include 19 | 20 | #include "./aslr.h" 21 | #include "./config.h" 22 | #include "./exc.h" 23 | #include "./namespace.h" 24 | #include "./posix.h" 25 | #include "./ptrace.h" 26 | #include "./symbol.h" 27 | 28 | #define FROB_FUNCS \ 29 | std::vector GetThreads(pid_t pid, PyAddresses addr, \ 30 | bool enable_threads); 31 | 32 | namespace pyflame { 33 | namespace { 34 | // locate within libpython 35 | PyAddresses AddressesFromLibPython(pid_t pid, const std::string &libpython, 36 | Namespace *ns, PyABI *abi) { 37 | std::string elf_path; 38 | const size_t offset = LocateLibPython(pid, libpython, &elf_path); 39 | if (offset == 0) { 40 | std::ostringstream ss; 41 | ss << "Failed to locate libpython named " << libpython; 42 | throw SymbolException(ss.str()); 43 | } 44 | 45 | ELF pyelf; 46 | pyelf.Open(elf_path, ns); 47 | pyelf.Parse(); 48 | const PyAddresses addrs = pyelf.GetAddresses(abi); 49 | if (addrs.empty()) { 50 | throw SymbolException("Failed to locate addresses"); 51 | } 52 | return addrs + offset; 53 | } 54 | 55 | PyAddresses Addrs(pid_t pid, Namespace *ns, PyABI *abi) { 56 | std::ostringstream ss; 57 | ss << "/proc/" << pid << "/exe"; 58 | ELF target; 59 | std::string exe = ReadLink(ss.str().c_str()); 60 | target.Open(exe, ns); 61 | target.Parse(); 62 | 63 | // There's two different cases here. The default way Python is compiled you 64 | // get a "static" build which means that you get a big several-megabytes 65 | // Python executable that has all of the symbols statically built in. For 66 | // instance, this is how Python is built on Debian and Ubuntu. This is the 67 | // easiest case to handle, since in this case there are no tricks, we just 68 | // need to find the symbol in the ELF file. 69 | // 70 | // There's also a configure option called --enable-shared where you get a 71 | // small several-kilobytes Python executable that links against a 72 | // several-megabytes libpython2.7.so. This is how Python is built on Fedora. 73 | // If that's the case we need to do some fiddly things to find the true symbol 74 | // location. 75 | // 76 | // The code here attempts to detect if the executable links against 77 | // libpython2.7.so, and if it does the libpython variable will be filled with 78 | // the full soname. That determines where we need to look to find our symbol 79 | // table. 80 | 81 | PyAddresses addrs = target.GetAddresses(abi); 82 | if (addrs) { 83 | if (addrs.pie) { 84 | // If Python executable is PIE, add offsets 85 | std::string elf_path; 86 | const size_t offset = LocateLibPython(pid, exe, &elf_path); 87 | return addrs + offset; 88 | } else { 89 | return addrs; 90 | } 91 | } 92 | 93 | std::string libpython; 94 | for (const auto &lib : target.NeededLibs()) { 95 | if (lib.find("libpython") != std::string::npos) { 96 | libpython = lib; 97 | break; 98 | } 99 | } 100 | if (!libpython.empty()) { 101 | return AddressesFromLibPython(pid, libpython, ns, abi); 102 | } 103 | // A process like uwsgi may use dlopen() to load libpython... let's just guess 104 | // that the DSO is called libpython2.7.so 105 | // 106 | // XXX: this won't work if the embedding language is Python 3 107 | return AddressesFromLibPython(pid, "libpython2.7.so", ns, abi); 108 | } 109 | } // namespace 110 | 111 | #ifdef ENABLE_PY26 112 | namespace py26 { 113 | FROB_FUNCS 114 | } 115 | #endif 116 | 117 | #ifdef ENABLE_PY34 118 | namespace py34 { 119 | FROB_FUNCS 120 | } 121 | #endif 122 | 123 | #ifdef ENABLE_PY36 124 | namespace py36 { 125 | FROB_FUNCS 126 | } 127 | #endif 128 | 129 | // Fill the addrs_ member 130 | int PyFrob::set_addrs_(PyABI *abi) { 131 | Namespace ns(pid_); 132 | try { 133 | addrs_ = Addrs(pid_, &ns, abi); 134 | } catch (const SymbolException &exc) { 135 | return 1; 136 | } 137 | #if ENABLE_THREADS 138 | // If we didn't find the interp_head address, but we did find the public 139 | // PyInterpreterState_Head 140 | // function, use evil non-portable ptrace tricks to call the function 141 | if (enable_threads_ && addrs_.interp_head_addr == 0 && 142 | addrs_.interp_head_hint == 0 && addrs_.interp_head_fn_addr != 0) { 143 | addrs_.interp_head_hint = 144 | PtraceCallFunction(pid_, addrs_.interp_head_fn_addr); 145 | } 146 | #endif 147 | return 0; 148 | } 149 | 150 | int PyFrob::DetectABI(PyABI abi) { 151 | // Set up the function pointers. By default, we auto-detect the ABI. If an ABI 152 | // is explicitly passed to us, then use that one (even though it could be 153 | // wrong)! 154 | if (set_addrs_(abi == PyABI::Unknown ? &abi : nullptr)) { 155 | return 1; 156 | } 157 | switch (abi) { 158 | case PyABI::Unknown: 159 | throw FatalException("Failed to detect a Python ABI."); 160 | break; 161 | #ifdef ENABLE_PY26 162 | case PyABI::Py26: 163 | get_threads_ = py26::GetThreads; 164 | break; 165 | #endif 166 | #ifdef ENABLE_PY34 167 | case PyABI::Py34: 168 | get_threads_ = py34::GetThreads; 169 | break; 170 | #endif 171 | #ifdef ENABLE_PY36 172 | case PyABI::Py36: 173 | get_threads_ = py36::GetThreads; 174 | break; 175 | #endif 176 | default: 177 | std::ostringstream os; 178 | os << "Target has Python ABI " << static_cast(abi) 179 | << ", which is not supported by this pyflame build."; 180 | throw FatalException(os.str()); 181 | } 182 | 183 | if (addrs_.empty()) { 184 | throw FatalException("DetectABI(): addrs_ is unexpectedly empty."); 185 | } 186 | return 0; 187 | } 188 | 189 | std::string PyFrob::Status() const { 190 | std::ostringstream os; 191 | os << "/proc/" << pid_ << "/stat"; 192 | std::ifstream statfile(os.str()); 193 | std::string line; 194 | std::getline(statfile, line); 195 | return line; 196 | } 197 | 198 | std::vector PyFrob::GetThreads(void) const { 199 | return get_threads_(pid_, addrs_, enable_threads_); 200 | } 201 | } // namespace pyflame 202 | -------------------------------------------------------------------------------- /src/pyfrob.h: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "./ptrace.h" 18 | #include "./symbol.h" 19 | #include "./thread.h" 20 | 21 | // This abstracts the representation of py2/py3 22 | namespace pyflame { 23 | 24 | // Get the threads. Each thread stack will be in reverse order (most recent 25 | // frame first). 26 | typedef std::vector (*get_threads_t)(pid_t, PyAddresses, bool); 27 | 28 | // Frobber to get python stack stuff; this encapsulates all of the Python 29 | // interpreter logic. 30 | class PyFrob { 31 | public: 32 | PyFrob(pid_t pid, bool enable_threads) 33 | : pid_(pid), enable_threads_(enable_threads) {} 34 | ~PyFrob() { PtraceCleanup(pid_); } 35 | 36 | // Must be called before GetThreads() to detect the Python ABI. 37 | int DetectABI(PyABI abi); 38 | 39 | // Get the current frame list. 40 | std::vector GetThreads(void) const; 41 | 42 | // Useful when debugging. 43 | std::string Status() const; 44 | 45 | private: 46 | pid_t pid_; 47 | PyAddresses addrs_; 48 | bool enable_threads_; 49 | get_threads_t get_threads_; 50 | 51 | // Fill the addrs_ member 52 | int set_addrs_(PyABI *abi); 53 | }; 54 | 55 | } // namespace pyflame 56 | -------------------------------------------------------------------------------- /src/setns.h: -------------------------------------------------------------------------------- 1 | #if __GLIBC__ == 2 && __GLIBC_MINOR__ < 14 2 | /* Define setns() if missing from the C library */ 3 | #include 4 | static inline int setns(int fd, int nstype) 5 | { 6 | #ifdef __NR_setns 7 | return syscall(__NR_setns, fd, nstype); 8 | #elif defined(__NR_set_ns) 9 | return syscall(__NR_set_ns, fd, nstype); 10 | #else 11 | errno = ENOSYS; 12 | return -1; 13 | #endif 14 | } 15 | #endif 16 | -------------------------------------------------------------------------------- /src/symbol.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "./symbol.h" 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | #include "./posix.h" 28 | 29 | namespace pyflame { 30 | void ELF::Close() { 31 | if (addr_ != nullptr) { 32 | munmap(addr_, length_); 33 | addr_ = nullptr; 34 | } 35 | } 36 | 37 | // mmap the file 38 | void ELF::Open(const std::string &target, Namespace *ns) { 39 | Close(); 40 | int fd; 41 | if (ns != nullptr) { 42 | fd = ns->Open(target.c_str()); 43 | } else { 44 | fd = open(target.c_str(), O_RDONLY); 45 | } 46 | if (fd == -1) { 47 | std::ostringstream ss; 48 | ss << "Failed to open ELF file " << target << ": " << strerror(errno); 49 | throw FatalException(ss.str()); 50 | } 51 | length_ = lseek(fd, 0, SEEK_END); 52 | addr_ = mmap(nullptr, length_, PROT_READ, MAP_SHARED, fd, 0); 53 | pyflame::Close(fd); 54 | if (addr_ == MAP_FAILED) { 55 | std::ostringstream ss; 56 | ss << "Failed to mmap " << target << ": " << strerror(errno); 57 | throw FatalException(ss.str()); 58 | } 59 | 60 | if (hdr()->e_ident[EI_MAG0] != ELFMAG0 || 61 | hdr()->e_ident[EI_MAG1] != ELFMAG1 || 62 | hdr()->e_ident[EI_MAG2] != ELFMAG2 || 63 | hdr()->e_ident[EI_MAG3] != ELFMAG3) { 64 | std::ostringstream ss; 65 | ss << "File " << target << " does not have correct ELF magic header"; 66 | throw FatalException(ss.str()); 67 | } 68 | int elf_class = hdr()->e_ident[EI_CLASS]; 69 | if (elf_class != ARCH_ELFCLASS) { 70 | std::ostringstream ss; 71 | ss << "Target ELF file has EI_CLASS=" << elf_class 72 | << ", but for this architecture we expected EI_CLASS=" << ARCH_ELFCLASS; 73 | throw FatalException(ss.str()); 74 | } 75 | } 76 | 77 | void ELF::Parse() { 78 | // skip the first section since it must be of type SHT_NULL 79 | for (uint16_t i = 1; i < hdr()->e_shnum; i++) { 80 | const shdr_t *s = shdr(i); 81 | switch (s->sh_type) { 82 | case SHT_STRTAB: 83 | if (strcmp(strtab(s->sh_name), ".dynstr") == 0) { 84 | dynstr_ = i; 85 | } else if (strcmp(strtab(s->sh_name), ".strtab") == 0) { 86 | strtab_ = i; 87 | } 88 | break; 89 | case SHT_DYNSYM: 90 | dynsym_ = i; 91 | break; 92 | case SHT_DYNAMIC: 93 | dynamic_ = i; 94 | break; 95 | case SHT_SYMTAB: 96 | symtab_ = i; 97 | break; 98 | } 99 | } 100 | if (dynamic_ == -1) { 101 | throw FatalException("Failed to find section .dynamic"); 102 | } else if (dynstr_ == -1) { 103 | throw FatalException("Failed to find section .dynstr"); 104 | } else if (dynsym_ == -1) { 105 | throw FatalException("Failed to find section .dynsym"); 106 | } 107 | } 108 | 109 | std::vector ELF::NeededLibs() { 110 | // Get all of the strings 111 | std::vector needed; 112 | const shdr_t *s = shdr(dynamic_); 113 | const shdr_t *d = shdr(dynstr_); 114 | for (uint16_t i = 0; i < s->sh_size / s->sh_entsize; i++) { 115 | const dyn_t *dyn = 116 | reinterpret_cast(p() + s->sh_offset + i * s->sh_entsize); 117 | if (dyn->d_tag == DT_NEEDED) { 118 | needed.push_back( 119 | reinterpret_cast(p() + d->sh_offset + dyn->d_un.d_val)); 120 | } 121 | } 122 | return needed; 123 | } 124 | 125 | PyABI ELF::WalkTable(int sym, int str, PyAddresses *addrs) { 126 | PyABI abi{}; 127 | bool have_abi = false; 128 | const shdr_t *s = shdr(sym); 129 | const shdr_t *d = shdr(str); 130 | for (uint16_t i = 0; i < s->sh_size / s->sh_entsize; i++) { 131 | if (have_abi && addrs->tstate_addr && addrs->interp_head_addr && 132 | addrs->interp_head_fn_addr) { 133 | break; 134 | } 135 | 136 | const sym_t *sym = 137 | reinterpret_cast(p() + s->sh_offset + i * s->sh_entsize); 138 | const char *name = 139 | reinterpret_cast(p() + d->sh_offset + sym->st_name); 140 | if (!addrs->tstate_addr && strcmp(name, "_PyThreadState_Current") == 0) { 141 | addrs->tstate_addr = static_cast(sym->st_value); 142 | } else if (!addrs->interp_head_addr && strcmp(name, "interp_head") == 0) { 143 | addrs->interp_head_addr = static_cast(sym->st_value); 144 | } else if (!addrs->interp_head_addr && 145 | strcmp(name, "PyInterpreterState_Head") == 0) { 146 | addrs->interp_head_fn_addr = static_cast(sym->st_value); 147 | } else if (!have_abi) { 148 | if (strcmp(name, "PyString_Type") == 0) { 149 | // If we find PyString_Type, this is some kind of Python 2. 150 | have_abi = true; 151 | abi = PyABI::Py26; 152 | } else if (strcmp(name, "PyBytes_Type") == 0) { 153 | // If we find PyBytes_Type, it's Python 3. Continue looping though, in 154 | // case we see a Python 3.6 symbol. 155 | abi = PyABI::Py34; 156 | } else if (strcmp(name, "_PyEval_RequestCodeExtraIndex") == 0 || 157 | strcmp(name, "_PyCode_GetExtra") == 0 || 158 | strcmp(name, "_PyCode_SetExtra") == 0) { 159 | // Symbols added for Python 3.6, see: 160 | // https://www.python.org/dev/peps/pep-0523/ 161 | have_abi = true; 162 | abi = PyABI::Py36; 163 | } 164 | } 165 | } 166 | return abi; 167 | } 168 | 169 | addr_t ELF::GetBaseAddress() { 170 | int32_t phnum = hdr()->e_phnum; 171 | int32_t i; 172 | for (i = 0; i < phnum && phdr(i)->p_type != PT_LOAD; i++) { 173 | } 174 | if (i == phnum) { 175 | throw FatalException("Failed to find PT_LOAD entry in program headers"); 176 | } 177 | return phdr(i)->p_vaddr; 178 | } 179 | 180 | PyAddresses ELF::GetAddresses(PyABI *abi) { 181 | PyAddresses addrs; 182 | PyABI detected_abi = WalkTable(dynsym_, dynstr_, &addrs); 183 | if (symtab_ >= 0 && strtab_ >= 0) { 184 | detected_abi = WalkTable(symtab_, strtab_, &addrs); 185 | } 186 | addrs.pie = (hdr()->e_type == ET_DYN); 187 | if (abi != nullptr) { 188 | *abi = detected_abi; 189 | } 190 | // Handle prelinked shared objects 191 | if (hdr()->e_type == ET_DYN) { 192 | return addrs - GetBaseAddress(); 193 | } 194 | return addrs; 195 | } 196 | } // namespace pyflame 197 | -------------------------------------------------------------------------------- /src/symbol.h: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "./config.h" 26 | #include "./exc.h" 27 | #include "./namespace.h" 28 | 29 | #if USE_ELF64 30 | #define ehdr_t Elf64_Ehdr 31 | #define phdr_t Elf64_Phdr 32 | #define shdr_t Elf64_Shdr 33 | #define dyn_t Elf64_Dyn 34 | #define sym_t Elf64_Sym 35 | #define addr_t Elf64_Addr 36 | #define ARCH_ELFCLASS ELFCLASS64 37 | #else 38 | #define ehdr_t Elf32_Ehdr 39 | #define phdr_t Elf32_Phdr 40 | #define shdr_t Elf32_Shdr 41 | #define dyn_t Elf32_Dyn 42 | #define sym_t Elf32_Sym 43 | #define addr_t Elf32_Addr 44 | #define ARCH_ELFCLASS ELFCLASS32 45 | #endif 46 | 47 | namespace pyflame { 48 | 49 | // The Python interpreter ABI. Some ABIs span multiple Python versions. In that 50 | // case, the convention is to name the ABI after the first Python release to 51 | // introduce the ABI. 52 | enum class PyABI { 53 | Unknown = 0, // Unknown Python ABI 54 | Py26 = 26, // ABI for Python 2.6/2.7 55 | Py34 = 34, // ABI for Python 3.4/3.5 56 | Py36 = 36 // ABI for Python 3.6 57 | }; 58 | 59 | // Symbols 60 | struct PyAddresses { 61 | unsigned long tstate_addr; 62 | unsigned long interp_head_addr; 63 | unsigned long interp_head_fn_addr; 64 | unsigned long interp_head_hint; 65 | bool pie; 66 | 67 | PyAddresses() 68 | : tstate_addr(0), 69 | interp_head_addr(0), 70 | interp_head_fn_addr(0), 71 | interp_head_hint(0), 72 | pie(false) {} 73 | 74 | PyAddresses operator-(const unsigned long base) const { 75 | PyAddresses res(*this); 76 | res.tstate_addr = this->tstate_addr == 0 ? 0 : this->tstate_addr - base; 77 | res.interp_head_addr = 78 | this->interp_head_addr == 0 ? 0 : this->interp_head_addr - base; 79 | res.interp_head_fn_addr = 80 | this->interp_head_fn_addr == 0 ? 0 : this->interp_head_fn_addr - base; 81 | return res; 82 | } 83 | 84 | PyAddresses operator+(const unsigned long base) const { 85 | PyAddresses res(*this); 86 | res.tstate_addr = this->tstate_addr == 0 ? 0 : this->tstate_addr + base; 87 | res.interp_head_addr = 88 | this->interp_head_addr == 0 ? 0 : this->interp_head_addr + base; 89 | res.interp_head_fn_addr = 90 | this->interp_head_fn_addr == 0 ? 0 : this->interp_head_fn_addr + base; 91 | return res; 92 | } 93 | 94 | // True indicates a non-empty struct. 95 | explicit operator bool() const { return !empty(); } 96 | 97 | // Empty means the struct hasn't been initialized. 98 | inline bool empty() const { return this->tstate_addr == 0; } 99 | }; 100 | 101 | // Representation of an ELF file. 102 | class ELF { 103 | public: 104 | ELF() 105 | : addr_(nullptr), 106 | length_(0), 107 | dynamic_(-1), 108 | dynstr_(-1), 109 | dynsym_(-1), 110 | strtab_(-1), 111 | symtab_(-1) {} 112 | ~ELF() { Close(); } 113 | 114 | // Open a file 115 | void Open(const std::string &target, Namespace *ns); 116 | 117 | // Close the file; normally the destructor will do this automatically. 118 | void Close(); 119 | 120 | // Parse the ELF sections. 121 | void Parse(); 122 | 123 | // Find the DT_NEEDED fields. This is similar to the ldd(1) command. 124 | std::vector NeededLibs(); 125 | 126 | // Get the address of _PyThreadState_Current & interp_head, and set the Python 127 | // ABI. 128 | PyAddresses GetAddresses(PyABI *abi); 129 | 130 | // Extract the base load address from the Program Header table 131 | addr_t GetBaseAddress(); 132 | 133 | private: 134 | void *addr_; 135 | size_t length_; 136 | int dynamic_, dynstr_, dynsym_, strtab_, symtab_; 137 | 138 | inline const ehdr_t *hdr() const { 139 | return reinterpret_cast(addr_); 140 | } 141 | 142 | inline const phdr_t *phdr(int idx) const { 143 | if (idx < 0) { 144 | std::ostringstream ss; 145 | ss << "Illegal phdr index: " << idx; 146 | throw FatalException(ss.str()); 147 | } 148 | return reinterpret_cast(p() + hdr()->e_phoff + 149 | idx * hdr()->e_phentsize); 150 | } 151 | 152 | inline const shdr_t *shdr(int idx) const { 153 | if (idx < 0) { 154 | std::ostringstream ss; 155 | ss << "Illegal shdr index: " << idx; 156 | throw FatalException(ss.str()); 157 | } 158 | return reinterpret_cast(p() + hdr()->e_shoff + 159 | idx * hdr()->e_shentsize); 160 | } 161 | 162 | inline unsigned long p() const { 163 | return reinterpret_cast(addr_); 164 | } 165 | 166 | inline const char *strtab(int offset) const { 167 | const shdr_t *strings = shdr(hdr()->e_shstrndx); 168 | return reinterpret_cast(p() + strings->sh_offset + offset); 169 | } 170 | 171 | inline const char *dynstr(int offset) const { 172 | const shdr_t *strings = shdr(dynstr_); 173 | return reinterpret_cast(p() + strings->sh_offset + offset); 174 | } 175 | 176 | // Walk the symbol table, and return the detected ABI. 177 | PyABI WalkTable(int sym, int str, PyAddresses *addrs); 178 | }; 179 | } // namespace pyflame 180 | -------------------------------------------------------------------------------- /src/thread.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "./thread.h" 16 | 17 | namespace pyflame { 18 | std::ostream &operator<<(std::ostream &os, const Thread &thread) { 19 | os << thread.id(); 20 | if (thread.is_current()) { 21 | os << '*'; 22 | } 23 | os << ':' << std::endl; 24 | for (const auto &frame : thread.frames()) { 25 | os << frame << std::endl; 26 | } 27 | return os; 28 | } 29 | } // namespace pyflame 30 | -------------------------------------------------------------------------------- /src/thread.h: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Uber Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "./frame.h" 26 | 27 | namespace pyflame { 28 | 29 | class Thread { 30 | public: 31 | Thread() = delete; 32 | Thread(const Thread &other) 33 | : id_(other.id_), 34 | is_current_(other.is_current_), 35 | frames_(other.frames_) {} 36 | Thread(const unsigned long id, const bool is_current, 37 | const std::vector frames) 38 | : id_(id), is_current_(is_current), frames_(frames) {} 39 | 40 | inline const unsigned long id() const { return id_; } 41 | inline const bool is_current() const { return is_current_; } 42 | inline const std::vector &frames() const { return frames_; } 43 | 44 | inline bool operator==(const Thread &other) const { 45 | return id_ == other.id_ && is_current_ == other.is_current_ && 46 | frames_ == other.frames_; 47 | } 48 | 49 | private: 50 | unsigned long id_; 51 | bool is_current_; 52 | std::vector frames_; 53 | }; 54 | 55 | std::ostream &operator<<(std::ostream &os, const Thread &thread); 56 | } // namespace pyflame 57 | -------------------------------------------------------------------------------- /tests/dijkstra.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Uber Technologies, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | import logging 17 | import os 18 | import random 19 | import sys 20 | import threading 21 | 22 | log = logging.getLogger('dijkstra') 23 | 24 | try: 25 | range = xrange 26 | except NameError: 27 | pass 28 | 29 | 30 | class Graph(object): 31 | """Representation of a sparse graph.""" 32 | 33 | def __init__(self, width, height): 34 | self.width = width 35 | self.height = height 36 | self.initial = None 37 | self.goal = None 38 | self.filled = set() 39 | 40 | @classmethod 41 | def generate(cls, width, height, count): 42 | graph = cls(width, height) 43 | for _ in range(count): 44 | while True: 45 | x, y = graph.random_unfilled() 46 | if (x, y) not in graph: 47 | break 48 | graph.fill_node(x, y) 49 | possibilities = [] 50 | for xx in (-1, 0, 1): 51 | for yy in (-1, 0, 1): 52 | possibilities.append((xx, yy)) 53 | added = 0 54 | random.shuffle(possibilities) 55 | for px, py in possibilities: 56 | xx = x + px 57 | yy = y + py 58 | if not graph.valid(xx, yy): 59 | continue 60 | if (xx, yy) not in graph: 61 | graph.fill_node(xx, yy) 62 | added += 1 63 | if added == 3: 64 | break 65 | x = xx 66 | y = yy 67 | graph.initial = graph.random_unfilled() 68 | while True: 69 | goal = graph.random_unfilled() 70 | if goal != graph.initial: 71 | graph.goal = goal 72 | break 73 | return graph 74 | 75 | def random_unfilled(self): 76 | while True: 77 | x = random.randint(0, self.width - 1) 78 | y = random.randint(0, self.height - 1) 79 | if (x, y) not in self.filled: 80 | return (x, y) 81 | 82 | def fill_node(self, x, y): 83 | self.filled.add((x, y)) 84 | 85 | def valid(self, x, y): 86 | if x < 0 or y < 0: 87 | return False 88 | if x >= self.width or y >= self.height: 89 | return False 90 | return True 91 | 92 | def dist(self, x, y): 93 | gx, gy = self.goal 94 | dx = gx - x 95 | dy = gy - y 96 | return dx * dx + dy * dy 97 | 98 | def __str__(self): 99 | return '%s(%d, %d, %s) initial=%s goal=%s' % (self.__class__.__name__, 100 | self.width, self.height, 101 | sorted(self.filled), 102 | self.initial, self.goal) 103 | 104 | def __contains__(self, elem): 105 | return elem in self.filled 106 | 107 | 108 | def dijkstra(graph): 109 | solution = None 110 | via = {graph.initial: None} 111 | candidates = [] 112 | x, y = graph.initial 113 | for xx in (-1, 0, 1): 114 | for yy in (-1, 0, 1): 115 | px = x + xx 116 | py = y + yy 117 | point = (px, py) 118 | if graph.valid(px, py) and point not in graph and point not in via: 119 | d = graph.dist(px, py) 120 | candidates.append((d, point)) 121 | via[point] = graph.initial 122 | while candidates: 123 | candidates.sort(reverse=True) 124 | d, point = candidates.pop() 125 | if d == 0: 126 | solution = [point] 127 | while True: 128 | next_point = via[point] 129 | solution.append(next_point) 130 | if next_point == graph.initial: 131 | break 132 | else: 133 | point = next_point 134 | solution.reverse() 135 | break 136 | else: 137 | x, y = point 138 | for xx in (-1, 0, 1): 139 | for yy in (-1, 0, 1): 140 | px = x + xx 141 | py = y + yy 142 | new_point = (px, py) 143 | if graph.valid(px, py)\ 144 | and new_point not in graph\ 145 | and new_point not in via: 146 | d = graph.dist(px, py) 147 | candidates.append((d, new_point)) 148 | via[new_point] = point 149 | return solution 150 | 151 | 152 | def run(): 153 | """Run Dijkstra's algorithm.""" 154 | graph = Graph.generate(100, 100, 80) 155 | log.info('initial = %s', graph.initial) 156 | log.info('goal = %s', graph.goal) 157 | solution = dijkstra(graph) 158 | solution_len = 0 if solution is None else len(solution) 159 | log.info('solution = %s, len = %d', solution, solution_len) 160 | 161 | 162 | def run_times(quiet, times): 163 | """Run Dijkstra's algorithm in a loop.""" 164 | if not quiet: 165 | sys.stdout.write('%d\n' % (os.getpid(), )) 166 | sys.stdout.flush() 167 | if times <= 0: 168 | while True: 169 | run() 170 | else: 171 | for _ in range(times): 172 | run() 173 | 174 | 175 | def main(): 176 | parser = argparse.ArgumentParser() 177 | parser.add_argument('-q', '--quiet', action='store_true', help='Be quiet') 178 | parser.add_argument( 179 | '-v', '--verbose', action='store_true', help='Be verbose') 180 | parser.add_argument( 181 | '-t', '--threads', type=int, default=1, help='Number of threads') 182 | parser.add_argument( 183 | '-n', '--num', type=int, default=0, help='Number of iterations') 184 | args = parser.parse_args() 185 | 186 | logging.basicConfig() 187 | if args.verbose: 188 | log.setLevel(logging.DEBUG) 189 | 190 | if args.threads == 1: 191 | run_times(args.quiet, args.num) 192 | else: 193 | threads = [] 194 | for _ in range(args.threads): 195 | t = threading.Thread(target=run_times, args=(args.quiet, args.num)) 196 | t.start() 197 | threads.append(t) 198 | for i, t in enumerate(threads): 199 | log.info('joined thread %d', i) 200 | t.join() 201 | 202 | 203 | if __name__ == '__main__': 204 | main() 205 | -------------------------------------------------------------------------------- /tests/exit_early.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Uber Technologies, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | import os 17 | import sys 18 | import time 19 | 20 | 21 | def main(): 22 | parser = argparse.ArgumentParser() 23 | parser.add_argument('-s', '--silent', action='store_true') 24 | args = parser.parse_args() 25 | if not args.silent: 26 | sys.stdout.write('%d\n' % (os.getpid(), )) 27 | sys.stdout.flush() 28 | max_time = time.time() + 2 29 | while True: 30 | time.sleep(0.1) 31 | target = time.time() + 0.1 32 | while True: 33 | now = time.time() 34 | if now >= max_time: 35 | return 36 | if now >= target: 37 | break 38 | 39 | 40 | if __name__ == '__main__': 41 | main() 42 | -------------------------------------------------------------------------------- /tests/forker.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Evan Klitzke 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | import os 17 | import time 18 | 19 | 20 | def spawn(count): 21 | # we are the child, do some stuff 22 | t0 = time.time() 23 | x = 0 24 | while time.time() < t0 + 0.1: 25 | x += 1 26 | 27 | # spawn a new process 28 | if count: 29 | pid = os.fork() 30 | if pid == 0: 31 | spawn(count - 1) 32 | else: 33 | os.waitpid(pid, 0) 34 | 35 | 36 | def main(): 37 | parser = argparse.ArgumentParser() 38 | parser.add_argument( 39 | '-c', '--count', type=int, default=5, help='How many times to fork') 40 | args = parser.parse_args() 41 | 42 | pid = os.fork() 43 | if pid == 0: 44 | spawn(args.count) 45 | else: 46 | os.waitpid(pid, 0) 47 | 48 | 49 | if __name__ == '__main__': 50 | main() 51 | -------------------------------------------------------------------------------- /tests/sleep.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2016 Uber Technologies, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | sleep 1 18 | -------------------------------------------------------------------------------- /tests/sleeper.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Uber Technologies, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | import os 17 | import sys 18 | import time 19 | 20 | 21 | def main(): 22 | parser = argparse.ArgumentParser() 23 | parser.add_argument( 24 | '-t', '--time', default=0, type=int, help='How long to run for') 25 | parser.add_argument( 26 | '-f', '--fork', action='store_true', help='Fork child processes') 27 | args = parser.parse_args() 28 | 29 | if not args.fork: 30 | sys.stdout.write('%d\n' % (os.getpid(), )) 31 | sys.stdout.flush() 32 | t0 = time.time() 33 | while True: 34 | time.sleep(0.1) 35 | target = time.time() + 0.1 36 | while time.time() < target: 37 | pass 38 | if args.fork: 39 | pid = os.fork() 40 | if pid == 0: 41 | sys.exit(0) 42 | else: 43 | os.waitpid(pid, 0) 44 | if args.time and time.time() - t0 >= args.time: 45 | break 46 | 47 | 48 | if __name__ == '__main__': 49 | main() 50 | -------------------------------------------------------------------------------- /tests/sleeper_ユニコード.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Uber Technologies, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import sys 17 | import time 18 | 19 | 20 | def _sleep(sleep_time): 21 | time.sleep(sleep_time) 22 | target = time.time() + sleep_time 23 | while time.time() < target: 24 | pass 25 | 26 | 27 | def låtìÑ1(sleep_time): 28 | _sleep(sleep_time) 29 | 30 | 31 | # Arabic 32 | def وظيفة(sleep_time): 33 | _sleep(sleep_time) 34 | 35 | 36 | # Japanese 37 | def 日本語はどうですか(sleep_time): 38 | _sleep(sleep_time) 39 | 40 | 41 | # Khmer 42 | def មុខងារ(sleep_time): 43 | _sleep(sleep_time) 44 | 45 | 46 | # Thai 47 | def ฟังก์ชัน(sleep_time): 48 | _sleep(sleep_time) 49 | 50 | 51 | def main(): 52 | sys.stdout.write('%d\n' % (os.getpid(), )) 53 | sys.stdout.flush() 54 | while True: 55 | låtìÑ1(0.1) 56 | وظيفة(0.1) 57 | 日本語はどうですか(0.1) 58 | មុខងារ(0.1) 59 | ฟังก์ชัน(0.1) 60 | 61 | 62 | if __name__ == '__main__': 63 | main() 64 | -------------------------------------------------------------------------------- /tests/test_end_to_end.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2016 Uber Technologies, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import contextlib 18 | import os 19 | import platform 20 | import pytest 21 | import re 22 | import subprocess 23 | import sys 24 | import time 25 | 26 | IDLE_RE = re.compile(r'^\(idle\) \d+$') 27 | FLAMEGRAPH_RE = re.compile( 28 | r'^((?:[^:]+:[^:]+:\d+)(?:;[^:]+:[^:]+:\d+)*) (\d+)$') 29 | FLAMEGRAPH_NONUMBER_RE = re.compile( 30 | r'^((?:[^:]+:[^:]+)(?:;[^:]+:[^:]+)*) (\d+)$') 31 | TS_IDLE_RE = re.compile(r'\(idle\)') 32 | # Matches strings of the form 33 | # './tests/sleeper.py::31;./tests/sleeper.py:main:26;' 34 | TS_FLAMEGRAPH_RE = re.compile(r'[^[^\d]+\d+;]*') 35 | TS_RE = re.compile(r'\d+') 36 | 37 | SLEEP_A_RE = re.compile(r'.*:sleep_a:.*') 38 | SLEEP_B_RE = re.compile(r'.*:sleep_b:.*') 39 | 40 | MISSING_THREADS = platform.machine() != 'x86_64' 41 | 42 | IS_DOCKER = False 43 | try: 44 | """ Returns: True if running in a Docker container, else False """ 45 | with open('/proc/1/cgroup', 'rt') as ifh: 46 | IS_DOCKER = 'docker' in ifh.read() 47 | except: 48 | # Ignore exception, since there's nothing we can do 49 | pass 50 | 51 | 52 | @pytest.mark.skipif( 53 | os.environ.get('TRAVIS') != 'true', 54 | reason='Sanity check is only run on Travis.') 55 | def test_travis_build_environment(): 56 | """Sanity checks of the Travis test environment itself.""" 57 | assert 'python%d.%d' % sys.version_info[:2] == os.environ['PYVERSION'] 58 | assert not sys.executable.startswith('/opt') 59 | 60 | 61 | @pytest.mark.skipif( 62 | os.environ.get('PYMAJORVERSION') not in '23', 63 | reason='PYMAJORVERSION not set.') 64 | def test_rpm_build_environment(): 65 | """Sanity checks of the RPM test environment.""" 66 | assert int(os.environ['PYMAJORVERSION']) == sys.version_info[0] 67 | 68 | 69 | @contextlib.contextmanager 70 | def proc(argv, wait_for_pid=True): 71 | # start the process and wait for it to print its pid... we explicitly do 72 | # this instead of using the pid attribute so we can ensure that the process 73 | # is initialized 74 | proc = subprocess.Popen(argv, stdout=subprocess.PIPE) 75 | if wait_for_pid: 76 | proc.stdout.readline() 77 | 78 | try: 79 | yield proc 80 | finally: 81 | proc.kill() 82 | 83 | 84 | def python_proc(test_file, *args): 85 | argv = [sys.executable, './tests/%s' % (test_file, )] 86 | return proc(argv + [str(x) for x in args]) 87 | 88 | 89 | @pytest.yield_fixture 90 | def dijkstra(): 91 | with python_proc('dijkstra.py') as p: 92 | yield p 93 | 94 | 95 | @pytest.yield_fixture 96 | def threaded_dijkstra(): 97 | with python_proc('dijkstra.py', '-t', 4) as p: 98 | yield p 99 | 100 | 101 | @pytest.yield_fixture 102 | def sleeper(): 103 | with python_proc('sleeper.py') as p: 104 | yield p 105 | 106 | 107 | @pytest.yield_fixture 108 | def unicode_sleeper(): 109 | with python_proc('sleeper_ユニコード.py') as p: 110 | yield p 111 | 112 | 113 | @pytest.yield_fixture 114 | def threaded_sleeper(): 115 | with python_proc('threaded_sleeper.py') as p: 116 | yield p 117 | 118 | 119 | @pytest.yield_fixture 120 | def threaded_busy(): 121 | with python_proc('threaded_busy.py') as p: 122 | yield p 123 | 124 | 125 | @pytest.yield_fixture 126 | def exit_early(): 127 | with python_proc('exit_early.py') as p: 128 | yield p 129 | 130 | 131 | @pytest.yield_fixture 132 | def not_python(): 133 | with proc(['./tests/sleep.sh'], wait_for_pid=False) as p: 134 | yield p 135 | 136 | 137 | def assert_flamegraph(line, allow_idle, line_re=FLAMEGRAPH_RE): 138 | if allow_idle and IDLE_RE.match(line): 139 | return 140 | m = line_re.match(line) 141 | assert m is not None, 'line {!r} did not match!'.format(line) 142 | parts, count = m.groups() 143 | count = int(count, 10) 144 | assert count >= 1 145 | 146 | for part in parts.split(';'): 147 | tokens = part.split(':') 148 | 149 | if len(tokens) == 2: 150 | fname, func = tokens 151 | line_num = 1 152 | else: 153 | fname, func, line_num = tokens 154 | line_num = int(line_num, 10) 155 | 156 | # Make a best effort to sanity check the line number. This logic could 157 | # definitely be improved, since right now an off-by-one error wouldn't 158 | # be caught by the test suite. 159 | if fname.startswith('./tests/'): 160 | assert 1 <= line_num < 300 161 | 162 | 163 | def assert_unique(lines, allow_idle=False, line_re=FLAMEGRAPH_RE): 164 | seen = set() 165 | for line in lines: 166 | if line in seen: 167 | assert False, 'saw line {!r} twice in lines {!r}'.format( 168 | line, lines) 169 | seen.add(line) 170 | assert_flamegraph(line, allow_idle=allow_idle, line_re=line_re) 171 | yield line 172 | 173 | 174 | def consume_unique(lines, allow_idle=False, line_re=FLAMEGRAPH_RE): 175 | for line in assert_unique(lines, allow_idle=allow_idle, line_re=line_re): 176 | pass 177 | 178 | 179 | def communicate(proc): 180 | out, err = proc.communicate() 181 | if isinstance(out, bytes): 182 | out = out.decode('utf-8') 183 | if isinstance(err, bytes): 184 | err = err.decode('utf-8') 185 | return out, err 186 | 187 | 188 | def path_to_pyflame(): 189 | """Path to pyflame. 190 | 191 | Generally we prefer the executable built in the src/ directory. On Conda 192 | the tests are run in a chroot without the source code, so we fall back to 193 | the "system" installation if it looks like no executable has been built. 194 | """ 195 | if os.path.exists('./src/pyflame'): 196 | return './src/pyflame' 197 | return 'pyflame' 198 | 199 | 200 | def test_monitor(dijkstra): 201 | """Basic test for the monitor mode.""" 202 | proc = subprocess.Popen( 203 | [path_to_pyflame(), '-p', str(dijkstra.pid)], 204 | stdout=subprocess.PIPE, 205 | stderr=subprocess.PIPE, 206 | universal_newlines=True) 207 | out, err = communicate(proc) 208 | assert not err 209 | assert proc.returncode == 0 210 | lines = out.split('\n') 211 | assert lines.pop(-1) == '' # output should end in a newline 212 | consume_unique(lines) 213 | 214 | 215 | def test_non_gil(sleeper): 216 | """Basic test for non-GIL/native code processes.""" 217 | proc = subprocess.Popen( 218 | [path_to_pyflame(), '-p', str(sleeper.pid)], 219 | stdout=subprocess.PIPE, 220 | stderr=subprocess.PIPE, 221 | universal_newlines=True) 222 | out, err = communicate(proc) 223 | assert not err 224 | assert proc.returncode == 0 225 | lines = out.split('\n') 226 | assert lines.pop(-1) == '' # output should end in a newline 227 | consume_unique(lines, allow_idle=True) 228 | 229 | 230 | @pytest.mark.skipif(MISSING_THREADS, reason='build does not have threads') 231 | def test_threaded(threaded_sleeper): 232 | """Basic test for non-GIL/native code processes.""" 233 | proc = subprocess.Popen( 234 | [path_to_pyflame(), '--threads', '-p', 235 | str(threaded_sleeper.pid)], 236 | stdout=subprocess.PIPE, 237 | stderr=subprocess.PIPE, 238 | universal_newlines=True) 239 | out, err = communicate(proc) 240 | assert not err 241 | assert proc.returncode == 0 242 | lines = out.split('\n') 243 | assert lines.pop(-1) == '' # output should end in a newline 244 | a_count = 0 245 | b_count = 0 246 | for line in assert_unique(lines): 247 | if SLEEP_A_RE.match(line): 248 | assert_flamegraph(line, True) 249 | a_count += 1 250 | elif SLEEP_B_RE.match(line): 251 | assert_flamegraph(line, True) 252 | b_count += 1 253 | 254 | # We must see both threads. 255 | assert a_count > 0 256 | assert b_count > 0 257 | 258 | # We should see them both *about* the same number of times. 259 | small = float(min(a_count, b_count)) 260 | big = float(max(a_count, b_count)) 261 | assert (small / big) >= 0.5 262 | 263 | 264 | def test_unthreaded(threaded_busy): 265 | """Test only one process is profiled by default.""" 266 | proc = subprocess.Popen( 267 | [path_to_pyflame(), '-s', '0', '-p', 268 | str(threaded_busy.pid)], 269 | stdout=subprocess.PIPE, 270 | stderr=subprocess.PIPE, 271 | universal_newlines=True) 272 | out, err = communicate(proc) 273 | assert not err 274 | assert proc.returncode == 0 275 | lines = out.strip().split('\n') 276 | assert len(lines) == 1 277 | 278 | 279 | def test_legacy_pid_handling(threaded_busy): 280 | # test PID parsing when -p is not used 281 | proc = subprocess.Popen( 282 | [path_to_pyflame(), '-s', '0', 283 | str(threaded_busy.pid)], 284 | stdout=subprocess.PIPE, 285 | stderr=subprocess.PIPE, 286 | universal_newlines=True) 287 | out, err = communicate(proc) 288 | assert err.startswith('WARNING: ') 289 | assert proc.returncode == 0 290 | lines = out.strip().split('\n') 291 | assert len(lines) == 1 292 | 293 | 294 | def test_legacy_pid_handling_too_many_pids(): 295 | # test PID parsing when -p is not used 296 | proc = subprocess.Popen( 297 | [path_to_pyflame(), '1', '2'], 298 | stdout=subprocess.PIPE, 299 | stderr=subprocess.PIPE, 300 | universal_newlines=True) 301 | out, err = communicate(proc) 302 | assert proc.returncode == 1 303 | assert 'Usage: ' in err 304 | 305 | 306 | def test_dash_t_and_dash_p(): 307 | proc = subprocess.Popen( 308 | [path_to_pyflame(), '-p', '1', '-t', sys.executable, '--version'], 309 | stdout=subprocess.PIPE, 310 | stderr=subprocess.PIPE, 311 | universal_newlines=True) 312 | out, err = communicate(proc) 313 | assert 'mutually compatible' in err 314 | assert proc.returncode == 1 315 | 316 | 317 | def test_unsupported_abi(): 318 | proc = subprocess.Popen( 319 | [path_to_pyflame(), '--abi=0', '-p', '1'], 320 | stdout=subprocess.PIPE, 321 | stderr=subprocess.PIPE, 322 | universal_newlines=True) 323 | out, err = communicate(proc) 324 | assert err.startswith('Unknown or unsupported ABI ') 325 | assert proc.returncode == 1 326 | 327 | 328 | def test_exclude_idle(sleeper): 329 | """Basic test for idle processes.""" 330 | proc = subprocess.Popen( 331 | [path_to_pyflame(), '-x', '-p', 332 | str(sleeper.pid)], 333 | stdout=subprocess.PIPE, 334 | stderr=subprocess.PIPE, 335 | universal_newlines=True) 336 | out, err = communicate(proc) 337 | assert not err 338 | assert proc.returncode == 0 339 | lines = out.split('\n') 340 | assert lines.pop(-1) == '' # output should end in a newline 341 | consume_unique(lines) 342 | 343 | 344 | @pytest.mark.skipif( 345 | sys.getfilesystemencoding().lower() != 'utf-8', 346 | reason='requires UTF-8 filesystem, see ' 347 | 'https://bugs.python.org/issue8242') 348 | @pytest.mark.skipif(sys.version_info < (3, 3), reason="requires Python 3.3+") 349 | def test_utf8_output(unicode_sleeper): 350 | proc = subprocess.Popen( 351 | [path_to_pyflame(), '-x', '-p', 352 | str(unicode_sleeper.pid)], 353 | stdout=subprocess.PIPE, 354 | stderr=subprocess.PIPE, 355 | universal_newlines=True) 356 | out, err = communicate(proc) 357 | assert not err 358 | assert proc.returncode == 0 359 | 360 | # The output is decoded assuming UTF-8. So here we check if we can 361 | # find our function names again. 362 | func_names = ["låtìÑ1", "وظيفة", "日本語はどうですか", "មុខងារ", "ฟังก์ชัน"] 363 | 364 | for f in func_names: 365 | assert f in out, "Could not find function '{}' in output".format(f) 366 | 367 | lines = out.split('\n') 368 | assert lines.pop(-1) == '' # output should end in a newline 369 | consume_unique(lines) 370 | 371 | 372 | def test_exit_early(exit_early): 373 | proc = subprocess.Popen( 374 | [path_to_pyflame(), '-s', '10', '-p', 375 | str(exit_early.pid)], 376 | stdout=subprocess.PIPE, 377 | stderr=subprocess.PIPE) 378 | out, err = communicate(proc) 379 | assert not err 380 | assert proc.returncode == 0 381 | lines = out.split('\n') 382 | assert lines.pop(-1) == '' # output should end in a newline 383 | consume_unique(lines, allow_idle=True) 384 | 385 | 386 | def test_sample_not_python(not_python): 387 | proc = subprocess.Popen( 388 | [path_to_pyflame(), '-p', str(not_python.pid)], 389 | stdout=subprocess.PIPE, 390 | stderr=subprocess.PIPE) 391 | out, err = communicate(proc) 392 | assert not out 393 | assert (err.startswith('Failed to locate libpython') 394 | or err.startswith('Target ELF file has EI_CLASS')) 395 | assert proc.returncode == 1 396 | 397 | 398 | @pytest.mark.parametrize('force_abi', [False, True]) 399 | @pytest.mark.parametrize('trace_threads', [False] 400 | if MISSING_THREADS else [False, True]) 401 | def test_trace(force_abi, trace_threads): 402 | args = [path_to_pyflame()] 403 | if force_abi: 404 | abi_string = '%d%d' % sys.version_info[:2] 405 | args.extend(['--abi', abi_string]) 406 | if trace_threads: 407 | args.append('--threads') 408 | args.extend(['-t', sys.executable, 'tests/exit_early.py', '-s']) 409 | 410 | proc = subprocess.Popen( 411 | args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 412 | out, err = communicate(proc) 413 | assert not err 414 | assert proc.returncode == 0 415 | lines = out.split('\n') 416 | assert lines.pop(-1) == '' # output should end in a newline 417 | consume_unique(lines, allow_idle=True) 418 | 419 | 420 | def test_trace_not_python(): 421 | proc = subprocess.Popen( 422 | [path_to_pyflame(), '-t', './tests/sleep.sh'], 423 | stdout=subprocess.PIPE, 424 | stderr=subprocess.PIPE) 425 | out, err = communicate(proc) 426 | assert not out 427 | assert (err.startswith('Failed to locate libpython') 428 | or err.startswith('Target ELF file has EI_CLASS')) 429 | assert proc.returncode == 1 430 | 431 | 432 | def test_pyflame_a_pyflame(): 433 | proc = subprocess.Popen( 434 | [path_to_pyflame(), '-t', path_to_pyflame()], 435 | stdout=subprocess.PIPE, 436 | stderr=subprocess.PIPE) 437 | out, err = communicate(proc) 438 | assert not out 439 | assert err.startswith('You tried to pyflame a pyflame') 440 | assert proc.returncode == 1 441 | 442 | 443 | def test_pyflame_nonexistent_file(): 444 | proc = subprocess.Popen( 445 | [path_to_pyflame(), '-t', '/no/such/file'], 446 | stdout=subprocess.PIPE, 447 | stderr=subprocess.PIPE) 448 | out, err = communicate(proc) 449 | assert not out 450 | assert 'Child process exited with status' in err 451 | assert proc.returncode == 1 452 | 453 | 454 | def test_trace_no_arg(): 455 | proc = subprocess.Popen( 456 | [path_to_pyflame(), '-t'], 457 | stdout=subprocess.PIPE, 458 | stderr=subprocess.PIPE) 459 | out, err = communicate(proc) 460 | assert not out 461 | assert 'Usage: ' in err 462 | assert proc.returncode == 1 463 | 464 | 465 | def test_sample_no_arg(): 466 | proc = subprocess.Popen( 467 | [path_to_pyflame()], stdout=subprocess.PIPE, stderr=subprocess.PIPE) 468 | out, err = communicate(proc) 469 | assert not out 470 | assert err.startswith('Usage: ') 471 | assert proc.returncode == 1 472 | 473 | 474 | def test_sample_extra_args(): 475 | proc = subprocess.Popen( 476 | [path_to_pyflame(), 'foo', 'bar'], 477 | stdout=subprocess.PIPE, 478 | stderr=subprocess.PIPE) 479 | out, err = communicate(proc) 480 | assert not out 481 | assert err.startswith('Usage: ') 482 | assert proc.returncode == 1 483 | 484 | 485 | @pytest.mark.skipif(IS_DOCKER, reason='There is not init process in Docker') 486 | def test_permission_error(): 487 | # we should not be allowed to trace init 488 | proc = subprocess.Popen( 489 | [path_to_pyflame(), '-p', '1'], 490 | stdout=subprocess.PIPE, 491 | stderr=subprocess.PIPE) 492 | out, err = communicate(proc) 493 | assert not out 494 | assert err.startswith('Failed to seize PID') 495 | assert proc.returncode == 1 496 | 497 | 498 | @pytest.mark.parametrize('pid', [-1, 0, 1 << 200, 'not a pid']) 499 | def test_invalid_pid(pid): 500 | # we should not be allowed to trace init 501 | proc = subprocess.Popen( 502 | [path_to_pyflame(), '-p', str(pid)], 503 | stdout=subprocess.PIPE, 504 | stderr=subprocess.PIPE) 505 | out, err = communicate(proc) 506 | assert not out 507 | assert err.startswith('Failed to seize PID ') or 'failed to parse' in err 508 | assert proc.returncode == 1 509 | 510 | 511 | def test_include_ts(sleeper): 512 | """Basic test for timestamp processes.""" 513 | proc = subprocess.Popen( 514 | [path_to_pyflame(), '--flamechart', '-p', 515 | str(sleeper.pid)], 516 | stdout=subprocess.PIPE, 517 | stderr=subprocess.PIPE, 518 | universal_newlines=True) 519 | out, err = proc.communicate() 520 | assert not err 521 | assert proc.returncode == 0 522 | lines = out.split('\n') 523 | assert lines.pop(-1) == '' # output should end in a newline 524 | for line in lines: # DO NOT USE assert_unique HERE 525 | assert TS_FLAMEGRAPH_RE.match(line) or TS_RE.match( 526 | line) or TS_IDLE_RE.match(line) 527 | 528 | 529 | def test_include_ts_exclude_idle(sleeper): 530 | """Basic test for timestamp processes.""" 531 | proc = subprocess.Popen( 532 | [path_to_pyflame(), '--flamechart', '-x', '-p', 533 | str(sleeper.pid)], 534 | stdout=subprocess.PIPE, 535 | stderr=subprocess.PIPE, 536 | universal_newlines=True) 537 | out, err = proc.communicate() 538 | assert not err 539 | assert proc.returncode == 0 540 | lines = out.split('\n') 541 | assert lines.pop(-1) == '' # output should end in a newline 542 | for line in lines: # DO NOT USE assert_unique HERE 543 | assert not TS_IDLE_RE.match(line) 544 | assert TS_FLAMEGRAPH_RE.match(line) or TS_RE.match(line) 545 | 546 | 547 | @pytest.mark.parametrize('flag', ['-v', '--version']) 548 | def test_version(flag): 549 | """Test the version flag.""" 550 | proc = subprocess.Popen( 551 | [path_to_pyflame(), flag], 552 | stdout=subprocess.PIPE, 553 | stderr=subprocess.PIPE, 554 | universal_newlines=True) 555 | out, err = communicate(proc) 556 | assert not err 557 | assert proc.returncode == 0 558 | 559 | version_re = re.compile( 560 | r'^pyflame \d+\.\d+\.\d+ (\(commit [\w]+\) )?\S+ \S+ \(ABI list: .+\)$' 561 | ) 562 | assert version_re.match(out.strip()) 563 | 564 | 565 | def test_trace_forker(): 566 | t0 = time.time() 567 | proc = subprocess.Popen( 568 | [path_to_pyflame(), '-t', sys.executable, 'tests/forker.py'], 569 | stdout=subprocess.PIPE, 570 | stderr=subprocess.PIPE, 571 | universal_newlines=True) 572 | out, err = communicate(proc) 573 | elapsed = time.time() - t0 574 | assert not err 575 | assert proc.returncode == 0 576 | lines = out.split('\n') 577 | assert lines.pop(-1) == '' # output should end in a newline 578 | consume_unique(lines, allow_idle=True) 579 | assert elapsed >= 0.5 580 | 581 | 582 | def test_sigchld(): 583 | t0 = time.time() 584 | proc = subprocess.Popen( 585 | [ 586 | path_to_pyflame(), '-t', sys.executable, './tests/sleeper.py', 587 | '-t', '2', '-f' 588 | ], 589 | stdout=subprocess.PIPE, 590 | stderr=subprocess.PIPE) 591 | out, err = communicate(proc) 592 | elapsed = time.time() - t0 593 | assert not err 594 | assert proc.returncode == 0 595 | lines = out.split('\n') 596 | assert lines.pop(-1) == '' # output should end in a newline 597 | consume_unique(lines, allow_idle=True) 598 | assert elapsed >= 2 599 | 600 | 601 | @pytest.mark.skipif(MISSING_THREADS, reason='build does not have threads') 602 | def test_thread_dump(threaded_dijkstra): 603 | time.sleep(0.5) 604 | proc = subprocess.Popen( 605 | [path_to_pyflame(), '-d', '-p', 606 | str(threaded_dijkstra.pid)], 607 | stdout=subprocess.PIPE, 608 | stderr=subprocess.PIPE) 609 | out, err = communicate(proc) 610 | assert not err 611 | 612 | THREAD_RE = re.compile(r'^\d+\*?:') 613 | threads = 0 614 | for line in out.split('\n'): 615 | print(line) 616 | if THREAD_RE.match(line): 617 | threads += 1 618 | assert threads == 5 619 | 620 | 621 | def test_no_line_numbers(dijkstra): 622 | """Basic test for --no-line-numbers""" 623 | proc = subprocess.Popen( 624 | [path_to_pyflame(), '-p', 625 | str(dijkstra.pid), "--no-line-numbers"], 626 | stdout=subprocess.PIPE, 627 | stderr=subprocess.PIPE, 628 | universal_newlines=True) 629 | out, err = communicate(proc) 630 | assert not err 631 | assert proc.returncode == 0 632 | lines = out.split('\n') 633 | assert lines.pop(-1) == '' # output should end in a newline 634 | 635 | # With no line numbers included there can be duplicate lines, 636 | # but flamegraph.pl performs deduplication as well 637 | for line in lines: 638 | assert_flamegraph( 639 | line, allow_idle=True, line_re=FLAMEGRAPH_NONUMBER_RE) 640 | -------------------------------------------------------------------------------- /tests/threaded_busy.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Uber Technologies, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import sys 17 | import threading 18 | 19 | 20 | def do_sleep(): 21 | while True: 22 | pass 23 | 24 | 25 | def main(): 26 | sys.stdout.write('%d\n' % (os.getpid(), )) 27 | sys.stdout.flush() 28 | thread = threading.Thread(target=do_sleep) 29 | thread.start() 30 | do_sleep() 31 | 32 | 33 | if __name__ == '__main__': 34 | main() 35 | -------------------------------------------------------------------------------- /tests/threaded_sleeper.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Uber Technologies, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import sys 17 | import time 18 | import threading 19 | 20 | 21 | def do_sleep(): 22 | while True: 23 | time.sleep(0.1) 24 | target = time.time() + 0.1 25 | while time.time() < target: 26 | pass 27 | 28 | 29 | def sleep_a(): 30 | do_sleep() 31 | 32 | 33 | def sleep_b(): 34 | do_sleep() 35 | 36 | 37 | def main(): 38 | sys.stdout.write('%d\n' % (os.getpid(), )) 39 | sys.stdout.flush() 40 | thread_a = threading.Thread(target=sleep_a) 41 | thread_a.start() 42 | thread_b = threading.Thread(target=sleep_b) 43 | thread_b.start() 44 | 45 | 46 | if __name__ == '__main__': 47 | main() 48 | -------------------------------------------------------------------------------- /utils/flame-chart-json: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import json 5 | 6 | """ Generate JSON for visualizing Flame Charts using Chrome CPU Profiler 7 | This takes input from Pyflame which is of the format 8 | timestamp_1 9 | callstack_1 10 | ... 11 | timestamp_n 12 | callstack_n 13 | 14 | With each callstack is of below format 15 | file_path:function_name:line_no;...file_path:function_name:line_no; 16 | 17 | Then converts them to JSON format which can be visualized as Flame Charts using 18 | Chrome CPU profiler 19 | 20 | USAGE: cat pyflame.out | flame-chart-json > flame_chart.cpuprofile 21 | You can also pipe pyflame output directly. 22 | Note: Chrome CPU profiler expects file with extension ".cpuprofile". This format 23 | is supported on Chrome Version 54.0.2840.71 24 | 25 | """ 26 | 27 | 28 | class FCJson(object): 29 | 30 | def __init__(self): 31 | self.node_id = 1 32 | self.prof = {} 33 | self.prev_ts = 0 34 | self.delta_ts = 2000 35 | self.IDLE_NODE_ID = 2 36 | 37 | # init root 38 | self.prof['nodes'] = [] 39 | node = {} 40 | self.fill_node(node, "(root)") 41 | node['children'] = [] 42 | self.prof['nodes'].append(node) 43 | 44 | # init idle node 45 | idle_node = {} 46 | self.fill_node(idle_node, "(idle)") 47 | idle_node['children'] = [] 48 | self.prof['nodes'].append(idle_node) 49 | self.prof['nodes'][0]['children'].append(self.node_id - 1) 50 | 51 | # init rest 52 | self.prof['startTime'] = 0 53 | self.prof['endTime'] = 5000 54 | self.prof['samples'] = [] 55 | self.prof['timeDeltas'] = [] 56 | 57 | def fill_node(self, node, func_name, fill_child_id=False): 58 | node['id'] = self.node_id 59 | node['callFrame'] = {} 60 | node['callFrame']['functionName'] = func_name 61 | node['callFrame']['scriptId'] = "0" 62 | node['callFrame']['url'] = "" 63 | node['hitCount'] = 1 64 | self.node_id += 1 65 | if fill_child_id is True: 66 | node['children'] = [] 67 | node['children'].append(self.node_id) 68 | 69 | def create_cs(self, cs): 70 | add_child = True 71 | cs_len = len(cs) 72 | # Do for each of the stack frames in the callstack 73 | for i, frame in enumerate(cs): 74 | # idle case can be optimized if required in future 75 | if frame == '': 76 | continue 77 | 78 | # Store the stack frame 79 | node = {} 80 | if (i >= cs_len - 2): 81 | add_child = False 82 | 83 | self.fill_node(node, frame, add_child) 84 | 85 | # Add the node to prof 86 | self.prof['nodes'].append(node) 87 | 88 | def create_node(self, ts, cs): 89 | prev_node_id = self.node_id 90 | 91 | self.create_cs(cs.strip().split(';')) 92 | 93 | self.prof['samples'].append(self.node_id - 1) 94 | if self.prev_ts is not 0: 95 | self.prof['timeDeltas'].append(ts-self.prev_ts-1000) 96 | else: 97 | self.prof['timeDeltas'].append(0) 98 | 99 | # Idle is made as the root of all the callstacks so that we display the 100 | # flame chart correctly. I did not find a better way to do this 101 | self.prof['samples'].append(self.IDLE_NODE_ID) 102 | self.prof['timeDeltas'].append(1000) 103 | 104 | self.prev_ts = ts 105 | # add the node_id of the first frame of cs to the children of root & 106 | # idle nodes. 107 | self.prof['nodes'][0]['children'].append(prev_node_id) 108 | self.prof['nodes'][1]['children'].append(prev_node_id) 109 | 110 | def start(self): 111 | first_ts = 0 112 | last_ts = 0 113 | is_first_ln = True 114 | for line in sys.stdin: 115 | # 1st line should be timestamp & 2nd line callstack 116 | if is_first_ln: 117 | ts = int(line) 118 | else: 119 | cs = line 120 | 121 | # Keep track of timestamps 122 | if first_ts == 0: 123 | first_ts = ts 124 | last_ts = ts 125 | 126 | if is_first_ln is False: 127 | if cs is None: 128 | break 129 | 130 | is_first_ln = True 131 | self.create_node(ts, cs) 132 | self.delta_ts += 1000 133 | else: 134 | is_first_ln = False 135 | 136 | # update the end time based on the timestamps, and reduce it by half 137 | # because Chrome is doubling the endTime while displaying the flamechart 138 | self.prof['endTime'] = (last_ts - first_ts)/2 + 1000 139 | 140 | print(json.dumps(self.prof)) 141 | 142 | if __name__ == "__main__": 143 | fc = FCJson() 144 | fc.start() 145 | --------------------------------------------------------------------------------