├── .clang-format ├── .github └── workflows │ ├── publish.yml │ ├── static_analysis.yml │ └── unit_tests.yml ├── .gitmodules ├── .travis.yml ├── CMakeLists.txt ├── DEVELOPING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── benchmark ├── benchmark.py ├── benchmark_helper.py ├── bm_baseline.py ├── bm_simplefunc.py ├── compare.py ├── compare_commits.sh └── run.sh ├── cmake └── FindScorep.cmake ├── pyproject.toml ├── scorep ├── __init__.py ├── __main__.py ├── _instrumenters │ ├── __init__.py │ ├── base_instrumenter.py │ ├── dummy.py │ ├── scorep_cProfile.py │ ├── scorep_cTrace.py │ ├── scorep_instrumenter.py │ ├── scorep_profile.py │ ├── scorep_trace.py │ └── utils.py ├── _version.py ├── helper.py ├── instrumenter.py ├── subsystem.py └── user.py ├── setup.cfg ├── setup.py ├── src ├── classes.cpp ├── classes.hpp ├── methods.cpp ├── methods.hpp ├── scorep.hpp ├── scorep_bindings.cpp └── scorepy │ ├── cInstrumenter.cpp │ ├── cInstrumenter.hpp │ ├── compat.hpp │ ├── events.cpp │ ├── events.hpp │ ├── pathUtils.cpp │ ├── pathUtils.hpp │ ├── pythonHelpers.cpp │ ├── pythonHelpers.hpp │ └── pythoncapi_compat.h └── test ├── cases ├── call_main.py ├── classes.py ├── classes2.py ├── context.py ├── decorator.py ├── error_region.py ├── external_instrumentation.py ├── file_io.py ├── force_finalize.py ├── instrumentation.py ├── instrumentation2.py ├── interrupt.py ├── miniasync.py ├── mpi.py ├── nosleep.py ├── numpy_dot.py ├── numpy_dot_large.py ├── reload.py ├── sleep.py ├── use_threads.py ├── user_instrumentation.py ├── user_regions.py └── user_rewind.py ├── conftest.py ├── test_helper.py ├── test_pathUtils.cpp ├── test_scorep.py ├── test_subsystem.py └── utils.py /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | BasedOnStyle: LLVM 3 | AccessModifierOffset: -4 4 | AlignEscapedNewlinesLeft: false 5 | AlignTrailingComments: true 6 | AllowAllParametersOfDeclarationOnNextLine: true 7 | AllowShortIfStatementsOnASingleLine: false 8 | AllowShortLoopsOnASingleLine: false 9 | AllowShortFunctionsOnASingleLine: false 10 | AlwaysBreakTemplateDeclarations: true 11 | AlwaysBreakBeforeMultilineStrings: false 12 | BinPackParameters: true 13 | BreakBeforeBinaryOperators: false 14 | BreakBeforeBraces: Allman 15 | BreakBeforeTernaryOperators: false 16 | BreakConstructorInitializersBeforeComma: false 17 | ColumnLimit: 100 18 | ConstructorInitializerIndentWidth: 0 19 | ConstructorInitializerAllOnOneLineOrOnePerLine: false 20 | Cpp11BracedListStyle: false 21 | DerivePointerBinding: false 22 | ExperimentalAutoDetectBinPacking: false 23 | IndentCaseLabels: false 24 | IndentWidth: 4 25 | IndentFunctionDeclarationAfterType: false 26 | MaxEmptyLinesToKeep: 1 27 | NamespaceIndentation: Inner 28 | 29 | PointerBindsToType: true 30 | 31 | PenaltyBreakBeforeFirstCallParameter: 19 32 | PenaltyBreakComment: 60 33 | PenaltyBreakString: 1000 34 | PenaltyBreakFirstLessLess: 120 35 | PenaltyExcessCharacter: 1000000 36 | PenaltyReturnTypeOnItsOwnLine: 60 37 | 38 | 39 | SpaceBeforeAssignmentOperators: true 40 | SpaceInEmptyParentheses: false 41 | SpacesBeforeTrailingComments: 1 42 | 43 | 44 | Standard: Cpp11 45 | TabWidth: 4 46 | UseTab: Never 47 | 48 | SpacesInParentheses: false 49 | SpacesInAngles: false 50 | SpaceInEmptyParentheses: false 51 | SpacesInCStyleCastParentheses: false 52 | SpaceAfterControlStatementKeyword: true 53 | SpaceBeforeAssignmentOperators: true 54 | ContinuationIndentWidth: 4 55 | ... 56 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publishing on PyPi 2 | on: 3 | release: 4 | types: [published] 5 | 6 | jobs: 7 | publish: 8 | name: Publish Python 🐍 distributions 📦 to PyPI 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@master 12 | 13 | 14 | - name: Set up Python 3.8 15 | uses: actions/setup-python@v4 16 | with: 17 | python-version: 3.8 18 | - name: Check Version 19 | run: test ${{ github.event.release.tag_name }} = `python -c "import scorep._version; print('v'+scorep._version.__version__)"` 20 | 21 | - name: Add Score-P repo 22 | run: sudo add-apt-repository ppa:andreasgocht/scorep 23 | 24 | - name: Install Score-P 25 | run: sudo apt-get -y install scorep 26 | 27 | - name: Setup environment 28 | run: echo "$HOME/scorep/bin" >> $GITHUB_PATH 29 | 30 | - name: Install pypa/build 31 | run: >- 32 | python -m 33 | pip install build --user 34 | - name: Build a source tarball 35 | run: >- 36 | python -m 37 | build --sdist --outdir dist/ . 38 | - name: Publish distribution 📦 to PyPI 39 | uses: pypa/gh-action-pypi-publish@master 40 | with: 41 | password: ${{ secrets.PYPI_API_TOKEN }} 42 | -------------------------------------------------------------------------------- /.github/workflows/static_analysis.yml: -------------------------------------------------------------------------------- 1 | name: Static analysis 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | Cpp: 6 | runs-on: ubuntu-22.04 7 | steps: 8 | - uses: actions/checkout@v2 9 | - name: Formatting 10 | uses: Flamefire/clang-format-lint-action@master 11 | with: 12 | source: src 13 | clangFormatVersion: 9 14 | 15 | Python: 16 | runs-on: ubuntu-22.04 17 | steps: 18 | - uses: actions/checkout@v2 19 | - uses: actions/setup-python@v4 20 | with: 21 | python-version: 3.8 22 | - name: Install Python packages 23 | run: | 24 | pip install --upgrade pip 25 | pip install --upgrade flake8 26 | - name: Run flake8 27 | run: flake8 benchmark scorep test 28 | -------------------------------------------------------------------------------- /.github/workflows/unit_tests.yml: -------------------------------------------------------------------------------- 1 | name: Unit tests 2 | on: [push, pull_request, workflow_dispatch] 3 | 4 | env: 5 | SCOREP_TIMER: clock_gettime # tsc causes warnings 6 | RDMAV_FORK_SAFE: 7 | IBV_FORK_SAFE: 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | python: ["3.8", "3.9", "3.10", "3.11", "3.12", 'pypy-2.7', 'pypy-3.7', 'pypy-3.9', 'pypy-3.10'] 15 | fail-fast: false 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - uses: actions/cache@v3 20 | with: 21 | path: ~/.cache/pip 22 | key: ${{ runner.os }}-pip 23 | 24 | - name: Add Score-P repo 25 | run: sudo add-apt-repository ppa:andreasgocht/scorep 26 | 27 | - name: Install Score-P 28 | run: sudo apt-get -y install scorep 29 | 30 | - name: Setup environment 31 | run: echo "$HOME/scorep/bin" >> $GITHUB_PATH 32 | - name: set up Python 33 | uses: actions/setup-python@v4 34 | with: 35 | python-version: ${{matrix.python}} 36 | architecture: x64 37 | - name: Install Python packages 38 | run: | 39 | pip install --upgrade pip 40 | pip install --upgrade setuptools 41 | pip install numpy mpi4py pytest 42 | 43 | - name: Build python bindings 44 | run: pip install . 45 | - name: Run tests 46 | working-directory: test 47 | run: pytest -vv 48 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/score-p/scorep_binding_python/3a1642a32934d31eeea40c423e62bc486c7d0e79/.gitmodules -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: bionic 2 | 3 | language: python 4 | cache: pip 5 | python: 6 | - "2.7" 7 | - "3.6" 8 | - "3.7" 9 | 10 | addons: 11 | apt: 12 | sources: 13 | - sourceline: "ppa:andreasgocht/scorep" 14 | packages: 15 | - scorep 16 | - openmpi-common 17 | - openmpi-bin 18 | - libopenmpi-dev 19 | 20 | install: 21 | - pip install mpi4py numpy pytest 22 | 23 | script: 24 | - pip install ./ && cd test && pytest 25 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.12) 2 | project(scorep_bindings) 3 | 4 | option(DEV_MODE "Set this to ON to use CMake. However: NO SUPPORT IS PROVIDED, PIP IS THE SUPPORTED INSTALL MODE" OFF) 5 | 6 | if(NOT DEV_MODE) 7 | message(SEND_ERROR 8 | "CMake is only useful to develop this project, no support is provided." 9 | "To install the module properly use pip. See the Readme for details." 10 | "To override this you might set DEV_MODE to ON" 11 | "However: AGAIN NO SUPPORT IS PROVIDED, PIP IS THE SUPPORTED INSTALL MODE") 12 | endif() 13 | 14 | list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") 15 | 16 | find_package(Scorep REQUIRED) 17 | find_package(Python REQUIRED COMPONENTS Interpreter Development) 18 | 19 | Python_add_library(_bindings 20 | src/methods.cpp src/scorep_bindings.cpp src/scorepy/events.cpp 21 | ) 22 | if(Python_VERSION_MAJOR GREATER_EQUAL 3 AND NOT Python_INTERPRETER_ID STREQUAL "PyPy") 23 | target_sources(_bindings PRIVATE 24 | src/classes.cpp 25 | src/scorepy/cInstrumenter.cpp 26 | src/scorepy/pathUtils.cpp 27 | src/scorepy/pythonHelpers.cpp 28 | ) 29 | target_compile_definitions(_bindings PRIVATE SCOREPY_ENABLE_CINSTRUMENTER=1) 30 | else() 31 | target_compile_definitions(_bindings PRIVATE SCOREPY_ENABLE_CINSTRUMENTER=0) 32 | endif() 33 | target_link_libraries(_bindings PRIVATE Scorep::Plugin) 34 | target_compile_features(_bindings PRIVATE cxx_std_11) 35 | target_compile_definitions(_bindings PRIVATE PY_SSIZE_T_CLEAN) 36 | target_include_directories(_bindings PRIVATE src) 37 | 38 | set_target_properties(_bindings PROPERTIES 39 | LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/site-packages/scorep 40 | ) 41 | add_custom_target(ScorepModule ALL 42 | ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_LIST_DIR}/scorep $ 43 | COMMENT "Copying module files to build tree" 44 | ) 45 | 46 | enable_testing() 47 | add_test(NAME ScorepPythonTests 48 | COMMAND Python::Interpreter -m pytest 49 | WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/test 50 | ) 51 | set(pythonPath ${CMAKE_CURRENT_BINARY_DIR}/site-packages) 52 | if(ENV{PYTHONPATH}) 53 | string(PREPEND pythonPath "$ENV{PYTHONPATH}:") 54 | endif() 55 | set_tests_properties(ScorepPythonTests PROPERTIES ENVIRONMENT "PYTHONPATH=${pythonPath}") 56 | add_executable(CppTests test/test_pathUtils.cpp src/scorepy/pathUtils.cpp) 57 | target_include_directories(CppTests PRIVATE src) 58 | add_test(NAME CppTests COMMAND CppTests) 59 | 60 | set(INSTALL_DIR "lib/python${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}/site-packages") 61 | 62 | install(DIRECTORY scorep DESTINATION ${INSTALL_DIR}) 63 | install(TARGETS _bindings DESTINATION ${INSTALL_DIR}/scorep) 64 | -------------------------------------------------------------------------------- /DEVELOPING.md: -------------------------------------------------------------------------------- 1 | # Developing 2 | We appreciate any contributions to the Score-P Python Bindings. 3 | However, there are a few policies we agreed on and which are relevant for contributions. 4 | These are detailed below. 5 | 6 | ## Formatting / Codestyle 7 | 8 | Readable and consistent code makes it easy to understand your changes. 9 | Therefore the CI system has checks using `clang-format-9` and `flake8` in place. 10 | Please make sure that these test pass, when making a Pull Request. 11 | These tests will tell you the issues and often also how to fix them. 12 | Prior to opening a Pull Request you can use the provided `.flake8` and `.clang-format` files, to check your code locally and run `clang-format-9` or `autopep8` to fix most of them automatically. 13 | 14 | ## Build system 15 | 16 | The official way to build and install this module is using `pip`. 17 | Please make sure that all changes, you introduce, work with `pip install .`. 18 | 19 | However, you might have noticed that there is a CMake-based build system in place as well. 20 | We do not support this build system, and it might be outdated or buggy. 21 | Although, we do not actively maintain the CMake build system, and will not help you fix issues related to it, Pull Requests against it might be accepted. 22 | 23 | You might find this build system helpful for development, especially if you are doing C/C++ things: 24 | * Include paths for C++ are correctly searched for and set up for use by IDEs or other tools. For example Visual Studio Code works out of the box, given the appropriate extensions (C++, Python, CMake) are installed. 25 | * A folder `site-packages` is created in the build folder where the C/C++ extension module and the scorep module are copied to on each build (e.g. `make`-call). Hence it is possible to add that folder to the PYTHONPATH environment variable, build the project and start debugging or execute the tests in test. 26 | * A `test` target exists which can be run to execute all tests. 27 | 28 | Please note, that changes to the Python source files are not reflected in the build folder unless a build is executed. 29 | Also, if you delete Python files, we recommended to clear and recreate the build folder. 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2022-2023, 2 | Andreas Gocht-Zech 3 | 4 | Copyright 2017-2022, Technische Universitaet Dresden, Germany, all rights reserved. 5 | Author: Andreas Gocht-Zech 6 | 7 | portions copyright 2001, Autonomous Zones Industries, Inc., all rights... 8 | err... reserved and offered to the public under the terms of the 9 | Python 2.2 license. 10 | Author: Zooko O'Whielacronx 11 | http://zooko.com/ 12 | mailto:zooko@zooko.com 13 | 14 | Copyright 2000, Mojam Media, Inc., all rights reserved. 15 | Author: Skip Montanaro 16 | 17 | Copyright 1999, Bioreason, Inc., all rights reserved. 18 | Author: Andrew Dalke 19 | 20 | Copyright 1995-1997, Automatrix, Inc., all rights reserved. 21 | Author: Skip Montanaro 22 | 23 | Copyright 1991-1995, Stichting Mathematisch Centrum, all rights reserved. 24 | 25 | Permission to use, copy, modify, and distribute this Python software and 26 | its associated documentation for any purpose without fee is hereby 27 | granted, provided that the above copyright notice appears in all copies, 28 | and that both that copyright notice and this permission notice appear in 29 | supporting documentation, and that the name of neither Automatrix, 30 | Bioreason, Mojam Media or TU Dresden be used in advertising or publicity 31 | pertaining to distribution of the software without specific, written 32 | prior permission. 33 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include src *.hpp pythoncapi_compat.h 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Unit tests](https://github.com/score-p/scorep_binding_python/workflows/Unit%20tests/badge.svg?branch=master)](https://github.com/score-p/scorep_binding_python/actions) 2 | [![Static analysis](https://github.com/score-p/scorep_binding_python/workflows/Static%20analysis/badge.svg?branch=master)](https://github.com/score-p/scorep_binding_python/actions) 3 | 4 | # scorep 5 | scorep is a module that allows tracing of python scripts using [Score-P](https://score-p.org/). 6 | 7 | # Table of Content 8 | 9 | - [scorep](#scorep) 10 | - [Table of Content](#table-of-content) 11 | - [Install](#install) 12 | - [Use](#use) 13 | * [Instrumenter](#instrumenter) 14 | + [Instrumenter Types](#instrumenter-types) 15 | + [Instrumenter User Interface](#instrumenter-user-interface) 16 | + [Instrumenter File](#instrumenter-file) 17 | * [MPI](#mpi) 18 | * [User Regions](#user-regions) 19 | * [Overview about Flags](#overview-about-flags) 20 | * [Backward Compatibility](#backward-compatibility) 21 | - [Compatibility](#compatibility) 22 | * [Working](#working) 23 | * [Not Working](#not-working) 24 | - [Citing](#citing) 25 | - [Acknowledgments](#acknowledgments) 26 | 27 | 28 | 29 | # Install 30 | You need at least Score-P 5.0, build with `--enable-shared` and the gcc compiler plugin. 31 | Please make sure that `scorep-config` is in your `PATH` variable. 32 | 33 | For Ubuntu LTS systems there is a non-official ppa of Score-P available: https://launchpad.net/~andreasgocht/+archive/ubuntu/scorep . 34 | 35 | Then install the package from PyPI 36 | 37 | ``` 38 | pip install scorep 39 | ``` 40 | 41 | or build them from source 42 | 43 | 44 | ``` 45 | pip install . 46 | ``` 47 | 48 | # Use 49 | 50 | To trace the full script, you need to run 51 | 52 | ``` 53 | python -m scorep 54 | ``` 55 | 56 | The usual Score-P environment Variables will be respected. 57 | Please have a look at: 58 | 59 | [score-p.org](https://score-p.org) 60 | 61 | and 62 | 63 | [Score-P Documentation](https://perftools.pages.jsc.fz-juelich.de/cicd/scorep/tags/latest/html/) 64 | 65 | There is also a small [HowTo](https://github.com/score-p/scorep_binding_python/wiki) in the wiki. 66 | 67 | Since version 0.9 it is possible to pass the traditional Score-P instrumentation commands to the Score-P bindings, e.g.: 68 | 69 | ``` 70 | python -m scorep --mpp=mpi --thread=pthread 71 | ``` 72 | 73 | Please be aware that these commands are forwarded to `scorep-config`, not to `scorep`. Flags like `--verbose` won't work. 74 | 75 | ## Instrumenter 76 | The instrumenter ist the key part of the bindings. 77 | It registers with the Python tracing interface, and cares about the fowarding of events to Score-P. 78 | There are currently five different instrumenter types available as described in the following section [Instrumenter Types](#instrumenter-types) . 79 | A user interface, to dynamically enable and disable the automatic instrumentation, using the python hooks, is also available and described under [Instrumenter User Interface](#instrumenter-user-interface) 80 | 81 | ### Instrumenter Types 82 | With version 2.0 of the python bindings, the term "instrumenter" is introduced. 83 | The instrumenter describes the class that maps the Python `trace` or `profile` events to Score-P. 84 | Please be aware, that `trace` and `profile` does not refer to the traditional Score-P terms of tracing and profiling, but to the Python functions [sys.settrace](https://docs.python.org/3/library/sys.html#sys.settrace) and [sys.setprofile](https://docs.python.org/3/library/sys.html#sys.setprofile). 85 | 86 | The instrumenter that shall be used for tracing can be specified using `--instrumenter-type=`. 87 | Currently there are the following instrumenters available: 88 | * `profile` (default) implements `call` and `return` 89 | * `trace` implements `call` and `return` 90 | * `cProfile` / `cTrace` are the same as the above but implemented in C++ 91 | * `dummy` does nothing, can be used without `-m scorep` (as done by user instrumentation) 92 | 93 | The `profile` instrumenter should have a smaller overhead than `trace`. 94 | Using the instrumenters implemented in C++ additionally reduces the overhead but those are only available in Python 3. 95 | 96 | It is possible to disable the instrumenter passing `--noinstrumenter`. 97 | However, the [Instrumenter User Interface](#instrumenter-user-interface) may override this flag. 98 | 99 | ### Instrumenter User Interface 100 | 101 | It is possible to enable or disable the instrumenter during the program runtime using a user interface: 102 | 103 | ``` 104 | with scorep.instrumenter.disable(): 105 | do_something() 106 | 107 | with scorep.instrumenter.enable(): 108 | do_something() 109 | ``` 110 | 111 | The main idea is to reduce the instrumentation overhead for regions that are not of interest. 112 | Whenever the instrumenter is disabled, function enter or exits will not be trace. 113 | However, user regions as described in [User Regions](#user-regions) are not affected. 114 | Both functions are also available as decorators. 115 | 116 | As an example: 117 | 118 | ``` 119 | import numpy as np 120 | 121 | [...] 122 | c = np.dot(a,b) 123 | [...] 124 | ``` 125 | 126 | You might not be interested, what happens during the import of numpy, but actually how long `dot` takes. 127 | If you change the code to 128 | 129 | ``` 130 | import numpy as np 131 | import scorep 132 | 133 | [...] 134 | with scorep.instrumenter.enable(): 135 | c = np.dot(a,b) 136 | [...] 137 | ``` 138 | and run the code with `python -m scorep --noinstrumenter run.py` only the call to np.dot and everything below will be instrumented. 139 | 140 | With version 3.1 the bindings support the annotation of regions where the instrumenter setting was changed. 141 | You can pass a `region_name` to the instrumenter calls, e.g. `scorep.instrumenter.enable("enabled_region_name")` or `scorep.instrumenter.disable("disabled_region_name")`. 142 | This might be useful if you do something expensive, and just want to know how long it takes, but you do not care what happens exactly e.g.: 143 | 144 | ``` 145 | [...] 146 | def fun_calls(n): 147 | if (n>0): 148 | fun_calls(n-1) 149 | 150 | with scorep.instrumenter.disable("my_fun_calls"): 151 | fun_calls(1000000) 152 | [...] 153 | ``` 154 | 155 | `my_fun_calls` will be present in the trace or profile but `fun_calls` will not. 156 | 157 | However, doing 158 | ``` 159 | [...] 160 | with scorep.instrumenter.disable(): 161 | with scorep.instrumenter.disable("my_fun_calls"): 162 | fun_calls(1000000) 163 | [...] 164 | ``` 165 | will only disable the instrumenter, but `my_fun_calls` will not appear in the trace or profile, as the second call to `scorep.instrumenter.disable` did not change the state of the instrumenter. 166 | Please look to [User Regions](#user-regions), if you want to annotate a region, no matter what the instrumenter state is. 167 | 168 | ### Instrumenter File 169 | 170 | Handing a Python file to `--instrumenter-file` allows the instrumentation of modules and functions without changing their code. 171 | The file handed to `--instrumenter-file` is executed before the script is executed so that the original function definition can be overwritten before the function is executed. 172 | However, using this approach, it is no longer possible to track the bring up of the module. 173 | 174 | To simplify the instrumentation, the user instrumentation contains two helper calls: 175 | ``` 176 | scorep.user.instrument_function(function, instrumenter_fun=scorep.user.region) 177 | scorep.user.instrument_module(module, instrumenter_fun=scorep.user.region): 178 | ``` 179 | while `instrumenter_fun` might be one of: 180 | * `scorep.user.region`, decorator as explained below 181 | * `scorep.instrumenter.enable`, decorator as explained above 182 | * `scorep.instrumenter.disable`, decorator as explained above 183 | 184 | Using the `scorep.instrumenter` decorators, the instrumentation can be enabled or disabled from the given function. 185 | The function is executed below `enable` or `disable`. 186 | Using `scorep.user.region`, it is possible to instrument a full python program. 187 | However, I discourage this usage, as the overhead of the user instrumentation is higher than the built-in instrumenters. 188 | 189 | Using `scorep.user.instrument_module`, all functions of the given Python Module are instrumented. 190 | 191 | An example instrumenter file might look like the following: 192 | ``` 193 | import scorep.user 194 | 195 | # import module that shall be instrumented 196 | import module_to_instrument 197 | import module 198 | 199 | # hand over the imported module, containing functions which shall be instrumented 200 | scorep.user.instrument_module(module_to_instrument) 201 | 202 | # hand the function to be instrumented, and overwrite the original definiton of that function 203 | module.function_to_instrument = scorep.user.instrument_function(module.function_to_instrument) 204 | 205 | ``` 206 | 207 | ## MPI 208 | 209 | To use trace an MPI parallel application, please specify 210 | 211 | ``` 212 | python -m scorep --mpp=mpi 213 | ``` 214 | 215 | ## User Regions 216 | Since version 2.0 the python bindings support context managers for user regions: 217 | 218 | ``` 219 | with scorep.user.region("region_name"): 220 | do_something() 221 | ``` 222 | 223 | Since version 2.1 the python bindings support also decorators for functions: 224 | 225 | ``` 226 | @scorep.user.region("region_name") 227 | def do_something(): 228 | #do some things 229 | ``` 230 | If no region name is given, the function name will be used e.g.: 231 | 232 | ``` 233 | @scorep.user.region() 234 | def do_something(): 235 | #do some things 236 | ``` 237 | 238 | will result in `__main__:do_something`. 239 | 240 | The traditional calls to define a region still exists, but the usage is discouraged: 241 | 242 | ``` 243 | scorep.user.region_begin("region_name") 244 | scorep.user.region_end("region_name") 245 | ``` 246 | 247 | User parameters can be used in any case: 248 | 249 | ``` 250 | scorep.user.parameter_int(name, val) 251 | scorep.user.parameter_uint(name, val) 252 | scorep.user.parameter_string(name, string) 253 | ``` 254 | 255 | where `name` defines the name of the parameter or region, while `val` or `string` represents the value that is passed to Score-P. 256 | 257 | Disabling the recording with Score-P is still also possible: 258 | 259 | ``` 260 | scorep.user.enable_recording() 261 | scorep.user.disable_recording() 262 | ``` 263 | 264 | However, please be aware that the runtime impact of disabling Score-P is rather small, as the instrumenter is still active. 265 | For details about the instrumenter, please see [Instrumenter](#Instrumenter). 266 | 267 | ## Overview about Flags 268 | 269 | The following flags are special to the python bindings: 270 | 271 | * `--noinstrumenter` disables the instrumentation of python code. Useful for user instrumentation and to trace only specific code regions using `scorep.instrumenter.enable`. 272 | * `--instrumenter-type=` choose an instrumenter. See [Instrumenter](#Instrumenter). 273 | * `--keep-files` temporary files are kept. 274 | 275 | ## Backward Compatibility 276 | 277 | To maintain backwards compatibility, the following flags are set per default: 278 | 279 | ``` 280 | python -m scorep --compiler --thread=pthread 281 | ``` 282 | 283 | The traditional `--mpi` does still work, and is similar to the following call: 284 | 285 | ``` 286 | python -m scorep --compiler --thread=pthread --mpp=mpi 287 | ``` 288 | 289 | To disable compiler interface, please specify: 290 | 291 | ``` 292 | python -m scorep --nocompiler 293 | ``` 294 | 295 | However, this will not remove any compiler instrumentation in any binary. 296 | 297 | For other thread schemes just specify `--thread=`. 298 | E.g. : 299 | 300 | ``` 301 | python -m scorep --thread=omp 302 | ``` 303 | 304 | Please be aware the `--user` is always passed to Score-P, as this is needed for the python instrumentation. 305 | 306 | # Compatibility 307 | ## Working 308 | * python3 309 | * python2.7, but not all features are supported 310 | * mpi using mpi4py 311 | * threaded applications 312 | 313 | 314 | ## Not Working 315 | * python multiprocessing 316 | * Score-P does currently only support MPI or SHMEM. Any other multiprocessing approach cannot be traced. 317 | * tracking `importlib.reload()` 318 | 319 | # Citing 320 | 321 | If you publish some work using the python bindings, we would appriciate, if you could cite one of the following paper: 322 | 323 | ``` 324 | Gocht A., Schöne R., Frenzel J. (2021) 325 | Advanced Python Performance Monitoring with Score-P. 326 | In: Mix H., Niethammer C., Zhou H., Nagel W.E., Resch M.M. (eds) Tools for High Performance Computing 2018 / 2019. Springer, Cham. 327 | https://doi.org/10.1007/978-3-030-66057-4_14 328 | ``` 329 | 330 | A preprint can be found at: 331 | http://arxiv.org/abs/2010.15444 332 | 333 | The full paper is available at: 334 | https://doi.org/10.1007/978-3-030-66057-4_14 335 | 336 | 337 | ``` 338 | Gocht-Zech A., Grund A. and Schöne R. (2021) 339 | Controlling the Runtime Overhead of Python Monitoring with Selective Instrumentation 340 | In: 2021 IEEE/ACM International Workshop on Programming and Performance Visualization Tools (ProTools) 341 | https://doi.org/10.1109/ProTools54808.2021.00008 342 | ``` 343 | 344 | The full paper is available at: 345 | 346 | https://doi.org/10.1109/ProTools54808.2021.00008 347 | 348 | 349 | # Acknowledgments 350 | The European Union initially supported this work as part of the European Union’s Horizon 2020 project READEX (grant agreement number 671657). 351 | -------------------------------------------------------------------------------- /benchmark/benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Created on 04.10.2019 4 | 5 | @author: gocht 6 | ''' 7 | import argparse 8 | import sys 9 | import benchmark_helper 10 | import pickle 11 | import numpy as np 12 | 13 | # Available tests 14 | tests = ["bm_baseline.py", "bm_simplefunc.py"] 15 | 16 | # Available instrumenters 17 | instrumenters = ["profile", "trace", "dummy", "None"] 18 | if sys.version_info.major >= 3: 19 | instrumenters.extend(["cProfile", "cTrace"]) 20 | 21 | # Default values for: How many times the instrumented code is run during 1 test run 22 | reps_x = { 23 | "bm_baseline.py": ["1000000", "2000000", "3000000", "4000000", "5000000"], 24 | "bm_simplefunc.py": ["100000", "200000", "300000", "400000", "500000"], 25 | } 26 | 27 | 28 | def str_to_int(s): 29 | return int(float(s)) 30 | 31 | 32 | parser = argparse.ArgumentParser(description='Benchmark the instrumenters.', 33 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 34 | parser.add_argument('--test', '-t', metavar='TEST', nargs='+', default=tests, 35 | choices=tests, help='Which test(s) to run') 36 | parser.add_argument('--repetitions', '-r', default=51, type=str_to_int, 37 | help='How many times a test invocation is repeated (number of timings per test instance)') 38 | parser.add_argument('--loop-count', '-l', type=str_to_int, nargs='+', 39 | help=('How many times the instrumented code is run during 1 test run. ' 40 | 'Can be repeated and will create 1 test instance per argument')) 41 | parser.add_argument('--instrumenter', '-i', metavar='INST', nargs='+', default=instrumenters, 42 | choices=instrumenters, help='The instrumenter(s) to use') 43 | parser.add_argument('--output', '-o', default='results.pkl', help='Output file for the results') 44 | parser.add_argument('--dry-run', action='store_true', help='Print parsed arguments and exit') 45 | args = parser.parse_args() 46 | 47 | if args.dry_run: 48 | print(args) 49 | sys.exit(0) 50 | 51 | bench = benchmark_helper.BenchmarkEnv(repetitions=args.repetitions) 52 | results = {} 53 | 54 | for test in args.test: 55 | results[test] = {} 56 | 57 | for instrumenter in args.instrumenter: 58 | results[test][instrumenter] = {} 59 | 60 | if instrumenter == "None": 61 | scorep_settings = [] 62 | else: 63 | scorep_settings = ["-m", "scorep", "--instrumenter-type={}".format(instrumenter)] 64 | 65 | print("#########") 66 | print("{}: {}".format(test, scorep_settings)) 67 | print("#########") 68 | max_reps_width = len(str(max(reps_x[test]))) 69 | loop_counts = args.loop_count if args.loop_count else reps_x[test] 70 | for reps in loop_counts: 71 | times = bench.call(test, [str(reps)], 72 | scorep_settings=scorep_settings) 73 | times = np.array(times) 74 | print("{:>{width}}: Range={:{prec}}-{:{prec}} Mean={:{prec}} Median={:{prec}}".format( 75 | reps, times.min(), times.max(), times.mean(), np.median(times), width=max_reps_width, prec='5.4f')) 76 | results[test][instrumenter][reps] = times 77 | 78 | with open(args.output, "wb") as f: 79 | pickle.dump(results, f) 80 | -------------------------------------------------------------------------------- /benchmark/benchmark_helper.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import os 3 | import shutil 4 | import sys 5 | import time 6 | import tempfile 7 | 8 | 9 | class BenchmarkEnv(): 10 | def __init__(self, repetitions=10): 11 | self.env = os.environ.copy() 12 | self.env["SCOREP_ENABLE_PROFILING"] = "false" 13 | self.env["SCOREP_ENABLE_TRACING"] = "false" 14 | self.env["SCOREP_PROFILING_MAX_CALLPATH_DEPTH"] = "98" 15 | self.env["SCOREP_TOTAL_MEMORY"] = "3G" 16 | self.exp_dir = tempfile.mkdtemp(prefix="benchmark_dir_") 17 | self.repetitions = repetitions 18 | 19 | def __del__(self): 20 | shutil.rmtree( 21 | self.exp_dir, 22 | ignore_errors=True) 23 | 24 | def call(self, script, ops=[], scorep_settings=[]): 25 | self.env["SCOREP_EXPERIMENT_DIRECTORY"] = self.exp_dir + \ 26 | "/{}-{}-{}".format(script, ops, scorep_settings) 27 | 28 | arguments = [sys.executable] 29 | arguments.extend(scorep_settings) 30 | arguments.append(script) 31 | arguments.extend(ops) 32 | 33 | runtimes = [] 34 | for _ in range(self.repetitions): 35 | begin = time.time() 36 | out = subprocess.run( 37 | arguments, 38 | env=self.env) 39 | end = time.time() 40 | assert out.returncode == 0 41 | 42 | runtime = end - begin 43 | runtimes.append(runtime) 44 | 45 | return runtimes 46 | -------------------------------------------------------------------------------- /benchmark/bm_baseline.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | result = 0 4 | iterations = int(sys.argv[1]) 5 | 6 | for i in range(iterations): 7 | result += 1 8 | 9 | assert result == iterations 10 | -------------------------------------------------------------------------------- /benchmark/bm_simplefunc.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | def add(val): 5 | return val + 1 6 | 7 | 8 | result = 0 9 | iterations = int(sys.argv[1]) 10 | 11 | for i in range(iterations): 12 | result = add(result) 13 | 14 | assert result == iterations 15 | -------------------------------------------------------------------------------- /benchmark/compare.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import pickle 3 | import numpy 4 | 5 | parser = argparse.ArgumentParser(description='Compare two benchmarks.', 6 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 7 | parser.add_argument('left', help='First input for comparison') 8 | parser.add_argument('right', help='Second input for comparison') 9 | parser.add_argument('-s', help='short output', action='store_false') 10 | args = parser.parse_args() 11 | 12 | 13 | with open(args.left, "rb") as f: 14 | left = pickle.load(f) 15 | 16 | with open(args.right, "rb") as f: 17 | right = pickle.load(f) 18 | 19 | if left.keys() != right.keys(): 20 | raise ValueError("Different Experiments") 21 | 22 | experiment = right.keys() 23 | 24 | for exp in experiment: 25 | print("\nExperiment: {}".format(exp)) 26 | if left[exp].keys() != right[exp].keys(): 27 | raise ValueError("Different Instrumenters") 28 | instrumenters = left[exp].keys() 29 | 30 | for inst in instrumenters: 31 | print("\n\tInstrumenter: {}".format(inst)) 32 | if left[exp][inst].keys() != right[exp][inst].keys(): 33 | raise ValueError("Different Iterations") 34 | iterations = left[exp][inst].keys() 35 | Y_left = [] 36 | Y_right = [] 37 | X = [] 38 | for it in iterations: 39 | left_val = left[exp][inst][it] 40 | right_val = right[exp][inst][it] 41 | if len(left_val) != len(right_val): 42 | raise ValueError("Different Repetitons") 43 | 44 | Y_left.append(numpy.mean(left_val)) 45 | Y_right.append(numpy.mean(right_val)) 46 | X.append(numpy.full([1], it)) 47 | 48 | if args.s: 49 | print("\t\tInterations {}".format(it)) 50 | print("\t\tMean: {:>7.4f} s {:>7.4f} s".format(numpy.mean(left_val), numpy.mean(right_val))) 51 | print("\t\tMedian: {:>7.4f} s {:>7.4f} s".format( 52 | numpy.quantile(left_val, 0.50), numpy.quantile(right_val, 0.50))) 53 | print("\t\t5%: {:>7.4f} s {:>7.4f} s".format( 54 | numpy.quantile(left_val, 0.05), numpy.quantile(right_val, 0.05))) 55 | print("\t\t95%: {:>7.4f} s {:>7.4f} s".format( 56 | numpy.quantile(left_val, 0.95), numpy.quantile(right_val, 0.95))) 57 | Y_left = numpy.asarray(Y_left, dtype=float).flatten() 58 | Y_right = numpy.asarray(Y_right, dtype=float).flatten() 59 | X = numpy.asarray(X, dtype=float).flatten() 60 | 61 | cost_left = numpy.polyfit(X, Y_left, 1) 62 | cost_right = numpy.polyfit(X, Y_right, 1) 63 | 64 | if args.s: 65 | print("") 66 | print("\tSlope {:>7.4f} us {:>7.4f} us".format(cost_left[0] * 1e6, cost_right[0] * 1e6)) 67 | print("\tIntercept {:>7.4f} s {:>7.4f} s".format(cost_left[1], cost_right[1])) 68 | -------------------------------------------------------------------------------- /benchmark/compare_commits.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | wd=`pwd` 4 | root_dir=`realpath "$wd/../"` 5 | 6 | echo $wd 7 | echo $root_dir 8 | 9 | function benchmark_branch { 10 | cd $root_dir 11 | git checkout $1 12 | head=`git rev-parse --short HEAD` 13 | pip install . 14 | cd $wd 15 | python benchmark.py -o result-$1-$head.pkl 16 | } 17 | 18 | sleep 5 19 | benchmark_branch $1 20 | sleep 5 21 | benchmark_branch $2 22 | 23 | -------------------------------------------------------------------------------- /benchmark/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #SBATCH --time=04:00:00 3 | #SBATCH --exclusive 4 | #SBATCH -p haswell 5 | #SBATCH -A p_readex 6 | #SBATCH -N 1 7 | #SBATCH --ntasks-per-node=1 8 | #SBATCH -c 1 9 | #SBATCH --comment=no_monitoring 10 | #SBATCH --job-name benchmark_python 11 | 12 | module load Python/3.8.6-GCCcore-10.2.0 13 | module load Score-P/7.0-gompic-2020b 14 | 15 | env_dir=~/virtenv/p-3.8.6-GCCcore-10.2.0-scorep-7.0-gompic-2020b/ 16 | 17 | if [ ! -d $env_dir ] 18 | then 19 | echo "Please create virtual env under: $env_dir" 20 | exit -1 21 | fi 22 | 23 | source $env_dir/bin/activate 24 | 25 | srun compare_commits.sh master faster 26 | -------------------------------------------------------------------------------- /cmake/FindScorep.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016, Technische Universität Dresden, Germany 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without modification, are permitted 5 | # provided that the following conditions are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright notice, this list of conditions 8 | # and the following disclaimer. 9 | # 10 | # 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions 11 | # and the following disclaimer in the documentation and/or other materials provided with the 12 | # distribution. 13 | # 14 | # 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse 15 | # or promote products derived from this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR 18 | # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 19 | # FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 20 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 23 | # IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 24 | # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | 26 | IF(SCOREP_CONFIG_PATH) 27 | FIND_PROGRAM(SCOREP_CONFIG NAMES scorep-config 28 | PATHS 29 | ${SCOREP_CONFIG_PATH} 30 | /opt/scorep/bin 31 | ) 32 | ELSE(SCOREP_CONFIG_PATH) 33 | FIND_PROGRAM(SCOREP_CONFIG NAMES scorep-config 34 | PATHS 35 | /opt/scorep/bin 36 | ) 37 | ENDIF(SCOREP_CONFIG_PATH) 38 | 39 | IF(NOT SCOREP_CONFIG) 40 | MESSAGE(STATUS "no scorep-config found") 41 | set(SCOREP_FOUND false) 42 | ELSE(NOT SCOREP_CONFIG) 43 | 44 | message(STATUS "SCOREP library found. (using ${SCOREP_CONFIG})") 45 | 46 | execute_process(COMMAND ${SCOREP_CONFIG} "--user" "--cppflags" OUTPUT_VARIABLE SCOREP_CONFIG_FLAGS) 47 | 48 | string(REGEX MATCHALL "-I[^ ]*" SCOREP_CONFIG_INCLUDES "${SCOREP_CONFIG_FLAGS}") 49 | foreach(inc ${SCOREP_CONFIG_INCLUDES}) 50 | string(SUBSTRING ${inc} 2 -1 inc) 51 | list(APPEND SCOREP_INCLUDE_DIRS ${inc}) 52 | endforeach() 53 | 54 | string(REGEX MATCHALL "(^| +)-[^I][^ ]*" SCOREP_CONFIG_CXXFLAGS "${SCOREP_CONFIG_FLAGS}") 55 | foreach(flag ${SCOREP_CONFIG_CXXFLAGS}) 56 | string(STRIP ${flag} flag) 57 | list(APPEND SCOREP_CXX_FLAGS ${flag}) 58 | endforeach() 59 | 60 | unset(SCOREP_CONFIG_FLAGS) 61 | unset(SCOREP_CONFIG_INCLUDES) 62 | unset(SCOREP_CONFIG_CXXFLAGS) 63 | 64 | execute_process(COMMAND ${SCOREP_CONFIG} "--user" "--ldflags" OUTPUT_VARIABLE _LINK_LD_ARGS) 65 | STRING( REPLACE " " ";" _LINK_LD_ARGS ${_LINK_LD_ARGS} ) 66 | FOREACH( _ARG ${_LINK_LD_ARGS} ) 67 | IF(${_ARG} MATCHES "^-L") 68 | STRING(REGEX REPLACE "^-L" "" _ARG ${_ARG}) 69 | SET(SCOREP_LINK_DIRS ${SCOREP_LINK_DIRS} ${_ARG}) 70 | ENDIF(${_ARG} MATCHES "^-L") 71 | ENDFOREACH(_ARG) 72 | 73 | execute_process(COMMAND ${SCOREP_CONFIG} "--user" "--libs" OUTPUT_VARIABLE _LINK_LD_ARGS) 74 | STRING( REPLACE " " ";" _LINK_LD_ARGS ${_LINK_LD_ARGS} ) 75 | FOREACH( _ARG ${_LINK_LD_ARGS} ) 76 | IF(${_ARG} MATCHES "^-l") 77 | STRING(REGEX REPLACE "^-l" "" _ARG ${_ARG}) 78 | FIND_LIBRARY(_SCOREP_LIB_FROM_ARG NAMES ${_ARG} 79 | PATHS 80 | ${SCOREP_LINK_DIRS} 81 | ) 82 | IF(_SCOREP_LIB_FROM_ARG) 83 | SET(SCOREP_LIBRARIES ${SCOREP_LIBRARIES} ${_SCOREP_LIB_FROM_ARG}) 84 | ENDIF(_SCOREP_LIB_FROM_ARG) 85 | UNSET(_SCOREP_LIB_FROM_ARG CACHE) 86 | ENDIF(${_ARG} MATCHES "^-l") 87 | ENDFOREACH(_ARG) 88 | 89 | set(SCOREP_FOUND true) 90 | ENDIF(NOT SCOREP_CONFIG) 91 | 92 | include (FindPackageHandleStandardArgs) 93 | FIND_PACKAGE_HANDLE_STANDARD_ARGS( 94 | Scorep DEFAULT_MSG 95 | SCOREP_CONFIG 96 | SCOREP_LIBRARIES 97 | SCOREP_INCLUDE_DIRS 98 | ) 99 | 100 | add_library(Scorep::Scorep INTERFACE IMPORTED) 101 | set_target_properties(Scorep::Scorep PROPERTIES 102 | INTERFACE_INCLUDE_DIRECTORIES "${SCOREP_INCLUDE_DIRS}" 103 | INTERFACE_LINK_LIBRARIES "${SCOREP_LIBRARIES}" 104 | ) 105 | 106 | add_library(Scorep::Plugin INTERFACE IMPORTED) 107 | set_target_properties(Scorep::Plugin PROPERTIES 108 | INTERFACE_INCLUDE_DIRECTORIES "${SCOREP_INCLUDE_DIRS}" 109 | ) 110 | 111 | mark_as_advanced(SCOREP_CONFIG) 112 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools >= 40.9.0", 4 | ] 5 | build-backend = "setuptools.build_meta:__legacy__" -------------------------------------------------------------------------------- /scorep/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ["user", "instrumenter", "__version__"] 2 | from scorep._version import __version__ 3 | -------------------------------------------------------------------------------- /scorep/__main__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import scorep.instrumenter 5 | import scorep.subsystem 6 | from scorep.helper import print_err 7 | 8 | 9 | def _err_exit(msg): 10 | print_err("scorep: " + msg) 11 | sys.exit(1) 12 | 13 | 14 | def scorep_main(argv=None): 15 | if argv is None: 16 | argv = sys.argv 17 | 18 | scorep_config = [] 19 | prog_argv = [] 20 | parse_scorep_commands = True 21 | 22 | keep_files = False 23 | verbose = False 24 | no_default_threads = False 25 | no_default_compiler = False 26 | no_instrumenter = False 27 | if scorep.instrumenter.has_c_instrumenter(): 28 | instrumenter_type = "cProfile" 29 | else: 30 | instrumenter_type = "profile" 31 | instrumenter_file = None 32 | 33 | for elem in argv[1:]: 34 | if parse_scorep_commands: 35 | if elem == "--": 36 | parse_scorep_commands = False 37 | elif elem == "--mpi": 38 | scorep_config.append("--mpp=mpi") 39 | elif elem == "--keep-files": 40 | keep_files = True 41 | elif elem == "--verbose" or elem == '-v': 42 | verbose = True 43 | elif "--thread=" in elem: 44 | scorep_config.append(elem) 45 | no_default_threads = True 46 | elif elem == "--nocompiler": 47 | scorep_config.append(elem) 48 | no_default_compiler = True 49 | elif elem == "--nopython": 50 | no_instrumenter = True 51 | elif elem == "--noinstrumenter": 52 | no_instrumenter = True 53 | elif "--instrumenter-type" in elem: 54 | param = elem.split("=") 55 | instrumenter_type = param[1] 56 | elif "--instrumenter-file" in elem: 57 | param = elem.split("=") 58 | instrumenter_file = param[1] 59 | elif elem[0] == "-": 60 | scorep_config.append(elem) 61 | else: 62 | prog_argv.append(elem) 63 | parse_scorep_commands = False 64 | else: 65 | prog_argv.append(elem) 66 | 67 | if not no_default_threads: 68 | scorep_config.append("--thread=pthread") 69 | 70 | if not no_default_compiler: 71 | scorep_config.append("--compiler") 72 | 73 | if len(prog_argv) == 0: 74 | _err_exit("Did not find a script to run") 75 | 76 | if os.environ.get("SCOREP_PYTHON_BINDINGS_INITIALISED") != "true": 77 | scorep.subsystem.init_environment(scorep_config, keep_files, verbose) 78 | os.environ["SCOREP_PYTHON_BINDINGS_INITIALISED"] = "true" 79 | """ 80 | python -m starts the module as skript. i.e. sys.argv will loke like: 81 | ['/home/gocht/Dokumente/code/scorep_python/scorep.py', '--mpi', 'mpi_test.py'] 82 | 83 | To restart python we need to remove this line, and add `python -m scorep ...` again 84 | """ 85 | new_args = [sys.executable, "-m", "scorep"] 86 | for elem in sys.argv: 87 | if "scorep/__main__.py" in elem: 88 | continue 89 | else: 90 | new_args.append(elem) 91 | 92 | os.execve(sys.executable, new_args, os.environ) 93 | else: 94 | scorep.subsystem.reset_preload() 95 | 96 | # everything is ready 97 | sys.argv = prog_argv 98 | progname = prog_argv[0] 99 | sys.path[0] = os.path.split(progname)[0] 100 | 101 | tracer = scorep.instrumenter.get_instrumenter(not no_instrumenter, 102 | instrumenter_type) 103 | 104 | if instrumenter_file: 105 | with open(instrumenter_file) as f: 106 | exec(f.read()) 107 | 108 | try: 109 | with open(progname) as fp: 110 | code = compile(fp.read(), progname, 'exec') 111 | # try to emulate __main__ namespace as much as possible 112 | globs = { 113 | '__file__': progname, 114 | '__name__': '__main__', 115 | '__package__': None, 116 | '__cached__': None, 117 | } 118 | 119 | tracer.run(code, globs, globs) 120 | except OSError as err: 121 | _err_exit("Cannot run file %r because: %s" % (sys.argv[0], err)) 122 | finally: 123 | scorep.subsystem.clean_up(keep_files) 124 | 125 | 126 | def main(argv=None): 127 | import traceback 128 | call_stack = traceback.extract_stack() 129 | call_stack_array = traceback.format_list(call_stack) 130 | call_stack_string = "" 131 | for elem in call_stack_array[:-1]: 132 | call_stack_string += elem 133 | _err_exit( 134 | "Someone called scorep.__main__.main(argv).\n" 135 | "This is not supposed to happen, but might be triggered, " 136 | "if your application calls \"sys.modules['__main__'].main\".\n" 137 | "This python stacktrace might be helpfull to find the reason:\n%s" % 138 | call_stack_string) 139 | 140 | 141 | if __name__ == '__main__': 142 | scorep_main() 143 | -------------------------------------------------------------------------------- /scorep/_instrumenters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/score-p/scorep_binding_python/3a1642a32934d31eeea40c423e62bc486c7d0e79/scorep/_instrumenters/__init__.py -------------------------------------------------------------------------------- /scorep/_instrumenters/base_instrumenter.py: -------------------------------------------------------------------------------- 1 | __all__ = ['BaseInstrumenter'] 2 | 3 | import abc 4 | import sys 5 | 6 | 7 | if sys.version_info >= (3, 4): 8 | class _BaseInstrumenter(abc.ABC): 9 | pass 10 | else: 11 | class _BaseInstrumenter(): 12 | __metaclass__ = abc.ABCMeta 13 | 14 | 15 | class BaseInstrumenter(_BaseInstrumenter): 16 | @abc.abstractmethod 17 | def register(self): 18 | pass 19 | 20 | @abc.abstractmethod 21 | def unregister(self): 22 | pass 23 | 24 | @abc.abstractmethod 25 | def get_registered(self): 26 | return None 27 | 28 | @abc.abstractmethod 29 | def run(self, cmd, globals=None, locals=None): 30 | pass 31 | 32 | @abc.abstractmethod 33 | def region_begin(self, module_name, function_name, file_name, line_number, code_object): 34 | pass 35 | 36 | @abc.abstractmethod 37 | def region_end(self, module_name, function_name, code_object): 38 | pass 39 | 40 | @abc.abstractmethod 41 | def rewind_begin(self, name, file_name=None, line_number=None): 42 | pass 43 | 44 | @abc.abstractmethod 45 | def rewind_end(self, name, value): 46 | pass 47 | 48 | @abc.abstractmethod 49 | def user_enable_recording(self): 50 | pass 51 | 52 | @abc.abstractmethod 53 | def user_disable_recording(self): 54 | pass 55 | 56 | @abc.abstractmethod 57 | def user_parameter_int(self, name, val): 58 | pass 59 | 60 | @abc.abstractmethod 61 | def user_parameter_uint(self, name, val): 62 | pass 63 | 64 | @abc.abstractmethod 65 | def user_parameter_string(self, name, string): 66 | pass 67 | 68 | @abc.abstractmethod 69 | def force_finalize(self): 70 | pass 71 | 72 | @abc.abstractmethod 73 | def reregister_exit_handler(self): 74 | pass 75 | -------------------------------------------------------------------------------- /scorep/_instrumenters/dummy.py: -------------------------------------------------------------------------------- 1 | __all__ = ['ScorepDummy'] 2 | 3 | import scorep._instrumenters.base_instrumenter as base_instrumenter 4 | 5 | 6 | class ScorepDummy(base_instrumenter.BaseInstrumenter): 7 | def __init__(self, enable_instrumenter=True): 8 | pass 9 | 10 | def register(self): 11 | pass 12 | 13 | def unregister(self): 14 | pass 15 | 16 | def get_registered(self): 17 | return None 18 | 19 | def run(self, cmd, globals=None, locals=None): 20 | if globals is None: 21 | globals = {} 22 | if locals is None: 23 | locals = {} 24 | exec(cmd, globals, locals) 25 | 26 | def try_region_begin(self, code_object): 27 | pass 28 | 29 | def region_begin(self, module_name, function_name, file_name, line_number, code_object=None): 30 | pass 31 | 32 | def try_region_end(self, code_object): 33 | pass 34 | 35 | def region_end(self, module_name, function_name, code_object=None): 36 | pass 37 | 38 | def rewind_begin(self, name, file_name=None, line_number=None): 39 | pass 40 | 41 | def rewind_end(self, name, value): 42 | pass 43 | 44 | def user_enable_recording(self): 45 | pass 46 | 47 | def user_disable_recording(self): 48 | pass 49 | 50 | def user_parameter_int(self, name, val): 51 | pass 52 | 53 | def user_parameter_uint(self, name, val): 54 | pass 55 | 56 | def user_parameter_string(self, name, string): 57 | pass 58 | 59 | def force_finalize(self): 60 | pass 61 | 62 | def reregister_exit_handler(self): 63 | pass 64 | -------------------------------------------------------------------------------- /scorep/_instrumenters/scorep_cProfile.py: -------------------------------------------------------------------------------- 1 | from scorep._instrumenters.scorep_instrumenter import ScorepInstrumenter 2 | import scorep._bindings 3 | 4 | 5 | class ScorepCProfile(scorep._bindings.CInstrumenter, ScorepInstrumenter): 6 | def __init__(self, enable_instrumenter): 7 | scorep._bindings.CInstrumenter.__init__(self, interface='Profile') 8 | ScorepInstrumenter.__init__(self, enable_instrumenter) 9 | -------------------------------------------------------------------------------- /scorep/_instrumenters/scorep_cTrace.py: -------------------------------------------------------------------------------- 1 | from scorep._instrumenters.scorep_instrumenter import ScorepInstrumenter 2 | import scorep._bindings 3 | 4 | 5 | class ScorepCTrace(scorep._bindings.CInstrumenter, ScorepInstrumenter): 6 | def __init__(self, enable_instrumenter): 7 | scorep._bindings.CInstrumenter.__init__(self, interface='Trace') 8 | ScorepInstrumenter.__init__(self, enable_instrumenter) 9 | -------------------------------------------------------------------------------- /scorep/_instrumenters/scorep_instrumenter.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import inspect 3 | import os 4 | from scorep._instrumenters import base_instrumenter 5 | import scorep._bindings 6 | 7 | 8 | class ScorepInstrumenter(base_instrumenter.BaseInstrumenter): 9 | """Base class for all instrumenters using Score-P""" 10 | 11 | def __init__(self, enable_instrumenter=True): 12 | """ 13 | @param enable_instrumenter true if the tracing shall be initialised. 14 | Please note, that it is still possible to enable the tracing later using register() 15 | """ 16 | self._tracer_registered = False 17 | self._enabled = enable_instrumenter 18 | 19 | @abc.abstractmethod 20 | def _enable_instrumenter(self): 21 | """Actually enable this instrumenter to collect events""" 22 | 23 | @abc.abstractmethod 24 | def _disable_instrumenter(self): 25 | """Stop this instrumenter from collecting events""" 26 | 27 | def register(self): 28 | """Register this instrumenter (collect events)""" 29 | if not self._tracer_registered: 30 | self._enable_instrumenter() 31 | self._tracer_registered = True 32 | 33 | def unregister(self): 34 | """Unregister this instrumenter (stop collecting events)""" 35 | if self._tracer_registered: 36 | self._disable_instrumenter() 37 | self._tracer_registered = False 38 | 39 | def get_registered(self): 40 | """Return whether this instrumenter is currently collecting events""" 41 | return self._tracer_registered 42 | 43 | def run(self, cmd, globals=None, locals=None): 44 | """Run the compiled command under this instrumenter. 45 | 46 | When the instrumenter is enabled it is registered prior to the invocation and unregistered afterwards 47 | """ 48 | if globals is None: 49 | globals = {} 50 | if locals is None: 51 | locals = {} 52 | if self._enabled: 53 | self.register() 54 | try: 55 | exec(cmd, globals, locals) 56 | finally: 57 | self.unregister() 58 | 59 | def try_region_begin(self, code_object): 60 | """Tries to record a region begin event. Retruns True on success""" 61 | return scorep._bindings.try_region_begin(code_object) 62 | 63 | def region_begin(self, module_name, function_name, file_name, line_number, code_object=None): 64 | """Record a region begin event""" 65 | scorep._bindings.region_begin( 66 | module_name, function_name, file_name, line_number, code_object) 67 | 68 | def try_region_end(self, code_object): 69 | """Tries to record a region end event. Retruns True on success""" 70 | return scorep._bindings.try_region_end(code_object) 71 | 72 | def region_end(self, module_name, function_name, code_object=None): 73 | """Record a region end event""" 74 | scorep._bindings.region_end(module_name, function_name, code_object) 75 | 76 | def rewind_begin(self, name, file_name=None, line_number=None): 77 | """ 78 | Begin of an Rewind region. If file_name or line_number is None, both will 79 | be determined automatically 80 | @param name name of the user region 81 | @param file_name file name of the user region 82 | @param line_number line number of the user region 83 | """ 84 | if file_name is None or line_number is None: 85 | frame = inspect.currentframe().f_back 86 | file_name = frame.f_globals.get('__file__', None) 87 | line_number = frame.f_lineno 88 | if file_name is not None: 89 | full_file_name = os.path.abspath(file_name) 90 | else: 91 | full_file_name = "None" 92 | 93 | scorep._bindings.rewind_begin(name, full_file_name, line_number) 94 | 95 | def rewind_end(self, name, value): 96 | """ 97 | End of an Rewind region. 98 | @param name name of the user region 99 | @param value True or False, whenether the region shall be rewinded or not. 100 | """ 101 | scorep._bindings.rewind_end(name, value) 102 | 103 | def user_enable_recording(self): 104 | """Enable writing of trace events in ScoreP""" 105 | scorep._bindings.enable_recording() 106 | 107 | def user_disable_recording(self): 108 | """Disable writing of trace events in ScoreP""" 109 | scorep._bindings.disable_recording() 110 | 111 | def user_parameter_int(self, name, val): 112 | """Record a parameter of type integer""" 113 | scorep._bindings.parameter_int(name, val) 114 | 115 | def user_parameter_uint(self, name, val): 116 | """Record a parameter of type unsigned integer""" 117 | scorep._bindings.parameter_string(name, val) 118 | 119 | def user_parameter_string(self, name, string): 120 | """Record a parameter of type string""" 121 | scorep._bindings.parameter_string(name, string) 122 | 123 | def force_finalize(self): 124 | scorep._bindings.force_finalize() 125 | 126 | def reregister_exit_handler(self): 127 | scorep._bindings.reregister_exit_handler() 128 | -------------------------------------------------------------------------------- /scorep/_instrumenters/scorep_profile.py: -------------------------------------------------------------------------------- 1 | __all__ = ['ScorepProfile'] 2 | 3 | import sys 4 | from scorep._instrumenters.utils import get_module_name 5 | from scorep._instrumenters.scorep_instrumenter import ScorepInstrumenter 6 | import scorep._bindings 7 | 8 | try: 9 | import threading 10 | except ImportError: 11 | _setprofile = sys.setprofile 12 | 13 | def _unsetprofile(): 14 | sys.setprofile(None) 15 | 16 | else: 17 | def _setprofile(func): 18 | threading.setprofile(func) 19 | sys.setprofile(func) 20 | 21 | def _unsetprofile(): 22 | sys.setprofile(None) 23 | threading.setprofile(None) 24 | 25 | 26 | class ScorepProfile(ScorepInstrumenter): 27 | def _enable_instrumenter(self): 28 | _setprofile(self._globaltrace) 29 | 30 | def _disable_instrumenter(self): 31 | _unsetprofile() 32 | 33 | def _globaltrace(self, frame, why, arg): 34 | """Handler for call events. 35 | 36 | If the code block being entered is to be ignored, returns `None', 37 | else returns self.localtrace. 38 | """ 39 | if why == 'call': 40 | code = frame.f_code 41 | if not scorep._bindings.try_region_begin(code): 42 | if not code.co_name == "_unsetprofile": 43 | modulename = get_module_name(frame) 44 | if not modulename[:6] == "scorep": 45 | file_name = code.co_filename 46 | line_number = code.co_firstlineno 47 | scorep._bindings.region_begin(modulename, code.co_name, file_name, line_number, code) 48 | elif why == 'return': 49 | code = frame.f_code 50 | if not scorep._bindings.try_region_end(code): 51 | if not code.co_name == "_unsetprofile": 52 | modulename = get_module_name(frame) 53 | if not modulename[:6] == "scorep": 54 | scorep._bindings.region_end(modulename, code.co_name, code) 55 | -------------------------------------------------------------------------------- /scorep/_instrumenters/scorep_trace.py: -------------------------------------------------------------------------------- 1 | __all__ = ['ScorepTrace'] 2 | 3 | import sys 4 | from scorep._instrumenters.utils import get_module_name 5 | from scorep._instrumenters.scorep_instrumenter import ScorepInstrumenter 6 | import scorep._bindings 7 | 8 | try: 9 | import threading 10 | except ImportError: 11 | _settrace = sys.settrace 12 | 13 | def _unsettrace(): 14 | sys.settrace(None) 15 | 16 | else: 17 | def _settrace(func): 18 | threading.settrace(func) 19 | sys.settrace(func) 20 | 21 | def _unsettrace(): 22 | sys.settrace(None) 23 | threading.settrace(None) 24 | 25 | 26 | class ScorepTrace(ScorepInstrumenter): 27 | def _enable_instrumenter(self): 28 | _settrace(self._globaltrace) 29 | 30 | def _disable_instrumenter(self): 31 | _unsettrace() 32 | 33 | def _globaltrace(self, frame, why, arg): 34 | """Handler for call events. 35 | @return self.localtrace or None 36 | """ 37 | if why == 'call': 38 | code = frame.f_code 39 | if not scorep._bindings.try_region_begin(code): 40 | if not code.co_name == "_unsetprofile": 41 | modulename = get_module_name(frame) 42 | if not modulename[:6] == "scorep": 43 | full_file_name = code.co_filename 44 | line_number = code.co_firstlineno 45 | scorep._bindings.region_begin(modulename, code.co_name, full_file_name, line_number, code) 46 | return self._localtrace 47 | return None 48 | 49 | def _localtrace(self, frame, why, arg): 50 | if why == 'return': 51 | code = frame.f_code 52 | if not scorep._bindings.try_region_end(code): 53 | if not code.co_name == "_unsetprofile": 54 | modulename = get_module_name(frame) 55 | if not modulename[:6] == "scorep": 56 | scorep._bindings.region_end(modulename, code.co_name, code) 57 | return self._localtrace 58 | -------------------------------------------------------------------------------- /scorep/_instrumenters/utils.py: -------------------------------------------------------------------------------- 1 | from scorep._bindings import abspath 2 | from scorep.instrumenter import has_c_instrumenter 3 | 4 | 5 | def get_module_name(frame): 6 | """Get the name of the module the given frame resides in""" 7 | modulename = frame.f_globals.get("__name__", None) 8 | if modulename is None: 9 | # this is a NUMPY special situation, see NEP-18, and Score-P Issue 10 | # issues #63 11 | if frame.f_code.co_filename == "<__array_function__ internals>": 12 | modulename = "numpy.__array_function__" 13 | else: 14 | modulename = "unkown" 15 | typeobject = frame.f_locals.get("self", None) 16 | if typeobject is not None: 17 | if has_c_instrumenter(): 18 | return ".".join([modulename, type(typeobject).__name__]) 19 | else: 20 | return ".".join([modulename, typeobject.__class__.__name__]) 21 | return modulename 22 | 23 | 24 | def get_file_name(frame): 25 | """Get the full path to the file the given frame resides in""" 26 | file_name = frame.f_code.co_filename 27 | if file_name is not None: 28 | full_file_name = abspath(file_name) 29 | else: 30 | full_file_name = "None" 31 | return full_file_name 32 | -------------------------------------------------------------------------------- /scorep/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "4.4.0" 2 | -------------------------------------------------------------------------------- /scorep/helper.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import sys 3 | import os 4 | import re 5 | 6 | 7 | def print_err(*args): 8 | """Print to stderr""" 9 | sys.stderr.write(' '.join(map(str, args)) + '\n') 10 | 11 | 12 | def call(arguments): 13 | """ 14 | return a triple with (returncode, stdout, stderr) from the call to subprocess 15 | """ 16 | if sys.version_info > (3, 5): 17 | out = subprocess.run( 18 | arguments, 19 | stdout=subprocess.PIPE, 20 | stderr=subprocess.PIPE) 21 | result = ( 22 | out.returncode, 23 | out.stdout.decode("utf-8"), 24 | out.stderr.decode("utf-8")) 25 | else: 26 | p = subprocess.Popen( 27 | arguments, 28 | stdout=subprocess.PIPE, 29 | stderr=subprocess.PIPE) 30 | stdout, stderr = p.communicate() 31 | p.wait() 32 | result = (p.returncode, stdout.decode("utf-8"), stderr.decode("utf-8")) 33 | return result 34 | 35 | 36 | def get_python_version(): 37 | version = "{}.{}".format( 38 | sys.version_info.major, 39 | sys.version_info.minor) 40 | return version 41 | 42 | 43 | def get_scorep_version(): 44 | (return_code, std_out, std_err) = call(["scorep", "--version"]) 45 | if (return_code != 0): 46 | raise RuntimeError("Cannot call Score-P, reason {}".format(std_err)) 47 | me = re.search("([0-9.]+)", std_out) 48 | version_str = me.group(1) 49 | try: 50 | version = float(version_str) 51 | except TypeError: 52 | raise RuntimeError( 53 | "Can not decode the Score-P Version. The version string is: \"{}\"".format(std_out)) 54 | return version 55 | 56 | 57 | def get_scorep_config(config_line=None): 58 | (return_code, std_out, std_err) = call(["scorep-info", "config-summary"]) 59 | if (return_code != 0): 60 | raise RuntimeError("Cannot call Score-P, reason {}".format(std_err)) 61 | if config_line is None: 62 | return std_out.split("\n") 63 | else: 64 | for line in std_out.split("\n"): 65 | if config_line in line: 66 | return line 67 | return None 68 | 69 | 70 | def add_to_ld_library_path(path): 71 | """ 72 | adds the path to the LD_LIBRARY_PATH. 73 | @param path path to be added 74 | """ 75 | library_path = os.environ.get("LD_LIBRARY_PATH", "") 76 | library_paths = library_path.split(":") if library_path else [] 77 | if path not in library_paths: 78 | os.environ["LD_LIBRARY_PATH"] = ':'.join([path] + library_paths) 79 | 80 | 81 | def generate_compile_deps(config): 82 | """ 83 | Generates the data needed for compilation. 84 | """ 85 | 86 | scorep_config = ["scorep-config"] + config 87 | 88 | (return_code, stdout, stderr) = call(scorep_config) 89 | if return_code != 0: 90 | raise ValueError( 91 | "given config {} is not supported\nstdout: {}\nstrerr: {}".format(config, stdout, stderr)) 92 | 93 | (_, ldflags, _) = call(scorep_config + ["--ldflags"]) 94 | (_, libs, _) = call(scorep_config + ["--libs"]) 95 | (_, mgmt_libs, _) = call(scorep_config + ["--mgmt-libs"]) 96 | (_, cflags, _) = call(scorep_config + ["--cflags"]) 97 | 98 | libs = " " + libs + " " + mgmt_libs 99 | ldflags = " " + ldflags 100 | cflags = " " + cflags 101 | 102 | lib_dir = re.findall(r" -L[/+-@.\w]*", ldflags) 103 | lib = re.findall(r" -l[/+-@.\w]*", libs) 104 | include = re.findall(r" -I[/+-@.\w]*", cflags) 105 | macro = re.findall(r" -D[/+-@.\w]*", cflags) 106 | linker_flags = re.findall(r" -Wl[/+-@.\w]*", ldflags) 107 | 108 | def remove_flag3(x): return x[3:] 109 | 110 | def remove_space1(x): return x[1:] 111 | 112 | lib_dir = list(map(remove_flag3, lib_dir)) 113 | lib = list(map(remove_space1, lib)) 114 | include = list(map(remove_flag3, include)) 115 | macro = list(map(remove_flag3, macro)) 116 | linker_flags = list(map(remove_space1, linker_flags)) 117 | 118 | macro = list(map(lambda x: tuple([x, 1]), macro)) 119 | 120 | return (include, lib, lib_dir, macro, linker_flags) 121 | -------------------------------------------------------------------------------- /scorep/instrumenter.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import os 3 | import platform 4 | import functools 5 | 6 | global_instrumenter = None 7 | 8 | 9 | def has_c_instrumenter(): 10 | """Return true if the C instrumenter(s) are available""" 11 | # We are using the UTF-8 string features from Python 3 12 | # The C Instrumenter functions are not available on PyPy 13 | return platform.python_implementation() != 'PyPy' 14 | 15 | 16 | def get_instrumenter(enable_instrumenter=False, 17 | instrumenter_type="dummy"): 18 | """ 19 | returns an instrumenter 20 | 21 | @param enable_instrumenter True if the Instrumenter should be enabled when run is called 22 | @param instrumenter_type which python tracing interface to use. 23 | Currently available: `profile` (default), `trace` and `dummy` 24 | """ 25 | global global_instrumenter 26 | if global_instrumenter is None: 27 | if instrumenter_type == "profile": 28 | from scorep._instrumenters.scorep_profile import ScorepProfile 29 | global_instrumenter = ScorepProfile(enable_instrumenter) 30 | elif instrumenter_type == "trace": 31 | from scorep._instrumenters.scorep_trace import ScorepTrace 32 | global_instrumenter = ScorepTrace(enable_instrumenter) 33 | elif instrumenter_type == "dummy": 34 | from scorep._instrumenters.dummy import ScorepDummy 35 | global_instrumenter = ScorepDummy(enable_instrumenter) 36 | elif instrumenter_type == "cTrace": 37 | from scorep._instrumenters.scorep_cTrace import ScorepCTrace 38 | global_instrumenter = ScorepCTrace(enable_instrumenter) 39 | elif instrumenter_type == "cProfile": 40 | from scorep._instrumenters.scorep_cProfile import ScorepCProfile 41 | global_instrumenter = ScorepCProfile(enable_instrumenter) 42 | else: 43 | raise RuntimeError('instrumenter_type "{}" unkown'.format(instrumenter_type)) 44 | 45 | return global_instrumenter 46 | 47 | 48 | def register(): 49 | """ 50 | Reenables the python-tracing. 51 | """ 52 | get_instrumenter().register() 53 | 54 | 55 | def unregister(): 56 | """ 57 | Disables the python-tracing. 58 | Disabling the python-tracing is more efficient than disable_recording, 59 | as python does no longer call the tracing module. 60 | However, all the other things that are traced by Score-P will still be recorded. 61 | Please call register() to enable tracing again. 62 | """ 63 | get_instrumenter().unregister() 64 | 65 | 66 | class enable(): 67 | """ 68 | Context manager to enable tracing in a certain region: 69 | ``` 70 | with enable(region_name=None): 71 | do stuff 72 | ``` 73 | This overides --noinstrumenter (--nopython legacy) 74 | If a region name is given, the region the contextmanager is active will be marked in the trace or profile 75 | """ 76 | 77 | def __init__(self, region_name=""): 78 | self.region_name = region_name 79 | if region_name == "": 80 | self.user_region_name = False 81 | else: 82 | self.user_region_name = True 83 | self.module_name = "" 84 | 85 | def _recreate_cm(self): 86 | return self 87 | 88 | def __call__(self, func): 89 | with disable(): 90 | @functools.wraps(func) 91 | def inner(*args, **kwds): 92 | with self._recreate_cm(): 93 | return func(*args, **kwds) 94 | return inner 95 | 96 | def __enter__(self): 97 | self.tracer_registered = get_instrumenter().get_registered() 98 | if not self.tracer_registered: 99 | if self.user_region_name: 100 | self.module_name = "user_instrumenter" 101 | frame = inspect.currentframe().f_back 102 | file_name = frame.f_globals.get('__file__', None) 103 | line_number = frame.f_lineno 104 | if file_name is not None: 105 | full_file_name = os.path.abspath(file_name) 106 | else: 107 | full_file_name = "None" 108 | 109 | get_instrumenter().region_begin( 110 | self.module_name, self.region_name, full_file_name, 111 | line_number) 112 | 113 | get_instrumenter().register() 114 | 115 | def __exit__(self, exc_type=None, exc_value=None, traceback=None): 116 | if not self.tracer_registered: 117 | get_instrumenter().unregister() 118 | 119 | if self.user_region_name: 120 | get_instrumenter().region_end( 121 | self.module_name, self.region_name) 122 | 123 | 124 | class disable(): 125 | """ 126 | Context manager to disable tracing in a certain region: 127 | ``` 128 | with disable(): 129 | do stuff 130 | ``` 131 | This overides --noinstrumenter (--nopython legacy) 132 | If a region name is given, the region the contextmanager is active will be marked in the trace or profile 133 | """ 134 | 135 | def __init__(self, region_name=""): 136 | self.region_name = region_name 137 | if region_name == "": 138 | self.user_region_name = False 139 | else: 140 | self.user_region_name = True 141 | self.module_name = "" 142 | self.func = None 143 | 144 | def _recreate_cm(self): 145 | return self 146 | 147 | def __call__(self, func): 148 | self.__enter__() 149 | try: 150 | @functools.wraps(func) 151 | def inner(*args, **kwds): 152 | with self._recreate_cm(): 153 | return func(*args, **kwds) 154 | finally: 155 | self.__exit__() 156 | return inner 157 | 158 | def __enter__(self): 159 | self.tracer_registered = get_instrumenter().get_registered() 160 | if self.tracer_registered: 161 | get_instrumenter().unregister() 162 | 163 | if self.user_region_name: 164 | self.module_name = "user_instrumenter" 165 | frame = inspect.currentframe().f_back 166 | file_name = frame.f_globals.get('__file__', None) 167 | line_number = frame.f_lineno 168 | if file_name is not None: 169 | full_file_name = os.path.abspath(file_name) 170 | else: 171 | full_file_name = "None" 172 | 173 | get_instrumenter().region_begin( 174 | self.module_name, self.region_name, full_file_name, 175 | line_number) 176 | 177 | def __exit__(self, exc_type=None, exc_value=None, traceback=None): 178 | if self.tracer_registered: 179 | if self.user_region_name: 180 | get_instrumenter().region_end( 181 | self.module_name, self.region_name) 182 | 183 | get_instrumenter().register() 184 | -------------------------------------------------------------------------------- /scorep/subsystem.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | import shutil 4 | 5 | import scorep.helper 6 | from scorep.helper import print_err 7 | 8 | 9 | def _print_info(msg): 10 | """Print an info message with a prefix""" 11 | print_err("scorep: " + msg) 12 | 13 | 14 | def generate_subsystem_lib_name(): 15 | """ 16 | generate the name for the subsystem lib. 17 | """ 18 | mpi_lib_name = "libscorep_init_subsystem-{}.so".format( 19 | scorep.helper.get_python_version()) 20 | return mpi_lib_name 21 | 22 | 23 | def generate_ld_preload(scorep_config): 24 | """ 25 | This functions generates a string that needs to be passed to $LD_PRELOAD. 26 | After this string is passed, the tracing needs to be restarted with this $LD_PRELOAD in env. 27 | 28 | @return ld_preload string which needs to be passed to LD_PRELOAD 29 | """ 30 | 31 | (_, preload, _) = scorep.helper.call(["scorep-config"] + scorep_config + ["--preload-libs"]) 32 | return preload.strip() 33 | 34 | 35 | def generate_subsystem_code(config): 36 | """ 37 | Generates the data needed to be preloaded. 38 | """ 39 | 40 | scorep_config = ["scorep-config"] + config 41 | 42 | (return_code, _, _) = scorep.helper.call(scorep_config) 43 | if return_code != 0: 44 | raise ValueError( 45 | "given config {} is not supported".format(scorep_config)) 46 | (_, scorep_adapter_init, _) = scorep.helper.call( 47 | scorep_config + ["--adapter-init"]) 48 | 49 | return scorep_adapter_init 50 | 51 | 52 | def generate(scorep_config, keep_files=False): 53 | """ 54 | Uses the scorep_config to compile the scorep subsystem. 55 | Returns the name of the compiled subsystem, and the path the the temp folder, where the lib is located 56 | 57 | @param scorep_config scorep configuration to build subsystem 58 | @param keep_files whether to keep the generated files, or not. 59 | """ 60 | 61 | (include, lib, lib_dir, macro, 62 | linker_flags_tmp) = scorep.helper.generate_compile_deps(scorep_config) 63 | scorep_adapter_init = generate_subsystem_code(scorep_config) 64 | if ("-lscorep_adapter_opari2_mgmt" in lib): 65 | scorep_adapter_init += "\n" 66 | scorep_adapter_init += "/* OPARI dependencies */\n" 67 | scorep_adapter_init += "void POMP2_Init_regions(){}\n" 68 | scorep_adapter_init += "size_t POMP2_Get_num_regions(){return 0;};\n" 69 | scorep_adapter_init += "void POMP2_USER_Init_regions(){};\n" 70 | scorep_adapter_init += "size_t POMP2_USER_Get_num_regions(){return 0;};\n" 71 | 72 | # add -Wl,-no-as-needed to tell the compiler that we really want to link these. Actually this sould be default. 73 | # as distutils adds extra args at the very end we need to add all the libs 74 | # after this and skipt the libs later in the extension module 75 | linker_flags = ["-Wl,-no-as-needed"] 76 | linker_flags.extend(lib) 77 | linker_flags.extend(linker_flags_tmp) 78 | 79 | temp_dir = tempfile.mkdtemp(prefix="scorep.") 80 | if keep_files: 81 | _print_info("Score-P files are kept at: " + temp_dir) 82 | 83 | with open(temp_dir + "/scorep_init.c", "w") as f: 84 | f.write(scorep_adapter_init) 85 | 86 | subsystem_lib_name = generate_subsystem_lib_name() 87 | 88 | # setuptools, which replaces distutils, calls uname in python < 3.9 during distutils bootstraping. 89 | # When LD_PRELOAD is set, this leads to preloading Score-P to uname, and crashes the later tracing. 90 | # To avoid this, we need to do the distutils bootstrap as late as possible. 91 | # Setuptools does not support ccompiler.new_compiler https://github.com/pypa/setuptools/issues/4540 92 | from setuptools._distutils.ccompiler import new_compiler 93 | cc = new_compiler() 94 | 95 | compiled_subsystem = cc.compile( 96 | [temp_dir + "/scorep_init.c"], output_dir=temp_dir) 97 | cc.link( 98 | "scorep_init_mpi", 99 | objects=compiled_subsystem, 100 | output_filename=subsystem_lib_name, 101 | output_dir=temp_dir, 102 | library_dirs=lib_dir, 103 | extra_postargs=linker_flags) 104 | 105 | os.environ["SCOREP_PYTHON_BINDINGS_TEMP_DIR"] = temp_dir 106 | return subsystem_lib_name, temp_dir 107 | 108 | 109 | def init_environment(scorep_config, keep_files=False, verbose=False): 110 | """ 111 | Set the inital needed environment variables, to get everything up an running. 112 | As a few variables interact with LD env vars, the program needs to be restarted after this. 113 | 114 | @param scorep_config configuration flags for score-p 115 | @param keep_files whether to keep the generated files, or not. 116 | @param verbose Set to True to output information about config used and environment variables set. 117 | """ 118 | 119 | if "libscorep" in os.environ.get("LD_PRELOAD", ""): 120 | raise RuntimeError("Score-P is already loaded. This should not happen at this point") 121 | 122 | if "--user" not in scorep_config: 123 | scorep_config.append("--user") 124 | 125 | if verbose: 126 | _print_info("Score-P config: %s" % scorep_config) 127 | 128 | old_env = os.environ.copy() 129 | 130 | subsystem_lib_name, temp_dir = generate(scorep_config, keep_files) 131 | scorep_ld_preload = generate_ld_preload(scorep_config) 132 | 133 | if not os.access(temp_dir + "/" + subsystem_lib_name, os.X_OK): 134 | clean_up(keep_files=keep_files) 135 | raise RuntimeError( 136 | "The Score-P Subsystem Library at {} cannot be executed. Changing $TMP might help. " 137 | "Directory erased, use --keep-files to inspect the situation.".format( 138 | temp_dir + "/" + subsystem_lib_name)) 139 | 140 | scorep.helper.add_to_ld_library_path(temp_dir) 141 | 142 | preload_str = scorep_ld_preload + " " + subsystem_lib_name 143 | if os.environ.get("LD_PRELOAD"): 144 | print_err("LD_PRELOAD is already specified. If Score-P is already loaded this might lead to errors.") 145 | preload_str = os.environ["LD_PRELOAD"] + " " + preload_str 146 | os.environ["SCOREP_LD_PRELOAD_BACKUP"] = os.environ["LD_PRELOAD"] 147 | else: 148 | os.environ["SCOREP_LD_PRELOAD_BACKUP"] = "" 149 | os.environ["LD_PRELOAD"] = preload_str 150 | 151 | if verbose: 152 | for var in ("LD_LIBRARY_PATH", "LD_PRELOAD"): 153 | # Shorten the setting to e.g.: FOO=new:$FOO 154 | old_val = old_env.get(var) 155 | new_val = os.environ[var] 156 | if old_val: 157 | new_val = new_val.replace(old_val, '$' + var) 158 | _print_info('%s="%s"' % (var, new_val)) 159 | 160 | 161 | def reset_preload(): 162 | """ 163 | resets the environment variable `LD_PRELOAD` to the value before init_environment was called. 164 | """ 165 | if "SCOREP_LD_PRELOAD_BACKUP" in os.environ: 166 | if os.environ["SCOREP_LD_PRELOAD_BACKUP"] == "": 167 | del os.environ["LD_PRELOAD"] 168 | else: 169 | os.environ["LD_PRELOAD"] = os.environ["SCOREP_LD_PRELOAD_BACKUP"] 170 | 171 | 172 | def clean_up(keep_files=True): 173 | """ 174 | deletes the files that are associated to subsystem 175 | 176 | @param keep_files do not delete the generated files. For debugging. 177 | """ 178 | if keep_files: 179 | return 180 | else: 181 | if os.environ.get("SCOREP_PYTHON_BINDINGS_TEMP_DIR"): 182 | shutil.rmtree(os.environ["SCOREP_PYTHON_BINDINGS_TEMP_DIR"]) 183 | -------------------------------------------------------------------------------- /scorep/user.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import os.path 3 | import scorep.instrumenter 4 | import functools 5 | 6 | 7 | def region_begin(name, file_name=None, line_number=None): 8 | """ 9 | Begin of an User region. If file_name or line_number is None, both will 10 | be determined automatically 11 | @param name name of the user region 12 | @param file_name file name of the user region 13 | @param line_number line number of the user region 14 | """ 15 | with scorep.instrumenter.disable(): 16 | if file_name is None or line_number is None: 17 | frame = inspect.currentframe().f_back 18 | file_name = frame.f_globals.get('__file__', None) 19 | line_number = frame.f_lineno 20 | if file_name is not None: 21 | full_file_name = os.path.abspath(file_name) 22 | else: 23 | full_file_name = "None" 24 | 25 | scorep.instrumenter.get_instrumenter().region_begin( 26 | "user", name, full_file_name, line_number) 27 | 28 | 29 | def region_end(name): 30 | scorep.instrumenter.get_instrumenter().region_end("user", name) 31 | 32 | 33 | class region(object): 34 | """ 35 | Context manager or decorator for regions: 36 | ``` 37 | with region("some name"): 38 | do stuff 39 | 40 | @region() 41 | def fun(): 42 | do stuff 43 | ``` 44 | 45 | details for decorator stuff: 46 | https://github.com/python/cpython/blob/3.8/Lib/contextlib.py#L71 47 | 48 | """ 49 | 50 | def __init__(self, region_name=""): 51 | self.region_name = region_name 52 | if region_name == "": 53 | self.user_region_name = False 54 | else: 55 | self.user_region_name = True 56 | self.module_name = "" 57 | self.func = None 58 | 59 | def _recreate_cm(self): 60 | return self 61 | 62 | def __call__(self, func): 63 | with scorep.instrumenter.disable(): 64 | self.func = func 65 | 66 | @functools.wraps(func) 67 | def inner(*args, **kwds): 68 | with self._recreate_cm(): 69 | return func(*args, **kwds) 70 | 71 | return inner 72 | 73 | def __enter__(self): 74 | initally_registered = scorep.instrumenter.get_instrumenter().get_registered() 75 | with scorep.instrumenter.disable(): 76 | if self.user_region_name: 77 | # The user did specify a region name, so its a user_region 78 | self.module_name = "user" 79 | frame = inspect.currentframe().f_back 80 | file_name = frame.f_globals.get('__file__', None) 81 | line_number = frame.f_lineno 82 | if file_name is not None: 83 | full_file_name = os.path.abspath(file_name) 84 | else: 85 | full_file_name = "None" 86 | 87 | scorep.instrumenter.get_instrumenter().region_begin( 88 | self.module_name, self.region_name, full_file_name, line_number) 89 | elif callable(self.func) and not initally_registered: 90 | # The user did not specify a region name, and it's a callable, so it's a semi instrumented region 91 | self.code_obj = self.func.__code__ 92 | if not scorep.instrumenter.get_instrumenter().try_region_begin(self.code_obj): 93 | self.region_name = self.func.__name__ 94 | self.module_name = self.func.__module__ 95 | file_name = self.func.__code__.co_filename 96 | line_number = self.func.__code__.co_firstlineno 97 | scorep.instrumenter.get_instrumenter().region_begin( 98 | self.module_name, self.region_name, file_name, line_number, self.code_obj) 99 | elif callable(self.func) and initally_registered: 100 | # The user did not specify a region name, and it's a callable, so it's a 101 | # semi instrumented region. However, the instrumenter is active, so there 102 | # is nothing to do. 103 | pass 104 | else: 105 | # The user did not specify a region name, and it's not a callable. So it 106 | # is a context region without a region name. Throw an error. 107 | raise RuntimeError("A region name needs to be specified.") 108 | 109 | return self 110 | 111 | def __exit__(self, exc_type, exc_value, traceback): 112 | initally_registered = scorep.instrumenter.get_instrumenter().get_registered() 113 | if self.user_region_name: 114 | # The user did specify a region name, so its a user_region 115 | scorep.instrumenter.get_instrumenter().region_end( 116 | self.module_name, self.region_name) 117 | elif callable(self.func) and not initally_registered: 118 | # The user did not specify a region name, and it's a callable, so it's a semi instrumented region 119 | if not scorep.instrumenter.get_instrumenter().try_region_end(self.code_obj): 120 | scorep.instrumenter.get_instrumenter().region_end(self.module_name, self.region_name, self.code_obj) 121 | elif callable(self.func) and initally_registered: 122 | # The user did not specify a region name, and it's a callable, so it's a 123 | # semi instrumented region. However, the instrumenter is active, so there 124 | # is nothing to do. 125 | pass 126 | else: 127 | # The user did not specify a region name, and it's not a callable. So it 128 | # is a context region without a region name. Throw an error. 129 | raise RuntimeError("Something wen't wrong. Please do a Bug Report.") 130 | return False 131 | 132 | 133 | def instrument_function(fun, instrumenter_fun=region): 134 | return instrumenter_fun()(fun) 135 | 136 | 137 | def instrument_module(module, instrumenter_fun=region): 138 | for elem in dir(module): 139 | module_fun = module.__dict__[elem] 140 | if inspect.isfunction(module_fun): 141 | module.__dict__[elem] = instrument_function(module_fun, instrumenter_fun) 142 | 143 | 144 | def rewind_begin(name, file_name=None, line_number=None): 145 | """ 146 | Begin of a Rewind region. If file_name or line_number is None, both will 147 | be determined automatically 148 | @param name name of the user region 149 | @param file_name file name of the user region 150 | @param line_number line number of the user region 151 | """ 152 | with scorep.instrumenter.disable(): 153 | if file_name is None or line_number is None: 154 | frame = inspect.currentframe().f_back 155 | file_name = frame.f_globals.get('__file__', None) 156 | line_number = frame.f_lineno 157 | if file_name is not None: 158 | full_file_name = os.path.abspath(file_name) 159 | else: 160 | full_file_name = "None" 161 | 162 | scorep.instrumenter.get_instrumenter().rewind_begin( 163 | name, full_file_name, line_number) 164 | 165 | 166 | def rewind_end(name, value): 167 | """ 168 | End of a Rewind region. 169 | @param name name of the user region 170 | @param value True or False, whenether the region shall be rewinded or not. 171 | """ 172 | scorep.instrumenter.get_instrumenter().rewind_end(name, value) 173 | 174 | 175 | def enable_recording(): 176 | scorep.instrumenter.get_instrumenter().user_enable_recording() 177 | 178 | 179 | def disable_recording(): 180 | scorep.instrumenter.get_instrumenter().user_disable_recording() 181 | 182 | 183 | def parameter_int(name, val): 184 | scorep.instrumenter.get_instrumenter().user_parameter_int(name, val) 185 | 186 | 187 | def parameter_uint(name, val): 188 | scorep.instrumenter.get_instrumenter().user_parameter_uint(name, val) 189 | 190 | 191 | def parameter_string(name, string): 192 | scorep.instrumenter.get_instrumenter().user_parameter_string(name, string) 193 | 194 | 195 | def force_finalize(): 196 | """ 197 | Forces a finalisation, which might trigger traces or profiles to be written. 198 | Events after a call to @force_finalize() might not be recorded. 199 | """ 200 | scorep.instrumenter.get_instrumenter().force_finalize() 201 | 202 | 203 | def reregister_exit_handler(): 204 | """ 205 | Registers necessary atexit handler again if they have been overwritten. 206 | """ 207 | scorep.instrumenter.get_instrumenter().reregister_exit_handler() 208 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | license_files = LICENSE 3 | [pycodestyle] 4 | max_line_length = 120 5 | indent-size = 4 6 | [flake8] 7 | max_line_length = 120 8 | [tool:pytest] 9 | addopts = -ra 10 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from setuptools import setup, Extension 4 | import scorep.helper 5 | from scorep.instrumenter import has_c_instrumenter 6 | 7 | if scorep.helper.get_scorep_version() < 5.0: 8 | raise RuntimeError("Score-P version less than 5.0, plase use Score-P >= 5.0") 9 | 10 | link_mode = scorep.helper.get_scorep_config("Link mode:") 11 | if not ("shared=yes" in link_mode): 12 | raise RuntimeError( 13 | 'Score-P not build with "--enable-shared". Link mode is:\n{}'.format(link_mode) 14 | ) 15 | 16 | check_compiler = scorep.helper.get_scorep_config("C99 compiler used:") 17 | if check_compiler is None: 18 | check_compiler = scorep.helper.get_scorep_config("C99 compiler:") 19 | if check_compiler is None: 20 | raise RuntimeError("Can not parse the C99 compiler, aborting!") 21 | if "gcc" in check_compiler: 22 | gcc_plugin = scorep.helper.get_scorep_config("GCC plug-in support:") 23 | if not ("yes" in gcc_plugin): 24 | raise RuntimeError( 25 | "Score-P uses GCC but is not build with GCC Compiler Plugin. " 26 | "GCC plug-in support is:\n{}".format(gcc_plugin) 27 | ) 28 | 29 | 30 | cmodules = [] 31 | (include, _, _, _, _) = scorep.helper.generate_compile_deps([]) 32 | src_folder = os.path.abspath("src") 33 | include += [src_folder] 34 | sources = [ 35 | "src/methods.cpp", 36 | "src/scorep_bindings.cpp", 37 | "src/scorepy/events.cpp", 38 | "src/scorepy/pathUtils.cpp", 39 | ] 40 | define_macros = [("PY_SSIZE_T_CLEAN", "1")] 41 | # We are using the UTF-8 string features from Python 3 42 | # The C Instrumenter functions are not available on PyPy 43 | if has_c_instrumenter(): 44 | sources.extend( 45 | [ 46 | "src/classes.cpp", 47 | "src/scorepy/cInstrumenter.cpp", 48 | "src/scorepy/pythonHelpers.cpp", 49 | ] 50 | ) 51 | define_macros.append(("SCOREPY_ENABLE_CINSTRUMENTER", "1")) 52 | else: 53 | define_macros.append(("SCOREPY_ENABLE_CINSTRUMENTER", "0")) 54 | 55 | cmodules.append( 56 | Extension( 57 | "scorep._bindings", 58 | include_dirs=include, 59 | define_macros=define_macros, 60 | extra_compile_args=["-std=c++17"], 61 | sources=sources, 62 | ) 63 | ) 64 | 65 | setup( 66 | name="scorep", 67 | version=scorep._version.__version__, 68 | description="This is a Score-P tracing package for python", 69 | author="Andreas Gocht", 70 | author_email="andreas.gocht@tu-dresden.de", 71 | url="https://github.com/score-p/scorep_binding_python", 72 | long_description=""" 73 | This package allows tracing of python code using Score-P. 74 | A working Score-P version is required. 75 | To enable tracing it uses LD_PRELOAD to load the Score-P runtime libraries. 76 | Besides this, it uses the traditional python-tracing infrastructure. 77 | """, 78 | packages=["scorep", "scorep._instrumenters"], 79 | ext_modules=cmodules, 80 | classifiers=[ 81 | "Development Status :: 5 - Production/Stable", 82 | "Environment :: Console", 83 | "Intended Audience :: Developers", 84 | "Topic :: Software Development :: Testing", 85 | "Topic :: Software Development :: Quality Assurance", 86 | "Programming Language :: Python :: 2", 87 | "Programming Language :: Python :: 2.7", 88 | "Programming Language :: Python :: 3", 89 | "Programming Language :: Python :: 3.4", 90 | "Programming Language :: Python :: 3.5", 91 | "Programming Language :: Python :: 3.6", 92 | "Programming Language :: Python :: 3.7", 93 | "Programming Language :: Python :: 3.8", 94 | "Programming Language :: Python :: 3.9", 95 | "Programming Language :: Python :: 3.10", 96 | "Programming Language :: Python :: Implementation :: CPython", 97 | "Programming Language :: Python :: Implementation :: PyPy", 98 | "Operating System :: POSIX", 99 | "Operating System :: Unix", 100 | ], 101 | ) 102 | -------------------------------------------------------------------------------- /src/classes.cpp: -------------------------------------------------------------------------------- 1 | #include "classes.hpp" 2 | #include "scorepy/cInstrumenter.hpp" 3 | #include "scorepy/pythonHelpers.hpp" 4 | #include 5 | #include 6 | 7 | static_assert(std::is_trivial::value, 8 | "Must be trivial or object creation by Python is UB"); 9 | static_assert(std::is_standard_layout::value, 10 | "Must be standard layout or object creation by Python is UB"); 11 | 12 | extern "C" 13 | { 14 | 15 | /// tp_new implementation that calls object.__new__ with not args to allow ABC classes to work 16 | static PyObject* call_object_new(PyTypeObject* type, PyObject*, PyObject*) 17 | { 18 | scorepy::PyRefObject empty_tuple(PyTuple_New(0), scorepy::adopt_object); 19 | if (!empty_tuple) 20 | { 21 | return nullptr; 22 | } 23 | scorepy::PyRefObject empty_dict(PyDict_New(), scorepy::adopt_object); 24 | if (!empty_dict) 25 | { 26 | return nullptr; 27 | } 28 | return PyBaseObject_Type.tp_new(type, empty_tuple, empty_dict); 29 | } 30 | 31 | static void CInstrumenter_dealloc(scorepy::CInstrumenter* self) 32 | { 33 | self->deinit(); 34 | Py_TYPE(self)->tp_free(self->to_PyObject()); 35 | } 36 | 37 | static int CInstrumenter_init(scorepy::CInstrumenter* self, PyObject* args, PyObject* kwds) 38 | { 39 | static const char* kwlist[] = { "interface", nullptr }; 40 | const char* interface_cstring; 41 | 42 | if (!PyArg_ParseTupleAndKeywords(args, kwds, "s", const_cast(kwlist), 43 | &interface_cstring)) 44 | { 45 | return -1; 46 | } 47 | 48 | const std::string interface_string = interface_cstring; 49 | scorepy::InstrumenterInterface interface; 50 | if (interface_string == "Trace") 51 | { 52 | interface = scorepy::InstrumenterInterface::Trace; 53 | } 54 | else if (interface_string == "Profile") 55 | { 56 | interface = scorepy::InstrumenterInterface::Profile; 57 | } 58 | else 59 | { 60 | PyErr_Format(PyExc_TypeError, "Expected 'Trace' or 'Profile', got '%s'", 61 | interface_cstring); 62 | return -1; 63 | } 64 | 65 | self->init(interface); 66 | return 0; 67 | } 68 | 69 | static PyObject* CInstrumenter_get_interface(scorepy::CInstrumenter* self, void*) 70 | { 71 | const char* result = 72 | self->interface == scorepy::InstrumenterInterface::Trace ? "Trace" : "Profile"; 73 | return PyUnicode_FromString(result); 74 | } 75 | 76 | static PyObject* CInstrumenter_enable_instrumenter(scorepy::CInstrumenter* self, PyObject*) 77 | { 78 | self->enable_instrumenter(); 79 | Py_RETURN_NONE; 80 | } 81 | 82 | static PyObject* CInstrumenter_disable_instrumenter(scorepy::CInstrumenter* self, PyObject*) 83 | { 84 | self->disable_instrumenter(); 85 | Py_RETURN_NONE; 86 | } 87 | 88 | static PyObject* CInstrumenter_call(scorepy::CInstrumenter* self, PyObject* args, 89 | PyObject* kwds) 90 | { 91 | static const char* kwlist[] = { "frame", "event", "arg", nullptr }; 92 | 93 | PyFrameObject* frame; 94 | const char* event; 95 | PyObject* arg; 96 | 97 | if (!PyArg_ParseTupleAndKeywords(args, kwds, "OsO", const_cast(kwlist), &frame, 98 | &event, &arg)) 99 | { 100 | return nullptr; 101 | } 102 | return (*self)(*frame, event, arg); 103 | } 104 | } 105 | 106 | namespace scorepy 107 | { 108 | 109 | PyTypeObject& getCInstrumenterType() 110 | { 111 | static PyMethodDef methods[] = { 112 | { "_enable_instrumenter", scorepy::cast_to_PyFunc(CInstrumenter_enable_instrumenter), 113 | METH_NOARGS, "Enable the instrumenter" }, 114 | { "_disable_instrumenter", scorepy::cast_to_PyFunc(CInstrumenter_disable_instrumenter), 115 | METH_NOARGS, "Disable the instrumenter" }, 116 | { nullptr } /* Sentinel */ 117 | }; 118 | static PyGetSetDef getseters[] = { 119 | { "interface", scorepy::cast_to_PyFunc(CInstrumenter_get_interface), nullptr, 120 | "Return the used interface for instrumentation", nullptr }, 121 | { nullptr } /* Sentinel */ 122 | }; 123 | // Sets the first few fields explicitely and remaining ones to zero 124 | static PyTypeObject type = { 125 | PyVarObject_HEAD_INIT(nullptr, 0) /* header */ 126 | "scorep._bindings.CInstrumenter", /* tp_name */ 127 | sizeof(CInstrumenter), /* tp_basicsize */ 128 | }; 129 | type.tp_new = call_object_new; 130 | type.tp_init = scorepy::cast_to_PyFunc(CInstrumenter_init); 131 | type.tp_dealloc = scorepy::cast_to_PyFunc(CInstrumenter_dealloc); 132 | type.tp_methods = methods; 133 | type.tp_call = scorepy::cast_to_PyFunc(CInstrumenter_call); 134 | type.tp_getset = getseters; 135 | type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; 136 | type.tp_doc = "Class for the C instrumenter interface of Score-P"; 137 | return type; 138 | } 139 | 140 | } // namespace scorepy 141 | -------------------------------------------------------------------------------- /src/classes.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace scorepy 6 | { 7 | /// Return the type info to define for the python module 8 | PyTypeObject& getCInstrumenterType(); 9 | } // namespace scorepy 10 | -------------------------------------------------------------------------------- /src/methods.cpp: -------------------------------------------------------------------------------- 1 | #include "methods.hpp" 2 | #include "scorepy/events.hpp" 3 | #include "scorepy/pathUtils.hpp" 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | extern "C" 11 | { 12 | 13 | extern const char* SCOREP_GetExperimentDirName(void); 14 | 15 | extern void SCOREP_RegisterExitHandler(void); 16 | extern void SCOREP_FinalizeMeasurement(void); 17 | 18 | static PyObject* enable_recording(PyObject* self, PyObject* args) 19 | { 20 | SCOREP_User_EnableRecording(); 21 | Py_RETURN_NONE; 22 | } 23 | 24 | static PyObject* disable_recording(PyObject* self, PyObject* args) 25 | { 26 | 27 | SCOREP_User_DisableRecording(); 28 | Py_RETURN_NONE; 29 | } 30 | 31 | static PyObject* try_region_begin(PyObject* self, PyObject* args) 32 | { 33 | PyObject* identifier = nullptr; 34 | if (!PyArg_ParseTuple(args, "O", &identifier)) 35 | { 36 | return NULL; 37 | } 38 | 39 | bool success = scorepy::try_region_begin(reinterpret_cast(identifier)); 40 | if (success) 41 | { 42 | Py_RETURN_TRUE; 43 | } 44 | else 45 | { 46 | Py_RETURN_FALSE; 47 | } 48 | } 49 | 50 | /** This code is not thread save. However, this does not matter as the python GIL is not 51 | * released. 52 | */ 53 | static PyObject* region_begin(PyObject* self, PyObject* args) 54 | { 55 | const char* module_cstr; 56 | const char* function_name_cstr; 57 | const char* file_name_cstr; 58 | Py_ssize_t module_len; 59 | Py_ssize_t function_name_len; 60 | Py_ssize_t file_name_len; 61 | 62 | PyObject* identifier = nullptr; 63 | std::uint64_t line_number = 0; 64 | 65 | if (!PyArg_ParseTuple(args, "s#s#s#KO", &module_cstr, &module_len, &function_name_cstr, 66 | &function_name_len, &file_name_cstr, &file_name_len, &line_number, 67 | &identifier)) 68 | { 69 | return NULL; 70 | } 71 | 72 | std::string module(module_cstr, module_len); 73 | std::string_view function_name(function_name_cstr, function_name_len); 74 | auto const file_name_abs = 75 | scorepy::abspath(std::string_view(file_name_cstr, file_name_len)); 76 | 77 | if (identifier == nullptr || identifier == Py_None) 78 | { 79 | scorepy::region_begin(function_name, std::move(module), std::move(file_name_abs), 80 | line_number); 81 | } 82 | else 83 | { 84 | scorepy::region_begin(function_name, std::move(module), std::move(file_name_abs), 85 | line_number, reinterpret_cast(identifier)); 86 | } 87 | 88 | Py_RETURN_NONE; 89 | } 90 | 91 | static PyObject* try_region_end(PyObject* self, PyObject* args) 92 | { 93 | PyObject* identifier = nullptr; 94 | if (!PyArg_ParseTuple(args, "O", &identifier)) 95 | { 96 | return NULL; 97 | } 98 | 99 | bool success = scorepy::try_region_end(reinterpret_cast(identifier)); 100 | if (success) 101 | { 102 | Py_RETURN_TRUE; 103 | } 104 | else 105 | { 106 | Py_RETURN_FALSE; 107 | } 108 | } 109 | 110 | /** This code is not thread save. However, this does not matter as the python GIL is not 111 | * released. 112 | */ 113 | static PyObject* region_end(PyObject* self, PyObject* args) 114 | { 115 | const char* module_cstr; 116 | const char* function_name_cstr; 117 | Py_ssize_t module_len; 118 | Py_ssize_t function_name_len; 119 | PyObject* identifier = nullptr; 120 | 121 | if (!PyArg_ParseTuple(args, "s#s#O", &module_cstr, &module_len, &function_name_cstr, 122 | &function_name_len, &identifier)) 123 | { 124 | return NULL; 125 | } 126 | 127 | std::string module(module_cstr, module_len); 128 | std::string_view function_name(function_name_cstr, function_name_len); 129 | 130 | if (identifier == nullptr || identifier == Py_None) 131 | { 132 | scorepy::region_end(function_name, std::move(module)); 133 | } 134 | else 135 | { 136 | scorepy::region_end(function_name, std::move(module), 137 | reinterpret_cast(identifier)); 138 | } 139 | 140 | Py_RETURN_NONE; 141 | } 142 | 143 | static PyObject* rewind_begin(PyObject* self, PyObject* args) 144 | { 145 | const char* region_name; 146 | const char* file_name; 147 | std::uint64_t line_number = 0; 148 | 149 | if (!PyArg_ParseTuple(args, "ssK", ®ion_name, &file_name, &line_number)) 150 | return NULL; 151 | 152 | scorepy::rewind_begin(region_name, file_name, line_number); 153 | 154 | Py_RETURN_NONE; 155 | } 156 | 157 | static PyObject* rewind_end(PyObject* self, PyObject* args) 158 | { 159 | const char* region_name; 160 | PyObject* value; // false C-Style 161 | 162 | if (!PyArg_ParseTuple(args, "sO", ®ion_name, &value)) 163 | return NULL; 164 | 165 | // TODO cover PyObject_IsTrue(value) == -1 (error case) 166 | scorepy::rewind_end(region_name, PyObject_IsTrue(value) == 1); 167 | 168 | Py_RETURN_NONE; 169 | } 170 | 171 | static PyObject* parameter_string(PyObject* self, PyObject* args) 172 | { 173 | const char* name; 174 | const char* value; 175 | 176 | if (!PyArg_ParseTuple(args, "ss", &name, &value)) 177 | return NULL; 178 | 179 | scorepy::parameter_string(name, value); 180 | 181 | Py_RETURN_NONE; 182 | } 183 | 184 | static PyObject* parameter_int(PyObject* self, PyObject* args) 185 | { 186 | const char* name; 187 | long long value; 188 | 189 | if (!PyArg_ParseTuple(args, "sL", &name, &value)) 190 | return NULL; 191 | 192 | scorepy::parameter_int(name, value); 193 | 194 | Py_RETURN_NONE; 195 | } 196 | 197 | static PyObject* parameter_uint(PyObject* self, PyObject* args) 198 | { 199 | const char* name; 200 | unsigned long long value; 201 | 202 | if (!PyArg_ParseTuple(args, "sK", &name, &value)) 203 | return NULL; 204 | 205 | scorepy::parameter_uint(name, value); 206 | 207 | Py_RETURN_NONE; 208 | } 209 | 210 | static PyObject* get_experiment_dir_name(PyObject* self, PyObject* args) 211 | { 212 | 213 | return PyUnicode_FromString(SCOREP_GetExperimentDirName()); 214 | } 215 | 216 | static PyObject* abspath(PyObject* self, PyObject* args) 217 | { 218 | const char* path; 219 | 220 | if (!PyArg_ParseTuple(args, "s", &path)) 221 | return NULL; 222 | 223 | return PyUnicode_FromString(scorepy::abspath(path).c_str()); 224 | } 225 | 226 | static PyObject* force_finalize(PyObject* self, PyObject* args) 227 | { 228 | SCOREP_FinalizeMeasurement(); 229 | Py_RETURN_NONE; 230 | } 231 | 232 | static PyObject* reregister_exit_handler(PyObject* self, PyObject* args) 233 | { 234 | SCOREP_RegisterExitHandler(); 235 | Py_RETURN_NONE; 236 | } 237 | 238 | static PyMethodDef ScorePMethods[] = { 239 | { "region_begin", region_begin, METH_VARARGS, "enter a region." }, 240 | { "try_region_begin", try_region_begin, METH_VARARGS, 241 | "Tries to begin a region, returns True on Sucess." }, 242 | { "region_end", region_end, METH_VARARGS, "exit a region." }, 243 | { "try_region_end", try_region_end, METH_VARARGS, 244 | "Tries to end a region, returns True on Sucess." }, 245 | { "rewind_begin", rewind_begin, METH_VARARGS, "rewind begin." }, 246 | { "rewind_end", rewind_end, METH_VARARGS, "rewind end." }, 247 | { "enable_recording", enable_recording, METH_VARARGS, "disable scorep recording." }, 248 | { "disable_recording", disable_recording, METH_VARARGS, "disable scorep recording." }, 249 | { "parameter_int", parameter_int, METH_VARARGS, "User parameter int." }, 250 | { "parameter_uint", parameter_uint, METH_VARARGS, "User parameter uint." }, 251 | { "parameter_string", parameter_string, METH_VARARGS, "User parameter string." }, 252 | { "get_experiment_dir_name", get_experiment_dir_name, METH_VARARGS, 253 | "Get the Score-P experiment dir." }, 254 | { "abspath", abspath, METH_VARARGS, "Estimates the absolute Path." }, 255 | { "force_finalize", force_finalize, METH_VARARGS, "triggers a finalize" }, 256 | { "reregister_exit_handler", reregister_exit_handler, METH_VARARGS, 257 | "register a new atexit handler" }, 258 | { NULL, NULL, 0, NULL } /* Sentinel */ 259 | }; 260 | } 261 | 262 | namespace scorepy 263 | { 264 | PyMethodDef* getMethodTable() 265 | { 266 | return ScorePMethods; 267 | } 268 | } // namespace scorepy 269 | -------------------------------------------------------------------------------- /src/methods.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace scorepy 6 | { 7 | /// Return the methods to define for the python module 8 | PyMethodDef* getMethodTable(); 9 | } // namespace scorepy 10 | -------------------------------------------------------------------------------- /src/scorep.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace scorepy 6 | { 7 | /// Return the methods to define for the python module 8 | PyMethodDef* getMethodTable(); 9 | } // namespace scorepy 10 | -------------------------------------------------------------------------------- /src/scorep_bindings.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | #include "classes.hpp" 6 | #include "methods.hpp" 7 | 8 | #if PY_VERSION_HEX < 0x03000000 9 | PyMODINIT_FUNC init_bindings(void) 10 | { 11 | PyObject* m; 12 | #if SCOREPY_ENABLE_CINSTRUMENTER 13 | static PyTypeObject ctracerType = scorepy::getCInstrumenterType(); 14 | if (PyType_Ready(&ctracerType) < 0) 15 | return; 16 | #endif 17 | m = Py_InitModule("_bindings", scorepy::getMethodTable()); 18 | #if SCOREPY_ENABLE_CINSTRUMENTER 19 | Py_INCREF(&ctracerType); 20 | PyModule_AddObject(m, "CInstrumenter", (PyObject*)&ctracerType); 21 | #endif 22 | } 23 | #else /*python 3*/ 24 | static struct PyModuleDef scorepmodule = { PyModuleDef_HEAD_INIT, "_bindings", /* name of module */ 25 | NULL, /* module documentation, may be NULL */ 26 | -1, /* size of per-interpreter state of the module, 27 | or -1 if the module keeps state in global 28 | variables. */ 29 | scorepy::getMethodTable() }; 30 | PyMODINIT_FUNC PyInit__bindings(void) 31 | { 32 | #if SCOREPY_ENABLE_CINSTRUMENTER 33 | auto* ctracerType = &scorepy::getCInstrumenterType(); 34 | if (PyType_Ready(ctracerType) < 0) 35 | return nullptr; 36 | #endif 37 | 38 | auto* m = PyModule_Create(&scorepmodule); 39 | if (!m) 40 | { 41 | return nullptr; 42 | } 43 | 44 | #if SCOREPY_ENABLE_CINSTRUMENTER 45 | Py_INCREF(ctracerType); 46 | if (PyModule_AddObject(m, "CInstrumenter", (PyObject*)ctracerType) < 0) 47 | { 48 | Py_DECREF(ctracerType); 49 | Py_DECREF(m); 50 | return nullptr; 51 | } 52 | #endif 53 | 54 | return m; 55 | } 56 | #endif /*python 3*/ 57 | -------------------------------------------------------------------------------- /src/scorepy/cInstrumenter.cpp: -------------------------------------------------------------------------------- 1 | #include "cInstrumenter.hpp" 2 | #include "events.hpp" 3 | #include "pythonHelpers.hpp" 4 | #include "pythoncapi_compat.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | namespace scorepy 11 | { 12 | 13 | void CInstrumenter::init(InstrumenterInterface interface) 14 | { 15 | this->interface = interface; 16 | threading_module = PyImport_ImportModule("threading"); 17 | if (threading_module) 18 | { 19 | const char* name = (interface == InstrumenterInterface::Trace) ? "settrace" : "setprofile"; 20 | threading_set_instrumenter = PyObject_GetAttrString(threading_module, name); 21 | } 22 | } 23 | 24 | void CInstrumenter::deinit() 25 | { 26 | Py_CLEAR(threading_module); 27 | Py_CLEAR(threading_set_instrumenter); 28 | } 29 | 30 | void CInstrumenter::enable_instrumenter() 31 | { 32 | const auto callback = [](PyObject* obj, PyFrameObject* frame, int what, PyObject* arg) -> int { 33 | return from_PyObject(obj)->on_event(*frame, what, arg) ? 0 : -1; 34 | }; 35 | if (threading_set_instrumenter) 36 | { 37 | PyRefObject result(PyObject_CallFunction(threading_set_instrumenter, "O", to_PyObject()), 38 | adopt_object); 39 | } 40 | if (interface == InstrumenterInterface::Trace) 41 | { 42 | PyEval_SetTrace(callback, to_PyObject()); 43 | } 44 | else 45 | { 46 | PyEval_SetProfile(callback, to_PyObject()); 47 | } 48 | } 49 | 50 | void CInstrumenter::disable_instrumenter() 51 | { 52 | if (interface == InstrumenterInterface::Trace) 53 | { 54 | PyEval_SetTrace(nullptr, nullptr); 55 | } 56 | else 57 | { 58 | PyEval_SetProfile(nullptr, nullptr); 59 | } 60 | if (threading_set_instrumenter) 61 | { 62 | PyRefObject result(PyObject_CallFunction(threading_set_instrumenter, "O", Py_None), 63 | adopt_object); 64 | } 65 | } 66 | 67 | /// Mapping of PyTrace_* to it's string representations 68 | /// List taken from CPythons sysmodule.c 69 | static const std::array WHAT_STRINGS = { "call", "exception", "line", 70 | "return", "c_call", "c_exception", 71 | "c_return", "opcode" }; 72 | 73 | template 74 | int index_of(TCollection&& col, const TElement& element) 75 | { 76 | const auto it = std::find(col.cbegin(), col.cend(), element); 77 | if (it == col.end()) 78 | { 79 | return -1; 80 | } 81 | else 82 | { 83 | return std::distance(col.begin(), it); 84 | } 85 | } 86 | 87 | // Required because: `sys.getprofile()` returns the user object (2nd arg to PyEval_SetTrace) 88 | // So `sys.setprofile(sys.getprofile())` will not round-trip as it will try to call the 89 | // 2nd arg through pythons dispatch function. Hence make the object callable. 90 | // See https://nedbatchelder.com/text/trace-function.html for details 91 | PyObject* CInstrumenter::operator()(PyFrameObject& frame, const char* what_string, PyObject* arg) 92 | { 93 | const int what = index_of(WHAT_STRINGS, what_string); 94 | // To speed up further event processing install this class directly as the handler 95 | // But we might be inside a `sys.settrace` call where the user wanted to set another function 96 | // which would then be overwritten here. Hence use the CALL event which avoids the problem 97 | if (what == PyTrace_CALL) 98 | { 99 | enable_instrumenter(); 100 | } 101 | if (on_event(frame, what, arg)) 102 | { 103 | Py_INCREF(to_PyObject()); 104 | return to_PyObject(); 105 | } 106 | else 107 | { 108 | return nullptr; 109 | } 110 | } 111 | 112 | bool CInstrumenter::on_event(PyFrameObject& frame, int what, PyObject*) 113 | { 114 | switch (what) 115 | { 116 | case PyTrace_CALL: 117 | { 118 | PyCodeObject* code = PyFrame_GetCode(&frame); 119 | bool success = try_region_begin(code); 120 | if (!success) 121 | { 122 | const auto name = compat::get_string_as_utf_8(code->co_name); 123 | const auto module_name = get_module_name(frame); 124 | if (name.compare("_unsetprofile") != 0 && module_name.compare(0, 6, "scorep") != 0) 125 | { 126 | const int line_number = code->co_firstlineno; 127 | const auto file_name = get_file_name(frame); 128 | region_begin(name, std::move(module_name), std::move(file_name), line_number, code); 129 | } 130 | } 131 | Py_DECREF(code); 132 | break; 133 | } 134 | case PyTrace_RETURN: 135 | { 136 | PyCodeObject* code = PyFrame_GetCode(&frame); 137 | bool success = try_region_end(code); 138 | if (!success) 139 | { 140 | const auto name = compat::get_string_as_utf_8(code->co_name); 141 | const auto module_name = get_module_name(frame); 142 | // TODO: Use string_view/CString comparison? 143 | if (name.compare("_unsetprofile") != 0 && module_name.compare(0, 6, "scorep") != 0) 144 | { 145 | region_end(name, std::move(module_name), code); 146 | } 147 | } 148 | Py_DECREF(code); 149 | break; 150 | } 151 | } 152 | return true; 153 | } 154 | 155 | } // namespace scorepy 156 | -------------------------------------------------------------------------------- /src/scorepy/cInstrumenter.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace scorepy 7 | { 8 | /// Interface to Python used to implement an instrumenter 9 | /// See sys.settrace/setprofile 10 | enum class InstrumenterInterface 11 | { 12 | Profile, 13 | Trace 14 | }; 15 | 16 | struct CInstrumenter 17 | { 18 | PyObject_HEAD; 19 | InstrumenterInterface interface; 20 | PyObject* threading_module; 21 | PyObject* threading_set_instrumenter; 22 | 23 | void init(InstrumenterInterface interface); 24 | void deinit(); 25 | void enable_instrumenter(); 26 | void disable_instrumenter(); 27 | 28 | /// Callback for when this object is called directly 29 | PyObject* operator()(PyFrameObject& frame, const char* what, PyObject* arg); 30 | 31 | /// These casts are valid as long as `PyObject_HEAD` is the first entry in this struct 32 | PyObject* to_PyObject() 33 | { 34 | return reinterpret_cast(this); 35 | } 36 | static CInstrumenter* from_PyObject(PyObject* o) 37 | { 38 | return reinterpret_cast(o); 39 | } 40 | 41 | private: 42 | /// Callback for Python trace/profile events. Return true for success 43 | bool on_event(PyFrameObject& frame, int what, PyObject* arg); 44 | }; 45 | 46 | } // namespace scorepy 47 | -------------------------------------------------------------------------------- /src/scorepy/compat.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | namespace scorepy 12 | { 13 | namespace compat 14 | { 15 | /// Region names that are known to have no region enter event and should not report an error 16 | /// on region exit 17 | static const std::array exit_region_whitelist = { 18 | #if PY_MAJOR_VERSION >= 3 19 | "threading.Thread:_bootstrap_inner", "threading.Thread:_bootstrap", 20 | "threading.TMonitor:_bootstrap_inner", "threading.TMonitor:_bootstrap" 21 | #else 22 | "threading.Thread:__bootstrap_inner", "threading.Thread:__bootstrap", 23 | "threading.TMonitor:__bootstrap_inner", "threading.TMonitor:__bootstrap" 24 | #endif 25 | }; 26 | 27 | inline std::string_view get_string_as_utf_8(PyObject* py_string) 28 | { 29 | Py_ssize_t size = 0; 30 | 31 | #if PY_MAJOR_VERSION >= 3 32 | const char* string; 33 | string = PyUnicode_AsUTF8AndSize(py_string, &size); 34 | #else 35 | char* string; 36 | PyString_AsStringAndSize(py_string, &string, &size); 37 | #endif 38 | return std::string_view(string, size); 39 | } 40 | 41 | using PyCodeObject = PyCodeObject; 42 | 43 | using destructor = destructor; 44 | using code_dealloc = std::add_pointer::type; // void(*)(PyCodeObject*) 45 | 46 | /** 47 | * @brief For CPython we need to make sure, that the we register our own dealloc function, so we 48 | * can handle the deleteion of code_objects in our code. 49 | */ 50 | struct RegisterCodeDealloc 51 | { 52 | RegisterCodeDealloc(std::function on_dealloc_fun) 53 | { 54 | external_on_dealloc_fun = on_dealloc_fun; 55 | // PyPy does not need this, as CodeObjects are compiled, and therefore live for the 56 | // programms lifetime 57 | #ifndef PYPY_VERSION 58 | if (!python_code_dealloc) 59 | { 60 | python_code_dealloc = 61 | reinterpret_cast(PyCode_Type.tp_dealloc); 62 | PyCode_Type.tp_dealloc = reinterpret_cast(dealloc_fun); 63 | } 64 | else 65 | { 66 | std::cerr << "WARNING: Score-P Python's code_dealloc is alredy registerd!" 67 | << std::endl; 68 | } 69 | #endif 70 | } 71 | 72 | static void dealloc_fun(PyCodeObject* co) 73 | { 74 | if (external_on_dealloc_fun && python_code_dealloc) 75 | { 76 | external_on_dealloc_fun(co); 77 | python_code_dealloc(co); 78 | } 79 | } 80 | 81 | inline static compat::code_dealloc python_code_dealloc; 82 | inline static std::function external_on_dealloc_fun; 83 | }; 84 | 85 | } // namespace compat 86 | } // namespace scorepy 87 | -------------------------------------------------------------------------------- /src/scorepy/events.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | #include "compat.hpp" 7 | #include "events.hpp" 8 | #include "pythonHelpers.hpp" 9 | 10 | namespace scorepy 11 | { 12 | 13 | std::unordered_map regions; 14 | static std::unordered_map user_regions; 15 | static std::unordered_map rewind_regions; 16 | 17 | /** 18 | * @brief when Python PyCodeObject is deallocated, remove it from our regions list. 19 | * 20 | * @param co code object to remove 21 | */ 22 | void on_dealloc(PyCodeObject* co) 23 | { 24 | regions.erase(co); 25 | } 26 | 27 | static compat::RegisterCodeDealloc register_dealloc(on_dealloc); 28 | 29 | // Used for regions, that have an identifier, aka a code object id. (instrumenter regions and 30 | // some decorated regions) 31 | void region_begin(std::string_view function_name, std::string module, std::string file_name, 32 | const std::uint64_t line_number, compat::PyCodeObject* identifier) 33 | { 34 | region_handle& region = regions[identifier]; 35 | 36 | if (region == uninitialised_region_handle) 37 | { 38 | auto region_name = make_region_name(std::move(module), function_name); 39 | SCOREP_User_RegionInit(®ion.value, NULL, NULL, region_name.c_str(), 40 | SCOREP_USER_REGION_TYPE_FUNCTION, file_name.c_str(), line_number); 41 | 42 | if (const auto pos = region_name.find(':'); pos != std::string::npos) 43 | { 44 | region_name.resize(pos); 45 | SCOREP_User_RegionSetGroup(region.value, region_name.c_str()); 46 | } 47 | } 48 | SCOREP_User_RegionEnter(region.value); 49 | } 50 | 51 | // Used for regions, that only have a function name, a module, a file and a line number (user 52 | // regions) 53 | void region_begin(std::string_view function_name, std::string module, std::string file_name, 54 | const std::uint64_t line_number) 55 | { 56 | auto region_name = make_region_name(std::move(module), function_name); 57 | region_handle& region = user_regions[region_name]; 58 | 59 | if (region == uninitialised_region_handle) 60 | { 61 | SCOREP_User_RegionInit(®ion.value, NULL, NULL, region_name.c_str(), 62 | SCOREP_USER_REGION_TYPE_FUNCTION, file_name.c_str(), line_number); 63 | 64 | if (const auto pos = region_name.find(':'); pos != std::string::npos) 65 | { 66 | region_name.resize(pos); 67 | SCOREP_User_RegionSetGroup(region.value, region_name.c_str()); 68 | } 69 | } 70 | SCOREP_User_RegionEnter(region.value); 71 | } 72 | 73 | // Used for regions, that have an identifier, aka a code object id. (instrumenter regions and 74 | // some decorated regions) 75 | void region_end(std::string_view function_name, std::string module, 76 | compat::PyCodeObject* identifier) 77 | { 78 | const auto it_region = regions.find(identifier); 79 | if (it_region != regions.end()) 80 | { 81 | SCOREP_User_RegionEnd(it_region->second.value); 82 | } 83 | else 84 | { 85 | const auto region_name = make_region_name(std::move(module), function_name); 86 | region_end_error_handling(std::move(region_name)); 87 | } 88 | } 89 | 90 | // Used for regions, that only have a function name, a module (user regions) 91 | void region_end(std::string_view function_name, std::string module) 92 | { 93 | const auto region_name = make_region_name(std::move(module), function_name); 94 | auto it_region = user_regions.find(region_name); 95 | if (it_region != user_regions.end()) 96 | { 97 | SCOREP_User_RegionEnd(it_region->second.value); 98 | } 99 | else 100 | { 101 | region_end_error_handling(std::move(region_name)); 102 | } 103 | } 104 | 105 | void region_end_error_handling(std::string region_name) 106 | { 107 | static region_handle error_region; 108 | static SCOREP_User_ParameterHandle scorep_param = SCOREP_USER_INVALID_PARAMETER; 109 | static bool error_printed = false; 110 | 111 | if (std::find(compat::exit_region_whitelist.begin(), compat::exit_region_whitelist.end(), 112 | region_name) != compat::exit_region_whitelist.end()) 113 | { 114 | return; 115 | } 116 | 117 | if (error_region.value == SCOREP_USER_INVALID_REGION) 118 | { 119 | SCOREP_User_RegionInit(&error_region.value, NULL, NULL, "error_region", 120 | SCOREP_USER_REGION_TYPE_FUNCTION, "scorep.cpp", 0); 121 | SCOREP_User_RegionSetGroup(error_region.value, "error"); 122 | } 123 | SCOREP_User_RegionEnter(error_region.value); 124 | SCOREP_User_ParameterString(&scorep_param, "leave-region", region_name.c_str()); 125 | SCOREP_User_RegionEnd(error_region.value); 126 | 127 | if (!error_printed) 128 | { 129 | std::cerr << "SCOREP_BINDING_PYTHON ERROR: There was a region exit without an enter!\n" 130 | << "SCOREP_BINDING_PYTHON ERROR: For details look for \"error_region\" in " 131 | "the trace or profile." 132 | << std::endl; 133 | error_printed = true; 134 | } 135 | } 136 | 137 | void rewind_begin(std::string region_name, std::string file_name, std::uint64_t line_number) 138 | { 139 | auto pair = rewind_regions.emplace(make_pair(region_name, region_handle())); 140 | bool inserted_new = pair.second; 141 | auto& handle = pair.first->second; 142 | if (inserted_new) 143 | { 144 | SCOREP_User_RegionInit(&handle.value, NULL, NULL, region_name.c_str(), 145 | SCOREP_USER_REGION_TYPE_FUNCTION, file_name.c_str(), line_number); 146 | } 147 | SCOREP_User_RewindRegionEnter(handle.value); 148 | } 149 | 150 | void rewind_end(std::string region_name, bool value) 151 | { 152 | auto& handle = rewind_regions.at(region_name); 153 | /* don't call SCOREP_ExitRewindRegion, as 154 | * SCOREP_User_RewindRegionEnd does some additional magic 155 | * */ 156 | SCOREP_User_RewindRegionEnd(handle.value, value); 157 | } 158 | 159 | void parameter_int(std::string name, int64_t value) 160 | { 161 | static SCOREP_User_ParameterHandle scorep_param = SCOREP_USER_INVALID_PARAMETER; 162 | SCOREP_User_ParameterInt64(&scorep_param, name.c_str(), value); 163 | } 164 | 165 | void parameter_uint(std::string name, uint64_t value) 166 | { 167 | static SCOREP_User_ParameterHandle scorep_param = SCOREP_USER_INVALID_PARAMETER; 168 | SCOREP_User_ParameterUint64(&scorep_param, name.c_str(), value); 169 | } 170 | 171 | void parameter_string(std::string name, std::string value) 172 | { 173 | static SCOREP_User_ParameterHandle scorep_param = SCOREP_USER_INVALID_PARAMETER; 174 | SCOREP_User_ParameterString(&scorep_param, name.c_str(), value.c_str()); 175 | } 176 | 177 | } // namespace scorepy 178 | -------------------------------------------------------------------------------- /src/scorepy/events.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | #include "compat.hpp" 10 | 11 | namespace scorepy 12 | { 13 | 14 | struct region_handle 15 | { 16 | constexpr region_handle() = default; 17 | ~region_handle() = default; 18 | constexpr bool operator==(const region_handle& other) 19 | { 20 | return this->value == other.value; 21 | } 22 | constexpr bool operator!=(const region_handle& other) 23 | { 24 | return this->value != other.value; 25 | } 26 | 27 | SCOREP_User_RegionHandle value = SCOREP_USER_INVALID_REGION; 28 | }; 29 | 30 | constexpr region_handle uninitialised_region_handle = region_handle(); 31 | 32 | /// Combine the arguments into a region name 33 | inline std::string make_region_name(std::string module_name, std::string_view name) 34 | { 35 | module_name += ':'; 36 | module_name += name; 37 | return std::move(module_name); 38 | } 39 | 40 | extern std::unordered_map regions; 41 | 42 | /** tries to enter a region. Return true on success 43 | * 44 | */ 45 | inline bool try_region_begin(compat::PyCodeObject* identifier) 46 | { 47 | auto it = regions.find(identifier); 48 | if (it != regions.end()) 49 | { 50 | SCOREP_User_RegionEnter(it->second.value); 51 | return true; 52 | } 53 | else 54 | { 55 | return false; 56 | } 57 | } 58 | 59 | void region_begin(std::string_view function_name, std::string module, std::string file_name, 60 | const std::uint64_t line_number, compat::PyCodeObject* identifier); 61 | void region_begin(std::string_view function_name, std::string module, std::string file_name, 62 | const std::uint64_t line_number); 63 | 64 | /** tries to end a region. Return true on success 65 | * 66 | */ 67 | inline bool try_region_end(compat::PyCodeObject* identifier) 68 | { 69 | auto it_region = regions.find(identifier); 70 | if (it_region != regions.end()) 71 | { 72 | SCOREP_User_RegionEnd(it_region->second.value); 73 | return true; 74 | } 75 | else 76 | { 77 | return false; 78 | } 79 | } 80 | 81 | void region_end(std::string_view function_name, std::string module, 82 | compat::PyCodeObject* identifier); 83 | void region_end(std::string_view function_name, std::string module); 84 | 85 | void region_end_error_handling(std::string region_name); 86 | 87 | void rewind_begin(std::string region_name, std::string file_name, std::uint64_t line_number); 88 | void rewind_end(std::string region_name, bool value); 89 | 90 | void parameter_int(std::string name, int64_t value); 91 | void parameter_uint(std::string name, uint64_t value); 92 | void parameter_string(std::string name, std::string value); 93 | } // namespace scorepy 94 | -------------------------------------------------------------------------------- /src/scorepy/pathUtils.cpp: -------------------------------------------------------------------------------- 1 | #include "pathUtils.hpp" 2 | #include 3 | #include 4 | #include 5 | 6 | namespace scorepy 7 | { 8 | static std::string getcwd() 9 | { 10 | std::string result; 11 | const char* cwd; 12 | constexpr size_t chunk_size = 512; 13 | do 14 | { 15 | const size_t new_size = result.size() + chunk_size; 16 | if (new_size > std::numeric_limits::max()) 17 | { 18 | cwd = nullptr; 19 | break; 20 | } 21 | result.resize(new_size); 22 | cwd = ::getcwd(&result.front(), result.size()); 23 | } while (!cwd && errno == ERANGE); 24 | if (cwd) 25 | result.resize(result.find('\0', result.size() - chunk_size)); 26 | else 27 | result.clear(); 28 | return std::move(result); 29 | } 30 | 31 | void normalize_path(std::string& path) 32 | { 33 | // 2 slashes are kept, 1 or more than 2 become 1 according to POSIX 34 | const size_t num_slashes = (path.find_first_not_of('/') == 2) ? 2 : 1; 35 | const size_t path_len = path.size(); 36 | size_t cur_out = num_slashes; 37 | for (size_t i = cur_out; i != path_len + 1; ++i) 38 | { 39 | if (i == path_len || path[i] == '/') 40 | { 41 | const char prior = path[cur_out - 1]; 42 | if (prior == '/') // Double slash -> ignore 43 | continue; 44 | if (prior == '.') 45 | { 46 | const char second_prior = path[cur_out - 2]; 47 | if (second_prior == '/') // '/./' 48 | { 49 | --cur_out; 50 | continue; 51 | } 52 | else if (second_prior == '.' && path[cur_out - 3] == '/') // '/../' 53 | { 54 | if (cur_out < 3 + num_slashes) // already behind root slash 55 | cur_out -= 2; 56 | else 57 | { 58 | const auto prior_slash = path.rfind('/', cur_out - 4); 59 | cur_out = prior_slash + 1; 60 | } 61 | continue; 62 | } 63 | } 64 | if (i == path_len) 65 | break; 66 | } 67 | path[cur_out++] = path[i]; 68 | } 69 | // Remove trailing slash 70 | if (cur_out > num_slashes && path[cur_out - 1] == '/') 71 | --cur_out; 72 | path.resize(cur_out); 73 | } 74 | 75 | std::string abspath(std::string_view input_path) 76 | { 77 | std::string result; 78 | if (input_path[0] != '/') 79 | { 80 | result = getcwd(); 81 | // On error exit 82 | if (result.empty()) 83 | return {}; 84 | // Prepend CWD 85 | result.append(1, '/').append(input_path); 86 | } 87 | else 88 | { 89 | result = input_path; 90 | } 91 | normalize_path(result); 92 | return std::move(result); 93 | } 94 | } // namespace scorepy 95 | -------------------------------------------------------------------------------- /src/scorepy/pathUtils.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace scorepy 6 | { 7 | /// A//B, A/./B and A/foo/../B all become A/B 8 | /// Assumes an absolute, non-empty path 9 | void normalize_path(std::string& path); 10 | /// Makes the path absolute and normalized, see Python os.path.abspath 11 | std::string abspath(std::string_view input_path); 12 | } // namespace scorepy 13 | -------------------------------------------------------------------------------- /src/scorepy/pythonHelpers.cpp: -------------------------------------------------------------------------------- 1 | #include "pythonHelpers.hpp" 2 | #include "compat.hpp" 3 | #include "pathUtils.hpp" 4 | #include "pythoncapi_compat.h" 5 | 6 | #include 7 | #include 8 | 9 | namespace scorepy 10 | { 11 | std::string get_module_name(PyFrameObject& frame) 12 | { 13 | const char* self_name = nullptr; 14 | PyObject* locals = PyFrame_GetLocals(&frame); 15 | PyObject* self = PyDict_GetItemString(locals, "self"); 16 | if (self) 17 | { 18 | Py_INCREF(self); 19 | PyTypeObject* type = Py_TYPE(self); 20 | self_name = _PyType_Name(type); 21 | Py_DECREF(self); 22 | } 23 | Py_DECREF(locals); 24 | 25 | PyObject* globals = PyFrame_GetGlobals(&frame); 26 | PyObject* module_name = PyDict_GetItemString(globals, "__name__"); 27 | Py_DECREF(globals); 28 | if (module_name) 29 | { 30 | std::stringstream result; 31 | result << compat::get_string_as_utf_8(module_name); 32 | if (self_name) 33 | result << '.' << self_name; 34 | return std::move(result).str(); 35 | } 36 | 37 | // this is a NUMPY special situation, see NEP-18, and Score-P issue #63 38 | // TODO: Use string_view/C-String to avoid creating 2 std::strings 39 | PyCodeObject* code = PyFrame_GetCode(&frame); 40 | std::string_view filename = compat::get_string_as_utf_8(code->co_filename); 41 | Py_DECREF(code); 42 | if ((filename.size() > 0) && (filename == "<__array_function__ internals>")) 43 | return std::move(std::string("numpy.__array_function__")); 44 | else 45 | return std::move(std::string("unkown")); 46 | } 47 | 48 | std::string get_file_name(PyFrameObject& frame) 49 | { 50 | PyCodeObject* code = PyFrame_GetCode(&frame); 51 | PyObject* filename = code->co_filename; 52 | Py_DECREF(code); 53 | if (filename == Py_None) 54 | { 55 | return std::move(std::string("None")); 56 | } 57 | const auto full_file_name = abspath(compat::get_string_as_utf_8(filename)); 58 | return !full_file_name.empty() ? std::move(full_file_name) : "ErrorPath"; 59 | } 60 | } // namespace scorepy 61 | -------------------------------------------------------------------------------- /src/scorepy/pythonHelpers.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | namespace scorepy 9 | { 10 | 11 | struct retain_object_t 12 | { 13 | }; 14 | struct adopt_object_t 15 | { 16 | }; 17 | /// Marker to indicate that an owner is added, i.e. refCnt is increased 18 | constexpr retain_object_t retain_object{}; 19 | /// Marker to take over ownership, not touching the refCnt 20 | constexpr adopt_object_t adopt_object{}; 21 | 22 | /// Slim, owning wrapper over a PyObject* 23 | /// Decays implictely to a PyObject* 24 | class PyRefObject 25 | { 26 | PyObject* o_; 27 | 28 | public: 29 | explicit PyRefObject(PyObject* o, adopt_object_t) noexcept : o_(o) 30 | { 31 | } 32 | explicit PyRefObject(PyObject* o, retain_object_t) noexcept : o_(o) 33 | { 34 | Py_XINCREF(o_); 35 | } 36 | PyRefObject(PyRefObject&& rhs) noexcept : o_(rhs.o_) 37 | { 38 | rhs.o_ = nullptr; 39 | } 40 | PyRefObject& operator=(PyRefObject&& rhs) noexcept 41 | { 42 | o_ = rhs.o_; 43 | rhs.o_ = nullptr; 44 | return *this; 45 | } 46 | ~PyRefObject() noexcept 47 | { 48 | Py_XDECREF(o_); 49 | } 50 | 51 | operator PyObject*() const noexcept 52 | { 53 | return o_; 54 | } 55 | }; 56 | 57 | namespace detail 58 | { 59 | template 60 | struct ReplaceArgsToPyObject; 61 | 62 | template 63 | using ReplaceArgsToPyObject_t = typename ReplaceArgsToPyObject::type; 64 | } // namespace detail 65 | 66 | /// Cast a function pointer to a python-bindings compatible function pointer 67 | /// Replaces all Foo* by PyObject* for all types Foo that are PyObject compatible 68 | template 69 | auto cast_to_PyFunc(TFunc* func) -> detail::ReplaceArgsToPyObject_t* 70 | { 71 | return reinterpret_cast*>(func); 72 | } 73 | 74 | /// Return the module name the frame belongs to. 75 | /// The pointer is valid for the lifetime of the frame 76 | std::string get_module_name(PyFrameObject& frame); 77 | /// Return the file name the frame belongs to 78 | std::string get_file_name(PyFrameObject& frame); 79 | 80 | // Implementation details 81 | namespace detail 82 | { 83 | 84 | template 85 | struct make_void 86 | { 87 | typedef void type; 88 | }; 89 | template 90 | using void_t = typename make_void::type; 91 | 92 | template 93 | struct IsPyObject : std::false_type 94 | { 95 | }; 96 | template 97 | struct IsPyObject : IsPyObject 98 | { 99 | }; 100 | 101 | template 102 | struct IsPyObject().to_PyObject())>> : std::true_type 103 | { 104 | }; 105 | 106 | template 107 | struct ReplaceArgsToPyObject 108 | { 109 | template 110 | using replace = typename std::conditional::value, PyObject*, T>::type; 111 | using type = TResult(replace...); 112 | }; 113 | } // namespace detail 114 | } // namespace scorepy 115 | -------------------------------------------------------------------------------- /src/scorepy/pythoncapi_compat.h: -------------------------------------------------------------------------------- 1 | // Header file providing new C API functions to old Python versions. 2 | // 3 | // File distributed under the Zero Clause BSD (0BSD) license. 4 | // Copyright Contributors to the pythoncapi_compat project. 5 | // 6 | // Homepage: 7 | // https://github.com/python/pythoncapi_compat 8 | // 9 | // Latest version: 10 | // https://raw.githubusercontent.com/python/pythoncapi_compat/master/pythoncapi_compat.h 11 | // 12 | // SPDX-License-Identifier: 0BSD 13 | 14 | #ifndef PYTHONCAPI_COMPAT 15 | #define PYTHONCAPI_COMPAT 16 | 17 | #ifdef __cplusplus 18 | extern "C" 19 | { 20 | #endif 21 | 22 | #include "frameobject.h" // PyFrameObject, PyFrame_GetBack() 23 | #include 24 | 25 | // Compatibility with Visual Studio 2013 and older which don't support 26 | // the inline keyword in C (only in C++): use __inline instead. 27 | #if (defined(_MSC_VER) && _MSC_VER < 1900 && !defined(__cplusplus) && !defined(inline)) 28 | #define PYCAPI_COMPAT_STATIC_INLINE(TYPE) static __inline TYPE 29 | #else 30 | #define PYCAPI_COMPAT_STATIC_INLINE(TYPE) static inline TYPE 31 | #endif 32 | 33 | #ifndef _Py_CAST 34 | #define _Py_CAST(type, expr) ((type)(expr)) 35 | #endif 36 | 37 | // On C++11 and newer, _Py_NULL is defined as nullptr on C++11, 38 | // otherwise it is defined as NULL. 39 | #ifndef _Py_NULL 40 | #if defined(__cplusplus) && __cplusplus >= 201103 41 | #define _Py_NULL nullptr 42 | #else 43 | #define _Py_NULL NULL 44 | #endif 45 | #endif 46 | 47 | // Cast argument to PyObject* type. 48 | #ifndef _PyObject_CAST 49 | #define _PyObject_CAST(op) _Py_CAST(PyObject*, op) 50 | #endif 51 | 52 | // bpo-42262 added Py_NewRef() to Python 3.10.0a3 53 | #if PY_VERSION_HEX < 0x030A00A3 && !defined(Py_NewRef) 54 | PYCAPI_COMPAT_STATIC_INLINE(PyObject*) 55 | _Py_NewRef(PyObject* obj) 56 | { 57 | Py_INCREF(obj); 58 | return obj; 59 | } 60 | #define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) 61 | #endif 62 | 63 | // bpo-42262 added Py_XNewRef() to Python 3.10.0a3 64 | #if PY_VERSION_HEX < 0x030A00A3 && !defined(Py_XNewRef) 65 | PYCAPI_COMPAT_STATIC_INLINE(PyObject*) 66 | _Py_XNewRef(PyObject* obj) 67 | { 68 | Py_XINCREF(obj); 69 | return obj; 70 | } 71 | #define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) 72 | #endif 73 | 74 | // bpo-39573 added Py_SET_REFCNT() to Python 3.9.0a4 75 | #if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_REFCNT) 76 | PYCAPI_COMPAT_STATIC_INLINE(void) 77 | _Py_SET_REFCNT(PyObject* ob, Py_ssize_t refcnt) 78 | { 79 | ob->ob_refcnt = refcnt; 80 | } 81 | #define Py_SET_REFCNT(ob, refcnt) _Py_SET_REFCNT(_PyObject_CAST(ob), refcnt) 82 | #endif 83 | 84 | // Py_SETREF() and Py_XSETREF() were added to Python 3.5.2. 85 | // It is excluded from the limited C API. 86 | #if (PY_VERSION_HEX < 0x03050200 && !defined(Py_SETREF)) && !defined(Py_LIMITED_API) 87 | #define Py_SETREF(dst, src) \ 88 | do \ 89 | { \ 90 | PyObject** _tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \ 91 | PyObject* _tmp_dst = (*_tmp_dst_ptr); \ 92 | *_tmp_dst_ptr = _PyObject_CAST(src); \ 93 | Py_DECREF(_tmp_dst); \ 94 | } while (0) 95 | 96 | #define Py_XSETREF(dst, src) \ 97 | do \ 98 | { \ 99 | PyObject** _tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \ 100 | PyObject* _tmp_dst = (*_tmp_dst_ptr); \ 101 | *_tmp_dst_ptr = _PyObject_CAST(src); \ 102 | Py_XDECREF(_tmp_dst); \ 103 | } while (0) 104 | #endif 105 | 106 | // bpo-43753 added Py_Is(), Py_IsNone(), Py_IsTrue() and Py_IsFalse() 107 | // to Python 3.10.0b1. 108 | #if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_Is) 109 | #define Py_Is(x, y) ((x) == (y)) 110 | #endif 111 | #if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_IsNone) 112 | #define Py_IsNone(x) Py_Is(x, Py_None) 113 | #endif 114 | #if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_IsTrue) 115 | #define Py_IsTrue(x) Py_Is(x, Py_True) 116 | #endif 117 | #if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_IsFalse) 118 | #define Py_IsFalse(x) Py_Is(x, Py_False) 119 | #endif 120 | 121 | // bpo-39573 added Py_SET_TYPE() to Python 3.9.0a4 122 | #if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_TYPE) 123 | PYCAPI_COMPAT_STATIC_INLINE(void) 124 | _Py_SET_TYPE(PyObject* ob, PyTypeObject* type) 125 | { 126 | ob->ob_type = type; 127 | } 128 | #define Py_SET_TYPE(ob, type) _Py_SET_TYPE(_PyObject_CAST(ob), type) 129 | #endif 130 | 131 | // bpo-39573 added Py_SET_SIZE() to Python 3.9.0a4 132 | #if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_SIZE) 133 | PYCAPI_COMPAT_STATIC_INLINE(void) 134 | _Py_SET_SIZE(PyVarObject* ob, Py_ssize_t size) 135 | { 136 | ob->ob_size = size; 137 | } 138 | #define Py_SET_SIZE(ob, size) _Py_SET_SIZE((PyVarObject*)(ob), size) 139 | #endif 140 | 141 | // bpo-40421 added PyFrame_GetCode() to Python 3.9.0b1 142 | #if PY_VERSION_HEX < 0x030900B1 || defined(PYPY_VERSION) 143 | PYCAPI_COMPAT_STATIC_INLINE(PyCodeObject*) 144 | PyFrame_GetCode(PyFrameObject* frame) 145 | { 146 | assert(frame != _Py_NULL); 147 | assert(frame->f_code != _Py_NULL); 148 | return _Py_CAST(PyCodeObject*, Py_NewRef(frame->f_code)); 149 | } 150 | #endif 151 | 152 | PYCAPI_COMPAT_STATIC_INLINE(PyCodeObject*) 153 | _PyFrame_GetCodeBorrow(PyFrameObject* frame) 154 | { 155 | PyCodeObject* code = PyFrame_GetCode(frame); 156 | Py_DECREF(code); 157 | return code; 158 | } 159 | 160 | // bpo-40421 added PyFrame_GetBack() to Python 3.9.0b1 161 | #if PY_VERSION_HEX < 0x030900B1 && !defined(PYPY_VERSION) 162 | PYCAPI_COMPAT_STATIC_INLINE(PyFrameObject*) 163 | PyFrame_GetBack(PyFrameObject* frame) 164 | { 165 | assert(frame != _Py_NULL); 166 | return _Py_CAST(PyFrameObject*, Py_XNewRef(frame->f_back)); 167 | } 168 | #endif 169 | 170 | #if !defined(PYPY_VERSION) 171 | PYCAPI_COMPAT_STATIC_INLINE(PyFrameObject*) 172 | _PyFrame_GetBackBorrow(PyFrameObject* frame) 173 | { 174 | PyFrameObject* back = PyFrame_GetBack(frame); 175 | Py_XDECREF(back); 176 | return back; 177 | } 178 | #endif 179 | 180 | // bpo-40421 added PyFrame_GetLocals() to Python 3.11.0a7 181 | #if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) 182 | PYCAPI_COMPAT_STATIC_INLINE(PyObject*) 183 | PyFrame_GetLocals(PyFrameObject* frame) 184 | { 185 | #if PY_VERSION_HEX >= 0x030400B1 186 | if (PyFrame_FastToLocalsWithError(frame) < 0) 187 | { 188 | return NULL; 189 | } 190 | #else 191 | PyFrame_FastToLocals(frame); 192 | #endif 193 | return Py_NewRef(frame->f_locals); 194 | } 195 | #endif 196 | 197 | // bpo-40421 added PyFrame_GetGlobals() to Python 3.11.0a7 198 | #if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) 199 | PYCAPI_COMPAT_STATIC_INLINE(PyObject*) 200 | PyFrame_GetGlobals(PyFrameObject* frame) 201 | { 202 | return Py_NewRef(frame->f_globals); 203 | } 204 | #endif 205 | 206 | // bpo-40421 added PyFrame_GetBuiltins() to Python 3.11.0a7 207 | #if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) 208 | PYCAPI_COMPAT_STATIC_INLINE(PyObject*) 209 | PyFrame_GetBuiltins(PyFrameObject* frame) 210 | { 211 | return Py_NewRef(frame->f_builtins); 212 | } 213 | #endif 214 | 215 | // bpo-40421 added PyFrame_GetLasti() to Python 3.11.0b1 216 | #if PY_VERSION_HEX < 0x030B00B1 && !defined(PYPY_VERSION) 217 | PYCAPI_COMPAT_STATIC_INLINE(int) 218 | PyFrame_GetLasti(PyFrameObject* frame) 219 | { 220 | #if PY_VERSION_HEX >= 0x030A00A7 221 | // bpo-27129: Since Python 3.10.0a7, f_lasti is an instruction offset, 222 | // not a bytes offset anymore. Python uses 16-bit "wordcode" (2 bytes) 223 | // instructions. 224 | if (frame->f_lasti < 0) 225 | { 226 | return -1; 227 | } 228 | return frame->f_lasti * 2; 229 | #else 230 | return frame->f_lasti; 231 | #endif 232 | } 233 | #endif 234 | 235 | // gh-91248 added PyFrame_GetVar() to Python 3.12.0a2 236 | #if PY_VERSION_HEX < 0x030C00A2 && !defined(PYPY_VERSION) 237 | PYCAPI_COMPAT_STATIC_INLINE(PyObject*) 238 | PyFrame_GetVar(PyFrameObject* frame, PyObject* name) 239 | { 240 | PyObject *locals, *value; 241 | 242 | locals = PyFrame_GetLocals(frame); 243 | if (locals == NULL) 244 | { 245 | return NULL; 246 | } 247 | #if PY_VERSION_HEX >= 0x03000000 248 | value = PyDict_GetItemWithError(locals, name); 249 | #else 250 | value = PyDict_GetItem(locals, name); 251 | #endif 252 | Py_DECREF(locals); 253 | 254 | if (value == NULL) 255 | { 256 | if (PyErr_Occurred()) 257 | { 258 | return NULL; 259 | } 260 | #if PY_VERSION_HEX >= 0x03000000 261 | PyErr_Format(PyExc_NameError, "variable %R does not exist", name); 262 | #else 263 | PyErr_SetString(PyExc_NameError, "variable does not exist"); 264 | #endif 265 | return NULL; 266 | } 267 | return Py_NewRef(value); 268 | } 269 | #endif 270 | 271 | // gh-91248 added PyFrame_GetVarString() to Python 3.12.0a2 272 | #if PY_VERSION_HEX < 0x030C00A2 && !defined(PYPY_VERSION) 273 | PYCAPI_COMPAT_STATIC_INLINE(PyObject*) 274 | PyFrame_GetVarString(PyFrameObject* frame, const char* name) 275 | { 276 | PyObject *name_obj, *value; 277 | name_obj = PyUnicode_FromString(name); 278 | if (name_obj == NULL) 279 | { 280 | return NULL; 281 | } 282 | value = PyFrame_GetVar(frame, name_obj); 283 | Py_DECREF(name_obj); 284 | return value; 285 | } 286 | #endif 287 | 288 | // bpo-39947 added PyThreadState_GetInterpreter() to Python 3.9.0a5 289 | #if PY_VERSION_HEX < 0x030900A5 || defined(PYPY_VERSION) 290 | PYCAPI_COMPAT_STATIC_INLINE(PyInterpreterState*) 291 | PyThreadState_GetInterpreter(PyThreadState* tstate) 292 | { 293 | assert(tstate != _Py_NULL); 294 | return tstate->interp; 295 | } 296 | #endif 297 | 298 | // bpo-40429 added PyThreadState_GetFrame() to Python 3.9.0b1 299 | #if PY_VERSION_HEX < 0x030900B1 && !defined(PYPY_VERSION) 300 | PYCAPI_COMPAT_STATIC_INLINE(PyFrameObject*) 301 | PyThreadState_GetFrame(PyThreadState* tstate) 302 | { 303 | assert(tstate != _Py_NULL); 304 | return _Py_CAST(PyFrameObject*, Py_XNewRef(tstate->frame)); 305 | } 306 | #endif 307 | 308 | #if !defined(PYPY_VERSION) 309 | PYCAPI_COMPAT_STATIC_INLINE(PyFrameObject*) 310 | _PyThreadState_GetFrameBorrow(PyThreadState* tstate) 311 | { 312 | PyFrameObject* frame = PyThreadState_GetFrame(tstate); 313 | Py_XDECREF(frame); 314 | return frame; 315 | } 316 | #endif 317 | 318 | // bpo-39947 added PyInterpreterState_Get() to Python 3.9.0a5 319 | #if PY_VERSION_HEX < 0x030900A5 || defined(PYPY_VERSION) 320 | PYCAPI_COMPAT_STATIC_INLINE(PyInterpreterState*) 321 | PyInterpreterState_Get(void) 322 | { 323 | PyThreadState* tstate; 324 | PyInterpreterState* interp; 325 | 326 | tstate = PyThreadState_GET(); 327 | if (tstate == _Py_NULL) 328 | { 329 | Py_FatalError("GIL released (tstate is NULL)"); 330 | } 331 | interp = tstate->interp; 332 | if (interp == _Py_NULL) 333 | { 334 | Py_FatalError("no current interpreter"); 335 | } 336 | return interp; 337 | } 338 | #endif 339 | 340 | // bpo-39947 added PyInterpreterState_Get() to Python 3.9.0a6 341 | #if 0x030700A1 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030900A6 && !defined(PYPY_VERSION) 342 | PYCAPI_COMPAT_STATIC_INLINE(uint64_t) 343 | PyThreadState_GetID(PyThreadState* tstate) 344 | { 345 | assert(tstate != _Py_NULL); 346 | return tstate->id; 347 | } 348 | #endif 349 | 350 | // bpo-43760 added PyThreadState_EnterTracing() to Python 3.11.0a2 351 | #if PY_VERSION_HEX < 0x030B00A2 && !defined(PYPY_VERSION) 352 | PYCAPI_COMPAT_STATIC_INLINE(void) 353 | PyThreadState_EnterTracing(PyThreadState* tstate) 354 | { 355 | tstate->tracing++; 356 | #if PY_VERSION_HEX >= 0x030A00A1 357 | tstate->cframe->use_tracing = 0; 358 | #else 359 | tstate->use_tracing = 0; 360 | #endif 361 | } 362 | #endif 363 | 364 | // bpo-43760 added PyThreadState_LeaveTracing() to Python 3.11.0a2 365 | #if PY_VERSION_HEX < 0x030B00A2 && !defined(PYPY_VERSION) 366 | PYCAPI_COMPAT_STATIC_INLINE(void) 367 | PyThreadState_LeaveTracing(PyThreadState* tstate) 368 | { 369 | int use_tracing = (tstate->c_tracefunc != _Py_NULL || tstate->c_profilefunc != _Py_NULL); 370 | tstate->tracing--; 371 | #if PY_VERSION_HEX >= 0x030A00A1 372 | tstate->cframe->use_tracing = use_tracing; 373 | #else 374 | tstate->use_tracing = use_tracing; 375 | #endif 376 | } 377 | #endif 378 | 379 | // bpo-37194 added PyObject_CallNoArgs() to Python 3.9.0a1 380 | #if PY_VERSION_HEX < 0x030900A1 || defined(PYPY_VERSION) 381 | PYCAPI_COMPAT_STATIC_INLINE(PyObject*) 382 | PyObject_CallNoArgs(PyObject* func) 383 | { 384 | return PyObject_CallFunctionObjArgs(func, NULL); 385 | } 386 | #endif 387 | 388 | // bpo-39245 made PyObject_CallOneArg() public (previously called 389 | // _PyObject_CallOneArg) in Python 3.9.0a4 390 | #if PY_VERSION_HEX < 0x030900A4 || defined(PYPY_VERSION) 391 | PYCAPI_COMPAT_STATIC_INLINE(PyObject*) 392 | PyObject_CallOneArg(PyObject* func, PyObject* arg) 393 | { 394 | return PyObject_CallFunctionObjArgs(func, arg, NULL); 395 | } 396 | #endif 397 | 398 | // bpo-1635741 added PyModule_AddObjectRef() to Python 3.10.0a3 399 | #if PY_VERSION_HEX < 0x030A00A3 400 | PYCAPI_COMPAT_STATIC_INLINE(int) 401 | PyModule_AddObjectRef(PyObject* module, const char* name, PyObject* value) 402 | { 403 | int res; 404 | Py_XINCREF(value); 405 | res = PyModule_AddObject(module, name, value); 406 | if (res < 0) 407 | { 408 | Py_XDECREF(value); 409 | } 410 | return res; 411 | } 412 | #endif 413 | 414 | // bpo-40024 added PyModule_AddType() to Python 3.9.0a5 415 | #if PY_VERSION_HEX < 0x030900A5 416 | PYCAPI_COMPAT_STATIC_INLINE(int) 417 | PyModule_AddType(PyObject* module, PyTypeObject* type) 418 | { 419 | const char *name, *dot; 420 | 421 | if (PyType_Ready(type) < 0) 422 | { 423 | return -1; 424 | } 425 | 426 | // inline _PyType_Name() 427 | name = type->tp_name; 428 | assert(name != _Py_NULL); 429 | dot = strrchr(name, '.'); 430 | if (dot != _Py_NULL) 431 | { 432 | name = dot + 1; 433 | } 434 | 435 | return PyModule_AddObjectRef(module, name, _PyObject_CAST(type)); 436 | } 437 | #endif 438 | 439 | // bpo-40241 added PyObject_GC_IsTracked() to Python 3.9.0a6. 440 | // bpo-4688 added _PyObject_GC_IS_TRACKED() to Python 2.7.0a2. 441 | #if PY_VERSION_HEX < 0x030900A6 && !defined(PYPY_VERSION) 442 | PYCAPI_COMPAT_STATIC_INLINE(int) 443 | PyObject_GC_IsTracked(PyObject* obj) 444 | { 445 | return (PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)); 446 | } 447 | #endif 448 | 449 | // bpo-40241 added PyObject_GC_IsFinalized() to Python 3.9.0a6. 450 | // bpo-18112 added _PyGCHead_FINALIZED() to Python 3.4.0 final. 451 | #if PY_VERSION_HEX < 0x030900A6 && PY_VERSION_HEX >= 0x030400F0 && !defined(PYPY_VERSION) 452 | PYCAPI_COMPAT_STATIC_INLINE(int) 453 | PyObject_GC_IsFinalized(PyObject* obj) 454 | { 455 | PyGC_Head* gc = _Py_CAST(PyGC_Head*, obj) - 1; 456 | return (PyObject_IS_GC(obj) && _PyGCHead_FINALIZED(gc)); 457 | } 458 | #endif 459 | 460 | // bpo-39573 added Py_IS_TYPE() to Python 3.9.0a4 461 | #if PY_VERSION_HEX < 0x030900A4 && !defined(Py_IS_TYPE) 462 | PYCAPI_COMPAT_STATIC_INLINE(int) 463 | _Py_IS_TYPE(PyObject* ob, PyTypeObject* type) 464 | { 465 | return Py_TYPE(ob) == type; 466 | } 467 | #define Py_IS_TYPE(ob, type) _Py_IS_TYPE(_PyObject_CAST(ob), type) 468 | #endif 469 | 470 | // bpo-46906 added PyFloat_Pack2() and PyFloat_Unpack2() to Python 3.11a7. 471 | // bpo-11734 added _PyFloat_Pack2() and _PyFloat_Unpack2() to Python 3.6.0b1. 472 | // Python 3.11a2 moved _PyFloat_Pack2() and _PyFloat_Unpack2() to the internal 473 | // C API: Python 3.11a2-3.11a6 versions are not supported. 474 | #if 0x030600B1 <= PY_VERSION_HEX && PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) 475 | PYCAPI_COMPAT_STATIC_INLINE(int) 476 | PyFloat_Pack2(double x, char* p, int le) 477 | { 478 | return _PyFloat_Pack2(x, (unsigned char*)p, le); 479 | } 480 | 481 | PYCAPI_COMPAT_STATIC_INLINE(double) 482 | PyFloat_Unpack2(const char* p, int le) 483 | { 484 | return _PyFloat_Unpack2((const unsigned char*)p, le); 485 | } 486 | #endif 487 | 488 | // bpo-46906 added PyFloat_Pack4(), PyFloat_Pack8(), PyFloat_Unpack4() and 489 | // PyFloat_Unpack8() to Python 3.11a7. 490 | // Python 3.11a2 moved _PyFloat_Pack4(), _PyFloat_Pack8(), _PyFloat_Unpack4() 491 | // and _PyFloat_Unpack8() to the internal C API: Python 3.11a2-3.11a6 versions 492 | // are not supported. 493 | #if PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) 494 | PYCAPI_COMPAT_STATIC_INLINE(int) 495 | PyFloat_Pack4(double x, char* p, int le) 496 | { 497 | return _PyFloat_Pack4(x, (unsigned char*)p, le); 498 | } 499 | 500 | PYCAPI_COMPAT_STATIC_INLINE(int) 501 | PyFloat_Pack8(double x, char* p, int le) 502 | { 503 | return _PyFloat_Pack8(x, (unsigned char*)p, le); 504 | } 505 | 506 | PYCAPI_COMPAT_STATIC_INLINE(double) 507 | PyFloat_Unpack4(const char* p, int le) 508 | { 509 | return _PyFloat_Unpack4((const unsigned char*)p, le); 510 | } 511 | 512 | PYCAPI_COMPAT_STATIC_INLINE(double) 513 | PyFloat_Unpack8(const char* p, int le) 514 | { 515 | return _PyFloat_Unpack8((const unsigned char*)p, le); 516 | } 517 | #endif 518 | 519 | // gh-92154 added PyCode_GetCode() to Python 3.11.0b1 520 | #if PY_VERSION_HEX < 0x030B00B1 && !defined(PYPY_VERSION) 521 | PYCAPI_COMPAT_STATIC_INLINE(PyObject*) 522 | PyCode_GetCode(PyCodeObject* code) 523 | { 524 | return Py_NewRef(code->co_code); 525 | } 526 | #endif 527 | 528 | // gh-95008 added PyCode_GetVarnames() to Python 3.11.0rc1 529 | #if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) 530 | PYCAPI_COMPAT_STATIC_INLINE(PyObject*) 531 | PyCode_GetVarnames(PyCodeObject* code) 532 | { 533 | return Py_NewRef(code->co_varnames); 534 | } 535 | #endif 536 | 537 | // gh-95008 added PyCode_GetFreevars() to Python 3.11.0rc1 538 | #if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) 539 | PYCAPI_COMPAT_STATIC_INLINE(PyObject*) 540 | PyCode_GetFreevars(PyCodeObject* code) 541 | { 542 | return Py_NewRef(code->co_freevars); 543 | } 544 | #endif 545 | 546 | // gh-95008 added PyCode_GetCellvars() to Python 3.11.0rc1 547 | #if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) 548 | PYCAPI_COMPAT_STATIC_INLINE(PyObject*) 549 | PyCode_GetCellvars(PyCodeObject* code) 550 | { 551 | return Py_NewRef(code->co_cellvars); 552 | } 553 | #endif 554 | 555 | // Py_UNUSED() was added to Python 3.4.0b2. 556 | #if PY_VERSION_HEX < 0x030400B2 && !defined(Py_UNUSED) 557 | #if defined(__GNUC__) || defined(__clang__) 558 | #define Py_UNUSED(name) _unused_##name __attribute__((unused)) 559 | #else 560 | #define Py_UNUSED(name) _unused_##name 561 | #endif 562 | #endif 563 | 564 | #ifdef __cplusplus 565 | } 566 | #endif 567 | #endif // PYTHONCAPI_COMPAT 568 | -------------------------------------------------------------------------------- /test/cases/call_main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | def main(argv=None): 5 | print("successfully called main") 6 | 7 | 8 | sys.modules['__main__'].main(sys.argv) 9 | -------------------------------------------------------------------------------- /test/cases/classes.py: -------------------------------------------------------------------------------- 1 | import scorep.user 2 | import scorep.instrumenter 3 | 4 | 5 | class TestClass: 6 | def foo(self): 7 | print("foo") 8 | 9 | def doo(arg): 10 | print("doo") 11 | arg.foo() 12 | 13 | 14 | class TestClass2: 15 | @scorep.user.region() 16 | def foo(self): 17 | print("foo-2") 18 | 19 | 20 | def foo(): 21 | print("bar") 22 | 23 | def doo(arg): 24 | print("asdgh") 25 | doo("test") 26 | 27 | 28 | if __name__ == "__main__": 29 | t = TestClass() 30 | t2 = TestClass2() 31 | 32 | t2.foo() 33 | t.doo() 34 | foo() 35 | 36 | with scorep.instrumenter.disable(): 37 | t2.foo() 38 | -------------------------------------------------------------------------------- /test/cases/classes2.py: -------------------------------------------------------------------------------- 1 | import classes 2 | 3 | t = classes.TestClass 4 | t().doo() 5 | -------------------------------------------------------------------------------- /test/cases/context.py: -------------------------------------------------------------------------------- 1 | import scorep.user 2 | import scorep.instrumenter 3 | 4 | 5 | def foo(): 6 | with scorep.user.region("test_region"): 7 | print("hello world") 8 | 9 | 10 | def bar(): 11 | with scorep.instrumenter.enable(): 12 | foo() 13 | with scorep.instrumenter.disable(): 14 | foo() 15 | 16 | 17 | foo() 18 | bar() 19 | -------------------------------------------------------------------------------- /test/cases/decorator.py: -------------------------------------------------------------------------------- 1 | import scorep.user 2 | 3 | 4 | @scorep.user.region() 5 | def foo(): 6 | print("hello world") 7 | 8 | 9 | foo() 10 | with scorep.instrumenter.disable(): 11 | foo() 12 | with scorep.instrumenter.enable(): 13 | foo() 14 | -------------------------------------------------------------------------------- /test/cases/error_region.py: -------------------------------------------------------------------------------- 1 | import scorep.user 2 | 3 | 4 | def foo(): 5 | scorep.user.region_end("test_region") 6 | scorep.user.region_end("test_region_2") 7 | 8 | 9 | foo() 10 | -------------------------------------------------------------------------------- /test/cases/external_instrumentation.py: -------------------------------------------------------------------------------- 1 | import scorep.user 2 | 3 | import instrumentation2 4 | 5 | scorep.user.instrument_module(instrumentation2) 6 | -------------------------------------------------------------------------------- /test/cases/file_io.py: -------------------------------------------------------------------------------- 1 | import os 2 | import scorep.instrumenter 3 | 4 | with scorep.instrumenter.enable("expect io"): 5 | with open("test.txt", "w") as f: 6 | f.write("test") 7 | 8 | with open("test.txt", "r") as f: 9 | data = f.read() 10 | print(data) 11 | 12 | os.remove("test.txt") 13 | -------------------------------------------------------------------------------- /test/cases/force_finalize.py: -------------------------------------------------------------------------------- 1 | import scorep.user 2 | 3 | 4 | def foo(): 5 | print("foo") 6 | 7 | 8 | def bar(): 9 | print("bar") 10 | 11 | 12 | foo() 13 | scorep.user.force_finalize() 14 | bar() 15 | -------------------------------------------------------------------------------- /test/cases/instrumentation.py: -------------------------------------------------------------------------------- 1 | import instrumentation2 2 | 3 | 4 | def foo(): 5 | print("hello world") 6 | instrumentation2.baz() 7 | instrumentation2.bar() 8 | 9 | 10 | foo() 11 | -------------------------------------------------------------------------------- /test/cases/instrumentation2.py: -------------------------------------------------------------------------------- 1 | def baz(): 2 | print("baz") 3 | 4 | 5 | def bar(): 6 | print("bar") 7 | -------------------------------------------------------------------------------- /test/cases/interrupt.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | def foo(): 5 | print("hello world") 6 | while True: 7 | time.sleep(1) 8 | print("By By.") 9 | 10 | 11 | foo() 12 | -------------------------------------------------------------------------------- /test/cases/miniasync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import asyncio 4 | 5 | work_size = 10000000 6 | 7 | 8 | def actual_work(i): 9 | return sum(range(i)) 10 | 11 | 12 | async def work1(): 13 | for i in range(5): 14 | print("work1 ", actual_work(i * work_size)) 15 | await asyncio.sleep(1) 16 | 17 | 18 | async def work2(): 19 | for i in range(5): 20 | print("work2 ", actual_work(i * work_size)) 21 | await asyncio.sleep(1) 22 | 23 | 24 | async def amain(): 25 | await asyncio.gather( 26 | work1(), 27 | work2(), 28 | asyncio.get_event_loop().getaddrinfo("tu-dresden.de", 80), 29 | asyncio.get_event_loop().getaddrinfo("www.tu-dresden.de", 80), 30 | ) 31 | 32 | 33 | def main(): 34 | asyncio.run(amain()) 35 | 36 | 37 | if __name__ == "__main__": 38 | main() 39 | -------------------------------------------------------------------------------- /test/cases/mpi.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import scorep 4 | from mpi4py import MPI 5 | import numpy as np 6 | import mpi4py 7 | import instrumentation2 8 | mpi4py.rc.thread_level = "funneled" 9 | 10 | scorep.instrumenter.register() 11 | 12 | comm = mpi4py.MPI.COMM_WORLD 13 | 14 | comm.Barrier() 15 | 16 | # Prepare a vector of N=5 elements to be broadcasted... 17 | N = 5 18 | if comm.rank == 0: 19 | A = np.arange(N, dtype=np.float64) # rank 0 has proper data 20 | instrumentation2.baz() 21 | else: 22 | instrumentation2.bar() 23 | A = np.empty(N, dtype=np.float64) # all other just an empty array 24 | 25 | # Broadcast A from rank 0 to everybody 26 | comm.Bcast([A, MPI.DOUBLE]) 27 | 28 | # Everybody should now have the same... 29 | print("[%02d] %s" % (comm.rank, A)) 30 | -------------------------------------------------------------------------------- /test/cases/nosleep.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | def pointless_sleep(): 5 | time.sleep(5) 6 | 7 | 8 | def baz(): 9 | print("Nice you are here.") 10 | 11 | 12 | def foo(sleep=False): 13 | print("Hello world.") 14 | if sleep: 15 | pointless_sleep() 16 | baz() 17 | print("Good by.") 18 | 19 | 20 | foo() 21 | -------------------------------------------------------------------------------- /test/cases/numpy_dot.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import scorep.instrumenter 3 | 4 | with scorep.instrumenter.enable(): 5 | a = [[1, 2], [3, 4]] 6 | b = [[1, 2], [3, 4]] 7 | 8 | c = numpy.dot(a, b) 9 | print(c) 10 | -------------------------------------------------------------------------------- /test/cases/numpy_dot_large.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import numpy.linalg 3 | import scorep.instrumenter 4 | 5 | with scorep.instrumenter.enable(): 6 | a = [] 7 | b = [] 8 | 9 | for i in range(1000): 10 | a.append([]) 11 | b.append([]) 12 | for j in range(1000): 13 | a[i].append(i * j) 14 | b[i].append(i * j) 15 | 16 | c = numpy.dot(a, b) 17 | c = numpy.matmul(a, c) 18 | q, r = numpy.linalg.qr(c) 19 | print(q, r) 20 | -------------------------------------------------------------------------------- /test/cases/reload.py: -------------------------------------------------------------------------------- 1 | import reload_test 2 | import importlib 3 | import os 4 | 5 | data1 = """ 6 | def foo(): 7 | print("foo1") 8 | """ 9 | 10 | data2 = """ 11 | def foo(arg): 12 | print(arg) 13 | def bar(): 14 | print("bar") 15 | """ 16 | 17 | 18 | with open("reload_test.py", "w") as f: 19 | f.write(data1) 20 | 21 | reload_test.foo() 22 | reload_test.foo() 23 | 24 | importlib.reload(reload_test) 25 | reload_test.foo() 26 | 27 | 28 | with open("reload_test.py", "w") as f: 29 | f.write(data2) 30 | 31 | importlib.reload(reload_test) 32 | reload_test.foo("foo2") 33 | reload_test.bar() 34 | 35 | os.remove("reload_test.py") 36 | -------------------------------------------------------------------------------- /test/cases/sleep.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | def pointless_sleep(): 5 | time.sleep(5) 6 | 7 | 8 | def baz(): 9 | print("Nice you are here.") 10 | 11 | 12 | def foo(): 13 | print("Hello world.") 14 | if True: 15 | pointless_sleep() 16 | baz() 17 | print("Good by.") 18 | 19 | 20 | foo() 21 | -------------------------------------------------------------------------------- /test/cases/use_threads.py: -------------------------------------------------------------------------------- 1 | import random 2 | import threading 3 | import time 4 | import instrumentation2 5 | 6 | 7 | lock = threading.Lock() 8 | 9 | 10 | def worker(id, func): 11 | with lock: 12 | print("Thread %s started" % id) 13 | # Use a random delay to add non-determinism to the output 14 | time.sleep(random.uniform(0.01, 0.9)) 15 | func() 16 | 17 | 18 | def foo(): 19 | print("hello world") 20 | t1 = threading.Thread(target=worker, args=(0, instrumentation2.bar)) 21 | t2 = threading.Thread(target=worker, args=(1, instrumentation2.baz)) 22 | t1.start() 23 | t2.start() 24 | t1.join() 25 | t2.join() 26 | 27 | 28 | foo() 29 | -------------------------------------------------------------------------------- /test/cases/user_instrumentation.py: -------------------------------------------------------------------------------- 1 | import scorep 2 | import instrumentation2 3 | 4 | 5 | def foo(): 6 | print("hello world") 7 | instrumentation2.bar() 8 | 9 | 10 | @scorep.instrumenter.enable() 11 | def foo2(): 12 | print("hello world2") 13 | instrumentation2.baz() 14 | 15 | 16 | with scorep.instrumenter.enable(): 17 | foo() 18 | 19 | foo2() 20 | -------------------------------------------------------------------------------- /test/cases/user_regions.py: -------------------------------------------------------------------------------- 1 | import scorep.user 2 | import scorep.instrumenter 3 | 4 | 5 | def foo(): 6 | scorep.user.region_begin("test_region") 7 | print("hello world") 8 | scorep.user.region_end("test_region") 9 | 10 | 11 | def foo2(): 12 | with scorep.user.region("test_region_2"): 13 | print("hello world") 14 | 15 | 16 | @scorep.user.region() 17 | def foo3(): 18 | print("hello world3") 19 | 20 | 21 | @scorep.user.region("test_region_4") 22 | def foo4(): 23 | print("hello world4") 24 | 25 | 26 | foo() 27 | foo2() 28 | with scorep.instrumenter.enable(): 29 | foo3() 30 | with scorep.instrumenter.disable(): 31 | foo3() 32 | foo4() 33 | -------------------------------------------------------------------------------- /test/cases/user_rewind.py: -------------------------------------------------------------------------------- 1 | import scorep.user 2 | 3 | 4 | def bar(): 5 | print("hello world") 6 | 7 | 8 | def foo(): 9 | bar() 10 | scorep.user.rewind_begin("test_region") 11 | bar() 12 | scorep.user.rewind_end("test_region", True) 13 | 14 | 15 | foo() 16 | -------------------------------------------------------------------------------- /test/conftest.py: -------------------------------------------------------------------------------- 1 | import utils 2 | 3 | 4 | def pytest_assertrepr_compare(op, left, right): 5 | if isinstance(left, utils.OTF2_Region) and isinstance(right, utils.OTF2_Trace): 6 | if op == "in": 7 | output = ["Region \"{}\" ENTER or LEAVE not found in trace:".format(left)] 8 | for line in str(right).split("\n"): 9 | output.append("\t" + line) 10 | return output 11 | elif op == "not in": 12 | output = ["Unexpected region \"{}\" ENTER or LEAVE found in trace:".format(left)] 13 | for line in str(right).split("\n"): 14 | output.append("\t" + line) 15 | return output 16 | elif isinstance(left, utils.OTF2_Parameter) and isinstance(right, utils.OTF2_Trace) and op == "in": 17 | output = ["Parameter \"{parameter}\" with Value \"{value}\" not found in trace:".format( 18 | parameter=left.parameter, 19 | value=left.value)] 20 | for line in str(right).split("\n"): 21 | output.append("\t" + line) 22 | return output 23 | -------------------------------------------------------------------------------- /test/test_helper.py: -------------------------------------------------------------------------------- 1 | import os 2 | import scorep.helper 3 | 4 | 5 | def test_add_to_ld_library_path(monkeypatch): 6 | # Previous value: Empty 7 | monkeypatch.setenv('LD_LIBRARY_PATH', '') 8 | scorep.helper.add_to_ld_library_path('/my/path') 9 | assert os.environ['LD_LIBRARY_PATH'] == '/my/path' 10 | # Don't add duplicates 11 | scorep.helper.add_to_ld_library_path('/my/path') 12 | assert os.environ['LD_LIBRARY_PATH'] == '/my/path' 13 | # Prepend 14 | scorep.helper.add_to_ld_library_path('/new/folder') 15 | assert os.environ['LD_LIBRARY_PATH'] == '/new/folder:/my/path' 16 | # also no duplicates: 17 | for p in ('/my/path', '/new/folder'): 18 | scorep.helper.add_to_ld_library_path(p) 19 | assert os.environ['LD_LIBRARY_PATH'] == '/new/folder:/my/path' 20 | 21 | # Add parent folder of existing one 22 | monkeypatch.setenv('LD_LIBRARY_PATH', '/some/folder:/parent/sub') 23 | scorep.helper.add_to_ld_library_path('/parent') 24 | assert os.environ['LD_LIBRARY_PATH'] == '/parent:/some/folder:/parent/sub' 25 | -------------------------------------------------------------------------------- /test/test_pathUtils.cpp: -------------------------------------------------------------------------------- 1 | #include "scorepy/pathUtils.hpp" 2 | #include 3 | #include 4 | 5 | #define TEST(condition) \ 6 | if (!(condition)) \ 7 | throw std::runtime_error(std::string("Test ") + #condition + " failed at " + __FILE__ + ":" + \ 8 | std::to_string(__LINE__)) 9 | 10 | int main() 11 | { 12 | using scorepy::abspath; 13 | // Multiple slashes at start collapsed to 1 14 | TEST(abspath("/abc") == "/abc"); 15 | TEST(abspath("//abc") == "//abc"); 16 | TEST(abspath("///abc") == "/abc"); 17 | TEST(abspath("////abc") == "/abc"); 18 | // Trailing slashes and multiple slashes removed 19 | TEST(abspath("/abc/") == "/abc"); 20 | TEST(abspath("/abc//") == "/abc"); 21 | TEST(abspath("/abc//de") == "/abc/de"); 22 | TEST(abspath("/abc//de///fg") == "/abc/de/fg"); 23 | TEST(abspath("/abc//de///fg/") == "/abc/de/fg"); 24 | TEST(abspath("/abc//de///fg////") == "/abc/de/fg"); 25 | TEST(abspath("//abc/") == "//abc"); 26 | TEST(abspath("//abc//") == "//abc"); 27 | TEST(abspath("//abc//de") == "//abc/de"); 28 | TEST(abspath("//abc//de///fg") == "//abc/de/fg"); 29 | TEST(abspath("//abc//de///fg/") == "//abc/de/fg"); 30 | TEST(abspath("//abc//de///fg////") == "//abc/de/fg"); 31 | // Single dots removed 32 | TEST(abspath("/./abc/./defgh/./ijkl/.") == "/abc/defgh/ijkl"); 33 | TEST(abspath("/./abc././def.gh/./ijkl././.mn/.") == "/abc./def.gh/ijkl./.mn"); 34 | // Going up 1 level removes prior folder 35 | TEST(abspath("/abc/..") == "/"); 36 | TEST(abspath("//abc/..") == "//"); 37 | TEST(abspath("///abc/..") == "/"); 38 | TEST(abspath("/abc/../de") == "/de"); 39 | TEST(abspath("//abc/../de") == "//de"); 40 | TEST(abspath("///abc/../de") == "/de"); 41 | TEST(abspath("/abc/de/../fg") == "/abc/fg"); 42 | TEST(abspath("/abc/de/../../fg") == "/fg"); 43 | TEST(abspath("/abc/de../../fg") == "/abc/fg"); 44 | TEST(abspath("/abc../de/../fg") == "/abc../fg"); 45 | // Going up from root does nothing 46 | TEST(abspath("/../ab") == "/ab"); 47 | TEST(abspath("//../ab") == "//ab"); 48 | TEST(abspath("///../ab") == "/ab"); 49 | TEST(abspath("/abc/defgh/../../ijkl") == "/ijkl"); 50 | TEST(abspath("//abc/defgh/../../ijkl") == "//ijkl"); 51 | TEST(abspath("///abc/defgh/../../ijkl") == "/ijkl"); 52 | } 53 | -------------------------------------------------------------------------------- /test/test_scorep.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env pytest 2 | 3 | import os 4 | import pkgutil 5 | import platform 6 | import pytest 7 | import re 8 | import sys 9 | import numpy 10 | 11 | import utils 12 | from utils import OTF2_Trace, OTF2_Region, OTF2_Parameter 13 | 14 | 15 | def version_tuple(v): 16 | return tuple(map(int, (v.split(".")))) 17 | 18 | 19 | def has_package(name): 20 | return len(pkgutil.extend_path([], name)) > 0 21 | 22 | 23 | def requires_package(name): 24 | return pytest.mark.skipif(not has_package(name), reason="%s is required" % name) 25 | 26 | 27 | def functions_in_trace(function, otf2_print): 28 | for event in ("ENTER", "LEAVE"): 29 | assert re.search( 30 | '%s[ ]*[0-9 ]*[0-9 ]*Region: "%s"' % (event, function), otf2_print 31 | ) 32 | 33 | 34 | cinstrumenter_skip_mark = pytest.mark.skipif( 35 | platform.python_implementation() == "PyPy", 36 | reason="CInstrumenter only available in CPython and not in PyPy", 37 | ) 38 | # All instrumenters (except dummy which isn't a real one) 39 | ALL_INSTRUMENTERS = [ 40 | "profile", 41 | "trace", 42 | pytest.param("cProfile", marks=cinstrumenter_skip_mark), 43 | pytest.param("cTrace", marks=cinstrumenter_skip_mark), 44 | ] 45 | 46 | foreach_instrumenter = pytest.mark.parametrize("instrumenter", ALL_INSTRUMENTERS) 47 | 48 | 49 | @pytest.fixture 50 | def scorep_env(tmp_path): 51 | env = os.environ.copy() 52 | env["SCOREP_ENABLE_PROFILING"] = "false" 53 | env["SCOREP_ENABLE_TRACING"] = "true" 54 | env["SCOREP_PROFILING_MAX_CALLPATH_DEPTH"] = "98" 55 | env["SCOREP_TOTAL_MEMORY"] = "3G" 56 | env["SCOREP_EXPERIMENT_DIRECTORY"] = str(tmp_path / "test_bindings_dir") 57 | return env 58 | 59 | 60 | def get_trace_path(env): 61 | """Return the path to the otf2 trace file given an environment dict""" 62 | return env["SCOREP_EXPERIMENT_DIRECTORY"] + "/traces.otf2" 63 | 64 | 65 | def test_has_version(): 66 | import scorep 67 | 68 | assert scorep.__version__ is not None 69 | 70 | 71 | @cinstrumenter_skip_mark 72 | def test_has_c_instrumenter(): 73 | from scorep.instrumenter import has_c_instrumenter 74 | 75 | assert has_c_instrumenter() 76 | 77 | 78 | @foreach_instrumenter 79 | def test_user_regions(scorep_env, instrumenter): 80 | trace_path = get_trace_path(scorep_env) 81 | 82 | std_out, std_err = utils.call_with_scorep( 83 | "cases/user_regions.py", 84 | ["--nopython", "--instrumenter-type=" + instrumenter], 85 | env=scorep_env, 86 | ) 87 | 88 | assert std_err == "" 89 | assert ( 90 | std_out 91 | == "hello world\nhello world\nhello world3\nhello world3\nhello world4\n" 92 | ) 93 | 94 | trace = OTF2_Trace(trace_path) 95 | assert OTF2_Region("user:test_region") in trace 96 | assert OTF2_Region("user:test_region_2") in trace 97 | assert len(trace.findall(OTF2_Region("__main__:foo3"))) == 4 98 | assert OTF2_Region("user:test_region_4") in trace 99 | 100 | 101 | @foreach_instrumenter 102 | def test_context(scorep_env, instrumenter): 103 | trace_path = get_trace_path(scorep_env) 104 | 105 | std_out, std_err = utils.call_with_scorep( 106 | "cases/context.py", 107 | ["--noinstrumenter", "--instrumenter-type=" + instrumenter], 108 | env=scorep_env, 109 | ) 110 | 111 | assert std_err == "" 112 | assert std_out == "hello world\nhello world\nhello world\n" 113 | 114 | trace = OTF2_Trace(trace_path) 115 | assert OTF2_Region("user:test_region") in trace 116 | assert OTF2_Region("__main__:foo") in trace 117 | 118 | 119 | @foreach_instrumenter 120 | def test_decorator(scorep_env, instrumenter): 121 | trace_path = get_trace_path(scorep_env) 122 | 123 | std_out, std_err = utils.call_with_scorep( 124 | "cases/decorator.py", 125 | ["--noinstrumenter", "--instrumenter-type=" + instrumenter], 126 | env=scorep_env, 127 | ) 128 | 129 | assert std_err == "" 130 | assert std_out == "hello world\nhello world\nhello world\n" 131 | 132 | trace = OTF2_Trace(trace_path) 133 | assert len(trace.findall(OTF2_Region("__main__:foo"))) == 6 134 | 135 | 136 | def test_user_regions_no_scorep(): 137 | std_out, std_err = utils.call([sys.executable, "cases/user_regions.py"]) 138 | 139 | assert std_err == "" 140 | assert ( 141 | std_out 142 | == "hello world\nhello world\nhello world3\nhello world3\nhello world4\n" 143 | ) 144 | 145 | 146 | @foreach_instrumenter 147 | def test_user_rewind(scorep_env, instrumenter): 148 | trace_path = get_trace_path(scorep_env) 149 | 150 | std_out, std_err = utils.call_with_scorep( 151 | "cases/user_rewind.py", ["--instrumenter-type=" + instrumenter], env=scorep_env 152 | ) 153 | 154 | assert std_err == "" 155 | assert std_out == "hello world\nhello world\n" 156 | 157 | trace = OTF2_Trace(trace_path) 158 | assert re.search("MEASUREMENT_ON_OFF[ ]*[0-9 ]*[0-9 ]*Mode: OFF", str(trace)) 159 | assert re.search("MEASUREMENT_ON_OFF[ ]*[0-9 ]*[0-9 ]*Mode: ON", str(trace)) 160 | 161 | 162 | @pytest.mark.parametrize("instrumenter", ALL_INSTRUMENTERS + [None]) 163 | def test_instrumentation(scorep_env, instrumenter): 164 | trace_path = get_trace_path(scorep_env) 165 | 166 | # Also test when no instrumenter is given 167 | instrumenter_type = ["--instrumenter-type=" + instrumenter] if instrumenter else [] 168 | std_out, std_err = utils.call_with_scorep( 169 | "cases/instrumentation.py", ["--nocompiler"] + instrumenter_type, env=scorep_env 170 | ) 171 | 172 | assert std_err == "" 173 | assert std_out == "hello world\nbaz\nbar\n" 174 | 175 | trace = OTF2_Trace(trace_path) 176 | assert OTF2_Region("__main__:foo") in trace 177 | assert OTF2_Region("instrumentation2:bar") in trace 178 | assert OTF2_Region("instrumentation2:baz") in trace 179 | 180 | 181 | @foreach_instrumenter 182 | def test_user_instrumentation(scorep_env, instrumenter): 183 | trace_path = get_trace_path(scorep_env) 184 | 185 | std_out, std_err = utils.call_with_scorep( 186 | "cases/user_instrumentation.py", 187 | ["--nocompiler", "--noinstrumenter", "--instrumenter-type=" + instrumenter], 188 | env=scorep_env, 189 | ) 190 | 191 | assert std_err == "" 192 | assert std_out == "hello world\nbar\nhello world2\nbaz\n" 193 | 194 | trace = OTF2_Trace(trace_path) 195 | assert OTF2_Region("__main__:foo") in trace 196 | assert OTF2_Region("__main__:foo2") in trace 197 | assert OTF2_Region("instrumentation2:bar") in trace 198 | assert OTF2_Region("instrumentation2:baz") in trace 199 | 200 | 201 | @foreach_instrumenter 202 | def test_external_user_instrumentation(scorep_env, instrumenter): 203 | trace_path = get_trace_path(scorep_env) 204 | 205 | std_out, std_err = utils.call_with_scorep( 206 | "cases/instrumentation.py", 207 | ["--nocompiler", "--noinstrumenter", "--instrumenter-type=" + 208 | instrumenter, "--instrumenter-file=cases/external_instrumentation.py"], 209 | env=scorep_env, 210 | ) 211 | 212 | assert std_err == "" 213 | assert std_out == "hello world\nbaz\nbar\n" 214 | 215 | trace = OTF2_Trace(trace_path) 216 | assert OTF2_Region("instrumentation2:bar") in trace 217 | assert OTF2_Region("instrumentation2:baz") in trace 218 | 219 | 220 | @foreach_instrumenter 221 | def test_error_region(scorep_env, instrumenter): 222 | trace_path = get_trace_path(scorep_env) 223 | 224 | std_out, std_err = utils.call_with_scorep( 225 | "cases/error_region.py", 226 | ["--nocompiler", "--noinstrumenter", "--instrumenter-type=" + instrumenter], 227 | env=scorep_env, 228 | ) 229 | 230 | assert ( 231 | std_err 232 | == "SCOREP_BINDING_PYTHON ERROR: There was a region exit without an enter!\n" 233 | + 'SCOREP_BINDING_PYTHON ERROR: For details look for "error_region" in the trace or profile.\n' 234 | ) 235 | assert std_out == "" 236 | 237 | trace = OTF2_Trace(trace_path) 238 | assert OTF2_Region("error_region") in trace 239 | assert OTF2_Parameter("leave-region", "user:test_region") in trace 240 | assert OTF2_Parameter("leave-region", "user:test_region_2") in trace 241 | 242 | 243 | @requires_package("mpi4py") 244 | @requires_package("numpy") 245 | @foreach_instrumenter 246 | def test_mpi(scorep_env, instrumenter): 247 | trace_path = get_trace_path(scorep_env) 248 | std_out, std_err = utils.call( 249 | [ 250 | "mpirun", 251 | "-n", 252 | "2", 253 | "-mca", 254 | "btl", 255 | "^openib", 256 | sys.executable, 257 | "-m", 258 | "scorep", 259 | "--mpp=mpi", 260 | "--nocompiler", 261 | "--noinstrumenter", 262 | "--instrumenter-type=" + instrumenter, 263 | "cases/mpi.py", 264 | ], 265 | env=scorep_env, 266 | ) 267 | 268 | assert re.search(r"\[Score-P\] [\w/.: ]*MPI_THREAD_\(SERIALIZED\|MULTIPLE\) ", std_err) 269 | assert "[00] [0. 1. 2. 3. 4.]\n" in std_out 270 | assert "[01] [0. 1. 2. 3. 4.]\n" in std_out 271 | assert "bar\n" in std_out 272 | assert "baz\n" in std_out 273 | 274 | trace = OTF2_Trace(trace_path) 275 | assert OTF2_Region("instrumentation2:bar") in trace 276 | assert OTF2_Region("instrumentation2:baz") in trace 277 | 278 | 279 | @foreach_instrumenter 280 | def test_call_main(scorep_env, instrumenter): 281 | std_out, std_err = utils.call_with_scorep( 282 | "cases/call_main.py", 283 | ["--nocompiler", "--instrumenter-type=" + instrumenter], 284 | expected_returncode=1, 285 | env=scorep_env, 286 | ) 287 | 288 | expected_std_err = r"scorep: Someone called scorep\.__main__\.main" 289 | expected_std_out = "" 290 | assert re.search(expected_std_err, std_err) 291 | assert std_out == expected_std_out 292 | 293 | 294 | @foreach_instrumenter 295 | def test_classes(scorep_env, instrumenter): 296 | trace_path = get_trace_path(scorep_env) 297 | std_out, std_err = utils.call_with_scorep( 298 | "cases/classes.py", 299 | ["--nocompiler", "--instrumenter-type=" + instrumenter], 300 | expected_returncode=0, 301 | env=scorep_env, 302 | ) 303 | 304 | expected_std_err = "" 305 | expected_std_out = "foo-2\ndoo\nfoo\nbar\nasdgh\nfoo-2\n" 306 | 307 | assert std_out == expected_std_out 308 | assert std_err == expected_std_err 309 | 310 | trace = OTF2_Trace(trace_path) 311 | 312 | assert OTF2_Region("__main__:foo") in trace 313 | assert OTF2_Region("__main__.TestClass:foo") in trace 314 | assert OTF2_Region("__main__.TestClass2:foo") in trace 315 | 316 | 317 | def test_dummy(scorep_env): 318 | std_out, std_err = utils.call_with_scorep( 319 | "cases/instrumentation.py", ["--instrumenter-type=dummy"], env=scorep_env 320 | ) 321 | 322 | assert std_err == "" 323 | assert std_out == "hello world\nbaz\nbar\n" 324 | assert os.path.exists( 325 | scorep_env["SCOREP_EXPERIMENT_DIRECTORY"] 326 | ), "Score-P directory exists for dummy test" 327 | 328 | 329 | @pytest.mark.skipif(sys.version_info.major < 3, reason="not tested for python 2") 330 | @requires_package("numpy") 331 | @pytest.mark.skipif(version_tuple(numpy.version.version) >= version_tuple("1.22.0"), 332 | reason="There are some changes regarding __array_function__ in 1.22.0," 333 | "so the test is no longer needed") 334 | @foreach_instrumenter 335 | def test_numpy_dot(scorep_env, instrumenter): 336 | trace_path = get_trace_path(scorep_env) 337 | 338 | std_out, std_err = utils.call_with_scorep( 339 | "cases/numpy_dot.py", 340 | ["--nocompiler", "--noinstrumenter", "--instrumenter-type=" + instrumenter], 341 | env=scorep_env, 342 | ) 343 | 344 | assert std_out == "[[ 7 10]\n [15 22]]\n" 345 | assert std_err == "" 346 | 347 | trace = OTF2_Trace(trace_path) 348 | assert OTF2_Region("numpy.__array_function__:dot") in trace 349 | 350 | 351 | @foreach_instrumenter 352 | def test_threads(scorep_env, instrumenter): 353 | trace_path = get_trace_path(scorep_env) 354 | 355 | std_out, std_err = utils.call_with_scorep( 356 | "cases/use_threads.py", 357 | ["--nocompiler", "--instrumenter-type=" + instrumenter], 358 | env=scorep_env, 359 | ) 360 | 361 | assert std_err == "" or "warning: Thread after main " in std_err 362 | assert "hello world\n" in std_out 363 | assert "Thread 0 started\n" in std_out 364 | assert "Thread 1 started\n" in std_out 365 | assert "bar\n" in std_out 366 | assert "baz\n" in std_out 367 | 368 | trace = OTF2_Trace(trace_path) 369 | assert OTF2_Region("__main__:foo") in trace 370 | assert OTF2_Region("instrumentation2:bar") in trace 371 | assert OTF2_Region("instrumentation2:baz") in trace 372 | 373 | 374 | @pytest.mark.skipif(sys.version_info.major < 3, reason="not tested for python 2") 375 | @foreach_instrumenter 376 | def test_io(scorep_env, instrumenter): 377 | trace_path = get_trace_path(scorep_env) 378 | 379 | print("start") 380 | std_out, std_err = utils.call_with_scorep( 381 | "cases/file_io.py", 382 | [ 383 | "--nocompiler", 384 | "--instrumenter-type=" + instrumenter, 385 | "--noinstrumenter", 386 | "--io=runtime:posix", 387 | ], 388 | env=scorep_env, 389 | ) 390 | 391 | assert std_err == "" 392 | assert "test\n" in std_out 393 | 394 | trace = utils.OTF2_Trace(trace_path) 395 | 396 | file_regex = "\\[POSIX I\\/O\\][ \\w:/]*test\\.txt" 397 | # print_regex = "STDOUT_FILENO" 398 | 399 | ops = { 400 | "open": {"ENTER": "open64", "IO_CREATE_HANDLE": file_regex, "LEAVE": "open64"}, 401 | "seek": {"ENTER": "lseek64", "IO_SEEK": file_regex, "LEAVE": "lseek64"}, 402 | "write": { 403 | "ENTER": "write", 404 | "IO_OPERATION_BEGIN": file_regex, 405 | "IO_OPERATION_COMPLETE": file_regex, 406 | "LEAVE": "write", 407 | }, 408 | "read": { 409 | "ENTER": "read", 410 | "IO_OPERATION_BEGIN": file_regex, 411 | "IO_OPERATION_COMPLETE": file_regex, 412 | "LEAVE": "read", 413 | }, 414 | # for some reason there is no print in pytest 415 | # "print": { 416 | # "ENTER": "read", 417 | # "IO_OPERATION_BEGIN": print_regex, 418 | # "IO_OPERATION_COMPLETE": print_regex, 419 | # "LEAVE": "read", 420 | # }, 421 | "close": {"ENTER": "close", "IO_DESTROY_HANDLE": file_regex, "LEAVE": "close"}, 422 | } 423 | 424 | io_trace = "" 425 | io_trace_after = "" 426 | in_expected_io = False 427 | after_expected_io = False 428 | 429 | for line in str(trace).split("\n"): 430 | if ("user_instrumenter:expect io" in line) and (in_expected_io is False): 431 | in_expected_io = True 432 | elif ("user_instrumenter:expect io" in line) and (in_expected_io is True): 433 | in_expected_io = False 434 | after_expected_io = True 435 | if in_expected_io: 436 | io_trace += line + "\n" 437 | if after_expected_io: 438 | io_trace_after += line + "\n" 439 | 440 | for _, details in ops.items(): 441 | for event, data in details.items(): 442 | regex_str = '{event:}[ ]*[0-9 ]*[0-9 ]*(Region|Handle): "{data:}"'.format( 443 | event=event, data=data 444 | ) 445 | print(regex_str) 446 | assert re.search(regex_str, io_trace) 447 | 448 | 449 | @foreach_instrumenter 450 | def test_force_finalize(scorep_env, instrumenter): 451 | trace_path = get_trace_path(scorep_env) 452 | 453 | # Also test when no instrumenter is given 454 | instrumenter_type = ["--instrumenter-type=" + instrumenter] 455 | std_out, std_err = utils.call_with_scorep( 456 | "cases/force_finalize.py", instrumenter_type, env=scorep_env 457 | ) 458 | 459 | assert std_err == "" 460 | assert std_out == "foo\nbar\n" 461 | 462 | trace = OTF2_Trace(trace_path) 463 | assert OTF2_Region("__main__:foo") in trace 464 | assert OTF2_Region("__main__:bar") not in trace 465 | -------------------------------------------------------------------------------- /test/test_subsystem.py: -------------------------------------------------------------------------------- 1 | import os 2 | from scorep import subsystem 3 | 4 | 5 | def test_reset_preload(monkeypatch): 6 | monkeypatch.setenv('LD_PRELOAD', '/some/value') 7 | # Nothing changes if the var is not present 8 | monkeypatch.delenv('SCOREP_LD_PRELOAD_BACKUP', raising=False) 9 | subsystem.reset_preload() 10 | assert os.environ['LD_PRELOAD'] == '/some/value' 11 | 12 | # Variable set -> Update 13 | monkeypatch.setenv('SCOREP_LD_PRELOAD_BACKUP', '/new/value') 14 | subsystem.reset_preload() 15 | assert os.environ['LD_PRELOAD'] == '/new/value' 16 | 17 | # Variable empty -> remove 18 | monkeypatch.setenv('SCOREP_LD_PRELOAD_BACKUP', '') 19 | subsystem.reset_preload() 20 | assert 'LD_PRELOAD' not in os.environ 21 | -------------------------------------------------------------------------------- /test/utils.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import subprocess 3 | import re 4 | 5 | 6 | def call(arguments, expected_returncode=0, env=None): 7 | """ 8 | Calls the command specificied by arguments and checks the returncode via assert 9 | return (stdout, stderr) from the call to subprocess 10 | """ 11 | if sys.version_info > (3, 5): 12 | out = subprocess.run( 13 | arguments, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE 14 | ) 15 | try: 16 | assert out.returncode == expected_returncode 17 | except AssertionError as e: 18 | e.args += ("stderr: {}".format(out.stderr.decode("utf-8")),) 19 | raise 20 | stdout, stderr = (out.stdout, out.stderr) 21 | else: 22 | p = subprocess.Popen( 23 | arguments, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE 24 | ) 25 | stdout, stderr = p.communicate() 26 | try: 27 | assert p.returncode == expected_returncode 28 | except AssertionError as e: 29 | e.args += ("stderr: {}".format(stderr.decode("utf-8")),) 30 | raise 31 | return stdout.decode("utf-8"), stderr.decode("utf-8") 32 | 33 | 34 | def call_with_scorep(file, scorep_arguments=None, expected_returncode=0, env=None): 35 | """ 36 | Shortcut for running a python file with the scorep module 37 | 38 | @return (stdout, stderr) from the call to subprocess 39 | """ 40 | arguments = [sys.executable, "-m", "scorep"] 41 | if scorep_arguments: 42 | arguments.extend(scorep_arguments) 43 | return call(arguments + [str(file)], expected_returncode=expected_returncode, env=env) 44 | 45 | 46 | def call_otf2_print(trace_path): 47 | trace, std_err = call(["otf2-print", str(trace_path)]) 48 | return trace, std_err 49 | 50 | 51 | class OTF2_Region: 52 | def __init__(self, region): 53 | self.region = region 54 | 55 | def __str__(self): 56 | return self.region 57 | 58 | 59 | class OTF2_Parameter: 60 | def __init__(self, parameter, value): 61 | self.parameter = parameter 62 | self.value = value 63 | 64 | def __str__(self): 65 | return "{}:{}".format(self.parameter, self.value) 66 | 67 | 68 | class OTF2_Trace: 69 | def __init__(self, trace_path): 70 | self.path = trace_path 71 | self.trace, self.std_err = call_otf2_print(self.path) 72 | assert self.std_err == "" 73 | 74 | def __contains__(self, otf2_element): 75 | result = [] 76 | if isinstance(otf2_element, OTF2_Region): 77 | for event in ("ENTER", "LEAVE"): 78 | search_str = "{event}[ ]*[0-9 ]*[0-9 ]*Region: \"{region}\"".format( 79 | event=event, region=otf2_element.region) 80 | search_res = re.search(search_str, self.trace) 81 | result.append(search_res is not None) 82 | elif isinstance(otf2_element, OTF2_Parameter): 83 | search_str = "PARAMETER_STRING[ ]*[0-9 ]*[0-9 ]*Parameter: \"{parameter}\" <[0-9]*>, Value: \"{value}\"" 84 | search_str = search_str.format(parameter=otf2_element.parameter, value=otf2_element.value) 85 | search_res = re.search(search_str, self.trace) 86 | result.append(search_res is not None) 87 | else: 88 | raise NotImplementedError 89 | return all(result) 90 | 91 | def findall(self, otf2_element): 92 | result = [] 93 | if isinstance(otf2_element, OTF2_Region): 94 | for event in ("ENTER", "LEAVE"): 95 | search_str = "{event}[ ]*[0-9 ]*[0-9 ]*Region: \"{region}\"".format( 96 | event=event, region=otf2_element.region) 97 | search_res = re.findall(search_str, self.trace) 98 | result.extend(search_res) 99 | else: 100 | raise NotImplementedError 101 | return result 102 | 103 | def __str__(self): 104 | return self.trace 105 | --------------------------------------------------------------------------------