├── test ├── __init__.py ├── low_level │ ├── __init__.py │ ├── util.py │ ├── test_custom_timer.py │ ├── test_setstatprofile.py │ ├── test_threaded.py │ └── test_context.py ├── util.py ├── conftest.py ├── test_threading.py ├── test_overflow.py ├── fake_time_util.py ├── test_stack_sampler.py ├── test_profiler.py ├── test_cmdline.py ├── test_profiler_async.py └── test_processors.py ├── joulehunter ├── vendor │ └── __init__.py ├── __init__.py ├── renderers │ ├── __init__.py │ ├── base.py │ ├── html.py │ ├── jsonrenderer.py │ └── console.py ├── typing.py ├── util.py ├── low_level │ └── stat_profile_python.py ├── middleware.py ├── energy.py ├── session.py ├── processors.py ├── stack_sampler.py ├── profiler.py └── frame.py ├── examples ├── django_example │ ├── .gitignore │ ├── django_example │ │ ├── __init__.py │ │ ├── templates │ │ │ ├── template_base.html │ │ │ └── template.html │ │ ├── urls.py │ │ ├── views.py │ │ └── settings.py │ ├── README.md │ └── manage.py ├── async_example_simple.py ├── np_c_function.py ├── async_experiment_1.py ├── async_experiment_3.py ├── c_sort.py ├── flask_hello.py ├── django_template_render.py └── wikipedia_article_word_count.py ├── bin ├── serve_docs ├── build_js_bundle.py └── bump_version.py ├── docs ├── img │ ├── screenshot.jpg │ └── async-context.svg ├── home.md ├── index.md ├── Makefile ├── conf.py ├── reference.md ├── guide.md └── how-it-works.md ├── html_renderer ├── src │ ├── assets │ │ └── favicon.png │ ├── main.js │ ├── appState.js │ ├── model │ │ ├── Group.js │ │ └── Frame.js │ ├── Header.vue │ ├── App.vue │ └── Frame.vue ├── .gitignore ├── public │ └── index.html ├── package.json └── vue.config.js ├── noxfile.py ├── MANIFEST.in ├── MAINTAINERS.md ├── pyproject.toml ├── .editorconfig ├── requirements-dev.txt ├── .gitignore ├── metrics ├── interrupt.py ├── overflow.py ├── overhead.py └── multi_overhead.py ├── setup.py ├── setup.cfg ├── .pre-commit-config.yaml ├── .github └── workflows │ ├── test.yaml │ ├── python-publish.yml │ └── python-publish-test.yaml ├── CITATION.cff ├── LICENSE └── README.md /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/low_level/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /joulehunter/vendor/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/django_example/.gitignore: -------------------------------------------------------------------------------- 1 | db.sqlite3 2 | -------------------------------------------------------------------------------- /examples/django_example/django_example/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bin/serve_docs: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd "$(dirname "$0")" 4 | cd .. 5 | 6 | exec make -C docs livehtml 7 | -------------------------------------------------------------------------------- /docs/img/screenshot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerapi-ng/joulehunter/HEAD/docs/img/screenshot.jpg -------------------------------------------------------------------------------- /examples/django_example/README.md: -------------------------------------------------------------------------------- 1 | 2 | This is a simple simple test rig to develop joulehunter's Django middleware 3 | -------------------------------------------------------------------------------- /html_renderer/src/assets/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerapi-ng/joulehunter/HEAD/html_renderer/src/assets/favicon.png -------------------------------------------------------------------------------- /examples/django_example/django_example/templates/template_base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {% block content %} 6 | {% endblock %} 7 | 8 | 9 | -------------------------------------------------------------------------------- /html_renderer/src/main.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import App from './App.vue' 3 | 4 | Vue.config.productionTip = false 5 | 6 | new Vue({ 7 | render: h => h(App) 8 | }).$mount('#app') 9 | -------------------------------------------------------------------------------- /noxfile.py: -------------------------------------------------------------------------------- 1 | import nox 2 | 3 | 4 | @nox.session(python=["3.7", "3.8", "3.9"]) 5 | def test(session): 6 | session.install("-r", "requirements-dev.txt") 7 | session.run("pytest") 8 | -------------------------------------------------------------------------------- /examples/django_example/django_example/templates/template.html: -------------------------------------------------------------------------------- 1 | {% extends "template_base.html" %} 2 | 3 | {% block content %} 4 | {% spaceless %} 5 | something 6 | {% endspaceless %} 7 | {% endblock content %} 8 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include LICENSE 3 | recursive-include joulehunter *.js 4 | recursive-include html_renderer *.html *.json *.js *.ts *.vue *.css *.png 5 | prune html_renderer/node_modules 6 | prune html_renderer/dist 7 | -------------------------------------------------------------------------------- /joulehunter/__init__.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | from joulehunter.profiler import Profiler 4 | 5 | __version__ = "v1.0.2" 6 | 7 | # enable deprecation warnings 8 | warnings.filterwarnings("once", ".*", DeprecationWarning, r"joulehunter\..*") 9 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # Releasing 2 | 3 | ``` 4 | bump2version 5 | # or, bump2version --new-version x.x.x 6 | git push && git push --tags 7 | ``` 8 | 9 | Deployment to PyPI is performed in GitHub Actions. 10 | -------------------------------------------------------------------------------- /joulehunter/renderers/__init__.py: -------------------------------------------------------------------------------- 1 | from joulehunter.renderers.base import Renderer 2 | from joulehunter.renderers.console import ConsoleRenderer 3 | from joulehunter.renderers.html import HTMLRenderer 4 | from joulehunter.renderers.jsonrenderer import JSONRenderer 5 | -------------------------------------------------------------------------------- /examples/django_example/django_example/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | from django.contrib import admin 3 | 4 | from . import views 5 | 6 | urlpatterns = [ 7 | url("admin/", admin.site.urls), 8 | url(r"^$", views.hello_world), 9 | ] 10 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 100 3 | 4 | [tool.pyright] 5 | include = ["joulehunter", "test"] 6 | ignore = ["joulehunter/vendor"] 7 | pythonVersion = "3.7" 8 | 9 | [tool.isort] 10 | profile = "black" 11 | multi_line_output = 3 12 | line_length = 100 13 | -------------------------------------------------------------------------------- /docs/home.md: -------------------------------------------------------------------------------- 1 | --- 2 | html_meta: 3 | title: Home 4 | hide-toc: 5 | --- 6 | 7 | # joulehunter 8 | 9 | ```{include} ../README.md 10 | --- 11 | relative-docs: docs/ 12 | relative-images: 13 | start-after: '' 14 | end-before: '' 15 | --- 16 | ``` 17 | -------------------------------------------------------------------------------- /examples/async_example_simple.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from joulehunter import Profiler 4 | 5 | 6 | async def main(): 7 | p = Profiler() 8 | with p: 9 | print("Hello ...") 10 | await asyncio.sleep(1) 11 | print("... World!") 12 | p.print() 13 | 14 | 15 | asyncio.run(main()) 16 | -------------------------------------------------------------------------------- /examples/django_example/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_example.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /html_renderer/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | /dist 4 | 5 | # local env files 6 | .env.local 7 | .env.*.local 8 | 9 | # Log files 10 | npm-debug.log* 11 | yarn-debug.log* 12 | yarn-error.log* 13 | 14 | # Editor directories and files 15 | .idea 16 | .vscode 17 | *.suo 18 | *.ntvs* 19 | *.njsproj 20 | *.sln 21 | *.sw* 22 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: https://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | [*] 7 | indent_style = space 8 | indent_size = 4 9 | end_of_line = lf 10 | charset = utf-8 11 | trim_trailing_whitespace = true 12 | insert_final_newline = true 13 | 14 | [html_renderer/**] 15 | indent_size = 2 16 | -------------------------------------------------------------------------------- /examples/django_example/django_example/views.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from django.http import HttpResponse 4 | 5 | 6 | def hello_world(request): 7 | # do some useless work to delay this call a bit 8 | y = 1 9 | for x in range(1, 10000): 10 | y *= x 11 | time.sleep(0.1) 12 | 13 | return HttpResponse("Hello, world!") 14 | -------------------------------------------------------------------------------- /html_renderer/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | joulehunter dev server 6 | 7 | 8 |
9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | pytest 3 | pytest-mock 4 | flaky 5 | trio 6 | flake8 7 | django # used by middleware 8 | sphinx==4.0.2 9 | myst-parser==0.15.1 10 | furo==2021.6.18b36 11 | sphinxcontrib-programoutput==0.17 12 | pytest-asyncio==0.12.0 # pinned to an older version due to an incompatibility with flaky 13 | greenlet # used by a test 14 | nox 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # virtualenv 2 | env/ 3 | env2/ 4 | env3*/ 5 | .Python 6 | /env 7 | /venv 8 | 9 | # python 10 | *.pyc 11 | __pycache__/ 12 | 13 | # C extensions 14 | *.so 15 | 16 | # distribution 17 | dist/ 18 | *.egg-info/ 19 | build 20 | .eggs 21 | 22 | # testing 23 | .cache 24 | .pytest_cache 25 | 26 | # editor 27 | .vscode 28 | 29 | # docs 30 | docs/_build 31 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | joulehunter 2 | ============ 3 | 4 | ```{toctree} 5 | --- 6 | maxdepth: 2 7 | caption: "Contents" 8 | --- 9 | Home 10 | guide.md 11 | how-it-works.md 12 | reference.md 13 | GitHub 14 | ``` 15 | 16 | Indices and tables 17 | ------------------ 18 | 19 | * {ref}`genindex` 20 | * {ref}`search` 21 | -------------------------------------------------------------------------------- /examples/np_c_function.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import numpy as np 4 | 5 | arr = np.random.randint(0, 10000, 10000) 6 | 7 | # def print_profiler(frame, event, arg): 8 | # print(event, arg, getattr(arg, '__qualname__', arg.__name__), arg.__module__, dir(arg)) 9 | 10 | # sys.setprofile(print_profiler) 11 | 12 | for i in range(10000): 13 | arr.cumsum() 14 | 15 | # sys.setprofile(None) 16 | -------------------------------------------------------------------------------- /html_renderer/src/appState.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue'; 2 | 3 | class AppState { 4 | constructor() { 5 | this.visibleGroups = {} 6 | } 7 | 8 | isGroupVisible(group) { 9 | return this.visibleGroups[group.id] === true; 10 | } 11 | 12 | setGroupVisibility(group, visible) { 13 | Vue.set(this.visibleGroups, group.id, visible); 14 | } 15 | } 16 | 17 | const appState = new AppState() 18 | export default appState; 19 | -------------------------------------------------------------------------------- /joulehunter/typing.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import TYPE_CHECKING, Union 3 | 4 | if TYPE_CHECKING: 5 | from typing_extensions import Literal 6 | 7 | LiteralStr = Literal 8 | else: 9 | # a type, that when subscripted, returns `str`. 10 | class _LiteralStr: 11 | def __getitem__(self, values): 12 | return str 13 | 14 | LiteralStr = _LiteralStr() 15 | 16 | PathOrStr = Union[str, "os.PathLike[str]"] 17 | -------------------------------------------------------------------------------- /metrics/interrupt.py: -------------------------------------------------------------------------------- 1 | from platform import platform 2 | 3 | from joulehunter import Profiler 4 | 5 | p = Profiler() 6 | 7 | p.start() 8 | 9 | 10 | def func(): 11 | fd = open("/dev/urandom", "rb") 12 | _ = fd.read(1024 * 1024) 13 | 14 | 15 | func() 16 | 17 | # this failed on ubuntu 12.04 18 | platform() 19 | 20 | p.stop() 21 | 22 | print(p.output_text()) 23 | 24 | with open("ioerror_out.html", "w") as f: 25 | f.write(p.output_html()) 26 | -------------------------------------------------------------------------------- /metrics/overflow.py: -------------------------------------------------------------------------------- 1 | from joulehunter import Profiler 2 | 3 | p = Profiler(use_signal=False) 4 | 5 | p.start() 6 | 7 | 8 | def func(num): 9 | if num == 0: 10 | return 11 | b = 0 12 | for x in range(1, 100000): 13 | b += x 14 | 15 | return func(num - 1) 16 | 17 | 18 | func(900) 19 | 20 | p.stop() 21 | 22 | print(p.output_text()) 23 | 24 | with open("overflow_out.html", "w") as f: 25 | f.write(p.output_html()) 26 | -------------------------------------------------------------------------------- /test/low_level/util.py: -------------------------------------------------------------------------------- 1 | import functools 2 | 3 | import pytest 4 | 5 | from joulehunter.low_level.stat_profile import setstatprofile as setstatprofile_c 6 | from joulehunter.low_level.stat_profile_python import setstatprofile as setstatprofile_python 7 | 8 | """ 9 | Parametrizes the test with both the C and Python setstatprofile, just to check 10 | that the Python one is up-to-date with the C version. 11 | """ 12 | parametrize_setstatprofile = pytest.mark.parametrize( 13 | "setstatprofile", 14 | [setstatprofile_c, setstatprofile_python], 15 | ) 16 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import distutils 2 | import os 3 | import subprocess 4 | import sys 5 | from setuptools import Extension, find_packages, setup 6 | 7 | setup( 8 | name="joulehunter", 9 | packages=find_packages(), 10 | ext_modules=[ 11 | Extension( 12 | "joulehunter.low_level.stat_profile", 13 | sources=["joulehunter/low_level/stat_profile.c"], 14 | ) 15 | ], 16 | keywords=["profiling", "profile", "profiler", 17 | "energy", "cpu", "time", "sampling"], 18 | install_requires=[], 19 | include_package_data=True, 20 | python_requires=">=3.7", 21 | entry_points={"console_scripts": [ 22 | "joulehunter = joulehunter.__main__:main"]}, 23 | zip_safe=False, 24 | 25 | ) 26 | -------------------------------------------------------------------------------- /html_renderer/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "html_js", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "serve": "vue-cli-service serve", 7 | "build": "vue-cli-service build", 8 | "lint": "vue-cli-service lint" 9 | }, 10 | "dependencies": { 11 | "html-webpack-inline-source-plugin": "0.0.10", 12 | "vue": "^2.6.12" 13 | }, 14 | "devDependencies": { 15 | "@vue/cli-plugin-babel": "^4.5.11", 16 | "@vue/cli-service": "^4.5.11", 17 | "vue-template-compiler": "^2.6.12" 18 | }, 19 | "postcss": { 20 | "plugins": { 21 | "autoprefixer": {} 22 | } 23 | }, 24 | "browserslist": [ 25 | "> 5%", 26 | "not ie <= 8" 27 | ], 28 | "babel": { 29 | "presets": [ 30 | "@vue/app" 31 | ] 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /test/low_level/test_custom_timer.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from .util import parametrize_setstatprofile 4 | 5 | 6 | class CallCounter: 7 | def __init__(self) -> None: 8 | self.count = 0 9 | 10 | def __call__(self, *args: Any, **kwds: Any) -> Any: 11 | self.count += 1 12 | 13 | 14 | @parametrize_setstatprofile 15 | def test_increment(setstatprofile): 16 | time = 0.0 17 | 18 | def fake_time(): 19 | return time 20 | 21 | def fake_sleep(duration): 22 | nonlocal time 23 | time += duration 24 | 25 | counter = CallCounter() 26 | 27 | setstatprofile(counter, timer_func=fake_time) 28 | 29 | for _ in range(100): 30 | fake_sleep(1.0) 31 | 32 | setstatprofile(None) 33 | 34 | assert counter.count == 100 35 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile livehtml 16 | 17 | livehtml: 18 | sphinx-autobuild -a --watch ../joulehunter "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 19 | 20 | # Catch-all target: route all unknown targets to Sphinx using the new 21 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 22 | %: Makefile 23 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 24 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test = pytest 3 | 4 | [tool:pytest] 5 | testpaths = test 6 | 7 | [metadata] 8 | name = joulehunter 9 | version = 1.0.2 10 | author = chakib belgaid 11 | author_email = chakib.belgaid@gmail.com 12 | description = detect where your code consumes energy the most so you can optimize those functions 13 | long_description = file: README.md 14 | long_description_content_type = text/markdown 15 | url = https://github.com/powerapi-ng/joulehunter 16 | project_urls = 17 | Bug Tracker = https://github.com/powerapi-ng/joulehunter/issues 18 | classifiers = 19 | Environment :: Console 20 | Environment :: Web Environment 21 | Intended Audience :: Developers 22 | License :: OSI Approved :: BSD License 23 | Operating System :: POSIX 24 | Topic :: Software Development :: Debuggers 25 | Topic :: Software Development :: Testing 26 | 27 | -------------------------------------------------------------------------------- /html_renderer/vue.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | configureWebpack: { 3 | optimization: { 4 | splitChunks: false 5 | }, 6 | }, 7 | filenameHashing: false, 8 | css: { 9 | extract: false, 10 | }, 11 | chainWebpack: config => { 12 | config.module 13 | .rule('images') 14 | .use('url-loader') 15 | .loader('url-loader') 16 | .tap(options => { 17 | // inline everything 18 | options.limit = undefined; 19 | return options 20 | }) 21 | config.module 22 | .rule('svg') 23 | .use('url-loader') 24 | .loader('url-loader') 25 | .tap(options => { 26 | options = options || {} 27 | // inline everything 28 | options.limit = undefined; 29 | return options 30 | }) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /examples/async_experiment_1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import time 3 | 4 | import joulehunter 5 | 6 | 7 | def do_nothing(): 8 | pass 9 | 10 | 11 | def busy_wait(duration): 12 | end_time = time.time() + duration 13 | 14 | while time.time() < end_time: 15 | do_nothing() 16 | 17 | 18 | async def say(what, when, profile=False): 19 | if profile: 20 | p = joulehunter.Profiler() 21 | p.start() 22 | 23 | busy_wait(0.1) 24 | sleep_start = time.time() 25 | await asyncio.sleep(when) 26 | print(f"slept for {time.time() - sleep_start:.3f} seconds") 27 | busy_wait(0.1) 28 | 29 | print(what) 30 | if profile: 31 | p.stop() 32 | p.print(show_all=True) 33 | 34 | 35 | loop = asyncio.get_event_loop() 36 | 37 | loop.create_task(say("first hello", 2, profile=True)) 38 | loop.create_task(say("second hello", 1, profile=True)) 39 | loop.create_task(say("third hello", 3, profile=True)) 40 | 41 | loop.run_forever() 42 | loop.close() 43 | -------------------------------------------------------------------------------- /examples/async_experiment_3.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import time 3 | 4 | import trio 5 | 6 | import joulehunter 7 | 8 | 9 | def do_nothing(): 10 | pass 11 | 12 | 13 | def busy_wait(duration): 14 | end_time = time.time() + duration 15 | 16 | while time.time() < end_time: 17 | do_nothing() 18 | 19 | 20 | async def say(what, when, profile=False): 21 | if profile: 22 | p = joulehunter.Profiler() 23 | p.start() 24 | 25 | busy_wait(0.1) 26 | sleep_start = time.time() 27 | await trio.sleep(when) 28 | print(f"slept for {time.time() - sleep_start:.3f} seconds") 29 | busy_wait(0.1) 30 | 31 | print(what) 32 | if profile: 33 | p.stop() 34 | p.print(show_all=True) 35 | 36 | 37 | async def task(): 38 | async with trio.open_nursery() as nursery: 39 | nursery.start_soon(say, "first hello", 2, True) 40 | nursery.start_soon(say, "second hello", 1, True) 41 | nursery.start_soon(say, "third hello", 3, True) 42 | 43 | 44 | trio.run(task) 45 | -------------------------------------------------------------------------------- /examples/c_sort.py: -------------------------------------------------------------------------------- 1 | """ 2 | list.sort is interesting in that it calls a C function, that calls back to a 3 | Python function. In an ideal world, we'd be able to record the time inside the 4 | Python function _inside_ list.sort, but it's not possible currently, due to 5 | the way that Python records frame objects. 6 | 7 | Perhaps one day we could add some functionality to joulehunter_cext to keep 8 | a parallel stack containing both C and Python frames. But for now, this is 9 | fine. 10 | """ 11 | import sys 12 | import time 13 | 14 | import numpy as np 15 | 16 | arr = np.random.randint(0, 10, 10) 17 | 18 | # def print_profiler(frame, event, arg): 19 | # if event.startswith('c_'): 20 | # print(event, arg, getattr(arg, '__qualname__', arg.__name__), arg.__module__) 21 | # else: 22 | # print(event, frame.f_code.co_name) 23 | 24 | # sys.setprofile(print_profiler) 25 | 26 | 27 | def slow_key(el): 28 | time.sleep(0.01) 29 | return 0 30 | 31 | 32 | for i in range(10): 33 | list(arr).sort(key=slow_key) 34 | 35 | # sys.setprofile(None) 36 | -------------------------------------------------------------------------------- /examples/flask_hello.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from joulehunter import Profiler 4 | 5 | try: 6 | from flask import Flask, g, make_response, request 7 | except ImportError: 8 | print("This example requires Flask.") 9 | print("Install using `pip install flask`.") 10 | exit(1) 11 | 12 | app = Flask(__name__) 13 | 14 | 15 | @app.before_request 16 | def before_request(): 17 | if "profile" in request.args: 18 | g.profiler = Profiler() 19 | g.profiler.start() 20 | 21 | 22 | @app.after_request 23 | def after_request(response): 24 | if not hasattr(g, "profiler"): 25 | return response 26 | g.profiler.stop() 27 | output_html = g.profiler.output_html() 28 | return make_response(output_html) 29 | 30 | 31 | @app.route("/") 32 | def hello_world(): 33 | return "Hello, World!" 34 | 35 | 36 | @app.route("/sleep") 37 | def sleep(): 38 | time.sleep(0.1) 39 | return "Good morning!" 40 | 41 | 42 | @app.route("/dosomething") 43 | def do_something(): 44 | import requests 45 | 46 | requests.get("http://google.com") 47 | return "Google says hello!" 48 | -------------------------------------------------------------------------------- /html_renderer/src/model/Group.js: -------------------------------------------------------------------------------- 1 | export default class Group { 2 | frames = [] 3 | 4 | constructor(id, rootFrame) { 5 | this.id = id; 6 | this.rootFrame = rootFrame; 7 | } 8 | 9 | addFrame(frame) { 10 | this.frames.push(frame); 11 | } 12 | 13 | get exitFrames() { 14 | // exit frames are frames inside this group that have children outside the group. 15 | const exitFrames = [] 16 | 17 | for (const frame of this.frames) { 18 | let isExit = false; 19 | for (const child of frame.children) { 20 | if (child.group != this) { 21 | isExit = true; 22 | break; 23 | } 24 | } 25 | 26 | if (isExit) { 27 | exitFrames.push(frame); 28 | } 29 | } 30 | 31 | return exitFrames; 32 | } 33 | 34 | get libraries() { 35 | const libraries = []; 36 | 37 | for (const frame of this.frames) { 38 | const library = /^[^\\/.]*/.exec(frame.filePathShort)[0] 39 | if (!libraries.includes(library)) { 40 | libraries.push(library); 41 | } 42 | } 43 | 44 | return libraries; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v3.2.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-yaml 8 | - id: check-added-large-files 9 | 10 | - repo: https://github.com/pycqa/isort 11 | rev: 5.8.0 12 | hooks: 13 | - id: isort 14 | name: isort (python) 15 | 16 | - repo: https://github.com/psf/black 17 | rev: 20.8b1 18 | hooks: 19 | - id: black 20 | language_version: python3 21 | 22 | - repo: local 23 | hooks: 24 | - id: pyright 25 | name: pyright 26 | entry: pyright --venv-path . 27 | language: node 28 | pass_filenames: false 29 | types: [python] 30 | additional_dependencies: ['pyright@1.1.134'] 31 | - id: build 32 | name: build js bundle 33 | entry: bin/build_js_bundle.py --force 34 | files: html_renderer/.* 35 | language: node 36 | pass_filenames: false 37 | 38 | exclude: ^joulehunter/renderers/html_resources/app.js$|^joulehunter/vendor 39 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: Unit tests 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | python-version: [3.7, 3.8, 3.9] 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v2 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install --upgrade pip 26 | pip install -r requirements-dev.txt 27 | - name: Lint with flake8 28 | run: | 29 | # stop the build if there are Python syntax errors or undefined names 30 | flake8 . --count --select=E9,F63,F7,F5 --show-source --statistics 31 | - name: Test with pytest 32 | run: | 33 | pytest 34 | -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | name: Upload JouleHunter to PypI 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | build_sdist: 9 | name: Build source distribution 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - uses: actions/setup-python@v2 14 | name: Install Python 15 | with: 16 | python-version: "3.8" 17 | 18 | - name: Build sdist 19 | run: python setup.py sdist 20 | - uses: actions/upload-artifact@v2 21 | with: 22 | path: dist/*.tar.gz 23 | 24 | publish: 25 | name: Build and publish Publish JouleHunter to PyPI 26 | runs-on: ubuntu-latest 27 | needs: [build_sdist] 28 | steps: 29 | - uses: actions/download-artifact@v2 30 | with: 31 | name: artifact 32 | path: dist 33 | 34 | - name: Publish distribution 📦 to PyPI 35 | uses: pypa/gh-action-pypi-publish@master 36 | with: 37 | password: ${{ secrets.PYPI_API_TOKEN }} 38 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | 2 | cff-version: 1.2.0 3 | title: "JouleHunter : an energy profiler for python applications" 4 | message: Make your python code green again 5 | type: software 6 | date-released: 2021-09-15 7 | authors: 8 | - given-names: Mohammed chakib 9 | family-names: Belgaid 10 | email: chakib.belgaid@gmail.com 11 | orcid: 'https://orcid.org/0000-0002-5264-7426' 12 | affiliation: Inria university of Lille 13 | - given-names: Alex 14 | family-names: Kaminetzky 15 | - given-names: Romain 16 | family-names: Rouvoy 17 | email: romain.rouvoy@inria.fr 18 | affiliation: inria university of lille 19 | orcid: 'https://orcid.org/0000-0003-1771-8791' 20 | - orcid: 'https://orcid.org/0000-0003-0006-6088' 21 | affiliation: 'Inria university of lille ' 22 | email: lionel.seinturier@univ-lille.fr 23 | family-names: Seinturier 24 | given-names: Lionel 25 | identifiers: 26 | - type: url 27 | value: https://github.com/powerapi-ng/joulehunter/ 28 | repository-code: 'https://github.com/powerapi-ng/joulehunter/' 29 | abstract: >- 30 | Joulehunter helps you find what part of your code is consuming considerable amounts of energy within python applications. 31 | -------------------------------------------------------------------------------- /examples/django_template_render.py: -------------------------------------------------------------------------------- 1 | import os 2 | from optparse import OptionParser 3 | 4 | try: 5 | import django 6 | except ImportError: 7 | print("This example requires Django.") 8 | print("Install using `pip install Django`.") 9 | exit(1) 10 | 11 | import django.conf 12 | import django.template.loader 13 | 14 | 15 | def main(): 16 | parser = OptionParser() 17 | parser.add_option( 18 | "-i", 19 | "--iterations", 20 | dest="iterations", 21 | action="store", 22 | type="int", 23 | help="number of template render calls to make", 24 | default=100, 25 | ) 26 | options, _ = parser.parse_args() 27 | 28 | os.chdir(os.path.dirname(__file__)) 29 | 30 | django.conf.settings.configure( 31 | INSTALLED_APPS=(), 32 | TEMPLATES=[ 33 | { 34 | "BACKEND": "django.template.backends.django.DjangoTemplates", 35 | "DIRS": ["./django_example/django_example/templates"], 36 | } 37 | ], 38 | ) 39 | django.setup() 40 | 41 | for _ in range(0, options.iterations): 42 | django.template.loader.render_to_string("template.html") 43 | 44 | 45 | if __name__ == "__main__": 46 | main() 47 | -------------------------------------------------------------------------------- /examples/wikipedia_article_word_count.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | try: 4 | from urllib.request import urlopen 5 | except ImportError: 6 | from urllib2 import urlopen 7 | 8 | import collections 9 | import operator 10 | import sys 11 | 12 | WIKIPEDIA_ARTICLE_API_URL = "https://en.wikipedia.org/w/api.php?action=query&titles=Spoon&prop=revisions&rvprop=content&format=json" 13 | 14 | 15 | def download(): 16 | return urlopen(WIKIPEDIA_ARTICLE_API_URL).read() 17 | 18 | 19 | def parse(json_data): 20 | return json.loads(json_data) 21 | 22 | 23 | def most_common_words(page): 24 | word_occurences = collections.defaultdict(int) 25 | 26 | for revision in page["revisions"]: 27 | article = revision["*"] 28 | 29 | for word in article.split(): 30 | if len(word) < 2: 31 | continue 32 | word_occurences[word] += 1 33 | 34 | word_list = sorted(word_occurences.items(), key=operator.itemgetter(1), reverse=True) 35 | 36 | return word_list[0:5] 37 | 38 | 39 | def main(): 40 | data = parse(download()) 41 | page = list(data["query"]["pages"].values())[0] 42 | 43 | sys.stderr.write("This most common words were %s\n" % most_common_words(page)) 44 | 45 | 46 | if __name__ == "__main__": 47 | main() 48 | -------------------------------------------------------------------------------- /test/util.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import os 3 | import time 4 | from typing import Generator, Generic, Iterable, Iterator, NoReturn, Optional, TypeVar 5 | 6 | import trio 7 | from flaky import flaky 8 | 9 | from joulehunter.frame import BaseFrame 10 | from joulehunter.profiler import Profiler 11 | 12 | if "CI" in os.environ: 13 | # a decorator that allows some test flakyness in CI environments, presumably 14 | # due to contention. Useful for tests that rely on real time measurments. 15 | flaky_in_ci = flaky(max_runs=5, min_passes=1) 16 | else: 17 | flaky_in_ci = lambda a: a 18 | 19 | 20 | def assert_never(x: NoReturn) -> NoReturn: 21 | raise AssertionError(f"Invalid value: {x!r}") 22 | 23 | 24 | def do_nothing(): 25 | pass 26 | 27 | 28 | def busy_wait(duration): 29 | end_time = time.time() + duration 30 | 31 | while time.time() < end_time: 32 | do_nothing() 33 | 34 | 35 | def walk_frames(frame: BaseFrame) -> Generator[BaseFrame, None, None]: 36 | yield frame 37 | 38 | for f in frame.children: 39 | yield from walk_frames(f) 40 | 41 | 42 | T = TypeVar("T") 43 | 44 | 45 | def first(iterator: Iterator[T]) -> Optional[T]: 46 | try: 47 | return next(iterator) 48 | except StopIteration: 49 | return None 50 | -------------------------------------------------------------------------------- /.github/workflows/python-publish-test.yaml: -------------------------------------------------------------------------------- 1 | name: Publish JouleHunter to TestPyPI 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | 7 | jobs: 8 | build_sdist: 9 | name: Build source distribution 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | 14 | - uses: actions/setup-python@v2 15 | name: Install Python 16 | with: 17 | python-version: "3.8" 18 | 19 | - name: Build sdist 20 | run: python setup.py sdist 21 | 22 | - uses: actions/upload-artifact@v2 23 | with: 24 | path: dist/*.tar.gz 25 | publish: 26 | name: Build and publish Publish JouleHunter to TestPyPI 27 | runs-on: ubuntu-latest 28 | needs: [build_sdist] 29 | steps: 30 | - uses: actions/download-artifact@v2 31 | with: 32 | name: artifact 33 | path: dist 34 | 35 | - name: Publish distribution 📦 to Test PyPI 36 | uses: pypa/gh-action-pypi-publish@master 37 | with: 38 | password: ${{ secrets.TEST_PYPI_API_TOKEN }} 39 | repository_url: https://test.pypi.org/legacy/ 40 | -------------------------------------------------------------------------------- /test/conftest.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import pytest 4 | 5 | from joulehunter import stack_sampler, Profiler 6 | from _pytest.monkeypatch import MonkeyPatch 7 | 8 | 9 | @pytest.fixture(autouse=True) 10 | def check_sampler_state(): 11 | assert sys.getprofile() is None 12 | assert len(stack_sampler.get_stack_sampler().subscribers) == 0 13 | 14 | try: 15 | yield 16 | assert sys.getprofile() is None 17 | assert len(stack_sampler.get_stack_sampler().subscribers) == 0 18 | finally: 19 | sys.setprofile(None) 20 | stack_sampler.thread_locals.__dict__.clear() 21 | 22 | 23 | def current_energy_generator(): 24 | i = 0 25 | while True: 26 | i += 1 27 | yield i 28 | 29 | 30 | def current_energy(): 31 | return next(current_energy_generator.generator) 32 | 33 | 34 | def new_init(self, async_mode="disabled"): 35 | self._interval = 0.001 36 | self._last_session = None 37 | self._active_session = None 38 | self._async_mode = async_mode 39 | self.current_energy = current_energy 40 | self.domain_names = ["0", "mockup"] 41 | 42 | 43 | @pytest.fixture(autouse=True) 44 | def mock_profiler(): 45 | MonkeyPatch().setattr(Profiler, "__init__", new_init) 46 | myprofiler = Profiler() 47 | return myprofiler 48 | -------------------------------------------------------------------------------- /test/test_threading.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time 3 | from test.fake_time_util import fake_time 4 | 5 | import pytest 6 | 7 | from joulehunter import Profiler 8 | 9 | from .util import do_nothing 10 | 11 | 12 | def test_profiler_access_from_multiple_threads(): 13 | profiler = Profiler() 14 | 15 | profiler.start() 16 | 17 | thread_exception = None 18 | 19 | def helper(): 20 | while profiler._active_session and len(profiler._active_session.frame_records) < 10: 21 | time.sleep(0.0001) 22 | 23 | try: 24 | profiler.stop() 25 | except Exception as e: 26 | nonlocal thread_exception 27 | thread_exception = e 28 | 29 | t1 = threading.Thread(target=helper) 30 | t1.start() 31 | 32 | while t1.is_alive(): 33 | do_nothing() 34 | t1.join() 35 | 36 | with pytest.raises(Exception) as excinfo: 37 | profiler.output_html() 38 | 39 | assert "this profiler is still running" in excinfo.value.args[0] 40 | 41 | assert thread_exception is not None 42 | assert ( 43 | "Failed to stop profiling. Make sure that you start/stop profiling on the same thread." 44 | in thread_exception.args[0] 45 | ) 46 | 47 | # the above stop failed. actually stop the profiler 48 | profiler.stop() 49 | -------------------------------------------------------------------------------- /test/low_level/test_setstatprofile.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | from typing import Any 4 | 5 | import pytest 6 | 7 | from ..util import busy_wait, flaky_in_ci 8 | from .util import parametrize_setstatprofile 9 | 10 | 11 | class CallCounter: 12 | def __init__(self) -> None: 13 | self.count = 0 14 | 15 | def __call__(self, *args: Any, **kwds: Any) -> Any: 16 | self.count += 1 17 | 18 | 19 | @flaky_in_ci 20 | @parametrize_setstatprofile 21 | def test_100ms(setstatprofile): 22 | counter = CallCounter() 23 | setstatprofile(counter, 0.1) 24 | busy_wait(1.0) 25 | setstatprofile(None) 26 | assert 8 < counter.count < 12 27 | 28 | 29 | @flaky_in_ci 30 | @parametrize_setstatprofile 31 | def test_10ms(setstatprofile): 32 | counter = CallCounter() 33 | setstatprofile(counter, 0.01) 34 | busy_wait(1.0) 35 | setstatprofile(None) 36 | assert 70 <= counter.count <= 130 37 | 38 | 39 | @parametrize_setstatprofile 40 | def test_internal_object_compatibility(setstatprofile): 41 | setstatprofile(CallCounter(), 1e6) 42 | 43 | profile_state = sys.getprofile() 44 | 45 | print(repr(profile_state)) 46 | print(str(profile_state)) 47 | print(profile_state) 48 | print(type(profile_state)) 49 | print(type(profile_state).__name__) 50 | 51 | setstatprofile(None) 52 | -------------------------------------------------------------------------------- /html_renderer/src/model/Frame.js: -------------------------------------------------------------------------------- 1 | import Group from './Group'; 2 | 3 | export default class Frame { 4 | constructor(jsonObject, parent = null, context = {groups:{}}) { 5 | this.parent = parent; 6 | this.function = jsonObject.function; 7 | this.filePathShort = jsonObject.file_path_short; 8 | this.filePath = jsonObject.file_path; 9 | this.lineNo = jsonObject.line_no; 10 | this.time = jsonObject.time; 11 | this.isApplicationCode = jsonObject.is_application_code 12 | 13 | if (jsonObject.group_id) { 14 | const groupId = jsonObject.group_id; 15 | let group = context.groups[groupId] 16 | if (!group) { 17 | group = context.groups[groupId] = new Group(groupId, this); 18 | } 19 | group.addFrame(this); 20 | this.group = context.groups[groupId]; 21 | } else { 22 | this.group = null; 23 | } 24 | 25 | this.children = jsonObject.children.map(f => new Frame(f, this, context)); 26 | } 27 | 28 | get proportionOfTotal() { 29 | if (this.parent) { 30 | return this.parent.proportionOfTotal * this.proportionOfParent; 31 | } else { 32 | return 1.0; 33 | } 34 | } 35 | 36 | get proportionOfParent() { 37 | if (this.parent) { 38 | return this.time / this.parent.time; 39 | } else { 40 | return 1.0; 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /test/test_overflow.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import sys 3 | import time 4 | 5 | import pytest 6 | 7 | from joulehunter import Profiler 8 | from joulehunter.renderers import ConsoleRenderer, HTMLRenderer, JSONRenderer 9 | 10 | # Utilities 11 | 12 | 13 | def recurse(depth): 14 | if depth == 0: 15 | time.sleep(0.1) 16 | return 17 | 18 | recurse(depth - 1) 19 | 20 | 21 | def current_stack_depth(): 22 | depth = 0 23 | frame = inspect.currentframe() 24 | while frame: 25 | frame = frame.f_back 26 | depth += 1 27 | return depth 28 | 29 | 30 | # Fixtures 31 | 32 | 33 | @pytest.fixture() 34 | # @pytest.mark.usefixtures('mock_profiler') 35 | def deep_profiler_session(): 36 | profiler = Profiler() 37 | profiler.start() 38 | 39 | # give 120 frames for joulehunter to do its work. 40 | recursion_depth = sys.getrecursionlimit() - current_stack_depth() - 120 41 | recurse(recursion_depth) 42 | 43 | profiler.stop() 44 | return profiler.last_session 45 | 46 | 47 | # Tests 48 | 49 | 50 | def test_console(deep_profiler_session): 51 | ConsoleRenderer().render(deep_profiler_session) 52 | 53 | 54 | # html now uses the json renderer, so it's xfail too. 55 | def test_html(deep_profiler_session): 56 | HTMLRenderer().render(deep_profiler_session) 57 | 58 | 59 | def test_json(deep_profiler_session): 60 | JSONRenderer().render(deep_profiler_session) 61 | -------------------------------------------------------------------------------- /test/low_level/test_threaded.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import threading 4 | import time 5 | from typing import Any, List 6 | from unittest import TestCase 7 | 8 | import pytest 9 | 10 | from joulehunter.low_level.stat_profile import setstatprofile 11 | 12 | from ..util import busy_wait, do_nothing 13 | 14 | 15 | class CallCounter: 16 | def __init__(self, thread) -> None: 17 | self.thread = thread 18 | self.count = 0 19 | 20 | def __call__(self, *args: Any, **kwds: Any) -> Any: 21 | assert self.thread is threading.current_thread() 22 | self.count += 1 23 | 24 | 25 | def test_threaded(): 26 | # assert that each thread gets its own callbacks, and check that it 27 | # doesn't crash! 28 | 29 | counters: list[CallCounter | None] = [None for _ in range(10)] 30 | stop = False 31 | 32 | def profile_a_busy_wait(i): 33 | thread = threads[i] 34 | counter = CallCounter(thread) 35 | counters[i] = counter 36 | 37 | setstatprofile(counter, 0.001) 38 | while not stop: 39 | do_nothing() 40 | setstatprofile(None) 41 | 42 | threads = [threading.Thread(target=profile_a_busy_wait, args=(i,)) for i in range(10)] 43 | for thread in threads: 44 | thread.start() 45 | 46 | while not stop: 47 | stop = all(c is not None and c.count > 10 for c in counters) 48 | 49 | for thread in threads: 50 | thread.join() 51 | -------------------------------------------------------------------------------- /bin/build_js_bundle.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import argparse 4 | import os 5 | import shutil 6 | import subprocess 7 | import sys 8 | 9 | HTML_RENDERER_DIR = "html_renderer" 10 | JS_BUNDLE = "joulehunter/renderers/html_resources/app.js" 11 | 12 | if __name__ == "__main__": 13 | # chdir to root of repo 14 | os.chdir(os.path.dirname(__file__)) 15 | os.chdir("..") 16 | 17 | parser = argparse.ArgumentParser() 18 | parser.add_argument("--force", action="store_true", help="force a rebuild of the bundle") 19 | 20 | args = parser.parse_args() 21 | 22 | js_source_mtime = 0 23 | for dirpath, dirnames, filenames in os.walk(HTML_RENDERER_DIR): 24 | if "node_modules" in dirnames: 25 | dirnames.remove("node_modules") 26 | 27 | for filename in filenames: 28 | file = os.path.join(dirpath, filename) 29 | js_source_mtime = max(js_source_mtime, os.path.getmtime(file)) 30 | 31 | js_bundle_is_up_to_date = ( 32 | os.path.exists(JS_BUNDLE) and os.path.getmtime(JS_BUNDLE) >= js_source_mtime 33 | ) 34 | 35 | if js_bundle_is_up_to_date and not args.force: 36 | print("Bundle up-to-date") 37 | sys.exit(0) 38 | 39 | if subprocess.call("npm --version", shell=True) != 0: 40 | raise RuntimeError("npm is required to build the HTML renderer.") 41 | 42 | subprocess.check_call("npm ci", cwd=HTML_RENDERER_DIR, shell=True) 43 | subprocess.check_call("npm run build", cwd=HTML_RENDERER_DIR, shell=True) 44 | 45 | shutil.copyfile(HTML_RENDERER_DIR + "/dist/js/app.js", JS_BUNDLE) 46 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014-2020, Joe Rickerby and contributors 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of the copyright holder nor the names of its contributors 15 | may be used to endorse or promote products derived from this software without 16 | specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /html_renderer/src/Header.vue: -------------------------------------------------------------------------------- 1 | 20 | 31 | 66 | -------------------------------------------------------------------------------- /test/low_level/test_context.py: -------------------------------------------------------------------------------- 1 | import contextvars 2 | import time 3 | 4 | import pytest 5 | 6 | from ..util import busy_wait 7 | from .util import parametrize_setstatprofile 8 | 9 | 10 | @parametrize_setstatprofile 11 | def test_context_type(setstatprofile): 12 | with pytest.raises(TypeError): 13 | setstatprofile(lambda f, e, a: 0, 1e6, "not a context var") 14 | setstatprofile(None) 15 | 16 | 17 | profiler_context_var = contextvars.ContextVar("profiler_context_var", default=None) 18 | 19 | 20 | @parametrize_setstatprofile 21 | def test_context_tracking(setstatprofile): 22 | profile_calls = [] 23 | 24 | def profile_callback(frame, event, arg): 25 | nonlocal profile_calls 26 | profile_calls.append((frame, event, arg)) 27 | 28 | profiler_1 = object() 29 | profiler_2 = object() 30 | 31 | context_1 = contextvars.copy_context() 32 | context_2 = contextvars.copy_context() 33 | 34 | context_1.run(profiler_context_var.set, profiler_1) 35 | context_2.run(profiler_context_var.set, profiler_2) 36 | 37 | setstatprofile( 38 | profile_callback, 39 | 1e10, # set large interval so we only get context_change events 40 | profiler_context_var, 41 | ) 42 | 43 | context_1.run(busy_wait, 0.001) 44 | context_2.run(busy_wait, 0.001) 45 | 46 | setstatprofile(None) 47 | 48 | assert all(c[1] == "context_changed" for c in profile_calls) 49 | assert len(profile_calls) == 4 50 | 51 | new, old, _ = profile_calls[0][2] 52 | assert old is None 53 | assert new is profiler_1 54 | 55 | new, old, _ = profile_calls[1][2] 56 | assert old is profiler_1 57 | assert new is None 58 | 59 | new, old, _ = profile_calls[2][2] 60 | assert old is None 61 | assert new is profiler_2 62 | 63 | new, old, _ = profile_calls[3][2] 64 | assert old is profiler_2 65 | assert new is None 66 | -------------------------------------------------------------------------------- /metrics/overhead.py: -------------------------------------------------------------------------------- 1 | import cProfile 2 | import profile 3 | from timeit import Timer 4 | 5 | import django.conf 6 | import django.template.loader 7 | 8 | import joulehunter 9 | 10 | django.conf.settings.configure( 11 | INSTALLED_APPS=(), 12 | TEMPLATES=[ 13 | { 14 | "BACKEND": "django.template.backends.django.DjangoTemplates", 15 | "DIRS": [ 16 | "./examples/django_example/django_example/templates", 17 | ], 18 | } 19 | ], 20 | ) 21 | django.setup() 22 | 23 | 24 | def test_func_template(): 25 | django.template.loader.render_to_string("template.html") 26 | 27 | 28 | t = Timer(stmt=test_func_template) 29 | test_func = lambda: t.repeat(number=4000) 30 | 31 | # base 32 | base_timings = test_func() 33 | 34 | # # profile 35 | # p = profile.Profile() 36 | # profile_timings = p.runcall(lambda: test_func()) 37 | 38 | # cProfile 39 | cp = cProfile.Profile() 40 | cProfile_timings = cp.runcall(test_func) 41 | 42 | # joulehunter 43 | profiler = joulehunter.Profiler() 44 | profiler.start() 45 | joulehunter_timings = test_func() 46 | profiler.stop() 47 | 48 | # joulehunter timeline 49 | # profiler = joulehunter.Profiler(timeline=True) 50 | # profiler.start() 51 | # joulehunter_timeline_timings = test_func() 52 | # profiler.stop() 53 | 54 | with open("out.html", "w") as f: 55 | f.write(profiler.output_html()) 56 | 57 | print(profiler.output_text(unicode=True, color=True)) 58 | 59 | graph_data = ( 60 | ("Base timings", min(base_timings)), 61 | # ('profile', min(profile_timings)), 62 | ("cProfile", min(cProfile_timings)), 63 | ("joulehunter", min(joulehunter_timings)), 64 | # ('joulehunter timeline', min(joulehunter_timeline_timings)), 65 | ) 66 | 67 | from ascii_graph import Pyasciigraph 68 | 69 | graph = Pyasciigraph(float_format="{0:.3f}") 70 | for line in graph.graph("Profiler overhead", graph_data): 71 | print(line) 72 | -------------------------------------------------------------------------------- /examples/django_example/django_example/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | BASE_DIR = os.path.dirname(os.path.dirname(__file__)) 4 | 5 | DATABASES = { 6 | "default": { 7 | "ENGINE": "django.db.backends.sqlite3", 8 | "NAME": os.path.join(BASE_DIR, "db.sqlite3"), 9 | } 10 | } 11 | 12 | DEBUG = True 13 | TEMPLATE_DEBUG = True 14 | 15 | SECRET_KEY = "qg7_r+b@)(--as*(4ls$j$$(9i(pl_@y$g0j0r+!=@&$he(+o%" 16 | 17 | ROOT_URLCONF = "django_example.urls" 18 | 19 | INSTALLED_APPS = ( 20 | "django_example", 21 | "django.contrib.admin", 22 | "django.contrib.contenttypes", 23 | "django.contrib.auth", 24 | "django.contrib.sessions", 25 | "django.contrib.messages", 26 | ) 27 | 28 | MIDDLEWARE = ( 29 | "django.contrib.sessions.middleware.SessionMiddleware", 30 | "django.middleware.common.CommonMiddleware", 31 | "django.middleware.csrf.CsrfViewMiddleware", 32 | "django.contrib.auth.middleware.AuthenticationMiddleware", 33 | "django.contrib.messages.middleware.MessageMiddleware", 34 | "django.middleware.clickjacking.XFrameOptionsMiddleware", 35 | "joulehunter.middleware.ProfilerMiddleware", 36 | ) 37 | 38 | TEMPLATES = [ 39 | { 40 | "BACKEND": "django.template.backends.django.DjangoTemplates", 41 | "APP_DIRS": True, 42 | "OPTIONS": { 43 | "context_processors": [ 44 | "django.template.context_processors.debug", 45 | "django.template.context_processors.request", 46 | "django.contrib.auth.context_processors.auth", 47 | "django.contrib.messages.context_processors.messages", 48 | "django.template.context_processors.i18n", 49 | "django.template.context_processors.media", 50 | "django.template.context_processors.csrf", 51 | "django.template.context_processors.tz", 52 | "django.template.context_processors.static", 53 | ], 54 | }, 55 | }, 56 | ] 57 | 58 | 59 | def custom_show_joulehunter(request): 60 | return request.user.is_superuser 61 | 62 | 63 | joulehunter_SHOW_CALLBACK = "%s.custom_show_joulehunter" % __name__ 64 | -------------------------------------------------------------------------------- /joulehunter/util.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | import importlib 3 | import os 4 | import sys 5 | import warnings 6 | from typing import IO 7 | 8 | from joulehunter.vendor.decorator import decorator 9 | 10 | 11 | def object_with_import_path(import_path): 12 | if "." not in import_path: 13 | raise ValueError( 14 | "Can't import '%s', it is not a valid import path" % import_path) 15 | module_path, object_name = import_path.rsplit(".", 1) 16 | 17 | module = importlib.import_module(module_path) 18 | return getattr(module, object_name) 19 | 20 | 21 | def truncate(string: str, max_length: int): 22 | if len(string) > max_length: 23 | return string[0: max_length - 3] + "..." 24 | return string 25 | 26 | 27 | @decorator 28 | def deprecated(func, *args, **kwargs): 29 | """ Marks a function as deprecated. """ 30 | warnings.warn( 31 | f"{func} is deprecated and should no longer be used.", 32 | DeprecationWarning, 33 | stacklevel=3, 34 | ) 35 | return func(*args, **kwargs) 36 | 37 | 38 | def deprecated_option(option_name, message=""): 39 | """ Marks an option as deprecated. """ 40 | 41 | def caller(func, *args, **kwargs): 42 | if option_name in kwargs: 43 | warnings.warn( 44 | f"{option_name} is deprecated. {message}", 45 | DeprecationWarning, 46 | stacklevel=3, 47 | ) 48 | 49 | return func(*args, **kwargs) 50 | 51 | return decorator(caller) 52 | 53 | 54 | def file_supports_color(file_obj: IO) -> bool: 55 | """ 56 | Returns True if the running system's terminal supports color. 57 | 58 | Borrowed from Django 59 | https://github.com/django/django/blob/master/django/core/management/color.py 60 | """ 61 | plat = sys.platform 62 | supported_platform = plat != "Pocket PC" and ( 63 | plat != "win32" or "ANSICON" in os.environ) 64 | 65 | is_a_tty = file_is_a_tty(file_obj) 66 | 67 | return supported_platform and is_a_tty 68 | 69 | 70 | def file_supports_unicode(file_obj: IO) -> bool: 71 | encoding = getattr(file_obj, "encoding", None) 72 | if not encoding: 73 | return False 74 | 75 | codec_info = codecs.lookup(encoding) 76 | 77 | return "utf" in codec_info.name 78 | 79 | 80 | def file_is_a_tty(file_obj: IO) -> bool: 81 | return hasattr(file_obj, "isatty") and file_obj.isatty() 82 | -------------------------------------------------------------------------------- /joulehunter/renderers/base.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Any, List 4 | 5 | from joulehunter import processors 6 | from joulehunter.frame import BaseFrame 7 | from joulehunter.session import Session 8 | 9 | # pyright: strict 10 | 11 | 12 | ProcessorList = List[processors.ProcessorType] 13 | 14 | 15 | class Renderer: 16 | """ 17 | Abstract base class for renderers. 18 | """ 19 | 20 | processors: ProcessorList 21 | """ 22 | Processors installed on this renderer. This property is defined on the 23 | base class to provide a common way for users to add and 24 | manipulate them before calling :func:`render`. 25 | """ 26 | 27 | processor_options: dict[str, Any] 28 | """ 29 | Dictionary containing processor options, passed to each processor. 30 | """ 31 | 32 | def __init__( 33 | self, 34 | show_all: bool = False, 35 | timeline: bool = False, 36 | processor_options: dict[str, Any] | None = None, 37 | ): 38 | """ 39 | :param show_all: Don't hide library frames - show everything that joulehunter captures. 40 | :param timeline: Instead of aggregating time, leave the samples in chronological order. 41 | :param processor_options: A dictionary of processor options. 42 | """ 43 | # processors is defined on the base class to provide a common way for users to 44 | # add to and manipulate them before calling render() 45 | self.processors = self.default_processors() 46 | self.processor_options = processor_options or {} 47 | 48 | if show_all: 49 | self.processors.remove(processors.group_library_frames_processor) 50 | if timeline: 51 | self.processors.remove(processors.aggregate_repeated_calls) 52 | 53 | def default_processors(self) -> ProcessorList: 54 | """ 55 | Return a list of processors that this renderer uses by default. 56 | """ 57 | raise NotImplementedError() 58 | 59 | def preprocess(self, root_frame: BaseFrame | None) -> BaseFrame | None: 60 | frame = root_frame 61 | for processor in self.processors: 62 | frame = processor(frame, options=self.processor_options) 63 | return frame 64 | 65 | def render(self, session: Session) -> str: 66 | """ 67 | Return a string that contains the rendered form of `frame`. 68 | """ 69 | raise NotImplementedError() 70 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | 17 | 18 | # -- Project information ----------------------------------------------------- 19 | 20 | project = "joulehunter" 21 | copyright = "2021, Joe Rickerby" 22 | author = "Joe Rickerby" 23 | 24 | # The full version, including alpha/beta/rc tags 25 | release = "4.0.3" 26 | 27 | 28 | # -- General configuration --------------------------------------------------- 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = ["myst_parser", "sphinx.ext.autodoc", "sphinxcontrib.programoutput"] 34 | 35 | # Add any paths that contain templates here, relative to this directory. 36 | templates_path = ["_templates"] 37 | 38 | # List of patterns, relative to source directory, that match files and 39 | # directories to ignore when looking for source files. 40 | # This pattern also affects html_static_path and html_extra_path. 41 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 42 | 43 | 44 | # -- Options for HTML output ------------------------------------------------- 45 | 46 | # The theme to use for HTML and HTML Help pages. See the documentation for 47 | # a list of builtin themes. 48 | # 49 | html_theme = "furo" 50 | 51 | # Add any paths that contain custom static files (such as style sheets) here, 52 | # relative to this directory. They are copied after the builtin static files, 53 | # so a file named "default.css" will overwrite the builtin "default.css". 54 | html_static_path = ["_static"] 55 | 56 | # -- Autodoc setup 57 | 58 | autoclass_content = "both" 59 | autodoc_member_order = "bysource" 60 | autodoc_typehints = "description" 61 | autodoc_typehints_description_target = "documented" 62 | # napoleon_google_docstring = True 63 | # napoleon_use_rtype = False 64 | -------------------------------------------------------------------------------- /test/fake_time_util.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import contextlib 3 | import functools 4 | import random 5 | from unittest import mock 6 | 7 | from trio.testing import MockClock 8 | 9 | from joulehunter import stack_sampler 10 | 11 | 12 | class FakeClock: 13 | def __init__(self) -> None: 14 | self.time = random.random() * 1e6 15 | 16 | def get_time(self): 17 | return self.time 18 | 19 | def sleep(self, duration): 20 | self.time += duration 21 | 22 | 23 | @contextlib.contextmanager 24 | def fake_time(fake_clock=None): 25 | fake_clock = fake_clock or FakeClock() 26 | stack_sampler.get_stack_sampler().timer_func = fake_clock.get_time 27 | 28 | try: 29 | with mock.patch("time.sleep", new=fake_clock.sleep): 30 | yield fake_clock 31 | finally: 32 | stack_sampler.get_stack_sampler().timer_func = None 33 | 34 | 35 | class FakeClockAsyncio: 36 | # this implementation mostly lifted from 37 | # https://aiotools.readthedocs.io/en/latest/_modules/aiotools/timer.html#VirtualClock 38 | # License: https://github.com/achimnol/aiotools/blob/800f7f1bce086b0c83658bad8377e6cb1908e22f/LICENSE 39 | # Copyright (c) 2017 Joongi Kim 40 | def __init__(self) -> None: 41 | self.time = random.random() * 1e6 42 | 43 | def get_time(self): 44 | return self.time 45 | 46 | def sleep(self, duration): 47 | self.time += duration 48 | 49 | def _virtual_select(self, orig_select, timeout): 50 | self.time += timeout 51 | return orig_select(0) # override the timeout to zero 52 | 53 | 54 | @contextlib.contextmanager 55 | def fake_time_asyncio(loop=None): 56 | loop = loop or asyncio.get_running_loop() 57 | fake_clock = FakeClockAsyncio() 58 | 59 | # fmt: off 60 | with mock.patch.object( 61 | loop._selector, # type: ignore 62 | "select", 63 | new=functools.partial(fake_clock._virtual_select, loop._selector.select), # type: ignore 64 | ), mock.patch.object( 65 | loop, 66 | "time", 67 | new=fake_clock.get_time 68 | ), fake_time(fake_clock): 69 | yield fake_clock 70 | # fmt: on 71 | 72 | 73 | class FakeClockTrio: 74 | def __init__(self, clock: MockClock) -> None: 75 | self.trio_clock = clock 76 | 77 | def get_time(self): 78 | return self.trio_clock.current_time() 79 | 80 | def sleep(self, duration): 81 | self.trio_clock.jump(duration) 82 | 83 | 84 | @contextlib.contextmanager 85 | def fake_time_trio(): 86 | trio_clock = MockClock(autojump_threshold=0) 87 | fake_clock = FakeClockTrio(trio_clock) 88 | 89 | with fake_time(fake_clock): 90 | yield fake_clock 91 | -------------------------------------------------------------------------------- /metrics/multi_overhead.py: -------------------------------------------------------------------------------- 1 | import cProfile 2 | import profile 3 | import re 4 | import sys 5 | import time 6 | from timeit import Timer 7 | 8 | import django.conf 9 | 10 | import joulehunter 11 | 12 | django.conf.settings.configure( 13 | INSTALLED_APPS=(), 14 | TEMPLATES=[ 15 | { 16 | "BACKEND": "django.template.backends.django.DjangoTemplates", 17 | "DIRS": [ 18 | "./examples/django_example/django_example/templates", 19 | ], 20 | } 21 | ], 22 | ) 23 | django.setup() 24 | 25 | 26 | def test_func_re(): 27 | re.compile( 28 | r"[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?" 29 | ) 30 | 31 | 32 | def test_func_template(): 33 | django.template.loader.render_to_string("template.html") 34 | 35 | 36 | # heat caches 37 | test_func_template() 38 | 39 | 40 | def time_base(function, repeats): 41 | timer = Timer(stmt=function) 42 | return timer.repeat(number=repeats) 43 | 44 | 45 | def time_profile(function, repeats): 46 | timer = Timer(stmt=function) 47 | p = profile.Profile() 48 | return p.runcall(lambda: timer.repeat(number=repeats)) 49 | 50 | 51 | def time_cProfile(function, repeats): 52 | timer = Timer(stmt=function) 53 | p = cProfile.Profile() 54 | return p.runcall(lambda: timer.repeat(number=repeats)) 55 | 56 | 57 | def time_joulehunter(function, repeats): 58 | timer = Timer(stmt=function) 59 | p = joulehunter.Profiler() 60 | p.start() 61 | result = timer.repeat(number=repeats) 62 | p.stop() 63 | return result 64 | 65 | 66 | profilers = ( 67 | ("Base", time_base), 68 | # ('profile', time_profile), 69 | ("cProfile", time_cProfile), 70 | ("joulehunter", time_joulehunter), 71 | ) 72 | 73 | tests = ( 74 | ("re.compile", test_func_re, 120000), 75 | ("django template render", test_func_template, 400), 76 | ) 77 | 78 | 79 | def timings_for_test(test_func, repeats): 80 | results = [] 81 | for profiler_tuple in profilers: 82 | time = profiler_tuple[1](test_func, repeats) 83 | results += (profiler_tuple[0], min(time)) 84 | 85 | return results 86 | 87 | 88 | # print header 89 | for column in [""] + [test[0] for test in tests]: 90 | sys.stdout.write(f"{column:>24}") 91 | 92 | sys.stdout.write("\n") 93 | 94 | for profiler_tuple in profilers: 95 | sys.stdout.write(f"{profiler_tuple[0]:>24}") 96 | sys.stdout.flush() 97 | for test_tuple in tests: 98 | time = min(profiler_tuple[1](test_tuple[1], test_tuple[2])) * 10 99 | sys.stdout.write(f"{time:>24.2f}") 100 | sys.stdout.flush() 101 | sys.stdout.write("\n") 102 | -------------------------------------------------------------------------------- /joulehunter/low_level/stat_profile_python.py: -------------------------------------------------------------------------------- 1 | import contextvars 2 | import sys 3 | import timeit 4 | import types 5 | from typing import Any, Callable, List, Optional, Type 6 | 7 | 8 | class PythonStatProfiler: 9 | await_stack: List[str] 10 | 11 | def __init__(self, target, interval, context_var, timer_func): 12 | self.target = target 13 | self.interval = interval 14 | self.timer_func = timer_func or timeit.default_timer 15 | self.last_invocation = self.timer_func() 16 | 17 | if context_var: 18 | # raise typeerror to match the C version 19 | if not isinstance(context_var, contextvars.ContextVar): 20 | raise TypeError("not a context var") 21 | 22 | self.context_var = context_var 23 | self.last_context_var_value = context_var.get() if context_var else None 24 | self.await_stack = [] 25 | 26 | def profile(self, frame: types.FrameType, event: str, arg: Any): 27 | now = self.timer_func() 28 | 29 | if self.context_var: 30 | context_var_value = self.context_var.get() 31 | last_context_var_value = self.last_context_var_value 32 | 33 | if context_var_value is not last_context_var_value: 34 | context_change_frame = frame.f_back if event == "call" else frame 35 | self.target( 36 | context_change_frame, 37 | "context_changed", 38 | (context_var_value, last_context_var_value, self.await_stack), 39 | ) 40 | self.last_context_var_value = context_var_value 41 | 42 | # 0x80 == CO_COROUTINE (i.e. defined with 'async def') 43 | if event == "return" and frame.f_code.co_flags & 0x80: 44 | self.await_stack.append( 45 | "%s\x00%s\x00%i" 46 | % ( 47 | frame.f_code.co_name, 48 | frame.f_code.co_filename, 49 | frame.f_code.co_firstlineno, 50 | ) 51 | ) 52 | else: 53 | self.await_stack.clear() 54 | 55 | if now < self.last_invocation + self.interval: 56 | return 57 | 58 | self.last_invocation = now 59 | return self.target(frame, event, arg) 60 | 61 | 62 | """ 63 | A reimplementation of setstatprofile in Python, for prototyping/reference 64 | purposes. Not used in normal execution. 65 | """ 66 | 67 | 68 | def setstatprofile(target, interval=0.001, context_var=None, timer_func=None): 69 | if target: 70 | profiler = PythonStatProfiler( 71 | target=target, 72 | interval=interval, 73 | context_var=context_var, 74 | timer_func=timer_func, 75 | ) 76 | sys.setprofile(profiler.profile) 77 | else: 78 | sys.setprofile(None) 79 | -------------------------------------------------------------------------------- /docs/reference.md: -------------------------------------------------------------------------------- 1 | # API Reference 2 | 3 | ## Command line interface 4 | 5 | ``joulehunter`` works just like ``python``, on the command line, so you can 6 | call your scripts like ``joulehunter script.py`` or ``joulehunter -m 7 | my_module``. 8 | 9 | When your script ends, or when you kill it with `ctrl-c`, joulehunter will 10 | print a profile report to the console. 11 | 12 | ```{program-output} joulehunter --help 13 | ``` 14 | 15 | ## Python API 16 | 17 | The Python API is also available, for calling joulehunter directly from 18 | Python and writing integrations with with other tools. 19 | 20 | ### The Profiler object 21 | 22 | ```{eval-rst} 23 | .. autoclass:: joulehunter.Profiler 24 | :members: 25 | :special-members: __enter__ 26 | ``` 27 | 28 | ### Sessions 29 | 30 | ```{eval-rst} 31 | .. autoclass:: joulehunter.session.Session 32 | :members: 33 | ``` 34 | 35 | ### Renderers 36 | 37 | Renderers transform a tree of {class}`Frame` objects into some form of output. 38 | 39 | Rendering has two steps: 40 | 41 | 1. First, the renderer will 'preprocess' the Frame tree, applying each processor in the ``processor`` property, in turn. 42 | 2. The resulting tree is renderered into the desired format. 43 | 44 | Therefore, rendering can be customised by changing the ``processors`` property. For example, you can disable time-aggregation (making the profile into a timeline) by removing {func}`aggregate_repeated_calls`. 45 | 46 | ```{eval-rst} 47 | .. autoclass:: joulehunter.renderers.Renderer 48 | :members: 49 | 50 | .. autoclass:: joulehunter.renderers.ConsoleRenderer 51 | 52 | .. autoclass:: joulehunter.renderers.HTMLRenderer 53 | 54 | .. autoclass:: joulehunter.renderers.JSONRenderer 55 | ``` 56 | 57 | ### Processors 58 | 59 | ```{eval-rst} 60 | .. automodule:: joulehunter.processors 61 | :members: 62 | ``` 63 | 64 | ### Internals notes 65 | 66 | Frames are recorded by the Profiler in a time-linear fashion. While profiling, 67 | the profiler builds a list of frame stacks, with the frames having in format: 68 | 69 | function_name filename function_line_number 70 | 71 | When profiling is complete, this list is turned into a tree structure of 72 | Frame objects. This tree contains all the information as gathered by the 73 | profiler, suitable for a flame render. 74 | 75 | #### Frame objects, the call tree, and processors 76 | 77 | The frames are assembled to a call tree by the profiler session. The 78 | time-linearity is retained at this stage. 79 | 80 | Before rendering, the call tree is then fed through a sequence of 'processors' 81 | to transform the tree for output. 82 | 83 | The most interesting is `aggregate_repeated_calls`, which combines different 84 | instances of function calls into the same frame. This is intuitive as a 85 | summary of where time was spent during execution. 86 | 87 | The rest of the processors focus on removing or hiding irrelevant Frames 88 | from the output. 89 | 90 | #### Self time frames vs. frame.self_time 91 | 92 | Self time nodes exist to record time spent in a node, but not in its children. 93 | But normal frame objects can have self_time too. Why? frame.self_time is used 94 | to store the self_time of any nodes that were removed during processing. 95 | -------------------------------------------------------------------------------- /html_renderer/src/App.vue: -------------------------------------------------------------------------------- 1 | 11 | 12 | 93 | 94 | 113 | -------------------------------------------------------------------------------- /joulehunter/renderers/html.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import codecs 4 | import os 5 | import tempfile 6 | import urllib.parse 7 | import webbrowser 8 | from typing import Any 9 | 10 | from joulehunter import processors 11 | from joulehunter.renderers.base import ProcessorList, Renderer 12 | from joulehunter.renderers.jsonrenderer import JSONRenderer 13 | from joulehunter.session import Session 14 | 15 | # pyright: strict 16 | 17 | 18 | class HTMLRenderer(Renderer): 19 | """ 20 | Renders a rich, interactive web page, as a string of HTML. 21 | """ 22 | 23 | def __init__(self, **kwargs: Any): 24 | super().__init__(**kwargs) 25 | 26 | def render(self, session: Session): 27 | resources_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "html_resources/") 28 | 29 | if not os.path.exists(os.path.join(resources_dir, "app.js")): 30 | raise RuntimeError( 31 | "Could not find app.js. If you are running " 32 | "joulehunter from a git checkout, run 'python " 33 | "setup.py build' to compile the Javascript " 34 | "(requires nodejs)." 35 | ) 36 | 37 | with open(os.path.join(resources_dir, "app.js"), encoding="utf-8") as f: 38 | js = f.read() 39 | 40 | session_json = self.render_json(session) 41 | 42 | page = """ 43 | 44 | 45 | 46 | 47 | 48 |
49 | 52 | 55 | 56 | """.format( 57 | js=js, session_json=session_json 58 | ) 59 | 60 | return page 61 | 62 | def open_in_browser(self, session: Session, output_filename: str | None = None): 63 | """ 64 | Open the rendered HTML in a webbrowser. 65 | 66 | If output_filename=None (the default), a tempfile is used. 67 | 68 | The filename of the HTML file is returned. 69 | 70 | """ 71 | if output_filename is None: 72 | output_file = tempfile.NamedTemporaryFile(suffix=".html", delete=False) 73 | output_filename = output_file.name 74 | with codecs.getwriter("utf-8")(output_file) as f: 75 | f.write(self.render(session)) 76 | else: 77 | with codecs.open(output_filename, "w", "utf-8") as f: 78 | f.write(self.render(session)) 79 | 80 | url = urllib.parse.urlunparse(("file", "", output_filename, "", "", "")) 81 | webbrowser.open(url) 82 | return output_filename 83 | 84 | def render_json(self, session: Session): 85 | json_renderer = JSONRenderer() 86 | json_renderer.processors = self.processors 87 | json_renderer.processor_options = self.processor_options 88 | return json_renderer.render(session) 89 | 90 | def default_processors(self) -> ProcessorList: 91 | return [ 92 | processors.remove_importlib, 93 | processors.merge_consecutive_self_time, 94 | processors.aggregate_repeated_calls, 95 | processors.group_library_frames_processor, 96 | processors.remove_unnecessary_self_time_nodes, 97 | processors.remove_irrelevant_nodes, 98 | ] 99 | -------------------------------------------------------------------------------- /joulehunter/renderers/jsonrenderer.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | from typing import Any, Callable 5 | 6 | from joulehunter import processors 7 | from joulehunter.frame import BaseFrame 8 | from joulehunter.renderers.base import ProcessorList, Renderer 9 | from joulehunter.session import Session 10 | 11 | # pyright: strict 12 | 13 | 14 | # note: this file is called jsonrenderer to avoid hiding built-in module 'json'. 15 | 16 | encode_str: Callable[[str], str] = json.encoder.encode_basestring # type: ignore 17 | 18 | 19 | def encode_bool(a_bool: bool): 20 | return "true" if a_bool else "false" 21 | 22 | 23 | class JSONRenderer(Renderer): 24 | """ 25 | Outputs a tree of JSON, containing processed frames. 26 | """ 27 | 28 | def __init__(self, **kwargs: Any): 29 | super().__init__(**kwargs) 30 | 31 | def render_frame(self, frame: BaseFrame | None): 32 | if frame is None: 33 | return "null" 34 | # we don't use the json module because it uses 2x stack frames, so 35 | # crashes on deep but valid call stacks 36 | 37 | property_decls: list[str] = [] 38 | property_decls.append('"function": %s' % encode_str(frame.function or "")) 39 | property_decls.append('"file_path_short": %s' % encode_str(frame.file_path_short or "")) 40 | property_decls.append('"file_path": %s' % encode_str(frame.file_path or "")) 41 | property_decls.append('"line_no": %d' % frame.line_no) 42 | property_decls.append('"time": %f' % frame.time()) 43 | property_decls.append('"await_time": %f' % frame.await_time()) 44 | property_decls.append( 45 | '"is_application_code": %s' % encode_bool(frame.is_application_code or False) 46 | ) 47 | 48 | # can't use list comprehension here because it uses two stack frames each time. 49 | children_jsons: list[str] = [] 50 | for child in frame.children: 51 | children_jsons.append(self.render_frame(child)) 52 | property_decls.append('"children": [%s]' % ",".join(children_jsons)) 53 | 54 | if frame.group: 55 | property_decls.append('"group_id": %s' % encode_str(frame.group.id)) 56 | 57 | return "{%s}" % ",".join(property_decls) 58 | 59 | def render(self, session: Session): 60 | frame = self.preprocess(session.root_frame()) 61 | 62 | property_decls: list[str] = [] 63 | property_decls.append('"start_time": %f' % session.start_time) 64 | property_decls.append('"duration": %f' % session.duration) 65 | property_decls.append('"sample_count": %d' % session.sample_count) 66 | property_decls.append('"program": %s' % encode_str(session.program)) 67 | property_decls.append( 68 | '"package": %s' % encode_str(session.domain_names[0])) 69 | if len(session.domain_names) == 2: 70 | property_decls.append( 71 | '"component": %s' % encode_str(session.domain_names[1])) 72 | else: 73 | property_decls.append('"component": null') 74 | property_decls.append('"root_frame": %s' % self.render_frame(frame)) 75 | 76 | return "{%s}\n" % ",".join(property_decls) 77 | 78 | def default_processors(self) -> ProcessorList: 79 | return [ 80 | processors.remove_importlib, 81 | processors.merge_consecutive_self_time, 82 | processors.aggregate_repeated_calls, 83 | processors.group_library_frames_processor, 84 | processors.remove_unnecessary_self_time_nodes, 85 | processors.remove_irrelevant_nodes, 86 | ] 87 | -------------------------------------------------------------------------------- /joulehunter/middleware.py: -------------------------------------------------------------------------------- 1 | import io 2 | import os 3 | import sys 4 | import time 5 | 6 | from django.conf import settings 7 | from django.http import HttpResponse 8 | from django.utils.module_loading import import_string 9 | 10 | from joulehunter import Profiler 11 | from joulehunter.renderers.html import HTMLRenderer 12 | 13 | try: 14 | from django.utils.deprecation import MiddlewareMixin 15 | except ImportError: 16 | MiddlewareMixin = object 17 | 18 | 19 | class ProfilerMiddleware(MiddlewareMixin): # type: ignore 20 | def process_request(self, request): 21 | profile_dir = getattr(settings, "JOULEHUNTER_PROFILE_DIR", None) 22 | 23 | func_or_path = getattr(settings, "JOULEHUNTER_SHOW_CALLBACK", None) 24 | if isinstance(func_or_path, str): 25 | show_joulehunter = import_string(func_or_path) 26 | elif callable(func_or_path): 27 | show_joulehunter = func_or_path 28 | else: 29 | show_joulehunter = lambda request: True 30 | 31 | if ( 32 | show_joulehunter(request) 33 | and getattr(settings, "JOULEHUNTER_URL_ARGUMENT", "profile") in request.GET 34 | ) or profile_dir: 35 | package = request.GET.get( 36 | "package", 37 | getattr(settings, "JOULEHUNTER_PACKAGE", None) 38 | ) 39 | component = request.GET.get( 40 | "component", 41 | getattr(settings, "JOULEHUNTER_COMPONENT", None) 42 | ) 43 | if package == '': 44 | package = None 45 | if component == '': 46 | component = None 47 | 48 | if package is not None: 49 | profiler = Profiler(package=package, 50 | component=component) 51 | else: 52 | profiler = Profiler(component=component) 53 | profiler.start() 54 | 55 | request.profiler = profiler 56 | 57 | def process_response(self, request, response): 58 | if hasattr(request, "profiler"): 59 | profile_session = request.profiler.stop() 60 | 61 | renderer = HTMLRenderer() 62 | output_html = renderer.render(profile_session) 63 | 64 | profile_dir = getattr(settings, "JOULEHUNTER_PROFILE_DIR", None) 65 | 66 | # Limit the length of the file name (255 characters is the max limit on major current OS, but it is rather 67 | # high and the other parts (see line 36) are to be taken into account; so a hundred will be fine here). 68 | path = request.get_full_path().replace("/", "_")[:100] 69 | 70 | # Swap ? for _qs_ on Windows, as it does not support ? in filenames. 71 | if sys.platform in ["win32", "cygwin"]: 72 | path = path.replace("?", "_qs_") 73 | 74 | if profile_dir: 75 | filename = "{total_time:.3f}s {path} {timestamp:.0f}.html".format( 76 | total_time=profile_session.duration, 77 | path=path, 78 | timestamp=time.time(), 79 | ) 80 | 81 | file_path = os.path.join(profile_dir, filename) 82 | 83 | if not os.path.exists(profile_dir): 84 | os.mkdir(profile_dir) 85 | 86 | with open(file_path, "w", encoding="utf-8") as f: 87 | f.write(output_html) 88 | 89 | if getattr(settings, "JOULEHUNTER_URL_ARGUMENT", "profile") in request.GET: 90 | return HttpResponse(output_html) 91 | else: 92 | return response 93 | else: 94 | return response 95 | -------------------------------------------------------------------------------- /test/test_stack_sampler.py: -------------------------------------------------------------------------------- 1 | import contextvars 2 | import sys 3 | import time 4 | 5 | import pytest 6 | 7 | from joulehunter import stack_sampler 8 | 9 | from .util import do_nothing 10 | 11 | 12 | class SampleCounter: 13 | count = 0 14 | 15 | def sample(self, stack, time, async_state): 16 | self.count += 1 17 | 18 | 19 | def test_create(): 20 | sampler = stack_sampler.get_stack_sampler() 21 | assert sampler is not None 22 | 23 | assert sampler is stack_sampler.get_stack_sampler() 24 | 25 | 26 | def test_get_samples(): 27 | sampler = stack_sampler.get_stack_sampler() 28 | counter = SampleCounter() 29 | 30 | assert sys.getprofile() is None 31 | sampler.subscribe(counter.sample, desired_interval=0.001, use_async_context=True) 32 | assert sys.getprofile() is not None 33 | assert len(sampler.subscribers) == 1 34 | 35 | start = time.time() 36 | while time.time() < start + 1 and counter.count == 0: 37 | do_nothing() 38 | 39 | assert counter.count > 0 40 | 41 | assert sys.getprofile() is not None 42 | sampler.unsubscribe(counter.sample) 43 | assert sys.getprofile() is None 44 | 45 | assert len(sampler.subscribers) == 0 46 | 47 | 48 | def test_multiple_samplers(): 49 | sampler = stack_sampler.get_stack_sampler() 50 | counter_1 = SampleCounter() 51 | counter_2 = SampleCounter() 52 | 53 | sampler.subscribe(counter_1.sample, desired_interval=0.001, use_async_context=False) 54 | sampler.subscribe(counter_2.sample, desired_interval=0.001, use_async_context=False) 55 | 56 | assert len(sampler.subscribers) == 2 57 | 58 | start = time.time() 59 | while time.time() < start + 1 and counter_1.count == 0 and counter_2.count == 0: 60 | do_nothing() 61 | 62 | assert counter_1.count > 0 63 | assert counter_2.count > 0 64 | 65 | assert sys.getprofile() is not None 66 | 67 | sampler.unsubscribe(counter_1.sample) 68 | sampler.unsubscribe(counter_2.sample) 69 | 70 | assert sys.getprofile() is None 71 | 72 | assert len(sampler.subscribers) == 0 73 | 74 | 75 | def test_multiple_samplers_async_error(): 76 | sampler = stack_sampler.get_stack_sampler() 77 | 78 | counter_1 = SampleCounter() 79 | counter_2 = SampleCounter() 80 | 81 | sampler.subscribe(counter_1.sample, desired_interval=0.001, use_async_context=True) 82 | 83 | with pytest.raises(RuntimeError): 84 | sampler.subscribe(counter_2.sample, desired_interval=0.001, use_async_context=True) 85 | 86 | sampler.unsubscribe(counter_1.sample) 87 | 88 | 89 | def test_multiple_contexts(): 90 | sampler = stack_sampler.get_stack_sampler() 91 | 92 | counter_1 = SampleCounter() 93 | counter_2 = SampleCounter() 94 | 95 | context_1 = contextvars.copy_context() 96 | context_2 = contextvars.copy_context() 97 | 98 | assert sys.getprofile() is None 99 | assert len(sampler.subscribers) == 0 100 | context_1.run(sampler.subscribe, counter_1.sample, 0.001, True) 101 | context_2.run(sampler.subscribe, counter_2.sample, 0.001, True) 102 | 103 | assert sys.getprofile() is not None 104 | assert len(sampler.subscribers) == 2 105 | 106 | start = time.time() 107 | while time.time() < start + 1 and counter_1.count == 0 and counter_2.count == 0: 108 | do_nothing() 109 | 110 | assert counter_1.count > 0 111 | assert counter_2.count > 0 112 | 113 | assert sys.getprofile() is not None 114 | 115 | context_1.run(sampler.unsubscribe, counter_1.sample) 116 | context_2.run(sampler.unsubscribe, counter_2.sample) 117 | 118 | assert sys.getprofile() is None 119 | 120 | assert len(sampler.subscribers) == 0 121 | -------------------------------------------------------------------------------- /joulehunter/energy.py: -------------------------------------------------------------------------------- 1 | # Allows the use of standard collection type hinting from Python 3.7 onwards 2 | from __future__ import annotations 3 | 4 | from collections.abc import Generator 5 | import os 6 | from typing import Any 7 | 8 | RAPL_API_DIR = "/sys/devices/virtual/powercap/intel-rapl" 9 | 10 | 11 | def available_domains() -> list[dict[str, Any]]: 12 | if not os.path.exists(RAPL_API_DIR): 13 | raise RuntimeError("RAPL API is not available on this machine") 14 | 15 | packages = [ 16 | {"dirname": dirname, 17 | "name": domain_name([dirname]), 18 | "components": []} 19 | for dirname in sorted(os.listdir(RAPL_API_DIR)) 20 | if dirname.startswith("intel-rapl")] 21 | 22 | for package in packages: 23 | package["components"] = [ 24 | {"dirname": dirname, 25 | "name": domain_name([package["dirname"], dirname])} 26 | for dirname 27 | in sorted(os.listdir(os.path.join(RAPL_API_DIR, 28 | package["dirname"]))) 29 | if dirname.startswith("intel-rapl")] 30 | 31 | return packages 32 | 33 | 34 | def domain_name(dirnames: list[str]) -> str: 35 | rapl_name_path = os.path.join(RAPL_API_DIR, *dirnames, "name") 36 | if not os.path.exists(rapl_name_path): 37 | raise RuntimeError("Domain not found") 38 | with open(rapl_name_path, "r") as file: 39 | return file.readline().strip() 40 | 41 | 42 | def stringify_domains(domains: list[dict[str, Any]]) -> str: 43 | text = "" 44 | for package_num, package in enumerate(domains): 45 | text += f"[{package_num}] {package['name']}\n" 46 | for component_num, component in enumerate(package["components"]): 47 | text += f" [{component_num}] {component['name']}\n" 48 | text = text[:-1] 49 | return text 50 | 51 | 52 | def package_name_to_num(domains: list[dict[str, Any]], name: str) -> str: 53 | for package in domains: 54 | if package['name'] == name: 55 | return package['dirname'].split(':')[-1] 56 | raise RuntimeError("Package name not found") 57 | 58 | 59 | def component_name_to_num(domains: list[dict[str, Any]], name: str) -> str: 60 | for package in domains: 61 | for component in package["components"]: 62 | if component['name'] == name: 63 | return component['dirname'].split(':')[-1] 64 | raise RuntimeError("Component name not found") 65 | 66 | 67 | def dirnames_to_names(domains: list[dict[str, Any]], 68 | dirnames: list[str]) -> list[str]: 69 | names = [] 70 | for dirname in dirnames: 71 | for package in domains: 72 | if package['dirname'] == dirname: 73 | names.append(package['name']) 74 | for component in package["components"]: 75 | if component['dirname'] == dirname: 76 | names.append(component['name']) 77 | return names 78 | 79 | 80 | def parse_domain(package, component): 81 | available_domains_ = available_domains() 82 | 83 | package = str(package) 84 | if not package.isnumeric(): 85 | package = package_name_to_num(available_domains_, package) 86 | 87 | domain = [f'intel-rapl:{package}'] 88 | 89 | if component is not None: 90 | component = str(component) 91 | if not component.isnumeric(): 92 | component = component_name_to_num(available_domains_, component) 93 | domain.append(f'intel-rapl:{package}:{component}') 94 | 95 | return domain 96 | 97 | 98 | class Energy: 99 | def __init__(self, dirnames: list[str]) -> None: 100 | self.generator = Energy.current_energy_generator(dirnames) 101 | 102 | @staticmethod 103 | def current_energy_generator(dirnames: list[str]) -> Generator[ 104 | float, None, None]: 105 | rapl_energy_path = os.path.join(RAPL_API_DIR, *dirnames, 'energy_uj') 106 | 107 | if not os.path.exists(rapl_energy_path): 108 | raise RuntimeError("Domain not found") 109 | with open(rapl_energy_path, 'r') as file: 110 | while True: 111 | energy = float(file.readline()[:-1]) / 10**6 112 | file.seek(0) 113 | yield energy 114 | 115 | def current_energy(self) -> float: 116 | return next(self.generator) 117 | -------------------------------------------------------------------------------- /docs/img/async-context.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | APP ENTRYPOINT 18 | 19 | 20 | APP ENTRYPOINT 21 | 22 | 23 | RUN LOOP 24 | 25 | 26 | RUN LOOP 27 | 28 | 29 | TASK 1 30 | 31 | 32 | TASK 1 33 | 34 | 35 | TASK 1 36 | 37 | 38 | TASK 1 39 | 40 | 41 | TASK 2 42 | 43 | 44 | TASK 2 45 | 46 | 47 | PROFILER STARTED HERE 48 | 49 | 50 | PROFILER STARTED HERE 51 | 52 | 53 | TASK 2 54 | 55 | 56 | TASK 2 57 | 58 | 59 | A profiler started in an async task is scoped to that async context. 60 | 61 | 62 | When async tasks are created, they inherit the context from the caller. So starting a profiler before the run loop causes all async tasks to be profiled. 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /bin/bump_version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | 4 | from __future__ import annotations 5 | 6 | import glob 7 | import os 8 | import subprocess 9 | import sys 10 | import urllib.parse 11 | from pathlib import Path 12 | 13 | import click 14 | from packaging.version import InvalidVersion, Version 15 | 16 | import joulehunter 17 | 18 | config = [ 19 | # file path, version find/replace format 20 | ("setup.py", 'version="{}"'), 21 | ("joulehunter/__init__.py", '__version__ = "{}"'), 22 | ("docs/conf.py", 'release = "{}"'), 23 | ] 24 | 25 | RED = "\u001b[31m" 26 | GREEN = "\u001b[32m" 27 | OFF = "\u001b[0m" 28 | 29 | 30 | @click.command() 31 | def bump_version() -> None: 32 | current_version = joulehunter.__version__ 33 | 34 | try: 35 | commit_date_str = subprocess.run( 36 | [ 37 | "git", 38 | "show", 39 | "--no-patch", 40 | "--pretty=format:%ci", 41 | f"v{current_version}^{{commit}}", 42 | ], 43 | check=True, 44 | capture_output=True, 45 | encoding="utf8", 46 | ).stdout 47 | cd_date, cd_time, cd_tz = commit_date_str.split(" ") 48 | 49 | url_opts = urllib.parse.urlencode({"q": f"is:pr merged:>{cd_date}T{cd_time}{cd_tz}"}) 50 | url = f"https://github.com/powerapi-ng/joulehunter/pulls?{url_opts}" 51 | 52 | print(f"PRs merged since last release:\n {url}") 53 | print() 54 | except subprocess.CalledProcessError as e: 55 | print(e) 56 | print("Failed to get previous version tag information.") 57 | 58 | git_changes_result = subprocess.run(["git diff-index --quiet HEAD --"], shell=True) 59 | repo_has_uncommitted_changes = git_changes_result.returncode != 0 60 | 61 | if repo_has_uncommitted_changes: 62 | print("error: Uncommitted changes detected.") 63 | sys.exit(1) 64 | 65 | # fmt: off 66 | print( 'Current version:', current_version) # noqa 67 | new_version = input(' New version: ').strip() 68 | # fmt: on 69 | 70 | try: 71 | Version(new_version) 72 | except InvalidVersion: 73 | print("error: This version doesn't conform to PEP440") 74 | print(" https://www.python.org/dev/peps/pep-0440/") 75 | sys.exit(1) 76 | 77 | actions = [] 78 | 79 | for path_pattern, version_pattern in config: 80 | paths = [Path(p) for p in glob.glob(path_pattern)] 81 | 82 | if not paths: 83 | print(f"error: Pattern {path_pattern} didn't match any files") 84 | sys.exit(1) 85 | 86 | find_pattern = version_pattern.format(current_version) 87 | replace_pattern = version_pattern.format(new_version) 88 | found_at_least_one_file_needing_update = False 89 | 90 | for path in paths: 91 | contents = path.read_text(encoding="utf8") 92 | if find_pattern in contents: 93 | found_at_least_one_file_needing_update = True 94 | actions.append( 95 | ( 96 | path, 97 | find_pattern, 98 | replace_pattern, 99 | ) 100 | ) 101 | 102 | if not found_at_least_one_file_needing_update: 103 | print(f'''error: Didn't find any occurrences of "{find_pattern}" in "{path_pattern}"''') 104 | sys.exit(1) 105 | 106 | print() 107 | print("Here's the plan:") 108 | print() 109 | 110 | for action in actions: 111 | path, find, replace = action 112 | print(f"{path} {RED}{find}{OFF} → {GREEN}{replace}{OFF}") 113 | 114 | print(f"Then commit, and tag as v{new_version}") 115 | 116 | answer = input("Proceed? [y/N] ").strip() 117 | 118 | if answer != "y": 119 | print("Aborted") 120 | sys.exit(1) 121 | 122 | for path, find, replace in actions: 123 | contents = path.read_text(encoding="utf8") 124 | contents = contents.replace(find, replace) 125 | path.write_text(contents, encoding="utf8") 126 | 127 | print("Files updated. If you want to update the changelog as part of this") 128 | print("commit, do that now.") 129 | print() 130 | 131 | while input('Type "done" to continue: ').strip().lower() != "done": 132 | pass 133 | 134 | subprocess.run( 135 | [ 136 | "git", 137 | "commit", 138 | "--all", 139 | f"--message=Bump version: v{new_version}", 140 | ], 141 | check=True, 142 | ) 143 | 144 | subprocess.run( 145 | [ 146 | "git", 147 | "tag", 148 | "--annotate", 149 | f"--message=v{new_version}", 150 | f"v{new_version}", 151 | ], 152 | check=True, 153 | ) 154 | 155 | print("Done.") 156 | 157 | 158 | if __name__ == "__main__": 159 | os.chdir(Path(__file__).parent.parent.resolve()) 160 | bump_version() 161 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | joulehunter 2 | =========== 3 | [![Unit tests](https://github.com/powerapi-ng/joulehunter/actions/workflows/test.yaml/badge.svg)](https://github.com/powerapi-ng/joulehunter/actions/workflows/test.yaml) 4 | [![PyPI version](https://badge.fury.io/py/joulehunter.svg)](https://badge.fury.io/py/joulehunter) 5 | ![screenshot](https://user-images.githubusercontent.com/11022568/134655797-3872379e-0e4e-48d6-a771-6a94c756fa67.png) 6 | 7 | Joulehunter helps you find what part of your code is consuming considerable amounts of energy. 8 | 9 | This repo is still a work in progress. 😄 10 | 11 | Compatibility 12 | ------------ 13 | 14 | Joulehunter runs on **Linux** machines with **Intel RAPL** support. This technology has been available since the Sandy Bridge generation. 15 | 16 | 17 | Installation 18 | ------------ 19 | 20 | You can install joulehunter with pip: ```pip install joulehunter```. 21 | 22 | You can also clone the repo and install it directly: 23 | 24 | git clone https://github.com/powerapi-ng/joulehunter.git 25 | cd joulehunter 26 | python setup.py install 27 | 28 | Usage 29 | ------------ 30 | 31 | Joulehunter works similarly to [pyinstrument](https://github.com/joerick/pyinstrument), as we forked the repo and replaced time measuring with energy measuring. Here's [pyinstrument's documentation](https://pyinstrument.readthedocs.io/). Whenever ```pyinstrument``` is present in a variable name, it should be replaced with ```joulehunter``` (for example, ```PYINSTRUMENT_PROFILE_DIR``` turns into ```JOULEHUNTER_PROFILE_DIR```). 32 | 33 | ### Command line 34 | 35 | ```joulehunter -l``` will list the available domains on this machine. These include the packages and their components, such as the DRAM and core. 36 | 37 | The command ```joulehunter main.py``` will execute ```main.py``` and measure the energy consumption of the first package (CPU). 38 | 39 | To select the package to analyze use the option ```-p``` or ```--package``` followed by the package number or the package name. The default value is 0. 40 | 41 | The options ```-c``` and ```--component``` allow you to measure the energy of an individual component by specifying their name or ID. If not specified, the entire package will be selected. 42 | 43 | 44 | #### Example 45 | 46 | Executing ```joulehunter -l``` could output this: 47 | 48 | [0] package-0 49 | [0] core 50 | [1] uncore 51 | [2] dram 52 | [1] package-1 53 | [0] core 54 | [1] uncore 55 | [2] dram 56 | 57 | If we run ```joulehunter -p package-1 -c 2 my_file.py```, joulehunter will execute ```my_file.py``` and measure the energy consumption of package-1's DRAM. 58 | 59 | ### Profiling chunks of code 60 | 61 | As [pyinstrument's documentation](https://pyinstrument.readthedocs.io/en/latest/guide.html#profile-a-specific-chunk-of-code) shows, it's also possible to profile specific chunks of code. 62 | 63 | Joulehunter's Profiler class can receive two additional arguments: ```package``` and ```component```. They receive the ID (as a string or integer) or name of the desired package/component. If ```package``` is not specified, ```package-0``` will be used. If ```component``` is ```None```, the entire package will be analyzed. 64 | 65 | ### Profiling web requests in Flask 66 | 67 | Please refer to [pyinstrument's documentation](https://pyinstrument.readthedocs.io/en/latest/guide.html#profile-a-web-request-in-flask) for instructions on how to profile web requests in Flask. As in the previous case, joulehunter's ```Profiler()``` accepts two additional arguments. 68 | 69 | ### Profiling web requests in Django 70 | 71 | Profiling web requests in Django as explained in [pyinstrument's documentation](https://pyinstrument.readthedocs.io/en/latest/guide.html#profile-a-web-request-in-django) selects package 0 as the default domain (don't forget to rename the ```pyinstrument``` in variable names with ```joulehunter```). 72 | 73 | The user can choose a particular package and component as follows: 74 | 75 | **Query component**: The query strings ```package``` and ```component``` are used to select the desired package and component. For example, including ```?profiler&package=0&component=dram``` at the end of a request URL will select the first package and the DRAM. If the component query component is present but empty, the package will be analyzed. 76 | 77 | 78 | **Variable in ```settings.py```:** The user's selection can also be defined in ```settings.py``` with the ```JOULEHUNTER_PACKAGE``` and ```JOULEHUNTER_COMPONENT``` variables. These are later passed to ```Package()```. 79 | 80 | If the package or component is defined both as a query component and in ```settings.py```, the one defined as a query component will be selected. 81 | 82 | 83 | Read permission 84 | --------------- 85 | 86 | Due to a [security vulnerability](https://platypusattack.com), only root has read permission for the energy files. In order to circumvent this, run the script as root or grant read permissions for the following files: 87 | 88 | /sys/devices/virtual/powercap/intel-rapl/intel-rapl:*/energy_uj 89 | /sys/devices/virtual/powercap/intel-rapl/intel-rapl:*/intel-rapl:*:*/energy_uj 90 | 91 | More info [here](https://github.com/powerapi-ng/pyJoules/issues/13). 92 | 93 | Acknowledgments 94 | ------------ 95 | 96 | Thanks to [Joe Rickerby](https://github.com/joerick) and all of [pyinstrument](https://github.com/joerick/pyinstrument)'s contributors. 97 | 98 | This fork is being developed by [Chakib Belgaid](https://github.com/chakib-belgaid) and [Alex Kaminetzky](https://github.com/akaminetzkyp). Feel free to ask us any questions! 99 | -------------------------------------------------------------------------------- /test/test_profiler.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import json 3 | import time 4 | from functools import partial 5 | from test.fake_time_util import fake_time 6 | from typing import Generator, Optional 7 | 8 | import pytest 9 | import trio 10 | 11 | from joulehunter import Profiler, renderers 12 | from joulehunter.frame import BaseFrame, Frame 13 | from joulehunter.session import Session 14 | 15 | from .util import assert_never, busy_wait, flaky_in_ci 16 | 17 | 18 | def long_function_a(): 19 | time.sleep(0.25) 20 | 21 | 22 | def long_function_b(): 23 | time.sleep(0.5) 24 | 25 | 26 | # Tests # 27 | 28 | 29 | def test_collapses_multiple_calls_by_default(): 30 | profiler = Profiler() 31 | 32 | with fake_time(): 33 | profiler.start() 34 | 35 | long_function_a() 36 | long_function_b() 37 | long_function_a() 38 | long_function_b() 39 | 40 | profiler.stop() 41 | 42 | text_output = profiler.output_text() 43 | print(text_output) 44 | 45 | # output should be something like: 46 | # 1.500 J [100.0%] test_collapses_multiple_calls_by_default test/test_profiler.py:61 47 | # |- 1.000 J [66.7%] long_function_b test/test_profiler.py:54 48 | # | `- 1.000 J [66.7%] sleep test/fake_time_util.py:19 49 | # `- 0.500 J [33.3%] long_function_a test/test_profiler.py:50 50 | # `- 0.500 J [33.3%] sleep test/fake_time_util.py:19 51 | 52 | assert text_output.count( 53 | "1.500 J [100.0%] test_collapses_multiple_calls_by_default") == 1 54 | assert text_output.count("0.500 J [33.3%] long_function_a") == 1 55 | assert text_output.count("1.000 J [66.7%] long_function_b") == 1 56 | 57 | 58 | def test_profiler_retains_multiple_calls(): 59 | profiler = Profiler() 60 | 61 | with fake_time(): 62 | profiler.start() 63 | 64 | long_function_a() 65 | long_function_b() 66 | long_function_a() 67 | long_function_b() 68 | 69 | profiler.stop() 70 | 71 | print(profiler.output_text()) 72 | 73 | assert profiler.last_session 74 | frame = profiler.last_session.root_frame() 75 | assert frame 76 | assert frame.function == "test_profiler_retains_multiple_calls" 77 | assert len(frame.children) == 4 78 | 79 | 80 | def test_two_functions(): 81 | profiler = Profiler() 82 | 83 | with fake_time(): 84 | profiler.start() 85 | 86 | long_function_a() 87 | long_function_b() 88 | 89 | profiler.stop() 90 | 91 | print(profiler.output_text()) 92 | 93 | assert profiler.last_session 94 | 95 | frame = profiler.last_session.root_frame() 96 | 97 | assert frame 98 | assert frame.function == "test_two_functions" 99 | assert len(frame.children) == 2 100 | 101 | frame_b, frame_a = sorted( 102 | frame.children, key=lambda f: f.time(), reverse=True) 103 | 104 | assert frame_a.function == "long_function_a" 105 | assert frame_b.function == "long_function_b" 106 | 107 | # busy CI runners can be slow to wake up from the sleep. So we relax the 108 | # ranges a bit 109 | assert frame_a.time() == pytest.approx(0.25, abs=0.1) 110 | assert frame_b.time() == pytest.approx(0.5, abs=0.2) 111 | 112 | 113 | def test_context_manager(): 114 | with fake_time(): 115 | with Profiler() as profiler: 116 | long_function_a() 117 | long_function_b() 118 | 119 | assert profiler.last_session 120 | frame = profiler.last_session.root_frame() 121 | assert frame 122 | assert frame.function == "test_context_manager" 123 | assert len(frame.children) == 2 124 | 125 | 126 | def test_json_output(): 127 | with fake_time(): 128 | with Profiler() as profiler: 129 | long_function_a() 130 | long_function_b() 131 | 132 | output_data = profiler.output(renderers.JSONRenderer()) 133 | 134 | output = json.loads(output_data) 135 | assert "root_frame" in output 136 | 137 | root_frame = output["root_frame"] 138 | 139 | assert root_frame["function"] == "test_json_output" 140 | assert len(root_frame["children"]) == 2 141 | 142 | 143 | def test_empty_profile(monkeypatch): 144 | with Profiler() as profiler: 145 | pass 146 | profiler.output(renderer=renderers.ConsoleRenderer()) 147 | 148 | 149 | @ flaky_in_ci 150 | def test_state_management(): 151 | profiler = Profiler() 152 | 153 | assert profiler.last_session is None 154 | assert profiler.is_running == False 155 | 156 | profiler.start() 157 | 158 | assert profiler.last_session is None 159 | assert profiler.is_running == True 160 | 161 | busy_wait(0.1) 162 | 163 | profiler.stop() 164 | 165 | assert profiler.is_running == False 166 | assert profiler.last_session is not None 167 | assert profiler.last_session.duration == pytest.approx(0.1, rel=0.2) 168 | 169 | # test a second session, does it merge with the first? 170 | profiler.start() 171 | 172 | assert profiler.is_running == True 173 | busy_wait(0.1) 174 | 175 | profiler.stop() 176 | 177 | assert profiler.is_running == False 178 | 179 | assert profiler.last_session is not None 180 | assert profiler.last_session.duration == pytest.approx(0.2, rel=0.2) 181 | 182 | # test a reset 183 | profiler.reset() 184 | assert profiler.last_session is None 185 | 186 | # test a reset while running 187 | profiler.start() 188 | assert profiler.is_running == True 189 | profiler.reset() 190 | assert profiler.is_running == False 191 | assert profiler.last_session is None 192 | -------------------------------------------------------------------------------- /docs/guide.md: -------------------------------------------------------------------------------- 1 | User guide 2 | ---------- 3 | 4 | ### Installation 5 | 6 | ```{include} ../README.md 7 | --- 8 | relative-docs: docs/ 9 | relative-images: 10 | start-after: '' 11 | end-before: '' 12 | --- 13 | ``` 14 | 15 | ### Profile a Python script 16 | 17 | Call joulehunter directly from the command line. Instead of writing 18 | `python script.py`, type `joulehunter script.py`. Your script will run as 19 | normal, and at the end (or when you press `^C`), joulehunter will output a 20 | colored summary showing where most of the time was spent. 21 | 22 | Here are the options you can use: 23 | 24 | Usage: joulehunter [options] scriptfile [arg] ... 25 | 26 | Options: 27 | --version show program's version number and exit 28 | -h, --help show this help message and exit 29 | --load-prev=ID instead of running a script, load a previous report 30 | -m MODULE_NAME run library module as a script, like 'python -m 31 | module' 32 | --from-path (POSIX only) instead of the working directory, look 33 | for scriptfile in the PATH environment variable 34 | -o OUTFILE, --outfile=OUTFILE 35 | save to 36 | -r RENDERER, --renderer=RENDERER 37 | how the report should be rendered. One of: 'text', 38 | 'html', 'json', or python import path to a renderer 39 | class 40 | -t, --timeline render as a timeline - preserve ordering and don't 41 | condense repeated calls 42 | --hide=EXPR glob-style pattern matching the file paths whose 43 | frames to hide. Defaults to '*/lib/*'. 44 | --hide-regex=REGEX regex matching the file paths whose frames to hide. 45 | Useful if --hide doesn't give enough control. 46 | --show=EXPR glob-style pattern matching the file paths whose 47 | frames to show, regardless of --hide or --hide-regex. 48 | For example, use --show '*//*' to show frames 49 | within a library that would otherwise be hidden. 50 | --show-regex=REGEX regex matching the file paths whose frames to always 51 | show. Useful if --show doesn't give enough control. 52 | --show-all show everything 53 | --unicode (text renderer only) force unicode text output 54 | --no-unicode (text renderer only) force ascii text output 55 | --color (text renderer only) force ansi color text output 56 | --no-color (text renderer only) force no color text output 57 | 58 | **Protip:** `-r html` will give you a interactive profile report as HTML - you 59 | can really explore this way! 60 | 61 | ### Profile a specific chunk of code 62 | 63 | joulehunter also has a Python API. Just surround your code with joulehunter, 64 | like this: 65 | 66 | ```python 67 | from joulehunter import Profiler 68 | 69 | profiler = Profiler() 70 | profiler.start() 71 | 72 | # code you want to profile 73 | 74 | profiler.stop() 75 | 76 | profiler.print() 77 | ``` 78 | 79 | If you get "No samples were recorded." because your code executed in under 80 | 1ms, hooray! If you **still** want to instrument the code, set an interval 81 | value smaller than the default 0.001 (1 millisecond) like this: 82 | 83 | ```python 84 | profiler = Profiler(interval=0.0001) 85 | ... 86 | ``` 87 | 88 | Experiment with the interval value to see different depths, but keep in mind 89 | that smaller intervals could affect the performance overhead of profiling. 90 | 91 | **Protip:** To explore the profile in a web browser, use 92 | {meth}`profiler.open_in_browser() `. To 93 | save this HTML for later, use 94 | {meth}`profiler.output_html() `. 95 | 96 | ### Profile a web request in Django 97 | 98 | To profile Django web requests, add 99 | `joulehunter.middleware.ProfilerMiddleware` to `MIDDLEWARE_CLASSES` in your 100 | `settings.py`. 101 | 102 | Once installed, add `?profile` to the end of a request URL to activate the 103 | profiler. Your request will run as normal, but instead of getting the response, 104 | you'll get joulehunter's analysis of the request in a web page. 105 | 106 | If you're writing an API, it's not easy to change the URL when you want to 107 | profile something. In this case, add `joulehunter_PROFILE_DIR = 'profiles'` 108 | to your `settings.py`. joulehunter will profile every request and save the 109 | HTML output to the folder `profiles` in your working directory. 110 | 111 | If you want to show the profiling page depending on the request you can define 112 | `joulehunter_SHOW_CALLBACK` as dotted path to a function used for determining 113 | whether the page should show or not. 114 | You can provide your own function callback(request) which returns True or False 115 | in your settings.py. 116 | 117 | ```python 118 | def custom_show_joulehunter(request): 119 | return request.user.is_superuser 120 | 121 | 122 | joulehunter_SHOW_CALLBACK = "%s.custom_show_joulehunter" % __name__ 123 | ``` 124 | 125 | ### Profile a web request in Flask 126 | 127 | A simple setup to profile a Flask application is the following: 128 | 129 | ```python 130 | from flask import Flask, g, make_response, request 131 | app = Flask(__name__) 132 | 133 | @app.before_request 134 | def before_request(): 135 | if "profile" in request.args: 136 | g.profiler = Profiler() 137 | g.profiler.start() 138 | 139 | 140 | @app.after_request 141 | def after_request(response): 142 | if not hasattr(g, "profiler"): 143 | return response 144 | g.profiler.stop() 145 | output_html = g.profiler.output_html() 146 | return make_response(output_html) 147 | ``` 148 | 149 | This will check for the `?profile` query param on each request and if found, 150 | it starts profiling. After each request where the profiler was running it 151 | creates the html output and returns that instead of the actual response. 152 | 153 | ### Profile something else? 154 | 155 | I'd love to have more ways to profile using joulehunter - e.g. other 156 | web frameworks. PRs are encouraged! 157 | -------------------------------------------------------------------------------- /test/test_cmdline.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import sys 4 | from pathlib import Path 5 | 6 | import pytest 7 | 8 | # this script just does a busywait for 0.25 seconds. 9 | BUSY_WAIT_SCRIPT = """ 10 | import time, sys 11 | 12 | def do_nothing(): 13 | pass 14 | 15 | def busy_wait(duration): 16 | end_time = time.time() + duration 17 | 18 | while time.time() < end_time: 19 | do_nothing() 20 | 21 | def main(): 22 | print('sys.argv: ', sys.argv) 23 | busy_wait(0.25) 24 | 25 | 26 | if __name__ == '__main__': 27 | main() 28 | """ 29 | 30 | EXECUTION_DETAILS_SCRIPT = f""" 31 | #!{sys.executable} 32 | import sys, os 33 | print('__name__', __name__, file=sys.stderr) 34 | print('sys.argv', sys.argv, file=sys.stderr) 35 | print('sys.path', sys.path, file=sys.stderr) 36 | print('sys.executable', os.path.realpath(sys.executable), file=sys.stderr) 37 | print('os.getcwd()', os.getcwd(), file=sys.stderr) 38 | """.strip() 39 | 40 | 41 | @pytest.mark.skip(reason="we can't get available domaines unless we are in linux, we plan to moke later") 42 | @pytest.mark.parametrize( 43 | "joulehunter_invocation", 44 | (["joulehunter"], [sys.executable, "-m", "joulehunter"]), 45 | ) 46 | class TestCommandLine: 47 | def test_command_line(self, joulehunter_invocation, tmp_path: Path): 48 | busy_wait_py = tmp_path / "busy_wait.py" 49 | busy_wait_py.write_text(BUSY_WAIT_SCRIPT) 50 | 51 | # need to wrap Paths with str() due to CPython bug 33617 (fixed in Python 3.8) 52 | output = subprocess.check_output( 53 | [*joulehunter_invocation, str(busy_wait_py)]) 54 | 55 | assert "busy_wait" in str(output) 56 | assert "do_nothing" in str(output) 57 | 58 | def test_module_running(self, joulehunter_invocation, tmp_path: Path): 59 | (tmp_path / "busy_wait_module").mkdir() 60 | (tmp_path / "busy_wait_module" / "__init__.py").touch() 61 | (tmp_path / "busy_wait_module" / "__main__.py").write_text(BUSY_WAIT_SCRIPT) 62 | 63 | output = subprocess.check_output( 64 | [*joulehunter_invocation, "-m", "busy_wait_module"], cwd=tmp_path 65 | ) 66 | 67 | assert "busy_wait" in str(output) 68 | assert "do_nothing" in str(output) 69 | 70 | def test_single_file_module_running(self, joulehunter_invocation, tmp_path: Path): 71 | busy_wait_py = tmp_path / "busy_wait.py" 72 | busy_wait_py.write_text(BUSY_WAIT_SCRIPT) 73 | 74 | output = subprocess.check_output( 75 | [*joulehunter_invocation, "-m", "busy_wait"], cwd=tmp_path 76 | ) 77 | 78 | assert "busy_wait" in str(output) 79 | assert "do_nothing" in str(output) 80 | 81 | def test_running_yourself_as_module(self, joulehunter_invocation): 82 | subprocess.check_call( 83 | [*joulehunter_invocation, "-m", "joulehunter"], 84 | ) 85 | 86 | def test_path(self, joulehunter_invocation, tmp_path: Path, monkeypatch): 87 | if sys.platform == "win32": 88 | pytest.skip("--from-path is not supported on Windows") 89 | 90 | program_path = tmp_path / "pyi_test_program" 91 | 92 | program_path.write_text(BUSY_WAIT_SCRIPT) 93 | program_path.chmod(0x755) 94 | monkeypatch.setenv("PATH", str(tmp_path), prepend=os.pathsep) 95 | 96 | subprocess.check_call( 97 | [*joulehunter_invocation, "--from-path", "--", "pyi_test_program"], 98 | ) 99 | 100 | def test_script_execution_details(self, joulehunter_invocation, tmp_path: Path): 101 | program_path = tmp_path / "program.py" 102 | program_path.write_text(EXECUTION_DETAILS_SCRIPT) 103 | 104 | process_pyi = subprocess.run( 105 | [*joulehunter_invocation, str(program_path), "arg1", "arg2"], 106 | stderr=subprocess.PIPE, 107 | check=True, 108 | universal_newlines=True, 109 | ) 110 | process_native = subprocess.run( 111 | [sys.executable, str(program_path), "arg1", "arg2"], 112 | stderr=subprocess.PIPE, 113 | check=True, 114 | universal_newlines=True, 115 | ) 116 | 117 | print("process_pyi.stderr", process_pyi.stderr) 118 | print("process_native.stderr", process_native.stderr) 119 | assert process_pyi.stderr == process_native.stderr 120 | 121 | def test_module_execution_details(self, joulehunter_invocation, tmp_path: Path): 122 | (tmp_path / "test_module").mkdir() 123 | (tmp_path / "test_module" / "__init__.py").touch() 124 | (tmp_path / "test_module" / "__main__.py").write_text(EXECUTION_DETAILS_SCRIPT) 125 | 126 | process_pyi = subprocess.run( 127 | [*joulehunter_invocation, "-m", "test_module", "arg1", "arg2"], 128 | # stderr=subprocess.PIPE, 129 | check=True, 130 | cwd=tmp_path, 131 | universal_newlines=True, 132 | ) 133 | process_native = subprocess.run( 134 | [sys.executable, "-m", "test_module", "arg1", "arg2"], 135 | # stderr=subprocess.PIPE, 136 | check=True, 137 | cwd=tmp_path, 138 | universal_newlines=True, 139 | ) 140 | 141 | print("process_pyi.stderr", process_pyi.stderr) 142 | print("process_native.stderr", process_native.stderr) 143 | assert process_pyi.stderr == process_native.stderr 144 | 145 | def test_path_execution_details(self, joulehunter_invocation, tmp_path: Path, monkeypatch): 146 | if sys.platform == "win32": 147 | pytest.skip("--from-path is not supported on Windows") 148 | 149 | program_path = tmp_path / "pyi_test_program" 150 | program_path.write_text(EXECUTION_DETAILS_SCRIPT) 151 | program_path.chmod(0x755) 152 | monkeypatch.setenv("PATH", str(tmp_path), prepend=os.pathsep) 153 | 154 | process_pyi = subprocess.run( 155 | [ 156 | *joulehunter_invocation, 157 | "--from-path", 158 | "--", 159 | "pyi_test_program", 160 | "arg1", 161 | "arg2", 162 | ], 163 | stderr=subprocess.PIPE, 164 | check=True, 165 | universal_newlines=True, 166 | ) 167 | process_native = subprocess.run( 168 | ["pyi_test_program", "arg1", "arg2"], 169 | stderr=subprocess.PIPE, 170 | check=True, 171 | universal_newlines=True, 172 | ) 173 | 174 | print("process_pyi.stderr", process_pyi.stderr) 175 | print("process_native.stderr", process_native.stderr) 176 | assert process_pyi.stderr == process_native.stderr 177 | -------------------------------------------------------------------------------- /html_renderer/src/Frame.vue: -------------------------------------------------------------------------------- 1 | 51 | 52 | 138 | 139 | 223 | 224 | 241 | -------------------------------------------------------------------------------- /joulehunter/renderers/console.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | import joulehunter 3 | from joulehunter import processors 4 | from joulehunter.frame import BaseFrame 5 | from joulehunter.renderers.base import ProcessorList, Renderer 6 | from joulehunter.session import Session 7 | from joulehunter.util import truncate 8 | 9 | # pyright: strict 10 | 11 | 12 | class ConsoleRenderer(Renderer): 13 | """ 14 | Produces text-based output, suitable for text files or ANSI-compatible 15 | consoles. 16 | """ 17 | 18 | def __init__(self, unicode: bool = False, color: bool = False, **kwargs: Any): 19 | """ 20 | :param unicode: Use unicode, like box-drawing characters in the output. 21 | :param color: Enable color support, using ANSI color sequences. 22 | """ 23 | super().__init__(**kwargs) 24 | 25 | self.unicode = unicode 26 | self.color = color 27 | self.colors = self.colors_enabled if color else self.colors_disabled 28 | 29 | def render(self, session: Session): 30 | result = self.render_preamble(session) 31 | 32 | frame = self.preprocess(session.root_frame()) 33 | 34 | if frame is None: 35 | result += "No samples were recorded.\n\n" 36 | return result 37 | 38 | self.root_frame = frame 39 | result += self.render_frame( 40 | self.root_frame, total_energy=self.root_frame.time()) 41 | result += "\n" 42 | 43 | return result 44 | 45 | # pylint: disable=W1401 46 | def render_preamble(self, session: Session): 47 | c = self.colors 48 | lines = [ 49 | "", 50 | f"{c.bold}{c.cyan} . _ / _ /_ _ _/_ _ _ {c.end}{c.end}", 51 | f"{c.bold}{c.cyan} / /_//_// /_'/ //_// // /_'/ {c.end}{c.end}", 52 | f"{c.bold}{c.cyan}|/ {c.end}{c.end}", 53 | ] 54 | lines[1] += f"{c.cyan}Duration:{c.end} {session.duration:<12.3f}" 55 | lines[2] += f"{c.cyan}Package:{c.end} {session.domain_names[0]:<12}" 56 | lines[3] += f"{c.cyan}Program:{c.end} {session.program}" 57 | 58 | lines[1] += f"{c.cyan}Samples:{c.end} {session.sample_count}" 59 | if len(session.domain_names) == 2: 60 | lines[2] += f"{c.cyan}Component:{c.end} {session.domain_names[1]}" 61 | lines.append("") 62 | lines.append("") 63 | 64 | return "\n".join(lines) 65 | 66 | def render_frame( 67 | self, frame: BaseFrame, 68 | indent: str = "", 69 | child_indent: str = "", 70 | total_energy: float = None) -> str: 71 | if not frame.group or ( 72 | frame.group.root == frame 73 | or frame.total_self_time > 0.2 * self.root_frame.time() 74 | or frame in frame.group.exit_frames 75 | ): 76 | time_str = (self._ansi_color_for_time(frame) 77 | + f"{frame.time():.3f} J") 78 | if total_energy: 79 | percentage = frame.time() / total_energy * 100 80 | time_str += f" [{percentage:.1f}%]" 81 | time_str += self.colors.end 82 | function_color = self._ansi_color_for_function(frame) 83 | result = "{indent}{time_str} {function_color}{function}{c.end} {c.faint}{code_position}{c.end}\n".format( 84 | indent=indent, 85 | time_str=time_str, 86 | function_color=function_color, 87 | function=frame.function, 88 | code_position=frame.code_position_short, 89 | c=self.colors, 90 | ) 91 | if self.unicode: 92 | indents = {"├": "├─ ", "│": "│ ", "└": "└─ ", " ": " "} 93 | else: 94 | indents = {"├": "|- ", "│": "| ", "└": "`- ", " ": " "} 95 | 96 | if frame.group and frame.group.root == frame: 97 | result += "{indent}[{count} frames hidden] {c.faint}{libraries}{c.end}\n".format( 98 | indent=child_indent + " ", 99 | count=len(frame.group.frames), 100 | libraries=truncate(", ".join(frame.group.libraries), 40), 101 | c=self.colors, 102 | ) 103 | for key in indents: 104 | indents[key] = " " 105 | else: 106 | result = "" 107 | indents = {"├": "", "│": "", "└": "", " ": ""} 108 | 109 | if frame.children: 110 | last_child = frame.children[-1] 111 | 112 | for child in frame.children: 113 | if child is not last_child: 114 | c_indent = child_indent + indents["├"] 115 | cc_indent = child_indent + indents["│"] 116 | else: 117 | c_indent = child_indent + indents["└"] 118 | cc_indent = child_indent + indents[" "] 119 | result += self.render_frame( 120 | child, indent=c_indent, child_indent=cc_indent, 121 | total_energy=total_energy) 122 | 123 | return result 124 | 125 | def _ansi_color_for_time(self, frame: BaseFrame): 126 | proportion_of_total = frame.time() / self.root_frame.time() 127 | 128 | if proportion_of_total > 0.6: 129 | return self.colors.red 130 | elif proportion_of_total > 0.2: 131 | return self.colors.yellow 132 | elif proportion_of_total > 0.05: 133 | return self.colors.green 134 | else: 135 | return self.colors.bright_green + self.colors.faint 136 | 137 | def _ansi_color_for_function(self, frame: BaseFrame): 138 | if frame.is_application_code: 139 | return self.colors.bg_dark_blue_255 + self.colors.white_255 140 | else: 141 | return "" 142 | 143 | def default_processors(self) -> ProcessorList: 144 | return [ 145 | processors.remove_importlib, 146 | processors.merge_consecutive_self_time, 147 | processors.aggregate_repeated_calls, 148 | processors.group_library_frames_processor, 149 | processors.remove_unnecessary_self_time_nodes, 150 | processors.remove_irrelevant_nodes, 151 | ] 152 | 153 | class colors_enabled: 154 | red = "\033[31m" 155 | green = "\033[32m" 156 | yellow = "\033[33m" 157 | blue = "\033[34m" 158 | cyan = "\033[36m" 159 | bright_green = "\033[92m" 160 | white = "\033[37m\033[97m" 161 | 162 | bg_dark_blue_255 = "\033[48;5;24m" 163 | white_255 = "\033[38;5;15m" 164 | 165 | bold = "\033[1m" 166 | faint = "\033[2m" 167 | 168 | end = "\033[0m" 169 | 170 | class colors_disabled: 171 | red = "" 172 | green = "" 173 | yellow = "" 174 | blue = "" 175 | cyan = "" 176 | bright_green = "" 177 | white = "" 178 | 179 | bg_dark_blue_255 = "" 180 | white_255 = "" 181 | 182 | bold = "" 183 | faint = "" 184 | 185 | end = "" 186 | -------------------------------------------------------------------------------- /docs/how-it-works.md: -------------------------------------------------------------------------------- 1 | How it works 2 | ============ 3 | 4 | joulehunter interrupts the program every 1ms[^interval] and records the entire stack at 5 | that point. It does this using a C extension and `PyEval_SetProfile`, but only 6 | taking readings every 1ms. Check out [this blog post](http://joerick.me/posts/2017/12/15/pyinstrument-20/) for more info. 7 | 8 | [^interval]: Or, your configured ``interval``. 9 | 10 | You might be surprised at how few samples make up a report, but don't worry, 11 | it won't decrease accuracy. The default interval of 1ms is a lower bound for 12 | recording a stackframe, but if there is a long time spent in a single function 13 | call, it will be recorded at the end of that call. So effectively those 14 | samples were 'bunched up' and recorded at the end. 15 | 16 | ## Statistical profiling (not tracing) 17 | 18 | joulehunter is a statistical profiler - it doesn't track every 19 | function call that your program makes. Instead, it's recording the call stack 20 | every 1ms. 21 | 22 | That gives some advantages over other profilers. Firstly, statistical 23 | profilers are much lower-overhead than tracing profilers. 24 | 25 | | | Django template render × 4000 | Overhead 26 | | -------------|:---------------------------------------------------|---------: 27 | | Base | `████████████████ ` 0.33s | 28 | | | | 29 | | joulehunter | `████████████████████ ` 0.43s | 30% 30 | | cProfile | `█████████████████████████████ ` 0.61s | 84% 31 | | profile | `██████████████████████████████████...██` 6.79s | 2057% 32 | 33 | But low overhead is also important because it can distort the results. When 34 | using a tracing profiler, code that makes a lot of Python function calls 35 | invokes the profiler a lot, making it slower. This distorts the 36 | results, and might lead you to optimise the wrong part of your program! 37 | 38 | ## Full-stack recording 39 | 40 | The standard Python profilers [`profile`][1] and [`cProfile`][2] show you a 41 | big list of functions, ordered by the time spent in each function. 42 | This is great, but it can be difficult to interpret _why_ those functions are 43 | getting called. It's more helpful to know why those functions are called, and 44 | which parts of user code were involved. 45 | 46 | [1]: http://docs.python.org/2/library/profile.html#module-profile 47 | [2]: http://docs.python.org/2/library/profile.html#module-cProfile 48 | 49 | For example, let's say I want to figure out why a web request in Django is 50 | slow. If I use cProfile, I might get this: 51 | 52 | 151940 function calls (147672 primitive calls) in 1.696 seconds 53 | 54 | Ordered by: cumulative time 55 | 56 | ncalls tottime percall cumtime percall filename:lineno(function) 57 | 1 0.000 0.000 1.696 1.696 profile:0( at 0x1053d6a30, file "./manage.py", line 2>) 58 | 1 0.001 0.001 1.693 1.693 manage.py:2() 59 | 1 0.000 0.000 1.586 1.586 __init__.py:394(execute_from_command_line) 60 | 1 0.000 0.000 1.586 1.586 __init__.py:350(execute) 61 | 1 0.000 0.000 1.142 1.142 __init__.py:254(fetch_command) 62 | 43 0.013 0.000 1.124 0.026 __init__.py:1() 63 | 388 0.008 0.000 1.062 0.003 re.py:226(_compile) 64 | 158 0.005 0.000 1.048 0.007 sre_compile.py:496(compile) 65 | 1 0.001 0.001 1.042 1.042 __init__.py:78(get_commands) 66 | 153 0.001 0.000 1.036 0.007 re.py:188(compile) 67 | 106/102 0.001 0.000 1.030 0.010 __init__.py:52(__getattr__) 68 | 1 0.000 0.000 1.029 1.029 __init__.py:31(_setup) 69 | 1 0.000 0.000 1.021 1.021 __init__.py:57(_configure_logging) 70 | 2 0.002 0.001 1.011 0.505 log.py:1() 71 | 72 | It's often hard to understand how your own code relates to these traces. 73 | 74 | joulehunter records the entire stack, so tracking expensive calls is much 75 | easier. It also hides library frames by default, letting you focus on your 76 | app/module is affecting performance. 77 | 78 | ``` 79 | _ ._ __/__ _ _ _ _ _/_ Recorded: 14:53:35 Samples: 131 80 | /_//_/// /_\ / //_// / //_'/ // Duration: 3.131 CPU time: 0.195 81 | / _/ v3.0.0b3 82 | 83 | Program: examples/django_example/manage.py runserver --nothreading --noreload 84 | 85 | 3.131 manage.py:2 86 | └─ 3.118 execute_from_command_line django/core/management/__init__.py:378 87 | [473 frames hidden] django, socketserver, selectors, wsgi... 88 | 2.836 select selectors.py:365 89 | 0.126 _get_response django/core/handlers/base.py:96 90 | └─ 0.126 hello_world django_example/views.py:4 91 | ``` 92 | 93 | ## 'Wall-clock' time (not CPU time) 94 | 95 | joulehunter records duration using 'wall-clock' time. When you're writing a 96 | program that downloads data, reads files, and talks to databases, all that 97 | time is *included* in the tracked time by joulehunter. 98 | 99 | That's really important when debugging performance problems, since Python is 100 | often used as a 'glue' language between other services. The problem might not 101 | be in your program, but you should still be able to find why it's slow. 102 | 103 | ## Async profiling 104 | 105 | joulehunter can profile async programs that use `async` and `await`. This 106 | async support works by tracking the 'context' of execution, as provided by the 107 | built-in [contextvars] module. 108 | 109 | [contextvars]: https://docs.python.org/3/library/contextvars.html 110 | 111 | When you start a Profiler with the {py:attr}`async_mode ` `enabled` or `strict` (not `disabled`), that Profiler is attached to the current async context. 112 | 113 | When profiling, joulehunter keeps an eye on the context. When execution exits 114 | the context, it captures the `await` stack that caused the context to exit. 115 | Any time spent outside the context is attributed to the that halted execution 116 | of the `await`. 117 | 118 | Async contexts are inherited, so tasks started when a profiler is active are 119 | also profiled. 120 | 121 |
122 | 123 | ![Async context inheritance](img/async-context.svg) 124 | 125 | joulehunter supports async mode with Asyncio and Trio, other `async`/`await` 126 | frameworks should work as long as they use [contextvars]. 127 | 128 | [Greenlet] doesn't use `async` and `await`, and alters the Python stack during 129 | execution, so is not fully supported. However, because greenlet also supports 130 | [contextvars], we can limit profiling to one green thread, using `strict` 131 | mode. In `strict` mode, whenever your green thread is halted the time will be 132 | tracked in an `` frame. Alternatively, if you want to see 133 | what's happening when your green thread is halted, you can use 134 | `async_mode='disabled'` - just be aware that readouts might be misleading if 135 | multiple tasks are running concurrently. 136 | 137 | [greenlet]: https://pypi.org/project/greenlet/ 138 | -------------------------------------------------------------------------------- /joulehunter/session.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | from collections import deque 5 | from typing import Any, List, Tuple, cast 6 | 7 | from joulehunter.frame import BaseFrame, DummyFrame, Frame, SelfTimeFrame 8 | from joulehunter.typing import PathOrStr 9 | 10 | # pyright: strict 11 | 12 | 13 | ASSERTION_MESSAGE = ( 14 | "Please raise an issue at https://github.com/powerapi-ng/joulehunter/issues and " 15 | "let me know how you caused this error!" 16 | ) 17 | 18 | FrameRecordType = Tuple[List[str], float] 19 | 20 | 21 | class Session: 22 | def __init__( 23 | self, 24 | frame_records: list[FrameRecordType], 25 | start_time: float, 26 | duration: float, 27 | sample_count: int, 28 | start_call_stack: list[str], 29 | program: str, 30 | domain_names: list[str], 31 | ): 32 | """Session() 33 | 34 | Represents a profile session, contains the data collected during a profile session. 35 | 36 | :meta private: 37 | """ 38 | self.frame_records = frame_records 39 | self.start_time = start_time 40 | self.duration = duration 41 | self.sample_count = sample_count 42 | self.start_call_stack = start_call_stack 43 | self.program = program 44 | self.domain_names = domain_names 45 | 46 | @staticmethod 47 | def load(filename: PathOrStr) -> Session: 48 | """ 49 | Load a previously saved session from disk. 50 | 51 | :param filename: The path to load from. 52 | :rtype: Session 53 | """ 54 | with open(filename) as f: 55 | return Session.from_json(json.load(f)) 56 | 57 | def save(self, filename: PathOrStr) -> None: 58 | """ 59 | Saves a Session object to disk, in a JSON format. 60 | 61 | :param filename: The path to save to. Using the ``.pyisession`` extension is recommended. 62 | """ 63 | with open(filename, "w") as f: 64 | json.dump(self.to_json(), f) 65 | 66 | def to_json(self): 67 | return { 68 | "frame_records": self.frame_records, 69 | "start_time": self.start_time, 70 | "duration": self.duration, 71 | "sample_count": self.sample_count, 72 | "start_call_stack": self.start_call_stack, 73 | "program": self.program, 74 | "domain_names": self.domain_names, 75 | } 76 | 77 | @staticmethod 78 | def from_json(json_dict: dict[str, Any]): 79 | return Session( 80 | frame_records=json_dict["frame_records"], 81 | start_time=json_dict["start_time"], 82 | duration=json_dict["duration"], 83 | sample_count=json_dict["sample_count"], 84 | start_call_stack=json_dict["start_call_stack"], 85 | program=json_dict["program"], 86 | domain_names=json_dict["domain_names"], 87 | ) 88 | 89 | @staticmethod 90 | def combine(session1: Session, session2: Session) -> Session: 91 | """ 92 | Combines two :class:`Session` objects. 93 | 94 | Sessions that are joined in this way probably shouldn't be interpreted 95 | as timelines, because the samples are simply concatenated. But 96 | aggregate views (the default) of this data will work. 97 | 98 | :rtype: Session 99 | """ 100 | if session1.start_time > session2.start_time: 101 | # swap them around so that session1 is the first one 102 | session1, session2 = session2, session1 103 | 104 | return Session( 105 | frame_records=session1.frame_records + session2.frame_records, 106 | start_time=session1.start_time, 107 | duration=session1.duration + session2.duration, 108 | sample_count=session1.sample_count + session2.sample_count, 109 | start_call_stack=session1.start_call_stack, 110 | program=session1.program, 111 | domain_names=session1.domain_names, 112 | ) 113 | 114 | def root_frame(self, trim_stem: bool = True) -> BaseFrame | None: 115 | """ 116 | Parses the internal frame records and returns a tree of :class:`Frame` 117 | objects. This object can be renderered using a :class:`Renderer` 118 | object. 119 | 120 | :rtype: A :class:`Frame` object, or None if the session is empty. 121 | """ 122 | root_frame = None 123 | 124 | frame_stack: list[BaseFrame] = [] 125 | 126 | for frame_tuple in self.frame_records: 127 | identifier_stack = frame_tuple[0] 128 | time = frame_tuple[1] 129 | 130 | stack_depth = 0 131 | 132 | # now we must create a stack of frame objects and assign this time to the leaf 133 | for stack_depth, frame_identifier in enumerate(identifier_stack): 134 | if stack_depth < len(frame_stack): 135 | if frame_identifier != frame_stack[stack_depth].identifier: 136 | # trim any frames after and including this one 137 | del frame_stack[stack_depth:] 138 | 139 | if stack_depth >= len(frame_stack): 140 | frame = BaseFrame.new_subclass_with_identifier(frame_identifier) 141 | frame_stack.append(frame) 142 | 143 | if stack_depth == 0: 144 | # There should only be one root frame, as far as I know 145 | assert root_frame is None, ASSERTION_MESSAGE 146 | root_frame = frame 147 | else: 148 | parent = cast(Frame, frame_stack[stack_depth - 1]) 149 | parent.add_child(frame) 150 | 151 | # trim any extra frames 152 | del frame_stack[stack_depth + 1 :] 153 | 154 | # assign the time to the final frame in the stack 155 | final_frame = frame_stack[-1] 156 | if isinstance(final_frame, DummyFrame): 157 | final_frame.self_time += time 158 | elif isinstance(final_frame, Frame): 159 | final_frame.add_child(SelfTimeFrame(self_time=time)) 160 | else: 161 | raise Exception("unknown frame type") 162 | 163 | if root_frame is None: 164 | return None 165 | 166 | if trim_stem: 167 | root_frame = self._trim_stem(root_frame) 168 | 169 | return root_frame 170 | 171 | def _trim_stem(self, frame: BaseFrame): 172 | # trim the start of the tree before any branches. 173 | # we also don't want to trim beyond the call to profiler.start() 174 | 175 | start_stack = deque(self.start_call_stack) 176 | if start_stack.popleft() != frame.identifier: 177 | # the frame doesn't match where the profiler was started. Don't trim. 178 | return frame 179 | 180 | while frame.self_time == 0 and len(frame.children) == 1: 181 | # check child matches the start_call_stack, otherwise stop descending 182 | if len(start_stack) == 0 or frame.children[0].identifier != start_stack.popleft(): 183 | break 184 | 185 | frame = frame.children[0] 186 | 187 | frame.remove_from_parent() 188 | return frame 189 | -------------------------------------------------------------------------------- /joulehunter/processors.py: -------------------------------------------------------------------------------- 1 | """ 2 | Processors are functions that take a Frame object, and mutate the tree to perform some task. 3 | 4 | They can mutate the tree in-place, but also can change the root frame, they should always be 5 | called like:: 6 | 7 | frame = processor(frame, options=...) 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import re 13 | from operator import methodcaller 14 | from typing import Any, Callable, Dict, Union 15 | 16 | from joulehunter.frame import BaseFrame, Frame, FrameGroup, SelfTimeFrame 17 | 18 | # pyright: strict 19 | 20 | 21 | ProcessorType = Callable[..., Union[BaseFrame, None]] 22 | ProcessorOptions = Dict[str, Any] 23 | 24 | 25 | def remove_importlib(frame: BaseFrame | None, options: ProcessorOptions) -> BaseFrame | None: 26 | """ 27 | Removes `` BaseFrame | None: 50 | """ 51 | Converts a timeline into a time-aggregate summary. 52 | 53 | Adds together calls along the same call stack, so that repeated calls appear as the same 54 | frame. Removes time-linearity - frames are sorted according to total time spent. 55 | 56 | Useful for outputs that display a summary of execution (e.g. text and html outputs) 57 | """ 58 | if frame is None: 59 | return None 60 | 61 | children_by_identifier: dict[str, BaseFrame] = {} 62 | 63 | # iterate over a copy of the children since it's going to mutate while we're iterating 64 | for child in frame.children: 65 | if child.identifier in children_by_identifier: 66 | aggregate_frame = children_by_identifier[child.identifier] 67 | 68 | # combine the two frames, putting the children and self_time into the aggregate frame. 69 | aggregate_frame.self_time += child.self_time 70 | if child.children: 71 | if not isinstance(aggregate_frame, Frame): 72 | raise Exception("cannot aggregate children into a DummyFrame") 73 | aggregate_frame.add_children(child.children) 74 | 75 | # remove this frame, it's been incorporated into aggregate_frame 76 | child.remove_from_parent() 77 | else: 78 | # never seen this identifier before. It becomes the aggregate frame. 79 | children_by_identifier[child.identifier] = child 80 | 81 | # recurse into the children 82 | for child in frame.children: 83 | aggregate_repeated_calls(child, options=options) 84 | 85 | # sort the children by time 86 | # it's okay to use the internal _children list, sinde we're not changing the tree 87 | # structure. 88 | frame._children.sort(key=methodcaller("time"), reverse=True) # type: ignore # noqa 89 | 90 | return frame 91 | 92 | 93 | def group_library_frames_processor( 94 | frame: BaseFrame | None, options: ProcessorOptions 95 | ) -> BaseFrame | None: 96 | """ 97 | Groups frames that should be hidden into :class:`FrameGroup` objects, 98 | according to ``hide_regex`` and ``show_regex`` in the options dict. If 99 | both match, 'show' has precedence. 100 | 101 | Single frames are not grouped, there must be at least two frames in a 102 | group. 103 | """ 104 | if frame is None: 105 | return None 106 | 107 | hide_regex: str | None = options.get("hide_regex") 108 | show_regex: str | None = options.get("show_regex") 109 | 110 | def should_be_hidden(frame: BaseFrame): 111 | frame_file_path = frame.file_path or "" 112 | 113 | should_show = (show_regex is not None) and re.match(show_regex, frame_file_path) 114 | should_hide = (hide_regex is not None) and re.match(hide_regex, frame_file_path) 115 | 116 | # check for explicit user show/hide rules. 'show' has precedence. 117 | if should_show: 118 | return False 119 | if should_hide: 120 | return True 121 | 122 | return not frame.is_application_code 123 | 124 | def add_frames_to_group(frame: BaseFrame, group: FrameGroup): 125 | group.add_frame(frame) 126 | for child in frame.children: 127 | if should_be_hidden(child): 128 | add_frames_to_group(child, group) 129 | 130 | for child in frame.children: 131 | if not child.group and ( 132 | should_be_hidden(child) and any(should_be_hidden(cc) for cc in child.children) 133 | ): 134 | group = FrameGroup(child) 135 | add_frames_to_group(child, group) 136 | 137 | group_library_frames_processor(child, options=options) 138 | 139 | return frame 140 | 141 | 142 | def merge_consecutive_self_time( 143 | frame: BaseFrame | None, options: ProcessorOptions 144 | ) -> BaseFrame | None: 145 | """ 146 | Combines consecutive 'self time' frames. 147 | """ 148 | if frame is None: 149 | return None 150 | 151 | previous_self_time_frame = None 152 | 153 | for child in frame.children: 154 | if isinstance(child, SelfTimeFrame): 155 | if previous_self_time_frame: 156 | # merge 157 | previous_self_time_frame.self_time += child.self_time 158 | child.remove_from_parent() 159 | else: 160 | # keep a reference, maybe it'll be added to on the next loop 161 | previous_self_time_frame = child 162 | else: 163 | previous_self_time_frame = None 164 | 165 | for child in frame.children: 166 | merge_consecutive_self_time(child, options=options) 167 | 168 | return frame 169 | 170 | 171 | def remove_unnecessary_self_time_nodes( 172 | frame: BaseFrame | None, options: ProcessorOptions 173 | ) -> BaseFrame | None: 174 | """ 175 | When a frame has only one child, and that is a self-time frame, remove 176 | that node and move the time to parent, since it's unnecessary - it 177 | clutters the output and offers no additional information. 178 | """ 179 | if frame is None: 180 | return None 181 | 182 | if len(frame.children) == 1 and isinstance(frame.children[0], SelfTimeFrame): 183 | child = frame.children[0] 184 | frame.self_time += child.self_time 185 | child.remove_from_parent() 186 | 187 | for child in frame.children: 188 | remove_unnecessary_self_time_nodes(child, options=options) 189 | 190 | return frame 191 | 192 | 193 | def remove_irrelevant_nodes( 194 | frame: BaseFrame | None, options: ProcessorOptions, total_time: float | None = None 195 | ) -> BaseFrame | None: 196 | """ 197 | Remove nodes that represent less than e.g. 1% of the output. 198 | """ 199 | if frame is None: 200 | return None 201 | 202 | if total_time is None: 203 | total_time = frame.time() 204 | 205 | filter_threshold = options.get("filter_threshold", 0.01) 206 | 207 | for child in frame.children: 208 | proportion_of_total = child.time() / total_time 209 | 210 | if proportion_of_total < filter_threshold: 211 | frame.self_time += child.time() 212 | child.remove_from_parent() 213 | 214 | for child in frame.children: 215 | remove_irrelevant_nodes(child, options=options, total_time=total_time) 216 | 217 | return frame 218 | -------------------------------------------------------------------------------- /joulehunter/stack_sampler.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import threading 4 | import timeit 5 | import types 6 | from contextvars import ContextVar 7 | from typing import Any, Callable, List, NamedTuple, Optional, Union 8 | 9 | from joulehunter.low_level.stat_profile import setstatprofile 10 | from joulehunter.typing import LiteralStr 11 | 12 | # pyright: strict 13 | 14 | 15 | thread_locals = threading.local() 16 | 17 | StackSamplerSubscriberTarget = Callable[[ 18 | List[str], float, Optional["AsyncState"]], None] 19 | 20 | 21 | class StackSamplerSubscriber: 22 | def __init__( 23 | self, 24 | *, 25 | target: StackSamplerSubscriberTarget, 26 | desired_interval: float, 27 | bound_to_async_context: bool, 28 | async_state: AsyncState | None, 29 | ) -> None: 30 | self.target = target 31 | self.desired_interval = desired_interval 32 | self.bound_to_async_context = bound_to_async_context 33 | self.async_state = async_state 34 | 35 | 36 | active_profiler_context_var: ContextVar[object | None] = ContextVar( 37 | "active_profiler_context_var", default=None 38 | ) 39 | 40 | 41 | class StackSampler: 42 | """Manages setstatprofile for Profilers on a single thread""" 43 | 44 | subscribers: list[StackSamplerSubscriber] 45 | current_sampling_interval: float | None 46 | last_profile_time: float 47 | timer_func: Callable[[], float] 48 | 49 | def __init__(self, timer_func) -> None: 50 | self.subscribers = [] 51 | self.current_sampling_interval = None 52 | self.last_profile_time = 0.0 53 | self.timer_func = timer_func 54 | 55 | def subscribe( 56 | self, 57 | target: StackSamplerSubscriberTarget, 58 | desired_interval: float, 59 | use_async_context: bool, 60 | ): 61 | if use_async_context: 62 | if active_profiler_context_var.get() is not None: 63 | raise RuntimeError( 64 | "There is already a profiler running. You cannot run multiple profilers in the same thread or async context, unless you disable async support." 65 | ) 66 | active_profiler_context_var.set(target) 67 | 68 | self.subscribers.append( 69 | StackSamplerSubscriber( 70 | target=target, 71 | desired_interval=desired_interval, 72 | bound_to_async_context=use_async_context, 73 | async_state=AsyncState( 74 | "in_context") if use_async_context else None, 75 | ) 76 | ) 77 | self._update() 78 | 79 | def unsubscribe(self, target: StackSamplerSubscriberTarget): 80 | try: 81 | subscriber = next( 82 | s for s in self.subscribers if s.target == target) # type: ignore 83 | except StopIteration: 84 | raise StackSampler.SubscriberNotFound() 85 | 86 | if subscriber.bound_to_async_context: 87 | # (don't need to use context_var.reset() because we verified it was 88 | # None before we started) 89 | active_profiler_context_var.set(None) 90 | 91 | self.subscribers.remove(subscriber) 92 | 93 | self._update() 94 | 95 | def _update(self): 96 | if len(self.subscribers) == 0: 97 | self._stop_sampling() 98 | return 99 | 100 | min_subscribers_interval = min( 101 | s.desired_interval for s in self.subscribers) 102 | 103 | if self.current_sampling_interval != min_subscribers_interval: 104 | self._start_sampling(interval=min_subscribers_interval) 105 | 106 | def _start_sampling(self, interval: float): 107 | self.current_sampling_interval = interval 108 | if self.last_profile_time == 0.0: 109 | self.last_profile_time = self._timer() 110 | setstatprofile(self._sample, interval, 111 | active_profiler_context_var, self.timer_func) 112 | 113 | def _stop_sampling(self): 114 | setstatprofile(None) 115 | self.current_sampling_interval = None 116 | self.last_profile_time = 0.0 117 | 118 | def _sample(self, frame: types.FrameType, event: str, arg: Any): 119 | if event == "context_changed": 120 | new, old, coroutine_stack = arg 121 | 122 | for subscriber in self.subscribers: 123 | if subscriber.target == old: 124 | assert subscriber.bound_to_async_context 125 | full_stack = build_call_stack(frame, event, arg) 126 | if coroutine_stack: 127 | full_stack.extend(reversed(coroutine_stack)) 128 | subscriber.async_state = AsyncState( 129 | "out_of_context_awaited", info=full_stack 130 | ) 131 | else: 132 | subscriber.async_state = AsyncState( 133 | "out_of_context_unknown", info=full_stack 134 | ) 135 | elif subscriber.target == new: 136 | assert subscriber.bound_to_async_context 137 | subscriber.async_state = AsyncState("in_context") 138 | else: 139 | now = self._timer() 140 | time_since_last_sample = now - self.last_profile_time 141 | 142 | call_stack = build_call_stack(frame, event, arg) 143 | 144 | for subscriber in self.subscribers: 145 | subscriber.target( 146 | call_stack, time_since_last_sample, subscriber.async_state) 147 | 148 | self.last_profile_time = now 149 | 150 | def _timer(self): 151 | return self.timer_func() 152 | 153 | class SubscriberNotFound(Exception): 154 | pass 155 | 156 | 157 | def get_stack_sampler(timer_func) -> StackSampler: 158 | """ 159 | Gets the stack sampler for the current thread, creating it if necessary 160 | """ 161 | if not hasattr(thread_locals, "stack_sampler"): 162 | thread_locals.stack_sampler = StackSampler(timer_func) 163 | return thread_locals.stack_sampler 164 | 165 | 166 | def build_call_stack(frame: types.FrameType | None, event: str, arg: Any) -> list[str]: 167 | call_stack: list[str] = [] 168 | 169 | if event == "call": 170 | # if we're entering a function, the time should be attributed to 171 | # the caller 172 | frame = frame.f_back if frame else None 173 | elif event == "c_return" or event == "c_exception": 174 | # if we're exiting a C function, we should add a frame before 175 | # any Python frames that attributes the time to that C function 176 | c_frame_identifier = "%s\x00%s\x00%i" % ( 177 | getattr(arg, "__qualname__", arg.__name__), 178 | "", 179 | 0, 180 | ) 181 | call_stack.append(c_frame_identifier) 182 | 183 | while frame is not None: 184 | identifier = "%s\x00%s\x00%i" % ( 185 | frame.f_code.co_name, 186 | frame.f_code.co_filename, 187 | frame.f_code.co_firstlineno, 188 | ) 189 | call_stack.append(identifier) 190 | frame = frame.f_back 191 | 192 | thread = threading.current_thread() 193 | thread_identifier = "%s\x00%s\x00%i" % ( 194 | thread.name, "", thread.ident) 195 | call_stack.append(thread_identifier) 196 | 197 | # we iterated from the leaf to the root, we actually want the call stack 198 | # starting at the root, so reverse this array 199 | call_stack.reverse() 200 | 201 | return call_stack 202 | 203 | 204 | class AsyncState(NamedTuple): 205 | 206 | state: LiteralStr["in_context", 207 | "out_of_context_awaited", "out_of_context_unknown"] 208 | """ 209 | Definitions: 210 | ``in_context``: indicates that the sample comes from the subscriber's 211 | context. 212 | 213 | ``out_of_context_awaited``: the sample comes from outside the 214 | subscriber's context, but we tracked the await that happened before the 215 | context exited. :attr:`info` contains the call stack of the await. 216 | 217 | ``out_of_context_unknown``: the sample comes from outside the 218 | subscriber's context, but the change of context didn't look like an 219 | await. :attr:`info` contains the call stack when the context changed. 220 | """ 221 | 222 | info: Any = None 223 | -------------------------------------------------------------------------------- /test/test_profiler_async.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import sys 3 | import time 4 | from functools import partial 5 | from test.fake_time_util import fake_time, fake_time_asyncio, fake_time_trio 6 | from typing import Optional 7 | 8 | import greenlet 9 | import pytest 10 | import trio 11 | import trio._core._run 12 | import trio.lowlevel 13 | 14 | from joulehunter import processors, stack_sampler 15 | from joulehunter.frame import AwaitTimeFrame, OutOfContextFrame 16 | from joulehunter.profiler import Profiler 17 | from joulehunter.session import Session 18 | 19 | from .util import assert_never, flaky_in_ci, walk_frames 20 | 21 | # Utilities # 22 | 23 | 24 | @pytest.fixture(autouse=True) 25 | def tidy_up_stack_sampler(): 26 | assert sys.getprofile() is None 27 | assert len(stack_sampler.get_stack_sampler().subscribers) == 0 28 | 29 | yield 30 | 31 | assert sys.getprofile() is None 32 | assert len(stack_sampler.get_stack_sampler().subscribers) == 0 33 | stack_sampler.thread_locals.__dict__.clear() 34 | 35 | 36 | # Tests # 37 | 38 | @pytest.mark.skip(reason="not working with async mode yet ") 39 | @pytest.mark.asyncio 40 | async def test_sleep(): 41 | profiler = Profiler() 42 | 43 | with fake_time_asyncio(): 44 | profiler.start() 45 | 46 | await asyncio.sleep(0.2) 47 | 48 | session = profiler.stop() 49 | 50 | assert len(session.frame_records) > 0 51 | 52 | root_frame = session.root_frame() 53 | assert root_frame 54 | 55 | assert root_frame.time() == pytest.approx(0.2, rel=0.1) 56 | assert root_frame.await_time() == pytest.approx(0.2, rel=0.1) 57 | 58 | sleep_frame = next(f for f in walk_frames( 59 | root_frame) if f.function == "sleep") 60 | assert sleep_frame.time() == pytest.approx(0.2, rel=0.1) 61 | assert sleep_frame.time() == pytest.approx(0.2, rel=0.1) 62 | 63 | 64 | @pytest.mark.skip(reason="not working with async mode yet ") 65 | def test_sleep_trio(): 66 | async def run(): 67 | profiler = Profiler() 68 | profiler.start() 69 | 70 | await trio.sleep(0.2) 71 | 72 | session = profiler.stop() 73 | assert len(session.frame_records) > 0 74 | 75 | root_frame = session.root_frame() 76 | assert root_frame 77 | 78 | assert root_frame.time() == pytest.approx(0.2) 79 | assert root_frame.await_time() == pytest.approx(0.2) 80 | 81 | sleep_frame = next(f for f in walk_frames( 82 | root_frame) if f.function == "sleep") 83 | assert sleep_frame.time() == pytest.approx(0.2) 84 | assert sleep_frame.time() == pytest.approx(0.2) 85 | 86 | with fake_time_trio() as fake_clock: 87 | trio.run(run, clock=fake_clock.trio_clock) 88 | 89 | 90 | @pytest.mark.skip(reason="not working with async mode yet ") 91 | @flaky_in_ci 92 | @pytest.mark.parametrize("engine", ["asyncio", "trio"]) 93 | def test_profiler_task_isolation(engine): 94 | profiler_session: Optional[Session] = None 95 | 96 | async def async_wait(sync_time, async_time, profile=False, engine="asyncio"): 97 | # an async function that has both sync work and async work 98 | profiler = None 99 | 100 | if profile: 101 | profiler = Profiler() 102 | profiler.start() 103 | 104 | time.sleep(sync_time / 2) 105 | 106 | if engine == "asyncio": 107 | await asyncio.sleep(async_time) 108 | else: 109 | await trio.sleep(async_time) 110 | 111 | time.sleep(sync_time / 2) 112 | 113 | if profiler: 114 | profiler.stop() 115 | profiler.print(show_all=True) 116 | return profiler.last_session 117 | 118 | if engine == "asyncio": 119 | loop = asyncio.new_event_loop() 120 | 121 | with fake_time_asyncio(loop): 122 | profile_task = loop.create_task(async_wait( 123 | sync_time=0.1, async_time=0.5, profile=True)) 124 | loop.create_task(async_wait(sync_time=0.1, async_time=0.3)) 125 | loop.create_task(async_wait(sync_time=0.1, async_time=0.3)) 126 | 127 | loop.run_until_complete(profile_task) 128 | loop.close() 129 | 130 | profiler_session = profile_task.result() 131 | elif engine == "trio": 132 | 133 | async def async_wait_and_capture(**kwargs): 134 | nonlocal profiler_session 135 | profiler_session = await async_wait(**kwargs) 136 | 137 | async def multi_task(): 138 | async with trio.open_nursery() as nursery: 139 | nursery.start_soon( 140 | partial( 141 | async_wait_and_capture, 142 | sync_time=0.1, 143 | async_time=0.5, 144 | engine="trio", 145 | profile=True, 146 | ) 147 | ) 148 | nursery.start_soon( 149 | partial(async_wait, sync_time=0.1, 150 | async_time=0.3, engine="trio") 151 | ) 152 | nursery.start_soon( 153 | partial(async_wait, sync_time=0.1, 154 | async_time=0.3, engine="trio") 155 | ) 156 | 157 | with fake_time_trio() as fake_clock: 158 | trio.run(multi_task, clock=fake_clock.trio_clock) 159 | else: 160 | assert_never(engine) 161 | 162 | assert profiler_session 163 | 164 | root_frame = profiler_session.root_frame() 165 | assert root_frame is not None 166 | fake_work_frame = next(f for f in walk_frames( 167 | root_frame) if f.function == "async_wait") 168 | assert fake_work_frame.time() == pytest.approx(0.1 + 0.5, rel=0.1) 169 | 170 | root_frame = processors.aggregate_repeated_calls(root_frame, {}) 171 | assert root_frame 172 | 173 | await_frames = [f for f in walk_frames( 174 | root_frame) if isinstance(f, AwaitTimeFrame)] 175 | 176 | assert sum(f.self_time for f in await_frames) == pytest.approx( 177 | 0.5, rel=0.1) 178 | assert sum(f.time() for f in await_frames) == pytest.approx(0.5, rel=0.1) 179 | 180 | 181 | PYTHON_3_10_OR_LATER = sys.version_info >= (3, 10) 182 | 183 | 184 | def test_greenlet(): 185 | profiler = Profiler() 186 | 187 | with fake_time(): 188 | profiler.start() 189 | 190 | def y(duration): 191 | time.sleep(duration) 192 | 193 | y(0.1) 194 | greenlet.greenlet(y).switch(0.1) 195 | 196 | session = profiler.stop() 197 | 198 | profiler.print() 199 | 200 | root_frame = session.root_frame() 201 | assert root_frame 202 | 203 | assert root_frame.time() == pytest.approx(0.2, rel=0.1) 204 | 205 | if PYTHON_3_10_OR_LATER: 206 | switch_frames = [f for f in walk_frames( 207 | root_frame) if f.function == "greenlet.switch"] 208 | assert len(switch_frames) == 1 209 | assert switch_frames[0].time() == pytest.approx(0.1, rel=0.1) 210 | 211 | sleep_frames = [f for f in walk_frames( 212 | root_frame) if f.function == "sleep"] 213 | assert len(sleep_frames) == 1 214 | assert sleep_frames[0].time() == pytest.approx(0.1, rel=0.1) 215 | else: 216 | sleep_frames = [f for f in walk_frames( 217 | root_frame) if f.function == "sleep"] 218 | assert len(sleep_frames) == 2 219 | assert sleep_frames[0].time() == pytest.approx(0.1, rel=0.1) 220 | assert sleep_frames[1].time() == pytest.approx(0.1, rel=0.1) 221 | 222 | 223 | def test_strict_with_greenlet(): 224 | profiler = Profiler(async_mode="strict") 225 | 226 | with fake_time(): 227 | profiler.start() 228 | 229 | def y(duration): 230 | time.sleep(duration) 231 | 232 | y(0.1) 233 | greenlet.greenlet(y).switch(0.1) 234 | 235 | session = profiler.stop() 236 | 237 | profiler.print() 238 | 239 | root_frame = session.root_frame() 240 | assert root_frame 241 | 242 | assert root_frame.time() == pytest.approx(0.2, rel=0.1) 243 | 244 | sleep_frames = [f for f in walk_frames( 245 | root_frame) if f.function == "sleep"] 246 | assert len(sleep_frames) == 1 247 | assert sleep_frames[0].time() == pytest.approx(0.1, rel=0.1) 248 | 249 | if PYTHON_3_10_OR_LATER: 250 | greenlet_frames = [f for f in walk_frames( 251 | root_frame) if f.function == "greenlet.switch"] 252 | else: 253 | greenlet_frames = [f for f in walk_frames( 254 | root_frame) if isinstance(f, OutOfContextFrame)] 255 | 256 | assert len(greenlet_frames) == 1 257 | assert greenlet_frames[0].time() == pytest.approx(0.1, rel=0.1) 258 | -------------------------------------------------------------------------------- /test/test_processors.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from pytest import approx 4 | 5 | from joulehunter import processors 6 | from joulehunter.frame import Frame, SelfTimeFrame 7 | 8 | all_processors = [ 9 | processors.aggregate_repeated_calls, 10 | processors.group_library_frames_processor, 11 | processors.merge_consecutive_self_time, 12 | processors.remove_importlib, 13 | processors.remove_unnecessary_self_time_nodes, 14 | processors.remove_irrelevant_nodes, 15 | ] 16 | 17 | 18 | def test_frame_passthrough_none(): 19 | for processor in all_processors: 20 | assert processor(None, options={}) is None 21 | 22 | 23 | def test_remove_importlib(): 24 | frame = Frame( 25 | identifier="\x00sympy/__init__.py\x0012", 26 | children=[ 27 | Frame( 28 | identifier="_handle_fromlist\x00../\x00997", 29 | self_time=0.1, 30 | children=[ 31 | Frame( 32 | identifier="_find_and_load\x00../\x00997", 33 | self_time=0.1, 34 | children=[ 35 | Frame( 36 | identifier="\x00sympy/polys/polyfuncs.py\x001", 37 | self_time=0.05, 38 | ), 39 | Frame( 40 | identifier="\x00sympy/polys/partfrac.py\x001", 41 | self_time=0.2, 42 | ), 43 | ], 44 | ), 45 | Frame( 46 | identifier="\x00sympy/polys/numberfields.py\x001", 47 | self_time=0.05, 48 | ), 49 | ], 50 | ) 51 | ], 52 | ) 53 | 54 | assert frame.self_time == 0.0 55 | assert frame.time() == approx(0.5) 56 | 57 | frame = processors.remove_importlib(frame, options={}) 58 | assert frame 59 | 60 | assert frame.self_time == approx(0.2) # the root gets the self_time from the importlib 61 | assert frame.time() == approx(0.5) 62 | assert len(frame.children) == 3 63 | assert frame.children[0].file_path == "sympy/polys/polyfuncs.py" 64 | assert frame.children[1].file_path == "sympy/polys/partfrac.py" 65 | assert frame.children[2].file_path == "sympy/polys/numberfields.py" 66 | 67 | 68 | def test_merge_consecutive_self_time(): 69 | frame = Frame( 70 | identifier="\x00cibuildwheel/__init__.py\x0012", 71 | children=[ 72 | Frame( 73 | identifier="strip_newlines\x00cibuildwheel/utils.py\x00997", 74 | self_time=0.1, 75 | ), 76 | SelfTimeFrame( 77 | self_time=0.2, 78 | ), 79 | SelfTimeFrame( 80 | self_time=0.1, 81 | ), 82 | Frame( 83 | identifier="calculate_metrics\x00cibuildwheel/utils.py\x007", 84 | self_time=0.1, 85 | ), 86 | SelfTimeFrame( 87 | self_time=0.05, 88 | ), 89 | ], 90 | ) 91 | 92 | assert frame.time() == approx(0.55) 93 | 94 | frame = processors.merge_consecutive_self_time(frame, options={}) 95 | assert frame 96 | 97 | assert frame.time() == approx(0.55) 98 | assert len(frame.children) == 4 99 | assert frame.children[0].self_time == approx(0.1) 100 | assert frame.children[1].self_time == approx(0.3) 101 | assert isinstance(frame.children[1], SelfTimeFrame) 102 | assert frame.children[2].self_time == approx(0.1) 103 | assert frame.children[3].self_time == approx(0.05) 104 | assert isinstance(frame.children[3], SelfTimeFrame) 105 | 106 | 107 | def test_aggregate_repeated_calls(): 108 | frame = Frame( 109 | identifier="\x00cibuildwheel/__init__.py\x0012", 110 | children=[ 111 | Frame( 112 | identifier="strip_newlines\x00cibuildwheel/utils.py\x00997", 113 | self_time=0.1, 114 | children=[ 115 | Frame( 116 | identifier="scan_string\x00cibuildwheel/utils.py\x0054", 117 | self_time=0.2, 118 | ), 119 | ], 120 | ), 121 | SelfTimeFrame( 122 | self_time=0.1, 123 | ), 124 | Frame( 125 | identifier="strip_newlines\x00cibuildwheel/utils.py\x00997", 126 | self_time=0.1, 127 | ), 128 | SelfTimeFrame( 129 | self_time=0.2, 130 | ), 131 | Frame( 132 | identifier="calculate_metrics\x00cibuildwheel/utils.py\x007", 133 | self_time=0.1, 134 | ), 135 | SelfTimeFrame( 136 | self_time=0.05, 137 | ), 138 | ], 139 | ) 140 | 141 | assert frame.time() == approx(0.85) 142 | 143 | frame = processors.aggregate_repeated_calls(frame, options={}) 144 | 145 | assert frame 146 | assert frame.time() == approx(0.85) 147 | # children should be sorted by time 148 | assert len(frame.children) == 3 149 | assert frame.children[0].function == "strip_newlines" 150 | assert frame.children[0].time() == 0.4 151 | assert frame.children[0].children[0].function == "scan_string" 152 | assert isinstance(frame.children[1], SelfTimeFrame) 153 | assert frame.children[1].time() == approx(0.35) 154 | 155 | 156 | def test_remove_irrelevant_nodes(): 157 | frame = Frame( 158 | identifier="\x00cibuildwheel/__init__.py\x0012", 159 | children=[ 160 | Frame( 161 | identifier="strip_newlines\x00cibuildwheel/utils.py\x00997", 162 | children=[ 163 | Frame( 164 | identifier="scan_string\x00cibuildwheel/utils.py\x0054", 165 | self_time=10, 166 | ), 167 | ], 168 | ), 169 | SelfTimeFrame( 170 | self_time=0.5, 171 | ), 172 | Frame( 173 | identifier="strip_newlines\x00cibuildwheel/utils.py\x00997", 174 | self_time=0.5, 175 | ), 176 | Frame( 177 | identifier="calculate_metrics\x00cibuildwheel/utils.py\x007", 178 | self_time=0.01, 179 | ), 180 | ], 181 | ) 182 | 183 | assert frame.time() == approx(11.01) 184 | 185 | frame = processors.remove_irrelevant_nodes(frame, options={}) 186 | 187 | assert frame 188 | assert frame.time() == approx(11.01) 189 | # check the calculate metrics function was deleted 190 | assert len(frame.children) == 3 191 | assert "calculate_metrics" not in [f.function for f in frame.children] 192 | 193 | 194 | def test_remove_unnecessary_self_time_nodes(): 195 | frame = Frame( 196 | identifier="\x00cibuildwheel/__init__.py\x0012", 197 | children=[ 198 | Frame( 199 | identifier="strip_newlines\x00cibuildwheel/utils.py\x00997", 200 | children=[ 201 | SelfTimeFrame( 202 | self_time=0.2, 203 | ), 204 | ], 205 | ), 206 | SelfTimeFrame( 207 | self_time=0.5, 208 | ), 209 | Frame( 210 | identifier="strip_newlines\x00cibuildwheel/utils.py\x00997", 211 | self_time=0.5, 212 | ), 213 | Frame( 214 | identifier="calculate_metrics\x00cibuildwheel/utils.py\x007", 215 | self_time=0.1, 216 | ), 217 | ], 218 | ) 219 | 220 | assert frame.time() == approx(1.3) 221 | 222 | frame = processors.remove_unnecessary_self_time_nodes(frame, options={}) 223 | 224 | assert frame 225 | assert frame.time() == approx(1.3) 226 | assert len(frame.children) == 4 227 | # check the self time node was deleted 228 | strip_newlines_frame = frame.children[0] 229 | assert strip_newlines_frame.function == "strip_newlines" 230 | assert len(strip_newlines_frame.children) == 0 231 | assert strip_newlines_frame.self_time == 0.2 232 | 233 | 234 | def test_group_library_frames_processor(): 235 | frame = Frame( 236 | identifier="\x00cibuildwheel/__init__.py\x0012", 237 | children=[ 238 | Frame( 239 | identifier="library_function\x00env/lib/python3.6/django/__init__.py\x00997", 240 | children=[ 241 | Frame( 242 | identifier="library_inner\x00env/lib/python3.6/django/http.py\x0054", 243 | children=[ 244 | Frame( 245 | identifier="library_callback\x00env/lib/python3.6/django/views.py\x0054", 246 | children=[ 247 | Frame( 248 | identifier="\x00cibuildwheel/views.py\x0012", 249 | self_time=0.3, 250 | ), 251 | ], 252 | ), 253 | ], 254 | ), 255 | ], 256 | ), 257 | SelfTimeFrame( 258 | self_time=0.5, 259 | ), 260 | Frame( 261 | identifier="strip_newlines\x00cibuildwheel/utils.py\x00997", 262 | self_time=0.5, 263 | ), 264 | Frame( 265 | identifier="calculate_metrics\x00cibuildwheel/utils.py\x007", 266 | self_time=0.1, 267 | ), 268 | ], 269 | ) 270 | 271 | assert frame.time() == approx(1.4) 272 | 273 | frame = processors.group_library_frames_processor(frame, options={}) 274 | 275 | assert frame 276 | assert frame.time() == approx(1.4) 277 | group_root = frame.children[0] 278 | 279 | group = group_root.group 280 | assert group 281 | assert group.root == group_root 282 | 283 | for frame in group.frames: 284 | assert frame.group == group 285 | 286 | assert group_root in group.frames 287 | assert group_root.children[0] in group.frames 288 | assert group_root.children[0].children[0] in group.frames 289 | assert group_root.children[0].children[0] in group.exit_frames 290 | assert group_root.children[0].children[0].children[0] not in group.frames 291 | 292 | old_sys_path = sys.path[:] 293 | sys.path.append("env/lib/python3.6") 294 | assert group.libraries == ["django"] 295 | sys.path[:] = old_sys_path 296 | -------------------------------------------------------------------------------- /joulehunter/profiler.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import inspect 4 | import sys 5 | import time 6 | import types 7 | from typing import IO, Any, Union 8 | 9 | import joulehunter.energy as energy 10 | 11 | from joulehunter import renderers 12 | from joulehunter.frame import AWAIT_FRAME_IDENTIFIER, OUT_OF_CONTEXT_FRAME_IDENTIFIER 13 | from joulehunter.session import Session 14 | from joulehunter.stack_sampler import AsyncState, StackSampler, build_call_stack, get_stack_sampler 15 | from joulehunter.typing import LiteralStr 16 | from joulehunter.util import file_supports_color, file_supports_unicode 17 | 18 | # pyright: strict 19 | 20 | 21 | class ActiveProfilerSession: 22 | frame_records: list[tuple[list[str], float]] 23 | 24 | def __init__( 25 | self, 26 | start_time: float, 27 | start_call_stack: list[str], 28 | ) -> None: 29 | self.start_time = start_time 30 | self.start_call_stack = start_call_stack 31 | self.frame_records = [] 32 | 33 | 34 | AsyncMode = LiteralStr["enabled", "disabled", "strict"] 35 | 36 | 37 | class Profiler: 38 | """ 39 | The profiler - this is the main way to use joulehunter. 40 | """ 41 | 42 | _last_session: Session | None 43 | _active_session: ActiveProfilerSession | None 44 | _interval: float 45 | _async_mode: AsyncMode 46 | 47 | def __init__( 48 | self, 49 | interval: float = 0.001, 50 | async_mode: AsyncMode = "disabled", 51 | package: Union[str, int] = 0, 52 | component: Union[str, int] = None 53 | ): 54 | """ 55 | Note the profiling will not start until :func:`start` is called. 56 | 57 | :param interval: See :attr:`interval`. 58 | :param async_mode: See :attr:`async_mode`. 59 | """ 60 | self._interval = interval 61 | self._last_session = None 62 | self._active_session = None 63 | self._async_mode = async_mode 64 | 65 | self.domain = energy.parse_domain(package, component) 66 | self.current_energy = energy.Energy(self.domain).current_energy 67 | 68 | @property 69 | def interval(self) -> float: 70 | """ 71 | The minimum time, in seconds, between each stack sample. This translates into the 72 | resolution of the sampling. 73 | """ 74 | return self._interval 75 | 76 | @property 77 | def async_mode(self) -> AsyncMode: 78 | """ 79 | Configures how this Profiler tracks time in a program that uses 80 | async/await. 81 | 82 | ``enabled`` 83 | When this profiler sees an ``await``, time is logged in the function 84 | that awaited, rather than observing other coroutines or the event 85 | loop. 86 | 87 | ``disabled`` 88 | This profiler doesn't attempt to track ``await``. In a program that 89 | uses async/await, this will interleave other coroutines and event 90 | loop machinery in the profile. Use this option if async support is 91 | causing issues in your use case, or if you want to run multiple 92 | profilers at once. 93 | 94 | ``strict`` 95 | Instructs the profiler to only profile the current 96 | `async context `_. 97 | Frames that are observed in an other context are ignored, tracked 98 | instead as ````. 99 | """ 100 | return self._async_mode 101 | 102 | @property 103 | def last_session(self) -> Session | None: 104 | """ 105 | The previous session recorded by the Profiler. 106 | """ 107 | return self._last_session 108 | 109 | @property 110 | def domain_names(self) -> list[str]: 111 | return [energy.domain_name(self.domain[:index+1]) 112 | for index in range(len(self.domain))] 113 | 114 | def start(self, caller_frame: types.FrameType | None = None): 115 | """ 116 | Instructs the profiler to start - to begin observing the program's execution and recording 117 | frames. 118 | 119 | The normal way to invoke ``start()`` is with a new instance, but you can restart a Profiler 120 | that was previously running, too. The sessions are combined. 121 | 122 | :param caller_frame: Set this to override the default behaviour of treating the caller of 123 | ``start()`` as the 'start_call_stack' - the instigator of the profile. Most 124 | renderers will trim the 'root' from the call stack up to this frame, to 125 | present a simpler output. 126 | 127 | You might want to set this to ``inspect.currentframe().f_back`` if you are 128 | writing a library that wraps joulehunter. 129 | """ 130 | 131 | if caller_frame is None: 132 | caller_frame = inspect.currentframe().f_back # type: ignore 133 | 134 | try: 135 | self._active_session = ActiveProfilerSession( 136 | start_time=time.time(), 137 | start_call_stack=build_call_stack( 138 | caller_frame, "initial", None), 139 | ) 140 | 141 | use_async_context = self.async_mode != "disabled" 142 | get_stack_sampler(self.current_energy).subscribe( 143 | self._sampler_saw_call_stack, self.interval, use_async_context 144 | ) 145 | except Exception as e: 146 | self._active_session = None 147 | raise e 148 | 149 | def stop(self) -> Session: 150 | """ 151 | Stops the profiler observing, and sets :attr:`last_session` 152 | to the captured session. 153 | 154 | :return: The captured session. 155 | """ 156 | if not self._active_session: 157 | raise RuntimeError("This profiler is not currently running.") 158 | 159 | try: 160 | get_stack_sampler(self.current_energy)\ 161 | .unsubscribe(self._sampler_saw_call_stack) 162 | except StackSampler.SubscriberNotFound: 163 | raise RuntimeError( 164 | "Failed to stop profiling. Make sure that you start/stop profiling on the same thread." 165 | ) 166 | 167 | session = Session( 168 | frame_records=self._active_session.frame_records, 169 | start_time=self._active_session.start_time, 170 | duration=time.time() - self._active_session.start_time, 171 | sample_count=len(self._active_session.frame_records), 172 | program=" ".join(sys.argv), 173 | start_call_stack=self._active_session.start_call_stack, 174 | domain_names=self.domain_names 175 | ) 176 | self._active_session = None 177 | 178 | if self.last_session is not None: 179 | # include the previous session's data too 180 | session = Session.combine(self.last_session, session) 181 | 182 | self._last_session = session 183 | 184 | return session 185 | 186 | @property 187 | def is_running(self): 188 | """ 189 | Returns `True` if this profiler is running - i.e. observing the program execution. 190 | """ 191 | return self._active_session is not None 192 | 193 | def reset(self): 194 | """ 195 | Resets the Profiler, clearing the `last_session`. 196 | """ 197 | if self.is_running: 198 | self.stop() 199 | 200 | self._last_session = None 201 | 202 | def __enter__(self): 203 | """ 204 | Context manager support. 205 | 206 | Profilers can be used in `with` blocks! See this example: 207 | 208 | .. code-block:: python 209 | 210 | with Profiler() as p: 211 | # your code here... 212 | do_some_work() 213 | 214 | # profiling has ended. let's print the output. 215 | p.print() 216 | """ 217 | self.start(caller_frame=inspect.currentframe().f_back) # type: ignore 218 | return self 219 | 220 | def __exit__(self, *args: Any): 221 | self.stop() 222 | 223 | # pylint: disable=W0613 224 | def _sampler_saw_call_stack( 225 | self, 226 | call_stack: list[str], 227 | time_since_last_sample: float, 228 | async_state: AsyncState | None, 229 | ): 230 | if not self._active_session: 231 | raise RuntimeError( 232 | "Received a call stack without an active session. Please file an issue on joulehunter Github describing how you made this happen!" 233 | ) 234 | 235 | if ( 236 | async_state 237 | and async_state.state == "out_of_context_awaited" 238 | and self._async_mode in ["enabled", "strict"] 239 | ): 240 | awaiting_coroutine_stack = async_state.info 241 | self._active_session.frame_records.append( 242 | ( 243 | awaiting_coroutine_stack + [AWAIT_FRAME_IDENTIFIER], 244 | time_since_last_sample, 245 | ) 246 | ) 247 | elif ( 248 | async_state 249 | and async_state.state == "out_of_context_unknown" 250 | and self._async_mode == "strict" 251 | ): 252 | context_exit_frame = async_state.info 253 | self._active_session.frame_records.append( 254 | ( 255 | context_exit_frame + [OUT_OF_CONTEXT_FRAME_IDENTIFIER], 256 | time_since_last_sample, 257 | ) 258 | ) 259 | else: 260 | # regular sync code 261 | self._active_session.frame_records.append( 262 | (call_stack, time_since_last_sample)) 263 | 264 | def print( 265 | self, 266 | file: IO[str] = sys.stdout, 267 | *, 268 | unicode: bool | None = None, 269 | color: bool | None = None, 270 | show_all: bool = False, 271 | timeline: bool = False, 272 | ): 273 | """print(file=sys.stdout, *, unicode=None, color=None, show_all=False, timeline=False) 274 | 275 | Print the captured profile to the console. 276 | 277 | :param file: the IO stream to write to. Could be a file descriptor or sys.stdout, sys.stderr. Defaults to sys.stdout. 278 | :param unicode: Override unicode support detection. 279 | :param color: Override ANSI color support detection. 280 | :param show_all: Sets the ``show_all`` parameter on the renderer. 281 | :param timeline: Sets the ``timeline`` parameter on the renderer. 282 | """ 283 | if unicode is None: 284 | unicode = file_supports_unicode(file) 285 | if color is None: 286 | color = file_supports_color(file) 287 | 288 | print( 289 | self.output_text( 290 | unicode=unicode, 291 | color=color, 292 | show_all=show_all, 293 | timeline=timeline, 294 | ), 295 | file=file, 296 | ) 297 | 298 | def output_text( 299 | self, 300 | unicode: bool = False, 301 | color: bool = False, 302 | show_all: bool = False, 303 | timeline: bool = False, 304 | ) -> str: 305 | """ 306 | Return the profile output as text, as rendered by :class:`ConsoleRenderer` 307 | """ 308 | return self.output( 309 | renderer=renderers.ConsoleRenderer( 310 | unicode=unicode, color=color, show_all=show_all, timeline=timeline 311 | ) 312 | ) 313 | 314 | def output_html(self, timeline: bool = False) -> str: 315 | """ 316 | Return the profile output as HTML, as rendered by :class:`HTMLRenderer` 317 | """ 318 | return self.output(renderer=renderers.HTMLRenderer(timeline=timeline)) 319 | 320 | def open_in_browser(self, timeline: bool = False): 321 | """ 322 | Opens the last profile session in your web browser. 323 | """ 324 | session = self._get_last_session_or_fail() 325 | 326 | return renderers.HTMLRenderer(timeline=timeline).open_in_browser(session) 327 | 328 | def output(self, renderer: renderers.Renderer) -> str: 329 | """ 330 | Returns the last profile session, as rendered by ``renderer``. 331 | 332 | :param renderer: The renderer to use. 333 | """ 334 | session = self._get_last_session_or_fail() 335 | 336 | return renderer.render(session) 337 | 338 | def _get_last_session_or_fail(self) -> Session: 339 | if self.is_running: 340 | raise Exception( 341 | "can't render profile output because this profiler is still running") 342 | 343 | if self.last_session is None: 344 | raise Exception( 345 | "can't render profile output because this profiler has not completed a profile session yet" 346 | ) 347 | 348 | return self.last_session 349 | -------------------------------------------------------------------------------- /joulehunter/frame.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | import sys 5 | import uuid 6 | from typing import Sequence 7 | 8 | # pyright: strict 9 | 10 | 11 | class BaseFrame: 12 | group: FrameGroup | None 13 | 14 | def __init__(self, parent: Frame | None = None, self_time: float = 0): 15 | self.parent = parent 16 | self._self_time = self_time 17 | self.group = None 18 | 19 | def remove_from_parent(self): 20 | """ 21 | Removes this frame from its parent, and nulls the parent link 22 | """ 23 | if self.parent: 24 | self.parent._children.remove(self) # type: ignore 25 | self.parent._invalidate_time_caches() # type: ignore 26 | self.parent = None 27 | 28 | @staticmethod 29 | def new_subclass_with_identifier(identifier: str) -> BaseFrame: 30 | if identifier == AWAIT_FRAME_IDENTIFIER: 31 | return AwaitTimeFrame() 32 | elif identifier == OUT_OF_CONTEXT_FRAME_IDENTIFIER: 33 | return OutOfContextFrame() 34 | else: 35 | return Frame(identifier=identifier) 36 | 37 | @property 38 | def proportion_of_parent(self) -> float: 39 | if self.parent: 40 | try: 41 | return self.time() / self.parent.time() 42 | except ZeroDivisionError: 43 | return float("nan") 44 | else: 45 | return 1.0 46 | 47 | @property 48 | def total_self_time(self) -> float: 49 | """ 50 | The total amount of self time in this frame (including self time recorded by SelfTimeFrame 51 | children, and await time from AwaitTimeFrame children) 52 | """ 53 | self_time = self.self_time 54 | for child in self.children: 55 | if isinstance(child, SelfTimeFrame) or isinstance(child, AwaitTimeFrame): 56 | self_time += child.self_time 57 | return self_time 58 | 59 | @property 60 | def self_time(self) -> float: 61 | return self._self_time 62 | 63 | @self_time.setter 64 | def self_time(self, self_time: float): 65 | self._self_time = self_time 66 | self._invalidate_time_caches() 67 | 68 | # invalidates the cache for the time() function. 69 | # called whenever self_time or _children is modified. 70 | def _invalidate_time_caches(self): 71 | pass 72 | 73 | # stylistically I'd rather this was a property, but using @property appears to use twice 74 | # as many stack frames, so I'm forced into using a function since this method is recursive 75 | # down the call tree. 76 | def time(self) -> float: 77 | """ 78 | Wall-clock time spent in the function. Includes time spent in 'await', 79 | if applicable. 80 | """ 81 | raise NotImplementedError() 82 | 83 | def await_time(self) -> float: 84 | raise NotImplementedError() 85 | 86 | @property 87 | def identifier(self) -> str: 88 | raise NotImplementedError() 89 | 90 | @property 91 | def function(self) -> str | None: 92 | raise NotImplementedError() 93 | 94 | @property 95 | def file_path(self) -> str | None: 96 | raise NotImplementedError() 97 | 98 | @property 99 | def line_no(self) -> int | None: 100 | raise NotImplementedError() 101 | 102 | @property 103 | def file_path_short(self) -> str | None: 104 | raise NotImplementedError() 105 | 106 | @property 107 | def is_application_code(self) -> bool | None: 108 | raise NotImplementedError() 109 | 110 | @property 111 | def code_position_short(self) -> str | None: 112 | raise NotImplementedError() 113 | 114 | @property 115 | def children(self) -> Sequence[BaseFrame]: 116 | raise NotImplementedError() 117 | 118 | 119 | class Frame(BaseFrame): 120 | """ 121 | Object that represents a stack frame in the parsed tree 122 | """ 123 | 124 | _children: list[BaseFrame] 125 | _time: float | None 126 | _await_time: float | None 127 | _identifier: str 128 | 129 | def __init__( 130 | self, 131 | identifier: str = "", 132 | parent: Frame | None = None, 133 | children: Sequence[BaseFrame] | None = None, 134 | self_time: float = 0, 135 | ): 136 | super().__init__(parent=parent, self_time=self_time) 137 | 138 | self._identifier = identifier 139 | self._children = [] 140 | 141 | self._time = None 142 | self._await_time = None 143 | 144 | if children: 145 | for child in children: 146 | self.add_child(child) 147 | 148 | def add_child(self, frame: BaseFrame, after: BaseFrame | None = None): 149 | """ 150 | Adds a child frame, updating the parent link. 151 | Optionally, insert the frame in a specific position by passing the frame to insert 152 | this one after. 153 | """ 154 | frame.remove_from_parent() 155 | frame.parent = self 156 | if after is None: 157 | self._children.append(frame) 158 | else: 159 | index = self._children.index(after) + 1 160 | self._children.insert(index, frame) 161 | 162 | self._invalidate_time_caches() 163 | 164 | def add_children(self, frames: Sequence[BaseFrame], after: BaseFrame | None = None): 165 | """ 166 | Convenience method to add multiple frames at once. 167 | """ 168 | if after is not None: 169 | # if there's an 'after' parameter, add the frames in reverse so the order is 170 | # preserved. 171 | for frame in reversed(frames): 172 | self.add_child(frame, after=after) 173 | else: 174 | for frame in frames: 175 | self.add_child(frame) 176 | 177 | @property 178 | def identifier(self) -> str: 179 | return self._identifier 180 | 181 | @property 182 | def children(self) -> Sequence[BaseFrame]: 183 | # Return an immutable copy (this property should only be mutated using methods) 184 | # Also, returning a copy avoid problems when mutating while iterating, which happens a lot 185 | # in processors! 186 | return tuple(self._children) 187 | 188 | @property 189 | def function(self) -> str | None: 190 | if self.identifier: 191 | return self.identifier.split("\x00")[0] 192 | 193 | @property 194 | def file_path(self) -> str | None: 195 | if self.identifier: 196 | return self.identifier.split("\x00")[1] 197 | 198 | @property 199 | def line_no(self) -> int | None: 200 | if self.identifier: 201 | return int(self.identifier.split("\x00")[2]) 202 | 203 | @property 204 | def file_path_short(self) -> str | None: 205 | """Return the path resolved against the closest entry in sys.path""" 206 | if not hasattr(self, "_file_path_short"): 207 | if self.file_path: 208 | result = None 209 | 210 | for path in sys.path: 211 | # On Windows, if self.file_path and path are on different drives, relpath 212 | # will result in exception, because it cannot compute a relpath in this case. 213 | # The root cause is that on Windows, there is no root dir like '/' on Linux. 214 | try: 215 | candidate = os.path.relpath(self.file_path, path) 216 | except ValueError: 217 | continue 218 | 219 | if not result or (len(candidate.split(os.sep)) < len(result.split(os.sep))): 220 | result = candidate 221 | 222 | self._file_path_short = result 223 | else: 224 | self._file_path_short = None 225 | 226 | return self._file_path_short 227 | 228 | @property 229 | def is_application_code(self) -> bool | None: 230 | if self.identifier: 231 | file_path = self.file_path 232 | 233 | if not file_path: 234 | return False 235 | 236 | if "/lib/" in file_path: 237 | return False 238 | 239 | if os.sep != "/": 240 | # windows uses back-slash too, so let's look for that too. 241 | if (f"{os.sep}lib{os.sep}") in file_path: 242 | return False 243 | 244 | if file_path.startswith("<"): 245 | if file_path.startswith(" str | None: 256 | if self.identifier: 257 | return "%s:%i" % (self.file_path_short, self.line_no) 258 | 259 | def time(self): 260 | if self._time is None: 261 | # can't use a sum() expression here sadly, because this method 262 | # recurses down the call tree, and the generator uses an extra stack frame, 263 | # meaning we hit the stack limit when the profiled code is 500 frames deep. 264 | self._time = self.self_time 265 | 266 | for child in self.children: 267 | self._time += child.time() 268 | 269 | return self._time 270 | 271 | def await_time(self): 272 | if self._await_time is None: 273 | await_time = 0 274 | 275 | for child in self.children: 276 | await_time += child.await_time() 277 | 278 | self._await_time = await_time 279 | 280 | return self._await_time 281 | 282 | # pylint: disable=W0212 283 | def _invalidate_time_caches(self): 284 | self._time = None 285 | self._await_time = None 286 | # null all the parent's caches also. 287 | frame = self 288 | while frame.parent is not None: 289 | frame = frame.parent 290 | frame._time = None 291 | frame._await_time = None 292 | 293 | def __repr__(self): 294 | return "Frame(identifier=%s, time=%f, len(children)=%d), group=%r" % ( 295 | self.identifier, 296 | self.time(), 297 | len(self.children), 298 | self.group, 299 | ) 300 | 301 | 302 | class DummyFrame(BaseFrame): 303 | """ 304 | Informational frame that doesn't represent a real Python frame, but 305 | represents something about how time was spent in a function 306 | """ 307 | 308 | @property 309 | def _children(self) -> list[BaseFrame]: 310 | return [] 311 | 312 | @property 313 | def children(self) -> list[BaseFrame]: 314 | return [] 315 | 316 | @property 317 | def file_path(self): 318 | if self.parent: 319 | return self.parent.file_path 320 | 321 | @property 322 | def line_no(self): 323 | if self.parent: 324 | return self.parent.line_no 325 | 326 | @property 327 | def file_path_short(self): 328 | return "" 329 | 330 | @property 331 | def is_application_code(self): 332 | return False 333 | 334 | @property 335 | def code_position_short(self): 336 | return "" 337 | 338 | 339 | class SelfTimeFrame(DummyFrame): 340 | """ 341 | Represents a time spent inside a function 342 | """ 343 | 344 | def time(self): 345 | return self.self_time 346 | 347 | def await_time(self): 348 | return 0 349 | 350 | @property 351 | def function(self): 352 | return "[self]" 353 | 354 | @property 355 | def identifier(self): 356 | return "[self]" 357 | 358 | 359 | class AwaitTimeFrame(DummyFrame): 360 | """ 361 | Represents a time spent in an await - waiting for a coroutine to 362 | reactivate 363 | """ 364 | 365 | def time(self): 366 | return self.self_time 367 | 368 | def await_time(self): 369 | return self.self_time 370 | 371 | @property 372 | def function(self): 373 | return "[await]" 374 | 375 | @property 376 | def identifier(self): 377 | return "[await]" 378 | 379 | 380 | AWAIT_FRAME_IDENTIFIER = "[await]\x00\x000" 381 | 382 | 383 | class OutOfContextFrame(DummyFrame): 384 | """ 385 | Represents a time spent out of the profiler's context. 386 | """ 387 | 388 | def time(self): 389 | return self.self_time 390 | 391 | @property 392 | def function(self): 393 | return "[out-of-context]" 394 | 395 | @property 396 | def identifier(self): 397 | return "[out-of-context]" 398 | 399 | 400 | OUT_OF_CONTEXT_FRAME_IDENTIFIER = "[out-of-context]\x00\x000" 401 | 402 | 403 | class FrameGroup: 404 | _libraries: list[str] | None 405 | _frames: list[BaseFrame] 406 | _exit_frames: list[BaseFrame] | None 407 | 408 | def __init__(self, root: BaseFrame): 409 | self.root = root 410 | self.id = str(uuid.uuid4()) 411 | self._frames = [] 412 | self._exit_frames = None 413 | self._libraries = None 414 | 415 | self.add_frame(root) 416 | 417 | @property 418 | def libraries(self) -> list[str]: 419 | if self._libraries is None: 420 | libraries: list[str] = [] 421 | 422 | for frame in self.frames: 423 | if frame.file_path_short: 424 | library = frame.file_path_short.split(os.sep)[0] 425 | library, _ = os.path.splitext(library) 426 | if library and library not in libraries: 427 | libraries.append(library) 428 | self._libraries = libraries 429 | 430 | return self._libraries 431 | 432 | @property 433 | def frames(self) -> Sequence[BaseFrame]: 434 | return tuple(self._frames) 435 | 436 | # pylint: disable=W0212 437 | def add_frame(self, frame: BaseFrame): 438 | if frame.group: 439 | frame.group._frames.remove(frame) 440 | 441 | self._frames.append(frame) 442 | frame.group = self 443 | 444 | @property 445 | def exit_frames(self): 446 | """ 447 | Returns a list of frames whose children include a frame outside of the group 448 | """ 449 | if self._exit_frames is None: 450 | exit_frames: list[BaseFrame] = [] 451 | for frame in self.frames: 452 | if any(c.group != self for c in frame.children): 453 | exit_frames.append(frame) 454 | self._exit_frames = exit_frames 455 | 456 | return self._exit_frames 457 | 458 | def __repr__(self): 459 | return "FrameGroup(len(frames)=%d)" % len(self.frames) 460 | --------------------------------------------------------------------------------