├── .editorconfig ├── .github └── ISSUE_TEMPLATE.md ├── .gitignore ├── .project ├── .pydevproject ├── .readthedocs.yaml ├── .settings └── org.eclipse.core.resources.prefs ├── .travis.yml ├── AUTHORS.rst ├── CMakeLists.txt ├── CONTRIBUTING.rst ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── build_all.sh ├── docs ├── .gitignore ├── Makefile ├── make.bat └── source │ ├── authors.rst │ ├── conf.py │ ├── contributing.rst │ ├── examples.rst │ ├── examples │ ├── c_py_mem_trace.rst │ ├── debug_malloc_stats.rst │ ├── dtrace.rst │ ├── images │ │ └── process.log_14129.svg │ ├── process.rst │ └── trace_malloc.rst │ ├── history.rst │ ├── index.rst │ ├── installation.rst │ ├── memory_leaks.rst │ ├── memory_leaks │ ├── introduction.rst │ ├── techniques.rst │ └── tools.rst │ ├── readme.rst │ ├── ref │ ├── c_mem_leak.rst │ ├── c_py_mem_trace.rst │ ├── debug_malloc_stats.rst │ ├── process.rst │ ├── redirect_stdout.rst │ └── trace_malloc.rst │ ├── reference.rst │ ├── tech_notes │ ├── cPyMemTrace.rst │ ├── dtrace.rst │ ├── images │ │ ├── LASToHTML.log_20328.svg │ │ ├── LASToHTML.log_3938.svg │ │ ├── LASToHTML.log_4147.svg │ │ ├── LASToHTML.log_76753.svg │ │ ├── LASToHTML.log_77077.svg │ │ ├── LASToHTML.log_77633.svg │ │ ├── LASToHTML.log_8631.svg │ │ ├── LASToHTML.log_8692.svg │ │ ├── LASToHTML.log_9236.svg │ │ ├── LASToHTML.log_9434.svg │ │ ├── LASToHTML.log_9552.svg │ │ └── LASToHTML.log_9685.svg │ └── rss_cost.rst │ └── technical_notes.rst ├── pymemtrace ├── __init__.py ├── debug_malloc_stats.py ├── examples │ ├── __init__.py │ ├── ex_cPyMemTrace.py │ ├── ex_debug_malloc_stats.py │ ├── ex_dtrace.py │ ├── ex_memory_exercise.py │ ├── ex_process.py │ ├── ex_trace_malloc.py │ └── example.py ├── parse_dtrace_output.py ├── process.py ├── redirect_stdout.py ├── src │ ├── c │ │ ├── get_rss.c │ │ └── pymemtrace_util.c │ ├── cpy │ │ ├── cCustom.c │ │ ├── cMemLeak.c │ │ └── cPyMemTrace.c │ ├── include │ │ ├── get_rss.h │ │ └── pymemtrace_util.h │ └── main.c ├── trace_malloc.py └── util │ └── gnuplot.py ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── _test_redirect_stdout.py ├── test_cMemLeak.py ├── test_cpymemtrace.py ├── test_debug_malloc_stats.py ├── test_process.py ├── test_settrace.py └── test_trace_malloc.py ├── toolkit ├── py_flow_malloc_free.d ├── py_flowinfo.d ├── py_flowinfo_rss.d ├── py_malloc.d ├── py_object_D_WITH_PYMALLOC.d ├── py_object_U_WITH_PYMALLOC.d └── py_syscolors.d ├── tox.ini └── travis_pypi_setup.py /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | [LICENSE] 18 | insert_final_newline = false 19 | 20 | [Makefile] 21 | indent_style = tab 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | * pymemtrace version: 2 | * Python version: 3 | * Operating System: 4 | 5 | ### Description 6 | 7 | Describe what you were trying to get done. 8 | Tell us what happened, what went wrong, and what you expected to happen. 9 | 10 | ### What I Did 11 | 12 | ``` 13 | Paste the command(s) you ran and the output. 14 | If there was a crash, please include the traceback here. 15 | ``` 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Compressed logs 58 | *.log.tgz 59 | 60 | # Flask stuff: 61 | instance/ 62 | .webassets-cache 63 | 64 | # Scrapy stuff: 65 | .scrapy 66 | 67 | # Sphinx documentation 68 | docs/_build/ 69 | 70 | # PyBuilder 71 | target/ 72 | 73 | # Jupyter Notebook 74 | .ipynb_checkpoints 75 | 76 | # pyenv 77 | .python-version 78 | 79 | # celery beat schedule file 80 | celerybeat-schedule 81 | 82 | # SageMath parsed files 83 | *.sage.py 84 | 85 | # dotenv 86 | .env 87 | 88 | # virtualenv 89 | .venv 90 | venv/ 91 | ENV/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # PyCharm 107 | .idea/ 108 | 109 | # Temporary files 110 | tmp/ 111 | 112 | # Mac OS X 113 | .DS_Store 114 | 115 | # pytest 116 | .pytest_cache 117 | 118 | # cmake files 119 | cmake-build-debug/ 120 | cmake-build-release/ 121 | cmake-build-debug-event-trace/ 122 | 123 | attic/ 124 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | pymemtrace 4 | 5 | 6 | 7 | 8 | 9 | org.python.pydev.PyDevBuilder 10 | 11 | 12 | 13 | 14 | 15 | org.python.pydev.pythonNature 16 | 17 | 18 | -------------------------------------------------------------------------------- /.pydevproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /${PROJECT_DIR_NAME} 5 | 6 | python 3.0 7 | pymemtrace_00 8 | 9 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.11" 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/source/conf.py 17 | 18 | # We recommend specifying your dependencies to enable reproducible builds: 19 | # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 20 | # python: 21 | # install: 22 | # - requirements: docs/requirements.txt -------------------------------------------------------------------------------- /.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding//docs/source/conf.py=utf-8 3 | encoding//pymemtrace/__init__.py=utf-8 4 | encoding//pymemtrace/plot/PlotMemTrace.py=utf-8 5 | encoding//pymemtrace/pymemtrace.py=utf-8 6 | encoding//tests/test_data.py=utf-8 7 | encoding//tests/test_pymemtrace.py=utf-8 8 | encoding/setup.py=utf-8 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Config file for automatic testing at travis-ci.org 2 | # This file will be regenerated if you run travis_pypi_setup.py 3 | 4 | language: python 5 | python: 6 | - 3.9 7 | - 3.8 8 | - 3.7 9 | - 3.6 10 | 11 | # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors 12 | install: pip install -U tox-travis 13 | 14 | # command to run tests, e.g. python setup.py test 15 | script: tox 16 | 17 | # After you create the Github repo and add it to Travis, run the 18 | # travis_pypi_setup.py script to finish PyPI deployment setup 19 | deploy: 20 | provider: pypi 21 | distributions: sdist bdist_wheel 22 | user: paulross 23 | password: 24 | secure: PLEASE_REPLACE_ME 25 | on: 26 | tags: true 27 | repo: paulross/pymemtrace 28 | python: 3.9 29 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | Authors 2 | ======= 3 | 4 | Development Lead 5 | ---------------- 6 | 7 | * Paul Ross 8 | 9 | Contributors 10 | ------------ 11 | 12 | None yet. Why not be the first? 13 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.13) 2 | project(cPyMemTrace) 3 | 4 | #set(CMAKE_CXX_STANDARD 17) 5 | set(CMAKE_C_FLAGS "-std=c99") 6 | 7 | #add_compile_definitions(SVF_THREAD_SAFE) 8 | 9 | IF(CMAKE_BUILD_TYPE MATCHES DEBUG) 10 | message("debug build") 11 | add_compile_definitions(DEBUG) 12 | ELSE() 13 | message("release build") 14 | ENDIF(CMAKE_BUILD_TYPE MATCHES DEBUG) 15 | 16 | add_compile_options( 17 | "-Wall" 18 | "-Wextra" 19 | "-Wpedantic" 20 | "-Werror" 21 | "-Wfatal-errors" 22 | # "-Wno-unused-variable" # Temporary 23 | # "-Wno-unused-parameter" # Temporary 24 | "-fexceptions" 25 | "$<$:-O0;-g3;-ggdb>" 26 | ) 27 | 28 | #link_directories( 29 | # /Library/Frameworks/Python.framework/Versions/3.8/lib 30 | #) 31 | 32 | add_executable( 33 | cPyMemTrace 34 | pymemtrace/src/main.c 35 | pymemtrace/src/include/get_rss.h 36 | pymemtrace/src/c/get_rss.c 37 | pymemtrace/src/cpy/cPyMemTrace.c 38 | pymemtrace/src/cpy/cCustom.c 39 | pymemtrace/src/cpy/cMemLeak.c 40 | pymemtrace/src/include/pymemtrace_util.h 41 | pymemtrace/src/c/pymemtrace_util.c 42 | ) 43 | 44 | include_directories( 45 | pymemtrace/src/include 46 | ) 47 | 48 | FIND_PACKAGE (Python3 3.11 EXACT REQUIRED COMPONENTS Interpreter Development) 49 | #FIND_PACKAGE(PythonLibs 3.11 EXACT REQUIRED) 50 | #SET(PythonLibs_DIR "/Library/Frameworks/Python.framework/Versions/3.8") 51 | #FIND_PACKAGE(PythonLibs 3.8 REQUIRED PATHS ("/Library/Frameworks/Python.framework/Versions/3.8")) 52 | #FindPythonLibs() 53 | IF (Python3_FOUND) 54 | INCLUDE_DIRECTORIES("${Python3_INCLUDE_DIRS}") 55 | get_filename_component(PYTHON_LINK_DIRECTORY ${PYTHON_LIBRARY} DIRECTORY) 56 | # See: https://cmake.org/cmake/help/latest/module/FindPython3.html#module:FindPython3 57 | message("Python3_VERSION: ${Python3_VERSION}") 58 | message("Python3_EXECUTABLE: ${Python3_EXECUTABLE}") 59 | message("Python3_INTERPRETER_ID: ${Python3_INTERPRETER_ID}") 60 | message("Python3_INCLUDE_DIRS: ${Python3_INCLUDE_DIRS}") 61 | message("Python3_STDLIB: ${Python3_STDLIB}") 62 | message("Python3_STDARCH: ${Python3_STDARCH}") 63 | message("Python3_LINK_OPTIONS: ${Python3_LINK_OPTIONS}") 64 | message("Python3_LIBRARIES: ${Python3_LIBRARIES}") 65 | ELSE () 66 | MESSAGE(FATAL_ERROR "Unable to find Python libraries.") 67 | ENDIF () 68 | 69 | #FIND_PACKAGE(PythonLibs 3.8 REQUIRED) 70 | #IF(PYTHONLIBS_FOUND) 71 | # message(status " Python: ${PYTHON_INCLUDE_DIRS}") 72 | # INCLUDE_DIRECTORIES("${PYTHON_INCLUDE_DIRS}") 73 | #ELSE() 74 | # MESSAGE(FATAL_ERROR "Unable to find Python libraries.") 75 | #ENDIF() 76 | 77 | link_directories(${PYTHON_LINK_LIBRARY}) 78 | 79 | target_link_libraries(${PROJECT_NAME} ${PYTHON_LIBRARY}) 80 | 81 | #target_link_libraries(cPyMemTrace python3.8) 82 | 83 | target_compile_options(cPyMemTrace PRIVATE -Wall -Wextra -Wno-c99-extensions -pedantic)# -Werror) 84 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============ 4 | Contributing 5 | ============ 6 | 7 | Contributions are welcome, and they are greatly appreciated! Every 8 | little bit helps, and credit will always be given. 9 | 10 | You can contribute in many ways: 11 | 12 | Types of Contributions 13 | ---------------------- 14 | 15 | Report Bugs 16 | ~~~~~~~~~~~ 17 | 18 | Report bugs at https://github.com/paulross/pymemtrace/issues. 19 | 20 | If you are reporting a bug, please include: 21 | 22 | * Your operating system name and version. 23 | * Any details about your local setup that might be helpful in troubleshooting. 24 | * Detailed steps to reproduce the bug. 25 | 26 | Fix Bugs 27 | ~~~~~~~~ 28 | 29 | Look through the GitHub issues for bugs. Anything tagged with "bug" 30 | and "help wanted" is open to whoever wants to implement it. 31 | 32 | Implement Features 33 | ~~~~~~~~~~~~~~~~~~ 34 | 35 | Look through the GitHub issues for features. Anything tagged with "enhancement" 36 | and "help wanted" is open to whoever wants to implement it. 37 | 38 | Write Documentation 39 | ~~~~~~~~~~~~~~~~~~~ 40 | 41 | pymemtrace could always use more documentation, whether as part of the 42 | official pymemtrace docs, in docstrings, or even on the web in blog posts, 43 | articles, and such. 44 | 45 | Submit Feedback 46 | ~~~~~~~~~~~~~~~ 47 | 48 | The best way to send feedback is to file an issue at https://github.com/paulross/pymemtrace/issues. 49 | 50 | If you are proposing a feature: 51 | 52 | * Explain in detail how it would work. 53 | * Keep the scope as narrow as possible, to make it easier to implement. 54 | * Remember that this is a volunteer-driven project, and that contributions 55 | are welcome :) 56 | 57 | Get Started! 58 | ------------ 59 | 60 | Ready to contribute? Here's how to set up `pymemtrace` for local development. 61 | 62 | 1. Fork the `pymemtrace` repo on GitHub. 63 | 2. Clone your fork locally:: 64 | 65 | $ git clone git@github.com:your_name_here/pymemtrace.git 66 | 67 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 68 | 69 | $ mkvirtualenv pymemtrace 70 | $ cd pymemtrace/ 71 | $ python setup.py develop 72 | 73 | 4. Create a branch for local development:: 74 | 75 | $ git checkout -b name-of-your-bugfix-or-feature 76 | 77 | Now you can make your changes locally. 78 | 79 | 5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox:: 80 | 81 | $ flake8 pymemtrace tests 82 | $ python setup.py test or py.test 83 | $ tox 84 | 85 | To get flake8 and tox, just pip install them into your virtualenv. 86 | 87 | 6. Commit your changes and push your branch to GitHub:: 88 | 89 | $ git add . 90 | $ git commit -m "Your detailed description of your changes." 91 | $ git push origin name-of-your-bugfix-or-feature 92 | 93 | 7. Submit a pull request through the GitHub website. 94 | 95 | Pull Request Guidelines 96 | ----------------------- 97 | 98 | Before you submit a pull request, check that it meets these guidelines: 99 | 100 | 1. The pull request should include tests. 101 | 2. If the pull request adds functionality, the docs should be updated. Put 102 | your new functionality into a function with a docstring, and add the 103 | feature to the list in README.rst. 104 | 3. The pull request should work for Python 3.6, 3.7, 3.8 and 3.9, and for PyPy. Check 105 | https://travis-ci.org/paulross/pymemtrace/pull_requests 106 | and make sure that the tests pass for all supported Python versions. 107 | 108 | Tips 109 | ---- 110 | 111 | To run a subset of tests:: 112 | 113 | $ py.test tests.test_pymemtrace 114 | 115 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | History 3 | ======= 4 | 5 | 0.2.0 (2024-11-17) 6 | --------------------- 7 | 8 | * cPyMemTrace: 9 | * Add P/T, stack depth and python version to log file name, example: 10 | "20241107_195847_62264_P_0_PY3.13.0b3.log" 11 | * Add stacking of trace/profile functions with linked list of tTraceFileWrapperLinkedList. 12 | * Add an option to log to a specific file. 13 | * Add an API write_to_log() to inject text into the log file. 14 | * Add an optional message to the log file in cPyMemTrace. 15 | * Add Python API to get log file being written to by cPyMemTrace. 16 | * Bug fixes in cPyMemTrace.c 17 | * Safety fix for file path name lengths. 18 | * Fix for log files where '#' was being concatenated. 19 | 20 | 0.1.7 (2024-09-12) 21 | ------------------ 22 | 23 | * Minor fix for a single test. 24 | 25 | 0.1.6 (2024-09-11) 26 | ------------------ 27 | 28 | * Add support for Python versions 3.12, 3.13. Now supports Python versions 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13. 29 | 30 | 0.1.5 (2023-06-21) 31 | ------------------ 32 | 33 | * Add support for Python versions 3.10, 3.11. Now supports Python versions 3.7, 3.8, 3.9, 3.10, 3.11. 34 | 35 | 0.1.4 (2022-03-19) 36 | ------------------ 37 | 38 | * Fix Linux build. 39 | 40 | 0.1.3 (2022-03-17) 41 | ------------------ 42 | 43 | * Fix some tests. 44 | 45 | 0.1.2 (2022-03-17) 46 | ------------------ 47 | 48 | * Fix source distribution that had missing headers. 49 | 50 | 0.1.1 (2020-11-17) 51 | ------------------ 52 | 53 | * Add cPyMemTrace the C level profiler. 54 | * Add DTrace scripts for low level tracing. 55 | * Add debug_malloc_stats the wrapper around sys._debugmallocstats. 56 | * Add process from the TotalDepth project. 57 | * Add redirect_stdout for debug_malloc_stats. 58 | * Add trace_malloc, a wrapper around the tracemalloc module. 59 | * Includes extensive documentation and performance measurement. 60 | * First release on PyPI. 61 | 62 | 0.1.0 (2017-12-04) 63 | ------------------ 64 | 65 | * Initial idea and implementation, never released. 66 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 paulross 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.rst 2 | include CONTRIBUTING.rst 3 | include HISTORY.rst 4 | include LICENSE 5 | include README.rst 6 | 7 | recursive-include tests * 8 | recursive-exclude * __pycache__ 9 | recursive-exclude * *.py[co] 10 | 11 | recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif *.svg 12 | 13 | graft pymemtrace/src/include 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean clean-test clean-pyc clean-build docs help 2 | .DEFAULT_GOAL := help 3 | define BROWSER_PYSCRIPT 4 | import os, webbrowser, sys 5 | try: 6 | from urllib import pathname2url 7 | except: 8 | from urllib.request import pathname2url 9 | 10 | webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) 11 | endef 12 | export BROWSER_PYSCRIPT 13 | 14 | define PRINT_HELP_PYSCRIPT 15 | import re, sys 16 | 17 | for line in sys.stdin: 18 | match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) 19 | if match: 20 | target, help = match.groups() 21 | print("%-20s %s" % (target, help)) 22 | endef 23 | export PRINT_HELP_PYSCRIPT 24 | BROWSER := python -c "$$BROWSER_PYSCRIPT" 25 | 26 | help: 27 | @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) 28 | 29 | clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts 30 | 31 | 32 | clean-build: ## remove build artifacts 33 | rm -fr build/ 34 | rm -fr dist/ 35 | rm -fr .eggs/ 36 | find . -name '*.egg-info' -exec rm -fr {} + 37 | find . -name '*.egg' -exec rm -f {} + 38 | 39 | clean-pyc: ## remove Python file artifacts 40 | find . -name '*.pyc' -exec rm -f {} + 41 | find . -name '*.pyo' -exec rm -f {} + 42 | find . -name '*~' -exec rm -f {} + 43 | find . -name '__pycache__' -exec rm -fr {} + 44 | 45 | clean-test: ## remove test and coverage artifacts 46 | rm -fr .tox/ 47 | rm -f .coverage 48 | rm -fr htmlcov/ 49 | 50 | lint: ## check style with flake8 51 | flake8 pymemtrace tests 52 | 53 | test: ## run tests quickly with the default Python 54 | py.test 55 | 56 | 57 | test-all: ## run tests on every Python version with tox 58 | tox 59 | 60 | coverage: ## check code coverage quickly with the default Python 61 | coverage run --source pymemtrace -m pytest 62 | coverage report -m 63 | coverage html 64 | $(BROWSER) htmlcov/index.html 65 | 66 | docs: ## generate Sphinx HTML documentation, including API docs 67 | rm -f docs/pymemtrace.rst 68 | rm -f docs/modules.rst 69 | sphinx-apidoc -o docs/ pymemtrace 70 | $(MAKE) -C docs clean 71 | $(MAKE) -C docs html 72 | $(BROWSER) docs/_build/html/index.html 73 | 74 | servedocs: docs ## compile the docs watching for changes 75 | watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . 76 | 77 | release: clean ## package and upload a release 78 | python setup.py sdist upload 79 | python setup.py bdist_wheel upload 80 | 81 | dist: clean ## builds source and wheel package 82 | python setup.py sdist 83 | python setup.py bdist_wheel 84 | ls -l dist 85 | 86 | install: clean ## install the package to the active Python's site-packages 87 | python setup.py install 88 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ******************* 2 | Introduction 3 | ******************* 4 | 5 | 6 | ``pymemtrace`` provides tools for tracking and understanding Python memory usage at different levels, at different 7 | granularities and with different runtime costs. 8 | 9 | Full documentation: https://pymemtrace.readthedocs.io 10 | 11 | pymemtrace Tools 12 | ====================== 13 | 14 | The tools provided by ``pymemtrace``: 15 | 16 | * ``process`` is a very lightweight way of logging the total memory usage at regular time intervals. 17 | It can plot memory over time with plotting programs such as ``gnuplot``. 18 | See `some process examples `_ 19 | * ``cPyMemTrace`` is a memory tracer written in C that can report total memory usage for every function call/return for 20 | both C and Python sections. 21 | See some `cPyMemTrace examples `_ 22 | and a `technical note on cPyMemTrace `_. 23 | * DTrace: Here are a number of D scripts that can trace the low level ``malloc()`` and ``free()`` system calls and 24 | report how much memory was allocated and by whom. 25 | See some `DTrace examples `_ 26 | and a `technical note on DTrace `_. 27 | * ``trace_malloc`` is a convenience wrapper around the Python standard library `tracemalloc` module. 28 | This can report Python memory usage by module and line compensating for the cost of ``tracemalloc``. 29 | This can take memory snapshots before and after code blocks and show the change on memory caused by that code. 30 | See some `trace_malloc examples `_ 31 | * ``debug_malloc_stats`` is a wrapper around the ``sys._debugmallocstats`` function that can take snapshots of 32 | memory before and after code execution and report the significant differences of the Python small object allocator. 33 | See some `debug_malloc_stats examples `_ 34 | 35 | 36 | Tool Characteristics 37 | ====================== 38 | 39 | Each tool can be characterised by: 40 | 41 | - *Memory Granularity*: In how much detail is a memory change is observed. 42 | An example of *coarse* memory granularity is measuring the 43 | `Resident Set Size `_ which is normally in chunks of 4096 bytes. 44 | An example of *fine* memory granularity is recording every ``malloc()`` and ``free()``. 45 | - *Execution Granularity*: In how much code detail is the memory change observed. 46 | An example of *coarse* execution granularity is measuring the memory usage every second. 47 | An example of *fine* execution granularity is recording the memory usage every Python line. 48 | - *Memory Cost*: How much extra memory the tool needs. 49 | - *Execution Cost*: How much the execution time is increased. 50 | 51 | Clearly there are trade-offs between these depending on the problem you are trying to solve. 52 | 53 | .. list-table:: **Tool Characteristics** 54 | :widths: 15 30 30 30 30 55 | :header-rows: 1 56 | 57 | * - Tool 58 | - Memory Granularity 59 | - Execution Granularity 60 | - Memory Cost 61 | - Execution Cost 62 | * - ``process`` 63 | - RSS (total Python and C memory). 64 | - Regular time intervals. 65 | - Near zero. 66 | - Near zero. 67 | * - ``cPyMemTrace`` 68 | - RSS (total Python and C memory). 69 | - Per Python line, Python function and C function call. 70 | - Near zero. 71 | - x10 to x20. 72 | * - DTrace 73 | - Every ``malloc()`` and ``free()``. 74 | - Per function call and return. 75 | - Minimal. 76 | - x90 to x100. 77 | * - ``trace_malloc`` 78 | - Every Python object. 79 | - Per Python line, per function call. 80 | - Significant but compensated. 81 | - x900 for small objects, x6 for large objects. 82 | * - ``debug_malloc_stats`` 83 | - Python memory pool. 84 | - Snapshots the CPython memory pool either side of a block of code. 85 | - Minimal. 86 | - x2000+ for small objects, x12 for large objects. 87 | 88 | Package Metadata 89 | ========================= 90 | 91 | .. image:: https://img.shields.io/pypi/v/pymemtrace.svg 92 | :target: https://pypi.python.org/pypi/pymemtrace 93 | 94 | .. image:: https://img.shields.io/travis/paulross/pymemtrace.svg 95 | :target: https://travis-ci.org/paulross/pymemtrace 96 | 97 | .. image:: https://readthedocs.org/projects/pymemtrace/badge/?version=latest 98 | :target: https://pymemtrace.readthedocs.io/en/latest/?badge=latest 99 | :alt: Documentation Status 100 | 101 | .. image:: https://pyup.io/repos/github/paulross/pymemtrace/shield.svg 102 | :target: https://pyup.io/repos/github/paulross/pymemtrace/ 103 | :alt: Updates 104 | 105 | 106 | Licence 107 | ----------------------- 108 | 109 | Python memory tracing. 110 | 111 | * Free software: MIT license 112 | * Documentation: https://pymemtrace.readthedocs.io. 113 | * Project: https://github.com/paulross/pymemtrace. 114 | 115 | Credits 116 | ----------------- 117 | 118 | Phil Smith (AHL) with whom a casual lunch time chat lead to the creation of an earlier, but quite different 119 | implementation, of ``cPyMemTrace`` in pure Python. 120 | 121 | This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. 122 | 123 | .. _Cookiecutter: https://github.com/audreyr/cookiecutter 124 | .. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage 125 | 126 | -------------------------------------------------------------------------------- /build_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Builds pymentrace for distribution 4 | # Ref: https://packaging.python.org/tutorials/packaging-projects/ 5 | # 6 | # Other references: 7 | # https://kvz.io/bash-best-practices.html 8 | # https://bertvv.github.io/cheat-sheets/Bash.html 9 | 10 | set -o errexit # abort on nonzero exitstatus 11 | set -o nounset # abort on unbound variable 12 | set -o pipefail # don't hide errors within pipes 13 | 14 | PYTHON_VERSIONS=('3.7' '3.8' '3.9' '3.10' '3.11' '3.12' '3.13') 15 | PYTHON_VENV_ROOT="${HOME}/pyenvs" 16 | # Used for venvs 17 | PROJECT_NAME="pymemtrace" 18 | 19 | #printf "%-8s %8s %10s %10s %12s\n" "Ext" "Files" "Lines" "Words" "Bytes" 20 | 21 | deactivate_virtual_environment() { 22 | # https://stackoverflow.com/questions/42997258/virtualenv-activate-script-wont-run-in-bash-script-with-set-euo 23 | set +u 24 | if command -v deactivate &>/dev/null; then 25 | deactivate 26 | fi 27 | set -u 28 | } 29 | 30 | create_virtual_environments() { 31 | deactivate_virtual_environment 32 | for version in ${PYTHON_VERSIONS[*]}; do 33 | echo "---> Create virtual environment for Python version ${version}" 34 | venv_path="${PYTHON_VENV_ROOT}/${PROJECT_NAME}_${version}" 35 | if [ ! -d "${venv_path}" ]; then 36 | # Control will enter here if directory not exists. 37 | echo "---> Creating virtual environment at: ${venv_path}" 38 | "python${version}" -m venv "${venv_path}" 39 | fi 40 | done 41 | } 42 | 43 | remove_virtual_environments() { 44 | deactivate_virtual_environment 45 | for version in ${PYTHON_VERSIONS[*]}; do 46 | echo "---> For Python version ${version}" 47 | venv_path="${PYTHON_VENV_ROOT}/${PROJECT_NAME}_${version}" 48 | if [ -d "${venv_path}" ]; then 49 | # Control will enter here if directory exists. 50 | echo "---> Removing virtual environment at: ${venv_path}" 51 | #rm --recursive --force -- "${venv_path}" 52 | rm -rf -- "${venv_path}" 53 | fi 54 | done 55 | } 56 | 57 | create_bdist_wheel() { 58 | echo "---> Creating bdist_wheel for all versions..." 59 | for version in ${PYTHON_VERSIONS[*]}; do 60 | echo "---> For Python version ${version}" 61 | deactivate_virtual_environment 62 | venv_path="${PYTHON_VENV_ROOT}/${PROJECT_NAME}_${version}" 63 | if [ ! -d "${venv_path}" ]; then 64 | # Control will enter here if directory doesn't exist. 65 | echo "---> Creating virtual environment at: ${venv_path}" 66 | "python${version}" -m venv "${venv_path}" 67 | fi 68 | # https://stackoverflow.com/questions/42997258/virtualenv-activate-script-wont-run-in-bash-script-with-set-euo 69 | set +u 70 | source "${venv_path}/bin/activate" 71 | set -u 72 | echo "---> Python version:" 73 | python -VV 74 | echo "---> Installing everything via pip:" 75 | pip install -U pip setuptools wheel 76 | pip install -r requirements.txt 77 | echo "---> Result of pip install:" 78 | pip list 79 | echo "---> Running python setup.py develop:" 80 | MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ python setup.py develop 81 | echo "---> Running tests:" 82 | # Fail fast with -x 83 | # For some reason we need -s as our redirection of stdout is interfering with/being interfered by pytest. 84 | pytest -s -x tests 85 | echo "---> Running setup for bdist_wheel:" 86 | python setup.py bdist_wheel 87 | done 88 | } 89 | 90 | create_sdist() { 91 | echo "---> Running setup for sdist:" 92 | python setup.py sdist 93 | } 94 | 95 | report_all_versions_and_setups() { 96 | echo "---> Reporting all versions..." 97 | for version in ${PYTHON_VERSIONS[*]}; do 98 | echo "---> For Python version ${version}" 99 | deactivate_virtual_environment 100 | venv_path="${PYTHON_VENV_ROOT}/${PROJECT_NAME}_${version}" 101 | if [ ! -d "${venv_path}" ]; then 102 | # Control will enter here if directory doesn't exist. 103 | echo "---> Creating virtual environment at: ${venv_path}" 104 | "python${version}" -m venv "${venv_path}" 105 | fi 106 | echo "---> Virtual environment at: ${venv_path}" 107 | # https://stackoverflow.com/questions/42997258/virtualenv-activate-script-wont-run-in-bash-script-with-set-euo 108 | set +u 109 | source "${venv_path}/bin/activate" 110 | set -u 111 | echo "---> Python version:" 112 | python -VV 113 | echo "---> pip list:" 114 | pip list 115 | done 116 | } 117 | 118 | show_results_of_dist() { 119 | echo "---> dist/:" 120 | ls -l "dist" 121 | echo "---> twine check dist/*:" 122 | twine check dist/* 123 | # Test from Test PyPi 124 | # pip install -i https://test.pypi.org/simple/orderedstructs 125 | echo "---> Ready for upload to test PyPi:" 126 | echo "---> pip install twine" 127 | echo "---> twine upload --repository testpypi dist/*" 128 | echo "---> Or PyPi:" 129 | echo "---> twine upload dist/*" 130 | } 131 | 132 | echo "===> Removing build/ and dist/" 133 | #rm --recursive --force -- "build" "dist" 134 | rm -rf -- "build" "dist" 135 | remove_virtual_environments 136 | create_virtual_environments 137 | create_bdist_wheel 138 | create_sdist 139 | report_all_versions_and_setups 140 | pip install twine 141 | show_results_of_dist 142 | echo "===> All done" 143 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | /pymemtrace.rst 2 | /pymemtrace.*.rst 3 | /modules.rst 4 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pymemtrace.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pymemtrace.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/pymemtrace" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pymemtrace" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\pymemtrace.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\pymemtrace.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /docs/source/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # pymemtrace documentation build configuration file, created by 5 | # sphinx-quickstart on Tue Jul 9 22:26:36 2013. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import sys 17 | import os 18 | 19 | # If extensions (or modules to document with autodoc) are in another 20 | # directory, add these directories to sys.path here. If the directory is 21 | # relative to the documentation root, use os.path.abspath to make it 22 | # absolute, like shown here. 23 | #sys.path.insert(0, os.path.abspath('.')) 24 | 25 | # Get the project root dir, which is the parent dir of this 26 | cwd = os.getcwd() 27 | project_root = os.path.dirname(cwd) 28 | 29 | # Insert the project root dir as the first element in the PYTHONPATH. 30 | # This lets us ensure that the source package is imported, and that its 31 | # version is used. 32 | sys.path.insert(0, project_root) 33 | sys.path.insert(0, os.path.abspath('../../')) 34 | # sys.path.insert(0, os.path.abspath('../../pymemtrace')) 35 | # sys.path.insert(0, os.path.abspath('../pymemtrace/plot')) 36 | 37 | import pymemtrace 38 | 39 | # -- General configuration --------------------------------------------- 40 | 41 | # If your documentation needs a minimal Sphinx version, state it here. 42 | #needs_sphinx = '1.0' 43 | 44 | # Add any Sphinx extension module names here, as strings. They can be 45 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 46 | extensions = [ 47 | 'sphinx.ext.autodoc', 48 | 'sphinx.ext.doctest', 49 | 'sphinx.ext.todo', 50 | 'sphinx.ext.coverage', 51 | 'sphinx.ext.imgmath', 52 | 'sphinx.ext.viewcode', 53 | ]# 'sphinx.ext.mathjax'] 54 | 55 | # Add any paths that contain templates here, relative to this directory. 56 | templates_path = ['_templates'] 57 | 58 | # The suffix of source filenames. 59 | source_suffix = '.rst' 60 | 61 | # The encoding of source files. 62 | #source_encoding = 'utf-8-sig' 63 | 64 | # The master toctree document. 65 | master_doc = 'index' 66 | 67 | # General information about the project. 68 | project = u'pymemtrace' 69 | copyright = u"2017-2024, Paul Ross" 70 | 71 | # The version info for the project you're documenting, acts as replacement 72 | # for |version| and |release|, also used in various other places throughout 73 | # the built documents. 74 | # 75 | # The short X.Y version. 76 | version = pymemtrace.__version__ 77 | # The full version, including alpha/beta/rc tags. 78 | release = pymemtrace.__version__ 79 | 80 | # The language for content autogenerated by Sphinx. Refer to documentation 81 | # for a list of supported languages. 82 | #language = None 83 | 84 | # There are two options for replacing |today|: either, you set today to 85 | # some non-false value, then it is used: 86 | #today = '' 87 | # Else, today_fmt is used as the format for a strftime call. 88 | #today_fmt = '%B %d, %Y' 89 | 90 | # List of patterns, relative to source directory, that match files and 91 | # directories to ignore when looking for source files. 92 | exclude_patterns = ['_build'] 93 | 94 | # The reST default role (used for this markup: `text`) to use for all 95 | # documents. 96 | #default_role = None 97 | 98 | # If true, '()' will be appended to :func: etc. cross-reference text. 99 | #add_function_parentheses = True 100 | 101 | # If true, the current module name will be prepended to all description 102 | # unit titles (such as .. function::). 103 | #add_module_names = True 104 | 105 | # If true, sectionauthor and moduleauthor directives will be shown in the 106 | # output. They are ignored by default. 107 | #show_authors = False 108 | 109 | # The name of the Pygments (syntax highlighting) style to use. 110 | pygments_style = 'sphinx' 111 | 112 | # A list of ignored prefixes for module index sorting. 113 | #modindex_common_prefix = [] 114 | 115 | # If true, keep warnings as "system message" paragraphs in the built 116 | # documents. 117 | #keep_warnings = False 118 | 119 | 120 | # -- Options for HTML output ------------------------------------------- 121 | 122 | # The theme to use for HTML and HTML Help pages. See the documentation for 123 | # a list of builtin themes. 124 | # See https://sphinx-themes.org 125 | html_theme = 'nature' 126 | 127 | # Theme options are theme-specific and customize the look and feel of a 128 | # theme further. For a list of options available for each theme, see the 129 | # documentation. 130 | #html_theme_options = {} 131 | 132 | # Add any paths that contain custom themes here, relative to this directory. 133 | #html_theme_path = [] 134 | 135 | # The name for this set of Sphinx documents. If None, it defaults to 136 | # " v documentation". 137 | #html_title = None 138 | 139 | # A shorter title for the navigation bar. Default is the same as 140 | # html_title. 141 | #html_short_title = None 142 | 143 | # The name of an image file (relative to this directory) to place at the 144 | # top of the sidebar. 145 | #html_logo = None 146 | 147 | # The name of an image file (within the static path) to use as favicon 148 | # of the docs. This file should be a Windows icon file (.ico) being 149 | # 16x16 or 32x32 pixels large. 150 | #html_favicon = None 151 | 152 | # Add any paths that contain custom static files (such as style sheets) 153 | # here, relative to this directory. They are copied after the builtin 154 | # static files, so a file named "default.css" will overwrite the builtin 155 | # "default.css". 156 | html_static_path = ['_static'] 157 | 158 | # If not '', a 'Last updated on:' timestamp is inserted at every page 159 | # bottom, using the given strftime format. 160 | #html_last_updated_fmt = '%b %d, %Y' 161 | 162 | # If true, SmartyPants will be used to convert quotes and dashes to 163 | # typographically correct entities. 164 | #html_use_smartypants = True 165 | 166 | # Custom sidebar templates, maps document names to template names. 167 | #html_sidebars = {} 168 | 169 | # Additional templates that should be rendered to pages, maps page names 170 | # to template names. 171 | #html_additional_pages = {} 172 | 173 | # If false, no module index is generated. 174 | #html_domain_indices = True 175 | 176 | # If false, no index is generated. 177 | #html_use_index = True 178 | 179 | # If true, the index is split into individual pages for each letter. 180 | #html_split_index = False 181 | 182 | # If true, links to the reST sources are added to the pages. 183 | #html_show_sourcelink = True 184 | 185 | # If true, "Created using Sphinx" is shown in the HTML footer. 186 | # Default is True. 187 | #html_show_sphinx = True 188 | 189 | # If true, "(C) Copyright ..." is shown in the HTML footer. 190 | # Default is True. 191 | #html_show_copyright = True 192 | 193 | # If true, an OpenSearch description file will be output, and all pages 194 | # will contain a tag referring to it. The value of this option 195 | # must be the base URL from which the finished HTML is served. 196 | #html_use_opensearch = '' 197 | 198 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 199 | #html_file_suffix = None 200 | 201 | # Output file base name for HTML help builder. 202 | htmlhelp_basename = 'pymemtracedoc' 203 | 204 | 205 | # -- Options for LaTeX output ------------------------------------------ 206 | 207 | latex_elements = { 208 | # The paper size ('letterpaper' or 'a4paper'). 209 | #'papersize': 'letterpaper', 210 | 211 | # The font size ('10pt', '11pt' or '12pt'). 212 | #'pointsize': '10pt', 213 | 214 | # Additional stuff for the LaTeX preamble. 215 | #'preamble': '', 216 | } 217 | 218 | # Grouping the document tree into LaTeX files. List of tuples 219 | # (source start file, target name, title, author, documentclass 220 | # [howto/manual]). 221 | latex_documents = [ 222 | ('index', 'pymemtrace.tex', 223 | u'pymemtrace Documentation', 224 | u'Paul Ross', 'manual'), 225 | ] 226 | 227 | # The name of an image file (relative to this directory) to place at 228 | # the top of the title page. 229 | #latex_logo = None 230 | 231 | # For "manual" documents, if this is true, then toplevel headings 232 | # are parts, not chapters. 233 | #latex_use_parts = False 234 | 235 | # If true, show page references after internal links. 236 | #latex_show_pagerefs = False 237 | 238 | # If true, show URL addresses after external links. 239 | #latex_show_urls = False 240 | 241 | # Documents to append as an appendix to all manuals. 242 | #latex_appendices = [] 243 | 244 | # If false, no module index is generated. 245 | #latex_domain_indices = True 246 | 247 | 248 | # -- Options for manual page output ------------------------------------ 249 | 250 | # One entry per manual page. List of tuples 251 | # (source start file, name, description, authors, manual section). 252 | man_pages = [ 253 | ('index', 'pymemtrace', 254 | u'pymemtrace Documentation', 255 | [u'Paul Ross'], 1) 256 | ] 257 | 258 | # If true, show URL addresses after external links. 259 | #man_show_urls = False 260 | 261 | 262 | # -- Options for Texinfo output ---------------------------------------- 263 | 264 | # Grouping the document tree into Texinfo files. List of tuples 265 | # (source start file, target name, title, author, 266 | # dir menu entry, description, category) 267 | texinfo_documents = [ 268 | ('index', 'pymemtrace', 269 | u'pymemtrace Documentation', 270 | u'Paul Ross', 271 | 'pymemtrace', 272 | 'Various ways of tracing Python memory usage.', 273 | 'Miscellaneous'), 274 | ] 275 | 276 | # Documents to append as an appendix to all manuals. 277 | #texinfo_appendices = [] 278 | 279 | # If false, no module index is generated. 280 | #texinfo_domain_indices = True 281 | 282 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 283 | #texinfo_show_urls = 'footnote' 284 | 285 | # If true, do not generate a @detailmenu in the "Top" node's menu. 286 | #texinfo_no_detailmenu = False 287 | -------------------------------------------------------------------------------- /docs/source/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /docs/source/examples.rst: -------------------------------------------------------------------------------- 1 | ******************* 2 | Examples 3 | ******************* 4 | 5 | Contents: 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | examples/process 11 | examples/c_py_mem_trace 12 | examples/dtrace 13 | examples/debug_malloc_stats 14 | examples/trace_malloc 15 | -------------------------------------------------------------------------------- /docs/source/examples/c_py_mem_trace.rst: -------------------------------------------------------------------------------- 1 | .. _examples-cpymemtrace: 2 | 3 | ``cPyMemTrace`` Examples 4 | =============================================== 5 | 6 | ``cPyMemTrace`` is a Python profiler written in 'C' that records the `Resident Set Size `_ 7 | for every Python and C call and return. 8 | It writes this data to a log file with a name of the form ``YYMMDD_HHMMSS_PID.log``. 9 | 10 | Logging Changes in RSS 11 | -------------------------------- 12 | 13 | Here is a simple example: 14 | 15 | .. code-block:: python 16 | 17 | from pymemtrace import cPyMemTrace 18 | 19 | def create_string(l: int) -> str: 20 | return ' ' * l 21 | 22 | with cPyMemTrace.Profile(): 23 | l = [] 24 | for i in range(8): 25 | l.append(create_string(1024**2)) 26 | while len(l): 27 | l.pop() 28 | 29 | This produces a log file in the current working directory: 30 | 31 | .. code-block:: text 32 | 33 | Event dEvent Clock What File #line Function RSS dRSS 34 | NEXT: 0 +0 0.066718 CALL test.py # 9 create_string 9101312 9101312 35 | NEXT: 1 +1 0.067265 RETURN test.py # 10 create_string 10153984 1052672 36 | PREV: 4 +3 0.067285 CALL test.py # 9 create_string 10153984 0 37 | NEXT: 5 +4 0.067777 RETURN test.py # 10 create_string 11206656 1052672 38 | PREV: 8 +3 0.067787 CALL test.py # 9 create_string 11206656 0 39 | NEXT: 9 +4 0.068356 RETURN test.py # 10 create_string 12259328 1052672 40 | PREV: 12 +3 0.068367 CALL test.py # 9 create_string 12259328 0 41 | NEXT: 13 +4 0.068944 RETURN test.py # 10 create_string 13312000 1052672 42 | PREV: 16 +3 0.068954 CALL test.py # 9 create_string 13312000 0 43 | NEXT: 17 +4 0.069518 RETURN test.py # 10 create_string 14364672 1052672 44 | PREV: 20 +3 0.069534 CALL test.py # 9 create_string 14364672 0 45 | NEXT: 21 +4 0.070101 RETURN test.py # 10 create_string 15417344 1052672 46 | PREV: 24 +3 0.070120 CALL test.py # 9 create_string 15417344 0 47 | NEXT: 25 +4 0.070663 RETURN test.py # 10 create_string 16470016 1052672 48 | PREV: 28 +3 0.070677 CALL test.py # 9 create_string 16470016 0 49 | NEXT: 29 +4 0.071211 RETURN test.py # 10 create_string 17522688 1052672 50 | 51 | By default not all events are recorded just any that increase the RSS by one page along with the immediately preceding event. 52 | 53 | Logging Every Event 54 | -------------------------------- 55 | 56 | If all events are needed then change the constructor argument to 0: 57 | 58 | .. code-block:: python 59 | 60 | with cPyMemTrace.Profile(0): 61 | # As before 62 | 63 | And the log file looks like this: 64 | 65 | .. code-block:: text 66 | 67 | Event dEvent Clock What File #line Function RSS dRSS 68 | NEXT: 0 +0 0.079408 CALL test.py # 9 create_string 9105408 9105408 69 | NEXT: 1 +1 0.079987 RETURN test.py # 10 create_string 10158080 1052672 70 | NEXT: 2 +1 0.079994 C_CALL test.py # 64 append 10158080 0 71 | NEXT: 3 +1 0.079998 C_RETURN test.py # 64 append 10158080 0 72 | NEXT: 4 +1 0.080003 CALL test.py # 9 create_string 10158080 0 73 | NEXT: 5 +1 0.080682 RETURN test.py # 10 create_string 11210752 1052672 74 | NEXT: 6 +1 0.080693 C_CALL test.py # 64 append 11210752 0 75 | NEXT: 7 +1 0.080698 C_RETURN test.py # 64 append 11210752 0 76 | NEXT: 8 +1 0.080704 CALL test.py # 9 create_string 11210752 0 77 | NEXT: 9 +1 0.081414 RETURN test.py # 10 create_string 12263424 1052672 78 | NEXT: 10 +1 0.081424 C_CALL test.py # 64 append 12263424 0 79 | NEXT: 11 +1 0.081429 C_RETURN test.py # 64 append 12263424 0 80 | NEXT: 12 +1 0.081434 CALL test.py # 9 create_string 12263424 0 81 | NEXT: 13 +1 0.081993 RETURN test.py # 10 create_string 13316096 1052672 82 | NEXT: 14 +1 0.081998 C_CALL test.py # 64 append 13316096 0 83 | ... 84 | NEXT: 59 +1 0.084531 C_RETURN test.py # 66 pop 17526784 0 85 | NEXT: 60 +1 0.084535 C_CALL test.py # 65 len 17526784 0 86 | NEXT: 61 +1 0.084539 C_RETURN test.py # 65 len 17526784 0 87 | NEXT: 62 +1 0.084541 C_CALL test.py # 66 pop 17526784 0 88 | NEXT: 63 +1 0.084561 C_RETURN test.py # 66 pop 17526784 0 89 | NEXT: 64 +1 0.084566 C_CALL test.py # 65 len 17526784 0 90 | NEXT: 65 +1 0.084568 C_RETURN test.py # 65 len 17526784 0 91 | 92 | There is some discussion about the performance of ``cPyMemTrace`` here :ref:`tech_notes-cpymemtrace` 93 | -------------------------------------------------------------------------------- /docs/source/examples/debug_malloc_stats.rst: -------------------------------------------------------------------------------- 1 | .. _examples-debug_malloc_stats: 2 | 3 | ``debug_malloc_stats`` Examples 4 | =================================== 5 | 6 | These Python examples are in :py:mod:`pymemtrace.examples.ex_debug_alloc_stats` 7 | 8 | Adding Small Strings 9 | ---------------------------- 10 | 11 | Here is an example of adding small strings to a list under the watchful eye of :py:class:`debug_malloc_stats.DiffSysDebugMallocStats`: 12 | 13 | .. code-block:: python 14 | 15 | print(f'example_debug_malloc_stats_for_documentation()') 16 | with debug_malloc_stats.DiffSysDebugMallocStats() as malloc_diff: 17 | for i in range(1, 9): 18 | list_of_strings.append(' ' * (i * 8)) 19 | print(f'DiffSysDebugMallocStats.diff():') 20 | print(f'{malloc_diff.diff()}') 21 | 22 | The output is: 23 | 24 | .. code-block:: text 25 | 26 | example_debug_malloc_stats_for_documentation() 27 | DiffSysDebugMallocStats.diff(): 28 | class size num pools blocks in use avail blocks 29 | ----- ---- --------- ------------- ------------ 30 | 1 32 +1 +52 +74 31 | 2 48 +0 +17 -17 32 | 3 64 +0 +33 -33 33 | 4 80 +1 +51 -1 34 | 5 96 +2 +34 +50 35 | 6 112 +0 +2 -2 36 | 7 128 +0 +1 -1 37 | 10 176 +0 +1 -1 38 | 12 208 +0 +1 -1 39 | 17 288 +0 +1 -1 40 | 18 304 +0 +2 -2 41 | 25 416 +0 +3 -3 42 | 26 432 +0 +3 -3 43 | 27 448 +0 +3 -3 44 | 29 480 +0 +3 -3 45 | 30 496 +0 +1 -1 46 | 31 512 +0 +1 -1 47 | 48 | # bytes in allocated blocks = +19,904 49 | # bytes in available blocks = -3,808 50 | -4 unused pools * 4096 bytes = -16,384 51 | # bytes lost to pool headers = +192 52 | # bytes lost to quantization = +96 53 | 54 | -1 free 1-sized PyTupleObjects * 32 bytes each = -32 55 | +1 free 5-sized PyTupleObjects * 64 bytes each = +64 56 | +2 free PyDictObjects * 48 bytes each = +96 57 | -2 free PyListObjects * 40 bytes each = -80 58 | +1 free PyMethodObjects * 48 bytes each = +48 59 | 60 | 61 | Cost of ``debug_malloc_stats`` 62 | ----------------------------------- 63 | 64 | In :py:mod:`pymemtrace.examples.ex_debug_alloc_stats` are some timing examples of creating a list of strings of varying size 65 | with and without ``debug_malloc_stats``. 66 | Here are some typical results: 67 | 68 | .. Commented out typical output: 69 | 70 | $ caffeinate python pymemtrace/examples/ex_debug_malloc_stats.py 71 | number=10,000 repeat=5 convert=1,000,000 72 | example_timeit_under_512 : 2.746, 2.584, 2.582, 2.664, 2.462 mean= 2.607 min= 2.462 max= 2.746 span= 0.284 73 | example_timeit_under_512_with_debug_malloc_stats : 5556.577, 6321.485, 6391.563, 6247.821, 7243.693 mean= 6352.228 min= 5556.577 max= 7243.693 span= 1687.116 x2436.232 74 | example_timeit_over_512 : 5.428, 4.661, 5.704, 6.326, 4.507 mean= 5.325 min= 4.507 max= 6.326 span= 1.819 75 | example_timeit_over_512_with_debug_malloc_stats : 7074.884, 6553.412, 7123.040, 6636.192, 6707.841 mean= 6819.074 min= 6553.412 max= 7123.040 span= 569.628 x1280.509 76 | example_timeit_well_over_512 : 639.517, 482.394, 562.109, 681.655, 598.415 mean= 592.818 min= 482.394 max= 681.655 span= 199.261 77 | example_timeit_well_over_512_with_debug_malloc_stats : 7322.035, 6952.874, 7611.174, 7739.893, 7302.739 mean= 7385.743 min= 6952.874 max= 7739.893 span= 787.019 x 12.459 78 | (pymemtrace_3.8_A) 79 | 80 | .. list-table:: **Times in µs** 81 | :widths: 25 25 25 25 82 | :header-rows: 1 83 | 84 | * - Task 85 | - Without ``debug_malloc_stats`` 86 | - With ``debug_malloc_stats`` 87 | - Ratio 88 | * - 128 byte strings 89 | - 2.6 90 | - 6400 91 | - x2400 92 | * - 1024 byte strings 93 | - 5.3 94 | - 6800 95 | - x1300 96 | * - 1Mb strings 97 | - 590 98 | - 7400 99 | - x12 100 | 101 | -------------------------------------------------------------------------------- /docs/source/examples/process.rst: -------------------------------------------------------------------------------- 1 | .. _examples-process: 2 | 3 | ``process`` Examples 4 | ============================== 5 | 6 | ``process`` is a very lightweight way of logging the total memory usage at regular time intervals. 7 | Here is an example: 8 | 9 | .. code-block:: python 10 | 11 | import logging 12 | import random 13 | import sys 14 | import time 15 | 16 | from pymemtrace import process 17 | 18 | logger = logging.getLogger(__file__) 19 | 20 | def main() -> int: 21 | logging.basicConfig( 22 | level=logging.INFO, 23 | format='%(asctime)s - %(filename)s#%(lineno)d - %(process)5d - (%(threadName)-10s) - %(levelname)-8s - %(message)s', 24 | ) 25 | logger.info('Demonstration of logging a process') 26 | # Log process data to the log file every 0.5 seconds. 27 | with process.log_process(interval=0.5, log_level=logger.getEffectiveLevel()): 28 | for i in range(8): 29 | size = random.randint(128, 128 + 256) * 1024 ** 2 30 | # Add a message to report in the next process write. 31 | process.add_message_to_queue(f'String of {size:,d} bytes') 32 | s = ' ' * size 33 | time.sleep(0.75 + random.random()) 34 | del s 35 | time.sleep(0.25 + random.random() / 2) 36 | return 0 37 | 38 | if __name__ == '__main__': 39 | sys.exit(main()) 40 | 41 | The output will be something like: 42 | 43 | .. code-block:: text 44 | 45 | $ python pymemtrace/examples/ex_process.py 46 | 2020-11-16 10:36:38,886 - ex_process.py#19 - 14052 - (MainThread) - INFO - Demonstration of logging a process 47 | 2020-11-16 10:36:38,887 - process.py#289 - 14052 - (ProcMon ) - INFO - ProcessLoggingThread-JSON-START {"timestamp": "2020-11-16 10:36:38.887407", "memory_info": {"rss": 11403264, "vms": 4376133632, "pfaults": 3417, "pageins": 0}, "cpu_times": {"user": 0.07780156, "system": 0.01763538, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 0.09744381904602051, "pid": 14052} 48 | 2020-11-16 10:36:39,392 - process.py#293 - 14052 - (ProcMon ) - INFO - ProcessLoggingThread-JSON {"timestamp": "2020-11-16 10:36:39.392076", "memory_info": {"rss": 209616896, "vms": 4574580736, "pfaults": 51809, "pageins": 0}, "cpu_times": {"user": 0.123138272, "system": 0.080602592, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 0.6022598743438721, "pid": 14052, "label": "String of 198,180,864 bytes"} 49 | 2020-11-16 10:36:39,892 - process.py#289 - 14052 - (ProcMon ) - INFO - ProcessLoggingThread-JSON {"timestamp": "2020-11-16 10:36:39.892747", "memory_info": {"rss": 209620992, "vms": 4574580736, "pfaults": 51810, "pageins": 0}, "cpu_times": {"user": 0.123503456, "system": 0.080648712, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 1.1028308868408203, "pid": 14052} 50 | 2020-11-16 10:36:40,397 - process.py#289 - 14052 - (ProcMon ) - INFO - ProcessLoggingThread-JSON {"timestamp": "2020-11-16 10:36:40.397231", "memory_info": {"rss": 11440128, "vms": 4376395776, "pfaults": 51811, "pageins": 0}, "cpu_times": {"user": 0.123984048, "system": 0.10224284, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 1.6074140071868896, "pid": 14052} 51 | 2020-11-16 10:36:40,901 - process.py#293 - 14052 - (ProcMon ) - INFO - ProcessLoggingThread-JSON {"timestamp": "2020-11-16 10:36:40.901329", "memory_info": {"rss": 320774144, "vms": 4685729792, "pfaults": 127332, "pageins": 0}, "cpu_times": {"user": 0.194056, "system": 0.191915568, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 2.1114120483398438, "pid": 14052, "label": "String of 309,329,920 bytes"} 52 | ... 53 | 54 | Note the additions of ``"label": "String of 198,180,864 bytes"`` in two places. 55 | 56 | The line: 57 | 58 | .. code-block:: python 59 | 60 | with process.log_process(interval=0.5, log_level=logger.getEffectiveLevel()): 61 | # As before. 62 | 63 | Instructs ``process`` to report every 0.5 seconds to the current log at the current log level. 64 | You can specify an actual log level so: 65 | 66 | .. code-block:: python 67 | 68 | with process.log_process(interval=0.5, logging.INFO): 69 | # As before. 70 | 71 | And that will suppress any ``process`` output if you have teh logging level set at, say, ERROR. 72 | 73 | 74 | Monitoring Another Process 75 | ----------------------------------- 76 | 77 | ``process`` can monitor another process from the command line: 78 | 79 | .. code-block:: bash 80 | 81 | $ python pymemtrace/process.py -p 71519 82 | 2020-11-10 20:46:41,687 - process.py#354 - 71869 - (MainThread) - INFO - Demonstration of logging a process 83 | Monitoring 71519 84 | 2020-11-10 20:46:41,689 - process.py#289 - 71869 - (ProcMon ) - INFO - ProcessLoggingThread-JSON-START {"timestamp": "2020-11-10 20:46:41.688480", "memory_info": {"rss": 12906496, "vms": 4359774208, "pfaults": 3310, "pageins": 960}, "cpu_times": {"user": 0.248923952, "system": 0.078601624, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 1396.3783469200134, "pid": 71519} 85 | 2020-11-10 20:46:42,693 - process.py#289 - 71869 - (ProcMon ) - INFO - ProcessLoggingThread-JSON {"timestamp": "2020-11-10 20:46:42.693520", "memory_info": {"rss": 12906496, "vms": 4359774208, "pfaults": 3310, "pageins": 960}, "cpu_times": {"user": 0.248923952, "system": 0.078601624, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 1397.3834369182587, "pid": 71519} 86 | 2020-11-10 20:46:43,697 - process.py#289 - 71869 - (ProcMon ) - INFO - ProcessLoggingThread-JSON {"timestamp": "2020-11-10 20:46:43.697247", "memory_info": {"rss": 12906496, "vms": 4359774208, "pfaults": 3310, "pageins": 960}, "cpu_times": {"user": 0.248923952, "system": 0.078601624, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 1398.3871541023254, "pid": 71519} 87 | 2020-11-10 20:46:44,701 - process.py#289 - 71869 - (ProcMon ) - INFO - ProcessLoggingThread-JSON {"timestamp": "2020-11-10 20:46:44.701290", "memory_info": {"rss": 12906496, "vms": 4359774208, "pfaults": 3310, "pageins": 960}, "cpu_times": {"user": 0.248923952, "system": 0.078601624, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 1399.391231060028, "pid": 71519} 88 | 2020-11-10 20:46:45,705 - process.py#289 - 71869 - (ProcMon ) - INFO - ProcessLoggingThread-JSON {"timestamp": "2020-11-10 20:46:45.705679", "memory_info": {"rss": 12906496, "vms": 4359774208, "pfaults": 3310, "pageins": 960}, "cpu_times": {"user": 0.248923952, "system": 0.078601624, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 1400.3956229686737, "pid": 71519} 89 | 2020-11-10 20:46:46,708 - process.py#289 - 71869 - (ProcMon ) - INFO - ProcessLoggingThread-JSON {"timestamp": "2020-11-10 20:46:46.708657", "memory_info": {"rss": 12906496, "vms": 4359774208, "pfaults": 3310, "pageins": 960}, "cpu_times": {"user": 0.248923952, "system": 0.078601624, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 1401.398586988449, "pid": 71519} 90 | ^CKeyboardInterrupt! 91 | 2020-11-10 20:46:47,626 - process.py#289 - 71869 - (MainThread) - INFO - ProcessLoggingThread-JSON-STOP {"timestamp": "2020-11-10 20:46:47.626020", "memory_info": {"rss": 12906496, "vms": 4359774208, "pfaults": 3310, "pageins": 960}, "cpu_times": {"user": 0.248923952, "system": 0.078601624, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 1402.3160009384155, "pid": 71519} 92 | Bye, bye! 93 | 94 | 95 | Using ``gnuplot`` on the Log File 96 | -------------------------------------- 97 | 98 | ``process`` can extract memory data from the log file and write the necessary files for plotting with ``gnuplot`` (which must be installed). 99 | 100 | 101 | .. code-block:: bash 102 | 103 | $ pwd 104 | ~/Documents/workspace/pymemtrace (master) 105 | $ mkdir tmp 106 | $ mkdir tmp/gnuplot 107 | $ python pymemtrace/examples/ex_process.py > tmp/process.log 2>&1 108 | $ python pymemtrace/process.py tmp/process.log tmp/gnuplot/ 109 | 2020-11-16 10:39:55,884 - gnuplot.py#114 - 14141 - (MainThread) - INFO - gnuplot stdout: None 110 | 2020-11-16 10:39:55,887 - gnuplot.py#67 - 14141 - (MainThread) - INFO - Writing gnuplot data "process.log_14129" in path tmp/gnuplot/ 111 | 2020-11-16 10:39:55,924 - gnuplot.py#85 - 14141 - (MainThread) - INFO - gnuplot stdout: None 112 | Bye, bye! 113 | $ ll tmp/gnuplot/ 114 | total 160 115 | -rw-r--r-- 1 user staff 4829 16 Nov 10:39 process.log_14129.dat 116 | -rw-r--r-- 1 user staff 2766 16 Nov 10:39 process.log_14129.plt 117 | -rw-r--r-- 1 user staff 32943 16 Nov 10:39 process.log_14129.svg 118 | -rw-r--r-- 1 user staff 32100 16 Nov 10:39 test.svg 119 | 120 | The file ``process.log_14129.svg`` will look like this: 121 | 122 | .. image:: images/process.log_14129.svg 123 | :alt: Example of process.py 124 | :width: 800 125 | :align: center 126 | 127 | 128 | 129 | -------------------------------------------------------------------------------- /docs/source/examples/trace_malloc.rst: -------------------------------------------------------------------------------- 1 | .. _examples-trace_malloc: 2 | 3 | ``trace_malloc`` Examples 4 | ============================== 5 | 6 | ``trace_malloc`` contains some utility wrappers around the :py:mod:`tracemalloc` module. 7 | It can compensate for the memory used by :py:mod:`tracemalloc` module. 8 | 9 | These Python examples are in :py:mod:`pymemtrace.examples.ex_trace_malloc` 10 | 11 | 12 | Using ``trace_malloc`` Directly 13 | ---------------------------------------- 14 | 15 | 16 | Adding 1Mb Strings 17 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 18 | 19 | Here is an example of adding 1Mb strings to a list under the watchful eye of :py:class:`trace_malloc.TraceMalloc`: 20 | 21 | .. code-block:: python 22 | 23 | from pymemtrace import trace_malloc 24 | 25 | list_of_strings = [] 26 | print(f'example_trace_malloc_for_documentation()') 27 | with trace_malloc.TraceMalloc('filename') as tm: 28 | for i in range(8): 29 | list_of_strings.append(' ' * 1024**2) 30 | print(f' tm.memory_start={tm.memory_start}') 31 | print(f'tm.memory_finish={tm.memory_finish}') 32 | print(f' tm.diff={tm.diff}') 33 | for stat in tm.statistics: 34 | print(stat) 35 | 36 | Typical output is: 37 | 38 | .. code-block:: text 39 | 40 | example_trace_malloc_for_documentation() 41 | tm.memory_start=13072 42 | tm.memory_finish=13800 43 | tm.diff=8388692 44 | pymemtrace/examples/ex_trace_malloc.py:0: size=8194 KiB (+8193 KiB), count=16 (+10), average=512 KiB 45 | /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/tracemalloc.py:0: size=6464 B (+504 B), count=39 (+10), average=166 B 46 | Documents/workspace/pymemtrace/pymemtrace/trace_malloc.py:0: size=3076 B (-468 B), count=10 (-1), average=308 B 47 | /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/logging/__init__.py:0: size=16.3 KiB (-128 B), count=49 (-2), average=340 B 48 | /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/abc.py:0: size=3169 B (+0 B), count=30 (+0), average=106 B 49 | /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/posixpath.py:0: size=480 B (+0 B), count=1 (+0), average=480 B 50 | /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/threading.py:0: size=168 B (+0 B), count=2 (+0), average=84 B 51 | /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/_weakrefset.py:0: size=72 B (+0 B), count=1 (+0), average=72 B 52 | 53 | 54 | To eliminate the lines that is caused by ``tracemalloc`` itself change the last two lines to: 55 | 56 | .. code-block:: python 57 | 58 | for stat in tm.net_statistics: 59 | print(stat) 60 | 61 | Which removes the line: 62 | 63 | .. code-block:: text 64 | 65 | /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/tracemalloc.py:0: size=6464 B (+504 B), count=39 (+10), average=166 B 66 | 67 | 68 | Using ``trace_malloc`` as a Decorator 69 | ---------------------------------------- 70 | 71 | ``trace_malloc`` provides a function decorator that can log the tracemalloc memory usage caused by execution a function. 72 | For example: 73 | 74 | .. code-block:: python 75 | 76 | from pymemtrace import trace_malloc 77 | 78 | @trace_malloc.trace_malloc_log(logging.INFO) 79 | def example_decorator_for_documentation(list_of_strings): 80 | for i in range(8): 81 | list_of_strings.append(create_string(1024**2)) 82 | 83 | list_of_strings = [] 84 | example_decorator_for_documentation(list_of_strings) 85 | 86 | Would log something like the following: 87 | 88 | .. code-block:: text 89 | 90 | 2020-11-15 18:37:39,194 - trace_malloc.py#87 - 10121 - (MainThread) - INFO - TraceMalloc memory delta: 8,389,548 for "example_decorator_for_documentation()" 91 | 92 | 93 | 94 | 95 | Cost of ``trace_malloc`` 96 | ----------------------------------- 97 | 98 | In :py:mod:`pymemtrace.examples.ex_trace_malloc` are some timing examples of creating a list of strings of varying size 99 | with and without :py:class:`trace_malloc.TraceMalloc`. 100 | Here are some typical results: 101 | 102 | .. Commented out typical output: 103 | 104 | $ /usr/bin/time -lp caffeinate python pymemtrace/examples/ex_trace_malloc.py 105 | number=10,000 repeat=5 convert=1,000,000 106 | example_timeit_under_512 : 8.139, 5.642, 4.479, 4.401, 5.994 mean= 5.731 min= 4.401 max= 8.139 span= 3.739 107 | example_timeit_under_512_with_trace_malloc('filename') : 4868.405, 4898.027, 4786.358, 4753.629, 4781.850 mean= 4817.654 min= 4753.629 max= 4898.027 span= 144.398 x 840.645 108 | example_timeit_under_512_with_trace_malloc('lineno') : 5050.222, 5043.958, 5034.344, 5031.117, 5021.919 mean= 5036.312 min= 5021.919 max= 5050.222 span= 28.303 x 878.799 109 | example_timeit_under_512_with_trace_malloc('traceback') : 5037.949, 5052.557, 5054.989, 5050.296, 5050.368 mean= 5049.232 min= 5037.949 max= 5054.989 span= 17.040 x 881.053 110 | example_timeit_over_512 : 18.541, 17.827, 17.576, 17.529, 17.595 mean= 17.814 min= 17.529 max= 18.541 span= 1.012 111 | example_timeit_over_512_with_trace_malloc('filename') : 5068.476, 5053.528, 5065.614, 5050.911, 5497.147 mean= 5147.135 min= 5050.911 max= 5497.147 span= 446.236 x 288.945 112 | example_timeit_over_512_with_trace_malloc('lineno') : 5470.068, 5237.150, 5166.904, 5162.868, 5170.988 mean= 5241.596 min= 5162.868 max= 5470.068 span= 307.201 x 294.248 113 | example_timeit_over_512_with_trace_malloc('traceback') : 5094.635, 5105.176, 5111.833, 5097.936, 5083.761 mean= 5098.668 min= 5083.761 max= 5111.833 span= 28.071 x 286.224 114 | example_timeit_well_over_512 : 1080.574, 1069.804, 1071.831, 1072.760, 1073.760 mean= 1073.746 min= 1069.804 max= 1080.574 span= 10.771 115 | example_timeit_well_over_512_with_trace_malloc('filename') : 6260.360, 6241.928, 6252.577, 6258.768, 6252.283 mean= 6253.183 min= 6241.928 max= 6260.360 span= 18.432 x 5.824 116 | example_timeit_well_over_512_with_trace_malloc('lineno') : 6370.560, 6388.218, 6390.206, 6383.660, 6387.620 mean= 6384.053 min= 6370.560 max= 6390.206 span= 19.646 x 5.946 117 | example_timeit_well_over_512_with_trace_malloc('traceback') : 6295.303, 6309.619, 6300.180, 6305.292, 6320.041 mean= 6306.087 min= 6295.303 max= 6320.041 span= 24.738 x 5.873 118 | real 2521.90 119 | user 2484.92 120 | sys 28.66 121 | 26484736 maximum resident set size 122 | 0 average shared memory size 123 | 0 average unshared data size 124 | 0 average unshared stack size 125 | 7366 page reclaims 126 | 670 page faults 127 | 0 swaps 128 | 0 block input operations 129 | 0 block output operations 130 | 0 messages sent 131 | 0 messages received 132 | 0 signals received 133 | 74 voluntary context switches 134 | 917533 involuntary context switches 135 | (pymemtrace_3.8_A) 136 | 137 | 138 | Using key_type 'filename' 139 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 140 | 141 | .. list-table:: **Times in µs tracing** ``filename`` 142 | :widths: 25 25 25 25 143 | :header-rows: 1 144 | 145 | * - Task 146 | - Without ``trace_malloc.TraceMalloc`` 147 | - With ``trace_malloc.TraceMalloc`` 148 | - Ratio 149 | * - 256 byte strings 150 | - 5.7 151 | - 4800 152 | - x840 153 | * - 1024 byte strings 154 | - 18 155 | - 5100 156 | - x290 157 | * - 1Mb strings 158 | - 1100 159 | - 6300 160 | - x5.8 161 | 162 | 163 | Using key_type 'lineno' 164 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 165 | 166 | .. list-table:: **Times in µs tracing** ``lineno`` 167 | :widths: 25 25 25 25 168 | :header-rows: 1 169 | 170 | * - Task 171 | - Without ``trace_malloc.TraceMalloc`` 172 | - With ``trace_malloc.TraceMalloc`` 173 | - Ratio 174 | * - 256 byte strings 175 | - 5.7 176 | - 5000 177 | - x880 178 | * - 1024 byte strings 179 | - 18 180 | - 5200 181 | - x290 182 | * - 1Mb strings 183 | - 1100 184 | - 6400 185 | - x5.9 186 | 187 | 188 | Using key_type 'traceback' 189 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 190 | 191 | .. list-table:: **Times in µs tracing** ``traceback`` 192 | :widths: 25 25 25 25 193 | :header-rows: 1 194 | 195 | * - Task 196 | - Without ``trace_malloc.TraceMalloc`` 197 | - With ``trace_malloc.TraceMalloc`` 198 | - Ratio 199 | * - 256 byte strings 200 | - 5.7 201 | - 5000 202 | - x880 203 | * - 1024 byte strings 204 | - 18 205 | - 5100 206 | - x290 207 | * - 1Mb strings 208 | - 1100 209 | - 6300 210 | - x5.9 211 | 212 | -------------------------------------------------------------------------------- /docs/source/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../HISTORY.rst 2 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | ##################################### 2 | pymemtrace's documentation 3 | ##################################### 4 | 5 | Contents: 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | readme 11 | installation 12 | examples 13 | technical_notes 14 | 15 | memory_leaks 16 | reference 17 | 18 | contributing 19 | authors 20 | history 21 | 22 | 23 | Indices and tables 24 | ================== 25 | 26 | * :ref:`genindex` 27 | * :ref:`modindex` 28 | * :ref:`search` 29 | -------------------------------------------------------------------------------- /docs/source/installation.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | Installation 4 | ============ 5 | 6 | 7 | Stable release 8 | -------------- 9 | 10 | To install pymemtrace, run this command in your terminal: 11 | 12 | .. code-block:: console 13 | 14 | $ pip install pymemtrace 15 | 16 | This is the preferred method to install pymemtrace, as it will always install the most recent stable release. 17 | 18 | If you don't have `pip`_ installed, this `Python installation guide`_ can guide 19 | you through the process. 20 | 21 | .. _pip: https://pip.pypa.io 22 | .. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ 23 | 24 | 25 | From sources 26 | ------------ 27 | 28 | The sources for pymemtrace can be downloaded from the `Github repo`_. 29 | 30 | You can either clone the public repository: 31 | 32 | .. code-block:: console 33 | 34 | $ git clone git://github.com/paulross/pymemtrace 35 | 36 | Or download the `tarball`_: 37 | 38 | .. code-block:: console 39 | 40 | $ curl -OL https://github.com/paulross/pymemtrace/tarball/master 41 | 42 | Once you have a copy of the source, you can install it with: 43 | 44 | .. code-block:: console 45 | 46 | $ python setup.py install 47 | 48 | Or for development: 49 | 50 | .. code-block:: console 51 | 52 | $ python setup.py develop 53 | 54 | 55 | .. _Github repo: https://github.com/paulross/pymemtrace 56 | .. _tarball: https://github.com/paulross/pymemtrace/tarball/master 57 | -------------------------------------------------------------------------------- /docs/source/memory_leaks.rst: -------------------------------------------------------------------------------- 1 | ************************** 2 | Tracking Down Memory Leaks 3 | ************************** 4 | 5 | Contents: 6 | 7 | .. toctree:: 8 | :maxdepth: 3 9 | 10 | memory_leaks/introduction 11 | memory_leaks/tools 12 | memory_leaks/techniques 13 | -------------------------------------------------------------------------------- /docs/source/memory_leaks/techniques.rst: -------------------------------------------------------------------------------- 1 | Techniques 2 | ==================================== 3 | 4 | This describes some of the techniques I have found useful. 5 | Bear in mind: 6 | 7 | * Tracking down memory leaks can take a long, long time. 8 | * Every memory leak is its own special little snowflake! 9 | So what works will be situation specific. 10 | 11 | High Level 12 | ------------------ 13 | 14 | It is worth spending a fair bit of time at high level before diving into the code since: 15 | 16 | * Working at high level is relatively cheap. 17 | * It is usually non-invasive. 18 | * It will quickly find out the *scale* of the problem. 19 | * It will quickly find out the *repeatability* of the problem. 20 | * You should be able to create the test that shows that the leak is **fixed**. 21 | 22 | At the end of this you should be able to state: 23 | 24 | * The *frequency* of the memory leak. 25 | * The *severity* of the memory leak. 26 | 27 | Relevant quote: **"Time spent on reconnaissance is seldom wasted."** 28 | 29 | 30 | Using Platform Tools 31 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 32 | 33 | The high level investigation will usually concentrate on using platform tools such as builtin memory management tools or 34 | Python tools such as ``pymentrace``'s :ref:`examples-process` or ``psutil`` will prove useful. 35 | 36 | 37 | Specific Tricks 38 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 39 | 40 | TODO: Finish this. 41 | 42 | Turn the GC Off 43 | """"""""""""""""""""" 44 | 45 | Turning the garbage collector off with ``gc.disable()`` is worth trying to see what effect, if any, it has. 46 | 47 | Medium Level 48 | ------------------ 49 | 50 | TODO: Finish this. 51 | 52 | Information From the ``sys`` Module 53 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 54 | 55 | ``pymentrace``'s :ref:`examples-debug_malloc_stats` is a very useful wrapper around 56 | :py:func:`sys._debugmallocstats` which can report changes to Python's small object allocator. 57 | 58 | 59 | ``tracemalloc`` 60 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 61 | 62 | ``pymentrace``'s :ref:`examples-trace_malloc` is a very useful wrapper around 63 | :py:mod:`tracemalloc` which can report changes to Python's memory allocator. 64 | 65 | ``objgraph`` 66 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 67 | 68 | TODO: Finish this. 69 | 70 | 71 | Specific Tricks 72 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 73 | 74 | TODO: Finish this. 75 | 76 | Finding Which Python Objects are Holding References to an Object 77 | """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 78 | 79 | TODO: Finish this. 80 | 81 | C/C++ Increasing Reference Count Excessively 82 | """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 83 | 84 | TODO: Finish this. 85 | 86 | Low Level 87 | ------------------ 88 | 89 | TODO: Finish this -------------------------------------------------------------------------------- /docs/source/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../README.rst 2 | -------------------------------------------------------------------------------- /docs/source/ref/c_mem_leak.rst: -------------------------------------------------------------------------------- 1 | ``pymemtrace.cMemLeak`` 2 | ================================= 3 | 4 | Module ``pymemtrace.cMemLeak`` 5 | ---------------------------------------- 6 | 7 | .. automodule:: pymemtrace.cMemLeak 8 | :members: 9 | :special-members: 10 | :private-members: 11 | 12 | 13 | Class ``pymemtrace.cMemLeak.CMalloc`` 14 | ---------------------------------------- 15 | 16 | .. autoclass:: pymemtrace.cMemLeak.CMalloc 17 | :members: 18 | :special-members: 19 | :private-members: 20 | 21 | 22 | Class ``pymemtrace.cMemLeak.PyRawMalloc`` 23 | -------------------------------------------- 24 | 25 | .. autoclass:: pymemtrace.cMemLeak.PyRawMalloc 26 | :members: 27 | :special-members: 28 | :private-members: 29 | 30 | 31 | Class ``pymemtrace.cMemLeak.PyMalloc`` 32 | ---------------------------------------- 33 | 34 | .. autoclass:: pymemtrace.cMemLeak.PyMalloc 35 | :members: 36 | :special-members: 37 | :private-members: 38 | -------------------------------------------------------------------------------- /docs/source/ref/c_py_mem_trace.rst: -------------------------------------------------------------------------------- 1 | ``pymemtrace.cPyMemTrace`` 2 | ================================= 3 | 4 | Module ``pymemtrace.cPyMemTrace`` 5 | ---------------------------------------- 6 | 7 | .. automodule:: pymemtrace.cPyMemTrace 8 | :members: 9 | :special-members: 10 | :private-members: 11 | 12 | 13 | Class ``pymemtrace.cPyMemTrace.Profile`` 14 | ---------------------------------------- 15 | 16 | 17 | .. autoclass:: pymemtrace.cPyMemTrace.Profile 18 | :members: 19 | :special-members: 20 | :private-members: 21 | 22 | 23 | Class ``pymemtrace.cPyMemTrace.Trace`` 24 | ---------------------------------------- 25 | 26 | .. autoclass:: pymemtrace.cPyMemTrace.Trace 27 | :members: 28 | :special-members: 29 | :private-members: 30 | -------------------------------------------------------------------------------- /docs/source/ref/debug_malloc_stats.rst: -------------------------------------------------------------------------------- 1 | ``pymemtrace.debug_malloc_stats`` 2 | =================================================== 3 | 4 | .. automodule:: pymemtrace.debug_malloc_stats 5 | :members: 6 | :special-members: 7 | :private-members: 8 | -------------------------------------------------------------------------------- /docs/source/ref/process.rst: -------------------------------------------------------------------------------- 1 | ``pymemtrace.process`` 2 | ================================= 3 | 4 | .. automodule:: pymemtrace.process 5 | :members: 6 | :special-members: 7 | :private-members: 8 | -------------------------------------------------------------------------------- /docs/source/ref/redirect_stdout.rst: -------------------------------------------------------------------------------- 1 | ``pymemtrace.redirect_stdout`` 2 | =================================================== 3 | 4 | .. automodule:: pymemtrace.redirect_stdout 5 | :members: 6 | :special-members: 7 | :private-members: 8 | -------------------------------------------------------------------------------- /docs/source/ref/trace_malloc.rst: -------------------------------------------------------------------------------- 1 | ``pymemtrace.trace_malloc`` 2 | =================================================== 3 | 4 | Module ``pymemtrace.trace_malloc`` 5 | ---------------------------------------- 6 | 7 | .. automodule:: pymemtrace.trace_malloc 8 | :members: 9 | :special-members: 10 | :private-members: 11 | -------------------------------------------------------------------------------- /docs/source/reference.rst: -------------------------------------------------------------------------------- 1 | ************************ 2 | pymemtrace Reference 3 | ************************ 4 | 5 | .. toctree:: 6 | :maxdepth: 3 7 | 8 | ref/process 9 | ref/c_py_mem_trace 10 | ref/debug_malloc_stats 11 | ref/trace_malloc 12 | ref/c_mem_leak 13 | ref/redirect_stdout 14 | -------------------------------------------------------------------------------- /docs/source/tech_notes/dtrace.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _tech_notes-dtrace: 3 | 4 | Technical Note on DTrace 5 | ========================== 6 | 7 | DTrace was also used on the same code and data that was used to test ``cPyMemTrace``. 8 | See :ref:`tech_notes-cpymemtrace_test_data` [#]_. 9 | This was traced with DTrace using ``toolkit/py_flow_malloc_free.d``: 10 | 11 | .. code-block:: bash 12 | 13 | sudo dtrace -s toolkit/py_flow_malloc_free.d -p -C 14 | 15 | 16 | Python Builds 17 | ---------------------------- 18 | 19 | Python 3.9 was configured and built with DTrace support. 20 | 21 | Debug Build 22 | ^^^^^^^^^^^^^^^^^^^^ 23 | 24 | .. code-block:: bash 25 | 26 | configure --with-pydebug --without-pymalloc --with-valgrind --with-dtrace 27 | 28 | .. note:: 29 | 30 | Since this bypasses Python's small object allocator (``pymalloc``) then every ``malloc()`` and ``free()`` can be 31 | seen by DTrace. 32 | This makes the DTrace logs very large. 33 | 34 | Release Build 35 | ^^^^^^^^^^^^^^^^^^^^ 36 | 37 | .. code-block:: bash 38 | 39 | configure --with-dtrace 40 | 41 | 42 | Baseline: Python 3.9 43 | --------------------------- 44 | 45 | 46 | This is using a standard build of Python 3.9 **without** DTrace support. It establishes a benchmark baseline: 47 | 48 | .. image:: images/LASToHTML.log_77077.svg 49 | :alt: Basic Python 3.9 (release) performance. 50 | :width: 800 51 | :align: center 52 | 53 | Using ``time`` gives: 54 | 55 | .. code-block:: text 56 | 57 | real 35.49 58 | user 29.72 59 | sys 2.03 60 | 61 | Python 3.9 Release with DTrace support, no Tracing 62 | --------------------------------------------------------- 63 | 64 | Python 3.9 (release) with DTrace support but *not* tracing with DTrace: 65 | 66 | 67 | .. image:: images/LASToHTML.log_76753.svg 68 | :alt: Python 3.9 (release) with DTrace capability. 69 | :width: 800 70 | :align: center 71 | 72 | 73 | Using ``time`` gives: 74 | 75 | .. code-block:: text 76 | 77 | real 49.54 78 | user 35.56 79 | sys 2.45 80 | 81 | So a DTrace capable build has roughly a 40% premium in ``real`` time even when not tracing. 82 | 83 | Python 3.9 Release with DTrace support, DTrace Tracing 84 | --------------------------------------------------------- 85 | 86 | Python 3.9 (release) with DTrace support and DTrace running: 87 | 88 | .. image:: images/LASToHTML.log_77633.svg 89 | :alt: Python 3.9 (release) with DTrace capability, DTrace runnning. 90 | :width: 800 91 | :align: center 92 | 93 | Using ``time`` gives: 94 | 95 | .. code-block:: text 96 | 97 | real 3220.38 98 | user 902.51 99 | sys 1949.83 100 | 101 | Note the increase in ``sys`` time caused by DTrace. 102 | This is a x65 increase in runtime over a release build (not tracing) and a x91 increase over the non-DTrace baseline. 103 | 104 | DTrace Log File 105 | ^^^^^^^^^^^^^^^^^^^^^^^ 106 | 107 | The log file [#]_ has 243,285 lines of which: 108 | 109 | * 94,882 calls to ``malloc()`` 110 | * 144,684 calls to ``free()``. 74,254 of these are to ``free(0x0)``. 111 | 112 | 113 | Python 3.9 Debug with DTrace support, no Tracing 114 | --------------------------------------------------------- 115 | 116 | This is running a debug, DTrace capable build: 117 | 118 | .. image:: images/LASToHTML.log_3938.svg 119 | :alt: Python 3.9 (debug) with DTrace capability, DTrace not tracing. 120 | :width: 800 121 | :align: center 122 | 123 | Using ``time`` gives: 124 | 125 | .. code-block:: text 126 | 127 | real 148.55 128 | user 139.99 129 | sys 1.93 130 | 131 | This is a x3 increase of runtime over a release DTrace capable build. This is typical for CPython debug builds. 132 | 133 | .. Commented out: 134 | 135 | (TotalDepth3.9_develop) 136 | $ tdprocess tmp/LAS/cPyMemTrace/LASToHtml_trace_DTraceD_B/LASToHTML.log tmp/LAS/cPyMemTrace/LASToHtml_trace_DTraceD_B/gnuplot/ 137 | 2020-11-12 11:32:27,943 - process.py - 5108 - (MainThread) - INFO - Extracting data from a log at tmp/LAS/cPyMemTrace/LASToHtml_trace_DTraceD_B/LASToHTML.log to tmp/LAS/cPyMemTrace/LASToHtml_trace_DTraceD_B/gnuplot/ 138 | 2020-11-12 11:32:27,981 - gnuplot.py - 5108 - (MainThread) - INFO - gnuplot stdout: None 139 | 2020-11-12 11:32:28,000 - gnuplot.py - 5108 - (MainThread) - INFO - Writing gnuplot data "LASToHTML.log_3938" in path tmp/LAS/cPyMemTrace/LASToHtml_trace_DTraceD_B/gnuplot/ 140 | 2020-11-12 11:32:28,084 - gnuplot.py - 5108 - (MainThread) - INFO - gnuplot stdout: None 141 | 142 | 143 | Python 3.9 Debug with DTrace support, DTrace Tracing 144 | --------------------------------------------------------- 145 | 146 | This is running a debug DTrace capable build *and* tracing with DTrace: 147 | 148 | .. image:: images/LASToHTML.log_4147.svg 149 | :alt: Python 3.9 (debug) with DTrace capability, DTrace tracing. 150 | :width: 800 151 | :align: center 152 | 153 | Using ``time`` gives: 154 | 155 | .. code-block:: text 156 | 157 | real 3520.61 158 | user 1183.36 159 | sys 2127.22 160 | 161 | 162 | This is a x24 increase in runtime over a debug build not tracing or a x99 increase in a non-DTrace build. 163 | 164 | DTrace Log File 165 | ^^^^^^^^^^^^^^^^^^^^^^^ 166 | 167 | This has 16m lines of which there are: 168 | 169 | * 8m calls to ``malloc()`` 170 | * 8m calls to ``free()``. 39,000 of these are to ``free(0x0)``. 171 | 172 | 173 | .. Commented out: 174 | 175 | (TotalDepth3.9_develop) 176 | $ tdprocess tmp/LAS/cPyMemTrace/LASToHtml_trace_DTraceD_C/LASToHTML.log tmp/LAS/cPyMemTrace/LASToHtml_trace_DTraceD_C/gnuplot/ 177 | 2020-11-12 11:32:42,854 - process.py - 5119 - (MainThread) - INFO - Extracting data from a log at tmp/LAS/cPyMemTrace/LASToHtml_trace_DTraceD_C/LASToHTML.log to tmp/LAS/cPyMemTrace/LASToHtml_trace_DTraceD_C/gnuplot/ 178 | 2020-11-12 11:32:42,892 - gnuplot.py - 5119 - (MainThread) - INFO - gnuplot stdout: None 179 | 2020-11-12 11:32:43,074 - gnuplot.py - 5119 - (MainThread) - INFO - Writing gnuplot data "LASToHTML.log_4147" in path tmp/LAS/cPyMemTrace/LASToHtml_trace_DTraceD_C/gnuplot/ 180 | 2020-11-12 11:32:43,202 - gnuplot.py - 5119 - (MainThread) - INFO - gnuplot stdout: None 181 | 182 | Summary 183 | ---------------------- 184 | 185 | Here is a summary of the performance cost of using different builds and tracing with DTrace: 186 | 187 | +-------------------------------------------------------------------+-----------+-----------+-----------+-------------------+ 188 | | Task | ``real`` | ``user`` | ``sys`` | ``real`` ratio | 189 | +===================================================================+===========+===========+===========+===================+ 190 | | Baseline | 35.5 | 29.7 | 2.03 | 1.0 | 191 | +-------------------------------------------------------------------+-----------+-----------+-----------+-------------------+ 192 | | DTrace, no tracing | 49.5 | 35.6 | 2.45 | x1.4 | 193 | | Python release build using ``pymalloc``. | | | | | 194 | +-------------------------------------------------------------------+-----------+-----------+-----------+-------------------+ 195 | | DTrace, trace ``malloc()``, ``free()``. | 3220 | 903 | 1950 | x91 | 196 | | Python release build using ``pymalloc``. | | | | | 197 | +-------------------------------------------------------------------+-----------+-----------+-----------+-------------------+ 198 | | DTrace, no tracing. Debug, not using ``pymalloc`` | 148 | 134 | 1.93 | x4.2 | 199 | +-------------------------------------------------------------------+-----------+-----------+-----------+-------------------+ 200 | | DTrace, trace ``malloc()``, ``free()``. | 3520 | 1180 | 2130 | x99 | 201 | | Python debug build, not using ``pymalloc``. | | | | | 202 | +-------------------------------------------------------------------+-----------+-----------+-----------+-------------------+ 203 | 204 | DTrace Log File 205 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 206 | 207 | Piping the DTrace output to a log file gives the following log files for this job. 208 | 209 | +-----------------------------------+---------------+---------------+-----------------------+ 210 | | Build | Release | Debug | Ratio Debug/Release | 211 | +===================================+===============+===============+=======================+ 212 | | Size | 16 Mb | 11,000 Mb | x68 | 213 | +-----------------------------------+---------------+---------------+-----------------------+ 214 | | Lines | 243k | 16m | x68 | 215 | +-----------------------------------+---------------+---------------+-----------------------+ 216 | | ``malloc()`` entries | 94,880 | 8,096,729 | x85 | 217 | +-----------------------------------+---------------+---------------+-----------------------+ 218 | | ``free()`` entries | 144,684 | 8,054,421 | x56 | 219 | +-----------------------------------+---------------+---------------+-----------------------+ 220 | | ``free(0x0)`` entries | 74,254 | 38,849 | x0.52 | 221 | +-----------------------------------+---------------+---------------+-----------------------+ 222 | 223 | .. rubric:: Footnotes 224 | .. [#] This uses the LASToHTML from the TotalDepth project. 225 | .. [#] Removing garbage from the DTrace log can be done with ``grep -o "[[:print:][:space:]]*" `` 226 | -------------------------------------------------------------------------------- /docs/source/tech_notes/rss_cost.rst: -------------------------------------------------------------------------------- 1 | .. _tech_notes-rss_cost: 2 | 3 | Cost of Calculating RSS 4 | ============================= 5 | 6 | Obtaining the Resident Set Size is not something that is done very frequently. 7 | Typically, monitoring software runs at a frequency of one second or so more so the cost of obtaining the RSS value is 8 | not significant. 9 | However with memory profiling the RSS is required per function call or per line and the cost of calculating RSS becomes 10 | a bottleneck. 11 | For example see the :ref:`tech_notes-cpymemtrace`. 12 | 13 | Here is a comparative look at what that cost is. 14 | The platform is a Mac mini (late 2014) 2.8 GHz Intel Core i5 running macOS Mojave 10.14.6. 15 | 16 | Using ``psutil`` 17 | ----------------------- 18 | 19 | Here is the cost of calculating the RSS with ``psutil``: 20 | 21 | .. code-block:: python 22 | 23 | >>> import timeit 24 | >>> timeit.repeat('p.memory_info().rss', setup='import psutil; p = psutil.Process()', number=1_000_000, repeat=5) 25 | [9.89, 14.32, 12.00, 14.67, 13.77] 26 | 27 | So that takes typically 13 µs (range 9.8 to 14.3). 28 | 29 | Using ``cPyMemTrace`` 30 | ----------------------- 31 | 32 | ``cPyMemTrace`` uses code in ``pymemtrace/src/c/get_rss.c``. 33 | This is accessed from Python in ``cPyMemTrace.c``. 34 | Here is the cost: 35 | 36 | .. code-block:: python 37 | 38 | >>> import timeit 39 | >>> timeit.repeat('cPyMemTrace.rss()', setup='import cPyMemTrace', number=1_000_000, repeat=5) 40 | [1.656, 1.649, 1.636, 1.626, 1.646] 41 | 42 | So 1.64 µs ± 0.015 µs which agrees very closely with our estimate of 1.5 µs from :ref:`tech_notes-cpymemtrace`. 43 | 44 | Peak RSS (not available in ``psutil``) is much faster for some reason: 45 | 46 | .. code-block:: python 47 | 48 | >>> timeit.repeat('cPyMemTrace.rss_peak()', setup='import cPyMemTrace', number=1_000_000, repeat=5) 49 | [0.650, 0.628, 0.638, 0.629, 0.633] 50 | 51 | So 0.636 µs ± 0.011 µs. 52 | 53 | It looks like this is the best we can do and x8 faster than psutil. 54 | 55 | -------------------------------------------------------------------------------- /docs/source/technical_notes.rst: -------------------------------------------------------------------------------- 1 | ******************* 2 | Technical Notes 3 | ******************* 4 | 5 | 6 | 7 | Contents: 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | tech_notes/cPyMemTrace 13 | tech_notes/dtrace 14 | tech_notes/rss_cost 15 | -------------------------------------------------------------------------------- /pymemtrace/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Top-level package for pymemtrace.""" 4 | 5 | __author__ = """Paul Ross""" 6 | __email__ = 'apaulross@gmail.com' 7 | __version__ = '0.2.0rc0' 8 | -------------------------------------------------------------------------------- /pymemtrace/examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paulross/pymemtrace/7dd4f07c3ff36866e8b3779d9787903e07895152/pymemtrace/examples/__init__.py -------------------------------------------------------------------------------- /pymemtrace/examples/ex_cPyMemTrace.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import psutil 4 | 5 | from pymemtrace import custom 6 | from pymemtrace import cPyMemTrace 7 | 8 | 9 | def create_string(l: int) -> str: 10 | return ' ' * l 11 | 12 | 13 | COUNT = 16 14 | 15 | 16 | def test_under_512(): 17 | print(f'test_under_512 {COUNT}') 18 | l = [] 19 | for i in range(COUNT): 20 | l.append(create_string(256)) 21 | while len(l): 22 | l.pop() 23 | 24 | 25 | def test_over_512(): 26 | print(f'test_over_512 {COUNT}') 27 | l = [] 28 | for i in range(COUNT): 29 | l.append(create_string(1024)) 30 | while len(l): 31 | l.pop() 32 | 33 | 34 | def test_well_over_512(): 35 | print(f'test_well_over_512 {COUNT}') 36 | l = [] 37 | for i in range(COUNT): 38 | l.append(create_string(1024**2)) 39 | while len(l): 40 | l.pop() 41 | 42 | 43 | def f(l): 44 | print('Hi') 45 | s = ' ' * l 46 | print('Bye') 47 | 48 | 49 | def g(): 50 | print(f'Creating Custom.') 51 | # pid = psutil.Process() 52 | # print(f'Creating Custom: {pid.memory_info()}') 53 | obj = custom.Custom('First', 'Last') 54 | print(obj.name()) 55 | # print(f'Done: {pid.memory_info()}') 56 | print(f'Done.') 57 | 58 | 59 | def example_for_documentation(): 60 | print(f'example_for_documentation()') 61 | with cPyMemTrace.Profile(0, message="MESSAGE"): 62 | print(f'Logging to {cPyMemTrace.get_log_file_path_profile()}') 63 | l = [] 64 | for i in range(8): 65 | l.append(create_string(1024**2)) 66 | while len(l): 67 | l.pop() 68 | 69 | 70 | def main(): 71 | # cPyMemTrace._attach_profile() 72 | # f(1024**2) 73 | # # f(1024**2) 74 | # g() 75 | # cPyMemTrace._detach_profile() 76 | 77 | # with cPyMemTrace.Profile(): 78 | # # f(1024**2) 79 | # # f(1024**2) 80 | # # g() 81 | # test_under_512() 82 | # test_over_512() 83 | # test_well_over_512() 84 | 85 | example_for_documentation() 86 | 87 | return 0 88 | 89 | 90 | if __name__ == '__main__': 91 | sys.exit(main()) 92 | 93 | -------------------------------------------------------------------------------- /pymemtrace/examples/ex_debug_malloc_stats.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import timeit 3 | 4 | import typing 5 | from pymemtrace import debug_malloc_stats 6 | 7 | def create_string(l: int) -> str: 8 | return ' ' * l 9 | 10 | 11 | def test_under_512(count: int, list_of_strings: typing.List[str]): 12 | for i in range(count): 13 | list_of_strings.append(create_string(256)) 14 | 15 | 16 | def test_over_512(count: int, list_of_strings: typing.List[str]): 17 | for i in range(count): 18 | list_of_strings.append(create_string(1024)) 19 | 20 | 21 | def test_well_over_512(count: int, list_of_strings: typing.List[str]): 22 | for i in range(count): 23 | list_of_strings.append(create_string(1024**2)) 24 | 25 | 26 | def example_debug_malloc_stats_for_documentation(list_of_strings): 27 | """An example of using the trace_malloc.trace_malloc_report decorator for logging memory usage. 28 | Typical output: 29 | 30 | .. code-block:: text 31 | 32 | example_trace_malloc_for_documentation() 33 | DiffSysDebugMallocStats.diff(): 34 | class size num pools blocks in use avail blocks 35 | ----- ---- --------- ------------- ------------ 36 | 1 32 +1 +52 +74 37 | 2 48 +0 +17 -17 38 | 3 64 +0 +33 -33 39 | 4 80 +1 +51 -1 40 | 5 96 +2 +34 +50 41 | 6 112 +0 +2 -2 42 | 7 128 +0 +1 -1 43 | 10 176 +0 +1 -1 44 | 12 208 +0 +1 -1 45 | 17 288 +0 +1 -1 46 | 18 304 +0 +2 -2 47 | 25 416 +0 +3 -3 48 | 26 432 +0 +3 -3 49 | 27 448 +0 +3 -3 50 | 29 480 +0 +3 -3 51 | 30 496 +0 +1 -1 52 | 31 512 +0 +1 -1 53 | 54 | # bytes in allocated blocks = +19,904 55 | # bytes in available blocks = -3,808 56 | -4 unused pools * 4096 bytes = -16,384 57 | # bytes lost to pool headers = +192 58 | # bytes lost to quantization = +96 59 | 60 | -1 free 1-sized PyTupleObjects * 32 bytes each = -32 61 | +1 free 5-sized PyTupleObjects * 64 bytes each = +64 62 | +2 free PyDictObjects * 48 bytes each = +96 63 | -2 free PyListObjects * 40 bytes each = -80 64 | +1 free PyMethodObjects * 48 bytes each = +48 65 | 66 | """ 67 | print(f'example_trace_malloc_for_documentation()') 68 | with debug_malloc_stats.DiffSysDebugMallocStats() as malloc_diff: 69 | for i in range(1, 9): 70 | list_of_strings.append(' ' * (i * 8)) 71 | print(f'DiffSysDebugMallocStats.diff():') 72 | print(f'{malloc_diff.diff()}') 73 | 74 | 75 | COUNT = 8 76 | 77 | def example(): 78 | for function in (test_under_512, test_over_512, test_well_over_512): 79 | print(f'Function: {function}'.center(75, '=')) 80 | list_of_strings = [] 81 | with debug_malloc_stats.DiffSysDebugMallocStats() as malloc_diff: 82 | function(COUNT, list_of_strings) 83 | print(f'DiffSysDebugMallocStats.diff():') 84 | print(f'{malloc_diff.diff()}') 85 | print(f'DONE: Function: {function}'.center(75, '=')) 86 | print() 87 | 88 | 89 | def example_timeit(): 90 | for function in (test_under_512, test_over_512, test_well_over_512): 91 | list_of_strings = [] 92 | function(COUNT, list_of_strings) 93 | 94 | 95 | def example_timeit_with_debug_malloc_stats(): 96 | for function in (test_under_512, test_over_512, test_well_over_512): 97 | list_of_strings = [] 98 | with debug_malloc_stats.DiffSysDebugMallocStats() as malloc_diff: 99 | function(COUNT, list_of_strings) 100 | 101 | 102 | def example_timeit_under_512(): 103 | list_of_strings = [] 104 | test_under_512(COUNT, list_of_strings) 105 | 106 | 107 | def example_timeit_under_512_with_debug_malloc_stats(): 108 | list_of_strings = [] 109 | with debug_malloc_stats.DiffSysDebugMallocStats() as malloc_diff: 110 | test_under_512(COUNT, list_of_strings) 111 | 112 | 113 | def example_timeit_over_512(): 114 | list_of_strings = [] 115 | test_over_512(COUNT, list_of_strings) 116 | 117 | 118 | def example_timeit_over_512_with_debug_malloc_stats(): 119 | list_of_strings = [] 120 | with debug_malloc_stats.DiffSysDebugMallocStats() as malloc_diff: 121 | test_over_512(COUNT, list_of_strings) 122 | 123 | 124 | def example_timeit_well_over_512(): 125 | list_of_strings = [] 126 | test_well_over_512(COUNT, list_of_strings) 127 | 128 | 129 | def example_timeit_well_over_512_with_debug_malloc_stats(): 130 | list_of_strings = [] 131 | with debug_malloc_stats.DiffSysDebugMallocStats() as malloc_diff: 132 | test_well_over_512(COUNT, list_of_strings) 133 | 134 | 135 | def main(): 136 | # example_debug_malloc_stats_for_documentation([]) 137 | print() 138 | # example() 139 | # print(timeit.repeat('p.memory_info().rss', setup='import psutil; p = psutil.Process()', number=1_000_000, repeat=5)) 140 | 141 | NUMBER = 10_000 142 | REPEAT = 5 143 | CONVERT = 1_000_000 144 | print(f'number={NUMBER:,d} repeat={REPEAT:,d} convert={CONVERT:,d}') 145 | for function in ( 146 | 'example_timeit_under_512', 147 | 'example_timeit_over_512', 148 | 'example_timeit_well_over_512', 149 | ): 150 | # t = timeit.timeit(f"{function}()", setup=f"from __main__ import {function}", number=NUMBER) / NUMBER 151 | # print(f'{function:60}: {t:9.9f}') 152 | times = timeit.repeat(f"{function}()", setup=f"from __main__ import {function}", number=NUMBER, repeat=REPEAT) 153 | times = [CONVERT * t / NUMBER for t in times] 154 | result = [f'{v:9.3f}' for v in times] 155 | times_mean = sum(times) / REPEAT 156 | print( 157 | f'{function:60}:' 158 | f' {", ".join(result)}' 159 | f' mean={times_mean:9.3f}' 160 | f' min={min(times):9.3f}' 161 | f' max={max(times):9.3f}' 162 | f' span={max(times) - min(times):9.3f}' 163 | ) 164 | # With _with_debug_malloc_stats 165 | times_with_debug_malloc_stats = timeit.repeat(f"{function}_with_debug_malloc_stats()", setup=f"from __main__ import {function}_with_debug_malloc_stats", number=NUMBER, repeat=REPEAT) 166 | times_with_debug_malloc_stats = [CONVERT * t / NUMBER for t in times_with_debug_malloc_stats] 167 | result = [f'{v:9.3f}' for v in times_with_debug_malloc_stats] 168 | times_mean_with_debug_malloc_stats = sum(times_with_debug_malloc_stats) / REPEAT 169 | print( 170 | f'{function + "_with_debug_malloc_stats":60}:' 171 | f' {", ".join(result)}' 172 | f' mean={times_mean_with_debug_malloc_stats:9.3f}' 173 | f' min={min(times_with_debug_malloc_stats):9.3f}' 174 | f' max={max(times_with_debug_malloc_stats):9.3f}' 175 | f' span={max(times_with_debug_malloc_stats) - min(times_with_debug_malloc_stats):9.3f}' 176 | f' x{times_mean_with_debug_malloc_stats / times_mean:>8.3f}' 177 | ) 178 | return 0 179 | 180 | 181 | if __name__ == '__main__': 182 | sys.exit(main()) 183 | 184 | -------------------------------------------------------------------------------- /pymemtrace/examples/ex_dtrace.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example code to exercise DTrace. 3 | """ 4 | import logging 5 | import os 6 | import pprint 7 | import sys 8 | import sysconfig 9 | import time 10 | 11 | from pymemtrace import cMemLeak 12 | 13 | logger = logging.getLogger(__file__) 14 | 15 | 16 | def create_c_buffer_and_del(size: int): 17 | input(f'Waiting to create buffer PID={os.getpid()}, size={size} to continue... ') 18 | b = cMemLeak.CMalloc(size) 19 | input(f'Waiting to del buffer PID={os.getpid()}, size={size} to continue... ') 20 | del b 21 | input(f'DONE del buffer PID={os.getpid()}, size={size} to continue... ') 22 | 23 | 24 | def create_c_buffer(size: int): 25 | b = cMemLeak.CMalloc(size) 26 | # logger.info(f'create_c_buffer() {b.size} at 0x{b.buffer:x}') 27 | print(f'create_c_buffer() {b.size} at 0x{b.buffer:x}') 28 | return b 29 | 30 | 31 | def exercise_c_memory(): 32 | logger.info('exercise_c_memory()') 33 | str_list = [] 34 | for i in range(8): 35 | str_list.append(create_c_buffer(1652)) 36 | # time.sleep(0.5) 37 | input(f'Waiting to pop PID={os.getpid()}, to continue... ') 38 | str_list.pop() 39 | input(f'Pop\'d PID={os.getpid()}, to continue... ') 40 | while len(str_list): 41 | str_list.pop() 42 | logger.info('DONE: exercise_c_ memory()') 43 | 44 | 45 | def create_cmalloc_list(): 46 | l = [] 47 | for i in range(4): 48 | block = cMemLeak.CMalloc(1477) 49 | print(f'Created CMalloc size={block.size:d} buffer=0x{block.buffer:x}') 50 | l.append(block) 51 | while len(l): 52 | # Remove in reverse order 53 | block = l.pop(0) 54 | print(f'Pop\'d CMalloc size={block.size:d} buffer=0x{block.buffer:x}') 55 | l.clear() 56 | 57 | 58 | def create_pyrawmalloc_list(): 59 | l = [] 60 | for i in range(4): 61 | block = cMemLeak.PyRawMalloc(128) 62 | print(f'Created PyRawMalloc size={block.size:d} buffer=0x{block.buffer:x}') 63 | l.append(block) 64 | while len(l): 65 | # Remove in reverse order 66 | block = l.pop(0) 67 | print(f'Pop\'d PyRawMalloc size={block.size:d} buffer=0x{block.buffer:x}') 68 | l.clear() 69 | 70 | 71 | def create_pymalloc_list(): 72 | print(f'Python at {sys.executable} is configured with CONFIG_ARGS: {sysconfig.get_config_var("CONFIG_ARGS")}') 73 | l = [] 74 | for i in range(4): 75 | block = cMemLeak.PyMalloc(371) 76 | print(f'Created PyMalloc size={block.size:d} buffer=0x{block.buffer:x}') 77 | l.append(block) 78 | while len(l): 79 | # Remove in reverse order 80 | block = l.pop(0) 81 | print(f'Pop\'d PyMalloc size={block.size:d} buffer=0x{block.buffer:x}') 82 | l.clear() 83 | 84 | 85 | def create_py_array_list(size: int): 86 | l = [] 87 | for i in range(4): 88 | block = b' ' * size 89 | print(f'Created {type(block)} size={len(block):d} buffer=0x{id(block):x}') 90 | l.append(block) 91 | while len(l): 92 | # Remove in reverse order 93 | block = l.pop(0) 94 | print(f'Pop\'d {type(block)} size={len(block):d} buffer=0x{id(block):x}') 95 | l.clear() 96 | 97 | 98 | def main(): 99 | with_dtrace = sysconfig.get_config_var('WITH_DTRACE') 100 | if with_dtrace is None or with_dtrace != 1: 101 | raise RuntimeError(f'Python at {sys.executable} must be build with DTrace support.') 102 | logging.basicConfig( 103 | level=20, 104 | format='%(asctime)s - %(filename)-16s - %(lineno)4d - %(process)5d - (%(threadName)-10s) - %(levelname)-8s - %(message)s', 105 | stream=sys.stdout, 106 | ) 107 | logger.info('Python at %s is configured with CONFIG_ARGS: %s', sys.executable, sysconfig.get_config_var('CONFIG_ARGS')) 108 | # Wait after logging initialised 109 | input(f'Waiting to start tracing PID: {os.getpid()} ( to continue):') 110 | # exercise_c_memory() 111 | 112 | # create_cmalloc_list() 113 | # create_pyrawmalloc_list() 114 | # create_pymalloc_list() 115 | create_py_array_list(27) 116 | return 0 117 | 118 | 119 | if __name__ == '__main__': 120 | sys.exit(main()) 121 | 122 | 123 | -------------------------------------------------------------------------------- /pymemtrace/examples/ex_memory_exercise.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import os 4 | import sys 5 | import time 6 | 7 | from pymemtrace import cMemLeak 8 | 9 | 10 | logger = logging.getLogger(__file__) 11 | 12 | 13 | def create_c_buffer_and_del(size: int): 14 | input(f'Waiting to create buffer PID={os.getpid()}, size={size} to continue... ') 15 | b = cMemLeak.CMalloc(size) 16 | input(f'Waiting to del buffer PID={os.getpid()}, size={size} to continue... ') 17 | del b 18 | input(f'DONE del buffer PID={os.getpid()}, size={size} to continue... ') 19 | 20 | 21 | def create_c_buffer(size: int): 22 | b = cMemLeak.CMalloc(size) 23 | # logger.info(f'create_c_buffer() {b.size} at 0x{b.buffer:x}') 24 | print(f'create_c_buffer() {b.size} at 0x{b.buffer:x}') 25 | return b 26 | 27 | 28 | def exercise_c_memory(): 29 | logger.info('exercise_c_memory()') 30 | str_list = [] 31 | for i in range(8): 32 | str_list.append(create_c_buffer(1652)) 33 | # time.sleep(0.5) 34 | input(f'Waiting to pop PID={os.getpid()}, to continue... ') 35 | str_list.pop() 36 | input(f'Pop\'d PID={os.getpid()}, to continue... ') 37 | while len(str_list): 38 | str_list.pop() 39 | logger.info('DONE: exercise_c_ memory()') 40 | 41 | 42 | def create_string(size: int) -> str: 43 | return ' ' * size 44 | 45 | 46 | def exercise_memory(): 47 | logger.info('exercise_memory()') 48 | str_list = [] 49 | for i in range(8): 50 | str_list.append(create_string(1024**2)) 51 | time.sleep(0.5) 52 | while len(str_list): 53 | str_list.pop() 54 | logger.info('DONE: exercise_memory()') 55 | 56 | 57 | def main(): 58 | parser = argparse.ArgumentParser( 59 | description='Excercise Python and C memory', 60 | # formatter_class=argparse.RawDescriptionHelpFormatter, 61 | ) 62 | # parser.add_argument("-s", "--subdir", type=str, dest="subdir", 63 | # default=SUB_DIR_FOR_COMMON_FILES, 64 | # help="Sub-directory for writing the common files. [default: %(default)s]") 65 | parser.add_argument("-p", "--pause", action="store_true", dest="pause", 66 | default=False, 67 | help="Pause before starting. [default: %(default)s]") 68 | # parser.add_argument("-v", "--verbose", action="store_true", dest="verbose", 69 | # default=False, 70 | # help="Verbose, lists duplicate files and sizes. [default: %(default)s]") 71 | parser.add_argument( 72 | "-l", "--log_level", 73 | type=int, 74 | dest="log_level", 75 | default=20, 76 | help="Log Level (debug=10, info=20, warning=30, error=40, critical=50)" 77 | " [default: %(default)s]" 78 | ) 79 | # parser.add_argument( 80 | # dest="path", 81 | # nargs=1, 82 | # help="Path to source directory. WARNING: This will be rewritten in-place." 83 | # ) 84 | args = parser.parse_args() 85 | if args.pause: 86 | input(f'Waiting to continue PID={os.getpid()}, to continue... ') 87 | 88 | clock_start = time.perf_counter() 89 | # Initialise logging etc. 90 | logging.basicConfig(level=args.log_level, 91 | format='%(asctime)s - %(filename)-16s - %(lineno)4d - %(process)5d - (%(threadName)-10s) - %(levelname)-8s - %(message)s', 92 | # datefmt='%y-%m-%d % %H:%M:%S', 93 | stream=sys.stdout) 94 | try: 95 | while True: 96 | # exercise_memory() 97 | exercise_c_memory() 98 | # create_c_buffer_and_del(1653) 99 | except KeyboardInterrupt: 100 | print('Interrupted!') 101 | print(f'Runtime: {time.perf_counter() - clock_start:.3f} (s)') 102 | print('Bye, bye!') 103 | return 0 104 | 105 | 106 | if __name__ == '__main__': 107 | sys.exit(main()) -------------------------------------------------------------------------------- /pymemtrace/examples/ex_process.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example of using process that logs process data to the current log. 3 | """ 4 | import logging 5 | import random 6 | import sys 7 | import time 8 | 9 | from pymemtrace import process 10 | 11 | logger = logging.getLogger(__file__) 12 | 13 | 14 | def main() -> int: 15 | logging.basicConfig( 16 | level=logging.INFO, 17 | format='%(asctime)s - %(filename)s#%(lineno)d - %(process)5d - (%(threadName)-10s) - %(levelname)-8s - %(message)s', 18 | ) 19 | logger.info('Demonstration of logging a process') 20 | # Log process data to the log file every 0.5 seconds. 21 | with process.log_process(interval=0.5, log_level=logger.getEffectiveLevel()): 22 | for i in range(8): 23 | size = random.randint(128, 128 + 256) * 1024 ** 2 24 | # Add a message to report in the next process write. 25 | process.add_message_to_queue(f'String of {size:,d} bytes') 26 | s = ' ' * size 27 | time.sleep(0.75 + random.random()) 28 | del s 29 | time.sleep(0.25 + random.random() / 2) 30 | return 0 31 | 32 | 33 | if __name__ == '__main__': 34 | sys.exit(main()) 35 | -------------------------------------------------------------------------------- /pymemtrace/examples/ex_trace_malloc.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | import timeit 4 | 5 | from pymemtrace import trace_malloc 6 | 7 | def create_string(l: int) -> str: 8 | return ' ' * l 9 | 10 | 11 | COUNT = 16 12 | 13 | 14 | def test_under_512(list_of_strings): 15 | # print(f'test_under_512 count={COUNT}') 16 | for i in range(COUNT): 17 | list_of_strings.append(create_string(256)) 18 | # while len(list_of_strings): 19 | # list_of_strings.pop() 20 | 21 | 22 | def test_over_512(list_of_strings): 23 | # print(f'test_over_512 count={COUNT}') 24 | for i in range(COUNT): 25 | list_of_strings.append(create_string(1024)) 26 | # while len(list_of_strings): 27 | # list_of_strings.pop() 28 | 29 | 30 | def test_well_over_512(list_of_strings): 31 | # print(f'test_well_over_512 count={COUNT}') 32 | for i in range(COUNT): 33 | list_of_strings.append(create_string(1024**2)) 34 | # while len(list_of_strings): 35 | # list_of_strings.pop() 36 | 37 | 38 | def example_trace_malloc_for_documentation(list_of_strings): 39 | """An example of using the trace_malloc.trace_malloc_report decorator for logging memory usage. 40 | Typical output:: 41 | 42 | example_trace_malloc_for_documentation() 43 | pymemtrace/pymemtrace/examples/example_trace_malloc.py:0: size=8194 KiB (+8193 KiB), count=16 (+10), average=512 KiB 44 | /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/tracemalloc.py:0: size=6720 B (+552 B), count=43 (+11), average=156 B 45 | pymemtrace/pymemtrace/trace_malloc.py:0: size=3076 B (-468 B), count=10 (-1), average=308 B 46 | /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/logging/__init__.py:0: size=16.3 KiB (-176 B), count=49 (-3), average=340 B 47 | /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/abc.py:0: size=3169 B (+0 B), count=30 (+0), average=106 B 48 | /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/posixpath.py:0: size=480 B (+0 B), count=1 (+0), average=480 B 49 | /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/threading.py:0: size=168 B (+0 B), count=2 (+0), average=84 B 50 | /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/_weakrefset.py:0: size=72 B (+0 B), count=1 (+0), average=72 B 51 | """ 52 | print(f'example_trace_malloc_for_documentation()') 53 | with trace_malloc.TraceMalloc('filename') as tm: 54 | for i in range(8): 55 | list_of_strings.append(create_string(1024**2)) 56 | # list_of_strings.append(create_string(128)) 57 | print(f' tm.memory_start={tm.memory_start}') 58 | print(f'tm.memory_finish={tm.memory_finish}') 59 | print(f' tm.diff={tm.diff}') 60 | for stat in tm.statistics: 61 | print(stat) 62 | print() 63 | for stat in tm.net_statistics(): 64 | print(stat) 65 | 66 | 67 | @trace_malloc.trace_malloc_log(logging.INFO) 68 | def example_decorator_for_documentation(list_of_strings): 69 | """An example of using the trace_malloc.trace_malloc_report decorator for logging memory usage. 70 | Typical output:: 71 | 72 | 2020-11-15 18:13:06,000 - trace_malloc.py#82 - 9689 - (MainThread) - INFO - TraceMalloc memory delta: 8,389,548 73 | """ 74 | print(f'example_decorator_for_documentation()') 75 | for i in range(8): 76 | list_of_strings.append(create_string(1024**2)) 77 | 78 | 79 | def example(): 80 | for function in (test_under_512, test_over_512, test_well_over_512): 81 | print(f'Function: {function}') 82 | list_of_strings = [] 83 | # with trace_malloc.TraceMalloc('filename') as tm: 84 | with trace_malloc.TraceMalloc('lineno') as tm: 85 | function(list_of_strings) 86 | print(f'tm.memory_start={tm.memory_start}') 87 | print(f'tm.memory_finish={tm.memory_finish}') 88 | for stat in tm.net_statistics(): 89 | print(stat) 90 | print() 91 | 92 | 93 | def example_timeit_under_512(): 94 | list_of_strings = [] 95 | test_under_512(list_of_strings) 96 | 97 | 98 | def example_timeit_under_512_with_trace_malloc(key_type): 99 | list_of_strings = [] 100 | with trace_malloc.TraceMalloc(key_type) as tm: 101 | test_under_512(list_of_strings) 102 | 103 | 104 | def example_timeit_over_512(): 105 | list_of_strings = [] 106 | test_over_512(list_of_strings) 107 | 108 | 109 | def example_timeit_over_512_with_trace_malloc(key_type): 110 | list_of_strings = [] 111 | with trace_malloc.TraceMalloc(key_type) as tm: 112 | test_over_512(list_of_strings) 113 | 114 | 115 | def example_timeit_well_over_512(): 116 | list_of_strings = [] 117 | test_well_over_512(list_of_strings) 118 | 119 | 120 | def example_timeit_well_over_512_with_trace_malloc(key_type): 121 | list_of_strings = [] 122 | with trace_malloc.TraceMalloc(key_type) as tm: 123 | test_well_over_512(list_of_strings) 124 | 125 | 126 | def run_timeit(): 127 | NUMBER = 10_000 128 | REPEAT = 5 129 | CONVERT = 1_000_000 130 | print(f'number={NUMBER:,d} repeat={REPEAT:,d} convert={CONVERT:,d}') 131 | for function in ( 132 | 'example_timeit_under_512', 133 | 'example_timeit_over_512', 134 | 'example_timeit_well_over_512', 135 | ): 136 | # t = timeit.timeit(f"{function}()", setup=f"from __main__ import {function}", number=NUMBER) / NUMBER 137 | # print(f'{function:60}: {t:9.9f}') 138 | times = timeit.repeat(f"{function}()", setup=f"from __main__ import {function}", number=NUMBER, repeat=REPEAT) 139 | times = [CONVERT * t / NUMBER for t in times] 140 | result = [f'{v:9.3f}' for v in times] 141 | times_mean = sum(times) / REPEAT 142 | print( 143 | f'{function:60}:' 144 | f' {", ".join(result)}' 145 | f' mean={times_mean:9.3f}' 146 | f' min={min(times):9.3f}' 147 | f' max={max(times):9.3f}' 148 | f' span={max(times) - min(times):9.3f}' 149 | ) 150 | for key_type in ('filename', 'lineno', 'traceback'): 151 | # With _with_trace_malloc 152 | times_with_trace_malloc = timeit.repeat(f"{function}_with_trace_malloc('{key_type}')", setup=f"from __main__ import {function}_with_trace_malloc", number=NUMBER, repeat=REPEAT) 153 | times_with_trace_malloc = [CONVERT * t / NUMBER for t in times_with_trace_malloc] 154 | result = [f'{v:9.3f}' for v in times_with_trace_malloc] 155 | times_mean_with_trace_malloc = sum(times_with_trace_malloc) / REPEAT 156 | function_str = function + f"_with_trace_malloc('{key_type}')" 157 | print( 158 | f'{function_str:60}:' 159 | f' {", ".join(result)}' 160 | f' mean={times_mean_with_trace_malloc:9.3f}' 161 | f' min={min(times_with_trace_malloc):9.3f}' 162 | f' max={max(times_with_trace_malloc):9.3f}' 163 | f' span={max(times_with_trace_malloc) - min(times_with_trace_malloc):9.3f}' 164 | f' x{times_mean_with_trace_malloc / times_mean:>8.3f}' 165 | ) 166 | 167 | 168 | def main() -> int: 169 | logging.basicConfig( 170 | level=logging.INFO, 171 | # format='%(asctime)s - %(filename)24s#%(lineno)-4d - %(funcName)24s - %(process)5d - (%(threadName)-10s) - %(levelname)-8s - %(message)s', 172 | format='%(asctime)s - %(filename)24s#%(lineno)-4d - %(process)5d - (%(threadName)-10s) - %(levelname)-8s - %(message)s', 173 | stream=sys.stdout 174 | ) 175 | print() 176 | example_decorator_for_documentation([]) 177 | print() 178 | example_trace_malloc_for_documentation([]) 179 | # print() 180 | # example() 181 | return 0 182 | 183 | 184 | if __name__ == '__main__': 185 | sys.exit(main()) 186 | 187 | -------------------------------------------------------------------------------- /pymemtrace/examples/example.py: -------------------------------------------------------------------------------- 1 | 2 | import sys 3 | import time 4 | 5 | def make_list_strings(n): 6 | lst = [] 7 | for _i in range(n): 8 | lst.append(' ' * 1024) 9 | time.sleep(0.25) 10 | return lst 11 | 12 | def trim_list(lst, length): 13 | while len(lst) > length: 14 | lst.pop() 15 | time.sleep(0.15) 16 | return lst 17 | 18 | def just_sleep(t): 19 | time.sleep(t) 20 | 21 | def main(): 22 | for _i in range(3): 23 | l = make_list_strings(1024 * 10) 24 | just_sleep(0.5) 25 | trim_list(l, 128) 26 | just_sleep(0.2) 27 | return 0 28 | 29 | if __name__ == '__main__': 30 | sys.exit(main()) 31 | -------------------------------------------------------------------------------- /pymemtrace/parse_dtrace_output.py: -------------------------------------------------------------------------------- 1 | """ 2 | Parses DTrace output. 3 | 4 | Example from toolkit/py_flow_malloc_free.d: 5 | dtrace:::BEGIN 6 | 77633 cmn_cmd_opts.py:141 -> set_log_level malloc(560) pntr 0x7fca83ef4240 7 | 77633 __init__.py:422 -> validate malloc(1114) pntr 0x7fca858e4200 8 | 77633 __init__.py:422 -> validate free(0x7fca858e4200) 9 | 77633 threading.py:817 -> __init__ malloc(576) pntr 0x7fca83ef4470 10 | 11 | """ 12 | import logging 13 | import pprint 14 | import re 15 | import sys 16 | import typing 17 | 18 | 19 | logger = logging.getLogger(__file__) 20 | 21 | 22 | #: Matches " 77633 cmn_cmd_opts.py:141 -> set_log_level malloc(560) pntr 0x7fca83ef4240" 23 | #: Six groups: 24 | #: To ('77633', 'cmn_cmd_opts.py', '141', 'set_log_level', '560', '0x7fca83ef4240') 25 | RE_PY_FLOW_MALLOC_FREE_MALLOC = re.compile(r'^\s+(\d+)\s+(.+?):(\d+)\s+-> (\S+) malloc\((\d+)\) pntr (.+)$') 26 | 27 | 28 | class Malloc(typing.NamedTuple): 29 | log_line: int 30 | pid: int 31 | file: str 32 | line: int 33 | function: str 34 | size: int 35 | address: int 36 | 37 | def __str__(self): 38 | return f'Malloc: {self.log_line} {self.file}:{self.line} {self.function} {self.size} 0x{self.address:x}' 39 | 40 | def match_to_malloc(line: int, m: re.match) -> Malloc: 41 | """Given a match that has groups: ('77633', 'cmn_cmd_opts.py', '141', 'set_log_level', '560', '0x7fca83ef4240') 42 | this returns a Malloc() object.""" 43 | return Malloc( 44 | line, 45 | int(m.group(1)), 46 | m.group(2), 47 | int(m.group(3)), 48 | m.group(4), 49 | int(m.group(5)), 50 | int(m.group(6), 16), 51 | ) 52 | 53 | 54 | #: Matches " 77633 __init__.py:422 -> validate free(0x7fca858e4200)" 55 | #: Five groups: 56 | #: To ('77633', '__init__.py', '422', 'validate', '0x7fca858e4200') 57 | RE_PY_FLOW_MALLOC_FREE_FREE = re.compile(r'^\s+(\d+)\s+(.+?):(\d+)\s+-> (\S+) free\((\S+)\)$') 58 | 59 | 60 | class Free(typing.NamedTuple): 61 | log_line: int 62 | pid: int 63 | file: str 64 | line: int 65 | function: str 66 | address: int 67 | 68 | def __str__(self): 69 | return f'Free: {self.log_line} {self.file}:{self.line} {self.function}'# 0x{self.address:x}' 70 | 71 | 72 | def match_to_free(line: int, m: re.match) -> Free: 73 | """Given a match that has groups: ('77633', '__init__.py', '422', 'validate', '0x7fca858e4200') 74 | this returns a Malloc() object.""" 75 | return Free( 76 | line, 77 | int(m.group(1)), 78 | m.group(2), 79 | int(m.group(3)), 80 | m.group(4), 81 | int(m.group(5), 16), 82 | ) 83 | 84 | 85 | def parse_py_flow_malloc_free(file: typing.BinaryIO) -> typing.Dict[int, Malloc]: 86 | file.seek(0) 87 | malloc_dict: typing.Dict[int, Malloc] = {} 88 | for l, bin_line in enumerate(file): 89 | line = bin_line.decode('ascii', 'ignore') 90 | # print(f'TRACE: {line!r}') 91 | m = RE_PY_FLOW_MALLOC_FREE_MALLOC.match(line) 92 | if m is not None: 93 | malloc = match_to_malloc(l, m) 94 | if malloc.address in malloc_dict: 95 | logger.error('Line %d malloc address 0x%x already in malloc dict', l, malloc.address) 96 | else: 97 | malloc_dict[malloc.address] = malloc 98 | else: 99 | m = RE_PY_FLOW_MALLOC_FREE_FREE.match(line) 100 | if m is not None: 101 | free = match_to_free(l, m) 102 | if free.address == 0: 103 | logger.error('Ignoring 0x0 %s', free) 104 | else: 105 | if free.address not in malloc_dict: 106 | logger.error('Line %d free address 0x%x not in malloc dict', l, free.address) 107 | else: 108 | logger.info('%s free\'d %s', malloc_dict[free.address], free) 109 | del malloc_dict[free.address] 110 | return malloc_dict 111 | 112 | 113 | 114 | def main(): 115 | logging.basicConfig( 116 | level=20, 117 | format='%(asctime)s - %(filename)-16s - %(lineno)4d - %(process)5d - (%(threadName)-10s) - %(levelname)-8s - %(message)s', 118 | stream=sys.stdout, 119 | ) 120 | with open(sys.argv[1], 'rb') as file: 121 | malloc_dict = parse_py_flow_malloc_free(file) 122 | pprint.pprint(malloc_dict) 123 | return 0 124 | 125 | 126 | if __name__ == '__main__': 127 | sys.exit(main()) 128 | -------------------------------------------------------------------------------- /pymemtrace/redirect_stdout.py: -------------------------------------------------------------------------------- 1 | """Taken from the excellent blog: 2 | https://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/ 3 | 4 | Changes: 5 | 6 | * Minor edits. 7 | * Move tfile = tempfile.TemporaryFile(mode='w+b') outside try block. 8 | * Duplicate for stderr 9 | 10 | TODO: Unite duplicate code. 11 | """ 12 | from contextlib import contextmanager 13 | import ctypes 14 | import io 15 | import os 16 | import sys 17 | import tempfile 18 | 19 | import typing 20 | 21 | libc = ctypes.CDLL(None) 22 | if sys.platform == 'darwin': 23 | c_stdout = ctypes.c_void_p.in_dll(libc, '__stdoutp') 24 | c_stderr = ctypes.c_void_p.in_dll(libc, '__stderrp') 25 | else: 26 | c_stdout = ctypes.c_void_p.in_dll(libc, 'stdout') 27 | c_stderr = ctypes.c_void_p.in_dll(libc, 'stderr') 28 | 29 | 30 | @contextmanager 31 | def stdout_redirector(stream: typing.BinaryIO): 32 | """A context manager that redirects Python stdout and C stdout to the given binary I/O stream.""" 33 | def _redirect_stdout(to_fd): 34 | """Redirect stdout to the given file descriptor.""" 35 | # Flush the C-level buffer stdout 36 | libc.fflush(c_stdout) 37 | # Flush and close sys.stdout - also closes the file descriptor (fd) 38 | sys.stdout.close() 39 | # Make original_stdout_fd point to the same file as to_fd 40 | os.dup2(to_fd, original_stdout_fd) 41 | # Create a new sys.stdout that points to the redirected fd 42 | sys.stdout = io.TextIOWrapper(os.fdopen(original_stdout_fd, 'wb')) 43 | 44 | # The original fd stdout points to. Usually 1 on POSIX systems. 45 | original_stdout_fd = sys.stdout.fileno() 46 | # Save a copy of the original stdout fd in saved_stdout_fd 47 | saved_stdout_fd = os.dup(original_stdout_fd) 48 | # Create a temporary file and redirect stdout to it 49 | tfile = tempfile.TemporaryFile(mode='w+b') 50 | try: 51 | _redirect_stdout(tfile.fileno()) 52 | # Yield to caller, then redirect stdout back to the saved fd 53 | yield 54 | _redirect_stdout(saved_stdout_fd) 55 | # Copy contents of temporary file to the given stream 56 | tfile.flush() 57 | tfile.seek(0, io.SEEK_SET) 58 | stream.write(tfile.read()) 59 | finally: 60 | tfile.close() 61 | os.close(saved_stdout_fd) 62 | 63 | 64 | @contextmanager 65 | def stderr_redirector(stream: typing.BinaryIO): 66 | """A context manager that redirects Python stderr and C stderr to the given binary I/O stream.""" 67 | def _redirect_stderr(to_fd): 68 | """Redirect stderr to the given file descriptor.""" 69 | # Flush the C-level buffer stderr 70 | libc.fflush(c_stderr) 71 | # Flush and close sys.stderr - also closes the file descriptor (fd) 72 | sys.stderr.close() 73 | # Make original_stderr_fd point to the same file as to_fd 74 | os.dup2(to_fd, original_stderr_fd) 75 | # Create a new sys.stderr that points to the redirected fd 76 | sys.stderr = io.TextIOWrapper(os.fdopen(original_stderr_fd, 'wb')) 77 | 78 | # The original fd stderr points to. Usually 2 on POSIX systems. 79 | original_stderr_fd = sys.stderr.fileno() 80 | # Save a copy of the original stderr fd in saved_stderr_fd 81 | saved_stderr_fd = os.dup(original_stderr_fd) 82 | # Create a temporary file and redirect stderr to it 83 | tfile = tempfile.TemporaryFile(mode='w+b') 84 | try: 85 | _redirect_stderr(tfile.fileno()) 86 | # Yield to caller, then redirect stderr back to the saved fd 87 | yield 88 | _redirect_stderr(saved_stderr_fd) 89 | # Copy contents of temporary file to the given stream 90 | tfile.flush() 91 | tfile.seek(0, io.SEEK_SET) 92 | stream.write(tfile.read()) 93 | finally: 94 | tfile.close() 95 | os.close(saved_stderr_fd) 96 | -------------------------------------------------------------------------------- /pymemtrace/src/c/get_rss.c: -------------------------------------------------------------------------------- 1 | // Origin: https://stackoverflow.com/questions/669438/how-to-get-memory-usage-at-runtime-using-c 2 | // This gives the link: http://nadeausoftware.com/articles/2012/07/c_c_tip_how_get_process_resident_set_size_physical_memory_use 3 | // However that no londger exists. 4 | // Archive link: https://web.archive.org/web/20190923225212/http://nadeausoftware.com/articles/2012/07/c_c_tip_how_get_process_resident_set_size_physical_memory_use 5 | // This code is reformatted from that code, lightly edited, added header file etc. 6 | 7 | /* 8 | * Author: David Robert Nadeau 9 | * Site: http://NadeauSoftware.com/ 10 | * License: Creative Commons Attribution 3.0 Unported License 11 | * http://creativecommons.org/licenses/by/3.0/deed.en_US 12 | */ 13 | 14 | #include "get_rss.h" 15 | 16 | #if defined(_WIN32) 17 | #include 18 | #include 19 | #elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__)) 20 | #include 21 | #include 22 | #if defined(__APPLE__) && defined(__MACH__) 23 | /* Added for faster (?) Mac OS X RSS value in getCurrentRSS_alternate. */ 24 | #include 25 | #include 26 | #elif (defined(_AIX) || defined(__TOS__AIX__)) || (defined(__sun__) || defined(__sun) || defined(sun) && (defined(__SVR4) || defined(__svr4__))) 27 | #include 28 | #include 29 | #elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__) 30 | #include 31 | #endif 32 | #else 33 | #error "Cannot define getPeakRSS( ) or getCurrentRSS( ) for an unknown OS." 34 | #endif 35 | 36 | /** 37 | * Returns the peak (maximum so far) resident set size (physical 38 | * memory use) measured in bytes, or zero if the value cannot be 39 | * determined on this OS. 40 | */ 41 | size_t getPeakRSS(void) { 42 | #if defined(_WIN32) 43 | /* Windows -------------------------------------------------- */ 44 | PROCESS_MEMORY_COUNTERS info; 45 | GetProcessMemoryInfo( GetCurrentProcess( ), &info, sizeof(info) ); 46 | return (size_t)info.PeakWorkingSetSize; 47 | #elif (defined(_AIX) || defined(__TOS__AIX__)) || (defined(__sun__) || defined(__sun) || defined(sun) && (defined(__SVR4) || defined(__svr4__))) 48 | /* AIX and Solaris ------------------------------------------ */ 49 | struct psinfo psinfo; 50 | int fd = -1; 51 | if ( (fd = open( "/proc/self/psinfo", O_RDONLY )) == -1 ) 52 | return (size_t)0L; /* Can't open? */ 53 | if ( read( fd, &psinfo, sizeof(psinfo) ) != sizeof(psinfo) ) 54 | { 55 | close( fd ); 56 | return (size_t)0L; /* Can't read? */ 57 | } 58 | close( fd ); 59 | return (size_t)(psinfo.pr_rssize * 1024L); 60 | #elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__)) 61 | /* BSD, Linux, and OSX -------------------------------------- */ 62 | struct rusage rusage; 63 | getrusage( RUSAGE_SELF, &rusage ); 64 | #if defined(__APPLE__) && defined(__MACH__) 65 | return (size_t)rusage.ru_maxrss; 66 | #else 67 | return (size_t)(rusage.ru_maxrss * 1024L); 68 | #endif 69 | 70 | #else 71 | /* Unknown OS ----------------------------------------------- */ 72 | return (size_t)0L; /* Unsupported. */ 73 | #endif 74 | } 75 | 76 | /** 77 | * Returns the current resident set size (physical memory use) measured 78 | * in bytes, or zero if the value cannot be determined on this OS. 79 | */ 80 | size_t getCurrentRSS(void) { 81 | #if defined(_WIN32) 82 | /* Windows -------------------------------------------------- */ 83 | PROCESS_MEMORY_COUNTERS info; 84 | GetProcessMemoryInfo( GetCurrentProcess( ), &info, sizeof(info) ); 85 | return (size_t)info.WorkingSetSize; 86 | #elif defined(__APPLE__) && defined(__MACH__) 87 | /* OSX ------------------------------------------------------ */ 88 | struct mach_task_basic_info info; 89 | mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; 90 | if ( task_info( mach_task_self( ), MACH_TASK_BASIC_INFO, 91 | (task_info_t)&info, &infoCount ) != KERN_SUCCESS ) 92 | return (size_t)0L; /* Can't access? */ 93 | return (size_t)info.resident_size; 94 | #elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__) 95 | /* Linux ---------------------------------------------------- */ 96 | long rss = 0L; 97 | FILE* fp = NULL; 98 | if ( (fp = fopen( "/proc/self/statm", "r" )) == NULL ) 99 | return (size_t)0L; /* Can't open? */ 100 | if ( fscanf( fp, "%*s%ld", &rss ) != 1 ) { 101 | fclose( fp ); 102 | return (size_t)0L; /* Can't read? */ 103 | } 104 | fclose( fp ); 105 | return (size_t)rss * (size_t)sysconf( _SC_PAGESIZE); 106 | #else 107 | /* AIX, BSD, Solaris, and Unknown OS ------------------------ */ 108 | return (size_t)0L; /* Unsupported. */ 109 | #endif 110 | } 111 | 112 | size_t getCurrentRSS_alternate(void) { 113 | #if defined(__APPLE__) 114 | /* OSX ------------------------------------------------------ */ 115 | /* Empty fields in struct. */ 116 | /* 117 | struct rusage rusage; 118 | getrusage( RUSAGE_SELF, &rusage ); 119 | return (size_t)(rusage.ru_ixrss + rusage.ru_idrss + rusage.ru_isrss); 120 | */ 121 | /* 122 | * This works though. 123 | * Could use PROC_PID_RUSAGE ? 124 | */ 125 | pid_t pid = getpid(); 126 | struct proc_taskinfo proc; 127 | int st = proc_pidinfo(pid, PROC_PIDTASKINFO, 0, &proc, PROC_PIDTASKINFO_SIZE); 128 | if (st == 0) { 129 | return 0; 130 | } 131 | return proc.pti_resident_size; 132 | #endif 133 | return getCurrentRSS(); 134 | } 135 | 136 | -------------------------------------------------------------------------------- /pymemtrace/src/c/pymemtrace_util.c: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Paul Ross on 03/11/2020. 3 | // 4 | #define PY_SSIZE_T_CLEAN 5 | 6 | #include 7 | 8 | #define _POSIX_C_SOURCE 200112L // For gmtime_r in 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "pymemtrace_util.h" 16 | 17 | /** 18 | * Creates a log file name with the timestamp (to the second), the process ID and the Python version. 19 | * 20 | * @param trace_type 'T' for a trace function, 'P' for a profile function. 21 | * @param trace_stack_depth The length of the linked list of trace functions starting from 0. 22 | * This discriminates log files when there is nested tracing. 23 | * @return The log file name or NULL on failure. For example "20241107_195847_62264_P_0_PY3.13.0b3.log". 24 | */ 25 | const char *create_filename(char trace_type, int trace_stack_depth) { 26 | /* Not thread safe. */ 27 | static char filename[PYMEMTRACE_FILE_NAME_MAX_LENGTH]; 28 | static struct tm now; 29 | time_t t = time(NULL); 30 | gmtime_r(&t, &now); 31 | size_t len = strftime(filename, PYMEMTRACE_FILE_NAME_MAX_LENGTH, "%Y%m%d_%H%M%S", &now); 32 | if (len == 0) { 33 | fprintf(stderr, "create_filename(): strftime failed."); 34 | return NULL; 35 | } 36 | pid_t pid = getpid(); 37 | if (snprintf(filename + len, PYMEMTRACE_FILE_NAME_MAX_LENGTH - len - 1, "_%d_%c_%d_PY%s.log", pid, trace_type, trace_stack_depth, PY_VERSION) == 0) { 38 | fprintf(stderr, "create_filename(): failed to add PID, stack depth and Python version."); 39 | return NULL; 40 | } 41 | return filename; 42 | } 43 | 44 | /** 45 | * Get the current working directory using \c getcwd(). 46 | * 47 | * @return The current working directory or NULL on failure. 48 | */ 49 | const char *current_working_directory(void) { 50 | static char cwd[PYMEMTRACE_PATH_NAME_MAX_LENGTH]; 51 | if (getcwd(cwd, sizeof(cwd)) == NULL) { 52 | fprintf(stderr, "Can not get current working directory.\n"); 53 | return NULL; 54 | } 55 | return cwd; 56 | } 57 | -------------------------------------------------------------------------------- /pymemtrace/src/cpy/cCustom.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Created by Paul Ross on 31/10/2020. 3 | * 4 | * Typical extension type from the Python documentation, lightly edited. 5 | * 6 | */ 7 | #define PY_SSIZE_T_CLEAN 8 | #include 9 | #include "structmember.h" 10 | 11 | typedef struct { 12 | PyObject_HEAD 13 | PyObject *first; /* first name */ 14 | PyObject *last; /* last name */ 15 | int number; 16 | } CustomObject; 17 | 18 | static void 19 | Custom_dealloc(CustomObject *self) { 20 | Py_XDECREF(self->first); 21 | Py_XDECREF(self->last); 22 | Py_TYPE(self)->tp_free((PyObject *) self); 23 | } 24 | 25 | static PyObject * 26 | Custom_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwds)) { 27 | CustomObject *self; 28 | self = (CustomObject *) type->tp_alloc(type, 0); 29 | if (self != NULL) { 30 | self->first = PyUnicode_FromString(""); 31 | if (self->first == NULL) { 32 | Py_DECREF(self); 33 | return NULL; 34 | } 35 | self->last = PyUnicode_FromString(""); 36 | if (self->last == NULL) { 37 | Py_DECREF(self); 38 | return NULL; 39 | } 40 | self->number = 0; 41 | } 42 | return (PyObject *) self; 43 | } 44 | 45 | static int 46 | Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) { 47 | static char *kwlist[] = {"first", "last", "number", NULL}; 48 | PyObject *first = NULL, *last = NULL, *tmp; 49 | 50 | if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist, 51 | &first, &last, 52 | &self->number)) 53 | return -1; 54 | 55 | if (first) { 56 | tmp = self->first; 57 | Py_INCREF(first); 58 | self->first = first; 59 | Py_DECREF(tmp); 60 | } 61 | if (last) { 62 | tmp = self->last; 63 | Py_INCREF(last); 64 | self->last = last; 65 | Py_DECREF(tmp); 66 | } 67 | return 0; 68 | } 69 | 70 | static PyMemberDef Custom_members[] = { 71 | {"number", T_INT, offsetof(CustomObject, number), 0, 72 | "custom number"}, 73 | {NULL, 0, 0, 0, NULL} /* Sentinel */ 74 | }; 75 | 76 | static PyObject * 77 | Custom_getfirst(CustomObject *self, void *Py_UNUSED(closure)) { 78 | Py_INCREF(self->first); 79 | return self->first; 80 | } 81 | 82 | static int 83 | Custom_setfirst(CustomObject *self, PyObject *value, void *Py_UNUSED(closure)) { 84 | PyObject *tmp; 85 | if (value == NULL) { 86 | PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); 87 | return -1; 88 | } 89 | if (!PyUnicode_Check(value)) { 90 | PyErr_SetString(PyExc_TypeError, 91 | "The first attribute value must be a string"); 92 | return -1; 93 | } 94 | tmp = self->first; 95 | Py_INCREF(value); 96 | self->first = value; 97 | Py_DECREF(tmp); 98 | return 0; 99 | } 100 | 101 | static PyObject * 102 | Custom_getlast(CustomObject *self, void *closure) { 103 | (void) closure; 104 | Py_INCREF(self->last); 105 | return self->last; 106 | } 107 | 108 | static int 109 | Custom_setlast(CustomObject *self, PyObject *value, void *Py_UNUSED(closure)) { 110 | PyObject *tmp; 111 | if (value == NULL) { 112 | PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute"); 113 | return -1; 114 | } 115 | if (!PyUnicode_Check(value)) { 116 | PyErr_SetString(PyExc_TypeError, 117 | "The last attribute value must be a string"); 118 | return -1; 119 | } 120 | tmp = self->last; 121 | Py_INCREF(value); 122 | self->last = value; 123 | Py_DECREF(tmp); 124 | return 0; 125 | } 126 | 127 | static PyGetSetDef Custom_getsetters[] = { 128 | {"first", (getter) Custom_getfirst, (setter) Custom_setfirst, 129 | "first name", NULL}, 130 | {"last", (getter) Custom_getlast, (setter) Custom_setlast, 131 | "last name", NULL}, 132 | {NULL, NULL, NULL, NULL, NULL} /* Sentinel */ 133 | }; 134 | 135 | static PyObject * 136 | Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) { 137 | return PyUnicode_FromFormat("%S %S", self->first, self->last); 138 | } 139 | 140 | static PyMethodDef Custom_methods[] = { 141 | {"name", (PyCFunction) Custom_name, METH_NOARGS, 142 | "Return the name, combining the first and last name" 143 | }, 144 | {NULL, NULL, 0, NULL} /* Sentinel */ 145 | }; 146 | 147 | static PyTypeObject CustomType = { 148 | PyVarObject_HEAD_INIT(NULL, 0) 149 | .tp_name = "custom.Custom", 150 | .tp_doc = "Custom objects", 151 | .tp_basicsize = sizeof(CustomObject), 152 | .tp_itemsize = 0, 153 | .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, 154 | .tp_new = Custom_new, 155 | .tp_init = (initproc) Custom_init, 156 | .tp_dealloc = (destructor) Custom_dealloc, 157 | .tp_members = Custom_members, 158 | .tp_methods = Custom_methods, 159 | .tp_getset = Custom_getsetters, 160 | }; 161 | 162 | static PyMethodDef CustomMethods[] = { 163 | {NULL, NULL, 0, NULL} /* Sentinel */ 164 | }; 165 | 166 | static PyModuleDef custommodule = { 167 | PyModuleDef_HEAD_INIT, 168 | .m_name = "custom", 169 | .m_doc = "Example module that creates an extension type.", 170 | .m_size = -1, 171 | .m_methods = CustomMethods, 172 | }; 173 | 174 | PyMODINIT_FUNC 175 | PyInit_custom(void) { 176 | PyObject *m; 177 | if (PyType_Ready(&CustomType) < 0) 178 | return NULL; 179 | 180 | m = PyModule_Create(&custommodule); 181 | if (m == NULL) 182 | return NULL; 183 | 184 | Py_INCREF(&CustomType); 185 | if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { 186 | Py_DECREF(&CustomType); 187 | Py_DECREF(m); 188 | return NULL; 189 | } 190 | return m; 191 | } 192 | -------------------------------------------------------------------------------- /pymemtrace/src/include/get_rss.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Paul Ross on 29/10/2020. 3 | // 4 | 5 | #ifndef CPYMEMTRACE_GET_RSS_H 6 | #define CPYMEMTRACE_GET_RSS_H 7 | 8 | #include 9 | 10 | size_t getPeakRSS(void); 11 | size_t getCurrentRSS(void); 12 | size_t getCurrentRSS_alternate(void); 13 | 14 | #endif //CPYMEMTRACE_GET_RSS_H 15 | -------------------------------------------------------------------------------- /pymemtrace/src/include/pymemtrace_util.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Paul Ross on 03/11/2020. 3 | // 4 | 5 | #ifndef CPYMEMTRACE_PYMEMTRACE_UTIL_H 6 | #define CPYMEMTRACE_PYMEMTRACE_UTIL_H 7 | 8 | #define PYMEMTRACE_PATH_NAME_MAX_LENGTH 4096 9 | #define PYMEMTRACE_FILE_NAME_MAX_LENGTH 1024 10 | 11 | const char *create_filename(char trace_type, int trace_stack_depth); 12 | const char *current_working_directory(void); 13 | 14 | #endif //CPYMEMTRACE_PYMEMTRACE_UTIL_H 15 | -------------------------------------------------------------------------------- /pymemtrace/src/main.c: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Paul Ross on 29/10/2020. 3 | // 4 | // Source: https://www.gnu.org/software/libc/manual/html_node/Example-of-Getopt.html 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | #include "get_rss.h" 13 | 14 | void macosx_get_pid_info(void) { 15 | printf("macosx_get_pid_info()\n"); 16 | pid_t pid = getpid(); 17 | struct proc_bsdinfo proc; 18 | int st = proc_pidinfo(pid, PROC_PIDTBSDINFO, 0, &proc, PROC_PIDTBSDINFO_SIZE); 19 | printf("Result: %d %lu\n", st, sizeof(proc)); 20 | printf("name: %s\n", proc.pbi_name); 21 | } 22 | 23 | /* PROC_PIDTASKINFO in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk/usr/include/sys/proc_info.h:647 */ 24 | void macosx_get_task_info(void) { 25 | printf("macosx_get_task_info()\n"); 26 | pid_t pid = getpid(); 27 | struct proc_taskinfo proc; 28 | int st = proc_pidinfo(pid, PROC_PIDTASKINFO, 0, &proc, PROC_PIDTASKINFO_SIZE); 29 | printf("Result: %d %lu\n", st, sizeof(proc)); 30 | printf("RSS: %llu\n", proc.pti_resident_size); 31 | } 32 | 33 | void macosx_get_taskall_info(void) { 34 | printf("macosx_get_taskall_info()\n"); 35 | pid_t pid = getpid(); 36 | struct proc_taskallinfo proc; 37 | int st = proc_pidinfo(pid, PROC_PIDTASKALLINFO, 0, &proc, PROC_PIDTASKALLINFO_SIZE); 38 | printf("Result: %d %lu\n", st, sizeof(proc)); 39 | printf("name: %s\n", proc.pbsd.pbi_name); 40 | } 41 | 42 | void macosx_get_just_rss_info(void) { 43 | printf("macosx_get_just_rss_info()\n"); 44 | pid_t pid = getpid(); 45 | struct proc_taskallinfo proc; 46 | int st = proc_pidinfo(pid, PROC_PID_RUSAGE, 0, &proc, PROC_PID_RUSAGE_SIZE); 47 | printf("Result: %d %lu\n", st, sizeof(proc)); 48 | printf("name: %s\n", proc.pbsd.pbi_name); 49 | } 50 | 51 | void macosx_get_short_pid_info(void) { 52 | pid_t pid = getpid(); 53 | struct proc_bsdshortinfo proc; 54 | 55 | int st = proc_pidinfo(pid, PROC_PIDT_SHORTBSDINFO, 0, 56 | &proc, PROC_PIDT_SHORTBSDINFO_SIZE); 57 | 58 | if (st != PROC_PIDT_SHORTBSDINFO_SIZE) { 59 | fprintf(stderr, "Cannot get process info\n"); 60 | } 61 | printf(" pid: %d\n", (int)proc.pbsi_pid); 62 | printf("ppid: %d\n", (int)proc.pbsi_ppid); 63 | printf("comm: %s\n", proc.pbsi_comm); 64 | //printf("name: %s\n", proc.pbsi_name); 65 | printf(" uid: %d\n", (int)proc.pbsi_uid); 66 | printf(" gid: %d\n", (int)proc.pbsi_gid); 67 | } 68 | 69 | #if 1 70 | int 71 | main (int argc, char **argv) 72 | { 73 | int aflag = 0; 74 | int bflag = 0; 75 | char *cvalue = NULL; 76 | int index; 77 | int c; 78 | 79 | opterr = 0; 80 | 81 | while ((c = getopt (argc, argv, "abc:")) != -1) 82 | switch (c) 83 | { 84 | case 'a': 85 | aflag = 1; 86 | break; 87 | case 'b': 88 | bflag = 1; 89 | break; 90 | case 'c': 91 | cvalue = optarg; 92 | break; 93 | case '?': 94 | if (optopt == 'c') 95 | fprintf (stderr, "Option -%c requires an argument.\n", optopt); 96 | else if (isprint (optopt)) 97 | fprintf (stderr, "Unknown option `-%c'.\n", optopt); 98 | else 99 | fprintf (stderr, 100 | "Unknown option character `\\x%x'.\n", 101 | optopt); 102 | return 1; 103 | default: 104 | abort (); 105 | } 106 | 107 | printf ("aflag = %d, bflag = %d, cvalue = %s\n", 108 | aflag, bflag, cvalue); 109 | 110 | for (index = optind; index < argc; index++) 111 | printf ("Non-option argument %s\n", argv[index]); 112 | 113 | size_t rss = getCurrentRSS(); 114 | size_t rss_peak = getPeakRSS(); 115 | printf("RSS: %zu Peak RSS: %zu\n", rss, rss_peak); 116 | 117 | printf("\n"); 118 | macosx_get_short_pid_info(); 119 | 120 | printf("\n"); 121 | macosx_get_pid_info(); 122 | 123 | printf("\n"); 124 | macosx_get_task_info(); 125 | 126 | printf("\n"); 127 | macosx_get_taskall_info(); 128 | 129 | printf("\n"); 130 | macosx_get_just_rss_info(); 131 | 132 | return 0; 133 | } 134 | #endif 135 | -------------------------------------------------------------------------------- /pymemtrace/trace_malloc.py: -------------------------------------------------------------------------------- 1 | """ 2 | A wrapper around the tracemalloc standard library module. 3 | """ 4 | import functools 5 | import logging 6 | import sys 7 | import tracemalloc 8 | 9 | import typing 10 | 11 | 12 | class TraceMalloc: 13 | """A wrapper around the tracemalloc module that can compensate for tracemalloc's memory usage.""" 14 | 15 | # Central flag to control all instances of TraceMalloc's 16 | TRACE_ON = True 17 | ALLOWED_GRANULARITY = ('filename', 'lineno', 'traceback') 18 | 19 | def __init__(self, statistics_granularity: str = 'lineno'): 20 | """statistics_granularity can be 'filename', 'lineno' or 'traceback'.""" 21 | if statistics_granularity not in self.ALLOWED_GRANULARITY: 22 | raise ValueError( 23 | f'statistics_granularity must be in {self.ALLOWED_GRANULARITY} not {statistics_granularity}' 24 | ) 25 | self.statistics_granularity = statistics_granularity 26 | if self.TRACE_ON: 27 | if not tracemalloc.is_tracing(): 28 | tracemalloc.start() 29 | self.tracemalloc_snapshot_start: typing.Optional[tracemalloc.Snapshot] = None 30 | self.tracemalloc_snapshot_finish: typing.Optional[tracemalloc.Snapshot] = None 31 | self.memory_start: typing.Optional[int] = None 32 | self.memory_finish: typing.Optional[int] = None 33 | self.statistics: typing.List[tracemalloc.StatisticDiff] = [] 34 | self._diff: typing.Optional[int] = None 35 | 36 | def __enter__(self): 37 | """Take a tracemalloc snapshot.""" 38 | if self.TRACE_ON: 39 | self.tracemalloc_snapshot_start = tracemalloc.take_snapshot() 40 | self.memory_start = tracemalloc.get_tracemalloc_memory() 41 | return self 42 | 43 | def __exit__(self, exc_type, exc_val, exc_tb): 44 | """Take a tracemalloc snapshot and subtract the initial snapshot. Also note the tracemalloc memory usage.""" 45 | if self.TRACE_ON: 46 | self.tracemalloc_snapshot_finish = tracemalloc.take_snapshot() 47 | self.memory_finish = tracemalloc.get_tracemalloc_memory() 48 | self.statistics = self.tracemalloc_snapshot_finish.compare_to( 49 | self.tracemalloc_snapshot_start, self.statistics_granularity 50 | ) 51 | self._diff = None 52 | return False 53 | 54 | @property 55 | def tracemalloc_memory_usage(self) -> typing.Optional[int]: 56 | """Returns the tracemalloc memory usage between snapshots of None of no tracing.""" 57 | if self.TRACE_ON: 58 | return self.memory_finish - self.memory_start 59 | 60 | @property 61 | def diff(self) -> int: 62 | """The net memory usage difference recorded by tracemalloc allowing for the memory usage of tracemalloc.""" 63 | if self.TRACE_ON: 64 | if self._diff is None: 65 | self._diff = sum(s.size_diff for s in self.statistics) - self.tracemalloc_memory_usage 66 | return self._diff 67 | return -sys.maxsize - 1 68 | 69 | def net_statistics(self): 70 | """Returns the list of statistics ignoring those from the tracemalloc module itself.""" 71 | ret = [] 72 | for statistic in self.statistics: 73 | file_name = statistic.traceback[0].filename 74 | if file_name != tracemalloc.__file__: 75 | ret.append(statistic) 76 | return ret 77 | 78 | 79 | def trace_malloc_log(log_level: int): 80 | """Decorator that logs the decorated function the use of Python memory in bytes at the desired log level. 81 | This can be switched to a NOP by setting TraceMalloc.TRACE_ON to False.""" 82 | def memory_inner(fn): 83 | @functools.wraps(fn) 84 | def wrapper(*args, **kwargs): 85 | with TraceMalloc() as tm: 86 | result = fn(*args, ** kwargs) 87 | logging.log(log_level, f'TraceMalloc memory delta: {tm.diff:,d} for "{fn.__name__}()"') 88 | return result 89 | return wrapper 90 | return memory_inner 91 | -------------------------------------------------------------------------------- /pymemtrace/util/gnuplot.py: -------------------------------------------------------------------------------- 1 | """ 2 | Provides gnuplot support to command line tools. 3 | """ 4 | import argparse 5 | import logging 6 | import os 7 | import subprocess 8 | import typing 9 | from functools import reduce 10 | 11 | 12 | logger = logging.getLogger(__file__) 13 | 14 | 15 | def add_gnuplot_to_argument_parser(parser: argparse.ArgumentParser) -> None: 16 | """Adds ``--gnuplot=`` to the argument parser as ``args.gnuplot``.""" 17 | v = version() 18 | logger.info(f'gnuplot version: "{v}"') 19 | print(f'gnuplot version: "{v}"') 20 | if not v: 21 | raise ValueError('--gnuplot option is requested but gnuplot is not installed.') 22 | parser.add_argument('--gnuplot', type=str, help='Directory to write the gnuplot data.') 23 | 24 | 25 | def version() -> bytes: 26 | """ 27 | For example: b'gnuplot 5.2 patchlevel 6' 28 | """ 29 | with subprocess.Popen(['gnuplot', '--version'], stdout=subprocess.PIPE) as proc: 30 | return proc.stdout.read().strip() 31 | 32 | 33 | def _num_columns(table: typing.Sequence[typing.Sequence[typing.Any]]) -> int: 34 | """ 35 | Returns the number of columns of the table. 36 | Will raise a ValueError if the table is uneven. 37 | """ 38 | num_colums_set = set(len(r) for r in table) 39 | if len(num_colums_set) != 1: 40 | raise ValueError(f'Not rectangular: {num_colums_set}.') 41 | return num_colums_set.pop() 42 | 43 | 44 | def create_gnuplot_dat(table: typing.Sequence[typing.Sequence[typing.Any]]) -> str: 45 | """ 46 | Returns a pretty formatted string of the data in the given table suitable for use as a gnuplot ``.dat`` file. 47 | """ 48 | num_columns = _num_columns(table) 49 | column_widths = reduce( 50 | lambda l, rows: [max(l, len(str(r)) + 2) for l, r in zip(l, rows)], table, [0,] * num_columns, 51 | ) 52 | result: typing.List[str] = [] 53 | for row in table: 54 | result.append(' '.join(f'{str(row[i]):<{column_widths[i]}}' for i in range(num_columns))) 55 | return '\n'.join(result) 56 | 57 | 58 | def invoke_gnuplot(path: str, name: str, table: typing.Sequence[typing.Sequence[typing.Any]], plt: str) -> int: 59 | """ 60 | Create the plot for name. 61 | path - the directory to write the data and plot files to. 62 | name - the name of those files. 63 | table - the table of values to write to the data file. 64 | 65 | Returns the gnuplot error code. 66 | """ 67 | logger.info('Writing gnuplot data "{}" in path {}'.format(name, path)) 68 | os.makedirs(path, exist_ok=True) 69 | with open(os.path.join(path, f'{name}.dat'), 'w') as outfile: 70 | outfile.write(create_gnuplot_dat(table)) 71 | with open(os.path.join(path, f'{name}.plt'), 'w') as outfile: 72 | outfile.write(plt) 73 | proc = subprocess.Popen( 74 | args=['gnuplot', '-p', f'{name}.plt'], 75 | shell=False, 76 | cwd=path, 77 | ) 78 | try: 79 | # Timeout 10 seconds as curve fitting can take a while. 80 | stdout, stderr = proc.communicate(timeout=10) 81 | except subprocess.TimeoutExpired as err: 82 | logger.exception(str(err)) 83 | proc.kill() 84 | stdout, stderr = proc.communicate() 85 | logging.info(f'gnuplot stdout: {stdout}') 86 | if proc.returncode or stderr: 87 | logging.error(f'gnuplot stderr: {stdout}') 88 | return proc.returncode 89 | 90 | 91 | def write_test_file(path: str, typ: str) -> int: 92 | """Writes out a Gnuplot test file.""" 93 | test_stdin = '\n'.join( 94 | [ 95 | f'set terminal {typ}', 96 | f'set output "test.{typ}"', 97 | 'test', 98 | ] 99 | ) 100 | proc = subprocess.Popen( 101 | args=['gnuplot'], 102 | shell=False, 103 | cwd=path, 104 | stdin=subprocess.PIPE, 105 | ) 106 | try: 107 | proc.stdin.write(bytes(test_stdin, 'ascii')) 108 | # proc.stdin.close() 109 | stdout, stderr = proc.communicate(timeout=1, ) 110 | except subprocess.TimeoutExpired as err: 111 | logger.exception() 112 | proc.kill() 113 | stdout, stderr = proc.communicate() 114 | logging.info(f'gnuplot stdout: {stdout}') 115 | if stderr: 116 | logging.error(f'gnuplot stderr: {stdout}') 117 | return proc.returncode 118 | 119 | # Gnuplot fragments 120 | 121 | PLOT = """set grid 122 | set title "{title}" 123 | 124 | set pointsize 1 125 | set datafile separator whitespace#" " 126 | set datafile missing "NaN" 127 | """ 128 | 129 | X_LOG = """set logscale x 130 | set xlabel "{label}" 131 | # set mxtics 5 132 | # set xrange [0:3000] 133 | # set xtics 134 | # set format x 135 | """ 136 | 137 | Y_LOG = """set logscale y 138 | set ylabel "{label}" 139 | # set yrange [1:1e5] 140 | # set ytics 20 141 | # set mytics 2 142 | # set ytics 8,35,3 143 | """ 144 | 145 | Y2_LOG = """set logscale y2 146 | set y2label "{label}" 147 | #set y2range [1e5:1e9] 148 | set y2tics 149 | """ 150 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | psutil 2 | pytest 3 | pytest-runner 4 | setuptools 5 | Sphinx 6 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.2.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | search = version='{current_version}' 8 | replace = version='{new_version}' 9 | 10 | [bumpversion:file:pymemtrace/__init__.py] 11 | search = __version__ = '{current_version}' 12 | replace = __version__ = '{new_version}' 13 | 14 | [bdist_wheel] 15 | universal = 1 16 | 17 | [flake8] 18 | exclude = docs 19 | 20 | [aliases] 21 | test = pytest 22 | # Define setup.py command aliases here 23 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """The setup script.""" 5 | import os 6 | import sys 7 | from setuptools import setup, find_packages 8 | from distutils.core import Extension 9 | 10 | with open('README.rst') as readme_file: 11 | readme = readme_file.read() 12 | 13 | with open('HISTORY.rst') as history_file: 14 | history = history_file.read() 15 | 16 | requirements = [ 17 | 'psutil', 18 | ] 19 | 20 | setup_requirements = [ 21 | 'pytest', 22 | 'pytest-runner', 23 | ] 24 | 25 | test_requirements = [ 26 | 'pytest', 27 | ] 28 | 29 | extra_compile_args = [ 30 | '-Wall', 31 | '-Wextra', 32 | '-Werror', 33 | '-Wfatal-errors', 34 | '-Wpedantic', 35 | # Some internal Python library code does not like this with C++11. 36 | # '-Wno-c++11-compat-deprecated-writable-strings', 37 | # '-std=c++11', 38 | '-std=c99', 39 | # Until we use m_coalesce 40 | # '-Wno-unused-private-field', 41 | 42 | # # Temporary 43 | # '-Wno-unused-variable', 44 | # '-Wno-unused-parameter', 45 | ] 46 | 47 | if sys.platform.startswith('linux'): 48 | extra_compile_args.extend( 49 | [ 50 | # Linux, GCC complains about casting PyCFunction. 51 | '-Wno-cast-function-type', 52 | ] 53 | ) 54 | 55 | # DEBUG = False 56 | DEBUG = True 57 | 58 | if DEBUG: 59 | extra_compile_args.extend(['-g3', '-O0', '-DDEBUG=1', '-UNDEBUG']) 60 | else: 61 | extra_compile_args.extend(['-O3', '-UDEBUG', '-DNDEBUG']) 62 | 63 | setup( 64 | name='pymemtrace', 65 | version='0.2.0', 66 | description="Python memory tracing.", 67 | long_description=readme + '\n\n' + history, 68 | long_description_content_type='text/x-rst', 69 | author="Paul Ross", 70 | author_email='apaulross@gmail.com', 71 | url='https://github.com/paulross/pymemtrace', 72 | packages=find_packages(), # include=['pymemtrace']), 73 | include_package_data=True, 74 | install_requires=requirements, 75 | license="MIT license", 76 | zip_safe=False, 77 | keywords='pymemtrace', 78 | # https://pypi.org/classifiers/ 79 | classifiers=[ 80 | 'Development Status :: 3 - Alpha', 81 | 'Intended Audience :: Developers', 82 | 'License :: OSI Approved :: MIT License', 83 | 'Natural Language :: English', 84 | 'Programming Language :: C', 85 | 'Programming Language :: Python :: 3 :: Only', 86 | 'Programming Language :: Python :: Implementation :: CPython', 87 | # https://devguide.python.org/versions/ 88 | 'Programming Language :: Python :: 3.7', 89 | 'Programming Language :: Python :: 3.8', 90 | 'Programming Language :: Python :: 3.9', 91 | 'Programming Language :: Python :: 3.10', 92 | 'Programming Language :: Python :: 3.11', 93 | 'Programming Language :: Python :: 3.12', 94 | 'Programming Language :: Python :: 3.13', 95 | 'Topic :: Software Development', 96 | ], 97 | test_suite='tests', 98 | tests_require=test_requirements, 99 | setup_requires=setup_requirements, 100 | # Extensions 101 | ext_modules=[ 102 | Extension( 103 | "pymemtrace.custom", 104 | sources=[ 105 | 'pymemtrace/src/cpy/cCustom.c', 106 | ], 107 | include_dirs=[ 108 | '/usr/local/include', 109 | os.path.join('pymemtrace', 'src', 'include'), 110 | ], 111 | library_dirs=[os.getcwd(), ], # path to .a or .so file(s) 112 | extra_compile_args=extra_compile_args, 113 | ), 114 | Extension( 115 | "pymemtrace.cPyMemTrace", 116 | sources=[ 117 | 'pymemtrace/src/c/get_rss.c', 118 | 'pymemtrace/src/c/pymemtrace_util.c', 119 | 'pymemtrace/src/cpy/cPyMemTrace.c', 120 | ], 121 | include_dirs=[ 122 | '/usr/local/include', 123 | os.path.join('pymemtrace', 'src', 'include'), 124 | ], 125 | library_dirs=[os.getcwd(), ], # path to .a or .so file(s) 126 | extra_compile_args=extra_compile_args, 127 | ), 128 | Extension( 129 | "pymemtrace.cMemLeak", 130 | sources=[ 131 | 'pymemtrace/src/cpy/cMemLeak.c', 132 | ], 133 | include_dirs=[ 134 | '/usr/local/include', 135 | os.path.join('pymemtrace', 'src', 'include'), 136 | ], 137 | library_dirs=[os.getcwd(), ], # path to .a or .so file(s) 138 | extra_compile_args=extra_compile_args, 139 | ), 140 | ] 141 | ) 142 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Unit test package for pymemtrace.""" 4 | -------------------------------------------------------------------------------- /tests/_test_redirect_stdout.py: -------------------------------------------------------------------------------- 1 | import ctypes 2 | import io 3 | import sys 4 | 5 | import pytest 6 | 7 | from pymemtrace import redirect_stdout 8 | 9 | 10 | def test_stdout_redirector_python_bytes(): 11 | stream = io.BytesIO() 12 | with redirect_stdout.stdout_redirector(stream): 13 | sys.stdout.write('Foo') 14 | result = stream.getvalue() 15 | assert result == b'Foo', f'Result: {result!r}' 16 | 17 | 18 | # def test_stdout_redirector_python_text(): 19 | # stream = io.StringIO() 20 | # with redirect_stdout.stdout_redirector(stream): 21 | # sys.stdout.write('Foo') 22 | # result = stream.getvalue() 23 | # assert result == b'Foo', f'Result: {result!r}' 24 | 25 | 26 | def test_stdout_redirector_c_bytes(): 27 | libc = ctypes.CDLL(None) 28 | stream = io.BytesIO() 29 | with redirect_stdout.stdout_redirector(stream): 30 | libc.puts(b'This comes from C') 31 | result = stream.getvalue() 32 | assert result == b'This comes from C\n', f'Result: {result!r}' 33 | 34 | 35 | def test_stdout_redirector_python_c_bytes(): 36 | libc = ctypes.CDLL(None) 37 | stream = io.BytesIO() 38 | with redirect_stdout.stdout_redirector(stream): 39 | print('foobar') 40 | sys.stdout.write('sys.stdout.write()\n') 41 | libc.puts(b'This comes from C') 42 | result = stream.getvalue() 43 | assert result == b'This comes from C\nfoobar\nsys.stdout.write()\n', f'Result: {result!r}' 44 | 45 | 46 | def test_stdout_redirector_python_c_bytes_flush(): 47 | libc = ctypes.CDLL(None) 48 | stream = io.BytesIO() 49 | with redirect_stdout.stdout_redirector(stream): 50 | print('foobar') 51 | sys.stdout.write('sys.stdout.write()\n') 52 | sys.stdout.flush() 53 | libc.puts(b'This comes from C') 54 | result = stream.getvalue() 55 | assert result == b'foobar\nsys.stdout.write()\nThis comes from C\n', f'Result: {result!r}' 56 | 57 | 58 | if __name__ == '__main__': 59 | test_stdout_redirector_python_bytes() 60 | # test_stdout_redirector_python_text() 61 | test_stdout_redirector_c_bytes() 62 | test_stdout_redirector_python_c_bytes() 63 | test_stdout_redirector_python_c_bytes_flush() 64 | -------------------------------------------------------------------------------- /tests/test_cMemLeak.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import pytest 4 | 5 | from pymemtrace import cMemLeak 6 | 7 | 8 | def test_cmalloc_object(): 9 | cobj = cMemLeak.CMalloc(1024) 10 | assert cobj.size == 1024 11 | -------------------------------------------------------------------------------- /tests/test_process.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import io 3 | import pprint 4 | 5 | import pytest 6 | 7 | 8 | from pymemtrace import process 9 | 10 | 11 | @pytest.mark.parametrize( 12 | 'line, expected', 13 | ( 14 | ( 15 | '2019-10-14 17:44:46,955 - 24098 - INFO - process.py - ProcessLoggingThread-JSON-START {"timestamp": "2019-10-14 17:44:46.955519"}', 16 | ('-START', '{"timestamp": "2019-10-14 17:44:46.955519"}'), 17 | ), 18 | ( 19 | '2019-10-14 17:44:46,955 - 24098 - INFO - process.py - ProcessLoggingThread-JSON {"timestamp": "2019-10-14 17:44:46.955519"}', 20 | (None, '{"timestamp": "2019-10-14 17:44:46.955519"}'), 21 | ), 22 | ( 23 | '2019-10-14 17:44:46,955 - 24098 - INFO - process.py - ProcessLoggingThread-JSON-STOP {"timestamp": "2019-10-14 17:44:46.955519"}', 24 | ('-STOP', '{"timestamp": "2019-10-14 17:44:46.955519"}'), 25 | ), 26 | ) 27 | ) 28 | def test_re_log_line(line, expected): 29 | match = process.RE_LOG_LINE.match(line) 30 | assert match is not None 31 | assert match.groups() == expected 32 | 33 | 34 | EXAMPLE_PROCESS_LOG = """Cmd: /Users/engun/venvs/TotalDepth37_00/bin/tdrp66v1scanhtml -r --frame-slice=1/64 --log-process=1.0 data/by_type/RP66V1/WAPIMS/2006-2008/W002844/ data/HTML/W002844_I 35 | gnuplot version: "b'gnuplot 5.2 patchlevel 6'" 36 | args: Namespace(encrypted=False, frame_slice='1/64', gnuplot=None, keep_going=False, log_level=20, log_process=1.0, path_in='data/by_type/RP66V1/WAPIMS/2006-2008/W002844/', path_out='data/HTML/W002844_I', recurse=True, verbose=0) 37 | 2019-10-14 17:44:46,955 - 24098 - INFO - process.py - ProcessLoggingThread-JSON-START {"pid": 24098, "timestamp": "2019-10-14 17:44:46.955519", "memory_info": {"rss": 28475392, "vms": 4595617792, "pfaults": 10272, "pageins": 0}, "cpu_times": {"user": 0.3174768, "system": 0.0577991, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 0.23358798027038574} 38 | 2019-10-14 17:44:46,955 - 24098 - INFO - ScanHTML.py - scan_dir_or_file(): "data/by_type/RP66V1/WAPIMS/2006-2008/W002844" to "data/HTML/W002844_I" recurse: True 39 | 2019-10-14 17:44:46,963 - 24098 - INFO - ScanHTML.py - ScanFileHTML.scan_a_single_file(): "data/by_type/RP66V1/WAPIMS/2006-2008/W002844/WIRELINE/S1R2_CMR_MDT-GR/MDT_OFA_CMR_083PTP.DLIS" 40 | 2019-10-14 17:44:47,960 - 24098 - INFO - process.py - ProcessLoggingThread-JSON {"pid": 24098, "timestamp": "2019-10-14 17:44:47.960414", "memory_info": {"rss": 55967744, "vms": 4621426688, "pfaults": 18940, "pageins": 8}, "cpu_times": {"user": 1.291334912, "system": 0.07683248, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 1.2385711669921875} 41 | 2019-10-14 17:44:48,943 - 24098 - INFO - ScanHTML.py - ScanFileHTML.scan_a_single_file(): "data/by_type/RP66V1/WAPIMS/2006-2008/W002844/WIRELINE/S1R2_CMR_MDT-GR/MDT_OFA_CMR_077PTP.DLIS" 42 | 2019-10-14 17:44:48,964 - 24098 - INFO - process.py - ProcessLoggingThread-JSON {"pid": 24098, "timestamp": "2019-10-14 17:44:48.963983", "memory_info": {"rss": 41500672, "vms": 4606386176, "pfaults": 19427, "pageins": 61}, "cpu_times": {"user": 2.2686848, "system": 0.087920424, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 2.242030143737793} 43 | 2019-10-14 17:44:50,013 - 24098 - INFO - process.py - ProcessLoggingThread-JSON {"pid": 24098, "timestamp": "2019-10-14 17:44:50.012988", "memory_info": {"rss": 56074240, "vms": 4620902400, "pfaults": 23014, "pageins": 61}, "cpu_times": {"user": 3.30531712, "system": 0.099596592, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 3.2910971641540527} 44 | 2019-10-14 17:44:50,878 - 24098 - INFO - ScanHTML.py - ScanFileHTML.scan_a_single_file(): "data/by_type/RP66V1/WAPIMS/2006-2008/W002844/WIRELINE/S1R2_CMR_MDT-GR/MDT_OFA_CMR_082PTP.DLIS" 45 | 2019-10-14 17:44:51,019 - 24098 - INFO - process.py - ProcessLoggingThread-JSON {"pid": 24098, "timestamp": "2019-10-14 17:44:51.019315", "memory_info": {"rss": 46026752, "vms": 4610617344, "pfaults": 23347, "pageins": 61}, "cpu_times": {"user": 4.29572096, "system": 0.108004144, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 4.2973692417144775} 46 | 2019-10-14 17:44:52,024 - 24098 - INFO - process.py - ProcessLoggingThread-JSON-STOP {"pid": 24098, "timestamp": "2019-10-14 17:44:52.024755", "memory_info": {"rss": 56565760, "vms": 4621070336, "pfaults": 25993, "pageins": 61}, "cpu_times": {"user": 5.269735424, "system": 0.116896328, "children_user": 0.0, "children_system": 0.0}, "elapsed_time": 5.3028600215911865} 47 | Other log lines 48 | that will be ignored. 49 | """ 50 | 51 | EXPECTED_EXAMPLE_PROCESS_LOG = [ 52 | {'cpu_times': {'children_system': 0.0, 53 | 'children_user': 0.0, 54 | 'system': 0.0577991, 55 | 'user': 0.3174768}, 56 | 'elapsed_time': 0.23358798027038574, 57 | 'memory_info': {'pageins': 0, 58 | 'pfaults': 10272, 59 | 'rss': 28475392, 60 | 'vms': 4595617792}, 61 | 'pid': 24098, 62 | 'timestamp': datetime.datetime(2019, 10, 14, 17, 44, 46, 955519)}, 63 | {'cpu_times': {'children_system': 0.0, 64 | 'children_user': 0.0, 65 | 'system': 0.07683248, 66 | 'user': 1.291334912}, 67 | 'elapsed_time': 1.2385711669921875, 68 | 'memory_info': {'pageins': 8, 69 | 'pfaults': 18940, 70 | 'rss': 55967744, 71 | 'vms': 4621426688}, 72 | 'pid': 24098, 73 | 'timestamp': datetime.datetime(2019, 10, 14, 17, 44, 47, 960414)}, 74 | {'cpu_times': {'children_system': 0.0, 75 | 'children_user': 0.0, 76 | 'system': 0.087920424, 77 | 'user': 2.2686848}, 78 | 'elapsed_time': 2.242030143737793, 79 | 'memory_info': {'pageins': 61, 80 | 'pfaults': 19427, 81 | 'rss': 41500672, 82 | 'vms': 4606386176}, 83 | 'pid': 24098, 84 | 'timestamp': datetime.datetime(2019, 10, 14, 17, 44, 48, 963983)}, 85 | {'cpu_times': {'children_system': 0.0, 86 | 'children_user': 0.0, 87 | 'system': 0.099596592, 88 | 'user': 3.30531712}, 89 | 'elapsed_time': 3.2910971641540527, 90 | 'memory_info': {'pageins': 61, 91 | 'pfaults': 23014, 92 | 'rss': 56074240, 93 | 'vms': 4620902400}, 94 | 'pid': 24098, 95 | 'timestamp': datetime.datetime(2019, 10, 14, 17, 44, 50, 12988)}, 96 | {'cpu_times': {'children_system': 0.0, 97 | 'children_user': 0.0, 98 | 'system': 0.108004144, 99 | 'user': 4.29572096}, 100 | 'elapsed_time': 4.2973692417144775, 101 | 'memory_info': {'pageins': 61, 102 | 'pfaults': 23347, 103 | 'rss': 46026752, 104 | 'vms': 4610617344}, 105 | 'pid': 24098, 106 | 'timestamp': datetime.datetime(2019, 10, 14, 17, 44, 51, 19315)}, 107 | {'cpu_times': {'children_system': 0.0, 108 | 'children_user': 0.0, 109 | 'system': 0.116896328, 110 | 'user': 5.269735424}, 111 | 'elapsed_time': 5.3028600215911865, 112 | 'memory_info': {'pageins': 61, 113 | 'pfaults': 25993, 114 | 'rss': 56565760, 115 | 'vms': 4621070336}, 116 | 'pid': 24098, 117 | 'timestamp': datetime.datetime(2019, 10, 14, 17, 44, 52, 24755)} 118 | ] 119 | 120 | 121 | def test_extract_json(): 122 | istream = io.StringIO(EXAMPLE_PROCESS_LOG) 123 | result = process.extract_json(istream) 124 | # pprint.pprint(result) 125 | assert result == EXPECTED_EXAMPLE_PROCESS_LOG 126 | 127 | 128 | def test_extract_json_as_table(): 129 | istream = io.StringIO(EXAMPLE_PROCESS_LOG) 130 | json_result = process.extract_json(istream) 131 | table, t_min, t_max, rss_min, rss_max = process.extract_json_as_table(json_result) 132 | # pprint.pprint(table) 133 | result = '\n'.join(' '.join(row) for row in table[24098]) 134 | # print(result) 135 | expected = """#t(s) RSS PageFaults/s User Mean_CPU% Inst_CPU% Timestamp PID Label 136 | 0.2 28475392 43974.865437 0.3 135.9% 135.9% 2019-10-14T17:44:46.955519 24098 # 137 | 1.2 55967744 8625.019915 1.3 104.3% 96.9% 2019-10-14T17:44:47.960414 24098 # 138 | 2.2 41500672 485.321285 2.3 101.2% 97.4% 2019-10-14T17:44:48.963983 24098 # 139 | 3.3 56074240 3419.228639 3.3 100.4% 98.8% 2019-10-14T17:44:50.012988 24098 # 140 | 4.3 46026752 330.924416 4.3 100.0% 98.4% 2019-10-14T17:44:51.019315 24098 # 141 | 5.3 56565760 2631.550734 5.3 99.4% 96.9% 2019-10-14T17:44:52.024755 24098 # """ 142 | assert result == expected 143 | assert t_min == {24098: 0.23358798027038574} 144 | assert t_max == {24098: 5.3028600215911865} 145 | assert rss_min == {24098: 28475392} 146 | assert rss_max == {24098: 56565760} 147 | -------------------------------------------------------------------------------- /tests/test_trace_malloc.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from pymemtrace import trace_malloc 4 | 5 | 6 | def test_trace_malloc_simple(): 7 | list_of_strings = [] 8 | # with trace_malloc.TraceMalloc('filename') as tm: 9 | with trace_malloc.TraceMalloc('lineno') as tm: 10 | list_of_strings.append(' ' * 1024) 11 | # print() 12 | # print(f'tm.memory_start={tm.memory_start}') 13 | # print(f'tm.memory_finish={tm.memory_finish}') 14 | # print(f'tm.net_statistics(): {tm.net_statistics()}') 15 | # for stat in tm.net_statistics(): 16 | # print(stat) 17 | # assert tm.diff == 0, f'tm.diff={tm.diff}' 18 | # Different versions of Python will have a different number of entries. 19 | assert len(tm.net_statistics()) in (3, 4, 5, 6,) 20 | assert len(tm.statistics) > 3 21 | 22 | 23 | if __name__ == '__main__': 24 | test_trace_malloc_simple() 25 | -------------------------------------------------------------------------------- /toolkit/py_flow_malloc_free.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Zs 2 | /* 3 | * py_flow_malloc.d - Python libc malloc analysis. Written for the Python DTrace provider. 4 | * 5 | * This reports the Python call stack and along the way any calls to system malloc() or free(). 6 | * It also reports aggregate memory allocation by Python function. 7 | * 8 | * It requires a Python build with at least (Mac OSX): 9 | * ../configure --with-dtrace --with-openssl=$(brew --prefix openssl) 10 | * 11 | * This will build a 'release' version of Python with pymalloc, the small memory allocator for memory <=512 bytes. 12 | * 13 | * For a 'debug' version that tracks all memory allocations build with something like: 14 | * ../configure --with-pydebug --without-pymalloc --with-valgrind --with-dtrace --with-openssl=$(brew --prefix openssl) 15 | * 16 | * USAGE (-C is to invoke the C preprocessor on this script): 17 | * sudo dtrace -C -s toolkit/py_flow_malloc_free.d -p 18 | * 19 | * Or for full path names: 20 | * sudo dtrace -C -s toolkit/py_flow_malloc_free.d -D FULL_FILE_PATH -p 21 | * 22 | * Use -D PYTHON_CALL_STACK if you want the Python call stack (verbose). 23 | * 24 | * Copyright (c) 2020 Paul Ross. 25 | * Acknowledgments to py_malloc.d which is Copyright (c) 2007 Brendan Gregg. 26 | * 27 | */ 28 | 29 | #pragma D option quiet 30 | //#pragma D option switchrate=10 31 | 32 | self int depth; 33 | 34 | dtrace:::BEGIN 35 | { 36 | printf("dtrace:::BEGIN\n"); 37 | #ifdef PYTHON_CALL_STACK 38 | printf("%s %6s %10s %16s:%-4s %-8s -- %s\n", "C", "PID", "DELTA(us)", 39 | "FILE", "LINE", "TYPE", "FUNC"); 40 | #endif 41 | } 42 | 43 | python$target:::function-entry 44 | { 45 | #ifdef PYTHON_CALL_STACK 46 | printf("%6d %16s:%-4d CALL %*s-> %s\n", pid, 47 | #ifdef FULL_FILE_PATH 48 | copyinstr(arg0), 49 | #else 50 | basename(copyinstr(arg0)), 51 | #endif 52 | arg2, 53 | self->depth * 2, "", 54 | copyinstr(arg1)); 55 | self->depth++; 56 | #endif 57 | 58 | #ifdef FULL_FILE_PATH 59 | self->file = copyinstr(arg0); 60 | #else 61 | self->file = basename(copyinstr(arg0)); 62 | #endif 63 | self->name = copyinstr(arg1); 64 | self->line = arg2; 65 | } 66 | 67 | python$target:::function-return 68 | { 69 | #ifdef PYTHON_CALL_STACK 70 | self->depth -= self->depth > 0 ? 1 : 0; 71 | printf("%6d %16s:%-4d RTN %*s<- %s\n", pid, 72 | #ifdef FULL_FILE_PATH 73 | copyinstr(arg0), 74 | #else 75 | basename(copyinstr(arg0)), 76 | #endif 77 | arg2, 78 | self->depth * 2, "", copyinstr(arg1)); 79 | #endif 80 | self->file = 0; 81 | self->name = 0; 82 | self->line = 0; 83 | } 84 | 85 | python$target:::line 86 | { 87 | #ifdef FULL_FILE_PATH 88 | self->file = copyinstr(arg0); 89 | #else 90 | self->file = basename(copyinstr(arg0)); 91 | #endif 92 | self->name = copyinstr(arg1); 93 | self->line = arg2; 94 | } 95 | 96 | pid$target::malloc:entry 97 | /self->file != NULL/ 98 | { 99 | /* So this is slightly not well understood. It seems that self-file and self-> name do not persist to 100 | * pid$target::malloc:return 101 | * They are often null or truncated in some way. 102 | * 103 | * Instead we report them here but without the terminating '\n' then pid$target::malloc:return can add the pointer 104 | * value onto the end of the line and terminate it. 105 | * 106 | * It seems to work in practice. 107 | */ 108 | 109 | /* 110 | * arg0 is the buffer size to allocate. 111 | */ 112 | printf("%6d %16s:%-4d -> %s malloc(%d)", pid, self->file, self->line, self->name, arg0); 113 | 114 | @malloc_func_size[self->file, self->name] = sum(arg0); 115 | @malloc_func_dist[self->file, self->name] = quantize(arg0); 116 | } 117 | 118 | pid$target::malloc:return 119 | /self->file != NULL/ 120 | { 121 | /* 122 | * arg0 is the program counter. 123 | * arg1 is the buffer pointer that has been allocated. 124 | */ 125 | printf(" pntr 0x%x\n", arg1); 126 | } 127 | 128 | pid$target::malloc:entry 129 | /self->name == NULL/ 130 | { 131 | @malloc_lib_size[usym(ucaller)] = sum(arg0); 132 | @malloc_lib_dist[usym(ucaller)] = quantize(arg0); 133 | } 134 | 135 | pid$target::free:entry 136 | /self->file != NULL/ 137 | { 138 | /* 139 | * arg0 is the address to free. 140 | */ 141 | printf("%6d %16s:%-4d -> %s free(0x%x)\n", pid, self->file, self->line, self->name, arg0); 142 | } 143 | 144 | dtrace:::END 145 | { 146 | printf("\ndtrace:::END\n"); 147 | printf("Python malloc byte distributions by engine caller:\n"); 148 | printa(" %A, total bytes = %@d %@d\n", @malloc_lib_size, @malloc_lib_dist); 149 | printf("\nPython malloc byte distributions by Python file and function:\n\n"); 150 | printa(" %s, %s, bytes total = %@d %@d\n", @malloc_func_size, @malloc_func_dist); 151 | } 152 | -------------------------------------------------------------------------------- /toolkit/py_flowinfo.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Zs 2 | /* 3 | * py_flowinfo.d - snoop Python function flow with info using DTrace. 4 | * Written for the Python DTrace provider. 5 | * 6 | * $Id: py_flowinfo.d 41 2007-09-17 02:20:10Z brendan $ 7 | * 8 | * This traces activity from all Python programs on the system that are 9 | * running with Python provider support. 10 | * 11 | * USAGE: py_flowinfo.d # hit Ctrl-C to end 12 | * 13 | * FIELDS: 14 | * C CPU-id 15 | * PID Process ID 16 | * DELTA(us) Elapsed time from previous line to this line 17 | * FILE Filename of the Python program 18 | * LINE Line number of filename 19 | * TYPE Type of call (func) 20 | * FUNC Python function 21 | * 22 | * LEGEND: 23 | * -> function entry 24 | * <- function return 25 | * 26 | * Filename and function names are printed if available. 27 | * 28 | * WARNING: Watch the first column carefully, it prints the CPU-id. If it 29 | * changes, then it is very likely that the output has been shuffled. 30 | * 31 | * COPYRIGHT: Copyright (c) 2007 Brendan Gregg. 32 | * 33 | * CDDL HEADER START 34 | * 35 | * The contents of this file are subject to the terms of the 36 | * Common Development and Distribution License, Version 1.0 only 37 | * (the "License"). You may not use this file except in compliance 38 | * with the License. 39 | * 40 | * You can obtain a copy of the license at Docs/cddl1.txt 41 | * or http://www.opensolaris.org/os/licensing. 42 | * See the License for the specific language governing permissions 43 | * and limitations under the License. 44 | * 45 | * CDDL HEADER END 46 | * 47 | * 09-Sep-2007 Brendan Gregg Created this. 48 | */ 49 | 50 | 51 | #pragma D option quiet 52 | #pragma D option switchrate=10 53 | 54 | self int depth; 55 | 56 | dtrace:::BEGIN 57 | { 58 | printf("%s %6s %10s %16s:%-4s %-8s -- %s\n", "C", "PID", "DELTA(us)", 59 | "FILE", "LINE", "TYPE", "FUNC"); 60 | } 61 | 62 | python*:::function-entry, 63 | python*:::function-return 64 | /self->last == 0/ 65 | { 66 | self->last = timestamp; 67 | } 68 | 69 | python*:::function-entry 70 | { 71 | this->delta = (timestamp - self->last) / 1000; 72 | printf("%d %6d %10d %16s:%-4d %-8s %*s-> %s\n", cpu, pid, this->delta, 73 | basename(copyinstr(arg0)), arg2, "func", self->depth * 2, "", 74 | copyinstr(arg1)); 75 | self->depth++; 76 | self->last = timestamp; 77 | } 78 | 79 | python*:::function-return 80 | { 81 | this->delta = (timestamp - self->last) / 1000; 82 | self->depth -= self->depth > 0 ? 1 : 0; 83 | printf("%d %6d %10d %16s:%-4d %-8s %*s<- %s\n", cpu, pid, this->delta, 84 | basename(copyinstr(arg0)), arg2, "func", self->depth * 2, "", 85 | copyinstr(arg1)); 86 | self->last = timestamp; 87 | } 88 | -------------------------------------------------------------------------------- /toolkit/py_flowinfo_rss.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Zs 2 | /* 3 | * py_flowinfo.d - snoop Python function flow with info using DTrace. 4 | * Written for the Python DTrace provider. 5 | * 6 | * $Id: py_flowinfo.d 41 2007-09-17 02:20:10Z brendan $ 7 | * 8 | * This traces activity from all Python programs on the system that are 9 | * running with Python provider support. 10 | * 11 | * USAGE: py_flowinfo.d # hit Ctrl-C to end 12 | * 13 | * FIELDS: 14 | * C CPU-id 15 | * PID Process ID 16 | * DELTA(us) Elapsed time from previous line to this line 17 | * FILE Filename of the Python program 18 | * LINE Line number of filename 19 | * TYPE Type of call (func) 20 | * FUNC Python function 21 | * 22 | * LEGEND: 23 | * -> function entry 24 | * <- function return 25 | * 26 | * Filename and function names are printed if available. 27 | * 28 | * WARNING: Watch the first column carefully, it prints the CPU-id. If it 29 | * changes, then it is very likely that the output has been shuffled. 30 | * 31 | * COPYRIGHT: Copyright (c) 2007 Brendan Gregg. 32 | * 33 | * CDDL HEADER START 34 | * 35 | * The contents of this file are subject to the terms of the 36 | * Common Development and Distribution License, Version 1.0 only 37 | * (the "License"). You may not use this file except in compliance 38 | * with the License. 39 | * 40 | * You can obtain a copy of the license at Docs/cddl1.txt 41 | * or http://www.opensolaris.org/os/licensing. 42 | * See the License for the specific language governing permissions 43 | * and limitations under the License. 44 | * 45 | * CDDL HEADER END 46 | * 47 | * 09-Sep-2007 Brendan Gregg Created this. 48 | */ 49 | 50 | #pragma D option quiet 51 | #pragma D option destructive 52 | #pragma D option switchrate=10 53 | 54 | self int depth; 55 | 56 | dtrace:::BEGIN 57 | { 58 | printf("%s %6s %10s %16s:%-4s %-8s -- %s\n", "C", "PID", "DELTA(us)", 59 | "FILE", "LINE", "TYPE", "FUNC"); 60 | } 61 | 62 | python*:::function-entry, 63 | python*:::function-return 64 | /self->last == 0/ 65 | { 66 | self->last = timestamp; 67 | } 68 | 69 | python*:::function-entry 70 | { 71 | this->delta = (timestamp - self->last) / 1000; 72 | printf("%d %6d %10d %16s:%-4d %-8s %*s-> %s\n", cpu, pid, this->delta, 73 | basename(copyinstr(arg0)), arg2, "func", self->depth * 2, "", 74 | copyinstr(arg1)); 75 | self->depth++; 76 | self->last = timestamp; 77 | printf("PS: "); 78 | system("ps -p %d -o rss=", pid); 79 | printf("\n"); 80 | } 81 | 82 | python*:::function-return 83 | { 84 | this->delta = (timestamp - self->last) / 1000; 85 | self->depth -= self->depth > 0 ? 1 : 0; 86 | printf("%d %6d %10d %16s:%-4d %-8s %*s<- %s\n", cpu, pid, this->delta, 87 | basename(copyinstr(arg0)), arg2, "func", self->depth * 2, "", 88 | copyinstr(arg1)); 89 | self->last = timestamp; 90 | } 91 | -------------------------------------------------------------------------------- /toolkit/py_malloc.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Zs 2 | /* 3 | * py_malloc.d - Python libc malloc analysis. 4 | * Written for the Python DTrace provider. 5 | * 6 | * $Id: py_malloc.d 19 2007-09-12 07:47:59Z brendan $ 7 | * 8 | * This is an expiremental script to identify who is calling malloc() for 9 | * memory allocation, and to print distribution plots of the requested bytes. 10 | * If a malloc() occured while in a Python function, then that function is 11 | * identified as responsible; else the caller of malloc() is identified as 12 | * responsible - which will be a function from the Python engine. 13 | * 14 | * USAGE: py_malloc.d { -p PID | -c cmd } # hit Ctrl-C to end 15 | * 16 | * Filename and function names are printed if available. 17 | * 18 | * COPYRIGHT: Copyright (c) 2007 Brendan Gregg. 19 | * 20 | * CDDL HEADER START 21 | * 22 | * The contents of this file are subject to the terms of the 23 | * Common Development and Distribution License, Version 1.0 only 24 | * (the "License"). You may not use this file except in compliance 25 | * with the License. 26 | * 27 | * You can obtain a copy of the license at Docs/cddl1.txt 28 | * or http://www.opensolaris.org/os/licensing. 29 | * See the License for the specific language governing permissions 30 | * and limitations under the License. 31 | * 32 | * CDDL HEADER END 33 | * 34 | * 09-Sep-2007 Brendan Gregg Created this. 35 | */ 36 | 37 | #pragma D option quiet 38 | 39 | dtrace:::BEGIN 40 | { 41 | printf("Tracing... Hit Ctrl-C to end.\n"); 42 | } 43 | 44 | python$target:::function-entry 45 | { 46 | self->file = basename(copyinstr(arg0)); 47 | self->name = copyinstr(arg1); 48 | } 49 | 50 | python$target:::function-return 51 | { 52 | self->file = 0; 53 | self->name = 0; 54 | } 55 | 56 | pid$target::malloc:entry 57 | /self->file != NULL/ 58 | { 59 | @malloc_func_size[self->file, self->name] = sum(arg0); 60 | @malloc_func_dist[self->file, self->name] = quantize(arg0); 61 | } 62 | 63 | pid$target::malloc:entry 64 | /self->name == NULL/ 65 | { 66 | @malloc_lib_size[usym(ucaller)] = sum(arg0); 67 | @malloc_lib_dist[usym(ucaller)] = quantize(arg0); 68 | } 69 | 70 | 71 | dtrace:::END 72 | { 73 | printf("\nPython malloc byte distributions by engine caller,\n\n"); 74 | printa(" %A, total bytes = %@d %@d\n", @malloc_lib_size, 75 | @malloc_lib_dist); 76 | 77 | printf("\nPython malloc byte distributions by Python file and "); 78 | printf("function,\n\n"); 79 | printa(" %s, %s, bytes total = %@d %@d\n", @malloc_func_size, 80 | @malloc_func_dist); 81 | } 82 | -------------------------------------------------------------------------------- /toolkit/py_object_D_WITH_PYMALLOC.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Zs 2 | /* 3 | * toolkit/py_object_D_WITH_PYMALLOC.d - Python libc malloc analysis. 4 | * Written for the Python DTrace provider. 5 | * This is for release builds of Python that have WITH_PYMALLOC defined. 6 | * 7 | * This reports the Python call stack and along the way any calls to system malloc() or free(). 8 | * It also reports aggregate memory allocation by Python function. 9 | * 10 | * It requires a Python build with at least (Mac OSX): 11 | * ../configure --with-dtrace --with-openssl=$(brew --prefix openssl) 12 | * 13 | * This will build a 'release' version of Python with pymalloc, the small memory allocator for memory <=512 bytes. 14 | * 15 | * USAGE (-C is to invoke the C preprocessor on this script): 16 | * sudo dtrace -C -s toolkit/py_object_D_WITH_PYMALLOC.d -p 17 | * 18 | * Or for full path names: 19 | * sudo dtrace -C -s toolkit/py_object_D_WITH_PYMALLOC.d -D FULL_FILE_PATH -p 20 | * 21 | * Use -D PYTHON_CALL_STACK if you want the Python call stack (verbose). 22 | * 23 | * From Objects/obmalloc.c: 24 | * 25 | * #define MALLOC_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree} 26 | * #ifdef WITH_PYMALLOC 27 | * # define PYMALLOC_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free} 28 | * #endif 29 | * 30 | * #define PYRAW_ALLOC MALLOC_ALLOC 31 | * #ifdef WITH_PYMALLOC 32 | * # define PYOBJ_ALLOC PYMALLOC_ALLOC 33 | * #else 34 | * # define PYOBJ_ALLOC MALLOC_ALLOC 35 | * #endif 36 | * #define PYMEM_ALLOC PYOBJ_ALLOC 37 | * 38 | * 39 | * Copyright (c) 2020 Paul Ross. 40 | * Acknowledgments to py_malloc.d which is Copyright (c) 2007 Brendan Gregg. 41 | * 42 | */ 43 | 44 | #pragma D option quiet 45 | //#pragma D option switchrate=10 46 | 47 | self int depth; 48 | 49 | dtrace:::BEGIN 50 | { 51 | printf("dtrace:::BEGIN\n"); 52 | #ifdef PYTHON_CALL_STACK 53 | printf("%s %6s %10s %16s:%-4s %-8s -- %s\n", "C", "PID", "DELTA(us)", 54 | "FILE", "LINE", "TYPE", "FUNC"); 55 | #endif 56 | } 57 | 58 | python$target:::function-entry 59 | { 60 | #ifdef PYTHON_CALL_STACK 61 | printf("%6d %16s:%-4d CALL %*s-> %s\n", pid, 62 | #ifdef FULL_FILE_PATH 63 | copyinstr(arg0), 64 | #else 65 | basename(copyinstr(arg0)), 66 | #endif 67 | arg2, 68 | self->depth * 2, "", 69 | copyinstr(arg1)); 70 | self->depth++; 71 | #endif 72 | 73 | #ifdef FULL_FILE_PATH 74 | self->file = copyinstr(arg0); 75 | #else 76 | self->file = basename(copyinstr(arg0)); 77 | #endif 78 | self->name = copyinstr(arg1); 79 | self->line = arg2; 80 | } 81 | 82 | python$target:::function-return 83 | { 84 | #ifdef PYTHON_CALL_STACK 85 | self->depth -= self->depth > 0 ? 1 : 0; 86 | printf("%6d %16s:%-4d RTN %*s<- %s\n", pid, 87 | #ifdef FULL_FILE_PATH 88 | copyinstr(arg0), 89 | #else 90 | basename(copyinstr(arg0)), 91 | #endif 92 | arg2, 93 | self->depth * 2, "", copyinstr(arg1)); 94 | #endif 95 | self->file = 0; 96 | self->name = 0; 97 | self->line = 0; 98 | } 99 | 100 | python$target:::line 101 | { 102 | #ifdef FULL_FILE_PATH 103 | self->file = copyinstr(arg0); 104 | #else 105 | self->file = basename(copyinstr(arg0)); 106 | #endif 107 | self->name = copyinstr(arg1); 108 | self->line = arg2; 109 | } 110 | 111 | /* 112 | /self->file != NULL/ 113 | */ 114 | 115 | /* For pymalloc calls of: _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free */ 116 | 117 | pid$target::_PyObject_Malloc:entry 118 | /self->file != NULL/ 119 | { 120 | /* arg1 is the buffer size to allocate. */ 121 | printf("%6d %16s:%-4d -> %s _PyObject_Malloc(%d)", pid, self->file, self->line, self->name, arg1); 122 | } 123 | 124 | pid$target::_PyObject_Malloc:return 125 | /self->file != NULL/ 126 | { 127 | /* arg0 is the PC, arg1 is the buffer location */ 128 | printf(" _PyObject_Malloc returns 0x%x\n", arg1); 129 | } 130 | 131 | pid$target::_PyObject_Calloc:entry 132 | /self->file != NULL/ 133 | { 134 | /* arg1 is the number of elements, arg2 is the element size to allocate. */ 135 | printf("%6d %16s:%-4d -> %s _PyObject_Calloc(%d, %d)", pid, self->file, self->line, self->name, arg1, arg2); 136 | } 137 | 138 | pid$target::_PyObject_Calloc:return 139 | /self->file != NULL/ 140 | { 141 | /* arg0 is the PC, arg1 is the buffer location */ 142 | printf(" _PyObject_Calloc returns 0x%x\n", arg1); 143 | } 144 | 145 | pid$target::_PyObject_Realloc:entry 146 | /self->file != NULL/ 147 | { 148 | /* arg1 is the existing buffer, arg2 is the buffer size. */ 149 | printf("%6d %16s:%-4d -> %s _PyObject_Realloc(0x%x, %d)\n", pid, self->file, self->line, self->name, arg1, arg2); 150 | } 151 | 152 | #if 0 153 | /* Probe not available. */ 154 | pid$target::_PyObject_Realloc:return 155 | { 156 | /* arg0 is the PC, arg1 is the buffer location */ 157 | printf(" _PyObject_Realloc returns 0x%x\n", arg1); 158 | } 159 | #endif 160 | 161 | pid$target::_PyObject_Free:entry 162 | /self->file != NULL/ 163 | { 164 | /* arg1 is the existing buffer. */ 165 | printf("%6d %16s:%-4d -> %s _PyObject_Free(0x%x)\n", pid, self->file, self->line, self->name, arg1); 166 | } 167 | 168 | 169 | dtrace:::END 170 | { 171 | printf("\ndtrace:::END\n"); 172 | } 173 | -------------------------------------------------------------------------------- /toolkit/py_object_U_WITH_PYMALLOC.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Zs 2 | /* 3 | * toolkit/py_object_U_WITH_PYMALLOC.d - Python libc malloc analysis. 4 | * Written for the Python DTrace provider. 5 | * This is for debug builds of Python that have WITH_PYMALLOC not defined. 6 | * 7 | * This reports the Python call stack and along the way any calls to system malloc() or free(). 8 | * It also reports aggregate memory allocation by Python function. 9 | * 10 | * It requires a Python build with at least (Mac OSX) that tracks all memory allocations build with something like: 11 | * ../configure --with-pydebug --without-pymalloc --with-valgrind --with-dtrace --with-openssl=$(brew --prefix openssl) 12 | * 13 | * USAGE (-C is to invoke the C preprocessor on this script): 14 | * sudo dtrace -C -s toolkit/py_object_U_WITH_PYMALLOC.d -p 15 | * 16 | * Or for full path names: 17 | * sudo dtrace -C -s toolkit/py_object_U_WITH_PYMALLOC.d -D FULL_FILE_PATH -p 18 | * 19 | * Use -D PYTHON_CALL_STACK if you want the Python call stack (verbose). 20 | * 21 | * From Objects/obmalloc.c: 22 | * 23 | * #define MALLOC_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree} 24 | * #ifdef WITH_PYMALLOC 25 | * # define PYMALLOC_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free} 26 | * #endif 27 | * 28 | * #define PYRAW_ALLOC MALLOC_ALLOC 29 | * #ifdef WITH_PYMALLOC 30 | * # define PYOBJ_ALLOC PYMALLOC_ALLOC 31 | * #else 32 | * # define PYOBJ_ALLOC MALLOC_ALLOC 33 | * #endif 34 | * #define PYMEM_ALLOC PYOBJ_ALLOC 35 | * 36 | * 37 | * Copyright (c) 2020 Paul Ross. 38 | * Acknowledgments to py_malloc.d which is Copyright (c) 2007 Brendan Gregg. 39 | * 40 | */ 41 | 42 | #pragma D option quiet 43 | //#pragma D option switchrate=10 44 | 45 | self int depth; 46 | 47 | dtrace:::BEGIN 48 | { 49 | printf("dtrace:::BEGIN\n"); 50 | #ifdef PYTHON_CALL_STACK 51 | printf("%s %6s %10s %16s:%-4s %-8s -- %s\n", "C", "PID", "DELTA(us)", 52 | "FILE", "LINE", "TYPE", "FUNC"); 53 | #endif 54 | } 55 | 56 | python$target:::function-entry 57 | { 58 | #ifdef PYTHON_CALL_STACK 59 | printf("%6d %16s:%-4d CALL %*s-> %s\n", pid, 60 | #ifdef FULL_FILE_PATH 61 | copyinstr(arg0), 62 | #else 63 | basename(copyinstr(arg0)), 64 | #endif 65 | arg2, 66 | self->depth * 2, "", 67 | copyinstr(arg1)); 68 | self->depth++; 69 | #endif 70 | 71 | #ifdef FULL_FILE_PATH 72 | self->file = copyinstr(arg0); 73 | #else 74 | self->file = basename(copyinstr(arg0)); 75 | #endif 76 | self->name = copyinstr(arg1); 77 | self->line = arg2; 78 | } 79 | 80 | python$target:::function-return 81 | { 82 | #ifdef PYTHON_CALL_STACK 83 | self->depth -= self->depth > 0 ? 1 : 0; 84 | printf("%6d %16s:%-4d RTN %*s<- %s\n", pid, 85 | #ifdef FULL_FILE_PATH 86 | copyinstr(arg0), 87 | #else 88 | basename(copyinstr(arg0)), 89 | #endif 90 | arg2, 91 | self->depth * 2, "", copyinstr(arg1)); 92 | #endif 93 | self->file = 0; 94 | self->name = 0; 95 | self->line = 0; 96 | } 97 | 98 | python$target:::line 99 | { 100 | #ifdef FULL_FILE_PATH 101 | self->file = copyinstr(arg0); 102 | #else 103 | self->file = basename(copyinstr(arg0)); 104 | #endif 105 | self->name = copyinstr(arg1); 106 | self->line = arg2; 107 | } 108 | 109 | /* 110 | /self->file != NULL/ 111 | */ 112 | 113 | /* For pymalloc calls of: _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree */ 114 | 115 | pid$target::_PyMem_RawMalloc:entry 116 | /self->file != NULL/ 117 | { 118 | /* arg1 is the buffer size to allocate. */ 119 | printf("%6d %16s:%-4d -> %s _PyMem_RawMalloc(%d)", pid, self->file, self->line, self->name, arg1); 120 | } 121 | 122 | pid$target::_PyMem_RawMalloc:return 123 | /self->file != NULL/ 124 | { 125 | /* arg0 is the PC, arg1 is the buffer location */ 126 | printf(" _PyMem_RawMalloc returns 0x%x\n", arg1); 127 | } 128 | 129 | pid$target::_PyMem_RawCalloc:entry 130 | /self->file != NULL/ 131 | { 132 | /* arg1 is the number of elements, arg2 is the element size to allocate. */ 133 | printf("%6d %16s:%-4d -> %s _PyMem_RawCalloc(%d, %d)", pid, self->file, self->line, self->name, arg1, arg2); 134 | } 135 | 136 | pid$target::_PyMem_RawCalloc:return 137 | /self->file != NULL/ 138 | { 139 | /* arg0 is the PC, arg1 is the buffer location */ 140 | printf(" _PyMem_RawCalloc returns 0x%x\n", arg1); 141 | } 142 | 143 | pid$target::_PyMem_RawRealloc:entry 144 | /self->file != NULL/ 145 | { 146 | /* arg1 is the existing buffer, arg2 is the buffer size. */ 147 | printf("%6d %16s:%-4d -> %s _PyMem_RawRealloc(0x%x, %d)", pid, self->file, self->line, self->name, arg1, arg2); 148 | } 149 | 150 | pid$target::_PyMem_RawRealloc:return 151 | { 152 | /* arg0 is the PC, arg1 is the buffer location */ 153 | printf(" _PyMem_RawRealloc returns 0x%x\n", arg1); 154 | } 155 | 156 | pid$target::_PyMem_RawFree:entry 157 | /self->file != NULL/ 158 | { 159 | /* arg1 is the existing buffer. */ 160 | printf("%6d %16s:%-4d -> %s _PyMem_RawFree(0x%x)\n", pid, self->file, self->line, self->name, arg1); 161 | } 162 | 163 | 164 | dtrace:::END 165 | { 166 | printf("\ndtrace:::END\n"); 167 | } 168 | -------------------------------------------------------------------------------- /toolkit/py_syscolors.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Zs 2 | /* 3 | * py_syscolors.d - trace Python function flow plus syscalls, in color. 4 | * Written for the Python DTrace provider. 5 | * 6 | * $Id: py_syscolors.d 27 2007-09-13 09:26:01Z brendan $ 7 | * 8 | * USAGE: py_syscolors.d { -p PID | -c cmd } # hit Ctrl-C to end 9 | * 10 | * This watches Python function entries and returns, and indents child 11 | * function calls. 12 | * 13 | * FIELDS: 14 | * C CPU-id 15 | * PID Process ID 16 | * DELTA(us) Elapsed time from previous line to this line 17 | * FILE Filename of the Python program 18 | * LINE Line number of filename 19 | * TYPE Type of call (func/syscall) 20 | * NAME Python function or syscall name 21 | * 22 | * Filename and function names are printed if available. 23 | * 24 | * WARNING: Watch the first column carefully, it prints the CPU-id. If it 25 | * changes, then it is very likely that the output has been shuffled. 26 | * 27 | * COPYRIGHT: Copyright (c) 2007 Brendan Gregg. 28 | * 29 | * CDDL HEADER START 30 | * 31 | * The contents of this file are subject to the terms of the 32 | * Common Development and Distribution License, Version 1.0 only 33 | * (the "License"). You may not use this file except in compliance 34 | * with the License. 35 | * 36 | * You can obtain a copy of the license at Docs/cddl1.txt 37 | * or http://www.opensolaris.org/os/licensing. 38 | * See the License for the specific language governing permissions 39 | * and limitations under the License. 40 | * 41 | * CDDL HEADER END 42 | * 43 | * 09-Sep-2007 Brendan Gregg Created this. 44 | */ 45 | 46 | #pragma D option quiet 47 | #pragma D option switchrate=10 48 | 49 | self int depth; 50 | 51 | dtrace:::BEGIN 52 | { 53 | color_python = "\033[2;35m"; /* violet, faint */ 54 | color_syscall = "\033[2;32m"; /* green, faint */ 55 | color_off = "\033[0m"; /* default */ 56 | 57 | self->depth = 0; 58 | printf("%s %6s %10s %16s:%-4s %-8s -- %s\n", "C", "PID", "DELTA(us)", 59 | "FILE", "LINE", "TYPE", "NAME"); 60 | } 61 | 62 | python$target:::function-entry, 63 | python$target:::function-return, 64 | syscall:::entry, 65 | syscall:::return 66 | /self->last == 0 && pid == $target/ 67 | { 68 | self->last = timestamp; 69 | } 70 | 71 | python$target:::function-entry 72 | { 73 | this->delta = (timestamp - self->last) / 1000; 74 | printf("%s%d %6d %10d %16s:%-4d %-8s %*s-> %s%s\n", color_python, 75 | cpu, pid, this->delta, basename(copyinstr(arg0)), arg2, "func", 76 | self->depth * 2, "", copyinstr(arg1), color_off); 77 | self->depth++; 78 | self->last = timestamp; 79 | } 80 | 81 | python$target:::function-return 82 | { 83 | this->delta = (timestamp - self->last) / 1000; 84 | this->name = strjoin(strjoin(copyinstr(arg0), "::"), copyinstr(arg1)); 85 | self->depth -= self->depth > 0 ? 1 : 0; 86 | printf("%s%d %6d %10d %16s:%-4d %-8s %*s<- %s%s\n", color_python, 87 | cpu, pid, this->delta, basename(copyinstr(arg0)), arg2, "func", 88 | self->depth * 2, "", copyinstr(arg1), color_off); 89 | self->last = timestamp; 90 | } 91 | 92 | syscall:::entry 93 | /pid == $target/ 94 | { 95 | this->delta = (timestamp - self->last) / 1000; 96 | printf("%s%d %6d %10d %16s:- %-8s %*s-> %s%s\n", color_syscall, 97 | cpu, pid, this->delta, "\"", "syscall", self->depth * 2, "", 98 | probefunc, color_off); 99 | self->last = timestamp; 100 | } 101 | 102 | syscall:::return 103 | /pid == $target/ 104 | { 105 | this->delta = (timestamp - self->last) / 1000; 106 | printf("%s%d %6d %10d %16s:- %-8s %*s<- %s%s\n", color_syscall, 107 | cpu, pid, this->delta, "\"", "syscall", self->depth * 2, "", 108 | probefunc, color_off); 109 | self->last = timestamp; 110 | } 111 | 112 | /* 113 | proc:::exit 114 | /pid == $target/ 115 | { 116 | exit(0); 117 | } 118 | */ 119 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py26, py27, py33, py34, py35, flake8 3 | 4 | [travis] 5 | python = 6 | 3.5: py35 7 | 3.4: py34 8 | 3.3: py33 9 | 2.7: py27 10 | 2.6: py26 11 | 12 | [testenv:flake8] 13 | basepython=python 14 | deps=flake8 15 | commands=flake8 pymemtrace 16 | 17 | [testenv] 18 | setenv = 19 | PYTHONPATH = {toxinidir} 20 | deps = 21 | -r{toxinidir}/requirements_dev.txt 22 | commands = 23 | pip install -U pip 24 | py.test --basetemp={envtmpdir} 25 | 26 | 27 | ; If you want to make tox run the tests with the same versions, create a 28 | ; requirements.txt with the pinned versions and uncomment the following lines: 29 | ; deps = 30 | ; -r{toxinidir}/requirements.txt 31 | -------------------------------------------------------------------------------- /travis_pypi_setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """Update encrypted deploy password in Travis config file.""" 4 | 5 | 6 | from __future__ import print_function 7 | import base64 8 | import json 9 | import os 10 | from getpass import getpass 11 | import yaml 12 | from cryptography.hazmat.primitives.serialization import load_pem_public_key 13 | from cryptography.hazmat.backends import default_backend 14 | from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 15 | 16 | 17 | try: 18 | from urllib import urlopen 19 | except ImportError: 20 | from urllib.request import urlopen 21 | 22 | 23 | GITHUB_REPO = 'paulross/pymemtrace' 24 | TRAVIS_CONFIG_FILE = os.path.join( 25 | os.path.dirname(os.path.abspath(__file__)), '.travis.yml') 26 | 27 | 28 | def load_key(pubkey): 29 | """Load public RSA key. 30 | 31 | Work around keys with incorrect header/footer format. 32 | 33 | Read more about RSA encryption with cryptography: 34 | https://cryptography.io/latest/hazmat/primitives/asymmetric/rsa/ 35 | """ 36 | try: 37 | return load_pem_public_key(pubkey.encode(), default_backend()) 38 | except ValueError: 39 | # workaround for https://github.com/travis-ci/travis-api/issues/196 40 | pubkey = pubkey.replace('BEGIN RSA', 'BEGIN').replace('END RSA', 'END') 41 | return load_pem_public_key(pubkey.encode(), default_backend()) 42 | 43 | 44 | def encrypt(pubkey, password): 45 | """Encrypt password using given RSA public key and encode it with base64. 46 | 47 | The encrypted password can only be decrypted by someone with the 48 | private key (in this case, only Travis). 49 | """ 50 | key = load_key(pubkey) 51 | encrypted_password = key.encrypt(password, PKCS1v15()) 52 | return base64.b64encode(encrypted_password) 53 | 54 | 55 | def fetch_public_key(repo): 56 | """Download RSA public key Travis will use for this repo. 57 | 58 | Travis API docs: http://docs.travis-ci.com/api/#repository-keys 59 | """ 60 | keyurl = 'https://api.travis-ci.org/repos/{0}/key'.format(repo) 61 | data = json.loads(urlopen(keyurl).read().decode()) 62 | if 'key' not in data: 63 | errmsg = "Could not find public key for repo: {}.\n".format(repo) 64 | errmsg += "Have you already added your GitHub repo to Travis?" 65 | raise ValueError(errmsg) 66 | return data['key'] 67 | 68 | 69 | def prepend_line(filepath, line): 70 | """Rewrite a file adding a line to its beginning.""" 71 | with open(filepath) as f: 72 | lines = f.readlines() 73 | 74 | lines.insert(0, line) 75 | 76 | with open(filepath, 'w') as f: 77 | f.writelines(lines) 78 | 79 | 80 | def load_yaml_config(filepath): 81 | """Load yaml config file at the given path.""" 82 | with open(filepath) as f: 83 | return yaml.load(f) 84 | 85 | 86 | def save_yaml_config(filepath, config): 87 | """Save yaml config file at the given path.""" 88 | with open(filepath, 'w') as f: 89 | yaml.dump(config, f, default_flow_style=False) 90 | 91 | 92 | def update_travis_deploy_password(encrypted_password): 93 | """Put `encrypted_password` into the deploy section of .travis.yml.""" 94 | config = load_yaml_config(TRAVIS_CONFIG_FILE) 95 | 96 | config['deploy']['password'] = dict(secure=encrypted_password) 97 | 98 | save_yaml_config(TRAVIS_CONFIG_FILE, config) 99 | 100 | line = ('# This file was autogenerated and will overwrite' 101 | ' each time you run travis_pypi_setup.py\n') 102 | prepend_line(TRAVIS_CONFIG_FILE, line) 103 | 104 | 105 | def main(args): 106 | """Add a PyPI password to .travis.yml so that Travis can deploy to PyPI. 107 | 108 | Fetch the Travis public key for the repo, and encrypt the PyPI password 109 | with it before adding, so that only Travis can decrypt and use the PyPI 110 | password. 111 | """ 112 | public_key = fetch_public_key(args.repo) 113 | password = args.password or getpass('PyPI password: ') 114 | update_travis_deploy_password(encrypt(public_key, password.encode())) 115 | print("Wrote encrypted password to .travis.yml -- you're ready to deploy") 116 | 117 | 118 | if '__main__' == __name__: 119 | import argparse 120 | parser = argparse.ArgumentParser(description=__doc__) 121 | parser.add_argument('--repo', default=GITHUB_REPO, 122 | help='GitHub repo (default: %s)' % GITHUB_REPO) 123 | parser.add_argument('--password', 124 | help='PyPI password (will prompt if not provided)') 125 | 126 | args = parser.parse_args() 127 | main(args) 128 | --------------------------------------------------------------------------------