├── .github └── workflows │ └── ci.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── examples ├── crash.ipynb ├── crash.py └── specific.py ├── pyproject.toml ├── setup.cfg ├── setup.py ├── src └── debuglater │ ├── __init__.py │ ├── cli.py │ ├── ipython.py │ └── pydump.py └── tests ├── conftest.py ├── test_dump.py └── test_save_dump.py /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | 8 | runs-on: ubuntu-latest 9 | strategy: 10 | matrix: 11 | python-version: [3.7, 3.8, 3.9, '3.10'] 12 | 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: Set up Python ${{ matrix.python-version }} 16 | uses: actions/setup-python@v2 17 | with: 18 | python-version: ${{ matrix.python-version }} 19 | - name: Install dependencies 20 | run: | 21 | python -m pip install --upgrade pip 22 | pip install . 23 | # check the package is importable 24 | python -c "import debuglater" 25 | pip install ".[dev]" 26 | - name: Lint 27 | run: | 28 | python -m pip install --upgrade pkgmt 29 | pkgmt lint 30 | 31 | - name: Test with pytest 32 | run: | 33 | pytest 34 | 35 | readme-test: 36 | 37 | runs-on: ubuntu-latest 38 | strategy: 39 | matrix: 40 | python-version: ['3.10'] 41 | 42 | steps: 43 | - uses: actions/checkout@v2 44 | - name: Set up Python ${{ matrix.python-version }} 45 | uses: actions/setup-python@v2 46 | with: 47 | python-version: ${{ matrix.python-version }} 48 | - name: Install jupyblog 49 | run: | 50 | pip install --upgrade pip 51 | pip install pkgmt nbclient jupytext ipykernel 52 | pip install '.[all]' 53 | - name: Test readme 54 | run: | 55 | pkgmt test-md --file README.md 56 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | *.py[cod] 3 | *.dump 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Packages 9 | *.egg 10 | *.egg-info 11 | dist 12 | build 13 | eggs 14 | parts 15 | bin 16 | var 17 | sdist 18 | develop-eggs 19 | .installed.cfg 20 | lib 21 | lib64 22 | MANIFEST 23 | 24 | # Installer logs 25 | pip-log.txt 26 | 27 | # Unit test / coverage reports 28 | .coverage 29 | .tox 30 | nosetests.xml 31 | 32 | # Translations 33 | *.mo 34 | 35 | # Mr Developer 36 | .mr.developer.cfg 37 | .project 38 | .pydevproject 39 | 40 | # Backup files 41 | *~ 42 | *.bak -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | 3 | ## 1.4.5dev 4 | 5 | ## 1.4.4 (2023-01-07) 6 | * `save_dump` creates all parent directories if they don't exist 7 | 8 | ## 1.4.3 (2022-08-22) 9 | * Adds `debug_dump` as top-level import 10 | 11 | ## 1.4.2 (2022-08-14) 12 | * Adds `debuglater.excepthook_factory` 13 | 14 | ## 1.4.1 (2022-08-03) 15 | * Updates warning message when missing `dill` 16 | 17 | ## 1.4 (2022-08-03) 18 | * Fixes error when serializing traceback objects in Python 3.10 with `pickle` 19 | 20 | ## 1.3.3 (2022-08-02) 21 | * Adds current path to `sys.path` when loading dump 22 | * Adds `echo` argument to `run` function to disable print output 23 | 24 | ## 1.3.2 (2022-07-31) 25 | * Makes `dill` an optional dependency 26 | * Prints message if missing dill 27 | * Adds argument to select the path to dump file when patching IPython 28 | 29 | ## 1.3.1 (2022-07-22) 30 | * Fixed output message 31 | * Adds color to output message 32 | 33 | ## 1.3 (2022-07-22) 34 | 35 | *First version released by Ploomber* 36 | 37 | * Renames package 38 | * Adds support for new versions of Python 3 39 | * Adds integration with IPython 40 | 41 | ## 1.2.0 42 | 43 | * Port to Python 3 44 | 45 | ## 1.1.1 46 | 47 | * Fixed a few small bugs. 48 | 49 | ## 1.1.0 50 | 51 | * Now storing built-in datatypes and custom class data members instead of their string representations. 52 | 53 | ## 1.0.0 54 | 55 | * First public version 56 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (C) 2012 Eli Finer 4 | Copyright (c) 2022-Present Ploomber Inc. 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # `debuglater`: Store Python traceback for later debugging 3 | 4 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) 5 | 6 |

7 | Community 8 | | 9 | Newsletter 10 | | 11 | Twitter 12 | | 13 | LinkedIn 14 | | 15 | Blog 16 | | 17 | Website 18 | | 19 | YouTube 20 |

21 | 22 | > [!TIP] 23 | > Deploy AI apps for free on [Ploomber Cloud!](https://ploomber.io/?utm_medium=github&utm_source=debuglater) 24 | 25 | - `debuglater` writes the traceback object so you can use it later for debugging 26 | - Works with `pdb`, `pudb`, `ipdb` and `pdbpp` 27 | - You can use it to debug on a different machine, no need to have access to the source code 28 | 29 | For support, feature requests, and product updates: [join our community](https://ploomber.io/community), subscribe to our [newsletter](https://share.hsforms.com/1E7Qa_OpcRPi_MV-segFsaAe6c2g) or follow us on [Twitter](https://twitter.com/ploomber)/[LinkedIn](https://www.linkedin.com/company/ploomber/). 30 | 31 | ![demo](https://ploomber.io/images/doc/debuglater-demo/debug.gif) 32 | 33 | 34 | [Click here to tell your friends on Twitter!](https://twitter.com/intent/tweet?text=I%20just%20discovered%20debuglater%20on%20GitHub%3A%20serialize%20Python%20tracebacks%20for%20later%20debugging%21%20%F0%9F%A4%AF&url=https://github.com/ploomber/debuglater/) 35 | 36 | [Click here to tell your friends on LinkedIn!](https://www.linkedin.com/sharing/share-offsite/?url=https://github.com/ploomber/debuglater/) 37 | 38 | ## Installation 39 | 40 | ```sh 41 | 42 | pip install debuglater 43 | 44 | # for better serialization support (via dill) 45 | pip install 'debuglater[all]' 46 | 47 | # ..or with conda 48 | conda install debuglater -c conda-forge 49 | ``` 50 | 51 | ## Usage 52 | 53 | ```python 54 | import sys 55 | import debuglater 56 | 57 | sys.excepthook = debuglater.excepthook_factory(__file__) 58 | ``` 59 | 60 | For more details and alternative usage, keep reading. 61 | 62 | 63 | 64 | ## Example 65 | 66 | ```sh 67 | # get the example 68 | curl -O https://raw.githubusercontent.com/ploomber/debuglater/master/examples/crash.py 69 | ``` 70 | 71 | ```sh tags=["raises-exception"] 72 | # crash 73 | python crash.py 74 | ``` 75 | 76 | 77 | Debug: 78 | 79 | ```sh 80 | dltr crash.dump 81 | ``` 82 | 83 | Upon initialization, try printing the variables `x` and `y`: 84 | 85 | ``` 86 | Starting pdb... 87 | > /Users/ploomber/debuglater/examples/crash.py(5)() 88 | -> x / y 89 | (Pdb) x 90 | 1 91 | (Pdb) y 92 | 0 93 | (Pdb) quit 94 | ``` 95 | 96 | *Note: you can also use:* `debuglater crash.py.dump` 97 | 98 | 99 | 100 | 101 | ## Integration with Jupyter/IPython 102 | 103 | > **Note** 104 | > For an integration with papermill, see [ploomber-engine](https://github.com/ploomber/ploomber-engine) 105 | 106 | Add this at the top of your notebook/script: 107 | 108 | ```python 109 | from debuglater import patch_ipython 110 | patch_ipython() 111 | ``` 112 | 113 | 114 | ```sh 115 | # get sample notebook 116 | curl -O https://raw.githubusercontent.com/ploomber/debuglater/master/examples/crash.ipynb 117 | 118 | # install package to run notebooks 119 | pip install nbclient 120 | ``` 121 | 122 | ```sh tags=["raises-exception"] 123 | # run the notebook 124 | jupyter execute crash.ipynb 125 | ``` 126 | 127 | Debug: 128 | 129 | ``` 130 | dltr jupyter.dump 131 | ``` 132 | 133 | Upon initialization, try printing the variables `x` and `y`: 134 | 135 | ``` 136 | Starting pdb... 137 | -> x / y 138 | (Pdb) x 139 | 1 140 | (Pdb) y 141 | 0 142 | (Pdb) quit 143 | ``` 144 | 145 | 146 | *Note: you can also use:* `debuglater jupyter.dump` 147 | 148 | ## Motivation 149 | 150 | The [Ploomber team](https://github.com/ploomber/ploomber) develops tools for 151 | data analysis. When data analysis code executes non-interactively 152 | (example: a daily cron job that generates a report), it becomes hard to debug 153 | since logs are often insufficient, forcing data practitioners to re-run the 154 | code from scratch, which can take a lot of time. 155 | 156 | However, `debuglater` is a generic tool that can be used for any use case to facilitate post-mortem debugging. 157 | 158 | ## Use cases 159 | 160 | * Debug long-running code (e.g., crashed Machine Learning job) 161 | * Debug multiprocessing code (generate one dump file for each process) 162 | 163 | ## Credits 164 | 165 | This project is a fork of [Eli Finer's pydump](https://github.com/elifiner/pydump). 166 | -------------------------------------------------------------------------------- /examples/crash.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from debuglater import patch_ipython\n", 10 | "\n", 11 | "patch_ipython()" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "x = 1" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "y = 0" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "x / y" 39 | ] 40 | } 41 | ], 42 | "metadata": { 43 | "kernelspec": { 44 | "display_name": "Python 3.9.13 64-bit", 45 | "language": "python", 46 | "name": "python3" 47 | }, 48 | "language_info": { 49 | "name": "python", 50 | "version": "3.9.13" 51 | }, 52 | "orig_nbformat": 4, 53 | "vscode": { 54 | "interpreter": { 55 | "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49" 56 | } 57 | } 58 | }, 59 | "nbformat": 4, 60 | "nbformat_minor": 2 61 | } 62 | -------------------------------------------------------------------------------- /examples/crash.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import debuglater 3 | 4 | sys.excepthook = debuglater.excepthook_factory(__file__) 5 | 6 | x = 1 7 | y = 0 8 | 9 | x / y 10 | -------------------------------------------------------------------------------- /examples/specific.py: -------------------------------------------------------------------------------- 1 | x = 1 2 | y = 0 3 | 4 | try: 5 | x / y 6 | except Exception: 7 | import debuglater 8 | 9 | debuglater.run(__file__) 10 | raise 11 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.pytest.ini_options] 2 | addopts = "--pdbcls=IPython.terminal.debugger:Pdb" 3 | 4 | [tool.pkgmt] 5 | github = "ploomber/debuglater" 6 | package_name = "debuglater" -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = build/ 3 | max-line-length = 88 4 | extend-ignore = E203 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import re 2 | import ast 3 | from glob import glob 4 | from os.path import basename, splitext 5 | 6 | from setuptools import find_packages 7 | from setuptools import setup 8 | 9 | _version_re = re.compile(r"__version__\s+=\s+(.*)") 10 | 11 | with open("src/debuglater/__init__.py", "rb") as f: 12 | VERSION = str( 13 | ast.literal_eval(_version_re.search(f.read().decode("utf-8")).group(1)) 14 | ) 15 | 16 | DESCRIPTION = """ 17 | debuglater allows post-mortem debugging for Python programs. 18 | 19 | It writes the traceback of an exception into a file and can later load 20 | it in a Python debugger. 21 | 22 | Works with the built-in pdb and with other popular debuggers 23 | (pudb, ipdb and pdbpp). 24 | """ 25 | 26 | # requirements 27 | REQUIRES = [ 28 | "colorama", 29 | ] 30 | 31 | # optional requirements 32 | ALL = [ 33 | "dill", 34 | ] 35 | 36 | # only needed for development 37 | DEV = [ 38 | "pytest", 39 | "yapf", 40 | "flake8", 41 | "invoke", 42 | "twine", 43 | "pkgmt", 44 | # for tests 45 | "pandas", 46 | "numpy", 47 | ] 48 | 49 | setup( 50 | name="debuglater", 51 | version=VERSION, 52 | description="Post-mortem debugging for Python programs", 53 | long_description=DESCRIPTION, 54 | author="Ploomber", 55 | license="MIT", 56 | author_email="contact@plooomber.io", 57 | url="https://github.com/ploomber/debuglater", 58 | packages=find_packages("src"), 59 | package_dir={"": "src"}, 60 | py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], 61 | extras_require={ 62 | "dev": DEV + ALL, 63 | "all": ALL, 64 | }, 65 | entry_points={ 66 | "console_scripts": [ 67 | "debuglater=debuglater.cli:main", 68 | "dltr=debuglater.cli:main", 69 | ], 70 | }, 71 | classifiers=[ 72 | "Development Status :: 4 - Beta", 73 | "Environment :: Console", 74 | "Intended Audience :: Developers", 75 | "License :: OSI Approved :: MIT License", 76 | "Natural Language :: English", 77 | "Operating System :: OS Independent", 78 | "Programming Language :: Python", 79 | "Topic :: Software Development :: Debuggers", 80 | ], 81 | install_requires=REQUIRES, 82 | ) 83 | -------------------------------------------------------------------------------- /src/debuglater/__init__.py: -------------------------------------------------------------------------------- 1 | from debuglater.pydump import run, excepthook_factory, debug_dump 2 | from debuglater.ipython import patch_ipython 3 | 4 | __all__ = ["run", "excepthook_factory", "patch_ipython", "debug_dump"] 5 | 6 | __version__ = "1.4.5dev" 7 | -------------------------------------------------------------------------------- /src/debuglater/cli.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from debuglater import __version__ 4 | from debuglater.pydump import debug_dump 5 | 6 | 7 | def main(): 8 | import argparse 9 | 10 | parser = argparse.ArgumentParser( 11 | description=( 12 | f"debuglater {__version__}: " "post-mortem debugging for Python programs" 13 | ) 14 | ) 15 | debugger_group = parser.add_mutually_exclusive_group(required=False) 16 | debugger_group.add_argument( 17 | "--pdb", 18 | action="store_const", 19 | const="pdb", 20 | dest="debugger", 21 | help="Use builtin pdb or pdb++", 22 | ) 23 | debugger_group.add_argument( 24 | "--pudb", 25 | action="store_const", 26 | const="pudb", 27 | dest="debugger", 28 | help="Use pudb visual debugger", 29 | ) 30 | debugger_group.add_argument( 31 | "--ipdb", 32 | action="store_const", 33 | const="ipdb", 34 | dest="debugger", 35 | help="Use ipdb IPython debugger", 36 | ) 37 | parser.add_argument("filename", help="dumped file") 38 | args = parser.parse_args() 39 | if not args.debugger: 40 | args.debugger = "pdb" 41 | 42 | print("Starting %s..." % args.debugger, file=sys.stderr) 43 | dbg = __import__(args.debugger) 44 | return debug_dump(args.filename, dbg.post_mortem) 45 | 46 | 47 | if __name__ == "__main__": 48 | sys.exit(main() or 0) 49 | -------------------------------------------------------------------------------- /src/debuglater/ipython.py: -------------------------------------------------------------------------------- 1 | # ***************************************************************************** 2 | # Copyright (C) 2001 Nathaniel Gray 3 | # Copyright (C) 2001-2004 Fernando Perez 4 | # Copyright (C) 2022 Eduardo Blancas 5 | # 6 | # Distributed under the terms of the BSD License. The full license is in 7 | # the file COPYING, distributed as part of this software. 8 | # ***************************************************************************** 9 | import sys 10 | import types 11 | from functools import partial 12 | 13 | from debuglater.pydump import save_dump 14 | 15 | 16 | def _dump_message(path_to_dump): 17 | return ( 18 | f"Serializing traceback to: {path_to_dump}\n" f"To debug: dltr {path_to_dump}" 19 | ) 20 | 21 | 22 | # NOTE: this is based on the IPython implementation 23 | def debugger(self, force: bool = False, path_to_dump: str = "jupyter.dump"): 24 | # IPython is an optional depdendency 25 | from IPython.core.display_trap import DisplayTrap 26 | 27 | if force or self.call_pdb: 28 | if self.pdb is None: 29 | self.pdb = self.debugger_cls() 30 | # the system displayhook may have changed, restore the original 31 | # for pdb 32 | display_trap = DisplayTrap(hook=sys.__displayhook__) 33 | with display_trap: 34 | self.pdb.reset() 35 | # Find the right frame so we don't pop up inside ipython itself 36 | if hasattr(self, "tb") and self.tb is not None: 37 | etb = self.tb 38 | else: 39 | etb = self.tb = sys.last_traceback 40 | while self.tb is not None and self.tb.tb_next is not None: 41 | assert self.tb.tb_next is not None 42 | self.tb = self.tb.tb_next 43 | if etb and etb.tb_next: 44 | etb = etb.tb_next 45 | self.pdb.botframe = etb.tb_frame 46 | 47 | save_dump(path_to_dump, etb) 48 | # self.pdb.interaction(None, etb) 49 | 50 | if hasattr(self, "tb"): 51 | del self.tb 52 | 53 | 54 | # taken from IPython interactive shell 55 | def _showtraceback_ipython( 56 | self, etype, evalue, stb: str, path_to_dump: str = "jupyter.dump" 57 | ): 58 | val = self.InteractiveTB.stb2text(stb) + "\n" + _dump_message(path_to_dump) 59 | 60 | try: 61 | print(val) 62 | except UnicodeEncodeError: 63 | print(val.encode("utf-8", "backslashreplace").decode()) 64 | 65 | 66 | # taken from ipykernel 67 | # https://github.com/ipython/ipykernel/blob/51a613d501a86073ea1cdbd8023a168646644c6a/ipykernel/zmqshell.py#L530 68 | def _showtraceback_jupyter(self, etype, evalue, stb, path_to_dump): 69 | from ipykernel.jsonutil import json_clean 70 | 71 | # try to preserve ordering of tracebacks and print statements 72 | sys.stdout.flush() 73 | sys.stderr.flush() 74 | 75 | stb = stb + [_dump_message(path_to_dump)] 76 | 77 | exc_content = { 78 | "traceback": stb, 79 | "ename": str(etype.__name__), 80 | "evalue": str(evalue), 81 | } 82 | 83 | dh = self.displayhook 84 | # Send exception info over pub socket for other clients than the caller 85 | # to pick up 86 | topic = None 87 | if dh.topic: 88 | topic = dh.topic.replace(b"execute_result", b"error") 89 | 90 | dh.session.send( 91 | dh.pub_socket, 92 | "error", 93 | json_clean(exc_content), 94 | dh.parent_header, 95 | ident=topic, 96 | ) 97 | 98 | # FIXME - Once we rely on Python 3, the traceback is stored on the 99 | # exception object, so we shouldn't need to store it here. 100 | self._last_traceback = stb 101 | 102 | 103 | def patch_ipython(path_to_dump="jupyter.dump"): 104 | # optional dependency 105 | import IPython 106 | 107 | term = IPython.get_ipython() 108 | term.run_line_magic("pdb", "on") 109 | debugger_ = partial(debugger, path_to_dump=path_to_dump) 110 | term.InteractiveTB.debugger = types.MethodType(debugger_, term.InteractiveTB) 111 | 112 | _showtraceback_ = partial( 113 | _showtraceback_jupyter 114 | if hasattr(term, "_last_traceback") 115 | else _showtraceback_ipython, 116 | path_to_dump=path_to_dump, 117 | ) 118 | term._showtraceback = types.MethodType(_showtraceback_, term) 119 | -------------------------------------------------------------------------------- /src/debuglater/pydump.py: -------------------------------------------------------------------------------- 1 | """ 2 | The MIT License (MIT) 3 | Copyright (C) 2012 Eli Finer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | """ 23 | import builtins 24 | from contextlib import contextmanager 25 | from pathlib import Path 26 | import os 27 | import sys 28 | import pdb 29 | import gzip 30 | import linecache 31 | from traceback import format_exception 32 | 33 | try: 34 | import cPickle as pickle 35 | except ImportError: 36 | import pickle 37 | 38 | from colorama import init 39 | from colorama import Fore, Style 40 | 41 | init() 42 | 43 | try: 44 | import dill 45 | except ImportError: 46 | dill = None 47 | 48 | DUMP_VERSION = 1 49 | 50 | 51 | def _print_not_dill(): 52 | print( 53 | "Using pickle: Only built-in objects will be serialized. " 54 | "To serialize everything: pip install 'debuglater[all]'\n" 55 | ) 56 | 57 | 58 | def save_dump(filename, tb=None): 59 | """ 60 | Saves a Python traceback in a pickled file. This function will usually be 61 | called from an except block to allow post-mortem debugging of a failed 62 | process. 63 | 64 | The saved file can be loaded with load_dump which creates a fake traceback 65 | object that can be passed to any reasonable Python debugger. 66 | """ 67 | if not tb: 68 | tb = sys.exc_info()[2] 69 | fake_tb = FakeTraceback(tb) 70 | _remove_builtins(fake_tb) 71 | 72 | dump = { 73 | "traceback": fake_tb, 74 | "files": _get_traceback_files(fake_tb), 75 | "dump_version": DUMP_VERSION, 76 | } 77 | 78 | Path(filename).parent.mkdir(exist_ok=True, parents=True) 79 | 80 | with gzip.open(filename, "wb") as f: 81 | if dill is not None: 82 | dill.dump(dump, f) 83 | else: 84 | _print_not_dill() 85 | pickle.dump(dump, f, protocol=pickle.HIGHEST_PROTOCOL) 86 | 87 | 88 | def load_dump(filename): 89 | # NOTE: I think we can get rid of this 90 | # ugly hack to handle running non-install debuglater 91 | if "debuglater.pydump" not in sys.modules: 92 | sys.modules["debuglater.pydump"] = sys.modules[__name__] 93 | with gzip.open(filename, "rb") as f: 94 | if dill is not None: 95 | try: 96 | return dill.load(f) 97 | except IOError: 98 | try: 99 | with open(filename, "rb") as f: 100 | return dill.load(f) 101 | except Exception: 102 | pass # dill load failed, try pickle instead 103 | else: 104 | _print_not_dill() 105 | 106 | try: 107 | return pickle.load(f) 108 | except IOError: 109 | with open(filename, "rb") as f: 110 | return pickle.load(f) 111 | 112 | 113 | def debug_dump(dump_filename, post_mortem_func=pdb.post_mortem): 114 | # monkey patching for pdb's longlist command 115 | import inspect 116 | import types 117 | 118 | inspect.isframe = ( 119 | lambda obj: isinstance(obj, types.FrameType) 120 | or obj.__class__.__name__ == "FakeFrame" 121 | ) 122 | inspect.iscode = ( 123 | lambda obj: isinstance(obj, types.CodeType) 124 | or obj.__class__.__name__ == "FakeCode" 125 | ) 126 | inspect.isclass = ( 127 | lambda obj: isinstance(obj, type) or obj.__class__.__name__ == "FakeClass" 128 | ) 129 | inspect.istraceback = ( 130 | lambda obj: isinstance(obj, types.TracebackType) 131 | or obj.__class__.__name__ == "FakeTraceback" 132 | ) 133 | 134 | with add_to_sys_path(".", chdir=False): 135 | dump = load_dump(dump_filename) 136 | 137 | _cache_files(dump["files"]) 138 | tb = dump["traceback"] 139 | _inject_builtins(tb) 140 | _old_checkcache = linecache.checkcache 141 | linecache.checkcache = lambda filename=None: None 142 | post_mortem_func(tb) 143 | linecache.checkcache = _old_checkcache 144 | 145 | 146 | class FakeClass(object): 147 | def __init__(self, repr, vars): 148 | self.__repr = repr 149 | self.__dict__.update(vars) 150 | 151 | def __repr__(self): 152 | return self.__repr 153 | 154 | 155 | class FakeCode(object): 156 | def __init__(self, code): 157 | self.co_filename = os.path.abspath(code.co_filename) 158 | self.co_name = code.co_name 159 | self.co_argcount = code.co_argcount 160 | self.co_consts = tuple( 161 | FakeCode(c) if hasattr(c, "co_filename") else c for c in code.co_consts 162 | ) 163 | self.co_firstlineno = code.co_firstlineno 164 | self.co_lnotab = code.co_lnotab 165 | self.co_varnames = code.co_varnames 166 | self.co_flags = code.co_flags 167 | self.co_code = code.co_code 168 | 169 | # co_lines was introduced in a recent version 170 | if hasattr(code, "co_lines"): 171 | self.co_lines = FakeCoLines(code.co_lines) 172 | 173 | 174 | class FakeCoLines: 175 | def __init__(self, co_lines) -> None: 176 | self._co_lines = list(co_lines()) 177 | 178 | def __call__(self): 179 | return iter(self._co_lines) 180 | 181 | 182 | class FakeFrame(object): 183 | def __init__(self, frame): 184 | self.f_code = FakeCode(frame.f_code) 185 | self.f_locals = _convert_dict(frame.f_locals) 186 | self.f_globals = _convert_dict(frame.f_globals) 187 | self.f_lineno = frame.f_lineno 188 | self.f_back = FakeFrame(frame.f_back) if frame.f_back else None 189 | 190 | if "self" in self.f_locals: 191 | self.f_locals["self"] = _convert_obj(frame.f_locals["self"]) 192 | 193 | 194 | class FakeTraceback(object): 195 | def __init__(self, traceback): 196 | self.tb_frame = FakeFrame(traceback.tb_frame) 197 | self.tb_lineno = traceback.tb_lineno 198 | self.tb_next = FakeTraceback(traceback.tb_next) if traceback.tb_next else None 199 | self.tb_lasti = 0 200 | 201 | 202 | def _remove_builtins(fake_tb): 203 | traceback = fake_tb 204 | while traceback: 205 | frame = traceback.tb_frame 206 | while frame: 207 | frame.f_globals = dict( 208 | (k, v) for k, v in frame.f_globals.items() if k not in dir(builtins) 209 | ) 210 | frame = frame.f_back 211 | traceback = traceback.tb_next 212 | 213 | 214 | def _inject_builtins(fake_tb): 215 | traceback = fake_tb 216 | while traceback: 217 | frame = traceback.tb_frame 218 | while frame: 219 | frame.f_globals.update(builtins.__dict__) 220 | frame = frame.f_back 221 | traceback = traceback.tb_next 222 | 223 | 224 | def _get_traceback_files(traceback): 225 | files = {} 226 | while traceback: 227 | frame = traceback.tb_frame 228 | while frame: 229 | filename = os.path.abspath(frame.f_code.co_filename) 230 | if filename not in files: 231 | try: 232 | files[filename] = open(filename).read() 233 | except IOError: 234 | files[filename] = ( 235 | "couldn't locate '%s' " "during dump" % frame.f_code.co_filename 236 | ) 237 | frame = frame.f_back 238 | traceback = traceback.tb_next 239 | return files 240 | 241 | 242 | def _safe_repr(v): 243 | try: 244 | return repr(v) 245 | except Exception as e: 246 | return "repr error: " + str(e) 247 | 248 | 249 | def _convert_obj(obj): 250 | try: 251 | return FakeClass(_safe_repr(obj), _convert_dict(obj.__dict__)) 252 | except Exception: 253 | return _convert(obj) 254 | 255 | 256 | def _convert_dict(v): 257 | return dict((_convert(k), _convert(i)) for (k, i) in v.items()) 258 | 259 | 260 | def _convert_seq(v): 261 | return (_convert(i) for i in v) 262 | 263 | 264 | def _convert(v): 265 | if dill is not None: 266 | try: 267 | dill.dumps(v) 268 | return v 269 | except Exception: 270 | return _safe_repr(v) 271 | else: 272 | from datetime import date, time, datetime, timedelta 273 | 274 | BUILTIN = (str, int, float, date, time, datetime, timedelta) 275 | # XXX: what about bytes and bytearray? 276 | 277 | if v is None: 278 | return v 279 | 280 | if type(v) in BUILTIN: 281 | return v 282 | 283 | if type(v) is tuple: 284 | return tuple(_convert_seq(v)) 285 | 286 | if type(v) is list: 287 | return list(_convert_seq(v)) 288 | 289 | if type(v) is set: 290 | return set(_convert_seq(v)) 291 | 292 | if type(v) is dict: 293 | return _convert_dict(v) 294 | 295 | return _safe_repr(v) 296 | 297 | 298 | def _cache_files(files): 299 | for name, data in files.items(): 300 | lines = [line + "\n" for line in data.splitlines()] 301 | linecache.cache[name] = (len(data), None, lines, name) 302 | 303 | 304 | def run(filename, echo=True, tb=None): 305 | out = Path(filename).with_suffix(".dump") 306 | 307 | if echo: 308 | print(Fore.RED + f"Exception caught, writing {out}\n") 309 | 310 | save_dump(out, tb=tb) 311 | 312 | if echo: 313 | print(f"To debug, run:\n dltr {out}") 314 | print(Style.RESET_ALL) 315 | 316 | 317 | def excepthook_factory(filename): 318 | def excepthook(type, value, traceback): 319 | print( 320 | "".join(format_exception(type, value, traceback)), file=sys.stderr, end="" 321 | ) 322 | 323 | if type is not KeyboardInterrupt: 324 | run(filename, tb=traceback) 325 | 326 | return excepthook 327 | 328 | 329 | @contextmanager 330 | def add_to_sys_path(path, chdir=False): 331 | cwd_old = os.getcwd() 332 | 333 | if path is not None: 334 | path = os.path.abspath(path) 335 | sys.path.insert(0, path) 336 | 337 | if chdir: 338 | os.chdir(path) 339 | 340 | try: 341 | yield 342 | finally: 343 | if path is not None: 344 | sys.path.remove(path) 345 | os.chdir(cwd_old) 346 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | import pytest 5 | 6 | 7 | @pytest.fixture 8 | def tmp_empty(tmp_path): 9 | """ 10 | Create temporary path using pytest native fixture, 11 | them move it, yield, and restore the original path 12 | """ 13 | old = os.getcwd() 14 | os.chdir(str(tmp_path)) 15 | yield str(Path(tmp_path).resolve()) 16 | os.chdir(old) 17 | -------------------------------------------------------------------------------- /tests/test_dump.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import subprocess 3 | from unittest.mock import Mock 4 | 5 | import numpy as np 6 | import pandas as pd 7 | import pytest 8 | 9 | from debuglater import pydump 10 | 11 | # we assume dill is installed for running these tests 12 | try: 13 | import dill 14 | 15 | dill # to keep flake8 happy 16 | except ModuleNotFoundError: 17 | raise Exception("tests require dill") 18 | 19 | 20 | def foo(): 21 | foovar = 7 # noqa 22 | bar() 23 | 24 | 25 | def bar(): 26 | barvar = "hello" # noqa 27 | list_sample = [1, 2, 3, 4] # noqa 28 | dict_sample = {"a": 1, "b": 2} # noqa 29 | baz() 30 | 31 | 32 | def baz(): 33 | momo = Momo() 34 | momo.raiser() 35 | 36 | 37 | def numpy(): 38 | x = np.array([1, 2, 3]) # noqa 39 | 1 / 0 40 | 41 | 42 | def pandas(): 43 | x = pd.DataFrame({"x": [1, 2, 3]}) # noqa 44 | 1 / 0 45 | 46 | 47 | class Momo(object): 48 | def __init__(self): 49 | self.momodata = "Some data" 50 | 51 | def raiser(self): 52 | x = 1 # noqa 53 | raise Exception("BOOM!") 54 | 55 | 56 | def test_dump(capsys, monkeypatch): 57 | try: 58 | foo() 59 | except Exception: 60 | filename = __file__ + ".dump" 61 | pydump.run(filename) 62 | 63 | mock = Mock(side_effect=['print(f"x is {x}")', "quit"]) 64 | 65 | with monkeypatch.context() as m: 66 | m.setattr("builtins.input", mock) 67 | pydump.debug_dump(filename) 68 | 69 | out, _ = capsys.readouterr() 70 | 71 | assert "x is 1" in out 72 | 73 | 74 | def test_excepthook(capsys, monkeypatch): 75 | if Path("examples.crash.dump").is_file(): 76 | Path("examples.crash.dump").unlink() 77 | 78 | subprocess.run(["python", "examples/crash.py"]) 79 | 80 | mock = Mock(side_effect=['print(f"x is {x}")', "quit"]) 81 | 82 | with monkeypatch.context() as m: 83 | m.setattr("builtins.input", mock) 84 | pydump.debug_dump("examples/crash.dump") 85 | 86 | out, _ = capsys.readouterr() 87 | 88 | assert "x is 1" in out 89 | 90 | 91 | @pytest.mark.parametrize( 92 | "function, result", 93 | [ 94 | [numpy, "type of x is "], 95 | [pandas, "type of x is "], 96 | ], 97 | ) 98 | def test_data_structures(capsys, monkeypatch, function, result): 99 | try: 100 | function() 101 | except Exception: 102 | filename = __file__ + ".dump" 103 | pydump.run(filename) 104 | 105 | mock = Mock(side_effect=['print(f"type of x is {type(x)}")', "quit"]) 106 | 107 | with monkeypatch.context() as m: 108 | m.setattr("builtins.input", mock) 109 | pydump.debug_dump(filename) 110 | 111 | out, _ = capsys.readouterr() 112 | 113 | assert result in out 114 | -------------------------------------------------------------------------------- /tests/test_save_dump.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import pytest 4 | 5 | from debuglater import pydump 6 | 7 | 8 | @pytest.mark.parametrize( 9 | "path", 10 | [ 11 | "name.dump", 12 | "path/to/file.dump", 13 | ], 14 | ) 15 | def test_save_dump_without_dill(tmp_empty, monkeypatch, path): 16 | # simulate dill is not installed 17 | monkeypatch.setattr(pydump, "dill", None) 18 | 19 | try: 20 | 1 / 0 21 | except Exception: 22 | tb = sys.exc_info()[2] 23 | 24 | pydump.save_dump(path, tb=tb) 25 | --------------------------------------------------------------------------------