├── .github └── workflows │ └── python-publish.yml ├── .gitignore ├── LICENSE ├── README.md ├── leaping ├── setup.py └── src │ └── leaping.py ├── pytest-leaping ├── .gitignore ├── LICENSE ├── README.rst ├── __init__.py ├── pyproject.toml ├── src │ ├── __init__.py │ ├── leaping_llm_wrapper.py │ ├── leaping_models.py │ ├── plugin.py │ └── simpletracer.py └── tox.ini ├── requirements.txt └── tracing └── test_file.py /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Upload Python Package 10 | 11 | on: 12 | release: 13 | types: [published] 14 | 15 | permissions: 16 | contents: read 17 | 18 | jobs: 19 | deploy: 20 | 21 | runs-on: ubuntu-latest 22 | 23 | steps: 24 | - uses: actions/checkout@v3 25 | - name: Set up Python 26 | uses: actions/setup-python@v3 27 | with: 28 | python-version: '3.x' 29 | - name: Install dependencies 30 | run: | 31 | python -m pip install --upgrade pip 32 | pip install build 33 | - name: Build package 34 | run: python -m build 35 | - name: Publish package 36 | uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 37 | with: 38 | user: __token__ 39 | password: ${{ secrets.PYPI_API_TOKEN }} 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | __pycache__ 3 | .idea/ 4 | package-lock.json 5 | package.json 6 | .venv 7 | .env 8 | .DS_Store 9 | leaping.egg-info/ 10 | dist/ 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Leaping 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Leaping 2 | 3 | Leaping's pytest debugger is a simple, fast and lightweight debugger for Python tests. Leaping traces the execution of your code 4 | and allows you to retroactively inspect the state of your program at any time, using an LLM-based debugger with natural language. 5 | 6 | It does this by keeping track of all of the variable changes and other sources of non-determinism from within your code. 7 | 8 | # Installation 9 | - ``pip install leaping`` 10 | - Please set the environment variable `OPENAI_API_KEY` to your GPT API key, if you plan on using GPT 11 | 12 | # Usage 13 | `` 14 | pytest --leaping 15 | `` 16 | By default, pytest automatically discovers all the python tests within your project and runs them. Once the test has been run, a CLI will open allowing you 17 | to interact with the debugger. 18 | 19 | When pytest starts up, you will be prompted to select a model. Right now, we support both Ollama and GPT-4. 20 | 21 | # Features 22 | 23 | You can ask Leaping questions like: 24 | - Why am I not hitting function x? 25 | - Why was variable y set to this value? 26 | - What was the value of variable x at this point? 27 | - What changes can I make to this code to make this test pass? 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /leaping/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="leaping", 5 | version="0.1.14", 6 | entry_points={ 7 | 'console_scripts': [ 8 | 'leaping=leaping:main', 9 | ], 10 | }, 11 | python_requires='>=3.0', 12 | install_requires=[ 13 | "pytest-leaping==0.1.14", 14 | "prompt_toolkit==3.0.20", 15 | "openai==1.12.0" 16 | ], 17 | ) 18 | -------------------------------------------------------------------------------- /leaping/src/leaping.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import select 3 | import signal 4 | import sys 5 | from socket import socket, AF_INET, SOCK_STREAM 6 | import threading 7 | import time 8 | 9 | global stop_spinner 10 | stop_spinner = threading.Event() 11 | 12 | 13 | def spinner_animation(message="Loading..."): 14 | spinner_chars = ['|', '/', '-', '\\'] 15 | idx = 0 16 | while not stop_spinner.is_set(): 17 | print(f"\r{message} \U0001F914 {spinner_chars[idx % len(spinner_chars)]}", end='') 18 | idx += 1 19 | time.sleep(0.1) 20 | print('\r', end='') 21 | 22 | def create_spinner(): 23 | global stop_spinner 24 | stop_spinner.clear() 25 | spinner_thread = threading.Thread(target=spinner_animation, args=("Thinking...",)) 26 | spinner_thread.start() 27 | return spinner_thread 28 | 29 | def stop_spinner_animation(spinner_thread): 30 | stop_spinner.set() 31 | spinner_thread.join() 32 | 33 | 34 | def main(): 35 | parser = argparse.ArgumentParser(description="W") 36 | parser.add_argument('-p', '--port', type=int, help='The temporary file generated by pytest-leaping', required=True) 37 | 38 | args = parser.parse_args() 39 | if not args.port: 40 | raise ValueError("Port number not provided. Exiting...") 41 | 42 | global stop_spinner 43 | 44 | print(""" 45 | _ _ 46 | | | ___ __ _ _ __ (_)_ __ __ _ 47 | | | / _ \\/ _` | '_ \\| | '_ \\ / _` | 48 | | |__| __/ (_| | |_) | | | | | (_| | 49 | |_____\\___|\\__,_| .__/|_|_| |_|\\__, | 50 | |_| |___/ 51 | """) 52 | 53 | sock = socket(AF_INET, SOCK_STREAM) 54 | sock.connect(('localhost', args.port)) 55 | 56 | sigint_received = False 57 | 58 | 59 | def signal_handler(sig, frame): 60 | global sigint_received 61 | sigint_received = True 62 | sys.exit(0) 63 | sys.exit(0) 64 | # You can also initiate cleanup here if needed 65 | 66 | spinner = create_spinner() 67 | stop_sent = False 68 | sock.setblocking(False) 69 | signal.signal(signal.SIGINT, signal_handler) 70 | 71 | def receive_output_from_server(): 72 | while not stop_sent and not sigint_received: 73 | # Check if the socket is ready for reading 74 | ready_to_read, _, _ = select.select([sock], [], [], 0.1) 75 | if ready_to_read: 76 | response = sock.recv(2048) 77 | if response: 78 | stop_spinner_animation(spinner) 79 | if response == b"LEAPING_STOP": 80 | break 81 | print(f"\033[61m{response.decode('utf-8')}\033[0m", end="") 82 | 83 | receive_output_from_server() 84 | print("\n") 85 | 86 | while True: 87 | user_input = input("\n If the explanation is wrong, say why and we'll try again. Press q to exit: \n> ") 88 | 89 | if user_input.strip() == "q" or user_input.strip() == "exit": 90 | sock.sendall(b"exit") 91 | break 92 | elif user_input.strip() == "": # Check if the input is just an Enter key press (empty string) 93 | continue # Skip the rest of the loop and prompt again 94 | sock.sendall(user_input.encode("utf-8")) 95 | spinner = create_spinner() 96 | receive_output_from_server() 97 | -------------------------------------------------------------------------------- /pytest-leaping/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | .venv/lib/python3.12/site-packages/pip/_internal/operations/build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | .pytest_cache 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask instance folder 58 | instance/ 59 | 60 | # Sphinx documentation 61 | docs/_build/ 62 | 63 | # MkDocs documentation 64 | /site/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | .idea/ 76 | .pypirc 77 | .venv/ 78 | dist/ 79 | 80 | -------------------------------------------------------------------------------- /pytest-leaping/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "{}" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright {yyyy} {name of copyright owner} 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /pytest-leaping/README.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | pytest-leaping 3 | ============== 4 | 5 | .. image:: https://img.shields.io/pypi/v/pytest-leaping.svg 6 | :target: https://pypi.org/project/pytest-leaping 7 | :alt: PyPI version 8 | 9 | .. image:: https://img.shields.io/pypi/pyversions/pytest-leaping.svg 10 | :target: https://pypi.org/project/pytest-leaping 11 | :alt: Python versions 12 | 13 | .. image:: https://github.com/leaping/pytest-leaping/actions/workflows/main.yml/badge.svg 14 | :target: https://github.com/leaping/pytest-leaping/actions/workflows/main.yml 15 | :alt: See Build Status on GitHub Actions 16 | 17 | A simple plugin to use with pytest 18 | 19 | ---- 20 | 21 | This `pytest`_ plugin was generated with `Cookiecutter`_ along with `@hackebrot`_'s `cookiecutter-pytest-plugin`_ template. 22 | 23 | 24 | Features 25 | -------- 26 | 27 | * TODO 28 | 29 | 30 | Requirements 31 | ------------ 32 | 33 | * TODO 34 | 35 | 36 | Installation 37 | ------------ 38 | 39 | You can install "pytest-leaping" via `pip`_ from `PyPI`_:: 40 | 41 | $ pip install pytest-leaping 42 | 43 | 44 | Usage 45 | ----- 46 | 47 | * TODO 48 | 49 | Contributing 50 | ------------ 51 | Contributions are very welcome. Tests can be run with `tox`_, please ensure 52 | the coverage at least stays the same before you submit a pull request. 53 | 54 | License 55 | ------- 56 | 57 | Distributed under the terms of the `Apache Software License 2.0`_ license, "pytest-leaping" is free and open source software 58 | 59 | 60 | Issues 61 | ------ 62 | 63 | If you encounter any problems, please `file an issue`_ along with a detailed description. 64 | 65 | .. _`Cookiecutter`: https://github.com/audreyr/cookiecutter 66 | .. _`@hackebrot`: https://github.com/hackebrot 67 | .. _`MIT`: https://opensource.org/licenses/MIT 68 | .. _`BSD-3`: https://opensource.org/licenses/BSD-3-Clause 69 | .. _`GNU GPL v3.0`: https://www.gnu.org/licenses/gpl-3.0.txt 70 | .. _`Apache Software License 2.0`: https://www.apache.org/licenses/LICENSE-2.0 71 | .. _`cookiecutter-pytest-plugin`: https://github.com/pytest-dev/cookiecutter-pytest-plugin 72 | .. _`file an issue`: https://github.com/leaping/pytest-leaping/issues 73 | .. _`pytest`: https://github.com/pytest-dev/pytest 74 | .. _`tox`: https://tox.readthedocs.io/en/latest/ 75 | .. _`pip`: https://pypi.org/project/pip/ 76 | .. _`PyPI`: https://pypi.org/project 77 | -------------------------------------------------------------------------------- /pytest-leaping/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leapingio/leaping/cc8c069c81e46ec0330094cdbb427d5e33cbf67a/pytest-leaping/__init__.py -------------------------------------------------------------------------------- /pytest-leaping/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools>=61.0.0", 4 | ] 5 | build-backend = "setuptools.build_meta" 6 | 7 | [project] 8 | name = "pytest-leaping" 9 | description = "A simple plugin to use with pytest" 10 | version = "0.1.14" 11 | readme = "README.rst" 12 | requires-python = ">=3.0" 13 | authors = [ 14 | { name = "Leaping Labs Inc.", email = "founders@leaping.io" }, 15 | ] 16 | maintainers = [ 17 | { name = "Leaping Labs Inc.", email = "founders@leaping.io" }, 18 | ] 19 | license = {file = "LICENSE"} 20 | classifiers = [ 21 | "Framework :: Pytest", 22 | "Development Status :: 4 - Beta", 23 | "Intended Audience :: Developers", 24 | "Topic :: Software Development :: Testing", 25 | "Operating System :: OS Independent", 26 | "Programming Language :: Python", 27 | "Programming Language :: Python :: 3.8", 28 | "Programming Language :: Python :: 3.9", 29 | "Programming Language :: Python :: 3.10", 30 | "Programming Language :: Python :: 3.11", 31 | "Programming Language :: Python :: 3.12", 32 | "Programming Language :: Python :: 3 :: Only", 33 | "Programming Language :: Python :: Implementation :: CPython", 34 | "Programming Language :: Python :: Implementation :: PyPy", 35 | "License :: OSI Approved :: Apache Software License", 36 | ] 37 | dependencies = [ 38 | "pytest>=6.2.0", 39 | "pexpect==4.9.0", 40 | "posthog==3.5.0", 41 | "openai>=1.10.0" 42 | ] 43 | [project.urls] 44 | Repository = "https://github.com/leaping/pytest-leaping" 45 | [project.entry-points.pytest11] 46 | django = "plugin" 47 | 48 | [[tool.poetry.source]] 49 | name = "test-pypi" 50 | url = "https://test.pypi.org/simple/" 51 | -------------------------------------------------------------------------------- /pytest-leaping/src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leapingio/leaping/cc8c069c81e46ec0330094cdbb427d5e33cbf67a/pytest-leaping/src/__init__.py -------------------------------------------------------------------------------- /pytest-leaping/src/leaping_llm_wrapper.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import dataclasses 3 | import os 4 | 5 | import ollama 6 | from openai import OpenAI 7 | 8 | 9 | @dataclasses.dataclass 10 | class LLM(abc.ABC): 11 | model_name: str 12 | messages: list[dict[str, str]] = dataclasses.field(default_factory=list) 13 | 14 | def add_message(self, role, prompt): 15 | self.messages.append({"role": role, "content": prompt}) 16 | 17 | @abc.abstractmethod 18 | def chat_completion(self, stream=False): 19 | pass 20 | 21 | 22 | @dataclasses.dataclass 23 | class Ollama(LLM): 24 | def chat_completion(self, stream=False): 25 | response = ollama.chat( 26 | model='llama2', 27 | messages=self.messages, 28 | stream=stream, 29 | ) 30 | 31 | if stream: 32 | for chunk in response: 33 | yield chunk['message']['content'] 34 | 35 | return response 36 | 37 | 38 | @dataclasses.dataclass 39 | class GPT(LLM): 40 | temperature: float = 0.1 41 | 42 | def __post_init__(self): 43 | api_key = os.getenv("OPENAI_API_KEY") 44 | if not api_key: 45 | raise ValueError("OPENAI_API_KEY environment variable is not set") 46 | self.client = OpenAI(api_key=api_key) 47 | 48 | def chat_completion(self, stream=False): 49 | response = self.client.chat.completions.create( 50 | model=self.model_name, 51 | messages=self.messages, 52 | temperature=self.temperature, 53 | stream=stream, 54 | ) 55 | if stream: 56 | for chunk in response: 57 | if chunk_delta := chunk.choices[0].delta.content: 58 | yield chunk_delta 59 | return 60 | 61 | response_content = response.choices[0].message.content 62 | self.add_message("assistant", response_content) 63 | 64 | return response_content 65 | -------------------------------------------------------------------------------- /pytest-leaping/src/leaping_models.py: -------------------------------------------------------------------------------- 1 | import collections 2 | from copy import copy 3 | 4 | allowed_types = [int, str, float, dict, type(None), bool] 5 | 6 | 7 | def can_trace_type(variable): 8 | current_type = type(variable) 9 | if current_type in allowed_types: 10 | return True 11 | 12 | return False 13 | 14 | class ExecutionCursor: 15 | function_name: str 16 | file: str 17 | line: int 18 | f_locals: list 19 | 20 | def __init__(self, file: str, line: int, function_name: str, f_locals): 21 | self.function_name = function_name 22 | self.file = file 23 | self.line = line 24 | self.f_locals = f_locals 25 | 26 | 27 | class StackFrame: 28 | # parent: StackFrame 29 | def __init__(self, parent, file, line: int, function_name: str, f_locals: list): 30 | self.parent = parent 31 | self.file = file 32 | self.line = line 33 | self.function_name = function_name 34 | self.f_locals = f_locals 35 | 36 | @classmethod 37 | def new(cls, parent, execution_cursor: ExecutionCursor): 38 | return StackFrame(parent, execution_cursor.file, execution_cursor.line, execution_cursor.function_name, execution_cursor.f_locals) 39 | 40 | @classmethod 41 | def clone(cls, origin): 42 | if not origin: 43 | return StackFrame.empty() 44 | return StackFrame(origin.parent, origin.file, origin.line, origin.function_name, origin.f_locals) 45 | 46 | @classmethod 47 | def empty(cls): 48 | return StackFrame(None, None, None, None, None) 49 | 50 | 51 | class CallStack: 52 | def __init__(self): 53 | self.stack = collections.deque() 54 | 55 | def enter_frame(self, execution_cursor: ExecutionCursor): 56 | parent_frame = self.get_parent_frame() 57 | frame = StackFrame.new(parent_frame, execution_cursor) 58 | self.stack.append(frame) 59 | 60 | def get_parent_frame(self): 61 | if len(self.stack) > 0: 62 | return self.stack[-1] 63 | return None 64 | 65 | def new_cursor_in_current_frame(self, new_cursor: ExecutionCursor): 66 | stack_frame: StackFrame = self.top_level_frame_as_clone() 67 | stack_frame.line = new_cursor.line 68 | stack_frame.file = new_cursor.file 69 | stack_frame.function_name = new_cursor.function_name 70 | stack_frame.f_locals = copy(new_cursor.f_locals) 71 | 72 | # line event. Pop top of stack if available and replace with new frame 73 | if len(self.stack) > 0: 74 | self.stack.pop() 75 | self.stack.append(stack_frame) 76 | else: 77 | self.stack.append(stack_frame) 78 | 79 | def exit_frame(self): 80 | self.stack.pop() 81 | 82 | def top_level_frame_as_clone(self): 83 | current: StackFrame = self.current_frame() 84 | return StackFrame.clone(current) 85 | 86 | def current_frame(self): 87 | frame = self.get_parent_frame() 88 | return frame 89 | 90 | 91 | # name and dependencies of variable assignment derived from AST parsing 92 | class ASTAssignment: 93 | def __init__(self, name, deps): 94 | self.name = name 95 | self.deps = deps 96 | 97 | # variable update from sys.settrace 98 | class RuntimeAssignment: 99 | def __init__(self, name, value, path): 100 | self.name = name 101 | self.value = value 102 | self.path = path # path inside python object 103 | 104 | 105 | class VariableAssignmentNode: 106 | def __init__(self, var_name, value, context_line): # todo: deal with object paths 107 | self.var_name = var_name 108 | self.value = value 109 | self.context_line = context_line 110 | 111 | 112 | class FunctionCallNode: 113 | def __init__(self, file_name, func_name, call_args): 114 | self.file_name = file_name 115 | self.func_name = func_name 116 | self.call_args = call_args 117 | self.children = [] 118 | -------------------------------------------------------------------------------- /pytest-leaping/src/plugin.py: -------------------------------------------------------------------------------- 1 | import traceback 2 | from collections import defaultdict 3 | import sys 4 | import threading 5 | from types import CodeType 6 | import datetime 7 | 8 | import pexpect 9 | 10 | from simpletracer import SimpleTracer 11 | import subprocess 12 | from leaping_llm_wrapper import GPT, Ollama, LLM 13 | from _pytest.runner import runtestprotocol 14 | import os 15 | import time 16 | from leaping_models import FunctionCallNode, VariableAssignmentNode 17 | from _pytest.capture import MultiCapture 18 | from posthog import Posthog 19 | 20 | posthog = Posthog(project_api_key='phc_D109xSvTkwTKvxK65CE6TdIxjPoJLXnkVERctUrompz', host='https://app.posthog.com') 21 | global_timestamp = datetime.datetime.now(datetime.UTC).timestamp() 22 | tracer = SimpleTracer() 23 | llm = None 24 | 25 | def pytest_configure(config): 26 | leaping_option = config.getoption('--leaping') 27 | if not leaping_option: 28 | return 29 | 30 | # Somewhere here, say that we're going into the hit our own server mode 31 | 32 | capture_manager = config.pluginmanager.getplugin('capturemanager') # force the -s option 33 | if capture_manager._global_capturing is not None: 34 | capture_manager._global_capturing.pop_outerr_to_orig() 35 | capture_manager._global_capturing.stop_capturing() 36 | capture_manager._global_capturing = MultiCapture(in_=None, out=None, err=None) 37 | 38 | try: 39 | project_dir = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], encoding='utf-8').strip() 40 | except Exception: 41 | project_dir = str(config.rootdir) 42 | 43 | posthog.capture(global_timestamp, 'leaping_started') 44 | while True: 45 | # TODO: fix copy 46 | user_input = input("Please enter the number of the model you'd like to use? \n 1. GPT-4 \n 2. Ollama \n> ") 47 | global llm 48 | if "1" in user_input: 49 | api_key = os.getenv("OPENAI_API_KEY") 50 | if not api_key: 51 | api_key = input("Please enter your OpenAI API key: ") 52 | os.environ["OPENAI_API_KEY"] = api_key # set key as env var 53 | llm = GPT("gpt-4-0125-preview", temperature=0.5) 54 | break 55 | elif "2" in user_input: 56 | llm = Ollama("llama2") 57 | break 58 | else: # invalid input 59 | print("Invalid input. Please enter '1' or '2'.") 60 | 61 | tracer.project_dir = project_dir 62 | 63 | config.addinivalue_line("filterwarnings", "ignore") 64 | 65 | 66 | def pytest_addoption(parser): 67 | group = parser.getgroup('leaping') 68 | group.addoption( 69 | '--leaping', 70 | action='store_true', 71 | default=False, 72 | help='Enable Leaping for failed tests' 73 | ) 74 | 75 | 76 | def pytest_collection_modifyitems(config, items): 77 | if not config.getoption('--leaping'): 78 | return 79 | total_tests_collected = len(items) 80 | setattr(config, 'total_tests_collected', total_tests_collected) 81 | 82 | 83 | def _should_trace(file_name: str, func_name: str) -> bool: 84 | leaping_specific_files = [ 85 | "plugin", 86 | "leaping_models", 87 | "leaping_gpt", # TODO: maybe make this LEAPING_ or something less likely to run into collisions 88 | ] 89 | if "<" in file_name: 90 | return False 91 | if any(leaping_specific_file in file_name for leaping_specific_file in leaping_specific_files): 92 | return False 93 | 94 | if (".pyenv" in file_name) or (".venv" in file_name) or ("site-packages" in file_name): 95 | return False 96 | 97 | if os.path.abspath(file_name).startswith(tracer.project_dir): 98 | return True 99 | return False 100 | 101 | 102 | def monitor_call_trace(code: CodeType, instruction_offset: int): 103 | global tracer 104 | 105 | func_name = code.co_name 106 | 107 | if tracer and tracer.test_name in func_name or (tracer and tracer.stack_size > 0): 108 | if _should_trace(code.co_filename, func_name): 109 | tracer.call_stack_history.append((code.co_filename, func_name, "CALL", tracer.stack_size)) 110 | tracer.stack_size += 1 111 | 112 | 113 | def monitor_return_trace(code: CodeType, instruction_offset: int, retval: object): 114 | global tracer 115 | 116 | if tracer and tracer.stack_size > 0 and _should_trace(code.co_filename, code.co_name): 117 | tracer.call_stack_history.append((code.co_filename, code.co_name, "RETURN", tracer.stack_size)) 118 | tracer.stack_size -= 1 119 | 120 | 121 | def pytest_runtest_setup(item): 122 | if not item.config.getoption('--leaping'): 123 | return 124 | 125 | global tracer 126 | 127 | tracer.test_name = item.name 128 | tracer.test_start = time.time() 129 | 130 | if tracer.scope or tracer.monitoring_possible: 131 | sys.settrace(tracer.simple_tracer) 132 | return 133 | elif sys.version_info >= (3, 12): 134 | sys.monitoring.use_tool_id(3, "Tracer") 135 | sys.monitoring.register_callback(3, sys.monitoring.events.PY_START, monitor_call_trace) 136 | sys.monitoring.register_callback(3, sys.monitoring.events.PY_RETURN, monitor_return_trace) 137 | 138 | sys.monitoring.set_events(3, sys.monitoring.events.PY_START | sys.monitoring.events.PY_RETURN) 139 | 140 | if sys.version_info < (3, 12): 141 | tracer.monitoring_possible = True 142 | 143 | 144 | def pytest_runtest_teardown(item, nextitem): 145 | leaping_option = item.config.getoption('--leaping') 146 | if not leaping_option: 147 | return 148 | sys.settrace(None) 149 | if sys.version_info >= (3, 12): 150 | sys.monitoring.free_tool_id(3) 151 | 152 | 153 | error_context_prompt = """I got an error. Here's the trace: 154 | 155 | {} 156 | 157 | Here's the source code that we got the trace from: 158 | 159 | {} 160 | 161 | If you are certain about the root cause, describe it as tersely as possible, in a single sentence. Don't start the sentence with 'The root cause of the error is', just say what it is. 162 | 163 | In addition, please output the exact series of steps that occurred to get to the erroring state, with their associated places in code. 164 | 165 | """ 166 | 167 | test_passing_prompt = """The test passed. Here's the trace: 168 | {} 169 | 170 | Here's the source code that we got the trace from: 171 | {} 172 | 173 | Please output the exact series of steps that occurred to get to the passing state, with their associated places in code. For all questions that I ask going forward, please back up your claims 174 | with the exact series of steps that occurred to get to the state you are describing, with their associated places in code. 175 | """ 176 | 177 | 178 | def pytest_runtest_makereport(item, call): 179 | leaping_option = item.config.getoption('--leaping') 180 | if not leaping_option: 181 | return 182 | global tracer 183 | 184 | if call.excinfo is not None: 185 | error_type, error_message, traceback = call.excinfo._excinfo 186 | 187 | tracer.error_type = error_type 188 | tracer.error_message = error_message 189 | tracer.traceback = traceback 190 | 191 | 192 | def add_deltas(tracer, key, stack, counter_map, line_no, greater_than=False): 193 | assignments = tracer.function_to_assign_mapping[key] # all assignments in that function (which we got from AST parsing) 194 | 195 | assignments_to_add_line_nos = set() 196 | if not assignments: 197 | return 198 | 199 | # from ast assignemnt we get name and line number 200 | # from deltas we get name and line number + value 201 | # but sometimes the line numbers don't match, so do we even need the line number from the deltas? 202 | # well technically, the variable could show up multiple times at different lines so we need to still have a notion of ordering. 203 | # but even the ordering gets funky when we start talking about loops, for example 204 | # for ex, b only updates at the first line of the loop, so the ordering gets messed up. 205 | # i guess we can assume that if we take the line numbers for one variable, the order should be fine, and we can default to the closest line number 206 | 207 | for assignment_line_no in assignments.keys(): 208 | if greater_than and assignment_line_no > line_no: # when we want to get all the assignments after line_no 209 | assignments_to_add_line_nos.add(assignment_line_no) 210 | if not greater_than and assignment_line_no < line_no: # when we want to get all the assignments before line_no 211 | assignments_to_add_line_nos.add(assignment_line_no) 212 | 213 | for assignment_line_no in assignments_to_add_line_nos: 214 | for ast_assignment in assignments[assignment_line_no]: # all the ast assignments at that line number 215 | var_name = ast_assignment.name 216 | value = None 217 | delta_list = None 218 | if counter_map[key] < len(tracer.function_to_deltas[key]): 219 | delta_list = tracer.function_to_deltas[key][counter_map[key]] 220 | elif tracer.function_to_deltas[key]: # this should only get hit to accomodate the case of last root function assignments (root call funcs should only be called once in an execution) 221 | delta_list = tracer.function_to_deltas[key][-1] 222 | 223 | if not delta_list: 224 | continue 225 | # deltas are runtime assignments, which means that since a function can get executed multiple times during an execution trace, we need to keep a 226 | # monotonically increasing counter (per function and per line number) to get the right deltas 227 | 228 | for runtime_assignment in delta_list: 229 | # This inner loop matches up static AST assignments with runtime deltas. We shouldn't need to have a loop, but would need to refactor some data structure. 230 | # Fine for now since there should rarely be more than one variable assignment per line 231 | if runtime_assignment == var_name: 232 | value = delta_list[runtime_assignment][assignment_line_no] 233 | if value: 234 | break 235 | else: # line numbers don't match up between deltas and AST parsing, look for closest line number as heuristic for now 236 | closest = float('inf') 237 | for line_no, deltas in delta_list[runtime_assignment].items(): 238 | if not deltas: 239 | continue 240 | if abs(assignment_line_no - line_no) < abs(assignment_line_no - closest): 241 | closest = line_no 242 | value = delta_list[runtime_assignment][closest] 243 | break 244 | 245 | if value: 246 | del delta_list[runtime_assignment][assignment_line_no] 247 | value_string = "" 248 | if len(value) > 1: 249 | if len(value) > 5: 250 | value_string = "Last 5 values in loop: [" + ", ".join(value[:5]) + "]" 251 | else: 252 | value_string = "Values in loop: [" + ", ".join(value) + "]" 253 | else: 254 | value_string = value[0] 255 | 256 | context_line = tracer.function_to_source[key].split("\n")[assignment_line_no - 1].strip() 257 | if len(stack) != 0: 258 | stack[-1].children.append(VariableAssignmentNode(var_name, value_string, context_line)) 259 | 260 | 261 | def build_call_hierarchy_interval(stack, trace_data, trace_data_index, file_name, func_name, function_to_call_mapping, function_to_call_args, counter_map): 262 | last_root_call_line = 0 263 | 264 | index = 0 265 | for index, (trace_file_name, trace_func_name, event_type, depth) in enumerate(trace_data[trace_data_index:]): # sequence of "CALL" and "RETURN" calls gathered from sys.monitoring, representing execution trace 266 | if len(stack) == 0: 267 | break 268 | key = (stack[-1].file_name, stack[-1].func_name) 269 | 270 | if event_type == 'CALL': # strategy here is to create VariableAssignmentObjects for all the lines up to the current call 271 | 272 | call_mapping = function_to_call_mapping[key] # if there is no call mapping, probably out of scope 273 | 274 | if call_mapping and call_mapping[trace_func_name]: 275 | line_nos = call_mapping[trace_func_name] # ascending list of line numbers where the function gets called (from AST parsing) 276 | line_no = line_nos[0] # grab the first one 277 | remaining_line_nos = line_nos[1:] 278 | call_mapping[trace_func_name] = remaining_line_nos # re-assign the rest of the line numbers to the dict such that next time this function gets called, we grab the next line number 279 | 280 | if not remaining_line_nos and key == (file_name, func_name): # this means we are the last call within the root function, and we want to save that line number (see add_deltas call after end of loop) 281 | last_root_call_line = line_no # save that line 282 | 283 | add_deltas(tracer, key, stack, counter_map, line_no) 284 | 285 | call_args_list = function_to_call_args[(trace_file_name, trace_func_name)] # list of call args 286 | if call_args_list: 287 | new_call = FunctionCallNode(trace_file_name, trace_func_name, call_args_list.pop(0)) # pop off the first item from the list of call args such that next time the list is accessed we'll pop off the 2nd element 288 | else: 289 | new_call = FunctionCallNode(trace_file_name, trace_func_name, []) 290 | 291 | stack[-1].children.append(new_call) 292 | stack.append(new_call) 293 | 294 | elif event_type == 'RETURN': 295 | add_deltas(tracer, key, stack, counter_map, 0, greater_than=True) 296 | counter_map[key] += 1 297 | stack.pop() 298 | 299 | # add the last variable assignments in the root func that happen after the last function call within root 300 | add_deltas(tracer, (file_name, func_name), stack, counter_map, last_root_call_line, greater_than=True) 301 | 302 | return index 303 | 304 | 305 | def build_call_hierarchy(tracer): 306 | trace_data = tracer.call_stack_history 307 | function_to_call_mapping = tracer.function_to_call_mapping 308 | function_to_call_args = tracer.function_to_call_args 309 | counter_map = defaultdict(int) # one function can get called multiple times throughout execution, so we keep an index to figure out which execution number we're at 310 | 311 | trace_data_index = 0 312 | 313 | full_stack = [] 314 | 315 | while trace_data_index < len(trace_data) - 1: 316 | file_name, func_name, event_type, _ = trace_data[trace_data_index] 317 | root_call = FunctionCallNode(file_name, func_name, []) # todo: can root level pytest functions/fixtures have call args? 318 | stack = [root_call] 319 | 320 | trace_data_index += 1 321 | 322 | trace_data_index += build_call_hierarchy_interval(stack, trace_data, trace_data_index, file_name, func_name, function_to_call_mapping, function_to_call_args, counter_map) 323 | 324 | full_stack.append(root_call) 325 | 326 | # add the erroring line to the trace 327 | traceback = tracer.traceback 328 | if traceback: 329 | while traceback.tb_next: 330 | traceback = traceback.tb_next 331 | frame = traceback.tb_frame 332 | file_path = frame.f_code.co_filename 333 | func_name = frame.f_code.co_name 334 | line_no = frame.f_lineno - frame.f_code.co_firstlineno + 1 335 | try: 336 | source_code = tracer.function_to_source[(file_path, func_name)] 337 | except KeyError: # we've likely hit library code 338 | return full_stack 339 | 340 | error_context_line = source_code.split("\n")[line_no - 1].strip() 341 | full_stack.append(VariableAssignmentNode(tracer.error_type, tracer.error_message, error_context_line)) # todo: this assume the error messages at the root pytest function call. is that true? 342 | 343 | return full_stack 344 | 345 | 346 | def output_call_hierarchy(nodes, output, indent=0): 347 | last_index = len(nodes) - 1 348 | for index, node in enumerate(nodes): 349 | branch_prefix = " " 350 | if indent > 0: 351 | branch_prefix = "| " * (indent - 1) + ("+--- " if index < last_index else "\\--- ") 352 | 353 | if isinstance(node, FunctionCallNode): 354 | formatted_args = ", ".join([arg.name + "=" + arg.value for arg in node.call_args]) 355 | line = f"{branch_prefix}Function: {node.func_name}({formatted_args})" # todo: maybe differentiate between def func and func(), since one means we are expanding inline, the second means we're not going further 356 | output.append(line) 357 | output_call_hierarchy(node.children, output, indent + 1) 358 | elif isinstance(node, VariableAssignmentNode): 359 | line = f"{branch_prefix}{node.context_line} # {node.var_name}: {node.value}" 360 | output.append(line) 361 | 362 | 363 | def fetch_source(key): 364 | func_source = "" 365 | try: 366 | if key in tracer.method_to_class_source: 367 | func_source = tracer.method_to_class_source[key] 368 | else: 369 | func_source = tracer.function_to_source[key] 370 | except KeyError: 371 | pass 372 | 373 | return func_source 374 | 375 | 376 | def generate_suggestion(llm: LLM, test_failed: bool): 377 | global tracer 378 | root = build_call_hierarchy(tracer) 379 | output = [] 380 | output_call_hierarchy(root, output) 381 | 382 | source_text = "" 383 | source_char_limit = 40000 # 10 cents 384 | seen_keys = set() 385 | if tracer.scope_list: 386 | for key in (tracer.scope_list[::-1]): 387 | if key not in seen_keys: 388 | func_source = fetch_source(key) 389 | if func_source: 390 | source_text += func_source + "\n\n" 391 | if len(source_text) > source_char_limit: 392 | break 393 | seen_keys.add(key) 394 | else: # case for < 3.12 395 | for file_path, func_name, _, _ in tracer.call_stack_history[::-1]: 396 | key = (file_path, func_name) 397 | if key not in seen_keys: 398 | func_source = fetch_source(key) 399 | if func_source: 400 | source_text += func_source + "\n\n" 401 | if len(source_text) > source_char_limit: 402 | break 403 | seen_keys.add(key) 404 | 405 | 406 | if test_failed: 407 | output_string = "" 408 | 409 | num_tokens = sum([len(line) for line in output]) / 4 410 | if num_tokens > 10000: # $10 per million tokens, limit to 10 cents, so 10000 tokens max 411 | output_index_within_limit = len(output) - int(len(output) * 10000/num_tokens) 412 | output_string = "\n".join(output[output_index_within_limit:]) 413 | else: 414 | output_string = "\n".join(output) 415 | 416 | prompt = error_context_prompt.format(output_string, source_text) 417 | else: 418 | prompt = test_passing_prompt.format("\n".join(output), source_text) 419 | 420 | llm.add_message("user", prompt) 421 | return llm.chat_completion(stream=True) 422 | 423 | 424 | # Before re-running failed test, need to add scope information to tracer so that we can instrument the re-run 425 | def add_scope(): 426 | global tracer 427 | 428 | scope = [] 429 | 430 | for file_name, func_name, call_type, depth in tracer.call_stack_history: 431 | if call_type != "RETURN": 432 | scope.append((file_name, func_name)) 433 | 434 | tracer.call_stack_history = [] # reset stack history since we are re-running test 435 | tracer.scope_list = scope 436 | tracer.scope = set(scope) 437 | 438 | 439 | def launch_cli(test_failed: bool): 440 | import socket 441 | global llm 442 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 443 | sock.bind(('localhost', 0)) # let the os pick a port 444 | sock.listen(1) 445 | port = sock.getsockname()[1] 446 | 447 | def handle_output(sock, child): 448 | connection, client_address = sock.accept() 449 | initial_message = generate_suggestion(llm, test_failed) 450 | if traceback_obj := tracer.traceback: 451 | error_type = tracer.error_type 452 | error_message = tracer.error_message 453 | exception_str = "".join(traceback.format_exception_only(error_type, error_message)) 454 | 455 | connection.sendall(b"\033[0mInvestigating the following error:\n") 456 | connection.sendall(f"{str(exception_str)} \n".encode('utf-8')) 457 | for chunk in initial_message: 458 | try: 459 | connection.sendall(chunk.encode('utf-8')) 460 | except BrokenPipeError: 461 | child.sendcontrol('c') # Send Ctrl+C to the child process 462 | child.terminate(force=False) # Try to gracefully terminate the child 463 | sys.exit(0) 464 | connection.sendall(b"LEAPING_STOP") 465 | 466 | exit_command_received = False 467 | while not exit_command_received: 468 | data = connection.recv(2048) 469 | if data == b'exit': 470 | break 471 | user_input = data.decode('utf-8') 472 | gpt.add_message("user", user_input) 473 | response = gpt.chat_completion(stream=True) 474 | for chunk in response: 475 | try: 476 | connection.sendall(chunk.encode('utf-8')) 477 | except BrokenPipeError: 478 | child.sendcontrol('c') # Send Ctrl+C to the child process 479 | child.terminate(force=False) # Try to gracefully terminate the child 480 | sys.exit(0) 481 | connection.sendall(b"LEAPING_STOP") 482 | 483 | connection.close() 484 | 485 | child = pexpect.spawn(f"leaping --port {port}") 486 | thread = threading.Thread(target=handle_output, args=(sock, child)) 487 | thread.start() 488 | child.interact() 489 | thread.join() 490 | sock.close() 491 | 492 | 493 | def pytest_runtest_protocol(item, nextitem): 494 | global tracer 495 | 496 | leaping_option = item.config.getoption('leaping') 497 | if not leaping_option: 498 | return 499 | reports = runtestprotocol(item, nextitem=nextitem, log=False) 500 | 501 | test_failed = any(report.failed for report in reports if report.when == 'call') 502 | 503 | if not test_failed and item.config.total_tests_collected > 1: # Only run leaping on passing tests if explicitly requested 504 | return None 505 | 506 | tracer.test_duration = time.time() - tracer.test_start 507 | tracer.test_start = None 508 | add_scope() 509 | runtestprotocol(item, nextitem=nextitem, log=False) 510 | launch_cli(test_failed) 511 | 512 | for report in reports: 513 | item.ihook.pytest_runtest_logreport(report=report) 514 | 515 | return True 516 | -------------------------------------------------------------------------------- /pytest-leaping/src/simpletracer.py: -------------------------------------------------------------------------------- 1 | from leaping_models import ExecutionCursor, CallStack, ASTAssignment, RuntimeAssignment 2 | import ast 3 | import inspect 4 | import textwrap 5 | from collections import defaultdict 6 | import os 7 | 8 | 9 | def compare_objects(obj1, obj2, diffs, path="", depth=0): 10 | if depth > 2: 11 | return 12 | 13 | if not isinstance(obj1, type(obj2)): 14 | diffs.append((path, obj2)) 15 | return 16 | 17 | if not hasattr(obj1, "__dict__") or not hasattr(obj2, "__dict__"): 18 | if hasattr(obj2, "__dict__"): 19 | diffs.append(path, obj2[path]) 20 | elif not hasattr(obj1, "__dict__"): 21 | if obj1 != obj2: 22 | diffs.append((path, obj2)) 23 | return 24 | 25 | obj1_dict, obj2_dict = obj1.__dict__, obj2.__dict__ 26 | all_keys = set(obj1_dict.keys()) | set(obj2_dict.keys()) 27 | 28 | for key in all_keys: 29 | new_path = ( 30 | f"{path}.{key}" if path else key 31 | ) # todo: square brackets if array, list, or tuple. add validation that that's correct 32 | 33 | # todo: maybe have prev value as well 34 | 35 | if key not in obj1_dict: 36 | diffs.append((new_path, obj2_dict[key])) 37 | elif key not in obj2_dict: 38 | diffs.append((new_path, obj1_dict[key])) 39 | else: 40 | value1, value2 = obj1_dict[key], obj2_dict[key] 41 | 42 | compare_objects(value1, value2, diffs, new_path, depth + 1) 43 | 44 | 45 | def get_deltas(prev_locals, curr_locals): 46 | deltas = [] 47 | 48 | for local, val in curr_locals.items(): 49 | if local in prev_locals: 50 | diffs = [] 51 | compare_objects(prev_locals[local], val, diffs) 52 | if not diffs: 53 | continue 54 | for path, change in diffs: 55 | deltas.append(RuntimeAssignment(name=local, value=str(change), path=path)) 56 | 57 | if local not in prev_locals: 58 | deltas.append(RuntimeAssignment(name=local, value=str(val), path="")) 59 | 60 | return deltas 61 | 62 | 63 | def create_ast_mapping(parsed_ast, assign_mapping, call_mapping): 64 | def process_node(node): 65 | if isinstance(node, ast.arguments): 66 | for arg in node.args: 67 | assign_mapping[arg.lineno].append(ASTAssignment(name=arg.arg, deps=[])) 68 | 69 | elif isinstance(node, ast.Assign): 70 | assignee = node.targets[0] 71 | if isinstance(assignee, ast.Name): 72 | variables = [n.id for n in ast.walk(node.value) if isinstance(n, ast.Name)] 73 | assign_mapping[node.lineno].append(ASTAssignment(name=assignee.id, deps=variables)) 74 | 75 | elif isinstance(node, ast.AugAssign) or isinstance(node, ast.AnnAssign): 76 | assign_mapping[node.lineno].append(ASTAssignment(name=node.target.id, deps=[])) 77 | 78 | elif isinstance(node, ast.Return): 79 | variables = [] 80 | if node.value: 81 | variables = [n.id for n in ast.walk(node.value) if isinstance(n, ast.Name)] 82 | assign_mapping[node.lineno].append(ASTAssignment(name="return", deps=variables)) 83 | 84 | elif isinstance(node, ast.Call): 85 | if hasattr(node.func, 'id'): # Direct calls 86 | call_mapping[node.func.id].append(node.lineno) 87 | elif hasattr(node.func, 'attr'): # Method calls 88 | call_mapping[node.func.attr].append(node.lineno) 89 | 90 | for node in ast.walk(parsed_ast): 91 | process_node(node) 92 | 93 | 94 | def get_mapping_from_source(source): 95 | parsed_ast = ast.parse(source).body[0] 96 | 97 | assign_mapping = defaultdict(list) # line number -> list of assigns 98 | call_mapping = defaultdict(list) # function name -> list of line numbers 99 | create_ast_mapping(parsed_ast, assign_mapping, call_mapping) 100 | return assign_mapping, call_mapping 101 | 102 | 103 | def get_function_source_from_frame(frame, method_to_class_source): 104 | func_name = frame.f_code.co_name 105 | source_code = None 106 | 107 | if func_name in frame.f_globals: 108 | source_code = inspect.getsource(frame.f_globals[func_name]) 109 | else: # deal with methods that are not part of global scope, so we have to look at instance methods of self, where self is in the local scope 110 | if 'self' in frame.f_locals: 111 | cls = frame.f_locals['self'].__class__ 112 | if hasattr(cls, func_name): 113 | method = getattr(cls, func_name) 114 | source_code = inspect.getsource(method) 115 | method_to_class_source[(frame.f_code.co_filename, func_name)] = inspect.getsource(cls) 116 | 117 | if not source_code: 118 | return 119 | 120 | dedented_source = textwrap.dedent(source_code) 121 | return dedented_source 122 | 123 | 124 | class SimpleTracer: 125 | def __init__(self): 126 | self.project_dir = "" 127 | self.call_stack = CallStack() 128 | self.test_name = None 129 | self.test_start = None 130 | self.test_duration = None 131 | self.function_to_assign_mapping = defaultdict(list) 132 | self.function_to_call_mapping = defaultdict(list) 133 | self.function_to_deltas = defaultdict(list) 134 | self.function_to_call_args = defaultdict(list) 135 | self.function_to_source = {} 136 | self.method_to_class_source = {} 137 | self.filename_to_path = {} 138 | self.error_message = "" 139 | self.error_type = "" 140 | self.traceback = None 141 | self.call_stack_history = [] 142 | self.stack_size = 0 143 | self.scope = set() 144 | self.scope_list = [] 145 | self.line_counter = defaultdict(int) 146 | self.monitoring_possible = False 147 | 148 | def simple_tracer(self, frame, event: str, arg): 149 | if frame.f_code.co_filename not in self.filename_to_path: 150 | self.filename_to_path[frame.f_code.co_filename] = os.path.abspath(frame.f_code.co_filename) 151 | current_file = self.filename_to_path[frame.f_code.co_filename] 152 | 153 | if frame.f_code.co_filename[0] == "<": 154 | return 155 | if current_file.endswith("plugin.py") or current_file.endswith("models.py") or current_file.endswith( 156 | "simpletracer.py"): # todo: change conftest to be in a diff folder to filter out better 157 | return 158 | 159 | if (".pyenv" in current_file) or ("site-packages" in current_file): 160 | return 161 | 162 | if not self.monitoring_possible and self.scope and (current_file, frame.f_code.co_name) not in self.scope: 163 | return 164 | 165 | if not current_file.startswith(self.project_dir): 166 | return 167 | 168 | 169 | self.process_events(frame, event, arg) 170 | 171 | return self.simple_tracer 172 | 173 | def process_events(self, frame, event, arg): 174 | file_path = frame.f_code.co_filename 175 | func_name = frame.f_code.co_name 176 | 177 | if event == "line": 178 | line_no = frame.f_lineno 179 | 180 | key = (file_path, func_name, line_no) 181 | if self.line_counter[key] > 10: 182 | # heuristic to stop tracking deltas once we've hit a line more than 10 times 183 | # should clear this dictionary if the functions gets called again separately (clear cache in 'call' event) 184 | # todo: 10 is arbitrary, might be tricks to produce a better number 185 | return 186 | 187 | self.line_counter[key] += 1 188 | 189 | if (file_path, func_name) not in self.function_to_source.keys(): # if we haven't yet gotten the source/ast parsed the function 190 | 191 | source = get_function_source_from_frame(frame, self.method_to_class_source) 192 | 193 | if source: 194 | self.function_to_source[(file_path, func_name)] = source 195 | 196 | assign_mapping, call_mapping = get_mapping_from_source( 197 | source) # through the AST parsing of the source code, get map of assignments and calls by line number 198 | 199 | if assign_mapping: # dict of line_no -> list of ASTAssignment objects 200 | self.function_to_assign_mapping[(file_path, func_name)] = assign_mapping 201 | if call_mapping: # dict of the name of a function being called -> list of line_no (since a function can be called at multiple lines within the same function) 202 | self.function_to_call_mapping[(file_path, func_name)] = call_mapping 203 | 204 | relative_line_no = line_no - frame.f_code.co_firstlineno 205 | 206 | current_frame = self.call_stack.current_frame() 207 | 208 | prev_locals = current_frame.f_locals if current_frame else [] 209 | curr_locals = frame.f_locals 210 | 211 | deltas: list[RuntimeAssignment] = get_deltas(prev_locals, curr_locals) 212 | 213 | delta_map = defaultdict(lambda: defaultdict(list)) 214 | 215 | for delta in deltas: 216 | delta_map[delta.name][relative_line_no].append(delta.value) 217 | 218 | if self.function_to_deltas[(file_path, func_name)][-1] == "NEW": 219 | self.function_to_deltas[(file_path, func_name)][-1] = delta_map 220 | else: 221 | last_map = self.function_to_deltas[(file_path, func_name)][-1] 222 | for var_name, line_delta_list in delta_map.items(): 223 | if var_name not in last_map: 224 | last_map[var_name] = line_delta_list 225 | else: 226 | for key, val in line_delta_list.items(): 227 | last_map[var_name][key].extend(val) 228 | 229 | self.function_to_deltas[(file_path, func_name)][-1] = last_map 230 | 231 | cursor = self.create_cursor(file_path, frame) 232 | self.call_stack.new_cursor_in_current_frame(cursor) 233 | 234 | if event == "call": 235 | if self.monitoring_possible: 236 | self.stack_size += 1 237 | self.call_stack_history.append((file_path, func_name, "CALL", self.stack_size)) 238 | arg_deltas: list[RuntimeAssignment] = get_deltas([], 239 | frame.f_locals) # these deltas represent the parameters at the start of a function 240 | if arg_deltas: 241 | self.function_to_call_args[(file_path, func_name)].append(arg_deltas) 242 | 243 | cursor = self.create_cursor(file_path, frame) 244 | self.call_stack.enter_frame(cursor) 245 | 246 | self.function_to_deltas[(file_path, func_name)].append("NEW") 247 | 248 | if event == "return": 249 | self.stack_size -= 1 250 | if self.monitoring_possible: 251 | self.call_stack_history.append((file_path, func_name, "RETURN", self.stack_size)) 252 | self.call_stack.exit_frame() 253 | 254 | def create_cursor(self, file_path, frame): 255 | cursor = ExecutionCursor(file_path, frame.f_lineno, frame.f_code.co_name, frame.f_locals) 256 | return cursor 257 | 258 | def get_variable_history(self, variable_name, file_path, func_name, first_line_no, max_depth=1, current_depth=0): 259 | if current_depth > max_depth: 260 | return "" 261 | 262 | history = "" 263 | # todo: change this once we actually use variable history 264 | for line_no, deltas in self.function_to_deltas.get((file_path, func_name), {}).items(): 265 | for delta in deltas: 266 | if delta.name == variable_name: 267 | line = None 268 | with open(file_path) as f: 269 | line = f.readlines()[first_line_no + line_no - 2].strip() 270 | 271 | history += f"At line '{line}' in {func_name}, '{delta.name}' was set to {delta.value}" 272 | 273 | if current_depth < max_depth: 274 | for dep in self.get_variable_dependencies(variable_name, file_path, func_name): 275 | history += self.get_variable_history(dep, file_path, func_name, max_depth, 276 | current_depth + 1) 277 | 278 | return history 279 | 280 | def get_variable_dependencies(self, variable_name, file_path, func_name): 281 | dependencies = [] 282 | for _, mappings in self.function_to_assign_mapping.get((file_path, func_name), {}).items(): 283 | for mapping in mappings: 284 | if mapping.name == variable_name: 285 | dependencies.extend(mapping.deps) 286 | return dependencies 287 | -------------------------------------------------------------------------------- /pytest-leaping/tox.ini: -------------------------------------------------------------------------------- 1 | # For more information about tox, see https://tox.readthedocs.io/en/latest/ 2 | [tox] 3 | envlist = py38,py39,py310,py311,py312,pypy3,flake8 4 | 5 | [testenv] 6 | deps = pytest>=6.2.0 7 | commands = pytest {posargs:tests} 8 | 9 | [testenv:flake8] 10 | skip_install = true 11 | deps = flake8 12 | commands = flake8 src tests 13 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | annotated-types==0.6.0 2 | anyio==4.3.0 3 | certifi==2024.2.2 4 | distro==1.9.0 5 | fastapi==0.101.1 6 | h11==0.14.0 7 | httpcore==1.0.4 8 | httpx~=0.25.2 9 | idna==3.6 10 | openai==1.12.0 11 | pydantic~=2.5.3 12 | pydantic_core==2.16.3 13 | pylspclient==0.0.3 14 | python-lsp-server==1.10.0 15 | rapidfuzz==3.5.2 16 | six==1.16.0 17 | sniffio==1.3.0 18 | starlette==0.27.0 19 | tqdm==4.66.2 20 | tree-sitter==0.20.4 21 | tree-sitter-languages==1.10.2 22 | treelib==1.7.0 23 | typing_extensions==4.9.0 24 | pytest-leaping==0.1.0 25 | 26 | 27 | requests~=2.31.0 28 | setuptools~=58.0.4 29 | pexpect~=4.9.0 30 | pytest~=8.0.2 -------------------------------------------------------------------------------- /tracing/test_file.py: -------------------------------------------------------------------------------- 1 | class Person: 2 | def __init__(self, x): 3 | self.x = x 4 | 5 | def speak(self): 6 | return "hey" 7 | 8 | def introduce(self): 9 | msg = self.speak() 10 | return f"{msg}, I'm a person with attribute x = {self.x}" 11 | 12 | def calculate(self): 13 | return self.x * 2 14 | 15 | def react(self): 16 | reaction = self.introduce() + ". Nice to meet you!" 17 | return reaction 18 | 19 | def conclude(self): 20 | conclusion = self.react() + " Let's calculate something: " + str(self.calculate()) 21 | return conclusion 22 | 23 | def func(x): 24 | y = 3 * x 25 | z = y + 5 26 | 27 | p = Person(3) 28 | result = p.conclude() # Modified to use the conclude method 29 | 30 | return z + p.x 31 | 32 | def func2(y): 33 | a = y * 2 34 | return func(a) 35 | 36 | def func3(z): 37 | return func2(z + 1) 38 | 39 | def func4(w): 40 | return func3(w * 2) 41 | 42 | def test_failure(): 43 | val = func4(3) # Changed to call func4 44 | k = 5 45 | for i in range(300000): 46 | i ^= 2 47 | x = 3 / (val - 26) 48 | assert val == 4 49 | --------------------------------------------------------------------------------