├── .gitignore ├── README.md ├── docs └── Lingxi Technical Report 2505.pdf ├── imgs ├── AddMsg.png ├── GraphSelect.png ├── HILenable.png ├── Lingxi_logo.png ├── graph1.png ├── graph2.png └── graph3.png ├── langgraph.json ├── pyproject.toml └── src └── agent ├── README.md ├── __init__.py ├── constant.py ├── github_utils.py ├── hierarchy_graph_demo.py ├── llm.py ├── parsers.py ├── prompt ├── __init__.py ├── context_manager.py ├── mam.py ├── problem_decoder.py ├── problem_solver.py ├── reviewer.py ├── solution_mapper.py └── supervisor.py ├── runtime_config.py ├── state.py ├── supervisor_graph_demo.py ├── tool_set ├── constant.py ├── context_tools.py ├── edit_history.py ├── edit_tool.py ├── linter │ ├── __init__.py │ ├── base.py │ ├── impl │ │ ├── python.py │ │ ├── treesitter.py │ │ └── treesitter_compat.py │ └── linter.py ├── oheditor.py ├── sepl_tools.py └── utils.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | .langgraph_api/ 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the 87 | # code is 88 | # intended to run in multiple environments; otherwise, check them in: 89 | # .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in 93 | # version control. 94 | # However, in case of collaboration, if having platform-specific dependencies 95 | # or dependencies 96 | # having no cross-platform support, pipenv may install dependencies that 97 | # don't work, or not 98 | # install all needed dependencies. 99 | #Pipfile.lock 100 | 101 | # UV 102 | # Similar to Pipfile.lock, it is generally recommended to include uv.lock in 103 | # version control. 104 | # This is especially recommended for binary packages to ensure 105 | # reproducibility, and is more 106 | # commonly ignored for libraries. 107 | #uv.lock 108 | 109 | # poetry 110 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock 111 | # in version control. 112 | # This is especially recommended for binary packages to ensure 113 | # reproducibility, and is more 114 | # commonly ignored for libraries. 115 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 116 | #poetry.lock 117 | 118 | # pdm 119 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in 120 | # version control. 121 | #pdm.lock 122 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended 123 | # to not include it 124 | # in version control. 125 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 126 | .pdm.toml 127 | .pdm-python 128 | .pdm-build/ 129 | 130 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and 131 | # github.com/pdm-project/pdm 132 | __pypackages__/ 133 | 134 | # Celery stuff 135 | celerybeat-schedule 136 | celerybeat.pid 137 | 138 | # SageMath parsed files 139 | *.sage.py 140 | 141 | # Environments 142 | .env 143 | .venv 144 | env/ 145 | venv/ 146 | ENV/ 147 | env.bak/ 148 | venv.bak/ 149 | 150 | # Spyder project settings 151 | .spyderproject 152 | .spyproject 153 | 154 | # Rope project settings 155 | .ropeproject 156 | 157 | # mkdocs documentation 158 | /site 159 | 160 | # mypy 161 | .mypy_cache/ 162 | .dmypy.json 163 | dmypy.json 164 | .langchain.db 165 | # Pyre type checker 166 | .pyre/ 167 | 168 | # pytype static type analyzer 169 | .pytype/ 170 | 171 | # Cython debug symbols 172 | cython_debug/ 173 | 174 | # PyCharm 175 | # JetBrains specific template is maintained in a separate JetBrains.gitignore 176 | # that can 177 | # be found at 178 | # https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 179 | # and can be added to the global gitignore or merged into this file. For a 180 | # more nuclear 181 | # option (not recommended) you can uncomment the following to ignore the 182 | # entire idea folder. 183 | #.idea/ 184 | 185 | # Ruff stuff: 186 | .ruff_cache/ 187 | 188 | # PyPI configuration file 189 | .pypirc) 190 | 191 | 192 | # Custom ====================================================================== 193 | Pipfile 194 | .DS_Store 195 | .vscode/ 196 | /cache 197 | 198 | uv.lock 199 | 200 | scripts/results/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Lingxi Logo 2 | 3 | [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/nimasteryang/Lingxi) 4 | # Intro 5 | **Lingxi** is an open‑source, multi‑agent framework designed to automate a broad range of software‑engineering tasks. 6 | 7 | # SWE-Bench support 8 | Lingxi's implementation of SWE-Bench will release soon, please see our technical report at [Lingxi Technical Report (PDF)](docs/Lingxi%20Technical%20Report%202505.pdf) 9 | 10 | # Setup 11 | 12 | ## Part 1: Dependencies 13 | - Dependencies are located in `pyproject.toml`, and can be installed with the following command: 14 | - `cd Lingxi && pip install -e .` 15 | - The installation command can be ignored if using `uv` package manager and `uv run` later on (see Run section for reference) 16 | - It is suggested to use some package manager (e.g., venv, pipenv, uv, ...) 17 | ## Part 2: Environment Config 18 | - The prototype requires the following environment variables to run: 19 | 1. `LLM_PROVIDER`: The provider of the LLM to be used(e.g., "anthropic", "openai", "deepseek") 20 | 2. `LLM_MODEL`: The model name of the LLM to be used (e.g., "claude-3-5-haiku-latest") 21 | 3. The corresponding `LLM_PROVIDER` API key to be used 22 | - `LLM_PROVIDER="anthropic"`, set `ANTHROPIC_API_KEY` 23 | - `LLM_PROVIDER="openai"`, set `OPENAI_API_KEY` 24 | - `LLM_PROVIDER="deepseek"`, set `DEEPSEEK_API_KEY` 25 | - Additional provider integrations can be added, see [Langchain Chat Models](https://python.langchain.com/docs/integrations/chat/) 26 | 4. `OPENAI_API_KEY`: The OpenAI API key to be used. Required for generating embeddings to build project knowledge. 27 | 5. `GITHUB_TOKEN`: A GitHub API token used to automatically collect issue report information. It can be generated using a GitHub account through the following menus: 28 | - Profile > Settings > Developer Settings > Personal access tokens > Fine-grained tokens 29 | - Place these environment variables in the `.env` file in the root of the project. This file will be loaded by the prototype on execution 30 | # Run 31 | - The prototype can be ran with either of the following commands: 32 | - `langgraph dev --no-reload` (if installed via `pip`) 33 | - `uv run --env-file .env langgraph dev --no-reload` (if using `uv`) 34 | - The prototype will create a local instance of LangGraph Studio and will open a browser instance to the page once the prototype is run 35 | - Once in the UI, select the graph to run at the top left 36 | 37 | ![Graph Select](imgs/GraphSelect.png) 38 | 39 | - Add a Message by clicking "+ Message" button, and add the Github issue URL. An example URL is: https://github.com/gitpython-developers/GitPython/issues/1977 40 | ![Add Msg](imgs/AddMsg.png) 41 | 42 | - Finally click the Submit button 43 | - Note that human feedback functionality is disabled be default, it can be enabled by clicking the checkbox in the LangGraph Studio UI before submitting the issue URL: 44 | ![Graph1](imgs/HILenable.png) 45 | # Agents 46 | - The prototype currently consists of five main agents across the graphs and implemented in various LangGraph nodes 47 | ## Multi-Agent Manager 48 | - The multi-agent manager coordinates between the issue resolver and reviewer agents to provide feedback and or guidance to each step. 49 | - Only used in `src/agent/hierarchy_graph_demo.py`. 50 | - Prompt in `src/agent/prompt/mam.py` 51 | ## Supervisor 52 | - The supervisor oversees the entire workflow of solving the issue, and decides which agent to route to depending on the current progress of the issue. 53 | - Prompt in `src/agent/prompt/supervisor.py` 54 | ## Problem Decoder 55 | - The problem decoder analyzes and understands the issue requirements to generate a problem statement consisting of a topic question, codebase representation (i.e., bug localization), the current behaviour, and the expected behaviour. 56 | - Uses tools `[view_directory, search_relevant_files, view_file_content]` 57 | - Prompt in `src/agent/prompt/problem_decoder.py` 58 | 59 | ## Solution Mapper 60 | - The solution mapper reviews the information from the problem decoder with respect to the localized bug and the codebase, to then generate a detailed code change plan for each of the files and or functions. 61 | - Uses tools `[view_directory, search_relevant_files, view_file_content]` 62 | - Prompt in `src/agent/prompt/solution_mapper.py` 63 | ## Problem Solver 64 | - The problem solver implements the proposed change plan from the solution mapper by modifying files 65 | - Uses tools `[view_directory, search_relevant_files, str_replace_editor]` 66 | - Prompt in `src/agent/prompt/problem_solver.py` 67 | ## Reviewer 68 | - The reviewer verifies the proposed fix resulting from the issue resolver agent by generating and executing test cases. 69 | - Only used in `src/agent/hierarchy_graph_demo.py` 70 | - Uses tools `[view_directory, search_relevant_files, view_file_content, run_shell_cmd]` 71 | - Prompt in `src/agent/prompt/reviewer.py` 72 | 73 | # Tools 74 | - There are five main tools are defined and used by multiple agents in the workflow of the prototype 75 | - Note that tools used by agents are: 76 | - Defined with the `@tool` wrapper 77 | - Defined while constructing the agent via `create_react_agent` using`tools` parameters 78 | 79 | ## view_directory 80 | - View the file structure of the repository, including directories (marked with /). 81 | Automatically reduces depth if entries exceed 50. 82 | Args: 83 | dir_path (str): Starting directory. Defaults to './'. 84 | depth (Optional[int]): Maximum depth. None for unlimited. Defaults to None. 85 | Returns: 86 | List[str]: Sorted list of directories (with /) and files. 87 | - Defined in `src/agent/tool_set/sepl_tools.py` 88 | ## view_file_content 89 | - Read the content of the specified file. 90 | Parameters: 91 | file_name (str): File name relative to the git root directory. 92 | view_range (Optional[List[int]): Optional list containing [start_line, end_line] to limit the lines displayed. 93 | Usage: 94 | - LLM should initially attempt to read the entire file content. 95 | - If the file is too large, LLM can use the `view_file_structure` tool to identify relevant code ranges, 96 | and then call this tool again specifying the `view_range` to read only the necessary lines. 97 | Returns: 98 | str: Content of the file or the specified line range. 99 | - Defined in `src/agent/tool_set/sepl_tools.py` 100 | 101 | ## run_shell_cmd 102 | - Run a list of shell commands in sequential order and return the stdout results, your working directory is the root of the project 103 | Args: 104 | commands (List[str]): A list of shell commands to be run in sequential order. 105 | config (RunnableConfig) The runtime configuration. 106 | Returns: 107 | str: Result of running the commands. 108 | - Defined in `src/agent/tool_set/sepl_tools.py` 109 | 110 | 111 | ## search_relevant_files 112 | - Given a query search string (for example, the issue report description, filenames, etc), search for relevant code snippets of files in the project by calculating embedding similarity between the query and code snippets in a vector database. 113 | Args: 114 | query: A search string (for example, the issue report description, filenames, etc), to be used to find relevant files and functions. 115 | Returns: 116 | explanations (str): Each retrieved file with an explanation of how the file is relevant to the query. 117 | - This tool builds a local Vector DB using `ChromaDB` at the location defined in `src/agent/config.py:RUNTIME_DIR` by indexing all Java/Python files in the project. 118 | - Used by the `search_relevant_files` tool. 119 | - Defined in `src/agent/tool_set/context_tools.py` 120 | ## str_replace_editor 121 | - Custom editing tool for viewing, creating and editing files in plain-text format 122 | * State is persistent across command calls and discussions with the user 123 | * If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep 124 | * The `create` command cannot be used if the specified `path` already exists as a file 125 | * If a `command` generates a long output, it will be truncated and marked with `` 126 | Args: 127 | command (str): The commands to run. Allowed options are: `view`, `create`, `str_replace`, `insert`. 128 | path (str): Absolute path to file or directory, e.g. `/workspace/file.py` or `/workspace`. 129 | file_text (Optional[str]): Required parameter of `create` command, with the content of the file to be created. 130 | old_str (Optional[str]): Required parameter of `str_replace` command containing the string in `path` to replace. 131 | new_str (Optional[str]): Optional parameter of `str_replace` command containing the new string (if not given, no string will be added). Required parameter of `insert` command containing the string to insert. 132 | insert_line (Optional[int]): Required parameter of `insert` command. The `new_str` will be inserted AFTER the line `insert_line` of `path`. 133 | view_range (Optional[List[int]): Optional parameter of `view` command when `path` points to a file. If none is given, the full file is shown. If provided, the file will be shown in the indicated line number range, e.g. [100, 600] will show content between line 100 and 600. Indexing at 1 to start. Setting `[start_line, -1]` shows all lines from `start_line` to the end of the file. Unless you are sure about the line numbers, otherwise, do not set this parameter and use the `view` command to view the whole file. 134 | - Defined in `src/agent/tool_set/edit_tool.py` 135 | 136 | 137 | # Graphs 138 | 139 | - There are currently two developed graphs in the prototype 140 | 1. `src/agent/supervisor_graph_demo.py` 141 | 2. `src/agent/hierarchy_graph_demo.py` 142 | 143 | ## Supervisor Graph 144 | ![Graph1](imgs/graph1.png) 145 | - This graph defines a workflow to resolve an issue given the issue requirements, with use of a supervisor agent and integrated with human feedback 146 | - The graph is defined in the `issue_resolve_graph` variable and contains the following six nodes: 147 | 1. `input_handler_node`: This node handles the start of the workflow by setting up the runtime environment given the Github issue report URL. 148 | 2. `supervisor_node`: This node executes the supervisor agent. 149 | 3. `problem_decoder`: This node executes the problem decoder agent. 150 | 4. `solution_mapper`: This node executes the solution mapper agent. 151 | 5. `problem_solver`: This node executes the problem solver agent. 152 | - Human feedback: After each node is executed, the result is provided to the user for human feedback if enabled 153 | - Recall human feedback is disabled by default, see Run section on how to enable it 154 | - If the human provides feedback, the same node will be re-executed with the additional feedback information 155 | - If the human provides no feedback, the workflow will continue 156 | 157 | 158 | ## Hierarchy Graph 159 | ![Graph2](imgs/graph2.png) 160 | - This graph defines a workflow also to resolve an issue, though also implements a multi-agent manager to coordinate between an issue resolver agent (i.e. the graph from [Lingxi Documentation#Supervisor Graph]) and a reviewer agent, to review the result of the issue resolver agent, and also includes human feedback. 161 | - The graph is defined in the `hierarchy_graph` variable and contains the following three nodes: 162 | 1. `mam_node`: This node executes the multi-agent manager agent. 163 | 2. `issue_resolve_graph`: This node executes the issue resolver agent, combining all nodes and agents defined in [Lingxi Documentation#Supervisor Graph] 164 | 3. `reviewer_node`: This node executes the reviewer agent. 165 | 166 | # Development 167 | 168 | ## Adding agents 169 | - Agents are generally defined via: 170 | 1. System prompt (optional) 171 | 2. User prompt or past conversation history 172 | 3. Tools (optional) 173 | - In Langgraph, there are multiple ways to create and define agents depending on their purpose 174 | - See [Langgraph Documentation: Build an Agent](https://python.langchain.com/docs/tutorials/agents/) for more information, or view the related graph files in our project 175 | ### Tool-less Agents 176 | - These agents are defined to be used without tools 177 | - In this case, the LLM can be used directly 178 | 179 | ```python 180 | from langchain_anthropic import ChatAnthropic 181 | llm = ChatAnthropic(model="claude-3-5-haiku-latest", temperature=0.0) 182 | response = llm.invoke("Tell me a joke") # Single string input 183 | response = llm.invoke(["What is 2+2", "What is the previous result times 5"]) # List of string as input 184 | ``` 185 | ### ReACT Agents 186 | - These agents are defined with tools which they will decide to use on their own time 187 | - In this case, the agent should be created using [`create_react_agent`](https://langchain-ai.github.io/langgraph/reference/prebuilt/#langgraph.prebuilt.chat_agent_executor.create_react_agent), and will expect different input (see [Lingxi Documentation#Agent Expected Input and State]) 188 | ```python 189 | from langchain_anthropic import ChatAnthropic 190 | from langgraph.prebuilt import create_react_agent 191 | from langchain_core.messages import HumanMessage 192 | 193 | llm = ChatAnthropic(model="claude-3-5-haiku-latest", temperature=0.0) # Create LLM 194 | tools = [math_tool] # Tool example 195 | problem_solver_agent = create_react_agent( 196 | llm, 197 | tools=problem_decoder_tools, # Required for ReACT agents 198 | prompt="You must solve the given problems!" # Optional system prompt 199 | ) 200 | problem_solver_agent.invoke({"messages": [ 201 | HumanMessage(content="What is 2+2") 202 | ]}) # How to invoke the agent using user prompt "What is 2+2" 203 | ``` 204 | - ReACT agents also create a subgraph that includes the looks like so: 205 | ![Graph3](imgs/graph3.png) 206 | ### Agent Expected Input and LangGraph State 207 | - Different agent creation methods will expect different inputs 208 | - An agent without tools used directly from a class such as `ChatAnthropic` can expect relatively flexible inputs: 209 | - A string 210 | - A list of strings 211 | - A list of `BaseMessage` - usually their subclasses (i.e., `SystemMessage`, `HumanMessage`, `AIMessage`, ..) 212 | - When defining and invoking a ReACT agent, it will instead expect more structured input involving a `MessagesState` object, known as the LangGraph state 213 | - The `MessagesState` holds and defines the conversation history of the current run, as well as any additional variables if a custom State class is defined (e.g., see `src/agent/state.py:CustomState) 214 | - The `MessagesState` must be a dictionary (`dict`, `AnnotatedDict`) with the `messages` (the convo history) key as a list of `BaseMessages` - usually their subclasses (i.e., `SystemMessage`, `HumanMessage`, `AIMessage`, ..) 215 | - The messages in the state is updated automatically by the Langgraph graph within the defined nodes in the graph 216 | - Custom variables can be updated when returning from each node using the `update` kwarg in the `Command` class (e.g., `Command(update={"custom_var": 2"})) 217 | - It is important to note that the `messages` of `MessagesState` follows strict structure when being invoked by ReACT agents and may be difficult to alter 218 | - The main reason is that if an agent uses a tool, the `AIMessage` of the agent will be proceeded by a `ToolMessage` containing information of the corresponding tool 219 | - If one or the other message is removed from the state, the workflow will break 220 | 221 | ### System Prompts 222 | - Note that a system prompt can take different forms in LangGraph: 223 | - `prompt` kwarg of `create_react_agent` (str): This is converted to a SystemMessage and added to the beginning of the list of messages in state["messages"]. 224 | - SystemMessage: this is added to the beginning of the list of messages in state["messages"]. 225 | - Callable: This function should take in full graph state and the output is then passed to the language model. 226 | - Runnable: This runnable should take in full graph state and the output is then passed to the language model. 227 | 228 | ## Adding tools 229 | - Tools can be considered as Python functions that can be used by agents/LLM (defined via `create_react_agent`) 230 | - These are defined using the `@tool` decorator (`from langchain_core.tools import tool`), and require docstring explaining the function, the args, and expected return info 231 | - This information is passed to the agent as context for using the function 232 | - Instead of defining args in the docstring, type hints can be used which will be parsed and sent to the agent as well 233 | ```python 234 | # tools.py 235 | from langchain_core.tools import tool 236 | 237 | # Via type hints 238 | @tool 239 | def multiply(a: int, b: int) -> int: 240 | """Multiply two numbers.""" 241 | return a * b 242 | 243 | from typing import Annotated, List 244 | # Via annotated type hints 245 | @tool 246 | def multiply_by_max( 247 | a: Annotated[int, "scale factor"], 248 | b: Annotated[List[int], "list of ints over which to take maximum"], 249 | ) -> int: 250 | """Multiply a by the maximum of b.""" 251 | return a * max(b) 252 | ``` 253 | 254 | - Once tools are created, they can be linked to agents via the `tools` keyword in the `create_react_agent` constructor: 255 | ```python 256 | # graph.py 257 | from langchain_anthropic import ChatAnthropic 258 | from langchain_core.messages import HumanMessage 259 | from langgraph.prebuilt import create_react_agent 260 | 261 | from tools import multiply, multiply_by_max # Load the example tools above 262 | 263 | llm = ChatAnthropic(model="claude-3-5-haiku-latest", temperature=0.0) 264 | tools = [multiply, multiply_by_max] # Define the tools for the agent using the created tools above 265 | problem_solver_agent = create_react_agent( 266 | llm, 267 | tools=problem_decoder_tools, # Required for ReACT agents 268 | prompt="You must solve the given problems!" # Optional system prompt 269 | ) 270 | 271 | # Example prompts 272 | problem_solver_agent.invoke({"messages": [ 273 | HumanMessage(content="What is 2*5") 274 | ]}) 275 | problem_solver_agent.invoke({"messages": [ 276 | HumanMessage(content="What is 2 mutlipled by the max of this list: [5, 10, 15]") 277 | ]}) 278 | ``` 279 | - Additional info can be read on [Langchain Documentation: How to create tools](https://python.langchain.com/docs/how_to/custom_tools/), or view the related tool files in our project 280 | 281 | ## Creating graphs 282 | - A graph defines a multi-agent workflow, that generally follows these steps: 283 | 1. Create a Python file for the graph 284 | 2. Define agents 285 | 3. Define nodes for each agents where they will be invoked, containing the logic that will happen before/after their individual invocation. 286 | 4. Build the graph using `StateGraph` (base class for graph), constructing edges between the defined nodes, and with a State (e.g., `MessagesState`), 287 | - Consider implementing an edge to `END` node (automatically included in every graph) to manually define when to terminate the workflow 288 | 5. Define a variable for the compiled graph using `graph.compile()` 289 | 6. Create a new entry in `langgraph.json` in the format of `path_to_graph_file.py:compiled_graph_var` 290 | 291 | ```python 292 | # 1. Create the graph file: graph.py 293 | from langchain_anthropic import ChatAnthropic 294 | from langchain_core.messages import HumanMessage 295 | from langgraph.prebuilt import create_react_agent 296 | from langgraph.graph import MessagesState, StateGraph, END 297 | 298 | # 2. Define agents 299 | from .tools import multiply, multiply_by_max # Load the example tools above 300 | 301 | llm = ChatAnthropic(model="claude-3-5-haiku-latest", temperature=0.0) 302 | tools = [multiply, multiply_by_max] # Define the tools for the agent using the created tools above 303 | problem_solver_agent = create_react_agent( 304 | llm, 305 | tools=problem_decoder_tools, # Required for ReACT agents 306 | prompt="You must solve the given problems!" # Optional system prompt 307 | ) 308 | 309 | # 3. Define nodes 310 | def problem_solver_node(state: MessagesState): 311 | response = problem_solver_agent.invoke(state) # Invoke the problem solver on the messages 312 | new_messages = response["messages"][len(state["messages"]):] # Simple processing: only get the new messages from this LLM invocation 313 | return new_messages 314 | 315 | # 4. Build the graph 316 | workflow = StateGraph(MessagesState) 317 | workflow.add_edge(START, "problem_solver") # Begin the workflow by entering the problem_solver 318 | # Add the problem_solver agent node 319 | workflow.add_node("problem_solver", problem_solver_node) 320 | # Add the END node to terminate the workflow after the problem_solver is done 321 | workflow.add_edge("problem_solver", END) 322 | 323 | # 5. Compile the graph 324 | graph = workflow.compile() 325 | 326 | # Example prompts 327 | problem_solver_agent.invoke({"messages": [ 328 | HumanMessage(content="What is 2*5") 329 | ]}) 330 | problem_solver_agent.invoke({"messages": [ 331 | HumanMessage(content="What is 2 mutlipled by the max of this list: [5, 10, 15]") 332 | ]}) 333 | ``` 334 | - Step 6: Define the graph in langgraph.json 335 | ```json 336 | { 337 | "dependencies": ["."], 338 | "graphs": { 339 | "demo_graph": "graph.py:graph", 340 | "hierarchy_graph": "./src/agent/hierarchy_graph_demo.py:hierarchy_graph" 341 | }, 342 | "env": ".env" 343 | } 344 | ``` 345 | 346 | - Additional information can be found via the Langgraph Documentation, or view the related graph files in our project 347 | 348 | ## Creating subgraphs 349 | - Subgraphs (graphs within the graph) can be created 350 | - An example of this is our Hierarchy Demo graph, which uses the Issue resolve graph as a subgraph 351 | ```python 352 | # hierarchy_graph_demo.py 353 | ... 354 | from langgraph.graph import END, START, StateGraph 355 | from agent.supervisor_graph_demo import issue_resolve_graph 356 | ... 357 | builder = StateGraph(CustomState) 358 | builder.add_node( 359 | "issue_resolve_graph", 360 | issue_resolve_graph, 361 | destinations=({"mam_node": "mam_node-issue_resolve_graph"}), 362 | ) # Creates the issue_resolve_graph node subgraph, and define an edge from mam_node to issue_resolve_graph 363 | builder.add_node( 364 | "mam_node", 365 | mam_node, 366 | destinations=( 367 | { 368 | "reviewer_node": "mam_node-reviewer_node", 369 | "issue_resolve_graph": "mam_node-issue_resolve_graph", 370 | } 371 | ), 372 | ) # Create the multi-agent manager node and define edges between mam_node to reviewer_node, and mam_node back to issue_resolve_graph 373 | builder.add_node( 374 | "reviewer_node", 375 | reviewer_node, 376 | destinations=({"mam_node": "mam_node-reviewer_node"}), 377 | ) # Create the reviewer node and define an edge from mam_node to reviewer node 378 | builder.add_edge(START, "mam_node") # Define start 379 | builder.add_edge("issue_resolve_graph", "mam_node") # Finally add the edge from issue_resolve_graph and mam_node 380 | hierarchy_graph = builder.compile() # Compile the graph 381 | ``` 382 | - Note that a subgraph is also created when creating a node that includes a ReACT based agent as mentioned above 383 | - A reACT agent subgraph consits of nodes including: 384 | - `__start__`: Defines the start of the subgraph 385 | - `agent`: The agent that was defined and will be used within the node 386 | - `tools`: The tools bound to the agent via the `tools` kwarg 387 | - `__end__`: Defines the end of the subgraph 388 | - Note that conditional edges are illustrated via dashed lines, and mean that the agent will decide on its own whether to transition to that node on its own depending on certain critera (e.g., if it recognizes it requires tool use) 389 | - More info can be found on the [Langgraph Documentation: How to use subgraphs](https://langchain-ai.github.io/langgraph/how-tos/subgraph/), or view the Multi-agent Manager related graph file in our project (`src/agent/hierarchy_graph_demo.py`) 390 | 391 | # Additional Documentation 392 | 393 | 394 | ## File Structure 395 | ``` 396 | . 397 | ├── langgraph.json 398 | ├── pyproject.toml 399 | ├── src 400 | │ └── agent 401 | │ ├── constant.py 402 | │ ├── github_utils.py 403 | │ ├── hierarchy_graph_demo.py 404 | │ ├── __init__.py 405 | │ ├── llm.py 406 | │ ├── parsers.py 407 | │ ├── prompt 408 | │ │ ├── context_manager.py 409 | │ │ ├── __init__.py 410 | │ │ ├── mam.py 411 | │ │ ├── problem_decoder.py 412 | │ │ ├── problem_solver.py 413 | │ │ ├── reviewer.py 414 | │ │ ├── solution_mapper.py 415 | │ │ └── supervisor.py 416 | │ ├── runtime_config.py 417 | │ ├── state.py 418 | │ ├── supervisor_graph_demo.py 419 | │ ├── tool_set 420 | │ │ ├── constant.py 421 | │ │ ├── context_tools.py 422 | │ │ ├── edit_history.py 423 | │ │ ├── edit_tool.py 424 | │ │ ├── linter 425 | │ │ │ ├── base.py 426 | │ │ │ ├── impl 427 | │ │ │ │ ├── python.py 428 | │ │ │ │ ├── treesitter_compat.py 429 | │ │ │ │ └── treesitter.py 430 | │ │ │ ├── __init__.py 431 | │ │ │ ├── linter.py 432 | │ │ ├── oheditor.py 433 | │ │ ├── sepl_tools.py 434 | │ │ └── utils.py 435 | │ └── utils.py 436 | ``` 437 | ## Dir / 438 | ### langgraph.json 439 | - Defines configuration for the execution of the `langgraph dev` command, currently including: 440 | - The Python file(s) containing LangGraph graphs and the variable name(s) in the corresponding file(s) defining the graph 441 | - The location of the `.env` file 442 | - More information can be found in [LangGraph Application Structure](https://langchain-ai.github.io/langgraph/concepts/application_structure/) 443 | ### pyproject.toml 444 | - Defines all dependencies to run the prototype, which should be changed accordingly to corresponding additional dependencies 445 | 446 | ## Dir src/agent/ 447 | - Contains files defining the main workflow and structure of the prototype 448 | ### constant.py 449 | - Defines several constant variables used throughout the prototype. 450 | - `RUNTIME_DIR`: Defines the local location where files will be stored 451 | - `PATCH_RESULT_DIR`: Defines where resulting patches will be stored. 452 | - `REQUEST_TIMEOUT`: Defines the amount of seconds before web requests via `requests` library timeout. 453 | - `PY_LANGUAGE`,`JAVA_LANGUAGE`: Defines tree-sitter parsers used for file indexing. 454 | ### github_utils.py 455 | - Defines functions for using Github API to collect git-based information (e.g., issue report) 456 | - `get_issue_description`: Retrieves the issue description given the owner, project name, and issue ID 457 | - `get_issue_close_commit`: Retrieves the commit that closed the pull request corresponding to a given issue 458 | - Remember to define `GITHUB_TOKEN` environment variable in the `.env` for expected behaviour of this functionality 459 | ### llm.py 460 | - Defines the LLM based on the `LLM_PROVIDER` and `LLM_MODEL` env vars. 461 | - `create_llm`: Creates the LLM according to `LLM_PROVIDER` and `LLM_MODEL` env vars 462 | - Remember to define `LLM_PROVIDER`, `LLM_MODEL`, and the corresponding API token environment variables in the `.env` for expected behaviour of this functionality 463 | ### parsers.py 464 | - Defines parsers used to extract information from LLM responses 465 | - `relevant_file_explanations_parser`: A parser to extract file paths and explanations from JSON formatted LLM responses 466 | ### runtime_config.py 467 | - Handles all runtime environment configuration setup. Currently supports loading runtime environment using a GitHub issue URL. 468 | - `RuntimeConfig`: Singleton class to hold and setup runtime configuration. Each configuration loading entry point starts with `load_from`. 469 | - `RuntimeConfig.load_from_github_issue_url`: Setup the runtime config based on a given issue UR; 470 | Args: 471 | issue_url: The given issue URL 472 | ### state.py 473 | - Defines the custom state structures for the prototype. 474 | ### utils.py 475 | - Defines various util functions for the prototype. 476 | ## Dir src/agent/prompt/ 477 | - Contains files defining the prompts for each agent, and prompts for other steps of the prototype (multi-agent manager, supervisor, problem decoder, solution mapper, problem solver, reviewer) 478 | 479 | ### context_manager.py 480 | - Contains prompts relevant to context management 481 | - `RELEVANT_FILE_EXPLANATION_SYSTEM_PROMPT`: A prompt used in `search_relevant_files` tool to ask LLM to explain the relevancy of files retrieved from the vector DB. 482 | 483 | ## Dir src/agent/tool_set/ 484 | - Contains files defining the tools used by agents 485 | ### constant.py 486 | - Defines several constant variables used throughout the tools. 487 | ### context_tools.py 488 | - Defines context management tools 489 | - `EMBEDDING_FUNCTION`: Defines the embedding model for the Vector DB with use of the `search_relevant_files` tool. Currently uses `OpenAIEmbeddings` and requires `OPENAI_API_KEY` environment variable in `.env`. 490 | - `PROJECT_KNOWLEDGE_TEXT_SPLITTER`: Defines the text splitter for the Vector DB with use of the `search_relevant_files` tool. 491 | - `create_project_knowledge`: Creates the project knowledge component. Indexes all Java and Python files in the directory of the corresponding issue. Builds a local VectorDB using `ChromaDB` at the location defined in `src/agent/config.py:RUNTIME_DIR`. The algorithm differentiates between functions/methods and other code by using `tree-sitter`. Used by the `search_relevant_files` tool. 492 | - `summarizer`: Summarizes the information of the chat history/workflow and aggregates it into detailed steps. Used at each step in the supervisor agent. 493 | 494 | ### edit_history.py 495 | - History management for file edits with disk-based storage and memory constraints for OHEditor 496 | - Adapted from OpenHands file editor. For more information refer to https://github.com/All-Hands-AI/openhands-aci/blob/main/openhands_aci/editor/editor.py 497 | 498 | ### edit_tool.py 499 | - Enables file edits via OHEditor 500 | - Adapted from OpenHands file editor. For more information refer to https://github.com/All-Hands-AI/openhands-aci/blob/main/openhands_aci/editor/editor.py 501 | 502 | ### oheditor.py 503 | - OHEditor is a filesystem editor tool that allows the agent to 504 | - view 505 | - create 506 | - navigate 507 | - edit files 508 | - Adapted from OpenHands file editor. For more information refer to https://github.com/All-Hands-AI/openhands-aci/blob/main/openhands_aci/editor/editor.py 509 | 510 | ### sepl_tools.py 511 | - Defines software Engineering Project Lifecycle tools 512 | - `extract_git_diff_local`: Executes and returns the `git diff` command in a local runtime environment. 513 | - `save_git_diff`: Exports the result of the `git diff` command. 514 | ### utils.py 515 | - Defines util functions used by OHEditor 516 | - Adapted from OpenHands file editor. For more information refer to https://github.com/All-Hands-AI/openhands-aci/blob/main/openhands_aci/editor/editor.py 517 | ## Dir src/agent/tool_set/linter/ 518 | - Defines functionality for the linter used in OHEditor 519 | - Adapted from Aider linter. For more information refer to https://github.com/paul-gauthier/aider/blob/main/aider/linter.py 520 | 521 | 522 | # Additional Notes 523 | 524 | ## Space Limitations 525 | ### Cloned Repos 526 | - When submitting a URL, a local copy of the repository will be cloned at a specific commit, either: 527 | - The most recent commit, or 528 | - The commit before the commit that was merged in the pull request fixing the issue (if applicable) 529 | - This will take up considerable amount of space overtime, so be sure to clean the directory corresponding to `src/agent/constant.py:RUNTIME_DIR` 530 | 531 | ### `search_relevant_files` Vector DBs 532 | - If using the `search_relevant_files` tool, a vector/Chroma DB will be built for each issue submitted 533 | - This will take up considerable amount of space overtime, so be sure to clean the directory corresponding to `src/agent/constant.py:RUNTIME_DIR` 534 | -------------------------------------------------------------------------------- /docs/Lingxi Technical Report 2505.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nimasteryang/Lingxi/8c2f28d1a53468fc56b2904e0aa4bead42da9c30/docs/Lingxi Technical Report 2505.pdf -------------------------------------------------------------------------------- /imgs/AddMsg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nimasteryang/Lingxi/8c2f28d1a53468fc56b2904e0aa4bead42da9c30/imgs/AddMsg.png -------------------------------------------------------------------------------- /imgs/GraphSelect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nimasteryang/Lingxi/8c2f28d1a53468fc56b2904e0aa4bead42da9c30/imgs/GraphSelect.png -------------------------------------------------------------------------------- /imgs/HILenable.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nimasteryang/Lingxi/8c2f28d1a53468fc56b2904e0aa4bead42da9c30/imgs/HILenable.png -------------------------------------------------------------------------------- /imgs/Lingxi_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nimasteryang/Lingxi/8c2f28d1a53468fc56b2904e0aa4bead42da9c30/imgs/Lingxi_logo.png -------------------------------------------------------------------------------- /imgs/graph1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nimasteryang/Lingxi/8c2f28d1a53468fc56b2904e0aa4bead42da9c30/imgs/graph1.png -------------------------------------------------------------------------------- /imgs/graph2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nimasteryang/Lingxi/8c2f28d1a53468fc56b2904e0aa4bead42da9c30/imgs/graph2.png -------------------------------------------------------------------------------- /imgs/graph3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nimasteryang/Lingxi/8c2f28d1a53468fc56b2904e0aa4bead42da9c30/imgs/graph3.png -------------------------------------------------------------------------------- /langgraph.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": ["."], 3 | "graphs": { 4 | "issue_resolve_graph": "./src/agent/supervisor_graph_demo.py:issue_resolve_graph", 5 | "hierarchy_graph": "./src/agent/hierarchy_graph_demo.py:hierarchy_graph" 6 | }, 7 | "env": ".env" 8 | } 9 | 10 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "codexray_lite" 3 | version = "0.1.1" 4 | description = "CodeXRay Lite for a multi-agent software engineering tasks automation framework" 5 | authors = [ 6 | { name = "Xu Yang" }, 7 | { name = "Wenhan Zhu" }, 8 | { name = "Michael Pacheco" }, 9 | { name = "Jiayuan Zhou" }, 10 | ] 11 | readme = "README.md" 12 | license = { text = "MIT" } 13 | requires-python = ">=3.12" 14 | dependencies = [ 15 | "langgraph>=0.2.75", 16 | "langsmith", 17 | "langgraph-cli[inmem]", 18 | "langchain_anthropic>=0.3.8", 19 | "langchain_openai", 20 | "langchain_community", 21 | "langchain_chroma", 22 | "langchain_experimental", 23 | "prompt_toolkit", 24 | "docker", 25 | "gitpython", 26 | "ruff", 27 | "pre-commit", 28 | "pytest", 29 | "python-dotenv>=1.0.1", 30 | "datasets", 31 | "tree-sitter", 32 | "tree-sitter-languages", 33 | "tree-sitter-python", 34 | "tree-sitter-java", 35 | "chromadb", 36 | "diskcache", 37 | "langchain_deepseek>=0.1.2", 38 | "langgraph-checkpoint-sqlite", 39 | "grep-ast", 40 | ] 41 | 42 | 43 | [project.optional-dependencies] 44 | dev = ["mypy>=1.11.1", "ruff>=0.6.1"] 45 | 46 | [build-system] 47 | requires = ["setuptools>=73.0.0", "wheel"] 48 | build-backend = "setuptools.build_meta" 49 | 50 | [tool.setuptools] 51 | packages = ["langgraph.templates.agent", "agent"] 52 | [tool.setuptools.package-dir] 53 | "langgraph.templates.agent" = "src/agent" 54 | "agent" = "src/agent" 55 | 56 | 57 | [tool.setuptools.package-data] 58 | "*" = ["py.typed"] 59 | 60 | [tool.ruff] 61 | lint.select = [ 62 | "E", # pycodestyle 63 | "F", # pyflakes 64 | "I", # isort 65 | # "D", # pydocstyle 66 | # "D401", # First line should be in imperative mood 67 | # "T201", # W `print` found 68 | "UP", 69 | ] 70 | lint.ignore = [ 71 | "UP006", 72 | "UP007", 73 | # We actually do want to import from typing_extensions 74 | "UP031", # str formatter 75 | "UP035", 76 | # Relax the convention by _not_ requiring documentation for every function parameter. 77 | "D417", 78 | "E501", 79 | "E402", # import at top 80 | "F841", # unused variable 81 | # "W503", # black's recommended settings 82 | ] 83 | line-length=109 84 | [tool.ruff.lint.per-file-ignores] 85 | "tests/*" = ["D", "UP"] 86 | [tool.ruff.lint.pydocstyle] 87 | convention = "google" 88 | -------------------------------------------------------------------------------- /src/agent/README.md: -------------------------------------------------------------------------------- 1 | # Setup 2 | 3 | ## Part 1: Dependencies 4 | - Dependencies are located in `pyproject.toml`, and can be installed with the following command: 5 | - `cd codexray_lite && pip install -e .` 6 | - The installation command can be ignored if using `uv` package manager and `uv run` later on (see Run section for reference) 7 | - It is suggested to use some package manager (e.g., venv, pipenv, uv, ...) 8 | ## Part 2: Environment Config 9 | - The prototype requires the following environment variables to run: 10 | 1. `LLM_PROVIDER`: The provider of the LLM to be used(e.g., "anthropic", "openai", "deepseek") 11 | 2. `LLM_MODEL`: The model name of the LLM to be used (e.g., "claude-3-5-haiku-latest") 12 | 3. The corresponding `LLM_PROVIDER` API key to be used 13 | - `LLM_PROVIDER="anthropic"`, set `ANTHROPIC_API_KEY` 14 | - `LLM_PROVIDER="openai"`, set `OPENAI_API_KEY` 15 | - `LLM_PROVIDER="deepseek"`, set `DEEPSEEK_API_KEY` 16 | - Additional provider integrations can be added, see [Langchain Chat Models](https://python.langchain.com/docs/integrations/chat/) 17 | 4. `OPENAI_API_KEY`: The OpenAI API key to be used. Required for generating embeddings to build project knowledge. 18 | 5. `GITHUB_TOKEN`: A GitHub API token used to automatically collect issue report information. It can be generated using a GitHub account through the following menus: 19 | - Profile > Settings > Developer Settings > Personal access tokens > Fine-grained tokens 20 | - Place these environment variables in the `.env` file in the root of the project. This file will be loaded by the prototype on execution 21 | # Run 22 | - The prototype can be ran with either of the following commands: 23 | - `langgraph dev --no-reload` (if installed via `pip`) 24 | - `uv run --env-file .env langgraph dev --no-reload` (if using `uv`) 25 | - The prototype will create a local instance of LangGraph Studio and will open a browser instance to the page once the prototype is run 26 | - Once in the UI, select the graph to run at the top left 27 | ![[GraphSelect.png]] 28 | 29 | - Add a Message by clicking "+ Message" button, and add the Github issue URL. An example URL is: https://github.com/gitpython-developers/GitPython/issues/1977 30 | ![[AddMsg.png]] 31 | 32 | - Finally click the Submit button 33 | - Note that human feedback functionality is disabled be default, it can be enabled by clicking the checkbox in the LangGraph Studio UI before submitting the issue URL: 34 | ![[Pasted image 20250402150326.png]] 35 | # Agents 36 | - The prototype currently consists of five main agents across the graphs and implemented in various LangGraph nodes 37 | ## Multi-Agent Manager 38 | - The multi-agent manager coordinates between the issue resolver and reviewer agents to provide feedback and or guidance to each step. 39 | - Only used in `src/agent/hierarchy_graph_demo.py`. 40 | - Prompt in `src/agent/prompt/mam.py` 41 | ## Supervisor 42 | - The supervisor oversees the entire workflow of solving the issue, and decides which agent to route to depending on the current progress of the issue. 43 | - Prompt in `src/agent/prompt/supervisor.py` 44 | ## Problem Decoder 45 | - The problem decoder analyzes and understands the issue requirements to generate a problem statement consisting of a topic question, codebase representation (i.e., bug localization), the current behaviour, and the expected behaviour. 46 | - Uses tools `[view_directory, search_relevant_files, view_file_content]` 47 | - Prompt in `src/agent/prompt/problem_decoder.py` 48 | 49 | ## Solution Mapper 50 | - The solution mapper reviews the information from the problem decoder with respect to the localized bug and the codebase, to then generate a detailed code change plan for each of the files and or functions. 51 | - Uses tools `[view_directory, search_relevant_files, view_file_content]` 52 | - Prompt in `src/agent/prompt/solution_mapper.py` 53 | ## Problem Solver 54 | - The problem solver implements the proposed change plan from the solution mapper by modifying files 55 | - Uses tools `[view_directory, search_relevant_files, str_replace_editor]` 56 | - Prompt in `src/agent/prompt/problem_solver.py` 57 | ## Reviewer 58 | - The reviewer verifies the proposed fix resulting from the issue resolver agent by generating and executing test cases. 59 | - Only used in `src/agent/hierarchy_graph_demo.py` 60 | - Uses tools `[view_directory, search_relevant_files, view_file_content, run_shell_cmd]` 61 | - Prompt in `src/agent/prompt/reviewer.py` 62 | 63 | # Tools 64 | - There are four main tools are defined and used by multiple agents in the workflow of the prototype 65 | - Note that tools used by agents are: 66 | - Defined with the `@tool` wrapper 67 | - Defined while constructing the agent via `create_react_agent` using`tools` parameters 68 | 69 | ## view_directory 70 | - View the file structure of the repository, including directories (marked with /). 71 | Automatically reduces depth if entries exceed 50. 72 | Args: 73 | dir_path (str): Starting directory. Defaults to './'. 74 | depth (Optional[int]): Maximum depth. None for unlimited. Defaults to None. 75 | Returns: 76 | List[str]: Sorted list of directories (with /) and files. 77 | - Defined in `src/agent/tool_set/sepl_tools.py` 78 | ## view_file_content 79 | - Read the content of the specified file. 80 | Parameters: 81 | file_name (str): File name relative to the git root directory. 82 | view_range (Optional[List[int]]): Optional list containing [start_line, end_line] to limit the lines displayed. 83 | Usage: 84 | - LLM should initially attempt to read the entire file content. 85 | - If the file is too large, LLM can use the `view_file_structure` tool to identify relevant code ranges, 86 | and then call this tool again specifying the `view_range` to read only the necessary lines. 87 | Returns: 88 | str: Content of the file or the specified line range. 89 | - Defined in `src/agent/tool_set/sepl_tools.py` 90 | 91 | ## run_shell_cmd 92 | - Run a list of shell commands in sequential order and return the stdout results, your working directory is the root of the project 93 | Args: 94 | commands (List[str]): A list of shell commands to be run in sequential order. 95 | config (RunnableConfig) The runtime configuration. 96 | Returns: 97 | str: Result of running the commands. 98 | - Defined in `src/agent/tool_set/sepl_tools.py` 99 | 100 | 101 | ## search_relevant_files 102 | - Given a query search string (for example, the issue report description, filenames, etc), search for relevant code snippets of files in the project by calculating embedding similarity between the query and code snippets in a vector database. 103 | Args: 104 | query: A search string (for example, the issue report description, filenames, etc), to be used to find relevant files and functions. 105 | Returns: 106 | explanations (str): Each retrieved file with an explanation of how the file is relevant to the query. 107 | - This tool builds a local Vector DB using `ChromaDB` at the location defined in `src/agent/config.py:RUNTIME_DIR` by indexing all Java/Python files in the project. 108 | - Used by the `search_relevant_files` tool. 109 | - Defined in `src/agent/tool_set/context_tools.py` 110 | ## str_replace_editor 111 | - Custom editing tool for viewing, creating and editing files in plain-text format 112 | * State is persistent across command calls and discussions with the user 113 | * If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep 114 | * The `create` command cannot be used if the specified `path` already exists as a file 115 | * If a `command` generates a long output, it will be truncated and marked with `` 116 | Args: 117 | command (str): The commands to run. Allowed options are: `view`, `create`, `str_replace`, `insert`. 118 | path (str): Absolute path to file or directory, e.g. `/workspace/file.py` or `/workspace`. 119 | file_text (Optional[str]): Required parameter of `create` command, with the content of the file to be created. 120 | old_str (Optional[str]): Required parameter of `str_replace` command containing the string in `path` to replace. 121 | new_str (Optional[str]): Optional parameter of `str_replace` command containing the new string (if not given, no string will be added). Required parameter of `insert` command containing the string to insert. 122 | insert_line (Optional[int]): Required parameter of `insert` command. The `new_str` will be inserted AFTER the line `insert_line` of `path`. 123 | view_range (Optional[List[int]]): Optional parameter of `view` command when `path` points to a file. If none is given, the full file is shown. If provided, the file will be shown in the indicated line number range, e.g. [100, 600] will show content between line 100 and 600. Indexing at 1 to start. Setting `[start_line, -1]` shows all lines from `start_line` to the end of the file. Unless you are sure about the line numbers, otherwise, do not set this parameter and use the `view` command to view the whole file. 124 | - Defined in `src/agent/tool_set/edit_tool.py` 125 | 126 | 127 | # Graphs 128 | 129 | - There are currently two developed graphs in the prototype 130 | 1. `src/agent/supervisor_graph_demo.py` 131 | 2. `src/agent/hierarchy_graph_demo.py` 132 | 133 | ## Supervisor Graph 134 | ![[Pasted image 20250402102105.png]] 135 | - This graph defines a workflow to resolve an issue given the issue requirements, with use of a supervisor agent and integrated with human feedback 136 | - The graph is defined in the `issue_resolve_graph` variable and contains the following six nodes: 137 | 1. `input_handler_node`: This node handles the start of the workflow by setting up the runtime environment given the Github issue report URL. 138 | 2. `supervisor_node`: This node executes the supervisor agent. 139 | 3. `problem_decoder`: This node executes the problem decoder agent. 140 | 4. `solution_mapper`: This node executes the solution mapper agent. 141 | 5. `problem_solver`: This node executes the problem solver agent. 142 | - Human feedback: After each node is executed, the result is provided to the user for human feedback if enabled 143 | - Recall human feedback is disabled by default, see Run section on how to enable it 144 | - If the human provides feedback, the same node will be re-executed with the additional feedback information 145 | - If the human provides no feedback, the workflow will continue 146 | 147 | 148 | ## Hierarchy Graph 149 | ![[Pasted image 20250402102205.png]] 150 | - This graph defines a workflow also to resolve an issue, though also implements a multi-agent manager to coordinate between an issue resolver agent (i.e. the graph from [[CodeXRay Lite Documentation#Supervisor Graph]]) and a reviewer agent, to review the result of the issue resolver agent, and also includes human feedback. 151 | - The graph is defined in the `hierarchy_graph` variable and contains the following three nodes: 152 | 1. `mam_node`: This node executes the multi-agent manager agent. 153 | 2. `issue_resolve_graph`: This node executes the issue resolver agent, combining all nodes and agents defined in [[CodeXRay Lite Documentation#Supervisor Graph]] 154 | 3. `reviewer_node`: This node executes the reviewer agent. 155 | 156 | # Development 157 | 158 | ## Adding agents 159 | - Agents are generally defined via: 160 | 1. System prompt (optional) 161 | 2. User prompt or past conversation history 162 | 3. Tools (optional) 163 | - In Langgraph, there are multiple ways to create and define agents depending on their purpose 164 | - See [Langgraph Documentation: Build an Agent](https://python.langchain.com/docs/tutorials/agents/) for more information, or view the related graph files in our project 165 | ### Tool-less Agents 166 | - These agents are defined to be used without tools 167 | - In this case, the LLM can be used directly 168 | 169 | ```python 170 | from langchain_anthropic import ChatAnthropic 171 | llm = ChatAnthropic(model="claude-3-5-haiku-latest", temperature=0.0) 172 | response = llm.invoke("Tell me a joke") # Single string input 173 | response = llm.invoke(["What is 2+2", "What is the previous result times 5"]) # List of string as input 174 | ``` 175 | ### ReACT Agents 176 | - These agents are defined with tools which they will decide to use on their own time 177 | - In this case, the agent should be created using [`create_react_agent`](https://langchain-ai.github.io/langgraph/reference/prebuilt/#langgraph.prebuilt.chat_agent_executor.create_react_agent), and will expect different input (see [[CodeXRay Lite Documentation#Agent Expected Input and State]]) 178 | ```python 179 | from langchain_anthropic import ChatAnthropic 180 | from langgraph.prebuilt import create_react_agent 181 | from langchain_core.messages import HumanMessage 182 | 183 | llm = ChatAnthropic(model="claude-3-5-haiku-latest", temperature=0.0) # Create LLM 184 | tools = [math_tool] # Tool example 185 | problem_solver_agent = create_react_agent( 186 | llm, 187 | tools=problem_decoder_tools, # Required for ReACT agents 188 | prompt="You must solve the given problems!" # Optional system prompt 189 | ) 190 | problem_solver_agent.invoke({"messages": [ 191 | HumanMessage(content="What is 2+2") 192 | ]}) # How to invoke the agent using user prompt "What is 2+2" 193 | ``` 194 | - ReACT agents also create a subgraph that includes the looks like so: 195 | ![[Pasted image 20250402135525.png]] 196 | ### Agent Expected Input and LangGraph State 197 | - Different agent creation methods will expect different inputs 198 | - An agent without tools used directly from a class such as `ChatAnthropic` can expect relatively flexible inputs: 199 | - A string 200 | - A list of strings 201 | - A list of `BaseMessage` - usually their subclasses (i.e., `SystemMessage`, `HumanMessage`, `AIMessage`, ..) 202 | - When defining and invoking a ReACT agent, it will instead expect more structured input involving a `MessagesState` object, known as the LangGraph state 203 | - The `MessagesState` holds and defines the conversation history of the current run, as well as any additional variables if a custom State class is defined (e.g., see `src/agent/state.py:CustomState) 204 | - The `MessagesState` must be a dictionary (`dict`, `AnnotatedDict`) with the `messages` (the convo history) key as a list of `BaseMessages` - usually their subclasses (i.e., `SystemMessage`, `HumanMessage`, `AIMessage`, ..) 205 | - The messages in the state is updated automatically by the Langgraph graph within the defined nodes in the graph 206 | - Custom variables can be updated when returning from each node using the `update` kwarg in the `Command` class (e.g., `Command(update={"custom_var": 2"})) 207 | - It is important to note that the `messages` of `MessagesState` follows strict structure when being invoked by ReACT agents and may be difficult to alter 208 | - The main reason is that if an agent uses a tool, the `AIMessage` of the agent will be proceeded by a `ToolMessage` containing information of the corresponding tool 209 | - If one or the other message is removed from the state, the workflow will break 210 | 211 | ### System Prompts 212 | - Note that a system prompt can take different forms in LangGraph: 213 | - `prompt` kwarg of `create_react_agent` (str): This is converted to a SystemMessage and added to the beginning of the list of messages in state["messages"]. 214 | - SystemMessage: this is added to the beginning of the list of messages in state["messages"]. 215 | - Callable: This function should take in full graph state and the output is then passed to the language model. 216 | - Runnable: This runnable should take in full graph state and the output is then passed to the language model. 217 | 218 | ## Adding tools 219 | - Tools can be considered as Python functions that can be used by agents/LLM (defined via `create_react_agent`) 220 | - These are defined using the `@tool` decorator (`from langchain_core.tools import tool`), and require docstring explaining the function, the args, and expected return info 221 | - This information is passed to the agent as context for using the function 222 | - Instead of defining args in the docstring, type hints can be used which will be parsed and sent to the agent as well 223 | ```python 224 | # tools.py 225 | from langchain_core.tools import tool 226 | 227 | # Via type hints 228 | @tool 229 | def multiply(a: int, b: int) -> int: 230 | """Multiply two numbers.""" 231 | return a * b 232 | 233 | from typing import Annotated, List 234 | # Via annotated type hints 235 | @tool 236 | def multiply_by_max( 237 | a: Annotated[int, "scale factor"], 238 | b: Annotated[List[int], "list of ints over which to take maximum"], 239 | ) -> int: 240 | """Multiply a by the maximum of b.""" 241 | return a * max(b) 242 | ``` 243 | 244 | - Once tools are created, they can be linked to agents via the `tools` keyword in the `create_react_agent` constructor: 245 | ```python 246 | # graph.py 247 | from langchain_anthropic import ChatAnthropic 248 | from langchain_core.messages import HumanMessage 249 | from langgraph.prebuilt import create_react_agent 250 | 251 | from tools import multiply, multiply_by_max # Load the example tools above 252 | 253 | llm = ChatAnthropic(model="claude-3-5-haiku-latest", temperature=0.0) 254 | tools = [multiply, multiply_by_max] # Define the tools for the agent using the created tools above 255 | problem_solver_agent = create_react_agent( 256 | llm, 257 | tools=problem_decoder_tools, # Required for ReACT agents 258 | prompt="You must solve the given problems!" # Optional system prompt 259 | ) 260 | 261 | # Example prompts 262 | problem_solver_agent.invoke({"messages": [ 263 | HumanMessage(content="What is 2*5") 264 | ]}) 265 | problem_solver_agent.invoke({"messages": [ 266 | HumanMessage(content="What is 2 mutlipled by the max of this list: [5, 10, 15]") 267 | ]}) 268 | ``` 269 | - Additional info can be read on [Langchain Documentation: How to create tools](https://python.langchain.com/docs/how_to/custom_tools/), or view the related tool files in our project 270 | 271 | ## Creating graphs 272 | - A graph defines a multi-agent workflow, that generally follows these steps: 273 | 1. Create a Python file for the graph 274 | 2. Define agents 275 | 3. Define nodes for each agents where they will be invoked, containing the logic that will happen before/after their individual invocation. 276 | 4. Build the graph using `StateGraph` (base class for graph), constructing edges between the defined nodes, and with a State (e.g., `MessagesState`), 277 | - Consider implementing an edge to `END` node (automatically included in every graph) to manually define when to terminate the workflow 278 | 5. Define a variable for the compiled graph using `graph.compile()` 279 | 6. Create a new entry in `langgraph.json` in the format of `path_to_graph_file.py:compiled_graph_var` 280 | 281 | ```python 282 | # 1. Create the graph file: graph.py 283 | from langchain_anthropic import ChatAnthropic 284 | from langchain_core.messages import HumanMessage 285 | from langgraph.prebuilt import create_react_agent 286 | from langgraph.graph import MessagesState, StateGraph, END 287 | 288 | # 2. Define agents 289 | from .tools import multiply, multiply_by_max # Load the example tools above 290 | 291 | llm = ChatAnthropic(model="claude-3-5-haiku-latest", temperature=0.0) 292 | tools = [multiply, multiply_by_max] # Define the tools for the agent using the created tools above 293 | problem_solver_agent = create_react_agent( 294 | llm, 295 | tools=problem_decoder_tools, # Required for ReACT agents 296 | prompt="You must solve the given problems!" # Optional system prompt 297 | ) 298 | 299 | # 3. Define nodes 300 | def problem_solver_node(state: MessagesState): 301 | response = problem_solver_agent.invoke(state) # Invoke the problem solver on the messages 302 | new_messages = response["messages"][len(state["messages"]):] # Simple processing: only get the new messages from this LLM invocation 303 | return new_messages 304 | 305 | # 4. Build the graph 306 | workflow = StateGraph(MessagesState) 307 | workflow.add_edge(START, "problem_solver") # Begin the workflow by entering the problem_solver 308 | # Add the problem_solver agent node 309 | workflow.add_node("problem_solver", problem_solver_node) 310 | # Add the END node to terminate the workflow after the problem_solver is done 311 | workflow.add_edge("problem_solver", END) 312 | 313 | # 5. Compile the graph 314 | graph = workflow.compile() 315 | 316 | # Example prompts 317 | problem_solver_agent.invoke({"messages": [ 318 | HumanMessage(content="What is 2*5") 319 | ]}) 320 | problem_solver_agent.invoke({"messages": [ 321 | HumanMessage(content="What is 2 mutlipled by the max of this list: [5, 10, 15]") 322 | ]}) 323 | ``` 324 | 325 | ```json 326 | // Step 6: Define the graph in langgraph.json 327 | { 328 | "dependencies": ["."], 329 | "graphs": { 330 | "demo_graph": "graph.py:graph", 331 | "hierarchy_graph": "./src/agent/hierarchy_graph_demo.py:hierarchy_graph" 332 | }, 333 | "env": ".env" 334 | } 335 | ``` 336 | 337 | - Additional information can be found via the Langgraph Documentation, or view the related graph files in our project 338 | 339 | ## Creating subgraphs 340 | - Subgraphs (graphs within the graph) can be created 341 | - An example of this is our Hierarchy Demo graph, which uses the Issue resolve graph as a subgraph 342 | ```python 343 | # hierarchy_graph_demo.py 344 | ... 345 | from langgraph.graph import END, START, StateGraph 346 | from agent.supervisor_graph_demo import issue_resolve_graph 347 | ... 348 | builder = StateGraph(CustomState) 349 | builder.add_node( 350 | "issue_resolve_graph", 351 | issue_resolve_graph, 352 | destinations=({"mam_node": "mam_node-issue_resolve_graph"}), 353 | ) # Creates the issue_resolve_graph node subgraph, and define an edge from mam_node to issue_resolve_graph 354 | builder.add_node( 355 | "mam_node", 356 | mam_node, 357 | destinations=( 358 | { 359 | "reviewer_node": "mam_node-reviewer_node", 360 | "issue_resolve_graph": "mam_node-issue_resolve_graph", 361 | } 362 | ), 363 | ) # Create the multi-agent manager node and define edges between mam_node to reviewer_node, and mam_node back to issue_resolve_graph 364 | builder.add_node( 365 | "reviewer_node", 366 | reviewer_node, 367 | destinations=({"mam_node": "mam_node-reviewer_node"}), 368 | ) # Create the reviewer node and define an edge from mam_node to reviewer node 369 | builder.add_edge(START, "mam_node") # Define start 370 | builder.add_edge("issue_resolve_graph", "mam_node") # Finally add the edge from issue_resolve_graph and mam_node 371 | hierarchy_graph = builder.compile() # Compile the graph 372 | ``` 373 | - Note that a subgraph is also created when creating a node that includes a ReACT based agent, and produces the following subgraph: 374 | ![[Pasted image 20250402135525.png]] 375 | - The subgraph has the nodes including: 376 | - `__start__`: Defines the start of the subgraph 377 | - `agent`: The agent that was defined and will be used within the node 378 | - `tools`: The tools bound to the agent via the `tools` kwarg 379 | - `__end__`: Defines the end of the subgraph 380 | - Note that conditional edges are illustrated via dashed lines, and mean that the agent will decide on its own whether to transition to that node on its own depending on certain critera (e.g., if it recognizes it requires tool use) 381 | - More info can be found on the [Langgraph Documentation: How to use subgraphs](https://langchain-ai.github.io/langgraph/how-tos/subgraph/), or view the Multi-agent Manager related graph file in our project (`src/agent/hierarchy_graph_demo.py`) 382 | 383 | # Additional Documentation 384 | 385 | 386 | ## File Structure 387 | 388 | . 389 | ├── langgraph.json 390 | ├── pyproject.toml 391 | ├── src 392 | │   └── agent 393 | │   ├── constant.py 394 | │   ├── github_utils.py 395 | │   ├── hierarchy_graph_demo.py 396 | │   ├── __init__.py 397 | │   ├── llm.py 398 | │   ├── parsers.py 399 | │   ├── prompt 400 | │   │   ├── context_manager.py 401 | │   │   ├── __init__.py 402 | │   │   ├── mam.py 403 | │   │   ├── problem_decoder.py 404 | │   │   ├── problem_solver.py 405 | │   │   ├── reviewer.py 406 | │   │   ├── solution_mapper.py 407 | │   │   └── supervisor.py 408 | │   ├── runtime_config.py 409 | │   ├── state.py 410 | │   ├── supervisor_graph_demo.py 411 | │   ├── tool_set 412 | │   │   ├── constant.py 413 | │   │   ├── context_tools.py 414 | │   │   ├── edit_history.py 415 | │   │   ├── edit_tool.py 416 | │   │   ├── linter 417 | │   │   │   ├── base.py 418 | │   │   │   ├── impl 419 | │   │   │   │   ├── python.py 420 | │   │   │   │   ├── treesitter_compat.py 421 | │   │   │   │   └── treesitter.py 422 | │   │   │   ├── __init__.py 423 | │   │   │   ├── linter.py 424 | │   │   ├── oheditor.py 425 | │   │   ├── sepl_tools.py 426 | │   │   └── utils.py 427 | │   └── utils.py 428 | 429 | ## / 430 | ### langgraph.json 431 | - Defines configuration for the execution of the `langgraph dev` command, currently including: 432 | - The Python file(s) containing LangGraph graphs and the variable name(s) in the corresponding file(s) defining the graph 433 | - The location of the `.env` file 434 | - More information can be found in [LangGraph Application Structure](https://langchain-ai.github.io/langgraph/concepts/application_structure/) 435 | ### pyproject.toml 436 | - Defines all dependencies to run the prototype, which should be changed accordingly to corresponding additional dependencies 437 | 438 | ## src/agent/ 439 | - Contains files defining the main workflow and structure of the prototype 440 | ### constant.py 441 | - Defines several constant variables used throughout the prototype. 442 | - `RUNTIME_DIR`: Defines the local location where files will be stored 443 | - `PATCH_RESULT_DIR`: Defines where resulting patches will be stored. 444 | - `REQUEST_TIMEOUT`: Defines the amount of seconds before web requests via `requests` library timeout. 445 | - `PY_LANGUAGE`,`JAVA_LANGUAGE`: Defines tree-sitter parsers used for file indexing. 446 | ### github_utils.py 447 | - Defines functions for using Github API to collect git-based information (e.g., issue report) 448 | - `get_issue_description`: Retrieves the issue description given the owner, project name, and issue ID 449 | - `get_issue_close_commit`: Retrieves the commit that closed the pull request corresponding to a given issue 450 | - Remember to define `GITHUB_TOKEN` environment variable in the `.env` for expected behaviour of this functionality 451 | ### llm.py 452 | - Defines the LLM based on the `LLM_PROVIDER` and `LLM_MODEL` env vars. 453 | - `create_llm`: Creates the LLM according to `LLM_PROVIDER` and `LLM_MODEL` env vars 454 | - Remember to define `LLM_PROVIDER`, `LLM_MODEL`, and the corresponding API token environment variables in the `.env` for expected behaviour of this functionality 455 | ### parsers.py 456 | - Defines parsers used to extract information from LLM responses 457 | - `relevant_file_explanations_parser`: A parser to extract file paths and explanations from JSON formatted LLM responses 458 | ### runtime_config.py 459 | - Handles all runtime environment configuration setup. Currently supports loading runtime environment using a GitHub issue URL. 460 | - `RuntimeConfig`: Singleton class to hold and setup runtime configuration. Each configuration loading entry point starts with `load_from`. 461 | - `RuntimeConfig.load_from_github_issue_url`: Setup the runtime config based on a given issue UR; 462 | Args: 463 | issue_url: The given issue URL 464 | ## state.py 465 | - Defines the custom state structures for the prototype. 466 | ## utils.py 467 | - Defines various util functions for the prototype. 468 | ## src/agent/prompt/ 469 | - Contains files defining the prompts for each agent, and prompts for other steps of the prototype (multi-agent manager, supervisor, problem decoder, solution mapper, problem solver, reviewer) 470 | 471 | ### context_manager.py 472 | - Contains prompts relevant to context management 473 | - `RELEVANT_FILE_EXPLANATION_SYSTEM_PROMPT`: A prompt used in `search_relevant_files` tool to ask LLM to explain the relevancy of files retrieved from the vector DB. 474 | 475 | ## src/agent/tool_set/ 476 | - Contains files defining the tools used by agents 477 | ### constant.py 478 | - Defines several constant variables used throughout the tools. 479 | ### context_tools.py 480 | - Defines context management tools 481 | - `EMBEDDING_FUNCTION`: Defines the embedding model for the Vector DB with use of the `search_relevant_files` tool. Currently uses `OpenAIEmbeddings` and requires `OPENAI_API_KEY` environment variable in `.env`. 482 | - `PROJECT_KNOWLEDGE_TEXT_SPLITTER`: Defines the text splitter for the Vector DB with use of the `search_relevant_files` tool. 483 | - `create_project_knowledge`: Creates the project knowledge component. Indexes all Java and Python files in the directory of the corresponding issue. Builds a local VectorDB using `ChromaDB` at the location defined in `src/agent/config.py:RUNTIME_DIR`. The algorithm differentiates between functions/methods and other code by using `tree-sitter`. Used by the `search_relevant_files` tool. 484 | - `summarizer`: Summarizes the information of the chat history/workflow and aggregates it into detailed steps. Used at each step in the supervisor agent. 485 | 486 | ### edit_history.py 487 | - History management for file edits with disk-based storage and memory constraints for OHEditor 488 | - Adapted from OpenHands file editor. For more information refer to https://github.com/All-Hands-AI/openhands-aci/blob/main/openhands_aci/editor/editor.py 489 | 490 | ### edit_tool.py 491 | - Enables file edits via OHEditor 492 | - Adapted from OpenHands file editor. For more information refer to https://github.com/All-Hands-AI/openhands-aci/blob/main/openhands_aci/editor/editor.py 493 | 494 | ### oheditor.py 495 | - OHEditor is a filesystem editor tool that allows the agent to 496 | - view 497 | - create 498 | - navigate 499 | - edit files 500 | - Adapted from OpenHands file editor. For more information refer to https://github.com/All-Hands-AI/openhands-aci/blob/main/openhands_aci/editor/editor.py 501 | 502 | ### sepl_tools.py 503 | - Defines software Engineering Project Lifecycle tools 504 | - `extract_git_diff_local`: Executes and returns the `git diff` command in a local runtime environment. 505 | - `save_git_diff`: Exports the result of the `git diff` command. 506 | ### utils.py 507 | - Defines util functions used by OHEditor 508 | - Adapted from OpenHands file editor. For more information refer to https://github.com/All-Hands-AI/openhands-aci/blob/main/openhands_aci/editor/editor.py 509 | ## src/agent/tool_set/linter/ 510 | - Defines functionality for the linter used in OHEditor 511 | - Adapted from Aider linter. For more information refer to https://github.com/paul-gauthier/aider/blob/main/aider/linter.py 512 | 513 | 514 | # Additional Notes 515 | 516 | ## Space Limitations 517 | ### Cloned Repos 518 | - When submitting a URL, a local copy of the repository will be cloned at a specific commit, either: 519 | - The most recent commit, or 520 | - The commit before the commit that was merged in the pull request fixing the issue (if applicable) 521 | - This will take up considerable amount of space overtime, so be sure to clean the directory corresponding to `src/agent/constant.py:RUNTIME_DIR` 522 | 523 | ### `search_relevant_files` Vector DBs 524 | - If using the `search_relevant_files` tool, a vector/Chroma DB will be built for each issue submitted 525 | - This will take up considerable amount of space overtime, so be sure to clean the directory corresponding to `src/agent/constant.py:RUNTIME_DIR` -------------------------------------------------------------------------------- /src/agent/__init__.py: -------------------------------------------------------------------------------- 1 | """New LangGraph Agent. 2 | 3 | This module defines a custom graph. 4 | """ 5 | 6 | # from agent.graph import graph 7 | 8 | # __all__ = ["graph"] 9 | -------------------------------------------------------------------------------- /src/agent/constant.py: -------------------------------------------------------------------------------- 1 | """ 2 | Defines several constant variables used throughout the prototype. 3 | These constants include configuration settings, default values, and 4 | other immutable parameters used across different components of the system. 5 | """ 6 | 7 | import os 8 | 9 | import tree_sitter_java as tsjava 10 | import tree_sitter_python as tspython 11 | from tree_sitter import Language, Parser 12 | 13 | RUNTIME_DIR = os.path.join(os.environ["HOME"], "Tmp", "swe-runtime") 14 | 15 | PATCH_RESULT_DIR = os.path.join(RUNTIME_DIR, "results") 16 | os.makedirs(PATCH_RESULT_DIR, exist_ok=True) 17 | 18 | REQUEST_TIMEOUT = 30 19 | 20 | # Tree-sitter parser and query definitions used for indexing 21 | PY_LANGUAGE = Language(tspython.language()) 22 | JAVA_LANGUAGE = Language(tsjava.language()) 23 | 24 | tree_sitter_parsers = { 25 | "py": Parser(PY_LANGUAGE), 26 | "java": Parser(JAVA_LANGUAGE), 27 | } 28 | 29 | query_py_func_defs = PY_LANGUAGE.query( 30 | """(function_definition) @defs 31 | """ 32 | ) 33 | query_py_func_details = PY_LANGUAGE.query( 34 | """ 35 | name: (identifier) @name 36 | parameters: (parameters) @args 37 | body: (block) @block 38 | """ 39 | ) 40 | 41 | query_java_method_decs = JAVA_LANGUAGE.query("(method_declaration) @defs") 42 | query_java_construcor_decs = JAVA_LANGUAGE.query("(constructor_declaration) @defs") 43 | query_java_method_details = JAVA_LANGUAGE.query( 44 | """ 45 | name: (identifier) @name 46 | (modifiers) @mods 47 | (void_type) @void_type 48 | parameters: (formal_parameters) @args 49 | body: (block) @block 50 | """ 51 | ) 52 | 53 | func_queries = {"py": query_py_func_defs, "java": query_java_method_decs} 54 | func_detail_queries = {"py": query_py_func_details, "java": query_java_method_details} 55 | 56 | PLACE_HOLDER_PATCH = """diff --git a/_random_file_1bx7.txt b/_random_file_1bx7.txt 57 | new file mode 100644 58 | index 00000000..3372b06d 59 | --- /dev/null 60 | +++ b/_random_file_1bx7.txt 61 | @@ -0,0 +1 @@ 62 | +random text fillering, no meaning 63 | """ 64 | -------------------------------------------------------------------------------- /src/agent/github_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility functions for interacting with the GitHub API. 3 | 4 | This module provides helper methods to collect and process 5 | git-related information such as issue reports, repository details, 6 | and other GitHub-specific data retrieval operations. 7 | 8 | Several Functions in this module require proper GitHub API authentication 9 | and some may depend on the external library`requests`. 10 | """ 11 | 12 | import logging 13 | import os 14 | import re 15 | 16 | import dotenv 17 | import requests 18 | 19 | from agent.constant import REQUEST_TIMEOUT 20 | 21 | dotenv.load_dotenv( 22 | os.path.join( 23 | os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 24 | ".env", 25 | ) 26 | ) 27 | 28 | assert ( 29 | "GITHUB_TOKEN" in os.environ 30 | ), "Please put your GITHUB_TOKEN in .env at project root!" 31 | 32 | logging.basicConfig(level=logging.WARNING) 33 | logger = logging.getLogger(__name__) 34 | 35 | # Authentication header 36 | headers = {"Authorization": f"token {os.environ['GITHUB_TOKEN']}"} 37 | 38 | 39 | def parse_github_issue_url(issue_url): 40 | pattern = r"https://github\.com/([^/]+)/([^/]+)/issues/(\d+)" 41 | 42 | match = re.match(pattern, issue_url) 43 | 44 | if match: 45 | owner = match.group(1) 46 | project = match.group(2) 47 | issue_number = match.group(3) 48 | return owner, project, issue_number 49 | else: 50 | # Return None if the URL doesn't match the expected format 51 | return None, None, None 52 | 53 | 54 | def get_issue_description(owner, project, issue): 55 | """Retrieve the issue description 56 | Args: 57 | owner (str): Owner of the project 58 | project (str): Name of the project 59 | issue (Union[str, int]): Issue ID 60 | Returns: 61 | issue_description (str): The corresponding issue description.""" 62 | issue_api_url = f"https://api.github.com/repos/{owner}/{project}/issues/{issue}" 63 | 64 | response = requests.get(issue_api_url, headers=headers, timeout=REQUEST_TIMEOUT) 65 | 66 | # Check for successful response (HTTP status 200) 67 | if response.status_code == 200: 68 | issue_data = response.json() 69 | issue_description = issue_data.get("body", "No description available.") 70 | return issue_description 71 | else: 72 | # Handle errors 73 | print(f"Error fetching issue details: {response.status_code}") 74 | print("Response content:", response.text) 75 | return None 76 | 77 | 78 | # Fetch issue events to find the one that closed the issue 79 | def get_issue_events(url_): 80 | response = requests.get(url_, headers=headers, timeout=REQUEST_TIMEOUT) 81 | response.raise_for_status() # Raise an error for bad responses 82 | return response.json() 83 | 84 | 85 | # Fetch issue details to check for a linked PR 86 | def get_issue_details(url_): 87 | response = requests.get(url_, headers=headers, timeout=REQUEST_TIMEOUT) 88 | response.raise_for_status() # Raise an error for bad responses 89 | return response.json() 90 | 91 | 92 | # Main logic 93 | def get_issue_close_commit(owner, project, issue): 94 | """Retrieves the commit that closed the pull request corresponding to a given issue. 95 | Args: 96 | owner (str): Owner of the project 97 | project (str): Name of the project 98 | issue (Union[str, int]): Issue ID 99 | Returns: 100 | commit_id_to_return (str): The corresponding commit SHA.""" 101 | # Fetch the issue details 102 | issue_url = f"https://api.github.com/repos/{owner}/{project}/issues/{issue}" 103 | event_url = f"https://api.github.com/repos/{owner}/{project}/issues/{issue}/events" 104 | issue = get_issue_details(issue_url) 105 | 106 | # Check if the issue has a linked pull request 107 | if "pull_request" in issue: 108 | pr_url = issue["pull_request"]["url"] 109 | logger.info(f"Pull Request that closed the issue: {pr_url}") 110 | 111 | # Fetch the pull request details 112 | pr_response = requests.get(pr_url, headers=headers, timeout=REQUEST_TIMEOUT) 113 | pr_response.raise_for_status() 114 | pr_details = pr_response.json() 115 | 116 | # Check for commit associated with the pull request 117 | if pr_details["merged_at"]: 118 | logger.info(f"Pull Request was merged at: {pr_details['merged_at']}") 119 | logger.info( 120 | f"Commit that closed the issue: {pr_details['merge_commit_sha']}" 121 | ) 122 | else: 123 | logger.info("The pull request is not merged yet.") 124 | else: 125 | logger.info("No pull request linked to the issue.") 126 | # Fetch the events of the issue 127 | events = get_issue_events(event_url) 128 | commit_id_to_return = "" 129 | for event in events: 130 | # print(event) 131 | if event["event"] == "closed": 132 | if "commit_id" in event: 133 | logger.info(f"Commit that closed the issue: {event['commit_id']}") 134 | commit_id_to_return = event["commit_id"] 135 | elif "pull_request" in event: 136 | pr_url = event["pull_request"]["url"] 137 | logger.info(f"Pull Request that closed the issue: {pr_url}") 138 | return commit_id_to_return 139 | 140 | 141 | if __name__ == "__main__": 142 | # Run the function to get the closing commit or PR 143 | get_issue_close_commit("tpope", "vim-sensible", "161") 144 | -------------------------------------------------------------------------------- /src/agent/hierarchy_graph_demo.py: -------------------------------------------------------------------------------- 1 | # %% 2 | import os 3 | from typing import Literal 4 | import uuid 5 | 6 | 7 | import dotenv 8 | from langgraph.graph import END, START, StateGraph 9 | from langgraph.prebuilt import create_react_agent 10 | from langgraph.types import Command 11 | from langchain_core.messages import AIMessage, HumanMessage 12 | from typing_extensions import TypedDict 13 | 14 | from agent.llm import llm 15 | from agent.prompt import ( 16 | ISSUE_RESOLVE_REVIEWER_SYSTEM_PROMPT, 17 | ISSUE_RESOLVE_MAM_SYSTEM_PROMPT, 18 | ) 19 | from agent.runtime_config import RuntimeConfig 20 | from agent.state import CustomState 21 | from agent.supervisor_graph_demo import issue_resolve_graph 22 | from agent.tool_set.context_tools import search_relevant_files 23 | from agent.tool_set.sepl_tools import view_file_content, view_directory, run_shell_cmd 24 | 25 | rc = RuntimeConfig() 26 | 27 | reviewer_tools = [ 28 | view_directory, 29 | search_relevant_files, 30 | view_file_content, 31 | run_shell_cmd, 32 | ] 33 | 34 | dotenv.load_dotenv( 35 | os.path.join( 36 | os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 37 | ".env", 38 | ) 39 | ) 40 | 41 | 42 | class MamRouter(TypedDict): 43 | """Worker to route to next. If no workers needed, route to FINISH.""" 44 | 45 | options = ["issue_resolver", "reviewer", "FINISH"] 46 | next_agent: Literal[*options] 47 | thought: str 48 | 49 | 50 | def mam_node( 51 | state: CustomState, 52 | ) -> Command[Literal["issue_resolve_graph", "reviewer_node"]]: 53 | messages = [ 54 | {"role": "system", "content": ISSUE_RESOLVE_MAM_SYSTEM_PROMPT}, 55 | ] + state["messages"] 56 | 57 | response = llm.with_structured_output(MamRouter, strict=True).invoke(messages) 58 | 59 | next_agent = response["next_agent"] 60 | print(f"Next agent: {next_agent}") 61 | if "reviewer" in next_agent: 62 | goto = "reviewer_node" 63 | elif "issue_resolver" in next_agent: 64 | goto = "issue_resolve_graph" 65 | elif "FINISH" in next_agent: 66 | goto = END 67 | else: 68 | raise ValueError(f"Invalid next agent: {next_agent}") 69 | 70 | return Command( 71 | update={ 72 | "messages": [ 73 | AIMessage( 74 | content="MAM:\nThought: " 75 | + response["thought"] 76 | + "\nNext: " 77 | + response["next_agent"] 78 | + ".", 79 | name="MAM", 80 | ) 81 | ], 82 | "next_agent": goto if goto != END else None, 83 | }, 84 | goto=goto, 85 | ) 86 | 87 | 88 | reviewer_agent = create_react_agent( 89 | llm, 90 | tools=reviewer_tools, 91 | state_modifier=ISSUE_RESOLVE_REVIEWER_SYSTEM_PROMPT, 92 | ) 93 | 94 | 95 | def reviewer_node(state: CustomState) -> Command[Literal["mam_node"]]: 96 | result = reviewer_agent.invoke(state) 97 | new_messages = result["messages"][len(state["messages"]) :] 98 | last_message = new_messages[-1] 99 | # Add name to each AI message 100 | for msg in new_messages: 101 | if isinstance(msg, AIMessage): 102 | msg.name = "reviewer_node" 103 | 104 | return Command( 105 | update={"messages": last_message}, 106 | goto="mam_node", 107 | ) 108 | 109 | 110 | builder = StateGraph(CustomState) 111 | builder.add_node( 112 | "issue_resolve_graph", 113 | issue_resolve_graph, 114 | destinations=({"mam_node": "mam_node-issue_resolve_graph"}), 115 | ) 116 | builder.add_node( 117 | "mam_node", 118 | mam_node, 119 | destinations=( 120 | { 121 | "reviewer_node": "mam_node-reviewer_node", 122 | "issue_resolve_graph": "mam_node-issue_resolve_graph", 123 | } 124 | ), 125 | ) 126 | builder.add_node( 127 | "reviewer_node", 128 | reviewer_node, 129 | destinations=({"mam_node": "mam_node-reviewer_node"}), 130 | ) 131 | builder.add_edge(START, "mam_node") 132 | builder.add_edge("issue_resolve_graph", "mam_node") 133 | 134 | 135 | hierarchy_graph = builder.compile() 136 | 137 | 138 | # # %% 139 | if __name__ == "__main__": 140 | rc = RuntimeConfig() 141 | 142 | os.environ["LANGSMITH_TRACING"] = "true" 143 | thread = { 144 | "recursion_limit": 100, 145 | "run_id": uuid.uuid4(), 146 | "tags": ["interrupt"], 147 | "configurable": {"thread_id": "1"}, 148 | } 149 | initial_input = { 150 | "messages": [ 151 | HumanMessage( 152 | content="https://github.com/gitpython-developers/GitPython/issues/1413" 153 | ) 154 | ], 155 | "preset": "https://github.com/gitpython-developers/GitPython/issues/1413", 156 | "human_in_the_loop": False, 157 | } 158 | 159 | for chunk in hierarchy_graph.stream( 160 | initial_input, config=thread, stream_mode="values" 161 | ): 162 | if "messages" in chunk and len(chunk["messages"]) > 0: 163 | chunk["messages"][-1].pretty_print() 164 | -------------------------------------------------------------------------------- /src/agent/llm.py: -------------------------------------------------------------------------------- 1 | """ 2 | Defines the LLM based on the `LLM_PROVIDER` and `LLM_MODEL` env vars. 3 | """ 4 | 5 | import os 6 | 7 | from langchain_anthropic import ChatAnthropic 8 | from langchain_community.cache import SQLiteCache 9 | from langchain_core.globals import set_llm_cache 10 | from langchain_deepseek import ChatDeepSeek 11 | from langchain_openai import ChatOpenAI 12 | 13 | from agent.utils import UndefinedValueError 14 | 15 | from agent.runtime_config import load_env_config 16 | 17 | set_llm_cache(SQLiteCache(database_path=".langchain.db")) 18 | 19 | load_env_config() 20 | 21 | 22 | def create_llm(): 23 | """Creates the LLM according to `LLM_PROVIDER` and `LLM_MODEL` env vars""" 24 | created_llm = None 25 | llm_provider = os.getenv("LLM_PROVIDER") 26 | if not llm_provider: 27 | raise UndefinedValueError("LLM_PROVIDER") 28 | llm_name = os.getenv("LLM_MODEL") 29 | if "openai" in llm_provider.lower(): 30 | created_llm = ChatOpenAI( 31 | model=llm_name, temperature=0.0, max_tokens=2048, cache=True 32 | ) 33 | elif "anthropic" in llm_provider.lower(): 34 | created_llm = ChatAnthropic( 35 | model=llm_name, temperature=0.0, max_tokens=2048, cache=True 36 | ) 37 | elif "deepseek" in llm_provider.lower(): 38 | created_llm = ChatDeepSeek( 39 | model=llm_name, temperature=0.0, max_tokens=2048, cache=True 40 | ) 41 | 42 | if not created_llm or not llm_name: 43 | raise UndefinedValueError("LLM_MODEL") 44 | return created_llm 45 | 46 | 47 | llm = create_llm() 48 | 49 | if __name__ == "__main__": 50 | print(llm.invoke("Tell me a joke")) 51 | -------------------------------------------------------------------------------- /src/agent/parsers.py: -------------------------------------------------------------------------------- 1 | """ 2 | Defines parsers used to extract information from LLM responses 3 | """ 4 | 5 | from typing import List 6 | 7 | from langchain_core.output_parsers import JsonOutputParser 8 | from pydantic import BaseModel, Field 9 | 10 | 11 | class RelevantFileExplanation(BaseModel): 12 | """Parse LLM output to construct a file path string and explanation string.""" 13 | 14 | file_path: str = Field(description="The filepath of the relevant file.") 15 | explanation: str = Field( 16 | description="The explanation of how the file is relevant to the query." 17 | ) 18 | 19 | 20 | class RelevantFileExplanations(BaseModel): 21 | """Parse LLM output to construct a list of RelevantFileExplanation""" 22 | 23 | relevant_file_explanations: List[RelevantFileExplanation] 24 | 25 | 26 | relevant_file_explanations_parser = JsonOutputParser( 27 | pydantic_object=RelevantFileExplanations 28 | ) 29 | -------------------------------------------------------------------------------- /src/agent/prompt/__init__.py: -------------------------------------------------------------------------------- 1 | from .mam import ( 2 | ISSUE_RESOLVE_MAM_SYSTEM_PROMPT, 3 | ) 4 | from .context_manager import RELEVANT_FILE_EXPLANATION_SYSTEM_PROMPT 5 | from .problem_decoder import ( 6 | ISSUE_RESOLVE_PROBLEM_DECODER_SYSTEM_PROMPT, 7 | ) 8 | from .problem_solver import ( 9 | ISSUE_RESOLVE_PROBLEM_SOLVER_SYSTEM_PROMPT, 10 | ) 11 | from .reviewer import ( 12 | ISSUE_RESOLVE_REVIEWER_SYSTEM_PROMPT, 13 | ) 14 | from .solution_mapper import ( 15 | ISSUE_RESOLVE_SOLUTION_MAPPER_SYSTEM_PROMPT, 16 | ) 17 | from .supervisor import ( 18 | ISSUE_RESOLVE_SUPERVISOR_SYSTEM_PROMPT, 19 | ) 20 | 21 | __all__ = [ 22 | "ISSUE_RESOLVE_PROBLEM_DECODER_SYSTEM_PROMPT", 23 | "ISSUE_RESOLVE_PROBLEM_SOLVER_SYSTEM_PROMPT", 24 | "ISSUE_RESOLVE_REVIEWER_SYSTEM_PROMPT", 25 | "ISSUE_RESOLVE_SOLUTION_MAPPER_SYSTEM_PROMPT", 26 | "ISSUE_RESOLVE_SUPERVISOR_SYSTEM_PROMPT", 27 | "ISSUE_RESOLVE_MAM_SYSTEM_PROMPT", 28 | "RELEVANT_FILE_EXPLANATION_SYSTEM_PROMPT", 29 | ] 30 | -------------------------------------------------------------------------------- /src/agent/prompt/context_manager.py: -------------------------------------------------------------------------------- 1 | from string import Template 2 | 3 | RELEVANT_FILE_EXPLANATION_SYSTEM_PROMPT = Template( 4 | """Given a search term ${search_term}, a vector database performing similarity search of embeddings between the search term and code snippets of files and functions/methods in the project returned ${k} relevant documents. For each document, provide a description explaining why the search term is relevant to the code retrieved from the database. Below is a list of the filepaths and their corresponding code snippets in JSON format: 5 | ```${full_result}``` 6 | 7 | Only respond with your result as a list of JSON with the "file_path" key and the "explanation" key for your corresponding explanation. An example of the format is below: 8 | ```[{\"file_path\": \"filepath1/file1.py\", \"explanation\": \"This file contains the keyword \"UIButton\" from the search term\"}]```""" 9 | ) 10 | -------------------------------------------------------------------------------- /src/agent/prompt/mam.py: -------------------------------------------------------------------------------- 1 | ISSUE_RESOLVE_MAM_SYSTEM_PROMPT = """ 2 | You are an autonomous multi-agent manager responsible for coordinating the resolution of coding issues. Your role is to manage two agents: the **Issue Resolver Agent** and the **Reviewer Agent**. 3 | 4 | Follow these steps **strictly in order**: 5 | 6 | 1. **Issue Resolver Agent**: At the very beginning, you must call the Issue Resolver Agent to resolve the issue. 7 | 2. **Reviewer Agent**: After the Issue Resolver completes the task, you must call the Reviewer Agent to review the work. 8 | 9 | **Rules and Workflow:** 10 | 11 | - Your **first action** must always be to call the Issue Resolver Agent with the response field `"next_agent": "issue_resolver"`. 12 | - Once the Issue Resolver has completed their response, you are **required** to call the Reviewer Agent with `"next_agent": "reviewer"`. 13 | - **Under no condition should you skip the Reviewer Agent. Even if the Issue Resolver provides a review, it is not sufficient.** The Reviewer Agent offers a more systematic and thorough evaluation and must always be involved. 14 | - After the Reviewer Agent completes their review, you must **end the conversation** by setting `"next_agent": "FINISH"`. 15 | 16 | ✅ **Enforcement Reminder:** 17 | You **must not proceed to `"next_agent": "FINISH"` unless the Reviewer Agent has been explicitly called and has provided their review.** This step is mandatory. 18 | 19 | At each step, provide relevant feedback or guidance to the next agent based on the previous output. 20 | """ 21 | -------------------------------------------------------------------------------- /src/agent/prompt/problem_decoder.py: -------------------------------------------------------------------------------- 1 | ISSUE_RESOLVE_PROBLEM_DECODER_SYSTEM_PROMPT = """ 2 | You are a problem_decoder. You shold follow the guidance or feedback provided by the supervisor. And you are tasked with distill the issue description and generating a problem statement. You should do the follow three tasks: 3 | 1. Genreate the topic question. 4 | 2. Genreate the codebase representation. 5 | 3. Generate current behavour of the codebase. 6 | 4. Generate the expected behaviour of the codebase. 7 | 8 | **State your role of problem_decoder at the beginning of your response** 9 | 10 | # Task 1 Description 11 | Given the issue, generate a topic question. The topic question should define a “topic” for the task, which takes the form of a question that can be posed against the codebase, and used to define the before/after success criteria of the issue. The topic is a way to distill the task down to its essence. The topic question should helps a reader better understand the task in the issue, and to focus on the most important aspects of the codebase that are relevant to the task. 12 | 13 | Procecure: 14 | 1. Start by understanding the issue content 15 | 2. Generate a summary statement of the issue 16 | 3. Turn the summary statement into a yes/no question 17 | 18 | Response requirements: 19 | 1. the topic question should be a does or do question that can be answered with a yes or no 20 | 2. the topic question should be concise and specific to the task in the issue 21 | 3. output the topic questions with the status like {'question': topic_question, 'status': 'yes' if the topic question is currently satisfied based on issue description and 'no' otherwise} 22 | 4. keep all technical terms in the question as is, do not simplify them 23 | 5. when generating topic question, be sure to include all necessary pre-conditions of the issue. E.g, if the issue happens under specific circumstance, be sure your statement and your question reflects that. 24 | 25 | # Task 2 Description 26 | 27 | Given the question, use the tools to navigate the codebase and locate the files that are relevant to the issue. Then, generate statements about the codebase relevant to the question. The representation of the codebase is a tree-like structure with file names printed on lines and the functions and classed defined within preceeding with --. 28 | 29 | Procedure: 30 | 1. generate one or more statements repeat that answers the question based on the current status 31 | 2. generate one or more statements about which files in the codebase are relevant to the question 32 | 3. generate one or more statements about which methods in the codebase are relevant to the question. It is normal that there is no methods in the codebase related to the question. In this case, indicate such. 33 | 4. Generate one or more statements how to the codebase so that we can answer the question. 34 | 35 | Response requirements: 36 | 1. For each statement, output your answer directly without stating assumptions 37 | 2. When referencing existing file names, include the full path. 38 | 3. Be sure that referenced file names come from the codebase representation. 39 | 40 | Example output of Task 2: 41 | ### 42 | 43 | - No, `is_dirty()` does not use `diff-files` and `diff-index` instead of `git diff`. 44 | - The `is_dirty()` function in `git/repo/base.py` uses `git diff` to check for changes in the index and working tree. 45 | - Lines 957-977 in `git/repo/base.py` show that `is_dirty()` uses `self.git.diff` to compare the index against HEAD and the working tree. 46 | - The function does not use `diff-files` or `diff-index` commands. 47 | - The current implementation of `is_dirty()` can be slow for large repositories with text conversion enabled for diffs. 48 | ### 49 | 50 | Given the issue, and a representation of the codebase, identify the files in the codebase that are relevant to the issue. 51 | The representation of the codebase is a mapping of relative_file_path to the functions and classes in the file. 52 | Your response requirements: 53 | 1. List each relevant file path followed by an explanation of why it's relevant to the issue 54 | 2. Ensure every file path in your output comes from the representation of the codebase 55 | 56 | # Task 3 Description 57 | Given a question and a codebase representation, assume the topic question and the status generated from the Task 1 has been hypothetical implemented, generate statements about the codebase relevant to the question. 58 | 59 | Procedure: 60 | 1. Understand the main topic question and the assumption, understand the main subject of the intended change, be very careful, as the question may contain comparison clause. Any subject in the comparison clause is notthe main subject of the intended change. 61 | 2. generate one or more statements repeat that answers the question based on the assumption 62 | 3. generate one or more statements about which files in the codebase should contain the code change based on the assumption 63 | 4. Does the codebase has relevant method to the assumption? if not, propose a name of the method. Output one or more statements about the name of this method and what the method does 64 | 5. Generate one or more statements about how to implement the hypothetical code 65 | 66 | Response requirements: 67 | 1. For each statement, output your answer directly without stating assumptions 68 | 2. Don't quote your output with ###, ### are used to show the boundary of example outputs 69 | 3. When referencing existing file names, use the full path. 70 | 4. Be sure that referenced file names come from the codebase representation. 71 | 5. Ensuring that the statements are relevant to the main subject of the intended change 72 | 73 | Example output of Task 3: 74 | ### 75 | - Yes, the `Commit` class now has a method to get patch text similar to `git diff`. 76 | - Added a `patch` method to the `Commit` class in `git/objects/commit.py` to get patch text. 77 | - The `patch` method uses the `diff` method from the `Diffable` class to generate the patch text. 78 | ### 79 | """ -------------------------------------------------------------------------------- /src/agent/prompt/problem_solver.py: -------------------------------------------------------------------------------- 1 | 2 | ISSUE_RESOLVE_PROBLEM_SOLVER_SYSTEM_PROMPT = """ 3 | You are a programmer. Please edit the file in the codebase according to the following code change plan wrapped in the tags. 4 | 5 | You have access to the following two tools: 6 | 7 | - view_directory 8 | - str_replace_editor 9 | 10 | After tool calling, you should perform an observation on the tool calling result and reason about the next step before you proceed the next move. Wrap your observation and next step reasoning in the tags. 11 | For example: 12 | 13 | I've examined the content of "user_authentication.py" and found the issue. Lines 45-52 contain the password validation function that's causing the bug. The function isn't properly handling special characters in passwords. I'll now implement the changes specified in the code change plan to fix this file. 14 | 15 | 16 | Guidelines for implementation: 17 | 1. Make only the changes specified in the plan 18 | 2. Maintain consistent code style with the existing codebase 19 | 3. Ensure the changes don't introduce new bugs 20 | 4. Ensure your code edits are free from syntax errors and logical errors 21 | 22 | Procedure: 23 | 1. Understand the proposed code change plan. 24 | 2. View the code in the file that needs modification. 25 | 3. Use the str_replace_editor tool to edit the file. 26 | 4. Verify your changes match the requirements in the code change plan. 27 | """ -------------------------------------------------------------------------------- /src/agent/prompt/reviewer.py: -------------------------------------------------------------------------------- 1 | 2 | ISSUE_RESOLVE_REVIEWER_SYSTEM_PROMPT = """ 3 | You are a reviewer. You are provided the latest code changes generated by the problem_solver. 4 | 5 | You are tasked with reviewing the solution_mapper's solution and the problem_solver's patch to ensure the solution is correct and the patch are able to solve the issues. 6 | 7 | You can assume that the patch are already applied to the codebase. You should first generate the test cases yourself, and then run the test cases 8 | 9 | If the evaluation tests passed, the generated patch fixes the issue and you should response with the "FIXED". Otherwise, you should analyze the reason and provide feedback to the supervisor and prompt the supervisor to rerun the problem_solver with the feedback. 10 | """ 11 | -------------------------------------------------------------------------------- /src/agent/prompt/solution_mapper.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | ISSUE_RESOLVE_SOLUTION_MAPPER_SYSTEM_PROMPT = """ 4 | You are a solution mapper. You should do the follow two tasks: 5 | 1. According to the result from problem decoder, your task is to browse the relevant files and functions related with the problem. To do this, you should first understand the files and their semantics, and then understand the proposed solution. 6 | 2. And then, from the current status and the expected status, reasoning and generate the detailed code change plan for each relevant files/functions. 7 | 8 | Tools: 9 | You have access to the tools of: view_directory and view_file_content, which have the ability for you to locate files and read the content of a file. 10 | 11 | Response requirements: 12 | 1. **State your role of solution_mapper at the beginning of your response** 13 | 2. Response with keys being the filepath and value being the proposed change plans to the file. List changes to a specific file in bullet points. 14 | 3. Be sure that you consider edge cases to ensure correct functionality. 15 | """ -------------------------------------------------------------------------------- /src/agent/prompt/supervisor.py: -------------------------------------------------------------------------------- 1 | ISSUE_RESOLVE_SUPERVISOR_SYSTEM_PROMPT = """ 2 | You are an autonomous software engineer tasked with solving coding issues. Your role is to coordinate between three workers: problem_decoder, solution_mapper and problem_solver. 3 | 4 | Do the following steps in the same order: 5 | 1. problem_decoder: call the problem_decoder agent to decode the user request into a problem statement, current behaviour and expected behaviour. 6 | 2. solution_mapper: call the solution_mapper agent to navigate the codebase, locate the relevant code, and generate the proposed code change plans. 7 | 3. problem_solver: call the problem_solver agent to generate the code with the proposed plans. 8 | 9 | Your should first call the problem_decoder agent to decode the user request into a problem statement. 10 | Then, call the solution_mapper agent to map the problem statement into a proposed code change solution. 11 | Finally, call the problem_solver agent to follow the solution_mapper's instructions to fix the problem. 12 | 13 | On each steps, if you think the agent did not finish their task, you should call the agent again to solve the problem again, with some feedback provided in the thought. 14 | 15 | 16 | If the problem is not fixed for 3 attempts, you should end the conversation with response "FIX FAILED". 17 | If the problem is fixed, you should end the conversation with response "FINISH". 18 | 19 | You should provide feedback or guidance to the next acting agent. 20 | """ -------------------------------------------------------------------------------- /src/agent/runtime_config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Handles all runtime environment configuration setup. 3 | Currently supports loading runtime environment using a GitHub issue URL. 4 | """ 5 | 6 | import os 7 | from enum import Enum 8 | 9 | from dotenv import load_dotenv 10 | from git import Repo 11 | 12 | from agent.constant import RUNTIME_DIR 13 | from agent.tool_set.sepl_tools import extract_git_diff_local 14 | 15 | 16 | class RuntimeType(Enum): 17 | LOCAL = 1, "LOCAL" 18 | 19 | def __int__(self): 20 | return self.value[0] 21 | 22 | def __str__(self): 23 | return self.value[1] 24 | 25 | @classmethod 26 | def _missing_(cls, value): 27 | if isinstance(value, int): 28 | for member in cls: 29 | if member.value[0] == value: 30 | return member 31 | raise ValueError(f"{value} is not a valid {cls.__name__}") 32 | 33 | 34 | def load_env_config(): 35 | env_file = os.path.join( 36 | os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env" 37 | ) 38 | load_dotenv(env_file) 39 | 40 | 41 | class RuntimeConfig: 42 | """ 43 | Singleton class to hold the runtime configuration 44 | 45 | Each configuration loading entry point starts with `load_from` 46 | """ 47 | 48 | _instance = None 49 | initialized = False 50 | 51 | preset = None 52 | 53 | runtime_dir = RUNTIME_DIR 54 | proj_name = None 55 | proj_path = None 56 | issue_desc = None 57 | commit_head = None 58 | 59 | runtime_type: RuntimeType = None 60 | 61 | def __new__(cls, force_new_instance=False): 62 | if cls._instance is None or force_new_instance: 63 | instance = super().__new__(cls) 64 | instance.__init__() # Initialize a new instance 65 | if not force_new_instance: 66 | cls._instance = instance 67 | return instance 68 | return cls._instance 69 | 70 | def load(self, owner, project, commit_id): 71 | self.proj_name = owner + "/" + project 72 | self.proj_path = os.path.join(self.runtime_dir, self.proj_name) 73 | self.commit_head = commit_id 74 | 75 | self.initialized = True 76 | self.runtime_type = RuntimeType.LOCAL 77 | self.runtime_setup() 78 | 79 | def load_from_local(self, path): 80 | self.proj_path = path 81 | self.proj_name = "/".join(os.path.split("/")[-2:]) 82 | self.initialized = True 83 | self.runtime_type = RuntimeType.LOCAL 84 | self.runtime_setup() 85 | 86 | def load_from_github_issue_url(self, issue_url): 87 | """Setup the runtime config based on a given issue UR; 88 | Args: 89 | issue_url (str): The given issue URL""" 90 | from agent.github_utils import ( 91 | get_issue_close_commit, 92 | get_issue_description, 93 | parse_github_issue_url, 94 | ) 95 | 96 | owner, project, issue = parse_github_issue_url(issue_url) 97 | # TODO cache fetch results from github 98 | 99 | if not owner: 100 | raise ValueError(f"Invalid GitHub issue URL passed in: {issue_url}") 101 | 102 | self.proj_name = owner + "/" + project 103 | self.proj_path = os.path.join(self.runtime_dir, self.proj_name) 104 | self.issue_desc = get_issue_description(owner, project, issue) 105 | self.commit_head = get_issue_close_commit(owner, project, issue) 106 | 107 | checkout_parent = False 108 | if self.commit_head: 109 | print(f"Located closing commit @ {self.commit_head} for\n\t{issue_url}") 110 | checkout_parent = True 111 | 112 | self.initialized = True 113 | 114 | self.runtime_type = RuntimeType.LOCAL 115 | 116 | self.runtime_setup() 117 | if checkout_parent: 118 | self.checkout_parent_commit() 119 | 120 | def runtime_setup(self): 121 | assert self.initialized 122 | 123 | # setup runtime if doesn't exist 124 | if not os.path.exists(self.runtime_dir): 125 | print(f"{self.runtime_dir} doesn't exist, creating...") 126 | os.makedirs(self.runtime_dir) 127 | 128 | if not os.path.exists(self.proj_path): 129 | git_url = f"https://github.com/{self.proj_name}" 130 | print(f"Cloning {self.proj_name} to\n\t{self.proj_path}") 131 | repo = Repo.clone_from(git_url, self.proj_path) 132 | else: 133 | repo = Repo(self.proj_path) 134 | 135 | if self.commit_head: 136 | try: 137 | repo.git.checkout(self.commit_head) 138 | except Exception: 139 | print( 140 | f"[E] Unable to checkout commit for {self.proj_name}\n\tUsing default commit" 141 | ) 142 | 143 | # reset repo 144 | repo.git.reset("--hard") 145 | repo.git.clean("-xdf") 146 | 147 | self.commit_head = repo.commit().hexsha 148 | 149 | def checkout_parent_commit(self): 150 | assert os.path.isdir(self.proj_path) 151 | 152 | try: 153 | repo = Repo(self.proj_path) 154 | except Exception: 155 | print(f"[E] unable to initialize {self.proj_name} at {self.proj_path}") 156 | 157 | try: 158 | parent = repo.commit().parents[0] 159 | repo.git.checkout(parent.hexsha) 160 | except Exception: 161 | print(f"[E] unable to checkout parent for {self.proj_name}") 162 | 163 | self.commit_head = repo.commit().hexsha 164 | 165 | def dump_config(self): 166 | if self.runtime_type == RuntimeType.LOCAL: 167 | extract_git_diff = extract_git_diff_local 168 | else: 169 | raise NotImplementedError 170 | 171 | return { 172 | "runtime_type": int(self.runtime_type), 173 | "preset": self.preset, 174 | "path": self.proj_path, 175 | "patch": extract_git_diff(), 176 | } 177 | 178 | def pretty_print_runtime(self): 179 | if self.runtime_type == RuntimeType.LOCAL: 180 | print("Current configuration type is LOCAL") 181 | print(f"Runtime Dir: {self.runtime_dir}") 182 | print(f"Project Name: {self.proj_name}") 183 | print(f"Project Path: {self.proj_path}") 184 | print(f"Current Commit: {self.commit_head}") 185 | 186 | 187 | if __name__ == "__main__": 188 | rc = RuntimeConfig() 189 | # config.load_from_dynamic_select_preset() 190 | rc.load_from_github_issue_url("https://github.com/tpope/vim-vinegar/issues/136") 191 | 192 | rc.pretty_print_runtime() 193 | -------------------------------------------------------------------------------- /src/agent/state.py: -------------------------------------------------------------------------------- 1 | """Defines the custom state structures for the prototype.""" 2 | 3 | from __future__ import annotations 4 | 5 | from dataclasses import dataclass 6 | from langgraph.graph import MessagesState, add_messages 7 | from langgraph.managed import IsLastStep, RemainingSteps 8 | from langchain_core.messages import AnyMessage 9 | from typing import Annotated, List, Optional 10 | 11 | 12 | @dataclass 13 | class CustomState(MessagesState): 14 | last_agent: Optional[str] = None 15 | next_agent: Optional[str] = None 16 | summary: Optional[str] = None 17 | human_in_the_loop: Optional[bool] = True # Default to True to skip HIL 18 | preset: Optional[str] = None 19 | issue_description: Optional[str] = None 20 | -------------------------------------------------------------------------------- /src/agent/supervisor_graph_demo.py: -------------------------------------------------------------------------------- 1 | """ 2 | Demonstrates the supervisor based graph for fixing issue reports. 3 | """ 4 | 5 | # %% 6 | import os 7 | from typing import Literal 8 | import uuid 9 | 10 | 11 | import dotenv 12 | from langgraph.graph import END, START, StateGraph 13 | from langgraph.prebuilt import create_react_agent 14 | from langgraph.types import Command, interrupt 15 | from langchain_core.messages import AIMessage, HumanMessage, RemoveMessage 16 | from typing_extensions import TypedDict 17 | 18 | from agent.llm import llm 19 | from agent.prompt import ( 20 | ISSUE_RESOLVE_PROBLEM_DECODER_SYSTEM_PROMPT, 21 | ISSUE_RESOLVE_PROBLEM_SOLVER_SYSTEM_PROMPT, 22 | ISSUE_RESOLVE_SOLUTION_MAPPER_SYSTEM_PROMPT, 23 | ISSUE_RESOLVE_SUPERVISOR_SYSTEM_PROMPT, 24 | ) 25 | from agent.runtime_config import RuntimeConfig 26 | from agent.state import CustomState 27 | from agent.tool_set.context_tools import search_relevant_files, summarizer 28 | from agent.tool_set.edit_tool import str_replace_editor 29 | from agent.tool_set.sepl_tools import save_git_diff, view_file_content, view_directory 30 | from agent.utils import stage_message_processor 31 | 32 | rc = RuntimeConfig() 33 | 34 | problem_decoder_tools = [view_directory, search_relevant_files, view_file_content] 35 | solution_mapper_tools = [view_directory, search_relevant_files, view_file_content] 36 | problem_solver_tools = [view_directory, search_relevant_files, str_replace_editor] 37 | reviewer_tools = [view_directory, search_relevant_files, view_file_content] 38 | 39 | dotenv.load_dotenv( 40 | os.path.join( 41 | os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 42 | ".env", 43 | ) 44 | ) 45 | 46 | 47 | members = ["problem_decoder", "solution_mapper", "problem_solver"] 48 | options = members + ["FINISH"] 49 | 50 | 51 | class Router(TypedDict): 52 | """Worker to route to next. If no workers needed, route to FINISH.""" 53 | 54 | next_agent: Literal[*options] 55 | thought: str 56 | 57 | 58 | def input_handler_node(state: CustomState) -> Command[Literal["supervisor"]]: 59 | """in issue solving, input handler will take input of 60 | 1.swe-bench id, 61 | 2.issue link and setup the env accordingly""" 62 | user_input = state["messages"][0].content 63 | if "/issues/" in user_input: 64 | # the input are github link 65 | rc.load_from_github_issue_url(user_input) 66 | else: 67 | print("error, enter a valid issue link") 68 | return Command(goto=END) 69 | issue_description = rc.issue_desc 70 | return Command( 71 | update={ 72 | "messages": [ 73 | RemoveMessage(id=state["messages"][0].id), 74 | HumanMessage(content=issue_description), 75 | ], 76 | "last_agent": "input_handler", 77 | }, 78 | goto="supervisor", 79 | ) 80 | 81 | 82 | def supervisor_node( 83 | state: CustomState, 84 | ) -> Command[ 85 | Literal[ 86 | "problem_decoder", "solution_mapper", "problem_solver", "human_feedback", END 87 | ] 88 | ]: 89 | messages = [ 90 | {"role": "system", "content": ISSUE_RESOLVE_SUPERVISOR_SYSTEM_PROMPT}, 91 | ] + state["messages"] 92 | 93 | response = llm.with_structured_output(Router, strict=True).invoke(messages) 94 | 95 | next_agent = response["next_agent"] 96 | goto = next_agent 97 | 98 | goto = END if "FINISH" in goto else goto 99 | last_agent = state["last_agent"] 100 | 101 | if "human_in_the_loop" in state: 102 | if ( 103 | state["human_in_the_loop"] 104 | and next_agent != last_agent 105 | and last_agent != "input_handler" 106 | ): 107 | stage_messages = stage_message_processor(state["messages"]) 108 | summary = summarizer(stage_messages) 109 | return Command( 110 | update={ 111 | "summary": summary, 112 | "messages": [ 113 | AIMessage( 114 | content="Supervisor:\nThought: " 115 | + response["thought"] 116 | + "\nNext: " 117 | + response["next_agent"] 118 | + ".", 119 | name="supervisor", 120 | ) 121 | ], 122 | "last_agent": last_agent, 123 | "next_agent": goto if goto != END else None, 124 | }, 125 | goto="human_feedback", 126 | ) 127 | return Command( 128 | update={ 129 | "messages": [ 130 | AIMessage( 131 | content="Supervisor:\nThought: " 132 | + response["thought"] 133 | + "\nNext: " 134 | + response["next_agent"] 135 | + ".", 136 | name="supervisor", 137 | ) 138 | ], 139 | "last_agent": last_agent, 140 | "next_agent": goto if goto != END else None, 141 | }, 142 | goto=goto, 143 | ) 144 | 145 | 146 | def human_feedback_node(state: CustomState) -> Command[Literal[*members]]: 147 | next_agent = state["next_agent"] 148 | last_agent = state["last_agent"] 149 | summary = state["summary"] 150 | show_to_human = f"{summary}\n Please provide feedback on the last agent: " 151 | human_feedback = interrupt(show_to_human) 152 | feedback = human_feedback["feedback"] 153 | rerun = bool(human_feedback["rerun"]) # rerun is 0 or 1, if 1, then rerun 154 | 155 | if rerun: 156 | # now we rerun the last agent 157 | print(f"Human decide to rerun the last agent: {last_agent}") 158 | return Command( 159 | update={ 160 | "messages": [ 161 | AIMessage(content=summary, name="conversation_summary"), 162 | HumanMessage(content=feedback, name="human_feedback"), 163 | ], 164 | "next_agent": None, 165 | }, 166 | goto=last_agent, 167 | ) 168 | 169 | print(f"Human decide to continue to the next agent: {next_agent}") 170 | return Command( 171 | update={ 172 | "messages": [ 173 | AIMessage(content=summary, name="conversation_summary"), 174 | HumanMessage(content=feedback, name="human_feedback"), 175 | ] 176 | }, 177 | goto=next_agent, 178 | ) 179 | 180 | 181 | problem_decoder_agent = create_react_agent( 182 | llm, 183 | tools=problem_decoder_tools, 184 | state_modifier=ISSUE_RESOLVE_PROBLEM_DECODER_SYSTEM_PROMPT, 185 | ) 186 | 187 | 188 | def problem_decoder_node(state: CustomState) -> Command[Literal["supervisor"]]: 189 | result = problem_decoder_agent.invoke(state) 190 | new_messages = result["messages"][len(state["messages"]) :] 191 | 192 | for msg in new_messages: 193 | if isinstance(msg, AIMessage): 194 | msg.name = "problem_decoder" 195 | 196 | return Command( 197 | update={"messages": new_messages, "last_agent": "problem_decoder"}, 198 | goto="supervisor", 199 | ) 200 | 201 | 202 | solution_mapper_agent = create_react_agent( 203 | llm, 204 | tools=solution_mapper_tools, 205 | state_modifier=ISSUE_RESOLVE_SOLUTION_MAPPER_SYSTEM_PROMPT, 206 | ) 207 | 208 | 209 | def solution_mapper_node(state: CustomState) -> Command[Literal["supervisor"]]: 210 | print("Solution mapper node is running ~") 211 | result = solution_mapper_agent.invoke(state) 212 | new_messages = result["messages"][len(state["messages"]) :] 213 | 214 | for msg in new_messages: 215 | if isinstance(msg, AIMessage): 216 | msg.name = "solution_mapper" 217 | 218 | return Command( 219 | update={"messages": new_messages, "last_agent": "solution_mapper"}, 220 | goto="supervisor", 221 | ) 222 | 223 | 224 | problem_solver_agent = create_react_agent( 225 | llm, 226 | tools=problem_solver_tools, 227 | state_modifier=ISSUE_RESOLVE_PROBLEM_SOLVER_SYSTEM_PROMPT, 228 | ) 229 | 230 | 231 | def problem_solver_node(state: CustomState) -> Command[Literal["supervisor"]]: 232 | result = problem_solver_agent.invoke(state) 233 | new_messages = result["messages"][len(state["messages"]) :] 234 | 235 | # Add name to each AI message 236 | for msg in new_messages: 237 | if isinstance(msg, AIMessage): 238 | msg.name = "problem_solver" 239 | 240 | latest_patch = "Below is the latest code changes:\n" + save_git_diff() 241 | latest_patch = latest_patch.rstrip() 242 | print(f"Latest patch: {latest_patch}") 243 | 244 | return Command( 245 | update={ 246 | "messages": new_messages 247 | + [ 248 | AIMessage( 249 | content=latest_patch, 250 | name="problem_solver", 251 | ) 252 | ], 253 | "last_agent": "problem_solver", 254 | }, 255 | goto="supervisor", 256 | ) 257 | 258 | 259 | supervisor_builder = StateGraph(CustomState) 260 | supervisor_builder.add_edge(START, "input_handler") 261 | supervisor_builder.add_node( 262 | "input_handler", 263 | input_handler_node, 264 | destinations=({"supervisor": "input_handler-supervisor"}), 265 | ) 266 | supervisor_builder.add_node( 267 | "human_feedback", 268 | human_feedback_node, 269 | destinations=( 270 | { 271 | "problem_decoder": "human_feedback-problem_decoder", 272 | "solution_mapper": "human_feedback-solution_mapper", 273 | "problem_solver": "human_feedback-problem_solver", 274 | } 275 | ), 276 | ) 277 | supervisor_builder.add_node( 278 | "problem_decoder", 279 | problem_decoder_node, 280 | destinations=({"supervisor": "decoder-supervisor"}), 281 | ) 282 | supervisor_builder.add_node( 283 | "solution_mapper", 284 | solution_mapper_node, 285 | destinations=({"supervisor": "mapper-supervisor"}), 286 | ) 287 | supervisor_builder.add_node( 288 | "problem_solver", 289 | problem_solver_node, 290 | destinations=({"supervisor": "solver-supervisor"}), 291 | ) 292 | supervisor_builder.add_node( 293 | "supervisor", 294 | supervisor_node, 295 | destinations=( 296 | { 297 | "human_feedback": "supervisor-human_feedback", 298 | "problem_decoder": "supervisor-decoder", 299 | "solution_mapper": "supervisor-mapper", 300 | "problem_solver": "supervisor-solver", 301 | END: "END", 302 | } 303 | ), 304 | ) 305 | 306 | 307 | issue_resolve_graph = supervisor_builder.compile() 308 | 309 | 310 | # # %% 311 | if __name__ == "__main__": 312 | # set os env of LANGSMITH_TRACING to true 313 | rc = RuntimeConfig() 314 | 315 | # when using input_handler_node, no need to initialized 316 | os.environ["LANGSMITH_TRACING"] = "true" 317 | thread = { 318 | "recursion_limit": 100, 319 | "run_id": uuid.uuid4(), 320 | "tags": ["interrupt"], 321 | "configurable": {"thread_id": "1"}, 322 | } 323 | initial_input = { 324 | "messages": [ 325 | HumanMessage( 326 | content="https://github.com/gitpython-developers/GitPython/issues/1413" 327 | ) 328 | ], 329 | "preset": "https://github.com/gitpython-developers/GitPython/issues/1413", 330 | "human_in_the_loop": False, 331 | } 332 | 333 | for chunk in issue_resolve_graph.stream( 334 | initial_input, config=thread, stream_mode="values" 335 | ): 336 | if "messages" in chunk and len(chunk["messages"]) > 0: 337 | chunk["messages"][-1].pretty_print() 338 | -------------------------------------------------------------------------------- /src/agent/tool_set/constant.py: -------------------------------------------------------------------------------- 1 | MAX_RESPONSE_LEN_CHAR: int = 32000 2 | 3 | SNIPPET_CONTEXT_WINDOW = 4 4 | 5 | CONTENT_TRUNCATED_NOTICE = "Due to the max output limit, only part of the full response has been shown to you." 6 | 7 | # FILE_CONTENT_TRUNCATED_NOTICE: str = 'Due to the max output limit, only part of this file has been shown to you. You should retry this tool after you have searched inside the file with the shell tool or view_file_structure tool in order to find the line numbers of what you are looking for and use this tool with view_range.' 8 | FILE_CONTENT_TRUNCATED_NOTICE = "Due to the max output limit, only part of this file has been shown to you. You should retry this tool after you have searched inside the file with the `search_file_by_keywords` tool or view the file structure below in order to find the line numbers of what you are looking for, and then use this tool with view_range." 9 | 10 | DIRECTORY_CONTENT_TRUNCATED_NOTICE: str = ( 11 | "Due to the max output limit, only part of this directory has been shown to you. You should use `ls -la` with the shell tool instead to view large directories incrementally." 12 | ) 13 | -------------------------------------------------------------------------------- /src/agent/tool_set/context_tools.py: -------------------------------------------------------------------------------- 1 | """Defines context management tools""" 2 | 3 | from glob import glob 4 | import os 5 | 6 | from langchain_chroma import Chroma 7 | from langchain_core.documents import Document 8 | from langchain_core.messages import HumanMessage 9 | from langchain_core.tools import tool 10 | from langchain.text_splitter import RecursiveCharacterTextSplitter 11 | from langchain_openai import OpenAIEmbeddings 12 | from tqdm import tqdm 13 | 14 | from agent import runtime_config 15 | from agent.constant import func_queries, query_java_construcor_decs, tree_sitter_parsers 16 | from agent.llm import llm 17 | from agent.parsers import relevant_file_explanations_parser 18 | from agent.prompt import ( 19 | RELEVANT_FILE_EXPLANATION_SYSTEM_PROMPT, 20 | ) 21 | from agent.runtime_config import load_env_config 22 | from agent.utils import UndefinedValueError 23 | 24 | load_env_config() 25 | 26 | if "OPENAI_API_KEY" not in os.environ: 27 | raise UndefinedValueError("OPENAI_API_KEY") 28 | 29 | EMBEDDING_FUNCTION = OpenAIEmbeddings( 30 | api_key=os.environ.get("OPENAI_API_KEY"), 31 | model="text-embedding-3-small", 32 | ) # Defines the embedding model to use for the VectorDB with use of the `search_relevant_files` tool. 33 | PROJECT_KNOWLEDGE_TEXT_SPLITTER = RecursiveCharacterTextSplitter( 34 | chunk_size=1000, chunk_overlap=256, separators=["\n"] 35 | ) # Defines the text splitter for the Vector DB with use of the `search_relevant_files` tool. 36 | 37 | 38 | def create_project_knowledge( 39 | project_dir: str, 40 | collection_name="project_knowledge_db", 41 | file_types=("*.java", "*.py"), 42 | batch_size=1000, 43 | ): 44 | """Creates the Project Knowledge component. Indexes all Python files in the given directory. 45 | 46 | Args: 47 | project_dir: The path of the project to index. 48 | """ 49 | 50 | print(f"Creating project knowledge for {project_dir!r}") 51 | repo = ( 52 | project_dir.split("/")[-1] 53 | if not project_dir.endswith("/") 54 | else project_dir.split("/")[-2] 55 | ) 56 | rc = runtime_config.RuntimeConfig() 57 | 58 | persist_directory = os.path.join(rc.runtime_dir, collection_name + "_" + repo) 59 | 60 | print(f"{persist_directory=}") 61 | if os.path.isdir(persist_directory): 62 | project_knowledge_db = Chroma( 63 | persist_directory=persist_directory, 64 | embedding_function=EMBEDDING_FUNCTION, 65 | collection_name=collection_name, 66 | ) 67 | else: 68 | print(f"Creating project knowledge for {repo} ({project_dir})") 69 | project_knowledge_db = None 70 | file_paths = [] 71 | 72 | for file_type in file_types: 73 | file_paths += glob( 74 | os.path.join(project_dir, "**/" + file_type), recursive=True 75 | ) 76 | 77 | total_files = len(file_paths) 78 | 79 | file_batches = [ 80 | file_paths[i : i + batch_size] 81 | for i in range(0, len(file_paths), batch_size) 82 | ] 83 | print( 84 | f"Preparing to process {total_files} total files in {len(file_batches)} batches" 85 | ) 86 | 87 | for file_batch_idx, file_batch in enumerate(tqdm(file_batches)): 88 | file_document_batch = [] 89 | func_document_batch = [] 90 | for file_path in tqdm(file_batch): 91 | with open(file_path, encoding="utf-8") as pyfile: 92 | file_content = pyfile.read() 93 | 94 | # File processing 95 | relative_file_path = file_path.replace(project_dir + "/", "") 96 | file_document_batch.append( 97 | Document( 98 | page_content=file_content, 99 | metadata={"file_path": relative_file_path, "type": "file"}, 100 | ) 101 | ) 102 | 103 | # Func processing 104 | file_type_ext = relative_file_path.split(".")[-1] 105 | parser = tree_sitter_parsers[file_type_ext] 106 | tree = parser.parse(file_content.encode()) 107 | 108 | func_defs = ( 109 | func_queries[file_type_ext].captures(tree.root_node).get("defs", []) 110 | ) 111 | if ( 112 | file_type_ext == "java" 113 | ): # Java contains a "constructor_declaration" node separate from the already queried "method_declarations" nodes 114 | constructor_defs = query_java_construcor_decs.captures( 115 | tree.root_node 116 | ).get("defs", []) 117 | func_defs = constructor_defs + func_defs 118 | for func_def in func_defs: 119 | func_content = func_def.text.decode() 120 | func_name = func_def.child_by_field_name("name").text.decode() 121 | func_document_batch.append( 122 | Document( 123 | page_content=func_content, 124 | metadata={ 125 | "file_path": relative_file_path, 126 | "func_name": func_name, 127 | "type": "func", 128 | }, 129 | ) 130 | ) 131 | 132 | # Chunk the docs 133 | file_document_batch_split = PROJECT_KNOWLEDGE_TEXT_SPLITTER.split_documents( 134 | file_document_batch 135 | ) 136 | func_document_batch_split = PROJECT_KNOWLEDGE_TEXT_SPLITTER.split_documents( 137 | func_document_batch 138 | ) 139 | 140 | print( 141 | f"Inserting {len(file_document_batch_split)} file and {len(func_document_batch_split)} func chunked documents for batch {file_batch_idx}" 142 | ) 143 | # Insert chunked docs Chroma 144 | if project_knowledge_db is None: 145 | project_knowledge_db = Chroma.from_documents( 146 | file_document_batch_split + func_document_batch_split, 147 | EMBEDDING_FUNCTION, 148 | collection_name=collection_name, 149 | persist_directory=persist_directory, 150 | ) 151 | else: 152 | project_knowledge_db.add_documents( 153 | file_document_batch_split + func_document_batch_split 154 | ) 155 | 156 | # Retrieve docs for log 157 | total_files, total_funcs = ( 158 | len(project_knowledge_db.get(where={"type": "file"})["ids"]), 159 | len(project_knowledge_db.get(where={"type": "func"})["ids"]), 160 | ) 161 | print( 162 | f"Connected to DB {persist_directory}:{collection_name} containing {total_files} total files and {total_funcs} total func documents." 163 | ) 164 | 165 | # create VectorStoreRetriever 166 | project_knowledge_retriever = project_knowledge_db.as_retriever() 167 | return project_knowledge_retriever, project_knowledge_db 168 | 169 | 170 | @tool 171 | def search_relevant_files(query: str, k=10): 172 | """Given a query search string (for example, the issue report description, filenames, etc), search for relevant code snippets of files in the project by calculating embedding similarity between the query and code snippets in a vector database. 173 | 174 | Args: 175 | query: A search string (for example, the issue report description, filenames, etc), to be used to find relevant files and functions. 176 | """ 177 | rc = runtime_config.RuntimeConfig() 178 | 179 | project_knowledge_retriever, _ = create_project_knowledge(rc.proj_path) 180 | 181 | relevant_docs = project_knowledge_retriever.get_relevant_documents(query, k=k) 182 | 183 | full_result = [] 184 | return_string = f"Top {k} most relevant files: \n\n" 185 | print("-----RELEVANT DOCS-----") 186 | for doc in relevant_docs: 187 | return_string += doc.metadata["file_path"] + "\n" 188 | # if "func_name" in doc.metadata and doc.metadata["type"] == "func": 189 | if "name" in doc.metadata: 190 | full_result.append( 191 | { 192 | # "file_path": doc.metadata["file_path"] + ":" + doc.metadata["name"] + "()", 193 | "file_path": doc.metadata["file_path"] + ":" + doc.metadata["name"], 194 | "code_snippet": doc.page_content, 195 | } 196 | ) 197 | 198 | else: 199 | full_result.append( 200 | { 201 | "file_path": doc.metadata["file_path"], 202 | "code_snippet": doc.page_content, 203 | } 204 | ) 205 | 206 | return_string = return_string.strip() 207 | 208 | explain_prompt = RELEVANT_FILE_EXPLANATION_SYSTEM_PROMPT.substitute( 209 | search_term=query, k=k, full_result=full_result 210 | ).strip() 211 | generate_explanation = llm.invoke([HumanMessage(explain_prompt)]) 212 | 213 | explanations = relevant_file_explanations_parser.invoke( 214 | generate_explanation.content 215 | ) 216 | return explanations 217 | 218 | 219 | def summarizer(stage_msgs_processed): 220 | """Summarize the information of previous chat history to gain addtiional information or remember what you were doing.""" 221 | stage_message_keys = list(stage_msgs_processed.keys()) 222 | stage_messages = {} 223 | for k in stage_message_keys: 224 | stage_messages[k] = stage_msgs_processed[k] 225 | 226 | summary_prompt = f"Summarize the messages in the following conversation. Be sure to include aggregated details of the key steps and or goals of the message. Include the names of the agents and tools features in the steps. If the agent did not describe its process but used a tool mention the used tool(s). Also include any raw content such as problem statements, solution plans, generated code and or patches if applicable. Be sure to only output your result. Here are the message(s):\n```{stage_messages}\n\nHere is an example of the result:\n```\nStep 1: The user submitted the issue to be resolved.\nStep 2. The supervisor delegated the task to the problem_decoder\nStep 3. The problem_decoder asked the context_manager for help\nStep 4. The context_manager searched for relevant files in the codebase, including file1.py, file2.py.\nStep 5. The context_manager viewed the file file5.py using `view_file_content`.```" 227 | 228 | response = llm.invoke( 229 | [ 230 | HumanMessage( 231 | summary_prompt.strip(), 232 | name="summarizer_agent", 233 | ) 234 | ] 235 | ) 236 | 237 | summary = response.content 238 | 239 | return summary 240 | -------------------------------------------------------------------------------- /src/agent/tool_set/edit_history.py: -------------------------------------------------------------------------------- 1 | """History management for file edits with disk-based storage and memory constraints.""" 2 | 3 | import tempfile 4 | from pathlib import Path 5 | from typing import Optional 6 | 7 | from diskcache import Cache 8 | 9 | 10 | class FileHistoryManager: 11 | """Manages file edit history with disk-based storage and memory constraints.""" 12 | 13 | def __init__( 14 | self, max_history_per_file: int = 5, history_dir: Optional[Path] = None 15 | ): 16 | """Initialize the history manager. 17 | 18 | Args: 19 | max_history_per_file: Maximum number of history entries to keep per file (default: 5) 20 | history_dir: Directory to store history files. If None, uses a temp directory 21 | 22 | Notes: 23 | - Each file's history is limited to the last N entries to conserve memory 24 | - The disk cache is limited to 500MB total to prevent excessive disk usage 25 | - Older entries are automatically removed when limits are exceeded 26 | """ 27 | self.max_history_per_file = max_history_per_file 28 | if history_dir is None: 29 | history_dir = Path(tempfile.mkdtemp(prefix='oh_editor_history_')) 30 | self.cache = Cache(str(history_dir), size_limit=5e8) # 500MB size limit 31 | 32 | def add_history(self, file_path: Path, content: str): 33 | """Add a new history entry for a file.""" 34 | key = str(file_path) 35 | # Get list of entry indices and counter for this file 36 | entries_key = f'{key}:entries' 37 | counter_key = f'{key}:counter' 38 | entries = self.cache.get(entries_key, []) 39 | counter = self.cache.get(counter_key, 0) 40 | 41 | # Add new entry with monotonically increasing counter 42 | entry_key = f'{key}:{counter}' 43 | self.cache.set(entry_key, content) 44 | entries.append(entry_key) 45 | counter += 1 46 | 47 | # Keep only last N entries 48 | if len(entries) > self.max_history_per_file: 49 | old_key = entries.pop(0) 50 | self.cache.delete(old_key) 51 | 52 | # Update entries list and counter 53 | self.cache.set(entries_key, entries) 54 | self.cache.set(counter_key, counter) 55 | 56 | def get_last_history(self, file_path: Path) -> Optional[str]: 57 | """Get the most recent history entry for a file.""" 58 | key = str(file_path) 59 | entries_key = f'{key}:entries' 60 | entries = self.cache.get(entries_key, []) 61 | 62 | if not entries: 63 | return None 64 | 65 | # Get and remove last entry 66 | last_key = entries.pop() 67 | content = self.cache.get(last_key) 68 | self.cache.delete(last_key) 69 | 70 | # Update entries list 71 | self.cache.set(entries_key, entries) 72 | return content 73 | 74 | def clear_history(self, file_path: Path): 75 | """Clear history for a given file.""" 76 | key = str(file_path) 77 | entries_key = f'{key}:entries' 78 | counter_key = f'{key}:counter' 79 | entries = self.cache.get(entries_key, []) 80 | 81 | # Delete all entries 82 | for entry_key in entries: 83 | self.cache.delete(entry_key) 84 | 85 | # Delete entries list and counter 86 | self.cache.delete(entries_key) 87 | self.cache.delete(counter_key) -------------------------------------------------------------------------------- /src/agent/tool_set/edit_tool.py: -------------------------------------------------------------------------------- 1 | from typing import Annotated, List, Optional 2 | 3 | from langchain_core.tools import tool 4 | from langgraph.prebuilt import InjectedState 5 | from langgraph.prebuilt.chat_agent_executor import AgentState 6 | from agent.state import CustomState 7 | 8 | from agent import runtime_config 9 | from agent.runtime_config import RuntimeConfig 10 | from agent.tool_set.sepl_tools import save_git_diff 11 | from agent.tool_set.oheditor import CLIResult, OHEditor 12 | from langchain_core.runnables import RunnableConfig 13 | _GLOBAL_EDITOR = OHEditor() 14 | 15 | 16 | def _make_cli_result(tool_result: CLIResult) -> str: 17 | """Convert an CLIResult to an API ToolResultBlockParam.""" 18 | if tool_result.error: 19 | return f"ERROR:\n{tool_result.error}" 20 | 21 | assert tool_result.output, "Expected output in file_editor." 22 | return tool_result.output 23 | 24 | 25 | @tool 26 | def str_replace_editor( 27 | command: Annotated[str, "The command to be executed (view, create, str_replace, insert)"], 28 | path: Annotated[str, "Relative path from root of the repository to file or directory, e.g., 'file.py' or 'workspace'"], 29 | config: RunnableConfig, 30 | file_text: Optional[str] = None, 31 | old_str: Optional[str] = None, 32 | new_str: Optional[str] = None, 33 | insert_line: Optional[int] = None, 34 | view_range: Optional[List[int]] = None, 35 | # runtime_info: Annotated[dict, InjectedState("runtime_info")] = None, 36 | ): 37 | """ 38 | Custom editing tool for viewing, creating and editing files in plain-text format 39 | * State is persistent across command calls and discussions with the user 40 | * If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep 41 | * The `create` command cannot be used if the specified `path` already exists as a file 42 | * If a `command` generates a long output, it will be truncated and marked with `` 43 | 44 | 45 | Before using this tool to edit a file: 46 | 1. Use the `view` command to understand the file's contents and context 47 | 2. Verify the directory path is correct (only applicable when creating new files): 48 | - Use the `view` command to verify the parent directory exists and is the correct location 49 | 50 | When making edits: 51 | - Ensure the edit results in idiomatic, correct code 52 | - Do not leave the code in a broken state 53 | - Always use relative file paths (starting with ./) 54 | 55 | CRITICAL REQUIREMENTS FOR USING THIS TOOL: 56 | 57 | 1. EXACT MATCHING: The `old_str` parameter must match EXACTLY one or more consecutive lines from the file, including all whitespace and indentation. The tool will fail if `old_str` matches multiple locations or doesn't match exactly with the file content. 58 | 59 | 2. UNIQUENESS: The `old_str` must uniquely identify a single instance in the file: 60 | - Include sufficient context before and after the change point (3-5 lines recommended) 61 | - If not unique, the replacement will not be performed 62 | 63 | 3. REPLACEMENT: The `new_str` parameter should contain the edited lines that replace the `old_str`. Both strings must be different. 64 | 65 | Remember: when making multiple file edits in a row to the same file, you should prefer to send all edits in a single message with multiple calls to this tool, rather than multiple messages with a single call each. 66 | 67 | 68 | Args: 69 | command (str): The commands to run. Allowed options are: `view`, `create`, `str_replace`, `insert`. 70 | path (str): Absolute path to file or directory, e.g. `/workspace/file.py` or `/workspace`. 71 | file_text (Optional[str]): Required parameter of `create` command, with the content of the file to be created. 72 | old_str (Optional[str]): Required parameter of `str_replace` command containing the string in `path` to replace. 73 | new_str (Optional[str]): Optional parameter of `str_replace` command containing the new string (if not given, no string will be added). Required parameter of `insert` command containing the string to insert. 74 | insert_line (Optional[int]): Required parameter of `insert` command. The `new_str` will be inserted AFTER the line `insert_line` of `path`. 75 | view_range (Optional[List[int]]): Optional parameter of `view` command when `path` points to a file. If none is given, the full file is shown. If provided, the file will be shown in the indicated line number range, e.g. [100, 600] will show content between line 100 and 600. Indexing at 1 to start. Setting `[start_line, -1]` shows all lines from `start_line` to the end of the file. Unless you are sure about the line numbers, otherwise, do not set this parameter and use the `view` command to view the whole file. 76 | 77 | """ 78 | # try to fetch project_path from config, it might not exist 79 | proj_path = config.get("configurable", {}).get("proj_path") 80 | if proj_path is None: 81 | rc = runtime_config.RuntimeConfig() 82 | assert rc.initialized 83 | proj_path = rc.proj_path 84 | print(f"use global runtime config project path: {proj_path}") 85 | else: 86 | print(f"use configrable config project path: {proj_path}") 87 | result = _GLOBAL_EDITOR( 88 | command=command, 89 | path=path, 90 | file_text=file_text, 91 | view_range=view_range, 92 | old_str=old_str, 93 | new_str=new_str, 94 | insert_line=insert_line, 95 | proj_path=proj_path, 96 | ) 97 | return _make_cli_result(result) 98 | 99 | 100 | if __name__ == "__main__": 101 | rc = runtime_config.RuntimeConfig() 102 | rc.load_from_preset("gitpython-developers+GitPython@1413.yaml") 103 | print("=" * 50) 104 | rc.pretty_print_runtime() 105 | print("=" * 50) 106 | # print(view_directory.invoke({})) 107 | print(str_replace_editor.invoke({"command": "view", "path": "./django/db/models/query.py"})) 108 | # print( 109 | # str_replace_editor.invoke( 110 | # { 111 | # "command": "str_replace", 112 | # "path": "django/contrib/messages/storage/cookie.py", 113 | # "old_str": "if obj.extra_tags:", 114 | # "new_str": "if obj.extra_tags is not None:", 115 | # } 116 | # ) 117 | # ) 118 | 119 | # mock a create 120 | # print(str_replace_editor.invoke({"command": "create", "path": "doc/generate_logos_new.py", "file_text": "print('Hello, world!')"})) 121 | 122 | # mock a insert 123 | # print(str_replace_editor.invoke({"command": "insert", "path": "doc/generate_logos_new.py", "insert_line": 0, "new_str": "print('Hey uou, world!')"})) 124 | 125 | # undo the insert 126 | # print(str_replace_editor.invoke({"command": "undo_edit", "path": "doc/generate_logos.py"})) 127 | 128 | latest_patch = save_git_diff() 129 | print(f"Latest patch:\n{latest_patch}") 130 | # mock a undo_edit 131 | # print(str_replace_editor.invoke({"command": "undo_edit", "path": "doc/generate_logos_new.py"})) 132 | -------------------------------------------------------------------------------- /src/agent/tool_set/linter/__init__.py: -------------------------------------------------------------------------------- 1 | """Linter module for OpenHands ACI. 2 | 3 | Part of this Linter module is adapted from Aider (Apache 2.0 License, [original code](https://github.com/paul-gauthier/aider/blob/main/aider/linter.py)). Please see the [original repository](https://github.com/paul-gauthier/aider) for more information. 4 | """ 5 | 6 | from .base import LintResult 7 | from .linter import DefaultLinter 8 | 9 | __all__ = ['DefaultLinter', 'LintResult'] 10 | -------------------------------------------------------------------------------- /src/agent/tool_set/linter/base.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | from pydantic import BaseModel 4 | 5 | 6 | class LintResult(BaseModel): 7 | file: str 8 | line: int # 1-indexed 9 | column: int # 1-indexed 10 | message: str 11 | 12 | def visualize(self, half_window: int = 3) -> str: 13 | """Visualize the lint result by print out all the lines where the lint result is found. 14 | 15 | Args: 16 | half_window: The number of context lines to display around the error on each side. 17 | """ 18 | with open(self.file, 'r') as f: 19 | file_lines = f.readlines() 20 | 21 | # Add line numbers 22 | _span_size = len(str(len(file_lines))) 23 | file_lines = [ 24 | f'{i + 1:>{_span_size}}|{line.rstrip()}' 25 | for i, line in enumerate(file_lines) 26 | ] 27 | 28 | # Get the window of lines to display 29 | assert self.line <= len(file_lines) and self.line > 0 30 | line_idx = self.line - 1 31 | begin_window = max(0, line_idx - half_window) 32 | end_window = min(len(file_lines), line_idx + half_window + 1) 33 | 34 | selected_lines = file_lines[begin_window:end_window] 35 | line_idx_in_window = line_idx - begin_window 36 | 37 | # Add character hint 38 | _character_hint = ( 39 | _span_size * ' ' 40 | + ' ' * (self.column) 41 | + '^' 42 | + ' ERROR HERE: ' 43 | + self.message 44 | ) 45 | selected_lines[line_idx_in_window] = ( 46 | f'\033[91m{selected_lines[line_idx_in_window]}\033[0m' 47 | + '\n' 48 | + _character_hint 49 | ) 50 | return '\n'.join(selected_lines) 51 | 52 | 53 | class LinterException(Exception): 54 | """Base class for all linter exceptions.""" 55 | 56 | pass 57 | 58 | 59 | class BaseLinter(ABC): 60 | """Base class for all linters. 61 | 62 | Each linter should be able to lint files of a specific type and return a list of (parsed) lint results. 63 | """ 64 | 65 | encoding: str = 'utf-8' 66 | 67 | @property 68 | @abstractmethod 69 | def supported_extensions(self) -> list[str]: 70 | """The file extensions that this linter supports, such as .py or .tsx.""" 71 | return [] 72 | 73 | @abstractmethod 74 | def lint(self, file_path: str) -> list[LintResult]: 75 | """Lint the given file. 76 | 77 | file_path: The path to the file to lint. Required to be absolute. 78 | """ 79 | pass 80 | -------------------------------------------------------------------------------- /src/agent/tool_set/linter/impl/python.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from agent.tool_set.utils import run_shell_local 3 | 4 | from ..base import BaseLinter, LintResult 5 | 6 | 7 | def python_compile_lint(fname: str) -> list[LintResult]: 8 | try: 9 | with open(fname, 'r') as f: 10 | code = f.read() 11 | compile(code, fname, 'exec') # USE TRACEBACK BELOW HERE 12 | return [] 13 | except SyntaxError as err: 14 | err_lineno = getattr(err, 'end_lineno', err.lineno) 15 | err_offset = getattr(err, 'end_offset', err.offset) 16 | if err_offset and err_offset < 0: 17 | err_offset = err.offset 18 | return [ 19 | LintResult( 20 | file=fname, line=err_lineno, column=err_offset or 1, message=err.msg 21 | ) 22 | ] 23 | 24 | 25 | def flake_lint(filepath: str) -> list[LintResult]: 26 | fatal = 'F821,F822,F831,E112,E113,E999,E902' 27 | flake8_cmd = f'flake8 --select={fatal} --isolated {filepath}' 28 | 29 | try: 30 | cmd_outputs = run_shell_local(flake8_cmd, truncate_after=None)[1] 31 | except FileNotFoundError: 32 | return [] 33 | results: list[LintResult] = [] 34 | if not cmd_outputs: 35 | return results 36 | for line in cmd_outputs.splitlines(): 37 | parts = line.split(':') 38 | if len(parts) >= 4: 39 | _msg = parts[3].strip() 40 | if len(parts) > 4: 41 | _msg += ': ' + parts[4].strip() 42 | 43 | try: 44 | line_num = int(parts[1]) 45 | except ValueError as e: 46 | print( 47 | f'Error parsing flake8 output for line: {e}. Parsed parts: {parts}. Skipping...' 48 | ) 49 | continue 50 | 51 | try: 52 | column_num = int(parts[2]) 53 | except ValueError as e: 54 | column_num = 1 55 | _msg = ( 56 | parts[2].strip() + ' ' + _msg 57 | ) # add the unparsed message to the original message 58 | print( 59 | f'Error parsing flake8 output for column: {e}. Parsed parts: {parts}. Using default column 1.' 60 | ) 61 | 62 | results.append( 63 | LintResult( 64 | file=filepath, 65 | line=line_num, 66 | column=column_num, 67 | message=_msg, 68 | ) 69 | ) 70 | return results 71 | 72 | 73 | class PythonLinter(BaseLinter): 74 | @property 75 | def supported_extensions(self) -> List[str]: 76 | return ['.py'] 77 | 78 | def lint(self, file_path: str) -> list[LintResult]: 79 | error = flake_lint(file_path) 80 | if not error: 81 | error = python_compile_lint(file_path) 82 | return error 83 | 84 | def compile_lint(self, file_path: str, code: str) -> List[LintResult]: 85 | try: 86 | compile(code, file_path, 'exec') 87 | return [] 88 | except SyntaxError as e: 89 | return [ 90 | LintResult( 91 | file=file_path, 92 | line=e.lineno, 93 | column=e.offset, 94 | message=str(e), 95 | rule='SyntaxError', 96 | ) 97 | ] 98 | -------------------------------------------------------------------------------- /src/agent/tool_set/linter/impl/treesitter.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | from grep_ast import TreeContext, filename_to_lang 4 | from grep_ast.parsers import PARSERS 5 | 6 | from ..base import BaseLinter, LintResult 7 | from .treesitter_compat import get_parser 8 | 9 | # tree_sitter is throwing a FutureWarning 10 | warnings.simplefilter('ignore', category=FutureWarning) 11 | 12 | 13 | def tree_context(fname, code, line_nums): 14 | context = TreeContext( 15 | fname, 16 | code, 17 | color=False, 18 | line_number=True, 19 | child_context=False, 20 | last_line=False, 21 | margin=0, 22 | mark_lois=True, 23 | loi_pad=3, 24 | # header_max=30, 25 | show_top_of_file_parent_scope=False, 26 | ) 27 | line_nums = set(line_nums) 28 | context.add_lines_of_interest(line_nums) 29 | context.add_context() 30 | output = context.format() 31 | return output 32 | 33 | 34 | def traverse_tree(node): 35 | """Traverses the tree to find errors.""" 36 | errors = [] 37 | if node.type == 'ERROR' or node.is_missing: 38 | line_no = node.start_point[0] + 1 39 | col_no = node.start_point[1] + 1 40 | error_type = 'Missing node' if node.is_missing else 'Syntax error' 41 | errors.append((line_no, col_no, error_type)) 42 | 43 | for child in node.children: 44 | errors += traverse_tree(child) 45 | 46 | return errors 47 | 48 | 49 | class TreesitterBasicLinter(BaseLinter): 50 | @property 51 | def supported_extensions(self) -> list[str]: 52 | return list(PARSERS.keys()) 53 | 54 | def lint(self, file_path: str) -> list[LintResult]: 55 | """Use tree-sitter to look for syntax errors, display them with tree context.""" 56 | lang = filename_to_lang(file_path) 57 | if not lang: 58 | return [] 59 | parser = get_parser(lang) 60 | with open(file_path, 'r') as f: 61 | code = f.read() 62 | tree = parser.parse(bytes(code, 'utf-8')) 63 | errors = traverse_tree(tree.root_node) 64 | if not errors: 65 | return [] 66 | return [ 67 | LintResult( 68 | file=file_path, 69 | line=int(line), 70 | column=int(col), 71 | message=error_details, 72 | ) 73 | for line, col, error_details in errors 74 | ] 75 | -------------------------------------------------------------------------------- /src/agent/tool_set/linter/impl/treesitter_compat.py: -------------------------------------------------------------------------------- 1 | """Compatibility layer for tree-sitter 0.24.0.""" 2 | 3 | import importlib 4 | 5 | from tree_sitter import Language, Parser 6 | 7 | # Cache of loaded languages 8 | _language_cache = {} 9 | 10 | 11 | def get_parser(language): 12 | """Get a Parser object for the given language name.""" 13 | if language not in _language_cache: 14 | # Try to import the language module 15 | module_name = f'tree_sitter_{language}' 16 | try: 17 | module = importlib.import_module(module_name) 18 | _language_cache[language] = Language(module.language()) 19 | except ImportError: 20 | raise ValueError( 21 | f'Language {language} is not supported. Please install {module_name} package.' 22 | ) 23 | 24 | return Parser(_language_cache[language]) 25 | -------------------------------------------------------------------------------- /src/agent/tool_set/linter/linter.py: -------------------------------------------------------------------------------- 1 | import os 2 | from collections import defaultdict 3 | from difflib import SequenceMatcher 4 | 5 | from ..linter.base import BaseLinter, LinterException, LintResult 6 | from ..linter.impl.python import PythonLinter 7 | from ..linter.impl.treesitter import TreesitterBasicLinter 8 | 9 | 10 | class DefaultLinter(BaseLinter): 11 | def __init__(self): 12 | self.linters: dict[str, list[BaseLinter]] = defaultdict(list) 13 | self.linters['.py'] = [PythonLinter()] 14 | 15 | # Add treesitter linter as a fallback for all linters 16 | self.basic_linter = TreesitterBasicLinter() 17 | for extension in self.basic_linter.supported_extensions: 18 | self.linters[extension].append(self.basic_linter) 19 | self._supported_extensions = list(self.linters.keys()) 20 | 21 | @property 22 | def supported_extensions(self) -> list[str]: 23 | return self._supported_extensions 24 | 25 | def lint(self, file_path: str) -> list[LintResult]: 26 | if not os.path.isabs(file_path): 27 | raise LinterException(f'File path {file_path} is not an absolute path') 28 | file_extension = os.path.splitext(file_path)[1] 29 | 30 | linters: list[BaseLinter] = self.linters.get(file_extension, []) 31 | for linter in linters: 32 | res = linter.lint(file_path) 33 | # We always return the first linter's result (higher priority) 34 | if res: 35 | return res 36 | return [] 37 | 38 | def lint_file_diff( 39 | self, original_file_path: str, updated_file_path: str 40 | ) -> list[LintResult]: 41 | """Only return lint errors that are introduced by the diff. 42 | 43 | Args: 44 | original_file_path: The original file path. 45 | updated_file_path: The updated file path. 46 | 47 | Returns: 48 | A list of lint errors that are introduced by the diff. 49 | """ 50 | # 1. Lint the original and updated file 51 | original_lint_errors: list[LintResult] = self.lint(original_file_path) 52 | updated_lint_errors: list[LintResult] = self.lint(updated_file_path) 53 | 54 | # 2. Load the original and updated file content 55 | with open(original_file_path, 'r') as f: 56 | old_lines = f.readlines() 57 | with open(updated_file_path, 'r') as f: 58 | new_lines = f.readlines() 59 | 60 | # 3. Get line numbers that are changed & unchanged 61 | # Map the line number of the original file to the updated file 62 | # NOTE: this only works for lines that are not changed (i.e., equal) 63 | old_to_new_line_no_mapping: dict[int, int] = {} 64 | replace_or_inserted_lines: list[int] = [] 65 | for ( 66 | tag, 67 | old_idx_start, 68 | old_idx_end, 69 | new_idx_start, 70 | new_idx_end, 71 | ) in SequenceMatcher( 72 | isjunk=None, 73 | a=old_lines, 74 | b=new_lines, 75 | ).get_opcodes(): 76 | if tag == 'equal': 77 | for idx, _ in enumerate(old_lines[old_idx_start:old_idx_end]): 78 | old_to_new_line_no_mapping[old_idx_start + idx + 1] = ( 79 | new_idx_start + idx + 1 80 | ) 81 | elif tag == 'replace' or tag == 'insert': 82 | for idx, _ in enumerate(old_lines[old_idx_start:old_idx_end]): 83 | replace_or_inserted_lines.append(new_idx_start + idx + 1) 84 | else: 85 | # omit the case of delete 86 | pass 87 | 88 | # 4. Get pre-existing errors in unchanged lines 89 | # increased error elsewhere introduced by the newlines 90 | # i.e., we omit errors that are already in original files and report new one 91 | new_line_no_to_original_errors: dict[int, list[LintResult]] = defaultdict(list) 92 | for error in original_lint_errors: 93 | if error.line in old_to_new_line_no_mapping: 94 | new_line_no_to_original_errors[ 95 | old_to_new_line_no_mapping[error.line] 96 | ].append(error) 97 | 98 | # 5. Select errors from lint results in new file to report 99 | selected_errors = [] 100 | for error in updated_lint_errors: 101 | # 5.1. Error introduced by replace/insert 102 | if error.line in replace_or_inserted_lines: 103 | selected_errors.append(error) 104 | # 5.2. Error introduced by modified lines that impacted 105 | # the unchanged lines that HAVE pre-existing errors 106 | elif error.line in new_line_no_to_original_errors: 107 | # skip if the error is already reported 108 | # or add if the error is new 109 | if not any( 110 | original_error.message == error.message 111 | and original_error.column == error.column 112 | for original_error in new_line_no_to_original_errors[error.line] 113 | ): 114 | selected_errors.append(error) 115 | # 5.3. Error introduced by modified lines that impacted 116 | # the unchanged lines that have NO pre-existing errors 117 | else: 118 | selected_errors.append(error) 119 | 120 | # 6. Sort errors by line and column 121 | selected_errors.sort(key=lambda x: (x.line, x.column)) 122 | return selected_errors 123 | -------------------------------------------------------------------------------- /src/agent/tool_set/oheditor.py: -------------------------------------------------------------------------------- 1 | # This file is is adapted from OpenHands 2 | # https://github.com/All-Hands-AI/openhands-aci/blob/main/openhands_aci/editor/editor.py 3 | import mimetypes 4 | import os 5 | import re 6 | import shutil 7 | import subprocess 8 | import tempfile 9 | import time 10 | from dataclasses import asdict, dataclass, fields 11 | from pathlib import Path 12 | from typing import Literal, get_args 13 | from agent.tool_set.linter import DefaultLinter 14 | from agent.tool_set.utils import run_shell_local, maybe_truncate 15 | from agent.tool_set.constant import * 16 | from agent import runtime_config 17 | # from agent.tool_set.edit_history import FileHistoryManager 18 | 19 | Command = Literal[ 20 | "view", 21 | "create", 22 | "str_replace", 23 | "insert", 24 | # "undo_edit", 25 | # 'jump_to_definition', TODO: 26 | # 'find_references' TODO: 27 | ] 28 | 29 | 30 | @dataclass 31 | class CLIResult: 32 | """A ToolResult that can be rendered as a CLI output.""" 33 | 34 | output: str | None = None 35 | error: str | None = None 36 | # Optional fields for file editing commands 37 | path: str | None = None 38 | prev_exist: bool = True 39 | old_content: str | None = None 40 | new_content: str | None = None 41 | 42 | def __bool__(self): 43 | return any(getattr(self, field.name) for field in fields(self)) 44 | 45 | def to_dict(self, extra_field: dict | None = None) -> dict: 46 | result = asdict(self) 47 | 48 | # Add extra fields if provided 49 | if extra_field: 50 | result.update(extra_field) 51 | return result 52 | 53 | 54 | class OHEditor: 55 | """ 56 | An filesystem editor tool that allows the agent to 57 | - view 58 | - create 59 | - navigate 60 | - edit files 61 | The tool parameters are defined by Anthropic and are not editable. 62 | 63 | Original implementation: https://github.com/anthropics/anthropic-quickstarts/blob/main/computer-use-demo/computer_use_demo/tools/edit.py 64 | """ 65 | 66 | TOOL_NAME = "oh_editor" 67 | MAX_FILE_SIZE_MB = 10 # Maximum file size in MB 68 | 69 | def __init__(self, max_file_size_mb: int | None = None): 70 | """Initialize the editor. 71 | 72 | Args: 73 | max_file_size_mb: Maximum file size in MB. If None, uses the default MAX_FILE_SIZE_MB. 74 | """ 75 | self._linter = DefaultLinter() 76 | # self._history_manager = FileHistoryManager(max_history_per_file=10) 77 | self._max_file_size = (max_file_size_mb or self.MAX_FILE_SIZE_MB) * 1024 * 1024 # Convert to bytes 78 | 79 | def __call__( 80 | self, 81 | *, 82 | command: Command, 83 | path: str, 84 | file_text: str | None = None, 85 | view_range: list[int] | None = None, 86 | old_str: str | None = None, 87 | new_str: str | None = None, 88 | insert_line: int | None = None, 89 | enable_linting: bool = False, 90 | proj_path: str | None = None, 91 | **kwargs, 92 | ) -> CLIResult: 93 | _path = Path(os.path.join(proj_path, path)) 94 | 95 | print( 96 | f"path: {_path}, command:{command}, file_text:{file_text}, view_range:{view_range}, old_str:{old_str}, new_str:{new_str}, insert_line:{insert_line}, linting:{enable_linting}" 97 | ) 98 | 99 | # if file ends with .py, enable linting 100 | if _path.suffix == ".py": 101 | enable_linting = True 102 | 103 | # code.interact('OH Editor', local=dict(globals(), **locals())) 104 | self.validate_path(command, _path) 105 | if command == "view": 106 | return self.view(_path, view_range) 107 | elif command == "create": 108 | if file_text is None: 109 | return f"Error: Missing parameter 'file_text' for command '{command}'" 110 | self.write_file(_path, file_text) 111 | # self._history_manager.add_history(_path, file_text) 112 | return CLIResult( 113 | path=str(_path), 114 | new_content=file_text, 115 | prev_exist=False, 116 | output=f"File created successfully at: {_path}", 117 | ) 118 | elif command == "str_replace": 119 | if old_str is None: 120 | return CLIResult( 121 | error=f"Error: Missing parameter 'old_str' for command '{command}'", 122 | path=str(_path), 123 | prev_exist=True, 124 | ) 125 | if new_str == old_str: 126 | return CLIResult( 127 | error="Error: `new_str` and `old_str` must be different.", 128 | path=str(_path), 129 | prev_exist=True, 130 | ) 131 | return self.str_replace(_path, old_str, new_str, enable_linting) 132 | elif command == "insert": 133 | if insert_line is None: 134 | return CLIResult( 135 | error=f"Error: Missing parameter 'insert_line' for command '{command}'", 136 | path=str(_path), 137 | prev_exist=True, 138 | ) 139 | if new_str is None: 140 | return CLIResult( 141 | error=f"Error: Missing parameter 'new_str' for command '{command}'", 142 | path=str(_path), 143 | prev_exist=True, 144 | ) 145 | return self.insert(_path, insert_line, new_str, enable_linting) 146 | # elif command == "undo_edit": 147 | # return self.undo_edit(_path) 148 | 149 | return CLIResult( 150 | error=f"Error: Unrecognized command {command}. The allowed commands for the {self.TOOL_NAME} tool are: {', '.join(get_args(Command))}", 151 | path=str(_path), 152 | prev_exist=True, 153 | ) 154 | 155 | def _count_lines(self, path: Path) -> int: 156 | """ 157 | Count the number of lines in a file safely. 158 | """ 159 | # print(f"path: {path}") 160 | assert path.exists() 161 | with open(path) as f: 162 | return sum(1 for _ in f) 163 | 164 | def str_replace(self, path: Path, old_str: str, new_str: str | None, enable_linting: bool) -> CLIResult: 165 | """ 166 | Implement the str_replace command, which replaces old_str with new_str in the file content. 167 | """ 168 | self.validate_file(path) 169 | old_str = old_str.expandtabs() 170 | new_str = new_str.expandtabs() if new_str is not None else "" 171 | 172 | # Read the entire file first to handle both single-line and multi-line replacements 173 | file_content = self.read_file(path).expandtabs() 174 | 175 | # Find all occurrences using regex 176 | # Escape special regex characters in old_str to match it literally 177 | pattern = re.escape(old_str) 178 | occurrences = [ 179 | ( 180 | file_content.count("\n", 0, match.start()) + 1, # line number 181 | match.group(), # matched text 182 | match.start(), # start position 183 | ) 184 | for match in re.finditer(pattern, file_content) 185 | ] 186 | 187 | if not occurrences: 188 | return CLIResult( 189 | error=f"Error: No replacement was performed, old_str `{old_str}` did not appear verbatim in {path}.", 190 | path=str(path), 191 | prev_exist=True, 192 | ) 193 | if len(occurrences) > 1: 194 | line_numbers = sorted(set(line for line, _, _ in occurrences)) 195 | return CLIResult( 196 | error=f"Error: No replacement was performed. Multiple occurrences of old_str `{old_str}` in lines {line_numbers}. Please ensure it is unique.", 197 | path=str(path), 198 | prev_exist=True, 199 | ) 200 | 201 | # We found exactly one occurrence 202 | replacement_line, matched_text, idx = occurrences[0] 203 | 204 | # Create new content by replacing just the matched text 205 | new_file_content = file_content[:idx] + new_str + file_content[idx + len(matched_text) :] 206 | 207 | # Write the new content to the file 208 | self.write_file(path, new_file_content) 209 | 210 | # Save the content to history 211 | # self._history_manager.add_history(path, file_content) 212 | 213 | # Create a snippet of the edited section 214 | start_line = max(0, replacement_line - SNIPPET_CONTEXT_WINDOW) 215 | end_line = replacement_line + SNIPPET_CONTEXT_WINDOW + new_str.count("\n") 216 | 217 | # Read just the snippet range 218 | snippet = self.read_file(path, start_line=start_line, end_line=end_line) 219 | 220 | # Prepare the success message 221 | success_message = f"The file {path} has been edited. " 222 | success_message += self._make_output(snippet, f"a snippet of {path}", start_line + 1) 223 | 224 | if enable_linting: 225 | # Run linting on the changes 226 | lint_results = self._run_linting(file_content, new_file_content, path) 227 | success_message += "\n" + lint_results + "\n" 228 | 229 | success_message += ( 230 | "Review the changes and make sure they are as expected. Edit the file again if necessary." 231 | ) 232 | return CLIResult( 233 | output=success_message, 234 | prev_exist=True, 235 | path=str(path), 236 | old_content=file_content, 237 | new_content=new_file_content, 238 | ) 239 | 240 | def view(self, path: Path, view_range: list[int] | None = None) -> CLIResult: 241 | # print(f"view path: {path}") 242 | """ 243 | View the contents of a file or a directory. 244 | """ 245 | if path.is_dir(): 246 | if view_range: 247 | return CLIResult( 248 | error="Error: The `view_range` parameter is not allowed when `path` points to a directory.", 249 | path=str(path), 250 | prev_exist=True, 251 | ) 252 | 253 | # First count hidden files/dirs in current directory only 254 | # -mindepth 1 excludes . and .. automatically 255 | # _, hidden_stdout, _ = subprocess.run([rf"find -L {path} -mindepth 1 -maxdepth 1 -name '.*'"]) 256 | # hidden_count = len(hidden_stdout.strip().split("\n")) if hidden_stdout.strip() else 0 257 | 258 | # Then get files/dirs up to 2 levels deep, excluding hidden entries at both depth 1 and 2 259 | _, stdout, stderr = run_shell_local( 260 | rf"find -L {path} -maxdepth 2 -not \( -path '{path}/\.*' -o -path '{path}/*/\.*' \) | sort", 261 | truncate_notice=DIRECTORY_CONTENT_TRUNCATED_NOTICE, 262 | ) 263 | if not stderr: 264 | # Add trailing slashes to directories 265 | paths = stdout.strip().split("\n") if stdout.strip() else [] 266 | formatted_paths = [] 267 | for p in paths: 268 | if Path(p).is_dir(): 269 | formatted_paths.append(f"{p}/") 270 | else: 271 | formatted_paths.append(p) 272 | 273 | msg = [ 274 | f"Here's the files and directories up to 2 levels deep in {path}, excluding hidden items:\n" 275 | + "\n".join(formatted_paths) 276 | ] 277 | # if hidden_count > 0: 278 | # msg.append( 279 | # f"\n{hidden_count} hidden files/directories in this directory are excluded. You can use 'ls -la {path}' to see them." 280 | # ) 281 | stdout = "\n".join(msg) 282 | return CLIResult( 283 | output=stdout, 284 | error=stderr, 285 | path=str(path), 286 | prev_exist=True, 287 | ) 288 | 289 | # Validate file and count lines 290 | self.validate_file(path) 291 | num_lines = self._count_lines(path) 292 | 293 | start_line = 1 294 | if not view_range: 295 | file_content = self.read_file(path) 296 | output = self._make_output(file_content, str(path), start_line) 297 | 298 | return CLIResult( 299 | output=output, 300 | path=str(path), 301 | prev_exist=True, 302 | ) 303 | 304 | if len(view_range) != 2 or not all(isinstance(i, int) for i in view_range): 305 | return CLIResult( 306 | error="Error: `view_range` should be a list of two integers.", 307 | path=str(path), 308 | prev_exist=True, 309 | ) 310 | 311 | start_line, end_line = view_range 312 | if start_line < 1 or start_line > num_lines: 313 | return CLIResult( 314 | error=f"Error: Its first element `{start_line}` should be within the range of lines of the file: {[1, num_lines]}.", 315 | path=str(path), 316 | prev_exist=True, 317 | ) 318 | 319 | if end_line > num_lines: 320 | return CLIResult( 321 | error=f"Error: Its second element `{end_line}` should be smaller than the number of lines in the file:{num_lines}.", 322 | path=str(path), 323 | prev_exist=True, 324 | ) 325 | 326 | if end_line != -1 and end_line < start_line: 327 | return CLIResult( 328 | error=f"Error: Its second element `{end_line}` should be greater than or equal to the first element `{start_line}`.", 329 | path=str(path), 330 | prev_exist=True, 331 | ) 332 | 333 | if end_line == -1: 334 | end_line = num_lines 335 | 336 | file_content = self.read_file(path, start_line=start_line, end_line=end_line) 337 | return CLIResult( 338 | path=str(path), 339 | output=self._make_output(file_content, str(path), start_line), 340 | prev_exist=True, 341 | ) 342 | 343 | def write_file(self, path: Path, file_text: str) -> None: 344 | """ 345 | Write the content of a file to a given path; raise a ToolError if an error occurs. 346 | """ 347 | self.validate_file(path) 348 | try: 349 | path.write_text(file_text) 350 | except Exception as e: 351 | return CLIResult( 352 | error=f"Error: Ran into {e} while trying to write to {path}", 353 | path=str(path), 354 | prev_exist=True, 355 | ) 356 | 357 | def insert(self, path: Path, insert_line: int, new_str: str, enable_linting: bool) -> CLIResult: 358 | """ 359 | Implement the insert command, which inserts new_str at the specified line in the file content. 360 | """ 361 | # Validate file and count lines 362 | self.validate_file(path) 363 | num_lines = self._count_lines(path) 364 | 365 | if insert_line < 0 or insert_line > num_lines: 366 | return CLIResult( 367 | error=f"Error: It should be within the range of lines of the file: {[0, num_lines]}", 368 | path=str(path), 369 | prev_exist=True, 370 | ) 371 | 372 | new_str = new_str.expandtabs() 373 | new_str_lines = new_str.split("\n") 374 | 375 | # Create temporary file for the new content 376 | with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file: 377 | # Copy lines before insert point and save them for history 378 | history_lines = [] 379 | with open(path, "r") as f: 380 | for i, line in enumerate(f, 1): 381 | if i > insert_line: 382 | break 383 | temp_file.write(line.expandtabs()) 384 | history_lines.append(line) 385 | 386 | # Insert new content 387 | for line in new_str_lines: 388 | temp_file.write(line + "\n") 389 | 390 | # Copy remaining lines and save them for history 391 | with open(path, "r") as f: 392 | for i, line in enumerate(f, 1): 393 | if i <= insert_line: 394 | continue 395 | temp_file.write(line.expandtabs()) 396 | history_lines.append(line) 397 | 398 | # Move temporary file to original location 399 | shutil.move(temp_file.name, path) 400 | 401 | # Read just the snippet range 402 | start_line = max(1, insert_line - SNIPPET_CONTEXT_WINDOW) 403 | end_line = min( 404 | num_lines + len(new_str_lines), 405 | insert_line + SNIPPET_CONTEXT_WINDOW + len(new_str_lines), 406 | ) 407 | snippet = self.read_file(path, start_line=start_line, end_line=end_line) 408 | 409 | # Save history - we already have the lines in memory 410 | file_text = "".join(history_lines) 411 | # self._history_manager.add_history(path, file_text) 412 | 413 | # Read new content for result 414 | new_file_text = self.read_file(path) 415 | 416 | success_message = f"The file {path} has been edited. " 417 | success_message += self._make_output( 418 | snippet, 419 | "a snippet of the edited file", 420 | max(1, insert_line - SNIPPET_CONTEXT_WINDOW + 1), 421 | ) 422 | 423 | if enable_linting: 424 | # Run linting on the changes 425 | lint_results = self._run_linting(file_text, new_file_text, path) 426 | success_message += "\n" + lint_results + "\n" 427 | 428 | success_message += "Review the changes and make sure they are as expected (correct indentation, no duplicate lines, etc). Edit the file again if necessary." 429 | return CLIResult( 430 | output=success_message, 431 | prev_exist=True, 432 | path=str(path), 433 | old_content=file_text, 434 | new_content=new_file_text, 435 | ) 436 | 437 | def validate_path(self, command: Command, path: Path) -> None: 438 | """ 439 | Check that the path/command combination is valid. 440 | """ 441 | # Check if its an absolute path 442 | # print(path) 443 | if not path.is_absolute(): 444 | suggested_path = Path.cwd() / path 445 | return CLIResult( 446 | error=f"Error: The path should be an absolute path, starting with `/`. Maybe you meant {suggested_path}?", 447 | path=str(path), 448 | prev_exist=True, 449 | ) 450 | # Check if path and command are compatible 451 | if command == "create" and path.exists(): 452 | return CLIResult( 453 | error=f"Error: File already exists at: {path}. Cannot overwrite files using command `create`.", 454 | path=str(path), 455 | prev_exist=True, 456 | ) 457 | if command != "create" and not path.exists(): 458 | return CLIResult( 459 | error=f"Error: The path {path} does not exist. Please provide a valid path.", 460 | path=str(path), 461 | prev_exist=True, 462 | ) 463 | if command != "view" and path.is_dir(): 464 | return CLIResult( 465 | error=f"Error: The path {path} is a directory and only the `view` command can be used on directories.", 466 | path=str(path), 467 | prev_exist=True, 468 | ) 469 | 470 | # def undo_edit(self, path: Path) -> CLIResult: 471 | # """ 472 | # Implement the undo_edit command. 473 | # """ 474 | # current_text = self.read_file(path).expandtabs() 475 | # old_text = self._history_manager.get_last_history(path) 476 | # if old_text is None: 477 | # return CLIResult( 478 | # error=f"Error: No edit history found for {path}.", 479 | # path=str(path), 480 | # prev_exist=True, 481 | # ) 482 | 483 | # self.write_file(path, old_text) 484 | 485 | # return CLIResult( 486 | # output=f"Last edit to {path} undone successfully. {self._make_output(old_text, str(path))}", 487 | # path=str(path), 488 | # prev_exist=True, 489 | # old_content=current_text, 490 | # new_content=old_text, 491 | # ) 492 | 493 | def validate_file(self, path: Path) -> None: 494 | """ 495 | Validate a file for reading or editing operations. 496 | 497 | Args: 498 | path: Path to the file to validate 499 | 500 | Raises: 501 | FileValidationError: If the file fails validation 502 | """ 503 | if not path.is_file(): 504 | return # Skip validation for directories 505 | 506 | # Check file size 507 | file_size = os.path.getsize(path) 508 | max_size = self._max_file_size 509 | if file_size > max_size: 510 | return CLIResult( 511 | error=f"Error: File is too large ({file_size / 1024 / 1024:.1f}MB). Maximum allowed size is {int(max_size / 1024 / 1024)}MB.", 512 | path=str(path), 513 | prev_exist=True, 514 | ) 515 | 516 | # Check if file is binary 517 | mime_type, _ = mimetypes.guess_type(str(path)) 518 | if mime_type is None: 519 | # If mime_type is None, try to detect if it's binary by reading first chunk 520 | try: 521 | chunk = open(path, "rb").read(1024) 522 | if b"\0" in chunk: # Common way to detect binary files 523 | return CLIResult( 524 | error="Error: File appears to be binary. Only text files can be edited.", 525 | path=str(path), 526 | prev_exist=True, 527 | ) 528 | except Exception as e: 529 | return CLIResult( 530 | error=f"Error: Error checking file type: {str(e)}", 531 | path=str(path), 532 | prev_exist=True, 533 | ) 534 | # Known non-text mime type 535 | return CLIResult( 536 | error=f"Error: File type {mime_type} is not supported. Only text files can be edited.", 537 | path=str(path), 538 | prev_exist=True, 539 | ) 540 | 541 | def read_file( 542 | self, 543 | path: Path, 544 | start_line: int | None = None, 545 | end_line: int | None = None, 546 | ) -> str: 547 | """ 548 | Read the content of a file from a given path; raise a ToolError if an error occurs. 549 | 550 | Args: 551 | path: Path to the file to read 552 | start_line: Optional start line number (1-based). If provided with end_line, only reads that range. 553 | end_line: Optional end line number (1-based). Must be provided with start_line. 554 | """ 555 | self.validate_file(path) 556 | try: 557 | if start_line is not None and end_line is not None: 558 | # Read only the specified line range 559 | lines = [] 560 | with open(path, "r") as f: 561 | for i, line in enumerate(f, 1): 562 | if i > end_line: 563 | break 564 | if i >= start_line: 565 | lines.append(line) 566 | return "".join(lines) 567 | elif start_line is not None or end_line is not None: 568 | raise ValueError("Both start_line and end_line must be provided together") 569 | else: 570 | # Use line-by-line reading to avoid loading entire file into memory 571 | with open(path, "r") as f: 572 | return "".join(f) 573 | except Exception as e: 574 | return f"Error: Ran into {e} while trying to read {path}" 575 | 576 | def _make_output( 577 | self, 578 | snippet_content: str, 579 | snippet_description: str, 580 | start_line: int = 1, 581 | expand_tabs: bool = True, 582 | ) -> str: 583 | """ 584 | Generate output for the CLI based on the content of a code snippet. 585 | """ 586 | snippet_content = maybe_truncate(snippet_content, truncate_notice=FILE_CONTENT_TRUNCATED_NOTICE) 587 | if expand_tabs: 588 | snippet_content = snippet_content.expandtabs() 589 | 590 | snippet_content = "\n".join( 591 | [f"{i + start_line:6}\t{line}" for i, line in enumerate(snippet_content.split("\n"))] 592 | ) 593 | return f"Here's the result of running `cat -n` on {snippet_description}:\n" + snippet_content + "\n" 594 | 595 | def _run_linting(self, old_content: str, new_content: str, path: Path) -> str: 596 | """ 597 | Run linting on file changes and return formatted results. 598 | """ 599 | # Create a temporary directory 600 | with tempfile.TemporaryDirectory() as temp_dir: 601 | # Create paths with exact filenames in temp directory 602 | temp_old = Path(temp_dir) / f"old.{path.name}" 603 | temp_new = Path(temp_dir) / f"new.{path.name}" 604 | 605 | # Write content to temporary files 606 | temp_old.write_text(old_content) 607 | temp_new.write_text(new_content) 608 | 609 | # Run linting on the changes 610 | results = self._linter.lint_file_diff(str(temp_old), str(temp_new)) 611 | 612 | if not results: 613 | return "No linting issues found in the changes." 614 | 615 | # Format results 616 | output = ["Linting issues found in the changes:"] 617 | for result in results: 618 | output.append(f"- Line {result.line}, Column {result.column}: {result.message}") 619 | return "\n".join(output) + "\n" 620 | -------------------------------------------------------------------------------- /src/agent/tool_set/sepl_tools.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | import subprocess 4 | import time 5 | import uuid 6 | from typing import Annotated, List, Optional 7 | 8 | from git import Repo 9 | from langchain_core.tools import tool 10 | from langchain_core.runnables import RunnableConfig 11 | 12 | from agent import runtime_config 13 | from agent.constant import PATCH_RESULT_DIR, RUNTIME_DIR 14 | 15 | MAX_LIST_FILES = 50 # the maximum number of files to return 16 | MAX_RESPONSE_LEN_CHAR: int = 32000 17 | 18 | 19 | @tool 20 | def view_directory(dir_path: str = "./", depth: Optional[int] = None) -> List[str]: 21 | """View the file structure of the repository, including directories (marked with /). 22 | Automatically reduces depth if entries exceed 50. 23 | 24 | Args: 25 | dir_path (str): Starting directory. Defaults to './'. 26 | depth (Optional[int]): Maximum depth. None for unlimited. Defaults to None. 27 | 28 | Returns: 29 | List[str]: Sorted list of directories (with /) and files. 30 | """ 31 | rc = runtime_config.RuntimeConfig() 32 | assert rc.initialized 33 | 34 | # Normalize dir_path to ensure proper filtering 35 | # 36 | if dir_path.startswith("./"): 37 | processed_dir = dir_path[2:] 38 | else: 39 | processed_dir = dir_path 40 | 41 | if processed_dir: 42 | processed_dir = processed_dir.rstrip("/") + "/" 43 | 44 | # Fetch all files in the repository 45 | file_list = [] 46 | if rc.runtime_type == runtime_config.RuntimeType.LOCAL: 47 | repo = Repo(rc.proj_path) 48 | file_list = [entry.path for entry in repo.commit().tree.traverse()] 49 | else: 50 | raise ValueError("Unsupported runtime type") 51 | 52 | # Collect files and directories with their depths 53 | all_files = [] # Format: (full_path, depth) 54 | all_dirs = set() # Format: (full_dir_path, depth) 55 | 56 | for path in file_list: 57 | # Filter files outside the target directory 58 | if not path.startswith(processed_dir): 59 | continue 60 | 61 | # Calculate file depth 62 | rel_path = path[len(processed_dir) :] if processed_dir else path 63 | file_depth = rel_path.count("/") 64 | all_files.append((path, file_depth)) 65 | 66 | # Generate parent directories from the file path 67 | dir_components = rel_path.split("/")[:-1] # Exclude filename 68 | current_dir = [] 69 | for component in dir_components: 70 | current_dir.append(component) 71 | dir_rel_path = "/".join(current_dir) 72 | dir_depth = dir_rel_path.count("/") # Depth is based on slashes 73 | full_dir_path = f"{processed_dir}{dir_rel_path}/" 74 | all_dirs.add((full_dir_path, dir_depth)) 75 | 76 | # Function to filter entries by depth 77 | def filter_entries(max_depth: Optional[int]) -> List[str]: 78 | # Filter files 79 | filtered_files = [ 80 | path for path, d in all_files if (max_depth is None) or (d <= max_depth) 81 | ] 82 | # Filter directories 83 | filtered_dirs = [ 84 | dir_path 85 | for dir_path, d in all_dirs 86 | if (max_depth is None) or (d <= max_depth) 87 | ] 88 | # Combine and deduplicate 89 | entries = list(set(filtered_dirs + filtered_files)) 90 | return sorted(entries) # Alphabetical order 91 | 92 | # Check initial entry count 93 | initial_entries = filter_entries(depth) 94 | if len(initial_entries) <= 50: 95 | return initial_entries 96 | 97 | # Automatically reduce depth 98 | start_depth = ( 99 | depth 100 | if depth is not None 101 | else max( 102 | max((d for _, d in all_files), default=0), 103 | max((d for _, d in all_dirs), default=0), 104 | ) 105 | ) 106 | 107 | for d in range(start_depth, -1, -1): 108 | adjusted_entries = filter_entries(d) 109 | if len(adjusted_entries) <= 50: 110 | print(f"Note: Reduced depth to {d} with {len(adjusted_entries)} entries") 111 | return [ 112 | f"Note: Reduced depth to {d} with {len(adjusted_entries)} entries" 113 | ] + adjusted_entries 114 | 115 | # Fallback (depth 0) 116 | final_entries = filter_entries(0) 117 | print(f"Note: Limited to depth 0 with {len(final_entries)} entries") 118 | return [ 119 | f"Note: Limited to depth 0 with {len(final_entries)} entries" 120 | ] + final_entries 121 | 122 | 123 | @tool 124 | def view_file_content( 125 | file_name: Annotated[ 126 | str, 127 | "File name relative to git root, candidates can be retrieved by `view_directory`", 128 | ], 129 | view_range: Annotated[ 130 | Optional[List[int]], 131 | "Optional parameter [start_line, end_line] to specify the range of lines to view", 132 | ] = None, 133 | ) -> str: 134 | """ 135 | Read the content of the specified file. 136 | Parameters: 137 | file_name (str): File name relative to the git root directory. 138 | view_range (Optional[List[int]]): Optional list containing [start_line, end_line] to limit the lines displayed. 139 | Usage: 140 | - LLM should initially attempt to read the entire file content. 141 | - If the file is too large, LLM can use the `view_file_structure` tool to identify relevant code ranges, 142 | and then call this tool again specifying the `view_range` to read only the necessary lines. 143 | Returns: 144 | str: Content of the file or the specified line range. 145 | """ 146 | rc = runtime_config.RuntimeConfig() 147 | assert rc.initialized 148 | print( 149 | 'view_file_content: path:%s file_name="%s" view_range=%s' 150 | % (rc.proj_path, file_name, view_range) 151 | ) 152 | if rc.runtime_type == runtime_config.RuntimeType.LOCAL: 153 | full_file_path = os.path.join(rc.proj_path, file_name) 154 | if not os.path.isfile(full_file_path): 155 | raise ValueError(f"file_name: '{file_name}' doesn't exist!") 156 | with open(full_file_path, encoding="utf-8") as f: 157 | lines = f.readlines() 158 | if view_range: 159 | start_line, end_line = view_range 160 | lines = lines[start_line - 1 : end_line] 161 | lines = [f"{line}" for i, line in enumerate(lines)] 162 | else: 163 | lines = [f"{i + 1}\t{line}" for i, line in enumerate(lines)] 164 | file_content = "".join(lines) 165 | else: 166 | raise NotImplementedError 167 | 168 | # FILE_CONTENT_TRUNCATED_NOTICE = 'Due to the max output limit, only part of this file has been shown to you. You should retry this tool after you have searched inside the file with the `search_file_by_keywords` tool or `view_file_structure` tool in order to find the line numbers of what you are looking for, and then use this tool with view_range.' 169 | FILE_CONTENT_TRUNCATED_NOTICE = "Due to the max output limit, only part of this file has been shown to you. You should retry this tool after you have searched inside the file with the `search_file_by_keywords` tool or view the file structure below in order to find the line numbers of what you are looking for, and then use this tool with view_range." 170 | if len(file_content) > MAX_RESPONSE_LEN_CHAR: 171 | truncated = True 172 | else: 173 | truncated = False 174 | snippet_content = ( 175 | file_content 176 | if not truncated 177 | else file_content[:MAX_RESPONSE_LEN_CHAR] + FILE_CONTENT_TRUNCATED_NOTICE 178 | ) 179 | snippet_content = snippet_content.expandtabs() 180 | 181 | if view_range: 182 | start_line, end_line = view_range 183 | snippet_content = "\n".join( 184 | [ 185 | f"{i + start_line:6}\t{line}" 186 | for i, line in enumerate(snippet_content.split("\n")) 187 | ] 188 | ) 189 | 190 | return snippet_content 191 | 192 | 193 | def extract_git_diff_local(): 194 | """Executes and returns the `git diff` command in a local runtime environment.""" 195 | rc = runtime_config.RuntimeConfig() 196 | print("extracting git diff local") 197 | rc.pretty_print_runtime() 198 | assert rc.initialized 199 | assert rc.runtime_type == runtime_config.RuntimeType.LOCAL 200 | 201 | import subprocess 202 | 203 | process = subprocess.Popen( 204 | "/bin/bash", 205 | cwd=rc.proj_path, 206 | stdin=subprocess.PIPE, 207 | stdout=subprocess.PIPE, 208 | text=True, 209 | shell=True, 210 | ) 211 | out, err = process.communicate( 212 | "git -c core.fileMode=false diff --exit-code --no-color" 213 | ) 214 | return out 215 | 216 | 217 | # %% 218 | def save_git_diff(): 219 | print("Saving git diff") 220 | rc = runtime_config.RuntimeConfig() 221 | 222 | git_diff_output_before = extract_git_diff_local() 223 | instance_id = rc.proj_name.replace("/", "+") 224 | 225 | patch_path = ( 226 | os.path.join(PATCH_RESULT_DIR, instance_id + "@" + str(int(time.time()))) 227 | + ".patch" 228 | ) 229 | 230 | with open(patch_path, "w", encoding="utf-8") as save_file: 231 | save_file.write(git_diff_output_before) 232 | # print(f"Saved patch content to {patch_path}") 233 | return git_diff_output_before 234 | 235 | 236 | # %% 237 | @tool 238 | def run_shell_cmd( 239 | commands: Annotated[ 240 | List[str], "A list of shell commands to be run in sequential order" 241 | ], 242 | config: RunnableConfig, 243 | ) -> str: 244 | """Run a list of shell commands in sequential order and return the stdout results, your working directory is the root of the project""" 245 | 246 | proj_path = config.get("configurable", {}).get("proj_path") 247 | if proj_path is None: 248 | rc = runtime_config.RuntimeConfig() 249 | assert rc.initialized 250 | proj_path = rc.proj_path 251 | print(f"use global runtime config project path: {proj_path}") 252 | else: 253 | print(f"use configrable config project path: {proj_path}") 254 | 255 | if rc.runtime_type == runtime_config.RuntimeType.LOCAL: 256 | import subprocess 257 | 258 | process = subprocess.Popen( 259 | "/bin/bash", 260 | cwd=proj_path, 261 | stdin=subprocess.PIPE, 262 | stdout=subprocess.PIPE, 263 | text=True, 264 | shell=True, 265 | ) 266 | out, err = process.communicate("\n".join(commands)) 267 | return out 268 | 269 | else: 270 | raise NotImplementedError 271 | 272 | 273 | if __name__ == "__main__": 274 | runtime_config = runtime_config.RuntimeConfig() 275 | runtime_config.load_from_github_issue_url( 276 | "https://github.com/gitpython-developers/GitPython/issues/1977" 277 | ) 278 | 279 | 280 | # %% 281 | -------------------------------------------------------------------------------- /src/agent/tool_set/utils.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import time 3 | from agent.tool_set.constant import * 4 | 5 | def maybe_truncate( 6 | content: str, 7 | truncate_after: int | None = MAX_RESPONSE_LEN_CHAR, 8 | truncate_notice: str = CONTENT_TRUNCATED_NOTICE, 9 | ) -> str: 10 | """ 11 | Truncate content and append a notice if content exceeds the specified length. 12 | """ 13 | return ( 14 | content 15 | if not truncate_after or len(content) <= truncate_after 16 | else content[:truncate_after] + truncate_notice 17 | ) 18 | 19 | def run_shell_local( 20 | cmd: str, 21 | timeout: float | None = 120.0, # seconds 22 | truncate_after: int | None = MAX_RESPONSE_LEN_CHAR, 23 | truncate_notice: str = CONTENT_TRUNCATED_NOTICE, 24 | ) -> tuple[int, str, str]: 25 | """Run a shell command synchronously with a timeout. 26 | 27 | Args: 28 | cmd: The shell command to run. 29 | timeout: The maximum time to wait for the command to complete. 30 | truncate_after: The maximum number of characters to return for stdout and stderr. 31 | 32 | Returns: 33 | A tuple containing the return code, stdout, and stderr. 34 | """ 35 | 36 | start_time = time.time() 37 | 38 | try: 39 | process = subprocess.Popen( 40 | cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True 41 | ) 42 | 43 | stdout, stderr = process.communicate(timeout=timeout) 44 | 45 | return ( 46 | process.returncode or 0, 47 | maybe_truncate(stdout, truncate_after=truncate_after, truncate_notice=truncate_notice), 48 | maybe_truncate( 49 | stderr, 50 | truncate_after=truncate_after, 51 | truncate_notice=CONTENT_TRUNCATED_NOTICE, # Use generic notice for stderr 52 | ), 53 | ) 54 | except subprocess.TimeoutExpired: 55 | process.kill() 56 | elapsed_time = time.time() - start_time 57 | raise TimeoutError(f"Command '{cmd}' timed out after {elapsed_time:.2f} seconds") 58 | -------------------------------------------------------------------------------- /src/agent/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Defines various util functions for the prototype.""" 3 | 4 | 5 | class UndefinedValueError(ValueError): 6 | """ 7 | A custom exception raised when a variable is not defined. 8 | 9 | Args: 10 | variable_name (str): The name of the undefined variable 11 | message (str, optional): Custom error message 12 | """ 13 | 14 | def __init__(self, variable_name, message=None): 15 | if message is None: 16 | message = f"`{variable_name}` is required and not defined in `.env` environment variables." 17 | 18 | self.variable_name = variable_name 19 | 20 | super().__init__(message) 21 | 22 | 23 | def stage_message_processor(messages): 24 | """Based on XY's `message_processor` with few additions. It is used for newest version of summarizer/summary.""" 25 | message_json = {} 26 | step = 0 27 | 28 | index = 0 29 | found_human_feedback = False 30 | messages = [ 31 | vars(message) if not isinstance(message, dict) else message 32 | for message in messages 33 | ] 34 | for message in messages: 35 | if message["name"] == "human_feedback": 36 | found_human_feedback = True 37 | index = messages.index(message) 38 | 39 | if not found_human_feedback: 40 | index = 0 41 | 42 | messages = messages[index:] 43 | 44 | for message in messages: 45 | if message["type"] != "tool": 46 | # if message['content'] is string: 47 | if isinstance(message["content"], str): 48 | if step == 0: 49 | step += 1 50 | continue # skip human input as its duplicated 51 | name = message["name"] if "name" in message else "" 52 | message_json[f"Step {step} {name}:"] = {"detail": message["content"]} 53 | else: 54 | detail_cnt = 1 55 | details = {} 56 | for content in message["content"]: 57 | details[f"Detail {detail_cnt} {content['type']}:"] = content 58 | detail_cnt += 1 59 | name = message["name"] if "name" in message else "" 60 | message_json[f"Step {step} {name}:"] = {"detail": details} 61 | 62 | step += 1 63 | return message_json 64 | --------------------------------------------------------------------------------