├── prompts ├── __init__.py ├── notetaker.py ├── programmer.py └── orchestrator.py ├── start.sh ├── .DS_Store ├── README.md ├── editor.py ├── .gitignore ├── app.py ├── requirements.txt ├── LICENSE ├── agent.py └── index.html /prompts/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | env WHERE_EXECUTE="asdf" gradio app.py -------------------------------------------------------------------------------- /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lordspline/DevOn/HEAD/.DS_Store -------------------------------------------------------------------------------- /prompts/notetaker.py: -------------------------------------------------------------------------------- 1 | notetaker_notes = """Important Notes: 2 | Don't write anything in the Note Title field. 3 | Whatever notes you are told to make, write them in one go, don't press enter or type multiple times, because everytime you write, it will replace the prevoius content. 4 | You do not need to Save the note. When asked to note something down, just write it on the notepad. That is enough.""" 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: DevOn 3 | app_file: app.py 4 | sdk: gradio 5 | sdk_version: 4.24.0 6 | --- 7 | # DevOn 8 | 9 | [Huggingface Space](https://huggingface.co/spaces/lordspline/devon) 10 | 11 | What if you tried to do what Devin does, but using MultiOn's agents? 12 | 13 | We use 3 separate MultiOn Agents, one each for programming, researching and notetaking. 14 | 15 | Their activities are orchestrated and supervised by an overarching GPT-4V 16 | 17 | Setup: 18 | 19 | - `pip install -r requirements.txt` 20 | - `bash start.sh` 21 | 22 | Barebones Demo: 23 | 24 | https://github.com/lordspline/DevOn/assets/74811063/6de8ba85-3f43-415b-8fd9-eff6b2ed29c5 25 | 26 | \*Note: This is a simplistic demo only meant to showcase the range of MultiOn's capabilities. 27 | -------------------------------------------------------------------------------- /prompts/programmer.py: -------------------------------------------------------------------------------- 1 | # programmer_notes = """Important Notes: 2 | # Do not refresh the page ever to check for anything. Only wait. Do not refresh. 3 | # You are working in a terminal environment. 4 | # You will do everything using the terminal and only the terminal. 5 | # If you need to create a new file, do so using the touch command on the terminal. 6 | # If you need to see files in the current directory, do so using ls. 7 | # If you need to view a files content, do so using the cat command on the terminal. 8 | # To enter code into a file, use a single printf command. After the printf command has been completely typed, press enter. Typing the command and pressing enter must be 2 separate steps. 9 | # Do not open a text editor like vim or nano. 10 | # If you need to install a new package, use pip install on the terminal. 11 | # Do not use the same command repeatedly. 12 | # When you write code into a file, write it once, cat it once, then stop. Do no attempt to write again unless it is wrong. 13 | # Remember that you need to press Enter after typing a command into the terminal. Only press enter after the command has been completely typed. Typing the command pressing enter must be 2 separate steps.""" 14 | 15 | programmer_notes = """Important Notes: 16 | You are a Programmer who works in a Replit Environment exclusively. If you need to install a package, use the Shell and not the Console. 17 | Do not refresh the page ever to check for anything. Only wait. Do not refresh. Do not create new files. Write your code in currently open editor window itself. Do not type double quotation marks. If you are asked to type code containing them, use single quotes instead.""" 18 | -------------------------------------------------------------------------------- /editor.py: -------------------------------------------------------------------------------- 1 | import gradio as gr 2 | import os 3 | import time 4 | import ngrok 5 | from dotenv import load_dotenv 6 | 7 | load_dotenv(".env.local") 8 | 9 | 10 | def new_file(file_name): 11 | # print(file_explorer) 12 | new_file_path = os.path.join(os.getcwd(), "dev", file_name) 13 | open(new_file_path, "w") 14 | 15 | 16 | def set_current_file(file): 17 | if file is None: 18 | return "" 19 | file_content = "" 20 | with open(file) as f: 21 | file_content = f.read() 22 | return file_content 23 | 24 | 25 | def update_file(current_file, editor_content): 26 | with open(current_file, "w") as f: 27 | f.write(editor_content) 28 | return current_file, editor_content, "" 29 | 30 | 31 | with gr.Blocks(css="footer {visibility: hidden}") as demo: 32 | with gr.Row(): 33 | with gr.Column() as c: 34 | file_explorer = gr.FileExplorer(file_count="single", root_dir="dev") 35 | new_file_name = gr.Textbox(label="New File Name") 36 | new_file_button = gr.Button( 37 | "Create New File (After creating a new file, you must refresh the page to see the changes.)" 38 | ) 39 | 40 | with gr.Column(): 41 | code_display = gr.Code( 42 | "Select a file to start", 43 | language="python", 44 | label="Code Display (Displays file's current state, cannot edit this.)", 45 | ) 46 | code_editor = gr.Code( 47 | "", 48 | language="python", 49 | label="Code Editor (Enter the updated code to put in the file here.)", 50 | ) 51 | update_button = gr.Button("Update File") 52 | 53 | file_explorer.change(set_current_file, file_explorer, code_display) 54 | update_button.click( 55 | update_file, 56 | [file_explorer, code_editor], 57 | [file_explorer, code_display, code_editor], 58 | ) 59 | new_file_button.click(new_file, new_file_name) 60 | # with gr.Row(): 61 | # save_button = gr.Button("Save Current File") 62 | 63 | if __name__ == "__main__": 64 | demo.queue() 65 | listener = ngrok.forward(9000, authtoken_from_env=True) 66 | print(f"Ingress established at {listener.url()}") 67 | demo.launch(server_port=9000) 68 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | orb.py 7 | orb_app.py 8 | dev 9 | prompts/orb.py 10 | screenshots 11 | temp 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | lib/ 25 | lib64/ 26 | parts/ 27 | sdist/ 28 | var/ 29 | wheels/ 30 | share/python-wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | MANIFEST 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .nox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | *.cover 56 | *.py,cover 57 | .hypothesis/ 58 | .pytest_cache/ 59 | cover/ 60 | 61 | # Translations 62 | *.mo 63 | *.pot 64 | 65 | # Django stuff: 66 | *.log 67 | local_settings.py 68 | db.sqlite3 69 | db.sqlite3-journal 70 | 71 | # Flask stuff: 72 | instance/ 73 | .webassets-cache 74 | 75 | # Scrapy stuff: 76 | .scrapy 77 | 78 | # Sphinx documentation 79 | docs/_build/ 80 | 81 | # PyBuilder 82 | .pybuilder/ 83 | target/ 84 | 85 | # Jupyter Notebook 86 | .ipynb_checkpoints 87 | 88 | # IPython 89 | profile_default/ 90 | ipython_config.py 91 | 92 | # pyenv 93 | # For a library or package, you might want to ignore these files since the code is 94 | # intended to run in multiple environments; otherwise, check them in: 95 | # .python-version 96 | 97 | # pipenv 98 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 99 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 100 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 101 | # install all needed dependencies. 102 | #Pipfile.lock 103 | 104 | # poetry 105 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 106 | # This is especially recommended for binary packages to ensure reproducibility, and is more 107 | # commonly ignored for libraries. 108 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 109 | #poetry.lock 110 | 111 | # pdm 112 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 113 | #pdm.lock 114 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 115 | # in version control. 116 | # https://pdm.fming.dev/#use-with-ide 117 | .pdm.toml 118 | 119 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 120 | __pypackages__/ 121 | 122 | # Celery stuff 123 | celerybeat-schedule 124 | celerybeat.pid 125 | 126 | # SageMath parsed files 127 | *.sage.py 128 | 129 | # Environments 130 | .env* 131 | .venv 132 | env/ 133 | venv/ 134 | ENV/ 135 | env.bak/ 136 | venv.bak/ 137 | 138 | # Spyder project settings 139 | .spyderproject 140 | .spyproject 141 | 142 | # Rope project settings 143 | .ropeproject 144 | 145 | # mkdocs documentation 146 | /site 147 | 148 | # mypy 149 | .mypy_cache/ 150 | .dmypy.json 151 | dmypy.json 152 | 153 | # Pyre type checker 154 | .pyre/ 155 | 156 | # pytype static type analyzer 157 | .pytype/ 158 | 159 | # Cython debug symbols 160 | cython_debug/ 161 | 162 | # PyCharm 163 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 164 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 165 | # and can be added to the global gitignore or merged into this file. For a more nuclear 166 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 167 | #.idea/ -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | import gradio as gr 2 | import os 3 | import time 4 | from agent import DevOn 5 | 6 | image_temp = "https://miro.medium.com/v2/resize:fit:1200/0*n-2bW82Z6m6U2bij.jpeg" 7 | # devon = DevOn( 8 | # editor_image=image_temp, browser_image=image_temp, scratchpad_image=image_temp 9 | # ) 10 | devon = None 11 | # multion_api_key = "" 12 | # openai_api_key = "" 13 | # replit_email = "" 14 | # replit_password = "" 15 | 16 | 17 | def add_message(history, message): 18 | # for x in message["files"]: 19 | # history.append(((x,), None)) 20 | if message["text"] is not None: 21 | history.append((message["text"], None)) 22 | return history, gr.MultimodalTextbox(value=None, interactive=False) 23 | 24 | 25 | # def multion_api_key_update(x): 26 | # # global multion_api_key 27 | # multion_api_key = x 28 | 29 | 30 | # def openai_api_key_update(x): 31 | # # global openai_api_key 32 | # openai_api_key = x 33 | 34 | 35 | # def replit_email_update(x): 36 | # # global replit_email 37 | # replit_email = x 38 | 39 | 40 | # def replit_password_update(x): 41 | # # global replit_password 42 | # replit_password = x 43 | 44 | 45 | def bot( 46 | history, 47 | multion_api_key_in, 48 | openai_api_key_in, 49 | replit_email_in, 50 | replit_password_in, 51 | local, 52 | ): 53 | if len(multion_api_key_in) == 0: 54 | raise gr.Error("MultiOn API Key is Required.") 55 | if len(openai_api_key_in) == 0: 56 | raise gr.Error("OpenAI API Key is Required.") 57 | start_time = time.time() 58 | devon = DevOn( 59 | editor_image=image_temp, 60 | browser_image=image_temp, 61 | scratchpad_image=image_temp, 62 | multion_api_key=multion_api_key_in, 63 | openai_api_key=openai_api_key_in, 64 | replit_email=replit_email_in, 65 | replit_password=replit_password_in, 66 | local=local, 67 | ) 68 | 69 | for r in devon.run(history[-1][0]): 70 | curr_time = time.time() 71 | print(curr_time - start_time) 72 | # if curr_time - start_time >= 300: 73 | # break 74 | text, editor_image, browser_image, scratchpad_image = r 75 | if type(text) == str: 76 | history.append((None, text)) 77 | if editor_image is None: 78 | editor_image = devon.editor_image 79 | browser_image = devon.browser_image 80 | scratchpad_image = devon.scratchpad_image 81 | yield history, editor_image, browser_image, scratchpad_image 82 | 83 | 84 | with gr.Blocks(css="footer {visibility: hidden}") as demo: 85 | md = gr.Markdown( 86 | """Notes: 87 | - Use "Execute Locally" for better results. 88 | - For local execution, you need to download the [MultiOn Browser Extension](https://chromewebstore.google.com/detail/multion/ddmjhdbknfidiopmbaceghhhbgbpenmm) and have "API Enabled" in the settings. 89 | - The Huggingface Spaces demo will timeout after 5 minutes by default. To test with longer tasks, [clone the repo](https://github.com/lordspline/DevOn) and run DevOn locally.""" 90 | ) 91 | with gr.Row(): 92 | with gr.Column(): 93 | multion_api_key_in = gr.Textbox(label="MultiOn API Key") 94 | openai_api_key_in = gr.Textbox(label="OpenAI API Key") 95 | with gr.Column(): 96 | replit_email_in = gr.Textbox(label="Replit Email") 97 | replit_password_in = gr.Textbox(label="Replit Password") 98 | with gr.Row(): 99 | with gr.Column(): 100 | chatbot = gr.Chatbot( 101 | [], elem_id="chatbot", bubble_full_width=False, height=300 102 | ) 103 | 104 | chat_input = gr.MultimodalTextbox( 105 | value={ 106 | "text": "benchmark the perplexity api's resposne time with the api key abcdef" 107 | }, 108 | interactive=True, 109 | file_types=["text"], 110 | placeholder="Enter message or upload file...", 111 | show_label=False, 112 | ) 113 | 114 | with gr.Row(): 115 | local = gr.Checkbox(True, label="Execute Locally") 116 | terminate = gr.Button("Terminate") 117 | with gr.Column(): 118 | if devon: 119 | editor_view = gr.Image( 120 | devon.editor_image, 121 | label="Editor", 122 | ) 123 | else: 124 | editor_view = gr.Image() 125 | with gr.Row(): 126 | with gr.Column(): 127 | if devon: 128 | browser_view = gr.Image( 129 | devon.browser_image, 130 | label="Browser", 131 | ) 132 | else: 133 | browser_view = gr.Image() 134 | with gr.Column(): 135 | if devon: 136 | scratchpad_view = gr.Image( 137 | devon.scratchpad_image, 138 | label="Scratchpad", 139 | ) 140 | else: 141 | scratchpad_view = gr.Image() 142 | 143 | chat_msg = chat_input.submit( 144 | add_message, [chatbot, chat_input], [chatbot, chat_input] 145 | ) 146 | bot_msg = chat_msg.then( 147 | bot, 148 | [ 149 | chatbot, 150 | multion_api_key_in, 151 | openai_api_key_in, 152 | replit_email_in, 153 | replit_password_in, 154 | local, 155 | ], 156 | [chatbot, editor_view, browser_view, scratchpad_view], 157 | api_name="bot_response", 158 | ) 159 | bot_msg.then(lambda: gr.MultimodalTextbox(interactive=True), None, [chat_input]) 160 | 161 | # multion_api_key_in.change(multion_api_key_update, multion_api_key_in) 162 | # openai_api_key_in.change(openai_api_key_update, openai_api_key_in) 163 | # replit_email_in.change(replit_email_update, replit_email_in) 164 | # replit_password_in.change(replit_password_update, replit_password_in) 165 | 166 | terminate.click(fn=None, inputs=None, outputs=None, cancels=[bot_msg]) 167 | 168 | # chatbot.like(print_like_dislike, None, None) 169 | 170 | if __name__ == "__main__": 171 | demo.queue() 172 | demo.launch(debug=True) 173 | -------------------------------------------------------------------------------- /prompts/orchestrator.py: -------------------------------------------------------------------------------- 1 | orchestrator_prompt = """**General** 2 | 3 | - You are DevOn, an expert Software Developer. 4 | - You will be asked to develop a new software project from scratch. You will primarily work in Python. You will deal with large software projects spanning multiple files and user requirements. 5 | - Your lifecycle will essentially circle around the Task, the State, your Plan, your Actions, and your Interns. Each of these are described in detail below. 6 | - To start with, your Plan will be empty. You will receive a State (in the form of 3 images, one from each of your Interns) and a Task. You will construct a Plan outlining the steps you will need to take to complete the Task, then ask your Interns to do things in order to incrementally fulfil the steps and complete the Task. 7 | - With each step, you will also provide an Explanation, explaining to the user what you are currently doing, so they may be able to keep track and monitor your progress. For example: 8 | - Explanation: I am currently updating the plan based on the current state and the Task. 9 | - Explanation: I am currently creating a file called utils.py which will contain utility functions. 10 | 11 | **State** 12 | 13 | **Interns** 14 | 15 | - You have 3 interns who will help you with different tasks - a Programmer, a Researcher, and a Planner. Here’s some info about them: 16 | - Programmer: the Programmer is great at writing code given very specific instructions but isn’t a good long term planner. The Programmer works on Replit. You can ask the Programmer to write some code in certain files, make new files, etc. You can even give loose instructions like “Make a new file and write basic skeleton for an Agent class in it.” Keep in mind that the Programmer works exclusively in an online Replit IDE environment. Make sure your Plan and your Actions take this into consideration. 17 | - Researcher: the Researcher is very handy with a browser and great at finding out technical details, documentation, examples, miscellaneous information, etc. You can ask it to do things like “Find out how to make an LLM call using the Perplexity API”. 18 | - Notetaker: the Notetaker has a notepad and can note down anything you want. You will be able to see the notepad at all times. Anytime you want anything written down just to keep track of it, ask the Planner to do so. 19 | 20 | **Plan** 21 | 22 | - You have a persistent object to keep track of things: a Plan. 23 | - If the plan is empty, you will create a plan using the current state of things and the given task. You will do so using the update_plan action described below. 24 | 25 | **Actions** 26 | 27 | - There are 6 actions that you can take at the current time step. You must always take a valid action. You will complete the task by taking actions. You are free to take as many actions as needed (even hundreds), don’t try to rush by compressing multiple actions into one. These are the available actions: 28 | - update_plan : Update Plan’s value to . This will replace the old value, not append to it. If there’s something from the old plan you wish to include in the updated one, make sure to include it in the you provide as an argument. Some examples how you can use this: 29 | - update_plan In order to carry out the task of creating a Flask web server, I will need to take the following steps: 30 | 1) … 31 | 2) … 32 | 3) … 33 | - programmer : Ask the Programmer to carry out a . Some examples of how you can use this: 34 | - programmer Create a new Python file for utils called utils.py and write a generate_random_number() function in it that takes no parameters and returns a random number. 35 | - programmer Go to the model.py file and import generate_random_number() from utils. 36 | - researcher : Ask the Researcher to carry out a . The Researcher will reply to you with the information you asked for. Some examples of how you can use this: 37 | - researcher Find out how the OpenAI API is used. 38 | - researcher What is a SERP API I could use? 39 | - notetaker : Ask the Notetaker to carry out a . Some examples of how you can use this: 40 | - notetaker Note down the following information: MULTION_API_KEY=… 41 | - notetaker Note down the following information: An example Chat Completions API call looks like the following: 42 | from openai import OpenAI 43 | client = OpenAI() 44 | 45 | response = client.chat.completions.create( 46 | model="gpt-3.5-turbo", 47 | messages=[ 48 | {"role": "system", "content": "You are a helpful assistant."}, 49 | {"role": "user", "content": "Who won the world series in 2020?"}, 50 | {"role": "assistant", "content": "The Los Angeles Dodgers won the World Series in 2020."}, 51 | {"role": "user", "content": "Where was it played?"} 52 | ] 53 | ) 54 | - clarify : Clarify something about the Task. Sometimes, there may be missing information, such as logins, api keys, or some requirements of the Task may be unclear. Use this Action to clarify things from the user by asking . Use this sparingly. Try and make decisions yourself. Some examples of how you can use this: 55 | - clarify The Task mentions that I need to benchmark the Perplexity API. Could you provide your API Key? 56 | - submit: The Task is completed and you are ready to submit the output (whatever the programmer has so far). This is end the execution. Only do this when you are completely sure. 57 | 58 | **Important Notes** 59 | 60 | - Respond only by taking an Action (and providing the accompanying Explanation). Any response from you must be one of the above Actions. No other text in the response, just the Action and the Explanation. You will structure your output as such: 61 | ”Action: \nExplanation: ” 62 | - You do not need to ask the Programmer to log in. 63 | - You can see all the Interns screens. If it seems like an Intern has made a mistake or encountered an error, you can tell them about it using the relevant action and ask them to correct it. This is especially important with the Programmer. 64 | - When you ask the programmer to write some code, ask it like this: "programmer memorize the following code and write it in the editor: " 65 | - When you ask the programmer to write some code, make sure the code does not include any double quotation marks, only single quotation marks. E.g. "hello world" should instead be 'hello world'. 66 | - Do not ask the programmer to create new files. 67 | - When writing code, it is preferable to keep it small and simple. Don't write too much fluff. 68 | - Remember to only use single quotation marks. 69 | """ 70 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aioconsole==0.7.0 2 | aiodns==3.1.1 3 | aiofiles==23.2.1 4 | aiohttp==3.9.3 5 | aiohttp-retry==2.8.3 6 | aioprocessing==2.0.1 7 | aiosignal==1.3.1 8 | altair==5.3.0 9 | annotated-types==0.6.0 10 | anthropic==0.23.1 11 | anyio==3.7.1 12 | appnope==0.1.3 13 | asgiref==3.7.2 14 | asttokens==2.4.1 15 | astunparse==1.6.3 16 | attrs==23.2.0 17 | backcall==0.2.0 18 | backoff==2.2.1 19 | bcrypt==4.1.2 20 | beautifulsoup4==4.12.2 21 | boto3==1.34.69 22 | botocore==1.34.69 23 | brave-search==0.1.8 24 | Brotli==1.1.0 25 | build==1.0.3 26 | CacheControl==0.13.1 27 | cachetools==5.3.2 28 | certifi==2023.11.17 29 | cffi==1.16.0 30 | charset-normalizer==3.3.2 31 | chroma-hnswlib==0.7.3 32 | chromadb==0.4.22 33 | clean-text==0.6.0 34 | click==8.1.7 35 | cloudpickle==3.0.0 36 | cognitojwt==1.4.1 37 | colorama==0.4.6 38 | coloredlogs==15.0.1 39 | comm==0.2.1 40 | contourpy==1.2.0 41 | cryptography==41.0.7 42 | cssselect2==0.7.0 43 | cycler==0.12.1 44 | dataclasses-json==0.6.3 45 | datasets==2.18.0 46 | debugpy==1.8.0 47 | decorator==5.1.1 48 | Deprecated==1.2.14 49 | dill==0.3.8 50 | diskcache==5.6.3 51 | distro==1.9.0 52 | dnspython==2.6.1 53 | docopt==0.6.2 54 | docstring-parser==0.15 55 | ecdsa==0.18.0 56 | email_validator==2.1.1 57 | emoji==1.7.0 58 | exa-py==1.0.8 59 | executing==2.0.1 60 | faiss-cpu==1.7.4 61 | fastapi==0.109.0 62 | ffmpy==0.3.2 63 | filelock==3.13.1 64 | flatbuffers==23.5.26 65 | fonttools==4.49.0 66 | frozenlist==1.4.1 67 | fsspec==2023.12.2 68 | ftfy==6.1.3 69 | google-api-core==2.15.0 70 | google-api-python-client==2.113.0 71 | google-auth==2.26.1 72 | google-auth-httplib2==0.2.0 73 | google-cloud-core==2.4.1 74 | google-cloud-firestore==2.14.0 75 | google-cloud-storage==2.14.0 76 | google-crc32c==1.5.0 77 | google-resumable-media==2.7.0 78 | googleapis-common-protos==1.62.0 79 | googlesearch-python==1.2.3 80 | gptcache==0.1.43 81 | gradio==4.24.0 82 | gradio_client==0.14.0 83 | greenlet==3.0.1 84 | groq==0.4.1 85 | grpcio==1.60.0 86 | grpcio-status==1.60.0 87 | h11==0.14.0 88 | html-sanitizer==2.3.1 89 | html2text==2024.2.26 90 | html5lib==1.1 91 | httpcore==1.0.2 92 | httplib2==0.22.0 93 | httptools==0.6.1 94 | httpx==0.25.2 95 | huggingface-hub==0.20.1 96 | humanfriendly==10.0 97 | idna==3.6 98 | importlib-metadata==6.11.0 99 | importlib-resources==6.1.1 100 | iniconfig==2.0.0 101 | inquirerpy==0.3.4 102 | inscriptis==2.5.0 103 | instructor==0.5.2 104 | interegular==0.3.3 105 | ipykernel==6.28.0 106 | ipython==7.34.0 107 | itsdangerous==2.1.2 108 | jedi==0.19.1 109 | Jinja2==3.1.2 110 | jmespath==1.0.1 111 | joblib==1.3.2 112 | jsonlines==4.0.0 113 | jsonpatch==1.33 114 | jsonpointer==2.4 115 | jsonschema==4.21.1 116 | jsonschema-specifications==2023.12.1 117 | jupyter_client==8.6.0 118 | jupyter_core==5.7.1 119 | kiwisolver==1.4.5 120 | kubernetes==28.1.0 121 | langchain==0.0.353 122 | langchain-community==0.0.7 123 | langchain-core==0.1.4 124 | langsmith==0.0.75 125 | lark==1.1.9 126 | llvmlite==0.42.0 127 | lxml==4.9.3 128 | markdown-it-py==3.0.0 129 | markdown2==2.4.13 130 | MarkupSafe==2.1.3 131 | marshmallow==3.20.1 132 | matplotlib==3.8.3 133 | matplotlib-inline==0.1.6 134 | mdurl==0.1.2 135 | mmh3==4.0.1 136 | monotonic==1.6 137 | mpmath==1.3.0 138 | msal==1.27.0 139 | msgpack==1.0.7 140 | multidict==6.0.4 141 | multion==1.0.1 142 | multiprocess==0.70.16 143 | mypy-extensions==1.0.0 144 | nest-asyncio==1.5.8 145 | networkx==3.2.1 146 | ngrok==1.2.0 147 | nltk==3.8.1 148 | numba==0.59.0 149 | numpy==1.26.2 150 | oauthlib==3.2.2 151 | onnxruntime==1.16.3 152 | openai==1.23.6 153 | opentelemetry-api==1.22.0 154 | opentelemetry-exporter-otlp-proto-common==1.22.0 155 | opentelemetry-exporter-otlp-proto-grpc==1.22.0 156 | opentelemetry-instrumentation==0.43b0 157 | opentelemetry-instrumentation-asgi==0.43b0 158 | opentelemetry-instrumentation-fastapi==0.43b0 159 | opentelemetry-proto==1.22.0 160 | opentelemetry-sdk==1.22.0 161 | opentelemetry-semantic-conventions==0.43b0 162 | opentelemetry-util-http==0.43b0 163 | ordered-set==4.1.0 164 | orjson==3.9.15 165 | overrides==7.4.0 166 | packaging==23.2 167 | pandas==2.2.1 168 | paramiko==3.4.0 169 | parso==0.8.3 170 | pdfkit==1.0.0 171 | pexpect==4.9.0 172 | pfzy==0.3.4 173 | pickleshare==0.7.5 174 | pillow==10.2.0 175 | pinecone-client==3.0.0 176 | platformdirs==4.1.0 177 | playwright==1.40.0 178 | pluggy==1.4.0 179 | posthog==3.1.0 180 | prettytable==3.10.0 181 | prompt-toolkit==3.0.43 182 | proto-plus==1.23.0 183 | protobuf==4.25.1 184 | psutil==5.9.7 185 | ptyprocess==0.7.0 186 | pulsar-client==3.4.0 187 | pure-eval==0.2.2 188 | py-cpuinfo==9.0.0 189 | pyarrow==15.0.2 190 | pyarrow-hotfix==0.6 191 | pyasn1==0.5.1 192 | pyasn1-modules==0.3.0 193 | pycares==4.4.0 194 | pycparser==2.21 195 | pydantic==2.5.3 196 | pydantic-extra-types==2.6.0 197 | pydantic-settings==2.2.1 198 | pydantic_core==2.14.6 199 | pydot==2.0.0 200 | pydub==0.25.1 201 | pydyf==0.9.0 202 | pyee==11.0.1 203 | pyformlang==1.0.7 204 | Pygments==2.17.2 205 | PyJWT==2.8.0 206 | pymongo==4.6.2 207 | PyNaCl==1.5.0 208 | pyparsing==3.1.1 209 | pypdf==4.0.0 210 | pyphen==0.14.0 211 | PyPika==0.48.9 212 | pyproject_hooks==1.0.0 213 | pytesseract==0.3.10 214 | pytest==8.1.1 215 | pytest-asyncio==0.23.6 216 | python-dateutil==2.8.2 217 | python-dotenv==1.0.0 218 | python-jose==3.3.0 219 | python-multipart==0.0.9 220 | pytz==2024.1 221 | PyYAML==6.0.1 222 | pyzmq==25.1.2 223 | readabilipy==0.2.0 224 | referencing==0.33.0 225 | regex==2023.12.25 226 | requests==2.31.0 227 | requests-oauthlib==1.3.1 228 | rich==13.7.0 229 | rpds-py==0.18.0 230 | rsa==4.9 231 | ruff==0.3.4 232 | s3transfer==0.10.1 233 | safetensors==0.4.1 234 | scikit-learn==1.3.2 235 | scipy==1.11.4 236 | semantic-version==2.10.0 237 | sentence-transformers==2.2.2 238 | sentencepiece==0.1.99 239 | sentry-sdk==1.40.6 240 | shellingham==1.5.4 241 | six==1.16.0 242 | sniffio==1.3.0 243 | soupsieve==2.5 244 | SQLAlchemy==2.0.24 245 | stack-data==0.6.3 246 | starlette==0.35.1 247 | sympy==1.12 248 | tenacity==8.2.3 249 | termcolor==2.4.0 250 | text-generation==0.6.1 251 | threadpoolctl==3.2.0 252 | tiktoken==0.5.2 253 | tinycss2==1.2.1 254 | tokenizers==0.15.0 255 | toml==0.10.2 256 | tomli==2.0.1 257 | tomlkit==0.12.0 258 | toolz==0.12.1 259 | torch==2.1.2 260 | torchvision==0.16.2 261 | tornado==6.4 262 | tqdm==4.66.1 263 | tqdm-loggable==0.2 264 | traitlets==5.14.1 265 | transformers==4.36.2 266 | typer==0.9.0 267 | typing-inspect==0.9.0 268 | typing_extensions==4.9.0 269 | tzdata==2024.1 270 | ujson==5.9.0 271 | uritemplate==4.1.1 272 | urllib3==1.26.18 273 | uvicorn==0.25.0 274 | uvloop==0.19.0 275 | watchdog==4.0.0 276 | watchfiles==0.21.0 277 | wcwidth==0.2.12 278 | weasyprint==61.1 279 | webencodings==0.5.1 280 | websocket-client==1.7.0 281 | websockets==11.0.3 282 | wrapt==1.16.0 283 | xxhash==3.4.1 284 | yarl==1.9.4 285 | zipp==3.17.0 286 | zopfli==0.2.3 287 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /agent.py: -------------------------------------------------------------------------------- 1 | from openai import OpenAI 2 | from prompts.orchestrator import orchestrator_prompt 3 | from prompts.programmer import programmer_notes 4 | from prompts.notetaker import notetaker_notes 5 | from dotenv import load_dotenv 6 | import time 7 | import multion 8 | from multion.client import MultiOn 9 | import os 10 | 11 | load_dotenv(".env.local") 12 | 13 | # replit_email = os.getenv("REPLIT_EMAIL") 14 | # replit_password = os.getenv("REPLIT_PASSWORD") 15 | 16 | # multion_api_key = os.getenv("MULTION_API_KEY") 17 | # multion.login(use_api=True, multion_api_key=multion_api_key) 18 | 19 | # runpod_url = os.getenv("RUNPOD_URL") 20 | 21 | image_temp = "https://miro.medium.com/v2/resize:fit:1200/0*n-2bW82Z6m6U2bij.jpeg" 22 | 23 | 24 | class DevOn: 25 | def __init__( 26 | self, 27 | editor_image, 28 | browser_image, 29 | scratchpad_image, 30 | multion_api_key, 31 | openai_api_key, 32 | replit_email, 33 | replit_password, 34 | local, 35 | ): 36 | print(multion_api_key, openai_api_key) 37 | self.editor_image = editor_image 38 | self.browser_image = browser_image 39 | self.scratchpad_image = scratchpad_image 40 | self.local = local 41 | # if os.getenv("WHERE_EXECUTE"): 42 | # self.local = local 43 | # else: 44 | # self.local = False 45 | 46 | self.multion = MultiOn(api_key=multion_api_key) 47 | 48 | self.replit_email = replit_email 49 | self.replit_password = replit_password 50 | 51 | self.programmer = self.multion.sessions.create( 52 | url="https://replit.com/login", local=self.local, include_screenshot=True 53 | ) 54 | self.programmer_logged_in = False 55 | # self.editor_image = self.programmer.screenshot 56 | self.editor_image = self.multion.sessions.screenshot( 57 | session_id=self.programmer.session_id 58 | ).screenshot 59 | print(self.editor_image) 60 | time.sleep(1) 61 | # print(self.programmer) 62 | 63 | self.researcher = self.multion.sessions.create( 64 | url="https://www.google.com", local=self.local, include_screenshot=True 65 | ) 66 | # self.browser_image = self.researcher.screenshot 67 | self.browser_image = self.multion.sessions.screenshot( 68 | session_id=self.researcher.session_id 69 | ).screenshot 70 | time.sleep(1) 71 | 72 | self.notetaker = self.multion.sessions.create( 73 | url="https://anotepad.com/", local=self.local, include_screenshot=True 74 | ) 75 | # self.scratchpad_image = self.notetaker.screenshot 76 | self.scratchpad_image = self.multion.sessions.screenshot( 77 | session_id=self.notetaker.session_id 78 | ).screenshot 79 | time.sleep(1) 80 | 81 | self.done = True 82 | self.task = "" 83 | self.plan = "" 84 | self.messages = [] 85 | self.client = OpenAI(api_key=openai_api_key) 86 | 87 | def programmer_login(self): 88 | if self.local: 89 | cmd = "Create a new Python REPL." 90 | else: 91 | cmd = "Log in with the email {email} and the password {password}. Then create a new Python REPL.".format( 92 | email=self.replit_email, password=self.replit_password 93 | ) 94 | while True: 95 | self.programmer = self.multion.sessions.step( 96 | self.programmer.session_id, 97 | cmd=cmd + "\n\n" + programmer_notes, 98 | url="https://replit.com/login", 99 | include_screenshot=True, 100 | ) 101 | print(self.programmer) 102 | print( 103 | self.multion.sessions.screenshot( 104 | session_id=self.programmer.session_id 105 | ).screenshot 106 | ) 107 | # time.sleep(1) 108 | # yield ("", self.editor_image, self.browser_image, self.scratchpad_image) 109 | # self.editor_image = self.programmer["screenshot"] 110 | if self.programmer.status in ["DONE", "NOT SURE"]: 111 | break 112 | 113 | self.editor_image = self.multion.sessions.screenshot( 114 | session_id=self.programmer.session_id 115 | ).screenshot 116 | time.sleep(1) 117 | 118 | def prepare_messages(self): 119 | messages = [ 120 | {"role": "user", "content": orchestrator_prompt}, 121 | { 122 | "role": "user", 123 | "content": "The Task given to you is: {task}".format(task=self.task), 124 | }, 125 | { 126 | "role": "user", 127 | "content": "The current Plan state is: {plan}".format(plan=""), 128 | }, 129 | ] 130 | for message in self.messages: 131 | messages.append(message) 132 | 133 | messages.append( 134 | { 135 | "role": "user", 136 | "content": [ 137 | { 138 | "type": "text", 139 | "text": "This is the current state of the Programmer Intern.", 140 | }, 141 | {"type": "image_url", "image_url": {"url": self.editor_image}}, 142 | ], 143 | } 144 | ) 145 | messages.append( 146 | { 147 | "role": "user", 148 | "content": [ 149 | { 150 | "type": "text", 151 | "text": "This is the current state of the Researcher Intern.", 152 | }, 153 | {"type": "image_url", "image_url": {"url": self.browser_image}}, 154 | ], 155 | } 156 | ) 157 | messages.append( 158 | { 159 | "role": "user", 160 | "content": [ 161 | { 162 | "type": "text", 163 | "text": "This is the current state of the Notetaker Intern.", 164 | }, 165 | {"type": "image_url", "image_url": {"url": self.scratchpad_image}}, 166 | ], 167 | } 168 | ) 169 | return messages 170 | 171 | def execute_action(self, action): 172 | 173 | action_func = action.split(" ", 1)[0] 174 | 175 | if action_func == "submit": 176 | self.done = True 177 | return 178 | elif action_func == "update_plan": 179 | action_arg = action.split(" ", 1)[1] 180 | self.plan = action_arg 181 | return 182 | elif action_func == "programmer": 183 | action_arg = action.split(" ", 1)[1] 184 | while True: 185 | self.programmer = self.multion.sessions.step( 186 | self.programmer.session_id, 187 | cmd=action_arg + "\n\n" + programmer_notes, 188 | url="https://replit.com/login", 189 | include_screenshot=True, 190 | ) 191 | print(self.programmer) 192 | # if self.programmer.status == "NOT SURE": 193 | # self.messages.append( 194 | # { 195 | # "role": "user", 196 | # "content": "The Programmer says: {message}\n\nYour next reply will go to the programmer.".format( 197 | # message=self.programmer.message 198 | # ), 199 | # } 200 | # ) 201 | # chat_completion = self.client.chat.completions.create( 202 | # messages=self.prepare_messages(), 203 | # model="gpt-4-vision-preview", 204 | # # max_tokens=200, 205 | # ) 206 | # action_arg = chat_completion.choices[0].message.content 207 | # self.messages.append({"role": "assistant", "content": action_arg}) 208 | # else: 209 | self.messages.append( 210 | { 211 | "role": "user", 212 | "content": "The Programmer says: {message}".format( 213 | message=self.programmer.message 214 | ), 215 | } 216 | ) 217 | # time.sleep(1) 218 | # self.editor_image = self.programmer["screenshot"] 219 | # yield ("", self.editor_image, self.browser_image, self.scratchpad_image) 220 | if self.programmer.status in ["DONE", "NOT SURE"]: 221 | break 222 | self.editor_image = self.multion.sessions.screenshot( 223 | session_id=self.programmer.session_id 224 | ).screenshot 225 | print(self.editor_image) 226 | time.sleep(1) 227 | return 228 | elif action_func == "researcher": 229 | action_arg = action.split(" ", 1)[1] 230 | while True: 231 | self.researcher = self.multion.sessions.step( 232 | self.researcher.session_id, 233 | cmd=action_arg, 234 | url="https://www.google.com", 235 | include_screenshot=True, 236 | ) 237 | print(self.researcher) 238 | self.messages.append( 239 | { 240 | "role": "user", 241 | "content": "The Researcher says: {message}".format( 242 | message=self.researcher.message 243 | ), 244 | } 245 | ) 246 | # time.sleep(1) 247 | # self.browser_image = self.researcher["screenshot"] 248 | # yield ("", self.editor_image, self.browser_image, self.scratchpad_image) 249 | if self.researcher.status == "DONE": 250 | break 251 | self.browser_image = self.multion.sessions.screenshot( 252 | session_id=self.researcher.session_id 253 | ).screenshot 254 | print(self.browser_image) 255 | time.sleep(1) 256 | return 257 | elif action_func == "notetaker": 258 | action_arg = action.split(" ", 1)[1] 259 | while True: 260 | self.notetaker = self.multion.sessions.step( 261 | self.notetaker.session_id, 262 | cmd=action_arg + "\n\n" + notetaker_notes, 263 | url="https://anotepad.com/", 264 | include_screenshot=True, 265 | ) 266 | print(self.notetaker) 267 | self.messages.append( 268 | { 269 | "role": "user", 270 | "content": "The Notetaker says: {message}".format( 271 | message=self.notetaker.message 272 | ), 273 | } 274 | ) 275 | # time.sleep(1) 276 | # self.scratchpad_image = self.notetaker["screenshot"] 277 | # yield ("", self.editor_image, self.browser_image, self.scratchpad_image) 278 | if self.notetaker.status == "DONE": 279 | break 280 | self.scratchpad_image = self.multion.sessions.screenshot( 281 | session_id=self.notetaker.session_id 282 | ).screenshot 283 | print(self.scratchpad_image) 284 | time.sleep(1) 285 | return 286 | elif action_func == "clarify": 287 | action_arg = action.split(" ", 1)[1] 288 | return 289 | 290 | def orchestrator(self): 291 | if not self.programmer_logged_in: 292 | self.programmer_login() 293 | self.programmer_logged_in = True 294 | messages = self.prepare_messages() 295 | chat_completion = self.client.chat.completions.create( 296 | messages=messages, 297 | model="gpt-4-vision-preview", 298 | # max_tokens=200, 299 | ) 300 | response = chat_completion.choices[0].message.content 301 | action, explanation = response.split("Explanation: ", 1) 302 | action = action.split("Action: ", 1)[1] 303 | 304 | self.messages.append({"role": "assistant", "content": response}) 305 | self.messages.append( 306 | { 307 | "role": "user", 308 | "content": "The current Plan state is: {plan}".format(plan=self.plan), 309 | } 310 | ) 311 | print(self.messages) 312 | 313 | # self.execute_action(action) 314 | 315 | action_func = action.split(" ", 1)[0] 316 | 317 | if action_func == "submit": 318 | self.done = True 319 | elif action_func == "update_plan": 320 | action_arg = action.split(" ", 1)[1] 321 | self.plan = action_arg 322 | elif action_func == "programmer": 323 | action_arg = action.split(" ", 1)[1] 324 | while True: 325 | self.programmer = self.multion.sessions.step( 326 | self.programmer.session_id, 327 | cmd=action_arg + "\n\n" + programmer_notes, 328 | url="https://replit.com/login", 329 | include_screenshot=True, 330 | ) 331 | print(self.programmer) 332 | self.messages.append( 333 | { 334 | "role": "user", 335 | "content": "The Programmer says: {message}".format( 336 | message=self.programmer.message 337 | ), 338 | } 339 | ) 340 | if self.programmer.status in ["DONE", "NOT SURE"]: 341 | break 342 | self.editor_image = self.multion.sessions.screenshot( 343 | session_id=self.programmer.session_id 344 | ).screenshot 345 | print(self.editor_image) 346 | time.sleep(1) 347 | elif action_func == "researcher": 348 | action_arg = action.split(" ", 1)[1] 349 | while True: 350 | self.researcher = self.multion.sessions.step( 351 | self.researcher.session_id, 352 | cmd=action_arg, 353 | url="https://www.google.com", 354 | include_screenshot=True, 355 | ) 356 | print(self.researcher) 357 | self.messages.append( 358 | { 359 | "role": "user", 360 | "content": "The Researcher says: {message}".format( 361 | message=self.researcher.message 362 | ), 363 | } 364 | ) 365 | # time.sleep(1) 366 | # self.browser_image = self.researcher["screenshot"] 367 | # yield ("", self.editor_image, self.browser_image, self.scratchpad_image) 368 | if self.researcher.status == "DONE": 369 | break 370 | self.browser_image = self.multion.sessions.screenshot( 371 | session_id=self.researcher.session_id 372 | ).screenshot 373 | print(self.browser_image) 374 | time.sleep(1) 375 | elif action_func == "notetaker": 376 | action_arg = action.split(" ", 1)[1] 377 | while True: 378 | self.notetaker = self.multion.sessions.step( 379 | self.notetaker.session_id, 380 | cmd=action_arg + "\n\n" + notetaker_notes, 381 | url="https://anotepad.com/", 382 | include_screenshot=True, 383 | ) 384 | print(self.notetaker) 385 | self.messages.append( 386 | { 387 | "role": "user", 388 | "content": "The Notetaker says: {message}".format( 389 | message=self.notetaker.message 390 | ), 391 | } 392 | ) 393 | # time.sleep(1) 394 | # self.scratchpad_image = self.notetaker["screenshot"] 395 | # yield ("", self.editor_image, self.browser_image, self.scratchpad_image) 396 | if self.notetaker.status == "DONE": 397 | break 398 | self.scratchpad_image = self.multion.sessions.screenshot( 399 | session_id=self.notetaker.session_id 400 | ).screenshot 401 | print(self.scratchpad_image) 402 | time.sleep(1) 403 | elif action_func == "clarify": 404 | action_arg = action.split(" ", 1)[1] 405 | 406 | return explanation 407 | 408 | def run(self, prompt): 409 | self.done = False 410 | self.task = prompt 411 | while not self.done: 412 | # curr_response = self.orchestrator() 413 | 414 | if not self.programmer_logged_in: 415 | # self.programmer_login() 416 | if self.local: 417 | cmd = "Create a new Python REPL." 418 | else: 419 | cmd = "Log in with the email {email} and the password {password}. Then create a new Python REPL.".format( 420 | email=self.replit_email, password=self.replit_password 421 | ) 422 | while True: 423 | self.programmer = self.multion.sessions.step( 424 | self.programmer.session_id, 425 | cmd=cmd + "\n\n" + programmer_notes, 426 | url="https://replit.com/login", 427 | include_screenshot=True, 428 | ) 429 | print(self.programmer) 430 | print( 431 | self.multion.sessions.screenshot( 432 | session_id=self.programmer.session_id 433 | ).screenshot 434 | ) 435 | # time.sleep(1) 436 | # yield ("", self.editor_image, self.browser_image, self.scratchpad_image) 437 | # self.editor_image = self.programmer["screenshot"] 438 | if self.programmer.status in ["DONE", "NOT SURE"]: 439 | break 440 | self.editor_image = self.multion.sessions.screenshot( 441 | session_id=self.programmer.session_id 442 | ).screenshot 443 | time.sleep(1) 444 | yield ( 445 | "I am setting up the programming environment", 446 | self.editor_image, 447 | self.browser_image, 448 | self.scratchpad_image, 449 | ) 450 | 451 | self.editor_image = self.multion.sessions.screenshot( 452 | session_id=self.programmer.session_id 453 | ).screenshot 454 | time.sleep(1) 455 | self.programmer_logged_in = True 456 | messages = self.prepare_messages() 457 | chat_completion = self.client.chat.completions.create( 458 | messages=messages, 459 | model="gpt-4-vision-preview", 460 | # max_tokens=200, 461 | ) 462 | response = chat_completion.choices[0].message.content 463 | action, explanation = response.split("Explanation: ", 1) 464 | action = action.split("Action: ", 1)[1] 465 | 466 | self.messages.append({"role": "assistant", "content": response}) 467 | self.messages.append( 468 | { 469 | "role": "user", 470 | "content": "The current Plan state is: {plan}".format( 471 | plan=self.plan 472 | ), 473 | } 474 | ) 475 | print(self.messages) 476 | 477 | # self.execute_action(action) 478 | 479 | action_func = action.split(" ", 1)[0] 480 | 481 | if action_func == "submit": 482 | self.done = True 483 | yield ( 484 | explanation, 485 | self.editor_image, 486 | self.browser_image, 487 | self.scratchpad_image, 488 | ) 489 | elif action_func == "update_plan": 490 | action_arg = action.split(" ", 1)[1] 491 | self.plan = action_arg 492 | yield ( 493 | explanation, 494 | self.editor_image, 495 | self.browser_image, 496 | self.scratchpad_image, 497 | ) 498 | elif action_func == "programmer": 499 | action_arg = action.split(" ", 1)[1] 500 | while True: 501 | self.programmer = self.multion.sessions.step( 502 | self.programmer.session_id, 503 | cmd=action_arg + "\n\n" + programmer_notes, 504 | url="https://replit.com/login", 505 | include_screenshot=True, 506 | ) 507 | print(self.programmer) 508 | self.messages.append( 509 | { 510 | "role": "user", 511 | "content": "The Programmer says: {message}".format( 512 | message=self.programmer.message 513 | ), 514 | } 515 | ) 516 | if self.programmer.status in ["DONE", "NOT SURE"]: 517 | break 518 | self.editor_image = self.multion.sessions.screenshot( 519 | session_id=self.programmer.session_id 520 | ).screenshot 521 | print(self.editor_image) 522 | time.sleep(1) 523 | yield ( 524 | explanation, 525 | self.editor_image, 526 | self.browser_image, 527 | self.scratchpad_image, 528 | ) 529 | self.editor_image = self.multion.sessions.screenshot( 530 | session_id=self.programmer.session_id 531 | ).screenshot 532 | print(self.editor_image) 533 | time.sleep(1) 534 | elif action_func == "researcher": 535 | action_arg = action.split(" ", 1)[1] 536 | while True: 537 | self.researcher = self.multion.sessions.step( 538 | self.researcher.session_id, 539 | cmd=action_arg, 540 | url="https://www.google.com", 541 | include_screenshot=True, 542 | ) 543 | print(self.researcher) 544 | self.messages.append( 545 | { 546 | "role": "user", 547 | "content": "The Researcher says: {message}".format( 548 | message=self.researcher.message 549 | ), 550 | } 551 | ) 552 | # time.sleep(1) 553 | # self.browser_image = self.researcher["screenshot"] 554 | # yield ("", self.editor_image, self.browser_image, self.scratchpad_image) 555 | if self.researcher.status == "DONE": 556 | break 557 | self.browser_image = self.multion.sessions.screenshot( 558 | session_id=self.researcher.session_id 559 | ).screenshot 560 | print(self.browser_image) 561 | time.sleep(1) 562 | yield ( 563 | explanation, 564 | self.editor_image, 565 | self.browser_image, 566 | self.scratchpad_image, 567 | ) 568 | self.browser_image = self.multion.sessions.screenshot( 569 | session_id=self.researcher.session_id 570 | ).screenshot 571 | print(self.browser_image) 572 | time.sleep(1) 573 | elif action_func == "notetaker": 574 | action_arg = action.split(" ", 1)[1] 575 | while True: 576 | self.notetaker = self.multion.sessions.step( 577 | self.notetaker.session_id, 578 | cmd=action_arg + "\n\n" + notetaker_notes, 579 | url="https://anotepad.com/", 580 | include_screenshot=True, 581 | ) 582 | print(self.notetaker) 583 | self.messages.append( 584 | { 585 | "role": "user", 586 | "content": "The Notetaker says: {message}".format( 587 | message=self.notetaker.message 588 | ), 589 | } 590 | ) 591 | # time.sleep(1) 592 | # self.scratchpad_image = self.notetaker["screenshot"] 593 | # yield ("", self.editor_image, self.browser_image, self.scratchpad_image) 594 | if self.notetaker.status == "DONE": 595 | break 596 | self.scratchpad_image = self.multion.sessions.screenshot( 597 | session_id=self.notetaker.session_id 598 | ).screenshot 599 | print(self.scratchpad_image) 600 | time.sleep(1) 601 | yield ( 602 | explanation, 603 | self.editor_image, 604 | self.browser_image, 605 | self.scratchpad_image, 606 | ) 607 | self.scratchpad_image = self.multion.sessions.screenshot( 608 | session_id=self.notetaker.session_id 609 | ).screenshot 610 | print(self.scratchpad_image) 611 | time.sleep(1) 612 | elif action_func == "clarify": 613 | action_arg = action.split(" ", 1)[1] 614 | yield ( 615 | explanation, 616 | self.editor_image, 617 | self.browser_image, 618 | self.scratchpad_image, 619 | ) 620 | 621 | # return explanation 622 | 623 | # yield ( 624 | # curr_response, 625 | # self.editor_image, 626 | # self.browser_image, 627 | # self.scratchpad_image, 628 | # ) 629 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | multion 13 | openai 14 | python-dotenv 15 | pyodide-http 16 | 17 | 18 | 19 | import pyodide_http 20 | pyodide_http.patch_all() 21 | import gradio as gr 22 | import os 23 | import time 24 | from agent import DevOn 25 | 26 | image_temp = "https://miro.medium.com/v2/resize:fit:1200/0*n-2bW82Z6m6U2bij.jpeg" 27 | # devon = DevOn( 28 | # editor_image=image_temp, browser_image=image_temp, scratchpad_image=image_temp 29 | # ) 30 | devon = None 31 | multion_api_key = "" 32 | openai_api_key = "" 33 | replit_email = "" 34 | replit_password = "" 35 | 36 | 37 | def add_message(history, message): 38 | for x in message["files"]: 39 | history.append(((x,), None)) 40 | if message["text"] is not None: 41 | history.append((message["text"], None)) 42 | return history, gr.MultimodalTextbox(value=None, interactive=False) 43 | 44 | 45 | def multion_api_key_update(x): 46 | global multion_api_key 47 | multion_api_key = x 48 | 49 | 50 | def openai_api_key_update(x): 51 | global openai_api_key 52 | openai_api_key = x 53 | 54 | 55 | def replit_email_update(x): 56 | global replit_email 57 | replit_email = x 58 | 59 | 60 | def replit_password_update(x): 61 | global replit_password 62 | replit_password = x 63 | 64 | 65 | def bot(history): 66 | devon = DevOn( 67 | editor_image=image_temp, 68 | browser_image=image_temp, 69 | scratchpad_image=image_temp, 70 | multion_api_key=multion_api_key, 71 | openai_api_key=openai_api_key, 72 | replit_email=replit_email, 73 | replit_password=replit_password, 74 | ) 75 | 76 | for r in devon.run(history[-1][0]): 77 | text, editor_image, browser_image, scratchpad_image = r 78 | if type(text) == str: 79 | history.append((None, text)) 80 | if editor_image is None: 81 | editor_image = devon.editor_image 82 | browser_image = devon.browser_image 83 | scratchpad_image = devon.scratchpad_image 84 | yield history, editor_image, browser_image, scratchpad_image 85 | 86 | 87 | with gr.Blocks(css="footer {visibility: hidden}") as demo: 88 | with gr.Row(): 89 | with gr.Column(): 90 | multion_api_key_in = gr.Textbox(label="MultiOn API Key") 91 | openai_api_key_in = gr.Textbox(label="OpenAI API Key") 92 | with gr.Column(): 93 | replit_email_in = gr.Textbox(label="Replit Email") 94 | replit_password_in = gr.Textbox(label="Replit Password") 95 | with gr.Row(): 96 | with gr.Column(): 97 | chatbot = gr.Chatbot( 98 | [], elem_id="chatbot", bubble_full_width=False, height=300 99 | ) 100 | 101 | chat_input = gr.MultimodalTextbox( 102 | value={ 103 | "text": "benchmark the perplexity api's resposne time with the api key abcdef" 104 | }, 105 | interactive=True, 106 | file_types=["image"], 107 | placeholder="Enter message or upload file...", 108 | show_label=False, 109 | ) 110 | with gr.Column(): 111 | if devon: 112 | editor_view = gr.Image( 113 | devon.editor_image, 114 | label="Editor", 115 | ) 116 | else: 117 | editor_view = gr.Image() 118 | with gr.Row(): 119 | with gr.Column(): 120 | if devon: 121 | browser_view = gr.Image( 122 | devon.browser_image, 123 | label="Browser", 124 | ) 125 | else: 126 | browser_view = gr.Image() 127 | with gr.Column(): 128 | if devon: 129 | scratchpad_view = gr.Image( 130 | devon.scratchpad_image, 131 | label="Scratchpad", 132 | ) 133 | else: 134 | scratchpad_view = gr.Image() 135 | 136 | chat_msg = chat_input.submit( 137 | add_message, [chatbot, chat_input], [chatbot, chat_input] 138 | ) 139 | bot_msg = chat_msg.then( 140 | bot, 141 | [chatbot], 142 | [chatbot, editor_view, browser_view, scratchpad_view], 143 | api_name="bot_response", 144 | ) 145 | bot_msg.then(lambda: gr.MultimodalTextbox(interactive=True), None, [chat_input]) 146 | 147 | multion_api_key_in.change(multion_api_key_update, multion_api_key_in) 148 | openai_api_key_in.change(openai_api_key_update, openai_api_key_in) 149 | replit_email_in.change(replit_email_update, replit_email_in) 150 | replit_password_in.change(replit_password_update, replit_password_in) 151 | 152 | # chatbot.like(print_like_dislike, None, None) 153 | 154 | if __name__ == "__main__": 155 | demo.queue() 156 | demo.launch() 157 | 158 | 159 | 160 | from openai import OpenAI 161 | from prompts import orchestrator_prompt 162 | from prompts import programmer_notes 163 | from prompts import notetaker_notes 164 | from dotenv import load_dotenv 165 | import time 166 | import multion 167 | from multion.client import MultiOn 168 | import os 169 | 170 | load_dotenv(".env.local") 171 | 172 | # replit_email = os.getenv("REPLIT_EMAIL") 173 | # replit_password = os.getenv("REPLIT_PASSWORD") 174 | 175 | # multion_api_key = os.getenv("MULTION_API_KEY") 176 | # multion.login(use_api=True, multion_api_key=multion_api_key) 177 | 178 | # runpod_url = os.getenv("RUNPOD_URL") 179 | 180 | image_temp = "https://miro.medium.com/v2/resize:fit:1200/0*n-2bW82Z6m6U2bij.jpeg" 181 | 182 | 183 | class DevOn: 184 | def __init__( 185 | self, 186 | editor_image, 187 | browser_image, 188 | scratchpad_image, 189 | multion_api_key, 190 | openai_api_key, 191 | replit_email, 192 | replit_password, 193 | ): 194 | print(multion_api_key, openai_api_key) 195 | self.editor_image = editor_image 196 | self.browser_image = browser_image 197 | self.scratchpad_image = scratchpad_image 198 | self.local = os.getenv("WHERE_EXECUTE") == "local" 199 | 200 | self.multion = MultiOn(api_key=multion_api_key) 201 | 202 | self.replit_email = replit_email 203 | self.replit_password = replit_password 204 | 205 | self.programmer = self.multion.sessions.create( 206 | url="https://replit.com/login", local=self.local, include_screenshot=True 207 | ) 208 | self.programmer_logged_in = False 209 | # self.editor_image = self.programmer.screenshot 210 | self.editor_image = self.multion.sessions.screenshot( 211 | session_id=self.programmer.session_id 212 | ).screenshot 213 | print(self.editor_image) 214 | time.sleep(1) 215 | # print(self.programmer) 216 | 217 | self.researcher = self.multion.sessions.create( 218 | url="https://www.google.com", local=self.local, include_screenshot=True 219 | ) 220 | # self.browser_image = self.researcher.screenshot 221 | self.browser_image = self.multion.sessions.screenshot( 222 | session_id=self.researcher.session_id 223 | ).screenshot 224 | time.sleep(1) 225 | 226 | self.notetaker = self.multion.sessions.create( 227 | url="https://anotepad.com/", local=self.local, include_screenshot=True 228 | ) 229 | # self.scratchpad_image = self.notetaker.screenshot 230 | self.scratchpad_image = self.multion.sessions.screenshot( 231 | session_id=self.notetaker.session_id 232 | ).screenshot 233 | time.sleep(1) 234 | 235 | self.done = True 236 | self.task = "" 237 | self.plan = "" 238 | self.messages = [] 239 | self.client = OpenAI(api_key=openai_api_key) 240 | 241 | def programmer_login(self): 242 | if self.local: 243 | cmd = "Create a new Python REPL." 244 | else: 245 | cmd = "Log in with the email {email} and the password {password}. Then create a new Python REPL.".format( 246 | email=self.replit_email, password=self.replit_password 247 | ) 248 | while True: 249 | self.programmer = self.multion.sessions.step( 250 | self.programmer.session_id, 251 | cmd=cmd + "\n\n" + programmer_notes, 252 | url="https://replit.com/login", 253 | include_screenshot=True, 254 | ) 255 | print(self.programmer) 256 | print( 257 | self.multion.sessions.screenshot( 258 | session_id=self.programmer.session_id 259 | ).screenshot 260 | ) 261 | # time.sleep(1) 262 | # yield ("", self.editor_image, self.browser_image, self.scratchpad_image) 263 | # self.editor_image = self.programmer["screenshot"] 264 | if self.programmer.status == "DONE": 265 | break 266 | 267 | self.editor_image = self.multion.sessions.screenshot( 268 | session_id=self.programmer.session_id 269 | ).screenshot 270 | time.sleep(1) 271 | 272 | def prepare_messages(self): 273 | messages = [ 274 | {"role": "user", "content": orchestrator_prompt}, 275 | { 276 | "role": "user", 277 | "content": "The Task given to you is: {task}".format(task=self.task), 278 | }, 279 | { 280 | "role": "user", 281 | "content": "The current Plan state is: {plan}".format(plan=""), 282 | }, 283 | ] 284 | for message in self.messages: 285 | messages.append(message) 286 | 287 | messages.append( 288 | { 289 | "role": "user", 290 | "content": [ 291 | { 292 | "type": "text", 293 | "text": "This is the current state of the Programmer Intern.", 294 | }, 295 | {"type": "image_url", "image_url": {"url": self.editor_image}}, 296 | ], 297 | } 298 | ) 299 | messages.append( 300 | { 301 | "role": "user", 302 | "content": [ 303 | { 304 | "type": "text", 305 | "text": "This is the current state of the Researcher Intern.", 306 | }, 307 | {"type": "image_url", "image_url": {"url": self.browser_image}}, 308 | ], 309 | } 310 | ) 311 | messages.append( 312 | { 313 | "role": "user", 314 | "content": [ 315 | { 316 | "type": "text", 317 | "text": "This is the current state of the Notetaker Intern.", 318 | }, 319 | {"type": "image_url", "image_url": {"url": self.scratchpad_image}}, 320 | ], 321 | } 322 | ) 323 | return messages 324 | 325 | def execute_action(self, action): 326 | 327 | action_func = action.split(" ", 1)[0] 328 | 329 | if action_func == "submit": 330 | self.done = True 331 | return 332 | elif action_func == "update_plan": 333 | action_arg = action.split(" ", 1)[1] 334 | self.plan = action_arg 335 | return 336 | elif action_func == "programmer": 337 | action_arg = action.split(" ", 1)[1] 338 | while True: 339 | self.programmer = self.multion.sessions.step( 340 | self.programmer.session_id, 341 | cmd=action_arg + "\n\n" + programmer_notes, 342 | url="https://replit.com/login", 343 | include_screenshot=True, 344 | ) 345 | print(self.programmer) 346 | if self.programmer.status == "NOT SURE": 347 | self.messages.append( 348 | { 349 | "role": "user", 350 | "content": "The Programmer says: {message}\n\nYour next reply will go to the programmer.".format( 351 | message=self.programmer.message 352 | ), 353 | } 354 | ) 355 | chat_completion = self.client.chat.completions.create( 356 | messages=self.prepare_messages(), 357 | model="gpt-4-vision-preview", 358 | # max_tokens=200, 359 | ) 360 | action_arg = chat_completion.choices[0].message.content 361 | self.messages.append({"role": "assistant", "content": action_arg}) 362 | else: 363 | self.messages.append( 364 | { 365 | "role": "user", 366 | "content": "The Programmer says: {message}".format( 367 | message=self.programmer.message 368 | ), 369 | } 370 | ) 371 | # time.sleep(1) 372 | # self.editor_image = self.programmer["screenshot"] 373 | # yield ("", self.editor_image, self.browser_image, self.scratchpad_image) 374 | if self.programmer.status == "DONE": 375 | break 376 | self.editor_image = self.multion.sessions.screenshot( 377 | session_id=self.programmer.session_id 378 | ).screenshot 379 | print(self.editor_image) 380 | time.sleep(1) 381 | return 382 | elif action_func == "researcher": 383 | action_arg = action.split(" ", 1)[1] 384 | while True: 385 | self.researcher = self.multion.sessions.step( 386 | self.researcher.session_id, 387 | cmd=action_arg, 388 | url="https://www.google.com", 389 | include_screenshot=True, 390 | ) 391 | print(self.researcher) 392 | self.messages.append( 393 | { 394 | "role": "user", 395 | "content": "The Researcher says: {message}".format( 396 | message=self.researcher.message 397 | ), 398 | } 399 | ) 400 | # time.sleep(1) 401 | # self.browser_image = self.researcher["screenshot"] 402 | # yield ("", self.editor_image, self.browser_image, self.scratchpad_image) 403 | if self.researcher.status == "DONE": 404 | break 405 | self.browser_image = self.multion.sessions.screenshot( 406 | session_id=self.researcher.session_id 407 | ).screenshot 408 | print(self.browser_image) 409 | time.sleep(1) 410 | return 411 | elif action_func == "notetaker": 412 | action_arg = action.split(" ", 1)[1] 413 | while True: 414 | self.notetaker = self.multion.sessions.step( 415 | self.notetaker.session_id, 416 | cmd=action_arg + "\n\n" + notetaker_notes, 417 | url="https://anotepad.com/", 418 | include_screenshot=True, 419 | ) 420 | print(self.notetaker) 421 | self.messages.append( 422 | { 423 | "role": "user", 424 | "content": "The Notetaker says: {message}".format( 425 | message=self.notetaker.message 426 | ), 427 | } 428 | ) 429 | # time.sleep(1) 430 | # self.scratchpad_image = self.notetaker["screenshot"] 431 | # yield ("", self.editor_image, self.browser_image, self.scratchpad_image) 432 | if self.notetaker.status == "DONE": 433 | break 434 | self.scratchpad_image = self.multion.sessions.screenshot( 435 | session_id=self.notetaker.session_id 436 | ).screenshot 437 | print(self.scratchpad_image) 438 | time.sleep(1) 439 | return 440 | elif action_func == "clarify": 441 | action_arg = action.split(" ", 1)[1] 442 | return 443 | 444 | def orchestrator(self): 445 | if not self.programmer_logged_in: 446 | self.programmer_login() 447 | self.programmer_logged_in = True 448 | messages = self.prepare_messages() 449 | chat_completion = self.client.chat.completions.create( 450 | messages=messages, 451 | model="gpt-4-vision-preview", 452 | # max_tokens=200, 453 | ) 454 | response = chat_completion.choices[0].message.content 455 | action, explanation = response.split("Explanation: ", 1) 456 | action = action.split("Action: ", 1)[1] 457 | 458 | self.messages.append({"role": "assistant", "content": response}) 459 | self.messages.append( 460 | { 461 | "role": "user", 462 | "content": "The current Plan state is: {plan}".format(plan=self.plan), 463 | } 464 | ) 465 | print(self.messages) 466 | 467 | self.execute_action(action) 468 | 469 | # temp 470 | # self.done = True 471 | return explanation 472 | 473 | def run(self, prompt): 474 | self.done = False 475 | self.task = prompt 476 | while not self.done: 477 | curr_response = self.orchestrator() 478 | yield ( 479 | curr_response, 480 | self.editor_image, 481 | self.browser_image, 482 | self.scratchpad_image, 483 | ) 484 | 485 | 486 | 487 | 488 | orchestrator_prompt = """**General** 489 | 490 | - You are DevOn, an expert Software Developer. 491 | - You will be asked to develop a new software project from scratch. You will primarily work in Python. You will deal with large software projects spanning multiple files and user requirements. 492 | - Your lifecycle will essentially circle around the Task, the State, your Plan, your Actions, and your Interns. Each of these are described in detail below. 493 | - To start with, your Plan will be empty. You will receive a State (in the form of 3 images, one from each of your Interns) and a Task. You will construct a Plan outlining the steps you will need to take to complete the Task, then ask your Interns to do things in order to incrementally fulfil the steps and complete the Task. 494 | - With each step, you will also provide an Explanation, explaining to the user what you are currently doing, so they may be able to keep track and monitor your progress. For example: 495 | - Explanation: I am currently updating the plan based on the current state and the Task. 496 | - Explanation: I am currently creating a file called utils.py which will contain utility functions. 497 | 498 | **State** 499 | 500 | **Interns** 501 | 502 | - You have 3 interns who will help you with different tasks - a Programmer, a Researcher, and a Planner. Here’s some info about them: 503 | - Programmer: the Programmer is great at writing code given very specific instructions but isn’t a good long term planner. The Programmer works on Replit. You can ask the Programmer to write some code in certain files, make new files, etc. You can even give loose instructions like “Make a new file and write basic skeleton for an Agent class in it.” Keep in mind that the Programmer works exclusively in an online Replit IDE environment. Make sure your Plan and your Actions take this into consideration. 504 | - Researcher: the Researcher is very handy with a browser and great at finding out technical details, documentation, examples, miscellaneous information, etc. You can ask it to do things like “Find out how to make an LLM call using the Perplexity API”. 505 | - Notetaker: the Notetaker has a notepad and can note down anything you want. You will be able to see the notepad at all times. Anytime you want anything written down just to keep track of it, ask the Planner to do so. 506 | 507 | **Plan** 508 | 509 | - You have a persistent object to keep track of things: a Plan. 510 | - If the plan is empty, you will create a plan using the current state of things and the given task. You will do so using the update_plan action described below. 511 | 512 | **Actions** 513 | 514 | - There are 6 actions that you can take at the current time step. You must always take a valid action. You will complete the task by taking actions. You are free to take as many actions as needed (even hundreds), don’t try to rush by compressing multiple actions into one. These are the available actions: 515 | - update_plan : Update Plan’s value to . This will replace the old value, not append to it. If there’s something from the old plan you wish to include in the updated one, make sure to include it in the you provide as an argument. Some examples how you can use this: 516 | - update_plan In order to carry out the task of creating a Flask web server, I will need to take the following steps: 517 | 1) … 518 | 2) … 519 | 3) … 520 | - programmer : Ask the Programmer to carry out a . Some examples of how you can use this: 521 | - programmer Create a new Python file for utils called utils.py and write a generate_random_number() function in it that takes no parameters and returns a random number. 522 | - programmer Go to the model.py file and import generate_random_number() from utils. 523 | - researcher : Ask the Researcher to carry out a . The Researcher will reply to you with the information you asked for. Some examples of how you can use this: 524 | - researcher Find out how the OpenAI API is used. 525 | - researcher What is a SERP API I could use? 526 | - notetaker : Ask the Notetaker to carry out a . Some examples of how you can use this: 527 | - notetaker Note down the following information: MULTION_API_KEY=… 528 | - notetaker Note down the following information: An example Chat Completions API call looks like the following: 529 | from openai import OpenAI 530 | client = OpenAI() 531 | 532 | response = client.chat.completions.create( 533 | model="gpt-3.5-turbo", 534 | messages=[ 535 | {"role": "system", "content": "You are a helpful assistant."}, 536 | {"role": "user", "content": "Who won the world series in 2020?"}, 537 | {"role": "assistant", "content": "The Los Angeles Dodgers won the World Series in 2020."}, 538 | {"role": "user", "content": "Where was it played?"} 539 | ] 540 | ) 541 | - clarify : Clarify something about the Task. Sometimes, there may be missing information, such as logins, api keys, or some requirements of the Task may be unclear. Use this Action to clarify things from the user by asking . Use this sparingly. Try and make decisions yourself. Some examples of how you can use this: 542 | - clarify The Task mentions that I need to benchmark the Perplexity API. Could you provide your API Key? 543 | - submit: The Task is completed and you are ready to submit the output (whatever the programmer has so far). This is end the execution. Only do this when you are completely sure. 544 | 545 | **Important Notes** 546 | 547 | - Respond only by taking an Action (and providing the accompanying Explanation). Any response from you must be one of the above Actions. No other text in the response, just the Action and the Explanation. You will structure your output as such: 548 | ”Action: \nExplanation: ” 549 | - You do not need to ask the Programmer to log in. 550 | - You can see all the Interns screens. If it seems like an Intern has made a mistake or encountered an error, you can tell them about it using the relevant action and ask them to correct it. This is especially important with the Programmer. 551 | - When you ask the programmer to write some code, ask it like this: "programmer memorize the following code and write it in the editor: " 552 | - When you ask the programmer to write some code, make sure the code does not include any double quotation marks, only single quotation marks. E.g. "hello world" should instead be 'hello world'. 553 | - Do not ask the programmer to create new files. 554 | - When writing code, it is preferable to keep it small and simple. Don't write too much fluff. 555 | - Remember to only use single quotation marks. 556 | """ 557 | # programmer_notes = """Important Notes: 558 | # Do not refresh the page ever to check for anything. Only wait. Do not refresh. 559 | # You are working in a terminal environment. 560 | # You will do everything using the terminal and only the terminal. 561 | # If you need to create a new file, do so using the touch command on the terminal. 562 | # If you need to see files in the current directory, do so using ls. 563 | # If you need to view a files content, do so using the cat command on the terminal. 564 | # To enter code into a file, use a single printf command. After the printf command has been completely typed, press enter. Typing the command and pressing enter must be 2 separate steps. 565 | # Do not open a text editor like vim or nano. 566 | # If you need to install a new package, use pip install on the terminal. 567 | # Do not use the same command repeatedly. 568 | # When you write code into a file, write it once, cat it once, then stop. Do no attempt to write again unless it is wrong. 569 | # Remember that you need to press Enter after typing a command into the terminal. Only press enter after the command has been completely typed. Typing the command pressing enter must be 2 separate steps.""" 570 | 571 | programmer_notes = """Important Notes: 572 | You are a Programmer who works in a Replit Environment exclusively. If you need to install a package, use the Shell and not the Console. 573 | Do not refresh the page ever to check for anything. Only wait. Do not refresh. Do not create new files. Write your code in currently open editor window itself. Do not type double quotation marks. If you are asked to type code containing them, use single quotes instead.""" 574 | 575 | notetaker_notes = """Important Notes: 576 | Don't write anything in the Note Title field. 577 | Whatever notes you are told to make, write them in one go, don't press enter or type multiple times, because everytime you write, it will replace the prevoius content. 578 | You do not need to Save the note. When asked to note something down, just write it on the notepad. That is enough.""" 579 | 580 | 581 | 582 | 583 | 584 | --------------------------------------------------------------------------------