├── operate
├── __init__.py
├── models
│ ├── __init__.py
│ ├── weights
│ │ ├── __init__.py
│ │ └── best.pt
│ ├── prompts.py
│ └── apis.py
├── utils
│ ├── __init__.py
│ ├── style.py
│ ├── misc.py
│ ├── screenshot.py
│ ├── operating_system.py
│ ├── ocr.py
│ └── label.py
├── exceptions.py
├── main.py
├── operate.py
└── config.py
├── requirements-audio.txt
├── readme
├── key.png
├── terminal-access-1.png
├── terminal-access-2.png
└── self-operating-computer.png
├── .github
├── ISSUE_TEMPLATE
│ ├── feature_request.md
│ └── bug_report.md
├── workflows
│ └── upload-package.yml
└── PULL_REQUEST_TEMPLATE.md
├── setup.py
├── LICENSE
├── requirements.txt
├── CONTRIBUTING.md
├── .gitignore
├── evaluate.py
└── README.md
/operate/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/operate/models/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/operate/utils/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/requirements-audio.txt:
--------------------------------------------------------------------------------
1 | whisper-mic
--------------------------------------------------------------------------------
/operate/models/weights/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/readme/key.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OthersideAI/self-operating-computer/HEAD/readme/key.png
--------------------------------------------------------------------------------
/readme/terminal-access-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OthersideAI/self-operating-computer/HEAD/readme/terminal-access-1.png
--------------------------------------------------------------------------------
/readme/terminal-access-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OthersideAI/self-operating-computer/HEAD/readme/terminal-access-2.png
--------------------------------------------------------------------------------
/operate/models/weights/best.pt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OthersideAI/self-operating-computer/HEAD/operate/models/weights/best.pt
--------------------------------------------------------------------------------
/readme/self-operating-computer.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/OthersideAI/self-operating-computer/HEAD/readme/self-operating-computer.png
--------------------------------------------------------------------------------
/operate/exceptions.py:
--------------------------------------------------------------------------------
1 | class ModelNotRecognizedException(Exception):
2 | """Exception raised for unrecognized models.
3 |
4 | Attributes:
5 | model -- the unrecognized model
6 | message -- explanation of the error
7 | """
8 |
9 | def __init__(self, model, message="Model not recognized"):
10 | self.model = model
11 | self.message = message
12 | super().__init__(self.message)
13 |
14 | def __str__(self):
15 | return f"{self.message} : {self.model} "
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/feature_request.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Feature request
3 | about: Suggest an idea for this project
4 | title: '[FEATURE] Short Description of the Feature'
5 | labels: enhancement
6 | assignees: ''
7 |
8 | ---
9 |
10 | ### Is your feature request related to a problem? Please describe.
11 |
12 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
13 |
14 | ### Describe the solution you'd like
15 | A clear and concise description of what you want to happen.
16 |
17 | ### Describe alternatives you've considered
18 | A clear and concise description of any alternative solutions or features you've considered.
19 |
20 | ### Additional context
21 | Add any other context or screenshots about the feature request here.
--------------------------------------------------------------------------------
/.github/workflows/upload-package.yml:
--------------------------------------------------------------------------------
1 | name: Upload Python Package
2 |
3 | on:
4 | push:
5 | tags:
6 | - 'v*'
7 |
8 | jobs:
9 | deploy:
10 | runs-on: ubuntu-latest
11 | steps:
12 | - uses: actions/checkout@v3
13 |
14 | - name: Set up Python
15 | uses: actions/setup-python@v3
16 | with:
17 | python-version: '3.8'
18 |
19 | - name: Install dependencies
20 | run: |
21 | python -m pip install --upgrade pip
22 | pip install setuptools wheel twine
23 |
24 | - name: Build and check package
25 | run: |
26 | python setup.py sdist bdist_wheel
27 | twine check dist/*
28 |
29 | - name: Upload to PyPi
30 | uses: pypa/gh-action-pypi-publish@v1.4.2
31 | with:
32 | user: __token__
33 | password: ${{ secrets.PYPI_API_TOKEN }}
34 |
35 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/bug_report.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Bug report
3 | about: Create a report to help us improve
4 | title: '[BUG] Brief Description of the Issue'
5 | labels: bug
6 | assignees: ''
7 |
8 | ---
9 |
10 | Found a bug? Please fill out the sections below. 👍
11 |
12 |
13 | ### Describe the bug
14 |
15 | A clear and concise description of what the bug is.
16 |
17 | ### Steps to Reproduce
18 |
19 | 1. (for ex.) went to...
20 | 2. clicked on this point
21 | 3. not working
22 |
23 | ### Expected Behavior
24 | A brief description of what you expected to happen.
25 |
26 | ### Actual Behavior:
27 | what actually happened.
28 |
29 | ### Environment
30 | - OS:
31 | - Model Used (e.g., GPT-4v, Gemini Pro Vision):
32 | - Framework Version (optional):
33 |
34 | ### Screenshots
35 | If applicable, add screenshots to help explain your problem.
36 |
37 | ### Additional context
38 | Add any other context about the problem here.
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup, find_packages
2 |
3 | # Read the contents of your requirements.txt file
4 | with open("requirements.txt") as f:
5 | required = f.read().splitlines()
6 |
7 | # Read the contents of your README.md file for the project description
8 | with open("README.md", "r", encoding="utf-8") as readme_file:
9 | long_description = readme_file.read()
10 |
11 | setup(
12 | name="self-operating-computer",
13 | version="1.5.8",
14 | packages=find_packages(),
15 | install_requires=required, # Add dependencies here
16 | entry_points={
17 | "console_scripts": [
18 | "operate=operate.main:main_entry",
19 | ],
20 | },
21 | package_data={
22 | # Include the file in the operate.models.weights package
23 | "operate.models.weights": ["best.pt"],
24 | },
25 | long_description=long_description, # Add project description here
26 | long_description_content_type="text/markdown", # Specify Markdown format
27 | # include any other necessary setup options here
28 | )
29 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2023 OthersideAI
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | annotated-types==0.6.0
2 | anyio==3.7.1
3 | certifi==2023.7.22
4 | charset-normalizer==3.3.2
5 | colorama==0.4.6
6 | contourpy==1.2.0
7 | cycler==0.12.1
8 | distro==1.8.0
9 | EasyProcess==1.1
10 | entrypoint2==1.1
11 | exceptiongroup==1.1.3
12 | fonttools==4.44.0
13 | h11==0.14.0
14 | httpcore==1.0.2
15 | httpx>=0.25.2
16 | idna==3.4
17 | importlib-resources==6.1.1
18 | kiwisolver==1.4.5
19 | matplotlib==3.8.1
20 | MouseInfo==0.1.3
21 | mss==9.0.1
22 | numpy==1.26.1
23 | openai==1.2.3
24 | packaging==23.2
25 | Pillow==10.1.0
26 | prompt-toolkit==3.0.39
27 | PyAutoGUI==0.9.54
28 | pydantic==2.4.2
29 | pydantic_core==2.10.1
30 | PyGetWindow==0.0.9
31 | PyMsgBox==1.0.9
32 | pyparsing==3.1.1
33 | pyperclip==1.8.2
34 | PyRect==0.2.0
35 | pyscreenshot==3.1
36 | PyScreeze==0.1.29
37 | python3-xlib==0.15
38 | python-dateutil==2.8.2
39 | python-dotenv==1.0.0
40 | pytweening==1.0.7
41 | requests==2.31.0
42 | rubicon-objc==0.4.7
43 | six==1.16.0
44 | sniffio==1.3.0
45 | tqdm==4.66.1
46 | typing_extensions==4.8.0
47 | urllib3==2.0.7
48 | wcwidth==0.2.9
49 | zipp==3.17.0
50 | google-generativeai==0.3.0
51 | aiohttp==3.9.1
52 | ultralytics==8.0.227
53 | easyocr==1.7.1
54 | ollama==0.1.6
55 | anthropic
--------------------------------------------------------------------------------
/.github/PULL_REQUEST_TEMPLATE.md:
--------------------------------------------------------------------------------
1 | ## What does this PR do?
2 |
3 |
4 |
5 | Fixes # (issue)
6 |
7 | ## Requirement/Documentation
8 |
9 |
10 |
11 | - If there is a requirement document, please, share it here.
12 |
13 | ## Type of change
14 |
15 |
16 |
17 | - [ ] Bug fix (non-breaking change which fixes an issue)
18 | - [ ] Chore (refactoring code, technical debt, workflow improvements)
19 | - [ ] New feature (non-breaking change which adds functionality)
20 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
21 | - [ ] Tests (Unit/Integration/E2E or any other test)
22 | - [ ] This change requires a documentation update
23 |
24 |
25 | ## Mandatory Tasks
26 |
27 | - [ ] Make sure you have self-reviewed the code. A decent size PR without self-review might be rejected. Make sure before submmiting this PR you run tests with evaluate.py
28 |
--------------------------------------------------------------------------------
/operate/utils/style.py:
--------------------------------------------------------------------------------
1 | import sys
2 | import platform
3 | import os
4 | from prompt_toolkit.styles import Style as PromptStyle
5 |
6 |
7 | # Define style
8 | style = PromptStyle.from_dict(
9 | {
10 | "dialog": "bg:#88ff88",
11 | "button": "bg:#ffffff #000000",
12 | "dialog.body": "bg:#44cc44 #ffffff",
13 | "dialog shadow": "bg:#003800",
14 | }
15 | )
16 |
17 |
18 | # Check if on a windows terminal that supports ANSI escape codes
19 | def supports_ansi():
20 | """
21 | Check if the terminal supports ANSI escape codes
22 | """
23 | plat = platform.system()
24 | supported_platform = plat != "Windows" or "ANSICON" in os.environ
25 | is_a_tty = hasattr(sys.stdout, "isatty") and sys.stdout.isatty()
26 | return supported_platform and is_a_tty
27 |
28 |
29 | # Define ANSI color codes
30 | ANSI_GREEN = "\033[32m" if supports_ansi() else "" # Standard green text
31 | ANSI_BRIGHT_GREEN = "\033[92m" if supports_ansi() else "" # Bright/bold green text
32 | ANSI_RESET = "\033[0m" if supports_ansi() else "" # Reset to default text color
33 | ANSI_BLUE = "\033[94m" if supports_ansi() else "" # Bright blue
34 | ANSI_YELLOW = "\033[33m" if supports_ansi() else "" # Standard yellow text
35 | ANSI_RED = "\033[31m" if supports_ansi() else ""
36 | ANSI_BRIGHT_MAGENTA = "\033[95m" if supports_ansi() else "" # Bright magenta text
37 |
--------------------------------------------------------------------------------
/operate/main.py:
--------------------------------------------------------------------------------
1 | """
2 | Self-Operating Computer
3 | """
4 | import argparse
5 | from operate.utils.style import ANSI_BRIGHT_MAGENTA
6 | from operate.operate import main
7 |
8 |
9 | def main_entry():
10 | parser = argparse.ArgumentParser(
11 | description="Run the self-operating-computer with a specified model."
12 | )
13 | parser.add_argument(
14 | "-m",
15 | "--model",
16 | help="Specify the model to use",
17 | required=False,
18 | default="gpt-4-with-ocr",
19 | )
20 |
21 | # Add a voice flag
22 | parser.add_argument(
23 | "--voice",
24 | help="Use voice input mode",
25 | action="store_true",
26 | )
27 |
28 | # Add a flag for verbose mode
29 | parser.add_argument(
30 | "--verbose",
31 | help="Run operate in verbose mode",
32 | action="store_true",
33 | )
34 |
35 | # Allow for direct input of prompt
36 | parser.add_argument(
37 | "--prompt",
38 | help="Directly input the objective prompt",
39 | type=str,
40 | required=False,
41 | )
42 |
43 | try:
44 | args = parser.parse_args()
45 | main(
46 | args.model,
47 | terminal_prompt=args.prompt,
48 | voice_mode=args.voice,
49 | verbose_mode=args.verbose
50 | )
51 | except KeyboardInterrupt:
52 | print(f"\n{ANSI_BRIGHT_MAGENTA}Exiting...")
53 |
54 |
55 | if __name__ == "__main__":
56 | main_entry()
57 |
--------------------------------------------------------------------------------
/operate/utils/misc.py:
--------------------------------------------------------------------------------
1 | import json
2 | import re
3 |
4 |
5 | def convert_percent_to_decimal(percent):
6 | try:
7 | # Remove the '%' sign and convert to float
8 | decimal_value = float(percent)
9 |
10 | # Convert to decimal (e.g., 20% -> 0.20)
11 | return decimal_value
12 | except ValueError as e:
13 | print(f"[convert_percent_to_decimal] error: {e}")
14 | return None
15 |
16 |
17 | def parse_operations(response):
18 | if response == "DONE":
19 | return {"type": "DONE", "data": None}
20 | elif response.startswith("CLICK"):
21 | # Adjust the regex to match the correct format
22 | click_data = re.search(r"CLICK \{ (.+) \}", response).group(1)
23 | click_data_json = json.loads(f"{{{click_data}}}")
24 | return {"type": "CLICK", "data": click_data_json}
25 |
26 | elif response.startswith("TYPE"):
27 | # Extract the text to type
28 | try:
29 | type_data = re.search(r"TYPE (.+)", response, re.DOTALL).group(1)
30 | except:
31 | type_data = re.search(r'TYPE "(.+)"', response, re.DOTALL).group(1)
32 | return {"type": "TYPE", "data": type_data}
33 |
34 | elif response.startswith("SEARCH"):
35 | # Extract the search query
36 | try:
37 | search_data = re.search(r'SEARCH "(.+)"', response).group(1)
38 | except:
39 | search_data = re.search(r"SEARCH (.+)", response).group(1)
40 | return {"type": "SEARCH", "data": search_data}
41 |
42 | return {"type": "UNKNOWN", "data": response}
43 |
--------------------------------------------------------------------------------
/operate/utils/screenshot.py:
--------------------------------------------------------------------------------
1 | import os
2 | import platform
3 | import subprocess
4 | import pyautogui
5 | from PIL import Image, ImageDraw, ImageGrab
6 | import Xlib.display
7 | import Xlib.X
8 | import Xlib.Xutil # not sure if Xutil is necessary
9 |
10 |
11 | def capture_screen_with_cursor(file_path):
12 | user_platform = platform.system()
13 |
14 | if user_platform == "Windows":
15 | screenshot = pyautogui.screenshot()
16 | screenshot.save(file_path)
17 | elif user_platform == "Linux":
18 | # Use xlib to prevent scrot dependency for Linux
19 | screen = Xlib.display.Display().screen()
20 | size = screen.width_in_pixels, screen.height_in_pixels
21 | screenshot = ImageGrab.grab(bbox=(0, 0, size[0], size[1]))
22 | screenshot.save(file_path)
23 | elif user_platform == "Darwin": # (Mac OS)
24 | # Use the screencapture utility to capture the screen with the cursor
25 | subprocess.run(["screencapture", "-C", file_path])
26 | else:
27 | print(f"The platform you're using ({user_platform}) is not currently supported")
28 |
29 |
30 | def compress_screenshot(raw_screenshot_filename, screenshot_filename):
31 | with Image.open(raw_screenshot_filename) as img:
32 | # Check if the image has an alpha channel (transparency)
33 | if img.mode in ('RGBA', 'LA') or (img.mode == 'P' and 'transparency' in img.info):
34 | # Create a white background image
35 | background = Image.new('RGB', img.size, (255, 255, 255))
36 | # Paste the image onto the background, using the alpha channel as mask
37 | background.paste(img, mask=img.split()[3]) # 3 is the alpha channel
38 | # Save the result as JPEG
39 | background.save(screenshot_filename, 'JPEG', quality=85) # Adjust quality as needed
40 | else:
41 | # If no alpha channel, simply convert and save
42 | img.convert('RGB').save(screenshot_filename, 'JPEG', quality=85)
43 |
--------------------------------------------------------------------------------
/operate/utils/operating_system.py:
--------------------------------------------------------------------------------
1 | import pyautogui
2 | import platform
3 | import time
4 | import math
5 |
6 | from operate.utils.misc import convert_percent_to_decimal
7 |
8 |
9 | class OperatingSystem:
10 | def write(self, content):
11 | try:
12 | content = content.replace("\\n", "\n")
13 | for char in content:
14 | pyautogui.write(char)
15 | except Exception as e:
16 | print("[OperatingSystem][write] error:", e)
17 |
18 | def press(self, keys):
19 | try:
20 | for key in keys:
21 | pyautogui.keyDown(key)
22 | time.sleep(0.1)
23 | for key in keys:
24 | pyautogui.keyUp(key)
25 | except Exception as e:
26 | print("[OperatingSystem][press] error:", e)
27 |
28 | def mouse(self, click_detail):
29 | try:
30 | x = convert_percent_to_decimal(click_detail.get("x"))
31 | y = convert_percent_to_decimal(click_detail.get("y"))
32 |
33 | if click_detail and isinstance(x, float) and isinstance(y, float):
34 | self.click_at_percentage(x, y)
35 |
36 | except Exception as e:
37 | print("[OperatingSystem][mouse] error:", e)
38 |
39 | def click_at_percentage(
40 | self,
41 | x_percentage,
42 | y_percentage,
43 | duration=0.2,
44 | circle_radius=50,
45 | circle_duration=0.5,
46 | ):
47 | try:
48 | screen_width, screen_height = pyautogui.size()
49 | x_pixel = int(screen_width * float(x_percentage))
50 | y_pixel = int(screen_height * float(y_percentage))
51 |
52 | pyautogui.moveTo(x_pixel, y_pixel, duration=duration)
53 |
54 | start_time = time.time()
55 | while time.time() - start_time < circle_duration:
56 | angle = ((time.time() - start_time) / circle_duration) * 2 * math.pi
57 | x = x_pixel + math.cos(angle) * circle_radius
58 | y = y_pixel + math.sin(angle) * circle_radius
59 | pyautogui.moveTo(x, y, duration=0.1)
60 |
61 | pyautogui.click(x_pixel, y_pixel)
62 | except Exception as e:
63 | print("[OperatingSystem][click_at_percentage] error:", e)
64 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # Contributing
2 | We appreciate your contributions!
3 |
4 | ## Process
5 | 1. Fork it
6 | 2. Create your feature branch (`git checkout -b my-new-feature`)
7 | 3. Commit your changes (`git commit -am 'Add some feature'`)
8 | 4. Push to the branch (`git push origin my-new-feature`)
9 | 5. Create new Pull Request
10 |
11 | ## Modifying and Running Code
12 | 1. Make changes in `operate/main.py`
13 | 2. Run `pip install .` again
14 | 3. Run `operate` to see your changes
15 |
16 | ## Testing Changes
17 | **After making significant changes, it's important to verify that SOC can still successfully perform a set of common test cases.**
18 | In the root directory of the project, run:
19 | ```
20 | python3 evaluate.py
21 | ```
22 | This will automatically prompt `operate` to perform several simple objectives.
23 | Upon completion of each objective, GPT-4v will give an evaluation and determine if the objective was successfully reached.
24 |
25 | `evaluate.py` will print out if each test case `[PASSED]` or `[FAILED]`. In addition, a justification will be given on why the pass/fail was given.
26 |
27 | It is recommended that a screenshot of the `evaluate.py` output is included in any PR which could impact the performance of SOC.
28 |
29 | ## Contribution Ideas
30 | - **Improve performance by finding optimal screenshot grid**: A primary element of the framework is that it overlays a percentage grid on the screenshot which GPT-4v uses to estimate click locations. If someone is able to find the optimal grid and some evaluation metrics to confirm it is an improvement on the current method then we will merge that PR.
31 | - **Improve the `SUMMARY_PROMPT`**
32 | - **Improve Linux and Windows compatibility**: There are still some issues with Linux and Windows compatibility. PRs to fix the issues are encouraged.
33 | - **Adding New Multimodal Models**: Integration of new multimodal models is welcomed. If you have a specific model in mind that you believe would be a valuable addition, please feel free to integrate it and submit a PR.
34 | - **Iterate `--accurate` flag functionality**: Look at https://github.com/OthersideAI/self-operating-computer/pull/57 for previous iteration
35 | - **Enhanced Security**: A feature request to implement a _robust security feature_ that prompts users for _confirmation before executing potentially harmful actions_. This feature aims to _prevent unintended actions_ and _safeguard user data_ as mentioned here in this [OtherSide#25](https://github.com/OthersideAI/self-operating-computer/issues/25)
36 |
37 |
38 | ## Guidelines
39 | This will primarily be a [Software 2.0](https://karpathy.medium.com/software-2-0-a64152b37c35) project. For this reason:
40 |
41 | - Let's try to hold off refactors into separate files until `main.py` is more than 1000 lines
42 |
43 |
--------------------------------------------------------------------------------
/operate/utils/ocr.py:
--------------------------------------------------------------------------------
1 | from operate.config import Config
2 | from PIL import Image, ImageDraw
3 | import os
4 | from datetime import datetime
5 |
6 | # Load configuration
7 | config = Config()
8 |
9 |
10 | def get_text_element(result, search_text, image_path):
11 | """
12 | Searches for a text element in the OCR results and returns its index. Also draws bounding boxes on the image.
13 | Args:
14 | result (list): The list of results returned by EasyOCR.
15 | search_text (str): The text to search for in the OCR results.
16 | image_path (str): Path to the original image.
17 |
18 | Returns:
19 | int: The index of the element containing the search text.
20 |
21 | Raises:
22 | Exception: If the text element is not found in the results.
23 | """
24 | if config.verbose:
25 | print("[get_text_element]")
26 | print("[get_text_element] search_text", search_text)
27 | # Create /ocr directory if it doesn't exist
28 | ocr_dir = "ocr"
29 | if not os.path.exists(ocr_dir):
30 | os.makedirs(ocr_dir)
31 |
32 | # Open the original image
33 | image = Image.open(image_path)
34 | draw = ImageDraw.Draw(image)
35 |
36 | found_index = None
37 | for index, element in enumerate(result):
38 | text = element[1]
39 | box = element[0]
40 |
41 | if config.verbose:
42 | # Draw bounding box in blue
43 | draw.polygon([tuple(point) for point in box], outline="blue")
44 |
45 | if search_text in text:
46 | found_index = index
47 | if config.verbose:
48 | print("[get_text_element][loop] found search_text, index:", index)
49 |
50 | if found_index is not None:
51 | if config.verbose:
52 | # Draw bounding box of the found text in red
53 | box = result[found_index][0]
54 | draw.polygon([tuple(point) for point in box], outline="red")
55 | # Save the image with bounding boxes
56 | datetime_str = datetime.now().strftime("%Y%m%d_%H%M%S")
57 | ocr_image_path = os.path.join(ocr_dir, f"ocr_image_{datetime_str}.png")
58 | image.save(ocr_image_path)
59 | print("[get_text_element] OCR image saved at:", ocr_image_path)
60 |
61 | return found_index
62 |
63 | raise Exception("The text element was not found in the image")
64 |
65 |
66 | def get_text_coordinates(result, index, image_path):
67 | """
68 | Gets the coordinates of the text element at the specified index as a percentage of screen width and height.
69 | Args:
70 | result (list): The list of results returned by EasyOCR.
71 | index (int): The index of the text element in the results list.
72 | image_path (str): Path to the screenshot image.
73 |
74 | Returns:
75 | dict: A dictionary containing the 'x' and 'y' coordinates as percentages of the screen width and height.
76 | """
77 | if index >= len(result):
78 | raise Exception("Index out of range in OCR results")
79 |
80 | # Get the bounding box of the text element
81 | bounding_box = result[index][0]
82 |
83 | # Calculate the center of the bounding box
84 | min_x = min([coord[0] for coord in bounding_box])
85 | max_x = max([coord[0] for coord in bounding_box])
86 | min_y = min([coord[1] for coord in bounding_box])
87 | max_y = max([coord[1] for coord in bounding_box])
88 |
89 | center_x = (min_x + max_x) / 2
90 | center_y = (min_y + max_y) / 2
91 |
92 | # Get image dimensions
93 | with Image.open(image_path) as img:
94 | width, height = img.size
95 |
96 | # Convert to percentages
97 | percent_x = round((center_x / width), 3)
98 | percent_y = round((center_y / height), 3)
99 |
100 | return {"x": percent_x, "y": percent_y}
101 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | share/python-wheels/
24 | *.egg-info/
25 | .installed.cfg
26 | *.egg
27 | MANIFEST
28 |
29 | # PyInstaller
30 | # Usually these files are written by a python script from a template
31 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
32 | *.manifest
33 | *.spec
34 |
35 | # Installer logs
36 | pip-log.txt
37 | pip-delete-this-directory.txt
38 |
39 | # Unit test / coverage reports
40 | htmlcov/
41 | .tox/
42 | .nox/
43 | .coverage
44 | .coverage.*
45 | .cache
46 | nosetests.xml
47 | coverage.xml
48 | *.cover
49 | *.py,cover
50 | .hypothesis/
51 | .pytest_cache/
52 | cover/
53 |
54 | # Translations
55 | *.mo
56 | *.pot
57 |
58 | # Django stuff:
59 | *.log
60 | local_settings.py
61 | db.sqlite3
62 | db.sqlite3-journal
63 |
64 | # Flask stuff:
65 | instance/
66 | .webassets-cache
67 |
68 | # Scrapy stuff:
69 | .scrapy
70 |
71 | # Sphinx documentation
72 | docs/_build/
73 |
74 | # PyBuilder
75 | .pybuilder/
76 | target/
77 |
78 | # Jupyter Notebook
79 | .ipynb_checkpoints
80 |
81 | # IPython
82 | profile_default/
83 | ipython_config.py
84 |
85 | # pyenv
86 | # For a library or package, you might want to ignore these files since the code is
87 | # intended to run in multiple environments; otherwise, check them in:
88 | # .python-version
89 |
90 | # pipenv
91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
94 | # install all needed dependencies.
95 | #Pipfile.lock
96 |
97 | # poetry
98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99 | # This is especially recommended for binary packages to ensure reproducibility, and is more
100 | # commonly ignored for libraries.
101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102 | #poetry.lock
103 |
104 | # pdm
105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106 | #pdm.lock
107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108 | # in version control.
109 | # https://pdm.fming.dev/#use-with-ide
110 | .pdm.toml
111 |
112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113 | __pypackages__/
114 |
115 | # Celery stuff
116 | celerybeat-schedule
117 | celerybeat.pid
118 |
119 | # SageMath parsed files
120 | *.sage.py
121 |
122 | # Environments
123 | .env
124 | .venv
125 | env/
126 | venv/
127 | ENV/
128 | env.bak/
129 | venv.bak/
130 |
131 | # Spyder project settings
132 | .spyderproject
133 | .spyproject
134 |
135 | # Rope project settings
136 | .ropeproject
137 |
138 | # mkdocs documentation
139 | /site
140 |
141 | # mypy
142 | .mypy_cache/
143 | .dmypy.json
144 | dmypy.json
145 |
146 | # Pyre type checker
147 | .pyre/
148 |
149 | # pytype static type analyzer
150 | .pytype/
151 |
152 | # Cython debug symbols
153 | cython_debug/
154 |
155 | # PyCharm
156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158 | # and can be added to the global gitignore or merged into this file. For a more nuclear
159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder.
160 | #.idea/
161 |
162 | .DS_Store
163 |
164 | # Avoid sending testing screenshots up
165 | *.png
166 | operate/screenshots/
167 |
--------------------------------------------------------------------------------
/evaluate.py:
--------------------------------------------------------------------------------
1 | import sys
2 | import os
3 | import subprocess
4 | import platform
5 | import base64
6 | import json
7 | import openai
8 | import argparse
9 |
10 | from dotenv import load_dotenv
11 |
12 | # "Objective for `operate`" : "Guideline for passing this test case given to GPT-4v"
13 | TEST_CASES = {
14 | "Go to Github.com": "A Github page is visible.",
15 | "Go to Youtube.com and play a video": "The YouTube video player is visible.",
16 | }
17 |
18 | EVALUATION_PROMPT = """
19 | Your job is to look at the given screenshot and determine if the following guideline is met in the image.
20 | You must respond in the following format ONLY. Do not add anything else:
21 | {{ "guideline_met": (true|false), "reason": "Explanation for why guideline was or wasn't met" }}
22 | guideline_met must be set to a JSON boolean. True if the image meets the given guideline.
23 | reason must be a string containing a justification for your decision.
24 |
25 | Guideline: {guideline}
26 | """
27 |
28 | SCREENSHOT_PATH = os.path.join("screenshots", "screenshot.png")
29 |
30 |
31 | # Check if on a windows terminal that supports ANSI escape codes
32 | def supports_ansi():
33 | """
34 | Check if the terminal supports ANSI escape codes
35 | """
36 | plat = platform.system()
37 | supported_platform = plat != "Windows" or "ANSICON" in os.environ
38 | is_a_tty = hasattr(sys.stdout, "isatty") and sys.stdout.isatty()
39 | return supported_platform and is_a_tty
40 |
41 |
42 | if supports_ansi():
43 | # Standard green text
44 | ANSI_GREEN = "\033[32m"
45 | # Bright/bold green text
46 | ANSI_BRIGHT_GREEN = "\033[92m"
47 | # Reset to default text color
48 | ANSI_RESET = "\033[0m"
49 | # ANSI escape code for blue text
50 | ANSI_BLUE = "\033[94m" # This is for bright blue
51 |
52 | # Standard yellow text
53 | ANSI_YELLOW = "\033[33m"
54 |
55 | ANSI_RED = "\033[31m"
56 |
57 | # Bright magenta text
58 | ANSI_BRIGHT_MAGENTA = "\033[95m"
59 | else:
60 | ANSI_GREEN = ""
61 | ANSI_BRIGHT_GREEN = ""
62 | ANSI_RESET = ""
63 | ANSI_BLUE = ""
64 | ANSI_YELLOW = ""
65 | ANSI_RED = ""
66 | ANSI_BRIGHT_MAGENTA = ""
67 |
68 |
69 | def format_evaluation_prompt(guideline):
70 | prompt = EVALUATION_PROMPT.format(guideline=guideline)
71 | return prompt
72 |
73 |
74 | def parse_eval_content(content):
75 | try:
76 | res = json.loads(content)
77 |
78 | print(res["reason"])
79 |
80 | return res["guideline_met"]
81 | except:
82 | print(
83 | "The model gave a bad evaluation response and it couldn't be parsed. Exiting..."
84 | )
85 | exit(1)
86 |
87 |
88 | def evaluate_final_screenshot(guideline):
89 | """Load the final screenshot and return True or False if it meets the given guideline."""
90 | with open(SCREENSHOT_PATH, "rb") as img_file:
91 | img_base64 = base64.b64encode(img_file.read()).decode("utf-8")
92 |
93 | eval_message = [
94 | {
95 | "role": "user",
96 | "content": [
97 | {"type": "text", "text": format_evaluation_prompt(guideline)},
98 | {
99 | "type": "image_url",
100 | "image_url": {"url": f"data:image/jpeg;base64,{img_base64}"},
101 | },
102 | ],
103 | }
104 | ]
105 |
106 | response = openai.chat.completions.create(
107 | model="gpt-4o",
108 | messages=eval_message,
109 | presence_penalty=1,
110 | frequency_penalty=1,
111 | temperature=0.7,
112 | )
113 |
114 | eval_content = response.choices[0].message.content
115 |
116 | return parse_eval_content(eval_content)
117 |
118 |
119 | def run_test_case(objective, guideline, model):
120 | """Returns True if the result of the test with the given prompt meets the given guideline for the given model."""
121 | # Run `operate` with the model to evaluate and the test case prompt
122 | subprocess.run(
123 | ["operate", "-m", model, "--prompt", f'"{objective}"'],
124 | stdout=subprocess.DEVNULL,
125 | )
126 |
127 | try:
128 | result = evaluate_final_screenshot(guideline)
129 | except OSError:
130 | print("[Error] Couldn't open the screenshot for evaluation")
131 | return False
132 |
133 | return result
134 |
135 |
136 | def get_test_model():
137 | parser = argparse.ArgumentParser(
138 | description="Run the self-operating-computer with a specified model."
139 | )
140 |
141 | parser.add_argument(
142 | "-m",
143 | "--model",
144 | help="Specify the model to evaluate.",
145 | required=False,
146 | default="gpt-4-with-ocr",
147 | )
148 |
149 | return parser.parse_args().model
150 |
151 |
152 | def main():
153 | load_dotenv()
154 | openai.api_key = os.getenv("OPENAI_API_KEY")
155 |
156 | model = get_test_model()
157 |
158 | print(f"{ANSI_BLUE}[EVALUATING MODEL `{model}`]{ANSI_RESET}")
159 | print(f"{ANSI_BRIGHT_MAGENTA}[STARTING EVALUATION]{ANSI_RESET}")
160 |
161 | passed = 0
162 | failed = 0
163 | for objective, guideline in TEST_CASES.items():
164 | print(f"{ANSI_BLUE}[EVALUATING]{ANSI_RESET} '{objective}'")
165 |
166 | result = run_test_case(objective, guideline, model)
167 | if result:
168 | print(f"{ANSI_GREEN}[PASSED]{ANSI_RESET} '{objective}'")
169 | passed += 1
170 | else:
171 | print(f"{ANSI_RED}[FAILED]{ANSI_RESET} '{objective}'")
172 | failed += 1
173 |
174 | print(
175 | f"{ANSI_BRIGHT_MAGENTA}[EVALUATION COMPLETE]{ANSI_RESET} {passed} test{'' if passed == 1 else 's'} passed, {failed} test{'' if failed == 1 else 's'} failed"
176 | )
177 |
178 |
179 | if __name__ == "__main__":
180 | main()
181 |
--------------------------------------------------------------------------------
/operate/utils/label.py:
--------------------------------------------------------------------------------
1 | import io
2 | import base64
3 | import json
4 | import os
5 | import time
6 | import asyncio
7 | from PIL import Image, ImageDraw
8 |
9 |
10 | def validate_and_extract_image_data(data):
11 | if not data or "messages" not in data:
12 | raise ValueError("Invalid request, no messages found")
13 |
14 | messages = data["messages"]
15 | if (
16 | not messages
17 | or not isinstance(messages, list)
18 | or not messages[-1].get("image_url")
19 | ):
20 | raise ValueError("No image provided or incorrect format")
21 |
22 | image_data = messages[-1]["image_url"]["url"]
23 | if not image_data.startswith("data:image"):
24 | raise ValueError("Invalid image format")
25 |
26 | return image_data.split("base64,")[-1], messages
27 |
28 |
29 | def get_label_coordinates(label, label_coordinates):
30 | """
31 | Retrieves the coordinates for a given label.
32 |
33 | :param label: The label to find coordinates for (e.g., "~1").
34 | :param label_coordinates: Dictionary containing labels and their coordinates.
35 | :return: Coordinates of the label or None if the label is not found.
36 | """
37 | return label_coordinates.get(label)
38 |
39 |
40 | def is_overlapping(box1, box2):
41 | x1_box1, y1_box1, x2_box1, y2_box1 = box1
42 | x1_box2, y1_box2, x2_box2, y2_box2 = box2
43 |
44 | # Check if there is no overlap
45 | if x1_box1 > x2_box2 or x1_box2 > x2_box1:
46 | return False
47 | if (
48 | y1_box1 > y2_box2 or y1_box2 > y2_box1
49 | ): # Adjusted to check 100px proximity above
50 | return False
51 |
52 | return True
53 |
54 |
55 | def add_labels(base64_data, yolo_model):
56 | image_bytes = base64.b64decode(base64_data)
57 | image_labeled = Image.open(io.BytesIO(image_bytes)) # Corrected this line
58 | image_debug = image_labeled.copy() # Create a copy for the debug image
59 | image_original = (
60 | image_labeled.copy()
61 | ) # Copy of the original image for base64 return
62 |
63 | results = yolo_model(image_labeled)
64 |
65 | draw = ImageDraw.Draw(image_labeled)
66 | debug_draw = ImageDraw.Draw(
67 | image_debug
68 | ) # Create a separate draw object for the debug image
69 | font_size = 45
70 |
71 | labeled_images_dir = "labeled_images"
72 | label_coordinates = {} # Dictionary to store coordinates
73 |
74 | if not os.path.exists(labeled_images_dir):
75 | os.makedirs(labeled_images_dir)
76 |
77 | counter = 0
78 | drawn_boxes = [] # List to keep track of boxes already drawn
79 | for result in results:
80 | if hasattr(result, "boxes"):
81 | for det in result.boxes:
82 | bbox = det.xyxy[0]
83 | x1, y1, x2, y2 = bbox.tolist()
84 |
85 | debug_label = "D_" + str(counter)
86 | debug_index_position = (x1, y1 - font_size)
87 | debug_draw.rectangle([(x1, y1), (x2, y2)], outline="blue", width=1)
88 | debug_draw.text(
89 | debug_index_position,
90 | debug_label,
91 | fill="blue",
92 | font_size=font_size,
93 | )
94 |
95 | overlap = any(
96 | is_overlapping((x1, y1, x2, y2), box) for box in drawn_boxes
97 | )
98 |
99 | if not overlap:
100 | draw.rectangle([(x1, y1), (x2, y2)], outline="red", width=1)
101 | label = "~" + str(counter)
102 | index_position = (x1, y1 - font_size)
103 | draw.text(
104 | index_position,
105 | label,
106 | fill="red",
107 | font_size=font_size,
108 | )
109 |
110 | # Add the non-overlapping box to the drawn_boxes list
111 | drawn_boxes.append((x1, y1, x2, y2))
112 | label_coordinates[label] = (x1, y1, x2, y2)
113 |
114 | counter += 1
115 |
116 | # Save the image
117 | timestamp = time.strftime("%Y%m%d-%H%M%S")
118 |
119 | output_path = os.path.join(labeled_images_dir, f"img_{timestamp}_labeled.png")
120 | output_path_debug = os.path.join(labeled_images_dir, f"img_{timestamp}_debug.png")
121 | output_path_original = os.path.join(
122 | labeled_images_dir, f"img_{timestamp}_original.png"
123 | )
124 |
125 | image_labeled.save(output_path)
126 | image_debug.save(output_path_debug)
127 | image_original.save(output_path_original)
128 |
129 | buffered_original = io.BytesIO()
130 | image_original.save(buffered_original, format="PNG") # I guess this is needed
131 | img_base64_original = base64.b64encode(buffered_original.getvalue()).decode("utf-8")
132 |
133 | # Convert image to base64 for return
134 | buffered_labeled = io.BytesIO()
135 | image_labeled.save(buffered_labeled, format="PNG") # I guess this is needed
136 | img_base64_labeled = base64.b64encode(buffered_labeled.getvalue()).decode("utf-8")
137 |
138 | return img_base64_labeled, label_coordinates
139 |
140 |
141 | def get_click_position_in_percent(coordinates, image_size):
142 | """
143 | Calculates the click position at the center of the bounding box and converts it to percentages.
144 |
145 | :param coordinates: A tuple of the bounding box coordinates (x1, y1, x2, y2).
146 | :param image_size: A tuple of the image dimensions (width, height).
147 | :return: A tuple of the click position in percentages (x_percent, y_percent).
148 | """
149 | if not coordinates or not image_size:
150 | return None
151 |
152 | # Calculate the center of the bounding box
153 | x_center = (coordinates[0] + coordinates[2]) / 2
154 | y_center = (coordinates[1] + coordinates[3]) / 2
155 |
156 | # Convert to percentages
157 | x_percent = x_center / image_size[0]
158 | y_percent = y_center / image_size[1]
159 |
160 | return x_percent, y_percent
161 |
--------------------------------------------------------------------------------
/operate/operate.py:
--------------------------------------------------------------------------------
1 | import sys
2 | import os
3 | import time
4 | import asyncio
5 | from prompt_toolkit.shortcuts import message_dialog
6 | from prompt_toolkit import prompt
7 | from operate.exceptions import ModelNotRecognizedException
8 | import platform
9 |
10 | # from operate.models.prompts import USER_QUESTION, get_system_prompt
11 | from operate.models.prompts import (
12 | USER_QUESTION,
13 | get_system_prompt,
14 | )
15 | from operate.config import Config
16 | from operate.utils.style import (
17 | ANSI_GREEN,
18 | ANSI_RESET,
19 | ANSI_YELLOW,
20 | ANSI_RED,
21 | ANSI_BRIGHT_MAGENTA,
22 | ANSI_BLUE,
23 | style,
24 | )
25 | from operate.utils.operating_system import OperatingSystem
26 | from operate.models.apis import get_next_action
27 |
28 | # Load configuration
29 | config = Config()
30 | operating_system = OperatingSystem()
31 |
32 |
33 | def main(model, terminal_prompt, voice_mode=False, verbose_mode=False):
34 | """
35 | Main function for the Self-Operating Computer.
36 |
37 | Parameters:
38 | - model: The model used for generating responses.
39 | - terminal_prompt: A string representing the prompt provided in the terminal.
40 | - voice_mode: A boolean indicating whether to enable voice mode.
41 |
42 | Returns:
43 | None
44 | """
45 |
46 | mic = None
47 | # Initialize `WhisperMic`, if `voice_mode` is True
48 |
49 | config.verbose = verbose_mode
50 | config.validation(model, voice_mode)
51 |
52 | if voice_mode:
53 | try:
54 | from whisper_mic import WhisperMic
55 |
56 | # Initialize WhisperMic if import is successful
57 | mic = WhisperMic()
58 | except ImportError:
59 | print(
60 | "Voice mode requires the 'whisper_mic' module. Please install it using 'pip install -r requirements-audio.txt'"
61 | )
62 | sys.exit(1)
63 |
64 | # Skip message dialog if prompt was given directly
65 | if not terminal_prompt:
66 | message_dialog(
67 | title="Self-Operating Computer",
68 | text="An experimental framework to enable multimodal models to operate computers",
69 | style=style,
70 | ).run()
71 |
72 | else:
73 | print("Running direct prompt...")
74 |
75 | # # Clear the console
76 | if platform.system() == "Windows":
77 | os.system("cls")
78 | else:
79 | print("\033c", end="")
80 |
81 | if terminal_prompt: # Skip objective prompt if it was given as an argument
82 | objective = terminal_prompt
83 | elif voice_mode:
84 | print(
85 | f"{ANSI_GREEN}[Self-Operating Computer]{ANSI_RESET} Listening for your command... (speak now)"
86 | )
87 | try:
88 | objective = mic.listen()
89 | except Exception as e:
90 | print(f"{ANSI_RED}Error in capturing voice input: {e}{ANSI_RESET}")
91 | return # Exit if voice input fails
92 | else:
93 | print(
94 | f"[{ANSI_GREEN}Self-Operating Computer {ANSI_RESET}|{ANSI_BRIGHT_MAGENTA} {model}{ANSI_RESET}]\n{USER_QUESTION}"
95 | )
96 | print(f"{ANSI_YELLOW}[User]{ANSI_RESET}")
97 | objective = prompt(style=style)
98 |
99 | system_prompt = get_system_prompt(model, objective)
100 | system_message = {"role": "system", "content": system_prompt}
101 | messages = [system_message]
102 |
103 | loop_count = 0
104 |
105 | session_id = None
106 |
107 | while True:
108 | if config.verbose:
109 | print("[Self Operating Computer] loop_count", loop_count)
110 | try:
111 | operations, session_id = asyncio.run(
112 | get_next_action(model, messages, objective, session_id)
113 | )
114 |
115 | stop = operate(operations, model)
116 | if stop:
117 | break
118 |
119 | loop_count += 1
120 | if loop_count > 10:
121 | break
122 | except ModelNotRecognizedException as e:
123 | print(
124 | f"{ANSI_GREEN}[Self-Operating Computer]{ANSI_RED}[Error] -> {e} {ANSI_RESET}"
125 | )
126 | break
127 | except Exception as e:
128 | print(
129 | f"{ANSI_GREEN}[Self-Operating Computer]{ANSI_RED}[Error] -> {e} {ANSI_RESET}"
130 | )
131 | break
132 |
133 |
134 | def operate(operations, model):
135 | if config.verbose:
136 | print("[Self Operating Computer][operate]")
137 | for operation in operations:
138 | if config.verbose:
139 | print("[Self Operating Computer][operate] operation", operation)
140 | # wait one second
141 | time.sleep(1)
142 | operate_type = operation.get("operation").lower()
143 | operate_thought = operation.get("thought")
144 | operate_detail = ""
145 | if config.verbose:
146 | print("[Self Operating Computer][operate] operate_type", operate_type)
147 |
148 | if operate_type == "press" or operate_type == "hotkey":
149 | keys = operation.get("keys")
150 | operate_detail = keys
151 | operating_system.press(keys)
152 | elif operate_type == "write":
153 | content = operation.get("content")
154 | operate_detail = content
155 | operating_system.write(content)
156 | elif operate_type == "click":
157 | x = operation.get("x")
158 | y = operation.get("y")
159 | click_detail = {"x": x, "y": y}
160 | operate_detail = click_detail
161 |
162 | operating_system.mouse(click_detail)
163 | elif operate_type == "done":
164 | summary = operation.get("summary")
165 |
166 | print(
167 | f"[{ANSI_GREEN}Self-Operating Computer {ANSI_RESET}|{ANSI_BRIGHT_MAGENTA} {model}{ANSI_RESET}]"
168 | )
169 | print(f"{ANSI_BLUE}Objective Complete: {ANSI_RESET}{summary}\n")
170 | return True
171 |
172 | else:
173 | print(
174 | f"{ANSI_GREEN}[Self-Operating Computer]{ANSI_RED}[Error] unknown operation response :({ANSI_RESET}"
175 | )
176 | print(
177 | f"{ANSI_GREEN}[Self-Operating Computer]{ANSI_RED}[Error] AI response {ANSI_RESET}{operation}"
178 | )
179 | return True
180 |
181 | print(
182 | f"[{ANSI_GREEN}Self-Operating Computer {ANSI_RESET}|{ANSI_BRIGHT_MAGENTA} {model}{ANSI_RESET}]"
183 | )
184 | print(f"{operate_thought}")
185 | print(f"{ANSI_BLUE}Action: {ANSI_RESET}{operate_type} {operate_detail}\n")
186 |
187 | return False
188 |
--------------------------------------------------------------------------------
/operate/config.py:
--------------------------------------------------------------------------------
1 | import os
2 | import sys
3 |
4 | import google.generativeai as genai
5 | from dotenv import load_dotenv
6 | from ollama import Client
7 | from openai import OpenAI
8 | import anthropic
9 | from prompt_toolkit.shortcuts import input_dialog
10 |
11 |
12 | class Config:
13 | """
14 | Configuration class for managing settings.
15 |
16 | Attributes:
17 | verbose (bool): Flag indicating whether verbose mode is enabled.
18 | openai_api_key (str): API key for OpenAI.
19 | google_api_key (str): API key for Google.
20 | ollama_host (str): url to ollama running remotely.
21 | """
22 |
23 | _instance = None
24 |
25 | def __new__(cls):
26 | if cls._instance is None:
27 | cls._instance = super(Config, cls).__new__(cls)
28 | # Put any initialization here
29 | return cls._instance
30 |
31 | def __init__(self):
32 | load_dotenv()
33 | self.verbose = False
34 | self.openai_api_key = (
35 | None # instance variables are backups in case saving to a `.env` fails
36 | )
37 | self.google_api_key = (
38 | None # instance variables are backups in case saving to a `.env` fails
39 | )
40 | self.ollama_host = (
41 | None # instance variables are backups in case savint to a `.env` fails
42 | )
43 | self.anthropic_api_key = (
44 | None # instance variables are backups in case saving to a `.env` fails
45 | )
46 | self.qwen_api_key = (
47 | None # instance variables are backups in case saving to a `.env` fails
48 | )
49 |
50 | def initialize_openai(self):
51 | if self.verbose:
52 | print("[Config][initialize_openai]")
53 |
54 | if self.openai_api_key:
55 | if self.verbose:
56 | print("[Config][initialize_openai] using cached openai_api_key")
57 | api_key = self.openai_api_key
58 | else:
59 | if self.verbose:
60 | print(
61 | "[Config][initialize_openai] no cached openai_api_key, try to get from env."
62 | )
63 | api_key = os.getenv("OPENAI_API_KEY")
64 |
65 | client = OpenAI(
66 | api_key=api_key,
67 | )
68 | client.api_key = api_key
69 | client.base_url = os.getenv("OPENAI_API_BASE_URL", client.base_url)
70 | return client
71 |
72 | def initialize_qwen(self):
73 | if self.verbose:
74 | print("[Config][initialize_qwen]")
75 |
76 | if self.qwen_api_key:
77 | if self.verbose:
78 | print("[Config][initialize_qwen] using cached qwen_api_key")
79 | api_key = self.qwen_api_key
80 | else:
81 | if self.verbose:
82 | print(
83 | "[Config][initialize_qwen] no cached qwen_api_key, try to get from env."
84 | )
85 | api_key = os.getenv("QWEN_API_KEY")
86 |
87 | client = OpenAI(
88 | api_key=api_key,
89 | base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
90 | )
91 | client.api_key = api_key
92 | client.base_url = "https://dashscope.aliyuncs.com/compatible-mode/v1"
93 | return client
94 |
95 | def initialize_google(self):
96 | if self.google_api_key:
97 | if self.verbose:
98 | print("[Config][initialize_google] using cached google_api_key")
99 | api_key = self.google_api_key
100 | else:
101 | if self.verbose:
102 | print(
103 | "[Config][initialize_google] no cached google_api_key, try to get from env."
104 | )
105 | api_key = os.getenv("GOOGLE_API_KEY")
106 | genai.configure(api_key=api_key, transport="rest")
107 | model = genai.GenerativeModel("gemini-pro-vision")
108 |
109 | return model
110 |
111 | def initialize_ollama(self):
112 | if self.ollama_host:
113 | if self.verbose:
114 | print("[Config][initialize_ollama] using cached ollama host")
115 | else:
116 | if self.verbose:
117 | print(
118 | "[Config][initialize_ollama] no cached ollama host. Assuming ollama running locally."
119 | )
120 | self.ollama_host = os.getenv("OLLAMA_HOST", None)
121 | model = Client(host=self.ollama_host)
122 | return model
123 |
124 | def initialize_anthropic(self):
125 | if self.anthropic_api_key:
126 | api_key = self.anthropic_api_key
127 | else:
128 | api_key = os.getenv("ANTHROPIC_API_KEY")
129 | return anthropic.Anthropic(api_key=api_key)
130 |
131 | def validation(self, model, voice_mode):
132 | """
133 | Validate the input parameters for the dialog operation.
134 | """
135 | self.require_api_key(
136 | "OPENAI_API_KEY",
137 | "OpenAI API key",
138 | model == "gpt-4"
139 | or voice_mode
140 | or model == "gpt-4-with-som"
141 | or model == "gpt-4-with-ocr"
142 | or model == "gpt-4.1-with-ocr"
143 | or model == "o1-with-ocr",
144 | )
145 | self.require_api_key(
146 | "GOOGLE_API_KEY", "Google API key", model == "gemini-pro-vision"
147 | )
148 | self.require_api_key(
149 | "ANTHROPIC_API_KEY", "Anthropic API key", model == "claude-3"
150 | )
151 | self.require_api_key("QWEN_API_KEY", "Qwen API key", model == "qwen-vl")
152 |
153 | def require_api_key(self, key_name, key_description, is_required):
154 | key_exists = bool(os.environ.get(key_name))
155 | if self.verbose:
156 | print("[Config] require_api_key")
157 | print("[Config] key_name", key_name)
158 | print("[Config] key_description", key_description)
159 | print("[Config] key_exists", key_exists)
160 | if is_required and not key_exists:
161 | self.prompt_and_save_api_key(key_name, key_description)
162 |
163 | def prompt_and_save_api_key(self, key_name, key_description):
164 | key_value = input_dialog(
165 | title="API Key Required", text=f"Please enter your {key_description}:"
166 | ).run()
167 |
168 | if key_value is None: # User pressed cancel or closed the dialog
169 | sys.exit("Operation cancelled by user.")
170 |
171 | if key_value:
172 | if key_name == "OPENAI_API_KEY":
173 | self.openai_api_key = key_value
174 | elif key_name == "GOOGLE_API_KEY":
175 | self.google_api_key = key_value
176 | elif key_name == "ANTHROPIC_API_KEY":
177 | self.anthropic_api_key = key_value
178 | elif key_name == "QWEN_API_KEY":
179 | self.qwen_api_key = key_value
180 | self.save_api_key_to_env(key_name, key_value)
181 | load_dotenv() # Reload environment variables
182 | # Update the instance attribute with the new key
183 |
184 | @staticmethod
185 | def save_api_key_to_env(key_name, key_value):
186 | with open(".env", "a") as file:
187 | file.write(f"\n{key_name}='{key_value}'")
188 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | ome
2 | Self-Operating Computer Framework
3 |
4 |
5 | A framework to enable multimodal models to operate a computer.
6 |
7 |
8 | Using the same inputs and outputs as a human operator, the model views the screen and decides on a series of mouse and keyboard actions to reach an objective. Released Nov 2023, the Self-Operating Computer Framework was one of the first examples of full computer-use.
9 |
10 |
11 |
12 |
13 |
43 |
44 |
49 |
50 |
51 |