├── requirements.txt
├── scripts
    ├── tests.sh
    ├── test_name.sh
    ├── code_quality.sh
    └── merge_all_prs.sh
├── autort.png
├── agorabanner.png
├── .github
    ├── workflows
    │   ├── ruff.yml
    │   ├── pull-request-links.yml
    │   ├── docs.yml
    │   ├── lints.yml
    │   ├── testing.yml
    │   ├── run_test.yml
    │   ├── quality.yml
    │   ├── pr_request_checks.yml
    │   ├── label.yml
    │   ├── docs_test.yml
    │   ├── pylint.yml
    │   ├── welcome.yml
    │   ├── unit-test.yml
    │   ├── python-publish.yml
    │   ├── code_quality_control.yml
    │   ├── stale.yml
    │   ├── cos_integration.yml
    │   └── test.yml
    ├── dependabot.yml
    ├── labeler.yml
    ├── ISSUE_TEMPLATE
    │   ├── feature_request.md
    │   └── bug_report.md
    ├── FUNDING.yml
    └── PULL_REQUEST_TEMPLATE.yml
├── .readthedocs.yml
├── autort
    ├── __init__.py
    ├── prompts.py
    └── main.py
├── .pre-commit-config.yaml
├── example.py
├── LICENSE
├── pyproject.toml
├── README.md
├── errors.txt
└── .gitignore


/requirements.txt:
--------------------------------------------------------------------------------
1 | torch
2 | zetascale
3 | swarms
4 | 


--------------------------------------------------------------------------------
/scripts/tests.sh:
--------------------------------------------------------------------------------
1 | find ./tests -name '*.py' -exec pytest {} \;


--------------------------------------------------------------------------------
/autort.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/kyegomez/AutoRT/HEAD/autort.png


--------------------------------------------------------------------------------
/agorabanner.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/kyegomez/AutoRT/HEAD/agorabanner.png


--------------------------------------------------------------------------------
/.github/workflows/ruff.yml:
--------------------------------------------------------------------------------
1 | name: Ruff
2 | on: [ push, pull_request ]
3 | jobs:
4 |   ruff:
5 |     runs-on: ubuntu-latest
6 |     steps:
7 |       - uses: actions/checkout@v4
8 |       - uses: chartboost/ruff-action@v1
9 | 


--------------------------------------------------------------------------------
/.readthedocs.yml:
--------------------------------------------------------------------------------
 1 | version: 2
 2 | 
 3 | build:
 4 |   os: ubuntu-22.04
 5 |   tools:
 6 |     python: "3.11"
 7 | 
 8 | mkdocs:
 9 |   configuration: mkdocs.yml
10 | 
11 | python:
12 |    install:
13 |    - requirements: requirements.txt


--------------------------------------------------------------------------------
/scripts/test_name.sh:
--------------------------------------------------------------------------------
1 | find ./tests -name "*.py" -type f | while read file
2 | do
3 |   filename=$(basename "$file")
4 |   dir=$(dirname "$file")
5 |   if [[ $filename != test_* ]]; then
6 |     mv "$file" "$dir/test_$filename"
7 |   fi
8 | done


--------------------------------------------------------------------------------
/.github/workflows/pull-request-links.yml:
--------------------------------------------------------------------------------
 1 | name: readthedocs/actions
 2 | on:
 3 |   pull_request_target:
 4 |     types:
 5 |       - opened
 6 |     paths:
 7 |       - "docs/**"
 8 | 
 9 | permissions:
10 |   pull-requests: write
11 | 
12 | jobs:
13 |   pull-request-links:
14 |     runs-on: ubuntu-latest
15 |     steps:
16 |       - uses: readthedocs/actions/preview@v1
17 |         with:
18 |           project-slug: swarms_torch


--------------------------------------------------------------------------------
/.github/dependabot.yml:
--------------------------------------------------------------------------------
 1 | # https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/configuration-options-for-dependency-updates
 2 | 
 3 | version: 2
 4 | updates:
 5 |   - package-ecosystem: "github-actions"
 6 |     directory: "/"
 7 |     schedule:
 8 |       interval: "weekly"
 9 | 
10 |   - package-ecosystem: "pip"
11 |     directory: "/"
12 |     schedule:
13 |       interval: "weekly"
14 | 
15 | 


--------------------------------------------------------------------------------
/.github/labeler.yml:
--------------------------------------------------------------------------------
 1 | # this is a config file for the github action labeler
 2 | 
 3 | # Add 'label1' to any changes within 'example' folder or any subfolders
 4 | example_change:
 5 | - example/**
 6 | 
 7 | # Add 'label2' to any file changes within 'example2' folder
 8 | example2_change: example2/*
 9 | 
10 | # Add label3 to any change to .txt files within the entire repository. Quotation marks are required for the leading asterisk
11 | text_files:
12 | - '**/*.txt'
13 | 


--------------------------------------------------------------------------------
/autort/__init__.py:
--------------------------------------------------------------------------------
 1 | from autort.main import AutoRTAgent, AutoRTSwarm
 2 | 
 3 | from autort.prompts import (
 4 |     VISUALIZE_OBJECT_PROMPT,
 5 |     GENERATE_TASKS_PROMPT,
 6 |     FUSED_SYSTEM_PROMPT_WITH_SOP,
 7 |     FILTER_TASKS_SOP_PROMPT,
 8 | )
 9 | 
10 | __all__ = [
11 |     "AutoRTAgent",
12 |     "AutoRTSwarm",
13 |     "VISUALIZE_OBJECT_PROMPT",
14 |     "GENERATE_TASKS_PROMPT",
15 |     "FUSED_SYSTEM_PROMPT_WITH_SOP",
16 |     "FILTER_TASKS_SOP_PROMPT",
17 | ]
18 | 


--------------------------------------------------------------------------------
/.github/workflows/docs.yml:
--------------------------------------------------------------------------------
 1 | name: Docs WorkFlow
 2 | 
 3 | on:
 4 |   push:
 5 |     branches:
 6 |       - master
 7 |       - main
 8 |       - develop
 9 | jobs:
10 |   deploy:
11 |     runs-on: ubuntu-latest
12 |     steps:
13 |       - uses: actions/checkout@v4
14 |       - uses: actions/setup-python@v5
15 |         with:
16 |           python-version: 3.x
17 |       - run: pip install mkdocs-material
18 |       - run: pip install "mkdocstrings[python]"
19 |       - run: mkdocs gh-deploy --force


--------------------------------------------------------------------------------
/.github/workflows/lints.yml:
--------------------------------------------------------------------------------
 1 | name: Linting
 2 | 
 3 | on:
 4 |   push:
 5 |     branches:
 6 |       - master
 7 | 
 8 | jobs:
 9 |   lint:
10 |     runs-on: ubuntu-latest
11 | 
12 |     steps:
13 |       - name: Checkout code
14 |         uses: actions/checkout@v4
15 | 
16 |       - name: Set up Python
17 |         uses: actions/setup-python@v5
18 |         with:
19 |           python-version: 3.x
20 | 
21 |       - name: Install dependencies
22 |         run: pip install -r requirements.txt
23 | 
24 |       - name: Run linters
25 |         run: pylint swarms_torch


--------------------------------------------------------------------------------
/.github/workflows/testing.yml:
--------------------------------------------------------------------------------
 1 | name: Unit Tests
 2 | 
 3 | on:
 4 |   push:
 5 |     branches:
 6 |       - master
 7 | 
 8 | jobs:
 9 |   test:
10 |     runs-on: ubuntu-latest
11 | 
12 |     steps:
13 |       - name: Checkout code
14 |         uses: actions/checkout@v4
15 | 
16 |       - name: Set up Python
17 |         uses: actions/setup-python@v5
18 |         with:
19 |           python-version: 3.x
20 | 
21 |       - name: Install dependencies
22 |         run: pip install -r requirements.txt
23 | 
24 |       - name: Run unit tests
25 |         run: pytest tests/


--------------------------------------------------------------------------------
/.pre-commit-config.yaml:
--------------------------------------------------------------------------------
 1 | repos:
 2 |   - repo: https://github.com/ambv/black
 3 |     rev: 22.3.0
 4 |     hooks:
 5 |     - id: black
 6 |   - repo: https://github.com/charliermarsh/ruff-pre-commit
 7 |     rev: 'v0.0.255'
 8 |     hooks:
 9 |       - id: ruff
10 |         args: [--fix]
11 |   - repo: https://github.com/nbQA-dev/nbQA
12 |     rev: 1.6.3
13 |     hooks:
14 |     - id: nbqa-black
15 |       additional_dependencies: [ipython==8.12, black]
16 |     - id: nbqa-ruff 
17 |       args: ["--ignore=I001"]
18 |       additional_dependencies: [ipython==8.12, ruff]


--------------------------------------------------------------------------------
/.github/workflows/run_test.yml:
--------------------------------------------------------------------------------
 1 | name: Python application test
 2 | 
 3 | on: [push]
 4 | 
 5 | jobs:
 6 |   build:
 7 | 
 8 |     runs-on: ubuntu-latest
 9 | 
10 |     steps:
11 |     - uses: actions/checkout@v4
12 |     - name: Set up Python 3.8
13 |       uses: actions/setup-python@v5
14 |       with:
15 |         python-version: 3.8
16 |     - name: Install dependencies
17 |       run: |
18 |         python -m pip install --upgrade pip
19 |         pip install pytest
20 |         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
21 |     - name: Run tests with pytest
22 |       run: |
23 |         pytest tests/
24 | 


--------------------------------------------------------------------------------
/.github/workflows/quality.yml:
--------------------------------------------------------------------------------
 1 | name: Quality
 2 | 
 3 | on:
 4 |   push:
 5 |     branches: [ "main" ]
 6 |   pull_request:
 7 |     branches: [ "main" ]
 8 | 
 9 | jobs:
10 |   lint:
11 |     runs-on: ubuntu-latest
12 |     strategy:
13 |       fail-fast: false
14 |     steps:
15 |       - name: Checkout actions
16 |         uses: actions/checkout@v4
17 |         with:
18 |           fetch-depth: 0
19 |       - name: Init environment 
20 |         uses: ./.github/actions/init-environment 
21 |       - name: Run linter
22 |         run: |
23 |           pylint `git diff --name-only --diff-filter=d origin/main HEAD | grep -E '\.py$' | tr '\n' ' '`


--------------------------------------------------------------------------------
/.github/workflows/pr_request_checks.yml:
--------------------------------------------------------------------------------
 1 | name: Pull Request Checks
 2 | 
 3 | on:
 4 |   pull_request:
 5 |     branches:
 6 |       - master
 7 | 
 8 | jobs:
 9 |   test:
10 |     runs-on: ubuntu-latest
11 | 
12 |     steps:
13 |       - name: Checkout code
14 |         uses: actions/checkout@v4
15 | 
16 |       - name: Set up Python
17 |         uses: actions/setup-python@v5
18 |         with:
19 |           python-version: 3.x
20 | 
21 |       - name: Install dependencies
22 |         run: pip install -r requirements.txt
23 | 
24 |       - name: Run tests and checks
25 |         run: |
26 |           pytest tests/
27 |           pylint swarms_torch


--------------------------------------------------------------------------------
/.github/workflows/label.yml:
--------------------------------------------------------------------------------
 1 | # This workflow will triage pull requests and apply a label based on the
 2 | # paths that are modified in the pull request.
 3 | #
 4 | # To use this workflow, you will need to set up a .github/labeler.yml
 5 | # file with configuration.  For more information, see:
 6 | # https://github.com/actions/labeler
 7 | 
 8 | name: Labeler
 9 | on: [pull_request_target]
10 | 
11 | jobs:
12 |   label:
13 | 
14 |     runs-on: ubuntu-latest
15 |     permissions:
16 |       contents: read
17 |       pull-requests: write
18 | 
19 |     steps:
20 |     - uses: actions/labeler@v5.0.0
21 |       with:
22 |         repo-token: "${{ secrets.GITHUB_TOKEN }}"
23 | 


--------------------------------------------------------------------------------
/.github/workflows/docs_test.yml:
--------------------------------------------------------------------------------
 1 | name: Documentation Tests
 2 | 
 3 | on:
 4 |   push:
 5 |     branches:
 6 |       - master
 7 | 
 8 | jobs:
 9 |   test:
10 |     runs-on: ubuntu-latest
11 | 
12 |     steps:
13 |       - name: Checkout code
14 |         uses: actions/checkout@v4
15 | 
16 |       - name: Set up Python
17 |         uses: actions/setup-python@v5
18 |         with:
19 |           python-version: 3.x
20 | 
21 |       - name: Install dependencies
22 |         run: pip install -r requirements.txt
23 | 
24 |       - name: Build documentation
25 |         run: make docs
26 | 
27 |       - name: Validate documentation
28 |         run: sphinx-build -b linkcheck docs build/docs


--------------------------------------------------------------------------------
/.github/workflows/pylint.yml:
--------------------------------------------------------------------------------
 1 | name: Pylint
 2 | 
 3 | on: [push]
 4 | 
 5 | jobs:
 6 |   build:
 7 |     runs-on: ubuntu-latest
 8 |     strategy:
 9 |       matrix:
10 |         python-version: ["3.8", "3.9", "3.10"]
11 |     steps:
12 |     - uses: actions/checkout@v4
13 |     - name: Set up Python ${{ matrix.python-version }}
14 |       uses: actions/setup-python@v5
15 |       with:
16 |         python-version: ${{ matrix.python-version }}
17 |     - name: Install dependencies
18 |       run: |
19 |         python -m pip install --upgrade pip
20 |         pip install pylint
21 |     - name: Analysing the code with pylint
22 |       run: |
23 |         pylint $(git ls-files '*.py')
24 | 


--------------------------------------------------------------------------------
/.github/workflows/welcome.yml:
--------------------------------------------------------------------------------
 1 | name: Welcome WorkFlow
 2 | 
 3 | on:
 4 |   issues:
 5 |     types: [opened]
 6 |   pull_request_target:
 7 |     types: [opened]
 8 | 
 9 | jobs:
10 |   build:
11 |     name: 👋 Welcome
12 |     permissions: write-all
13 |     runs-on: ubuntu-latest
14 |     steps:
15 |       - uses: actions/first-interaction@v1.3.0
16 |         with:
17 |           repo-token: ${{ secrets.GITHUB_TOKEN }}
18 |           issue-message: "Hello there, thank you for opening an Issue ! 🙏🏻 The team was notified and they will get back to you asap."
19 |           pr-message:  "Hello there, thank you for opening an PR ! 🙏🏻 The team was notified and they will get back to you asap."


--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/feature_request.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | name: Feature request
 3 | about: Suggest an idea for this project
 4 | title: ''
 5 | labels: ''
 6 | assignees: 'kyegomez'
 7 | 
 8 | ---
 9 | 
10 | **Is your feature request related to a problem? Please describe.**
11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
12 | 
13 | **Describe the solution you'd like**
14 | A clear and concise description of what you want to happen.
15 | 
16 | **Describe alternatives you've considered**
17 | A clear and concise description of any alternative solutions or features you've considered.
18 | 
19 | **Additional context**
20 | Add any other context or screenshots about the feature request here.
21 | 


--------------------------------------------------------------------------------
/.github/FUNDING.yml:
--------------------------------------------------------------------------------
 1 | # These are supported funding model platforms
 2 | 
 3 | github: [kyegomez]
 4 | patreon: # Replace with a single Patreon username
 5 | open_collective: # Replace with a single Open Collective username
 6 | ko_fi: # Replace with a single Ko-fi username
 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
 9 | liberapay: # Replace with a single Liberapay username
10 | issuehunt: # Replace with a single IssueHunt username
11 | otechie: # Replace with a single Otechie username
12 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
13 | custom: #Nothing
14 | 


--------------------------------------------------------------------------------
/.github/workflows/unit-test.yml:
--------------------------------------------------------------------------------
 1 | name: build
 2 | 
 3 | on:
 4 |   push:
 5 |     branches: [ main ]
 6 |   pull_request:
 7 |     branches: [ main ]
 8 | 
 9 | jobs:
10 | 
11 |   build:
12 | 
13 |     runs-on: ubuntu-latest
14 | 
15 |     steps:
16 |     - uses: actions/checkout@v4
17 | 
18 |     - name: Setup Python
19 |       uses: actions/setup-python@v5
20 |       with:
21 |         python-version: '3.10'
22 | 
23 |     - name: Install dependencies
24 |       run: pip install -r requirements.txt
25 | 
26 |     - name: Run Python unit tests
27 |       run: python3 -m unittest tests/
28 | 
29 |     - name: Verify that the Docker image for the action builds
30 |       run: docker build . --file Dockerfile
31 | 
32 |     - name: Verify integration test results
33 |       run: python3 -m unittest tests/
34 | 


--------------------------------------------------------------------------------
/.github/workflows/python-publish.yml:
--------------------------------------------------------------------------------
 1 | 
 2 | name: Upload Python Package
 3 | 
 4 | on:
 5 |   release:
 6 |     types: [published]
 7 | 
 8 | permissions:
 9 |   contents: read
10 | 
11 | jobs:
12 |   deploy:
13 | 
14 |     runs-on: ubuntu-latest
15 | 
16 |     steps:
17 |     - uses: actions/checkout@v4
18 |     - name: Set up Python
19 |       uses: actions/setup-python@v5
20 |       with:
21 |         python-version: '3.x'
22 |     - name: Install dependencies
23 |       run: |
24 |         python -m pip install --upgrade pip
25 |         pip install build
26 |     - name: Build package
27 |       run: python -m build
28 |     - name: Publish package
29 |       uses: pypa/gh-action-pypi-publish@2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf
30 |       with:
31 |         user: __token__
32 |         password: ${{ secrets.PYPI_API_TOKEN }}


--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/bug_report.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | name: Bug report
 3 | about: Create a detailed report on the bug and it's root cause. Conduct root cause error analysis
 4 | title: "[BUG] "
 5 | labels: bug
 6 | assignees: kyegomez
 7 | 
 8 | ---
 9 | 
10 | **Describe the bug**
11 | A clear and concise description of what the bug is and what the main root cause error is. Test very thoroughly before submitting.
12 | 
13 | **To Reproduce**
14 | Steps to reproduce the behavior:
15 | 1. Go to '...'
16 | 2. Click on '....'
17 | 3. Scroll down to '....'
18 | 4. See error
19 | 
20 | **Expected behavior**
21 | A clear and concise description of what you expected to happen.
22 | 
23 | **Screenshots**
24 | If applicable, add screenshots to help explain your problem.
25 | 
26 | **Additional context**
27 | Add any other context about the problem here.
28 | 


--------------------------------------------------------------------------------
/.github/workflows/code_quality_control.yml:
--------------------------------------------------------------------------------
 1 | name: Linting and Formatting
 2 | 
 3 | on:
 4 |   push:
 5 |     branches:
 6 |       - main
 7 | 
 8 | jobs:
 9 |   lint_and_format:
10 |     runs-on: ubuntu-latest
11 | 
12 |     steps:
13 |       - name: Checkout code
14 |         uses: actions/checkout@v4
15 | 
16 |       - name: Set up Python
17 |         uses: actions/setup-python@v5
18 |         with:
19 |           python-version: 3.x
20 | 
21 |       - name: Install dependencies
22 |         run: pip install -r requirements.txt
23 | 
24 |       - name: Find Python files
25 |         run: find swarms_torch -name "*.py" -type f -exec autopep8 --in-place --aggressive --aggressive {} +
26 | 
27 |       - name: Push changes
28 |         uses: ad-m/github-push-action@master
29 |         with:
30 |           github_token: ${{ secrets.GITHUB_TOKEN }}


--------------------------------------------------------------------------------
/.github/workflows/stale.yml:
--------------------------------------------------------------------------------
 1 | # This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
 2 | #
 3 | # You can adjust the behavior by modifying this file.
 4 | # For more information, see:
 5 | # https://github.com/actions/stale
 6 | name: Mark stale issues and pull requests
 7 | 
 8 | on:
 9 |   schedule:
10 |   - cron: '26 12 * * *'
11 | 
12 | jobs:
13 |   stale:
14 | 
15 |     runs-on: ubuntu-latest
16 |     permissions:
17 |       issues: write
18 |       pull-requests: write
19 | 
20 |     steps:
21 |     - uses: actions/stale@v9
22 |       with:
23 |         repo-token: ${{ secrets.GITHUB_TOKEN }}
24 |         stale-issue-message: 'Stale issue message'
25 |         stale-pr-message: 'Stale pull request message'
26 |         stale-issue-label: 'no-issue-activity'
27 |         stale-pr-label: 'no-pr-activity'


--------------------------------------------------------------------------------
/scripts/code_quality.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # Navigate to the directory containing the 'package' folder
 4 | # cd /path/to/your/code/directory
 5 | 
 6 | # Run autopep8 with max aggressiveness (-aaa) and in-place modification (-i)
 7 | # on all Python files (*.py) under the 'package' directory.
 8 | autopep8 --in-place --aggressive --aggressive --recursive --experimental --list-fixes package/
 9 | 
10 | # Run black with default settings, since black does not have an aggressiveness level.
11 | # Black will format all Python files it finds in the 'package' directory.
12 | black --experimental-string-processing package/
13 | 
14 | # Run ruff on the 'package' directory.
15 | # Add any additional flags if needed according to your version of ruff.
16 | ruff --unsafe_fix
17 | 
18 | # YAPF
19 | yapf --recursive --in-place --verbose --style=google --parallel package
20 | 


--------------------------------------------------------------------------------
/example.py:
--------------------------------------------------------------------------------
 1 | # Import necessary modules
 2 | import os
 3 | from autort import AutoRTSwarm, AutoRTAgent
 4 | 
 5 | # Set the OpenAI API key
 6 | openai_api_key = os.getenv("OPENAI_API_KEY")
 7 | 
 8 | # Define a list of AutoRTAgent instances
 9 | agents = [
10 |     AutoRTAgent(openai_api_key, max_tokens=1000),
11 |     AutoRTAgent(openai_api_key, max_tokens=1000),
12 | ]
13 | 
14 | # Create an instance of AutoRTSwarm with the agents and datastore
15 | autort_swarm = AutoRTSwarm(agents=agents)
16 | 
17 | # Run the AutoRTSwarm with the given inputs
18 | autort_swarm.run(
19 |     "There is a bottle on the table.",
20 |     "https://i.imgur.com/2qY9f8U.png",
21 | )
22 | 
23 | 
24 | # Run a single agent in the swarm
25 | run_agent = autort_swarm.run_single_agent(
26 |     autort_swarm.agents[0].id,
27 |     (
28 |         "Pick up the bottle and place it on the shelf."
29 |         "https://i.imgur.com/2qY9f8U.png"
30 |     ),
31 | )
32 | 


--------------------------------------------------------------------------------
/scripts/merge_all_prs.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # Check if we are inside a Git repository
 4 | if ! git rev-parse --git-dir > /dev/null 2>&1; then
 5 |     echo "Error: Must be run inside a Git repository."
 6 |     exit 1
 7 | fi
 8 | 
 9 | # Fetch all open pull requests
10 | echo "Fetching open PRs..."
11 | prs=$(gh pr list --state open --json number --jq '.[].number')
12 | 
13 | # Check if there are PRs to merge
14 | if [ -z "$prs" ]; then
15 |     echo "No open PRs to merge."
16 |     exit 0
17 | fi
18 | 
19 | echo "Found PRs: $prs"
20 | 
21 | # Loop through each pull request number and merge it
22 | for pr in $prs; do
23 |     echo "Attempting to merge PR #$pr"
24 |     merge_output=$(gh pr merge $pr --auto --merge)
25 |     merge_status=$?
26 |     if [ $merge_status -ne 0 ]; then
27 |         echo "Failed to merge PR #$pr. Error: $merge_output"
28 |     else
29 |         echo "Successfully merged PR #$pr"
30 |     fi
31 | done
32 | 
33 | echo "Processing complete."
34 | 


--------------------------------------------------------------------------------
/.github/workflows/cos_integration.yml:
--------------------------------------------------------------------------------
 1 | name: Continuous Integration
 2 | 
 3 | on:
 4 |   push:
 5 |     branches:
 6 |       - main
 7 | 
 8 | jobs:
 9 |   test:
10 |     runs-on: ubuntu-latest
11 |     steps:
12 |       - name: Checkout code
13 |         uses: actions/checkout@v4
14 | 
15 |       - name: Set up Python
16 |         uses: actions/setup-python@v5
17 |         with:
18 |           python-version: 3.x
19 | 
20 |       - name: Install dependencies
21 |         run: pip install -r requirements.txt
22 | 
23 |       - name: Run unit tests
24 |         run: pytest tests/unit
25 | 
26 |       - name: Run integration tests
27 |         run: pytest tests/integration
28 | 
29 |       - name: Run code coverage
30 |         run: pytest --cov=swarms tests/
31 | 
32 |       - name: Run linters
33 |         run: pylint swarms
34 | 
35 |       - name: Build documentation
36 |         run: make docs
37 | 
38 |       - name: Validate documentation
39 |         run: sphinx-build -b linkcheck docs build/docs
40 | 
41 |       - name: Run performance tests
42 |         run: pytest tests/performance


--------------------------------------------------------------------------------
/.github/PULL_REQUEST_TEMPLATE.yml:
--------------------------------------------------------------------------------
 1 | <!-- Thank you for contributing to Zeta!
 2 | 
 3 | Replace this comment with:
 4 |   - Description: a description of the change, 
 5 |   - Issue: the issue # it fixes (if applicable),
 6 |   - Dependencies: any dependencies required for this change,
 7 |   - Tag maintainer: for a quicker response, tag the relevant maintainer (see below),
 8 |   - Twitter handle: we announce bigger features on Twitter. If your PR gets announced and you'd like a mention, we'll gladly shout you out!
 9 | 
10 | If you're adding a new integration, please include:
11 |   1. a test for the integration, preferably unit tests that do not rely on network access,
12 |   2. an example notebook showing its use.
13 | 
14 | Maintainer responsibilities:
15 |   - nn / Misc / if you don't know who to tag: kye@apac.ai
16 |   - tokenizers: kye@apac.ai
17 |   - training / Prompts: kye@apac.ai
18 |   - models: kye@apac.ai
19 | 
20 | If no one reviews your PR within a few days, feel free to kye@apac.ai
21 | 
22 | See contribution guidelines for more information on how to write/run tests, lint, etc: https://github.com/kyegomez/zeta


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2023 Eternal Reclaimer
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/.github/workflows/test.yml:
--------------------------------------------------------------------------------
 1 | name: test
 2 | 
 3 | on:
 4 |   push:
 5 |     branches: [master]
 6 |   pull_request:
 7 |   workflow_dispatch:
 8 | 
 9 | env:
10 |   POETRY_VERSION: "1.4.2"
11 | 
12 | jobs:
13 |   build:
14 |     runs-on: ubuntu-latest
15 |     strategy:
16 |       matrix:
17 |         python-version:
18 |           - "3.8"
19 |           - "3.9"
20 |           - "3.10"
21 |           - "3.11"
22 |         test_type:
23 |           - "core"
24 |           - "extended"
25 |     name: Python ${{ matrix.python-version }} ${{ matrix.test_type }}
26 |     steps:
27 |       - uses: actions/checkout@v4
28 |       - name: Set up Python ${{ matrix.python-version }}
29 |         uses: "./.github/actions/poetry_setup"
30 |         with:
31 |           python-version: ${{ matrix.python-version }}
32 |           poetry-version: "1.4.2"
33 |           cache-key: ${{ matrix.test_type }}
34 |           install-command: |
35 |               if [ "${{ matrix.test_type }}" == "core" ]; then
36 |                 echo "Running core tests, installing dependencies with poetry..."
37 |                 poetry install
38 |               else
39 |                 echo "Running extended tests, installing dependencies with poetry..."
40 |                 poetry install -E extended_testing
41 |               fi
42 |       - name: Run ${{matrix.test_type}} tests
43 |         run: |
44 |           if [ "${{ matrix.test_type }}" == "core" ]; then
45 |             make test
46 |           else
47 |             make extended_tests
48 |           fi
49 |         shell: bash


--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
 1 | [build-system]
 2 | requires = ["poetry-core>=1.0.0"]
 3 | build-backend = "poetry.core.masonry.api"
 4 | 
 5 | [tool.poetry]
 6 | name = "autort-swarms"
 7 | version = "0.0.6"
 8 | description = "AutoRT - Pytorch"
 9 | license = "MIT"
10 | authors = ["Kye Gomez <kye@apac.ai>"]
11 | homepage = "https://github.com/kyegomez/AutoRT"
12 | documentation = "https://github.com/kyegomez/AutoRT"  # Add this if you have documentation.
13 | readme = "README.md"  # Assuming you have a README.md
14 | repository = "https://github.com/kyegomez/AutoRT"
15 | keywords = ["artificial intelligence", "deep learning", "optimizers", "Prompt Engineering"]
16 | classifiers = [
17 |     "Development Status :: 4 - Beta",
18 |     "Intended Audience :: Developers",
19 |     "Topic :: Scientific/Engineering :: Artificial Intelligence",
20 |     "License :: OSI Approved :: MIT License",
21 |     "Programming Language :: Python :: 3.9"
22 | ]
23 | packages = [
24 |     { include = "autort" },
25 |     { include = "autort/**/*.py" },
26 | ]
27 | 
28 | 
29 | [tool.poetry.dependencies]
30 | python = "^3.6"
31 | torch = "^1.9.0"
32 | torchvision = "^0.10.0"
33 | einops = "*"
34 | zetascale = "*"
35 | swarms = "*"
36 | 
37 | 
38 | 
39 | 
40 | 
41 | [tool.poetry.group.lint.dependencies]
42 | ruff = "^0.1.6"
43 | types-toml = "^0.10.8.1"
44 | types-redis = "^4.3.21.6"
45 | types-pytz = "^2023.3.0.0"
46 | black = "^23.1.0"
47 | types-chardet = "^5.0.4.6"
48 | mypy-protobuf = "^3.0.0"
49 | 
50 | 
51 | [tool.autopep8]
52 | max_line_length = 80
53 | ignore = "E501,W6"  # or ["E501", "W6"]
54 | in-place = true
55 | recursive = true
56 | aggressive = 3
57 | 
58 | 
59 | [tool.ruff]
60 | line-length = 70
61 | 
62 | [tool.black]
63 | line-length = 70
64 | target-version = ['py38']
65 | preview = true
66 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | [![Multi-Modality](agorabanner.png)](https://discord.gg/qUtxnK2NMf)
 2 | 
 3 | # AutoRT
 4 | ![AutoRTImage](autort.png)
 5 | Implementation of AutoRT: "AutoRT: Embodied Foundation Models for Large Scale Orchestration of Robotic Agents". This repo will implement the multi agent system that transforms a scene into a list of ranked and priortized tasks for an robotic action model to execute. This is an very effective setup that I personally believe is the future for swarming robotic foundation models!
 6 | 
 7 | This project will be implemented using Swarms, for the various llms and use the official RT-1 as the robotic action model.
 8 | 
 9 | [PAPER LINK](https://auto-rt.github.io/static/pdf/AutoRT.pdf)
10 | 
11 | ## Install
12 | `$ pip3 install autort-swarms`
13 | 
14 | 
15 | ## Usage
16 | 
17 | 
18 | ### AutoRTAgent
19 | A single AutoRT agent that: analyzes a scene using visual COT -> generate tasks -> filter tasks -> execute it with a robotic transformer.
20 | ```python
21 | # Import necessary modules
22 | import os
23 | from autort import AutoRTSwarm, AutoRTAgent
24 | 
25 | # Set the OpenAI API key
26 | openai_api_key = os.getenv("OPENAI_API_KEY")
27 | 
28 | # Define a list of AutoRTAgent instances
29 | agents = [
30 |     AutoRTAgent(openai_api_key, max_tokens=1000),
31 |     AutoRTAgent(openai_api_key, max_tokens=1000),
32 | ]
33 | 
34 | # Create an instance of AutoRTSwarm with the agents and datastore
35 | autort_swarm = AutoRTSwarm(agents)
36 | 
37 | # Run the AutoRTSwarm with the given inputs
38 | autort_swarm.run(
39 |     "There is a bottle on the table.",
40 |     "https://i.imgur.com/2qY9f8U.png",
41 | )
42 | ```
43 | 
44 | 
45 | ### AutoRTSwarm
46 | A team of AutoRT agents where you can plug in and play any number of `AutoRTAgents` with customization. First, the task will be routed to each agent and then all of their outputs will be saved.
47 | ```python
48 | # Import necessary modules
49 | import os
50 | from autort import AutoRTSwarm, AutoRTAgent
51 | 
52 | # Set the OpenAI API key
53 | openai_api_key = os.getenv("OPENAI_API_KEY")
54 | 
55 | # Define a list of AutoRTAgent instances
56 | agents = [
57 |     AutoRTAgent(openai_api_key, max_tokens=1000),
58 |     AutoRTAgent(openai_api_key, max_tokens=1000),
59 | ]
60 | 
61 | # Create an instance of AutoRTSwarm with the agents and datastore
62 | autort_swarm = AutoRTSwarm(agents)
63 | 
64 | # Run the AutoRTSwarm with the given inputs
65 | autort_swarm.run(
66 |     "There is a bottle on the table.",
67 |     "https://i.imgur.com/2qY9f8U.png",
68 | )
69 | ```
70 | 
71 | ## Citation
72 | ```bibtex
73 | @inproceedings{
74 |     anonymous2023autort,
75 |     title={Auto{RT}: Embodied Foundation Models for Large Scale Orchestration of Robotic Agents},
76 |     author={Anonymous},
77 |     booktitle={Submitted to The Twelfth International Conference on Learning Representations},
78 |     year={2023},
79 |     url={https://openreview.net/forum?id=xVlcbh0poD},
80 |     note={under review}
81 | }
82 | 
83 | ```
84 | 
85 | 
86 | # License
87 | MIT
88 | 
89 | 
90 | 
91 | # Todo
92 | - [ ] Implement a run method into `AutoRTSwarm` that runs all the agents with APIs.
93 | - [ ] Make it able to send commands to a certain agent using the swarm network.
94 | - [ ] Send a task to all agents in the swarm network
95 | - [ ] 


--------------------------------------------------------------------------------
/errors.txt:
--------------------------------------------------------------------------------
 1 | /Library/Frameworks/Python.framework/Error generating response: 'choices'
 2 | Error generating response: 'choices'
 3 | Warning: distutils Version classes are deprecated. Use packaging.version instead.
 4 |   if LooseVersion(torch.__version__) < LooseVersion("1.0.0"):
 5 | /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/thop/profile.py:68: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.
 6 |   if LooseVersion(torch.__version__) >= LooseVersion("1.1.0"):
 7 | distutils Version classes are deprecated. Use packaging.version instead.
 8 | distutils Version classes are deprecated. Use packaging.version instead.
 9 | Error generating response: 'choices'
10 | Error generating response: 'choices'
11 | Error generating response: 'choices'
12 | Traceback (most recent call last):
13 |   File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/swarms/structs/agent.py", line 522, in run
14 |     response = self.llm(
15 |                ^^^^^^^^^
16 |   File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/swarms/models/gpt4_vision_api.py", line 376, in __call__
17 |     raise error
18 |   File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/swarms/models/gpt4_vision_api.py", line 361, in __call__
19 |     content = out["choices"][0]["message"]["content"]
20 |               ~~~^^^^^^^^^^^
21 | KeyError: 'choices'
22 | 
23 | During handling of the above exception, another exception occurred:
24 | 
25 | Traceback (most recent call last):
26 |   File "/Users/defalt/Desktop/Athena/research/AutoRT/example.py", line 18, in <module>
27 |     autort_swarm.run(
28 |   File "/Users/defalt/Desktop/Athena/research/AutoRT/autort/main.py", line 223, in run
29 |     out = self.network.run_many_agents(text, img, *args, **kwargs)
30 |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
31 |   File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/swarms/structs/swarm_net.py", line 188, in run_many_agents
32 |     return [
33 |            ^
34 |   File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/swarms/structs/swarm_net.py", line 189, in <listcomp>
35 |     agent.run(task, *args, **kwargs)
36 |   File "/Users/defalt/Desktop/Athena/research/AutoRT/autort/main.py", line 151, in run
37 |     vllm = self.vllm.run(text, img, *args, **kwargs)
38 |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
39 |   File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/swarms/structs/agent.py", line 557, in run
40 |     time.sleep(self.retry_interval)
41 | KeyboardInterrupt
42 | Implicitly cleaning up <TemporaryDirectory '/var/folders/zl/0vgkvtm52vjcs1skpnlqnssw0000gn/T/tmpwnz2mkqtwandb'>
43 | Implicitly cleaning up <TemporaryDirectory '/var/folders/zl/0vgkvtm52vjcs1skpnlqnssw0000gn/T/tmpjr_v40b8wandb-artifacts'>
44 | Implicitly cleaning up <TemporaryDirectory '/var/folders/zl/0vgkvtm52vjcs1skpnlqnssw0000gn/T/tmpcpop6k5pwandb-media'>
45 | Implicitly cleaning up <TemporaryDirectory '/var/folders/zl/0vgkvtm52vjcs1skpnlqnssw0000gn/T/tmpnnfam_xjwandb-media'>
46 | Implicitly cleaning up <TemporaryDirectory '/var/folders/zl/0vgkvtm52vjcs1skpnlqnssw0000gn/T/tmpc6ugrlkh'>
47 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | .vscode/
  9 | .vscode
 10 | 
 11 | # Distribution / packaging
 12 | .Python
 13 | build/
 14 | develop-eggs/
 15 | dist/
 16 | downloads/
 17 | eggs/
 18 | .eggs/
 19 | lib/
 20 | lib64/
 21 | parts/
 22 | .ruff_cache/
 23 | sdist/
 24 | var/
 25 | wheels/
 26 | share/python-wheels/
 27 | *.egg-info/
 28 | .installed.cfg
 29 | *.egg
 30 | MANIFEST
 31 | 
 32 | # PyInstaller
 33 | #  Usually these files are written by a python script from a template
 34 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 35 | *.manifest
 36 | *.spec
 37 | 
 38 | # Installer logs
 39 | pip-log.txt
 40 | pip-delete-this-directory.txt
 41 | 
 42 | # Unit test / coverage reports
 43 | htmlcov/
 44 | .tox/
 45 | .nox/
 46 | .coverage
 47 | .coverage.*
 48 | .cache
 49 | nosetests.xml
 50 | coverage.xml
 51 | *.cover
 52 | *.py,cover
 53 | .hypothesis/
 54 | .pytest_cache/
 55 | cover/
 56 | 
 57 | # Translations
 58 | *.mo
 59 | *.pot
 60 | 
 61 | # Django stuff:
 62 | *.log
 63 | local_settings.py
 64 | db.sqlite3
 65 | db.sqlite3-journal
 66 | 
 67 | # Flask stuff:
 68 | instance/
 69 | .webassets-cache
 70 | 
 71 | # Scrapy stuff:
 72 | .scrapy
 73 | 
 74 | # Sphinx documentation
 75 | docs/_build/
 76 | 
 77 | # PyBuilder
 78 | .pybuilder/
 79 | target/
 80 | 
 81 | # Jupyter Notebook
 82 | .ipynb_checkpoints
 83 | 
 84 | # IPython
 85 | profile_default/
 86 | ipython_config.py
 87 | 
 88 | # pyenv
 89 | #   For a library or package, you might want to ignore these files since the code is
 90 | #   intended to run in multiple environments; otherwise, check them in:
 91 | # .python-version
 92 | 
 93 | # pipenv
 94 | #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 95 | #   However, in case of collaboration, if having platform-specific dependencies or dependencies
 96 | #   having no cross-platform support, pipenv may install dependencies that don't work, or not
 97 | #   install all needed dependencies.
 98 | #Pipfile.lock
 99 | 
100 | # poetry
101 | #   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
102 | #   This is especially recommended for binary packages to ensure reproducibility, and is more
103 | #   commonly ignored for libraries.
104 | #   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
105 | #poetry.lock
106 | 
107 | # pdm
108 | #   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
109 | #pdm.lock
110 | #   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
111 | #   in version control.
112 | #   https://pdm.fming.dev/#use-with-ide
113 | .pdm.toml
114 | 
115 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
116 | __pypackages__/
117 | 
118 | # Celery stuff
119 | celerybeat-schedule
120 | celerybeat.pid
121 | 
122 | # SageMath parsed files
123 | *.sage.py
124 | 
125 | # Environments
126 | .env
127 | .venv
128 | env/
129 | venv/
130 | ENV/
131 | env.bak/
132 | venv.bak/
133 | 
134 | # Spyder project settings
135 | .spyderproject
136 | .spyproject
137 | 
138 | # Rope project settings
139 | .ropeproject
140 | 
141 | # mkdocs documentation
142 | /site
143 | 
144 | # mypy
145 | .mypy_cache/
146 | .dmypy.json
147 | dmypy.json
148 | 
149 | # Pyre type checker
150 | .pyre/
151 | 
152 | # pytype static type analyzer
153 | .pytype/
154 | 
155 | # Cython debug symbols
156 | cython_debug/
157 | 
158 | # PyCharm
159 | #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
160 | #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
161 | #  and can be added to the global gitignore or merged into this file.  For a more nuclear
162 | #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
163 | #.idea/
164 | 


--------------------------------------------------------------------------------
/autort/prompts.py:
--------------------------------------------------------------------------------
  1 | VISUALIZE_OBJECT_PROMPT = """
  2 | "Your task is to analyze and describe the visual scene presented to you. Pay attention to the following key elements:
  3 | 1. **Identify All Major and Minor Objects:** Describe each object in the scene, including their size, shape, color, and any notable features. 
  4 | 2. **Spatial Relationships:** Explain where each object is located in relation to others. Are they overlapping, adjacent to, or distant from one another? 
  5 | 3. **Colors and Textures:** Note the colors and textures present in the scene. Are there any patterns or unique visual elements?
  6 | 4. **Lighting and Shadows:** Observe the source of light, its intensity, and direction. How do the lighting and shadows affect the appearance of objects?
  7 | 5. **Any Motion or Activity:** If there is any movement or action happening in the scene, describe it in detail. What is moving, and how?
  8 | 6. **Atmosphere and Mood:** If applicable, comment on the overall atmosphere or mood conveyed by the scene. Is it cheerful, gloomy, chaotic, serene?
  9 | 7. **Background and Context:** Provide information about the background and any contextual elements that give a sense of place or time.
 10 | 
 11 | Your description should be thorough and precise, enabling a clear understanding of the scene for someone who cannot see it. Think of your description as painting a picture with words."
 12 | 
 13 | This prompt guides the multimodal model to cover all critical aspects of a scene, ensuring a comprehensive description for the vision model.
 14 | """
 15 | 
 16 | 
 17 | GENERATE_TASKS_PROMPT = """
 18 | To create an effective introduction for the multi-shot instruction prompt designed for a language model (LLM) to generate tasks for a robot, the introduction should set the context, define the purpose, and outline the structure of the tasks. Here's an example introduction:
 19 | 
 20 | Introduction to Robot Task Generation Using Language Model
 21 | 
 22 | Welcome to the task generation module for robotic operations. This guide is designed to instruct a language model in generating specific tasks for a robot to execute. The goal is to create clear, actionable, and safe directives that a robot can understand and carry out efficiently in various environments.
 23 | 
 24 | Each task is structured in a multi-shot format, providing a comprehensive blueprint for the robot's operation. This format includes:
 25 | 
 26 | [Task][Number][Task Name]: A unique identifier for each task, facilitating easy reference and organization.
 27 | Objective: A brief description of the overall goal or purpose of the task.
 28 | Steps: A detailed, step-by-step guide that the robot will follow to complete the task.
 29 | Safety Measures: Critical considerations and precautions to ensure the robot's operation is safe for both the robot and its surroundings.
 30 | These tasks are designed to be adaptable and can be customized according to different robotic capabilities and operational requirements. The instructions are formulated to be easily parsed by advanced language models and translated into executable commands for robots.
 31 | 
 32 | ---------------EXAMPLES ----------
 33 | [Task][1][Pick and Place Objects]
 34 | 
 35 | Objective: The robot is required to pick up specific objects from a designated area and place them in a predefined location.
 36 | Steps:
 37 | Identify the object to be picked up using visual or sensor-based recognition.
 38 | Navigate to the object's location.
 39 | Use the robotic arm to grasp the object securely.
 40 | Transport the object to the designated drop-off location.
 41 | Gently release the object at the drop-off point.
 42 | Safety Measures: Ensure that the grip is firm but not damaging to the object. Avoid collisions with other objects and people during navigation.
 43 | [Task][2][Environmental Monitoring]
 44 | 
 45 | Objective: Continuously monitor environmental parameters like temperature, humidity, and air quality within a specified area.
 46 | Steps:
 47 | Navigate through the area following a predefined path.
 48 | Use onboard sensors to record environmental data.
 49 | Transmit the data to the central monitoring system at regular intervals.
 50 | Alert the central system if any parameter goes beyond the set thresholds.
 51 | Safety Measures: The robot should maintain a safe distance from obstacles and avoid restricted zones.
 52 | [Task][3][Surveillance and Security]
 53 | 
 54 | Objective: Perform routine surveillance rounds in a designated area to ensure security.
 55 | Steps:
 56 | Patrol the area following a set route.
 57 | Use cameras and sensors to detect unusual activities or intrusions.
 58 | Record and send real-time footage to the security control room.
 59 | If an intrusion is detected, sound an alarm and notify the control room.
 60 | Safety Measures: Avoid direct confrontation and maintain a safe distance from any identified threat.
 61 | 
 62 | """
 63 | 
 64 | 
 65 | def FOUNDATIONAL_RULES_RPOMPT(
 66 |     guidance: str = "Do not interact with humans or animals.",
 67 | ):
 68 |     FOUNDATIONAL_RULES = f"""
 69 |     The following are rules you must follow to successfully fulfill your purpose 
 70 |     of automating tasks. These rules are divided into four categories: foundational,
 71 |     safety, embodiment, and guidance. 
 72 |     
 73 |     ------------- RULES -----------------
 74 |     FOUNDATIONAL RULES =
 75 |     F1. A robot may not injure a human being.
 76 |     F2. A robot must protect its own existence as long as such protection does not conflict with F1.
 77 |     F3. A robot must obey orders given it by human beings except where such orders would
 78 |     conflict with F1 or F2.
 79 |     
 80 |     SAFETY RULES =
 81 |     S1. This robot shall not attempt tasks involving humans, animals or living things.
 82 |     S2. This robot shall not interact with objects that are sharp, such as a knife.
 83 |     S3. This robot shall not interact with objects that are electrical, such as a computer or tablet.
 84 |     
 85 |     EMBODIMENT RULES =
 86 |     E1. This robot shall not attempt to lift objects that are heavier than a book. For example, it
 87 |     cannot move a couch but it can push plastic chairs.
 88 |     E2. This robot only has one arm, and thus cannot perform tasks requiring two arms. For
 89 |     example, it cannot open a bottle.
 90 |     
 91 |     GUIDANCE RULES =
 92 |     G1. The human command, which the robot should follow if given: {guidance}
 93 |     """
 94 |     return FOUNDATIONAL_RULES
 95 | 
 96 | 
 97 | def FUSED_SYSTEM_PROMPT_WITH_SOP(guidance: str, sop: str):
 98 |     # Fused together the rules and the SOP
 99 |     return FOUNDATIONAL_RULES_RPOMPT(guidance) + sop
100 | 
101 | 
102 | FILTER_TASKS_SOP_PROMPT = """
103 | 
104 | 
105 | This SOP is designed to guide the language model in generating and categorizing tasks for robotic execution. 
106 | The tasks are evaluated based on their complexity and the necessity of human assistance. 
107 | The system aims to autonomously filter and rank tasks, identifying which can be 
108 | performed independently by the robot and which require human intervention.
109 | 
110 | Instruction Format:
111 | Each task should be formatted as follows:
112 | 
113 | [Step][Task Name][Human Needed: Yes/No]
114 | Process:
115 | 
116 | Task Generation: The model will generate a list of potential tasks for the robot.
117 | Self-Reflection Filtering: Each task is evaluated for feasibility and safety. The model will categorize tasks into those that require human assistance and those that do not.
118 | Task Sampling: A valid task is selected from the filtered list for the robot to attempt.
119 | Execution and Review: The robot attempts the task. If human assistance is required, the task is paused until assistance is provided.
120 | 
121 | 
122 | 
123 | ########  Examples #########
124 | 
125 | [Step][1][Visual Inspection of Machinery][Human Needed: No]
126 | 
127 | Task: The robot will inspect machinery for any signs of wear or damage using its visual sensors.
128 | Self-Reflection Filter: The task is within the robot's capability and does not require human intervention.
129 | [Step][2][Precision Welding of Components][Human Needed: Yes]
130 | 
131 | Task: Perform precision welding on specific components in the assembly line.
132 | Self-Reflection Filter: Task requires human guidance for setup and quality control, due to the high precision and risk involved.
133 | [Step][3][Inventory Stocktaking][Human Needed: No]
134 | 
135 | Task: Count and organize inventory in the warehouse.
136 | Self-Reflection Filter: The task is straightforward and can be autonomously completed by the robot.
137 | [Step][4][Complex Circuitry Repair][Human Needed: Yes]
138 | 
139 | Task: Repair complex circuitry in electronic equipment.
140 | Self-Reflection Filter: Due to the intricate nature of the task, human expertise is required for guidance and verification.
141 | 
142 | 
143 | """
144 | 


--------------------------------------------------------------------------------
/autort/main.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | from dotenv import load_dotenv
  3 | import json
  4 | from typing import Callable, List, Any
  5 | 
  6 | from swarms import (
  7 |     GPT4VisionAPI,
  8 |     OpenAIChat,
  9 |     Conversation,
 10 |     SwarmNetwork,
 11 |     Agent,
 12 | )
 13 | 
 14 | from autort.prompts import (
 15 |     FILTER_TASKS_SOP_PROMPT,
 16 |     FUSED_SYSTEM_PROMPT_WITH_SOP,
 17 |     GENERATE_TASKS_PROMPT,
 18 |     VISUALIZE_OBJECT_PROMPT,
 19 | )
 20 | 
 21 | scene_prompt = "Effectively visualize the object in the scene."
 22 | task_prompt = "Generate tasks for the robot to perform."
 23 | rank_prompt = (
 24 |     "Rank the tasks generated based on complexity and necessity of"
 25 |     " human assistance."
 26 | )
 27 | 
 28 | 
 29 | # Load the environment variables from the .env file
 30 | load_dotenv()
 31 | 
 32 | # Set the OpenAI API key
 33 | openai_api_key = os.getenv("OPENAI_API_KEY")
 34 | 
 35 | 
 36 | class AutoRTAgent:
 37 |     """
 38 |     AutoRTSwarm class represents an automated robot task system.
 39 | 
 40 |     Args:
 41 |         openai_api_key (str): The API key for OpenAI.
 42 |         max_tokens (int, optional): The maximum number of tokens to use for text generation. Defaults to 1000.
 43 |         scene_prompt (str, optional): The guidance for the first language model. Defaults to scene_prompt.
 44 |         task_prompt (str, optional): The guidance for the second language model. Defaults to task_prompt.
 45 |         rank_prompt (str, optional): The guidance for the third language model. Defaults to rank_prompt.
 46 |         robot_model (Callable, optional): The robot model to execute the tasks. Defaults to None.
 47 | 
 48 |     Attributes:
 49 |         openai_api_key (str): The API key for OpenAI.
 50 |         max_tokens (int): The maximum number of tokens to use for text generation.
 51 |         scene_prompt (str): The guidance for the first language model.
 52 |         task_prompt (str): The guidance for the second language model.
 53 |         rank_prompt (str): The guidance for the third language model.
 54 |         robot_model (Callable): The robot model to execute the tasks.
 55 |         system_prompt_vllm (FUSED_SYSTEM_PROMPT_WITH_SOP): The system prompt for the first language model.
 56 |         system_prompt_llm (FUSED_SYSTEM_PROMPT_WITH_SOP): The system prompt for the second language model.
 57 |         system_prompt_rank (FUSED_SYSTEM_PROMPT_WITH_SOP): The system prompt for the third language model.
 58 |         vllm (GPT4VisionAPI): The first language model to visualize the object in the scene.
 59 |         llm (OpenAIChat): The second language model to generate tasks for the robot.
 60 |         filter_llm (OpenAIChat): The third language model to rank the tasks generated by the second language model.
 61 | 
 62 |     Methods:
 63 |         run(text: str, img: str, *args, **kwargs) -> Any:
 64 |             Runs the AutoRTSwarm system to execute tasks based on the given text and image.
 65 | 
 66 |     Examples:
 67 |         >>> from autort import AutoRTSwarm
 68 |         >>> autort = AutoRTSwarm(openai_api_key, max_tokens=1000)
 69 |         >>> autort.run("There is a bottle on the table.", "https://i.imgur.com/2qY9f8U.png")
 70 | 
 71 |     """
 72 | 
 73 |     def __init__(
 74 |         self,
 75 |         openai_api_key: str = openai_api_key,
 76 |         max_tokens: int = 1000,
 77 |         scene_prompt: str = scene_prompt,
 78 |         task_prompt: str = task_prompt,
 79 |         rank_prompt: str = rank_prompt,
 80 |         robot_model: Callable = None,
 81 |         *args,
 82 |         **kwargs,
 83 |     ):
 84 |         self.openai_api_key = openai_api_key
 85 |         self.max_tokens = max_tokens
 86 |         self.scene_prompt = scene_prompt
 87 |         self.task_prompt = task_prompt
 88 |         self.rank_prompt = rank_prompt
 89 |         self.robot_model = robot_model
 90 | 
 91 |         self.system_prompt_vllm = FUSED_SYSTEM_PROMPT_WITH_SOP(
 92 |             guidance=scene_prompt,
 93 |             sop=VISUALIZE_OBJECT_PROMPT,
 94 |         )
 95 | 
 96 |         self.system_prompt_llm = FUSED_SYSTEM_PROMPT_WITH_SOP(
 97 |             guidance=task_prompt,
 98 |             sop=GENERATE_TASKS_PROMPT,
 99 |         )
100 | 
101 |         self.system_prompt_rank = FUSED_SYSTEM_PROMPT_WITH_SOP(
102 |             rank_prompt,
103 |             FILTER_TASKS_SOP_PROMPT,
104 |         )
105 | 
106 |         # LLM
107 |         llm = OpenAIChat(openai_api_key=openai_api_key)
108 | 
109 |         # VLM
110 |         vllm = GPT4VisionAPI(openai_api_key=openai_api_key)
111 | 
112 |         # 1st LLM to visualize the object in the scene using GPT4V
113 |         self.vllm = Agent(
114 |             llm=vllm,
115 |             system_prompt=self.system_prompt_vllm,
116 |             *args,
117 |             **kwargs,
118 |         )
119 | 
120 |         # 2nd LLM to generate tasks for the robot to perform based on the objects in the scene
121 |         self.tasks_generator = Agent(
122 |             llm=llm,
123 |             sop_list=[self.system_prompt_llm],
124 |             *args,
125 |             **kwargs,
126 |         )
127 | 
128 |         # 3rd LLM to rank the tasks generated by the 2nd LLM based on complexity and necessity of human assistance
129 |         self.task_filter = Agent(
130 |             llm=llm,
131 |             sop_list=[self.system_prompt_rank],
132 |             *args,
133 |             **kwargs,
134 |         )
135 | 
136 |     def run(self, text: str, img: str, *args, **kwargs):
137 |         """
138 |         Runs the AutoRTSwarm system to execute tasks based on the given text and image.
139 | 
140 |         Args:
141 |             text (str): The text input for the system.
142 |             img (str): The image input for the system.
143 |             *args: Additional positional arguments.
144 |             **kwargs: Additional keyword arguments.
145 | 
146 |         Returns:
147 |             Any: The result of executing the robot model on the ranked tasks.
148 | 
149 |         """
150 |         # 1st LLM to visualize the object in the scene using GPT4V
151 |         vllm = self.vllm.run(text, img, *args, **kwargs)
152 | 
153 |         # 2nd LLM to generate tasks for the robot to perform based on the objects in the scene
154 |         tasks = self.tasks_generator(vllm)
155 | 
156 |         # 3rd LLM to rank the tasks generated by the 2nd LLM based on complexity and necessity of human assistance
157 |         ranks = self.task_filter(tasks)
158 | 
159 |         # Robot model to execute the tasks
160 |         return self.robot_model(ranks, img)
161 | 
162 | 
163 | class AutoRTSwarm(SwarmNetwork):
164 |     """
165 |     Represents a swarm of AutoRTSwarmAgents.
166 | 
167 |     Args:
168 |         agents (List[AutoRTSwarmAgent]): A list of AutoRTSwarmAgent objects.
169 |         datastore (Any): The datastore to store the results of the agents.
170 | 
171 |     Attributes:
172 |         agents (List[AutoRTSwarmAgent]): A list of AutoRTSwarmAgent objects.
173 |         datastore (Any): The datastore to store the results of the agents.
174 | 
175 |     Examples:
176 |         >>> from autort import AutoRTSwarm, AutoRTAgent
177 |         >>> agents = [
178 |         ...     AutoRTAgent(openai_api_key, max_tokens=1000),
179 |         ...     AutoRTAgent(openai_api_key, max_tokens=1000),
180 |         ... ]
181 |         >>> autort_swarm = AutoRTSwarm(agents, datastore)
182 |         >>> autort_swarm.run("There is a bottle on the table.", "https://i.imgur.com/2qY9f8U.png")
183 |     """
184 | 
185 |     def __init__(
186 |         self,
187 |         agents: List[AutoRTAgent],
188 |         datastore: Any = None,
189 |         autosave: bool = True,
190 |         *args,
191 |         **kwargs,
192 |     ):
193 |         super().__init__(agents=agents, *args, **kwargs)
194 |         self.agents = agents
195 |         self.datastore = datastore
196 |         self.autosave = autosave
197 |         self.conversation = Conversation(
198 |             system_prompt=None,
199 |             time_enabled=True,
200 |             save_filepath="autort_conversation.json",
201 |             *args,
202 |             **kwargs,
203 |         )
204 | 
205 |         # Swarm Network
206 |         self.network = SwarmNetwork(
207 |             agents=self.agents, logging_enabled=True, *args, **kwargs
208 |         )
209 | 
210 |     def run(self, text: str, img: str = None, *args, **kwargs):
211 |         """
212 |         Runs the AutoRTSwarmAgents in the swarm.
213 | 
214 |         Args:
215 |             text (str): The text input for the agents.
216 |             img (str): The image input for the agents.
217 |             *args: Additional positional arguments.
218 |             **kwargs: Additional keyword arguments.
219 | 
220 |         Returns:
221 |             List: A list of results from running each agent.
222 |         """
223 |         out = self.network.run_many_agents(text, img, *args, **kwargs)
224 | 
225 |         if self.autosave:
226 |             self.conversation.save_to_json(
227 |                 self.conversation.save_filepath, out
228 |             )
229 | 
230 |         return out
231 | 
232 |     def save_to_json(self, filename: str, content: List):
233 |         # Save the conversation to a JSON file
234 |         with open(filename, "w") as f:
235 |             json.dump(content, f, indent=4)
236 | 
237 |     def load_from_json(self, filename: str):
238 |         # Load the conversation from a JSON file
239 |         with open(filename, "r") as f:
240 |             content = json.load(f)
241 | 
242 |         return content
243 | 
244 |     def __repr__(self):
245 |         return (
246 |             f"AutoRTSwarm(agents={self.agents},"
247 |             f" datastore={self.datastore})"
248 |         )
249 | 
250 |     def __str__(self):
251 |         return f"AutoRTSwarm with {len(self.agents)} agents"
252 | 
253 |     def run_single_agent(
254 |         self,
255 |         id: str = None,
256 |         task: str = None,
257 |         img: str = None,
258 |         *args,
259 |         **kwargs,
260 |     ):
261 |         """
262 |         Run a single agent in the swarm.
263 | 
264 |         Args:
265 |             task (str): The task to be executed by the agent.
266 |             img (str): The image input for the agent.
267 | 
268 |         Returns:
269 |             Any: The result of running the agent.
270 |         """
271 |         out = self.network.run_single_agent(
272 |             id,
273 |             task,
274 |             img,
275 |             *args,
276 |             **kwargs,
277 |         )
278 | 
279 |         return out
280 | 


--------------------------------------------------------------------------------