├── .all-contributorsrc ├── .cursorignore ├── .dockerignore ├── .github └── workflows │ ├── ci-lume.yml │ ├── publish-agent.yml │ ├── publish-computer-server.yml │ ├── publish-computer.yml │ ├── publish-core.yml │ ├── publish-lume.yml │ ├── publish-mcp-server.yml │ ├── publish-pylume.yml │ ├── publish-som.yml │ └── reusable-publish.yml ├── .gitignore ├── .vscode ├── launch.json ├── lume.code-workspace ├── lumier.code-workspace └── py.code-workspace ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE.md ├── README.md ├── docs ├── Developer-Guide.md ├── FAQ.md └── Telemetry.md ├── examples ├── agent_examples.py ├── agent_ui_examples.py ├── computer_examples.py ├── computer_ui_examples.py ├── pylume_examples.py ├── som_examples.py └── utils.py ├── img ├── agent.png ├── agent_gradio_ui.png ├── cli.png ├── computer.png ├── logo_black.png └── logo_white.png ├── libs ├── agent │ ├── README.md │ ├── agent │ │ ├── __init__.py │ │ ├── core │ │ │ ├── __init__.py │ │ │ ├── agent.py │ │ │ ├── base.py │ │ │ ├── callbacks.py │ │ │ ├── experiment.py │ │ │ ├── factory.py │ │ │ ├── messages.py │ │ │ ├── provider_config.py │ │ │ ├── telemetry.py │ │ │ ├── tools.py │ │ │ ├── tools │ │ │ │ ├── __init__.py │ │ │ │ ├── base.py │ │ │ │ ├── bash.py │ │ │ │ ├── collection.py │ │ │ │ ├── computer.py │ │ │ │ ├── edit.py │ │ │ │ └── manager.py │ │ │ ├── types.py │ │ │ └── visualization.py │ │ ├── providers │ │ │ ├── __init__.py │ │ │ ├── anthropic │ │ │ │ ├── __init__.py │ │ │ │ ├── api │ │ │ │ │ ├── client.py │ │ │ │ │ └── logging.py │ │ │ │ ├── api_handler.py │ │ │ │ ├── callbacks │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── manager.py │ │ │ │ ├── loop.py │ │ │ │ ├── prompts.py │ │ │ │ ├── response_handler.py │ │ │ │ ├── tools │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── base.py │ │ │ │ │ ├── bash.py │ │ │ │ │ ├── collection.py │ │ │ │ │ ├── computer.py │ │ │ │ │ ├── edit.py │ │ │ │ │ ├── manager.py │ │ │ │ │ └── run.py │ │ │ │ ├── types.py │ │ │ │ └── utils.py │ │ │ ├── omni │ │ │ │ ├── __init__.py │ │ │ │ ├── api_handler.py │ │ │ │ ├── clients │ │ │ │ │ ├── anthropic.py │ │ │ │ │ ├── base.py │ │ │ │ │ ├── oaicompat.py │ │ │ │ │ ├── ollama.py │ │ │ │ │ ├── openai.py │ │ │ │ │ └── utils.py │ │ │ │ ├── image_utils.py │ │ │ │ ├── loop.py │ │ │ │ ├── parser.py │ │ │ │ ├── prompts.py │ │ │ │ ├── tools │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── base.py │ │ │ │ │ ├── bash.py │ │ │ │ │ ├── computer.py │ │ │ │ │ └── manager.py │ │ │ │ └── utils.py │ │ │ ├── openai │ │ │ │ ├── __init__.py │ │ │ │ ├── api_handler.py │ │ │ │ ├── loop.py │ │ │ │ ├── response_handler.py │ │ │ │ ├── tools │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── base.py │ │ │ │ │ ├── computer.py │ │ │ │ │ └── manager.py │ │ │ │ ├── types.py │ │ │ │ └── utils.py │ │ │ └── uitars │ │ │ │ ├── __init__.py │ │ │ │ ├── clients │ │ │ │ ├── base.py │ │ │ │ ├── mlxvlm.py │ │ │ │ └── oaicompat.py │ │ │ │ ├── loop.py │ │ │ │ ├── prompts.py │ │ │ │ ├── tools │ │ │ │ ├── __init__.py │ │ │ │ ├── computer.py │ │ │ │ └── manager.py │ │ │ │ └── utils.py │ │ ├── telemetry.py │ │ └── ui │ │ │ ├── __init__.py │ │ │ └── gradio │ │ │ ├── __init__.py │ │ │ └── app.py │ ├── poetry.toml │ └── pyproject.toml ├── computer-server │ ├── README.md │ ├── computer_server │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── cli.py │ │ ├── diorama │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── diorama.py │ │ │ ├── diorama_computer.py │ │ │ ├── draw.py │ │ │ ├── macos.py │ │ │ └── safezone.py │ │ ├── handlers │ │ │ ├── base.py │ │ │ ├── factory.py │ │ │ ├── linux.py │ │ │ └── macos.py │ │ ├── main.py │ │ └── server.py │ ├── examples │ │ ├── __init__.py │ │ └── usage_example.py │ ├── pyproject.toml │ ├── run_server.py │ └── test_connection.py ├── computer │ ├── README.md │ ├── computer │ │ ├── __init__.py │ │ ├── computer.py │ │ ├── diorama_computer.py │ │ ├── interface │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── factory.py │ │ │ ├── linux.py │ │ │ ├── macos.py │ │ │ └── models.py │ │ ├── logger.py │ │ ├── models.py │ │ ├── providers │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── cloud │ │ │ │ ├── __init__.py │ │ │ │ └── provider.py │ │ │ ├── factory.py │ │ │ ├── lume │ │ │ │ ├── __init__.py │ │ │ │ └── provider.py │ │ │ ├── lume_api.py │ │ │ └── lumier │ │ │ │ ├── __init__.py │ │ │ │ └── provider.py │ │ ├── telemetry.py │ │ ├── ui │ │ │ ├── __init__.py │ │ │ └── gradio │ │ │ │ ├── __init__.py │ │ │ │ └── app.py │ │ └── utils.py │ ├── poetry.toml │ └── pyproject.toml ├── core │ ├── README.md │ ├── core │ │ ├── __init__.py │ │ └── telemetry │ │ │ ├── __init__.py │ │ │ ├── client.py │ │ │ ├── models.py │ │ │ ├── posthog_client.py │ │ │ ├── sender.py │ │ │ └── telemetry.py │ ├── poetry.toml │ └── pyproject.toml ├── lume │ ├── .cursorignore │ ├── CONTRIBUTING.md │ ├── Package.resolved │ ├── Package.swift │ ├── README.md │ ├── docs │ │ ├── API-Reference.md │ │ ├── Development.md │ │ └── FAQ.md │ ├── img │ │ ├── cli.png │ │ ├── logo_black.png │ │ └── logo_white.png │ ├── resources │ │ └── lume.entitlements │ ├── scripts │ │ ├── build │ │ │ ├── build-debug.sh │ │ │ ├── build-release-notarized.sh │ │ │ └── build-release.sh │ │ └── install.sh │ ├── src │ │ ├── Commands │ │ │ ├── Clone.swift │ │ │ ├── Config.swift │ │ │ ├── Create.swift │ │ │ ├── Delete.swift │ │ │ ├── Get.swift │ │ │ ├── IPSW.swift │ │ │ ├── Images.swift │ │ │ ├── List.swift │ │ │ ├── Logs.swift │ │ │ ├── Options │ │ │ │ └── FormatOption.swift │ │ │ ├── Prune.swift │ │ │ ├── Pull.swift │ │ │ ├── Push.swift │ │ │ ├── Run.swift │ │ │ ├── Serve.swift │ │ │ ├── Set.swift │ │ │ └── Stop.swift │ │ ├── ContainerRegistry │ │ │ ├── ImageContainerRegistry.swift │ │ │ ├── ImageList.swift │ │ │ └── ImagesPrinter.swift │ │ ├── Errors │ │ │ └── Errors.swift │ │ ├── FileSystem │ │ │ ├── Home.swift │ │ │ ├── Settings.swift │ │ │ ├── VMConfig.swift │ │ │ ├── VMDirectory.swift │ │ │ └── VMLocation.swift │ │ ├── LumeController.swift │ │ ├── Main.swift │ │ ├── Server │ │ │ ├── HTTP.swift │ │ │ ├── Handlers.swift │ │ │ ├── Requests.swift │ │ │ ├── Responses.swift │ │ │ └── Server.swift │ │ ├── Utils │ │ │ ├── CommandRegistry.swift │ │ │ ├── CommandUtils.swift │ │ │ ├── Logger.swift │ │ │ ├── NetworkUtils.swift │ │ │ ├── Path.swift │ │ │ ├── ProcessRunner.swift │ │ │ ├── ProgressLogger.swift │ │ │ ├── String.swift │ │ │ └── Utils.swift │ │ ├── VM │ │ │ ├── DarwinVM.swift │ │ │ ├── LinuxVM.swift │ │ │ ├── VM.swift │ │ │ ├── VMDetails.swift │ │ │ ├── VMDetailsPrinter.swift │ │ │ ├── VMDisplayResolution.swift │ │ │ └── VMFactory.swift │ │ ├── VNC │ │ │ ├── PassphraseGenerator.swift │ │ │ └── VNCService.swift │ │ └── Virtualization │ │ │ ├── DHCPLeaseParser.swift │ │ │ ├── DarwinImageLoader.swift │ │ │ ├── ImageLoaderFactory.swift │ │ │ └── VMVirtualizationService.swift │ └── tests │ │ ├── Mocks │ │ ├── MockVM.swift │ │ ├── MockVMVirtualizationService.swift │ │ └── MockVNCService.swift │ │ ├── VM │ │ └── VMDetailsPrinterTests.swift │ │ ├── VMTests.swift │ │ ├── VMVirtualizationServiceTests.swift │ │ └── VNCServiceTests.swift ├── lumier │ ├── .dockerignore │ ├── Dockerfile │ ├── README.md │ └── src │ │ ├── bin │ │ └── entry.sh │ │ ├── config │ │ └── constants.sh │ │ ├── hooks │ │ └── on-logon.sh │ │ └── lib │ │ ├── utils.sh │ │ └── vm.sh ├── mcp-server │ ├── README.md │ ├── mcp_server │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── server.py │ ├── pyproject.toml │ └── scripts │ │ ├── install_mcp_server.sh │ │ └── start_mcp_server.sh ├── pylume │ ├── README.md │ ├── __init__.py │ ├── pylume │ │ ├── __init__.py │ │ ├── client.py │ │ ├── exceptions.py │ │ ├── lume │ │ ├── models.py │ │ ├── pylume.py │ │ └── server.py │ └── pyproject.toml └── som │ ├── README.md │ ├── poetry.toml │ ├── pyproject.toml │ ├── som │ ├── __init__.py │ ├── detect.py │ ├── detection.py │ ├── models.py │ ├── ocr.py │ ├── util │ │ └── utils.py │ └── visualization.py │ └── tests │ └── test_omniparser.py ├── notebooks ├── agent_nb.ipynb ├── blog │ ├── build-your-own-operator-on-macos-1.ipynb │ └── build-your-own-operator-on-macos-2.ipynb ├── computer_nb.ipynb ├── computer_server_nb.ipynb └── pylume_nb.ipynb ├── pyproject.toml ├── pyrightconfig.json └── scripts ├── build.sh ├── cleanup.sh ├── playground.sh └── run-docker-dev.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | # Version control 2 | .git 3 | .github 4 | .gitignore 5 | 6 | # Environment and cache 7 | .venv 8 | .env 9 | .env.local 10 | __pycache__ 11 | *.pyc 12 | *.pyo 13 | *.pyd 14 | .Python 15 | .pytest_cache 16 | .pdm-build 17 | 18 | # Distribution / packaging 19 | dist 20 | build 21 | *.egg-info 22 | 23 | # Development 24 | .vscode 25 | .idea 26 | *.swp 27 | *.swo 28 | 29 | # Docs 30 | docs/site 31 | 32 | # Notebooks 33 | notebooks/.ipynb_checkpoints 34 | 35 | # Docker 36 | Dockerfile 37 | .dockerignore -------------------------------------------------------------------------------- /.github/workflows/ci-lume.yml: -------------------------------------------------------------------------------- 1 | name: lume 2 | on: 3 | push: 4 | branches: 5 | - "main" 6 | pull_request: {} 7 | 8 | concurrency: 9 | group: lume-${{ github.workflow }}-${{ github.ref }} 10 | cancel-in-progress: true 11 | 12 | # Runner images: https://github.com/actions/runner-images 13 | 14 | jobs: 15 | test: 16 | name: Test 17 | runs-on: macos-15 18 | steps: 19 | - uses: actions/checkout@v4 20 | - run: uname -a 21 | - run: sudo xcode-select -s /Applications/Xcode_16.app # Swift 6.0 22 | - run: swift test 23 | working-directory: ./libs/lume 24 | build: 25 | name: Release build 26 | runs-on: macos-15 27 | steps: 28 | - uses: actions/checkout@v4 29 | - run: uname -a 30 | - run: sudo xcode-select -s /Applications/Xcode_16.app # Swift 6.0 31 | - run: swift build --configuration release 32 | working-directory: ./libs/lume 33 | -------------------------------------------------------------------------------- /.github/workflows/publish-computer-server.yml: -------------------------------------------------------------------------------- 1 | name: Publish Computer Server Package 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'computer-server-v*' 7 | workflow_dispatch: 8 | inputs: 9 | version: 10 | description: 'Version to publish (without v prefix)' 11 | required: true 12 | default: '0.1.0' 13 | workflow_call: 14 | inputs: 15 | version: 16 | description: 'Version to publish' 17 | required: true 18 | type: string 19 | outputs: 20 | version: 21 | description: "The version that was published" 22 | value: ${{ jobs.prepare.outputs.version }} 23 | 24 | # Adding permissions at workflow level 25 | permissions: 26 | contents: write 27 | 28 | jobs: 29 | prepare: 30 | runs-on: macos-latest 31 | outputs: 32 | version: ${{ steps.get-version.outputs.version }} 33 | steps: 34 | - uses: actions/checkout@v4 35 | 36 | - name: Determine version 37 | id: get-version 38 | run: | 39 | if [ "${{ github.event_name }}" == "push" ]; then 40 | # Extract version from tag (for package-specific tags) 41 | if [[ "${{ github.ref }}" =~ ^refs/tags/computer-server-v([0-9]+\.[0-9]+\.[0-9]+) ]]; then 42 | VERSION=${BASH_REMATCH[1]} 43 | else 44 | echo "Invalid tag format for computer-server" 45 | exit 1 46 | fi 47 | elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then 48 | # Use version from workflow dispatch 49 | VERSION=${{ github.event.inputs.version }} 50 | else 51 | # Use version from workflow_call 52 | VERSION=${{ inputs.version }} 53 | fi 54 | echo "VERSION=$VERSION" 55 | echo "version=$VERSION" >> $GITHUB_OUTPUT 56 | 57 | - name: Set up Python 58 | uses: actions/setup-python@v4 59 | with: 60 | python-version: '3.10' 61 | 62 | publish: 63 | needs: prepare 64 | uses: ./.github/workflows/reusable-publish.yml 65 | with: 66 | package_name: "computer-server" 67 | package_dir: "libs/computer-server" 68 | version: ${{ needs.prepare.outputs.version }} 69 | is_lume_package: false 70 | base_package_name: "cua-computer-server" 71 | secrets: 72 | PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} 73 | 74 | set-env-variables: 75 | needs: [prepare, publish] 76 | runs-on: macos-latest 77 | steps: 78 | - name: Set environment variables for use in other jobs 79 | run: | 80 | echo "COMPUTER_VERSION=${{ needs.prepare.outputs.version }}" >> $GITHUB_ENV -------------------------------------------------------------------------------- /.github/workflows/publish-core.yml: -------------------------------------------------------------------------------- 1 | name: Publish Core Package 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'core-v*' 7 | workflow_dispatch: 8 | inputs: 9 | version: 10 | description: 'Version to publish (without v prefix)' 11 | required: true 12 | default: '0.1.0' 13 | workflow_call: 14 | inputs: 15 | version: 16 | description: 'Version to publish' 17 | required: true 18 | type: string 19 | 20 | # Adding permissions at workflow level 21 | permissions: 22 | contents: write 23 | 24 | jobs: 25 | prepare: 26 | runs-on: macos-latest 27 | outputs: 28 | version: ${{ steps.get-version.outputs.version }} 29 | steps: 30 | - uses: actions/checkout@v4 31 | 32 | - name: Determine version 33 | id: get-version 34 | run: | 35 | if [ "${{ github.event_name }}" == "push" ]; then 36 | # Extract version from tag (for package-specific tags) 37 | if [[ "${{ github.ref }}" =~ ^refs/tags/core-v([0-9]+\.[0-9]+\.[0-9]+) ]]; then 38 | VERSION=${BASH_REMATCH[1]} 39 | else 40 | echo "Invalid tag format for core" 41 | exit 1 42 | fi 43 | elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then 44 | # Use version from workflow dispatch 45 | VERSION=${{ github.event.inputs.version }} 46 | else 47 | # Use version from workflow_call 48 | VERSION=${{ inputs.version }} 49 | fi 50 | echo "VERSION=$VERSION" 51 | echo "version=$VERSION" >> $GITHUB_OUTPUT 52 | 53 | publish: 54 | needs: prepare 55 | uses: ./.github/workflows/reusable-publish.yml 56 | with: 57 | package_name: "core" 58 | package_dir: "libs/core" 59 | version: ${{ needs.prepare.outputs.version }} 60 | is_lume_package: false 61 | base_package_name: "cua-core" 62 | secrets: 63 | PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/publish-pylume.yml: -------------------------------------------------------------------------------- 1 | name: Publish Pylume Package 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'pylume-v*' 7 | workflow_dispatch: 8 | inputs: 9 | version: 10 | description: 'Version to publish (without v prefix)' 11 | required: true 12 | default: '0.1.0' 13 | workflow_call: 14 | inputs: 15 | version: 16 | description: 'Version to publish' 17 | required: true 18 | type: string 19 | outputs: 20 | version: 21 | description: "The version that was published" 22 | value: ${{ jobs.determine-version.outputs.version }} 23 | 24 | # Adding permissions at workflow level 25 | permissions: 26 | contents: write 27 | 28 | jobs: 29 | determine-version: 30 | runs-on: macos-latest 31 | outputs: 32 | version: ${{ steps.get-version.outputs.version }} 33 | steps: 34 | - uses: actions/checkout@v4 35 | 36 | - name: Determine version 37 | id: get-version 38 | run: | 39 | if [ "${{ github.event_name }}" == "push" ]; then 40 | # Extract version from tag (for package-specific tags) 41 | if [[ "${{ github.ref }}" =~ ^refs/tags/pylume-v([0-9]+\.[0-9]+\.[0-9]+) ]]; then 42 | VERSION=${BASH_REMATCH[1]} 43 | else 44 | echo "Invalid tag format for pylume" 45 | exit 1 46 | fi 47 | elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then 48 | # Use version from workflow dispatch 49 | VERSION=${{ github.event.inputs.version }} 50 | else 51 | # Use version from workflow_call 52 | VERSION=${{ inputs.version }} 53 | fi 54 | echo "VERSION=$VERSION" 55 | echo "version=$VERSION" >> $GITHUB_OUTPUT 56 | 57 | validate-version: 58 | runs-on: macos-latest 59 | needs: determine-version 60 | steps: 61 | - uses: actions/checkout@v4 62 | - name: Validate version 63 | id: validate-version 64 | run: | 65 | CODE_VERSION=$(grep '__version__' libs/pylume/pylume/__init__.py | cut -d'"' -f2) 66 | if [ "${{ needs.determine-version.outputs.version }}" != "$CODE_VERSION" ]; then 67 | echo "Version mismatch: expected $CODE_VERSION, got ${{ needs.determine-version.outputs.version }}" 68 | exit 1 69 | fi 70 | echo "Version validated: $CODE_VERSION" 71 | 72 | publish: 73 | needs: determine-version 74 | uses: ./.github/workflows/reusable-publish.yml 75 | with: 76 | package_name: "pylume" 77 | package_dir: "libs/pylume" 78 | version: ${{ needs.determine-version.outputs.version }} 79 | is_lume_package: true 80 | base_package_name: "pylume" 81 | secrets: 82 | PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/publish-som.yml: -------------------------------------------------------------------------------- 1 | name: Publish SOM Package 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'som-v*' 7 | workflow_dispatch: 8 | inputs: 9 | version: 10 | description: 'Version to publish (without v prefix)' 11 | required: true 12 | default: '0.1.0' 13 | workflow_call: 14 | inputs: 15 | version: 16 | description: 'Version to publish' 17 | required: true 18 | type: string 19 | outputs: 20 | version: 21 | description: "The version that was published" 22 | value: ${{ jobs.determine-version.outputs.version }} 23 | 24 | # Adding permissions at workflow level 25 | permissions: 26 | contents: write 27 | 28 | jobs: 29 | determine-version: 30 | runs-on: macos-latest 31 | outputs: 32 | version: ${{ steps.get-version.outputs.version }} 33 | steps: 34 | - uses: actions/checkout@v4 35 | 36 | - name: Determine version 37 | id: get-version 38 | run: | 39 | if [ "${{ github.event_name }}" == "push" ]; then 40 | # Extract version from tag (for package-specific tags) 41 | if [[ "${{ github.ref }}" =~ ^refs/tags/som-v([0-9]+\.[0-9]+\.[0-9]+) ]]; then 42 | VERSION=${BASH_REMATCH[1]} 43 | else 44 | echo "Invalid tag format for som" 45 | exit 1 46 | fi 47 | elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then 48 | # Use version from workflow dispatch 49 | VERSION=${{ github.event.inputs.version }} 50 | else 51 | # Use version from workflow_call 52 | VERSION=${{ inputs.version }} 53 | fi 54 | echo "VERSION=$VERSION" 55 | echo "version=$VERSION" >> $GITHUB_OUTPUT 56 | 57 | publish: 58 | needs: determine-version 59 | uses: ./.github/workflows/reusable-publish.yml 60 | with: 61 | package_name: "som" 62 | package_dir: "libs/som" 63 | version: ${{ needs.determine-version.outputs.version }} 64 | is_lume_package: false 65 | base_package_name: "cua-som" 66 | secrets: 67 | PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} -------------------------------------------------------------------------------- /.vscode/lumier.code-workspace: -------------------------------------------------------------------------------- 1 | { 2 | "folders": [ 3 | { 4 | "name": "lumier", 5 | "path": "../libs/lumier" 6 | }, 7 | { 8 | "name": "lume", 9 | "path": "../libs/lume" 10 | } 11 | ], 12 | "settings": { 13 | "files.exclude": { 14 | "**/.git": true, 15 | "**/.svn": true, 16 | "**/.hg": true, 17 | "**/CVS": true, 18 | "**/.DS_Store": true 19 | } 20 | }, 21 | "tasks": { 22 | "version": "2.0.0", 23 | "tasks": [ 24 | ] 25 | }, 26 | "launch": { 27 | "configurations": [ 28 | ] 29 | } 30 | } -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to cua 2 | 3 | We deeply appreciate your interest in contributing to cua! Whether you're reporting bugs, suggesting enhancements, improving docs, or submitting pull requests, your contributions help improve the project for everyone. 4 | 5 | ## Reporting Bugs 6 | 7 | If you've encountered a bug in the project, we encourage you to report it. Please follow these steps: 8 | 9 | 1. **Check the Issue Tracker**: Before submitting a new bug report, please check our issue tracker to see if the bug has already been reported. 10 | 2. **Create a New Issue**: If the bug hasn't been reported, create a new issue with: 11 | - A clear title and detailed description 12 | - Steps to reproduce the issue 13 | - Expected vs actual behavior 14 | - Your environment (macOS version, lume version) 15 | - Any relevant logs or error messages 16 | 3. **Label Your Issue**: Label your issue as a `bug` to help maintainers identify it quickly. 17 | 18 | ## Suggesting Enhancements 19 | 20 | We're always looking for suggestions to make lume better. If you have an idea: 21 | 22 | 1. **Check Existing Issues**: See if someone else has already suggested something similar. 23 | 2. **Create a New Issue**: If your enhancement is new, create an issue describing: 24 | - The problem your enhancement solves 25 | - How your enhancement would work 26 | - Any potential implementation details 27 | - Why this enhancement would benefit lume users 28 | 29 | ## Code Formatting 30 | 31 | We follow strict code formatting guidelines to ensure consistency across the codebase. Before submitting any code: 32 | 33 | 1. **Review Our Format Guide**: Please review our [Code Formatting Standards](docs/Developer-Guide.md#code-formatting-standards) section in the Getting Started guide. 34 | 2. **Configure Your IDE**: We recommend using the workspace settings provided in `.vscode/` for automatic formatting. 35 | 3. **Run Formatting Tools**: Always run the formatting tools before submitting a PR: 36 | ```bash 37 | # For Python code 38 | pdm run black . 39 | pdm run ruff check --fix . 40 | ``` 41 | 4. **Validate Your Code**: Ensure your code passes all checks: 42 | ```bash 43 | pdm run mypy . 44 | ``` 45 | 46 | ## Documentation 47 | 48 | Documentation improvements are always welcome. You can: 49 | - Fix typos or unclear explanations 50 | - Add examples and use cases 51 | - Improve API documentation 52 | - Add tutorials or guides 53 | 54 | For detailed instructions on setting up your development environment and submitting code contributions, please see our [Developer-Guide](./docs/Developer-Guide.md) guide. 55 | 56 | Feel free to join our [Discord community](https://discord.com/invite/mVnXXpdE85) to discuss ideas or get help with your contributions. -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim 2 | 3 | # Set environment variables 4 | ENV PYTHONUNBUFFERED=1 \ 5 | PYTHONDONTWRITEBYTECODE=1 \ 6 | PIP_NO_CACHE_DIR=1 \ 7 | PIP_DISABLE_PIP_VERSION_CHECK=1 \ 8 | PYTHONPATH="/app/libs/core:/app/libs/computer:/app/libs/agent:/app/libs/som:/app/libs/pylume:/app/libs/computer-server" 9 | 10 | # Install system dependencies for ARM architecture 11 | RUN apt-get update && apt-get install -y --no-install-recommends \ 12 | git \ 13 | build-essential \ 14 | libgl1-mesa-glx \ 15 | libglib2.0-0 \ 16 | libxcb-xinerama0 \ 17 | libxkbcommon-x11-0 \ 18 | cmake \ 19 | pkg-config \ 20 | curl \ 21 | iputils-ping \ 22 | net-tools \ 23 | sed \ 24 | && apt-get clean \ 25 | && rm -rf /var/lib/apt/lists/* 26 | 27 | # Set working directory 28 | WORKDIR /app 29 | 30 | # Copy the entire project temporarily 31 | # We'll mount the real source code over this at runtime 32 | COPY . /app/ 33 | 34 | # Create a simple .env.local file for build.sh 35 | RUN echo "PYTHON_BIN=python" > /app/.env.local 36 | 37 | # Modify build.sh to skip virtual environment creation 38 | RUN sed -i 's/python -m venv .venv/echo "Skipping venv creation in Docker"/' /app/scripts/build.sh && \ 39 | sed -i 's/source .venv\/bin\/activate/echo "Skipping venv activation in Docker"/' /app/scripts/build.sh && \ 40 | sed -i 's/find . -type d -name ".venv" -exec rm -rf {} +/echo "Skipping .venv removal in Docker"/' /app/scripts/build.sh && \ 41 | chmod +x /app/scripts/build.sh 42 | 43 | # Run the build script to install dependencies 44 | RUN cd /app && ./scripts/build.sh 45 | 46 | # Clean up the source files now that dependencies are installed 47 | # When we run the container, we'll mount the actual source code 48 | RUN rm -rf /app/* /app/.??* 49 | 50 | # Note: This Docker image doesn't contain the lume executable (macOS-specific) 51 | # Instead, it relies on connecting to a lume server running on the host machine 52 | # via host.docker.internal:7777 53 | 54 | # Default command 55 | CMD ["bash"] -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 trycua 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /docs/Telemetry.md: -------------------------------------------------------------------------------- 1 | # Telemetry in CUA 2 | 3 | This document explains how telemetry works in CUA libraries and how you can control it. 4 | 5 | CUA tracks anonymized usage and error report statistics; we ascribe to Posthog's approach as detailed [here](https://posthog.com/blog/open-source-telemetry-ethical). If you would like to opt out of sending anonymized info, you can set `telemetry_enabled` to false. 6 | 7 | ## What telemetry data we collect 8 | 9 | CUA libraries collect minimal anonymous usage data to help improve our software. The telemetry data we collect is specifically limited to: 10 | 11 | - Basic system information: 12 | - Operating system (e.g., 'darwin', 'win32', 'linux') 13 | - Python version (e.g., '3.11.0') 14 | - Module initialization events: 15 | - When a module (like 'computer' or 'agent') is imported 16 | - Version of the module being used 17 | 18 | We do NOT collect: 19 | - Personal information 20 | - Contents of files 21 | - Specific text being typed 22 | - Actual screenshots or screen contents 23 | - User-specific identifiers 24 | - API keys 25 | - File contents 26 | - Application data or content 27 | - User interactions with the computer 28 | - Information about files being accessed 29 | 30 | ## Controlling Telemetry 31 | 32 | We are committed to transparency and user control over telemetry. There are two ways to control telemetry: 33 | 34 | ## 1. Environment Variable (Global Control) 35 | 36 | Telemetry is enabled by default. To disable telemetry, set the `CUA_TELEMETRY_ENABLED` environment variable to a falsy value (`0`, `false`, `no`, or `off`): 37 | 38 | ```bash 39 | # Disable telemetry before running your script 40 | export CUA_TELEMETRY_ENABLED=false 41 | 42 | # Or as part of the command 43 | CUA_TELEMETRY_ENABLED=1 python your_script.py 44 | 45 | ``` 46 | Or from Python: 47 | ```python 48 | import os 49 | os.environ["CUA_TELEMETRY_ENABLED"] = "false" 50 | ``` 51 | 52 | ## 2. Instance-Level Control 53 | 54 | You can control telemetry for specific CUA instances by setting `telemetry_enabled` when creating them: 55 | 56 | ```python 57 | # Disable telemetry for a specific Computer instance 58 | computer = Computer(telemetry_enabled=False) 59 | 60 | # Enable telemetry for a specific Agent instance 61 | agent = ComputerAgent(telemetry_enabled=True) 62 | ``` 63 | 64 | You can check if telemetry is enabled for an instance: 65 | 66 | ```python 67 | print(computer.telemetry_enabled) # Will print True or False 68 | ``` 69 | 70 | Note that telemetry settings must be configured during initialization and cannot be changed after the object is created. 71 | 72 | ## Transparency 73 | 74 | We believe in being transparent about the data we collect. If you have any questions about our telemetry practices, please open an issue on our GitHub repository. -------------------------------------------------------------------------------- /examples/agent_ui_examples.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Simple example script for the Computer-Use Agent Gradio UI. 4 | 5 | This script launches the advanced Gradio UI for the Computer-Use Agent 6 | with full model selection and configuration options. 7 | It can be run directly from the command line. 8 | """ 9 | 10 | 11 | from utils import load_dotenv_files 12 | 13 | load_dotenv_files() 14 | 15 | # Import the create_gradio_ui function 16 | from agent.ui.gradio.app import create_gradio_ui 17 | 18 | if __name__ == "__main__": 19 | print("Launching Computer-Use Agent Gradio UI with advanced features...") 20 | app = create_gradio_ui() 21 | app.launch(share=False) 22 | -------------------------------------------------------------------------------- /examples/computer_ui_examples.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Simple example script for the Computer Interface Gradio UI. 4 | 5 | This script launches the advanced Gradio UI for the Computer Interface 6 | with full model selection and configuration options. 7 | It can be run directly from the command line. 8 | """ 9 | 10 | 11 | from utils import load_dotenv_files 12 | 13 | load_dotenv_files() 14 | 15 | # Import the create_gradio_ui function 16 | from computer.ui.gradio.app import create_gradio_ui 17 | 18 | if __name__ == "__main__": 19 | print("Launching Computer Interface Gradio UI with advanced features...") 20 | app = create_gradio_ui() 21 | app.launch(share=False) 22 | 23 | # Optional: Using the saved dataset 24 | # import datasets 25 | # from computer.ui.utils import convert_to_unsloth 26 | # ds = datasets.load_dataset("ddupont/highquality-cua-demonstrations") 27 | # ds = convert_to_unsloth(ds) -------------------------------------------------------------------------------- /examples/pylume_examples.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from pylume import PyLume, ImageRef, VMRunOpts, SharedDirectory, VMConfig, VMUpdateOpts 3 | 4 | 5 | async def main(): 6 | """Example usage of PyLume.""" 7 | async with PyLume(port=7777, use_existing_server=False, debug=True) as pylume: 8 | 9 | # Get latest IPSW URL 10 | print("\n=== Getting Latest IPSW URL ===") 11 | url = await pylume.get_latest_ipsw_url() 12 | print("Latest IPSW URL:", url) 13 | 14 | # Create a new VM 15 | print("\n=== Creating a new VM ===") 16 | vm_config = VMConfig( 17 | name="lume-vm-new", 18 | os="macOS", 19 | cpu=2, 20 | memory="4GB", 21 | disk_size="64GB", # type: ignore 22 | display="1024x768", 23 | ipsw="latest", 24 | ) 25 | await pylume.create_vm(vm_config) 26 | 27 | # Get latest IPSW URL 28 | print("\n=== Getting Latest IPSW URL ===") 29 | url = await pylume.get_latest_ipsw_url() 30 | print("Latest IPSW URL:", url) 31 | 32 | # List available images 33 | print("\n=== Listing Available Images ===") 34 | images = await pylume.get_images() 35 | print("Available Images:", images) 36 | 37 | # List all VMs to verify creation 38 | print("\n=== Listing All VMs ===") 39 | vms = await pylume.list_vms() 40 | print("VMs:", vms) 41 | 42 | # Get specific VM details 43 | print("\n=== Getting VM Details ===") 44 | vm = await pylume.get_vm("lume-vm") 45 | print("VM Details:", vm) 46 | 47 | # Update VM settings 48 | print("\n=== Updating VM Settings ===") 49 | update_opts = VMUpdateOpts(cpu=8, memory="4GB") 50 | await pylume.update_vm("lume-vm", update_opts) 51 | 52 | # Pull an image 53 | image_ref = ImageRef( 54 | image="macos-sequoia-vanilla", tag="latest", registry="ghcr.io", organization="trycua" 55 | ) 56 | await pylume.pull_image(image_ref, name="lume-vm-pulled") 57 | 58 | # Run with shared directory 59 | run_opts = VMRunOpts( 60 | no_display=False, # type: ignore 61 | shared_directories=[ # type: ignore 62 | SharedDirectory(host_path="~/shared", read_only=False) # type: ignore 63 | ], 64 | ) 65 | await pylume.run_vm("lume-vm", run_opts) 66 | 67 | # Or simpler: 68 | await pylume.run_vm("lume-vm") 69 | 70 | # Clone VM 71 | print("\n=== Cloning VM ===") 72 | await pylume.clone_vm("lume-vm", "lume-vm-cloned") 73 | 74 | # Stop VM 75 | print("\n=== Stopping VM ===") 76 | await pylume.stop_vm("lume-vm") 77 | 78 | # Delete VM 79 | print("\n=== Deleting VM ===") 80 | await pylume.delete_vm("lume-vm-cloned") 81 | 82 | 83 | if __name__ == "__main__": 84 | asyncio.run(main()) 85 | -------------------------------------------------------------------------------- /examples/utils.py: -------------------------------------------------------------------------------- 1 | """Utility functions for example scripts.""" 2 | 3 | import os 4 | import sys 5 | import signal 6 | from pathlib import Path 7 | from typing import Optional 8 | 9 | 10 | def load_env_file(path: Path) -> bool: 11 | """Load environment variables from a file. 12 | 13 | Args: 14 | path: Path to the .env file 15 | 16 | Returns: 17 | True if file was loaded successfully, False otherwise 18 | """ 19 | if not path.exists(): 20 | return False 21 | 22 | print(f"Loading environment from {path}") 23 | with open(path, "r") as f: 24 | for line in f: 25 | line = line.strip() 26 | if not line or line.startswith("#"): 27 | continue 28 | 29 | key, value = line.split("=", 1) 30 | os.environ[key] = value 31 | 32 | return True 33 | 34 | 35 | def load_dotenv_files(): 36 | """Load environment variables from .env files. 37 | 38 | Tries to load from .env.local first, then .env if .env.local doesn't exist. 39 | """ 40 | # Get the project root directory (parent of the examples directory) 41 | project_root = Path(__file__).parent.parent 42 | 43 | # Try loading .env.local first, then .env if .env.local doesn't exist 44 | env_local_path = project_root / ".env.local" 45 | env_path = project_root / ".env" 46 | 47 | # Load .env.local if it exists, otherwise try .env 48 | if not load_env_file(env_local_path): 49 | load_env_file(env_path) 50 | 51 | 52 | def handle_sigint(signum, frame): 53 | """Handle SIGINT (Ctrl+C) gracefully.""" 54 | print("\nExiting gracefully...") 55 | sys.exit(0) 56 | -------------------------------------------------------------------------------- /img/agent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trycua/cua/99b979ef114345fae36dba627f33577388759f47/img/agent.png -------------------------------------------------------------------------------- /img/agent_gradio_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trycua/cua/99b979ef114345fae36dba627f33577388759f47/img/agent_gradio_ui.png -------------------------------------------------------------------------------- /img/cli.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trycua/cua/99b979ef114345fae36dba627f33577388759f47/img/cli.png -------------------------------------------------------------------------------- /img/computer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trycua/cua/99b979ef114345fae36dba627f33577388759f47/img/computer.png -------------------------------------------------------------------------------- /img/logo_black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trycua/cua/99b979ef114345fae36dba627f33577388759f47/img/logo_black.png -------------------------------------------------------------------------------- /img/logo_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trycua/cua/99b979ef114345fae36dba627f33577388759f47/img/logo_white.png -------------------------------------------------------------------------------- /libs/agent/agent/__init__.py: -------------------------------------------------------------------------------- 1 | """CUA (Computer Use) Agent for AI-driven computer interaction.""" 2 | 3 | import sys 4 | import logging 5 | 6 | __version__ = "0.1.0" 7 | 8 | # Initialize logging 9 | logger = logging.getLogger("cua.agent") 10 | 11 | # Initialize telemetry when the package is imported 12 | try: 13 | # Import from core telemetry for basic functions 14 | from core.telemetry import ( 15 | is_telemetry_enabled, 16 | flush, 17 | record_event, 18 | ) 19 | 20 | # Import set_dimension from our own telemetry module 21 | from .core.telemetry import set_dimension 22 | 23 | # Check if telemetry is enabled 24 | if is_telemetry_enabled(): 25 | logger.info("Telemetry is enabled") 26 | 27 | # Record package initialization 28 | record_event( 29 | "module_init", 30 | { 31 | "module": "agent", 32 | "version": __version__, 33 | "python_version": sys.version, 34 | }, 35 | ) 36 | 37 | # Set the package version as a dimension 38 | set_dimension("agent_version", __version__) 39 | 40 | # Flush events to ensure they're sent 41 | flush() 42 | else: 43 | logger.info("Telemetry is disabled") 44 | except ImportError as e: 45 | # Telemetry not available 46 | logger.warning(f"Telemetry not available: {e}") 47 | except Exception as e: 48 | # Other issues with telemetry 49 | logger.warning(f"Error initializing telemetry: {e}") 50 | 51 | from .core.types import LLMProvider, LLM 52 | from .core.factory import AgentLoop 53 | from .core.agent import ComputerAgent 54 | 55 | __all__ = ["AgentLoop", "LLMProvider", "LLM", "ComputerAgent"] 56 | -------------------------------------------------------------------------------- /libs/agent/agent/core/__init__.py: -------------------------------------------------------------------------------- 1 | """Core agent components.""" 2 | 3 | from .factory import BaseLoop 4 | from .messages import ( 5 | BaseMessageManager, 6 | ImageRetentionConfig, 7 | ) 8 | from .callbacks import ( 9 | CallbackManager, 10 | CallbackHandler, 11 | BaseCallbackManager, 12 | ContentCallback, 13 | ToolCallback, 14 | APICallback, 15 | ) 16 | 17 | __all__ = [ 18 | "BaseLoop", 19 | "CallbackManager", 20 | "CallbackHandler", 21 | "BaseMessageManager", 22 | "ImageRetentionConfig", 23 | "BaseCallbackManager", 24 | "ContentCallback", 25 | "ToolCallback", 26 | "APICallback", 27 | ] 28 | -------------------------------------------------------------------------------- /libs/agent/agent/core/provider_config.py: -------------------------------------------------------------------------------- 1 | """Provider-specific configurations and constants.""" 2 | 3 | from .types import LLMProvider 4 | 5 | # Default models for different providers 6 | DEFAULT_MODELS = { 7 | LLMProvider.OPENAI: "gpt-4o", 8 | LLMProvider.ANTHROPIC: "claude-3-7-sonnet-20250219", 9 | LLMProvider.OLLAMA: "gemma3:4b-it-q4_K_M", 10 | LLMProvider.OAICOMPAT: "Qwen2.5-VL-7B-Instruct", 11 | LLMProvider.MLXVLM: "mlx-community/UI-TARS-1.5-7B-4bit", 12 | } 13 | 14 | # Map providers to their environment variable names 15 | ENV_VARS = { 16 | LLMProvider.OPENAI: "OPENAI_API_KEY", 17 | LLMProvider.ANTHROPIC: "ANTHROPIC_API_KEY", 18 | LLMProvider.OLLAMA: "none", 19 | LLMProvider.OAICOMPAT: "none", # OpenAI-compatible API typically doesn't require an API key 20 | LLMProvider.MLXVLM: "none", # MLX VLM typically doesn't require an API key 21 | } 22 | -------------------------------------------------------------------------------- /libs/agent/agent/core/tools.py: -------------------------------------------------------------------------------- 1 | """Tool-related type definitions.""" 2 | 3 | from enum import StrEnum 4 | from typing import Dict, Any, Optional 5 | from pydantic import BaseModel, ConfigDict 6 | 7 | class ToolInvocationState(StrEnum): 8 | """States for tool invocation.""" 9 | CALL = 'call' 10 | PARTIAL_CALL = 'partial-call' 11 | RESULT = 'result' 12 | 13 | class ToolInvocation(BaseModel): 14 | """Tool invocation type.""" 15 | model_config = ConfigDict(extra='forbid') 16 | state: Optional[str] = None 17 | toolCallId: str 18 | toolName: Optional[str] = None 19 | args: Optional[Dict[str, Any]] = None 20 | 21 | class ClientAttachment(BaseModel): 22 | """Client attachment type.""" 23 | name: str 24 | contentType: str 25 | url: str 26 | 27 | class ToolResult(BaseModel): 28 | """Result of a tool execution.""" 29 | model_config = ConfigDict(extra='forbid') 30 | output: Optional[str] = None 31 | error: Optional[str] = None 32 | metadata: Optional[Dict[str, Any]] = None 33 | -------------------------------------------------------------------------------- /libs/agent/agent/core/tools/__init__.py: -------------------------------------------------------------------------------- 1 | """Core tools package.""" 2 | 3 | from .base import BaseTool, ToolResult, ToolError, ToolFailure, CLIResult 4 | from .bash import BaseBashTool 5 | from .collection import ToolCollection 6 | from .computer import BaseComputerTool 7 | from .edit import BaseEditTool 8 | from .manager import BaseToolManager 9 | 10 | __all__ = [ 11 | "BaseTool", 12 | "ToolResult", 13 | "ToolError", 14 | "ToolFailure", 15 | "CLIResult", 16 | "BaseBashTool", 17 | "BaseComputerTool", 18 | "BaseEditTool", 19 | "ToolCollection", 20 | "BaseToolManager", 21 | ] 22 | -------------------------------------------------------------------------------- /libs/agent/agent/core/tools/base.py: -------------------------------------------------------------------------------- 1 | """Abstract base classes for tools that can be used with any provider.""" 2 | 3 | from abc import ABCMeta, abstractmethod 4 | from dataclasses import dataclass, fields, replace 5 | from typing import Any, Dict 6 | 7 | 8 | class BaseTool(metaclass=ABCMeta): 9 | """Abstract base class for provider-agnostic tools.""" 10 | 11 | name: str 12 | 13 | @abstractmethod 14 | async def __call__(self, **kwargs) -> Any: 15 | """Executes the tool with the given arguments.""" 16 | ... 17 | 18 | @abstractmethod 19 | def to_params(self) -> Dict[str, Any]: 20 | """Convert tool to provider-specific API parameters. 21 | 22 | Returns: 23 | Dictionary with tool parameters specific to the LLM provider 24 | """ 25 | raise NotImplementedError 26 | 27 | 28 | @dataclass(kw_only=True, frozen=True) 29 | class ToolResult: 30 | """Represents the result of a tool execution.""" 31 | 32 | output: str | None = None 33 | error: str | None = None 34 | base64_image: str | None = None 35 | system: str | None = None 36 | content: list[dict] | None = None 37 | 38 | def __bool__(self): 39 | return any(getattr(self, field.name) for field in fields(self)) 40 | 41 | def __add__(self, other: "ToolResult"): 42 | def combine_fields(field: str | None, other_field: str | None, concatenate: bool = True): 43 | if field and other_field: 44 | if concatenate: 45 | return field + other_field 46 | raise ValueError("Cannot combine tool results") 47 | return field or other_field 48 | 49 | return ToolResult( 50 | output=combine_fields(self.output, other.output), 51 | error=combine_fields(self.error, other.error), 52 | base64_image=combine_fields(self.base64_image, other.base64_image, False), 53 | system=combine_fields(self.system, other.system), 54 | content=self.content or other.content, # Use first non-None content 55 | ) 56 | 57 | def replace(self, **kwargs): 58 | """Returns a new ToolResult with the given fields replaced.""" 59 | return replace(self, **kwargs) 60 | 61 | 62 | class CLIResult(ToolResult): 63 | """A ToolResult that can be rendered as a CLI output.""" 64 | 65 | 66 | class ToolFailure(ToolResult): 67 | """A ToolResult that represents a failure.""" 68 | 69 | 70 | class ToolError(Exception): 71 | """Raised when a tool encounters an error.""" 72 | 73 | def __init__(self, message): 74 | self.message = message 75 | -------------------------------------------------------------------------------- /libs/agent/agent/core/tools/bash.py: -------------------------------------------------------------------------------- 1 | """Abstract base bash/shell tool implementation.""" 2 | 3 | import asyncio 4 | import logging 5 | from abc import abstractmethod 6 | from typing import Any, Dict, Tuple 7 | 8 | from computer.computer import Computer 9 | 10 | from .base import BaseTool, ToolResult 11 | 12 | 13 | class BaseBashTool(BaseTool): 14 | """Base class for bash/shell command execution tools across different providers.""" 15 | 16 | name = "bash" 17 | logger = logging.getLogger(__name__) 18 | computer: Computer 19 | 20 | def __init__(self, computer: Computer): 21 | """Initialize the BashTool. 22 | 23 | Args: 24 | computer: Computer instance, may be used for related operations 25 | """ 26 | self.computer = computer 27 | 28 | async def run_command(self, command: str) -> Tuple[int, str, str]: 29 | """Run a shell command and return exit code, stdout, and stderr. 30 | 31 | Args: 32 | command: Shell command to execute 33 | 34 | Returns: 35 | Tuple containing (exit_code, stdout, stderr) 36 | """ 37 | try: 38 | process = await asyncio.create_subprocess_shell( 39 | command, 40 | stdout=asyncio.subprocess.PIPE, 41 | stderr=asyncio.subprocess.PIPE, 42 | ) 43 | stdout, stderr = await process.communicate() 44 | return process.returncode or 0, stdout.decode(), stderr.decode() 45 | except Exception as e: 46 | self.logger.error(f"Error running command: {str(e)}") 47 | return 1, "", str(e) 48 | 49 | @abstractmethod 50 | async def __call__(self, **kwargs) -> ToolResult: 51 | """Execute the tool with the provided arguments.""" 52 | raise NotImplementedError 53 | -------------------------------------------------------------------------------- /libs/agent/agent/core/tools/collection.py: -------------------------------------------------------------------------------- 1 | """Collection classes for managing multiple tools.""" 2 | 3 | from typing import Any, Dict, List, Type 4 | 5 | from .base import ( 6 | BaseTool, 7 | ToolError, 8 | ToolFailure, 9 | ToolResult, 10 | ) 11 | 12 | 13 | class ToolCollection: 14 | """A collection of tools that can be used with any provider.""" 15 | 16 | def __init__(self, *tools: BaseTool): 17 | self.tools = tools 18 | self.tool_map = {tool.name: tool for tool in tools} 19 | 20 | def to_params(self) -> List[Dict[str, Any]]: 21 | """Convert all tools to provider-specific parameters. 22 | 23 | Returns: 24 | List of dictionaries with tool parameters 25 | """ 26 | return [tool.to_params() for tool in self.tools] 27 | 28 | async def run(self, *, name: str, tool_input: Dict[str, Any]) -> ToolResult: 29 | """Run a tool with the given input. 30 | 31 | Args: 32 | name: Name of the tool to run 33 | tool_input: Input parameters for the tool 34 | 35 | Returns: 36 | Result of the tool execution 37 | """ 38 | tool = self.tool_map.get(name) 39 | if not tool: 40 | return ToolFailure(error=f"Tool {name} is invalid") 41 | try: 42 | return await tool(**tool_input) 43 | except ToolError as e: 44 | return ToolFailure(error=e.message) 45 | except Exception as e: 46 | return ToolFailure(error=f"Unexpected error in tool {name}: {str(e)}") 47 | -------------------------------------------------------------------------------- /libs/agent/agent/core/tools/edit.py: -------------------------------------------------------------------------------- 1 | """Abstract base edit tool implementation.""" 2 | 3 | import asyncio 4 | import logging 5 | import os 6 | from abc import abstractmethod 7 | from pathlib import Path 8 | from typing import Any, Dict, Optional 9 | 10 | from computer.computer import Computer 11 | 12 | from .base import BaseTool, ToolError, ToolResult 13 | 14 | 15 | class BaseEditTool(BaseTool): 16 | """Base class for text editor tools across different providers.""" 17 | 18 | name = "edit" 19 | logger = logging.getLogger(__name__) 20 | computer: Computer 21 | 22 | def __init__(self, computer: Computer): 23 | """Initialize the EditTool. 24 | 25 | Args: 26 | computer: Computer instance, may be used for related operations 27 | """ 28 | self.computer = computer 29 | 30 | async def read_file(self, path: str) -> str: 31 | """Read a file and return its contents. 32 | 33 | Args: 34 | path: Path to the file to read 35 | 36 | Returns: 37 | File contents as a string 38 | """ 39 | try: 40 | path_obj = Path(path) 41 | if not path_obj.exists(): 42 | raise ToolError(f"File does not exist: {path}") 43 | return path_obj.read_text() 44 | except Exception as e: 45 | self.logger.error(f"Error reading file: {str(e)}") 46 | raise ToolError(f"Failed to read file: {str(e)}") 47 | 48 | async def write_file(self, path: str, content: str) -> None: 49 | """Write content to a file. 50 | 51 | Args: 52 | path: Path to the file to write 53 | content: Content to write to the file 54 | """ 55 | try: 56 | path_obj = Path(path) 57 | # Create parent directories if they don't exist 58 | path_obj.parent.mkdir(parents=True, exist_ok=True) 59 | path_obj.write_text(content) 60 | except Exception as e: 61 | self.logger.error(f"Error writing file: {str(e)}") 62 | raise ToolError(f"Failed to write file: {str(e)}") 63 | 64 | @abstractmethod 65 | async def __call__(self, **kwargs) -> ToolResult: 66 | """Execute the tool with the provided arguments.""" 67 | raise NotImplementedError 68 | -------------------------------------------------------------------------------- /libs/agent/agent/core/tools/manager.py: -------------------------------------------------------------------------------- 1 | """Tool manager for initializing and running tools.""" 2 | 3 | from abc import ABC, abstractmethod 4 | from typing import Any, Dict, List 5 | 6 | from computer.computer import Computer 7 | 8 | from .base import BaseTool, ToolResult 9 | from .collection import ToolCollection 10 | 11 | 12 | class BaseToolManager(ABC): 13 | """Base class for tool managers across different providers.""" 14 | 15 | def __init__(self, computer: Computer): 16 | """Initialize the tool manager. 17 | 18 | Args: 19 | computer: Computer instance for computer-related tools 20 | """ 21 | self.computer = computer 22 | self.tools: ToolCollection | None = None 23 | 24 | @abstractmethod 25 | def _initialize_tools(self) -> ToolCollection: 26 | """Initialize all available tools.""" 27 | ... 28 | 29 | async def initialize(self) -> None: 30 | """Initialize tool-specific requirements and create tool collection.""" 31 | await self._initialize_tools_specific() 32 | self.tools = self._initialize_tools() 33 | 34 | @abstractmethod 35 | async def _initialize_tools_specific(self) -> None: 36 | """Initialize provider-specific tool requirements.""" 37 | ... 38 | 39 | @abstractmethod 40 | def get_tool_params(self) -> List[Dict[str, Any]]: 41 | """Get tool parameters for API calls.""" 42 | ... 43 | 44 | async def execute_tool(self, name: str, tool_input: Dict[str, Any]) -> ToolResult: 45 | """Execute a tool with the given input. 46 | 47 | Args: 48 | name: Name of the tool to execute 49 | tool_input: Input parameters for the tool 50 | 51 | Returns: 52 | Result of the tool execution 53 | """ 54 | if self.tools is None: 55 | raise RuntimeError("Tools not initialized. Call initialize() first.") 56 | return await self.tools.run(name=name, tool_input=tool_input) 57 | -------------------------------------------------------------------------------- /libs/agent/agent/core/types.py: -------------------------------------------------------------------------------- 1 | """Core type definitions.""" 2 | 3 | from typing import Any, Dict, List, Optional, TypedDict, Union 4 | from enum import StrEnum 5 | from dataclasses import dataclass 6 | 7 | 8 | class AgentLoop(StrEnum): 9 | """Enumeration of available loop types.""" 10 | 11 | ANTHROPIC = "anthropic" # Anthropic implementation 12 | OMNI = "omni" # OmniLoop implementation 13 | OPENAI = "openai" # OpenAI implementation 14 | OLLAMA = "ollama" # OLLAMA implementation 15 | UITARS = "uitars" # UI-TARS implementation 16 | # Add more loop types as needed 17 | 18 | 19 | class LLMProvider(StrEnum): 20 | """Supported LLM providers.""" 21 | 22 | ANTHROPIC = "anthropic" 23 | OPENAI = "openai" 24 | OLLAMA = "ollama" 25 | OAICOMPAT = "oaicompat" 26 | MLXVLM= "mlxvlm" 27 | 28 | 29 | @dataclass 30 | class LLM: 31 | """Configuration for LLM model and provider.""" 32 | 33 | provider: LLMProvider 34 | name: Optional[str] = None 35 | provider_base_url: Optional[str] = None 36 | 37 | def __post_init__(self): 38 | """Set default model name if not provided.""" 39 | if self.name is None: 40 | from .provider_config import DEFAULT_MODELS 41 | 42 | self.name = DEFAULT_MODELS.get(self.provider) 43 | 44 | # Set default provider URL if none provided 45 | if self.provider_base_url is None and self.provider == LLMProvider.OAICOMPAT: 46 | # Default for vLLM 47 | self.provider_base_url = "http://localhost:8000/v1" 48 | # Common alternatives: 49 | # - LM Studio: "http://localhost:1234/v1" 50 | # - LocalAI: "http://localhost:8080/v1" 51 | # - Ollama with OpenAI compatible API: "http://localhost:11434/v1" 52 | 53 | 54 | # For backward compatibility 55 | LLMModel = LLM 56 | Model = LLM 57 | 58 | 59 | class AgentResponse(TypedDict, total=False): 60 | """Agent response format.""" 61 | 62 | id: str 63 | object: str 64 | created_at: int 65 | status: str 66 | error: Optional[str] 67 | incomplete_details: Optional[Any] 68 | instructions: Optional[Any] 69 | max_output_tokens: Optional[int] 70 | model: str 71 | output: List[Dict[str, Any]] 72 | parallel_tool_calls: bool 73 | previous_response_id: Optional[str] 74 | reasoning: Dict[str, str] 75 | store: bool 76 | temperature: float 77 | text: Dict[str, Dict[str, str]] 78 | tool_choice: str 79 | tools: List[Dict[str, Union[str, int]]] 80 | top_p: float 81 | truncation: str 82 | usage: Dict[str, Any] 83 | user: Optional[str] 84 | metadata: Dict[str, Any] 85 | response: Dict[str, List[Dict[str, Any]]] 86 | # Additional fields for error responses 87 | role: str 88 | content: Union[str, List[Dict[str, Any]]] 89 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/__init__.py: -------------------------------------------------------------------------------- 1 | """Provider implementations for different AI services.""" 2 | 3 | # Import specific providers only when needed to avoid circular imports 4 | __all__ = [] # Let each provider module handle its own exports -------------------------------------------------------------------------------- /libs/agent/agent/providers/anthropic/__init__.py: -------------------------------------------------------------------------------- 1 | """Anthropic provider implementation.""" 2 | 3 | from .loop import AnthropicLoop 4 | from .types import LLMProvider 5 | 6 | __all__ = ["AnthropicLoop", "LLMProvider"] 7 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/anthropic/callbacks/__init__.py: -------------------------------------------------------------------------------- 1 | """Anthropic callbacks package.""" 2 | 3 | from .manager import CallbackManager 4 | 5 | __all__ = ["CallbackManager"] 6 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/anthropic/callbacks/manager.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Protocol 2 | import httpx 3 | from anthropic.types.beta import BetaContentBlockParam 4 | from ..tools import ToolResult 5 | 6 | 7 | class APICallback(Protocol): 8 | """Protocol for API callbacks.""" 9 | 10 | def __call__( 11 | self, 12 | request: httpx.Request | None, 13 | response: httpx.Response | object | None, 14 | error: Exception | None, 15 | ) -> None: ... 16 | 17 | 18 | class ContentCallback(Protocol): 19 | """Protocol for content callbacks.""" 20 | 21 | def __call__(self, content: BetaContentBlockParam) -> None: ... 22 | 23 | 24 | class ToolCallback(Protocol): 25 | """Protocol for tool callbacks.""" 26 | 27 | def __call__(self, result: ToolResult, tool_id: str) -> None: ... 28 | 29 | 30 | class CallbackManager: 31 | """Manages various callbacks for the agent system.""" 32 | 33 | def __init__( 34 | self, 35 | content_callback: ContentCallback, 36 | tool_callback: ToolCallback, 37 | api_callback: APICallback, 38 | ): 39 | """Initialize the callback manager. 40 | 41 | Args: 42 | content_callback: Callback for content updates 43 | tool_callback: Callback for tool execution results 44 | api_callback: Callback for API interactions 45 | """ 46 | self.content_callback = content_callback 47 | self.tool_callback = tool_callback 48 | self.api_callback = api_callback 49 | 50 | def on_content(self, content: BetaContentBlockParam) -> None: 51 | """Handle content updates.""" 52 | self.content_callback(content) 53 | 54 | def on_tool_result(self, result: ToolResult, tool_id: str) -> None: 55 | """Handle tool execution results.""" 56 | self.tool_callback(result, tool_id) 57 | 58 | def on_api_interaction( 59 | self, 60 | request: httpx.Request | None, 61 | response: httpx.Response | object | None, 62 | error: Exception | None, 63 | ) -> None: 64 | """Handle API interactions.""" 65 | self.api_callback(request, response, error) 66 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/anthropic/prompts.py: -------------------------------------------------------------------------------- 1 | """System prompts for Anthropic provider.""" 2 | 3 | from datetime import datetime 4 | import platform 5 | 6 | today = datetime.today() 7 | today = f"{today.strftime('%A, %B')} {today.day}, {today.year}" 8 | 9 | SYSTEM_PROMPT = f""" 10 | * You are utilising a macOS virtual machine using ARM architecture with internet access and Safari as default browser. 11 | * You can feel free to install macOS applications with your bash tool. Use curl instead of wget. 12 | * Using bash tool you can start GUI applications. GUI apps run with bash tool will appear within your desktop environment, but they may take some time to appear. Take a screenshot to confirm it did. 13 | * When using your bash tool with commands that are expected to output very large quantities of text, redirect into a tmp file and use str_replace_editor or `grep -n -B -A ` to confirm output. 14 | * When viewing a page it can be helpful to zoom out so that you can see everything on the page. Either that, or make sure you scroll down to see everything before deciding something isn't available. 15 | * When using your computer function calls, they take a while to run and send back to you. Where possible/feasible, try to chain multiple of these calls all into one function calls request. 16 | * The current date is {today}. 17 | 18 | 19 | 20 | * Plan at maximum 1 step each time, and evaluate the result of each step before proceeding. Hold back if you're not sure about the result of the step. 21 | * If you're not sure about the location of an application, use start the app using the bash tool. 22 | * If the item you are looking at is a pdf, if after taking a single screenshot of the pdf it seems that you want to read the entire document instead of trying to continue to read the pdf from your screenshots + navigation, determine the URL, use curl to download the pdf, install and use pdftotext to convert it to a text file, and then read that text file directly with your StrReplaceEditTool. 23 | """ 24 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/anthropic/tools/__init__.py: -------------------------------------------------------------------------------- 1 | """Anthropic-specific tools for agent.""" 2 | 3 | from .base import ( 4 | BaseAnthropicTool, 5 | ToolResult, 6 | ToolError, 7 | ToolFailure, 8 | CLIResult, 9 | AnthropicToolResult, 10 | AnthropicToolError, 11 | AnthropicToolFailure, 12 | AnthropicCLIResult, 13 | ) 14 | from .bash import BashTool 15 | from .computer import ComputerTool 16 | from .edit import EditTool 17 | from .manager import ToolManager 18 | 19 | __all__ = [ 20 | "BaseAnthropicTool", 21 | "ToolResult", 22 | "ToolError", 23 | "ToolFailure", 24 | "CLIResult", 25 | "AnthropicToolResult", 26 | "AnthropicToolError", 27 | "AnthropicToolFailure", 28 | "AnthropicCLIResult", 29 | "BashTool", 30 | "ComputerTool", 31 | "EditTool", 32 | "ToolManager", 33 | ] 34 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/anthropic/tools/base.py: -------------------------------------------------------------------------------- 1 | """Anthropic-specific tool base classes.""" 2 | 3 | from abc import ABCMeta, abstractmethod 4 | from dataclasses import dataclass, fields, replace 5 | from typing import Any, Dict 6 | 7 | from anthropic.types.beta import BetaToolUnionParam 8 | 9 | from ....core.tools.base import BaseTool 10 | 11 | 12 | class BaseAnthropicTool(BaseTool, metaclass=ABCMeta): 13 | """Abstract base class for Anthropic-defined tools.""" 14 | 15 | def __init__(self): 16 | """Initialize the base Anthropic tool.""" 17 | # No specific initialization needed yet, but included for future extensibility 18 | pass 19 | 20 | @abstractmethod 21 | async def __call__(self, **kwargs) -> Any: 22 | """Executes the tool with the given arguments.""" 23 | ... 24 | 25 | @abstractmethod 26 | def to_params(self) -> Dict[str, Any]: 27 | """Convert tool to Anthropic-specific API parameters. 28 | 29 | Returns: 30 | Dictionary with tool parameters for Anthropic API 31 | """ 32 | raise NotImplementedError 33 | 34 | 35 | @dataclass(kw_only=True, frozen=True) 36 | class ToolResult: 37 | """Represents the result of a tool execution.""" 38 | 39 | output: str | None = None 40 | error: str | None = None 41 | base64_image: str | None = None 42 | system: str | None = None 43 | content: list[dict] | None = None 44 | 45 | def __bool__(self): 46 | return any(getattr(self, field.name) for field in fields(self)) 47 | 48 | def __add__(self, other: "ToolResult"): 49 | def combine_fields(field: str | None, other_field: str | None, concatenate: bool = True): 50 | if field and other_field: 51 | if concatenate: 52 | return field + other_field 53 | raise ValueError("Cannot combine tool results") 54 | return field or other_field 55 | 56 | return ToolResult( 57 | output=combine_fields(self.output, other.output), 58 | error=combine_fields(self.error, other.error), 59 | base64_image=combine_fields(self.base64_image, other.base64_image, False), 60 | system=combine_fields(self.system, other.system), 61 | content=self.content or other.content, # Use first non-None content 62 | ) 63 | 64 | def replace(self, **kwargs): 65 | """Returns a new ToolResult with the given fields replaced.""" 66 | return replace(self, **kwargs) 67 | 68 | 69 | class CLIResult(ToolResult): 70 | """A ToolResult that can be rendered as a CLI output.""" 71 | 72 | 73 | class ToolFailure(ToolResult): 74 | """A ToolResult that represents a failure.""" 75 | 76 | 77 | class ToolError(Exception): 78 | """Raised when a tool encounters an error.""" 79 | 80 | def __init__(self, message): 81 | self.message = message 82 | 83 | 84 | # Re-export the core tool classes with Anthropic-specific names for backward compatibility 85 | AnthropicToolResult = ToolResult 86 | AnthropicToolError = ToolError 87 | AnthropicToolFailure = ToolFailure 88 | AnthropicCLIResult = CLIResult 89 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/anthropic/tools/bash.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import os 3 | from typing import ClassVar, Literal, Dict, Any 4 | from computer.computer import Computer 5 | 6 | from .base import BaseAnthropicTool, CLIResult, ToolError, ToolResult 7 | from ....core.tools.bash import BaseBashTool 8 | 9 | 10 | class BashTool(BaseBashTool, BaseAnthropicTool): 11 | """ 12 | A tool that allows the agent to run bash commands. 13 | The tool parameters are defined by Anthropic and are not editable. 14 | """ 15 | 16 | name: ClassVar[Literal["bash"]] = "bash" 17 | api_type: ClassVar[Literal["bash_20250124"]] = "bash_20250124" 18 | _timeout: float = 120.0 # seconds 19 | 20 | def __init__(self, computer: Computer): 21 | """Initialize the bash tool. 22 | 23 | Args: 24 | computer: Computer instance for executing commands 25 | """ 26 | # Initialize the base bash tool first 27 | BaseBashTool.__init__(self, computer) 28 | # Then initialize the Anthropic tool 29 | BaseAnthropicTool.__init__(self) 30 | # Initialize bash session 31 | 32 | async def __call__(self, command: str | None = None, restart: bool = False, **kwargs): 33 | """Execute a bash command. 34 | 35 | Args: 36 | command: The command to execute 37 | restart: Whether to restart the shell (not used with computer interface) 38 | 39 | Returns: 40 | Tool execution result 41 | 42 | Raises: 43 | ToolError: If command execution fails 44 | """ 45 | if restart: 46 | return ToolResult(system="Restart not needed with computer interface.") 47 | 48 | if command is None: 49 | raise ToolError("no command provided.") 50 | 51 | try: 52 | async with asyncio.timeout(self._timeout): 53 | stdout, stderr = await self.computer.interface.run_command(command) 54 | return CLIResult(output=stdout or "", error=stderr or "") 55 | except asyncio.TimeoutError as e: 56 | raise ToolError(f"Command timed out after {self._timeout} seconds") from e 57 | except Exception as e: 58 | raise ToolError(f"Failed to execute command: {str(e)}") 59 | 60 | def to_params(self) -> Dict[str, Any]: 61 | """Convert tool to API parameters. 62 | 63 | Returns: 64 | Dictionary with tool parameters 65 | """ 66 | return {"name": self.name, "type": self.api_type} 67 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/anthropic/tools/collection.py: -------------------------------------------------------------------------------- 1 | """Collection classes for managing multiple tools.""" 2 | 3 | from typing import Any, cast 4 | 5 | from anthropic.types.beta import BetaToolUnionParam 6 | 7 | from .base import ( 8 | BaseAnthropicTool, 9 | ToolError, 10 | ToolFailure, 11 | ToolResult, 12 | ) 13 | 14 | 15 | class ToolCollection: 16 | """A collection of anthropic-defined tools.""" 17 | 18 | def __init__(self, *tools: BaseAnthropicTool): 19 | self.tools = tools 20 | self.tool_map = {tool.to_params()["name"]: tool for tool in tools} 21 | 22 | def to_params( 23 | self, 24 | ) -> list[BetaToolUnionParam]: 25 | return cast(list[BetaToolUnionParam], [tool.to_params() for tool in self.tools]) 26 | 27 | async def run(self, *, name: str, tool_input: dict[str, Any]) -> ToolResult: 28 | tool = self.tool_map.get(name) 29 | if not tool: 30 | return ToolFailure(error=f"Tool {name} is invalid") 31 | try: 32 | return await tool(**tool_input) 33 | except ToolError as e: 34 | return ToolFailure(error=e.message) 35 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/anthropic/tools/manager.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, List, cast 2 | from anthropic.types.beta import BetaToolUnionParam 3 | from computer.computer import Computer 4 | 5 | from ....core.tools import BaseToolManager, ToolResult 6 | from ....core.tools.collection import ToolCollection 7 | 8 | from .bash import BashTool 9 | from .computer import ComputerTool 10 | from .edit import EditTool 11 | 12 | 13 | class ToolManager(BaseToolManager): 14 | """Manages Anthropic-specific tool initialization and execution.""" 15 | 16 | def __init__(self, computer: Computer): 17 | """Initialize the tool manager. 18 | 19 | Args: 20 | computer: Computer instance for computer-related tools 21 | """ 22 | super().__init__(computer) 23 | # Initialize Anthropic-specific tools 24 | self.computer_tool = ComputerTool(self.computer) 25 | self.bash_tool = BashTool(self.computer) 26 | self.edit_tool = EditTool(self.computer) 27 | 28 | def _initialize_tools(self) -> ToolCollection: 29 | """Initialize all available tools.""" 30 | return ToolCollection(self.computer_tool, self.bash_tool, self.edit_tool) 31 | 32 | async def _initialize_tools_specific(self) -> None: 33 | """Initialize Anthropic-specific tool requirements.""" 34 | await self.computer_tool.initialize_dimensions() 35 | 36 | def get_tool_params(self) -> List[BetaToolUnionParam]: 37 | """Get tool parameters for Anthropic API calls.""" 38 | if self.tools is None: 39 | raise RuntimeError("Tools not initialized. Call initialize() first.") 40 | return cast(List[BetaToolUnionParam], self.tools.to_params()) 41 | 42 | async def execute_tool(self, name: str, tool_input: dict[str, Any]) -> ToolResult: 43 | """Execute a tool with the given input. 44 | 45 | Args: 46 | name: Name of the tool to execute 47 | tool_input: Input parameters for the tool 48 | 49 | Returns: 50 | Result of the tool execution 51 | """ 52 | if self.tools is None: 53 | raise RuntimeError("Tools not initialized. Call initialize() first.") 54 | return await self.tools.run(name=name, tool_input=tool_input) 55 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/anthropic/tools/run.py: -------------------------------------------------------------------------------- 1 | """Utility to run shell commands asynchronously with a timeout.""" 2 | 3 | import asyncio 4 | 5 | TRUNCATED_MESSAGE: str = "To save on context only part of this file has been shown to you. You should retry this tool after you have searched inside the file with `grep -n` in order to find the line numbers of what you are looking for." 6 | MAX_RESPONSE_LEN: int = 16000 7 | 8 | 9 | def maybe_truncate(content: str, truncate_after: int | None = MAX_RESPONSE_LEN): 10 | """Truncate content and append a notice if content exceeds the specified length.""" 11 | return ( 12 | content 13 | if not truncate_after or len(content) <= truncate_after 14 | else content[:truncate_after] + TRUNCATED_MESSAGE 15 | ) 16 | 17 | 18 | async def run( 19 | cmd: str, 20 | timeout: float | None = 120.0, # seconds 21 | truncate_after: int | None = MAX_RESPONSE_LEN, 22 | ): 23 | """Run a shell command asynchronously with a timeout.""" 24 | process = await asyncio.create_subprocess_shell( 25 | cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE 26 | ) 27 | 28 | try: 29 | stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout) 30 | return ( 31 | process.returncode or 0, 32 | maybe_truncate(stdout.decode(), truncate_after=truncate_after), 33 | maybe_truncate(stderr.decode(), truncate_after=truncate_after), 34 | ) 35 | except asyncio.TimeoutError as exc: 36 | try: 37 | process.kill() 38 | except ProcessLookupError: 39 | pass 40 | raise TimeoutError( 41 | f"Command '{cmd}' timed out after {timeout} seconds" 42 | ) from exc 43 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/anthropic/types.py: -------------------------------------------------------------------------------- 1 | from enum import StrEnum 2 | 3 | 4 | class LLMProvider(StrEnum): 5 | """Enum for supported API providers.""" 6 | 7 | ANTHROPIC = "anthropic" 8 | BEDROCK = "bedrock" 9 | VERTEX = "vertex" 10 | 11 | 12 | PROVIDER_TO_DEFAULT_MODEL_NAME: dict[LLMProvider, str] = { 13 | LLMProvider.ANTHROPIC: "claude-3-7-sonnet-20250219", 14 | LLMProvider.BEDROCK: "anthropic.claude-3-7-sonnet-20250219-v2:0", 15 | LLMProvider.VERTEX: "claude-3-5-sonnet-v2@20241022", 16 | } 17 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/omni/__init__.py: -------------------------------------------------------------------------------- 1 | """Omni provider implementation.""" 2 | 3 | from ...core.types import LLMProvider 4 | from .image_utils import ( 5 | decode_base64_image, 6 | ) 7 | 8 | __all__ = ["LLMProvider", "decode_base64_image"] 9 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/omni/api_handler.py: -------------------------------------------------------------------------------- 1 | """API handling for Omni provider.""" 2 | 3 | import logging 4 | from typing import Any, Dict, List 5 | 6 | from .prompts import SYSTEM_PROMPT 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | class OmniAPIHandler: 12 | """Handler for Omni API calls.""" 13 | 14 | def __init__(self, loop): 15 | """Initialize the API handler. 16 | 17 | Args: 18 | loop: Parent loop instance 19 | """ 20 | self.loop = loop 21 | 22 | async def make_api_call( 23 | self, messages: List[Dict[str, Any]], system_prompt: str = SYSTEM_PROMPT 24 | ) -> Any: 25 | """Make an API call to the appropriate provider. 26 | 27 | Args: 28 | messages: List of messages in standard OpenAI format 29 | system_prompt: System prompt to use 30 | 31 | Returns: 32 | API response 33 | """ 34 | if not self.loop._make_api_call: 35 | raise RuntimeError("Loop does not have _make_api_call method") 36 | 37 | try: 38 | # Use the loop's _make_api_call method with standard messages 39 | return await self.loop._make_api_call(messages=messages, system_prompt=system_prompt) 40 | except Exception as e: 41 | logger.error(f"Error making API call: {str(e)}") 42 | raise 43 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/omni/clients/base.py: -------------------------------------------------------------------------------- 1 | """Base client implementation for Omni providers.""" 2 | 3 | import logging 4 | from typing import Dict, List, Optional, Any, Tuple 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class BaseOmniClient: 10 | """Base class for provider-specific clients.""" 11 | 12 | def __init__(self, api_key: Optional[str] = None, model: Optional[str] = None): 13 | """Initialize base client. 14 | 15 | Args: 16 | api_key: Optional API key 17 | model: Optional model name 18 | """ 19 | self.api_key = api_key 20 | self.model = model 21 | 22 | async def run_interleaved( 23 | self, messages: List[Dict[str, Any]], system: str, max_tokens: Optional[int] = None 24 | ) -> Dict[str, Any]: 25 | """Run interleaved chat completion. 26 | 27 | Args: 28 | messages: List of message dicts 29 | system: System prompt 30 | max_tokens: Optional max tokens override 31 | 32 | Returns: 33 | Response dict 34 | """ 35 | raise NotImplementedError 36 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/omni/clients/utils.py: -------------------------------------------------------------------------------- 1 | import base64 2 | 3 | def is_image_path(text: str) -> bool: 4 | """Check if a text string is an image file path. 5 | 6 | Args: 7 | text: Text string to check 8 | 9 | Returns: 10 | True if text ends with image extension, False otherwise 11 | """ 12 | image_extensions = (".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif") 13 | return text.endswith(image_extensions) 14 | 15 | def encode_image(image_path: str) -> str: 16 | """Encode image file to base64. 17 | 18 | Args: 19 | image_path: Path to image file 20 | 21 | Returns: 22 | Base64 encoded image string 23 | """ 24 | with open(image_path, "rb") as image_file: 25 | return base64.b64encode(image_file.read()).decode("utf-8") -------------------------------------------------------------------------------- /libs/agent/agent/providers/omni/image_utils.py: -------------------------------------------------------------------------------- 1 | """Image processing utilities for the Cua provider.""" 2 | 3 | import base64 4 | import logging 5 | import re 6 | from io import BytesIO 7 | from typing import Optional, Tuple 8 | from PIL import Image 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | def decode_base64_image(img_base64: str) -> Optional[Image.Image]: 14 | """Decode a base64 encoded image to a PIL Image. 15 | 16 | Args: 17 | img_base64: Base64 encoded image, may include data URL prefix 18 | 19 | Returns: 20 | PIL Image or None if decoding fails 21 | """ 22 | try: 23 | # Remove data URL prefix if present 24 | if img_base64.startswith("data:image"): 25 | img_base64 = img_base64.split(",")[1] 26 | 27 | # Decode base64 to bytes 28 | img_data = base64.b64decode(img_base64) 29 | 30 | # Convert bytes to PIL Image 31 | return Image.open(BytesIO(img_data)) 32 | except Exception as e: 33 | logger.error(f"Error decoding base64 image: {str(e)}") 34 | return None 35 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/omni/tools/__init__.py: -------------------------------------------------------------------------------- 1 | """Omni provider tools - compatible with multiple LLM providers.""" 2 | 3 | from ....core.tools import BaseTool, ToolResult, ToolError, ToolFailure, CLIResult 4 | from .base import BaseOmniTool 5 | from .computer import ComputerTool 6 | from .bash import BashTool 7 | from .manager import ToolManager 8 | 9 | # Re-export the tools with Omni-specific names for backward compatibility 10 | OmniToolResult = ToolResult 11 | OmniToolError = ToolError 12 | OmniToolFailure = ToolFailure 13 | OmniCLIResult = CLIResult 14 | 15 | # We'll export specific tools once implemented 16 | __all__ = [ 17 | "BaseTool", 18 | "BaseOmniTool", 19 | "ToolResult", 20 | "ToolError", 21 | "ToolFailure", 22 | "CLIResult", 23 | "OmniToolResult", 24 | "OmniToolError", 25 | "OmniToolFailure", 26 | "OmniCLIResult", 27 | "ComputerTool", 28 | "BashTool", 29 | "ToolManager", 30 | ] 31 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/omni/tools/base.py: -------------------------------------------------------------------------------- 1 | """Omni-specific tool base classes.""" 2 | 3 | from abc import ABCMeta, abstractmethod 4 | from typing import Any, Dict 5 | 6 | from ....core.tools.base import BaseTool 7 | 8 | 9 | class BaseOmniTool(BaseTool, metaclass=ABCMeta): 10 | """Abstract base class for Omni provider tools.""" 11 | 12 | def __init__(self): 13 | """Initialize the base Omni tool.""" 14 | # No specific initialization needed yet, but included for future extensibility 15 | pass 16 | 17 | @abstractmethod 18 | async def __call__(self, **kwargs) -> Any: 19 | """Executes the tool with the given arguments.""" 20 | ... 21 | 22 | @abstractmethod 23 | def to_params(self) -> Dict[str, Any]: 24 | """Convert tool to Omni provider-specific API parameters. 25 | 26 | Returns: 27 | Dictionary with tool parameters for the specific API 28 | """ 29 | raise NotImplementedError 30 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/omni/tools/bash.py: -------------------------------------------------------------------------------- 1 | """Bash tool for Omni provider.""" 2 | 3 | import logging 4 | from typing import Any, Dict 5 | 6 | from computer import Computer 7 | from ....core.tools import ToolResult, ToolError 8 | from .base import BaseOmniTool 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class BashTool(BaseOmniTool): 14 | """Tool for executing bash commands.""" 15 | 16 | name = "bash" 17 | description = "Execute bash commands on the system" 18 | 19 | def __init__(self, computer: Computer): 20 | """Initialize the bash tool. 21 | 22 | Args: 23 | computer: Computer instance 24 | """ 25 | super().__init__() 26 | self.computer = computer 27 | 28 | def to_params(self) -> Dict[str, Any]: 29 | """Convert tool to API parameters. 30 | 31 | Returns: 32 | Dictionary with tool parameters 33 | """ 34 | return { 35 | "type": "function", 36 | "function": { 37 | "name": self.name, 38 | "description": self.description, 39 | "parameters": { 40 | "type": "object", 41 | "properties": { 42 | "command": { 43 | "type": "string", 44 | "description": "The bash command to execute", 45 | }, 46 | }, 47 | "required": ["command"], 48 | }, 49 | }, 50 | } 51 | 52 | async def __call__(self, **kwargs) -> ToolResult: 53 | """Execute bash command. 54 | 55 | Args: 56 | **kwargs: Command parameters 57 | 58 | Returns: 59 | Tool execution result 60 | """ 61 | try: 62 | command = kwargs.get("command", "") 63 | if not command: 64 | return ToolResult(error="No command specified") 65 | 66 | # The true implementation would use the actual method to run terminal commands 67 | # Since we're getting linter errors, we'll just implement a placeholder that will 68 | # be replaced with the correct implementation when this tool is fully integrated 69 | logger.info(f"Would execute command: {command}") 70 | return ToolResult(output=f"Command executed (placeholder): {command}") 71 | 72 | except Exception as e: 73 | logger.error(f"Error in bash tool: {str(e)}") 74 | return ToolResult(error=f"Error: {str(e)}") 75 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/omni/tools/manager.py: -------------------------------------------------------------------------------- 1 | """Tool manager for the Omni provider.""" 2 | 3 | from typing import Any, Dict, List 4 | from computer.computer import Computer 5 | 6 | from ....core.tools import BaseToolManager, ToolResult 7 | from ....core.tools.collection import ToolCollection 8 | from .computer import ComputerTool 9 | from .bash import BashTool 10 | from ....core.types import LLMProvider 11 | 12 | 13 | class ToolManager(BaseToolManager): 14 | """Manages Omni provider tool initialization and execution.""" 15 | 16 | def __init__(self, computer: Computer, provider: LLMProvider): 17 | """Initialize the tool manager. 18 | 19 | Args: 20 | computer: Computer instance for computer-related tools 21 | provider: The LLM provider being used 22 | """ 23 | super().__init__(computer) 24 | self.provider = provider 25 | # Initialize Omni-specific tools 26 | self.computer_tool = ComputerTool(self.computer) 27 | self.bash_tool = BashTool(self.computer) 28 | 29 | def _initialize_tools(self) -> ToolCollection: 30 | """Initialize all available tools.""" 31 | return ToolCollection(self.computer_tool, self.bash_tool) 32 | 33 | async def _initialize_tools_specific(self) -> None: 34 | """Initialize Omni provider-specific tool requirements.""" 35 | await self.computer_tool.initialize_dimensions() 36 | 37 | def get_tool_params(self) -> List[Dict[str, Any]]: 38 | """Get tool parameters for API calls. 39 | 40 | Returns: 41 | List of tool parameters for the current provider's API 42 | """ 43 | if self.tools is None: 44 | raise RuntimeError("Tools not initialized. Call initialize() first.") 45 | 46 | return self.tools.to_params() 47 | 48 | async def execute_tool(self, name: str, tool_input: dict[str, Any]) -> ToolResult: 49 | """Execute a tool with the given input. 50 | 51 | Args: 52 | name: Name of the tool to execute 53 | tool_input: Input parameters for the tool 54 | 55 | Returns: 56 | Result of the tool execution 57 | """ 58 | if self.tools is None: 59 | raise RuntimeError("Tools not initialized. Call initialize() first.") 60 | 61 | return await self.tools.run(name=name, tool_input=tool_input) 62 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/openai/__init__.py: -------------------------------------------------------------------------------- 1 | """OpenAI Agent Response API provider for computer control.""" 2 | 3 | from .types import LLMProvider 4 | from .loop import OpenAILoop 5 | 6 | __all__ = ["OpenAILoop", "LLMProvider"] 7 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/openai/tools/__init__.py: -------------------------------------------------------------------------------- 1 | """OpenAI tools module for computer control.""" 2 | 3 | from .manager import ToolManager 4 | from .computer import ComputerTool 5 | from .base import BaseOpenAITool, ToolResult, ToolError, ToolFailure, CLIResult 6 | 7 | __all__ = [ 8 | "ToolManager", 9 | "ComputerTool", 10 | "BaseOpenAITool", 11 | "ToolResult", 12 | "ToolError", 13 | "ToolFailure", 14 | "CLIResult", 15 | ] 16 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/openai/tools/base.py: -------------------------------------------------------------------------------- 1 | """OpenAI-specific tool base classes.""" 2 | 3 | from abc import ABCMeta, abstractmethod 4 | from dataclasses import dataclass, fields, replace 5 | from typing import Any, Dict, List, Optional 6 | 7 | from ....core.tools.base import BaseTool 8 | 9 | 10 | class BaseOpenAITool(BaseTool, metaclass=ABCMeta): 11 | """Abstract base class for OpenAI-defined tools.""" 12 | 13 | def __init__(self): 14 | """Initialize the base OpenAI tool.""" 15 | # No specific initialization needed yet, but included for future extensibility 16 | pass 17 | 18 | @abstractmethod 19 | async def __call__(self, **kwargs) -> Any: 20 | """Executes the tool with the given arguments.""" 21 | ... 22 | 23 | @abstractmethod 24 | def to_params(self) -> Dict[str, Any]: 25 | """Convert tool to OpenAI-specific API parameters. 26 | 27 | Returns: 28 | Dictionary with tool parameters for OpenAI API 29 | """ 30 | raise NotImplementedError 31 | 32 | 33 | @dataclass(kw_only=True, frozen=True) 34 | class ToolResult: 35 | """Represents the result of a tool execution.""" 36 | 37 | output: str | None = None 38 | error: str | None = None 39 | base64_image: str | None = None 40 | system: str | None = None 41 | content: list[dict] | None = None 42 | 43 | def __bool__(self): 44 | return any(getattr(self, field.name) for field in fields(self)) 45 | 46 | def __add__(self, other: "ToolResult"): 47 | def combine_fields(field: str | None, other_field: str | None, concatenate: bool = True): 48 | if field and other_field: 49 | if concatenate: 50 | return field + other_field 51 | raise ValueError("Cannot combine tool results") 52 | return field or other_field 53 | 54 | return ToolResult( 55 | output=combine_fields(self.output, other.output), 56 | error=combine_fields(self.error, other.error), 57 | base64_image=combine_fields(self.base64_image, other.base64_image, False), 58 | system=combine_fields(self.system, other.system), 59 | content=self.content or other.content, # Use first non-None content 60 | ) 61 | 62 | def replace(self, **kwargs): 63 | """Returns a new ToolResult with the given fields replaced.""" 64 | return replace(self, **kwargs) 65 | 66 | 67 | class CLIResult(ToolResult): 68 | """A ToolResult that can be rendered as a CLI output.""" 69 | 70 | 71 | class ToolFailure(ToolResult): 72 | """A ToolResult that represents a failure.""" 73 | 74 | 75 | class ToolError(Exception): 76 | """Raised when a tool encounters an error.""" 77 | 78 | def __init__(self, message): 79 | self.message = message 80 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/openai/types.py: -------------------------------------------------------------------------------- 1 | """Type definitions for the OpenAI provider.""" 2 | 3 | from enum import StrEnum, auto 4 | from typing import Dict, List, Optional, Union, Any 5 | from dataclasses import dataclass 6 | 7 | 8 | class LLMProvider(StrEnum): 9 | """OpenAI LLM provider types.""" 10 | 11 | OPENAI = "openai" 12 | 13 | 14 | class ResponseItemType(StrEnum): 15 | """Types of items in OpenAI Agent Response output.""" 16 | 17 | MESSAGE = "message" 18 | COMPUTER_CALL = "computer_call" 19 | COMPUTER_CALL_OUTPUT = "computer_call_output" 20 | REASONING = "reasoning" 21 | 22 | 23 | @dataclass 24 | class ComputerAction: 25 | """Represents a computer action to be performed.""" 26 | 27 | type: str 28 | x: Optional[int] = None 29 | y: Optional[int] = None 30 | text: Optional[str] = None 31 | button: Optional[str] = None 32 | keys: Optional[List[str]] = None 33 | ms: Optional[int] = None 34 | scroll_x: Optional[int] = None 35 | scroll_y: Optional[int] = None 36 | path: Optional[List[Dict[str, int]]] = None 37 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/uitars/__init__.py: -------------------------------------------------------------------------------- 1 | """UI-TARS Agent provider package.""" 2 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/uitars/clients/base.py: -------------------------------------------------------------------------------- 1 | """Base client implementation for Omni providers.""" 2 | 3 | import logging 4 | from typing import Dict, List, Optional, Any, Tuple 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class BaseUITarsClient: 10 | """Base class for provider-specific clients.""" 11 | 12 | def __init__(self, api_key: Optional[str] = None, model: Optional[str] = None): 13 | """Initialize base client. 14 | 15 | Args: 16 | api_key: Optional API key 17 | model: Optional model name 18 | """ 19 | self.api_key = api_key 20 | self.model = model 21 | 22 | async def run_interleaved( 23 | self, messages: List[Dict[str, Any]], system: str, max_tokens: Optional[int] = None 24 | ) -> Dict[str, Any]: 25 | """Run interleaved chat completion. 26 | 27 | Args: 28 | messages: List of message dicts 29 | system: System prompt 30 | max_tokens: Optional max tokens override 31 | 32 | Returns: 33 | Response dict 34 | """ 35 | raise NotImplementedError 36 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/uitars/prompts.py: -------------------------------------------------------------------------------- 1 | """Prompts for UI-TARS agent.""" 2 | 3 | MAC_SPECIFIC_NOTES = """ 4 | (You are operating on macOS, use 'cmd' instead of 'ctrl' for most shortcuts e.g., hotkey(key='cmd c') for copy, hotkey(key='cmd v') for paste, hotkey(key='cmd t') for new tab).) 5 | """ 6 | 7 | SYSTEM_PROMPT = "You are a helpful assistant." 8 | 9 | COMPUTER_USE = """You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. 10 | 11 | ## Output Format 12 | ``` 13 | Thought: ... 14 | Action: ... 15 | ``` 16 | 17 | ## Action Space 18 | 19 | click(start_box='<|box_start|>(x1,y1)<|box_end|>') 20 | left_double(start_box='<|box_start|>(x1,y1)<|box_end|>') 21 | right_single(start_box='<|box_start|>(x1,y1)<|box_end|>') 22 | drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>') 23 | hotkey(key='') 24 | type(content='xxx') # Use escape characters \\', \\\", and \\n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\n at the end of content. 25 | scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left') 26 | wait() #Sleep for 5s and take a screenshot to check for any changes. 27 | finished(content='xxx') # Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format. 28 | 29 | 30 | ## Note 31 | - Use {language} in `Thought` part. 32 | - Write a small plan and finally summarize your next action (with its target element) in one sentence in `Thought` part. 33 | 34 | ## User Instruction 35 | {instruction} 36 | """ 37 | 38 | MOBILE_USE = """You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. 39 | ## Output Format 40 | ``` 41 | Thought: ... 42 | Action: ... 43 | ``` 44 | ## Action Space 45 | 46 | click(start_box='<|box_start|>(x1,y1)<|box_end|>') 47 | long_press(start_box='<|box_start|>(x1,y1)<|box_end|>') 48 | type(content='') #If you want to submit your input, use "\\n" at the end of `content`. 49 | scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left') 50 | open_app(app_name=\'\') 51 | drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>') 52 | press_home() 53 | press_back() 54 | finished(content='xxx') # Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format. 55 | 56 | 57 | ## Note 58 | - Use {language} in `Thought` part. 59 | - Write a small plan and finally summarize your next action (with its target element) in one sentence in `Thought` part. 60 | 61 | ## User Instruction 62 | {instruction} 63 | """ 64 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/uitars/tools/__init__.py: -------------------------------------------------------------------------------- 1 | """UI-TARS tools package.""" 2 | -------------------------------------------------------------------------------- /libs/agent/agent/providers/uitars/tools/manager.py: -------------------------------------------------------------------------------- 1 | """Tool manager for the UI-TARS provider.""" 2 | 3 | import logging 4 | from typing import Any, Dict, List, Optional 5 | 6 | from computer import Computer 7 | from ....core.tools import BaseToolManager 8 | from ....core.tools.collection import ToolCollection 9 | from .computer import ComputerTool 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | class ToolManager(BaseToolManager): 15 | """Manages UI-TARS provider tool initialization and execution.""" 16 | 17 | def __init__(self, computer: Computer): 18 | """Initialize the tool manager. 19 | 20 | Args: 21 | computer: Computer instance for computer-related tools 22 | """ 23 | super().__init__(computer) 24 | # Initialize UI-TARS-specific tools 25 | self.computer_tool = ComputerTool(self.computer) 26 | self._initialized = False 27 | 28 | def _initialize_tools(self) -> ToolCollection: 29 | """Initialize all available tools.""" 30 | return ToolCollection(self.computer_tool) 31 | 32 | async def _initialize_tools_specific(self) -> None: 33 | """Initialize UI-TARS provider-specific tool requirements.""" 34 | await self.computer_tool.initialize_dimensions() 35 | 36 | def get_tool_params(self) -> List[Dict[str, Any]]: 37 | """Get tool parameters for API calls. 38 | 39 | Returns: 40 | List of tool parameters for the current provider's API 41 | """ 42 | if self.tools is None: 43 | raise RuntimeError("Tools not initialized. Call initialize() first.") 44 | 45 | return self.tools.to_params() 46 | 47 | async def execute_tool(self, name: str, tool_input: dict[str, Any]) -> Any: 48 | """Execute a tool with the given input. 49 | 50 | Args: 51 | name: Name of the tool to execute 52 | tool_input: Input parameters for the tool 53 | 54 | Returns: 55 | Result of the tool execution 56 | """ 57 | if self.tools is None: 58 | raise RuntimeError("Tools not initialized. Call initialize() first.") 59 | 60 | return await self.tools.run(name=name, tool_input=tool_input) 61 | -------------------------------------------------------------------------------- /libs/agent/agent/telemetry.py: -------------------------------------------------------------------------------- 1 | """Telemetry support for Agent class.""" 2 | 3 | import os 4 | import platform 5 | import sys 6 | import time 7 | from typing import Any, Dict, Optional 8 | 9 | from core.telemetry import ( 10 | record_event, 11 | is_telemetry_enabled, 12 | flush, 13 | get_telemetry_client, 14 | increment, 15 | ) 16 | 17 | # System information used for telemetry 18 | SYSTEM_INFO = { 19 | "os": sys.platform, 20 | "python_version": platform.python_version(), 21 | } 22 | -------------------------------------------------------------------------------- /libs/agent/agent/ui/__init__.py: -------------------------------------------------------------------------------- 1 | """UI modules for the Computer-Use Agent.""" 2 | -------------------------------------------------------------------------------- /libs/agent/agent/ui/gradio/__init__.py: -------------------------------------------------------------------------------- 1 | """Gradio UI for Computer-Use Agent.""" 2 | 3 | import gradio as gr 4 | from typing import Optional 5 | 6 | from .app import create_gradio_ui 7 | 8 | 9 | def registry(name: str = "cua:gpt-4o") -> gr.Blocks: 10 | """Create and register a Gradio UI for the Computer-Use Agent. 11 | 12 | Args: 13 | name: The name to use for the Gradio app, in format 'provider:model' 14 | 15 | Returns: 16 | A Gradio Blocks application 17 | """ 18 | provider, model = name.split(":", 1) if ":" in name else ("openai", name) 19 | 20 | # Create and return the Gradio UI 21 | return create_gradio_ui(provider_name=provider, model_name=model) 22 | -------------------------------------------------------------------------------- /libs/agent/poetry.toml: -------------------------------------------------------------------------------- 1 | [virtualenvs] 2 | in-project = true 3 | -------------------------------------------------------------------------------- /libs/computer-server/README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 |
4 | 5 | 6 | 7 | Shows my svg 8 | 9 |
10 | 11 | [![Python](https://img.shields.io/badge/Python-333333?logo=python&logoColor=white&labelColor=333333)](#) 12 | [![macOS](https://img.shields.io/badge/macOS-000000?logo=apple&logoColor=F0F0F0)](#) 13 | [![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?&logo=discord&logoColor=white)](https://discord.com/invite/mVnXXpdE85) 14 | [![PyPI](https://img.shields.io/pypi/v/cua-computer-server?color=333333)](https://pypi.org/project/cua-computer-server/) 15 |

16 |
17 | 18 | **Computer Server** is the server component for the Computer-Use Interface (CUI) framework powering Cua for interacting with local macOS and Linux sandboxes, PyAutoGUI-compatible, and pluggable with any AI agent systems (Cua, Langchain, CrewAI, AutoGen). 19 | 20 | ## Features 21 | 22 | - WebSocket API for computer-use 23 | - Cross-platform support (macOS, Linux) 24 | - Integration with CUA computer library for screen control, keyboard/mouse automation, and accessibility 25 | 26 | ## Install 27 | 28 | To install the Computer-Use Interface (CUI): 29 | 30 | ```bash 31 | pip install cua-computer-server 32 | ``` 33 | 34 | ## Run 35 | 36 | Refer to this notebook for a step-by-step guide on how to use the Computer-Use Server on the host system or VM: 37 | 38 | - [Computer-Use Server](../../notebooks/computer_server_nb.ipynb) -------------------------------------------------------------------------------- /libs/computer-server/computer_server/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Computer API package. 3 | Provides a server interface for the Computer API. 4 | """ 5 | 6 | from __future__ import annotations 7 | 8 | __version__: str = "0.1.0" 9 | 10 | # Explicitly export Server for static type checkers 11 | from .server import Server as Server # noqa: F401 12 | 13 | __all__ = ["Server", "run_cli"] 14 | 15 | 16 | def run_cli() -> None: 17 | """Entry point for CLI""" 18 | from .cli import main 19 | 20 | main() 21 | -------------------------------------------------------------------------------- /libs/computer-server/computer_server/__main__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Main entry point for running the Computer Server as a module. 3 | This allows the server to be started with `python -m computer_server`. 4 | """ 5 | 6 | import sys 7 | from .cli import main 8 | 9 | if __name__ == "__main__": 10 | sys.exit(main()) 11 | -------------------------------------------------------------------------------- /libs/computer-server/computer_server/cli.py: -------------------------------------------------------------------------------- 1 | """ 2 | Command-line interface for the Computer API server. 3 | """ 4 | 5 | import argparse 6 | import logging 7 | import sys 8 | from typing import List, Optional 9 | 10 | from .server import Server 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace: 16 | """Parse command-line arguments.""" 17 | parser = argparse.ArgumentParser(description="Start the Computer API server") 18 | parser.add_argument( 19 | "--host", default="0.0.0.0", help="Host to bind the server to (default: 0.0.0.0)" 20 | ) 21 | parser.add_argument( 22 | "--port", type=int, default=8000, help="Port to bind the server to (default: 8000)" 23 | ) 24 | parser.add_argument( 25 | "--log-level", 26 | choices=["debug", "info", "warning", "error", "critical"], 27 | default="info", 28 | help="Logging level (default: info)", 29 | ) 30 | parser.add_argument( 31 | "--ssl-keyfile", 32 | type=str, 33 | help="Path to SSL private key file (enables HTTPS)", 34 | ) 35 | parser.add_argument( 36 | "--ssl-certfile", 37 | type=str, 38 | help="Path to SSL certificate file (enables HTTPS)", 39 | ) 40 | 41 | return parser.parse_args(args) 42 | 43 | 44 | def main() -> None: 45 | """Main entry point for the CLI.""" 46 | args = parse_args() 47 | 48 | # Configure logging 49 | logging.basicConfig( 50 | level=getattr(logging, args.log_level.upper()), 51 | format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", 52 | ) 53 | 54 | # Create and start the server 55 | logger.info(f"Starting CUA Computer API server on {args.host}:{args.port}...") 56 | 57 | # Handle SSL configuration 58 | ssl_args = {} 59 | if args.ssl_keyfile and args.ssl_certfile: 60 | ssl_args = { 61 | "ssl_keyfile": args.ssl_keyfile, 62 | "ssl_certfile": args.ssl_certfile, 63 | } 64 | logger.info("HTTPS mode enabled with SSL certificates") 65 | elif args.ssl_keyfile or args.ssl_certfile: 66 | logger.warning("Both --ssl-keyfile and --ssl-certfile are required for HTTPS. Running in HTTP mode.") 67 | else: 68 | logger.info("HTTP mode (no SSL certificates provided)") 69 | 70 | server = Server(host=args.host, port=args.port, log_level=args.log_level, **ssl_args) 71 | 72 | try: 73 | server.start() 74 | except KeyboardInterrupt: 75 | logger.info("Server stopped by user") 76 | sys.exit(0) 77 | except Exception as e: 78 | logger.error(f"Error starting server: {e}") 79 | sys.exit(1) 80 | 81 | 82 | if __name__ == "__main__": 83 | main() 84 | -------------------------------------------------------------------------------- /libs/computer-server/computer_server/diorama/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trycua/cua/99b979ef114345fae36dba627f33577388759f47/libs/computer-server/computer_server/diorama/__init__.py -------------------------------------------------------------------------------- /libs/computer-server/computer_server/diorama/base.py: -------------------------------------------------------------------------------- 1 | class BaseDioramaHandler: 2 | """Base Diorama handler for unsupported OSes.""" 3 | async def diorama_cmd(self, action: str, arguments: dict = None) -> dict: 4 | return {"success": False, "error": "Diorama is not supported on this OS yet."} 5 | -------------------------------------------------------------------------------- /libs/computer-server/computer_server/diorama/diorama_computer.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | class DioramaComputer: 4 | """ 5 | A minimal Computer-like interface for Diorama, compatible with ComputerAgent. 6 | Implements _initialized, run(), and __aenter__ for agent compatibility. 7 | """ 8 | def __init__(self, diorama): 9 | self.diorama = diorama 10 | self.interface = self.diorama.interface 11 | self._initialized = False 12 | 13 | async def __aenter__(self): 14 | # Ensure the event loop is running (for compatibility) 15 | try: 16 | asyncio.get_running_loop() 17 | except RuntimeError: 18 | asyncio.set_event_loop(asyncio.new_event_loop()) 19 | self._initialized = True 20 | return self 21 | 22 | async def run(self): 23 | # This is a stub for compatibility 24 | if not self._initialized: 25 | await self.__aenter__() 26 | return self 27 | -------------------------------------------------------------------------------- /libs/computer-server/computer_server/diorama/macos.py: -------------------------------------------------------------------------------- 1 | import platform 2 | import sys 3 | import platform 4 | import inspect 5 | from computer_server.diorama.diorama import Diorama 6 | from computer_server.diorama.base import BaseDioramaHandler 7 | from typing import Optional 8 | 9 | class MacOSDioramaHandler(BaseDioramaHandler): 10 | """Handler for Diorama commands on macOS, using local diorama module.""" 11 | async def diorama_cmd(self, action: str, arguments: Optional[dict] = None) -> dict: 12 | if platform.system().lower() != "darwin": 13 | return {"success": False, "error": "Diorama is only supported on macOS."} 14 | try: 15 | app_list = arguments.get("app_list") if arguments else None 16 | if not app_list: 17 | return {"success": False, "error": "Missing 'app_list' in arguments"} 18 | diorama = Diorama(app_list) 19 | interface = diorama.interface 20 | if not hasattr(interface, action): 21 | return {"success": False, "error": f"Unknown diorama action: {action}"} 22 | method = getattr(interface, action) 23 | # Remove app_list from arguments before calling the method 24 | filtered_arguments = dict(arguments) 25 | filtered_arguments.pop("app_list", None) 26 | if inspect.iscoroutinefunction(method): 27 | result = await method(**(filtered_arguments or {})) 28 | else: 29 | result = method(**(filtered_arguments or {})) 30 | return {"success": True, "result": result} 31 | except Exception as e: 32 | import traceback 33 | return {"success": False, "error": str(e), "trace": traceback.format_exc()} 34 | -------------------------------------------------------------------------------- /libs/computer-server/computer_server/handlers/factory.py: -------------------------------------------------------------------------------- 1 | import platform 2 | import subprocess 3 | from typing import Tuple, Type 4 | from .base import BaseAccessibilityHandler, BaseAutomationHandler 5 | from computer_server.diorama.base import BaseDioramaHandler 6 | 7 | # Conditionally import platform-specific handlers 8 | system = platform.system().lower() 9 | if system == 'darwin': 10 | from .macos import MacOSAccessibilityHandler, MacOSAutomationHandler 11 | from computer_server.diorama.macos import MacOSDioramaHandler 12 | elif system == 'linux': 13 | from .linux import LinuxAccessibilityHandler, LinuxAutomationHandler 14 | 15 | class HandlerFactory: 16 | """Factory for creating OS-specific handlers.""" 17 | 18 | @staticmethod 19 | def _get_current_os() -> str: 20 | """Determine the current OS. 21 | 22 | Returns: 23 | str: The OS type ('darwin' for macOS or 'linux' for Linux) 24 | 25 | Raises: 26 | RuntimeError: If unable to determine the current OS 27 | """ 28 | try: 29 | # Use platform.system() as primary method 30 | system = platform.system().lower() 31 | if system in ['darwin', 'linux', 'windows']: 32 | return 'darwin' if system == 'darwin' else 'linux' if system == 'linux' else 'windows' 33 | 34 | # Fallback to uname if platform.system() doesn't return expected values 35 | result = subprocess.run(['uname', '-s'], capture_output=True, text=True) 36 | if result.returncode != 0: 37 | raise RuntimeError(f"uname command failed: {result.stderr}") 38 | return result.stdout.strip().lower() 39 | except Exception as e: 40 | raise RuntimeError(f"Failed to determine current OS: {str(e)}") 41 | 42 | @staticmethod 43 | def create_handlers() -> Tuple[BaseAccessibilityHandler, BaseAutomationHandler, BaseDioramaHandler]: 44 | """Create and return appropriate handlers for the current OS. 45 | 46 | Returns: 47 | Tuple[BaseAccessibilityHandler, BaseAutomationHandler, BaseDioramaHandler]: A tuple containing 48 | the appropriate accessibility, automation, and diorama handlers for the current OS. 49 | 50 | Raises: 51 | NotImplementedError: If the current OS is not supported 52 | RuntimeError: If unable to determine the current OS 53 | """ 54 | os_type = HandlerFactory._get_current_os() 55 | 56 | if os_type == 'darwin': 57 | return MacOSAccessibilityHandler(), MacOSAutomationHandler(), MacOSDioramaHandler() 58 | elif os_type == 'linux': 59 | return LinuxAccessibilityHandler(), LinuxAutomationHandler(), BaseDioramaHandler() 60 | else: 61 | raise NotImplementedError(f"OS '{os_type}' is not supported") -------------------------------------------------------------------------------- /libs/computer-server/examples/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Examples package for the CUA Computer API. 3 | """ 4 | -------------------------------------------------------------------------------- /libs/computer-server/examples/usage_example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Example showing how to use the CUA Computer API as an imported package. 4 | """ 5 | 6 | import asyncio 7 | import logging 8 | from typing import TYPE_CHECKING 9 | 10 | # For type checking only 11 | if TYPE_CHECKING: 12 | from computer_api import Server 13 | 14 | # Setup logging 15 | logging.basicConfig( 16 | level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" 17 | ) 18 | logger = logging.getLogger(__name__) 19 | 20 | 21 | # Example 1: Synchronous usage (blocks until server is stopped) 22 | def example_sync(): 23 | """ 24 | Example of synchronous server usage. This will block until interrupted. 25 | Run with: python3 -m examples.usage_example sync 26 | """ 27 | # Import directly to avoid any confusion 28 | from computer_api.server import Server 29 | 30 | server = Server(port=8080) 31 | print("Server started at http://localhost:8080") 32 | print("Press Ctrl+C to stop the server") 33 | 34 | try: 35 | server.start() # This will block until the server is stopped 36 | except KeyboardInterrupt: 37 | print("Server stopped by user") 38 | 39 | 40 | # Example 2: Asynchronous usage 41 | async def example_async(): 42 | """ 43 | Example of asynchronous server usage. This will start the server in the background 44 | and allow other operations to run concurrently. 45 | Run with: python3 -m examples.usage_example async 46 | """ 47 | # Import directly to avoid any confusion 48 | from computer_api.server import Server 49 | 50 | server = Server(port=8080) 51 | 52 | # Start the server in the background 53 | await server.start_async() 54 | 55 | print("Server is running in the background") 56 | print("Performing other tasks...") 57 | 58 | # Do other things while the server is running 59 | for i in range(5): 60 | print(f"Doing work iteration {i+1}/5...") 61 | await asyncio.sleep(2) 62 | 63 | print("Work complete, stopping server...") 64 | 65 | # Stop the server when done 66 | await server.stop() 67 | print("Server stopped") 68 | 69 | 70 | if __name__ == "__main__": 71 | import sys 72 | 73 | if len(sys.argv) > 1 and sys.argv[1] == "async": 74 | asyncio.run(example_async()) 75 | else: 76 | example_sync() 77 | -------------------------------------------------------------------------------- /libs/computer-server/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["pdm-backend"] 3 | build-backend = "pdm.backend" 4 | 5 | [project] 6 | name = "cua-computer-server" 7 | version = "0.1.0" 8 | description = "Server component for the Computer-Use Interface (CUI) framework powering Cua" 9 | authors = [ 10 | { name = "TryCua", email = "gh@trycua.com" } 11 | ] 12 | readme = "README.md" 13 | license = { text = "MIT" } 14 | requires-python = ">=3.9" 15 | dependencies = [ 16 | "fastapi>=0.111.0", 17 | "uvicorn[standard]>=0.27.0", 18 | "pydantic>=2.0.0", 19 | "pyautogui>=0.9.54", 20 | "pillow>=10.2.0", 21 | "aiohttp>=3.9.1" 22 | ] 23 | 24 | [project.optional-dependencies] 25 | macos = [ 26 | "pyobjc-framework-Cocoa>=10.1", 27 | "pyobjc-framework-Quartz>=10.1", 28 | "pyobjc-framework-ApplicationServices>=10.1" 29 | ] 30 | linux = [ 31 | "python-xlib>=0.33" 32 | ] 33 | 34 | [project.urls] 35 | homepage = "https://github.com/trycua/cua" 36 | repository = "https://github.com/trycua/cua" 37 | 38 | [project.scripts] 39 | cua-computer-server = "computer_server:run_cli" 40 | 41 | [tool.pdm] 42 | distribution = true 43 | 44 | [tool.pdm.build] 45 | includes = ["computer_server"] 46 | package-data = {"computer_server" = ["py.typed"]} 47 | 48 | [tool.pdm.dev-dependencies] 49 | test = [ 50 | "pytest>=7.0.0", 51 | "pytest-asyncio>=0.23.0" 52 | ] 53 | format = [ 54 | "black>=23.0.0", 55 | "isort>=5.12.0" 56 | ] 57 | dev = [ 58 | "ruff>=0.0.241", 59 | "mypy>=0.971" 60 | ] 61 | 62 | [tool.pdm.scripts] 63 | api = "python -m computer_server" 64 | 65 | [tool.ruff] 66 | line-length = 100 67 | target-version = "py310" 68 | select = ["E", "F", "B", "I"] 69 | fix = true 70 | 71 | [tool.ruff.format] 72 | docstring-code-format = true 73 | 74 | [tool.mypy] 75 | strict = true 76 | python_version = "3.10" 77 | ignore_missing_imports = true 78 | disallow_untyped_defs = true 79 | check_untyped_defs = true 80 | warn_return_any = true 81 | show_error_codes = true 82 | warn_unused_ignores = false -------------------------------------------------------------------------------- /libs/computer-server/run_server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Entrypoint script for the Computer Server. 4 | 5 | This script provides a simple way to start the Computer Server from the command line 6 | or using a launch configuration in an IDE. 7 | 8 | Usage: 9 | python run_server.py [--host HOST] [--port PORT] [--log-level LEVEL] 10 | """ 11 | 12 | import sys 13 | from computer_server.cli import main 14 | 15 | if __name__ == "__main__": 16 | sys.exit(main()) 17 | -------------------------------------------------------------------------------- /libs/computer-server/test_connection.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Connection test script for Computer Server. 4 | 5 | This script tests the WebSocket connection to the Computer Server and keeps 6 | it alive, allowing you to verify the server is running correctly. 7 | """ 8 | 9 | import asyncio 10 | import json 11 | import websockets 12 | import argparse 13 | import sys 14 | 15 | 16 | async def test_connection(host="localhost", port=8000, keep_alive=False): 17 | """Test connection to the Computer Server.""" 18 | uri = f"ws://{host}:{port}/ws" 19 | print(f"Connecting to {uri}...") 20 | 21 | try: 22 | async with websockets.connect(uri) as websocket: 23 | print("Connection established!") 24 | 25 | # Send a test command to get screen size 26 | await websocket.send(json.dumps({"command": "get_screen_size", "params": {}})) 27 | response = await websocket.recv() 28 | print(f"Response: {response}") 29 | 30 | if keep_alive: 31 | print("\nKeeping connection alive. Press Ctrl+C to exit...") 32 | while True: 33 | # Send a command every 5 seconds to keep the connection alive 34 | await asyncio.sleep(5) 35 | await websocket.send( 36 | json.dumps({"command": "get_cursor_position", "params": {}}) 37 | ) 38 | response = await websocket.recv() 39 | print(f"Cursor position: {response}") 40 | except websockets.exceptions.ConnectionClosed as e: 41 | print(f"Connection closed: {e}") 42 | return False 43 | except ConnectionRefusedError: 44 | print(f"Connection refused. Is the server running at {host}:{port}?") 45 | return False 46 | except Exception as e: 47 | print(f"Error: {e}") 48 | return False 49 | 50 | return True 51 | 52 | 53 | def parse_args(): 54 | parser = argparse.ArgumentParser(description="Test connection to Computer Server") 55 | parser.add_argument("--host", default="localhost", help="Host address (default: localhost)") 56 | parser.add_argument("--port", type=int, default=8000, help="Port number (default: 8000)") 57 | parser.add_argument("--keep-alive", action="store_true", help="Keep connection alive") 58 | return parser.parse_args() 59 | 60 | 61 | async def main(): 62 | args = parse_args() 63 | success = await test_connection(args.host, args.port, args.keep_alive) 64 | return 0 if success else 1 65 | 66 | 67 | if __name__ == "__main__": 68 | try: 69 | sys.exit(asyncio.run(main())) 70 | except KeyboardInterrupt: 71 | print("\nExiting...") 72 | sys.exit(0) 73 | -------------------------------------------------------------------------------- /libs/computer/computer/__init__.py: -------------------------------------------------------------------------------- 1 | """CUA Computer Interface for cross-platform computer control.""" 2 | 3 | import logging 4 | import sys 5 | 6 | __version__ = "0.1.0" 7 | 8 | # Initialize logging 9 | logger = logging.getLogger("cua.computer") 10 | 11 | # Initialize telemetry when the package is imported 12 | try: 13 | # Import from core telemetry 14 | from core.telemetry import ( 15 | is_telemetry_enabled, 16 | flush, 17 | record_event, 18 | ) 19 | 20 | # Check if telemetry is enabled 21 | if is_telemetry_enabled(): 22 | logger.info("Telemetry is enabled") 23 | 24 | # Record package initialization 25 | record_event( 26 | "module_init", 27 | { 28 | "module": "computer", 29 | "version": __version__, 30 | "python_version": sys.version, 31 | }, 32 | ) 33 | 34 | # Flush events to ensure they're sent 35 | flush() 36 | else: 37 | logger.info("Telemetry is disabled") 38 | except ImportError as e: 39 | # Telemetry not available 40 | logger.warning(f"Telemetry not available: {e}") 41 | except Exception as e: 42 | # Other issues with telemetry 43 | logger.warning(f"Error initializing telemetry: {e}") 44 | 45 | # Core components 46 | from .computer import Computer 47 | 48 | # Provider components 49 | from .providers.base import VMProviderType 50 | 51 | __all__ = ["Computer", "VMProviderType"] 52 | -------------------------------------------------------------------------------- /libs/computer/computer/interface/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Interface package for Computer SDK. 3 | """ 4 | 5 | from .factory import InterfaceFactory 6 | from .base import BaseComputerInterface 7 | from .macos import MacOSComputerInterface 8 | 9 | __all__ = [ 10 | "InterfaceFactory", 11 | "BaseComputerInterface", 12 | "MacOSComputerInterface", 13 | ] -------------------------------------------------------------------------------- /libs/computer/computer/interface/factory.py: -------------------------------------------------------------------------------- 1 | """Factory for creating computer interfaces.""" 2 | 3 | from typing import Literal, Optional 4 | from .base import BaseComputerInterface 5 | 6 | class InterfaceFactory: 7 | """Factory for creating OS-specific computer interfaces.""" 8 | 9 | @staticmethod 10 | def create_interface_for_os( 11 | os: Literal['macos', 'linux'], 12 | ip_address: str, 13 | api_key: Optional[str] = None, 14 | vm_name: Optional[str] = None 15 | ) -> BaseComputerInterface: 16 | """Create an interface for the specified OS. 17 | 18 | Args: 19 | os: Operating system type ('macos' or 'linux') 20 | ip_address: IP address of the computer to control 21 | api_key: Optional API key for cloud authentication 22 | vm_name: Optional VM name for cloud authentication 23 | 24 | Returns: 25 | BaseComputerInterface: The appropriate interface for the OS 26 | 27 | Raises: 28 | ValueError: If the OS type is not supported 29 | """ 30 | # Import implementations here to avoid circular imports 31 | from .macos import MacOSComputerInterface 32 | from .linux import LinuxComputerInterface 33 | 34 | if os == 'macos': 35 | return MacOSComputerInterface(ip_address, api_key=api_key, vm_name=vm_name) 36 | elif os == 'linux': 37 | return LinuxComputerInterface(ip_address, api_key=api_key, vm_name=vm_name) 38 | else: 39 | raise ValueError(f"Unsupported OS type: {os}") -------------------------------------------------------------------------------- /libs/computer/computer/logger.py: -------------------------------------------------------------------------------- 1 | """Logging utilities for the Computer module.""" 2 | 3 | import logging 4 | from enum import IntEnum 5 | 6 | 7 | # Keep LogLevel for backward compatibility, but it will be deprecated 8 | class LogLevel(IntEnum): 9 | """Log levels for logging. Deprecated - use standard logging levels instead.""" 10 | 11 | QUIET = 0 # Only warnings and errors 12 | NORMAL = 1 # Info level, standard output 13 | VERBOSE = 2 # More detailed information 14 | DEBUG = 3 # Full debug information 15 | 16 | 17 | # Map LogLevel to standard logging levels for backward compatibility 18 | LOGLEVEL_MAP = { 19 | LogLevel.QUIET: logging.WARNING, 20 | LogLevel.NORMAL: logging.INFO, 21 | LogLevel.VERBOSE: logging.DEBUG, 22 | LogLevel.DEBUG: logging.DEBUG, 23 | } 24 | 25 | 26 | class Logger: 27 | """Logger class for Computer.""" 28 | 29 | def __init__(self, name: str, verbosity: int): 30 | """Initialize the logger. 31 | 32 | Args: 33 | name: The name of the logger. 34 | verbosity: The log level (use standard logging levels like logging.INFO). 35 | For backward compatibility, LogLevel enum values are also accepted. 36 | """ 37 | self.logger = logging.getLogger(name) 38 | 39 | # Convert LogLevel enum to standard logging level if needed 40 | if isinstance(verbosity, LogLevel): 41 | self.verbosity = LOGLEVEL_MAP.get(verbosity, logging.INFO) 42 | else: 43 | self.verbosity = verbosity 44 | 45 | self._configure() 46 | 47 | def _configure(self): 48 | """Configure the logger based on log level.""" 49 | # Set the logging level directly 50 | self.logger.setLevel(self.verbosity) 51 | 52 | # Log the verbosity level that was set 53 | if self.verbosity <= logging.DEBUG: 54 | self.logger.info("Logger set to DEBUG level") 55 | elif self.verbosity <= logging.INFO: 56 | self.logger.info("Logger set to INFO level") 57 | elif self.verbosity <= logging.WARNING: 58 | self.logger.warning("Logger set to WARNING level") 59 | elif self.verbosity <= logging.ERROR: 60 | self.logger.warning("Logger set to ERROR level") 61 | elif self.verbosity <= logging.CRITICAL: 62 | self.logger.warning("Logger set to CRITICAL level") 63 | 64 | def debug(self, message: str): 65 | """Log a debug message if log level is DEBUG or lower.""" 66 | self.logger.debug(message) 67 | 68 | def info(self, message: str): 69 | """Log an info message if log level is INFO or lower.""" 70 | self.logger.info(message) 71 | 72 | def verbose(self, message: str): 73 | """Log a verbose message between INFO and DEBUG levels.""" 74 | # Since there's no standard verbose level, 75 | # use debug level with [VERBOSE] prefix for backward compatibility 76 | self.logger.debug(f"[VERBOSE] {message}") 77 | 78 | def warning(self, message: str): 79 | """Log a warning message.""" 80 | self.logger.warning(message) 81 | 82 | def error(self, message: str): 83 | """Log an error message.""" 84 | self.logger.error(message) 85 | -------------------------------------------------------------------------------- /libs/computer/computer/models.py: -------------------------------------------------------------------------------- 1 | """Models for computer configuration.""" 2 | 3 | from dataclasses import dataclass 4 | from typing import Optional, Any, Dict 5 | 6 | # Import base provider interface 7 | from .providers.base import BaseVMProvider 8 | 9 | @dataclass 10 | class Display: 11 | """Display configuration.""" 12 | width: int 13 | height: int 14 | 15 | @dataclass 16 | class Image: 17 | """VM image configuration.""" 18 | image: str 19 | tag: str 20 | name: str 21 | 22 | @dataclass 23 | class Computer: 24 | """Computer configuration.""" 25 | image: str 26 | tag: str 27 | name: str 28 | display: Display 29 | memory: str 30 | cpu: str 31 | vm_provider: Optional[BaseVMProvider] = None 32 | 33 | # @property # Remove the property decorator 34 | async def get_ip(self) -> Optional[str]: 35 | """Get the IP address of the VM.""" 36 | if not self.vm_provider: 37 | return None 38 | 39 | vm = await self.vm_provider.get_vm(self.name) 40 | # Handle both object attribute and dictionary access for ip_address 41 | if vm: 42 | if isinstance(vm, dict): 43 | return vm.get("ip_address") 44 | else: 45 | # Access as attribute for object-based return values 46 | return getattr(vm, "ip_address", None) 47 | return None -------------------------------------------------------------------------------- /libs/computer/computer/providers/__init__.py: -------------------------------------------------------------------------------- 1 | """Provider implementations for different VM backends.""" 2 | 3 | # Import specific providers only when needed to avoid circular imports 4 | __all__ = [] # Let each provider module handle its own exports 5 | -------------------------------------------------------------------------------- /libs/computer/computer/providers/cloud/__init__.py: -------------------------------------------------------------------------------- 1 | """CloudProvider module for interacting with cloud-based virtual machines.""" 2 | 3 | from .provider import CloudProvider 4 | 5 | __all__ = ["CloudProvider"] 6 | -------------------------------------------------------------------------------- /libs/computer/computer/providers/cloud/provider.py: -------------------------------------------------------------------------------- 1 | """Cloud VM provider implementation. 2 | 3 | This module contains a stub implementation for a future cloud VM provider. 4 | """ 5 | 6 | import logging 7 | from typing import Dict, List, Optional, Any 8 | 9 | from ..base import BaseVMProvider, VMProviderType 10 | 11 | # Setup logging 12 | logger = logging.getLogger(__name__) 13 | 14 | import asyncio 15 | import aiohttp 16 | from urllib.parse import urlparse 17 | 18 | class CloudProvider(BaseVMProvider): 19 | """Cloud VM Provider implementation.""" 20 | def __init__( 21 | self, 22 | api_key: str, 23 | verbose: bool = False, 24 | **kwargs, 25 | ): 26 | """ 27 | Args: 28 | api_key: API key for authentication 29 | name: Name of the VM 30 | verbose: Enable verbose logging 31 | """ 32 | assert api_key, "api_key required for CloudProvider" 33 | self.api_key = api_key 34 | self.verbose = verbose 35 | 36 | @property 37 | def provider_type(self) -> VMProviderType: 38 | return VMProviderType.CLOUD 39 | 40 | async def __aenter__(self): 41 | return self 42 | 43 | async def __aexit__(self, exc_type, exc_val, exc_tb): 44 | pass 45 | 46 | async def get_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]: 47 | """Get VM VNC URL by name using the cloud API.""" 48 | return {"name": name, "hostname": f"{name}.containers.cloud.trycua.com"} 49 | 50 | async def list_vms(self) -> List[Dict[str, Any]]: 51 | logger.warning("CloudProvider.list_vms is not implemented") 52 | return [] 53 | 54 | async def run_vm(self, image: str, name: str, run_opts: Dict[str, Any], storage: Optional[str] = None) -> Dict[str, Any]: 55 | logger.warning("CloudProvider.run_vm is not implemented") 56 | return {"name": name, "status": "unavailable", "message": "CloudProvider is not implemented"} 57 | 58 | async def stop_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]: 59 | logger.warning("CloudProvider.stop_vm is not implemented") 60 | return {"name": name, "status": "stopped", "message": "CloudProvider is not implemented"} 61 | 62 | async def update_vm(self, name: str, update_opts: Dict[str, Any], storage: Optional[str] = None) -> Dict[str, Any]: 63 | logger.warning("CloudProvider.update_vm is not implemented") 64 | return {"name": name, "status": "unchanged", "message": "CloudProvider is not implemented"} 65 | 66 | async def get_ip(self, name: Optional[str] = None, storage: Optional[str] = None, retry_delay: int = 2) -> str: 67 | """ 68 | Return the VM's IP address as '{container_name}.containers.cloud.trycua.com'. 69 | Uses the provided 'name' argument (the VM name requested by the caller), 70 | falling back to self.name only if 'name' is None. 71 | Retries up to 3 times with retry_delay seconds if hostname is not available. 72 | """ 73 | if name is None: 74 | raise ValueError("VM name is required for CloudProvider.get_ip") 75 | return f"{name}.containers.cloud.trycua.com" 76 | -------------------------------------------------------------------------------- /libs/computer/computer/providers/lume/__init__.py: -------------------------------------------------------------------------------- 1 | """Lume VM provider implementation.""" 2 | 3 | try: 4 | from .provider import LumeProvider 5 | HAS_LUME = True 6 | __all__ = ["LumeProvider"] 7 | except ImportError: 8 | HAS_LUME = False 9 | __all__ = [] 10 | -------------------------------------------------------------------------------- /libs/computer/computer/providers/lumier/__init__.py: -------------------------------------------------------------------------------- 1 | """Lumier VM provider implementation.""" 2 | 3 | try: 4 | # Use the same import approach as in the Lume provider 5 | from .provider import LumierProvider 6 | HAS_LUMIER = True 7 | except ImportError: 8 | HAS_LUMIER = False 9 | -------------------------------------------------------------------------------- /libs/computer/computer/ui/__init__.py: -------------------------------------------------------------------------------- 1 | """UI modules for the Computer Interface.""" 2 | -------------------------------------------------------------------------------- /libs/computer/computer/ui/gradio/__init__.py: -------------------------------------------------------------------------------- 1 | """Gradio UI for Computer UI.""" 2 | 3 | import gradio as gr 4 | from typing import Optional 5 | 6 | from .app import create_gradio_ui 7 | -------------------------------------------------------------------------------- /libs/computer/poetry.toml: -------------------------------------------------------------------------------- 1 | [virtualenvs] 2 | in-project = true 3 | -------------------------------------------------------------------------------- /libs/computer/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["pdm-backend"] 3 | build-backend = "pdm.backend" 4 | 5 | [project] 6 | name = "cua-computer" 7 | version = "0.1.0" 8 | description = "Computer-Use Interface (CUI) framework powering Cua" 9 | readme = "README.md" 10 | authors = [ 11 | { name = "TryCua", email = "gh@trycua.com" } 12 | ] 13 | dependencies = [ 14 | "pillow>=10.0.0", 15 | "websocket-client>=1.8.0", 16 | "websockets>=12.0", 17 | "aiohttp>=3.9.0", 18 | "cua-core>=0.1.0,<0.2.0", 19 | "pydantic>=2.11.1" 20 | ] 21 | requires-python = ">=3.11" 22 | 23 | [project.optional-dependencies] 24 | lume = [ 25 | ] 26 | lumier = [ 27 | ] 28 | ui = [ 29 | "gradio>=5.23.3,<6.0.0", 30 | "python-dotenv>=1.0.1,<2.0.0", 31 | "datasets>=3.6.0,<4.0.0", 32 | ] 33 | all = [ 34 | # Include all optional dependencies 35 | "gradio>=5.23.3,<6.0.0", 36 | "python-dotenv>=1.0.1,<2.0.0", 37 | "datasets>=3.6.0,<4.0.0", 38 | ] 39 | 40 | [tool.pdm] 41 | distribution = true 42 | 43 | [tool.pdm.build] 44 | includes = ["computer/"] 45 | source-includes = ["tests/", "README.md", "LICENSE"] 46 | 47 | [tool.black] 48 | line-length = 100 49 | target-version = ["py311"] 50 | 51 | [tool.ruff] 52 | line-length = 100 53 | target-version = "py311" 54 | select = ["E", "F", "B", "I"] 55 | fix = true 56 | 57 | [tool.ruff.format] 58 | docstring-code-format = true 59 | 60 | [tool.mypy] 61 | strict = true 62 | python_version = "3.11" 63 | ignore_missing_imports = true 64 | disallow_untyped_defs = true 65 | check_untyped_defs = true 66 | warn_return_any = true 67 | show_error_codes = true 68 | warn_unused_ignores = false 69 | 70 | [tool.pytest.ini_options] 71 | asyncio_mode = "auto" 72 | testpaths = ["tests"] 73 | python_files = "test_*.py" -------------------------------------------------------------------------------- /libs/core/README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 |
4 | 5 | 6 | 7 | Shows my svg 8 | 9 |
10 | 11 | [![Python](https://img.shields.io/badge/Python-333333?logo=python&logoColor=white&labelColor=333333)](#) 12 | [![macOS](https://img.shields.io/badge/macOS-000000?logo=apple&logoColor=F0F0F0)](#) 13 | [![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?&logo=discord&logoColor=white)](https://discord.com/invite/mVnXXpdE85) 14 | [![PyPI](https://img.shields.io/pypi/v/cua-core?color=333333)](https://pypi.org/project/cua-core/) 15 |

16 |
17 | 18 | **Cua Core** provides essential shared functionality and utilities used across the Cua ecosystem: 19 | 20 | - Privacy-focused telemetry system for transparent usage analytics 21 | - Common helper functions and utilities used by other Cua packages 22 | - Core infrastructure components shared between modules 23 | 24 | ## Installation 25 | 26 | ```bash 27 | pip install cua-core 28 | ``` -------------------------------------------------------------------------------- /libs/core/core/__init__.py: -------------------------------------------------------------------------------- 1 | """Core functionality shared across Cua components.""" 2 | 3 | __version__ = "0.1.0" 4 | -------------------------------------------------------------------------------- /libs/core/core/telemetry/__init__.py: -------------------------------------------------------------------------------- 1 | """This module provides the core telemetry functionality for CUA libraries. 2 | 3 | It provides a low-overhead way to collect anonymous usage data. 4 | """ 5 | 6 | from core.telemetry.telemetry import ( 7 | UniversalTelemetryClient, 8 | enable_telemetry, 9 | disable_telemetry, 10 | flush, 11 | get_telemetry_client, 12 | increment, 13 | record_event, 14 | is_telemetry_enabled, 15 | is_telemetry_globally_disabled, 16 | ) 17 | 18 | 19 | __all__ = [ 20 | "UniversalTelemetryClient", 21 | "enable_telemetry", 22 | "disable_telemetry", 23 | "flush", 24 | "get_telemetry_client", 25 | "increment", 26 | "record_event", 27 | "is_telemetry_enabled", 28 | "is_telemetry_globally_disabled", 29 | ] 30 | -------------------------------------------------------------------------------- /libs/core/core/telemetry/models.py: -------------------------------------------------------------------------------- 1 | """Models for telemetry data.""" 2 | 3 | from __future__ import annotations 4 | 5 | from datetime import datetime 6 | from typing import Any, Dict, List, Optional 7 | 8 | from pydantic import BaseModel, Field 9 | 10 | 11 | class TelemetryEvent(BaseModel): 12 | """A telemetry event with properties.""" 13 | 14 | name: str 15 | properties: Dict[str, Any] = Field(default_factory=dict) 16 | timestamp: float = Field(default_factory=lambda: datetime.now().timestamp()) 17 | 18 | 19 | class TelemetryPayload(BaseModel): 20 | """Telemetry payload sent to the server.""" 21 | 22 | version: str 23 | installation_id: str 24 | counters: Dict[str, int] = Field(default_factory=dict) 25 | events: List[TelemetryEvent] = Field(default_factory=list) 26 | duration: float = 0 27 | timestamp: float = Field(default_factory=lambda: datetime.now().timestamp()) 28 | 29 | 30 | class UserRecord(BaseModel): 31 | """User record stored in the telemetry database.""" 32 | 33 | id: str 34 | version: Optional[str] = None 35 | created_at: Optional[datetime] = None 36 | last_seen_at: Optional[datetime] = None 37 | is_ci: bool = False 38 | -------------------------------------------------------------------------------- /libs/core/core/telemetry/sender.py: -------------------------------------------------------------------------------- 1 | """Telemetry sender module for sending anonymous usage data.""" 2 | 3 | import logging 4 | from typing import Any, Dict 5 | 6 | logger = logging.getLogger("cua.telemetry") 7 | 8 | 9 | def send_telemetry(payload: Dict[str, Any]) -> bool: 10 | """Send telemetry data to collection endpoint. 11 | 12 | Args: 13 | payload: Telemetry data to send 14 | 15 | Returns: 16 | bool: True if sending was successful, False otherwise 17 | """ 18 | try: 19 | # For now, just log the payload and return success 20 | logger.debug(f"Would send telemetry: {payload}") 21 | return True 22 | except Exception as e: 23 | logger.debug(f"Error sending telemetry: {e}") 24 | return False 25 | -------------------------------------------------------------------------------- /libs/core/poetry.toml: -------------------------------------------------------------------------------- 1 | [virtualenvs] 2 | in-project = true -------------------------------------------------------------------------------- /libs/core/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["pdm-backend"] 3 | build-backend = "pdm.backend" 4 | 5 | [project] 6 | name = "cua-core" 7 | version = "0.1.0" 8 | description = "Core functionality for Cua including telemetry and shared utilities" 9 | readme = "README.md" 10 | authors = [ 11 | { name = "TryCua", email = "gh@trycua.com" } 12 | ] 13 | dependencies = [ 14 | "pydantic>=2.0.0", 15 | "httpx>=0.24.0", 16 | "posthog>=3.20.0" 17 | ] 18 | requires-python = ">=3.11" 19 | 20 | [tool.pdm] 21 | distribution = true 22 | 23 | [tool.pdm.build] 24 | includes = ["core/"] 25 | source-includes = ["tests/", "README.md", "LICENSE"] 26 | 27 | [tool.black] 28 | line-length = 100 29 | target-version = ["py311"] 30 | 31 | [tool.ruff] 32 | line-length = 100 33 | target-version = "py311" 34 | select = ["E", "F", "B", "I"] 35 | fix = true 36 | 37 | [tool.ruff.format] 38 | docstring-code-format = true 39 | 40 | [tool.mypy] 41 | strict = true 42 | python_version = "3.11" 43 | ignore_missing_imports = true 44 | disallow_untyped_defs = true 45 | check_untyped_defs = true 46 | warn_return_any = true 47 | show_error_codes = true 48 | warn_unused_ignores = false 49 | 50 | [tool.pytest.ini_options] 51 | asyncio_mode = "auto" 52 | testpaths = ["tests"] 53 | python_files = "test_*.py" 54 | [dependency-groups] 55 | dev = [ 56 | "pytest>=8.3.5", 57 | ] 58 | -------------------------------------------------------------------------------- /libs/lume/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to lume 2 | 3 | We deeply appreciate your interest in contributing to lume! Whether you're reporting bugs, suggesting enhancements, improving docs, or submitting pull requests, your contributions help improve the project for everyone. 4 | 5 | ## Reporting Bugs 6 | 7 | If you've encountered a bug in the project, we encourage you to report it. Please follow these steps: 8 | 9 | 1. **Check the Issue Tracker**: Before submitting a new bug report, please check our issue tracker to see if the bug has already been reported. 10 | 2. **Create a New Issue**: If the bug hasn't been reported, create a new issue with: 11 | - A clear title and detailed description 12 | - Steps to reproduce the issue 13 | - Expected vs actual behavior 14 | - Your environment (macOS version, lume version) 15 | - Any relevant logs or error messages 16 | 3. **Label Your Issue**: Label your issue as a `bug` to help maintainers identify it quickly. 17 | 18 | ## Suggesting Enhancements 19 | 20 | We're always looking for suggestions to make lume better. If you have an idea: 21 | 22 | 1. **Check Existing Issues**: See if someone else has already suggested something similar. 23 | 2. **Create a New Issue**: If your enhancement is new, create an issue describing: 24 | - The problem your enhancement solves 25 | - How your enhancement would work 26 | - Any potential implementation details 27 | - Why this enhancement would benefit lume users 28 | 29 | ## Documentation 30 | 31 | Documentation improvements are always welcome. You can: 32 | - Fix typos or unclear explanations 33 | - Add examples and use cases 34 | - Improve API documentation 35 | - Add tutorials or guides 36 | 37 | For detailed instructions on setting up your development environment and submitting code contributions, please see our [Development.md](docs/Development.md) guide. 38 | 39 | Feel free to join our [Discord community](https://discord.com/invite/mVnXXpdE85) to discuss ideas or get help with your contributions. -------------------------------------------------------------------------------- /libs/lume/Package.resolved: -------------------------------------------------------------------------------- 1 | { 2 | "originHash" : "81a9d169da3c391b981b894044911091d11285486aab463e32222490c931ba45", 3 | "pins" : [ 4 | { 5 | "identity" : "dynamic", 6 | "kind" : "remoteSourceControl", 7 | "location" : "https://github.com/mhdhejazi/Dynamic", 8 | "state" : { 9 | "branch" : "master", 10 | "revision" : "772883073d044bc754d401cabb6574624eb3778f" 11 | } 12 | }, 13 | { 14 | "identity" : "swift-argument-parser", 15 | "kind" : "remoteSourceControl", 16 | "location" : "https://github.com/apple/swift-argument-parser", 17 | "state" : { 18 | "revision" : "41982a3656a71c768319979febd796c6fd111d5c", 19 | "version" : "1.5.0" 20 | } 21 | }, 22 | { 23 | "identity" : "swift-atomics", 24 | "kind" : "remoteSourceControl", 25 | "location" : "https://github.com/apple/swift-atomics.git", 26 | "state" : { 27 | "revision" : "cd142fd2f64be2100422d658e7411e39489da985", 28 | "version" : "1.2.0" 29 | } 30 | }, 31 | { 32 | "identity" : "swift-cmark", 33 | "kind" : "remoteSourceControl", 34 | "location" : "https://github.com/apple/swift-cmark.git", 35 | "state" : { 36 | "revision" : "3ccff77b2dc5b96b77db3da0d68d28068593fa53", 37 | "version" : "0.5.0" 38 | } 39 | }, 40 | { 41 | "identity" : "swift-format", 42 | "kind" : "remoteSourceControl", 43 | "location" : "https://github.com/apple/swift-format.git", 44 | "state" : { 45 | "branch" : "release/5.10", 46 | "revision" : "3191b8f3109730af449c6332d0b1ca6653b857a0" 47 | } 48 | }, 49 | { 50 | "identity" : "swift-markdown", 51 | "kind" : "remoteSourceControl", 52 | "location" : "https://github.com/apple/swift-markdown.git", 53 | "state" : { 54 | "revision" : "8f79cb175981458a0a27e76cb42fee8e17b1a993", 55 | "version" : "0.5.0" 56 | } 57 | }, 58 | { 59 | "identity" : "swift-syntax", 60 | "kind" : "remoteSourceControl", 61 | "location" : "https://github.com/apple/swift-syntax.git", 62 | "state" : { 63 | "branch" : "release/5.10", 64 | "revision" : "cdd571f366a4298bb863a9dcfe1295bb595041d5" 65 | } 66 | } 67 | ], 68 | "version" : 3 69 | } 70 | -------------------------------------------------------------------------------- /libs/lume/Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version: 6.0 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "lume", 8 | platforms: [ 9 | .macOS(.v14) 10 | ], 11 | dependencies: [ 12 | .package(url: "https://github.com/apple/swift-argument-parser", from: "1.3.1"), 13 | .package(url: "https://github.com/apple/swift-format.git", branch: ("release/5.10")), 14 | .package(url: "https://github.com/apple/swift-atomics.git", .upToNextMajor(from: "1.2.0")), 15 | .package(url: "https://github.com/mhdhejazi/Dynamic", branch: "master") 16 | ], 17 | targets: [ 18 | // Targets are the basic building blocks of a package, defining a module or a test suite. 19 | // Targets can depend on other targets in this package and products from dependencies. 20 | .executableTarget( 21 | name: "lume", 22 | dependencies: [ 23 | .product(name: "ArgumentParser", package: "swift-argument-parser"), 24 | .product(name: "Atomics", package: "swift-atomics"), 25 | .product(name: "Dynamic", package: "Dynamic") 26 | ], 27 | path: "src"), 28 | .testTarget( 29 | name: "lumeTests", 30 | dependencies: [ 31 | "lume" 32 | ], 33 | path: "tests") 34 | ] 35 | ) 36 | -------------------------------------------------------------------------------- /libs/lume/docs/Development.md: -------------------------------------------------------------------------------- 1 | # Development Guide 2 | 3 | This guide will help you set up your development environment and understand the process for contributing code to lume. 4 | 5 | ## Environment Setup 6 | 7 | Lume development requires: 8 | - Swift 6 or higher 9 | - Xcode 15 or higher 10 | - macOS Sequoia 15.2 or higher 11 | - (Optional) VS Code with Swift extension 12 | 13 | ## Setting Up the Repository Locally 14 | 15 | 1. **Fork the Repository**: Create your own fork of lume 16 | 2. **Clone the Repository**: 17 | ```bash 18 | git clone https://github.com/trycua/lume.git 19 | cd lume 20 | ``` 21 | 3. **Install Dependencies**: 22 | ```bash 23 | swift package resolve 24 | ``` 25 | 4. **Build the Project**: 26 | ```bash 27 | swift build 28 | ``` 29 | 30 | ## Development Workflow 31 | 32 | 1. Create a new branch for your changes 33 | 2. Make your changes 34 | 3. Run the tests: `swift test` 35 | 4. Build and test your changes locally 36 | 5. Commit your changes with clear commit messages 37 | 38 | ## Submitting Pull Requests 39 | 40 | 1. Push your changes to your fork 41 | 2. Open a Pull Request with: 42 | - A clear title and description 43 | - Reference to any related issues 44 | - Screenshots or logs if relevant 45 | 3. Respond to any feedback from maintainers 46 | -------------------------------------------------------------------------------- /libs/lume/img/cli.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trycua/cua/99b979ef114345fae36dba627f33577388759f47/libs/lume/img/cli.png -------------------------------------------------------------------------------- /libs/lume/img/logo_black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trycua/cua/99b979ef114345fae36dba627f33577388759f47/libs/lume/img/logo_black.png -------------------------------------------------------------------------------- /libs/lume/img/logo_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trycua/cua/99b979ef114345fae36dba627f33577388759f47/libs/lume/img/logo_white.png -------------------------------------------------------------------------------- /libs/lume/resources/lume.entitlements: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | com.apple.security.virtualization 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /libs/lume/scripts/build/build-debug.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | swift build --product lume 4 | codesign --force --entitlement resources/lume.entitlements --sign - .build/debug/lume 5 | -------------------------------------------------------------------------------- /libs/lume/scripts/build/build-release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | pushd ../../ 4 | 5 | swift build -c release --product lume 6 | codesign --force --entitlement ./resources/lume.entitlements --sign - .build/release/lume 7 | 8 | mkdir -p ./.release 9 | cp -f .build/release/lume ./.release/lume 10 | 11 | # Install to user-local bin directory (standard location) 12 | USER_BIN="$HOME/.local/bin" 13 | mkdir -p "$USER_BIN" 14 | cp -f ./.release/lume "$USER_BIN/lume" 15 | 16 | # Advise user to add to PATH if not present 17 | if ! echo "$PATH" | grep -q "$USER_BIN"; then 18 | echo "[lume build] Note: $USER_BIN is not in your PATH. Add 'export PATH=\"$USER_BIN:\$PATH\"' to your shell profile." 19 | fi 20 | 21 | popd -------------------------------------------------------------------------------- /libs/lume/src/Commands/Clone.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | 4 | struct Clone: AsyncParsableCommand { 5 | static let configuration = CommandConfiguration( 6 | abstract: "Clone an existing virtual machine" 7 | ) 8 | 9 | @Argument(help: "Name of the source virtual machine", completion: .custom(completeVMName)) 10 | var name: String 11 | 12 | @Argument(help: "Name for the cloned virtual machine") 13 | var newName: String 14 | 15 | @Option(name: .customLong("source-storage"), help: "Source VM storage location") 16 | var sourceStorage: String? 17 | 18 | @Option(name: .customLong("dest-storage"), help: "Destination VM storage location") 19 | var destStorage: String? 20 | 21 | init() {} 22 | 23 | @MainActor 24 | func run() async throws { 25 | let vmController = LumeController() 26 | try vmController.clone( 27 | name: name, 28 | newName: newName, 29 | sourceLocation: sourceStorage, 30 | destLocation: destStorage 31 | ) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /libs/lume/src/Commands/Create.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | import Virtualization 4 | 5 | // MARK: - Create Command 6 | 7 | struct Create: AsyncParsableCommand { 8 | static let configuration = CommandConfiguration( 9 | abstract: "Create a new virtual machine" 10 | ) 11 | 12 | @Argument(help: "Name for the virtual machine") 13 | var name: String 14 | 15 | @Option( 16 | help: "Operating system to install. Defaults to macOS.", 17 | completion: .list(["macOS", "linux"])) 18 | var os: String = "macOS" 19 | 20 | @Option(help: "Number of CPU cores", transform: { Int($0) ?? 4 }) 21 | var cpu: Int = 4 22 | 23 | @Option( 24 | help: "Memory size, e.g., 8192MB or 8GB. Defaults to 8GB.", transform: { try parseSize($0) } 25 | ) 26 | var memory: UInt64 = 8 * 1024 * 1024 * 1024 27 | 28 | @Option( 29 | help: "Disk size, e.g., 20480MB or 20GB. Defaults to 50GB.", 30 | transform: { try parseSize($0) }) 31 | var diskSize: UInt64 = 50 * 1024 * 1024 * 1024 32 | 33 | @Option(help: "Display resolution in format WIDTHxHEIGHT. Defaults to 1024x768.") 34 | var display: VMDisplayResolution = VMDisplayResolution(string: "1024x768")! 35 | 36 | @Option( 37 | help: 38 | "Path to macOS restore image (IPSW), or 'latest' to download the latest supported version. Required for macOS VMs.", 39 | completion: .file(extensions: ["ipsw"]) 40 | ) 41 | var ipsw: String? 42 | 43 | @Option(name: .customLong("storage"), help: "VM storage location to use or direct path to VM location") 44 | var storage: String? 45 | 46 | init() { 47 | } 48 | 49 | @MainActor 50 | func run() async throws { 51 | let controller = LumeController() 52 | try await controller.create( 53 | name: name, 54 | os: os, 55 | diskSize: diskSize, 56 | cpuCount: cpu, 57 | memorySize: memory, 58 | display: display.string, 59 | ipsw: ipsw, 60 | storage: storage 61 | ) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /libs/lume/src/Commands/Delete.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | 4 | struct Delete: AsyncParsableCommand { 5 | static let configuration = CommandConfiguration( 6 | abstract: "Delete a virtual machine" 7 | ) 8 | 9 | @Argument(help: "Name of the virtual machine to delete", completion: .custom(completeVMName)) 10 | var name: String 11 | 12 | @Flag(name: .long, help: "Force deletion without confirmation") 13 | var force = false 14 | 15 | @Option(name: .customLong("storage"), help: "VM storage location to use or direct path to VM location") 16 | var storage: String? 17 | 18 | init() {} 19 | 20 | @MainActor 21 | func run() async throws { 22 | if !force { 23 | print( 24 | "Are you sure you want to delete the virtual machine '\(name)'? [y/N] ", 25 | terminator: "") 26 | guard let response = readLine()?.lowercased(), 27 | response == "y" || response == "yes" 28 | else { 29 | print("Deletion cancelled") 30 | return 31 | } 32 | } 33 | 34 | let vmController = LumeController() 35 | try await vmController.delete(name: name, storage: storage) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /libs/lume/src/Commands/Get.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | 4 | struct Get: AsyncParsableCommand { 5 | static let configuration = CommandConfiguration( 6 | abstract: "Get detailed information about a virtual machine" 7 | ) 8 | 9 | @Argument(help: "Name of the virtual machine", completion: .custom(completeVMName)) 10 | var name: String 11 | 12 | @Option(name: [.long, .customShort("f")], help: "Output format (json|text)") 13 | var format: FormatOption = .text 14 | 15 | @Option(name: .customLong("storage"), help: "VM storage location to use or direct path to VM location") 16 | var storage: String? 17 | 18 | init() { 19 | } 20 | 21 | @MainActor 22 | func run() async throws { 23 | let vmController = LumeController() 24 | let vm = try vmController.get(name: name, storage: storage) 25 | try VMDetailsPrinter.printStatus([vm.details], format: self.format) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /libs/lume/src/Commands/IPSW.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | 4 | struct IPSW: AsyncParsableCommand { 5 | static let configuration = CommandConfiguration( 6 | abstract: "Get macOS restore image IPSW URL", 7 | discussion: "Download IPSW file manually, then use in create command with --ipsw" 8 | ) 9 | 10 | init() { 11 | 12 | } 13 | 14 | @MainActor 15 | func run() async throws { 16 | let vmController = LumeController() 17 | let url = try await vmController.getLatestIPSWURL() 18 | print(url.absoluteString) 19 | } 20 | } -------------------------------------------------------------------------------- /libs/lume/src/Commands/Images.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | 4 | struct Images: AsyncParsableCommand { 5 | static let configuration = CommandConfiguration( 6 | abstract: "List available macOS images from local cache" 7 | ) 8 | 9 | @Option(help: "Organization to list from. Defaults to trycua") 10 | var organization: String = "trycua" 11 | 12 | init() {} 13 | 14 | @MainActor 15 | func run() async throws { 16 | let vmController = LumeController() 17 | _ = try await vmController.getImages(organization: organization) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /libs/lume/src/Commands/List.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | 4 | struct List: AsyncParsableCommand { 5 | static let configuration: CommandConfiguration = CommandConfiguration( 6 | commandName: "ls", 7 | abstract: "List virtual machines" 8 | ) 9 | 10 | @Option(name: [.long, .customShort("f")], help: "Output format (json|text)") 11 | var format: FormatOption = .text 12 | 13 | @Option(name: .long, help: "Filter by storage location name") 14 | var storage: String? 15 | 16 | init() { 17 | } 18 | 19 | @MainActor 20 | func run() async throws { 21 | let manager = LumeController() 22 | let vms = try manager.list(storage: self.storage) 23 | if vms.isEmpty && self.format == .text { 24 | if let storageName = self.storage { 25 | print("No virtual machines found in storage '\(storageName)'") 26 | } else { 27 | print("No virtual machines found") 28 | } 29 | } else { 30 | try VMDetailsPrinter.printStatus(vms, format: self.format) 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /libs/lume/src/Commands/Options/FormatOption.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | 3 | enum FormatOption: String, ExpressibleByArgument { 4 | case json 5 | case text 6 | } 7 | -------------------------------------------------------------------------------- /libs/lume/src/Commands/Prune.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | 4 | struct Prune: AsyncParsableCommand { 5 | static let configuration: CommandConfiguration = CommandConfiguration( 6 | commandName: "prune", 7 | abstract: "Remove cached images" 8 | ) 9 | 10 | init() { 11 | } 12 | 13 | @MainActor 14 | func run() async throws { 15 | let manager = LumeController() 16 | try await manager.pruneImages() 17 | print("Successfully removed cached images") 18 | } 19 | } -------------------------------------------------------------------------------- /libs/lume/src/Commands/Pull.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | 4 | struct Pull: AsyncParsableCommand { 5 | static let configuration = CommandConfiguration( 6 | abstract: "Pull a macOS image from GitHub Container Registry" 7 | ) 8 | 9 | @Argument(help: "Image to pull (format: name:tag)") 10 | var image: String 11 | 12 | @Argument( 13 | help: "Name for the VM (defaults to image name without tag)", transform: { Optional($0) }) 14 | var name: String? 15 | 16 | @Option(help: "Github Container Registry to pull from. Defaults to ghcr.io") 17 | var registry: String = "ghcr.io" 18 | 19 | @Option(help: "Organization to pull from. Defaults to trycua") 20 | var organization: String = "trycua" 21 | 22 | @Option(name: .customLong("storage"), help: "VM storage location to use or direct path to VM location") 23 | var storage: String? 24 | 25 | init() {} 26 | 27 | @MainActor 28 | func run() async throws { 29 | let controller = LumeController() 30 | try await controller.pullImage( 31 | image: image, 32 | name: name, 33 | registry: registry, 34 | organization: organization, 35 | storage: storage 36 | ) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /libs/lume/src/Commands/Push.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | 4 | struct Push: AsyncParsableCommand { 5 | static let configuration = CommandConfiguration( 6 | abstract: "Push a macOS VM to GitHub Container Registry" 7 | ) 8 | 9 | @Argument(help: "Name of the VM to push") 10 | var name: String 11 | 12 | @Argument(help: "Image tag to push (format: name:tag)") 13 | var image: String 14 | 15 | @Option(parsing: .upToNextOption, help: "Additional tags to push the same image to") 16 | var additionalTags: [String] = [] 17 | 18 | @Option(help: "Github Container Registry to push to. Defaults to ghcr.io") 19 | var registry: String = "ghcr.io" 20 | 21 | @Option(help: "Organization to push to. Defaults to trycua") 22 | var organization: String = "trycua" 23 | 24 | @Option(name: .customLong("storage"), help: "VM storage location to use") 25 | var storage: String? 26 | 27 | @Option(help: "Chunk size for large files in MB. Defaults to 512.") 28 | var chunkSizeMb: Int = 512 29 | 30 | @Flag(name: .long, help: "Enable verbose logging") 31 | var verbose: Bool = false 32 | 33 | @Flag(name: .long, help: "Prepare files without uploading to registry") 34 | var dryRun: Bool = false 35 | 36 | @Flag(name: .long, help: "In dry-run mode, also reassemble chunks to verify integrity") 37 | var reassemble: Bool = true 38 | 39 | init() {} 40 | 41 | @MainActor 42 | func run() async throws { 43 | let controller = LumeController() 44 | 45 | // Parse primary image name and tag 46 | let components = image.split(separator: ":") 47 | guard components.count == 2, let primaryTag = components.last else { 48 | throw ValidationError("Invalid primary image format. Expected format: name:tag") 49 | } 50 | let imageName = String(components.first!) 51 | 52 | // Combine primary and additional tags, ensuring uniqueness 53 | var allTags: Swift.Set = [] 54 | allTags.insert(String(primaryTag)) 55 | allTags.formUnion(additionalTags) 56 | 57 | guard !allTags.isEmpty else { 58 | throw ValidationError("At least one tag must be provided.") 59 | } 60 | 61 | try await controller.pushImage( 62 | name: name, 63 | imageName: imageName, // Pass base image name 64 | tags: Array(allTags), // Pass array of all unique tags 65 | registry: registry, 66 | organization: organization, 67 | storage: storage, 68 | chunkSizeMb: chunkSizeMb, 69 | verbose: verbose, 70 | dryRun: dryRun, 71 | reassemble: reassemble 72 | ) 73 | } 74 | } -------------------------------------------------------------------------------- /libs/lume/src/Commands/Serve.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | 4 | struct Serve: AsyncParsableCommand { 5 | static let configuration = CommandConfiguration( 6 | abstract: "Start the VM management server" 7 | ) 8 | 9 | @Option(help: "Port to listen on") 10 | var port: UInt16 = 7777 11 | 12 | func run() async throws { 13 | let server = await Server(port: port) 14 | 15 | Logger.info("Starting server", metadata: ["port": "\(port)"]) 16 | 17 | // Using custom error handling to prevent ArgumentParser from printing additional error messages 18 | do { 19 | try await server.start() 20 | } catch let error as PortError { 21 | // For port errors, just log once with the suggestion 22 | let suggestedPort = port + 1 23 | 24 | // Create a user-friendly error message that includes the suggestion 25 | let message = """ 26 | \(error.localizedDescription) 27 | Try using a different port: lume serve --port \(suggestedPort) 28 | """ 29 | 30 | // Log the message (without the "ERROR:" prefix that ArgumentParser will add) 31 | Logger.error(message) 32 | 33 | // Exit with a custom code to prevent ArgumentParser from printing the error again 34 | Foundation.exit(1) 35 | } catch { 36 | // For other errors, log once 37 | Logger.error("Failed to start server", metadata: ["error": error.localizedDescription]) 38 | throw error 39 | } 40 | } 41 | } -------------------------------------------------------------------------------- /libs/lume/src/Commands/Set.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | 4 | struct Set: AsyncParsableCommand { 5 | static let configuration = CommandConfiguration( 6 | abstract: "Set new values for CPU, memory, and disk size of a virtual machine" 7 | ) 8 | 9 | @Argument(help: "Name of the virtual machine", completion: .custom(completeVMName)) 10 | var name: String 11 | 12 | @Option(help: "New number of CPU cores") 13 | var cpu: Int? 14 | 15 | @Option(help: "New memory size, e.g., 8192MB or 8GB.", transform: { try parseSize($0) }) 16 | var memory: UInt64? 17 | 18 | @Option(help: "New disk size, e.g., 20480MB or 20GB.", transform: { try parseSize($0) }) 19 | var diskSize: UInt64? 20 | 21 | @Option(help: "New display resolution in format WIDTHxHEIGHT.") 22 | var display: VMDisplayResolution? 23 | 24 | @Option(name: .customLong("storage"), help: "VM storage location to use or direct path to VM location") 25 | var storage: String? 26 | 27 | init() { 28 | } 29 | 30 | @MainActor 31 | func run() async throws { 32 | let vmController = LumeController() 33 | try vmController.updateSettings( 34 | name: name, 35 | cpu: cpu, 36 | memory: memory, 37 | diskSize: diskSize, 38 | display: display?.string, 39 | storage: storage 40 | ) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /libs/lume/src/Commands/Stop.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | 4 | struct Stop: AsyncParsableCommand { 5 | static let configuration = CommandConfiguration( 6 | abstract: "Stop a virtual machine" 7 | ) 8 | 9 | @Argument(help: "Name of the virtual machine", completion: .custom(completeVMName)) 10 | var name: String 11 | 12 | @Option(name: .customLong("storage"), help: "VM storage location to use or direct path to VM location") 13 | var storage: String? 14 | 15 | init() { 16 | } 17 | 18 | @MainActor 19 | func run() async throws { 20 | let vmController = LumeController() 21 | try await vmController.stopVM(name: name, storage: storage) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /libs/lume/src/ContainerRegistry/ImageList.swift: -------------------------------------------------------------------------------- 1 | public struct ImageList: Codable { 2 | public let local: [String] 3 | public let remote: [String] 4 | } -------------------------------------------------------------------------------- /libs/lume/src/ContainerRegistry/ImagesPrinter.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | struct ImagesPrinter { 4 | private struct Column: Sendable { 5 | let header: String 6 | let width: Int 7 | let getValue: @Sendable (String) -> String 8 | } 9 | 10 | private static let columns: [Column] = [ 11 | Column(header: "name", width: 28) { $0.split(separator: ":").first.map(String.init) ?? $0 }, 12 | Column(header: "image_id", width: 16) { $0.split(separator: ":").last.map(String.init) ?? "-" } 13 | ] 14 | 15 | static func print(images: [String]) { 16 | if images.isEmpty { 17 | Swift.print("No images found") 18 | return 19 | } 20 | 21 | printHeader() 22 | images.sorted().forEach(printImage) 23 | } 24 | 25 | private static func printHeader() { 26 | let paddedHeaders = columns.map { $0.header.paddedToWidth($0.width) } 27 | Swift.print(paddedHeaders.joined()) 28 | } 29 | 30 | private static func printImage(_ image: String) { 31 | let paddedColumns = columns.map { column in 32 | column.getValue(image).paddedToWidth(column.width) 33 | } 34 | Swift.print(paddedColumns.joined()) 35 | } 36 | } 37 | 38 | private extension String { 39 | func paddedToWidth(_ width: Int) -> String { 40 | padding(toLength: width, withPad: " ", startingAt: 0) 41 | } 42 | } -------------------------------------------------------------------------------- /libs/lume/src/FileSystem/VMLocation.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Represents a location where VMs can be stored 4 | struct VMLocation: Codable, Equatable, Sendable { 5 | let name: String 6 | let path: String 7 | 8 | var expandedPath: String { 9 | (path as NSString).expandingTildeInPath 10 | } 11 | 12 | /// Validates the location path exists and is writable 13 | func validate() throws { 14 | let fullPath = expandedPath 15 | var isDir: ObjCBool = false 16 | 17 | if FileManager.default.fileExists(atPath: fullPath, isDirectory: &isDir) { 18 | if !isDir.boolValue { 19 | throw VMLocationError.notADirectory(path: fullPath) 20 | } 21 | 22 | if !FileManager.default.isWritableFile(atPath: fullPath) { 23 | throw VMLocationError.directoryNotWritable(path: fullPath) 24 | } 25 | } else { 26 | // Try to create the directory 27 | do { 28 | try FileManager.default.createDirectory( 29 | atPath: fullPath, 30 | withIntermediateDirectories: true 31 | ) 32 | } catch { 33 | throw VMLocationError.directoryCreationFailed(path: fullPath, error: error) 34 | } 35 | } 36 | } 37 | } 38 | 39 | // MARK: - Errors 40 | 41 | enum VMLocationError: Error, LocalizedError { 42 | case notADirectory(path: String) 43 | case directoryNotWritable(path: String) 44 | case directoryCreationFailed(path: String, error: Error) 45 | case locationNotFound(name: String) 46 | case duplicateLocationName(name: String) 47 | case invalidLocationName(name: String) 48 | case defaultLocationCannotBeRemoved(name: String) 49 | 50 | var errorDescription: String? { 51 | switch self { 52 | case .notADirectory(let path): 53 | return "Path is not a directory: \(path)" 54 | case .directoryNotWritable(let path): 55 | return "Directory is not writable: \(path)" 56 | case .directoryCreationFailed(let path, let error): 57 | return "Failed to create directory at \(path): \(error.localizedDescription)" 58 | case .locationNotFound(let name): 59 | return "VM location not found: \(name)" 60 | case .duplicateLocationName(let name): 61 | return "VM location with name '\(name)' already exists" 62 | case .invalidLocationName(let name): 63 | return 64 | "Invalid location name: \(name). Names should be alphanumeric with underscores or dashes." 65 | case .defaultLocationCannotBeRemoved(let name): 66 | return "Cannot remove the default location '\(name)'. Set a new default location first." 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /libs/lume/src/Main.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | 4 | @main 5 | struct Lume: AsyncParsableCommand { 6 | static var configuration: CommandConfiguration { 7 | CommandConfiguration( 8 | commandName: "lume", 9 | abstract: "A lightweight CLI and local API server to build, run and manage macOS VMs.", 10 | version: Version.current, 11 | subcommands: CommandRegistry.allCommands, 12 | helpNames: .long 13 | ) 14 | } 15 | } 16 | 17 | // MARK: - Version Management 18 | extension Lume { 19 | enum Version { 20 | static let current: String = "0.1.0" 21 | } 22 | } 23 | 24 | // MARK: - Command Execution 25 | extension Lume { 26 | public static func main() async { 27 | do { 28 | try await executeCommand() 29 | } catch { 30 | exit(withError: error) 31 | } 32 | } 33 | 34 | private static func executeCommand() async throws { 35 | var command = try parseAsRoot() 36 | 37 | if var asyncCommand = command as? AsyncParsableCommand { 38 | try await asyncCommand.run() 39 | } else { 40 | try command.run() 41 | } 42 | } 43 | } -------------------------------------------------------------------------------- /libs/lume/src/Server/Responses.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | struct APIError: Codable { 4 | let message: String 5 | } 6 | 7 | // Helper struct to encode mixed-type dictionaries 8 | struct AnyEncodable: Encodable { 9 | private let value: Encodable 10 | 11 | init(_ value: Encodable) { 12 | self.value = value 13 | } 14 | 15 | func encode(to encoder: Encoder) throws { 16 | try value.encode(to: encoder) 17 | } 18 | } 19 | 20 | extension HTTPResponse { 21 | static func json(_ value: T) throws -> HTTPResponse { 22 | let data = try JSONEncoder().encode(value) 23 | return HTTPResponse( 24 | statusCode: .ok, 25 | headers: ["Content-Type": "application/json"], 26 | body: data 27 | ) 28 | } 29 | 30 | static func badRequest(message: String) -> HTTPResponse { 31 | let error = APIError(message: message) 32 | return try! HTTPResponse( 33 | statusCode: .badRequest, 34 | headers: ["Content-Type": "application/json"], 35 | body: JSONEncoder().encode(error) 36 | ) 37 | } 38 | } -------------------------------------------------------------------------------- /libs/lume/src/Utils/CommandRegistry.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | 3 | enum CommandRegistry { 4 | static var allCommands: [ParsableCommand.Type] { 5 | [ 6 | Create.self, 7 | Pull.self, 8 | Push.self, 9 | Images.self, 10 | Clone.self, 11 | Get.self, 12 | Set.self, 13 | List.self, 14 | Run.self, 15 | Stop.self, 16 | IPSW.self, 17 | Serve.self, 18 | Delete.self, 19 | Prune.self, 20 | Config.self, 21 | Logs.self, 22 | ] 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /libs/lume/src/Utils/CommandUtils.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | 4 | func completeVMName(_ arguments: [String]) -> [String] { 5 | (try? Home().getAllVMDirectories().map { $0.directory.name }) ?? [] 6 | } 7 | -------------------------------------------------------------------------------- /libs/lume/src/Utils/Logger.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | struct Logger { 4 | typealias Metadata = [String: String] 5 | 6 | enum Level: String { 7 | case info 8 | case error 9 | case debug 10 | } 11 | 12 | static func info(_ message: String, metadata: Metadata = [:]) { 13 | log(.info, message, metadata) 14 | } 15 | 16 | static func error(_ message: String, metadata: Metadata = [:]) { 17 | log(.error, message, metadata) 18 | } 19 | 20 | static func debug(_ message: String, metadata: Metadata = [:]) { 21 | log(.debug, message, metadata) 22 | } 23 | 24 | private static func log(_ level: Level, _ message: String, _ metadata: Metadata) { 25 | let timestamp = ISO8601DateFormatter().string(from: Date()) 26 | let metadataString = metadata.isEmpty ? "" : " " + metadata.map { "\($0.key)=\($0.value)" }.joined(separator: " ") 27 | print("[\(timestamp)] \(level.rawValue.uppercased()): \(message)\(metadataString)") 28 | } 29 | } -------------------------------------------------------------------------------- /libs/lume/src/Utils/NetworkUtils.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | enum NetworkUtils { 4 | /// Checks if an IP address is reachable by sending a ping 5 | /// - Parameter ipAddress: The IP address to check 6 | /// - Returns: true if the IP is reachable, false otherwise 7 | static func isReachable(ipAddress: String) -> Bool { 8 | let process = Process() 9 | process.executableURL = URL(fileURLWithPath: "/sbin/ping") 10 | process.arguments = ["-c", "1", "-t", "1", ipAddress] 11 | 12 | let pipe = Pipe() 13 | process.standardOutput = pipe 14 | process.standardError = pipe 15 | 16 | do { 17 | try process.run() 18 | process.waitUntilExit() 19 | return process.terminationStatus == 0 20 | } catch { 21 | return false 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /libs/lume/src/Utils/Path.swift: -------------------------------------------------------------------------------- 1 | import ArgumentParser 2 | import Foundation 3 | 4 | struct Path: CustomStringConvertible, ExpressibleByArgument { 5 | let url: URL 6 | 7 | init(_ path: String) { 8 | url = URL(filePath: NSString(string: path).expandingTildeInPath).standardizedFileURL 9 | } 10 | 11 | init(_ url: URL) { 12 | self.url = url 13 | } 14 | 15 | init(argument: String) { 16 | self.init(argument) 17 | } 18 | 19 | func file(_ path: String) -> Path { 20 | return Path(url.appendingPathComponent(path, isDirectory: false)) 21 | } 22 | 23 | func directory(_ path: String) -> Path { 24 | return Path(url.appendingPathComponent(path, isDirectory: true)) 25 | } 26 | 27 | func exists() -> Bool { 28 | return FileManager.default.fileExists(atPath: url.standardizedFileURL.path(percentEncoded: false)) 29 | } 30 | 31 | func writable() -> Bool { 32 | return FileManager.default.isWritableFile(atPath: url.standardizedFileURL.path(percentEncoded: false)) 33 | } 34 | 35 | var name: String { 36 | return url.lastPathComponent 37 | } 38 | 39 | var path: String { 40 | return url.standardizedFileURL.path(percentEncoded: false) 41 | } 42 | 43 | var description: String { 44 | return url.path() 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /libs/lume/src/Utils/ProcessRunner.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Protocol for process execution 4 | protocol ProcessRunner { 5 | func run(executable: String, arguments: [String]) throws 6 | } 7 | 8 | class DefaultProcessRunner: ProcessRunner { 9 | func run(executable: String, arguments: [String]) throws { 10 | let process = Process() 11 | process.executableURL = URL(fileURLWithPath: executable) 12 | process.arguments = arguments 13 | try process.run() 14 | } 15 | } -------------------------------------------------------------------------------- /libs/lume/src/Utils/ProgressLogger.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | struct ProgressLogger { 4 | private var lastLoggedProgress: Double = 0.0 5 | private let threshold: Double 6 | 7 | init(threshold: Double = 0.05) { 8 | self.threshold = threshold 9 | } 10 | 11 | mutating func logProgress(current: Double, context: String) { 12 | if current - lastLoggedProgress >= threshold { 13 | lastLoggedProgress = current 14 | let percentage = Int(current * 100) 15 | Logger.info("\(context) Progress: \(percentage)%") 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /libs/lume/src/Utils/String.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | extension String { 4 | func padding(_ toLength: Int) -> String { 5 | return self.padding(toLength: toLength, withPad: " ", startingAt: 0) 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /libs/lume/src/Utils/Utils.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import ArgumentParser 3 | 4 | extension Collection { 5 | subscript (safe index: Index) -> Element? { 6 | indices.contains(index) ? self[index] : nil 7 | } 8 | } 9 | 10 | func resolveBinaryPath(_ name: String) -> URL? { 11 | guard let path = ProcessInfo.processInfo.environment["PATH"] else { 12 | return nil 13 | } 14 | 15 | for pathComponent in path.split(separator: ":") { 16 | let url = URL(fileURLWithPath: String(pathComponent)) 17 | .appendingPathComponent(name, isDirectory: false) 18 | 19 | if FileManager.default.fileExists(atPath: url.path) { 20 | return url 21 | } 22 | } 23 | 24 | return nil 25 | } 26 | 27 | // Helper function to parse size strings 28 | func parseSize(_ input: String) throws -> UInt64 { 29 | let lowercased = input.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() 30 | let multiplier: Double 31 | let valueString: String 32 | 33 | if lowercased.hasSuffix("tb") { 34 | multiplier = 1024 * 1024 * 1024 * 1024 35 | valueString = String(lowercased.dropLast(2)) 36 | } else if lowercased.hasSuffix("gb") { 37 | multiplier = 1024 * 1024 * 1024 38 | valueString = String(lowercased.dropLast(2)) 39 | } else if lowercased.hasSuffix("mb") { 40 | multiplier = 1024 * 1024 41 | valueString = String(lowercased.dropLast(2)) 42 | } else if lowercased.hasSuffix("kb") { 43 | multiplier = 1024 44 | valueString = String(lowercased.dropLast(2)) 45 | } else { 46 | multiplier = 1024 * 1024 47 | valueString = lowercased 48 | } 49 | 50 | guard let value = Double(valueString.trimmingCharacters(in: .whitespacesAndNewlines)) else { 51 | throw ValidationError("Malformed size input: \(input). Could not parse numeric value.") 52 | } 53 | 54 | let bytesAsDouble = (value * multiplier).rounded() 55 | 56 | guard bytesAsDouble >= 0 && bytesAsDouble <= Double(UInt64.max) else { 57 | throw ValidationError("Calculated size out of bounds for UInt64: \(input)") 58 | } 59 | 60 | let val = UInt64(bytesAsDouble) 61 | 62 | return val 63 | } 64 | -------------------------------------------------------------------------------- /libs/lume/src/VM/LinuxVM.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Linux-specific virtual machine implementation 4 | @MainActor 5 | final class LinuxVM: VM { 6 | override init( 7 | vmDirContext: VMDirContext, 8 | virtualizationServiceFactory: @escaping (VMVirtualizationServiceContext) throws -> VMVirtualizationService = { try LinuxVirtualizationService(configuration: $0) }, 9 | vncServiceFactory: @escaping (VMDirectory) -> VNCService = { DefaultVNCService(vmDirectory: $0) } 10 | ) { 11 | super.init( 12 | vmDirContext: vmDirContext, 13 | virtualizationServiceFactory: virtualizationServiceFactory, 14 | vncServiceFactory: vncServiceFactory 15 | ) 16 | } 17 | 18 | override func getOSType() -> String { 19 | return "linux" 20 | } 21 | 22 | override func setup( 23 | ipswPath: String, 24 | cpuCount: Int, 25 | memorySize: UInt64, 26 | diskSize: UInt64, 27 | display: String 28 | ) async throws { 29 | 30 | try setDiskSize(diskSize) 31 | 32 | let service = try virtualizationServiceFactory( 33 | try createVMVirtualizationServiceContext( 34 | cpuCount: cpuCount, 35 | memorySize: memorySize, 36 | display: display 37 | ) 38 | ) 39 | guard let linuxService = service as? LinuxVirtualizationService else { 40 | throw VMError.internalError("Installation requires LinuxVirtualizationService") 41 | } 42 | 43 | try updateVMConfig(vmConfig: try VMConfig( 44 | os: getOSType(), 45 | cpuCount: cpuCount, 46 | memorySize: memorySize, 47 | diskSize: diskSize, 48 | macAddress: linuxService.generateMacAddress(), 49 | display: display 50 | )) 51 | 52 | // Create NVRAM store for EFI 53 | try linuxService.createNVRAM(at: vmDirContext.nvramPath) 54 | } 55 | } -------------------------------------------------------------------------------- /libs/lume/src/VM/VMDetails.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Network 3 | 4 | struct DiskSize: Codable { 5 | let allocated: UInt64 6 | let total: UInt64 7 | } 8 | 9 | extension DiskSize { 10 | var formattedAllocated: String { 11 | formatBytes(allocated) 12 | } 13 | 14 | var formattedTotal: String { 15 | formatBytes(total) 16 | } 17 | 18 | private func formatBytes(_ bytes: UInt64) -> String { 19 | let units = ["B", "KB", "MB", "GB", "TB"] 20 | var size = Double(bytes) 21 | var unitIndex = 0 22 | 23 | while size >= 1024 && unitIndex < units.count - 1 { 24 | size /= 1024 25 | unitIndex += 1 26 | } 27 | 28 | return String(format: "%.1f%@", size, units[unitIndex]) 29 | } 30 | } 31 | 32 | struct VMDetails: Codable { 33 | let name: String 34 | let os: String 35 | let cpuCount: Int 36 | let memorySize: UInt64 37 | let diskSize: DiskSize 38 | let display: String 39 | let status: String 40 | let vncUrl: String? 41 | let ipAddress: String? 42 | let locationName: String 43 | let sharedDirectories: [SharedDirectory]? 44 | 45 | init( 46 | name: String, 47 | os: String, 48 | cpuCount: Int, 49 | memorySize: UInt64, 50 | diskSize: DiskSize, 51 | display: String, 52 | status: String, 53 | vncUrl: String?, 54 | ipAddress: String?, 55 | locationName: String, 56 | sharedDirectories: [SharedDirectory]? = nil 57 | ) { 58 | self.name = name 59 | self.os = os 60 | self.cpuCount = cpuCount 61 | self.memorySize = memorySize 62 | self.diskSize = diskSize 63 | self.display = display 64 | self.status = status 65 | self.vncUrl = vncUrl 66 | self.ipAddress = ipAddress 67 | self.locationName = locationName 68 | self.sharedDirectories = sharedDirectories 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /libs/lume/src/VM/VMDisplayResolution.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import ArgumentParser 3 | 4 | struct VMDisplayResolution: Codable, ExpressibleByArgument { 5 | let width: Int 6 | let height: Int 7 | 8 | init?(string: String) { 9 | let components = string.components(separatedBy: "x") 10 | guard components.count == 2, 11 | let width = Int(components[0]), 12 | let height = Int(components[1]), 13 | width > 0, height > 0 else { 14 | return nil 15 | } 16 | self.width = width 17 | self.height = height 18 | } 19 | 20 | var string: String { 21 | "\(width)x\(height)" 22 | } 23 | 24 | init?(argument: String) { 25 | guard let resolution = VMDisplayResolution(string: argument) else { return nil } 26 | self = resolution 27 | } 28 | } -------------------------------------------------------------------------------- /libs/lume/src/VM/VMFactory.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Virtualization 3 | 4 | enum VMType: String { 5 | case darwin = "macOS" 6 | case linux = "linux" 7 | } 8 | 9 | protocol VMFactory { 10 | @MainActor 11 | func createVM( 12 | vmDirContext: VMDirContext, 13 | imageLoader: ImageLoader? 14 | ) throws -> VM 15 | } 16 | 17 | class DefaultVMFactory: VMFactory { 18 | @MainActor 19 | func createVM( 20 | vmDirContext: VMDirContext, 21 | imageLoader: ImageLoader? 22 | ) throws -> VM { 23 | let osType = vmDirContext.config.os.lowercased() 24 | 25 | switch osType { 26 | case "macos", "darwin": 27 | guard let imageLoader = imageLoader else { 28 | throw VMError.internalError("ImageLoader required for macOS VM") 29 | } 30 | return DarwinVM(vmDirContext: vmDirContext, imageLoader: imageLoader) 31 | case "linux": 32 | return LinuxVM(vmDirContext: vmDirContext) 33 | default: 34 | throw VMError.unsupportedOS(osType) 35 | } 36 | } 37 | } -------------------------------------------------------------------------------- /libs/lume/src/VNC/PassphraseGenerator.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import CryptoKit 3 | 4 | final class PassphraseGenerator { 5 | private let words: [String] 6 | 7 | init(words: [String] = PassphraseGenerator.defaultWords) { 8 | self.words = words 9 | } 10 | 11 | func prefix(_ count: Int) -> [String] { 12 | guard count > 0 else { return [] } 13 | 14 | // Use secure random number generation 15 | var result: [String] = [] 16 | for _ in 0.. ImageLoader 7 | } 8 | 9 | /// Default implementation of ImageLoaderFactory that creates appropriate loaders based on image type 10 | final class DefaultImageLoaderFactory: ImageLoaderFactory { 11 | func createImageLoader() -> ImageLoader { 12 | // For now, we only support Darwin images 13 | // In the future, this can be extended to support other OS types 14 | // by analyzing the image path or having explicit OS type parameter 15 | return DarwinImageLoader() 16 | } 17 | } -------------------------------------------------------------------------------- /libs/lume/tests/Mocks/MockVM.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | @testable import lume 4 | 5 | @MainActor 6 | class MockVM: VM { 7 | private var mockIsRunning = false 8 | 9 | override func getOSType() -> String { 10 | return "mock-os" 11 | } 12 | 13 | override func setup( 14 | ipswPath: String, cpuCount: Int, memorySize: UInt64, diskSize: UInt64, display: String 15 | ) async throws { 16 | // Mock setup implementation 17 | vmDirContext.config.setCpuCount(cpuCount) 18 | vmDirContext.config.setMemorySize(memorySize) 19 | vmDirContext.config.setDiskSize(diskSize) 20 | vmDirContext.config.setMacAddress("00:11:22:33:44:55") 21 | try vmDirContext.saveConfig() 22 | } 23 | 24 | override func run( 25 | noDisplay: Bool, sharedDirectories: [SharedDirectory], mount: Path?, vncPort: Int = 0, 26 | recoveryMode: Bool = false, usbMassStoragePaths: [Path]? = nil 27 | ) async throws { 28 | mockIsRunning = true 29 | try await super.run( 30 | noDisplay: noDisplay, sharedDirectories: sharedDirectories, mount: mount, 31 | vncPort: vncPort, recoveryMode: recoveryMode, 32 | usbMassStoragePaths: usbMassStoragePaths 33 | ) 34 | } 35 | 36 | override func stop() async throws { 37 | mockIsRunning = false 38 | try await super.stop() 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /libs/lume/tests/Mocks/MockVMVirtualizationService.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Virtualization 3 | @testable import lume 4 | 5 | @MainActor 6 | final class MockVMVirtualizationService: VMVirtualizationService { 7 | private(set) var currentState: VZVirtualMachine.State = .stopped 8 | private(set) var startCallCount = 0 9 | private(set) var stopCallCount = 0 10 | private(set) var pauseCallCount = 0 11 | private(set) var resumeCallCount = 0 12 | 13 | var state: VZVirtualMachine.State { 14 | currentState 15 | } 16 | 17 | private var _shouldFailNextOperation = false 18 | private var _operationError: Error = VMError.internalError("Mock operation failed") 19 | 20 | nonisolated func configure(shouldFail: Bool, error: Error = VMError.internalError("Mock operation failed")) async { 21 | await setConfiguration(shouldFail: shouldFail, error: error) 22 | } 23 | 24 | @MainActor 25 | private func setConfiguration(shouldFail: Bool, error: Error) { 26 | _shouldFailNextOperation = shouldFail 27 | _operationError = error 28 | } 29 | 30 | func start() async throws { 31 | startCallCount += 1 32 | if _shouldFailNextOperation { 33 | throw _operationError 34 | } 35 | currentState = .running 36 | } 37 | 38 | func stop() async throws { 39 | stopCallCount += 1 40 | if _shouldFailNextOperation { 41 | throw _operationError 42 | } 43 | currentState = .stopped 44 | } 45 | 46 | func pause() async throws { 47 | pauseCallCount += 1 48 | if _shouldFailNextOperation { 49 | throw _operationError 50 | } 51 | currentState = .paused 52 | } 53 | 54 | func resume() async throws { 55 | resumeCallCount += 1 56 | if _shouldFailNextOperation { 57 | throw _operationError 58 | } 59 | currentState = .running 60 | } 61 | 62 | func getVirtualMachine() -> Any { 63 | return "mock_vm" 64 | } 65 | } -------------------------------------------------------------------------------- /libs/lume/tests/Mocks/MockVNCService.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | @testable import lume 3 | 4 | @MainActor 5 | final class MockVNCService: VNCService { 6 | private(set) var url: String? 7 | private(set) var isRunning = false 8 | private(set) var clientOpenCount = 0 9 | private var _attachedVM: Any? 10 | private let vmDirectory: VMDirectory 11 | 12 | init(vmDirectory: VMDirectory) { 13 | self.vmDirectory = vmDirectory 14 | } 15 | 16 | nonisolated var attachedVM: String? { 17 | get async { 18 | await Task { @MainActor in 19 | _attachedVM as? String 20 | }.value 21 | } 22 | } 23 | 24 | func start(port: Int, virtualMachine: Any?) async throws { 25 | isRunning = true 26 | url = "vnc://localhost:\(port)" 27 | _attachedVM = virtualMachine 28 | } 29 | 30 | func stop() { 31 | isRunning = false 32 | url = nil 33 | _attachedVM = nil 34 | } 35 | 36 | func openClient(url: String) async throws { 37 | guard isRunning else { 38 | throw VMError.vncNotConfigured 39 | } 40 | clientOpenCount += 1 41 | } 42 | } -------------------------------------------------------------------------------- /libs/lume/tests/VMVirtualizationServiceTests.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Testing 3 | import Virtualization 4 | @testable import lume 5 | 6 | @Test("VMVirtualizationService starts correctly") 7 | func testVMVirtualizationServiceStart() async throws { 8 | let service = MockVMVirtualizationService() 9 | 10 | // Initial state 11 | #expect(await service.state == .stopped) 12 | #expect(await service.startCallCount == 0) 13 | 14 | // Start service 15 | try await service.start() 16 | #expect(await service.state == .running) 17 | #expect(await service.startCallCount == 1) 18 | } 19 | 20 | @Test("VMVirtualizationService stops correctly") 21 | func testVMVirtualizationServiceStop() async throws { 22 | let service = MockVMVirtualizationService() 23 | 24 | // Start then stop 25 | try await service.start() 26 | try await service.stop() 27 | 28 | #expect(await service.state == .stopped) 29 | #expect(await service.stopCallCount == 1) 30 | } 31 | 32 | @Test("VMVirtualizationService handles pause and resume") 33 | func testVMVirtualizationServicePauseResume() async throws { 34 | let service = MockVMVirtualizationService() 35 | 36 | // Start and pause 37 | try await service.start() 38 | try await service.pause() 39 | #expect(await service.state == .paused) 40 | #expect(await service.pauseCallCount == 1) 41 | 42 | // Resume 43 | try await service.resume() 44 | #expect(await service.state == .running) 45 | #expect(await service.resumeCallCount == 1) 46 | } 47 | 48 | @Test("VMVirtualizationService handles operation failures") 49 | func testVMVirtualizationServiceFailures() async throws { 50 | let service = MockVMVirtualizationService() 51 | await service.configure(shouldFail: true) 52 | 53 | // Test start failure 54 | do { 55 | try await service.start() 56 | #expect(Bool(false), "Expected start to throw") 57 | } catch let error as VMError { 58 | switch error { 59 | case .internalError(let message): 60 | #expect(message == "Mock operation failed") 61 | default: 62 | #expect(Bool(false), "Unexpected error type: \(error)") 63 | } 64 | } 65 | 66 | #expect(await service.state == .stopped) 67 | #expect(await service.startCallCount == 1) 68 | } -------------------------------------------------------------------------------- /libs/lume/tests/VNCServiceTests.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Testing 3 | @testable import lume 4 | 5 | @Test("VNCService starts correctly") 6 | func testVNCServiceStart() async throws { 7 | let tempDir = try createTempDirectory() 8 | let vmDir = VMDirectory(Path(tempDir.path)) 9 | let service = await MockVNCService(vmDirectory: vmDir) 10 | 11 | // Initial state 12 | let isRunning = await service.isRunning 13 | let url = await service.url 14 | #expect(!isRunning) 15 | #expect(url == nil) 16 | 17 | // Start service 18 | try await service.start(port: 5900, virtualMachine: nil) 19 | #expect(await service.isRunning) 20 | #expect(await service.url?.contains("5900") ?? false) 21 | } 22 | 23 | @Test("VNCService stops correctly") 24 | func testVNCServiceStop() async throws { 25 | let tempDir = try createTempDirectory() 26 | let vmDir = VMDirectory(Path(tempDir.path)) 27 | let service = await MockVNCService(vmDirectory: vmDir) 28 | try await service.start(port: 5900, virtualMachine: nil) 29 | 30 | await service.stop() 31 | let isRunning = await service.isRunning 32 | let url = await service.url 33 | #expect(!isRunning) 34 | #expect(url == nil) 35 | } 36 | 37 | @Test("VNCService handles client operations") 38 | func testVNCServiceClient() async throws { 39 | let tempDir = try createTempDirectory() 40 | let vmDir = VMDirectory(Path(tempDir.path)) 41 | let service = await MockVNCService(vmDirectory: vmDir) 42 | 43 | // Should fail when not started 44 | do { 45 | try await service.openClient(url: "vnc://localhost:5900") 46 | #expect(Bool(false), "Expected openClient to throw when not started") 47 | } catch VMError.vncNotConfigured { 48 | // Expected error 49 | } catch { 50 | #expect(Bool(false), "Expected vncNotConfigured error but got \(error)") 51 | } 52 | 53 | // Start and try client operations 54 | try await service.start(port: 5900, virtualMachine: nil) 55 | try await service.openClient(url: "vnc://localhost:5900") 56 | #expect(await service.clientOpenCount == 1) 57 | 58 | // Stop and verify client operations fail 59 | await service.stop() 60 | do { 61 | try await service.openClient(url: "vnc://localhost:5900") 62 | #expect(Bool(false), "Expected openClient to throw after stopping") 63 | } catch VMError.vncNotConfigured { 64 | // Expected error 65 | } catch { 66 | #expect(Bool(false), "Expected vncNotConfigured error but got \(error)") 67 | } 68 | } 69 | 70 | @Test("VNCService handles virtual machine attachment") 71 | func testVNCServiceVMAttachment() async throws { 72 | let tempDir = try createTempDirectory() 73 | let vmDir = VMDirectory(Path(tempDir.path)) 74 | let service = await MockVNCService(vmDirectory: vmDir) 75 | let mockVM = "mock_vm" 76 | 77 | try await service.start(port: 5900, virtualMachine: mockVM) 78 | let attachedVM = await service.attachedVM 79 | #expect(attachedVM == mockVM) 80 | } 81 | 82 | private func createTempDirectory() throws -> URL { 83 | let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) 84 | try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) 85 | return tempDir 86 | } -------------------------------------------------------------------------------- /libs/lumier/.dockerignore: -------------------------------------------------------------------------------- 1 | # Ignore macOS system files and trash 2 | .DS_Store 3 | .Trashes 4 | **/.Trashes 5 | **/.* 6 | 7 | # Ignore Python cache 8 | __pycache__/ 9 | *.pyc 10 | *.pyo 11 | 12 | # Ignore virtual environments 13 | .venv/ 14 | venv/ 15 | 16 | # Ignore editor/project files 17 | .vscode/ 18 | .idea/ 19 | *.swp 20 | 21 | # Ignore test artifacts 22 | test-results/ 23 | 24 | # Ignore anything else you don't want in the Docker build context 25 | ./examples -------------------------------------------------------------------------------- /libs/lumier/Dockerfile: -------------------------------------------------------------------------------- 1 | # Base image using Debian for arm64 architecture (optimized for Apple Silicon) 2 | FROM debian:bullseye-slim AS lumier-base 3 | 4 | # Set environment variables for Lume API server configuration 5 | ENV LUME_API_HOST="host.docker.internal" 6 | 7 | # Default VM configuration (can be overridden at runtime) 8 | ENV VERSION="ghcr.io/trycua/macos-sequoia-vanilla:latest" 9 | ENV RAM_SIZE="8192" 10 | ENV CPU_CORES="4" 11 | ENV DISK_SIZE="100" 12 | ENV DISPLAY="1024x768" 13 | ENV VM_NAME="lumier" 14 | ENV HOST_SHARED_PATH="" 15 | ENV LUMIER_DEBUG="0" 16 | 17 | # Install necessary tools and noVNC dependencies 18 | RUN apt-get update && \ 19 | apt-get install -y \ 20 | netcat-traditional \ 21 | curl \ 22 | sshpass \ 23 | wget \ 24 | unzip \ 25 | git \ 26 | python3 \ 27 | python3-pip \ 28 | python3-numpy \ 29 | procps && \ 30 | rm -rf /var/lib/apt/lists/* 31 | 32 | # Download and install noVNC without caching 33 | RUN wget https://github.com/trycua/noVNC/archive/refs/heads/master.zip -O master1.zip && \ 34 | unzip master1.zip && \ 35 | mv noVNC-master /opt/noVNC && \ 36 | rm master1.zip 37 | 38 | # Set environment variables for noVNC 39 | ENV NOVNC_PATH="/opt/noVNC" 40 | 41 | # Create necessary directories 42 | RUN mkdir -p /run/bin /run/lib /run/config /run/hooks /run/lifecycle 43 | 44 | # Copy scripts to the container 45 | COPY src/config/constants.sh /run/config/ 46 | COPY src/bin/entry.sh /run/bin/entry.sh 47 | 48 | # Copy library files if they exist 49 | COPY src/lib/ /run/lib/ 50 | COPY src/hooks/ /run/hooks/ 51 | 52 | # Copy on-logon script to lifecycle directory 53 | COPY src/hooks/on-logon.sh /run/lifecycle/ 54 | 55 | # Make scripts executable 56 | RUN chmod +x \ 57 | /run/bin/* \ 58 | /run/hooks/* \ 59 | /run/lifecycle/* 2>/dev/null || true 60 | 61 | # Expose ports for noVNC and Lume API 62 | EXPOSE 8006 63 | 64 | # VOLUME setup 65 | VOLUME [ "/storage" ] 66 | VOLUME [ "/data" ] 67 | 68 | # Default entrypoint 69 | ENTRYPOINT ["/run/bin/entry.sh"] -------------------------------------------------------------------------------- /libs/lumier/src/config/constants.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Port configuration 4 | TUNNEL_PORT=8080 5 | VNC_PORT=8006 6 | 7 | # Host configuration 8 | TUNNEL_HOST="host.docker.internal" 9 | 10 | # Default VM configuration 11 | DEFAULT_RAM_SIZE="8192" 12 | DEFAULT_CPU_CORES="4" 13 | DEFAULT_DISK_SIZE="100" 14 | DEFAULT_VM_NAME="lumier" 15 | DEFAULT_VM_VERSION="ghcr.io/trycua/macos-sequoia-vanilla:latest" 16 | 17 | # Paths 18 | NOVNC_PATH="/opt/noVNC" 19 | LIFECYCLE_HOOKS_DIR="/run/hooks" 20 | 21 | # VM connection details 22 | HOST_USER="lume" 23 | HOST_PASSWORD="lume" 24 | SSH_RETRY_ATTEMPTS=20 25 | SSH_RETRY_INTERVAL=5 -------------------------------------------------------------------------------- /libs/lumier/src/hooks/on-logon.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Arguments passed from execute_remote_script in vm.sh 4 | # $1: VNC_PASSWORD 5 | # $2: HOST_SHARED_PATH (Path inside VM where host shared dir is mounted, e.g., /Volumes/My Shared Files) 6 | 7 | VNC_PASSWORD="$1" 8 | # IMPORTANT: In the VM, the shared folder is always mounted at this fixed location 9 | HOST_SHARED_PATH="/Volumes/My Shared Files" 10 | 11 | # Set default value for VNC_DEBUG if not provided 12 | VNC_DEBUG=${VNC_DEBUG:-0} 13 | 14 | # Define the path to the user's optional on-logon script within the shared folder 15 | USER_ON_LOGON_SCRIPT_PATH="$HOST_SHARED_PATH/lifecycle/on-logon.sh" 16 | 17 | # Show basic information when debug is enabled 18 | if [ "$VNC_DEBUG" = "1" ]; then 19 | echo "[VM] Lumier lifecycle script starting" 20 | echo "[VM] Looking for user script: $USER_ON_LOGON_SCRIPT_PATH" 21 | fi 22 | 23 | # Check if the user-provided script exists 24 | if [ -f "$USER_ON_LOGON_SCRIPT_PATH" ]; then 25 | if [ "$VNC_DEBUG" = "1" ]; then 26 | echo "[VM] Found user script: $USER_ON_LOGON_SCRIPT_PATH" 27 | fi 28 | 29 | # Always show what script we're executing 30 | echo "[VM] Executing user lifecycle script" 31 | 32 | # Make script executable 33 | chmod +x "$USER_ON_LOGON_SCRIPT_PATH" 34 | 35 | # Execute the user script in a subshell with error output captured 36 | "$USER_ON_LOGON_SCRIPT_PATH" "$VNC_PASSWORD" "$HOST_SHARED_PATH" 2>&1 37 | 38 | # Capture exit code 39 | USER_SCRIPT_EXIT_CODE=$? 40 | 41 | # Always report script execution results 42 | if [ $USER_SCRIPT_EXIT_CODE -eq 0 ]; then 43 | echo "[VM] User lifecycle script completed successfully" 44 | else 45 | echo "[VM] User lifecycle script failed with exit code: $USER_SCRIPT_EXIT_CODE" 46 | fi 47 | 48 | # Check results (only in debug mode) 49 | if [ "$VNC_DEBUG" = "1" ]; then 50 | # List any files created by the script 51 | echo "[VM] Files created by user script:" 52 | ls -la /Users/lume/Desktop/hello_*.txt 2>/dev/null || echo "[VM] No script-created files found" 53 | fi 54 | else 55 | if [ "$VNC_DEBUG" = "1" ]; then 56 | echo "[VM] No user lifecycle script found" 57 | fi 58 | fi 59 | 60 | exit 0 # Ensure the entry point script exits cleanly 61 | -------------------------------------------------------------------------------- /libs/mcp-server/mcp_server/__init__.py: -------------------------------------------------------------------------------- 1 | """MCP Server for Computer-Use Agent (CUA).""" 2 | 3 | import sys 4 | import os 5 | 6 | # Add detailed debugging at import time 7 | with open("/tmp/mcp_server_debug.log", "w") as f: 8 | f.write(f"Python executable: {sys.executable}\n") 9 | f.write(f"Python version: {sys.version}\n") 10 | f.write(f"Working directory: {os.getcwd()}\n") 11 | f.write(f"Python path:\n{chr(10).join(sys.path)}\n") 12 | f.write(f"Environment variables:\n") 13 | for key, value in os.environ.items(): 14 | f.write(f"{key}={value}\n") 15 | 16 | from .server import server, main 17 | 18 | __version__ = "0.1.0" 19 | __all__ = ["server", "main"] 20 | -------------------------------------------------------------------------------- /libs/mcp-server/mcp_server/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Entry point for the MCP server module.""" 3 | 4 | from .server import main 5 | 6 | if __name__ == "__main__": 7 | main() 8 | -------------------------------------------------------------------------------- /libs/mcp-server/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["pdm-backend"] 3 | build-backend = "pdm.backend" 4 | 5 | [project] 6 | name = "cua-mcp-server" 7 | description = "MCP Server for Computer-Use Agent (CUA)" 8 | readme = "README.md" 9 | requires-python = ">=3.11" 10 | version = "0.1.0" 11 | authors = [ 12 | {name = "TryCua", email = "gh@trycua.com"} 13 | ] 14 | dependencies = [ 15 | "mcp>=1.6.0,<2.0.0", 16 | "cua-agent[all]>=0.2.0,<0.3.0", 17 | "cua-computer>=0.2.0,<0.3.0", 18 | ] 19 | 20 | [project.scripts] 21 | cua-mcp-server = "mcp_server.server:main" 22 | 23 | [tool.pdm] 24 | distribution = true 25 | 26 | [tool.pdm.dev-dependencies] 27 | dev = [ 28 | "black>=23.9.1", 29 | "ruff>=0.0.292", 30 | ] 31 | 32 | [tool.black] 33 | line-length = 100 34 | target-version = ["py311"] 35 | 36 | [tool.ruff] 37 | line-length = 100 38 | target-version = "py311" 39 | select = ["E", "F", "B", "I"] 40 | fix = true 41 | -------------------------------------------------------------------------------- /libs/mcp-server/scripts/install_mcp_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | # Create the ~/.cua directory if it doesn't exist 6 | mkdir -p "$HOME/.cua" 7 | 8 | # Create start_mcp_server.sh script in ~/.cua directory 9 | cat > "$HOME/.cua/start_mcp_server.sh" << 'EOF' 10 | #!/bin/bash 11 | 12 | set -e 13 | 14 | # Function to check if a directory is writable 15 | is_writable() { 16 | [ -w "$1" ] 17 | } 18 | 19 | # Function to check if a command exists (silent) 20 | command_exists() { 21 | command -v "$1" >/dev/null 2>&1 22 | } 23 | 24 | # Find a writable directory for the virtual environment 25 | if is_writable "$HOME"; then 26 | VENV_DIR="$HOME/.cua-mcp-venv" 27 | elif is_writable "/tmp"; then 28 | VENV_DIR="/tmp/.cua-mcp-venv" 29 | else 30 | # Try to create a directory in the current working directory 31 | TEMP_DIR="$(pwd)/.cua-mcp-venv" 32 | if is_writable "$(pwd)"; then 33 | VENV_DIR="$TEMP_DIR" 34 | else 35 | echo "Error: Cannot find a writable directory for the virtual environment." >&2 36 | exit 1 37 | fi 38 | fi 39 | 40 | # Check if Python is installed 41 | if ! command_exists python3; then 42 | echo "Error: Python 3 is not installed." >&2 43 | exit 1 44 | fi 45 | 46 | # Check if pip is installed 47 | if ! command_exists pip3; then 48 | echo "Error: pip3 is not installed." >&2 49 | exit 1 50 | fi 51 | 52 | # Create virtual environment if it doesn't exist 53 | if [ ! -d "$VENV_DIR" ]; then 54 | # Redirect output to prevent JSON parsing errors in Claude 55 | python3 -m venv "$VENV_DIR" >/dev/null 2>&1 56 | fi 57 | 58 | # Activate virtual environment 59 | source "$VENV_DIR/bin/activate" 60 | 61 | # Always install/upgrade the latest version of cua-mcp-server 62 | pip install --upgrade "cua-mcp-server" 63 | 64 | # Run the MCP server with isolation from development paths 65 | cd "$VENV_DIR" # Change to venv directory to avoid current directory in path 66 | 67 | python3 -c "from mcp_server.server import main; main()" 68 | EOF 69 | 70 | # Make the script executable 71 | chmod +x "$HOME/.cua/start_mcp_server.sh" 72 | 73 | echo "MCP server startup script created at $HOME/.cua/start_mcp_server.sh" 74 | -------------------------------------------------------------------------------- /libs/mcp-server/scripts/start_mcp_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | # Set the CUA repository path based on script location 6 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" 7 | CUA_REPO_DIR="$( cd "$SCRIPT_DIR/../../.." &> /dev/null && pwd )" 8 | PYTHON_PATH="${CUA_REPO_DIR}/.venv/bin/python" 9 | 10 | # Set Python path to include all necessary libraries 11 | export PYTHONPATH="${CUA_REPO_DIR}/libs/mcp-server:${CUA_REPO_DIR}/libs/agent:${CUA_REPO_DIR}/libs/computer:${CUA_REPO_DIR}/libs/core:${CUA_REPO_DIR}/libs/pylume" 12 | 13 | # Run the MCP server directly as a module 14 | $PYTHON_PATH -m mcp_server.server -------------------------------------------------------------------------------- /libs/pylume/README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 |
4 | 5 | 6 | 7 | Shows my svg 8 | 9 |
10 | 11 | [![Python](https://img.shields.io/badge/Python-333333?logo=python&logoColor=white&labelColor=333333)](#) 12 | [![macOS](https://img.shields.io/badge/macOS-000000?logo=apple&logoColor=F0F0F0)](#) 13 | [![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?&logo=discord&logoColor=white)](https://discord.com/invite/mVnXXpdE85) 14 | [![PyPI](https://img.shields.io/pypi/v/pylume?color=333333)](https://pypi.org/project/pylume/) 15 |

16 |
17 | 18 | 19 | **pylume** is a lightweight Python library based on [lume](https://github.com/trycua/lume) to create, run and manage macOS and Linux virtual machines (VMs) natively on Apple Silicon. 20 | 21 |
22 | lume-py 23 |
24 | 25 | 26 | ```bash 27 | pip install pylume 28 | ``` 29 | 30 | ## Usage 31 | 32 | Please refer to this [Notebook](./samples/nb.ipynb) for a quickstart. More details about the underlying API used by pylume are available [here](https://github.com/trycua/lume/docs/API-Reference.md). 33 | 34 | ## Prebuilt Images 35 | 36 | Pre-built images are available on [ghcr.io/trycua](https://github.com/orgs/trycua/packages). 37 | These images come pre-configured with an SSH server and auto-login enabled. 38 | 39 | ## Contributing 40 | 41 | We welcome and greatly appreciate contributions to lume! Whether you're improving documentation, adding new features, fixing bugs, or adding new VM images, your efforts help make pylume better for everyone. 42 | 43 | Join our [Discord community](https://discord.com/invite/mVnXXpdE85) to discuss ideas or get assistance. 44 | 45 | ## License 46 | 47 | lume is open-sourced under the MIT License - see the [LICENSE](LICENSE) file for details. 48 | 49 | ## Stargazers over time 50 | 51 | [![Stargazers over time](https://starchart.cc/trycua/pylume.svg?variant=adaptive)](https://starchart.cc/trycua/pylume) 52 | -------------------------------------------------------------------------------- /libs/pylume/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | PyLume Python SDK - A client library for managing macOS VMs with PyLume. 3 | """ 4 | 5 | from pylume.pylume import * 6 | from pylume.models import * 7 | from pylume.exceptions import * 8 | 9 | __version__ = "0.1.0" 10 | -------------------------------------------------------------------------------- /libs/pylume/pylume/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | PyLume Python SDK - A client library for managing macOS VMs with PyLume. 3 | 4 | Example: 5 | >>> from pylume import PyLume, VMConfig 6 | >>> client = PyLume() 7 | >>> config = VMConfig(name="my-vm", cpu=4, memory="8GB", disk_size="64GB") 8 | >>> client.create_vm(config) 9 | >>> client.run_vm("my-vm") 10 | """ 11 | 12 | # Import exceptions then all models 13 | from .exceptions import ( 14 | LumeConfigError, 15 | LumeConnectionError, 16 | LumeError, 17 | LumeImageError, 18 | LumeNotFoundError, 19 | LumeServerError, 20 | LumeTimeoutError, 21 | LumeVMError, 22 | ) 23 | from .models import ( 24 | CloneSpec, 25 | ImageInfo, 26 | ImageList, 27 | ImageRef, 28 | SharedDirectory, 29 | VMConfig, 30 | VMRunOpts, 31 | VMStatus, 32 | VMUpdateOpts, 33 | ) 34 | 35 | # Import main class last to avoid circular imports 36 | from .pylume import PyLume 37 | 38 | __version__ = "0.2.2" 39 | 40 | __all__ = [ 41 | "PyLume", 42 | "VMConfig", 43 | "VMStatus", 44 | "VMRunOpts", 45 | "VMUpdateOpts", 46 | "ImageRef", 47 | "CloneSpec", 48 | "SharedDirectory", 49 | "ImageList", 50 | "ImageInfo", 51 | "LumeError", 52 | "LumeServerError", 53 | "LumeConnectionError", 54 | "LumeTimeoutError", 55 | "LumeNotFoundError", 56 | "LumeConfigError", 57 | "LumeVMError", 58 | "LumeImageError", 59 | ] 60 | -------------------------------------------------------------------------------- /libs/pylume/pylume/exceptions.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | class LumeError(Exception): 4 | """Base exception for all PyLume errors.""" 5 | pass 6 | 7 | class LumeServerError(LumeError): 8 | """Raised when there's an error with the PyLume server.""" 9 | def __init__(self, message: str, status_code: Optional[int] = None, response_text: Optional[str] = None): 10 | self.status_code = status_code 11 | self.response_text = response_text 12 | super().__init__(message) 13 | 14 | class LumeConnectionError(LumeError): 15 | """Raised when there's an error connecting to the PyLume server.""" 16 | pass 17 | 18 | class LumeTimeoutError(LumeError): 19 | """Raised when a request to the PyLume server times out.""" 20 | pass 21 | 22 | class LumeNotFoundError(LumeError): 23 | """Raised when a requested resource is not found.""" 24 | pass 25 | 26 | class LumeConfigError(LumeError): 27 | """Raised when there's an error with the configuration.""" 28 | pass 29 | 30 | class LumeVMError(LumeError): 31 | """Raised when there's an error with a VM operation.""" 32 | pass 33 | 34 | class LumeImageError(LumeError): 35 | """Raised when there's an error with an image operation.""" 36 | pass -------------------------------------------------------------------------------- /libs/pylume/pylume/lume: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trycua/cua/99b979ef114345fae36dba627f33577388759f47/libs/pylume/pylume/lume -------------------------------------------------------------------------------- /libs/pylume/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | build-backend = "pdm.backend" 3 | requires = ["pdm-backend"] 4 | 5 | [project] 6 | authors = [{ name = "TryCua", email = "gh@trycua.com" }] 7 | classifiers = [ 8 | "Intended Audience :: Developers", 9 | "License :: OSI Approved :: MIT License", 10 | "Operating System :: MacOS :: MacOS X", 11 | "Programming Language :: Python :: 3", 12 | "Programming Language :: Python :: 3.10", 13 | "Programming Language :: Python :: 3.11", 14 | "Programming Language :: Python :: 3.12", 15 | ] 16 | dependencies = ["pydantic>=2.11.1"] 17 | description = "Python SDK for lume - run macOS and Linux VMs on Apple Silicon" 18 | dynamic = ["version"] 19 | keywords = ["apple-silicon", "macos", "virtualization", "vm"] 20 | license = { text = "MIT" } 21 | name = "pylume" 22 | readme = "README.md" 23 | requires-python = ">=3.9" 24 | 25 | [tool.pdm.version] 26 | path = "pylume/__init__.py" 27 | source = "file" 28 | 29 | [project.urls] 30 | homepage = "https://github.com/trycua/pylume" 31 | repository = "https://github.com/trycua/pylume" 32 | 33 | [tool.pdm] 34 | distribution = true 35 | 36 | [tool.pdm.dev-dependencies] 37 | dev = [ 38 | "black>=23.0.0", 39 | "isort>=5.12.0", 40 | "pytest-asyncio>=0.23.0", 41 | "pytest>=7.0.0", 42 | ] 43 | 44 | [tool.black] 45 | line-length = 100 46 | target-version = ["py311"] 47 | 48 | [tool.ruff] 49 | fix = true 50 | line-length = 100 51 | select = ["B", "E", "F", "I"] 52 | target-version = "py311" 53 | 54 | [tool.ruff.format] 55 | docstring-code-format = true 56 | 57 | [tool.mypy] 58 | check_untyped_defs = true 59 | disallow_untyped_defs = true 60 | ignore_missing_imports = true 61 | python_version = "3.11" 62 | show_error_codes = true 63 | strict = true 64 | warn_return_any = true 65 | warn_unused_ignores = false 66 | 67 | [tool.pytest.ini_options] 68 | asyncio_mode = "auto" 69 | python_files = "test_*.py" 70 | testpaths = ["tests"] 71 | 72 | [tool.pdm.build] 73 | includes = ["pylume/"] 74 | source-includes = ["LICENSE", "README.md", "tests/"] 75 | -------------------------------------------------------------------------------- /libs/som/poetry.toml: -------------------------------------------------------------------------------- 1 | [virtualenvs] 2 | in-project = true 3 | -------------------------------------------------------------------------------- /libs/som/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["pdm-backend"] 3 | build-backend = "pdm.backend" 4 | 5 | [project] 6 | name = "cua-som" 7 | version = "0.1.0" 8 | description = "Computer Vision and OCR library for detecting and analyzing UI elements" 9 | authors = [ 10 | { name = "TryCua", email = "gh@trycua.com" } 11 | ] 12 | dependencies = [ 13 | "torch>=2.2.1", 14 | "torchvision>=0.17.1", 15 | "ultralytics>=8.1.28", 16 | "easyocr>=1.7.1", 17 | "numpy>=1.26.4", 18 | "pillow>=10.2.0", 19 | "setuptools>=75.8.1", 20 | "opencv-python-headless>=4.11.0.86", 21 | "matplotlib>=3.8.3", 22 | "huggingface-hub>=0.21.4", 23 | "supervision>=0.25.1", 24 | "typing-extensions>=4.9.0", 25 | "pydantic>=2.6.3" 26 | ] 27 | requires-python = ">=3.11" 28 | readme = "README.md" 29 | license = {text = "MIT"} 30 | keywords = ["computer-vision", "ocr", "ui-analysis", "icon-detection"] 31 | classifiers = [ 32 | "Development Status :: 4 - Beta", 33 | "Intended Audience :: Developers", 34 | "License :: OSI Approved :: MIT License", 35 | "Programming Language :: Python :: 3", 36 | "Programming Language :: Python :: 3.11", 37 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 38 | "Topic :: Scientific/Engineering :: Image Recognition" 39 | ] 40 | 41 | [project.urls] 42 | Homepage = "https://github.com/trycua/cua" 43 | Repository = "https://github.com/trycua/cua" 44 | Documentation = "https://github.com/trycua/cua/tree/main/docs" 45 | 46 | [tool.pdm] 47 | distribution = true 48 | package-type = "library" 49 | src-layout = false 50 | 51 | [tool.pdm.build] 52 | includes = ["som/"] 53 | source-includes = ["tests/", "README.md", "LICENSE"] 54 | 55 | [tool.black] 56 | line-length = 100 57 | target-version = ["py311"] 58 | 59 | [tool.ruff] 60 | line-length = 100 61 | target-version = "py311" 62 | select = ["E", "F", "B", "I"] 63 | fix = true 64 | 65 | [tool.ruff.format] 66 | docstring-code-format = true 67 | 68 | [tool.mypy] 69 | strict = true 70 | python_version = "3.11" 71 | ignore_missing_imports = true 72 | disallow_untyped_defs = true 73 | check_untyped_defs = true 74 | warn_return_any = true 75 | show_error_codes = true 76 | warn_unused_ignores = false 77 | 78 | [tool.pytest.ini_options] 79 | asyncio_mode = "auto" 80 | testpaths = ["tests"] 81 | python_files = "test_*.py" 82 | -------------------------------------------------------------------------------- /libs/som/som/__init__.py: -------------------------------------------------------------------------------- 1 | """SOM - Computer Vision and OCR library for detecting and analyzing UI elements.""" 2 | 3 | __version__ = "0.1.0" 4 | 5 | from .detect import OmniParser 6 | from .models import ( 7 | BoundingBox, 8 | UIElement, 9 | IconElement, 10 | TextElement, 11 | ParserMetadata, 12 | ParseResult 13 | ) 14 | 15 | __all__ = [ 16 | "OmniParser", 17 | "BoundingBox", 18 | "UIElement", 19 | "IconElement", 20 | "TextElement", 21 | "ParserMetadata", 22 | "ParseResult" 23 | ] -------------------------------------------------------------------------------- /libs/som/tests/test_omniparser.py: -------------------------------------------------------------------------------- 1 | # """Basic tests for the omniparser package.""" 2 | 3 | # import pytest 4 | # from omniparser import IconDetector 5 | 6 | # def test_icon_detector_import(): 7 | # """Test that we can import the IconDetector class.""" 8 | # assert IconDetector is not None 9 | 10 | # def test_icon_detector_init(): 11 | # """Test that we can create an IconDetector instance.""" 12 | # detector = IconDetector(force_cpu=True) 13 | # assert detector is not None 14 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | build-backend = "pdm.backend" 3 | requires = ["pdm-backend"] 4 | 5 | [project] 6 | authors = [{ name = "TryCua", email = "gh@trycua.com" }] 7 | dependencies = [] 8 | description = "CUA (Computer Use Agent) mono-repo" 9 | license = { text = "MIT" } 10 | name = "cua-workspace" 11 | readme = "README.md" 12 | requires-python = ">=3.11" 13 | version = "0.1.0" 14 | 15 | [project.urls] 16 | repository = "https://github.com/trycua/cua" 17 | 18 | [dependency-groups] 19 | dev = [] 20 | examples = [] 21 | 22 | [tool.pdm] 23 | distribution = false 24 | 25 | [tool.pdm.dev-dependencies] 26 | dev = [ 27 | "-e agent @ file:///${PROJECT_ROOT}/libs/agent", 28 | "-e computer @ file:///${PROJECT_ROOT}/libs/computer", 29 | "-e computer-server @ file:///${PROJECT_ROOT}/libs/computer-server", 30 | "-e cua-som @ file:///${PROJECT_ROOT}/libs/som", 31 | "-e mcp-server @ file:///${PROJECT_ROOT}/libs/mcp-server", 32 | "-e pylume @ file:///${PROJECT_ROOT}/libs/pylume", 33 | "black>=23.0.0", 34 | "ipykernel>=6.29.5", 35 | "jedi>=0.19.2", 36 | "jupyter>=1.0.0", 37 | "mypy>=1.10.0", 38 | "ruff>=0.9.2", 39 | "types-requests>=2.31.0", 40 | ] 41 | docs = ["mkdocs-material>=9.2.0", "mkdocs>=1.5.0"] 42 | test = [ 43 | "aioresponses>=0.7.4", 44 | "pytest-asyncio>=0.21.1", 45 | "pytest-cov>=4.1.0", 46 | "pytest-mock>=3.10.0", 47 | "pytest-xdist>=3.6.1", 48 | "pytest>=8.0.0", 49 | ] 50 | 51 | [tool.pdm.resolution] 52 | respect-source-order = true 53 | 54 | [tool.black] 55 | line-length = 100 56 | target-version = ["py311"] 57 | 58 | [tool.ruff] 59 | fix = true 60 | line-length = 100 61 | select = ["B", "E", "F", "I"] 62 | target-version = "py311" 63 | 64 | [tool.ruff.format] 65 | docstring-code-format = true 66 | 67 | [tool.mypy] 68 | check_untyped_defs = true 69 | disallow_untyped_defs = true 70 | ignore_missing_imports = true 71 | python_version = "3.11" 72 | show_error_codes = true 73 | strict = true 74 | warn_return_any = true 75 | warn_unused_ignores = false 76 | 77 | [tool.pytest.ini_options] 78 | asyncio_mode = "auto" 79 | python_files = "test_*.py" 80 | testpaths = ["libs/*/tests"] 81 | -------------------------------------------------------------------------------- /pyrightconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "include": [ 3 | "**/*.py" 4 | ], 5 | "exclude": [ 6 | "**/node_modules/**", 7 | "**/__pycache__/**", 8 | "**/.*/**", 9 | "**/venv/**", 10 | "**/.venv/**", 11 | "**/dist/**", 12 | "**/build/**", 13 | ".pdm-build/**", 14 | "**/.git/**", 15 | "examples/**", 16 | "notebooks/**", 17 | "logs/**", 18 | "screenshots/**" 19 | ], 20 | "typeCheckingMode": "basic", 21 | "useLibraryCodeForTypes": true, 22 | "reportMissingImports": false, 23 | "reportMissingModuleSource": false 24 | } 25 | -------------------------------------------------------------------------------- /scripts/cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Exit on error 4 | set -e 5 | 6 | # Colors for output 7 | RED='\033[0;31m' 8 | GREEN='\033[0;32m' 9 | BLUE='\033[0;34m' 10 | NC='\033[0m' # No Color 11 | 12 | # Function to print step information 13 | print_step() { 14 | echo -e "${BLUE}==> $1${NC}" 15 | } 16 | 17 | # Function to print success message 18 | print_success() { 19 | echo -e "${GREEN}==> Success: $1${NC}" 20 | } 21 | 22 | # Function to print error message 23 | print_error() { 24 | echo -e "${RED}==> Error: $1${NC}" >&2 25 | } 26 | 27 | # Get the script's directory 28 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 29 | PROJECT_ROOT="$SCRIPT_DIR/.." 30 | 31 | # Change to project root 32 | cd "$PROJECT_ROOT" 33 | 34 | print_step "Starting cleanup of all caches and virtual environments..." 35 | 36 | # Remove all virtual environments 37 | print_step "Removing virtual environments..." 38 | find . -type d -name ".venv" -exec rm -rf {} + 39 | print_success "Virtual environments removed" 40 | 41 | # Remove all Python cache files and directories 42 | print_step "Removing Python cache files and directories..." 43 | find . -type d -name "__pycache__" -exec rm -rf {} + 44 | find . -type d -name ".pytest_cache" -exec rm -rf {} + 45 | find . -type d -name ".mypy_cache" -exec rm -rf {} + 46 | find . -type d -name ".ruff_cache" -exec rm -rf {} + 47 | find . -name "*.pyc" -delete 48 | find . -name "*.pyo" -delete 49 | find . -name "*.pyd" -delete 50 | print_success "Python cache files removed" 51 | 52 | # Remove all build artifacts 53 | print_step "Removing build artifacts..." 54 | find . -type d -name "build" -exec rm -rf {} + 55 | find . -type d -name "dist" -exec rm -rf {} + 56 | find . -type d -name "*.egg-info" -exec rm -rf {} + 57 | find . -type d -name "*.egg" -exec rm -rf {} + 58 | print_success "Build artifacts removed" 59 | 60 | # Remove PDM-related files and directories 61 | print_step "Removing PDM-related files and directories..." 62 | find . -name "pdm.lock" -delete 63 | find . -type d -name ".pdm-build" -exec rm -rf {} + 64 | find . -name ".pdm-python" -delete # .pdm-python is a file, not a directory 65 | print_success "PDM-related files removed" 66 | 67 | # Remove MCP-related files 68 | print_step "Removing MCP-related files..." 69 | find . -name "mcp_server.log" -delete 70 | print_success "MCP-related files removed" 71 | 72 | # Remove .env file 73 | print_step "Removing .env file..." 74 | rm -f .env 75 | print_success ".env file removed" 76 | 77 | # Remove typings directory 78 | print_step "Removing typings directory..." 79 | rm -rf .vscode/typings 80 | print_success "Typings directory removed" 81 | 82 | # Clean up any temporary files 83 | print_step "Removing temporary files..." 84 | find . -name "*.tmp" -delete 85 | find . -name "*.bak" -delete 86 | find . -name "*.swp" -delete 87 | print_success "Temporary files removed" 88 | 89 | print_success "Cleanup complete! All caches and virtual environments have been removed." 90 | print_step "To rebuild the project, run: bash scripts/build.sh" 91 | --------------------------------------------------------------------------------