├── .github └── workflows │ └── build.yaml ├── .gitignore ├── LICENSE ├── README.md ├── docs ├── api-reference │ ├── agent │ │ ├── tasks.mdx │ │ └── ui.mdx │ ├── computer-use │ │ ├── examples.mdx │ │ └── unified-endpoint.mdx │ ├── endpoint │ │ ├── create.mdx │ │ ├── delete.mdx │ │ ├── get.mdx │ │ └── webhook.mdx │ ├── introduction.mdx │ └── openapi.json ├── core-concepts │ ├── agent-system.mdx │ ├── architecture.mdx │ └── desktop-environment.mdx ├── docs.json ├── favicon.svg ├── images │ ├── agent-architecture.png │ └── core-container.png ├── introduction.mdx ├── logo │ ├── bytebot_transparent_logo_dark.svg │ └── bytebot_transparent_logo_white.svg ├── quickstart.mdx └── rest-api │ ├── computer-use.mdx │ ├── examples.mdx │ └── introduction.mdx ├── infrastructure └── docker │ ├── .dockerignore │ ├── Dockerfile │ ├── docker-compose.core.yml │ ├── docker-compose.yml │ ├── supervisord.conf │ └── xfce4 │ ├── desktop │ ├── icons.screen.latest.rc │ └── icons.screen0-1264x673.rc │ ├── helpers.rc │ ├── terminal │ └── accels.scm │ └── xfconf │ └── xfce-perchannel-xml │ ├── displays.xml │ ├── thunar.xml │ ├── xfce4-appfinder.xml │ ├── xfce4-desktop.xml │ ├── xfce4-keyboard-shortcuts.xml │ ├── xfce4-notifyd.xml │ ├── xfce4-panel.xml │ └── xfwm4.xml ├── packages ├── bytebot-agent │ ├── .dockerignore │ ├── .env.example │ ├── .gitignore │ ├── .prettierrc │ ├── Dockerfile │ ├── eslint.config.mjs │ ├── nest-cli.json │ ├── package-lock.json │ ├── package.json │ ├── prisma │ │ ├── migrations │ │ │ ├── 20250328022708_initial_migration │ │ │ │ └── migration.sql │ │ │ ├── 20250413053912_message_role │ │ │ │ └── migration.sql │ │ │ ├── 20250522200556_updated_task_structure │ │ │ │ └── migration.sql │ │ │ ├── 20250523162632_add_scheduling │ │ │ │ └── migration.sql │ │ │ ├── 20250529003255_tasks_control │ │ │ │ └── migration.sql │ │ │ ├── 20250530012753_tasks_control │ │ │ │ └── migration.sql │ │ │ └── migration_lock.toml │ │ └── schema.prisma │ ├── src │ │ ├── agent │ │ │ ├── agent.module.ts │ │ │ ├── agent.processor.ts │ │ │ └── agent.scheduler.ts │ │ ├── anthropic │ │ │ ├── anthropic.constants.ts │ │ │ ├── anthropic.module.ts │ │ │ ├── anthropic.service.ts │ │ │ └── anthropic.tools.ts │ │ ├── app.controller.ts │ │ ├── app.module.ts │ │ ├── app.service.ts │ │ ├── common │ │ │ └── utils.ts │ │ ├── main.ts │ │ ├── messages │ │ │ ├── messages.module.ts │ │ │ └── messages.service.ts │ │ ├── prisma │ │ │ ├── prisma.module.ts │ │ │ └── prisma.service.ts │ │ └── tasks │ │ │ ├── dto │ │ │ ├── create-task.dto.ts │ │ │ ├── guide-task.dto.ts │ │ │ └── update-task.dto.ts │ │ │ ├── tasks.controller.ts │ │ │ ├── tasks.gateway.ts │ │ │ ├── tasks.module.ts │ │ │ └── tasks.service.ts │ ├── tsconfig.build.json │ └── tsconfig.json ├── bytebot-ui │ ├── .dockerignore │ ├── .gitignore │ ├── .prettierrc.json │ ├── Dockerfile │ ├── components.json │ ├── eslint.config.mjs │ ├── next.config.ts │ ├── package-lock.json │ ├── package.json │ ├── postcss.config.mjs │ ├── public │ │ ├── bytebot_square_light.svg │ │ ├── bytebot_transparent_logo_dark.svg │ │ ├── bytebot_transparent_logo_white.svg │ │ └── stock-1.png │ ├── src │ │ ├── app │ │ │ ├── favicon.ico │ │ │ ├── globals.css │ │ │ ├── layout.tsx │ │ │ ├── page.tsx │ │ │ └── tasks │ │ │ │ ├── [id] │ │ │ │ └── page.tsx │ │ │ │ └── page.tsx │ │ ├── components │ │ │ ├── layout │ │ │ │ ├── BrowserHeader.tsx │ │ │ │ └── Header.tsx │ │ │ ├── messages │ │ │ │ ├── ChatContainer.tsx │ │ │ │ ├── ChatInput.tsx │ │ │ │ └── MessageGroup.tsx │ │ │ ├── screenshot │ │ │ │ └── ScreenshotViewer.tsx │ │ │ ├── tasks │ │ │ │ ├── TaskItem.tsx │ │ │ │ └── TaskList.tsx │ │ │ ├── ui │ │ │ │ ├── TopicPopover.tsx │ │ │ │ ├── button.tsx │ │ │ │ ├── card.tsx │ │ │ │ ├── input.tsx │ │ │ │ ├── popover.tsx │ │ │ │ ├── scroll-area.tsx │ │ │ │ ├── select.tsx │ │ │ │ ├── separator.tsx │ │ │ │ ├── switch.tsx │ │ │ │ └── text-shimmer.tsx │ │ │ └── vnc │ │ │ │ └── VncViewer.tsx │ │ ├── hooks │ │ │ ├── useChatSession.ts │ │ │ ├── useScrollScreenshot.ts │ │ │ └── useWebSocket.ts │ │ ├── lib │ │ │ └── utils.ts │ │ ├── types │ │ │ └── index.ts │ │ └── utils │ │ │ ├── screenshotUtils.ts │ │ │ ├── stringUtils.ts │ │ │ └── taskUtils.ts │ └── tsconfig.json ├── bytebotd │ ├── .prettierrc │ ├── eslint.config.mjs │ ├── nest-cli.json │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── app.controller.ts │ │ ├── app.module.ts │ │ ├── app.service.ts │ │ ├── computer-use │ │ │ ├── computer-use.controller.ts │ │ │ ├── computer-use.module.ts │ │ │ ├── computer-use.service.ts │ │ │ └── dto │ │ │ │ ├── base.dto.ts │ │ │ │ ├── computer-action-validation.pipe.ts │ │ │ │ └── computer-action.dto.ts │ │ ├── main.ts │ │ └── nut │ │ │ ├── nut.module.ts │ │ │ └── nut.service.ts │ ├── tsconfig.build.json │ └── tsconfig.json └── shared │ ├── package-lock.json │ ├── package.json │ ├── src │ ├── index.ts │ ├── types │ │ └── messageContent.types.ts │ └── utils │ │ └── messageContent.utils.ts │ └── tsconfig.json ├── scripts ├── build.sh ├── run.sh └── teardown.sh └── static ├── background.jpg └── bytebot-logo.png /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - "infrastructure/docker/**" 9 | - "packages/bytebotd/**" 10 | 11 | permissions: 12 | contents: read 13 | packages: write 14 | 15 | jobs: 16 | docker: 17 | runs-on: ubuntu-22.04 18 | 19 | steps: 20 | # 1. Check out code 21 | - uses: actions/checkout@v4 22 | 23 | # 2. Enable QEMU so the amd64 runner can cross‑build arm64 24 | - uses: docker/setup-qemu-action@v3 25 | 26 | # 3. Set up Buildx builder 27 | - uses: docker/setup-buildx-action@v3 28 | 29 | # 4. Generate OCI labels + the single "edge" tag 30 | - name: Docker meta 31 | id: meta 32 | uses: docker/metadata-action@v5 33 | with: 34 | images: ghcr.io/bytebot-ai/bytebot 35 | tags: type=edge 36 | 37 | # 5. Log in to GHCR 38 | - name: Login to GitHub Container Registry 39 | uses: docker/login-action@v3 40 | with: 41 | registry: ghcr.io 42 | username: ${{ github.actor }} 43 | password: ${{ secrets.GITHUB_TOKEN }} 44 | 45 | # 6. Build & push a multi‑arch image 46 | - name: Build and push 47 | uses: docker/build-push-action@v6 48 | env: 49 | BUILDX_NO_DEFAULT_ATTESTATIONS: 1 # hide "unknown/unknown" in GHCR 50 | DOCKER_BUILD_SUMMARY: false # keep logs concise 51 | DOCKER_BUILD_RECORD_UPLOAD: false 52 | with: 53 | context: . 54 | file: ./infrastructure/docker/Dockerfile 55 | platforms: linux/amd64,linux/arm64 # build both archs in one go 56 | push: true 57 | cache-from: type=gha 58 | cache-to: type=gha,mode=max 59 | tags: ${{ steps.meta.outputs.tags }} 60 | labels: ${{ steps.meta.outputs.labels }} 61 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | .pnpm-debug.log* 9 | 10 | # Diagnostic reports (https://nodejs.org/api/report.html) 11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 12 | 13 | # Runtime data 14 | pids 15 | *.pid 16 | *.seed 17 | *.pid.lock 18 | 19 | # Directory for instrumented libs generated by jscoverage/JSCover 20 | lib-cov 21 | 22 | # Coverage directory used by tools like istanbul 23 | coverage 24 | *.lcov 25 | 26 | # nyc test coverage 27 | .nyc_output 28 | 29 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 30 | .grunt 31 | 32 | # Bower dependency directory (https://bower.io/) 33 | bower_components 34 | 35 | # node-waf configuration 36 | .lock-wscript 37 | 38 | # Compiled binary addons (https://nodejs.org/api/addons.html) 39 | build/Release 40 | 41 | # Dependency directories 42 | node_modules/ 43 | jspm_packages/ 44 | 45 | # Snowpack dependency directory (https://snowpack.dev/) 46 | web_modules/ 47 | 48 | # TypeScript cache 49 | *.tsbuildinfo 50 | 51 | # Optional npm cache directory 52 | .npm 53 | 54 | # Optional eslint cache 55 | .eslintcache 56 | 57 | # Optional stylelint cache 58 | .stylelintcache 59 | 60 | # Microbundle cache 61 | .rpt2_cache/ 62 | .rts2_cache_cjs/ 63 | .rts2_cache_es/ 64 | .rts2_cache_umd/ 65 | 66 | # Optional REPL history 67 | .node_repl_history 68 | 69 | # Output of 'npm pack' 70 | *.tgz 71 | 72 | # Yarn Integrity file 73 | .yarn-integrity 74 | 75 | # dotenv environment variable files 76 | .env 77 | .env.development.local 78 | .env.test.local 79 | .env.production.local 80 | .env.local 81 | 82 | # parcel-bundler cache (https://parceljs.org/) 83 | .cache 84 | .parcel-cache 85 | 86 | # Next.js build output 87 | .next 88 | out 89 | 90 | # Nuxt.js build / generate output 91 | .nuxt 92 | dist 93 | 94 | # Gatsby files 95 | .cache/ 96 | # Comment in the public line in if your project uses Gatsby and not Next.js 97 | # https://nextjs.org/blog/next-9-1#public-directory-support 98 | # public 99 | 100 | # vuepress build output 101 | .vuepress/dist 102 | 103 | # vuepress v2.x temp and cache directory 104 | .temp 105 | .cache 106 | 107 | # Docusaurus cache and generated files 108 | .docusaurus 109 | 110 | # Serverless directories 111 | .serverless/ 112 | 113 | # FuseBox cache 114 | .fusebox/ 115 | 116 | # DynamoDB Local files 117 | .dynamodb/ 118 | 119 | # TernJS port file 120 | .tern-port 121 | 122 | # Stores VSCode versions used for testing VSCode extensions 123 | .vscode-test 124 | 125 | # yarn v2 126 | .yarn/cache 127 | .yarn/unplugged 128 | .yarn/build-state.yml 129 | .yarn/install-state.gz 130 | .pnp.* 131 | 132 | 133 | *.qcow2 134 | *.iso 135 | *.img 136 | *.vdi 137 | *.vmdk 138 | *.vhdx 139 | *.vhd 140 | 141 | 142 | # compiled output 143 | agent/dist 144 | agent/node_modules 145 | agent/build 146 | 147 | # Logs 148 | logs 149 | *.log 150 | npm-debug.log* 151 | pnpm-debug.log* 152 | yarn-debug.log* 153 | yarn-error.log* 154 | lerna-debug.log* 155 | 156 | # OS 157 | .DS_Store 158 | 159 | # Tests 160 | agent/coverage 161 | agent/.nyc_output 162 | 163 | # IDEs and editors 164 | agent/.idea 165 | agent/.project 166 | agent/.classpath 167 | .c9/ 168 | *.launch 169 | .settings/ 170 | *.sublime-workspace 171 | 172 | # IDE - VSCode 173 | .vscode/* 174 | !.vscode/settings.json 175 | !.vscode/tasks.json 176 | !.vscode/launch.json 177 | !.vscode/extensions.json 178 | 179 | # dotenv environment variable files 180 | .env.development.local 181 | .env.test.local 182 | .env.production.local 183 | .env.local 184 | 185 | # temp directory 186 | .temp 187 | .tmp 188 | 189 | # Runtime data 190 | pids 191 | *.pid 192 | *.seed 193 | *.pid.lock 194 | 195 | # Diagnostic reports (https://nodejs.org/api/report.html) 196 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 197 | 198 | # QEMU 199 | *.qcow2 200 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Tantl Labs, Inc 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | Bytebot Logo 4 | 5 | [🌐 Website](https://bytebot.ai) • [📚 Docs](https://docs.bytebot.ai) • [💬 Discord](https://discord.com/invite/zcb5wA2t4u) • [𝕏 Twitter](https://x.com/bytebot_ai) 6 | 7 | ## Bytebot – **The Easiest Way to Build Desktop Agents** 8 | 9 |
10 | 11 | ## ✨ Why Bytebot? 12 | 13 | Bytebot spins up a containerized Linux desktop with a task-driven agent ready for automation. Chat with it through the web UI or control it programmatically for scraping, CI tasks and remote work. 14 | 15 | ## Examples 16 | 17 | 18 | 19 | https://github.com/user-attachments/assets/32a76e83-ea3a-4d5e-b34b-3b57f3604948 20 | 21 | 22 | 23 | 24 | https://github.com/user-attachments/assets/5f946df9-9161-4e7e-8262-9eda83ee7d22 25 | 26 | 27 | 28 | ## 🚀 Features 29 | 30 | - 📦 **Containerized Desktop** – XFCE4 on Ubuntu 22.04 in a single Docker image 31 | - 🌍 **Access Anywhere** – VNC & browser‑based **noVNC** built‑in 32 | - 🛠️ **Unified API** – Script every click & keystroke with a clean REST interface 33 | - ⚙️ **Ready‑to‑Go Tools** – Firefox & essentials pre‑installed 34 | - 🤖 **Task-Driven Agent** – Manage tasks via REST or Chat UI and watch them run 35 | 36 | ## 🧠 Agent System 37 | 38 | Bytebot's agent stack is orchestrated with `docker-compose`. It starts: 39 | 40 | - `bytebot-desktop` – the Linux desktop and automation daemon 41 | - `bytebot-agent` – NestJS service processing tasks with Anthropic's Claude 42 | - `bytebot-ui` – Next.js chat interface 43 | - `postgres` – stores tasks and conversation history 44 | 45 | Open `http://localhost:9992` to give the agent a task and watch it work. 46 | 47 | ## 📖 Documentation 48 | 49 | Dive deeper at [**docs.bytebot.ai**](https://docs.bytebot.ai). 50 | 51 | ## ⚡ Quick Start 52 | 53 | ### 🛠️ Prerequisites 54 | 55 | - Docker ≥ 20.10 56 | 57 | ### 🐳 Run Bytebot 58 | 59 | #### 🤖 Full Agent Stack (fastest way) 60 | 61 | ```bash 62 | echo "ANTHROPIC_API_KEY=your_api_key_here" > infrastructure/docker/.env 63 | 64 | docker-compose -f infrastructure/docker/docker-compose.yml \ 65 | --env-file infrastructure/docker/.env up -d # start desktop, agent & UI 66 | ``` 67 | Once running, open `http://localhost:9992` to chat with the agent. 68 | 69 | Stop: 70 | 71 | ```bash 72 | docker-compose -f infrastructure/docker/docker-compose.yml \ 73 | --env-file infrastructure/docker/.env down 74 | ``` 75 | 76 | #### Core Container 77 | 78 | ```bash 79 | docker-compose -f infrastructure/docker/docker-compose.core.yml pull # pull latest remote image 80 | 81 | docker-compose -f infrastructure/docker/docker-compose.core.yml up -d --no-build # start container 82 | ``` 83 | 84 | Build locally instead: 85 | 86 | ```bash 87 | 88 | docker-compose -f infrastructure/docker/docker-compose.core.yml up -d --build # build image and start container 89 | ``` 90 | 91 | Stop: 92 | 93 | ```bash 94 | docker-compose -f infrastructure/docker/docker-compose.core.yml down 95 | ``` 96 | 97 | More details in the [**Quickstart Guide**](https://docs.bytebot.ai/quickstart). 98 | 99 | ### 🔑 Connect 100 | 101 | | Interface | URL / Port | Notes | 102 | | ------------- | --------------------------- | ------------------------ | 103 | | 💬 Chat UI | `http://localhost:9992` | Agent UI | 104 | | 🤖 Agent API | `http://localhost:9991` | REST API | 105 | | 🌐 noVNC | `http://localhost:9990/vnc` | open in any browser | 106 | | 🖥️ VNC Client | `localhost:5900` | password‑less by default | 107 | 108 | 109 | 110 | 111 | ## 🤖 Automation API 112 | 113 | Control Bytebot with a single endpoint. Read the [**REST reference**](https://docs.bytebot.ai/rest-api/computer-use). Supported actions: 114 | 115 | | 🎮 Action | Description | 116 | | ----------------- | -------------------------- | 117 | | `move_mouse` | Move cursor to coordinates | 118 | | `trace_mouse` | Draw a path | 119 | | `click_mouse` | Click (left/right/middle) | 120 | | `press_mouse` | Press / release button | 121 | | `drag_mouse` | Drag along path | 122 | | `scroll` | Scroll direction & amount | 123 | | `type_keys` | Type sequence of keys | 124 | | `press_keys` | Press / release keys | 125 | | `type_text` | Type a string | 126 | | `wait` | Wait milliseconds | 127 | | `screenshot` | Capture screen | 128 | | `cursor_position` | Return cursor position | 129 | 130 | _(See docs for parameter details.)_ 131 | 132 | ## 🙌 Contributing 133 | 134 | 1. 🍴 Fork & branch from `main` 135 | 2. 💡 Commit small, focused changes 136 | 3. 📩 Open a PR with details 137 | 4. 🔍 Address review feedback 138 | 5. 🎉 Merge & celebrate! 139 | 140 | ## 💬 Support 141 | 142 | Questions or ideas? Join us on [**Discord**](https://discord.com/invite/zcb5wA2t4u). 143 | 144 | ## 🙏 Acknowledgments 145 | 146 | Powered by [**nutjs**](https://github.com/nut-tree/nut.js) and inspired by Anthropic's [**computer‑use demo**](https://github.com/anthropics/anthropic-quickstarts/tree/main/computer-use-demo). 147 | 148 | ## 📄 License 149 | 150 | MIT © 2025 Tantl Labs, Inc. 151 | -------------------------------------------------------------------------------- /docs/api-reference/agent/ui.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Chat UI' 3 | description: 'Documentation for the Bytebot Chat UI' 4 | --- 5 | 6 | ## Bytebot Chat UI 7 | 8 | The Bytebot Chat UI provides a web-based interface for interacting with the Bytebot agent system. It combines a chat interface with an embedded noVNC viewer, allowing you to communicate with the agent and watch it perform tasks on the desktop in real-time. 9 | 10 | Bytebot Chat UI Overview 11 | 12 | ## Accessing the UI 13 | 14 | When running the full Bytebot agent system, the Chat UI is available at: 15 | 16 | ``` 17 | http://localhost:9992 18 | ``` 19 | 20 | ## UI Components 21 | 22 | ### Task Management Panel 23 | 24 | The task management panel allows you to: 25 | 26 | - Create new tasks 27 | - View existing tasks 28 | - See task status and priority 29 | - Select a task to work on 30 | 31 | Task Management Panel 32 | 33 | ### Chat Interface 34 | 35 | The main chat interface provides: 36 | 37 | - Conversation history with the agent 38 | - Message input for sending new instructions 39 | - Support for markdown formatting in messages 40 | - Automatic scrolling to new messages 41 | 42 | ### Desktop Viewer 43 | 44 | The embedded noVNC viewer displays: 45 | 46 | - Real-time view of the desktop environment 47 | - Visual feedback of agent actions 48 | - Option to expand to full-screen view 49 | - Connection status indicator 50 | 51 | ## Features 52 | 53 | ### Task Creation 54 | 55 | To create a new task: 56 | 57 | 1. Click the "New Task" button in the task panel 58 | 2. Enter a description for the task 59 | 3. Click "Create Task" 60 | 61 | ### Conversation Controls 62 | 63 | The chat interface supports: 64 | 65 | - Text messages with markdown formatting 66 | - Viewing image content in messages 67 | - Displaying tool use actions 68 | - Showing tool results 69 | 70 | ### Desktop Interaction 71 | 72 | While primarily for viewing, the desktop panel allows: 73 | 74 | - Expanding to full-screen view 75 | - Viewing desktop screenshots taken by the agent 76 | - Real-time monitoring of agent actions 77 | 78 | ## Message Types 79 | 80 | The chat interface displays different types of messages based on Anthropic's content block structure: 81 | 82 | - **User Messages**: Your instructions and queries 83 | - **Assistant Messages**: Responses from the agent, which may include: 84 | - **Text Content Blocks**: Markdown-formatted text responses 85 | - **Image Content Blocks**: Images generated or captured 86 | - **Tool Use Content Blocks**: Computer actions being performed 87 | - **Tool Result Content Blocks**: Results of computer actions 88 | 89 | The message content structure follows this format: 90 | 91 | ```typescript 92 | interface Message { 93 | id: string; 94 | content: MessageContentBlock[]; 95 | role: MessageRole; // "USER" or "ASSISTANT" 96 | createdAt?: string; 97 | } 98 | 99 | interface MessageContentBlock { 100 | type: string; 101 | [key: string]: any; 102 | } 103 | 104 | interface TextContentBlock extends MessageContentBlock { 105 | type: "text"; 106 | text: string; 107 | } 108 | 109 | interface ImageContentBlock extends MessageContentBlock { 110 | type: "image"; 111 | source: { 112 | type: "base64"; 113 | media_type: string; 114 | data: string; 115 | }; 116 | } 117 | ``` 118 | 119 | ## Technical Details 120 | 121 | The Bytebot Chat UI is built with: 122 | 123 | - **Next.js**: React framework for the frontend 124 | - **Tailwind CSS**: For styling 125 | - **ReactMarkdown**: For rendering markdown content 126 | - **noVNC**: For the embedded desktop viewer 127 | 128 | ## Troubleshooting 129 | 130 | ### Connection Issues 131 | 132 | If you experience connection issues: 133 | 134 | 1. Ensure all Bytebot services are running 135 | 2. Check that ports 9990, 9991, and 9992 are accessible 136 | 3. Try refreshing the browser 137 | 4. Check browser console for error messages 138 | 139 | ### Desktop Viewer Issues 140 | 141 | If the desktop viewer is not displaying: 142 | 143 | 1. Ensure the Bytebot container is running 144 | 2. Check that the noVNC service is accessible at port 9990 145 | 3. Try clicking the "Reconnect" button 146 | 4. Verify that no other VNC client is connected exclusively 147 | 148 | ### Message Display Issues 149 | 150 | If messages are not displaying correctly: 151 | 152 | 1. Check that the message content is properly formatted 153 | 2. Ensure the agent service is processing tasks correctly 154 | 3. Check the browser console for any rendering errors 155 | 4. Try refreshing the browser 156 | -------------------------------------------------------------------------------- /docs/api-reference/endpoint/create.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Create Plant' 3 | openapi: 'POST /plants' 4 | --- 5 | -------------------------------------------------------------------------------- /docs/api-reference/endpoint/delete.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Delete Plant' 3 | openapi: 'DELETE /plants/{id}' 4 | --- 5 | -------------------------------------------------------------------------------- /docs/api-reference/endpoint/get.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Get Plants' 3 | openapi: 'GET /plants' 4 | --- 5 | -------------------------------------------------------------------------------- /docs/api-reference/endpoint/webhook.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'New Plant' 3 | openapi: 'WEBHOOK /plant/webhook' 4 | --- 5 | -------------------------------------------------------------------------------- /docs/api-reference/introduction.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "API Reference" 3 | description: "Overview of the Bytebot API endpoints" 4 | --- 5 | 6 | ## Computer Use API 7 | 8 | Bytebot's core functionality is exposed through its Computer Use API, which provides a unified endpoint for all interactions with the desktop environment. The API allows for programmatic control of mouse movement, keyboard input, and screen capture. 9 | 10 | ### Authentication 11 | 12 | The Bytebot API does not require authentication by default when accessed locally. For remote access, standard network security practices should be implemented. 13 | 14 | ### Base URL 15 | 16 | All API endpoints are relative to the base URL: 17 | 18 | ``` 19 | http://localhost:9990 20 | ``` 21 | 22 | The port can be configured when running the container. 23 | 24 | ### API Endpoints 25 | 26 | 27 | 32 | Single endpoint for all desktop interactions including mouse, keyboard, and 33 | screen operations 34 | 35 | 40 | Code examples and snippets for common automation scenarios 41 | 42 | 43 | 44 | ### Response Format 45 | 46 | All API responses follow a standard JSON format: 47 | 48 | ```json 49 | { 50 | "success": true, 51 | "data": { ... }, // Response data specific to the action 52 | "error": null // Error message if success is false 53 | } 54 | ``` 55 | 56 | ### Error Handling 57 | 58 | When an error occurs, the API returns: 59 | 60 | ```json 61 | { 62 | "success": false, 63 | "data": null, 64 | "error": "Detailed error message" 65 | } 66 | ``` 67 | 68 | Common HTTP status codes: 69 | 70 | | Status Code | Description | 71 | | ----------- | -------------------------------- | 72 | | 200 | Success | 73 | | 400 | Bad Request - Invalid parameters | 74 | | 500 | Internal Server Error | 75 | 76 | ### Rate Limiting 77 | 78 | The API currently does not implement rate limiting, but excessive requests may impact performance of the virtual desktop environment. 79 | -------------------------------------------------------------------------------- /docs/core-concepts/architecture.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Architecture" 3 | description: "Overview of the Bytebot architecture and components" 4 | --- 5 | 6 | ## Bytebot Architecture 7 | 8 | Bytebot is designed with a modular architecture that can be run as a standalone desktop container or as a full-featured agent system with a web UI. 9 | 10 | Bytebot Architecture Diagram 15 | 16 | ## Core Components 17 | 18 | ### Container Base 19 | 20 | - **Ubuntu 22.04** serves as the base operating system 21 | - Provides a stable foundation for the desktop environment and tools 22 | 23 | ### Desktop Environment 24 | 25 | - **XFCE4** desktop environment 26 | - Lightweight and customizable 27 | - Comes pre-configured with sensible defaults 28 | - Includes default user account: `bytebot` with sudo privileges 29 | 30 | ### Automation Daemon (bytebotd) 31 | 32 | - **bytebotd daemon** is the core service that enables automation 33 | - Built on top of nutjs for desktop automation 34 | - Exposes a REST API for remote control 35 | - Provides unified endpoint for all computer actions 36 | - Accessible at `localhost:9990` 37 | 38 | ### Browser and Tools 39 | 40 | - **Firefox** pre-installed and configured 41 | - Essential utilities for development and testing 42 | - Default applications for common tasks 43 | 44 | ### Remote Access 45 | 46 | - **VNC server** for direct desktop access 47 | - **noVNC** for browser-based desktop access 48 | 49 | ## Agent System Components 50 | 51 | When running the full Bytebot system using docker-compose, the following additional components are available: 52 | 53 | ### Bytebot Agent 54 | 55 | - **Agent service** that manages tasks and AI-driven automation 56 | - Built with NestJS for reliable API services 57 | - Implements a task processing system with queues via BullMQ 58 | - Integrates with Anthropic's Claude for AI capabilities 59 | - Accessible at `localhost:9991` 60 | 61 | ### Databases 62 | 63 | - **PostgreSQL database** for storing tasks, messages, and agent state 64 | - Provides persistence for tasks and conversations 65 | 66 | ### Chat UI 67 | 68 | - **NextJS web application** for interacting with the agent 69 | - Provides a chat interface for communicating with the AI 70 | - Includes an embedded noVNC view for observing desktop actions 71 | - Accessible at `localhost:9992` 72 | 73 | ## Task Management 74 | 75 | The agent system implements a task-based workflow: 76 | 77 | 1. **Tasks** are the primary unit of work with properties like status, priority, and description 78 | 2. **Messages** represent the conversation between user and assistant 79 | 3. **Summaries** capture the state and progress of tasks 80 | 81 | ## Communication Flow 82 | 83 | ### Standalone Mode 84 | 85 | 1. **External Application** makes requests to the Bytebot API 86 | 2. **bytebotd daemon** receives and processes these requests 87 | 3. **Desktop Automation** is performed using nutjs 88 | 4. **Results/Screenshots** are returned to the calling application 89 | 90 | ### Agent Mode 91 | 92 | 1. **User** creates tasks and sends messages via the Chat UI 93 | 2. **Agent service** processes tasks and messages through a queue system 94 | 3. **AI Integration** with Claude generates responses and computer actions 95 | 4. **Computer Use API** executes actions on the Bytebot desktop 96 | 5. **Results** are returned to the user through the Chat UI 97 | 98 | ## Security Considerations 99 | 100 | 101 | The default container configuration is intended for development and testing 102 | purposes only. It should **not** be used in production environments without 103 | security hardening. 104 | 105 | 106 | Security aspects to consider before deploying in production: 107 | 108 | 1. The container runs with a default user account that has sudo privileges 109 | 2. Remote access protocols (VNC, noVNC) are not encrypted by default 110 | 3. The REST API does not implement authentication by default 111 | 4. Container networking exposes several ports that should be secured 112 | 5. API keys (like ANTHROPIC_API_KEY) should be properly secured 113 | 114 | ## Customization Points 115 | 116 | Bytebot is designed to be customizable for different use cases: 117 | 118 | - **Docker base image** can be modified for different Linux distributions 119 | - **Desktop environment** can be replaced with alternatives (GNOME, KDE, etc.) 120 | - **Pre-installed applications** can be customized for specific testing needs 121 | - **API endpoints** can be extended for additional functionality 122 | - **Agent system** can be extended with custom tools and integrations 123 | -------------------------------------------------------------------------------- /docs/core-concepts/desktop-environment.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Desktop Environment" 3 | description: "Details about the Bytebot containerized desktop environment" 4 | --- 5 | 6 | ## Containerized Desktop 7 | 8 | Bytebot's containerized desktop environment provides a lightweight yet fully-functional Linux desktop inside a Docker container. This approach ensures consistency across different host systems and simplifies deployment. 9 | 10 | ## Components 11 | 12 | ### XFCE4 Desktop 13 | 14 | The desktop environment in Bytebot is based on XFCE4, a lightweight and efficient desktop environment for Unix-like operating systems: 15 | 16 | - **Lightweight**: Requires minimal system resources 17 | - **Customizable**: Easily adapted for different use cases 18 | - **Fast**: Provides responsive desktop experience 19 | - **Compatible**: Works well with automation tools 20 | 21 | ### Base System 22 | 23 | - **Ubuntu 22.04** (Jammy Jellyfish) serves as the base operating system 24 | - Default user account: `bytebot` with sudo privileges 25 | - Pre-configured locale and timezone settings 26 | 27 | ### Pre-installed Applications 28 | 29 | The Bytebot container comes with essential software pre-installed: 30 | 31 | - **Firefox** web browser 32 | - **1Password** password manager 33 | - **Thunderbird** email client 34 | - **Terminal emulator** for command-line access 35 | - **Text editor** for viewing and editing files 36 | - **File manager** for navigating the filesystem 37 | - **Basic system utilities** (calculator, image viewer, etc.) 38 | 39 | ## Desktop Configuration 40 | 41 | The desktop environment is configured with automation in mind: 42 | 43 | - **Simplified layout**: Clean desktop with minimal distractions 44 | - **Predictable element positioning**: Consistent locations for UI elements 45 | - **Auto-login**: Desktop environment starts automatically 46 | - **Resolution control**: Configurable display settings 47 | 48 | ## Remote Access 49 | 50 | ### VNC Server 51 | 52 | The container runs a VNC server that allows direct access to the desktop: 53 | 54 | - Accessible on port 5900 (default VNC port) 55 | - Compatible with standard VNC clients (RealVNC, TightVNC, etc.) 56 | - Supports standard VNC authentication 57 | 58 | ### noVNC 59 | 60 | For browser-based access, noVNC is included: 61 | 62 | - Accessible via web browser at `http://localhost:9990/vnc` 63 | - No client software required 64 | - Works across different platforms and devices 65 | 66 | ## Display Server 67 | 68 | Bytebot uses Xvfb (X Virtual Framebuffer) as its display server: 69 | 70 | - Creates virtual displays in memory without hardware 71 | - Suitable for headless environments 72 | - Configurable resolution and color depth 73 | - Compatible with standard X11 applications 74 | 75 | ## Using the Desktop Environment 76 | 77 | ### Manual Interaction 78 | 79 | You can directly interact with the desktop environment using: 80 | 81 | 1. **VNC client** connected to `localhost:5900` 82 | 2. **Web browser** navigated to `http://localhost:9990/vnc` 83 | 84 | ### Programmatic Interaction 85 | 86 | The primary purpose of the Bytebot desktop is programmatic control through the Computer Use API: 87 | 88 | - Control mouse and keyboard inputs 89 | - Capture screenshots 90 | - Interact with desktop applications 91 | - Automate workflows 92 | 93 | ## Performance Considerations 94 | 95 | The containerized desktop has some performance characteristics to be aware of: 96 | 97 | - **Resource usage**: XFCE4 is lightweight but still requires CPU and memory resources 98 | - **Graphics performance**: Limited 3D acceleration compared to native desktop 99 | - **Network overhead**: Remote access adds some latency 100 | - **Disk I/O**: Container storage may be slower than native filesystem 101 | 102 | ## Customization 103 | 104 | You can customize the desktop environment by: 105 | 106 | 1. Modifying the Dockerfile to install additional software 107 | 2. Adjusting the XFCE4 configuration files for different layouts 108 | 3. Adding custom startup scripts 109 | 4. Setting environment variables to control behavior 110 | 111 | ## Security Notes 112 | 113 | 114 | The default desktop environment provides convenience at the expense of 115 | security. For production use, additional hardening is recommended. 116 | 117 | 118 | Security considerations: 119 | 120 | - User has sudo privileges by default 121 | - VNC password should be changed from default 122 | - Desktop autologin removes authentication requirement 123 | - NoVNC web access may expose the desktop to unauthorized users 124 | -------------------------------------------------------------------------------- /docs/docs.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://mintlify.com/docs.json", 3 | "theme": "mint", 4 | "name": "Bytebot Documentation", 5 | "colors": { 6 | "primary": "#000000", 7 | "light": "#fbfaf9", 8 | "dark": "#000000" 9 | }, 10 | "favicon": "/favicon.svg", 11 | "navigation": { 12 | "tabs": [ 13 | { 14 | "tab": "Guides", 15 | "groups": [ 16 | { 17 | "group": "Get Started", 18 | "pages": ["introduction", "quickstart"] 19 | }, 20 | { 21 | "group": "Core Concepts", 22 | "pages": [ 23 | "core-concepts/architecture", 24 | "core-concepts/desktop-environment", 25 | "core-concepts/agent-system" 26 | ] 27 | } 28 | ] 29 | }, 30 | { 31 | "tab": "REST API", 32 | "groups": [ 33 | { 34 | "group": "Overview", 35 | "pages": ["rest-api/introduction"] 36 | }, 37 | { 38 | "group": "Endpoints", 39 | "pages": ["rest-api/computer-use", "rest-api/examples"] 40 | } 41 | ] 42 | } 43 | ], 44 | "global": { 45 | "anchors": [ 46 | { 47 | "anchor": "GitHub", 48 | "href": "https://github.com/bytebot-ai/bytebot", 49 | "icon": "github" 50 | }, 51 | { 52 | "anchor": "Discord", 53 | "href": "https://discord.gg/6nxuF6cs", 54 | "icon": "discord" 55 | }, 56 | { 57 | "anchor": "Twitter", 58 | "href": "https://x.com/bytebot_ai", 59 | "icon": "twitter" 60 | }, 61 | { 62 | "anchor": "Blog", 63 | "href": "https://bytebot.ai/blog", 64 | "icon": "newspaper" 65 | } 66 | ] 67 | } 68 | }, 69 | "logo": { 70 | "light": "/logo/bytebot_transparent_logo_dark.svg", 71 | "dark": "/logo/bytebot_transparent_logo_white.svg" 72 | }, 73 | "navbar": { 74 | "links": [ 75 | { 76 | "label": "Support", 77 | "href": "mailto:support@bytebot.ai" 78 | } 79 | ], 80 | "primary": { 81 | "type": "button", 82 | "label": "GitHub", 83 | "href": "https://github.com/bytebot-ai/bytebot" 84 | } 85 | }, 86 | "footer": { 87 | "socials": { 88 | "github": "https://github.com/bytebot-ai/bytebot", 89 | "twitter": "https://twitter.com/bytebotai" 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /docs/favicon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /docs/images/agent-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytebot-ai/bytebot/afb98c7a2021aa95b5feac5715a40567dbb32e91/docs/images/agent-architecture.png -------------------------------------------------------------------------------- /docs/images/core-container.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytebot-ai/bytebot/afb98c7a2021aa95b5feac5715a40567dbb32e91/docs/images/core-container.png -------------------------------------------------------------------------------- /docs/introduction.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: Introduction 3 | description: "A containerized computer use environment with an integrated XFCE4 desktop and automation daemon" 4 | --- 5 | 6 |

7 | Bytebot Logo 13 | Bytebot Logo 19 |

20 | 21 | ## What is Bytebot? 22 | 23 | Bytebot provides a complete, self-contained environment for computer use automation. It encapsulates a lightweight XFCE4 desktop environment inside a Docker container with the bytebotd daemon for programmatic control, making it easy to deploy across different platforms. 24 | 25 | ## Key Features 26 | 27 | 28 | 33 | Runs a lightweight XFCE4 desktop on Ubuntu 22.04 with pre-installed tools 34 | 35 | 40 | Control the desktop environment programmatically through a unified REST API 41 | 42 | 43 | Works on any system that supports Docker with simple setup 44 | 45 | 46 | View and interact with the desktop through VNC or browser-based noVNC 47 | 48 | 49 | 50 | ## Architecture Overview 51 | 52 | Bytebot is designed as a single, integrated container that provides both a desktop environment and the tools to control it: 53 | 54 | Bytebot Core Container 55 | 56 | ## Getting Started 57 | 58 | Get up and running with Bytebot in minutes: 59 | 60 | 61 | 62 | Set up and run Bytebot on your system 63 | 64 | 65 | Learn how to programmatically control the Bytebot environment 66 | 67 | 68 | 69 | 70 | The default container configuration is intended for development and testing 71 | purposes only. It should **not** be used in production environments without 72 | security hardening. 73 | 74 | -------------------------------------------------------------------------------- /docs/logo/bytebot_transparent_logo_dark.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /docs/logo/bytebot_transparent_logo_white.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /docs/rest-api/introduction.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Introduction" 3 | description: "Overview of the Bytebot REST API" 4 | --- 5 | 6 | ## Bytebot REST API 7 | 8 | Bytebot's core functionality is exposed through its REST API, which provides endpoints for interacting with the desktop environment. The API allows for programmatic control of mouse movement, keyboard input, and screen capture. 9 | 10 | ### Base URL 11 | 12 | All API endpoints are relative to the base URL: 13 | 14 | ``` 15 | http://localhost:9990 16 | ``` 17 | 18 | The port can be configured when running the container. 19 | 20 | ### Authentication 21 | 22 | The Bytebot API does not require authentication by default when accessed locally. For remote access, standard network security practices should be implemented. 23 | 24 | ### Response Format 25 | 26 | All API responses follow a standard JSON format: 27 | 28 | ```json 29 | { 30 | "success": true, 31 | "data": { ... }, // Response data specific to the action 32 | "error": null // Error message if success is false 33 | } 34 | ``` 35 | 36 | ### Error Handling 37 | 38 | When an error occurs, the API returns: 39 | 40 | ```json 41 | { 42 | "success": false, 43 | "data": null, 44 | "error": "Detailed error message" 45 | } 46 | ``` 47 | 48 | Common HTTP status codes: 49 | 50 | | Status Code | Description | 51 | | ----------- | -------------------------------- | 52 | | 200 | Success | 53 | | 400 | Bad Request - Invalid parameters | 54 | | 500 | Internal Server Error | 55 | 56 | ### Available Endpoints 57 | 58 | 59 | 64 | Execute desktop automation actions like mouse movements, clicks, keyboard 65 | input, and screenshots 66 | 67 | 68 | Code examples and snippets for common automation scenarios 69 | 70 | 71 | 72 | ### Rate Limiting 73 | 74 | The API currently does not implement rate limiting, but excessive requests may impact performance of the virtual desktop environment. 75 | -------------------------------------------------------------------------------- /infrastructure/docker/.dockerignore: -------------------------------------------------------------------------------- 1 | **/node_modules 2 | **/dist 3 | **/.next 4 | **/.git 5 | **/.vscode 6 | **/.env* 7 | **/npm-debug.log 8 | **/yarn-debug.log 9 | **/yarn-error.log 10 | **/package-lock.json -------------------------------------------------------------------------------- /infrastructure/docker/docker-compose.core.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | 3 | services: 4 | bytebot-desktop: 5 | # Build from source 6 | build: 7 | context: ../../ 8 | dockerfile: infrastructure/docker/Dockerfile 9 | # Use pre-built image 10 | image: ghcr.io/bytebot-ai/bytebot:edge 11 | 12 | container_name: bytebot-desktop 13 | restart: unless-stopped 14 | hostname: computer 15 | privileged: true 16 | ports: 17 | - "9990:9990" # bytebotd service 18 | - "5900:5900" # VNC display 19 | - "6080:6080" # noVNC client 20 | - "6081:6081" # noVNC HTTP proxy 21 | environment: 22 | - DISPLAY=:0 23 | -------------------------------------------------------------------------------- /infrastructure/docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | name: bytebot 2 | 3 | services: 4 | bytebot-desktop: 5 | # Build from source 6 | build: 7 | context: ../../ 8 | dockerfile: infrastructure/docker/Dockerfile 9 | # Use pre-built image 10 | image: ghcr.io/bytebot-ai/bytebot:edge 11 | shm_size: "2g" 12 | container_name: bytebot-desktop 13 | restart: unless-stopped 14 | hostname: computer 15 | privileged: true 16 | ports: 17 | - "9990:9990" # bytebotd service 18 | - "5900:5900" # VNC display 19 | - "6080:6080" # noVNC client 20 | - "6081:6081" # noVNC HTTP proxy 21 | environment: 22 | - DISPLAY=:0 23 | networks: 24 | - bytebot-network 25 | 26 | postgres: 27 | image: postgres:16-alpine 28 | container_name: bytebot-postgres 29 | restart: unless-stopped 30 | ports: 31 | - "5432:5432" 32 | environment: 33 | - POSTGRES_PASSWORD=postgres 34 | - POSTGRES_USER=postgres 35 | - POSTGRES_DB=bytebotdb 36 | networks: 37 | - bytebot-network 38 | volumes: 39 | - postgres_data:/var/lib/postgresql/data 40 | 41 | bytebot-agent: 42 | build: 43 | context: ../../packages/ 44 | dockerfile: bytebot-agent/Dockerfile 45 | container_name: bytebot-agent 46 | restart: unless-stopped 47 | ports: 48 | - "9991:9991" 49 | environment: 50 | - DATABASE_URL=${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/bytebotdb} 51 | - BYTEBOT_DESKTOP_BASE_URL=${BYTEBOT_DESKTOP_BASE_URL:-http://bytebot-desktop:9990} 52 | - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} 53 | depends_on: 54 | - postgres 55 | networks: 56 | - bytebot-network 57 | 58 | bytebot-ui: 59 | build: 60 | context: ../../packages/ 61 | dockerfile: bytebot-ui/Dockerfile 62 | args: 63 | - NEXT_PUBLIC_BYTEBOT_AGENT_BASE_URL=${BYTEBOT_AGENT_BASE_URL:-http://localhost:9991} 64 | - NEXT_PUBLIC_BYTEBOT_DESKTOP_VNC_URL=${BYTEBOT_DESKTOP_VNC_URL:-ws://localhost:6080} 65 | container_name: bytebot-ui 66 | restart: unless-stopped 67 | ports: 68 | - "9992:9992" 69 | environment: 70 | - NODE_ENV=production 71 | depends_on: 72 | - bytebot-agent 73 | networks: 74 | - bytebot-network 75 | 76 | networks: 77 | bytebot-network: 78 | driver: bridge 79 | 80 | volumes: 81 | postgres_data: 82 | -------------------------------------------------------------------------------- /infrastructure/docker/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon=true 3 | logfile=/dev/stdout 4 | logfile_maxbytes=0 5 | loglevel=info 6 | 7 | [program:set-hostname] 8 | command=bash -c "sudo hostname computer" 9 | autostart=true 10 | autorestart=false 11 | startsecs=0 12 | priority=1 13 | stdout_logfile=/dev/stdout 14 | stdout_logfile_maxbytes=0 15 | redirect_stderr=true 16 | 17 | [program:dbus] 18 | command=/usr/bin/dbus-daemon --system --nofork 19 | priority=1 20 | autostart=true 21 | autorestart=true 22 | stdout_logfile=/dev/stdout 23 | stdout_logfile_maxbytes=0 24 | redirect_stderr=true 25 | 26 | [program:xvfb] 27 | command=Xvfb :0 -screen 0 1280x960x24 -ac -nolisten tcp 28 | autostart=true 29 | autorestart=true 30 | startsecs=5 31 | priority=10 32 | stdout_logfile=/dev/stdout 33 | stdout_logfile_maxbytes=0 34 | redirect_stderr=true 35 | 36 | [program:xfce4] 37 | command=sh -c 'sleep 5 && startxfce4' 38 | environment=DISPLAY=":0" 39 | autostart=true 40 | autorestart=true 41 | startsecs=5 42 | priority=20 43 | stdout_logfile=/dev/stdout 44 | stdout_logfile_maxbytes=0 45 | redirect_stderr=true 46 | depends_on=xvfb 47 | 48 | [program:x11vnc] 49 | command=x11vnc -display :0 -N -forever -shared -rfbport 5900 50 | autostart=true 51 | autorestart=true 52 | startsecs=5 53 | priority=30 54 | environment=DISPLAY=":0" 55 | stdout_logfile=/dev/stdout 56 | stdout_logfile_maxbytes=0 57 | redirect_stderr=true 58 | depends_on=xfce4 59 | 60 | [program:websockify] 61 | command=websockify 6080 localhost:5900 62 | autostart=true 63 | autorestart=true 64 | startsecs=5 65 | priority=40 66 | stdout_logfile=/dev/stdout 67 | stdout_logfile_maxbytes=0 68 | redirect_stderr=true 69 | depends_on=x11vnc 70 | 71 | [program:novnc-http] 72 | command=python3 -m http.server 6081 --directory /opt/noVNC 73 | autostart=true 74 | autorestart=true 75 | startsecs=5 76 | priority=50 77 | stdout_logfile=/dev/stdout 78 | stdout_logfile_maxbytes=0 79 | redirect_stderr=true 80 | depends_on=websockify 81 | 82 | [program:bytebotd] 83 | command=node /bytebotd/dist/main.js 84 | directory=/bytebotd 85 | autostart=true 86 | autorestart=true 87 | startsecs=5 88 | priority=60 89 | environment=DISPLAY=":0" 90 | stdout_logfile=/dev/stdout 91 | stdout_logfile_maxbytes=0 92 | redirect_stderr=true 93 | depends_on=novnc-http 94 | 95 | [eventlistener:startup] 96 | command=echo "All services started successfully" 97 | events=PROCESS_STATE_RUNNING 98 | buffer_size=100 -------------------------------------------------------------------------------- /infrastructure/docker/xfce4/desktop/icons.screen.latest.rc: -------------------------------------------------------------------------------- 1 | /home/bytebot/.config/xfce4/desktop/icons.screen0-1264x673.rc -------------------------------------------------------------------------------- /infrastructure/docker/xfce4/desktop/icons.screen0-1264x673.rc: -------------------------------------------------------------------------------- 1 | [xfdesktop-version-4.10.3+-rcfile_format] 2 | 4.10.3+=true 3 | 4 | [/home/bytebot/Desktop/firefox.desktop] 5 | row=0 6 | col=0 7 | 8 | [/home/bytebot/Desktop/thunderbird.desktop] 9 | row=1 10 | col=0 11 | 12 | [/home/bytebot/Desktop/1password.desktop] 13 | row=2 14 | col=0 15 | 16 | [/home/bytebot/Desktop/xfce4-terminal-emulator.desktop] 17 | row=3 18 | col=0 19 | 20 | 21 | [/home/bytebot] 22 | row=4 23 | col=0 24 | 25 | 26 | [Trash] 27 | row=5 28 | col=0 29 | -------------------------------------------------------------------------------- /infrastructure/docker/xfce4/helpers.rc: -------------------------------------------------------------------------------- 1 | TerminalEmulator=xfce4-terminal 2 | WebBrowser=firefox 3 | FileManager=thunar 4 | 5 | -------------------------------------------------------------------------------- /infrastructure/docker/xfce4/terminal/accels.scm: -------------------------------------------------------------------------------- 1 | ; xfce4-terminal GtkAccelMap rc-file -*- scheme -*- 2 | ; this file is an automated accelerator map dump 3 | ; 4 | (gtk_accel_path "/terminal-window/goto-tab-2" "2") 5 | (gtk_accel_path "/terminal-window/goto-tab-6" "6") 6 | ; (gtk_accel_path "/terminal-window/copy-input" "") 7 | ; (gtk_accel_path "/terminal-window/close-other-tabs" "") 8 | ; (gtk_accel_path "/terminal-window/move-tab-right" "Page_Down") 9 | (gtk_accel_path "/terminal-window/goto-tab-7" "7") 10 | ; (gtk_accel_path "/terminal-window/set-title-color" "") 11 | ; (gtk_accel_path "/terminal-window/edit-menu" "") 12 | ; (gtk_accel_path "/terminal-window/zoom-menu" "") 13 | (gtk_accel_path "/terminal-window/goto-tab-1" "1") 14 | ; (gtk_accel_path "/terminal-window/fullscreen" "F11") 15 | ; (gtk_accel_path "/terminal-window/read-only" "") 16 | (gtk_accel_path "/terminal-window/goto-tab-5" "5") 17 | ; (gtk_accel_path "/terminal-window/preferences" "") 18 | ; (gtk_accel_path "/terminal-window/reset-and-clear" "") 19 | ; (gtk_accel_path "/terminal-window/about" "") 20 | (gtk_accel_path "/terminal-window/goto-tab-4" "4") 21 | ; (gtk_accel_path "/terminal-window/close-window" "q") 22 | ; (gtk_accel_path "/terminal-window/reset" "") 23 | ; (gtk_accel_path "/terminal-window/save-contents" "") 24 | (gtk_accel_path "/terminal-window/toggle-menubar" "F10") 25 | ; (gtk_accel_path "/terminal-window/copy" "c") 26 | ; (gtk_accel_path "/terminal-window/copy-html" "") 27 | ; (gtk_accel_path "/terminal-window/last-active-tab" "") 28 | ; (gtk_accel_path "/terminal-window/show-borders" "") 29 | ; (gtk_accel_path "/terminal-window/view-menu" "") 30 | ; (gtk_accel_path "/terminal-window/detach-tab" "d") 31 | ; (gtk_accel_path "/terminal-window/scroll-on-output" "") 32 | ; (gtk_accel_path "/terminal-window/show-toolbar" "") 33 | ; (gtk_accel_path "/terminal-window/next-tab" "Page_Down") 34 | ; (gtk_accel_path "/terminal-window/tabs-menu" "") 35 | ; (gtk_accel_path "/terminal-window/search-next" "") 36 | ; (gtk_accel_path "/terminal-window/search-prev" "") 37 | ; (gtk_accel_path "/terminal-window/undo-close-tab" "") 38 | ; (gtk_accel_path "/terminal-window/set-title" "s") 39 | ; (gtk_accel_path "/terminal-window/contents" "F1") 40 | ; (gtk_accel_path "/terminal-window/zoom-reset" "0") 41 | ; (gtk_accel_path "/terminal-window/close-tab" "w") 42 | ; (gtk_accel_path "/terminal-window/new-tab" "t") 43 | ; (gtk_accel_path "/terminal-window/new-window" "n") 44 | ; (gtk_accel_path "/terminal-window/terminal-menu" "") 45 | ; (gtk_accel_path "/terminal-window/show-menubar" "") 46 | ; (gtk_accel_path "/terminal-window/select-all" "a") 47 | ; (gtk_accel_path "/terminal-window/paste" "v") 48 | (gtk_accel_path "/terminal-window/goto-tab-9" "9") 49 | ; (gtk_accel_path "/terminal-window/move-tab-left" "Page_Up") 50 | ; (gtk_accel_path "/terminal-window/search" "f") 51 | ; (gtk_accel_path "/terminal-window/file-menu" "") 52 | ; (gtk_accel_path "/terminal-window/prev-tab" "Page_Up") 53 | ; (gtk_accel_path "/terminal-window/paste-selection" "") 54 | ; (gtk_accel_path "/terminal-window/zoom-in" "plus") 55 | ; (gtk_accel_path "/terminal-window/zoom-out" "minus") 56 | (gtk_accel_path "/terminal-window/goto-tab-8" "8") 57 | ; (gtk_accel_path "/terminal-window/help-menu" "") 58 | (gtk_accel_path "/terminal-window/goto-tab-3" "3") 59 | -------------------------------------------------------------------------------- /infrastructure/docker/xfce4/xfconf/xfce-perchannel-xml/displays.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /infrastructure/docker/xfce4/xfconf/xfce-perchannel-xml/thunar.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /infrastructure/docker/xfce4/xfconf/xfce-perchannel-xml/xfce4-appfinder.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /infrastructure/docker/xfce4/xfconf/xfce-perchannel-xml/xfce4-desktop.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /infrastructure/docker/xfce4/xfconf/xfce-perchannel-xml/xfce4-notifyd.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /infrastructure/docker/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /infrastructure/docker/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /packages/bytebot-agent/.dockerignore: -------------------------------------------------------------------------------- 1 | **/node_modules 2 | **/dist 3 | **/.git 4 | **/.vscode 5 | **/.env* 6 | **/npm-debug.log 7 | **/yarn-debug.log 8 | **/yarn-error.log 9 | **/package-lock.json -------------------------------------------------------------------------------- /packages/bytebot-agent/.env.example: -------------------------------------------------------------------------------- 1 | DATABASE_URL=postgresql://postgres:postgres@postgres:5432/bytebotdb 2 | -------------------------------------------------------------------------------- /packages/bytebot-agent/.gitignore: -------------------------------------------------------------------------------- 1 | # compiled output 2 | /dist 3 | /node_modules 4 | /build 5 | 6 | # Logs 7 | logs 8 | *.log 9 | npm-debug.log* 10 | pnpm-debug.log* 11 | yarn-debug.log* 12 | yarn-error.log* 13 | lerna-debug.log* 14 | 15 | # OS 16 | .DS_Store 17 | 18 | # Tests 19 | /coverage 20 | /.nyc_output 21 | 22 | # IDEs and editors 23 | /.idea 24 | .project 25 | .classpath 26 | .c9/ 27 | *.launch 28 | .settings/ 29 | *.sublime-workspace 30 | 31 | # IDE - VSCode 32 | .vscode/* 33 | !.vscode/settings.json 34 | !.vscode/tasks.json 35 | !.vscode/launch.json 36 | !.vscode/extensions.json 37 | 38 | # dotenv environment variable files 39 | .env 40 | .env.development.local 41 | .env.test.local 42 | .env.production.local 43 | .env.local 44 | 45 | # temp directory 46 | .temp 47 | .tmp 48 | 49 | # Runtime data 50 | pids 51 | *.pid 52 | *.seed 53 | *.pid.lock 54 | 55 | # Diagnostic reports (https://nodejs.org/api/report.html) 56 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 57 | -------------------------------------------------------------------------------- /packages/bytebot-agent/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "trailingComma": "all" 4 | } -------------------------------------------------------------------------------- /packages/bytebot-agent/Dockerfile: -------------------------------------------------------------------------------- 1 | # Base image 2 | FROM node:20-alpine 3 | 4 | # Create app directory 5 | WORKDIR /app 6 | 7 | # Copy app source 8 | COPY ./shared ./shared 9 | COPY ./bytebot-agent/ ./bytebot-agent/ 10 | 11 | WORKDIR /app/bytebot-agent 12 | 13 | # Install dependencies 14 | RUN npm install 15 | 16 | 17 | RUN npm run build 18 | 19 | # Run the application 20 | CMD ["npm", "run", "start:prod"] 21 | 22 | 23 | -------------------------------------------------------------------------------- /packages/bytebot-agent/eslint.config.mjs: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | import eslint from '@eslint/js'; 3 | import eslintPluginPrettierRecommended from 'eslint-plugin-prettier/recommended'; 4 | import globals from 'globals'; 5 | import tseslint from 'typescript-eslint'; 6 | 7 | export default tseslint.config( 8 | { 9 | ignores: ['eslint.config.mjs'], 10 | }, 11 | eslint.configs.recommended, 12 | ...tseslint.configs.recommendedTypeChecked, 13 | eslintPluginPrettierRecommended, 14 | { 15 | languageOptions: { 16 | globals: { 17 | ...globals.node, 18 | ...globals.jest, 19 | }, 20 | ecmaVersion: 5, 21 | sourceType: 'module', 22 | parserOptions: { 23 | projectService: true, 24 | tsconfigRootDir: import.meta.dirname, 25 | }, 26 | }, 27 | }, 28 | { 29 | rules: { 30 | '@typescript-eslint/no-explicit-any': 'off', 31 | '@typescript-eslint/no-floating-promises': 'warn', 32 | '@typescript-eslint/no-unsafe-argument': 'warn' 33 | }, 34 | }, 35 | ); -------------------------------------------------------------------------------- /packages/bytebot-agent/nest-cli.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/nest-cli", 3 | "collection": "@nestjs/schematics", 4 | "sourceRoot": "src", 5 | "compilerOptions": { 6 | "deleteOutDir": true 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /packages/bytebot-agent/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bytebot-agent", 3 | "version": "0.0.1", 4 | "description": "", 5 | "author": "", 6 | "private": true, 7 | "license": "UNLICENSED", 8 | "scripts": { 9 | "prisma:dev": "npx prisma migrate dev && npx prisma generate", 10 | "prisma:prod": "npx prisma migrate deploy && npx prisma generate", 11 | "build": "npm run build --prefix ../shared && npx prisma generate && nest build", 12 | "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", 13 | "start": "nest start", 14 | "start:dev": "nest start --watch", 15 | "start:debug": "nest start --debug --watch", 16 | "start:prod": "npx prisma migrate deploy && npx prisma generate && node dist/main", 17 | "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", 18 | "test": "jest", 19 | "test:watch": "jest --watch", 20 | "test:cov": "jest --coverage", 21 | "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", 22 | "test:e2e": "jest --config ./test/jest-e2e.json" 23 | }, 24 | "dependencies": { 25 | "@anthropic-ai/sdk": "^0.39.0", 26 | "@bytebot/shared": "../shared", 27 | "@nestjs/common": "^11.0.1", 28 | "@nestjs/config": "^4.0.2", 29 | "@nestjs/core": "^11.0.1", 30 | "@nestjs/platform-express": "^11.1.2", 31 | "@nestjs/platform-socket.io": "^11.1.1", 32 | "@nestjs/schedule": "^6.0.0", 33 | "@nestjs/websockets": "^11.1.1", 34 | "@prisma/client": "^6.6.0", 35 | "class-validator": "^0.14.2", 36 | "reflect-metadata": "^0.2.2", 37 | "rxjs": "^7.8.1", 38 | "socket.io": "^4.8.1" 39 | }, 40 | "devDependencies": { 41 | "@eslint/eslintrc": "^3.2.0", 42 | "@eslint/js": "^9.18.0", 43 | "@nestjs/cli": "^11.0.0", 44 | "@nestjs/schematics": "^11.0.0", 45 | "@nestjs/testing": "^11.0.1", 46 | "@swc/cli": "^0.6.0", 47 | "@swc/core": "^1.10.7", 48 | "@types/express": "^5.0.0", 49 | "@types/jest": "^29.5.14", 50 | "@types/node": "^22.10.7", 51 | "@types/supertest": "^6.0.2", 52 | "eslint": "^9.18.0", 53 | "eslint-config-prettier": "^10.0.1", 54 | "eslint-plugin-prettier": "^5.2.2", 55 | "globals": "^15.14.0", 56 | "jest": "^29.7.0", 57 | "prettier": "^3.4.2", 58 | "source-map-support": "^0.5.21", 59 | "supertest": "^7.0.0", 60 | "ts-jest": "^29.2.5", 61 | "ts-loader": "^9.5.2", 62 | "ts-node": "^10.9.2", 63 | "tsconfig-paths": "^4.2.0", 64 | "typescript": "^5.7.3", 65 | "typescript-eslint": "^8.20.0" 66 | }, 67 | "jest": { 68 | "moduleFileExtensions": [ 69 | "js", 70 | "json", 71 | "ts" 72 | ], 73 | "rootDir": "src", 74 | "testRegex": ".*\\.spec\\.ts$", 75 | "transform": { 76 | "^.+\\.(t|j)s$": "ts-jest" 77 | }, 78 | "collectCoverageFrom": [ 79 | "**/*.(t|j)s" 80 | ], 81 | "coverageDirectory": "../coverage", 82 | "testEnvironment": "node" 83 | }, 84 | "engines": { 85 | "node": "20" 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /packages/bytebot-agent/prisma/migrations/20250328022708_initial_migration/migration.sql: -------------------------------------------------------------------------------- 1 | -- CreateEnum 2 | CREATE TYPE "TaskStatus" AS ENUM ('PENDING', 'IN_PROGRESS', 'NEEDS_HELP', 'NEEDS_REVIEW', 'COMPLETED', 'CANCELLED', 'FAILED'); 3 | 4 | -- CreateEnum 5 | CREATE TYPE "TaskPriority" AS ENUM ('LOW', 'MEDIUM', 'HIGH', 'URGENT'); 6 | 7 | -- CreateEnum 8 | CREATE TYPE "MessageType" AS ENUM ('USER', 'ASSISTANT'); 9 | 10 | -- CreateTable 11 | CREATE TABLE "Task" ( 12 | "id" TEXT NOT NULL, 13 | "description" TEXT NOT NULL, 14 | "status" "TaskStatus" NOT NULL DEFAULT 'PENDING', 15 | "priority" "TaskPriority" NOT NULL DEFAULT 'MEDIUM', 16 | "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, 17 | "updatedAt" TIMESTAMP(3) NOT NULL, 18 | 19 | CONSTRAINT "Task_pkey" PRIMARY KEY ("id") 20 | ); 21 | 22 | -- CreateTable 23 | CREATE TABLE "Summary" ( 24 | "id" TEXT NOT NULL, 25 | "content" TEXT NOT NULL, 26 | "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, 27 | "updatedAt" TIMESTAMP(3) NOT NULL, 28 | "taskId" TEXT NOT NULL, 29 | "parentId" TEXT, 30 | 31 | CONSTRAINT "Summary_pkey" PRIMARY KEY ("id") 32 | ); 33 | 34 | -- CreateTable 35 | CREATE TABLE "Message" ( 36 | "id" TEXT NOT NULL, 37 | "content" JSONB NOT NULL, 38 | "type" "MessageType" NOT NULL DEFAULT 'ASSISTANT', 39 | "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, 40 | "updatedAt" TIMESTAMP(3) NOT NULL, 41 | "taskId" TEXT NOT NULL, 42 | "summaryId" TEXT, 43 | 44 | CONSTRAINT "Message_pkey" PRIMARY KEY ("id") 45 | ); 46 | 47 | -- AddForeignKey 48 | ALTER TABLE "Summary" ADD CONSTRAINT "Summary_taskId_fkey" FOREIGN KEY ("taskId") REFERENCES "Task"("id") ON DELETE RESTRICT ON UPDATE CASCADE; 49 | 50 | -- AddForeignKey 51 | ALTER TABLE "Summary" ADD CONSTRAINT "Summary_parentId_fkey" FOREIGN KEY ("parentId") REFERENCES "Summary"("id") ON DELETE SET NULL ON UPDATE CASCADE; 52 | 53 | -- AddForeignKey 54 | ALTER TABLE "Message" ADD CONSTRAINT "Message_taskId_fkey" FOREIGN KEY ("taskId") REFERENCES "Task"("id") ON DELETE RESTRICT ON UPDATE CASCADE; 55 | 56 | -- AddForeignKey 57 | ALTER TABLE "Message" ADD CONSTRAINT "Message_summaryId_fkey" FOREIGN KEY ("summaryId") REFERENCES "Summary"("id") ON DELETE SET NULL ON UPDATE CASCADE; 58 | -------------------------------------------------------------------------------- /packages/bytebot-agent/prisma/migrations/20250413053912_message_role/migration.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Warnings: 3 | 4 | - You are about to drop the column `type` on the `Message` table. All the data in the column will be lost. 5 | 6 | */ 7 | -- CreateEnum 8 | CREATE TYPE "MessageRole" AS ENUM ('USER', 'ASSISTANT'); 9 | 10 | -- AlterTable 11 | ALTER TABLE "Message" DROP COLUMN "type", 12 | ADD COLUMN "role" "MessageRole" NOT NULL DEFAULT 'ASSISTANT'; 13 | 14 | -- DropEnum 15 | DROP TYPE "MessageType"; 16 | -------------------------------------------------------------------------------- /packages/bytebot-agent/prisma/migrations/20250522200556_updated_task_structure/migration.sql: -------------------------------------------------------------------------------- 1 | 2 | -- CreateEnum 3 | CREATE TYPE "Role" AS ENUM ('USER', 'ASSISTANT'); 4 | 5 | -- CreateEnum 6 | CREATE TYPE "TaskType" AS ENUM ('IMMEDIATE', 'SCHEDULED'); 7 | 8 | -- AlterEnum 9 | BEGIN; 10 | CREATE TYPE "TaskStatus_new" AS ENUM ('PENDING', 'RUNNING', 'NEEDS_HELP', 'NEEDS_REVIEW', 'COMPLETED', 'CANCELLED', 'FAILED'); 11 | ALTER TABLE "Task" ALTER COLUMN "status" DROP DEFAULT; 12 | ALTER TABLE "Task" ALTER COLUMN "status" TYPE "TaskStatus_new" USING (CASE "status"::text WHEN 'IN_PROGRESS' THEN 'RUNNING' ELSE "status"::text END::"TaskStatus_new"); 13 | ALTER TYPE "TaskStatus" RENAME TO "TaskStatus_old"; 14 | ALTER TYPE "TaskStatus_new" RENAME TO "TaskStatus"; 15 | DROP TYPE "TaskStatus_old"; 16 | ALTER TABLE "Task" ALTER COLUMN "status" SET DEFAULT 'PENDING'; 17 | COMMIT; 18 | 19 | -- DropForeignKey 20 | ALTER TABLE "Message" DROP CONSTRAINT "Message_taskId_fkey"; 21 | 22 | -- DropForeignKey 23 | ALTER TABLE "Summary" DROP CONSTRAINT "Summary_taskId_fkey"; 24 | 25 | -- AlterTable 26 | ALTER TABLE "Message" ADD COLUMN "new_role" "Role" NOT NULL DEFAULT 'ASSISTANT'; 27 | UPDATE "Message" 28 | SET "new_role" = CASE 29 | WHEN lower("role"::text) = 'user' THEN 'USER'::"Role" 30 | WHEN lower("role"::text) = 'assistant' THEN 'ASSISTANT'::"Role" 31 | ELSE 'ASSISTANT'::"Role" 32 | END; 33 | 34 | -- Step 3: Drop the old 'role' column. 35 | ALTER TABLE "Message" DROP COLUMN "role"; 36 | 37 | -- Step 4: Rename 'new_role' to 'role'. 38 | ALTER TABLE "Message" RENAME COLUMN "new_role" TO "role"; 39 | 40 | -- AlterTable 41 | ALTER TABLE "Task" ADD COLUMN "completedAt" TIMESTAMP(3), 42 | ADD COLUMN "createdBy" "Role" NOT NULL DEFAULT 'USER', 43 | ADD COLUMN "error" TEXT, 44 | ADD COLUMN "executedAt" TIMESTAMP(3), 45 | ADD COLUMN "result" JSONB, 46 | ADD COLUMN "type" "TaskType" NOT NULL DEFAULT 'IMMEDIATE'; 47 | 48 | -- DropEnum 49 | DROP TYPE "MessageRole"; 50 | 51 | -- AddForeignKey 52 | ALTER TABLE "Summary" ADD CONSTRAINT "Summary_taskId_fkey" FOREIGN KEY ("taskId") REFERENCES "Task"("id") ON DELETE CASCADE ON UPDATE CASCADE; 53 | 54 | -- AddForeignKey 55 | ALTER TABLE "Message" ADD CONSTRAINT "Message_taskId_fkey" FOREIGN KEY ("taskId") REFERENCES "Task"("id") ON DELETE CASCADE ON UPDATE CASCADE; 56 | -------------------------------------------------------------------------------- /packages/bytebot-agent/prisma/migrations/20250523162632_add_scheduling/migration.sql: -------------------------------------------------------------------------------- 1 | -- AlterTable 2 | ALTER TABLE "Task" ADD COLUMN "queuedAt" TIMESTAMP(3), 3 | ADD COLUMN "scheduledFor" TIMESTAMP(3); 4 | -------------------------------------------------------------------------------- /packages/bytebot-agent/prisma/migrations/20250529003255_tasks_control/migration.sql: -------------------------------------------------------------------------------- 1 | -- AlterTable 2 | ALTER TABLE "Task" ADD COLUMN "control" "Role" NOT NULL DEFAULT 'USER'; 3 | -------------------------------------------------------------------------------- /packages/bytebot-agent/prisma/migrations/20250530012753_tasks_control/migration.sql: -------------------------------------------------------------------------------- 1 | -- AlterTable 2 | ALTER TABLE "Task" ALTER COLUMN "control" SET DEFAULT 'ASSISTANT'; 3 | -------------------------------------------------------------------------------- /packages/bytebot-agent/prisma/migrations/migration_lock.toml: -------------------------------------------------------------------------------- 1 | # Please do not edit this file manually 2 | # It should be added in your version-control system (e.g., Git) 3 | provider = "postgresql" 4 | -------------------------------------------------------------------------------- /packages/bytebot-agent/prisma/schema.prisma: -------------------------------------------------------------------------------- 1 | // This is your Prisma schema file, 2 | // learn more about it in the docs: https://pris.ly/d/prisma-schema 3 | 4 | // Looking for ways to speed up your queries, or scale easily with your serverless or edge functions? 5 | // Try Prisma Accelerate: https://pris.ly/cli/accelerate-init 6 | 7 | generator client { 8 | provider = "prisma-client-js" 9 | } 10 | 11 | datasource db { 12 | provider = "postgresql" 13 | url = env("DATABASE_URL") 14 | } 15 | 16 | enum TaskStatus { 17 | PENDING 18 | RUNNING 19 | NEEDS_HELP 20 | NEEDS_REVIEW 21 | COMPLETED 22 | CANCELLED 23 | FAILED 24 | } 25 | 26 | enum TaskPriority { 27 | LOW 28 | MEDIUM 29 | HIGH 30 | URGENT 31 | } 32 | 33 | enum Role { 34 | USER 35 | ASSISTANT 36 | } 37 | 38 | enum TaskType { 39 | IMMEDIATE 40 | SCHEDULED 41 | } 42 | 43 | model Task { 44 | id String @id @default(uuid()) 45 | description String 46 | type TaskType @default(IMMEDIATE) 47 | status TaskStatus @default(PENDING) 48 | priority TaskPriority @default(MEDIUM) 49 | control Role @default(ASSISTANT) 50 | createdAt DateTime @default(now()) 51 | createdBy Role @default(USER) 52 | scheduledFor DateTime? 53 | updatedAt DateTime @updatedAt 54 | executedAt DateTime? 55 | completedAt DateTime? 56 | queuedAt DateTime? 57 | error String? 58 | result Json? 59 | messages Message[] 60 | summaries Summary[] 61 | } 62 | 63 | model Summary { 64 | id String @id @default(uuid()) 65 | content String 66 | createdAt DateTime @default(now()) 67 | updatedAt DateTime @updatedAt 68 | messages Message[] // One-to-many relationship: one Summary has many Messages 69 | 70 | task Task @relation(fields: [taskId], references: [id], onDelete: Cascade) 71 | taskId String 72 | 73 | // Self-referential relationship 74 | parentSummary Summary? @relation("SummaryHierarchy", fields: [parentId], references: [id]) 75 | parentId String? 76 | childSummaries Summary[] @relation("SummaryHierarchy") 77 | } 78 | 79 | model Message { 80 | id String @id @default(uuid()) 81 | // Content field follows Anthropic's content blocks structure 82 | // Example: 83 | // [ 84 | // {"type": "text", "text": "Hello world"}, 85 | // {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": "..."}} 86 | // ] 87 | content Json 88 | role Role @default(ASSISTANT) 89 | createdAt DateTime @default(now()) 90 | updatedAt DateTime @updatedAt 91 | task Task @relation(fields: [taskId], references: [id], onDelete: Cascade) 92 | taskId String 93 | summary Summary? @relation(fields: [summaryId], references: [id]) 94 | summaryId String? // Optional foreign key to Summary 95 | } -------------------------------------------------------------------------------- /packages/bytebot-agent/src/agent/agent.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { TasksModule } from '../tasks/tasks.module'; 3 | import { MessagesModule } from '../messages/messages.module'; 4 | import { AnthropicModule } from '../anthropic/anthropic.module'; 5 | import { AgentProcessor } from './agent.processor'; 6 | import { ConfigModule } from '@nestjs/config'; 7 | import { AgentScheduler } from './agent.scheduler'; 8 | 9 | @Module({ 10 | imports: [ConfigModule, TasksModule, MessagesModule, AnthropicModule], 11 | providers: [AgentProcessor, AgentScheduler], 12 | exports: [AgentProcessor], 13 | }) 14 | export class AgentModule {} 15 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/agent/agent.scheduler.ts: -------------------------------------------------------------------------------- 1 | import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; 2 | import { Cron, CronExpression } from '@nestjs/schedule'; 3 | import { TasksService } from '../tasks/tasks.service'; 4 | import { AgentProcessor } from './agent.processor'; 5 | import { TaskStatus } from '@prisma/client'; 6 | 7 | @Injectable() 8 | export class AgentScheduler implements OnModuleInit { 9 | private readonly logger = new Logger(AgentScheduler.name); 10 | 11 | constructor( 12 | private readonly tasksService: TasksService, 13 | private readonly agentProcessor: AgentProcessor, 14 | ) {} 15 | 16 | async onModuleInit() { 17 | this.logger.log('AgentScheduler initialized'); 18 | await this.handleCron(); 19 | } 20 | 21 | @Cron(CronExpression.EVERY_5_SECONDS) 22 | async handleCron() { 23 | const now = new Date(); 24 | const scheduledTasks = await this.tasksService.findScheduledTasks(); 25 | for (const scheduledTask of scheduledTasks) { 26 | if (scheduledTask.scheduledFor && scheduledTask.scheduledFor < now) { 27 | this.logger.debug( 28 | `Task ID: ${scheduledTask.id} is scheduled for ${scheduledTask.scheduledFor}, queuing it`, 29 | ); 30 | await this.tasksService.update(scheduledTask.id, { 31 | queuedAt: now, 32 | }); 33 | } 34 | } 35 | 36 | if (this.agentProcessor.isRunning()) { 37 | return; 38 | } 39 | // Find the highest priority task to execute 40 | const task = await this.tasksService.findNextTask(); 41 | if (task) { 42 | await this.tasksService.update(task.id, { 43 | status: TaskStatus.RUNNING, 44 | executedAt: new Date(), 45 | }); 46 | this.logger.debug(`Processing task ID: ${task.id}`); 47 | await this.agentProcessor.processTask(task.id); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/anthropic/anthropic.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { ConfigModule } from '@nestjs/config'; 3 | import { AnthropicService } from './anthropic.service'; 4 | 5 | @Module({ 6 | imports: [ConfigModule], 7 | providers: [AnthropicService], 8 | exports: [AnthropicService], 9 | }) 10 | export class AnthropicModule {} 11 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/anthropic/anthropic.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable, Logger } from '@nestjs/common'; 2 | import { ConfigService } from '@nestjs/config'; 3 | import Anthropic from '@anthropic-ai/sdk'; 4 | import { 5 | MessageContentBlock, 6 | MessageContentType, 7 | TextContentBlock, 8 | ToolUseContentBlock, 9 | } from '@bytebot/shared'; 10 | import { AGENT_SYSTEM_PROMPT, DEFAULT_MODEL } from './anthropic.constants'; 11 | import { Message, Role } from '@prisma/client'; 12 | import { anthropicTools } from './anthropic.tools'; 13 | 14 | @Injectable() 15 | export class AnthropicService { 16 | private readonly anthropic: Anthropic; 17 | private readonly logger = new Logger(AnthropicService.name); 18 | 19 | constructor(private readonly configService: ConfigService) { 20 | const apiKey = this.configService.get('ANTHROPIC_API_KEY'); 21 | 22 | if (!apiKey) { 23 | this.logger.warn( 24 | 'ANTHROPIC_API_KEY is not set. AnthropicService will not work properly.', 25 | ); 26 | } 27 | 28 | this.anthropic = new Anthropic({ 29 | apiKey: apiKey || 'dummy-key-for-initialization', 30 | }); 31 | } 32 | 33 | /** 34 | * Sends a message to Anthropic Claude and returns the response 35 | * 36 | * @param messages Array of message content blocks representing the conversation 37 | * @param options Additional options for the API call 38 | * @returns The AI response as an array of message content blocks 39 | */ 40 | async sendMessage(messages: Message[]): Promise { 41 | try { 42 | const model = DEFAULT_MODEL; 43 | const maxTokens = 8192; 44 | const system = AGENT_SYSTEM_PROMPT; 45 | 46 | // Convert our message content blocks to Anthropic's expected format 47 | const anthropicMessages = this.formatMessagesForAnthropic(messages); 48 | 49 | // Make the API call 50 | const response = await this.anthropic.beta.messages.create({ 51 | model, 52 | max_tokens: maxTokens, 53 | system, 54 | messages: anthropicMessages, 55 | tools: anthropicTools, 56 | }); 57 | 58 | // Convert Anthropic's response to our message content blocks format 59 | return this.formatAnthropicResponse(response.content); 60 | } catch (error) { 61 | this.logger.error( 62 | `Error sending message to Anthropic: ${error.message}`, 63 | error.stack, 64 | ); 65 | throw error; 66 | } 67 | } 68 | 69 | /** 70 | * Convert our MessageContentBlock format to Anthropic's message format 71 | */ 72 | private formatMessagesForAnthropic( 73 | messages: Message[], 74 | ): Anthropic.MessageParam[] { 75 | const anthropicMessages: Anthropic.MessageParam[] = []; 76 | 77 | // Process each message content block 78 | for (const message of messages) { 79 | const messageContentBlocks = message.content as MessageContentBlock[]; 80 | const content: Anthropic.ContentBlockParam[] = messageContentBlocks.map( 81 | (block) => block as Anthropic.ContentBlockParam, 82 | ); 83 | anthropicMessages.push({ 84 | role: message.role === Role.USER ? 'user' : 'assistant', 85 | content: content, 86 | }); 87 | } 88 | 89 | return anthropicMessages; 90 | } 91 | 92 | /** 93 | * Convert Anthropic's response content to our MessageContentBlock format 94 | */ 95 | private formatAnthropicResponse( 96 | content: Anthropic.ContentBlock[], 97 | ): MessageContentBlock[] { 98 | return content.map((block) => { 99 | switch (block.type) { 100 | case 'text': 101 | return { 102 | type: MessageContentType.Text, 103 | text: block.text, 104 | } as TextContentBlock; 105 | 106 | case 'tool_use': 107 | return { 108 | type: MessageContentType.ToolUse, 109 | id: block.id, 110 | name: block.name, 111 | input: block.input, 112 | } as ToolUseContentBlock; 113 | 114 | default: 115 | this.logger.warn( 116 | `Unknown content block type from Anthropic: ${block.type}`, 117 | ); 118 | return { 119 | type: MessageContentType.Text, 120 | text: JSON.stringify(block), 121 | } as TextContentBlock; 122 | } 123 | }); 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/app.controller.ts: -------------------------------------------------------------------------------- 1 | import { Controller, Get } from '@nestjs/common'; 2 | import { AppService } from './app.service'; 3 | 4 | @Controller() 5 | export class AppController { 6 | constructor(private readonly appService: AppService) {} 7 | 8 | @Get() 9 | getHello(): string { 10 | return this.appService.getHello(); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/app.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { AppController } from './app.controller'; 3 | import { AppService } from './app.service'; 4 | import { AgentModule } from './agent/agent.module'; 5 | import { TasksModule } from './tasks/tasks.module'; 6 | import { MessagesModule } from './messages/messages.module'; 7 | import { AnthropicModule } from './anthropic/anthropic.module'; 8 | import { PrismaModule } from './prisma/prisma.module'; 9 | import { ConfigModule } from '@nestjs/config'; 10 | import { ScheduleModule } from '@nestjs/schedule'; 11 | 12 | @Module({ 13 | imports: [ 14 | ScheduleModule.forRoot(), 15 | ConfigModule.forRoot({ 16 | isGlobal: true, 17 | }), 18 | AgentModule, 19 | TasksModule, 20 | MessagesModule, 21 | AnthropicModule, 22 | PrismaModule, 23 | ], 24 | controllers: [AppController], 25 | providers: [AppService], 26 | }) 27 | export class AppModule {} 28 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/app.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@nestjs/common'; 2 | 3 | @Injectable() 4 | export class AppService { 5 | getHello(): string { 6 | return 'Hello World!'; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/common/utils.ts: -------------------------------------------------------------------------------- 1 | export const getSystemCapabilityString = (): string => { 2 | // Get current date in format like "Wednesday, April 16, 2025" 3 | const today = new Date(); 4 | const options: Intl.DateTimeFormatOptions = { 5 | weekday: 'long', 6 | year: 'numeric', 7 | month: 'long', 8 | day: 'numeric', 9 | }; 10 | const formattedDate = today.toLocaleDateString('en-US', options); 11 | 12 | // Use process.arch to get architecture (similar to platform.machine() in Python) 13 | const architecture = process.arch; // Returns 'arm64', 'x64', etc. 14 | 15 | return ` 16 | * You are utilising an Ubuntu virtual machine using ${architecture} architecture with internet access. 17 | * You can feel free to install Ubuntu applications with your bash tool. Use curl instead of wget. 18 | * To open firefox, please just click on the firefox icon. Note, firefox-esr is what is installed on your system. 19 | * Using bash tool you can start GUI applications, but you need to set export DISPLAY=:0 and use a subshell. For example "(DISPLAY=:0 xterm &)". GUI apps run with bash tool will appear within your desktop environment, but they may take some time to appear. Take a screenshot to confirm it did. 20 | * When using your bash tool with commands that are expected to output very large quantities of text, redirect into a tmp file and use str_replace_editor or \`grep -n -B -A \` to confirm output. 21 | * When viewing a page it can be helpful to zoom out so that you can see everything on the page. Either that, or make sure you scroll down to see everything before deciding something isn't available. 22 | * When using your computer function calls, they take a while to run and send back to you. Where possible/feasible, try to chain multiple of these calls all into one function calls request. 23 | * The current date is ${formattedDate}. 24 | 25 | 26 | 27 | * When using Firefox, if a startup wizard appears, IGNORE IT. Do not even click "skip this step". Instead, click on the address bar where it says "Search or enter address", and enter the appropriate search term or URL there. 28 | * If the item you are looking at is a pdf, if after taking a single screenshot of the pdf it seems that you want to read the entire document instead of trying to continue to read the pdf from your screenshots + navigation, determine the URL, use curl to download the pdf, install and use pdftotext to convert it to a text file, and then read that text file directly. 29 | `; 30 | }; 31 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/main.ts: -------------------------------------------------------------------------------- 1 | import { NestFactory } from '@nestjs/core'; 2 | import { AppModule } from './app.module'; 3 | import { webcrypto } from 'crypto'; 4 | 5 | // Polyfill for crypto global (required by @nestjs/schedule) 6 | if (!globalThis.crypto) { 7 | globalThis.crypto = webcrypto as any; 8 | } 9 | 10 | async function bootstrap() { 11 | console.log('Starting bytebot-agent application...'); 12 | 13 | try { 14 | const app = await NestFactory.create(AppModule); 15 | 16 | // Enable CORS 17 | app.enableCors({ 18 | origin: "*", 19 | methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'], 20 | credentials: true, 21 | }); 22 | 23 | await app.listen(process.env.PORT ?? 9991); 24 | } catch (error) { 25 | console.error('Error starting application:', error); 26 | } 27 | } 28 | bootstrap(); 29 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/messages/messages.module.ts: -------------------------------------------------------------------------------- 1 | import { Module, forwardRef } from '@nestjs/common'; 2 | import { MessagesService } from './messages.service'; 3 | import { PrismaModule } from '../prisma/prisma.module'; 4 | import { TasksModule } from '../tasks/tasks.module'; 5 | 6 | @Module({ 7 | imports: [PrismaModule, forwardRef(() => TasksModule)], 8 | providers: [MessagesService], 9 | exports: [MessagesService], 10 | }) 11 | export class MessagesModule {} 12 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/messages/messages.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable, NotFoundException, Inject, forwardRef } from '@nestjs/common'; 2 | import { PrismaService } from '../prisma/prisma.service'; 3 | import { Message, Role, Prisma } from '@prisma/client'; 4 | import { MessageContentBlock } from '@bytebot/shared'; 5 | import { TasksGateway } from '../tasks/tasks.gateway'; 6 | 7 | @Injectable() 8 | export class MessagesService { 9 | constructor( 10 | private prisma: PrismaService, 11 | @Inject(forwardRef(() => TasksGateway)) 12 | private readonly tasksGateway: TasksGateway, 13 | ) {} 14 | 15 | async create(data: { 16 | content: MessageContentBlock[]; 17 | role: Role; 18 | taskId: string; 19 | }): Promise { 20 | const message = await this.prisma.message.create({ 21 | data: { 22 | content: data.content as Prisma.InputJsonValue, 23 | role: data.role, 24 | taskId: data.taskId, 25 | }, 26 | }); 27 | 28 | this.tasksGateway.emitNewMessage(data.taskId, message); 29 | 30 | return message; 31 | } 32 | 33 | async findAll(taskId: string): Promise { 34 | return this.prisma.message.findMany({ 35 | where: { 36 | taskId, 37 | }, 38 | orderBy: { 39 | createdAt: 'asc', 40 | }, 41 | }); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/prisma/prisma.module.ts: -------------------------------------------------------------------------------- 1 | import { Global, Module } from "@nestjs/common"; 2 | 3 | import { PrismaService } from "./prisma.service"; 4 | 5 | @Global() 6 | @Module({ 7 | providers: [PrismaService], 8 | exports: [PrismaService], 9 | }) 10 | export class PrismaModule {} 11 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/prisma/prisma.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable, OnModuleInit } from "@nestjs/common"; 2 | import { PrismaClient } from "@prisma/client"; 3 | 4 | @Injectable() 5 | export class PrismaService extends PrismaClient implements OnModuleInit { 6 | constructor() { 7 | super(); 8 | } 9 | 10 | async onModuleInit() { 11 | await this.$connect(); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/tasks/dto/create-task.dto.ts: -------------------------------------------------------------------------------- 1 | import { IsDate, IsNotEmpty, IsOptional, IsString } from 'class-validator'; 2 | import { Role, TaskPriority, TaskType } from '@prisma/client'; 3 | 4 | export class CreateTaskDto { 5 | @IsNotEmpty() 6 | @IsString() 7 | description: string; 8 | 9 | @IsOptional() 10 | @IsString() 11 | type?: TaskType; 12 | 13 | @IsOptional() 14 | @IsDate() 15 | scheduledFor?: Date; 16 | 17 | @IsOptional() 18 | @IsString() 19 | priority?: TaskPriority; 20 | 21 | @IsOptional() 22 | @IsString() 23 | createdBy?: Role; 24 | } 25 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/tasks/dto/guide-task.dto.ts: -------------------------------------------------------------------------------- 1 | import { IsNotEmpty, IsString } from 'class-validator'; 2 | 3 | export class GuideTaskDto { 4 | @IsNotEmpty() 5 | @IsString() 6 | message: string; 7 | } 8 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/tasks/dto/update-task.dto.ts: -------------------------------------------------------------------------------- 1 | import { IsEnum, IsOptional } from 'class-validator'; 2 | import { TaskPriority, TaskStatus } from '@prisma/client'; 3 | 4 | export class UpdateTaskDto { 5 | @IsOptional() 6 | @IsEnum(TaskStatus) 7 | status?: TaskStatus; 8 | 9 | @IsOptional() 10 | @IsEnum(TaskPriority) 11 | priority?: TaskPriority; 12 | 13 | @IsOptional() 14 | queuedAt?: Date; 15 | 16 | @IsOptional() 17 | executedAt?: Date; 18 | 19 | @IsOptional() 20 | completedAt?: Date; 21 | } 22 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/tasks/tasks.controller.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Controller, 3 | Get, 4 | Post, 5 | Body, 6 | Param, 7 | Patch, 8 | Delete, 9 | HttpStatus, 10 | HttpCode, 11 | } from '@nestjs/common'; 12 | import { TasksService } from './tasks.service'; 13 | import { CreateTaskDto } from './dto/create-task.dto'; 14 | import { UpdateTaskDto } from './dto/update-task.dto'; 15 | import { Message, Task } from '@prisma/client'; 16 | import { GuideTaskDto } from './dto/guide-task.dto'; 17 | import { MessagesService } from 'src/messages/messages.service'; 18 | 19 | @Controller('tasks') 20 | export class TasksController { 21 | constructor( 22 | private readonly tasksService: TasksService, 23 | private readonly messagesService: MessagesService, 24 | ) {} 25 | 26 | @Post() 27 | @HttpCode(HttpStatus.CREATED) 28 | async create(@Body() createTaskDto: CreateTaskDto): Promise { 29 | return this.tasksService.create(createTaskDto); 30 | } 31 | 32 | @Get() 33 | async findAll(): Promise { 34 | return this.tasksService.findAll(); 35 | } 36 | 37 | @Get(':id') 38 | async findById(@Param('id') id: string): Promise { 39 | return this.tasksService.findById(id); 40 | } 41 | 42 | @Get(':id/messages') 43 | async taskMessages(@Param('id') taskId: string): Promise { 44 | const messages = await this.messagesService.findAll(taskId); 45 | return messages; 46 | } 47 | 48 | @Patch(':id') 49 | async update( 50 | @Param('id') id: string, 51 | @Body() updateTaskDto: UpdateTaskDto, 52 | ): Promise { 53 | return this.tasksService.update(id, updateTaskDto); 54 | } 55 | 56 | @Delete(':id') 57 | @HttpCode(HttpStatus.NO_CONTENT) 58 | async delete(@Param('id') id: string): Promise { 59 | await this.tasksService.delete(id); 60 | } 61 | 62 | @Post(':id/guide') 63 | @HttpCode(HttpStatus.CREATED) 64 | async guideTask( 65 | @Param('id') taskId: string, 66 | @Body() guideTaskDto: GuideTaskDto, 67 | ): Promise { 68 | return this.tasksService.guideTask(taskId, guideTaskDto); 69 | } 70 | 71 | @Post(':id/takeover') 72 | @HttpCode(HttpStatus.OK) 73 | async takeOver(@Param('id') taskId: string): Promise { 74 | return this.tasksService.takeOver(taskId); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /packages/bytebot-agent/src/tasks/tasks.gateway.ts: -------------------------------------------------------------------------------- 1 | import { 2 | WebSocketGateway, 3 | WebSocketServer, 4 | SubscribeMessage, 5 | OnGatewayConnection, 6 | OnGatewayDisconnect, 7 | } from '@nestjs/websockets'; 8 | import { Server, Socket } from 'socket.io'; 9 | import { Injectable } from '@nestjs/common'; 10 | 11 | @Injectable() 12 | @WebSocketGateway({ 13 | cors: { 14 | origin: '*', 15 | methods: ['GET', 'POST'], 16 | }, 17 | }) 18 | export class TasksGateway implements OnGatewayConnection, OnGatewayDisconnect { 19 | @WebSocketServer() 20 | server: Server; 21 | 22 | handleConnection(client: Socket) { 23 | console.log(`Client connected: ${client.id}`); 24 | } 25 | 26 | handleDisconnect(client: Socket) { 27 | console.log(`Client disconnected: ${client.id}`); 28 | } 29 | 30 | @SubscribeMessage('join_task') 31 | handleJoinTask(client: Socket, taskId: string) { 32 | client.join(`task_${taskId}`); 33 | console.log(`Client ${client.id} joined task ${taskId}`); 34 | } 35 | 36 | @SubscribeMessage('leave_task') 37 | handleLeaveTask(client: Socket, taskId: string) { 38 | client.leave(`task_${taskId}`); 39 | console.log(`Client ${client.id} left task ${taskId}`); 40 | } 41 | 42 | emitTaskUpdate(taskId: string, task: any) { 43 | this.server.to(`task_${taskId}`).emit('task_updated', task); 44 | } 45 | 46 | emitNewMessage(taskId: string, message: any) { 47 | this.server.to(`task_${taskId}`).emit('new_message', message); 48 | } 49 | 50 | emitTaskCreated(task: any) { 51 | this.server.emit('task_created', task); 52 | } 53 | 54 | emitTaskDeleted(taskId: string) { 55 | this.server.emit('task_deleted', taskId); 56 | } 57 | } -------------------------------------------------------------------------------- /packages/bytebot-agent/src/tasks/tasks.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { TasksController } from './tasks.controller'; 3 | import { TasksService } from './tasks.service'; 4 | import { TasksGateway } from './tasks.gateway'; 5 | import { PrismaModule } from '../prisma/prisma.module'; 6 | import { MessagesModule } from 'src/messages/messages.module'; 7 | 8 | @Module({ 9 | imports: [PrismaModule, MessagesModule], 10 | controllers: [TasksController], 11 | providers: [TasksService, TasksGateway], 12 | exports: [TasksService, TasksGateway], 13 | }) 14 | export class TasksModule {} 15 | -------------------------------------------------------------------------------- /packages/bytebot-agent/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["node_modules", "test", "dist", "**/*spec.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /packages/bytebot-agent/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "declaration": true, 5 | "removeComments": true, 6 | "emitDecoratorMetadata": true, 7 | "experimentalDecorators": true, 8 | "allowSyntheticDefaultImports": true, 9 | "target": "ES2021", 10 | "sourceMap": true, 11 | "outDir": "./dist", 12 | "baseUrl": "./", 13 | "incremental": true, 14 | "skipLibCheck": true, 15 | "strictNullChecks": true, 16 | "forceConsistentCasingInFileNames": true, 17 | "noImplicitAny": false, 18 | "strictBindCallApply": false, 19 | "noFallthroughCasesInSwitch": false 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /packages/bytebot-ui/.dockerignore: -------------------------------------------------------------------------------- 1 | **/node_modules 2 | **/dist 3 | **/.next 4 | **/.git 5 | **/.vscode 6 | **/.env* 7 | **/npm-debug.log 8 | **/yarn-debug.log 9 | **/yarn-error.log 10 | **/package-lock.json -------------------------------------------------------------------------------- /packages/bytebot-ui/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.* 7 | .yarn/* 8 | !.yarn/patches 9 | !.yarn/plugins 10 | !.yarn/releases 11 | !.yarn/versions 12 | 13 | # testing 14 | /coverage 15 | 16 | # next.js 17 | /.next/ 18 | /out/ 19 | 20 | # production 21 | /build 22 | 23 | # misc 24 | .DS_Store 25 | *.pem 26 | 27 | # debug 28 | npm-debug.log* 29 | yarn-debug.log* 30 | yarn-error.log* 31 | .pnpm-debug.log* 32 | 33 | # env files (can opt-in for committing if needed) 34 | .env 35 | 36 | # vercel 37 | .vercel 38 | 39 | # typescript 40 | *.tsbuildinfo 41 | next-env.d.ts 42 | -------------------------------------------------------------------------------- /packages/bytebot-ui/.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "plugins": ["prettier-plugin-tailwindcss"] 3 | } 4 | -------------------------------------------------------------------------------- /packages/bytebot-ui/Dockerfile: -------------------------------------------------------------------------------- 1 | # Base image 2 | FROM node:20-alpine 3 | 4 | # Declare build arguments 5 | ARG NEXT_PUBLIC_BYTEBOT_AGENT_BASE_URL 6 | ARG NEXT_PUBLIC_BYTEBOT_DESKTOP_VNC_URL 7 | 8 | # Set environment variables for the build process 9 | ENV NEXT_PUBLIC_BYTEBOT_AGENT_BASE_URL=${NEXT_PUBLIC_BYTEBOT_AGENT_BASE_URL} 10 | ENV NEXT_PUBLIC_BYTEBOT_DESKTOP_VNC_URL=${NEXT_PUBLIC_BYTEBOT_DESKTOP_VNC_URL} 11 | 12 | # Create app directory 13 | WORKDIR /app 14 | 15 | # Copy app source 16 | COPY ./shared ./shared 17 | COPY ./bytebot-ui/ ./bytebot-ui 18 | 19 | WORKDIR /app/bytebot-ui 20 | 21 | # Install dependencies 22 | RUN npm install 23 | 24 | RUN npm run build 25 | 26 | # Run the application 27 | CMD ["npm", "run", "start"] 28 | 29 | 30 | -------------------------------------------------------------------------------- /packages/bytebot-ui/components.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://ui.shadcn.com/schema.json", 3 | "style": "new-york", 4 | "rsc": true, 5 | "tsx": true, 6 | "tailwind": { 7 | "config": "", 8 | "css": "src/app/globals.css", 9 | "baseColor": "stone", 10 | "cssVariables": true, 11 | "prefix": "" 12 | }, 13 | "aliases": { 14 | "components": "@/components", 15 | "utils": "@/lib/utils", 16 | "ui": "@/components/ui", 17 | "lib": "@/lib", 18 | "hooks": "@/hooks" 19 | }, 20 | "iconLibrary": "lucide" 21 | } -------------------------------------------------------------------------------- /packages/bytebot-ui/eslint.config.mjs: -------------------------------------------------------------------------------- 1 | import { dirname } from "path"; 2 | import { fileURLToPath } from "url"; 3 | import { FlatCompat } from "@eslint/eslintrc"; 4 | 5 | const __filename = fileURLToPath(import.meta.url); 6 | const __dirname = dirname(__filename); 7 | 8 | const compat = new FlatCompat({ 9 | baseDirectory: __dirname, 10 | }); 11 | 12 | const eslintConfig = [ 13 | ...compat.extends("next/core-web-vitals", "next/typescript"), 14 | ]; 15 | 16 | export default eslintConfig; 17 | -------------------------------------------------------------------------------- /packages/bytebot-ui/next.config.ts: -------------------------------------------------------------------------------- 1 | import type { NextConfig } from "next"; 2 | 3 | const nextConfig: NextConfig = { 4 | /* config options here */ 5 | transpilePackages: ["@bytebot/shared"], 6 | }; 7 | 8 | export default nextConfig; 9 | -------------------------------------------------------------------------------- /packages/bytebot-ui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bytebot-ui", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev -p 9992", 7 | "build": "npm run build --prefix ../shared && next build", 8 | "start": "next start -p 9992", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "@anthropic-ai/sdk": "^0.39.0", 13 | "@bytebot/shared": "../shared", 14 | "@hugeicons/core-free-icons": "^1.0.14", 15 | "@hugeicons/react": "^1.0.5", 16 | "@prisma/client": "^6.5.0", 17 | "@radix-ui/react-dropdown-menu": "^2.1.6", 18 | "@radix-ui/react-popover": "^1.1.11", 19 | "@radix-ui/react-scroll-area": "^1.2.3", 20 | "@radix-ui/react-select": "^2.2.2", 21 | "@radix-ui/react-separator": "^1.1.2", 22 | "@radix-ui/react-slot": "^1.1.2", 23 | "@radix-ui/react-switch": "^1.1.3", 24 | "@types/express": "^5.0.1", 25 | "class-variance-authority": "^0.7.1", 26 | "clsx": "^2.1.1", 27 | "date-fns": "^4.1.0", 28 | "dotenv": "^16.4.7", 29 | "express": "^4.21.2", 30 | "motion": "^12.12.1", 31 | "next": ">=15.2.4", 32 | "next-themes": "^0.4.6", 33 | "next-transpile-modules": "^10.0.1", 34 | "react": "^19.0.0", 35 | "react-dom": "^19.0.0", 36 | "react-markdown": "^10.1.0", 37 | "react-vnc": "^3.1.0", 38 | "socket.io-client": "^4.8.1", 39 | "tailwind-merge": "^3.0.2", 40 | "tsx": "^4.19.3", 41 | "tw-animate-css": "^1.2.4", 42 | "zod": "^3.24.2" 43 | }, 44 | "devDependencies": { 45 | "@eslint/eslintrc": "^3", 46 | "@tailwindcss/postcss": "^4.1.3", 47 | "@types/node": "^20.17.27", 48 | "@types/react": "^19", 49 | "@types/react-dom": "^19", 50 | "eslint": "^9", 51 | "eslint-config-next": "15.2.3", 52 | "prettier": "^3.5.3", 53 | "prettier-plugin-tailwindcss": "^0.6.11", 54 | "prisma": "^6.5.0", 55 | "tailwindcss": "^4", 56 | "typescript": "^5" 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /packages/bytebot-ui/postcss.config.mjs: -------------------------------------------------------------------------------- 1 | const config = { 2 | plugins: ["@tailwindcss/postcss"], 3 | }; 4 | 5 | export default config; 6 | -------------------------------------------------------------------------------- /packages/bytebot-ui/public/bytebot_square_light.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /packages/bytebot-ui/public/bytebot_transparent_logo_dark.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /packages/bytebot-ui/public/bytebot_transparent_logo_white.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /packages/bytebot-ui/public/stock-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytebot-ai/bytebot/afb98c7a2021aa95b5feac5715a40567dbb32e91/packages/bytebot-ui/public/stock-1.png -------------------------------------------------------------------------------- /packages/bytebot-ui/src/app/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytebot-ai/bytebot/afb98c7a2021aa95b5feac5715a40567dbb32e91/packages/bytebot-ui/src/app/favicon.ico -------------------------------------------------------------------------------- /packages/bytebot-ui/src/app/layout.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react" 2 | import type { Metadata } from "next" 3 | import { Inter } from "next/font/google" 4 | import "./globals.css" 5 | 6 | const inter = Inter({ subsets: ["latin"] }) 7 | 8 | export const metadata: Metadata = { 9 | title: "Bytebot", 10 | description: "Bytebot is the container for desktop agents.", 11 | } 12 | 13 | export default function RootLayout({ 14 | children, 15 | }: Readonly<{ 16 | children: React.ReactNode 17 | }>) { 18 | return ( 19 | 20 | 21 | {children} 22 | 23 | 24 | ) 25 | } 26 | 27 | -------------------------------------------------------------------------------- /packages/bytebot-ui/src/app/tasks/page.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import React, { useEffect, useState } from "react"; 4 | import { Header } from "@/components/layout/Header"; 5 | import { TaskItem } from "@/components/tasks/TaskItem"; 6 | import { fetchTasks } from "@/utils/taskUtils"; 7 | import { Task } from "@/types"; 8 | import { Button } from "@/components/ui/button"; 9 | import Link from "next/link"; 10 | 11 | export default function Tasks() { 12 | const [tasks, setTasks] = useState([]); 13 | const [isLoading, setIsLoading] = useState(true); 14 | 15 | useEffect(() => { 16 | const loadTasks = async () => { 17 | setIsLoading(true); 18 | try { 19 | const fetchedTasks = await fetchTasks(); 20 | setTasks(fetchedTasks); 21 | } catch (error) { 22 | console.error("Failed to load tasks:", error); 23 | } finally { 24 | setIsLoading(false); 25 | } 26 | }; 27 | 28 | loadTasks(); 29 | }, []); 30 | 31 | return ( 32 |
33 |
34 | 35 |
36 |
37 |

Tasks

38 | 39 | {isLoading ? ( 40 |
41 |
42 |

Loading tasks...

43 |
44 | ) : tasks.length === 0 ? ( 45 |
46 |
47 |

48 | No tasks yet 49 |

50 |

51 | Get started by creating a first task 52 |

53 | 54 | 57 | 58 |
59 |
60 | ) : ( 61 |
62 | {tasks.map((task) => ( 63 | 64 | ))} 65 |
66 | )} 67 |
68 |
69 |
70 | ); 71 | } 72 | -------------------------------------------------------------------------------- /packages/bytebot-ui/src/components/layout/BrowserHeader.tsx: -------------------------------------------------------------------------------- 1 | export const BrowserHeader: React.FC = () => { 2 | return ( 3 |
4 |
5 |
6 |
7 | 8 | 12 |
13 |
14 | ); 15 | }; 16 | -------------------------------------------------------------------------------- /packages/bytebot-ui/src/components/layout/Header.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from "react"; 2 | import Link from "next/link"; 3 | import Image from "next/image"; 4 | import { useTheme } from "next-themes"; 5 | import { HugeiconsIcon } from '@hugeicons/react' 6 | import { DocumentCodeIcon, TaskDaily01Icon, Home01Icon } from '@hugeicons/core-free-icons' 7 | import { usePathname } from 'next/navigation'; 8 | 9 | // Uncommenting interface if needed in the future 10 | // interface HeaderProps { 11 | // currentTaskId?: string | null; 12 | // onNewConversation?: () => void; 13 | // } 14 | 15 | export function Header() { 16 | const { resolvedTheme } = useTheme(); 17 | const [mounted, setMounted] = useState(false); 18 | const pathname = usePathname(); 19 | 20 | // After mounting, we can safely show the theme-dependent content 21 | useEffect(() => { 22 | setMounted(true); 23 | }, []); 24 | 25 | // Function to determine if a link is active 26 | const isActive = (path: string) => { 27 | if (path === '/') { 28 | return pathname === '/'; 29 | } 30 | return pathname?.startsWith(path); 31 | }; 32 | 33 | // Get classes for navigation links based on active state 34 | const getLinkClasses = (path: string) => { 35 | const baseClasses = "flex items-center gap-1.5 transition-colors px-3 py-1.5 rounded-lg"; 36 | const activeClasses = "bg-bytebot-bronze-light-a3 text-bytebot-bronze-light-12"; 37 | const inactiveClasses = "text-bytebot-bronze-dark-9 hover:bg-bytebot-bronze-light-a1 hover:text-bytebot-bronze-light-12"; 38 | 39 | return `${baseClasses} ${isActive(path) ? activeClasses : inactiveClasses}`; 40 | }; 41 | 42 | return ( 43 |
44 |
45 | {/* Logo without link */} 46 |
47 | {mounted ? ( 48 | Bytebot Logo 59 | ) : ( 60 |
61 | )} 62 |
63 |
64 |
65 | 66 | 67 | Home 68 | 69 | 70 | 71 | Tasks 72 | 73 | 74 | 75 | Docs 76 | 77 |
78 |
79 |
80 |
81 | ); 82 | } 83 | -------------------------------------------------------------------------------- /packages/bytebot-ui/src/components/messages/ChatContainer.tsx: -------------------------------------------------------------------------------- 1 | import React, { useRef, useEffect } from "react"; 2 | import { Message, Role, TaskStatus } from "@/types"; 3 | import { MessageGroup } from "./MessageGroup"; 4 | import { isToolResultContentBlock } from "@bytebot/shared"; 5 | import { TextShimmer } from "../ui/text-shimmer"; 6 | 7 | interface ChatContainerProps { 8 | taskStatus: TaskStatus; 9 | control: Role; 10 | messages: Message[]; 11 | isLoadingSession: boolean; 12 | scrollRef?: React.RefObject; 13 | } 14 | 15 | export interface GroupedMessages { 16 | role: Role; 17 | messages: Message[]; 18 | } 19 | 20 | /** 21 | * Groups back-to-back messages from the same role in a conversation JSON 22 | * 23 | * @param {Object} conversation - The conversation JSON object with messages array 24 | * @returns {Object} A new conversation object with grouped messages 25 | */ 26 | function groupBackToBackMessages(messages: Message[]): GroupedMessages[] { 27 | const groupedConversation: GroupedMessages[] = []; 28 | 29 | let currentGroup: GroupedMessages | null = null; 30 | 31 | for (const message of messages) { 32 | const role = message.role; 33 | 34 | // If this is the first message or role is different from the previous group 35 | if (!currentGroup || currentGroup.role !== role) { 36 | // Save the previous group if it exists 37 | if (currentGroup) { 38 | groupedConversation.push(currentGroup); 39 | } 40 | 41 | // Start a new group 42 | currentGroup = { 43 | role: role, 44 | messages: [message], 45 | }; 46 | } else { 47 | // Same role as previous, merge the content 48 | currentGroup.messages.push(message); 49 | } 50 | } 51 | 52 | // Add the last group 53 | if (currentGroup) { 54 | groupedConversation.push(currentGroup); 55 | } 56 | 57 | return groupedConversation; 58 | } 59 | 60 | function filterMessages(messages: Message[]): Message[] { 61 | const filteredMessages: Message[] = []; 62 | for (const message of messages) { 63 | const contentBlocks = message.content; 64 | 65 | // If the role is a user message and all the content blocks are tool result blocks 66 | if ( 67 | message.role === Role.USER && 68 | contentBlocks.every((block) => isToolResultContentBlock(block)) 69 | ) { 70 | message.role = Role.ASSISTANT; 71 | } 72 | 73 | filteredMessages.push(message); 74 | } 75 | return filteredMessages; 76 | } 77 | 78 | export function ChatContainer({ 79 | taskStatus, 80 | control, 81 | messages, 82 | isLoadingSession, 83 | scrollRef, 84 | }: ChatContainerProps) { 85 | const messagesEndRef = useRef(null); 86 | // Group back-to-back messages from the same role 87 | const groupedConversation = groupBackToBackMessages(filterMessages(messages)); 88 | 89 | // This effect runs whenever the messages array changes 90 | useEffect(() => { 91 | if ( 92 | taskStatus === TaskStatus.RUNNING || 93 | taskStatus === TaskStatus.NEEDS_HELP 94 | ) { 95 | scrollToBottom(); 96 | } 97 | }, [taskStatus, messages]); 98 | 99 | // Function to scroll to the bottom of the messages 100 | const scrollToBottom = () => { 101 | messagesEndRef.current?.scrollIntoView({ behavior: "smooth" }); 102 | }; 103 | 104 | return ( 105 |
106 | {isLoadingSession ? ( 107 |
108 |
109 |
110 | ) : messages.length > 0 ? ( 111 | <> 112 | {groupedConversation.map((group, groupIndex) => ( 113 |
114 | 115 |
116 | ))} 117 | {taskStatus === TaskStatus.RUNNING && control === Role.ASSISTANT && ( 118 | 119 | Bytebot is working... 120 | 121 | )} 122 | 123 | ) : ( 124 |
125 |

No messages yet...

126 |
127 | )} 128 | {/* This empty div is the target for scrolling */} 129 |
130 |
131 | ); 132 | } 133 | -------------------------------------------------------------------------------- /packages/bytebot-ui/src/components/messages/ChatInput.tsx: -------------------------------------------------------------------------------- 1 | import React, { useRef, useEffect } from 'react'; 2 | import { Button } from '@/components/ui/button'; 3 | import { HugeiconsIcon } from '@hugeicons/react' 4 | import { ArrowRight02Icon } from '@hugeicons/core-free-icons' 5 | import { cn } from '@/lib/utils'; 6 | 7 | interface ChatInputProps { 8 | input: string; 9 | isLoading: boolean; 10 | onInputChange: (value: string) => void; 11 | onSend: () => void; 12 | minLines?: number; 13 | placeholder?: string; 14 | } 15 | 16 | export function ChatInput({ input, isLoading, onInputChange, onSend, minLines = 1, placeholder = "Ask me to do something..." }: ChatInputProps) { 17 | const textareaRef = useRef(null); 18 | 19 | const handleSubmit = (e: React.FormEvent) => { 20 | e.preventDefault(); 21 | onSend(); 22 | }; 23 | 24 | // Auto-resize textarea based on content 25 | useEffect(() => { 26 | const textarea = textareaRef.current; 27 | if (!textarea) return; 28 | 29 | // Reset height to auto to get the correct scrollHeight 30 | textarea.style.height = 'auto'; 31 | 32 | // Calculate minimum height based on minLines 33 | const lineHeight = 24; // approximate line height in pixels 34 | const minHeight = lineHeight * minLines + 12; 35 | 36 | // Set height to scrollHeight or minHeight, whichever is larger 37 | const newHeight = Math.max(textarea.scrollHeight, minHeight); 38 | textarea.style.height = `${newHeight}px`; 39 | }, [input, minLines]); 40 | 41 | // Determine button position based on minLines 42 | const buttonPositionClass = minLines > 1 ? "bottom-1.5" : "top-1/2 -translate-y-1/2"; 43 | 44 | return ( 45 |
46 |
47 |