├── docs ├── assets │ ├── favicon.ico │ ├── favicon-16x16.png │ ├── favicon-32x32.png │ ├── apple-touch-icon.png │ ├── android-chrome-192x192.png │ ├── android-chrome-512x512.png │ ├── glyphicons-halflings-regular.eot │ ├── glyphicons-halflings-regular.ttf │ ├── glyphicons-halflings-regular.woff │ ├── glyphicons-halflings-regular.woff2 │ ├── prism-diff-highlight.css │ ├── prism-toolbar.css │ ├── prism.css │ └── main.css └── index.html ├── .env.example ├── apidoc.json ├── run.sh ├── package.json ├── todos.md ├── .gitignore ├── LICENSE ├── index.js ├── services ├── browserbase-service.js ├── session-expiry-service.js └── stagehand-service.js ├── README.md ├── routes ├── session-routes.js └── browser-routes.js └── streamlit_ui.py /docs/assets/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kortix-ai/wbac/HEAD/docs/assets/favicon.ico -------------------------------------------------------------------------------- /docs/assets/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kortix-ai/wbac/HEAD/docs/assets/favicon-16x16.png -------------------------------------------------------------------------------- /docs/assets/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kortix-ai/wbac/HEAD/docs/assets/favicon-32x32.png -------------------------------------------------------------------------------- /docs/assets/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kortix-ai/wbac/HEAD/docs/assets/apple-touch-icon.png -------------------------------------------------------------------------------- /docs/assets/android-chrome-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kortix-ai/wbac/HEAD/docs/assets/android-chrome-192x192.png -------------------------------------------------------------------------------- /docs/assets/android-chrome-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kortix-ai/wbac/HEAD/docs/assets/android-chrome-512x512.png -------------------------------------------------------------------------------- /docs/assets/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kortix-ai/wbac/HEAD/docs/assets/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /docs/assets/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kortix-ai/wbac/HEAD/docs/assets/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /docs/assets/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kortix-ai/wbac/HEAD/docs/assets/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /docs/assets/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kortix-ai/wbac/HEAD/docs/assets/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | BROWSERBASE_API_KEY=your_api_key 2 | BROWSERBASE_PROJECT_ID=your_project_id 3 | OPENAI_API_KEY=your_openai_key 4 | ANTHROPIC_API_KEY=your_anthropic_key 5 | -------------------------------------------------------------------------------- /apidoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Web Browser AI-Control API", 3 | "version": "1.0.0", 4 | "description": "API documentation for AI-powered browser control", 5 | "title": "Browser Control API Documentation", 6 | "url": "http://localhost:3000" 7 | } -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Start the server and UI 4 | echo "Starting Stagehand server..." 5 | npm run dev & 6 | SERVER_PID=$! 7 | 8 | echo "Starting Streamlit UI..." 9 | streamlit run streamlit_ui.py 10 | 11 | # Cleanup on exit 12 | trap "kill $SERVER_PID" EXIT -------------------------------------------------------------------------------- /docs/assets/prism-diff-highlight.css: -------------------------------------------------------------------------------- 1 | pre.diff-highlight > code .token.deleted:not(.prefix), 2 | pre > code.diff-highlight .token.deleted:not(.prefix) { 3 | background-color: rgba(255, 0, 0, .1); 4 | color: inherit; 5 | display: block; 6 | } 7 | 8 | pre.diff-highlight > code .token.inserted:not(.prefix), 9 | pre > code.diff-highlight .token.inserted:not(.prefix) { 10 | background-color: rgba(0, 255, 128, .1); 11 | color: inherit; 12 | display: block; 13 | } 14 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "web-browser-ai-control", 3 | "version": "1.0.0", 4 | "scripts": { 5 | "start": "node index.js", 6 | "dev": "nodemon index.js", 7 | "generate-docs": "apidoc -i routes/ -o docs/" 8 | }, 9 | "dependencies": { 10 | "@browserbasehq/sdk": "^2.0.0", 11 | "@browserbasehq/stagehand": "^1.1.1", 12 | "cors": "^2.8.5", 13 | "express": "^4.21.1", 14 | "nodemon": "^3.1.7", 15 | "playwright-core": "^1.48.2", 16 | "zod": "^3.23.8" 17 | }, 18 | "devDependencies": { 19 | "apidoc": "^1.2.0" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /todos.md: -------------------------------------------------------------------------------- 1 | 2 | [] execute playwright code directly endpoints 3 | 4 | [] browserbase api key & project id as params to the API 5 | 6 | [] Make error messages in API responses actually work – Error in console is like that – but not returned in API response. 7 | [stagehand:Init] Failed to resume session bca19dd8-b80e-4827-9388-858a183eefb5: Session bca19dd8-b80e-4827-9388-858a183eefb5 is not running (status: COMPLETED) 8 | Error in init: Error: Session bca19dd8-b80e-4827-9388-858a183eefb5 is not running (status: COMPLETED) 9 | at /Users/markokraemer/Projects/ap-stagehand/stagehand/dist/index.js:1344:19 10 | at Generator.next () 11 | at fulfilled (/Users/markokraemer/Projects/ap-stagehand/stagehand/dist/index.js:61:24) 12 | at process.processTicksAndRejections (node:internal/process/task_queues:105:5) -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | node_modules/ 3 | package-lock.json 4 | yarn.lock 5 | 6 | # Environment variables 7 | .env 8 | .env.local 9 | .env.*.local 10 | 11 | # Logs 12 | logs/ 13 | *.log 14 | npm-debug.log* 15 | yarn-debug.log* 16 | yarn-error.log* 17 | 18 | # Runtime data 19 | pids/ 20 | *.pid 21 | *.seed 22 | *.pid.lock 23 | 24 | # Testing 25 | coverage/ 26 | .nyc_output/ 27 | 28 | # Production build 29 | dist/ 30 | build/ 31 | out/ 32 | 33 | # IDE and editor files 34 | .idea/ 35 | .vscode/ 36 | *.swp 37 | *.swo 38 | .DS_Store 39 | Thumbs.db 40 | 41 | # Optional npm cache directory 42 | .npm 43 | 44 | # Optional eslint cache 45 | .eslintcache 46 | 47 | # Optional REPL history 48 | .node_repl_history 49 | 50 | # Debug files 51 | .debug/ 52 | debug.log 53 | 54 | # Temporary files 55 | tmp/ 56 | temp/ 57 | 58 | # Stagehand repo 59 | # stagehand/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Kortix AI Corp 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const express = require('express'); 2 | const cors = require('cors'); 3 | const sessionRoutes = require('./routes/session-routes'); 4 | const browserRoutes = require('./routes/browser-routes'); 5 | const sessionExpiryManager = require('./services/session-expiry-service'); 6 | 7 | const app = express(); 8 | app.use(express.json()); 9 | app.use(cors({ 10 | origin: /^http:\/\/localhost(:[0-9]+)?$/, 11 | optionsSuccessStatus: 200 12 | })); 13 | 14 | // Routes 15 | app.use('/api/sessions', sessionRoutes); 16 | app.use('/api/browser', browserRoutes); 17 | 18 | // Health check 19 | app.get('/health', (req, res) => { 20 | res.json({ status: 'ok' }); 21 | }); 22 | 23 | const PORT = process.env.PORT || 3000; 24 | app.listen(PORT, () => { 25 | console.log(`Server running on port ${PORT}`); 26 | }); 27 | 28 | // Handle shutdown 29 | process.on('SIGINT', async () => { 30 | try { 31 | console.log('\nShutting down server...'); 32 | 33 | // Stop all sessions 34 | await sessionExpiryManager.stopAllSessions(); 35 | 36 | console.log('Cleanup complete. Exiting...'); 37 | process.exit(0); 38 | } catch (error) { 39 | console.error('Error during shutdown:', error); 40 | process.exit(1); 41 | } 42 | }); -------------------------------------------------------------------------------- /docs/assets/prism-toolbar.css: -------------------------------------------------------------------------------- 1 | div.code-toolbar { 2 | position: relative; 3 | } 4 | 5 | div.code-toolbar > .toolbar { 6 | position: absolute; 7 | z-index: 10; 8 | top: .3em; 9 | right: .2em; 10 | transition: opacity 0.3s ease-in-out; 11 | opacity: 0; 12 | } 13 | 14 | div.code-toolbar:hover > .toolbar { 15 | opacity: 1; 16 | } 17 | 18 | /* Separate line b/c rules are thrown out if selector is invalid. 19 | IE11 and old Edge versions don't support :focus-within. */ 20 | div.code-toolbar:focus-within > .toolbar { 21 | opacity: 1; 22 | } 23 | 24 | div.code-toolbar > .toolbar > .toolbar-item { 25 | display: inline-block; 26 | } 27 | 28 | div.code-toolbar > .toolbar > .toolbar-item > a { 29 | cursor: pointer; 30 | } 31 | 32 | div.code-toolbar > .toolbar > .toolbar-item > button { 33 | background: none; 34 | border: 0; 35 | color: inherit; 36 | font: inherit; 37 | line-height: normal; 38 | overflow: visible; 39 | padding: 0; 40 | -webkit-user-select: none; /* for button */ 41 | -moz-user-select: none; 42 | -ms-user-select: none; 43 | } 44 | 45 | div.code-toolbar > .toolbar > .toolbar-item > a, 46 | div.code-toolbar > .toolbar > .toolbar-item > button, 47 | div.code-toolbar > .toolbar > .toolbar-item > span { 48 | color: #bbb; 49 | font-size: .8em; 50 | padding: 0 .5em; 51 | background: #f5f2f0; 52 | background: rgba(224, 224, 224, 0.2); 53 | box-shadow: 0 2px 0 0 rgba(0,0,0,0.2); 54 | border-radius: .5em; 55 | } 56 | 57 | div.code-toolbar > .toolbar > .toolbar-item > a:hover, 58 | div.code-toolbar > .toolbar > .toolbar-item > a:focus, 59 | div.code-toolbar > .toolbar > .toolbar-item > button:hover, 60 | div.code-toolbar > .toolbar > .toolbar-item > button:focus, 61 | div.code-toolbar > .toolbar > .toolbar-item > span:hover, 62 | div.code-toolbar > .toolbar > .toolbar-item > span:focus { 63 | color: inherit; 64 | text-decoration: none; 65 | } 66 | -------------------------------------------------------------------------------- /docs/assets/prism.css: -------------------------------------------------------------------------------- 1 | /** 2 | * prism.js tomorrow night eighties for JavaScript, CoffeeScript, CSS and HTML 3 | * Based on https://github.com/chriskempson/tomorrow-theme 4 | * @author Rose Pritchard 5 | */ 6 | 7 | code[class*="language-"], 8 | pre[class*="language-"] { 9 | color: #ccc; 10 | background: none; 11 | font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; 12 | font-size: 1em; 13 | text-align: left; 14 | white-space: pre; 15 | word-spacing: normal; 16 | word-break: normal; 17 | word-wrap: normal; 18 | line-height: 1.5; 19 | 20 | -moz-tab-size: 4; 21 | -o-tab-size: 4; 22 | tab-size: 4; 23 | 24 | -webkit-hyphens: none; 25 | -moz-hyphens: none; 26 | -ms-hyphens: none; 27 | hyphens: none; 28 | 29 | } 30 | 31 | /* Code blocks */ 32 | pre[class*="language-"] { 33 | padding: 1em; 34 | margin: .5em 0; 35 | overflow: auto; 36 | } 37 | 38 | :not(pre) > code[class*="language-"], 39 | pre[class*="language-"] { 40 | background: #2d2d2d; 41 | } 42 | 43 | /* Inline code */ 44 | :not(pre) > code[class*="language-"] { 45 | padding: .1em; 46 | border-radius: .3em; 47 | white-space: normal; 48 | } 49 | 50 | .token.comment, 51 | .token.block-comment, 52 | .token.prolog, 53 | .token.doctype, 54 | .token.cdata { 55 | color: #999; 56 | } 57 | 58 | .token.punctuation { 59 | color: #ccc; 60 | } 61 | 62 | .token.tag, 63 | .token.attr-name, 64 | .token.namespace, 65 | .token.deleted { 66 | color: #e2777a; 67 | } 68 | 69 | .token.function-name { 70 | color: #6196cc; 71 | } 72 | 73 | .token.boolean, 74 | .token.number, 75 | .token.function { 76 | color: #f08d49; 77 | } 78 | 79 | .token.property, 80 | .token.class-name, 81 | .token.constant, 82 | .token.symbol { 83 | color: #f8c555; 84 | } 85 | 86 | .token.selector, 87 | .token.important, 88 | .token.atrule, 89 | .token.keyword, 90 | .token.builtin { 91 | color: #cc99cd; 92 | } 93 | 94 | .token.string, 95 | .token.char, 96 | .token.attr-value, 97 | .token.regex, 98 | .token.variable { 99 | color: #7ec699; 100 | } 101 | 102 | .token.operator, 103 | .token.entity, 104 | .token.url { 105 | color: #67cdcc; 106 | } 107 | 108 | .token.important, 109 | .token.bold { 110 | font-weight: bold; 111 | } 112 | .token.italic { 113 | font-style: italic; 114 | } 115 | 116 | .token.entity { 117 | cursor: help; 118 | } 119 | 120 | .token.inserted { 121 | color: green; 122 | } 123 | -------------------------------------------------------------------------------- /services/browserbase-service.js: -------------------------------------------------------------------------------- 1 | const Browserbase = require('@browserbasehq/sdk'); 2 | 3 | class BrowserbaseService { 4 | constructor() { 5 | this.apiKey = process.env.BROWSERBASE_API_KEY; 6 | this.projectId = process.env.BROWSERBASE_PROJECT_ID; 7 | 8 | if (!this.apiKey || !this.projectId) { 9 | throw new Error('BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID environment variables are required'); 10 | } 11 | 12 | this.client = new Browserbase({ apiKey: this.apiKey }); 13 | } 14 | 15 | async listRunningSessions() { 16 | try { 17 | const sessions = await this.client.sessions.list({ 18 | status: 'RUNNING', 19 | projectId: this.projectId 20 | }); 21 | return sessions; 22 | } catch (error) { 23 | console.error('Error listing Browserbase sessions:', error); 24 | throw error; 25 | } 26 | } 27 | 28 | async createSession() { 29 | try { 30 | const session = await this.client.sessions.create({ 31 | projectId: this.projectId, 32 | keepAlive: true, 33 | timeout: 10800 // 3 hours in seconds timeout 34 | }); 35 | return session; 36 | } catch (error) { 37 | console.error('Error creating Browserbase session:', error); 38 | throw error; 39 | } 40 | } 41 | 42 | async stopSession(sessionId) { 43 | try { 44 | await this.client.sessions.update(sessionId, { 45 | status: "REQUEST_RELEASE", 46 | projectId: this.projectId 47 | }); 48 | return true; 49 | } catch (error) { 50 | console.error('Error stopping Browserbase session:', error); 51 | throw error; 52 | } 53 | } 54 | 55 | async getSession(sessionId) { 56 | try { 57 | const session = await this.client.sessions.retrieve(sessionId); 58 | return session; 59 | } catch (error) { 60 | console.error('Error retrieving Browserbase session:', error); 61 | throw error; 62 | } 63 | } 64 | 65 | async getSessionDebugUrls(sessionId) { 66 | try { 67 | const debugUrls = await this.client.sessions.debug(sessionId); 68 | return debugUrls; 69 | } catch (error) { 70 | console.error('Error retrieving session debug URLs:', error); 71 | throw error; 72 | } 73 | } 74 | } 75 | 76 | module.exports = new BrowserbaseService(); -------------------------------------------------------------------------------- /services/session-expiry-service.js: -------------------------------------------------------------------------------- 1 | const browserbaseService = require('./browserbase-service'); 2 | const stagehandService = require('./stagehand-service'); 3 | 4 | class SessionExpiryManager { 5 | constructor() { 6 | this.expiryTasks = new Map(); // sessionId -> timeout handle 7 | this.EXPIRY_TIMEOUT = 10 * 60 * 1000; // 10 minutes in milliseconds 8 | } 9 | 10 | async trackSession(sessionId) { 11 | try { 12 | // Cancel existing expiry task if any 13 | this.resetExpiry(sessionId); 14 | 15 | // Verify session is running 16 | const session = await browserbaseService.getSession(sessionId); 17 | if (!session || session.status !== 'RUNNING') { 18 | console.warn(`Session ${sessionId} is not running (status: ${session?.status})`); 19 | return false; 20 | } 21 | 22 | return true; 23 | } catch (error) { 24 | console.error(`Error tracking session ${sessionId}:`, error); 25 | return false; 26 | } 27 | } 28 | 29 | resetExpiry(sessionId) { 30 | // Clear existing timeout if any 31 | if (this.expiryTasks.has(sessionId)) { 32 | clearTimeout(this.expiryTasks.get(sessionId)); 33 | } 34 | 35 | // Set new expiry timeout 36 | const timeoutHandle = setTimeout(async () => { 37 | await this.stopSession(sessionId); 38 | }, this.EXPIRY_TIMEOUT); 39 | 40 | this.expiryTasks.set(sessionId, timeoutHandle); 41 | } 42 | 43 | async stopSession(sessionId) { 44 | try { 45 | console.log(`Session ${sessionId} expired, stopping...`); 46 | 47 | // Stop the session 48 | await browserbaseService.stopSession(sessionId); 49 | await stagehandService.cleanupInstance(sessionId); 50 | 51 | // Clean up expiry task 52 | if (this.expiryTasks.has(sessionId)) { 53 | clearTimeout(this.expiryTasks.get(sessionId)); 54 | this.expiryTasks.delete(sessionId); 55 | } 56 | 57 | console.log(`Session ${sessionId} stopped successfully`); 58 | } catch (error) { 59 | console.error(`Error stopping session ${sessionId}:`, error); 60 | } 61 | } 62 | 63 | async stopAllSessions() { 64 | try { 65 | const sessions = await browserbaseService.listRunningSessions(); 66 | 67 | if (sessions && sessions.length > 0) { 68 | console.log(`Stopping ${sessions.length} expired sessions...`); 69 | 70 | await Promise.all(sessions.map(session => this.stopSession(session.id))); 71 | } 72 | } catch (error) { 73 | console.error('Error stopping all sessions:', error); 74 | } 75 | } 76 | } 77 | 78 | // Create singleton instance 79 | const sessionExpiryManager = new SessionExpiryManager(); 80 | module.exports = sessionExpiryManager; -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Web Browser AI-Control API Server 2 | 3 | A hosted API server for AI agents to control web browsers in the cloud using [Browserbase](https://www.browserbase.com/), [Stagehand](https://github.com/browserbase/stagehand), and [Playwright](https://playwright.dev/). 4 | 5 | ## Overview 6 | 7 | This server provides a REST API for AI agents to: 8 | - Create and manage browser sessions in the cloud 9 | - Control browsers using natural language commands 10 | - Extract structured data from web pages 11 | - Monitor browser activity and errors 12 | - Take screenshots and inspect DOM state 13 | - Execute custom Playwright automation scripts 14 | 15 | Built on top of: 16 | - [Browserbase](https://www.browserbase.com/) for cloud browser infrastructure 17 | - [Stagehand](https://github.com/browserbase/stagehand) for AI-powered browser control 18 | - [Playwright](https://playwright.dev/) for low-level browser automation 19 | - Express.js for the API server 20 | 21 | ## API Reference 22 | 23 | For complete API documentation, visit [wbac-api-docs.netlify.app](https://wbac-api-docs.netlify.app/) 24 | 25 | ### Session Management 26 | 27 | - `POST /api/sessions/create-session` - Create new browser session 28 | - `POST /api/sessions/stop-session/:sessionId` - Stop session 29 | - `GET /api/sessions/running-sessions` - List active sessions 30 | - `GET /api/sessions/session/:sessionId` - Get session information 31 | - `GET /api/sessions/debug/:sessionId` - Get session debug URLs 32 | 33 | ### Browser Control 34 | 35 | - `POST /api/browser/navigate/:sessionId` - Navigate to URL 36 | - `POST /api/browser/act/:sessionId` - Perform action via natural language 37 | - `POST /api/browser/extract/:sessionId` - Extract structured data 38 | - `POST /api/browser/observe/:sessionId` - Get possible actions 39 | 40 | ### Monitoring 41 | 42 | - `GET /api/browser/console-logs/:sessionId` - Get console logs 43 | - `GET /api/browser/network-logs/:sessionId` - Get network logs 44 | - `GET /api/browser/dom-state/:sessionId` - Get DOM state 45 | - `POST /api/browser/screenshot/:sessionId` - Take screenshot 46 | - `POST /api/browser/clear-logs/:sessionId` - Clear logs 47 | 48 | ## Key Features 49 | 50 | ### Browser Session Management 51 | - Create new browser sessions 52 | - Resume existing sessions 53 | - List running sessions 54 | - Stop/cleanup sessions 55 | 56 | ### AI-Powered Browser Control 57 | - Natural language actions via `act()` 58 | - Structured data extraction via `extract()` 59 | - Page observation via `observe()` 60 | - Vision-based interaction support 61 | 62 | 63 | ### Monitoring & Debugging 64 | - Console log monitoring 65 | - Network request/response logging 66 | - Error tracking 67 | - Screenshot capture 68 | - DOM state inspection 69 | 70 | ## Getting Started 71 | 72 | ### Prerequisites 73 | 74 | - Node.js 16+ 75 | - Browserbase account and credentials 76 | - OpenAI or Anthropic API key for AI features 77 | 78 | ### Installation 79 | 80 | ``` 81 | git clone https://github.com/kortix-ai/wbac 82 | cd wbac 83 | npm i 84 | ``` 85 | 86 | ### Configuration 87 | 88 | Create a `.env` file with: 89 | 90 | ``` 91 | BROWSERBASE_API_KEY=your_api_key 92 | BROWSERBASE_PROJECT_ID=your_project_id 93 | OPENAI_API_KEY=your_openai_key 94 | ANTHROPIC_API_KEY=your_anthropic_key 95 | ``` 96 | 97 | ### Running the Server 98 | 99 | ``` 100 | npm start 101 | ``` 102 | 103 | ## UI Interface 104 | 105 | A Streamlit-based UI is included for testing and debugging: 106 | 107 | ``` 108 | pip install streamlit 109 | streamlit run streamlit_ui.py 110 | ``` 111 | 112 | ## Use Cases 113 | 114 | - AI agents that need web browsing capabilities 115 | - Automated web testing with AI assistance 116 | - Web scraping with natural language commands 117 | - Browser automation monitoring and debugging 118 | 119 | ## Contributing 120 | 121 | Contributions welcome! Please read our contributing guidelines and submit pull requests. 122 | 123 | ## License 124 | 125 | MIT License - see LICENSE file for details 126 | 127 | ## Acknowledgements 128 | 129 | - [Browserbase](https://www.browserbase.com/) for cloud browser infrastructure 130 | - [Stagehand](https://github.com/browserbase/stagehand) for AI browser control capabilities 131 | - [Playwright](https://playwright.dev/) for powerful browser automation 132 | -------------------------------------------------------------------------------- /routes/session-routes.js: -------------------------------------------------------------------------------- 1 | const express = require('express'); 2 | const router = express.Router(); 3 | const browserbaseService = require('../services/browserbase-service'); 4 | const stagehandService = require('../services/stagehand-service'); 5 | const sessionExpiryManager = require('../services/session-expiry-service'); 6 | 7 | /** 8 | * @api {post} /api/sessions/create-session Create Browser Session 9 | * @apiName CreateSession 10 | * @apiGroup Sessions 11 | * @apiVersion 1.0.0 12 | * 13 | * @apiDescription Creates a new browser session in the cloud. 14 | * 15 | * @apiSuccess {Boolean} success Indicates if operation was successful 16 | * @apiSuccess {String} sessionId Unique identifier for the created session 17 | * 18 | * @apiError (Error 500) {Object} error Error object with message 19 | */ 20 | router.post('/create-session', async (req, res) => { 21 | try { 22 | const session = await browserbaseService.createSession(); 23 | if (!session?.id) { 24 | return res.status(500).json({ 25 | success: false, 26 | error: 'Failed to create browser session' 27 | }); 28 | } 29 | 30 | await sessionExpiryManager.trackSession(session.id); 31 | 32 | res.json({ 33 | success: true, 34 | sessionId: session.id 35 | }); 36 | } catch (error) { 37 | res.status(500).json({ 38 | success: false, 39 | error: error.message, 40 | stack: process.env.NODE_ENV === 'development' ? error.stack : undefined 41 | }); 42 | } 43 | }); 44 | 45 | /** 46 | * @api {post} /api/sessions/stop-session/:sessionId Stop Browser Session 47 | * @apiName StopSession 48 | * @apiGroup Sessions 49 | * @apiVersion 1.0.0 50 | * 51 | * @apiDescription Stops and cleans up an existing browser session. 52 | * 53 | * @apiParam {String} sessionId Session's unique identifier 54 | * 55 | * @apiSuccess {Boolean} success Indicates if operation was successful 56 | * 57 | * @apiError (Error 500) {Object} error Error object with message 58 | */ 59 | router.post('/stop-session/:sessionId', async (req, res) => { 60 | try { 61 | const { sessionId } = req.params; 62 | if (!sessionId) { 63 | return res.status(400).json({ 64 | success: false, 65 | error: 'Session ID is required' 66 | }); 67 | } 68 | 69 | const session = await browserbaseService.getSession(sessionId); 70 | if (!session) { 71 | return res.status(404).json({ 72 | success: false, 73 | error: `Session ${sessionId} not found` 74 | }); 75 | } 76 | 77 | await browserbaseService.stopSession(sessionId); 78 | await stagehandService.cleanupInstance(sessionId); 79 | 80 | res.json({ success: true }); 81 | } catch (error) { 82 | res.status(500).json({ 83 | success: false, 84 | error: error.message, 85 | stack: process.env.NODE_ENV === 'development' ? error.stack : undefined 86 | }); 87 | } 88 | }); 89 | 90 | /** 91 | * @api {get} /api/sessions/running-sessions List Running Sessions 92 | * @apiName GetRunningSessions 93 | * @apiGroup Sessions 94 | * @apiVersion 1.0.0 95 | * 96 | * @apiDescription Retrieves a list of all running browser sessions. 97 | * 98 | * @apiSuccess {Boolean} success Indicates if operation was successful 99 | * @apiSuccess {Object[]} sessions List of session objects 100 | * 101 | * @apiError (Error 500) {Object} error Error object with message 102 | */ 103 | router.get('/running-sessions', async (req, res) => { 104 | try { 105 | const sessions = await browserbaseService.listRunningSessions(); 106 | res.json({ 107 | success: true, 108 | sessions: sessions.map(session => ({ 109 | id: session.id, 110 | createdAt: session.createdAt, 111 | region: session.region, 112 | status: session.status 113 | })) 114 | }); 115 | } catch (error) { 116 | res.status(500).json({ error: error.message }); 117 | } 118 | }); 119 | 120 | /** 121 | * @api {get} /api/sessions/session/:sessionId Get Session Details 122 | * @apiName GetSession 123 | * @apiGroup Sessions 124 | * @apiVersion 1.0.0 125 | * 126 | * @apiDescription Retrieves details for a specific Browserbase session. 127 | * 128 | * @apiParam {String} sessionId Session's unique identifier 129 | * 130 | * @apiSuccess {Object} session Session details 131 | * @apiSuccess {Boolean} success Operation success status 132 | * 133 | * @apiError (Error 500) {Object} error Error object with message 134 | */ 135 | router.get('/session/:sessionId', async (req, res) => { 136 | try { 137 | const session = await browserbaseService.getSession(req.params.sessionId); 138 | res.json({ 139 | success: true, 140 | session 141 | }); 142 | } catch (error) { 143 | res.status(500).json({ 144 | success: false, 145 | error: error.message 146 | }); 147 | } 148 | }); 149 | 150 | /** 151 | * @api {get} /api/sessions/debug/:sessionId Get Session Debug URLs 152 | * @apiName GetSessionDebugUrls 153 | * @apiGroup Sessions 154 | * @apiVersion 1.0.0 155 | * 156 | * @apiDescription Retrieves debug URLs for a specific Browserbase session. 157 | * 158 | * @apiParam {String} sessionId Session's unique identifier 159 | * 160 | * @apiSuccess {Object} debugUrls Debug URLs object containing debuggerUrl, debuggerFullscreenUrl, wsUrl, and pages 161 | * @apiSuccess {Boolean} success Operation success status 162 | * 163 | * @apiError (Error 500) {Object} error Error object with message 164 | */ 165 | router.get('/debug/:sessionId', async (req, res) => { 166 | try { 167 | const debugUrls = await browserbaseService.getSessionDebugUrls(req.params.sessionId); 168 | res.json({ 169 | success: true, 170 | debugUrls 171 | }); 172 | } catch (error) { 173 | res.status(500).json({ 174 | success: false, 175 | error: error.message, 176 | stack: process.env.NODE_ENV === 'development' ? error.stack : undefined 177 | }); 178 | } 179 | }); 180 | 181 | /** 182 | * @api {post} /api/sessions/reset-expiry/:sessionId Reset Session Expiry 183 | * @apiName ResetSessionExpiry 184 | * @apiGroup Sessions 185 | * @apiVersion 1.0.0 186 | * 187 | * @apiDescription Resets the expiry timer for a specific Browserbase session. 188 | * 189 | * @apiParam {String} sessionId Session's unique identifier 190 | * 191 | * @apiSuccess {Boolean} success Indicates if operation was successful 192 | * 193 | * @apiError (Error 500) {Object} error Error object with message 194 | */ 195 | router.post('/reset-expiry/:sessionId', async (req, res) => { 196 | try { 197 | const { sessionId } = req.params; 198 | 199 | const isActive = await sessionExpiryManager.trackSession(sessionId); 200 | if (!isActive) { 201 | return res.status(404).json({ 202 | success: false, 203 | error: 'Session not found or not running' 204 | }); 205 | } 206 | 207 | sessionExpiryManager.resetExpiry(sessionId); 208 | 209 | res.json({ success: true }); 210 | } catch (error) { 211 | res.status(500).json({ 212 | success: false, 213 | error: error.message 214 | }); 215 | } 216 | }); 217 | 218 | module.exports = router; -------------------------------------------------------------------------------- /services/stagehand-service.js: -------------------------------------------------------------------------------- 1 | const { Stagehand } = require('@browserbasehq/stagehand'); 2 | 3 | class StagehandService { 4 | constructor() { 5 | this.instances = new Map(); 6 | this.logs = new Map(); 7 | } 8 | 9 | async getOrCreateInstance(sessionId, options = {}) { 10 | if (this.instances.has(sessionId)) { 11 | return this.instances.get(sessionId); 12 | } 13 | 14 | const config = { 15 | env: 'BROWSERBASE', 16 | verbose: options.verbose || 1, 17 | debugDom: options.debugDom || true, 18 | browserbaseResumeSessionID: sessionId, 19 | domSettleTimeoutMs: 30000, 20 | enableCaching: false, 21 | apiKey: process.env.BROWSERBASE_API_KEY, 22 | projectId: process.env.BROWSERBASE_PROJECT_ID 23 | }; 24 | 25 | const stagehand = new Stagehand(config); 26 | await stagehand.init({ modelName: options.modelName }); 27 | 28 | this.logs.set(sessionId, { 29 | console: [], 30 | network: [] 31 | }); 32 | 33 | await this._setupMonitoring(sessionId, stagehand, options.monitorSettings); 34 | 35 | this.instances.set(sessionId, stagehand); 36 | return stagehand; 37 | } 38 | 39 | async _setupMonitoring(sessionId, stagehand, settings = {}) { 40 | const sessionLogs = this.logs.get(sessionId); 41 | 42 | stagehand.page.on('console', message => { 43 | const type = message.type(); 44 | sessionLogs.console.push({ 45 | type, 46 | message: message.text(), 47 | path: message.location()?.url || 'unknown', 48 | timestamp: new Date().toISOString(), 49 | stackTrace: message.stackTrace?.() || null 50 | }); 51 | }); 52 | 53 | stagehand.page.on('response', async response => { 54 | try { 55 | const request = response.request(); 56 | const networkLog = { 57 | url: response.url(), 58 | method: request.method(), 59 | status: response.status(), 60 | timestamp: new Date().toISOString(), 61 | request: { 62 | headers: request.headers(), 63 | body: await request.postData() || null, 64 | queryParams: new URL(response.url()).searchParams.toString(), 65 | }, 66 | response: { 67 | headers: response.headers(), 68 | body: null 69 | } 70 | }; 71 | 72 | try { 73 | const contentType = response.headers()['content-type'] || ''; 74 | if (contentType.includes('application/json')) { 75 | networkLog.response.body = await response.json(); 76 | } else { 77 | networkLog.response.body = await response.text(); 78 | } 79 | } catch (e) { 80 | networkLog.response.body = 'Could not parse response body'; 81 | } 82 | 83 | sessionLogs.network.push(networkLog); 84 | } catch (error) { 85 | console.error('Error logging network response:', error); 86 | } 87 | }); 88 | } 89 | 90 | getInstance(sessionId) { 91 | return this.instances.get(sessionId); 92 | } 93 | 94 | getLogs(sessionId) { 95 | return this.logs.get(sessionId); 96 | } 97 | 98 | async cleanupInstance(sessionId) { 99 | const instance = this.instances.get(sessionId); 100 | if (instance) { 101 | await instance.context.close(); 102 | this.instances.delete(sessionId); 103 | this.logs.delete(sessionId); 104 | } 105 | } 106 | 107 | _normalizeConsoleLogFilters(filters = {}) { 108 | return { 109 | levels: { 110 | error: filters.levels?.error ?? true, 111 | warning: filters.levels?.warning ?? false, 112 | info: filters.levels?.info ?? false, 113 | trace: filters.levels?.trace ?? false 114 | }, 115 | excludeStringFilters: Array.isArray(filters.excludeStringFilters) ? 116 | filters.excludeStringFilters.map(f => f.toLowerCase()) : [], 117 | includeStringFilters: Array.isArray(filters.includeStringFilters) ? 118 | filters.includeStringFilters.map(f => f.toLowerCase()) : [], 119 | startTime: filters.startTime ? new Date(filters.startTime) : null, 120 | endTime: filters.endTime ? new Date(filters.endTime) : null, 121 | truncateLength: filters.truncateLength || 500 122 | }; 123 | } 124 | 125 | async getConsoleLogs(sessionId, filters = {}) { 126 | const logs = this.getLogs(sessionId); 127 | if (!logs) throw new Error('Session not initialized'); 128 | 129 | const normalizedFilters = this._normalizeConsoleLogFilters(filters); 130 | 131 | return logs.console 132 | .filter(log => { 133 | if (!log) return false; 134 | 135 | if (normalizedFilters.startTime && new Date(log.timestamp) < normalizedFilters.startTime) return false; 136 | if (normalizedFilters.endTime && new Date(log.timestamp) > normalizedFilters.endTime) return false; 137 | 138 | const typeMatch = ( 139 | (log.type === 'error' && normalizedFilters.levels.error) || 140 | (log.type === 'warning' && normalizedFilters.levels.warning) || 141 | ((log.type === 'info' || log.type === 'log') && normalizedFilters.levels.info) || 142 | (log.type === 'trace' && normalizedFilters.levels.trace) 143 | ); 144 | if (!typeMatch) return false; 145 | 146 | const searchableContent = [ 147 | log.message?.toLowerCase() || '', 148 | log.path?.toLowerCase() || '', 149 | log.type?.toLowerCase() || '', 150 | log.stackTrace?.toLowerCase() || '', 151 | ...(log.args || []).map(arg => String(arg).toLowerCase()) 152 | ].join(' '); 153 | 154 | for (const excludeFilter of normalizedFilters.excludeStringFilters) { 155 | if (searchableContent.includes(excludeFilter)) { 156 | return false; 157 | } 158 | } 159 | 160 | if (normalizedFilters.includeStringFilters.length > 0) { 161 | return normalizedFilters.includeStringFilters.some(filter => searchableContent.includes(filter)); 162 | } 163 | 164 | return true; 165 | }) 166 | .map(log => this._formatConsoleLog(log, normalizedFilters)); 167 | } 168 | 169 | _formatConsoleLog(log, filters) { 170 | const truncate = (str, length) => { 171 | if (!str) return str; 172 | if (str.length <= length) return str; 173 | return str.substring(0, length) + '... (truncated)'; 174 | }; 175 | 176 | return { 177 | type: log.type, 178 | timestamp: log.timestamp, 179 | message: truncate(log.message, filters.truncateLength), 180 | path: log.path, 181 | stackTrace: truncate(log.stackTrace, filters.truncateLength), 182 | args: log.args ? log.args.map(arg => truncate(String(arg), filters.truncateLength)) : undefined 183 | }; 184 | } 185 | 186 | _normalizeLogFilters(filters = {}) { 187 | return { 188 | statusCodes: { 189 | info: filters.statusCodes?.info ?? true, 190 | success: filters.statusCodes?.success ?? true, 191 | redirect: filters.statusCodes?.redirect ?? true, 192 | clientError: filters.statusCodes?.clientError ?? true, 193 | serverError: filters.statusCodes?.serverError ?? true 194 | }, 195 | includeHeaders: filters.includeHeaders ?? false, 196 | includeBody: filters.includeBody ?? true, 197 | includeQueryParams: filters.includeQueryParams ?? true, 198 | excludeStringFilters: Array.isArray(filters.excludeStringFilters) ? 199 | filters.excludeStringFilters.map(f => f.toLowerCase()) : [], 200 | includeStringFilters: Array.isArray(filters.includeStringFilters) ? 201 | filters.includeStringFilters.map(f => f.toLowerCase()) : [], 202 | startTime: filters.startTime ? new Date(filters.startTime) : null, 203 | endTime: filters.endTime ? new Date(filters.endTime) : null, 204 | truncateLength: filters.truncateLength || 5000 205 | }; 206 | } 207 | 208 | async getNetworkLogs(sessionId, filters = {}) { 209 | const logs = this.getLogs(sessionId); 210 | if (!logs) throw new Error('Session not initialized'); 211 | 212 | const normalizedFilters = this._normalizeLogFilters(filters); 213 | 214 | const filteredLogs = logs.network 215 | .filter(log => { 216 | if (!log) return false; 217 | 218 | if (normalizedFilters.startTime && new Date(log.timestamp) < normalizedFilters.startTime) return false; 219 | if (normalizedFilters.endTime && new Date(log.timestamp) > normalizedFilters.endTime) return false; 220 | 221 | const status = log.status; 222 | const statusMatch = ( 223 | (status >= 100 && status <= 199 && normalizedFilters.statusCodes.info) || 224 | (status >= 200 && status <= 299 && normalizedFilters.statusCodes.success) || 225 | (status >= 300 && status <= 399 && normalizedFilters.statusCodes.redirect) || 226 | (status >= 400 && status <= 499 && normalizedFilters.statusCodes.clientError) || 227 | (status >= 500 && status <= 599 && normalizedFilters.statusCodes.serverError) 228 | ); 229 | 230 | if (!statusMatch) return false; 231 | 232 | const searchableContent = [ 233 | log.url?.toLowerCase() || '', 234 | log.method?.toLowerCase() || '', 235 | JSON.stringify(log.request?.headers || {}).toLowerCase(), 236 | JSON.stringify(log.response?.headers || {}).toLowerCase(), 237 | (log.request?.body ? String(log.request.body).toLowerCase() : ''), 238 | (log.response?.body ? String(log.response.body).toLowerCase() : '') 239 | ].join(' '); 240 | 241 | for (const excludeFilter of normalizedFilters.excludeStringFilters) { 242 | if (searchableContent.includes(excludeFilter)) { 243 | // console.log(`Excluding log with URL ${log.url} due to filter: ${excludeFilter}`); 244 | return false; 245 | } 246 | } 247 | 248 | if (normalizedFilters.includeStringFilters.length > 0) { 249 | return normalizedFilters.includeStringFilters.some(filter => searchableContent.includes(filter)); 250 | } 251 | 252 | return true; 253 | }) 254 | .map(log => this._formatNetworkLog(log, normalizedFilters)); 255 | 256 | // console.log('Filtered network logs count:', filteredLogs.length); 257 | 258 | return filteredLogs; 259 | } 260 | 261 | _formatNetworkLog(log, filters) { 262 | const truncate = (str, length) => { 263 | if (!str) return str; 264 | if (typeof str === 'object') str = JSON.stringify(str); 265 | if (str.length <= length) return str; 266 | return str.substring(0, length) + '... (truncated)'; 267 | }; 268 | 269 | return { 270 | url: log.url, 271 | method: log.method, 272 | status: log.status, 273 | timestamp: log.timestamp, 274 | request: { 275 | method: log.method, 276 | url: log.url, 277 | queryParams: filters.includeQueryParams ? log.request?.queryParams : undefined, 278 | headers: filters.includeHeaders ? log.request?.headers : undefined, 279 | body: filters.includeBody ? truncate(log.request?.body, filters.truncateLength) : undefined 280 | }, 281 | response: { 282 | status: log.status, 283 | headers: filters.includeHeaders ? log.response?.headers : undefined, 284 | body: filters.includeBody ? truncate(log.response?.body, filters.truncateLength) : undefined 285 | } 286 | }; 287 | } 288 | } 289 | 290 | module.exports = new StagehandService(); -------------------------------------------------------------------------------- /docs/assets/main.css: -------------------------------------------------------------------------------- 1 | /** 2 | * apidoc main css file 3 | */ 4 | 5 | /** 6 | * Define colors 7 | */ 8 | :root { 9 | --primary: #0088cc; 10 | --white: #fff; 11 | --light-gray: #ccc; 12 | --main-gray: #777; 13 | --dark-gray: #2d2d2d; 14 | --hover-gray: #666; 15 | --meth-get: green; 16 | --meth-put: #e5c500; 17 | --meth-post: #4070ec; 18 | --meth-delete: #ed0039; 19 | --red: #dc3545; 20 | } 21 | 22 | .color-primary { 23 | color: var(--primary); 24 | } 25 | 26 | .bg-primary { 27 | background-color: var(--primary); 28 | } 29 | 30 | .bg-red { 31 | color: var(--white); 32 | background-color: var(--red); 33 | } 34 | 35 | .border-danger { 36 | border: 1px solid var(--red); 37 | } 38 | 39 | /** for some reason the iOS safari style is applied on date inputs */ 40 | input[type="date"] { 41 | line-height: 1.4 !important; 42 | } 43 | 44 | /* ------------------------------------------------------------------------------------------ 45 | * Content 46 | * ------------------------------------------------------------------------------------------ */ 47 | @font-face { 48 | font-family: 'Glyphicons Halflings'; 49 | src: url('./glyphicons-halflings-regular.eot'); 50 | src: url('./glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'), 51 | url('./glyphicons-halflings-regular.woff') format('woff'), 52 | url('./glyphicons-halflings-regular.woff2') format('woff2'), 53 | url('./glyphicons-halflings-regular.ttf') format('truetype'), 54 | url('./glyphicons-halflings-regular.svg#glyphicons-halflingsregular') format('svg'); 55 | } 56 | 57 | /* Hide vertical scrollbar on off canvas animation ("left" positioning) */ 58 | html { 59 | overflow-x: hidden; 60 | } 61 | 62 | body { 63 | font-family: "Source Sans Pro", sans-serif; 64 | } 65 | 66 | a:focus { 67 | background-color: var(--primary); 68 | } 69 | 70 | #content { 71 | margin-top: 10px; 72 | padding-left: 10px; 73 | } 74 | 75 | p { 76 | font-size: 130%; 77 | color: var(--main-gray); 78 | } 79 | 80 | section { 81 | padding: 30px 0; 82 | } 83 | 84 | article { 85 | border-top: 1px solid var(--light-gray); 86 | padding: 14px 0 30px 0; 87 | } 88 | 89 | table { 90 | border-collapse: collapse; 91 | width: 100%; 92 | margin: 0 0 20px 0; 93 | } 94 | 95 | th { 96 | background-color: var(--main-gray); 97 | color: var(--white); 98 | text-align: left; 99 | padding: 5px 8px; 100 | border: 1px solid var(--main-gray); 101 | } 102 | 103 | td { 104 | padding: 5px; 105 | border: 1px solid var(--main-gray); 106 | } 107 | 108 | td.code { 109 | font-family: "Source Code Pro", monospace; 110 | font-weight: 600; 111 | } 112 | 113 | .label { 114 | float: right; 115 | margin-top: 4px; 116 | user-select: none; 117 | } 118 | 119 | .label.optional { 120 | background-color: grey; 121 | } 122 | 123 | .label.required { 124 | background-color: var(--red); 125 | } 126 | 127 | .default-value, 128 | .type-size { 129 | font-style: italic; 130 | font-size: 95%; 131 | } 132 | 133 | .open-left { 134 | right: 0; 135 | left: auto; 136 | } 137 | 138 | .invisible { 139 | visibility: hidden; 140 | } 141 | 142 | .input-group-addon.sample-request-select { 143 | padding: 0 6px; 144 | } 145 | 146 | .input-group-addon.sample-request-select select { 147 | width: auto; 148 | height: 32px; 149 | } 150 | 151 | .sample-request-input-Boolean-container { 152 | width: 40px; 153 | height: 34px; 154 | background: var(--white); 155 | border: 1px solid var(--light-gray); 156 | } 157 | 158 | .sample-request-input-Boolean-container > div { 159 | margin-top: 7px; 160 | text-align: center; 161 | } 162 | 163 | .sample-request-input-Boolean-container > div input { 164 | margin: 0; 165 | } 166 | 167 | /* ------------------------------------------------------------------------------------------ 168 | * Request method (HTTP verb) 169 | * ------------------------------------------------------------------------------------------ */ 170 | .method { 171 | font-weight: 600; 172 | font-size: 15px; 173 | display: inline-block; 174 | margin: 0 0 5px 0; 175 | padding: 4px 5px; 176 | border-radius: 6px; 177 | text-transform: uppercase; 178 | background-color: var(--main-gray); 179 | color: var(--white); 180 | } 181 | 182 | .meth-get { 183 | background-color: var(--meth-get); 184 | } 185 | 186 | .meth-put { 187 | background-color: var(--meth-put); 188 | } 189 | 190 | .meth-post { 191 | background-color: var(--meth-post); 192 | } 193 | 194 | .meth-delete { 195 | background-color: var(--meth-delete); 196 | } 197 | 198 | /* ------------------------------------------------------------------------------------------ 199 | * Sidenav 200 | * ------------------------------------------------------------------------------------------ */ 201 | #scrollingNav { 202 | position: fixed; 203 | top: 0; 204 | left: 0; 205 | bottom: 0; 206 | z-index: 10; 207 | background-color: var(--dark-gray); 208 | box-shadow: 0 2px 5px 0 rgb(0 0 0 / 16%), 0 2px 10px 0 rgb(0 0 0 / 12%); 209 | } 210 | 211 | .sidenav { 212 | color: var(--white); 213 | position: absolute; 214 | top: 50px; 215 | left: 0; 216 | right: 0; 217 | bottom: 0; 218 | overflow-x: hidden; 219 | overflow-y: hidden; 220 | } 221 | 222 | .sidenav:hover { 223 | overflow-x: auto; 224 | overflow-y: auto; 225 | } 226 | 227 | .sidenav > li > a { 228 | color: var(--white); 229 | display: block; 230 | padding: 8px 13px; 231 | } 232 | 233 | /* non active sidenav link are highlighted on hover */ 234 | .sidenav > li:not(.active) > a:hover { 235 | background-color: var(--hover-gray); 236 | } 237 | 238 | .sidenav > li.nav-header { 239 | margin-top: 8px; 240 | margin-bottom: 8px; 241 | } 242 | 243 | .sidenav > li.nav-header > a { 244 | padding: 5px 15px; 245 | font-weight: 700; 246 | font-size: 16px; 247 | background-color: var(--main-gray); 248 | } 249 | 250 | 251 | .sidenav > li.active > a { 252 | position: relative; 253 | background-color: var(--primary); 254 | color: var(--white); 255 | } 256 | 257 | /** 258 | * TODO: commented out for the moment 259 | .sidenav > li.has-modifications a { 260 | border-right: 4px solid var(--main-gray); 261 | } 262 | 263 | .nav-list-item :not(.is-new) { 264 | border-left: 4px solid var(--main-gray); 265 | } 266 | 267 | .sidenav > li.is-new a { 268 | border-left: 4px solid var(--primary); 269 | } 270 | */ 271 | 272 | /* 273 | * Off Canvas 274 | * -------------------------------------------------- 275 | */ 276 | @media screen and (max-width: 767px) { 277 | #content { 278 | margin-top: 58px; 279 | } 280 | 281 | .row-offcanvas { 282 | position: relative; 283 | -webkit-transition: all .25s ease-out; 284 | -o-transition: all .25s ease-out; 285 | transition: all .25s ease-out; 286 | left: 0; 287 | } 288 | 289 | .row-offcanvas, 290 | .row-offcanvas * { 291 | transition: all 0.5s ease-out; 292 | } 293 | 294 | .row-offcanvas .sidebar-offcanvas { 295 | position: absolute; 296 | top: 0; 297 | left: -200px !important; /* 6 columns */ 298 | width: 100%; /* 6 columns */ 299 | max-width: 200px; 300 | } 301 | 302 | .nav-toggle { 303 | position: fixed; 304 | left: 0; 305 | background: var(--dark-gray); 306 | width: 100%; 307 | } 308 | 309 | .nav-toggle .btn { 310 | margin: 10px 14px; 311 | } 312 | .nav-toggle .icon-bar { 313 | display: block; 314 | width: 22px; 315 | height: 2px; 316 | border-radius: 1px; 317 | background-color: var(--white); 318 | } 319 | .nav-toggle .icon-bar + .icon-bar { 320 | margin-top: 4px; 321 | } 322 | 323 | .row-offcanvas.active .sidebar-offcanvas { 324 | left: 0 !important; /* 6 columns */ 325 | } 326 | .row-offcanvas.active, .row-offcanvas.active .nav-toggle { 327 | left: 200px; 328 | } 329 | /* Styling the three lines to make it an X */ 330 | .row-offcanvas.active .nav-toggle .btn > .icon-bar { 331 | transform: rotate(45deg) translate(-4px, -4px); 332 | } 333 | .row-offcanvas.active .nav-toggle .btn .icon-bar:nth-child(2) { 334 | display: none; 335 | } 336 | .row-offcanvas.active .nav-toggle .btn .icon-bar:nth-child(3) { 337 | transform: rotate(-45deg); 338 | } 339 | } 340 | 341 | /* ------------------------------------------------------------------------------------------ 342 | * Side nav search 343 | * ------------------------------------------------------------------------------------------ */ 344 | .sidenav-search { 345 | padding: 16px 10px 10px; 346 | background-color: var(--dark-gray); 347 | } 348 | 349 | .sidenav-search .search { 350 | height: 26px; 351 | } 352 | 353 | .search-reset { 354 | position: absolute; 355 | display: block; 356 | cursor: pointer; 357 | width: 20px; 358 | height: 20px; 359 | text-align: center; 360 | right: 28px; 361 | top: 18px; 362 | background-color: #fff; 363 | } 364 | 365 | /* ------------------------------------------------------------------------------------------ 366 | * Prism - Toolbar 367 | * ------------------------------------------------------------------------------------------ */ 368 | div.code-toolbar.code-toolbar > .toolbar { 369 | top: .4rem; 370 | right: .4rem; 371 | } 372 | div.code-toolbar.code-toolbar > .toolbar > .toolbar-item > button:hover, 373 | div.code-toolbar.code-toolbar > .toolbar > .toolbar-item > button:focus { 374 | color: var(--white); 375 | } 376 | div.code-toolbar.code-toolbar > .toolbar > .toolbar-item > button { 377 | color: var(--light-gray); 378 | padding: .5em; 379 | background: var(--hover-gray); 380 | box-shadow: 0 2px 1px 1px rgba(0,0,0,.5); 381 | } 382 | 383 | /* ------------------------------------------------------------------------------------------ 384 | * Compare 385 | * ------------------------------------------------------------------------------------------ */ 386 | 387 | ins { 388 | background: #60d060; 389 | text-decoration: none; 390 | color: #000000; 391 | } 392 | 393 | del { 394 | background: #f05050; 395 | color: #000000; 396 | } 397 | 398 | .label-ins { 399 | background-color: #60d060; 400 | } 401 | 402 | .label-del { 403 | background-color: #f05050; 404 | text-decoration: line-through; 405 | } 406 | 407 | pre.ins { 408 | background-color: #60d060; 409 | } 410 | 411 | pre.del { 412 | background-color: #f05050; 413 | text-decoration: line-through; 414 | } 415 | 416 | table.ins th, 417 | table.ins td { 418 | background-color: #60d060; 419 | } 420 | 421 | table.del th, 422 | table.del td { 423 | background-color: #f05050; 424 | text-decoration: line-through; 425 | } 426 | 427 | tr.ins td { 428 | background-color: #60d060; 429 | } 430 | 431 | tr.del td { 432 | background-color: #f05050; 433 | text-decoration: line-through; 434 | } 435 | 436 | /* ------------------------------------------------------------------------------------------ 437 | * Spinner 438 | * ------------------------------------------------------------------------------------------ */ 439 | 440 | #loader { 441 | position: absolute; 442 | width: 100%; 443 | } 444 | 445 | #loader p { 446 | padding-top: 80px; 447 | margin-left: -4px; 448 | } 449 | 450 | .spinner { 451 | margin: 200px auto; 452 | width: 60px; 453 | height: 60px; 454 | position: relative; 455 | } 456 | 457 | .container1 > div, .container2 > div, .container3 > div { 458 | width: 14px; 459 | height: 14px; 460 | background-color: #0088cc; 461 | 462 | border-radius: 100%; 463 | position: absolute; 464 | -webkit-animation: bouncedelay 1.2s infinite ease-in-out; 465 | animation: bouncedelay 1.2s infinite ease-in-out; 466 | /* Prevent first frame from flickering when animation starts */ 467 | -webkit-animation-fill-mode: both; 468 | animation-fill-mode: both; 469 | } 470 | 471 | .spinner .spinner-container { 472 | position: absolute; 473 | width: 100%; 474 | height: 100%; 475 | } 476 | 477 | .container2 { 478 | -webkit-transform: rotateZ(45deg); 479 | transform: rotateZ(45deg); 480 | } 481 | 482 | .container3 { 483 | -webkit-transform: rotateZ(90deg); 484 | transform: rotateZ(90deg); 485 | } 486 | 487 | .circle1 { top: 0; left: 0; } 488 | .circle2 { top: 0; right: 0; } 489 | .circle3 { right: 0; bottom: 0; } 490 | .circle4 { left: 0; bottom: 0; } 491 | 492 | .container2 .circle1 { 493 | -webkit-animation-delay: -1.1s; 494 | animation-delay: -1.1s; 495 | } 496 | 497 | .container3 .circle1 { 498 | -webkit-animation-delay: -1.0s; 499 | animation-delay: -1.0s; 500 | } 501 | 502 | .container1 .circle2 { 503 | -webkit-animation-delay: -0.9s; 504 | animation-delay: -0.9s; 505 | } 506 | 507 | .container2 .circle2 { 508 | -webkit-animation-delay: -0.8s; 509 | animation-delay: -0.8s; 510 | } 511 | 512 | .container3 .circle2 { 513 | -webkit-animation-delay: -0.7s; 514 | animation-delay: -0.7s; 515 | } 516 | 517 | .container1 .circle3 { 518 | -webkit-animation-delay: -0.6s; 519 | animation-delay: -0.6s; 520 | } 521 | 522 | .container2 .circle3 { 523 | -webkit-animation-delay: -0.5s; 524 | animation-delay: -0.5s; 525 | } 526 | 527 | .container3 .circle3 { 528 | -webkit-animation-delay: -0.4s; 529 | animation-delay: -0.4s; 530 | } 531 | 532 | .container1 .circle4 { 533 | -webkit-animation-delay: -0.3s; 534 | animation-delay: -0.3s; 535 | } 536 | 537 | .container2 .circle4 { 538 | -webkit-animation-delay: -0.2s; 539 | animation-delay: -0.2s; 540 | } 541 | 542 | .container3 .circle4 { 543 | -webkit-animation-delay: -0.1s; 544 | animation-delay: -0.1s; 545 | } 546 | 547 | @-webkit-keyframes bouncedelay { 548 | 0%, 80%, 100% { -webkit-transform: scale(0.0) } 549 | 40% { -webkit-transform: scale(1.0) } 550 | } 551 | 552 | @keyframes bouncedelay { 553 | 0%, 80%, 100% { 554 | transform: scale(0.0); 555 | -webkit-transform: scale(0.0); 556 | } 40% { 557 | transform: scale(1.0); 558 | -webkit-transform: scale(1.0); 559 | } 560 | } 561 | 562 | /* ------------------------------------------------------------------------------------------ 563 | * Tabs 564 | * ------------------------------------------------------------------------------------------ */ 565 | ul.nav-tabs { 566 | margin: 0; 567 | } 568 | 569 | p.deprecated span{ 570 | color: var(--red); 571 | font-weight: bold; 572 | text-decoration: underline; 573 | } 574 | 575 | /** 576 | * Footer 577 | */ 578 | #generator { 579 | padding: 10px 0; 580 | } 581 | 582 | /* ------------------------------------------------------------------------------------------ 583 | * Print 584 | * ------------------------------------------------------------------------------------------ */ 585 | 586 | @media print { 587 | 588 | #sidenav, 589 | #version, 590 | #versions, 591 | section .version, 592 | section .versions { 593 | display: none; 594 | } 595 | 596 | #content { 597 | margin-left: 0; 598 | } 599 | 600 | a { 601 | text-decoration: none; 602 | color: inherit; 603 | } 604 | 605 | a:after { 606 | content: " [" attr(href) "] "; 607 | } 608 | 609 | p { 610 | color: #000000 611 | } 612 | 613 | pre { 614 | background-color: #ffffff; 615 | color: #000000; 616 | padding: 10px; 617 | border: #808080 1px solid; 618 | border-radius: 6px; 619 | position: relative; 620 | margin: 10px 0 20px 0; 621 | } 622 | 623 | } /* /@media print */ 624 | -------------------------------------------------------------------------------- /routes/browser-routes.js: -------------------------------------------------------------------------------- 1 | const express = require('express'); 2 | const router = express.Router(); 3 | const stagehandService = require('../services/stagehand-service'); 4 | const sessionExpiryManager = require('../services/session-expiry-service'); 5 | const { z } = require('zod'); 6 | 7 | // Helper function to ensure Stagehand instance exists 8 | async function ensureStagehand(sessionId, options = {}) { 9 | try { 10 | // Reset session expiry on any browser activity 11 | await sessionExpiryManager.trackSession(sessionId); 12 | sessionExpiryManager.resetExpiry(sessionId); 13 | 14 | return await stagehandService.getOrCreateInstance(sessionId, options); 15 | } catch (error) { 16 | throw new Error(`Failed to initialize browser session: ${error.message}`); 17 | } 18 | } 19 | 20 | /** 21 | * @api {post} /api/browser/navigate/:sessionId Navigate Browser 22 | * @apiName NavigateBrowser 23 | * @apiGroup Browser 24 | * @apiVersion 1.0.0 25 | * 26 | * @apiDescription Navigates the browser to a specified URL with network idle waiting. 27 | * 28 | * @apiParam {String} sessionId Session's unique identifier 29 | * @apiBody {String} url URL to navigate to 30 | * 31 | * @apiSuccess {Boolean} success Indicates if operation was successful 32 | * @apiSuccess {String} url Current browser URL after navigation 33 | * 34 | * @apiError (Error 400) {Object} error URL is required 35 | * @apiError (Error 500) {Object} error Error object with message and optional stack trace 36 | */ 37 | router.post('/navigate/:sessionId', async (req, res) => { 38 | try { 39 | const { url } = req.body; 40 | if (!url) { 41 | return res.status(400).json({ 42 | success: false, 43 | error: 'URL is required for navigation' 44 | }); 45 | } 46 | 47 | const stagehand = await ensureStagehand(req.params.sessionId); 48 | if (!stagehand?.page) { 49 | return res.status(500).json({ 50 | success: false, 51 | error: 'Browser page not initialized' 52 | }); 53 | } 54 | 55 | await stagehand.page.goto(url, { 56 | waitUntil: 'networkidle0', 57 | timeout: 30000 58 | }); 59 | 60 | res.json({ 61 | success: true, 62 | url: url, 63 | }); 64 | } catch (error) { 65 | console.error('Navigation error:', error); 66 | res.status(500).json({ 67 | success: false, 68 | error: error.message, 69 | stack: process.env.NODE_ENV === 'development' ? error.stack : undefined 70 | }); 71 | } 72 | }); 73 | 74 | /** 75 | * @api {post} /api/browser/act/:sessionId Perform Browser Action 76 | * @apiName PerformAction 77 | * @apiGroup Browser 78 | * @apiVersion 1.0.0 79 | * 80 | * @apiDescription Performs a natural language action in the browser with optional vision capabilities. 81 | * 82 | * @apiParam {String} sessionId Session's unique identifier 83 | * @apiBody {String} action Natural language action to perform 84 | * @apiBody {String} [useVision='fallback'] Vision mode: 'fallback' 85 | * @apiBody {String} [modelName] Optional AI model name to use 86 | * @apiBody {Boolean} [includeLogs] Include logs from the action execution 87 | * @apiBody {Object} [logFilters] Filters for included logs 88 | * @apiBody {Object} [logFilters.console] Console log filters 89 | * @apiBody {Boolean} [logFilters.console.includeErrors=true] Include error logs 90 | * @apiBody {Boolean} [logFilters.console.includeWarnings=false] Include warning logs 91 | * @apiBody {Boolean} [logFilters.console.includeInfo=false] Include info logs 92 | * @apiBody {Boolean} [logFilters.console.includeTrace=false] Include trace logs 93 | * @apiBody {String[]} [logFilters.console.includeStringFilters] Strings to include in console logs 94 | * @apiBody {String[]} [logFilters.console.excludeStringFilters] Strings to exclude from console logs 95 | * @apiBody {Number} [logFilters.console.truncateLength=500] Maximum length for console messages 96 | * @apiBody {Object} [logFilters.network] Network log filters 97 | * @apiBody {Object} [logFilters.network.statusCodes] Status code filters 98 | * @apiBody {Boolean} [logFilters.network.statusCodes.info=true] Include 1xx responses 99 | * @apiBody {Boolean} [logFilters.network.statusCodes.success=true] Include 2xx responses 100 | * @apiBody {Boolean} [logFilters.network.statusCodes.redirect=true] Include 3xx responses 101 | * @apiBody {Boolean} [logFilters.network.statusCodes.clientError=true] Include 4xx responses 102 | * @apiBody {Boolean} [logFilters.network.statusCodes.serverError=true] Include 5xx responses 103 | * @apiBody {Boolean} [logFilters.network.includeHeaders=false] Include headers 104 | * @apiBody {Boolean} [logFilters.network.includeBody=true] Include bodies 105 | * @apiBody {Boolean} [logFilters.network.includeQueryParams=true] Include query parameters 106 | * @apiBody {String[]} [logFilters.network.includeStringFilters] Strings to include in network logs 107 | * @apiBody {String[]} [logFilters.network.excludeStringFilters] Strings to exclude from network logs 108 | * @apiBody {Number} [logFilters.network.truncateLength=500] Maximum length for bodies 109 | * 110 | * @apiSuccess {Object} result Action execution result 111 | * @apiSuccess {Object} [logs] Filtered logs if requested 112 | * 113 | * @apiError (Error 500) {Object} error Error object with message 114 | */ 115 | router.post('/act/:sessionId', async (req, res) => { 116 | try { 117 | const { action, useVision, modelName, includeLogs, logFilters } = req.body; 118 | 119 | if (!action) { 120 | return res.status(400).json({ 121 | success: false, 122 | error: 'Action instruction is required' 123 | }); 124 | } 125 | 126 | const stagehand = await ensureStagehand(req.params.sessionId, { modelName }); 127 | if (!stagehand) { 128 | return res.status(500).json({ 129 | success: false, 130 | error: 'Failed to initialize browser session' 131 | }); 132 | } 133 | 134 | const actionStartTime = new Date().toISOString(); 135 | 136 | const result = await stagehand.act({ 137 | action, 138 | useVision: useVision || 'fallback', 139 | modelName 140 | }); 141 | 142 | let logs = null; 143 | if (includeLogs) { 144 | const [consoleLogs, networkLogs] = await Promise.all([ 145 | stagehandService.getConsoleLogs(req.params.sessionId, { 146 | levels: { 147 | error: logFilters?.console?.levels?.error ?? true, 148 | warning: logFilters?.console?.levels?.warning ?? false, 149 | info: logFilters?.console?.levels?.info ?? false, 150 | trace: logFilters?.console?.levels?.trace ?? false 151 | }, 152 | includeStringFilters: logFilters?.console?.includeStringFilters, 153 | excludeStringFilters: logFilters?.console?.excludeStringFilters, 154 | startTime: actionStartTime, 155 | truncateLength: logFilters?.console?.truncateLength 156 | }), 157 | stagehandService.getNetworkLogs(req.params.sessionId, { 158 | statusCodes: { 159 | info: logFilters?.network?.statusCodes?.info ?? true, 160 | success: logFilters?.network?.statusCodes?.success ?? true, 161 | redirect: logFilters?.network?.statusCodes?.redirect ?? true, 162 | clientError: logFilters?.network?.statusCodes?.clientError ?? true, 163 | serverError: logFilters?.network?.statusCodes?.serverError ?? true 164 | }, 165 | includeHeaders: logFilters?.network?.includeHeaders ?? false, 166 | includeBody: logFilters?.network?.includeBody ?? true, 167 | includeQueryParams: logFilters?.network?.includeQueryParams ?? true, 168 | includeStringFilters: logFilters?.network?.includeStringFilters, 169 | excludeStringFilters: logFilters?.network?.excludeStringFilters, 170 | startTime: actionStartTime, 171 | truncateLength: logFilters?.network?.truncateLength 172 | }) 173 | ]); 174 | 175 | logs = { console: consoleLogs, network: networkLogs }; 176 | } 177 | 178 | res.json({ 179 | success: true, 180 | result, 181 | ...(includeLogs && { logs }) 182 | }); 183 | } catch (error) { 184 | res.status(500).json({ 185 | success: false, 186 | error: error.message, 187 | stack: process.env.NODE_ENV === 'development' ? error.stack : undefined 188 | }); 189 | } 190 | }); 191 | 192 | /** 193 | * @api {post} /api/browser/extract/:sessionId Extract Data 194 | * @apiName ExtractData 195 | * @apiGroup Browser 196 | * @apiVersion 1.0.0 197 | * 198 | * @apiDescription Extracts structured data from the current page using a Zod schema. 199 | * 200 | * @apiParam {String} sessionId Session's unique identifier 201 | * @apiBody {String} instruction Natural language instruction for extraction 202 | * @apiBody {Object} schema Zod schema definition 203 | * @apiBody {String} [modelName] Optional AI model name to use 204 | * 205 | * @apiSuccess {Object} result Extracted data matching schema 206 | * 207 | * @apiError (Error 500) {Object} error Error object with message 208 | */ 209 | router.post('/extract/:sessionId', async (req, res) => { 210 | try { 211 | const { instruction, schema, modelName } = req.body; 212 | const stagehand = await ensureStagehand(req.params.sessionId, { modelName }); 213 | 214 | const result = await stagehand.extract({ 215 | instruction, 216 | schema: z.object(schema), 217 | modelName 218 | }); 219 | 220 | res.json(result); 221 | } catch (error) { 222 | res.status(500).json({ error: error.message }); 223 | } 224 | }); 225 | 226 | /** 227 | * @api {post} /api/browser/observe/:sessionId Observe Page 228 | * @apiName ObservePage 229 | * @apiGroup Browser 230 | * @apiVersion 1.0.0 231 | * 232 | * @apiDescription Analyzes the page and returns possible actions based on content. 233 | * 234 | * @apiParam {String} sessionId Session's unique identifier 235 | * @apiBody {String} [instruction] Optional instruction to guide observation 236 | * @apiBody {String} [useVision='fallback'] Vision mode: 'fallback' 237 | * @apiBody {String} [modelName] Optional AI model name to use 238 | * 239 | * @apiSuccess {Object} actions List of possible actions and observations 240 | * 241 | * @apiError (Error 500) {Object} error Error object with message 242 | */ 243 | router.post('/observe/:sessionId', async (req, res) => { 244 | try { 245 | const { instruction, useVision, modelName } = req.body; 246 | const stagehand = await ensureStagehand(req.params.sessionId, { modelName }); 247 | 248 | const actions = await stagehand.observe({ 249 | instruction, 250 | useVision: useVision || 'fallback', 251 | modelName 252 | }); 253 | 254 | res.json(actions); 255 | } catch (error) { 256 | res.status(500).json({ error: error.message }); 257 | } 258 | }); 259 | 260 | 261 | /** 262 | * @api {post} /api/browser/screenshot/:sessionId Take Screenshot 263 | * @apiName TakeScreenshot 264 | * @apiGroup Browser 265 | * @apiVersion 1.0.0 266 | * 267 | * @apiDescription Captures a screenshot of the current page state. 268 | * 269 | * @apiParam {String} sessionId Session's unique identifier 270 | * 271 | * @apiSuccess {Binary} image JPEG image data 272 | * 273 | * @apiError (Error 500) {Object} error Error object with message 274 | */ 275 | router.post('/screenshot/:sessionId', async (req, res) => { 276 | try { 277 | const stagehand = await ensureStagehand(req.params.sessionId); 278 | const screenshot = await stagehand.page.screenshot({ 279 | type: 'jpeg', 280 | quality: 80 281 | }); 282 | 283 | res.writeHead(200, { 284 | 'Content-Type': 'image/jpeg', 285 | 'Content-Length': screenshot.length 286 | }); 287 | res.end(screenshot); 288 | } catch (error) { 289 | res.status(500).json({ error: error.message }); 290 | } 291 | }); 292 | 293 | /** 294 | * @api {get} /api/browser/dom-state/:sessionId Get DOM State 295 | * @apiName GetDOMState 296 | * @apiGroup Browser 297 | * @apiVersion 1.0.0 298 | * 299 | * @apiDescription Retrieves the current DOM state as HTML. 300 | * 301 | * @apiParam {String} sessionId Session's unique identifier 302 | * 303 | * @apiSuccess {Object} state Current page HTML 304 | * 305 | * @apiError (Error 500) {Object} error Error object with message 306 | */ 307 | router.get('/dom-state/:sessionId', async (req, res) => { 308 | try { 309 | const stagehand = await ensureStagehand(req.params.sessionId); 310 | const state = await stagehand.page.evaluate(() => document.documentElement.outerHTML); 311 | res.json({ state }); 312 | } catch (error) { 313 | res.status(500).json({ error: error.message }); 314 | } 315 | }); 316 | 317 | /** 318 | * @api {get} /api/browser/console-logs/:sessionId Get Console Logs 319 | * @apiName GetConsoleLogs 320 | * @apiGroup Browser 321 | * @apiVersion 1.0.0 322 | * 323 | * @apiDescription Retrieves filtered console logs from the browser session. 324 | * 325 | * @apiParam {String} sessionId Session's unique identifier 326 | * @apiQuery {Boolean} [levels.error=true] Include error logs 327 | * @apiQuery {Boolean} [levels.warning=false] Include warning logs 328 | * @apiQuery {Boolean} [levels.info=false] Include info logs 329 | * @apiQuery {Boolean} [levels.trace=false] Include trace logs 330 | * @apiQuery {String[]} [includeStringFilters] Array of strings to include (matches message, path, type) 331 | * @apiQuery {String[]} [excludeStringFilters] Array of strings to exclude (matches message, path, type) 332 | * @apiQuery {String} [startTime] Filter logs after this ISO timestamp 333 | * @apiQuery {String} [endTime] Filter logs before this ISO timestamp 334 | * @apiQuery {Number} [truncateLength=500] Maximum length for log messages before truncation 335 | * 336 | * @apiSuccess {Object[]} logs Filtered console logs 337 | * 338 | * @apiError (Error 500) {Object} error Error object with message 339 | */ 340 | router.get('/console-logs/:sessionId', async (req, res) => { 341 | try { 342 | const stagehand = await ensureStagehand(req.params.sessionId); 343 | const logs = await stagehandService.getConsoleLogs(req.params.sessionId, { 344 | levels: { 345 | error: req.query.error !== 'false', 346 | warning: req.query.warning === 'true', 347 | info: req.query.info === 'true', 348 | trace: req.query.trace === 'true' 349 | }, 350 | includeStringFilters: req.query.includeStringFilters, 351 | excludeStringFilters: req.query.excludeStringFilters, 352 | startTime: req.query.startTime, 353 | endTime: req.query.endTime, 354 | truncateLength: req.query.truncateLength ? parseInt(req.query.truncateLength) : undefined 355 | }); 356 | res.json({ success: true, logs }); 357 | } catch (error) { 358 | console.error('Error getting console logs:', error); 359 | res.status(500).json({ success: false, error: error.message }); 360 | } 361 | }); 362 | 363 | /** 364 | * @api {get} /api/browser/network-logs/:sessionId Get Network Logs 365 | * @apiName GetNetworkLogs 366 | * @apiGroup Browser 367 | * @apiVersion 1.0.0 368 | * 369 | * @apiDescription Retrieves filtered network request/response logs. 370 | * 371 | * @apiParam {String} sessionId Session's unique identifier 372 | * @apiQuery {Boolean} [includeHeaders=false] Include request/response headers 373 | * @apiQuery {Boolean} [includeBody=true] Include request/response bodies 374 | * @apiQuery {Boolean} [includeInfo=true] Include informational responses (100-199) 375 | * @apiQuery {Boolean} [includeSuccess=true] Include successful responses (200-299) 376 | * @apiQuery {Boolean} [includeRedirect=true] Include redirection responses (300-399) 377 | * @apiQuery {Boolean} [includeClientError=true] Include client error responses (400-499) 378 | * @apiQuery {Boolean} [includeServerError=true] Include server error responses (500-599) 379 | * @apiQuery {String[]} [includeStringFilters] Array of strings to include (matches URL, method, or headers) 380 | * @apiQuery {String[]} [excludeStringFilters] Array of strings to exclude (matches URL, method, or headers) 381 | * @apiQuery {String} [startTime] Filter logs after this ISO timestamp 382 | * @apiQuery {String} [endTime] Filter logs before this ISO timestamp 383 | * @apiQuery {Number} [truncateLength=500] Maximum length for request/response bodies before truncation 384 | * 385 | * @apiSuccess {Object[]} logs Filtered network logs 386 | */ 387 | router.get('/network-logs/:sessionId', async (req, res) => { 388 | try { 389 | const stagehand = await ensureStagehand(req.params.sessionId); 390 | const logs = await stagehandService.getNetworkLogs(req.params.sessionId, { 391 | statusCodes: { 392 | info: req.query.includeInfo !== 'false', 393 | success: req.query.includeSuccess !== 'false', 394 | redirect: req.query.includeRedirect !== 'false', 395 | clientError: req.query.includeClientError !== 'false', 396 | serverError: req.query.includeServerError !== 'false' 397 | }, 398 | includeHeaders: req.query.includeHeaders === 'true', 399 | includeBody: req.query.includeBody !== 'false', 400 | includeQueryParams: req.query.includeQueryParams !== 'false', 401 | includeStringFilters: req.query.includeStringFilters, 402 | excludeStringFilters: req.query.excludeStringFilters, 403 | startTime: req.query.startTime, 404 | endTime: req.query.endTime, 405 | truncateLength: req.query.truncateLength ? parseInt(req.query.truncateLength) : undefined 406 | }); 407 | 408 | res.json({ success: true, logs }); 409 | } catch (error) { 410 | console.error('Error getting network logs:', error); 411 | res.status(500).json({ success: false, error: error.message }); 412 | } 413 | }); 414 | 415 | /** 416 | * @api {post} /api/browser/clear-logs/:sessionId Clear Logs 417 | * @apiName ClearLogs 418 | * @apiGroup Browser 419 | * @apiVersion 1.0.0 420 | * 421 | * @apiDescription Clears all logs for the specified session. 422 | * 423 | * @apiParam {String} sessionId Session's unique identifier 424 | * 425 | * @apiSuccess {Boolean} success Operation success status 426 | * 427 | * @apiError (Error 500) {Object} error Error object with message 428 | */ 429 | router.post('/clear-logs/:sessionId', (req, res) => { 430 | try { 431 | const logs = stagehandService.getLogs(req.params.sessionId); 432 | if (!logs) throw new Error('Session not initialized'); 433 | 434 | logs.console = []; 435 | logs.network = []; 436 | 437 | res.json({ success: true }); 438 | } catch (error) { 439 | res.status(500).json({ error: error.message }); 440 | } 441 | }); 442 | 443 | 444 | 445 | module.exports = router; -------------------------------------------------------------------------------- /streamlit_ui.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | import requests 3 | import json 4 | from PIL import Image 5 | from io import BytesIO 6 | from datetime import datetime 7 | 8 | API_URL = "http://localhost:3000/api" 9 | 10 | # Add constants for default values 11 | DEFAULT_NAVIGATION_TIMEOUT = 30000 12 | DEFAULT_VISION_MODE = 'fallback' 13 | 14 | # Default log filter values matching browser-routes.js 15 | DEFAULT_LOG_FILTERS = { 16 | 'console': { 17 | 'levels': { 18 | 'error': True, 19 | 'warning': False, 20 | 'info': False, 21 | 'trace': False 22 | }, 23 | 'truncateLength': 500, 24 | 'includeStringFilters': [], 25 | 'excludeStringFilters': [] 26 | }, 27 | 'network': { 28 | 'statusCodes': { 29 | 'info': True, 30 | 'success': True, 31 | 'redirect': True, 32 | 'clientError': True, 33 | 'serverError': True 34 | }, 35 | 'includeHeaders': False, 36 | 'includeBody': True, 37 | 'includeQueryParams': True, 38 | 'truncateLength': 500, 39 | 'includeStringFilters': [], 40 | 'excludeStringFilters': [] 41 | } 42 | } 43 | 44 | def create_session(): 45 | response = requests.post(f"{API_URL}/sessions/create-session") 46 | return response.json() 47 | 48 | def stop_session(session_id): 49 | response = requests.post(f"{API_URL}/sessions/stop-session/{session_id}") 50 | return response.json() 51 | 52 | def navigate(session_id, url): 53 | """Navigate browser with default timeout""" 54 | response = requests.post( 55 | f"{API_URL}/browser/navigate/{session_id}", 56 | json={ 57 | "url": url, 58 | "timeout": DEFAULT_NAVIGATION_TIMEOUT 59 | } 60 | ) 61 | return response.json() 62 | 63 | def perform_action(session_id, action, use_vision=DEFAULT_VISION_MODE, model_name=None, include_logs=False, log_filters=None): 64 | """Perform browser action with default vision mode and log filters""" 65 | payload = { 66 | "action": action, 67 | "useVision": use_vision or DEFAULT_VISION_MODE, 68 | "modelName": model_name, 69 | "includeLogs": include_logs, 70 | "logFilters": log_filters or DEFAULT_LOG_FILTERS if include_logs else None 71 | } 72 | response = requests.post(f"{API_URL}/browser/act/{session_id}", json=payload) 73 | return response.json() 74 | 75 | def extract_data(session_id, instruction, schema, model_name=None): 76 | response = requests.post(f"{API_URL}/browser/extract/{session_id}", json={ 77 | "instruction": instruction, 78 | "schema": schema, 79 | "modelName": model_name 80 | }) 81 | return response.json() 82 | 83 | def observe_page(session_id, instruction=None, use_vision=DEFAULT_VISION_MODE, model_name=None): 84 | """Observe page with default vision mode""" 85 | payload = { 86 | "instruction": instruction, 87 | "useVision": use_vision or DEFAULT_VISION_MODE, 88 | "modelName": model_name 89 | } 90 | response = requests.post(f"{API_URL}/browser/observe/{session_id}", json=payload) 91 | return response.json() 92 | 93 | def get_screenshot(session_id): 94 | response = requests.post(f"{API_URL}/browser/screenshot/{session_id}") 95 | if response.status_code == 200: 96 | return Image.open(BytesIO(response.content)) 97 | return None 98 | 99 | def get_dom_state(session_id): 100 | response = requests.get(f"{API_URL}/browser/dom-state/{session_id}") 101 | return response.json() 102 | 103 | def get_console_logs(session_id, filters=None): 104 | """Get console logs with default filters""" 105 | params = { 106 | 'error': str(filters.get('levels', {}).get('error', DEFAULT_LOG_FILTERS['console']['levels']['error'])).lower(), 107 | 'warning': str(filters.get('levels', {}).get('warning', DEFAULT_LOG_FILTERS['console']['levels']['warning'])).lower(), 108 | 'info': str(filters.get('levels', {}).get('info', DEFAULT_LOG_FILTERS['console']['levels']['info'])).lower(), 109 | 'trace': str(filters.get('levels', {}).get('trace', DEFAULT_LOG_FILTERS['console']['levels']['trace'])).lower(), 110 | 'truncateLength': filters.get('truncateLength', DEFAULT_LOG_FILTERS['console']['truncateLength']) 111 | } 112 | 113 | # Properly handle string filters as arrays 114 | if filters.get('includeStringFilters'): 115 | # Convert to list if it's a string 116 | if isinstance(filters['includeStringFilters'], str): 117 | filters['includeStringFilters'] = [f.strip() for f in filters['includeStringFilters'].split('\n') if f.strip()] 118 | params['includeStringFilters[]'] = filters['includeStringFilters'] 119 | 120 | if filters.get('excludeStringFilters'): 121 | # Convert to list if it's a string 122 | if isinstance(filters['excludeStringFilters'], str): 123 | filters['excludeStringFilters'] = [f.strip() for f in filters['excludeStringFilters'].split('\n') if f.strip()] 124 | params['excludeStringFilters[]'] = filters['excludeStringFilters'] 125 | 126 | response = requests.get(f"{API_URL}/browser/console-logs/{session_id}", params=params) 127 | return response.json() 128 | 129 | def get_network_logs(session_id, filters=None): 130 | """Get network logs with default filters""" 131 | default_network = DEFAULT_LOG_FILTERS['network'] 132 | params = { 133 | 'includeInfo': str(filters.get('statusCodes', {}).get('info', default_network['statusCodes']['info'])).lower(), 134 | 'includeSuccess': str(filters.get('statusCodes', {}).get('success', default_network['statusCodes']['success'])).lower(), 135 | 'includeRedirect': str(filters.get('statusCodes', {}).get('redirect', default_network['statusCodes']['redirect'])).lower(), 136 | 'includeClientError': str(filters.get('statusCodes', {}).get('clientError', default_network['statusCodes']['clientError'])).lower(), 137 | 'includeServerError': str(filters.get('statusCodes', {}).get('serverError', default_network['statusCodes']['serverError'])).lower(), 138 | 'includeHeaders': str(filters.get('includeHeaders', default_network['includeHeaders'])).lower(), 139 | 'includeBody': str(filters.get('includeBody', default_network['includeBody'])).lower(), 140 | 'includeQueryParams': str(filters.get('includeQueryParams', default_network['includeQueryParams'])).lower(), 141 | 'truncateLength': filters.get('truncateLength', default_network['truncateLength']) 142 | } 143 | 144 | # Properly handle string filters as arrays 145 | if filters.get('includeStringFilters'): 146 | # Convert to list if it's a string 147 | if isinstance(filters['includeStringFilters'], str): 148 | filters['includeStringFilters'] = [f.strip() for f in filters['includeStringFilters'].split('\n') if f.strip()] 149 | params['includeStringFilters[]'] = filters['includeStringFilters'] 150 | 151 | if filters.get('excludeStringFilters'): 152 | # Convert to list if it's a string 153 | if isinstance(filters['excludeStringFilters'], str): 154 | filters['excludeStringFilters'] = [f.strip() for f in filters['excludeStringFilters'].split('\n') if f.strip()] 155 | params['excludeStringFilters[]'] = filters['excludeStringFilters'] 156 | 157 | response = requests.get(f"{API_URL}/browser/network-logs/{session_id}", params=params) 158 | return response.json() 159 | 160 | def clear_logs(session_id): 161 | response = requests.post(f"{API_URL}/browser/clear-logs/{session_id}") 162 | return response.json() 163 | 164 | 165 | 166 | def get_running_sessions(): 167 | response = requests.get(f"{API_URL}/sessions/running-sessions") 168 | return response.json() 169 | 170 | def get_session_details(session_id): 171 | response = requests.get(f"{API_URL}/sessions/session/{session_id}") 172 | return response.json() 173 | 174 | def render_network_log(log, parent_container): 175 | """Renders a single network log entry with expandable sections""" 176 | status = log.get('status', 0) 177 | status_color = { 178 | range(100, 200): 'blue', 179 | range(200, 300): 'green', 180 | range(300, 400): 'orange', 181 | range(400, 500): 'red', 182 | range(500, 600): 'purple' 183 | } 184 | log_color = next((color for range_obj, color in status_color.items() 185 | if status in range_obj), 'black') 186 | 187 | parent_container.markdown(f""" 188 |
189 | [{status}] 190 | {log.get('method', '')} {log.get('url', '')}
191 | ({log.get('timestamp', '')}) 192 |
193 | """, unsafe_allow_html=True) 194 | 195 | # Request Details 196 | cols = parent_container.columns([1, 1, 1]) 197 | 198 | # Column 1: Request Details 199 | with cols[0]: 200 | st.markdown("**Request Details:**") 201 | if log.get('request', {}).get('queryParams'): 202 | st.markdown("*Query Parameters:*") 203 | st.code(log['request']['queryParams']) 204 | 205 | if log.get('request', {}).get('headers'): 206 | st.markdown("*Headers:*") 207 | st.json(log['request']['headers']) 208 | 209 | if log.get('request', {}).get('body'): 210 | st.markdown("*Body:*") 211 | try: 212 | st.json(json.loads(log['request']['body'])) 213 | except: 214 | st.code(log['request']['body']) 215 | 216 | # Column 2: Response Details 217 | with cols[1]: 218 | st.markdown("**Response Details:**") 219 | if log.get('response', {}).get('headers'): 220 | st.markdown("*Headers:*") 221 | st.json(log['response']['headers']) 222 | 223 | if log.get('response', {}).get('body'): 224 | st.markdown("*Body:*") 225 | try: 226 | st.json(json.loads(log['response']['body'])) 227 | except: 228 | st.code(log['response']['body']) 229 | 230 | # Column 3: Timing Information 231 | with cols[2]: 232 | if log.get('timing'): 233 | st.markdown("**Timing Information:**") 234 | st.json(log['timing']) 235 | 236 | parent_container.markdown("---") 237 | 238 | def render_console_log(log, parent_container): 239 | """Renders a single console log entry with proper formatting""" 240 | log_color = { 241 | 'error': 'red', 242 | 'warning': 'orange', 243 | 'info': 'blue', 244 | 'log': 'green', 245 | 'trace': 'gray' 246 | }.get(log.get('type', 'log'), 'black') 247 | 248 | parent_container.markdown(f""" 249 |
250 | [{log.get('type', '').upper()}] 251 | ({log.get('timestamp', '')})
252 | Message: {log.get('message', '')}
253 | Path: {log.get('path', '')} 254 |
255 | """, unsafe_allow_html=True) 256 | 257 | cols = parent_container.columns([1, 1]) 258 | 259 | # Column 1: Stack Trace 260 | with cols[0]: 261 | if log.get('stackTrace'): 262 | st.markdown("**Stack Trace:**") 263 | st.code(log['stackTrace']) 264 | 265 | # Column 2: Arguments 266 | with cols[1]: 267 | if log.get('args'): 268 | st.markdown("**Arguments:**") 269 | st.json(log['args']) 270 | 271 | parent_container.markdown("---") 272 | 273 | def render_action_result(result): 274 | """Renders an action result with associated logs""" 275 | # Main result 276 | st.success("Action completed successfully") 277 | st.json(result.get('result', {})) 278 | 279 | # Logs section 280 | if result.get('logs'): 281 | st.markdown("### Action Logs") 282 | 283 | # Console logs tab and Network logs tab 284 | log_tabs = st.tabs(["Console Logs", "Network Logs"]) 285 | 286 | # Console logs 287 | with log_tabs[0]: 288 | if result['logs'].get('console'): 289 | for log in result['logs']['console']: 290 | render_console_log(log, st) 291 | else: 292 | st.info("No console logs available") 293 | 294 | # Network logs 295 | with log_tabs[1]: 296 | if result['logs'].get('network'): 297 | for log in result['logs']['network']: 298 | render_network_log(log, st) 299 | else: 300 | st.info("No network logs available") 301 | 302 | def main(): 303 | st.title("Web Browser AI-Control API Server") 304 | 305 | # Session Management 306 | if 'session_id' not in st.session_state: 307 | st.session_state.session_id = None 308 | 309 | # Sidebar for session management and configuration 310 | st.sidebar.subheader("Session Management") 311 | 312 | # Session creation/resumption options 313 | session_mode = st.sidebar.radio( 314 | "Session Mode", 315 | ["Create New Session", "Resume Existing Session"] 316 | ) 317 | 318 | if session_mode == "Resume Existing Session": 319 | # Add this section to show running sessions 320 | try: 321 | running_sessions = get_running_sessions() 322 | if running_sessions.get('success') and running_sessions.get('sessions'): 323 | st.sidebar.subheader("Running Sessions") 324 | sessions = running_sessions['sessions'] 325 | 326 | # Create a formatted selection for each session 327 | session_options = { 328 | f"Session {s['id']} ({s['region']})": s['id'] 329 | for s in sessions 330 | } 331 | 332 | selected_session = st.sidebar.selectbox( 333 | "Select Running Session", 334 | options=list(session_options.keys()), 335 | help="Select an existing running session" 336 | ) 337 | 338 | if selected_session: 339 | manual_session_id = session_options[selected_session] 340 | 341 | except Exception as e: 342 | st.sidebar.error(f"Error fetching running sessions: {str(e)}") 343 | 344 | # Keep the manual input as fallback 345 | manual_session_id = st.sidebar.text_input( 346 | "Or Enter Session ID Manually", 347 | value=manual_session_id if 'manual_session_id' in locals() else "", 348 | help="Enter an existing Browserbase session ID" 349 | ) 350 | 351 | if st.sidebar.button("Resume Session"): 352 | try: 353 | st.session_state.session_id = manual_session_id 354 | st.sidebar.success(f"Resumed session: {manual_session_id}") 355 | 356 | # Display Browserbase session URL 357 | browserbase_url = f"https://www.browserbase.com/sessions/{manual_session_id}" 358 | st.sidebar.markdown("### Session URLs") 359 | st.sidebar.markdown(f"💻 [Browserbase Session]({browserbase_url})") 360 | 361 | 362 | st.rerun() 363 | except Exception as e: 364 | st.sidebar.error(f"Error resuming session: {str(e)}") 365 | else: 366 | if not st.session_state.session_id and st.sidebar.button("Create New Session"): 367 | try: 368 | result = create_session() 369 | if result.get('success'): 370 | st.session_state.session_id = result['sessionId'] 371 | st.sidebar.success("Session created successfully") 372 | 373 | # Display Browserbase session URL 374 | browserbase_url = f"https://www.browserbase.com/sessions/{result['sessionId']}" 375 | st.sidebar.markdown("### Session URLs") 376 | st.sidebar.markdown(f"💻 [Browserbase Session]({browserbase_url})") 377 | 378 | 379 | st.rerun() 380 | else: 381 | st.sidebar.error("Failed to create session") 382 | except Exception as e: 383 | st.sidebar.error(f"Error creating session: {str(e)}") 384 | 385 | # Display selected session info 386 | if st.session_state.session_id: 387 | st.sidebar.success(f"Selected Session: {st.session_state.session_id}") 388 | 389 | # Add session details 390 | try: 391 | session_details = get_session_details(st.session_state.session_id) 392 | if session_details.get('success'): 393 | session = session_details['session'] 394 | st.sidebar.markdown("### Session Details") 395 | st.sidebar.markdown(f""" 396 | - **Status**: {session.get('status')} 397 | - **Region**: {session.get('region')} 398 | - **Created**: {session.get('createdAt')} 399 | - **Last Updated**: {session.get('updatedAt')} 400 | """) 401 | except Exception as e: 402 | st.sidebar.error(f"Error fetching session details: {str(e)}") 403 | 404 | # Display Browserbase and debug URLs in sidebar 405 | browserbase_url = f"https://www.browserbase.com/sessions/{st.session_state.session_id}" 406 | st.sidebar.markdown("### Session URLs") 407 | st.sidebar.markdown(f"💻 [Browserbase Session]({browserbase_url})") 408 | 409 | if st.sidebar.button("Stop Session"): 410 | try: 411 | # Always call the stop session API regardless of session mode 412 | result = stop_session(st.session_state.session_id) 413 | if result.get('success'): 414 | st.session_state.session_id = None 415 | st.sidebar.success("Session stopped successfully") 416 | st.rerun() 417 | else: 418 | st.sidebar.error("Failed to stop session") 419 | except Exception as e: 420 | st.sidebar.error(f"Error stopping session: {str(e)}") 421 | # Log the full error for debugging 422 | print(f"Session stop error: {str(e)}") 423 | 424 | # Model selection 425 | st.sidebar.subheader("Model Configuration") 426 | model_options = [ 427 | "claude-3-5-sonnet-latest", 428 | "claude-3-5-sonnet-20240620", 429 | "claude-3-5-sonnet-20241022", 430 | "gpt-4o", 431 | "gpt-4o-mini", 432 | "gpt-4o-2024-08-06", 433 | ] 434 | selected_model = st.sidebar.selectbox("Select Model", model_options) 435 | 436 | # Main content area - only show if we have an active session 437 | if not st.session_state.session_id: 438 | st.warning("Please select or create a browser session.") 439 | return 440 | 441 | # Navigation section 442 | st.subheader("Navigation") 443 | url = st.text_input("Enter URL") 444 | if st.button("Navigate"): 445 | with st.spinner("Navigating..."): 446 | result = navigate(st.session_state.session_id, url) 447 | if result.get("success"): 448 | st.success("Navigation successful!") 449 | else: 450 | st.error(f"Navigation failed: {result.get('error', 'Unknown error')}") 451 | 452 | # Enhanced Action section 453 | st.subheader("Perform Action") 454 | 455 | action = st.text_input("Enter action to perform") 456 | col1, col2 = st.columns(2) 457 | with col1: 458 | use_vision = st.checkbox("Use Vision", key="action_use_vision") 459 | with col2: 460 | include_logs = st.checkbox("Include Action Logs", 461 | help="Include logs generated during the action execution", 462 | key="action_include_logs" 463 | ) 464 | 465 | # Add log filters when including logs 466 | log_filters = None 467 | if include_logs: 468 | with st.expander("Log Filters", expanded=True): 469 | st.subheader("Console Log Filters") 470 | col1, col2 = st.columns(2) 471 | with col1: 472 | console_truncate_length = st.number_input( 473 | "Console Truncate Length", 474 | value=500, 475 | step=10, 476 | help="Maximum length of console messages before truncation", 477 | key="action_console_truncate_length" 478 | ) 479 | console_include_errors = st.checkbox("Include Errors", value=True, key="action_console_include_errors") 480 | console_include_warnings = st.checkbox("Include Warnings", key="action_console_include_warnings") 481 | console_include_info = st.checkbox("Include Info", key="action_console_include_info") 482 | console_include_trace = st.checkbox("Include Trace", key="action_console_include_trace") 483 | 484 | with col2: 485 | console_include_string_filters = st.text_area( 486 | "Include Strings (one per line)", 487 | help="Enter strings to include in results. Logs matching any string will be shown.", 488 | key="action_console_include_string_filters" 489 | ) 490 | 491 | console_exclude_string_filters = st.text_area( 492 | "Exclude Strings (one per line)", 493 | help="Enter strings to exclude from results. Logs matching any string will be hidden.", 494 | key="action_console_exclude_string_filters" 495 | ) 496 | 497 | st.subheader("Network Log Filters") 498 | col1, col2 = st.columns(2) 499 | with col1: 500 | network_truncate_length = st.number_input( 501 | "Network Truncate Length", 502 | value=500, 503 | step=10, 504 | help="Maximum length of request/response bodies before truncation", 505 | key="action_network_truncate_length" 506 | ) 507 | st.write("Status Codes:") 508 | network_include_info = st.checkbox("1xx (Informational)", value=False, key="action_network_include_info") 509 | network_include_success = st.checkbox("2xx (Success)", value=False, key="action_network_include_success") 510 | network_include_redirect = st.checkbox("3xx (Redirect)", value=False, key="action_network_include_redirect") 511 | network_include_client_error = st.checkbox("4xx (Client Error)", value=True, key="action_network_include_client_error") 512 | network_include_server_error = st.checkbox("5xx (Server Error)", value=True, key="action_network_include_server_error") 513 | 514 | with col2: 515 | st.write("Request Details:") 516 | network_include_request_headers = st.checkbox("Include Request Headers", value=False, key="action_network_include_request_headers") 517 | network_include_request_body = st.checkbox("Include Request Body", value=False, key="action_network_include_request_body") 518 | network_include_request_params = st.checkbox("Include Query Parameters", value=False, key="action_network_include_request_params") 519 | 520 | st.write("Response Details:") 521 | network_include_response_headers = st.checkbox("Include Response Headers", value=False, key="action_network_include_response_headers") 522 | network_include_response_body = st.checkbox("Include Response Body", value=False, key="action_network_include_response_body") 523 | 524 | network_include_string_filters = st.text_area( 525 | "Include Strings (one per line)", 526 | help="Enter strings to include in results. Requests matching any string will be shown.", 527 | key="action_network_include_string_filters" 528 | ) 529 | 530 | network_exclude_string_filters = st.text_area( 531 | "Exclude Strings (one per line)", 532 | help="Enter strings to exclude from results. Requests matching any string will be hidden.", 533 | key="action_network_exclude_string_filters" 534 | ) 535 | 536 | log_filters = { 537 | "console": { 538 | "includeErrors": console_include_errors, 539 | "includeWarnings": console_include_warnings, 540 | "includeInfo": console_include_info, 541 | "includeTrace": console_include_trace, 542 | "truncateLength": console_truncate_length, 543 | "includeStringFilters": [f.strip() for f in console_include_string_filters.split('\n') if f.strip()], 544 | "excludeStringFilters": [f.strip() for f in console_exclude_string_filters.split('\n') if f.strip()] 545 | }, 546 | "network": { 547 | "statusCodes": { 548 | "info": network_include_info, 549 | "success": network_include_success, 550 | "redirect": network_include_redirect, 551 | "clientError": network_include_client_error, 552 | "serverError": network_include_server_error 553 | }, 554 | "request": { 555 | "includeHeaders": network_include_request_headers, 556 | "includeBody": network_include_request_body, 557 | "includeQueryParams": network_include_request_params 558 | }, 559 | "response": { 560 | "includeHeaders": network_include_response_headers, 561 | "includeBody": network_include_response_body 562 | }, 563 | "truncateLength": network_truncate_length, 564 | "includeStringFilters": [f.strip() for f in network_include_string_filters.split('\n') if f.strip()], 565 | "excludeStringFilters": [f.strip() for f in network_exclude_string_filters.split('\n') if f.strip()] 566 | } 567 | } 568 | 569 | if st.button("Execute Action"): 570 | with st.spinner("Executing action..."): 571 | result = perform_action( 572 | st.session_state.session_id, 573 | action, 574 | use_vision, 575 | selected_model, 576 | include_logs, 577 | log_filters 578 | ) 579 | render_action_result(result) 580 | 581 | # Extract Data section 582 | st.subheader("Extract Data") 583 | instruction = st.text_input("Enter extraction instruction") 584 | schema = st.text_area("Enter schema (JSON)", value='{"example": "string"}') 585 | if st.button("Extract"): 586 | try: 587 | schema_dict = json.loads(schema) 588 | with st.spinner("Extracting data..."): 589 | result = extract_data( 590 | st.session_state.session_id, 591 | instruction, 592 | schema_dict, 593 | selected_model 594 | ) 595 | st.json(result) 596 | except json.JSONDecodeError: 597 | st.error("Invalid JSON schema") 598 | 599 | # Observe Page section 600 | st.subheader("Observe Page") 601 | observe_instruction = st.text_input("Enter observation instruction (optional)") 602 | use_vision_observe = st.checkbox("Use Vision for Observation", key="observe_use_vision") 603 | if st.button("Observe"): 604 | with st.spinner("Observing page..."): 605 | result = observe_page( 606 | st.session_state.session_id, 607 | observe_instruction, 608 | use_vision_observe, 609 | selected_model 610 | ) 611 | st.json(result) 612 | 613 | # DOM State section 614 | st.subheader("DOM State") 615 | if st.button("View DOM State"): 616 | with st.spinner("Fetching DOM state..."): 617 | result = get_dom_state(st.session_state.session_id) 618 | if result.get("state"): 619 | with st.expander("DOM State", expanded=True): 620 | st.code(result["state"], language="html") 621 | 622 | # Screenshot section 623 | st.subheader("Page Screenshot") 624 | if st.button("Take Screenshot"): 625 | with st.spinner("Taking screenshot..."): 626 | screenshot = get_screenshot(st.session_state.session_id) 627 | if screenshot: 628 | st.image(screenshot, use_column_width=True) 629 | 630 | # Logs section 631 | st.subheader("Logs") 632 | log_type = st.radio("Log Type", ["Console", "Network"]) 633 | 634 | with st.expander("Log Filters", expanded=True): 635 | if log_type == "Console": 636 | col1, col2 = st.columns(2) 637 | with col1: 638 | truncate_length = st.number_input( 639 | "Truncate Length", 640 | value=DEFAULT_LOG_FILTERS['console']['truncateLength'], 641 | step=10, 642 | help="Maximum length of log messages before truncation", 643 | key="console_truncate_length" 644 | ) 645 | include_errors = st.checkbox( 646 | "Include Errors", 647 | value=DEFAULT_LOG_FILTERS['console']['levels']['error'], 648 | key="console_include_errors" 649 | ) 650 | include_warnings = st.checkbox( 651 | "Include Warnings", 652 | value=DEFAULT_LOG_FILTERS['console']['levels']['warning'], 653 | key="console_include_warnings" 654 | ) 655 | include_info = st.checkbox( 656 | "Include Info", 657 | value=DEFAULT_LOG_FILTERS['console']['levels']['info'], 658 | key="console_include_info" 659 | ) 660 | include_trace = st.checkbox( 661 | "Include Trace", 662 | value=DEFAULT_LOG_FILTERS['console']['levels']['trace'], 663 | key="console_include_trace" 664 | ) 665 | 666 | with col2: 667 | include_string_filters = st.text_area( 668 | "Include Strings (one per line)", 669 | help="Enter strings to include in results. Logs matching any string will be shown.", 670 | key="console_include_string_filters" 671 | ) 672 | 673 | exclude_string_filters = st.text_area( 674 | "Exclude Strings (one per line)", 675 | help="Enter strings to exclude from results. Logs matching any string will be hidden.", 676 | key="console_exclude_string_filters" 677 | ) 678 | 679 | use_time_filter = st.checkbox("Filter by Time", value=False, key="console_use_time_filter") 680 | if use_time_filter: 681 | start_time = st.time_input("Start Time", key="console_start_time") 682 | end_time = st.time_input("End Time", key="console_end_time") 683 | today = datetime.now().date() 684 | start_datetime = datetime.combine(today, start_time).isoformat() if start_time else None 685 | end_datetime = datetime.combine(today, end_time).isoformat() if end_time else None 686 | 687 | filters = { 688 | 'levels': { 689 | 'error': include_errors, 690 | 'warning': include_warnings, 691 | 'info': include_info, 692 | 'trace': include_trace 693 | }, 694 | 'truncateLength': truncate_length, 695 | 'includeStringFilters': [f.strip() for f in include_string_filters.split('\n') if f.strip()], 696 | 'excludeStringFilters': [f.strip() for f in exclude_string_filters.split('\n') if f.strip()] 697 | } 698 | if use_time_filter: 699 | filters.update({ 700 | 'startTime': start_datetime, 701 | 'endTime': end_datetime 702 | }) 703 | 704 | else: # Network logs 705 | col1, col2 = st.columns(2) 706 | with col1: 707 | truncate_length = st.number_input( 708 | "Truncate Length", 709 | value=DEFAULT_LOG_FILTERS['network']['truncateLength'], 710 | step=10, 711 | help="Maximum length of request/response bodies before truncation", 712 | key="network_truncate_length" 713 | ) 714 | st.write("Status Codes:") 715 | include_info = st.checkbox( 716 | "1xx (Informational)", 717 | value=DEFAULT_LOG_FILTERS['network']['statusCodes']['info'], 718 | key="network_include_info" 719 | ) 720 | include_success = st.checkbox( 721 | "2xx (Success)", 722 | value=DEFAULT_LOG_FILTERS['network']['statusCodes']['success'], 723 | key="network_include_success" 724 | ) 725 | include_redirect = st.checkbox( 726 | "3xx (Redirect)", 727 | value=DEFAULT_LOG_FILTERS['network']['statusCodes']['redirect'], 728 | key="network_include_redirect" 729 | ) 730 | include_client_error = st.checkbox( 731 | "4xx (Client Error)", 732 | value=DEFAULT_LOG_FILTERS['network']['statusCodes']['clientError'], 733 | key="network_include_client_error" 734 | ) 735 | include_server_error = st.checkbox( 736 | "5xx (Server Error)", 737 | value=DEFAULT_LOG_FILTERS['network']['statusCodes']['serverError'], 738 | key="network_include_server_error" 739 | ) 740 | 741 | with col2: 742 | st.write("Request Details:") 743 | include_request_headers = st.checkbox( 744 | "Include Request Headers", 745 | value=DEFAULT_LOG_FILTERS['network']['includeHeaders'], 746 | key="network_include_request_headers" 747 | ) 748 | include_request_body = st.checkbox( 749 | "Include Request Body", 750 | value=DEFAULT_LOG_FILTERS['network']['includeBody'], 751 | key="network_include_request_body" 752 | ) 753 | include_request_params = st.checkbox( 754 | "Include Query Parameters", 755 | value=DEFAULT_LOG_FILTERS['network']['includeQueryParams'], 756 | key="network_include_request_params" 757 | ) 758 | 759 | st.write("Response Details:") 760 | include_response_headers = st.checkbox( 761 | "Include Response Headers", 762 | value=DEFAULT_LOG_FILTERS['network']['includeHeaders'], 763 | key="network_include_response_headers" 764 | ) 765 | include_response_body = st.checkbox( 766 | "Include Response Body", 767 | value=DEFAULT_LOG_FILTERS['network']['includeBody'], 768 | key="network_include_response_body" 769 | ) 770 | 771 | include_string_filters = st.text_area( 772 | "Include Strings (one per line)", 773 | help="Enter strings to include in results. Requests matching any string will be shown.", 774 | key="logs_network_include_string_filters" 775 | ) 776 | 777 | exclude_string_filters = st.text_area( 778 | "Exclude Strings (one per line)", 779 | help="Enter strings to exclude from results. Requests matching any string will be hidden.", 780 | key="logs_network_exclude_string_filters" 781 | ) 782 | 783 | filters = { 784 | 'statusCodes': { 785 | 'info': include_info, 786 | 'success': include_success, 787 | 'redirect': include_redirect, 788 | 'clientError': include_client_error, 789 | 'serverError': include_server_error 790 | }, 791 | 'includeHeaders': include_request_headers or include_response_headers, 792 | 'includeBody': include_request_body or include_response_body, 793 | 'includeQueryParams': include_request_params, 794 | 'truncateLength': truncate_length, 795 | 'includeStringFilters': [f.strip() for f in include_string_filters.split('\n') if f.strip()], 796 | 'excludeStringFilters': [f.strip() for f in exclude_string_filters.split('\n') if f.strip()] 797 | } 798 | 799 | if st.button("View Logs"): 800 | with st.spinner("Fetching logs..."): 801 | if log_type == "Console": 802 | result = get_console_logs(st.session_state.session_id, filters) 803 | if result.get('success') and result.get('logs'): 804 | for log in result['logs']: 805 | render_console_log(log, st) 806 | else: 807 | st.info("No console logs found") 808 | else: 809 | result = get_network_logs(st.session_state.session_id, filters) 810 | if result.get('success') and result.get('logs'): 811 | for log in result['logs']: 812 | render_network_log(log, st) 813 | else: 814 | st.info("No network logs found") 815 | 816 | if st.button("Clear Logs"): 817 | with st.spinner("Clearing logs..."): 818 | result = clear_logs(st.session_state.session_id) 819 | if result.get("success"): 820 | st.success("Logs cleared successfully") 821 | else: 822 | st.error("Failed to clear logs") 823 | 824 | if __name__ == "__main__": 825 | main() -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Browser Control API Documentation 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 55 | 56 | 57 | 85 | 86 | 91 | 92 | 97 | 98 | 109 | 110 | 123 | 124 | 213 | 214 | 259 | 260 | 311 | 312 | 381 | 382 | 604 | 605 | 721 | 722 | 790 | 791 | 900 | 901 | 918 | 919 | 936 | 937 | 1016 | 1017 |
1018 |
1019 |
1020 |
1021 |
1022 | 1023 |
1024 | 1025 |
1026 |
1027 |
1028 |
1029 | 1030 |
1031 | 1043 |
1044 | 1045 | 1046 | 1047 | 1048 | --------------------------------------------------------------------------------