|
37 |
38 |
55 | Event (0)
56 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 | |
57 | 58 | 59 | Target 60 | | 61 |62 | Details 63 | | 64 |65 | 66 | 67 | | 68 |
|---|
(disconnected)
15 | 32 |
6 | 
7 |
8 |
7 | 9 |
10 | 
12 | 
14 | 
16 |
17 |
18 |
19 |
20 | OTA WAP Browser Action Capturer
21 | ======================
22 |
23 | OTA browser action capturer is a simple tool which helps you to collect the interactions with browser such as click, typing etc. and transforms them to well structured data for generating LLM-powered "records and replay" instructions. The action data will be organized in a JSON format and sent to your local WAP server of data collection.
24 |
25 | To deploy the local WAP server, please refer to: https://github.com/OTA-Tech-AI/webagentprotocol
26 | WAP (Web Agent protocol) is our standard protocol for AI Agent record-and-play inferencing.
27 |
28 |
29 | Installation
30 | -----
31 |
32 | Install our public **Chrome extension** at [WAP Browser Action Capturer](https://chromewebstore.google.com/detail/wap-browser-action-captur/chikiefojkdpmijbhepipdkadcljlbmh).
33 |
34 | If you want to install this extension locally, please refer to: https://developer.chrome.com/docs/extensions/get-started/tutorial/hello-world
35 |
36 | Usage of Action Capturer
37 | -----
38 |
39 | ### Perpare
40 |
41 | Open Chrome DevTools and navigate to the **"OTA Action Capturer"** panel and you will see:
42 |
43 |
12 |
14 |
16 | 17 |
44 |
45 | Make sure the IP address and port in Settings are correct:
46 |
47 | 
48 |
49 | ### Start a record
50 |
51 | Clearly describe the task you will be working on and click "START RECORD":
52 |
53 | 
54 |
55 | The capturer will record actions such as clicking, typing, navigating etc. only in the current page.
56 |
57 | If the HTML content in the page changed, the event table will present the added/removed/changed nodes. The information of changing will be collected and sent to your local WAP server.
58 |
59 | 
60 |
61 | An example of the formatted data which you will received in the WAP backend server is like:
62 |
63 | ```json
64 | {
65 | "taskId": "MkCAhQsHgXn7YgaK",
66 | "type": "click",
67 | "actionTimestamp": 1746325231479,
68 | "eventTarget": {
69 | "type": "click",
70 | "target": "\n...",
71 | "targetId": "mntl-card-list-card--extendable_3-0",
72 | "targetClass": "comp mntl-card-list-card--extendable mntl-universal-card mntl-document-card mntl-card card card--no-image"
73 | },
74 | "allEvents": {},
75 | "pageHTMLContent": "
..."
76 | }
77 | ```
78 |
79 | The extension utilized MutationObserver to capture any node changes in the page, please refer to [MutationObserver](https://developer.mozilla.org/en/docs/Web/API/MutationObserver) for more details.
80 |
81 | ### Pause
82 | During the record, you can click on "PAUSE" to pause the capturer so that no actions will be recorded until you hit "RESUME":
83 |
84 | 
85 |
86 | Thanks to
87 | ------
88 |
89 | OTA action capturer is built on top of [DOMListenerExtension](https://github.com/kdzwinel/DOMListenerExtension).
90 |
91 | License
92 | -------
93 |
94 | This program is free software: you can redistribute it and/or modify
95 | it under the terms of the GNU General Public License as published by
96 | the Free Software Foundation, either version 3 of the License, or
97 | (at your option) any later version.
98 |
99 | This program is distributed in the hope that it will be useful,
100 | but WITHOUT ANY WARRANTY; without even the implied warranty of
101 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
102 | GNU General Public License for more details.
103 |
104 | You should have received a copy of the GNU General Public License
105 | along with this program. If not, see .md
78 | output_dir = Path(output_path)
79 | output_dir.mkdir(parents=True, exist_ok=True)
80 | out_path = output_dir / f"subgoal_{subtask_name}_{grouped_items['action_type']}.md"
81 | out_path.write_text(filled_markdown, encoding="utf-8")
82 |
83 | return out_path
84 |
85 | # ---------------------------------------------------------------------------
86 | # helper: locate exactly one task-start file and return its taskDescription
87 | # ---------------------------------------------------------------------------
88 | def find_task_prompt(data_dir: str | Path) -> str:
89 | data_dir = Path(data_dir)
90 |
91 | # 0️⃣ Does the path exist?
92 | if not data_dir.exists():
93 | sys.exit(f"[OTA error] given path does not exist: {data_dir}")
94 |
95 | # 0️⃣b Is it a directory?
96 | if not data_dir.is_dir():
97 | sys.exit(f"[OTA error] path is not a directory: {data_dir}")
98 |
99 | # 1️⃣ Gather every *.json recursively (sub-folders included)
100 | json_paths = sorted(data_dir.rglob("*.json"))
101 | if not json_paths:
102 | sys.exit(f"[OTA error] no *.json files found under {data_dir}")
103 |
104 | # 2️⃣ First file (by name) must be task-start
105 | first = json.loads(json_paths[0].read_text(encoding="utf-8"))
106 | if first.get("type") != "task-start":
107 | sys.exit("[OTA error] first JSON file is not a task-start record")
108 |
109 | # 3️⃣ Collect *all* task-start files
110 | task_start_files = [
111 | p for p in json_paths
112 | if json.loads(p.read_text(encoding="utf-8")).get("type") == "task-start"
113 | ]
114 | if len(task_start_files) == 0:
115 | sys.exit("[OTA error] no task-start file found")
116 | if len(task_start_files) > 1:
117 | names = ", ".join(p.name for p in task_start_files)
118 | sys.exit(f"[OTA error] multiple task-start files detected: {names}")
119 |
120 | # 4️⃣ Extract taskDescription
121 | task_json = json.loads(task_start_files[0].read_text(encoding="utf-8"))
122 | task_id = task_json.get("taskId")
123 | task_desc = task_json.get("taskDescription")
124 | if not task_desc or not task_id:
125 | sys.exit(f"[OTA error] task-start file {task_start_files[0].name} "
126 | "has no taskDescription or taskId")
127 |
128 | return task_desc, task_id
129 |
130 |
131 | def load_event_json(path: str | Path) -> Dict[str, Any]:
132 | """Read the given JSON file and return it as a Python dict."""
133 | path = Path(path)
134 | if not path.is_file():
135 | raise FileNotFoundError(f"Cannot find JSON file: {path}")
136 | with path.open("r", encoding="utf-8") as f:
137 | return json.load(f)
--------------------------------------------------------------------------------
/utils/subgoal_generator.py:
--------------------------------------------------------------------------------
1 | """Sub‑goal batch generator.
2 |
3 | Given a directory path, this helper loads every `*.md` file (each file is
4 | assumed to contain a prompt) and sends the content to OpenAI (via
5 | `ask_llm`). It returns a list of dicts with filename, prompt, and reply.
6 | """
7 | from __future__ import annotations
8 |
9 | import json, re
10 | from pathlib import Path
11 | from typing import List, Dict, Any, Optional
12 |
13 | from utils.llm import ask_llm
14 |
15 | __all__ = ["generate_subgoals_from_dir"]
16 |
17 |
18 | def _load_prompts(dir_path: str | Path) -> List[tuple[Path, str]]:
19 | """Return a list of (path, text) for every .md file in *dir_path*."""
20 | dir_path = Path(dir_path)
21 | paths = sorted(dir_path.glob("*.md"))
22 | prompts: List[tuple[Path, str]] = []
23 | for p in paths:
24 | text = p.read_text(encoding="utf-8").strip()
25 | if text:
26 | prompts.append((p, text))
27 | return prompts
28 |
29 |
30 | def generate_subgoals_from_dir(
31 | dir_path: str | Path,
32 | *,
33 | system_prompt: Optional[str] = None,
34 | model: str = "gpt-4o-mini",
35 | temperature: float = 0.2,
36 | save_jsonl: Optional[str | Path] = None,
37 | ) -> List[Dict[str, Any]]:
38 | """Load all .md files under *dir_path*, query the LLM, and return results.
39 |
40 | Parameters
41 | ----------
42 | dir_path : str | Path
43 | Directory containing `*.md` prompt files.
44 | system_prompt : str | None
45 | Optional system message for the LLM.
46 | model : str
47 | OpenAI model name.
48 | temperature : float
49 | Sampling temperature.
50 | save_jsonl : str | Path | None
51 | If given, write a JSON‑lines file with each result.
52 |
53 | Returns
54 | -------
55 | list[dict]
56 | Each dict contains {"file", "prompt", "reply"}.
57 | """
58 | prompts = _load_prompts(dir_path)
59 | if not prompts:
60 | raise FileNotFoundError(f"No .md files found in {dir_path}")
61 |
62 | if save_jsonl:
63 | save_path = Path(save_jsonl)
64 | if save_path.exists():
65 | save_path.unlink()
66 |
67 | results: List[Dict[str, Any]] = []
68 |
69 | for idx, (path, prompt_text) in enumerate(prompts, 1):
70 | print(f"[{idx}/{len(prompts)}] Querying LLM for {path.name} …")
71 | reply = ask_llm(
72 | prompt_text,
73 | system_prompt=system_prompt,
74 | model=model,
75 | temperature=temperature,
76 | )
77 | result = {
78 | "file": path.name,
79 | "prompt": prompt_text,
80 | "reply": reply,
81 | }
82 | results.append(result)
83 |
84 | # Optionally append to JSONL file incrementally
85 | if save_jsonl:
86 | with Path(save_jsonl).open("a", encoding="utf-8") as f:
87 | f.write(json.dumps(result, ensure_ascii=False) + "\n")
88 |
89 | return results
90 |
91 | # ---------------------------------------------------------------------------
92 | # JSONL "reply" → next_goal extractor
93 | # ---------------------------------------------------------------------------
94 |
95 | def _clean_reply(raw_reply: str) -> str:
96 | """Remove markdown fences and whitespace from a reply string."""
97 | # strip triple back‑tick blocks if present
98 | fenced = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE)
99 | m = fenced.search(raw_reply)
100 | if m:
101 | return m.group(1).strip()
102 | return raw_reply.strip()
103 |
104 |
105 | def wap_subgoal_list_generation(
106 | ultimate_goal: str,
107 | task_id: str,
108 | jsonl_path: str | Path,
109 | out_path: str | Path = "wap_subgoal.json",
110 | ) -> List[str]:
111 | """Read *jsonl_path*, extract the `next_goal` from each line's `reply`,
112 | and write the list of next goals to *out_path* (as a JSON array).
113 |
114 | Returns the list for immediate use.
115 | """
116 | jsonl_path = Path(jsonl_path)
117 | if not jsonl_path.is_file():
118 | raise FileNotFoundError(jsonl_path)
119 |
120 | goals: List[Dict[str, str]] = [{"index": 0, "subgoal": "task starts, go for the next sub-goal"}]
121 | with jsonl_path.open("r", encoding="utf-8") as fh:
122 | for line_no, line in enumerate(fh, 1):
123 | if not line.strip():
124 | continue
125 | try:
126 | record = json.loads(line)
127 | except json.JSONDecodeError:
128 | print(f"[extract] line {line_no}: malformed JSON – skipped")
129 | continue
130 |
131 | raw_reply = str(record.get("reply", ""))
132 | cleaned = _clean_reply(raw_reply)
133 | try:
134 | reply_json = json.loads(cleaned)
135 | except json.JSONDecodeError:
136 | print(f"[extract] line {line_no}: reply not valid JSON – skipped")
137 | continue
138 |
139 | goal_text = reply_json.get("next_goal")
140 | if goal_text:
141 | goals.append({"index": len(goals), "subgoal": goal_text})
142 | else:
143 | print(f"[extract] line {line_no}: no 'next_goal' key – skipped")
144 |
145 | goals.append({"index": len(goals), "subgoal": "task done"})
146 |
147 | final_output = {
148 | "ultimate_goal": ultimate_goal,
149 | "task_id": task_id,
150 | "type": "smart_replay",
151 | "subgoal_list": goals
152 | }
153 | Path(out_path).write_text(json.dumps(final_output, ensure_ascii=False, indent=2), encoding="utf-8")
154 | print(f"[extract] wrote {len(goals)} sub‑goals → {out_path}")
155 | return goals
156 |
--------------------------------------------------------------------------------
/wap_replay/generate_mcp_server.py:
--------------------------------------------------------------------------------
1 | from mcp.server.fastmcp import FastMCP
2 | import httpx
3 | import os
4 | import json
5 | import argparse
6 | from typing import Optional
7 | import utils.llm
8 | import glob
9 | from dotenv import load_dotenv
10 |
11 | load_dotenv()
12 |
13 | def extract_ultimate_goal(task_id: str) -> str:
14 | """
15 | Try to extract ultimate_goal from either exact_replay or smart_replay file.
16 |
17 | Args:
18 | task_id: The task ID to look for in the replay files
19 |
20 | Returns:
21 | The ultimate_goal string if found
22 |
23 | Raises:
24 | FileNotFoundError: If neither replay file exists
25 | ValueError: If ultimate_goal field is not found
26 | """
27 | exact_replay_path = os.path.join(".", "data_processed", "exact_replay", f"wap_exact_replay_list_{task_id}.json")
28 | smart_replay_path = os.path.join(".", "data_processed", "smart_replay", f"wap_smart_replay_list_{task_id}.json")
29 |
30 | for file_path in [exact_replay_path, smart_replay_path]:
31 | if os.path.exists(file_path):
32 | try:
33 | with open(file_path, 'r') as f:
34 | data = json.load(f)
35 | return data['ultimate_goal']
36 | except (json.JSONDecodeError, KeyError):
37 | continue
38 |
39 | raise ValueError(f"Could not find ultimate_goal in replay files for task_id {task_id}")
40 |
41 | def summarize_goal(ultimate_goal: str) -> str:
42 | """
43 | Generate a function name from the ultimate goal.
44 | This is a placeholder - in practice you would call OpenAI API here.
45 | """
46 | return utils.llm.ask_llm(f"Summarize the following to a single function name with underscore in plaintext: {ultimate_goal}")
47 |
48 | def create_mcp_server(ultimate_goal: str, function_name: str, task_id: str) -> str:
49 | """
50 | Creates an MCP server file with the specified parameters.
51 |
52 | Args:
53 | ultimate_goal: The goal description for the MCP
54 | function_name: The base name for the functions
55 | task_id: The task ID used in file paths
56 |
57 | Returns:
58 | The complete Python code as a string
59 | """
60 | exact_replay_path = os.path.join(".", "data_processed", "exact_replay", f"wap_exact_replay_list_{task_id}.json")
61 | smart_replay_path = os.path.join(".", "data_processed", "smart_replay", f"wap_smart_replay_list_{task_id}.json")
62 |
63 | smart_docstring = f"smart replay: {ultimate_goal}"
64 | exact_docstring = f"exact replay: {ultimate_goal}"
65 |
66 | code = f'''
67 | from mcp.server.fastmcp import FastMCP
68 | import httpx
69 |
70 | mcp = FastMCP("{ultimate_goal}")
71 | '''
72 |
73 | # Only include the tool function for the existing replay file
74 | if os.path.exists(smart_replay_path):
75 | code += f'''
76 | @mcp.tool()
77 | async def {function_name}_smart_replay() -> str:
78 | """{smart_docstring}"""
79 | async with httpx.AsyncClient(timeout=600.0) as client:
80 | response = await client.get(
81 | "http://localhost:3089/replay",
82 | params={{
83 | "concurrent": 1,
84 | "model": "openai",
85 | "file_path": 'data_processed/smart_replay/wap_smart_replay_list_{task_id}.json'
86 | }}
87 | )
88 | return response.text
89 | return "FAILED"
90 | '''
91 | if os.path.exists(exact_replay_path):
92 | code += f'''
93 | @mcp.tool()
94 | async def {function_name}_exact_replay() -> str:
95 | """{exact_docstring}"""
96 | async with httpx.AsyncClient(timeout=600.0) as client:
97 | response = await client.get(
98 | "http://localhost:3089/replay",
99 | params={{
100 | "concurrent": 1,
101 | "model": "openai",
102 | "file_path": 'data_processed/exact_replay/wap_exact_replay_list_{task_id}.json'
103 | }}
104 | )
105 | return response.text
106 | return "FAILED"
107 | '''
108 |
109 | code += '''
110 | if __name__ == "__main__":
111 | mcp.run(transport="stdio")
112 | '''
113 | return code
114 |
115 | def main():
116 | parser = argparse.ArgumentParser(description='Create MCP server file from replay data')
117 | parser.add_argument('--task_id', required=True, help='Task ID to process')
118 | args = parser.parse_args()
119 |
120 | try:
121 | # Extract ultimate_goal from replay files
122 | ultimate_goal = extract_ultimate_goal(args.task_id)
123 |
124 | # Generate function name
125 | function_name = summarize_goal(ultimate_goal)
126 |
127 | # Generate the code
128 | server_code = create_mcp_server(ultimate_goal, function_name, args.task_id)
129 |
130 | # Create mcp_servers directory if it doesn't exist
131 | os.makedirs("mcp_servers", exist_ok=True)
132 |
133 | # Check for existing files with the same task_id
134 | existing_files = glob.glob(os.path.join("mcp_servers", f"*_{args.task_id}_mcp_server.py"))
135 |
136 | # If duplicates exist, remove them
137 | for existing_file in existing_files:
138 | os.remove(existing_file)
139 | print(f"Removed duplicate: {existing_file}")
140 |
141 | # Save to file in the mcp_servers folder
142 | filename = os.path.join("mcp_servers", f"{function_name}_{args.task_id}_mcp_server.py")
143 | with open(filename, "w") as file:
144 | file.write(server_code)
145 |
146 | print(f"Successfully created/updated {filename}")
147 | except Exception as e:
148 | print(f"Error: {str(e)}")
149 | exit(1)
150 |
151 | if __name__ == "__main__":
152 | main()
--------------------------------------------------------------------------------
/browser_use/dom/tests/extraction_test.py:
--------------------------------------------------------------------------------
1 | import asyncio
2 | import time
3 |
4 | from langchain_openai import ChatOpenAI
5 |
6 | from browser_use.browser.browser import Browser, BrowserConfig
7 | from browser_use.browser.context import BrowserContext, BrowserContextConfig
8 | from browser_use.dom.service import DomService
9 | from browser_use.utils import time_execution_sync
10 |
11 |
12 | def count_string_tokens(string: str, model: str) -> int:
13 | """Count the number of tokens in a string using a specified model."""
14 | llm = ChatOpenAI(model=model)
15 | return llm.count_tokens(string)
16 |
17 |
18 | async def test_process_html_file():
19 | config = BrowserContextConfig(
20 | cookies_file='cookies3.json',
21 | disable_security=True,
22 | wait_for_network_idle_page_load_time=2,
23 | )
24 |
25 | browser = Browser(
26 | config=BrowserConfig(
27 | # chrome_instance_path='/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
28 | )
29 | )
30 | context = BrowserContext(browser=browser, config=config) # noqa: F821
31 |
32 | websites = [
33 | 'https://kayak.com/flights',
34 | 'https://immobilienscout24.de',
35 | 'https://google.com',
36 | 'https://amazon.com',
37 | 'https://github.com',
38 | ]
39 |
40 | async with context as context:
41 | page = await context.get_current_page()
42 | dom_service = DomService(page)
43 |
44 | for website in websites:
45 | print(f'\n{"=" * 50}\nTesting {website}\n{"=" * 50}')
46 | await page.goto(website)
47 | time.sleep(2) # Additional wait for dynamic content
48 |
49 | async def test_viewport(expansion: int, description: str):
50 | print(f'\n{description}:')
51 | dom_state = await time_execution_sync(f'get_clickable_elements ({description})')(
52 | dom_service.get_clickable_elements
53 | )(highlight_elements=True, viewport_expansion=expansion)
54 |
55 | elements = dom_state.element_tree
56 | selector_map = dom_state.selector_map
57 | element_count = len(selector_map.keys())
58 | token_count = count_string_tokens(elements.clickable_elements_to_string(), model='gpt-4o')
59 |
60 | print(f'Number of elements: {element_count}')
61 | print(f'Token count: {token_count}')
62 | return element_count, token_count
63 |
64 | expansions = [0, 100, 200, 300, 400, 500, 600, 1000, -1, -200]
65 | results = []
66 |
67 | for i, expansion in enumerate(expansions):
68 | description = (
69 | f'{i + 1}. Expansion {expansion}px' if expansion >= 0 else f'{i + 1}. All elements ({expansion} expansion)'
70 | )
71 | count, tokens = await test_viewport(expansion, description)
72 | results.append((count, tokens))
73 | input('Press Enter to continue...')
74 | await page.evaluate('document.getElementById("playwright-highlight-container")?.remove()')
75 |
76 | # Print comparison summary
77 | print('\nComparison Summary:')
78 | for i, (count, tokens) in enumerate(results):
79 | expansion = expansions[i]
80 | description = f'Expansion {expansion}px' if expansion >= 0 else 'All elements (-1)'
81 | initial_count, initial_tokens = results[0]
82 | print(f'{description}: {count} elements (+{count - initial_count}), {tokens} tokens')
83 |
84 | input('\nPress Enter to continue to next website...')
85 |
86 | # Clear highlights before next website
87 | await page.evaluate('document.getElementById("playwright-highlight-container")?.remove()')
88 |
89 |
90 | async def test_focus_vs_all_elements():
91 | config = BrowserContextConfig(
92 | # cookies_file='cookies3.json',
93 | disable_security=True,
94 | wait_for_network_idle_page_load_time=2,
95 | )
96 |
97 | browser = Browser(
98 | config=BrowserConfig(
99 | # browser_binary_path='/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
100 | )
101 | )
102 | context = BrowserContext(browser=browser, config=config) # noqa: F821
103 |
104 | websites = [
105 | 'https://en.wikipedia.org/wiki/Humanist_Party_of_Ontario',
106 | 'https://www.google.com/travel/flights?tfs=CBwQARoJagcIARIDTEpVGglyBwgBEgNMSlVAAUgBcAGCAQsI____________AZgBAQ&tfu=KgIIAw&hl=en-US&gl=US',
107 | # 'https://www.concur.com/?&cookie_preferences=cpra',
108 | 'https://immobilienscout24.de',
109 | 'https://docs.google.com/spreadsheets/d/1INaIcfpYXlMRWO__de61SHFCaqt1lfHlcvtXZPItlpI/edit',
110 | 'https://www.zeiss.com/career/en/job-search.html?page=1',
111 | 'https://www.mlb.com/yankees/stats/',
112 | 'https://www.amazon.com/s?k=laptop&s=review-rank&crid=1RZCEJ289EUSI&qid=1740202453&sprefix=laptop%2Caps%2C166&ref=sr_st_review-rank&ds=v1%3A4EnYKXVQA7DIE41qCvRZoNB4qN92Jlztd3BPsTFXmxU',
113 | 'https://codepen.io/geheimschriftstift/pen/mPLvQz',
114 | 'https://reddit.com',
115 | 'https://www.google.com/search?q=google+hi&oq=google+hi&gs_lcrp=EgZjaHJvbWUyBggAEEUYOTIGCAEQRRhA0gEIMjI2NmowajSoAgCwAgE&sourceid=chrome&ie=UTF-8',
116 | 'https://kayak.com/flights',
117 | 'https://google.com',
118 | 'https://amazon.com',
119 | 'https://github.com',
120 | ]
121 |
122 | async with context as context:
123 | page = await context.get_current_page()
124 | dom_service = DomService(page)
125 |
126 | for website in websites:
127 | # sleep 2
128 | await page.goto(website)
129 | time.sleep(2)
130 |
131 | while True:
132 | try:
133 | print(f'\n{"=" * 50}\nTesting {website}\n{"=" * 50}')
134 | # time.sleep(2) # Additional wait for dynamic content
135 |
136 | # First get all elements
137 | print('\nGetting all elements:')
138 | all_elements_state = await time_execution_sync('get_all_elements')(dom_service.get_clickable_elements)(
139 | highlight_elements=True, viewport_expansion=1000
140 | )
141 |
142 | selector_map = all_elements_state.selector_map
143 | total_elements = len(selector_map.keys())
144 | print(f'Total number of elements: {total_elements}')
145 |
146 | print(all_elements_state.element_tree.clickable_elements_to_string())
147 |
148 | answer = input('Press Enter to clear highlights and continue...')
149 | if answer == 'q':
150 | break
151 |
152 | await page.evaluate('document.getElementById("playwright-highlight-container")?.remove()')
153 |
154 | except Exception as e:
155 | print(f'Error: {e}')
156 | pass
157 |
158 |
159 | if __name__ == '__main__':
160 | asyncio.run(test_focus_vs_all_elements())
161 | asyncio.run(test_process_html_file())
162 |
--------------------------------------------------------------------------------
/browser_use/agent/tests.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | from browser_use.agent.views import (
4 | ActionResult,
5 | AgentBrain,
6 | AgentHistory,
7 | AgentHistoryList,
8 | AgentOutput,
9 | )
10 | from browser_use.browser.views import BrowserState, BrowserStateHistory, TabInfo
11 | from browser_use.controller.registry.service import Registry
12 | from browser_use.controller.views import ClickElementAction, DoneAction, ExtractPageContentAction
13 | from browser_use.dom.views import DOMElementNode
14 |
15 |
16 | @pytest.fixture
17 | def sample_browser_state():
18 | return BrowserState(
19 | url='https://example.com',
20 | title='Example Page',
21 | tabs=[TabInfo(url='https://example.com', title='Example Page', page_id=1)],
22 | screenshot='screenshot1.png',
23 | element_tree=DOMElementNode(
24 | tag_name='root',
25 | is_visible=True,
26 | parent=None,
27 | xpath='',
28 | attributes={},
29 | children=[],
30 | ),
31 | selector_map={},
32 | )
33 |
34 |
35 | @pytest.fixture
36 | def action_registry():
37 | registry = Registry()
38 |
39 | # Register the actions we need for testing
40 | @registry.action(description='Click an element', param_model=ClickElementAction)
41 | def click_element(params: ClickElementAction, browser=None):
42 | pass
43 |
44 | @registry.action(
45 | description='Extract page content',
46 | param_model=ExtractPageContentAction,
47 | )
48 | def extract_page_content(params: ExtractPageContentAction, browser=None):
49 | pass
50 |
51 | @registry.action(description='Mark task as done', param_model=DoneAction)
52 | def done(params: DoneAction):
53 | pass
54 |
55 | # Create the dynamic ActionModel with all registered actions
56 | return registry.create_action_model()
57 |
58 |
59 | @pytest.fixture
60 | def sample_history(action_registry):
61 | # Create actions with nested params structure
62 | click_action = action_registry(click_element={'index': 1})
63 |
64 | extract_action = action_registry(extract_page_content={'value': 'text'})
65 |
66 | done_action = action_registry(done={'text': 'Task completed'})
67 |
68 | histories = [
69 | AgentHistory(
70 | model_output=AgentOutput(
71 | current_state=AgentBrain(
72 | evaluation_previous_goal='None',
73 | memory='Started task',
74 | next_goal='Click button',
75 | ),
76 | action=[click_action],
77 | ),
78 | result=[ActionResult(is_done=False)],
79 | state=BrowserStateHistory(
80 | url='https://example.com',
81 | title='Page 1',
82 | tabs=[TabInfo(url='https://example.com', title='Page 1', page_id=1)],
83 | screenshot='screenshot1.png',
84 | interacted_element=[{'xpath': '//button[1]'}],
85 | ),
86 | ),
87 | AgentHistory(
88 | model_output=AgentOutput(
89 | current_state=AgentBrain(
90 | evaluation_previous_goal='Clicked button',
91 | memory='Button clicked',
92 | next_goal='Extract content',
93 | ),
94 | action=[extract_action],
95 | ),
96 | result=[
97 | ActionResult(
98 | is_done=False,
99 | extracted_content='Extracted text',
100 | error='Failed to extract completely',
101 | )
102 | ],
103 | state=BrowserStateHistory(
104 | url='https://example.com/page2',
105 | title='Page 2',
106 | tabs=[TabInfo(url='https://example.com/page2', title='Page 2', page_id=2)],
107 | screenshot='screenshot2.png',
108 | interacted_element=[{'xpath': '//div[1]'}],
109 | ),
110 | ),
111 | AgentHistory(
112 | model_output=AgentOutput(
113 | current_state=AgentBrain(
114 | evaluation_previous_goal='Extracted content',
115 | memory='Content extracted',
116 | next_goal='Finish task',
117 | ),
118 | action=[done_action],
119 | ),
120 | result=[ActionResult(is_done=True, extracted_content='Task completed', error=None)],
121 | state=BrowserStateHistory(
122 | url='https://example.com/page2',
123 | title='Page 2',
124 | tabs=[TabInfo(url='https://example.com/page2', title='Page 2', page_id=2)],
125 | screenshot='screenshot3.png',
126 | interacted_element=[{'xpath': '//div[1]'}],
127 | ),
128 | ),
129 | ]
130 | return AgentHistoryList(history=histories)
131 |
132 |
133 | def test_last_model_output(sample_history: AgentHistoryList):
134 | last_output = sample_history.last_action()
135 | print(last_output)
136 | assert last_output == {'done': {'text': 'Task completed'}}
137 |
138 |
139 | def test_get_errors(sample_history: AgentHistoryList):
140 | errors = sample_history.errors()
141 | assert len(errors) == 1
142 | assert errors[0] == 'Failed to extract completely'
143 |
144 |
145 | def test_final_result(sample_history: AgentHistoryList):
146 | assert sample_history.final_result() == 'Task completed'
147 |
148 |
149 | def test_is_done(sample_history: AgentHistoryList):
150 | assert sample_history.is_done() is True
151 |
152 |
153 | def test_urls(sample_history: AgentHistoryList):
154 | urls = sample_history.urls()
155 | assert 'https://example.com' in urls
156 | assert 'https://example.com/page2' in urls
157 |
158 |
159 | def test_all_screenshots(sample_history: AgentHistoryList):
160 | screenshots = sample_history.screenshots()
161 | assert len(screenshots) == 3
162 | assert screenshots == ['screenshot1.png', 'screenshot2.png', 'screenshot3.png']
163 |
164 |
165 | def test_all_model_outputs(sample_history: AgentHistoryList):
166 | outputs = sample_history.model_actions()
167 | print(f'DEBUG: {outputs[0]}')
168 | assert len(outputs) == 3
169 | # get first key value pair
170 | assert dict([next(iter(outputs[0].items()))]) == {'click_element': {'index': 1}}
171 | assert dict([next(iter(outputs[1].items()))]) == {'extract_page_content': {'value': 'text'}}
172 | assert dict([next(iter(outputs[2].items()))]) == {'done': {'text': 'Task completed'}}
173 |
174 |
175 | def test_all_model_outputs_filtered(sample_history: AgentHistoryList):
176 | filtered = sample_history.model_actions_filtered(include=['click_element'])
177 | assert len(filtered) == 1
178 | assert filtered[0]['click_element']['index'] == 1
179 |
180 |
181 | def test_empty_history():
182 | empty_history = AgentHistoryList(history=[])
183 | assert empty_history.last_action() is None
184 | assert empty_history.final_result() is None
185 | assert empty_history.is_done() is False
186 | assert len(empty_history.urls()) == 0
187 |
188 |
189 | # Add a test to verify action creation
190 | def test_action_creation(action_registry):
191 | click_action = action_registry(click_element={'index': 1})
192 |
193 | assert click_action.model_dump(exclude_none=True) == {'click_element': {'index': 1}}
194 |
195 |
196 | # run this with:
197 | # pytest browser_use/agent/tests.py
198 |
--------------------------------------------------------------------------------
/browser_use/dom/views.py:
--------------------------------------------------------------------------------
1 | from dataclasses import dataclass
2 | from functools import cached_property
3 | from typing import TYPE_CHECKING, Dict, List, Optional
4 |
5 | from browser_use.dom.history_tree_processor.view import CoordinateSet, HashedDomElement, ViewportInfo
6 | from browser_use.utils import time_execution_sync
7 |
8 | # Avoid circular import issues
9 | if TYPE_CHECKING:
10 | from .views import DOMElementNode
11 |
12 |
13 | @dataclass(frozen=False)
14 | class DOMBaseNode:
15 | is_visible: bool
16 | # Use None as default and set parent later to avoid circular reference issues
17 | parent: Optional['DOMElementNode']
18 |
19 |
20 | @dataclass(frozen=False)
21 | class DOMTextNode(DOMBaseNode):
22 | text: str
23 | type: str = 'TEXT_NODE'
24 |
25 | def has_parent_with_highlight_index(self) -> bool:
26 | current = self.parent
27 | while current is not None:
28 | # stop if the element has a highlight index (will be handled separately)
29 | if current.highlight_index is not None:
30 | return True
31 |
32 | current = current.parent
33 | return False
34 |
35 | def is_parent_in_viewport(self) -> bool:
36 | if self.parent is None:
37 | return False
38 | return self.parent.is_in_viewport
39 |
40 | def is_parent_top_element(self) -> bool:
41 | if self.parent is None:
42 | return False
43 | return self.parent.is_top_element
44 |
45 |
46 | @dataclass(frozen=False)
47 | class DOMElementNode(DOMBaseNode):
48 | """
49 | xpath: the xpath of the element from the last root node (shadow root or iframe OR document if no shadow root or iframe).
50 | To properly reference the element we need to recursively switch the root node until we find the element (work you way up the tree with `.parent`)
51 | """
52 |
53 | tag_name: str
54 | xpath: str
55 | attributes: Dict[str, str]
56 | children: List[DOMBaseNode]
57 | is_interactive: bool = False
58 | is_top_element: bool = False
59 | is_in_viewport: bool = False
60 | is_ota_interactive_element: bool = False
61 | shadow_root: bool = False
62 | highlight_index: Optional[int] = None
63 | viewport_coordinates: Optional[CoordinateSet] = None
64 | page_coordinates: Optional[CoordinateSet] = None
65 | viewport_info: Optional[ViewportInfo] = None
66 |
67 | def __repr__(self) -> str:
68 | tag_str = f'<{self.tag_name}'
69 |
70 | # Add attributes
71 | for key, value in self.attributes.items():
72 | tag_str += f' {key}="{value}"'
73 | tag_str += '>'
74 |
75 | # Add extra info
76 | extras = []
77 | if self.is_interactive:
78 | extras.append('interactive')
79 | if self.is_top_element:
80 | extras.append('top')
81 | if self.shadow_root:
82 | extras.append('shadow-root')
83 | if self.highlight_index is not None:
84 | extras.append(f'highlight:{self.highlight_index}')
85 | if self.is_in_viewport:
86 | extras.append('in-viewport')
87 | if self.is_ota_interactive_element:
88 | extras.append('ota-interactive-element')
89 |
90 | if extras:
91 | tag_str += f' [{", ".join(extras)}]'
92 |
93 | return tag_str
94 |
95 | @cached_property
96 | def hash(self) -> HashedDomElement:
97 | from browser_use.dom.history_tree_processor.service import (
98 | HistoryTreeProcessor,
99 | )
100 |
101 | return HistoryTreeProcessor._hash_dom_element(self)
102 |
103 | def get_all_text_till_next_clickable_element(self, max_depth: int = -1) -> str:
104 | text_parts = []
105 |
106 | def collect_text(node: DOMBaseNode, current_depth: int) -> None:
107 | if max_depth != -1 and current_depth > max_depth:
108 | return
109 |
110 | # Skip this branch if we hit a highlighted element (except for the current node)
111 | if isinstance(node, DOMElementNode) and node != self and node.highlight_index is not None:
112 | return
113 |
114 | if isinstance(node, DOMTextNode):
115 | text_parts.append(node.text)
116 | elif isinstance(node, DOMElementNode):
117 | for child in node.children:
118 | collect_text(child, current_depth + 1)
119 |
120 | collect_text(self, 0)
121 | return '\n'.join(text_parts).strip()
122 |
123 | @time_execution_sync('--clickable_elements_to_string')
124 | def clickable_elements_to_string(self, include_attributes: list[str] | None = None) -> str:
125 | """Convert the processed DOM content to HTML."""
126 | formatted_text = []
127 |
128 | def process_node(node: DOMBaseNode, depth: int) -> None:
129 | if isinstance(node, DOMElementNode):
130 | # Add element with highlight_index
131 | if node.highlight_index is not None:
132 | attributes_str = ''
133 | text = node.get_all_text_till_next_clickable_element()
134 | if include_attributes:
135 | attributes = list(
136 | set(
137 | [
138 | f"{str(key)}={str(value)}"
139 | for key, value in node.attributes.items()
140 | if key in include_attributes and value != node.tag_name
141 | ]
142 | )
143 | )
144 | if text in attributes:
145 | attributes.remove(text)
146 | attributes_str = ';'.join(attributes)
147 | line = f'[{node.highlight_index}]<{node.tag_name} '
148 | if attributes_str:
149 | line += f'{attributes_str}'
150 | if text:
151 | if attributes_str:
152 | line += f'>{text}'
153 | else:
154 | line += f'{text}'
155 | line += '/>'
156 | formatted_text.append(line)
157 |
158 | # Process children regardless
159 | for child in node.children:
160 | process_node(child, depth + 1)
161 |
162 | elif isinstance(node, DOMTextNode):
163 | # Add text only if it doesn't have a highlighted parent
164 | if not node.has_parent_with_highlight_index() and node.is_visible: # and node.is_parent_top_element()
165 | formatted_text.append(f'{node.text}')
166 |
167 | process_node(self, 0)
168 | return '\n'.join(formatted_text)
169 |
170 | def get_file_upload_element(self, check_siblings: bool = True) -> Optional['DOMElementNode']:
171 | # Check if current element is a file input
172 | if self.tag_name == 'input' and self.attributes.get('type') == 'file':
173 | return self
174 |
175 | # Check children
176 | for child in self.children:
177 | if isinstance(child, DOMElementNode):
178 | result = child.get_file_upload_element(check_siblings=False)
179 | if result:
180 | return result
181 |
182 | # Check siblings only for the initial call
183 | if check_siblings and self.parent:
184 | for sibling in self.parent.children:
185 | if sibling is not self and isinstance(sibling, DOMElementNode):
186 | result = sibling.get_file_upload_element(check_siblings=False)
187 | if result:
188 | return result
189 |
190 | return None
191 |
192 |
193 | SelectorMap = dict[int, DOMElementNode]
194 |
195 |
196 | @dataclass
197 | class DOMState:
198 | element_tree: DOMElementNode
199 | selector_map: SelectorMap
200 |
--------------------------------------------------------------------------------
/browser_use/dom/service.py:
--------------------------------------------------------------------------------
1 | import gc
2 | import json
3 | import logging
4 | from dataclasses import dataclass
5 | from importlib import resources
6 | from typing import TYPE_CHECKING, Optional
7 | from urllib.parse import urlparse
8 |
9 | if TYPE_CHECKING:
10 | from playwright.async_api import Page
11 |
12 | from browser_use.dom.views import (
13 | DOMBaseNode,
14 | DOMElementNode,
15 | DOMState,
16 | DOMTextNode,
17 | SelectorMap,
18 | )
19 | from browser_use.utils import time_execution_async
20 |
21 | logger = logging.getLogger(__name__)
22 |
23 |
24 | @dataclass
25 | class ViewportInfo:
26 | width: int
27 | height: int
28 |
29 |
30 | class DomService:
31 | def __init__(self, page: 'Page'):
32 | self.page = page
33 | self.xpath_cache = {}
34 |
35 | self.js_code = resources.files('browser_use.dom').joinpath('buildDomTree.js').read_text()
36 |
37 | # region - Clickable elements
38 | @time_execution_async('--get_clickable_elements')
39 | async def get_clickable_elements(
40 | self,
41 | highlight_elements: bool = True,
42 | focus_element: int = -1,
43 | viewport_expansion: int = 0,
44 | ) -> DOMState:
45 | element_tree, selector_map = await self._build_dom_tree(highlight_elements, focus_element, viewport_expansion)
46 | return DOMState(element_tree=element_tree, selector_map=selector_map)
47 |
48 | @time_execution_async('--get_cross_origin_iframes')
49 | async def get_cross_origin_iframes(self) -> list[str]:
50 | # invisible cross-origin iframes are used for ads and tracking, dont open those
51 | hidden_frame_urls = await self.page.locator('iframe').filter(visible=False).evaluate_all('e => e.map(e => e.src)')
52 |
53 | is_ad_url = lambda url: any(
54 | domain in urlparse(url).netloc for domain in ('doubleclick.net', 'adroll.com', 'googletagmanager.com')
55 | )
56 |
57 | return [
58 | frame.url
59 | for frame in self.page.frames
60 | if urlparse(frame.url).netloc # exclude data:urls and about:blank
61 | and urlparse(frame.url).netloc != urlparse(self.page.url).netloc # exclude same-origin iframes
62 | and frame.url not in hidden_frame_urls # exclude hidden frames
63 | and not is_ad_url(frame.url) # exclude most common ad network tracker frame URLs
64 | ]
65 |
66 | @time_execution_async('--build_dom_tree')
67 | async def _build_dom_tree(
68 | self,
69 | highlight_elements: bool,
70 | focus_element: int,
71 | viewport_expansion: int,
72 | ) -> tuple[DOMElementNode, SelectorMap]:
73 | if await self.page.evaluate('1+1') != 2:
74 | raise ValueError('The page cannot evaluate javascript code properly')
75 |
76 | if self.page.url == 'about:blank':
77 | # short-circuit if the page is a new empty tab for speed, no need to inject buildDomTree.js
78 | return (
79 | DOMElementNode(
80 | tag_name='body',
81 | xpath='',
82 | attributes={},
83 | children=[],
84 | is_visible=False,
85 | parent=None,
86 | ),
87 | {},
88 | )
89 |
90 | # NOTE: We execute JS code in the browser to extract important DOM information.
91 | # The returned hash map contains information about the DOM tree and the
92 | # relationship between the DOM elements.
93 | debug_mode = logger.getEffectiveLevel() == logging.DEBUG
94 | args = {
95 | 'doHighlightElements': highlight_elements,
96 | 'focusHighlightIndex': focus_element,
97 | 'viewportExpansion': viewport_expansion,
98 | 'debugMode': debug_mode,
99 | }
100 |
101 | try:
102 | eval_page: dict = await self.page.evaluate(self.js_code, args)
103 | except Exception as e:
104 | logger.error('Error evaluating JavaScript: %s', e)
105 | raise
106 |
107 | # Only log performance metrics in debug mode
108 | if debug_mode and 'perfMetrics' in eval_page:
109 | logger.debug(
110 | 'DOM Tree Building Performance Metrics for: %s\n%s',
111 | self.page.url,
112 | json.dumps(eval_page['perfMetrics'], indent=2),
113 | )
114 |
115 | return await self._construct_dom_tree(eval_page)
116 |
117 | @time_execution_async('--construct_dom_tree')
118 | async def _construct_dom_tree(
119 | self,
120 | eval_page: dict,
121 | ) -> tuple[DOMElementNode, SelectorMap]:
122 | js_node_map = eval_page['map']
123 | js_root_id = eval_page['rootId']
124 |
125 | selector_map = {}
126 | node_map = {}
127 |
128 | for id, node_data in js_node_map.items():
129 | node, children_ids = self._parse_node(node_data)
130 | if node is None:
131 | continue
132 |
133 | node_map[id] = node
134 |
135 | if isinstance(node, DOMElementNode) and node.highlight_index is not None:
136 | selector_map[node.highlight_index] = node
137 |
138 | # NOTE: We know that we are building the tree bottom up
139 | # and all children are already processed.
140 | if isinstance(node, DOMElementNode):
141 | for child_id in children_ids:
142 | if child_id not in node_map:
143 | continue
144 |
145 | child_node = node_map[child_id]
146 |
147 | child_node.parent = node
148 | node.children.append(child_node)
149 |
150 | html_to_dict = node_map[str(js_root_id)]
151 |
152 | del node_map
153 | del js_node_map
154 | del js_root_id
155 |
156 | gc.collect()
157 |
158 | if html_to_dict is None or not isinstance(html_to_dict, DOMElementNode):
159 | raise ValueError('Failed to parse HTML to dictionary')
160 |
161 | return html_to_dict, selector_map
162 |
163 | def _parse_node(
164 | self,
165 | node_data: dict,
166 | ) -> tuple[Optional[DOMBaseNode], list[int]]:
167 | if not node_data:
168 | return None, []
169 |
170 | # Process text nodes immediately
171 | if node_data.get('type') == 'TEXT_NODE':
172 | text_node = DOMTextNode(
173 | text=node_data['text'],
174 | is_visible=node_data['isVisible'],
175 | parent=None,
176 | )
177 | return text_node, []
178 |
179 | # Process coordinates if they exist for element nodes
180 |
181 | viewport_info = None
182 |
183 | if 'viewport' in node_data:
184 | viewport_info = ViewportInfo(
185 | width=node_data['viewport']['width'],
186 | height=node_data['viewport']['height'],
187 | )
188 |
189 | element_node = DOMElementNode(
190 | tag_name=node_data['tagName'],
191 | xpath=node_data['xpath'],
192 | attributes=node_data.get('attributes', {}),
193 | children=[],
194 | is_visible=node_data.get('isVisible', False),
195 | is_interactive=node_data.get('isInteractive', False),
196 | is_top_element=node_data.get('isTopElement', False),
197 | is_in_viewport=node_data.get('isInViewport', False),
198 | is_ota_interactive_element=node_data.get('isOTAInteractiveElement', False),
199 | highlight_index=node_data.get('highlightIndex'),
200 | shadow_root=node_data.get('shadowRoot', False),
201 | parent=None,
202 | viewport_info=viewport_info,
203 | )
204 |
205 | children_ids = node_data.get('children', [])
206 |
207 | return element_node, children_ids
208 |
--------------------------------------------------------------------------------
/browser_use/agent/prompts.py:
--------------------------------------------------------------------------------
1 | import importlib.resources
2 | from datetime import datetime
3 | from typing import TYPE_CHECKING, List, Optional, Union
4 |
5 | from langchain_core.messages import HumanMessage, SystemMessage
6 |
7 | if TYPE_CHECKING:
8 | from browser_use.agent.views import ActionResult, AgentStepInfo
9 | from browser_use.browser.views import BrowserState
10 |
11 |
12 | class SystemPrompt:
13 | def __init__(
14 | self,
15 | action_description: str,
16 | max_actions_per_step: int = 10,
17 | override_system_message: Optional[str] = None,
18 | extend_system_message: Optional[str] = None,
19 | replay_mode: str = None
20 | ):
21 | self.default_action_description = action_description
22 | self.max_actions_per_step = max_actions_per_step
23 | self.system_prompt_path = 'system_prompt_wap_replay.md' if replay_mode == "smart_replay" else 'system_prompt.md'
24 | prompt = ''
25 | if override_system_message:
26 | prompt = override_system_message
27 | else:
28 | self._load_prompt_template()
29 | prompt = self.prompt_template.format(max_actions=self.max_actions_per_step)
30 |
31 | if extend_system_message:
32 | prompt += f'\n{extend_system_message}'
33 |
34 | self.system_message = SystemMessage(content=prompt)
35 |
36 | def _load_prompt_template(self) -> None:
37 | """Load the prompt template from the markdown file."""
38 | try:
39 | # This works both in development and when installed as a package
40 | with importlib.resources.files('browser_use.agent').joinpath(self.system_prompt_path).open('r') as f:
41 | self.prompt_template = f.read()
42 | except Exception as e:
43 | raise RuntimeError(f'Failed to load system prompt template: {e}')
44 |
45 | def get_system_message(self) -> SystemMessage:
46 | """
47 | Get the system prompt for the agent.
48 |
49 | Returns:
50 | SystemMessage: Formatted system prompt
51 | """
52 | return self.system_message
53 |
54 |
55 | # Functions:
56 | # {self.default_action_description}
57 |
58 | # Example:
59 | # {self.example_response()}
60 | # Your AVAILABLE ACTIONS:
61 | # {self.default_action_description}
62 |
63 |
64 | class AgentMessagePrompt:
65 | def __init__(
66 | self,
67 | state: 'BrowserState',
68 | result: Optional[List['ActionResult']] = None,
69 | include_attributes: list[str] = [],
70 | step_info: Optional['AgentStepInfo'] = None,
71 | subgoals: Optional[list] = None,
72 | ):
73 | self.state = state
74 | self.result = result
75 | self.include_attributes = include_attributes
76 | self.step_info = step_info
77 | self.subgoals = subgoals or []
78 |
79 | def get_user_message(self, use_vision: bool = True, is_smart_replay: bool = False) -> HumanMessage:
80 | elements_text = self.state.element_tree.clickable_elements_to_string(include_attributes=self.include_attributes)
81 |
82 | has_content_above = (self.state.pixels_above or 0) > 0
83 | has_content_below = (self.state.pixels_below or 0) > 0
84 |
85 | if elements_text != '':
86 | if has_content_above:
87 | elements_text = (
88 | f'... {self.state.pixels_above} pixels above - scroll or extract content to see more ...\n{elements_text}'
89 | )
90 | else:
91 | elements_text = f'[Start of page]\n{elements_text}'
92 | if has_content_below:
93 | elements_text = (
94 | f'{elements_text}\n... {self.state.pixels_below} pixels below - scroll or extract content to see more ...'
95 | )
96 | else:
97 | elements_text = f'{elements_text}\n[End of page]'
98 | else:
99 | elements_text = 'empty page'
100 |
101 | if self.step_info:
102 | step_info_description = f'Current step: {self.step_info.step_number + 1}/{self.step_info.max_steps}'
103 | else:
104 | step_info_description = ''
105 | time_str = datetime.now().strftime('%Y-%m-%d %H:%M')
106 | step_info_description += f'Current date and time: {time_str}'
107 |
108 |
109 | state_description = ""
110 |
111 | if not is_smart_replay:
112 | state_description = f"""
113 | [Task history memory ends]
114 | [Current state starts here]
115 | The following is one-time information - if you need to remember it write it to memory:
116 | Current url: {self.state.url}
117 | Available tabs:
118 | {self.state.tabs}
119 | Interactive elements from top layer of the current page inside the viewport:
120 | {elements_text}
121 | {step_info_description}
122 | """
123 | else:
124 | state_description = f"""
125 | [Task history memory ends]
126 | [sub-goals start here]
127 | Previous sub-goal: {self.subgoals[0]}
128 | Current sub-goal: {self.subgoals[1]}
129 | [sub-goals end]
130 | [Current state starts here]
131 | The following is one-time information - if you need to remember it write it to memory:
132 | Current url: {self.state.url}
133 | Available tabs:
134 | {self.state.tabs}
135 | Interactive elements from top layer of the current page inside the viewport:
136 | {elements_text}
137 | {step_info_description}
138 | """
139 |
140 | if self.result:
141 | for i, result in enumerate(self.result):
142 | if result.extracted_content:
143 | state_description += f'\nAction result {i + 1}/{len(self.result)}: {result.extracted_content}'
144 | if result.error:
145 | # only use last line of error
146 | error = result.error.split('\n')[-1]
147 | state_description += f'\nAction error {i + 1}/{len(self.result)}: ...{error}'
148 |
149 | if self.state.screenshot and use_vision is True:
150 | # Format message for vision model
151 | return HumanMessage(
152 | content=[
153 | {'type': 'text', 'text': state_description},
154 | {
155 | 'type': 'image_url',
156 | 'image_url': {'url': f'data:image/png;base64,{self.state.screenshot}'}, # , 'detail': 'low'
157 | },
158 | ]
159 | )
160 |
161 | return HumanMessage(content=state_description)
162 |
163 |
164 | class PlannerPrompt(SystemPrompt):
165 | def get_system_message(self, is_planner_reasoning) -> Union[SystemMessage, HumanMessage]:
166 | planner_prompt_text = """You are a planning agent that helps break down tasks into smaller steps and reason about the current state.
167 | Your role is to:
168 | 1. Analyze the current state and history
169 | 2. Evaluate progress towards the ultimate goal
170 | 3. Identify potential challenges or roadblocks
171 | 4. Suggest the next high-level steps to take
172 |
173 | Inside your messages, there will be AI messages from different agents with different formats.
174 |
175 | Your output format should be always a JSON object with the following fields:
176 | {
177 | "state_analysis": "Brief analysis of the current state and what has been done so far",
178 | "progress_evaluation": "Evaluation of progress towards the ultimate goal (as percentage and description)",
179 | "challenges": "List any potential challenges or roadblocks",
180 | "next_steps": "List 2-3 concrete next steps to take",
181 | "reasoning": "Explain your reasoning for the suggested next steps"
182 | }
183 |
184 | Ignore the other AI messages output structures.
185 |
186 | Keep your responses concise and focused on actionable insights."""
187 |
188 | if is_planner_reasoning:
189 | return HumanMessage(content=planner_prompt_text)
190 | else:
191 | return SystemMessage(content=planner_prompt_text)
192 |
--------------------------------------------------------------------------------
/chrome-extension/css/panel.css:
--------------------------------------------------------------------------------
1 | @font-face {
2 | font-family: 'Raleway';
3 | font-style: normal;
4 | font-weight: 400;
5 | src: local('Raleway'), url(../other/Raleway.woff2) format('woff2');
6 | unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02C6, U+02DA, U+02DC, U+2000-206F, U+2074, U+20AC, U+2212, U+2215, U+E0FF, U+EFFD, U+F000;
7 | }
8 |
9 | * {
10 | box-sizing: border-box;
11 | }
12 |
13 | body {
14 | padding: 10px;
15 | background: white;
16 | }
17 |
18 | .intro {
19 | position: absolute;
20 | left: 50%;
21 | top: 50%;
22 | transform: translate(-50%, -50%);
23 |
24 | display: flex;
25 | flex-direction: column;
26 | align-items: center;
27 | text-align: center;
28 | }
29 |
30 | nav {
31 | position: fixed;
32 | z-index: 1;
33 | bottom: 0;
34 | left: 0;
35 | right: 0;
36 | background: rgba(255, 255, 255, 0.7);
37 | padding: 10px;
38 | }
39 |
40 | nav > button {
41 | margin-bottom: 0;
42 | background: white;
43 | }
44 |
45 | .top {
46 | opacity: 1;
47 | transition: opacity 300ms;
48 | }
49 |
50 | .top.hidden {
51 | pointer-events: none;
52 | opacity: 0;
53 | }
54 |
55 | .status {
56 | position: fixed;
57 | top: 10px;
58 | right: 10px;
59 | color: rgba(139, 0, 0, 0.5);
60 | font-size: small;
61 | z-index: 1;
62 | cursor: help;
63 | }
64 |
65 | .status.connected {
66 | display: none;
67 | }
68 |
69 | .events {
70 | margin-bottom: 50px;
71 | table-layout: fixed;
72 | }
73 |
74 | .events thead th {
75 | vertical-align: bottom;
76 | }
77 |
78 | .events thead th input[type=text] {
79 | margin-bottom: 9px;
80 | height: 30px;
81 | font-weight: normal;
82 | width: 100%;
83 | min-width: 55px;
84 | }
85 |
86 | .events thead th .checkbox {
87 | margin-bottom: 9px;
88 | }
89 |
90 | .events td {
91 | word-wrap: break-word;
92 | }
93 |
94 | .events .time {
95 | display: block;
96 | color: #b6b6b6;
97 | font-size: small;
98 | }
99 |
100 | .events td:first-child {
101 | border-left: solid transparent 5px;
102 | padding-left: 4px;
103 | }
104 |
105 | .events tr.nodes-added td:first-child {
106 | border-left-color: rgb(138,219,246);
107 | }
108 |
109 | .events tr.nodes-removed td:first-child {
110 | border-left-color: rgb(255,198,139);
111 | }
112 |
113 | .events tr.text-changed td:first-child {
114 | border-left-color: rgb(254,239,139);
115 | }
116 |
117 | .events tr.attribute-changed td:first-child {
118 | border-left-color: rgb(179,146,248);
119 | }
120 |
121 | .events th:nth-child(1), .events td:nth-child(1) {
122 | min-width: 155px;
123 | }
124 |
125 | .events th:nth-child(3), .events td:nth-child(3) {
126 | /* width: 60%; */
127 | }
128 |
129 | .events td > div {
130 | max-height: 85px;
131 | overflow: auto;
132 | }
133 |
134 | .events td:nth-child(3) hr {
135 | margin-top: 2px;
136 | margin-bottom: 2px;
137 | }
138 |
139 | .events thead .counter {
140 | color: #b6b6b6;
141 | }
142 |
143 | .node:hover {
144 | text-decoration: underline;
145 | cursor: pointer;
146 | }
147 |
148 | /*
149 | Filter
150 | */
151 |
152 | .events tbody tr {
153 | display: none;
154 | }
155 |
156 | .events tbody.nodes-added-visible tr.target-match.nodes-added,
157 | .events tbody.nodes-removed-visible tr.target-match.nodes-removed,
158 | .events tbody.text-changed-visible tr.target-match.text-changed,
159 | .events tbody.attribute-changed-visible tr.target-match.attribute-changed {
160 | display: table-row;
161 | }
162 |
163 | /*
164 | Checkbox
165 | */
166 |
167 | .checkbox {
168 | width: 20px;
169 | position: relative;
170 | display: inline-block;
171 | }
172 |
173 | .checkbox label {
174 | cursor: pointer;
175 | position: absolute;
176 | width: 20px;
177 | height: 20px;
178 | top: 0;
179 | border-radius: 4px;
180 | margin: 0;
181 | }
182 |
183 | .checkbox label:after {
184 | opacity: 0;
185 | content: '';
186 | position: absolute;
187 | width: 9px;
188 | height: 5px;
189 | background: transparent;
190 | top: 4px;
191 | left: 4px;
192 | border: 3px solid #fcfff4;
193 | border-top: none;
194 | border-right: none;
195 |
196 | transform: rotate(-45deg);
197 | }
198 |
199 | .checkbox label:hover::after {
200 | opacity: 0.3;
201 | }
202 |
203 | .checkbox input {
204 | visibility: hidden;
205 | margin: 0;
206 | }
207 |
208 | .checkbox input[type=checkbox]:checked + label:after {
209 | opacity: 1;
210 | }
211 |
212 | .checkbox.nodes-added label {
213 | background: rgb(138,219,246);
214 | }
215 |
216 | .checkbox.nodes-removed label {
217 | background: rgb(255,198,139);
218 | }
219 |
220 | .checkbox.text-changed label {
221 | background: rgb(254,239,139);
222 | }
223 |
224 | .checkbox.attribute-changed label {
225 | background: rgb(179,146,248);
226 | }
227 |
228 | .task-description-input {
229 | width: 100%;
230 | padding: 8px;
231 | font-size: 14px;
232 | min-height: 20px;
233 | margin-bottom: 0;
234 | resize: none;
235 | overflow: hidden;
236 | box-sizing: border-box;
237 | line-height: 1.4;
238 | }
239 |
240 | .task-description-input.invalid {
241 | border: 1px solid #e74c3c;
242 | }
243 |
244 | .task-description-section {
245 | padding-bottom: 10px;
246 | }
247 |
248 | .task-description-label {
249 | display: none; /* Default hidden */
250 | font-size: 16px;
251 | font-weight: bold;
252 | margin-top: 10px;
253 | margin-bottom: 100px;
254 | padding: 5px 10px;
255 | background-color: #f0f0f0;
256 | color: #333;
257 | border: 1px solid #ccc;
258 | border-radius: 5px;
259 | white-space: pre-wrap; /* Allow line breaks if task description is long */
260 | }
261 |
262 | .task-description-task-id {
263 | display: none; /* Default hidden */
264 | font-size: 16px;
265 | font-weight: bold;
266 | margin-top: 10px;
267 | margin-bottom: 100px;
268 | padding: 5px 10px;
269 | background-color: hsl(52, 100%, 57%);
270 | color: #333;
271 | border: 1px solid #ccc;
272 | border-radius: 5px;
273 | white-space: pre-wrap; /* Allow line breaks if task description is long */
274 | }
275 |
276 | .hidden { display: none !important; }
277 |
278 | .settings-panel {
279 | padding: 10px;
280 | border-top: 1px solid #ddd;
281 | }
282 |
283 | .settings-panel label {
284 | display: block;
285 | margin: 6px 0;
286 | }
287 |
288 | /* Make the header a positioning context */
289 | table.events thead {
290 | position: relative;
291 | }
292 |
293 | /* Pin the button */
294 | .settings-btn {
295 | position: absolute;
296 | top: 4px;
297 | right: 8px;
298 | background-color: #e5e5e5;
299 | }
300 |
301 | .social-buttons {
302 | display: flex;
303 | align-items: center;
304 | gap: 8px;
305 | }
306 |
307 | .website-btn {
308 | font-family: inherit;
309 | font-size: 10px;
310 | font-weight: 600;
311 | line-height: 1;
312 | padding: 5px 10px;
313 | color: #fff;
314 | background: #000d1d;
315 | border-radius: 15px;
316 | text-decoration: none;
317 | }
318 |
319 | .website-btn:hover,
320 | .website-btn:focus {
321 | color: #fff;
322 | background: #00223d;
323 | outline: none;
324 | }
325 |
326 | .website-btn:active {
327 | color: #fff;
328 | background: #00294e;
329 | }
--------------------------------------------------------------------------------
/chrome-extension/js/EventTable.js:
--------------------------------------------------------------------------------
1 | (function () {
2 | "use strict";
3 |
4 | function formatNode(node) {
5 | return '' + node.selector + '';
6 | }
7 |
8 | function formatValue(value) {
9 | if (value === null) {
10 | return 'null';
11 | } else if (value === undefined) {
12 | return 'undefined';
13 | } else {
14 | return '"' + value + '"';
15 | }
16 | }
17 |
18 | function momentJS(date, timeOnly) {
19 | let hours = date.getHours();
20 | let minutes = date.getMinutes();
21 | let seconds = date.getSeconds();
22 | hours = hours % 24;
23 | minutes = minutes < 10 ? '0' + minutes : minutes;
24 | seconds = seconds < 10 ? '0' + seconds : seconds;
25 | const strTime = hours + ':' + minutes + ':' + seconds;
26 |
27 | if (timeOnly) {
28 | return strTime;
29 | }
30 |
31 | return date.getMonth()+1 + "/" + date.getDate() + "/" + date.getFullYear() + " " + strTime + ":" + date.getMilliseconds();
32 | }
33 |
34 | function formatDate(timestamp) {
35 | const date = new Date(timestamp);
36 |
37 | return `${momentJS(date, true)}`;
38 | }
39 |
40 | function formatEventDetails(event) {
41 | var details = "";
42 | switch (event.type) {
43 | case "nodes added":
44 | details = event.nodes.length + ' node(s) added: ' +
45 | '' + (event.nodes.map(formatNode)).join(', ') + '';
46 | break;
47 | case "nodes removed":
48 | details = event.nodes.length + ' node(s) removed: ' +
49 | '' + (event.nodes.map(formatNode)).join(', ') + '';
50 | break;
51 | case "attribute changed":
52 | details = '"' + event.attribute + '" ';
53 |
54 | if (event.oldValue === null && event.newValue === "") {
55 | details += ' was added';
56 | } else if (event.newValue === null && event.oldValue === "") {
57 | details += ' was removed';
58 | } else {
59 | details += 'changed from ' + formatValue(event.oldValue) + ' ' +
60 | 'to ' + formatValue(event.newValue) + '';
61 | }
62 |
63 | break;
64 | case "text changed":
65 | details = 'text changed ' +
66 | 'from ' + formatValue(event.oldValue) + ' ' +
67 | 'to ' + formatValue(event.newValue) + '';
68 | break;
69 | }
70 |
71 | return details;
72 | }
73 |
74 | function EventTable(table) {
75 | this._tableHead = table.tHead;
76 | this._tableBody = table.tBodies[0];
77 | this._counter = this._tableHead.querySelector('.counter');
78 | this._targetFilter = (this._tableHead).querySelector('.target-filter');
79 | this._count = this._tableBody.children.length;
80 |
81 | //FILTERS
82 | var thead = this._tableHead;
83 | var tbody = this._tableBody;
84 | var targetFilter = this._targetFilter;
85 | var typeFilters = (this._tableHead).querySelectorAll('.type-filters input');
86 |
87 | function updateTypeFilters() {
88 | var nodesAdded = thead.querySelector('.nodes-added input').checked;
89 | var nodesRemoved = thead.querySelector('.nodes-removed input').checked;
90 | var textChanged = thead.querySelector('.text-changed input').checked;
91 | var attributeChanged = thead.querySelector('.attribute-changed input').checked;
92 |
93 | if (nodesAdded) {
94 | tbody.classList.add('nodes-added-visible');
95 | } else {
96 | tbody.classList.remove('nodes-added-visible');
97 | }
98 |
99 | if (nodesRemoved) {
100 | tbody.classList.add('nodes-removed-visible');
101 | } else {
102 | tbody.classList.remove('nodes-removed-visible');
103 | }
104 |
105 | if (textChanged) {
106 | tbody.classList.add('text-changed-visible');
107 | } else {
108 | tbody.classList.remove('text-changed-visible');
109 | }
110 |
111 | if (attributeChanged) {
112 | tbody.classList.add('attribute-changed-visible');
113 | } else {
114 | tbody.classList.remove('attribute-changed-visible');
115 | }
116 | }
117 |
118 | updateTypeFilters();
119 |
120 | function updateTargetFilter() {
121 | var query = (targetFilter.value).trim();
122 |
123 | for (var i = 0, l = tbody.children.length; i < l; i++) {
124 | var tr = tbody.children[i];
125 | var targetTd = tr.children[1];
126 |
127 | if (!query || targetTd.innerText.indexOf(query) > -1) {
128 | tr.classList.add('target-match');
129 | } else {
130 | tr.classList.remove('target-match');
131 | }
132 | }
133 | }
134 |
135 | updateTargetFilter();
136 |
137 | targetFilter.addEventListener('keyup', updateTargetFilter);
138 |
139 | for (var i = 0, l = typeFilters.length; i < l; i++) {
140 | typeFilters[i].addEventListener('change', updateTypeFilters);
141 | }
142 | }
143 |
144 | EventTable.prototype._updateEventCounter = function () {
145 | (this._counter).innerText = '(' + this._count + ')';
146 | };
147 |
148 | EventTable.prototype.clear = function () {
149 | (this._tableBody).innerHTML = '';
150 |
151 | this._count = 0;
152 | this._updateEventCounter();
153 | };
154 |
155 | EventTable.prototype.addEvent = function (event) {
156 | var tr = (this._tableBody).firstChild;
157 | var tdAction, tdDetails, tdTarget;
158 |
159 | //check if events should be grouped together
160 | if (tr && parseInt(tr.dataset.targetNodeId, 10) === event.target.nodeId && tr.dataset.eventType === event.type) {
161 | tdAction = tr.querySelector('td:nth-child(1)');
162 | tdDetails = tr.querySelector('td:nth-child(3)');
163 |
164 | tr.dataset.count = parseInt(tr.dataset.count || "1", 10) + 1;
165 |
166 | tdAction.innerHTML = tr.dataset.count + ' x ' + event.type + formatDate(event.date);
167 |
168 | tdDetails.querySelector('div').innerHTML += '
' + formatEventDetails(event); 169 | 170 | this._count++; 171 | return; 172 | } 173 | 174 | tr = document.createElement('tr'); 175 | tdAction = document.createElement('td'); 176 | tdTarget = document.createElement('td'); 177 | tdDetails = document.createElement('td'); 178 | 179 | tr.dataset.targetNodeId = event.target.nodeId; 180 | tr.dataset.eventType = event.type; 181 | 182 | tr.classList.add(event.type.replace(' ', '-')); 183 | 184 | tdAction.innerHTML = event.type + formatDate(event.date); 185 | tdTarget.innerHTML = '
9 |.json
97 | ```
98 |
99 | An example of the formatted data which you will received in the WAP backend server is like:
100 |
101 | ```json
102 | {
103 | "taskId": "MkCAhQsHgXn7YgaK",
104 | "type": "click",
105 | "actionTimestamp": 1746325231479,
106 | "eventTarget": {
107 | "type": "click",
108 | "target": "\n
85 |
86 | Thanks to
87 | ------
88 |
89 | OTA action capturer is built on top of [DOMListenerExtension](https://github.com/kdzwinel/DOMListenerExtension).
90 |
91 | License
92 | -------
93 |
94 | This program is free software: you can redistribute it and/or modify
95 | it under the terms of the GNU General Public License as published by
96 | the Free Software Foundation, either version 3 of the License, or
97 | (at your option) any later version.
98 |
99 | This program is distributed in the hope that it will be useful,
100 | but WITHOUT ANY WARRANTY; without even the implied warranty of
101 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
102 | GNU General Public License for more details.
103 |
104 | You should have received a copy of the GNU General Public License
105 | along with this program. If not, see ' + formatEventDetails(event); 169 | 170 | this._count++; 171 | return; 172 | } 173 | 174 | tr = document.createElement('tr'); 175 | tdAction = document.createElement('td'); 176 | tdTarget = document.createElement('td'); 177 | tdDetails = document.createElement('td'); 178 | 179 | tr.dataset.targetNodeId = event.target.nodeId; 180 | tr.dataset.eventType = event.type; 181 | 182 | tr.classList.add(event.type.replace(' ', '-')); 183 | 184 | tdAction.innerHTML = event.type + formatDate(event.date); 185 | tdTarget.innerHTML = '
' + formatNode(event.target) + '
';
186 | tdDetails.innerHTML = '' + formatEventDetails(event) + '
';
187 |
188 | tr.appendChild(tdAction);
189 | tr.appendChild(tdTarget);
190 | tr.appendChild(tdDetails);
191 |
192 | //check if it matches current query
193 | var query = ((this._targetFilter).value).trim();
194 | if (!query || tdTarget.innerText.indexOf(query) > -1) {
195 | tr.classList.add('target-match');
196 | }
197 |
198 | //insert at the top/beginning
199 | (this._tableBody).insertBefore(tr, this._tableBody.firstChild);
200 |
201 | tr.animate([
202 | {opacity: 0},
203 | {opacity: 1}
204 | ], 300);
205 |
206 | this._count++;
207 | this._updateEventCounter();
208 | };
209 |
210 | window.EventTable = EventTable;
211 | })();
212 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | 
7 |
8 |
7 | 9 |
10 |
12 |
14 | 
16 |
17 |
18 |
19 | # Web Agent Protocol
20 |
21 | ## Overview
22 |
23 | The Web Agent Protocol (WAP) is a standardized framework designed to enable seamless interaction between users, web agents, and browsers by recording and replaying browser actions. It separates the concerns of action recording and execution, allowing for efficient automation and reusability. The Python SDK for WAP implements the full specification, making it easy to:
24 |
25 | 1. **Collect** user‑interaction data with the [OTA‑WAP Chrome extension](https://github.com/OTA-Tech-AI/webagentprotocol/tree/main/chrome-extension).
26 | 2. **Convert** the raw event stream into either **_exact‑replay_** or **_smart‑replay_** action lists.
27 | 3. **Convert** recorded actions into **_MCP_** servers for reuse by any agent or user
28 | 4. **Replay** those lists using the **_WAP-Replay_** protocol to ensure accurate browser operations.
29 |
30 | ### WAP FULL DEMO
31 |
32 | [](https://www.youtube.com/watch?v=joh9FXJfnwk)
33 |
34 | ### Without WAP
35 | 
36 |
37 | ### WAP Record
38 | 
39 |
40 | ### WAP Replay
41 | 
42 |
43 | ## Example using WAP
44 | 
45 |
46 | ## Setup
47 | Install the dependencies with the following command:
48 |
49 | Create a conda env
50 |
51 | ```bash
52 | conda create -n WAP python=3.11
53 | ```
54 |
55 | Activate the conda env
56 |
57 | ```bash
58 | conda activate WAP
59 | ```
60 |
61 | Install the dependencies
62 |
63 | ```bash
64 | pip install -r requirements.txt
65 | ```
66 |
67 | Setup your repo source path:
68 | ```
69 | set PYTHONPATH=C:/path/to/webagentprotocol # for Windows
70 | export PYTHONPATH=/path/to/webagentprotocol # for Linux
71 | ```
72 |
73 | Create **.env** file under the repo root directory with your own API keys:
74 | ```
75 | OPENAI_API_KEY=sk-proj-...
76 | DEEPSEEK_API_KEY=sk-...
77 | ```
78 |
79 | ## Record
80 |
81 | ### WAP record extension
82 | Please refer to [OTA‑WAP Chrome Extension](https://github.com/OTA-Tech-AI/webagentprotocol/tree/main/chrome-extension) to setup action capturer in your Chrome browser.
83 |
84 | ### Start data‑collection server
85 |
86 | Run the following command to start the server to collect data from the extension:
87 | ```bash
88 | python action_collect_server.py
89 | ```
90 | **Once the server is up, you can start to record from the page using WAP Chrome extension.**
91 |
92 | The server listens on http://localhost:4934/action-data by default, please make sure the Host and Port in the extension settings match this server config.
93 | Each session will be saved to:
94 |
95 | ```bash
96 | data/YYYYMMDD/taskid/summary_event_
12 |
14 |
16 | 17 |
...",
109 | "targetId": "mntl-card-list-card--extendable_3-0",
110 | "targetClass": "comp mntl-card-list-card--extendable mntl-universal-card mntl-document-card mntl-card card card--no-image"
111 | },
112 | "allEvents": {},
113 | "pageHTMLContent": "
..."
114 | }
115 | ```
116 |
117 |
118 | ## Generate replay lists
119 |
120 | | Mode | Command |
121 | | -------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
122 | | **Exact replay** – exactly reproduce every action | `python wap_replay/generate_exact_replay_list.py --data_dir_path data// --output_dir_path data_processed/exact_replay` |
123 | | **Smart replay** – condensed goal‑oriented steps | `python wap_replay/generate_smart_replay_list.py --data_dir_path data// --output_dir_path data_processed/smart_replay` |
124 |
125 | Replace **** with the folder produced by the extension
126 | (e.g. em3h6UBDZykz0gnH).
127 |
128 | Output structure:
129 | ```bash
130 | data_processed/smart_replay/
131 | ├─ subgoals_/ # intermediate prompts & replies
132 | └─ wap_smart_replay_list_.json # final smart replay list for the agent
133 |
134 | data_processed/exact_replay/
135 | └─ wap_smart_replay_list_.json # final exact replay list for the agent
136 | ```
137 |
138 | ## Replay
139 | ```bash
140 | python run_replay.py --model-provider openai --wap_replay_list data_processed/exact_replay/wap_exact_replay_list_.json --max-concurrent 1
141 | ```
142 | For **smart-replay**, replace the path with a smart‑replay JSON to test this mode.
143 |
144 | ## Convert to MCP Server
145 |
146 | ```bash
147 | python wap_replay\generate_mcp_server.py --task_id
148 | ```
149 |
150 | converted MCP servers will be located under ``` mcp_servers ``` folder
151 |
152 | ## Replay with MCP
153 |
154 | You would need 2 terminals to replay with MCP. In the first termnial
155 | ```bash
156 | python wap_service.py
157 | ```
158 |
159 | In the second termnial
160 | ```bash
161 | python mcp_client.py
162 | ```
163 |
164 | Then enter your prompt in the second terminal
165 |
166 | ```bash
167 | example: find a top rated keyboard on amazon.ca using smart replay
168 | ```
169 |
170 | ## Replay with our Desktop App
171 |
172 | We provide out-of-box desktop app for running replay lists. It is easy to install and you don't need any extra steps for setup and deployments. Visit [WAP Replay Tool releases](https://github.com/OTA-Tech-AI/web-agent-protocol/releases) for more details.
173 |
174 | 
175 |
176 |
177 | ## Troubleshooting
178 |
179 | **ModuleNotFoundError** – run commands from the project root or export PYTHONPATH=. (set PYTHONPATH=. for Windows).
180 |
181 | “no task‑start file” – ensure the extension recorded a full session;
182 | the generators require exactly one task-start and one task-finish record.
183 |
184 | ## Acknowledgement
185 |
186 | Browser-Use: https://github.com/browser-use/browser-use
187 |
188 | MCP: https://github.com/modelcontextprotocol/python-sdk
189 |
190 | DOM Extension: https://github.com/kdzwinel/DOMListenerExtension
191 |
--------------------------------------------------------------------------------
/browser_use/agent/message_manager/tests.py:
--------------------------------------------------------------------------------
1 | import pytest
2 | from langchain_anthropic import ChatAnthropic
3 | from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
4 | from langchain_openai import AzureChatOpenAI, ChatOpenAI
5 |
6 | from browser_use.agent.message_manager.service import MessageManager, MessageManagerSettings
7 | from browser_use.agent.views import ActionResult
8 | from browser_use.browser.views import BrowserState, TabInfo
9 | from browser_use.dom.views import DOMElementNode, DOMTextNode
10 |
11 |
12 | @pytest.fixture(
13 | params=[
14 | ChatOpenAI(model='gpt-4o-mini'),
15 | AzureChatOpenAI(model='gpt-4o', api_version='2024-02-15-preview'),
16 | ChatAnthropic(model_name='claude-3-5-sonnet-20240620', timeout=100, temperature=0.0, stop=None),
17 | ],
18 | ids=['gpt-4o-mini', 'gpt-4o', 'claude-3-5-sonnet'],
19 | )
20 | def message_manager(request: pytest.FixtureRequest):
21 | task = 'Test task'
22 | action_descriptions = 'Test actions'
23 | return MessageManager(
24 | task=task,
25 | system_message=SystemMessage(content=action_descriptions),
26 | settings=MessageManagerSettings(
27 | max_input_tokens=1000,
28 | estimated_characters_per_token=3,
29 | image_tokens=800,
30 | ),
31 | )
32 |
33 |
34 | def test_initial_messages(message_manager: MessageManager):
35 | """Test that message manager initializes with system and task messages"""
36 | messages = message_manager.get_messages()
37 | assert len(messages) == 2
38 | assert isinstance(messages[0], SystemMessage)
39 | assert isinstance(messages[1], HumanMessage)
40 | assert 'Test task' in messages[1].content
41 |
42 |
43 | def test_add_state_message(message_manager: MessageManager):
44 | """Test adding browser state message"""
45 | state = BrowserState(
46 | url='https://test.com',
47 | title='Test Page',
48 | element_tree=DOMElementNode(
49 | tag_name='div',
50 | attributes={},
51 | children=[],
52 | is_visible=True,
53 | parent=None,
54 | xpath='//div',
55 | ),
56 | selector_map={},
57 | tabs=[TabInfo(page_id=1, url='https://test.com', title='Test Page')],
58 | )
59 | message_manager.add_state_message(state)
60 |
61 | messages = message_manager.get_messages()
62 | assert len(messages) == 3
63 | assert isinstance(messages[2], HumanMessage)
64 | assert 'https://test.com' in messages[2].content
65 |
66 |
67 | def test_add_state_with_memory_result(message_manager: MessageManager):
68 | """Test adding state with result that should be included in memory"""
69 | state = BrowserState(
70 | url='https://test.com',
71 | title='Test Page',
72 | element_tree=DOMElementNode(
73 | tag_name='div',
74 | attributes={},
75 | children=[],
76 | is_visible=True,
77 | parent=None,
78 | xpath='//div',
79 | ),
80 | selector_map={},
81 | tabs=[TabInfo(page_id=1, url='https://test.com', title='Test Page')],
82 | )
83 | result = ActionResult(extracted_content='Important content', include_in_memory=True)
84 |
85 | message_manager.add_state_message(state, [result])
86 | messages = message_manager.get_messages()
87 |
88 | # Should have system, task, extracted content, and state messages
89 | assert len(messages) == 4
90 | assert 'Important content' in messages[2].content
91 | assert isinstance(messages[2], HumanMessage)
92 | assert isinstance(messages[3], HumanMessage)
93 | assert 'Important content' not in messages[3].content
94 |
95 |
96 | def test_add_state_with_non_memory_result(message_manager: MessageManager):
97 | """Test adding state with result that should not be included in memory"""
98 | state = BrowserState(
99 | url='https://test.com',
100 | title='Test Page',
101 | element_tree=DOMElementNode(
102 | tag_name='div',
103 | attributes={},
104 | children=[],
105 | is_visible=True,
106 | parent=None,
107 | xpath='//div',
108 | ),
109 | selector_map={},
110 | tabs=[TabInfo(page_id=1, url='https://test.com', title='Test Page')],
111 | )
112 | result = ActionResult(extracted_content='Temporary content', include_in_memory=False)
113 |
114 | message_manager.add_state_message(state, [result])
115 | messages = message_manager.get_messages()
116 |
117 | # Should have system, task, and combined state+result message
118 | assert len(messages) == 3
119 | assert 'Temporary content' in messages[2].content
120 | assert isinstance(messages[2], HumanMessage)
121 |
122 |
123 | @pytest.mark.skip('not sure how to fix this')
124 | @pytest.mark.parametrize('max_tokens', [100000, 10000, 5000])
125 | def test_token_overflow_handling_with_real_flow(message_manager: MessageManager, max_tokens):
126 | """Test handling of token overflow in a realistic message flow"""
127 | # Set more realistic token limit
128 | message_manager.settings.max_input_tokens = max_tokens
129 |
130 | # Create a long sequence of interactions
131 | for i in range(200): # Simulate 40 steps of interaction
132 | # Create state with varying content length
133 | state = BrowserState(
134 | url=f'https://test{i}.com',
135 | title=f'Test Page {i}',
136 | element_tree=DOMElementNode(
137 | tag_name='div',
138 | attributes={},
139 | children=[
140 | DOMTextNode(
141 | text=f'Content {j} ' * (10 + i), # Increasing content length
142 | is_visible=True,
143 | parent=None,
144 | )
145 | for j in range(5) # Multiple DOM items
146 | ],
147 | is_visible=True,
148 | parent=None,
149 | xpath='//div',
150 | ),
151 | selector_map={j: f'//div[{j}]' for j in range(5)},
152 | tabs=[TabInfo(page_id=1, url=f'https://test{i}.com', title=f'Test Page {i}')],
153 | )
154 |
155 | # Alternate between different types of results
156 | result = None
157 | if i % 2 == 0: # Every other iteration
158 | result = ActionResult(
159 | extracted_content=f'Important content from step {i}' * 5,
160 | include_in_memory=i % 4 == 0, # Include in memory every 4th message
161 | )
162 |
163 | # Add state message
164 | if result:
165 | message_manager.add_state_message(state, [result])
166 | else:
167 | message_manager.add_state_message(state)
168 |
169 | try:
170 | messages = message_manager.get_messages()
171 | except ValueError as e:
172 | if 'Max token limit reached - history is too long' in str(e):
173 | return # If error occurs, end the test
174 | else:
175 | raise e
176 |
177 | assert message_manager.state.history.current_tokens <= message_manager.settings.max_input_tokens + 100
178 |
179 | last_msg = messages[-1]
180 | assert isinstance(last_msg, HumanMessage)
181 |
182 | if i % 4 == 0:
183 | assert isinstance(message_manager.state.history.messages[-2].message, HumanMessage)
184 | if i % 2 == 0 and not i % 4 == 0:
185 | if isinstance(last_msg.content, list):
186 | assert 'Current url: https://test' in last_msg.content[0]['text']
187 | else:
188 | assert 'Current url: https://test' in last_msg.content
189 |
190 | # Add model output every time
191 | from browser_use.agent.views import AgentBrain, AgentOutput
192 | from browser_use.controller.registry.views import ActionModel
193 |
194 | output = AgentOutput(
195 | current_state=AgentBrain(
196 | evaluation_previous_goal=f'Success in step {i}',
197 | memory=f'Memory from step {i}',
198 | next_goal=f'Goal for step {i + 1}',
199 | ),
200 | action=[ActionModel()],
201 | )
202 | message_manager._remove_last_state_message()
203 | message_manager.add_model_output(output)
204 |
205 | # Get messages and verify after each addition
206 | messages = [m.message for m in message_manager.state.history.messages]
207 |
208 | # Verify token limit is respected
209 |
210 | # Verify essential messages are preserved
211 | assert isinstance(messages[0], SystemMessage) # System prompt always first
212 | assert isinstance(messages[1], HumanMessage) # Task always second
213 | assert 'Test task' in messages[1].content
214 |
215 | # Verify structure of latest messages
216 | assert isinstance(messages[-1], AIMessage) # Last message should be model output
217 | assert f'step {i}' in messages[-1].content # Should contain current step info
218 |
219 | # Log token usage for debugging
220 | token_usage = message_manager.state.history.current_tokens
221 | token_limit = message_manager.settings.max_input_tokens
222 | # print(f'Step {i}: Using {token_usage}/{token_limit} tokens')
223 |
224 | # go through all messages and verify that the token count and total tokens is correct
225 | total_tokens = 0
226 | real_tokens = []
227 | stored_tokens = []
228 | for msg in message_manager.state.history.messages:
229 | total_tokens += msg.metadata.tokens
230 | stored_tokens.append(msg.metadata.tokens)
231 | real_tokens.append(message_manager._count_tokens(msg.message))
232 | assert total_tokens == sum(real_tokens)
233 | assert stored_tokens == real_tokens
234 | assert message_manager.state.history.current_tokens == total_tokens
235 |
236 |
237 | # pytest -s browser_use/agent/message_manager/tests.py
238 |
--------------------------------------------------------------------------------
/browser_use/controller/registry/service.py:
--------------------------------------------------------------------------------
1 | import asyncio
2 | from inspect import iscoroutinefunction, signature
3 | from typing import Any, Callable, Dict, Generic, Optional, Type, TypeVar
4 |
5 | from langchain_core.language_models.chat_models import BaseChatModel
6 | from pydantic import BaseModel, Field, create_model
7 |
8 | from browser_use.browser.context import BrowserContext
9 | from browser_use.controller.registry.views import (
10 | ActionModel,
11 | ActionRegistry,
12 | RegisteredAction,
13 | )
14 | from browser_use.telemetry.service import ProductTelemetry
15 | from browser_use.telemetry.views import (
16 | ControllerRegisteredFunctionsTelemetryEvent,
17 | RegisteredFunction,
18 | )
19 | from browser_use.utils import time_execution_async, time_execution_sync
20 |
21 | Context = TypeVar('Context')
22 |
23 |
24 | class Registry(Generic[Context]):
25 | """Service for registering and managing actions"""
26 |
27 | def __init__(self, exclude_actions: list[str] | None = None):
28 | self.registry = ActionRegistry()
29 | self.telemetry = ProductTelemetry()
30 | self.exclude_actions = exclude_actions if exclude_actions is not None else []
31 |
32 | @time_execution_sync('--create_param_model')
33 | def _create_param_model(self, function: Callable) -> Type[BaseModel]:
34 | """Creates a Pydantic model from function signature"""
35 | sig = signature(function)
36 | params = {
37 | name: (param.annotation, ... if param.default == param.empty else param.default)
38 | for name, param in sig.parameters.items()
39 | if name != 'browser' and name != 'page_extraction_llm' and name != 'available_file_paths'
40 | }
41 | # TODO: make the types here work
42 | return create_model(
43 | f'{function.__name__}_parameters',
44 | __base__=ActionModel,
45 | **params, # type: ignore
46 | )
47 |
48 | def action(
49 | self,
50 | description: str,
51 | param_model: Optional[Type[BaseModel]] = None,
52 | domains: Optional[list[str]] = None,
53 | page_filter: Optional[Callable[[Any], bool]] = None,
54 | ):
55 | """Decorator for registering actions"""
56 |
57 | def decorator(func: Callable):
58 | # Skip registration if action is in exclude_actions
59 | if func.__name__ in self.exclude_actions:
60 | return func
61 |
62 | # Create param model from function if not provided
63 | actual_param_model = param_model or self._create_param_model(func)
64 |
65 | # Wrap sync functions to make them async
66 | if not iscoroutinefunction(func):
67 |
68 | async def async_wrapper(*args, **kwargs):
69 | return await asyncio.to_thread(func, *args, **kwargs)
70 |
71 | # Copy the signature and other metadata from the original function
72 | async_wrapper.__signature__ = signature(func)
73 | async_wrapper.__name__ = func.__name__
74 | async_wrapper.__annotations__ = func.__annotations__
75 | wrapped_func = async_wrapper
76 | else:
77 | wrapped_func = func
78 |
79 | action = RegisteredAction(
80 | name=func.__name__,
81 | description=description,
82 | function=wrapped_func,
83 | param_model=actual_param_model,
84 | domains=domains,
85 | page_filter=page_filter,
86 | )
87 | self.registry.actions[func.__name__] = action
88 | return func
89 |
90 | return decorator
91 |
92 | @time_execution_async('--execute_action')
93 | async def execute_action(
94 | self,
95 | action_name: str,
96 | params: dict,
97 | browser: Optional[BrowserContext] = None,
98 | page_extraction_llm: Optional[BaseChatModel] = None,
99 | sensitive_data: Optional[Dict[str, str]] = None,
100 | available_file_paths: Optional[list[str]] = None,
101 | #
102 | context: Context | None = None,
103 | ) -> Any:
104 | """Execute a registered action"""
105 | if action_name not in self.registry.actions:
106 | raise ValueError(f'Action {action_name} not found')
107 |
108 | action = self.registry.actions[action_name]
109 | try:
110 | # Create the validated Pydantic model
111 | validated_params = action.param_model(**params)
112 |
113 | # Check if the first parameter is a Pydantic model
114 | sig = signature(action.function)
115 | parameters = list(sig.parameters.values())
116 | is_pydantic = parameters and issubclass(parameters[0].annotation, BaseModel)
117 | parameter_names = [param.name for param in parameters]
118 |
119 | if sensitive_data:
120 | validated_params = self._replace_sensitive_data(validated_params, sensitive_data)
121 |
122 | # Check if the action requires browser
123 | if 'browser' in parameter_names and not browser:
124 | raise ValueError(f'Action {action_name} requires browser but none provided.')
125 | if 'page_extraction_llm' in parameter_names and not page_extraction_llm:
126 | raise ValueError(f'Action {action_name} requires page_extraction_llm but none provided.')
127 | if 'available_file_paths' in parameter_names and not available_file_paths:
128 | raise ValueError(f'Action {action_name} requires available_file_paths but none provided.')
129 |
130 | if 'context' in parameter_names and not context:
131 | raise ValueError(f'Action {action_name} requires context but none provided.')
132 |
133 | # Prepare arguments based on parameter type
134 | extra_args = {}
135 | if 'context' in parameter_names:
136 | extra_args['context'] = context
137 | if 'browser' in parameter_names:
138 | extra_args['browser'] = browser
139 | if 'page_extraction_llm' in parameter_names:
140 | extra_args['page_extraction_llm'] = page_extraction_llm
141 | if 'available_file_paths' in parameter_names:
142 | extra_args['available_file_paths'] = available_file_paths
143 | if action_name == 'input_text' and sensitive_data:
144 | extra_args['has_sensitive_data'] = True
145 | if is_pydantic:
146 | return await action.function(validated_params, **extra_args)
147 | return await action.function(**validated_params.model_dump(), **extra_args)
148 |
149 | except Exception as e:
150 | raise RuntimeError(f'Error executing action {action_name}: {str(e)}') from e
151 |
152 | def _replace_sensitive_data(self, params: BaseModel, sensitive_data: Dict[str, str]) -> BaseModel:
153 | """Replaces the sensitive data in the params"""
154 | # if there are any str with placeholder in the params, replace them with the actual value from sensitive_data
155 |
156 | import re
157 |
158 | secret_pattern = re.compile(r'(.*?) ')
159 |
160 | def replace_secrets(value):
161 | if isinstance(value, str):
162 | matches = secret_pattern.findall(value)
163 | for placeholder in matches:
164 | if placeholder in sensitive_data:
165 | value = value.replace(f'{placeholder} ', sensitive_data[placeholder])
166 | return value
167 | elif isinstance(value, dict):
168 | return {k: replace_secrets(v) for k, v in value.items()}
169 | elif isinstance(value, list):
170 | return [replace_secrets(v) for v in value]
171 | return value
172 |
173 | for key, value in params.model_dump().items():
174 | params.__dict__[key] = replace_secrets(value)
175 | return params
176 |
177 | @time_execution_sync('--create_action_model')
178 | def create_action_model(self, include_actions: Optional[list[str]] = None, page=None) -> Type[ActionModel]:
179 | """Creates a Pydantic model from registered actions, used by LLM APIs that support tool calling & enforce a schema"""
180 |
181 | # Filter actions based on page if provided:
182 | # if page is None, only include actions with no filters
183 | # if page is provided, only include actions that match the page
184 |
185 | available_actions = {}
186 | for name, action in self.registry.actions.items():
187 | if include_actions is not None and name not in include_actions:
188 | continue
189 |
190 | # If no page provided, only include actions with no filters
191 | if page is None:
192 | if action.page_filter is None and action.domains is None:
193 | available_actions[name] = action
194 | continue
195 |
196 | # Check page_filter if present
197 | domain_is_allowed = self.registry._match_domains(action.domains, page.url)
198 | page_is_allowed = self.registry._match_page_filter(action.page_filter, page)
199 |
200 | # Include action if both filters match (or if either is not present)
201 | if domain_is_allowed and page_is_allowed:
202 | available_actions[name] = action
203 |
204 | fields = {
205 | name: (
206 | Optional[action.param_model],
207 | Field(default=None, description=action.description),
208 | )
209 | for name, action in available_actions.items()
210 | }
211 |
212 | self.telemetry.capture(
213 | ControllerRegisteredFunctionsTelemetryEvent(
214 | registered_functions=[
215 | RegisteredFunction(name=name, params=action.param_model.model_json_schema())
216 | for name, action in available_actions.items()
217 | ]
218 | )
219 | )
220 |
221 | return create_model('ActionModel', __base__=ActionModel, **fields) # type:ignore
222 |
223 | def get_prompt_description(self, page=None) -> str:
224 | """Get a description of all actions for the prompt
225 |
226 | If page is provided, only include actions that are available for that page
227 | based on their filter_func
228 | """
229 | return self.registry.get_prompt_description(page=page)
230 |
--------------------------------------------------------------------------------
/chrome-extension/css/normalize.css:
--------------------------------------------------------------------------------
1 | /*! normalize.css v3.0.2 | MIT License | git.io/normalize */
2 |
3 | /**
4 | * 1. Set default font family to sans-serif.
5 | * 2. Prevent iOS text size adjust after orientation change, without disabling
6 | * user zoom.
7 | */
8 |
9 | html {
10 | font-family: sans-serif; /* 1 */
11 | -ms-text-size-adjust: 100%; /* 2 */
12 | -webkit-text-size-adjust: 100%; /* 2 */
13 | }
14 |
15 | /**
16 | * Remove default margin.
17 | */
18 |
19 | body {
20 | margin: 0;
21 | }
22 |
23 | /* HTML5 display definitions
24 | ========================================================================== */
25 |
26 | /**
27 | * Correct `block` display not defined for any HTML5 element in IE 8/9.
28 | * Correct `block` display not defined for `details` or `summary` in IE 10/11
29 | * and Firefox.
30 | * Correct `block` display not defined for `main` in IE 11.
31 | */
32 |
33 | article,
34 | aside,
35 | details,
36 | figcaption,
37 | figure,
38 | footer,
39 | header,
40 | hgroup,
41 | main,
42 | menu,
43 | nav,
44 | section,
45 | summary {
46 | display: block;
47 | }
48 |
49 | /**
50 | * 1. Correct `inline-block` display not defined in IE 8/9.
51 | * 2. Normalize vertical alignment of `progress` in Chrome, Firefox, and Opera.
52 | */
53 |
54 | audio,
55 | canvas,
56 | progress,
57 | video {
58 | display: inline-block; /* 1 */
59 | vertical-align: baseline; /* 2 */
60 | }
61 |
62 | /**
63 | * Prevent modern browsers from displaying `audio` without controls.
64 | * Remove excess height in iOS 5 devices.
65 | */
66 |
67 | audio:not([controls]) {
68 | display: none;
69 | height: 0;
70 | }
71 |
72 | /**
73 | * Address `[hidden]` styling not present in IE 8/9/10.
74 | * Hide the `template` element in IE 8/9/11, Safari, and Firefox < 22.
75 | */
76 |
77 | [hidden],
78 | template {
79 | display: none;
80 | }
81 |
82 | /* Links
83 | ========================================================================== */
84 |
85 | /**
86 | * Remove the gray background color from active links in IE 10.
87 | */
88 |
89 | a {
90 | background-color: transparent;
91 | }
92 |
93 | /**
94 | * Improve readability when focused and also mouse hovered in all browsers.
95 | */
96 |
97 | a:active,
98 | a:hover {
99 | outline: 0;
100 | }
101 |
102 | /* Text-level semantics
103 | ========================================================================== */
104 |
105 | /**
106 | * Address styling not present in IE 8/9/10/11, Safari, and Chrome.
107 | */
108 |
109 | abbr[title] {
110 | border-bottom: 1px dotted;
111 | }
112 |
113 | /**
114 | * Address style set to `bolder` in Firefox 4+, Safari, and Chrome.
115 | */
116 |
117 | b,
118 | strong {
119 | font-weight: bold;
120 | }
121 |
122 | /**
123 | * Address styling not present in Safari and Chrome.
124 | */
125 |
126 | dfn {
127 | font-style: italic;
128 | }
129 |
130 | /**
131 | * Address variable `h1` font-size and margin within `section` and `article`
132 | * contexts in Firefox 4+, Safari, and Chrome.
133 | */
134 |
135 | h1 {
136 | font-size: 2em;
137 | margin: 0.67em 0;
138 | }
139 |
140 | /**
141 | * Address styling not present in IE 8/9.
142 | */
143 |
144 | mark {
145 | background: #ff0;
146 | color: #000;
147 | }
148 |
149 | /**
150 | * Address inconsistent and variable font size in all browsers.
151 | */
152 |
153 | small {
154 | font-size: 80%;
155 | }
156 |
157 | /**
158 | * Prevent `sub` and `sup` affecting `line-height` in all browsers.
159 | */
160 |
161 | sub,
162 | sup {
163 | font-size: 75%;
164 | line-height: 0;
165 | position: relative;
166 | vertical-align: baseline;
167 | }
168 |
169 | sup {
170 | top: -0.5em;
171 | }
172 |
173 | sub {
174 | bottom: -0.25em;
175 | }
176 |
177 | /* Embedded content
178 | ========================================================================== */
179 |
180 | /**
181 | * Remove border when inside `a` element in IE 8/9/10.
182 | */
183 |
184 | img {
185 | border: 0;
186 | }
187 |
188 | /**
189 | * Correct overflow not hidden in IE 9/10/11.
190 | */
191 |
192 | svg:not(:root) {
193 | overflow: hidden;
194 | }
195 |
196 | /* Grouping content
197 | ========================================================================== */
198 |
199 | /**
200 | * Address margin not present in IE 8/9 and Safari.
201 | */
202 |
203 | figure {
204 | margin: 1em 40px;
205 | }
206 |
207 | /**
208 | * Address differences between Firefox and other browsers.
209 | */
210 |
211 | hr {
212 | -moz-box-sizing: content-box;
213 | box-sizing: content-box;
214 | height: 0;
215 | }
216 |
217 | /**
218 | * Contain overflow in all browsers.
219 | */
220 |
221 | pre {
222 | overflow: auto;
223 | }
224 |
225 | /**
226 | * Address odd `em`-unit font size rendering in all browsers.
227 | */
228 |
229 | code,
230 | kbd,
231 | pre,
232 | samp {
233 | font-family: monospace, monospace;
234 | font-size: 1em;
235 | }
236 |
237 | /* Forms
238 | ========================================================================== */
239 |
240 | /**
241 | * Known limitation: by default, Chrome and Safari on OS X allow very limited
242 | * styling of `select`, unless a `border` property is set.
243 | */
244 |
245 | /**
246 | * 1. Correct color not being inherited.
247 | * Known issue: affects color of disabled elements.
248 | * 2. Correct font properties not being inherited.
249 | * 3. Address margins set differently in Firefox 4+, Safari, and Chrome.
250 | */
251 |
252 | button,
253 | input,
254 | optgroup,
255 | select,
256 | textarea {
257 | color: inherit; /* 1 */
258 | font: inherit; /* 2 */
259 | margin: 0; /* 3 */
260 | }
261 |
262 | /**
263 | * Address `overflow` set to `hidden` in IE 8/9/10/11.
264 | */
265 |
266 | button {
267 | overflow: visible;
268 | }
269 |
270 | /**
271 | * Address inconsistent `text-transform` inheritance for `button` and `select`.
272 | * All other form control elements do not inherit `text-transform` values.
273 | * Correct `button` style inheritance in Firefox, IE 8/9/10/11, and Opera.
274 | * Correct `select` style inheritance in Firefox.
275 | */
276 |
277 | button,
278 | select {
279 | text-transform: none;
280 | }
281 |
282 | /**
283 | * 1. Avoid the WebKit bug in Android 4.0.* where (2) destroys native `audio`
284 | * and `video` controls.
285 | * 2. Correct inability to style clickable `input` types in iOS.
286 | * 3. Improve usability and consistency of cursor style between image-type
287 | * `input` and others.
288 | */
289 |
290 | button,
291 | html input[type="button"], /* 1 */
292 | input[type="reset"],
293 | input[type="submit"] {
294 | -webkit-appearance: button; /* 2 */
295 | cursor: pointer; /* 3 */
296 | }
297 |
298 | /**
299 | * Re-set default cursor for disabled elements.
300 | */
301 |
302 | button[disabled],
303 | html input[disabled] {
304 | cursor: default;
305 | }
306 |
307 | /**
308 | * Remove inner padding and border in Firefox 4+.
309 | */
310 |
311 | button::-moz-focus-inner,
312 | input::-moz-focus-inner {
313 | border: 0;
314 | padding: 0;
315 | }
316 |
317 | /**
318 | * Address Firefox 4+ setting `line-height` on `input` using `!important` in
319 | * the UA stylesheet.
320 | */
321 |
322 | input {
323 | line-height: normal;
324 | }
325 |
326 | /**
327 | * It's recommended that you don't attempt to style these elements.
328 | * Firefox's implementation doesn't respect box-sizing, padding, or width.
329 | *
330 | * 1. Address box sizing set to `content-box` in IE 8/9/10.
331 | * 2. Remove excess padding in IE 8/9/10.
332 | */
333 |
334 | input[type="checkbox"],
335 | input[type="radio"] {
336 | box-sizing: border-box; /* 1 */
337 | padding: 0; /* 2 */
338 | }
339 |
340 | /**
341 | * Fix the cursor style for Chrome's increment/decrement buttons. For certain
342 | * `font-size` values of the `input`, it causes the cursor style of the
343 | * decrement button to change from `default` to `text`.
344 | */
345 |
346 | input[type="number"]::-webkit-inner-spin-button,
347 | input[type="number"]::-webkit-outer-spin-button {
348 | height: auto;
349 | }
350 |
351 | /**
352 | * 1. Address `appearance` set to `searchfield` in Safari and Chrome.
353 | * 2. Address `box-sizing` set to `border-box` in Safari and Chrome
354 | * (include `-moz` to future-proof).
355 | */
356 |
357 | input[type="search"] {
358 | -webkit-appearance: textfield; /* 1 */
359 | -moz-box-sizing: content-box;
360 | -webkit-box-sizing: content-box; /* 2 */
361 | box-sizing: content-box;
362 | }
363 |
364 | /**
365 | * Remove inner padding and search cancel button in Safari and Chrome on OS X.
366 | * Safari (but not Chrome) clips the cancel button when the search input has
367 | * padding (and `textfield` appearance).
368 | */
369 |
370 | input[type="search"]::-webkit-search-cancel-button,
371 | input[type="search"]::-webkit-search-decoration {
372 | -webkit-appearance: none;
373 | }
374 |
375 | /**
376 | * Define consistent border, margin, and padding.
377 | */
378 |
379 | fieldset {
380 | border: 1px solid #c0c0c0;
381 | margin: 0 2px;
382 | padding: 0.35em 0.625em 0.75em;
383 | }
384 |
385 | /**
386 | * 1. Correct `color` not being inherited in IE 8/9/10/11.
387 | * 2. Remove padding so people aren't caught out if they zero out fieldsets.
388 | */
389 |
390 | legend {
391 | border: 0; /* 1 */
392 | padding: 0; /* 2 */
393 | }
394 |
395 | /**
396 | * Remove default vertical scrollbar in IE 8/9/10/11.
397 | */
398 |
399 | textarea {
400 | overflow: auto;
401 | }
402 |
403 | /**
404 | * Don't inherit the `font-weight` (applied by a rule above).
405 | * NOTE: the default cannot safely be changed in Chrome and Safari on OS X.
406 | */
407 |
408 | optgroup {
409 | font-weight: bold;
410 | }
411 |
412 | /* Tables
413 | ========================================================================== */
414 |
415 | /**
416 | * Remove most spacing between table cells.
417 | */
418 |
419 | table {
420 | border-collapse: collapse;
421 | border-spacing: 0;
422 | }
423 |
424 | td,
425 | th {
426 | padding: 0;
427 | }
--------------------------------------------------------------------------------
175 |
176 |
177 | ## Troubleshooting
178 |
179 | **ModuleNotFoundError** – run commands from the project root or export PYTHONPATH=. (set PYTHONPATH=. for Windows).
180 |
181 | “no task‑start file” – ensure the extension recorded a full session;
182 | the generators require exactly one task-start and one task-finish record.
183 |
184 | ## Acknowledgement
185 |
186 | Browser-Use: https://github.com/browser-use/browser-use
187 |
188 | MCP: https://github.com/modelcontextprotocol/python-sdk
189 |
190 | DOM Extension: https://github.com/kdzwinel/DOMListenerExtension
191 |
--------------------------------------------------------------------------------
/browser_use/agent/message_manager/tests.py:
--------------------------------------------------------------------------------
1 | import pytest
2 | from langchain_anthropic import ChatAnthropic
3 | from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
4 | from langchain_openai import AzureChatOpenAI, ChatOpenAI
5 |
6 | from browser_use.agent.message_manager.service import MessageManager, MessageManagerSettings
7 | from browser_use.agent.views import ActionResult
8 | from browser_use.browser.views import BrowserState, TabInfo
9 | from browser_use.dom.views import DOMElementNode, DOMTextNode
10 |
11 |
12 | @pytest.fixture(
13 | params=[
14 | ChatOpenAI(model='gpt-4o-mini'),
15 | AzureChatOpenAI(model='gpt-4o', api_version='2024-02-15-preview'),
16 | ChatAnthropic(model_name='claude-3-5-sonnet-20240620', timeout=100, temperature=0.0, stop=None),
17 | ],
18 | ids=['gpt-4o-mini', 'gpt-4o', 'claude-3-5-sonnet'],
19 | )
20 | def message_manager(request: pytest.FixtureRequest):
21 | task = 'Test task'
22 | action_descriptions = 'Test actions'
23 | return MessageManager(
24 | task=task,
25 | system_message=SystemMessage(content=action_descriptions),
26 | settings=MessageManagerSettings(
27 | max_input_tokens=1000,
28 | estimated_characters_per_token=3,
29 | image_tokens=800,
30 | ),
31 | )
32 |
33 |
34 | def test_initial_messages(message_manager: MessageManager):
35 | """Test that message manager initializes with system and task messages"""
36 | messages = message_manager.get_messages()
37 | assert len(messages) == 2
38 | assert isinstance(messages[0], SystemMessage)
39 | assert isinstance(messages[1], HumanMessage)
40 | assert 'Test task' in messages[1].content
41 |
42 |
43 | def test_add_state_message(message_manager: MessageManager):
44 | """Test adding browser state message"""
45 | state = BrowserState(
46 | url='https://test.com',
47 | title='Test Page',
48 | element_tree=DOMElementNode(
49 | tag_name='div',
50 | attributes={},
51 | children=[],
52 | is_visible=True,
53 | parent=None,
54 | xpath='//div',
55 | ),
56 | selector_map={},
57 | tabs=[TabInfo(page_id=1, url='https://test.com', title='Test Page')],
58 | )
59 | message_manager.add_state_message(state)
60 |
61 | messages = message_manager.get_messages()
62 | assert len(messages) == 3
63 | assert isinstance(messages[2], HumanMessage)
64 | assert 'https://test.com' in messages[2].content
65 |
66 |
67 | def test_add_state_with_memory_result(message_manager: MessageManager):
68 | """Test adding state with result that should be included in memory"""
69 | state = BrowserState(
70 | url='https://test.com',
71 | title='Test Page',
72 | element_tree=DOMElementNode(
73 | tag_name='div',
74 | attributes={},
75 | children=[],
76 | is_visible=True,
77 | parent=None,
78 | xpath='//div',
79 | ),
80 | selector_map={},
81 | tabs=[TabInfo(page_id=1, url='https://test.com', title='Test Page')],
82 | )
83 | result = ActionResult(extracted_content='Important content', include_in_memory=True)
84 |
85 | message_manager.add_state_message(state, [result])
86 | messages = message_manager.get_messages()
87 |
88 | # Should have system, task, extracted content, and state messages
89 | assert len(messages) == 4
90 | assert 'Important content' in messages[2].content
91 | assert isinstance(messages[2], HumanMessage)
92 | assert isinstance(messages[3], HumanMessage)
93 | assert 'Important content' not in messages[3].content
94 |
95 |
96 | def test_add_state_with_non_memory_result(message_manager: MessageManager):
97 | """Test adding state with result that should not be included in memory"""
98 | state = BrowserState(
99 | url='https://test.com',
100 | title='Test Page',
101 | element_tree=DOMElementNode(
102 | tag_name='div',
103 | attributes={},
104 | children=[],
105 | is_visible=True,
106 | parent=None,
107 | xpath='//div',
108 | ),
109 | selector_map={},
110 | tabs=[TabInfo(page_id=1, url='https://test.com', title='Test Page')],
111 | )
112 | result = ActionResult(extracted_content='Temporary content', include_in_memory=False)
113 |
114 | message_manager.add_state_message(state, [result])
115 | messages = message_manager.get_messages()
116 |
117 | # Should have system, task, and combined state+result message
118 | assert len(messages) == 3
119 | assert 'Temporary content' in messages[2].content
120 | assert isinstance(messages[2], HumanMessage)
121 |
122 |
123 | @pytest.mark.skip('not sure how to fix this')
124 | @pytest.mark.parametrize('max_tokens', [100000, 10000, 5000])
125 | def test_token_overflow_handling_with_real_flow(message_manager: MessageManager, max_tokens):
126 | """Test handling of token overflow in a realistic message flow"""
127 | # Set more realistic token limit
128 | message_manager.settings.max_input_tokens = max_tokens
129 |
130 | # Create a long sequence of interactions
131 | for i in range(200): # Simulate 40 steps of interaction
132 | # Create state with varying content length
133 | state = BrowserState(
134 | url=f'https://test{i}.com',
135 | title=f'Test Page {i}',
136 | element_tree=DOMElementNode(
137 | tag_name='div',
138 | attributes={},
139 | children=[
140 | DOMTextNode(
141 | text=f'Content {j} ' * (10 + i), # Increasing content length
142 | is_visible=True,
143 | parent=None,
144 | )
145 | for j in range(5) # Multiple DOM items
146 | ],
147 | is_visible=True,
148 | parent=None,
149 | xpath='//div',
150 | ),
151 | selector_map={j: f'//div[{j}]' for j in range(5)},
152 | tabs=[TabInfo(page_id=1, url=f'https://test{i}.com', title=f'Test Page {i}')],
153 | )
154 |
155 | # Alternate between different types of results
156 | result = None
157 | if i % 2 == 0: # Every other iteration
158 | result = ActionResult(
159 | extracted_content=f'Important content from step {i}' * 5,
160 | include_in_memory=i % 4 == 0, # Include in memory every 4th message
161 | )
162 |
163 | # Add state message
164 | if result:
165 | message_manager.add_state_message(state, [result])
166 | else:
167 | message_manager.add_state_message(state)
168 |
169 | try:
170 | messages = message_manager.get_messages()
171 | except ValueError as e:
172 | if 'Max token limit reached - history is too long' in str(e):
173 | return # If error occurs, end the test
174 | else:
175 | raise e
176 |
177 | assert message_manager.state.history.current_tokens <= message_manager.settings.max_input_tokens + 100
178 |
179 | last_msg = messages[-1]
180 | assert isinstance(last_msg, HumanMessage)
181 |
182 | if i % 4 == 0:
183 | assert isinstance(message_manager.state.history.messages[-2].message, HumanMessage)
184 | if i % 2 == 0 and not i % 4 == 0:
185 | if isinstance(last_msg.content, list):
186 | assert 'Current url: https://test' in last_msg.content[0]['text']
187 | else:
188 | assert 'Current url: https://test' in last_msg.content
189 |
190 | # Add model output every time
191 | from browser_use.agent.views import AgentBrain, AgentOutput
192 | from browser_use.controller.registry.views import ActionModel
193 |
194 | output = AgentOutput(
195 | current_state=AgentBrain(
196 | evaluation_previous_goal=f'Success in step {i}',
197 | memory=f'Memory from step {i}',
198 | next_goal=f'Goal for step {i + 1}',
199 | ),
200 | action=[ActionModel()],
201 | )
202 | message_manager._remove_last_state_message()
203 | message_manager.add_model_output(output)
204 |
205 | # Get messages and verify after each addition
206 | messages = [m.message for m in message_manager.state.history.messages]
207 |
208 | # Verify token limit is respected
209 |
210 | # Verify essential messages are preserved
211 | assert isinstance(messages[0], SystemMessage) # System prompt always first
212 | assert isinstance(messages[1], HumanMessage) # Task always second
213 | assert 'Test task' in messages[1].content
214 |
215 | # Verify structure of latest messages
216 | assert isinstance(messages[-1], AIMessage) # Last message should be model output
217 | assert f'step {i}' in messages[-1].content # Should contain current step info
218 |
219 | # Log token usage for debugging
220 | token_usage = message_manager.state.history.current_tokens
221 | token_limit = message_manager.settings.max_input_tokens
222 | # print(f'Step {i}: Using {token_usage}/{token_limit} tokens')
223 |
224 | # go through all messages and verify that the token count and total tokens is correct
225 | total_tokens = 0
226 | real_tokens = []
227 | stored_tokens = []
228 | for msg in message_manager.state.history.messages:
229 | total_tokens += msg.metadata.tokens
230 | stored_tokens.append(msg.metadata.tokens)
231 | real_tokens.append(message_manager._count_tokens(msg.message))
232 | assert total_tokens == sum(real_tokens)
233 | assert stored_tokens == real_tokens
234 | assert message_manager.state.history.current_tokens == total_tokens
235 |
236 |
237 | # pytest -s browser_use/agent/message_manager/tests.py
238 |
--------------------------------------------------------------------------------
/browser_use/controller/registry/service.py:
--------------------------------------------------------------------------------
1 | import asyncio
2 | from inspect import iscoroutinefunction, signature
3 | from typing import Any, Callable, Dict, Generic, Optional, Type, TypeVar
4 |
5 | from langchain_core.language_models.chat_models import BaseChatModel
6 | from pydantic import BaseModel, Field, create_model
7 |
8 | from browser_use.browser.context import BrowserContext
9 | from browser_use.controller.registry.views import (
10 | ActionModel,
11 | ActionRegistry,
12 | RegisteredAction,
13 | )
14 | from browser_use.telemetry.service import ProductTelemetry
15 | from browser_use.telemetry.views import (
16 | ControllerRegisteredFunctionsTelemetryEvent,
17 | RegisteredFunction,
18 | )
19 | from browser_use.utils import time_execution_async, time_execution_sync
20 |
21 | Context = TypeVar('Context')
22 |
23 |
24 | class Registry(Generic[Context]):
25 | """Service for registering and managing actions"""
26 |
27 | def __init__(self, exclude_actions: list[str] | None = None):
28 | self.registry = ActionRegistry()
29 | self.telemetry = ProductTelemetry()
30 | self.exclude_actions = exclude_actions if exclude_actions is not None else []
31 |
32 | @time_execution_sync('--create_param_model')
33 | def _create_param_model(self, function: Callable) -> Type[BaseModel]:
34 | """Creates a Pydantic model from function signature"""
35 | sig = signature(function)
36 | params = {
37 | name: (param.annotation, ... if param.default == param.empty else param.default)
38 | for name, param in sig.parameters.items()
39 | if name != 'browser' and name != 'page_extraction_llm' and name != 'available_file_paths'
40 | }
41 | # TODO: make the types here work
42 | return create_model(
43 | f'{function.__name__}_parameters',
44 | __base__=ActionModel,
45 | **params, # type: ignore
46 | )
47 |
48 | def action(
49 | self,
50 | description: str,
51 | param_model: Optional[Type[BaseModel]] = None,
52 | domains: Optional[list[str]] = None,
53 | page_filter: Optional[Callable[[Any], bool]] = None,
54 | ):
55 | """Decorator for registering actions"""
56 |
57 | def decorator(func: Callable):
58 | # Skip registration if action is in exclude_actions
59 | if func.__name__ in self.exclude_actions:
60 | return func
61 |
62 | # Create param model from function if not provided
63 | actual_param_model = param_model or self._create_param_model(func)
64 |
65 | # Wrap sync functions to make them async
66 | if not iscoroutinefunction(func):
67 |
68 | async def async_wrapper(*args, **kwargs):
69 | return await asyncio.to_thread(func, *args, **kwargs)
70 |
71 | # Copy the signature and other metadata from the original function
72 | async_wrapper.__signature__ = signature(func)
73 | async_wrapper.__name__ = func.__name__
74 | async_wrapper.__annotations__ = func.__annotations__
75 | wrapped_func = async_wrapper
76 | else:
77 | wrapped_func = func
78 |
79 | action = RegisteredAction(
80 | name=func.__name__,
81 | description=description,
82 | function=wrapped_func,
83 | param_model=actual_param_model,
84 | domains=domains,
85 | page_filter=page_filter,
86 | )
87 | self.registry.actions[func.__name__] = action
88 | return func
89 |
90 | return decorator
91 |
92 | @time_execution_async('--execute_action')
93 | async def execute_action(
94 | self,
95 | action_name: str,
96 | params: dict,
97 | browser: Optional[BrowserContext] = None,
98 | page_extraction_llm: Optional[BaseChatModel] = None,
99 | sensitive_data: Optional[Dict[str, str]] = None,
100 | available_file_paths: Optional[list[str]] = None,
101 | #
102 | context: Context | None = None,
103 | ) -> Any:
104 | """Execute a registered action"""
105 | if action_name not in self.registry.actions:
106 | raise ValueError(f'Action {action_name} not found')
107 |
108 | action = self.registry.actions[action_name]
109 | try:
110 | # Create the validated Pydantic model
111 | validated_params = action.param_model(**params)
112 |
113 | # Check if the first parameter is a Pydantic model
114 | sig = signature(action.function)
115 | parameters = list(sig.parameters.values())
116 | is_pydantic = parameters and issubclass(parameters[0].annotation, BaseModel)
117 | parameter_names = [param.name for param in parameters]
118 |
119 | if sensitive_data:
120 | validated_params = self._replace_sensitive_data(validated_params, sensitive_data)
121 |
122 | # Check if the action requires browser
123 | if 'browser' in parameter_names and not browser:
124 | raise ValueError(f'Action {action_name} requires browser but none provided.')
125 | if 'page_extraction_llm' in parameter_names and not page_extraction_llm:
126 | raise ValueError(f'Action {action_name} requires page_extraction_llm but none provided.')
127 | if 'available_file_paths' in parameter_names and not available_file_paths:
128 | raise ValueError(f'Action {action_name} requires available_file_paths but none provided.')
129 |
130 | if 'context' in parameter_names and not context:
131 | raise ValueError(f'Action {action_name} requires context but none provided.')
132 |
133 | # Prepare arguments based on parameter type
134 | extra_args = {}
135 | if 'context' in parameter_names:
136 | extra_args['context'] = context
137 | if 'browser' in parameter_names:
138 | extra_args['browser'] = browser
139 | if 'page_extraction_llm' in parameter_names:
140 | extra_args['page_extraction_llm'] = page_extraction_llm
141 | if 'available_file_paths' in parameter_names:
142 | extra_args['available_file_paths'] = available_file_paths
143 | if action_name == 'input_text' and sensitive_data:
144 | extra_args['has_sensitive_data'] = True
145 | if is_pydantic:
146 | return await action.function(validated_params, **extra_args)
147 | return await action.function(**validated_params.model_dump(), **extra_args)
148 |
149 | except Exception as e:
150 | raise RuntimeError(f'Error executing action {action_name}: {str(e)}') from e
151 |
152 | def _replace_sensitive_data(self, params: BaseModel, sensitive_data: Dict[str, str]) -> BaseModel:
153 | """Replaces the sensitive data in the params"""
154 | # if there are any str with