├── .env-example ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── demo.py ├── loop.py ├── pyproject.toml ├── reset.sh └── src └── playwright_computer_use ├── __init__.py ├── assets ├── __init__.py └── cursor.png ├── async_api.py └── sync_api.py /.env-example: -------------------------------------------------------------------------------- 1 | ANTHROPIC_API_KEY="sk-..." 2 | INVARIANT_API_KEY="inv-..." 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # UV 98 | # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | #uv.lock 102 | 103 | # poetry 104 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 108 | #poetry.lock 109 | 110 | # pdm 111 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 112 | #pdm.lock 113 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 114 | # in version control. 115 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 116 | .pdm.toml 117 | .pdm-python 118 | .pdm-build/ 119 | 120 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 121 | __pypackages__/ 122 | 123 | # Celery stuff 124 | celerybeat-schedule 125 | celerybeat.pid 126 | 127 | # SageMath parsed files 128 | *.sage.py 129 | 130 | # Environments 131 | .env 132 | .venv 133 | env/ 134 | venv/ 135 | ENV/ 136 | env.bak/ 137 | venv.bak/ 138 | 139 | # Spyder project settings 140 | .spyderproject 141 | .spyproject 142 | 143 | # Rope project settings 144 | .ropeproject 145 | 146 | # mkdocs documentation 147 | /site 148 | 149 | # mypy 150 | .mypy_cache/ 151 | .dmypy.json 152 | dmypy.json 153 | 154 | # Pyre type checker 155 | .pyre/ 156 | 157 | # pytype static type analyzer 158 | .pytype/ 159 | 160 | # Cython debug symbols 161 | cython_debug/ 162 | 163 | # PyCharm 164 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 165 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 166 | # and can be added to the global gitignore or merged into this file. For a more nuclear 167 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 168 | #.idea/ 169 | 170 | # PyPI configuration file 171 | .pypirc 172 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/astral-sh/ruff-pre-commit 3 | # Ruff version. 4 | rev: v0.9.1 5 | hooks: 6 | # Run the linter. 7 | - id: ruff 8 | args: [ --fix ] 9 | files: .*\.py$ 10 | # Run the formatter. 11 | - id: ruff-format 12 | files: .*\.py$ 13 | - repo: https://github.com/pre-commit/mirrors-mypy 14 | rev: 'v1.14.1' # Use the sha / tag you want to point at 15 | hooks: 16 | - id: mypy 17 | files: ^src/.*\.py$ # -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2025 Invariant Labs AG 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Playwright Computer Use 2 | 3 | Easily use the Claude `computer` tool to let an agent interact with a web browser on your machine (playwright). 4 | 5 | https://github.com/user-attachments/assets/3d876280-4822-4679-9dd1-689a0f596041 6 | 7 | This repo contains the required code to connect a Playwright browser to Claude's computer use capabilities. This enables you to use a browser as a tool for your agent, to interact with web pages, and achieve tasks that require a browser. 8 | 9 | **We now also support Claude 3.7!** 10 | 11 | ## Quickstart 12 | 13 | Clone the Repo 14 | ``` 15 | git clone https://github.com/invariantlabs-ai/playwright-computer-use.git 16 | ``` 17 | 18 | Install the dependencies: 19 | ``` 20 | cd playwright-computer-use 21 | pip install -e . 22 | ``` 23 | 24 | Create a `.env` basing on `.env-example` ([Anthropic Key](https://console.anthropic.com) and an optional [Invariant Key](https://explorer.invariantlabs.ai) for tracing). Then run: 25 | 26 | ``` 27 | python demo.py "How long does it take to travel from Zurich to Milan?" 28 | ``` 29 | 30 | This will spawn an agent on your machine that attempts to achieve whatever task you have in mind in the browser. 31 | 32 | ## Install As Package 33 | 34 | ``` 35 | pip install git://git@github.com/invariantlabs-ai/playwright-computer-use.git 36 | ``` 37 | 38 | ## Using the PlaywrightToolbox as a Library 39 | 40 | You can also include the `PlaywrightToolbox` as a tool for `Claude`, to enable the use of a playwright browser in an existing agent. 41 | 42 | ```python 43 | from playwright_computer_use.sync_api import PlaywrightToolbox #Use sync api when working with sync Playwright page, use async otherwise 44 | 45 | tools = PlaywrightToolbox(page=page, use_cursor=True, beta_version="20250124") 46 | 47 | # Give Claude access to computer use tool 48 | response = anthropic_client.beta.messages.create( 49 | ... 50 | model="claude-3-7-sonnet-20250219", 51 | tools=tools.to_params(), 52 | betas=["computer-use-2025-01-24"], 53 | ) 54 | 55 | # Run computer use tool on playwright 56 | tools.run_tool(**response.content[0].model_dump()) 57 | ``` 58 | For a more in-depth example look at `demo.py` 59 | -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- 1 | """Demo of Claude Agent running using Playwright.""" 2 | 3 | import asyncio 4 | from playwright.async_api import async_playwright, Playwright 5 | from loop import sampling_loop, anthropic_to_invariant 6 | from playwright_computer_use.async_api import PlaywrightToolbox 7 | from anthropic import Anthropic 8 | from invariant_sdk.client import Client as InvariantClient 9 | from dotenv import load_dotenv 10 | import os 11 | import sys 12 | 13 | 14 | load_dotenv() 15 | MODEL = "claude-3-7-sonnet-20250219" 16 | 17 | model_to_beta = { 18 | "claude-3-7-sonnet-20250219": "20250124", 19 | "claude-3-5-sonnet-20241022": "20241022", 20 | } 21 | 22 | anthropic_client = Anthropic() 23 | invariant_client = InvariantClient() if "INVARIANT_API_KEY" in os.environ else None 24 | 25 | 26 | async def run(playwright: Playwright, prompt: str): 27 | """Setup tools and run loop.""" 28 | browser = await playwright.firefox.launch(headless=False) 29 | context = await browser.new_context() 30 | page = await context.new_page() 31 | await page.set_viewport_size({"width": 1024, "height": 768}) # Computer-use default 32 | await page.goto("https://www.google.com") 33 | playwright_tools = PlaywrightToolbox( 34 | page, use_cursor=True, beta_version=model_to_beta[MODEL] 35 | ) 36 | messages = await sampling_loop( 37 | model=MODEL, 38 | anthropic_client=anthropic_client, 39 | messages=[{"role": "user", "content": prompt}], 40 | tools=playwright_tools, 41 | page=page, 42 | verbose=True, 43 | only_n_most_recent_images=10, 44 | ) 45 | print(messages[-1]["content"][0]["text"]) 46 | if invariant_client is not None: 47 | response = invariant_client.create_request_and_push_trace( 48 | messages=[anthropic_to_invariant(messages)], 49 | dataset="playwright_computer_use_trace", 50 | ) 51 | url = f"{invariant_client.api_url}/trace/{response.id[0]}" 52 | print(f"View the trace at {url}") 53 | else: 54 | print( 55 | "No INVARIANT_API_KEY found. Add it to your .env file to push the trace to Invariant explorer https://explorer.invariantlabs.ai." 56 | ) 57 | await browser.close() 58 | 59 | 60 | prompt = sys.argv[1] if len(sys.argv) > 1 else "What is the capital of France?" 61 | 62 | 63 | async def main(): 64 | """Run the Agent loop.""" 65 | async with async_playwright() as playwright: 66 | await run(playwright, prompt) 67 | 68 | 69 | asyncio.run(main()) 70 | -------------------------------------------------------------------------------- /loop.py: -------------------------------------------------------------------------------- 1 | """Agentic sampling loop that calls the Anthropic API and local implementation of anthropic-defined computer use tools.""" 2 | 3 | import sys 4 | 5 | from collections.abc import Callable 6 | from datetime import datetime 7 | from typing import cast 8 | 9 | 10 | import httpx 11 | from anthropic import ( 12 | Anthropic, 13 | AnthropicBedrock, 14 | AnthropicVertex, 15 | APIError, 16 | APIResponseValidationError, 17 | APIStatusError, 18 | ) 19 | from playwright.sync_api import Page 20 | from anthropic.types.beta import ( 21 | BetaCacheControlEphemeralParam, 22 | BetaMessage, 23 | BetaMessageParam, 24 | BetaTextBlock, 25 | BetaTextBlockParam, 26 | BetaToolResultBlockParam, 27 | BetaToolUseBlockParam, 28 | ) 29 | 30 | from playwright_computer_use.async_api import PlaywrightToolbox, ToolResult 31 | 32 | COMPUTER_USE_BETA_FLAG = { 33 | "20241022": "computer-use-2024-10-22", 34 | "20250124": "computer-use-2025-01-24", 35 | } 36 | PROMPT_CACHING_BETA_FLAG = "prompt-caching-2024-07-31" 37 | 38 | 39 | # This system prompt is optimized for the Docker environment in this repository and 40 | # specific tool combinations enabled. 41 | # We encourage modifying this system prompt to ensure the model has context for the 42 | # environment it is running in, and to provide any additional information that may be 43 | # helpful for the task at hand. 44 | SYSTEM_PROMPT = f""" 45 | * You are utilising an firefox browser with internet access. The entirity of the task you are given can be solved by navigating from this web page. 46 | * You can only use one page, and you can't open new tabs. 47 | * When viewing a page it can be helpful to zoom out so that you can see everything on the page. Either that, or make sure you scroll down to see everything before deciding something isn't available. 48 | * When using your computer function calls, they take a while to run and send back to you. Where possible/feasible, try to chain multiple of these calls all into one function calls request. At the end always ask for a screenshot, to make sure the state of the page is as you expect. 49 | * The current date is {datetime.today().strftime("%A, %B %-d, %Y")}. 50 | 51 | """ 52 | 53 | 54 | async def sampling_loop( 55 | *, 56 | model: str, 57 | anthropic_client: Anthropic, 58 | system_prompt: str = SYSTEM_PROMPT, 59 | messages: list[BetaMessageParam], 60 | page: Page, 61 | tools: PlaywrightToolbox, 62 | only_n_most_recent_images: int | None = None, 63 | max_tokens: int = 4096, 64 | enable_prompt_caching: bool = True, 65 | verbose: bool = False, 66 | ): 67 | """Agentic sampling loop for the assistant/tool interaction of computer use.""" 68 | assert page is not None, "playwright page must be provided" 69 | 70 | system = BetaTextBlockParam( 71 | type="text", 72 | text=system_prompt, 73 | ) 74 | if verbose: 75 | for message in messages: 76 | if message["role"] == "user": 77 | print(f"user > {message['content']}") 78 | if message["role"] == "assistant": 79 | for content_block in message["content"]: 80 | if content_block["type"] == "text": 81 | print(f"assistant > {content_block['text']}") 82 | if message["role"] == "tool": 83 | print( 84 | f"tool call > {content_block['name']} {content_block['input']}" 85 | ) 86 | while True: 87 | enable_prompt_caching = False 88 | betas = [COMPUTER_USE_BETA_FLAG[tools.beta_version]] 89 | image_truncation_threshold = only_n_most_recent_images or 0 90 | 91 | if enable_prompt_caching: 92 | betas.append(PROMPT_CACHING_BETA_FLAG) 93 | _inject_prompt_caching(messages) 94 | # Because cached reads are 10% of the price, we don't think it's 95 | # ever sensible to break the cache by truncating images 96 | only_n_most_recent_images = 0 97 | system["cache_control"] = {"type": "ephemeral"} 98 | 99 | if only_n_most_recent_images: 100 | _maybe_filter_to_n_most_recent_images( 101 | messages, 102 | only_n_most_recent_images, 103 | min_removal_threshold=image_truncation_threshold, 104 | ) 105 | 106 | # Call the API 107 | # we use raw_response to provide debug information to streamlit. Your 108 | # implementation may be able call the SDK directly with: 109 | # `response = client.messages.create(...)` instead. 110 | try: 111 | if verbose: 112 | sys.stdout.write("Calling Model") 113 | sys.stdout.flush() 114 | response = anthropic_client.beta.messages.create( 115 | max_tokens=max_tokens, 116 | messages=messages, 117 | model=model, 118 | system=[system], 119 | tools=tools.to_params(), 120 | betas=betas, 121 | ) 122 | if verbose: 123 | sys.stdout.write( 124 | "\r\033[K" 125 | ) # Move to the beginning of the line and clear it 126 | sys.stdout.flush() 127 | except (APIStatusError, APIResponseValidationError) as e: 128 | raise e 129 | except APIError as e: 130 | return [{"role": "system", "content": system_prompt}] + messages 131 | 132 | response_params = _response_to_params(response) 133 | messages.append( 134 | { 135 | "role": "assistant", 136 | "content": response_params, 137 | } 138 | ) 139 | 140 | tool_result_content: list[BetaToolResultBlockParam] = [] 141 | for content_block in response_params: 142 | if content_block["type"] == "tool_use": 143 | if verbose: 144 | print( 145 | f"tool call > {content_block['name']} {content_block['input']}" 146 | ) 147 | result = await tools.run_tool( 148 | name=content_block["name"], 149 | input=content_block["input"], 150 | tool_use_id=content_block["id"], 151 | ) 152 | tool_result_content.append(result) 153 | if verbose and content_block["type"] == "text": 154 | print(f"assistant > {content_block['text']}") 155 | 156 | if not tool_result_content: 157 | return [{"role": "system", "content": system_prompt}] + messages 158 | 159 | messages.append({"content": tool_result_content, "role": "user"}) 160 | 161 | 162 | def anthropic_to_invariant( 163 | messages: list[dict], keep_empty_tool_response: bool = False 164 | ) -> list[dict]: 165 | """Converts a list of messages from the Anthropic API to the Invariant API format.""" 166 | output = [] 167 | for message in messages: 168 | if message["role"] == "system": 169 | output.append({"role": "system", "content": message["content"]}) 170 | if message["role"] == "user": 171 | if isinstance(message["content"], list): 172 | for sub_message in message["content"]: 173 | assert sub_message["type"] == "tool_result" 174 | if sub_message["content"]: 175 | assert len(sub_message["content"]) == 1 176 | assert sub_message["content"][0]["type"] == "image" 177 | output.append( 178 | { 179 | "role": "tool", 180 | "content": "local_base64_img: " 181 | + sub_message["content"][0]["source"]["data"], 182 | "tool_id": sub_message["tool_use_id"], 183 | } 184 | ) 185 | else: 186 | if keep_empty_tool_response and any( 187 | [sub_message[k] for k in sub_message] 188 | ): 189 | output.append( 190 | { 191 | "role": "tool", 192 | "content": {"is_error": True} 193 | if sub_message["is_error"] 194 | else {}, 195 | "tool_id": sub_message["tool_use_id"], 196 | } 197 | ) 198 | else: 199 | output.append({"role": "user", "content": message["content"]}) 200 | if message["role"] == "assistant": 201 | for sub_message in message["content"]: 202 | if sub_message["type"] == "text": 203 | output.append( 204 | {"role": "assistant", "content": sub_message.get("text")} 205 | ) 206 | if sub_message["type"] == "tool_use": 207 | output.append( 208 | { 209 | "role": "assistant", 210 | "content": None, 211 | "tool_calls": [ 212 | { 213 | "tool_id": sub_message.get("id"), 214 | "type": "function", 215 | "function": { 216 | "name": sub_message.get("name"), 217 | "arguments": sub_message.get("input"), 218 | }, 219 | } 220 | ], 221 | } 222 | ) 223 | return output 224 | 225 | 226 | def _maybe_filter_to_n_most_recent_images( 227 | messages: list[BetaMessageParam], 228 | images_to_keep: int, 229 | min_removal_threshold: int, 230 | ): 231 | """Only keep latest images. 232 | 233 | With the assumption that images are screenshots that are of diminishing value as 234 | the conversation progresses, remove all but the final `images_to_keep` tool_result 235 | images in place, with a chunk of min_removal_threshold to reduce the amount we 236 | break the implicit prompt cache. 237 | """ 238 | if images_to_keep is None: 239 | return messages 240 | 241 | tool_result_blocks = cast( 242 | list[BetaToolResultBlockParam], 243 | [ 244 | item 245 | for message in messages 246 | for item in ( 247 | message["content"] if isinstance(message["content"], list) else [] 248 | ) 249 | if isinstance(item, dict) and item.get("type") == "tool_result" 250 | ], 251 | ) 252 | 253 | total_images = sum( 254 | 1 255 | for tool_result in tool_result_blocks 256 | for content in tool_result.get("content", []) 257 | if isinstance(content, dict) and content.get("type") == "image" 258 | ) 259 | 260 | images_to_remove = total_images - images_to_keep 261 | # for better cache behavior, we want to remove in chunks 262 | images_to_remove -= images_to_remove % min_removal_threshold 263 | 264 | for tool_result in tool_result_blocks: 265 | if isinstance(tool_result.get("content"), list): 266 | new_content = [] 267 | for content in tool_result.get("content", []): 268 | if isinstance(content, dict) and content.get("type") == "image": 269 | if images_to_remove > 0: 270 | images_to_remove -= 1 271 | continue 272 | new_content.append(content) 273 | tool_result["content"] = new_content 274 | 275 | 276 | def _response_to_params( 277 | response: BetaMessage, 278 | ) -> list[BetaTextBlockParam | BetaToolUseBlockParam]: 279 | res: list[BetaTextBlockParam | BetaToolUseBlockParam] = [] 280 | for block in response.content: 281 | if isinstance(block, BetaTextBlock): 282 | res.append({"type": "text", "text": block.text}) 283 | else: 284 | res.append(cast(BetaToolUseBlockParam, block.model_dump())) 285 | return res 286 | 287 | 288 | def _inject_prompt_caching( 289 | messages: list[BetaMessageParam], 290 | ): 291 | """Set cache breakpoints for the 3 most recent turns. 292 | 293 | One cache breakpoint is left for tools/system prompt, to be shared across sessions 294 | """ 295 | breakpoints_remaining = 3 296 | for message in reversed(messages): 297 | if message["role"] == "user" and isinstance( 298 | content := message["content"], list 299 | ): 300 | if breakpoints_remaining: 301 | breakpoints_remaining -= 1 302 | content[-1]["cache_control"] = BetaCacheControlEphemeralParam( 303 | {"type": "ephemeral"} 304 | ) 305 | else: 306 | content[-1].pop("cache_control", None) 307 | # we'll only every have one extra turn per loop 308 | break 309 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools>=61.0", 4 | ] 5 | 6 | 7 | build-backend = "setuptools.build_meta" 8 | 9 | [project] 10 | name = "playwright-computer-use" 11 | version = "0.0.1" 12 | authors = [ 13 | { name="Marco Milanta", email="marco@invariantlabs.ai" }, 14 | ] 15 | description = "A package to connect Claude computer use with Playwright." 16 | readme = "README.md" 17 | requires-python = ">=3.9" 18 | classifiers = [ 19 | "Programming Language :: Python :: 3", 20 | "License :: OSI Approved :: MIT License", 21 | "Operating System :: OS Independent", 22 | ] 23 | dependencies = [ 24 | "anthropic", 25 | "python-dotenv", 26 | "playwright", 27 | "Pillow>=10.1.0,<11", 28 | "invariant-sdk", 29 | ] 30 | [project.urls] 31 | Homepage = "https://github.com/invariantlabs-ai/playwright-computer-use" 32 | Issues = "https://github.com/invariantlabs-ai/playwright-computer-use/issues" 33 | 34 | 35 | [tool.setuptools.package-data] 36 | "playwright_computer_use" = ["assets/*.png"] 37 | 38 | 39 | [tool.ruff] 40 | select = ["D", "G"] # Include docstring rules (D) and Google-style docstring rules (G) 41 | ignore = [] # List rules to ignore, if any 42 | extend-ignore = [] # Optionally extend the ignore list 43 | 44 | # Optionally, specify additional settings for Ruff 45 | line-length = 88 # Adjust line length if needed 46 | 47 | [tool.ruff.lint.pydocstyle] 48 | convention = "google" 49 | 50 | [tool.mypy] 51 | files = "src" 52 | -------------------------------------------------------------------------------- /reset.sh: -------------------------------------------------------------------------------- 1 | deactivate 2 | rm -r venv 3 | python3 -m venv venv 4 | source venv/bin/activate 5 | pip install . -------------------------------------------------------------------------------- /src/playwright_computer_use/__init__.py: -------------------------------------------------------------------------------- 1 | """General utilities for the project.""" 2 | -------------------------------------------------------------------------------- /src/playwright_computer_use/assets/__init__.py: -------------------------------------------------------------------------------- 1 | """Assets.""" 2 | -------------------------------------------------------------------------------- /src/playwright_computer_use/assets/cursor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/invariantlabs-ai/playwright-computer-use/41b901c8f9b33d463c909f83c34ecd93dc59fb70/src/playwright_computer_use/assets/cursor.png -------------------------------------------------------------------------------- /src/playwright_computer_use/async_api.py: -------------------------------------------------------------------------------- 1 | """This module contains the PlaywrightToolbox class to be used with an Async Playwright Page.""" 2 | 3 | import importlib.resources 4 | import base64 5 | from typing import Literal, TypedDict, get_args, Type, cast 6 | from playwright.async_api import Page 7 | from asyncio import sleep 8 | from PIL import Image 9 | import io 10 | from anthropic.types.beta import ( 11 | BetaToolComputerUse20241022Param, 12 | BetaToolComputerUse20250124Param, 13 | BetaToolParam, 14 | BetaToolResultBlockParam, 15 | BetaTextBlockParam, 16 | BetaImageBlockParam, 17 | ) 18 | from dataclasses import dataclass 19 | 20 | TYPING_DELAY_MS = 12 21 | SCROLL_MULTIPLIER_FACTOR = 500 22 | TYPING_GROUP_SIZE = 50 23 | 24 | Action_20241022 = Literal[ 25 | "key", 26 | "type", 27 | "mouse_move", 28 | "left_click", 29 | "left_click_drag", 30 | "right_click", 31 | "middle_click", 32 | "double_click", 33 | "screenshot", 34 | "cursor_position", 35 | ] 36 | 37 | Action_20250124 = ( 38 | Action_20241022 39 | | Literal[ 40 | "left_mouse_down", 41 | "left_mouse_up", 42 | "scroll", 43 | "hold_key", 44 | "wait", 45 | "triple_click", 46 | ] 47 | ) 48 | 49 | ScrollDirection = Literal["up", "down", "left", "right"] 50 | 51 | 52 | class ComputerToolOptions(TypedDict): 53 | """Options for the computer tool.""" 54 | 55 | display_height_px: int 56 | display_width_px: int 57 | display_number: int | None 58 | 59 | 60 | def chunks(s: str, chunk_size: int) -> list[str]: 61 | """Split a string into chunks of a specific size.""" 62 | return [s[i : i + chunk_size] for i in range(0, len(s), chunk_size)] 63 | 64 | 65 | @dataclass(kw_only=True, frozen=True) 66 | class ToolResult: 67 | """Represents the result of a tool execution.""" 68 | 69 | output: str | None = None 70 | error: str | None = None 71 | base64_image: str | None = None 72 | 73 | 74 | class ToolError(Exception): 75 | """Raised when a tool encounters an error.""" 76 | 77 | def __init__(self, message): 78 | """Create a new ToolError.""" 79 | self.message = message 80 | 81 | 82 | class PlaywrightToolbox: 83 | """Toolbox for interaction between Claude and Async Playwright Page.""" 84 | 85 | def __init__( 86 | self, 87 | page: Page, 88 | use_cursor: bool = True, 89 | screenshot_wait_until: Literal["load", "domcontentloaded", "networkidle"] 90 | | None = None, 91 | beta_version: Literal["20241022", "20250124"] = "20250124", 92 | ): 93 | """Create a new PlaywrightToolbox. 94 | 95 | Args: 96 | page: The Async Playwright page to interact with. 97 | use_cursor: Whether to display the cursor in the screenshots or not. 98 | screenshot_wait_until: Optional, wait until the page is in a specific state before taking a screenshot. Default does not wait 99 | beta_version: The version of the beta to use. Default is the latest version (Claude3.7) 100 | """ 101 | self.page = page 102 | self.beta_version = beta_version 103 | computer_tool_map: dict[str, Type[BasePlaywrightComputerTool]] = { 104 | "20241022": PlaywrightComputerTool20241022, 105 | "20250124": PlaywrightComputerTool20250124, 106 | } 107 | ComputerTool = computer_tool_map[beta_version] 108 | self.tools: list[ 109 | BasePlaywrightComputerTool | PlaywrightSetURLTool | PlaywrightBackTool 110 | ] = [ 111 | ComputerTool( 112 | page, use_cursor=use_cursor, screenshot_wait_until=screenshot_wait_until 113 | ), 114 | PlaywrightSetURLTool(page), 115 | PlaywrightBackTool(page), 116 | ] 117 | 118 | def to_params(self) -> list[BetaToolParam]: 119 | """Expose the params of all the tools in the toolbox.""" 120 | return [tool.to_params() for tool in self.tools] 121 | 122 | async def run_tool( 123 | self, name: str, input: dict, tool_use_id: str 124 | ) -> BetaToolResultBlockParam: 125 | """Pick the right tool using `name` and run it.""" 126 | if name not in [tool.name for tool in self.tools]: 127 | return ToolError(message=f"Unknown tool {name}, only computer use allowed") 128 | tool = next(tool for tool in self.tools if tool.name == name) 129 | result = await tool(**input) 130 | return _make_api_tool_result(tool_use_id=tool_use_id, result=result) 131 | 132 | 133 | class PlaywrightSetURLTool: 134 | """Tool to navigate to a specific URL.""" 135 | 136 | name: Literal["set_url"] = "set_url" 137 | 138 | def __init__(self, page: Page): 139 | """Create a new PlaywrightSetURLTool. 140 | 141 | Args: 142 | page: The Async Playwright page to interact with. 143 | """ 144 | super().__init__() 145 | self.page = page 146 | 147 | def to_params(self) -> BetaToolParam: 148 | """Params describing the tool. Description used by Claude to understand how to this use tool.""" 149 | return BetaToolParam( 150 | name=self.name, 151 | description="This tool allows to go directly to a specified URL.", 152 | input_schema={ 153 | "type": "object", 154 | "properties": { 155 | "url": { 156 | "type": "string", 157 | "description": "URL of the web page to navigate to.", 158 | } 159 | }, 160 | "required": ["url"], 161 | }, 162 | ) 163 | 164 | async def __call__(self, *, url: str): 165 | """Trigger goto the chosen url.""" 166 | try: 167 | await self.page.goto(url) 168 | return ToolResult() 169 | except Exception as e: 170 | return ToolResult(error=str(e)) 171 | 172 | 173 | class PlaywrightBackTool: 174 | """Tool to navigate to the previous page.""" 175 | 176 | name: Literal["previous_page"] = "previous_page" 177 | 178 | def __init__(self, page: Page): 179 | """Create a new PlaywrightBackTool. 180 | 181 | Args: 182 | page: The Async Playwright page to interact with. 183 | """ 184 | super().__init__() 185 | self.page = page 186 | 187 | def to_params(self) -> BetaToolParam: 188 | """Params describing the tool. Description used by Claude to understand how to this use tool.""" 189 | return BetaToolParam( 190 | name=self.name, 191 | description="This tool navigate to the previous page.", 192 | input_schema={ 193 | "type": "object", 194 | "properties": {}, 195 | "required": [], 196 | }, 197 | ) 198 | 199 | async def __call__(self): 200 | """Trigger the back button in the browser.""" 201 | try: 202 | await self.page.go_back() 203 | return ToolResult() 204 | except Exception as e: 205 | return ToolResult(error=str(e)) 206 | 207 | 208 | class BasePlaywrightComputerTool: 209 | """A tool that allows the agent to interact with Async Playwright Page.""" 210 | 211 | name: Literal["computer"] = "computer" 212 | 213 | @property 214 | def width(self) -> int: 215 | """The width of the Playwright page in pixels.""" 216 | return self.page.viewport_size["width"] 217 | 218 | @property 219 | def height(self) -> int: 220 | """The height of the Playwright page in pixels.""" 221 | return self.page.viewport_size["height"] 222 | 223 | @property 224 | def options(self) -> ComputerToolOptions: 225 | """The options of the tool.""" 226 | return { 227 | "display_width_px": self.width, 228 | "display_height_px": self.height, 229 | "display_number": 1, # hardcoded 230 | } 231 | 232 | def to_params(self): 233 | """Params describing the tool. Used by Claude to understand this is a computer use tool.""" 234 | raise NotImplementedError("to_params must be implemented in the subclass") 235 | 236 | def __init__( 237 | self, 238 | page: Page, 239 | use_cursor: bool = True, 240 | screenshot_wait_until: Literal["load", "domcontentloaded", "networkidle"] 241 | | None = None, 242 | ): 243 | """Initializes the PlaywrightComputerTool. 244 | 245 | Args: 246 | page: The Async Playwright page to interact with. 247 | use_cursor: Whether to display the cursor in the screenshots or not. 248 | screenshot_wait_until: Optional, wait until the page is in a specific state before taking a screenshot. Default does not wait 249 | """ 250 | super().__init__() 251 | self.page = page 252 | self.use_cursor = use_cursor 253 | self.mouse_position: tuple[int, int] = (0, 0) 254 | self.screenshot_wait_until = screenshot_wait_until 255 | 256 | async def __call__( 257 | self, 258 | *, 259 | action: Action_20241022, 260 | text: str | None = None, 261 | coordinate: tuple[int, int] | None = None, 262 | **kwargs, 263 | ): 264 | """Run an action. text and coordinate are potential additional parameters.""" 265 | if action in ("mouse_move", "left_click_drag"): 266 | if coordinate is None: 267 | raise ToolError(f"coordinate is required for {action}") 268 | if text is not None: 269 | raise ToolError(f"text is not accepted for {action}") 270 | if not isinstance(coordinate, list) or len(coordinate) != 2: 271 | raise ToolError(f"{coordinate} must be a tuple of length 2") 272 | if not all(isinstance(i, int) and i >= 0 for i in coordinate): 273 | raise ToolError(f"{coordinate} must be a tuple of non-negative ints") 274 | 275 | x, y = coordinate 276 | 277 | if action == "mouse_move": 278 | await self.page.mouse.move(x, y) 279 | self.mouse_position = (x, y) 280 | return ToolResult(output=None, error=None, base64_image=None) 281 | elif action == "left_click_drag": 282 | raise NotImplementedError("left_click_drag is not implemented yet") 283 | 284 | if action in ("key", "type"): 285 | if text is None: 286 | raise ToolError(f"text is required for {action}") 287 | if coordinate is not None: 288 | raise ToolError(f"coordinate is not accepted for {action}") 289 | if not isinstance(text, str): 290 | raise ToolError(output=f"{text} must be a string") 291 | 292 | if action == "key": 293 | # hande shifts 294 | await self.press_key(text) 295 | return ToolResult() 296 | elif action == "type": 297 | for chunk in chunks(text, TYPING_GROUP_SIZE): 298 | await self.page.keyboard.type(chunk) 299 | return await self.screenshot() 300 | 301 | if action in ( 302 | "left_click", 303 | "right_click", 304 | "double_click", 305 | "middle_click", 306 | "screenshot", 307 | "cursor_position", 308 | ): 309 | if text is not None: 310 | raise ToolError(f"text is not accepted for {action}") 311 | if coordinate is not None: 312 | raise ToolError(f"coordinate is not accepted for {action}") 313 | 314 | if action == "screenshot": 315 | return await self.screenshot() 316 | elif action == "cursor_position": 317 | return ToolResult( 318 | output=f"X={self.mouse_position[0]},Y={self.mouse_position[1]}" 319 | ) 320 | else: 321 | click_arg = { 322 | "left_click": {"button": "left", "click_count": 1}, 323 | "right_click": {"button": "right", "click_count": 1}, 324 | "middle_click": {"button": "middle", "click_count": 1}, 325 | "double_click": {"button": "left", "click_count": 2, "delay": 100}, 326 | }[action] 327 | await self.page.mouse.click( 328 | self.mouse_position[0], self.mouse_position[1], **click_arg 329 | ) 330 | return ToolResult() 331 | 332 | raise ToolError(f"Invalid action: {action}") 333 | 334 | async def screenshot(self) -> ToolResult: 335 | """Take a screenshot of the current screen and return the base64 encoded image.""" 336 | if self.screenshot_wait_until is not None: 337 | await self.page.wait_for_load_state(self.screenshot_wait_until) 338 | await self.page.wait_for_load_state() 339 | screenshot = await self.page.screenshot() 340 | image = Image.open(io.BytesIO(screenshot)) 341 | img_small = image.resize((self.width, self.height), Image.LANCZOS) 342 | if self.use_cursor: 343 | cursor = load_cursor_image() 344 | img_small.paste(cursor, self.mouse_position, cursor) 345 | buffered = io.BytesIO() 346 | img_small.save(buffered, format="PNG") 347 | base64_image = base64.b64encode(buffered.getvalue()).decode() 348 | return ToolResult(base64_image=base64_image) 349 | 350 | async def press_key(self, key: str): 351 | """Press a key on the keyboard. Handle + shifts. Eg: Ctrl+Shift+T.""" 352 | shifts = [] 353 | if "+" in key: 354 | shifts += key.split("+")[:-1] 355 | key = key.split("+")[-1] 356 | for shift in shifts: 357 | await self.page.keyboard.down(shift) 358 | await self.page.keyboard.press(to_playwright_key(key)) 359 | for shift in shifts: 360 | await self.page.keyboard.up(shift) 361 | 362 | 363 | class PlaywrightComputerTool20241022(BasePlaywrightComputerTool): 364 | """Tool to interact with the computer using Playwright (Beta 22/10/2024).""" 365 | 366 | api_type: Literal["computer_20241022"] = "computer_20241022" 367 | 368 | def to_params(self) -> BetaToolComputerUse20241022Param: 369 | """Params describing the tool. Used by Claude to understand this is a computer use tool.""" 370 | return {"name": self.name, "type": self.api_type, **self.options} 371 | 372 | 373 | class PlaywrightComputerTool20250124(BasePlaywrightComputerTool): 374 | """Tool to interact with the computer using Playwright (Beta 24/01/2025).""" 375 | 376 | api_type: Literal["computer_20250124"] = "computer_20250124" 377 | 378 | def to_params(self) -> BetaToolComputerUse20250124Param: 379 | """Params describing the tool. Used by Claude to understand this is a computer use tool.""" 380 | return {"name": self.name, "type": self.api_type, **self.options} 381 | 382 | async def __call__( 383 | self, 384 | *, 385 | action: Action_20250124, 386 | text: str | None = None, 387 | coordinate: tuple[int, int] | None = None, 388 | scroll_direction: ScrollDirection | None = None, 389 | scroll_amount: int | None = None, 390 | duration: int | float | None = None, 391 | key: str | None = None, 392 | **kwargs, 393 | ): 394 | """Run an action. text, coordinate, scroll_directions, scroll_amount, duration, key are potential additional parameters.""" 395 | if action in ("left_mouse_down", "left_mouse_up"): 396 | if coordinate is not None: 397 | raise ToolError(f"coordinate is not accepted for {action=}.") 398 | await ( 399 | self.page.mouse.down() 400 | ) if action == "left_mouse_down" else await self.page.mouse.up() 401 | return ToolResult() 402 | if action == "scroll": 403 | if scroll_direction is None or scroll_direction not in get_args( 404 | ScrollDirection 405 | ): 406 | raise ToolError( 407 | f"{scroll_direction=} must be 'up', 'down', 'left', or 'right'" 408 | ) 409 | if not isinstance(scroll_amount, int) or scroll_amount < 0: 410 | raise ToolError(f"{scroll_amount=} must be a non-negative int") 411 | if coordinate is not None: 412 | x, y = coordinate 413 | await self.page.mouse.move(x, y) 414 | self.mouse_position = (x, y) 415 | scroll_amount *= SCROLL_MULTIPLIER_FACTOR 416 | scroll_params = { 417 | "up": {"delta_y": -scroll_amount, "delta_x": 0}, 418 | "down": {"delta_y": scroll_amount, "delta_x": 0}, 419 | "left": {"delta_y": 0, "delta_x": scroll_amount}, 420 | "right": {"delta_y": 0, "delta_x": -scroll_amount}, 421 | }[scroll_direction] 422 | 423 | await self.page.mouse.wheel(**scroll_params) 424 | return ToolResult() 425 | 426 | if action in ("hold_key", "wait"): 427 | if duration is None or not isinstance(duration, (int, float)): 428 | raise ToolError(f"{duration=} must be a number") 429 | if duration < 0: 430 | raise ToolError(f"{duration=} must be non-negative") 431 | if duration > 100: 432 | raise ToolError(f"{duration=} is too long.") 433 | 434 | if action == "hold_key": 435 | if text is None: 436 | raise ToolError(f"text is required for {action}") 437 | await self.page.keyboard.press(to_playwright_key(text), delay=duration) 438 | return ToolResult() 439 | 440 | if action == "wait": 441 | await sleep(duration) 442 | return await self.screenshot() 443 | 444 | if action in ( 445 | "left_click", 446 | "right_click", 447 | "double_click", 448 | "triple_click", 449 | "middle_click", 450 | ): 451 | if text is not None: 452 | raise ToolError(f"text is not accepted for {action}") 453 | mouse_move_part = "" 454 | if coordinate is not None: 455 | x, y = coordinate 456 | await self.page.mouse.move(x, y) 457 | self.mouse_position = (x, y) 458 | 459 | click_arg = { 460 | "left_click": {"button": "left", "click_count": 1}, 461 | "right_click": {"button": "right", "click_count": 1}, 462 | "middle_click": {"button": "middle", "click_count": 1}, 463 | "double_click": {"button": "left", "click_count": 2, "delay": 10}, 464 | "double_click": {"button": "left", "click_count": 3, "delay": 10}, 465 | }[action] 466 | if key: 467 | self.page.keyboard.down(to_playwright_key(key)) 468 | await self.page.mouse.click( 469 | self.mouse_position[0], self.mouse_position[1], **click_arg 470 | ) 471 | if key: 472 | self.page.keyboard.up(to_playwright_key(key)) 473 | 474 | return ToolResult() 475 | action = cast(Action_20241022, action) 476 | return await super().__call__( 477 | action=action, text=text, coordinate=coordinate, key=key, **kwargs 478 | ) 479 | 480 | 481 | def to_playwright_key(key: str) -> str: 482 | """Convert a key to the Playwright key format.""" 483 | valid_keys = ( 484 | ["F{i}" for i in range(1, 13)] 485 | + ["Digit{i}" for i in range(10)] 486 | + ["Key{i}" for i in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"] 487 | + [i for i in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"] 488 | + [i.lower() for i in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"] 489 | + [ 490 | "Backquote", 491 | "Minus", 492 | "Equal", 493 | "Backslash", 494 | "Backspace", 495 | "Tab", 496 | "Delete", 497 | "Escape", 498 | "ArrowDown", 499 | "End", 500 | "Enter", 501 | "Home", 502 | "Insert", 503 | "PageDown", 504 | "PageUp", 505 | "ArrowRight", 506 | "ArrowUp", 507 | ] 508 | ) 509 | if key in valid_keys: 510 | return key 511 | if key == "Return": 512 | return "Enter" 513 | if key == "Page_Down": 514 | return "PageDown" 515 | if key == "Page_Up": 516 | return "PageUp" 517 | if key == "Left": 518 | return "ArrowLeft" 519 | if key == "Right": 520 | return "ArrowRight" 521 | if key == "Up": 522 | return "ArrowUp" 523 | if key == "Down": 524 | return "ArrowDown" 525 | if key == "BackSpace": 526 | return "Backspace" 527 | if key == "alt": 528 | return "Alt" 529 | print(f"Key {key} is not properly mapped into playwright") 530 | return key 531 | 532 | 533 | def load_cursor_image(): 534 | """Access the cursor.png file in the assets directory.""" 535 | with importlib.resources.open_binary( 536 | "playwright_computer_use.assets", "cursor.png" 537 | ) as img_file: 538 | image = Image.open(img_file) 539 | image.load() # Ensure the image is fully loaded into memory 540 | return image 541 | 542 | 543 | def _make_api_tool_result( 544 | result: ToolResult, tool_use_id: str 545 | ) -> BetaToolResultBlockParam: 546 | """Convert an agent ToolResult to an API ToolResultBlockParam.""" 547 | if result.error: 548 | return BetaToolResultBlockParam( 549 | tool_use_id=tool_use_id, 550 | is_error=True, 551 | content=result.error, 552 | type="tool_result", 553 | ) 554 | else: 555 | tool_result_content: list[BetaTextBlockParam | BetaImageBlockParam] = [] 556 | if result.output: 557 | tool_result_content.append( 558 | BetaTextBlockParam( 559 | type="text", 560 | text=result.output, 561 | ) 562 | ) 563 | if result.base64_image: 564 | tool_result_content.append( 565 | BetaImageBlockParam( 566 | type="image", 567 | source={ 568 | "type": "base64", 569 | "media_type": "image/png", 570 | "data": result.base64_image, 571 | }, 572 | ) 573 | ) 574 | return BetaToolResultBlockParam( 575 | tool_use_id=tool_use_id, 576 | is_error=False, 577 | content=tool_result_content, 578 | type="tool_result", 579 | ) 580 | -------------------------------------------------------------------------------- /src/playwright_computer_use/sync_api.py: -------------------------------------------------------------------------------- 1 | """This module contains the PlaywrightToolbox class to be used with an Async Playwright Page.""" 2 | 3 | from playwright.sync_api import Page 4 | from anthropic.types.beta import ( 5 | BetaToolComputerUse20241022Param, 6 | BetaToolParam, 7 | BetaToolComputerUse20250124Param, 8 | ) 9 | from typing import Literal, get_args, cast, Type 10 | from PIL import Image 11 | import importlib.resources 12 | from time import sleep 13 | import io 14 | import base64 15 | from playwright_computer_use.async_api import ( 16 | ToolError, 17 | ToolResult, 18 | ComputerToolOptions, 19 | Action_20250124, 20 | Action_20241022, 21 | ScrollDirection, 22 | chunks, 23 | TYPING_GROUP_SIZE, 24 | SCROLL_MULTIPLIER_FACTOR, 25 | to_playwright_key, 26 | load_cursor_image, 27 | _make_api_tool_result, 28 | ) 29 | 30 | 31 | class PlaywrightToolbox: 32 | """Toolbox for interaction between Claude and Async Playwright Page.""" 33 | 34 | def __init__( 35 | self, 36 | page: Page, 37 | use_cursor: bool = True, 38 | screenshot_wait_until: Literal["load", "domcontentloaded", "networkidle"] 39 | | None = None, 40 | beta_version: Literal["20241022", "20250124"] = "20250124", 41 | ): 42 | """Create a new PlaywrightToolbox. 43 | 44 | Args: 45 | page: The Async Playwright page to interact with. 46 | use_cursor: Whether to display the cursor in the screenshots or not. 47 | screenshot_wait_until: Optional, wait until the page is in a specific state before taking a screenshot. Default does not wait 48 | beta_version: The version of the beta to use. Default is the latest version (Claude3.7) 49 | """ 50 | self.page = page 51 | self.beta_version = beta_version 52 | computer_tool_map: dict[str, Type[BasePlaywrightComputerTool]] = { 53 | "20241022": PlaywrightComputerTool20241022, 54 | "20250124": PlaywrightComputerTool20250124, 55 | } 56 | ComputerTool = computer_tool_map[beta_version] 57 | self.tools: list[ 58 | BasePlaywrightComputerTool | PlaywrightSetURLTool | PlaywrightBackTool 59 | ] = [ 60 | ComputerTool( 61 | page, use_cursor=use_cursor, screenshot_wait_until=screenshot_wait_until 62 | ), 63 | PlaywrightSetURLTool(page), 64 | PlaywrightBackTool(page), 65 | ] 66 | 67 | def to_params(self) -> list[BetaToolParam]: 68 | """Expose the params of all the tools in the toolbox.""" 69 | return [tool.to_params() for tool in self.tools] 70 | 71 | def run_tool(self, name: str, input: dict, tool_use_id: str): 72 | """Pick the right tool using `name` and run it.""" 73 | if name not in [tool.name for tool in self.tools]: 74 | return ToolError(message=f"Unknown tool {name}, only computer use allowed") 75 | tool = next(tool for tool in self.tools if tool.name == name) 76 | result = tool(**input) 77 | return _make_api_tool_result(tool_use_id=tool_use_id, result=result) 78 | 79 | 80 | class PlaywrightSetURLTool: 81 | """Tool to navigate to a specific URL.""" 82 | 83 | name: Literal["set_url"] = "set_url" 84 | 85 | def __init__(self, page: Page): 86 | """Create a new PlaywrightSetURLTool. 87 | 88 | Args: 89 | page: The Sync Playwright page to interact with. 90 | """ 91 | super().__init__() 92 | self.page = page 93 | 94 | def to_params(self) -> BetaToolParam: 95 | """Params describing the tool. Description used by Claude to understand how to this use tool.""" 96 | return BetaToolParam( 97 | name=self.name, 98 | description="This tool allows to go directly to a specified URL.", 99 | input_schema={ 100 | "type": "object", 101 | "properties": { 102 | "url": { 103 | "type": "string", 104 | "description": "URL of the web page to navigate to.", 105 | } 106 | }, 107 | "required": ["url"], 108 | }, 109 | ) 110 | 111 | def __call__(self, *, url: str): 112 | """Trigger goto the chosen url.""" 113 | try: 114 | self.page.goto(url) 115 | return ToolResult() 116 | except Exception as e: 117 | return ToolResult(error=str(e)) 118 | 119 | 120 | class PlaywrightBackTool: 121 | """Tool to navigate to the previous page.""" 122 | 123 | name: Literal["previous_page"] = "previous_page" 124 | 125 | def __init__(self, page: Page): 126 | """Create a new PlaywrightBackTool. 127 | 128 | Args: 129 | page: The Sync Playwright page to interact with. 130 | """ 131 | super().__init__() 132 | self.page = page 133 | 134 | def to_params(self) -> BetaToolParam: 135 | """Params describing the tool. Description used by Claude to understand how to this use tool.""" 136 | return BetaToolParam( 137 | name=self.name, 138 | description="This tool navigate to the previous page.", 139 | input_schema={ 140 | "type": "object", 141 | "properties": {}, 142 | "required": [], 143 | }, 144 | ) 145 | 146 | def __call__(self): 147 | """Trigger the back button in the browser.""" 148 | try: 149 | self.page.go_back() 150 | return ToolResult() 151 | except Exception as e: 152 | return ToolResult(error=str(e)) 153 | 154 | 155 | class BasePlaywrightComputerTool: 156 | """A tool that allows the agent to interact with Sync Playwright Page.""" 157 | 158 | name: Literal["computer"] = "computer" 159 | 160 | @property 161 | def width(self) -> int: 162 | """The width of the Playwright page in pixels.""" 163 | return self.page.viewport_size["width"] 164 | 165 | @property 166 | def height(self) -> int: 167 | """The height of the Playwright page in pixels.""" 168 | return self.page.viewport_size["height"] 169 | 170 | @property 171 | def options(self) -> ComputerToolOptions: 172 | """The options of the tool.""" 173 | return { 174 | "display_width_px": self.width, 175 | "display_height_px": self.height, 176 | "display_number": 0, # hardcoded 177 | } 178 | 179 | def to_params(self): 180 | """Params describing the tool. Used by Claude to understand this is a computer use tool.""" 181 | raise NotImplementedError("to_params must be implemented in the subclass") 182 | 183 | def __init__( 184 | self, 185 | page: Page, 186 | use_cursor: bool = True, 187 | screenshot_wait_until: Literal["load", "domcontentloaded", "networkidle"] 188 | | None = None, 189 | ): 190 | """Initializes the PlaywrightComputerTool. 191 | 192 | Args: 193 | page: The Sync Playwright page to interact with. 194 | use_cursor: Whether to display the cursor in the screenshots or not. 195 | screenshot_wait_until: Optional, wait until the page is in a specific state before taking a screenshot. Default does not wait 196 | """ 197 | super().__init__() 198 | self.page = page 199 | self.use_cursor = use_cursor 200 | self.mouse_position: tuple[int, int] = (0, 0) 201 | self.screenshot_wait_until = screenshot_wait_until 202 | 203 | def __call__( 204 | self, 205 | *, 206 | action: Action_20241022, 207 | text: str | None = None, 208 | coordinate: tuple[int, int] | None = None, 209 | **kwargs, 210 | ): 211 | """Run an action. text and coordinate are potential additional parameters.""" 212 | if action in ("mouse_move", "left_click_drag"): 213 | if coordinate is None: 214 | raise ToolError(f"coordinate is required for {action}") 215 | if text is not None: 216 | raise ToolError(f"text is not accepted for {action}") 217 | if not isinstance(coordinate, list) or len(coordinate) != 2: 218 | raise ToolError(f"{coordinate} must be a tuple of length 2") 219 | if not all(isinstance(i, int) and i >= 0 for i in coordinate): 220 | raise ToolError(f"{coordinate} must be a tuple of non-negative ints") 221 | 222 | x, y = coordinate 223 | 224 | if action == "mouse_move": 225 | action = self.page.mouse.move(x, y) 226 | self.mouse_position = (x, y) 227 | return ToolResult(output=None, error=None, base64_image=None) 228 | elif action == "left_click_drag": 229 | raise NotImplementedError("left_click_drag is not implemented yet") 230 | 231 | if action in ("key", "type"): 232 | if text is None: 233 | raise ToolError(f"text is required for {action}") 234 | if coordinate is not None: 235 | raise ToolError(f"coordinate is not accepted for {action}") 236 | if not isinstance(text, str): 237 | raise ToolError(output=f"{text} must be a string") 238 | 239 | if action == "key": 240 | # hande shifts 241 | self.press_key(text) 242 | return ToolResult() 243 | elif action == "type": 244 | for chunk in chunks(text, TYPING_GROUP_SIZE): 245 | self.page.keyboard.type(chunk) 246 | return self.screenshot() 247 | 248 | if action in ( 249 | "left_click", 250 | "right_click", 251 | "double_click", 252 | "middle_click", 253 | "screenshot", 254 | "cursor_position", 255 | ): 256 | if text is not None: 257 | raise ToolError(f"text is not accepted for {action}") 258 | if coordinate is not None: 259 | raise ToolError(f"coordinate is not accepted for {action}") 260 | 261 | if action == "screenshot": 262 | return self.screenshot() 263 | elif action == "cursor_position": 264 | return ToolResult( 265 | output=f"X={self.mouse_position[0]},Y={self.mouse_position[1]}" 266 | ) 267 | else: 268 | click_arg = { 269 | "left_click": {"button": "left", "click_count": 1}, 270 | "right_click": {"button": "right", "click_count": 1}, 271 | "middle_click": {"button": "middle", "click_count": 1}, 272 | "double_click": {"button": "left", "click_count": 2, "delay": 100}, 273 | }[action] 274 | self.page.mouse.click( 275 | self.mouse_position[0], self.mouse_position[1], **click_arg 276 | ) 277 | return ToolResult() 278 | 279 | raise ToolError(f"Invalid action: {action}") 280 | 281 | def screenshot(self) -> ToolResult: 282 | """Take a screenshot of the current screen and return the base64 encoded image.""" 283 | if self.screenshot_wait_until is not None: 284 | self.page.wait_for_load_state(self.screenshot_wait_until) 285 | screenshot = self.page.screenshot() 286 | image = Image.open(io.BytesIO(screenshot)) 287 | img_small = image.resize((self.width, self.height), Image.LANCZOS) 288 | 289 | if self.use_cursor: 290 | cursor = load_cursor_image() 291 | img_small.paste(cursor, self.mouse_position, cursor) 292 | buffered = io.BytesIO() 293 | img_small.save(buffered, format="PNG") 294 | base64_image = base64.b64encode(buffered.getvalue()).decode() 295 | return ToolResult(base64_image=base64_image) 296 | 297 | def press_key(self, key: str): 298 | """Press a key on the keyboard. Handle + shifts. Eg: Ctrl+Shift+T.""" 299 | shifts = [] 300 | if "+" in key: 301 | shifts += key.split("+")[:-1] 302 | key = key.split("+")[-1] 303 | for shift in shifts: 304 | self.page.keyboard.down(shift) 305 | self.page.keyboard.press(to_playwright_key(key)) 306 | for shift in shifts: 307 | self.page.keyboard.up(shift) 308 | 309 | 310 | class PlaywrightComputerTool20241022(BasePlaywrightComputerTool): 311 | """Tool to interact with the computer using Playwright (Beta 22/10/2024).""" 312 | 313 | api_type: Literal["computer_20241022"] = "computer_20241022" 314 | 315 | def to_params(self) -> BetaToolComputerUse20241022Param: 316 | """Params describing the tool. Used by Claude to understand this is a computer use tool.""" 317 | return {"name": self.name, "type": self.api_type, **self.options} 318 | 319 | 320 | class PlaywrightComputerTool20250124(BasePlaywrightComputerTool): 321 | """Tool to interact with the computer using Playwright (Beta 24/01/2025).""" 322 | 323 | api_type: Literal["computer_20250124"] = "computer_20250124" 324 | 325 | def to_params(self) -> BetaToolComputerUse20250124Param: 326 | """Params describing the tool. Used by Claude to understand this is a computer use tool.""" 327 | return {"name": self.name, "type": self.api_type, **self.options} 328 | 329 | def __call__( 330 | self, 331 | *, 332 | action: Action_20250124, 333 | text: str | None = None, 334 | coordinate: tuple[int, int] | None = None, 335 | scroll_direction: ScrollDirection | None = None, 336 | scroll_amount: int | None = None, 337 | duration: int | float | None = None, 338 | key: str | None = None, 339 | **kwargs, 340 | ): 341 | """Run an action. text, coordinate, scroll_directions, scroll_amount, duration, key are potential additional parameters.""" 342 | if action in ("left_mouse_down", "left_mouse_up"): 343 | if coordinate is not None: 344 | raise ToolError(f"coordinate is not accepted for {action=}.") 345 | self.page.mouse.down() if action == "left_mouse_down" else self.page.mouse.up() 346 | return ToolResult() 347 | if action == "scroll": 348 | if scroll_direction is None or scroll_direction not in get_args( 349 | ScrollDirection 350 | ): 351 | raise ToolError( 352 | f"{scroll_direction=} must be 'up', 'down', 'left', or 'right'" 353 | ) 354 | if not isinstance(scroll_amount, int) or scroll_amount < 0: 355 | raise ToolError(f"{scroll_amount=} must be a non-negative int") 356 | if coordinate is not None: 357 | x, y = coordinate 358 | self.page.mouse.move(x, y) 359 | self.mouse_position = (x, y) 360 | scroll_amount *= SCROLL_MULTIPLIER_FACTOR 361 | scroll_params = { 362 | "up": {"delta_y": -scroll_amount, "delta_x": 0}, 363 | "down": {"delta_y": scroll_amount, "delta_x": 0}, 364 | "left": {"delta_y": 0, "delta_x": scroll_amount}, 365 | "right": {"delta_y": 0, "delta_x": -scroll_amount}, 366 | }[scroll_direction] 367 | 368 | self.page.mouse.wheel(**scroll_params) 369 | return ToolResult() 370 | 371 | if action in ("hold_key", "wait"): 372 | if duration is None or not isinstance(duration, (int, float)): 373 | raise ToolError(f"{duration=} must be a number") 374 | if duration < 0: 375 | raise ToolError(f"{duration=} must be non-negative") 376 | if duration > 100: 377 | raise ToolError(f"{duration=} is too long.") 378 | 379 | if action == "hold_key": 380 | if text is None: 381 | raise ToolError(f"text is required for {action}") 382 | self.page.keyboard.press(to_playwright_key(text), delay=duration) 383 | return ToolResult() 384 | 385 | if action == "wait": 386 | sleep(duration) 387 | return self.screenshot() 388 | 389 | if action in ( 390 | "left_click", 391 | "right_click", 392 | "double_click", 393 | "triple_click", 394 | "middle_click", 395 | ): 396 | if text is not None: 397 | raise ToolError(f"text is not accepted for {action}") 398 | if coordinate is not None: 399 | x, y = coordinate 400 | self.page.mouse.move(x, y) 401 | self.mouse_position = (x, y) 402 | 403 | click_arg = { 404 | "left_click": {"button": "left", "click_count": 1}, 405 | "right_click": {"button": "right", "click_count": 1}, 406 | "middle_click": {"button": "middle", "click_count": 1}, 407 | "double_click": {"button": "left", "click_count": 2, "delay": 10}, 408 | "double_click": {"button": "left", "click_count": 3, "delay": 10}, 409 | }[action] 410 | if key: 411 | self.page.keyboard.down(to_playwright_key(key)) 412 | self.page.mouse.click( 413 | self.mouse_position[0], self.mouse_position[1], **click_arg 414 | ) 415 | if key: 416 | self.page.keyboard.up(to_playwright_key(key)) 417 | 418 | return ToolResult() 419 | 420 | action = cast(Action_20241022, action) 421 | return super().__call__( 422 | action=action, text=text, coordinate=coordinate, key=key, **kwargs 423 | ) 424 | --------------------------------------------------------------------------------