├── .gitignore ├── LICENSE ├── README.md ├── SECURITY.md ├── demo.ipynb ├── docs └── Evaluation.md ├── eval ├── logs_sspro_omniv2.json └── ss_pro_gpt4o_omniv2.py ├── gradio_demo.py ├── imgs ├── demo_image.jpg ├── demo_image_som.jpg ├── excel.png ├── google_page.png ├── gradioicon.png ├── header_bar.png ├── header_bar_thin.png ├── ios.png ├── logo.png ├── mobile.png ├── omni3.jpg ├── omniboxicon.png ├── omniparsericon.png ├── onenote.png ├── saved_image_demo.png ├── som_overlaid_omni.png ├── teams.png ├── windows.png ├── windows_home.png ├── windows_multitab.png ├── windows_vm.png └── word.png ├── omnitool ├── gradio │ ├── .gitignore │ ├── __init__.py │ ├── agent │ │ ├── anthropic_agent.py │ │ ├── llm_utils │ │ │ ├── groqclient.py │ │ │ ├── oaiclient.py │ │ │ ├── omniparserclient.py │ │ │ └── utils.py │ │ ├── vlm_agent.py │ │ └── vlm_agent_with_orchestrator.py │ ├── app.py │ ├── app_new.py │ ├── app_streamlit.py │ ├── executor │ │ └── anthropic_executor.py │ ├── loop.py │ └── tools │ │ ├── __init__.py │ │ ├── base.py │ │ ├── collection.py │ │ ├── computer.py │ │ └── screen_capture.py ├── omnibox │ ├── .gitignore │ ├── Dockerfile │ ├── compose.yml │ ├── scripts │ │ ├── manage_vm.ps1 │ │ └── manage_vm.sh │ └── vm │ │ ├── buildcontainer │ │ ├── define.sh │ │ ├── entry.sh │ │ ├── install.sh │ │ ├── power.sh │ │ └── samba.sh │ │ ├── win11def │ │ └── win11x64-enterprise-eval.xml │ │ ├── win11iso │ │ └── README.md │ │ └── win11setup │ │ ├── firstboot │ │ └── install.bat │ │ └── setupscripts │ │ ├── on-logon.ps1 │ │ ├── server │ │ ├── cursor.png │ │ ├── main.py │ │ └── requirements.txt │ │ ├── setup-tools.psm1 │ │ ├── setup.ps1 │ │ └── tools_config.json ├── omniparserserver │ └── omniparserserver.py └── readme.md ├── requirements.txt └── util ├── __init__.py ├── box_annotator.py ├── omniparser.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | weights/icon_caption_blip2 2 | weights/icon_caption_florence 3 | weights/icon_detect/ 4 | weights/icon_detect_v1_5/ 5 | weights/icon_detect_v1_5_2/ 6 | .gradio 7 | __pycache__/ 8 | debug.ipynb 9 | util/__pycache__/ 10 | index.html?linkid=2289031 11 | wget-log 12 | weights/icon_caption_florence_v2/ 13 | omnitool/gradio/uploads/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Attribution 4.0 International 2 | 3 | ======================================================================= 4 | 5 | Creative Commons Corporation ("Creative Commons") is not a law firm and 6 | does not provide legal services or legal advice. Distribution of 7 | Creative Commons public licenses does not create a lawyer-client or 8 | other relationship. Creative Commons makes its licenses and related 9 | information available on an "as-is" basis. Creative Commons gives no 10 | warranties regarding its licenses, any material licensed under their 11 | terms and conditions, or any related information. Creative Commons 12 | disclaims all liability for damages resulting from their use to the 13 | fullest extent possible. 14 | 15 | Using Creative Commons Public Licenses 16 | 17 | Creative Commons public licenses provide a standard set of terms and 18 | conditions that creators and other rights holders may use to share 19 | original works of authorship and other material subject to copyright 20 | and certain other rights specified in the public license below. The 21 | following considerations are for informational purposes only, are not 22 | exhaustive, and do not form part of our licenses. 23 | 24 | Considerations for licensors: Our public licenses are 25 | intended for use by those authorized to give the public 26 | permission to use material in ways otherwise restricted by 27 | copyright and certain other rights. Our licenses are 28 | irrevocable. Licensors should read and understand the terms 29 | and conditions of the license they choose before applying it. 30 | Licensors should also secure all rights necessary before 31 | applying our licenses so that the public can reuse the 32 | material as expected. Licensors should clearly mark any 33 | material not subject to the license. This includes other CC- 34 | licensed material, or material used under an exception or 35 | limitation to copyright. More considerations for licensors: 36 | wiki.creativecommons.org/Considerations_for_licensors 37 | 38 | Considerations for the public: By using one of our public 39 | licenses, a licensor grants the public permission to use the 40 | licensed material under specified terms and conditions. If 41 | the licensor's permission is not necessary for any reason--for 42 | example, because of any applicable exception or limitation to 43 | copyright--then that use is not regulated by the license. Our 44 | licenses grant only permissions under copyright and certain 45 | other rights that a licensor has authority to grant. Use of 46 | the licensed material may still be restricted for other 47 | reasons, including because others have copyright or other 48 | rights in the material. A licensor may make special requests, 49 | such as asking that all changes be marked or described. 50 | Although not required by our licenses, you are encouraged to 51 | respect those requests where reasonable. More_considerations 52 | for the public: 53 | wiki.creativecommons.org/Considerations_for_licensees 54 | 55 | ======================================================================= 56 | 57 | Creative Commons Attribution 4.0 International Public License 58 | 59 | By exercising the Licensed Rights (defined below), You accept and agree 60 | to be bound by the terms and conditions of this Creative Commons 61 | Attribution 4.0 International Public License ("Public License"). To the 62 | extent this Public License may be interpreted as a contract, You are 63 | granted the Licensed Rights in consideration of Your acceptance of 64 | these terms and conditions, and the Licensor grants You such rights in 65 | consideration of benefits the Licensor receives from making the 66 | Licensed Material available under these terms and conditions. 67 | 68 | 69 | Section 1 -- Definitions. 70 | 71 | a. Adapted Material means material subject to Copyright and Similar 72 | Rights that is derived from or based upon the Licensed Material 73 | and in which the Licensed Material is translated, altered, 74 | arranged, transformed, or otherwise modified in a manner requiring 75 | permission under the Copyright and Similar Rights held by the 76 | Licensor. For purposes of this Public License, where the Licensed 77 | Material is a musical work, performance, or sound recording, 78 | Adapted Material is always produced where the Licensed Material is 79 | synched in timed relation with a moving image. 80 | 81 | b. Adapter's License means the license You apply to Your Copyright 82 | and Similar Rights in Your contributions to Adapted Material in 83 | accordance with the terms and conditions of this Public License. 84 | 85 | c. Copyright and Similar Rights means copyright and/or similar rights 86 | closely related to copyright including, without limitation, 87 | performance, broadcast, sound recording, and Sui Generis Database 88 | Rights, without regard to how the rights are labeled or 89 | categorized. For purposes of this Public License, the rights 90 | specified in Section 2(b)(1)-(2) are not Copyright and Similar 91 | Rights. 92 | 93 | d. Effective Technological Measures means those measures that, in the 94 | absence of proper authority, may not be circumvented under laws 95 | fulfilling obligations under Article 11 of the WIPO Copyright 96 | Treaty adopted on December 20, 1996, and/or similar international 97 | agreements. 98 | 99 | e. Exceptions and Limitations means fair use, fair dealing, and/or 100 | any other exception or limitation to Copyright and Similar Rights 101 | that applies to Your use of the Licensed Material. 102 | 103 | f. Licensed Material means the artistic or literary work, database, 104 | or other material to which the Licensor applied this Public 105 | License. 106 | 107 | g. Licensed Rights means the rights granted to You subject to the 108 | terms and conditions of this Public License, which are limited to 109 | all Copyright and Similar Rights that apply to Your use of the 110 | Licensed Material and that the Licensor has authority to license. 111 | 112 | h. Licensor means the individual(s) or entity(ies) granting rights 113 | under this Public License. 114 | 115 | i. Share means to provide material to the public by any means or 116 | process that requires permission under the Licensed Rights, such 117 | as reproduction, public display, public performance, distribution, 118 | dissemination, communication, or importation, and to make material 119 | available to the public including in ways that members of the 120 | public may access the material from a place and at a time 121 | individually chosen by them. 122 | 123 | j. Sui Generis Database Rights means rights other than copyright 124 | resulting from Directive 96/9/EC of the European Parliament and of 125 | the Council of 11 March 1996 on the legal protection of databases, 126 | as amended and/or succeeded, as well as other essentially 127 | equivalent rights anywhere in the world. 128 | 129 | k. You means the individual or entity exercising the Licensed Rights 130 | under this Public License. Your has a corresponding meaning. 131 | 132 | 133 | Section 2 -- Scope. 134 | 135 | a. License grant. 136 | 137 | 1. Subject to the terms and conditions of this Public License, 138 | the Licensor hereby grants You a worldwide, royalty-free, 139 | non-sublicensable, non-exclusive, irrevocable license to 140 | exercise the Licensed Rights in the Licensed Material to: 141 | 142 | a. reproduce and Share the Licensed Material, in whole or 143 | in part; and 144 | 145 | b. produce, reproduce, and Share Adapted Material. 146 | 147 | 2. Exceptions and Limitations. For the avoidance of doubt, where 148 | Exceptions and Limitations apply to Your use, this Public 149 | License does not apply, and You do not need to comply with 150 | its terms and conditions. 151 | 152 | 3. Term. The term of this Public License is specified in Section 153 | 6(a). 154 | 155 | 4. Media and formats; technical modifications allowed. The 156 | Licensor authorizes You to exercise the Licensed Rights in 157 | all media and formats whether now known or hereafter created, 158 | and to make technical modifications necessary to do so. The 159 | Licensor waives and/or agrees not to assert any right or 160 | authority to forbid You from making technical modifications 161 | necessary to exercise the Licensed Rights, including 162 | technical modifications necessary to circumvent Effective 163 | Technological Measures. For purposes of this Public License, 164 | simply making modifications authorized by this Section 2(a) 165 | (4) never produces Adapted Material. 166 | 167 | 5. Downstream recipients. 168 | 169 | a. Offer from the Licensor -- Licensed Material. Every 170 | recipient of the Licensed Material automatically 171 | receives an offer from the Licensor to exercise the 172 | Licensed Rights under the terms and conditions of this 173 | Public License. 174 | 175 | b. No downstream restrictions. You may not offer or impose 176 | any additional or different terms or conditions on, or 177 | apply any Effective Technological Measures to, the 178 | Licensed Material if doing so restricts exercise of the 179 | Licensed Rights by any recipient of the Licensed 180 | Material. 181 | 182 | 6. No endorsement. Nothing in this Public License constitutes or 183 | may be construed as permission to assert or imply that You 184 | are, or that Your use of the Licensed Material is, connected 185 | with, or sponsored, endorsed, or granted official status by, 186 | the Licensor or others designated to receive attribution as 187 | provided in Section 3(a)(1)(A)(i). 188 | 189 | b. Other rights. 190 | 191 | 1. Moral rights, such as the right of integrity, are not 192 | licensed under this Public License, nor are publicity, 193 | privacy, and/or other similar personality rights; however, to 194 | the extent possible, the Licensor waives and/or agrees not to 195 | assert any such rights held by the Licensor to the limited 196 | extent necessary to allow You to exercise the Licensed 197 | Rights, but not otherwise. 198 | 199 | 2. Patent and trademark rights are not licensed under this 200 | Public License. 201 | 202 | 3. To the extent possible, the Licensor waives any right to 203 | collect royalties from You for the exercise of the Licensed 204 | Rights, whether directly or through a collecting society 205 | under any voluntary or waivable statutory or compulsory 206 | licensing scheme. In all other cases the Licensor expressly 207 | reserves any right to collect such royalties. 208 | 209 | 210 | Section 3 -- License Conditions. 211 | 212 | Your exercise of the Licensed Rights is expressly made subject to the 213 | following conditions. 214 | 215 | a. Attribution. 216 | 217 | 1. If You Share the Licensed Material (including in modified 218 | form), You must: 219 | 220 | a. retain the following if it is supplied by the Licensor 221 | with the Licensed Material: 222 | 223 | i. identification of the creator(s) of the Licensed 224 | Material and any others designated to receive 225 | attribution, in any reasonable manner requested by 226 | the Licensor (including by pseudonym if 227 | designated); 228 | 229 | ii. a copyright notice; 230 | 231 | iii. a notice that refers to this Public License; 232 | 233 | iv. a notice that refers to the disclaimer of 234 | warranties; 235 | 236 | v. a URI or hyperlink to the Licensed Material to the 237 | extent reasonably practicable; 238 | 239 | b. indicate if You modified the Licensed Material and 240 | retain an indication of any previous modifications; and 241 | 242 | c. indicate the Licensed Material is licensed under this 243 | Public License, and include the text of, or the URI or 244 | hyperlink to, this Public License. 245 | 246 | 2. You may satisfy the conditions in Section 3(a)(1) in any 247 | reasonable manner based on the medium, means, and context in 248 | which You Share the Licensed Material. For example, it may be 249 | reasonable to satisfy the conditions by providing a URI or 250 | hyperlink to a resource that includes the required 251 | information. 252 | 253 | 3. If requested by the Licensor, You must remove any of the 254 | information required by Section 3(a)(1)(A) to the extent 255 | reasonably practicable. 256 | 257 | 4. If You Share Adapted Material You produce, the Adapter's 258 | License You apply must not prevent recipients of the Adapted 259 | Material from complying with this Public License. 260 | 261 | 262 | Section 4 -- Sui Generis Database Rights. 263 | 264 | Where the Licensed Rights include Sui Generis Database Rights that 265 | apply to Your use of the Licensed Material: 266 | 267 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right 268 | to extract, reuse, reproduce, and Share all or a substantial 269 | portion of the contents of the database; 270 | 271 | b. if You include all or a substantial portion of the database 272 | contents in a database in which You have Sui Generis Database 273 | Rights, then the database in which You have Sui Generis Database 274 | Rights (but not its individual contents) is Adapted Material; and 275 | 276 | c. You must comply with the conditions in Section 3(a) if You Share 277 | all or a substantial portion of the contents of the database. 278 | 279 | For the avoidance of doubt, this Section 4 supplements and does not 280 | replace Your obligations under this Public License where the Licensed 281 | Rights include other Copyright and Similar Rights. 282 | 283 | 284 | Section 5 -- Disclaimer of Warranties and Limitation of Liability. 285 | 286 | a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE 287 | EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS 288 | AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF 289 | ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, 290 | IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, 291 | WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR 292 | PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, 293 | ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT 294 | KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT 295 | ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. 296 | 297 | b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE 298 | TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, 299 | NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, 300 | INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, 301 | COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR 302 | USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN 303 | ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR 304 | DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR 305 | IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. 306 | 307 | c. The disclaimer of warranties and limitation of liability provided 308 | above shall be interpreted in a manner that, to the extent 309 | possible, most closely approximates an absolute disclaimer and 310 | waiver of all liability. 311 | 312 | 313 | Section 6 -- Term and Termination. 314 | 315 | a. This Public License applies for the term of the Copyright and 316 | Similar Rights licensed here. However, if You fail to comply with 317 | this Public License, then Your rights under this Public License 318 | terminate automatically. 319 | 320 | b. Where Your right to use the Licensed Material has terminated under 321 | Section 6(a), it reinstates: 322 | 323 | 1. automatically as of the date the violation is cured, provided 324 | it is cured within 30 days of Your discovery of the 325 | violation; or 326 | 327 | 2. upon express reinstatement by the Licensor. 328 | 329 | For the avoidance of doubt, this Section 6(b) does not affect any 330 | right the Licensor may have to seek remedies for Your violations 331 | of this Public License. 332 | 333 | c. For the avoidance of doubt, the Licensor may also offer the 334 | Licensed Material under separate terms or conditions or stop 335 | distributing the Licensed Material at any time; however, doing so 336 | will not terminate this Public License. 337 | 338 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public 339 | License. 340 | 341 | 342 | Section 7 -- Other Terms and Conditions. 343 | 344 | a. The Licensor shall not be bound by any additional or different 345 | terms or conditions communicated by You unless expressly agreed. 346 | 347 | b. Any arrangements, understandings, or agreements regarding the 348 | Licensed Material not stated herein are separate from and 349 | independent of the terms and conditions of this Public License. 350 | 351 | 352 | Section 8 -- Interpretation. 353 | 354 | a. For the avoidance of doubt, this Public License does not, and 355 | shall not be interpreted to, reduce, limit, restrict, or impose 356 | conditions on any use of the Licensed Material that could lawfully 357 | be made without permission under this Public License. 358 | 359 | b. To the extent possible, if any provision of this Public License is 360 | deemed unenforceable, it shall be automatically reformed to the 361 | minimum extent necessary to make it enforceable. If the provision 362 | cannot be reformed, it shall be severed from this Public License 363 | without affecting the enforceability of the remaining terms and 364 | conditions. 365 | 366 | c. No term or condition of this Public License will be waived and no 367 | failure to comply consented to unless expressly agreed to by the 368 | Licensor. 369 | 370 | d. Nothing in this Public License constitutes or may be interpreted 371 | as a limitation upon, or waiver of, any privileges and immunities 372 | that apply to the Licensor or You, including from the legal 373 | processes of any jurisdiction or authority. 374 | 375 | 376 | ======================================================================= 377 | 378 | Creative Commons is not a party to its public 379 | licenses. Notwithstanding, Creative Commons may elect to apply one of 380 | its public licenses to material it publishes and in those instances 381 | will be considered the “Licensor.” The text of the Creative Commons 382 | public licenses is dedicated to the public domain under the CC0 Public 383 | Domain Dedication. Except for the limited purpose of indicating that 384 | material is shared under a Creative Commons public license or as 385 | otherwise permitted by the Creative Commons policies published at 386 | creativecommons.org/policies, Creative Commons does not authorize the 387 | use of the trademark "Creative Commons" or any other trademark or logo 388 | of Creative Commons without its prior written consent including, 389 | without limitation, in connection with any unauthorized modifications 390 | to any of its public licenses or any other arrangements, 391 | understandings, or agreements concerning use of licensed material. For 392 | the avoidance of doubt, this paragraph does not form part of the 393 | public licenses. 394 | 395 | Creative Commons may be contacted at creativecommons.org. 396 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OmniParser: Screen Parsing tool for Pure Vision Based GUI Agent 2 | 3 |

4 | Logo 5 |

6 | 7 | 8 | [![arXiv](https://img.shields.io/badge/Paper-green)](https://arxiv.org/abs/2408.00203) 9 | [![License](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) 10 | 11 | 📢 [[Project Page](https://microsoft.github.io/OmniParser/)] [[V2 Blog Post](https://www.microsoft.com/en-us/research/articles/omniparser-v2-turning-any-llm-into-a-computer-use-agent/)] [[Models V2](https://huggingface.co/microsoft/OmniParser-v2.0)] [[Models V1.5](https://huggingface.co/microsoft/OmniParser)] [[HuggingFace Space Demo](https://huggingface.co/spaces/microsoft/OmniParser-v2)] 12 | 13 | **OmniParser** is a comprehensive method for parsing user interface screenshots into structured and easy-to-understand elements, which significantly enhances the ability of GPT-4V to generate actions that can be accurately grounded in the corresponding regions of the interface. 14 | 15 | ## News 16 | - [2025/3] We support local logging of trajecotry so that you can use OmniParser+OmniTool to build training data pipeline for your favorate agent in your domain. [Documentation WIP] 17 | - [2025/3] We are gradually adding multi agents orchstration and improving user interface in OmniTool for better experience. 18 | - [2025/2] We release OmniParser V2 [checkpoints](https://huggingface.co/microsoft/OmniParser-v2.0). [Watch Video](https://1drv.ms/v/c/650b027c18d5a573/EWXbVESKWo9Buu6OYCwg06wBeoM97C6EOTG6RjvWLEN1Qg?e=alnHGC) 19 | - [2025/2] We introduce OmniTool: Control a Windows 11 VM with OmniParser + your vision model of choice. OmniTool supports out of the box the following large language models - OpenAI (4o/o1/o3-mini), DeepSeek (R1), Qwen (2.5VL) or Anthropic Computer Use. [Watch Video](https://1drv.ms/v/c/650b027c18d5a573/EehZ7RzY69ZHn-MeQHrnnR4BCj3by-cLLpUVlxMjF4O65Q?e=8LxMgX) 20 | - [2025/1] V2 is coming. We achieve new state of the art results 39.5% on the new grounding benchmark [Screen Spot Pro](https://github.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding/tree/main) with OmniParser v2 (will be released soon)! Read more details [here](https://github.com/microsoft/OmniParser/tree/master/docs/Evaluation.md). 21 | - [2024/11] We release an updated version, OmniParser V1.5 which features 1) more fine grained/small icon detection, 2) prediction of whether each screen element is interactable or not. Examples in the demo.ipynb. 22 | - [2024/10] OmniParser was the #1 trending model on huggingface model hub (starting 10/29/2024). 23 | - [2024/10] Feel free to checkout our demo on [huggingface space](https://huggingface.co/spaces/microsoft/OmniParser)! (stay tuned for OmniParser + Claude Computer Use) 24 | - [2024/10] Both Interactive Region Detection Model and Icon functional description model are released! [Hugginface models](https://huggingface.co/microsoft/OmniParser) 25 | - [2024/09] OmniParser achieves the best performance on [Windows Agent Arena](https://microsoft.github.io/WindowsAgentArena/)! 26 | 27 | ## Install 28 | First clone the repo, and then install environment: 29 | ```python 30 | cd OmniParser 31 | conda create -n "omni" python==3.12 32 | conda activate omni 33 | pip install -r requirements.txt 34 | ``` 35 | 36 | Ensure you have the V2 weights downloaded in weights folder (ensure caption weights folder is called icon_caption_florence). If not download them with: 37 | ``` 38 | # download the model checkpoints to local directory OmniParser/weights/ 39 | for f in icon_detect/{train_args.yaml,model.pt,model.yaml} icon_caption/{config.json,generation_config.json,model.safetensors}; do huggingface-cli download microsoft/OmniParser-v2.0 "$f" --local-dir weights; done 40 | mv weights/icon_caption weights/icon_caption_florence 41 | ``` 42 | 43 | 54 | 55 | ## Examples: 56 | We put together a few simple examples in the demo.ipynb. 57 | 58 | ## Gradio Demo 59 | To run gradio demo, simply run: 60 | ```python 61 | python gradio_demo.py 62 | ``` 63 | 64 | ## Model Weights License 65 | For the model checkpoints on huggingface model hub, please note that icon_detect model is under AGPL license since it is a license inherited from the original yolo model. And icon_caption_blip2 & icon_caption_florence is under MIT license. Please refer to the LICENSE file in the folder of each model: https://huggingface.co/microsoft/OmniParser. 66 | 67 | ## 📚 Citation 68 | Our technical report can be found [here](https://arxiv.org/abs/2408.00203). 69 | If you find our work useful, please consider citing our work: 70 | ``` 71 | @misc{lu2024omniparserpurevisionbased, 72 | title={OmniParser for Pure Vision Based GUI Agent}, 73 | author={Yadong Lu and Jianwei Yang and Yelong Shen and Ahmed Awadallah}, 74 | year={2024}, 75 | eprint={2408.00203}, 76 | archivePrefix={arXiv}, 77 | primaryClass={cs.CV}, 78 | url={https://arxiv.org/abs/2408.00203}, 79 | } 80 | ``` 81 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /docs/Evaluation.md: -------------------------------------------------------------------------------- 1 | # Eval setup for ScreenSpot Pro 2 | We adapt the eval code from ScreenSpot Pro (ss pro) official [repo](https://github.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding/tree/main). This folder contains the inference script/results on this benchmark. We going through legal review proces to release omniparser v2. Once it is done, we will update the file so that it can load the v2 model. 3 | 1. eval/ss_pro_gpt4o_omniv2.py: contains the prompt we use, it can be dropped in replacement for this [file](https://github.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding/blob/main/models/gpt4x.py) in the original ss pro repo. 4 | 2. eval/logs_sspro_omniv2.json: contains the inferenced results for ss pro using GPT4o+OmniParserv2. 5 | -------------------------------------------------------------------------------- /gradio_demo.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import gradio as gr 4 | import numpy as np 5 | import torch 6 | from PIL import Image 7 | import io 8 | 9 | 10 | import base64, os 11 | from util.utils import check_ocr_box, get_yolo_model, get_caption_model_processor, get_som_labeled_img 12 | import torch 13 | from PIL import Image 14 | 15 | yolo_model = get_yolo_model(model_path='weights/icon_detect/model.pt') 16 | caption_model_processor = get_caption_model_processor(model_name="florence2", model_name_or_path="weights/icon_caption_florence") 17 | # caption_model_processor = get_caption_model_processor(model_name="blip2", model_name_or_path="weights/icon_caption_blip2") 18 | 19 | MARKDOWN = """ 20 | # OmniParser for Pure Vision Based General GUI Agent 🔥 21 |
22 | 23 | Arxiv 24 | 25 |
26 | 27 | OmniParser is a screen parsing tool to convert general GUI screen to structured elements. 28 | """ 29 | 30 | DEVICE = torch.device('cuda') 31 | 32 | # @spaces.GPU 33 | # @torch.inference_mode() 34 | # @torch.autocast(device_type="cuda", dtype=torch.bfloat16) 35 | def process( 36 | image_input, 37 | box_threshold, 38 | iou_threshold, 39 | use_paddleocr, 40 | imgsz 41 | ) -> Optional[Image.Image]: 42 | 43 | box_overlay_ratio = image_input.size[0] / 3200 44 | draw_bbox_config = { 45 | 'text_scale': 0.8 * box_overlay_ratio, 46 | 'text_thickness': max(int(2 * box_overlay_ratio), 1), 47 | 'text_padding': max(int(3 * box_overlay_ratio), 1), 48 | 'thickness': max(int(3 * box_overlay_ratio), 1), 49 | } 50 | # import pdb; pdb.set_trace() 51 | 52 | ocr_bbox_rslt, is_goal_filtered = check_ocr_box(image_input, display_img = False, output_bb_format='xyxy', goal_filtering=None, easyocr_args={'paragraph': False, 'text_threshold':0.9}, use_paddleocr=use_paddleocr) 53 | text, ocr_bbox = ocr_bbox_rslt 54 | dino_labled_img, label_coordinates, parsed_content_list = get_som_labeled_img(image_input, yolo_model, BOX_TRESHOLD = box_threshold, output_coord_in_ratio=True, ocr_bbox=ocr_bbox,draw_bbox_config=draw_bbox_config, caption_model_processor=caption_model_processor, ocr_text=text,iou_threshold=iou_threshold, imgsz=imgsz,) 55 | image = Image.open(io.BytesIO(base64.b64decode(dino_labled_img))) 56 | print('finish processing') 57 | parsed_content_list = '\n'.join([f'icon {i}: ' + str(v) for i,v in enumerate(parsed_content_list)]) 58 | # parsed_content_list = str(parsed_content_list) 59 | return image, str(parsed_content_list) 60 | 61 | with gr.Blocks() as demo: 62 | gr.Markdown(MARKDOWN) 63 | with gr.Row(): 64 | with gr.Column(): 65 | image_input_component = gr.Image( 66 | type='pil', label='Upload image') 67 | # set the threshold for removing the bounding boxes with low confidence, default is 0.05 68 | box_threshold_component = gr.Slider( 69 | label='Box Threshold', minimum=0.01, maximum=1.0, step=0.01, value=0.05) 70 | # set the threshold for removing the bounding boxes with large overlap, default is 0.1 71 | iou_threshold_component = gr.Slider( 72 | label='IOU Threshold', minimum=0.01, maximum=1.0, step=0.01, value=0.1) 73 | use_paddleocr_component = gr.Checkbox( 74 | label='Use PaddleOCR', value=True) 75 | imgsz_component = gr.Slider( 76 | label='Icon Detect Image Size', minimum=640, maximum=1920, step=32, value=640) 77 | submit_button_component = gr.Button( 78 | value='Submit', variant='primary') 79 | with gr.Column(): 80 | image_output_component = gr.Image(type='pil', label='Image Output') 81 | text_output_component = gr.Textbox(label='Parsed screen elements', placeholder='Text Output') 82 | 83 | submit_button_component.click( 84 | fn=process, 85 | inputs=[ 86 | image_input_component, 87 | box_threshold_component, 88 | iou_threshold_component, 89 | use_paddleocr_component, 90 | imgsz_component 91 | ], 92 | outputs=[image_output_component, text_output_component] 93 | ) 94 | 95 | # demo.launch(debug=False, show_error=True, share=True) 96 | demo.launch(share=True, server_port=7861, server_name='0.0.0.0') 97 | -------------------------------------------------------------------------------- /imgs/demo_image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/demo_image.jpg -------------------------------------------------------------------------------- /imgs/demo_image_som.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/demo_image_som.jpg -------------------------------------------------------------------------------- /imgs/excel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/excel.png -------------------------------------------------------------------------------- /imgs/google_page.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/google_page.png -------------------------------------------------------------------------------- /imgs/gradioicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/gradioicon.png -------------------------------------------------------------------------------- /imgs/header_bar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/header_bar.png -------------------------------------------------------------------------------- /imgs/header_bar_thin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/header_bar_thin.png -------------------------------------------------------------------------------- /imgs/ios.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/ios.png -------------------------------------------------------------------------------- /imgs/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/logo.png -------------------------------------------------------------------------------- /imgs/mobile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/mobile.png -------------------------------------------------------------------------------- /imgs/omni3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/omni3.jpg -------------------------------------------------------------------------------- /imgs/omniboxicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/omniboxicon.png -------------------------------------------------------------------------------- /imgs/omniparsericon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/omniparsericon.png -------------------------------------------------------------------------------- /imgs/onenote.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/onenote.png -------------------------------------------------------------------------------- /imgs/saved_image_demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/saved_image_demo.png -------------------------------------------------------------------------------- /imgs/som_overlaid_omni.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/som_overlaid_omni.png -------------------------------------------------------------------------------- /imgs/teams.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/teams.png -------------------------------------------------------------------------------- /imgs/windows.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/windows.png -------------------------------------------------------------------------------- /imgs/windows_home.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/windows_home.png -------------------------------------------------------------------------------- /imgs/windows_multitab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/windows_multitab.png -------------------------------------------------------------------------------- /imgs/windows_vm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/windows_vm.png -------------------------------------------------------------------------------- /imgs/word.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/imgs/word.png -------------------------------------------------------------------------------- /omnitool/gradio/.gitignore: -------------------------------------------------------------------------------- 1 | tmp/ -------------------------------------------------------------------------------- /omnitool/gradio/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/omnitool/gradio/__init__.py -------------------------------------------------------------------------------- /omnitool/gradio/agent/anthropic_agent.py: -------------------------------------------------------------------------------- 1 | """ 2 | Agentic sampling loop that calls the Anthropic API and local implenmentation of anthropic-defined computer use tools. 3 | """ 4 | import asyncio 5 | import platform 6 | from collections.abc import Callable 7 | from datetime import datetime 8 | from enum import StrEnum 9 | from typing import Any, cast 10 | 11 | from anthropic import Anthropic, AnthropicBedrock, AnthropicVertex, APIResponse 12 | from anthropic.types import ( 13 | ToolResultBlockParam, 14 | ) 15 | from anthropic.types.beta import ( 16 | BetaContentBlock, 17 | BetaContentBlockParam, 18 | BetaImageBlockParam, 19 | BetaMessage, 20 | BetaMessageParam, 21 | BetaTextBlockParam, 22 | BetaToolResultBlockParam, 23 | ) 24 | from anthropic.types import TextBlock 25 | from anthropic.types.beta import BetaMessage, BetaTextBlock, BetaToolUseBlock 26 | 27 | from tools import ComputerTool, ToolCollection, ToolResult 28 | 29 | from PIL import Image 30 | from io import BytesIO 31 | import gradio as gr 32 | from typing import Dict 33 | 34 | BETA_FLAG = "computer-use-2024-10-22" 35 | 36 | class APIProvider(StrEnum): 37 | ANTHROPIC = "anthropic" 38 | BEDROCK = "bedrock" 39 | VERTEX = "vertex" 40 | 41 | SYSTEM_PROMPT = f""" 42 | * You are utilizing a Windows system with internet access. 43 | * The current date is {datetime.today().strftime('%A, %B %d, %Y')}. 44 | 45 | """ 46 | 47 | class AnthropicActor: 48 | def __init__( 49 | self, 50 | model: str, 51 | provider: APIProvider, 52 | api_key: str, 53 | api_response_callback: Callable[[APIResponse[BetaMessage]], None], 54 | max_tokens: int = 4096, 55 | only_n_most_recent_images: int | None = None, 56 | print_usage: bool = True, 57 | ): 58 | self.model = model 59 | self.provider = provider 60 | self.api_key = api_key 61 | self.api_response_callback = api_response_callback 62 | self.max_tokens = max_tokens 63 | self.only_n_most_recent_images = only_n_most_recent_images 64 | 65 | self.tool_collection = ToolCollection(ComputerTool()) 66 | 67 | self.system = SYSTEM_PROMPT 68 | 69 | self.total_token_usage = 0 70 | self.total_cost = 0 71 | self.print_usage = print_usage 72 | 73 | # Instantiate the appropriate API client based on the provider 74 | if provider == APIProvider.ANTHROPIC: 75 | self.client = Anthropic(api_key=api_key) 76 | elif provider == APIProvider.VERTEX: 77 | self.client = AnthropicVertex() 78 | elif provider == APIProvider.BEDROCK: 79 | self.client = AnthropicBedrock() 80 | 81 | def __call__( 82 | self, 83 | *, 84 | messages: list[BetaMessageParam] 85 | ): 86 | """ 87 | Generate a response given history messages. 88 | """ 89 | if self.only_n_most_recent_images: 90 | _maybe_filter_to_n_most_recent_images(messages, self.only_n_most_recent_images) 91 | 92 | # Call the API synchronously 93 | raw_response = self.client.beta.messages.with_raw_response.create( 94 | max_tokens=self.max_tokens, 95 | messages=messages, 96 | model=self.model, 97 | system=self.system, 98 | tools=self.tool_collection.to_params(), 99 | betas=["computer-use-2024-10-22"], 100 | ) 101 | 102 | self.api_response_callback(cast(APIResponse[BetaMessage], raw_response)) 103 | 104 | response = raw_response.parse() 105 | print(f"AnthropicActor response: {response}") 106 | 107 | self.total_token_usage += response.usage.input_tokens + response.usage.output_tokens 108 | self.total_cost += (response.usage.input_tokens * 3 / 1000000 + response.usage.output_tokens * 15 / 1000000) 109 | 110 | if self.print_usage: 111 | print(f"Claude total token usage so far: {self.total_token_usage}, total cost so far: $USD{self.total_cost}") 112 | 113 | return response 114 | 115 | 116 | def _maybe_filter_to_n_most_recent_images( 117 | messages: list[BetaMessageParam], 118 | images_to_keep: int, 119 | min_removal_threshold: int = 10, 120 | ): 121 | """ 122 | With the assumption that images are screenshots that are of diminishing value as 123 | the conversation progresses, remove all but the final `images_to_keep` tool_result 124 | images in place, with a chunk of min_removal_threshold to reduce the amount we 125 | break the implicit prompt cache. 126 | """ 127 | if images_to_keep is None: 128 | return messages 129 | 130 | tool_result_blocks = cast( 131 | list[ToolResultBlockParam], 132 | [ 133 | item 134 | for message in messages 135 | for item in ( 136 | message["content"] if isinstance(message["content"], list) else [] 137 | ) 138 | if isinstance(item, dict) and item.get("type") == "tool_result" 139 | ], 140 | ) 141 | 142 | total_images = sum( 143 | 1 144 | for tool_result in tool_result_blocks 145 | for content in tool_result.get("content", []) 146 | if isinstance(content, dict) and content.get("type") == "image" 147 | ) 148 | 149 | images_to_remove = total_images - images_to_keep 150 | # for better cache behavior, we want to remove in chunks 151 | images_to_remove -= images_to_remove % min_removal_threshold 152 | 153 | for tool_result in tool_result_blocks: 154 | if isinstance(tool_result.get("content"), list): 155 | new_content = [] 156 | for content in tool_result.get("content", []): 157 | if isinstance(content, dict) and content.get("type") == "image": 158 | if images_to_remove > 0: 159 | images_to_remove -= 1 160 | continue 161 | new_content.append(content) 162 | tool_result["content"] = new_content -------------------------------------------------------------------------------- /omnitool/gradio/agent/llm_utils/groqclient.py: -------------------------------------------------------------------------------- 1 | from groq import Groq 2 | import os 3 | from .utils import is_image_path 4 | 5 | def run_groq_interleaved(messages: list, system: str, model_name: str, api_key: str, max_tokens=256, temperature=0.6): 6 | """ 7 | Run a chat completion through Groq's API, ignoring any images in the messages. 8 | """ 9 | api_key = api_key or os.environ.get("GROQ_API_KEY") 10 | if not api_key: 11 | raise ValueError("GROQ_API_KEY is not set") 12 | 13 | client = Groq(api_key=api_key) 14 | # avoid using system messages for R1 15 | final_messages = [{"role": "user", "content": system}] 16 | 17 | if isinstance(messages, list): 18 | for item in messages: 19 | if isinstance(item, dict): 20 | # For dict items, concatenate all text content, ignoring images 21 | text_contents = [] 22 | for cnt in item["content"]: 23 | if isinstance(cnt, str): 24 | if not is_image_path(cnt): # Skip image paths 25 | text_contents.append(cnt) 26 | else: 27 | text_contents.append(str(cnt)) 28 | 29 | if text_contents: # Only add if there's text content 30 | message = {"role": "user", "content": " ".join(text_contents)} 31 | final_messages.append(message) 32 | else: # str 33 | message = {"role": "user", "content": item} 34 | final_messages.append(message) 35 | 36 | elif isinstance(messages, str): 37 | final_messages.append({"role": "user", "content": messages}) 38 | 39 | try: 40 | completion = client.chat.completions.create( 41 | model="deepseek-r1-distill-llama-70b", 42 | messages=final_messages, 43 | temperature=0.6, 44 | max_completion_tokens=max_tokens, 45 | top_p=0.95, 46 | stream=False, 47 | reasoning_format="raw" 48 | ) 49 | 50 | response = completion.choices[0].message.content 51 | final_answer = response.split('\n')[-1] if '' in response else response 52 | final_answer = final_answer.replace("", "").replace("", "") 53 | token_usage = completion.usage.total_tokens 54 | 55 | return final_answer, token_usage 56 | except Exception as e: 57 | print(f"Error in interleaved Groq: {e}") 58 | 59 | return str(e), 0 -------------------------------------------------------------------------------- /omnitool/gradio/agent/llm_utils/oaiclient.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | import base64 4 | import requests 5 | from .utils import is_image_path, encode_image 6 | 7 | def run_oai_interleaved(messages: list, system: str, model_name: str, api_key: str, max_tokens=256, temperature=0, provider_base_url: str = "https://api.openai.com/v1"): 8 | headers = {"Content-Type": "application/json", 9 | "Authorization": f"Bearer {api_key}"} 10 | final_messages = [{"role": "system", "content": system}] 11 | 12 | if type(messages) == list: 13 | for item in messages: 14 | contents = [] 15 | if isinstance(item, dict): 16 | for cnt in item["content"]: 17 | if isinstance(cnt, str): 18 | if is_image_path(cnt) and 'o3-mini' not in model_name: 19 | # 03 mini does not support images 20 | base64_image = encode_image(cnt) 21 | content = {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}} 22 | else: 23 | content = {"type": "text", "text": cnt} 24 | else: 25 | # in this case it is a text block from anthropic 26 | content = {"type": "text", "text": str(cnt)} 27 | 28 | contents.append(content) 29 | 30 | message = {"role": 'user', "content": contents} 31 | else: # str 32 | contents.append({"type": "text", "text": item}) 33 | message = {"role": "user", "content": contents} 34 | 35 | final_messages.append(message) 36 | 37 | 38 | elif isinstance(messages, str): 39 | final_messages = [{"role": "user", "content": messages}] 40 | 41 | payload = { 42 | "model": model_name, 43 | "messages": final_messages, 44 | } 45 | if 'o1' in model_name or 'o3-mini' in model_name: 46 | payload['reasoning_effort'] = 'low' 47 | payload['max_completion_tokens'] = max_tokens 48 | else: 49 | payload['max_tokens'] = max_tokens 50 | 51 | response = requests.post( 52 | f"{provider_base_url}/chat/completions", headers=headers, json=payload 53 | ) 54 | 55 | 56 | try: 57 | text = response.json()['choices'][0]['message']['content'] 58 | token_usage = int(response.json()['usage']['total_tokens']) 59 | return text, token_usage 60 | except Exception as e: 61 | print(f"Error in interleaved openAI: {e}. This may due to your invalid API key. Please check the response: {response.json()} ") 62 | return response.json() -------------------------------------------------------------------------------- /omnitool/gradio/agent/llm_utils/omniparserclient.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import base64 3 | from pathlib import Path 4 | from tools.screen_capture import get_screenshot 5 | from agent.llm_utils.utils import encode_image 6 | 7 | OUTPUT_DIR = "./tmp/outputs" 8 | 9 | class OmniParserClient: 10 | def __init__(self, 11 | url: str) -> None: 12 | self.url = url 13 | 14 | def __call__(self,): 15 | screenshot, screenshot_path = get_screenshot() 16 | screenshot_path = str(screenshot_path) 17 | image_base64 = encode_image(screenshot_path) 18 | response = requests.post(self.url, json={"base64_image": image_base64}) 19 | response_json = response.json() 20 | print('omniparser latency:', response_json['latency']) 21 | 22 | som_image_data = base64.b64decode(response_json['som_image_base64']) 23 | screenshot_path_uuid = Path(screenshot_path).stem.replace("screenshot_", "") 24 | som_screenshot_path = f"{OUTPUT_DIR}/screenshot_som_{screenshot_path_uuid}.png" 25 | with open(som_screenshot_path, "wb") as f: 26 | f.write(som_image_data) 27 | 28 | response_json['width'] = screenshot.size[0] 29 | response_json['height'] = screenshot.size[1] 30 | response_json['original_screenshot_base64'] = image_base64 31 | response_json['screenshot_uuid'] = screenshot_path_uuid 32 | response_json = self.reformat_messages(response_json) 33 | return response_json 34 | 35 | def reformat_messages(self, response_json: dict): 36 | screen_info = "" 37 | for idx, element in enumerate(response_json["parsed_content_list"]): 38 | element['idx'] = idx 39 | if element['type'] == 'text': 40 | screen_info += f'ID: {idx}, Text: {element["content"]}\n' 41 | elif element['type'] == 'icon': 42 | screen_info += f'ID: {idx}, Icon: {element["content"]}\n' 43 | response_json['screen_info'] = screen_info 44 | return response_json -------------------------------------------------------------------------------- /omnitool/gradio/agent/llm_utils/utils.py: -------------------------------------------------------------------------------- 1 | import base64 2 | 3 | def is_image_path(text): 4 | image_extensions = (".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif") 5 | if text.endswith(image_extensions): 6 | return True 7 | else: 8 | return False 9 | 10 | def encode_image(image_path): 11 | """Encode image file to base64.""" 12 | with open(image_path, "rb") as image_file: 13 | return base64.b64encode(image_file.read()).decode("utf-8") -------------------------------------------------------------------------------- /omnitool/gradio/agent/vlm_agent.py: -------------------------------------------------------------------------------- 1 | import json 2 | from collections.abc import Callable 3 | from typing import cast, Callable 4 | import uuid 5 | from PIL import Image, ImageDraw 6 | import base64 7 | from io import BytesIO 8 | 9 | from anthropic import APIResponse 10 | from anthropic.types import ToolResultBlockParam 11 | from anthropic.types.beta import BetaMessage, BetaTextBlock, BetaToolUseBlock, BetaMessageParam, BetaUsage 12 | 13 | from agent.llm_utils.oaiclient import run_oai_interleaved 14 | from agent.llm_utils.groqclient import run_groq_interleaved 15 | from agent.llm_utils.utils import is_image_path 16 | import time 17 | import re 18 | 19 | OUTPUT_DIR = "./tmp/outputs" 20 | 21 | def extract_data(input_string, data_type): 22 | # Regular expression to extract content starting from '```python' until the end if there are no closing backticks 23 | pattern = f"```{data_type}" + r"(.*?)(```|$)" 24 | # Extract content 25 | # re.DOTALL allows '.' to match newlines as well 26 | matches = re.findall(pattern, input_string, re.DOTALL) 27 | # Return the first match if exists, trimming whitespace and ignoring potential closing backticks 28 | return matches[0][0].strip() if matches else input_string 29 | 30 | class VLMAgent: 31 | def __init__( 32 | self, 33 | model: str, 34 | provider: str, 35 | api_key: str, 36 | output_callback: Callable, 37 | api_response_callback: Callable, 38 | max_tokens: int = 4096, 39 | only_n_most_recent_images: int | None = None, 40 | print_usage: bool = True, 41 | ): 42 | if model == "omniparser + gpt-4o": 43 | self.model = "gpt-4o-2024-11-20" 44 | elif model == "omniparser + R1": 45 | self.model = "deepseek-r1-distill-llama-70b" 46 | elif model == "omniparser + qwen2.5vl": 47 | self.model = "qwen2.5-vl-72b-instruct" 48 | elif model == "omniparser + o1": 49 | self.model = "o1" 50 | elif model == "omniparser + o3-mini": 51 | self.model = "o3-mini" 52 | else: 53 | raise ValueError(f"Model {model} not supported") 54 | 55 | 56 | self.provider = provider 57 | self.api_key = api_key 58 | self.api_response_callback = api_response_callback 59 | self.max_tokens = max_tokens 60 | self.only_n_most_recent_images = only_n_most_recent_images 61 | self.output_callback = output_callback 62 | 63 | self.print_usage = print_usage 64 | self.total_token_usage = 0 65 | self.total_cost = 0 66 | self.step_count = 0 67 | 68 | self.system = '' 69 | 70 | def __call__(self, messages: list, parsed_screen: list[str, list, dict]): 71 | self.step_count += 1 72 | image_base64 = parsed_screen['original_screenshot_base64'] 73 | latency_omniparser = parsed_screen['latency'] 74 | self.output_callback(f'-- Step {self.step_count}: --', sender="bot") 75 | screen_info = str(parsed_screen['screen_info']) 76 | screenshot_uuid = parsed_screen['screenshot_uuid'] 77 | screen_width, screen_height = parsed_screen['width'], parsed_screen['height'] 78 | 79 | boxids_and_labels = parsed_screen["screen_info"] 80 | system = self._get_system_prompt(boxids_and_labels) 81 | 82 | # drop looping actions msg, byte image etc 83 | planner_messages = messages 84 | _remove_som_images(planner_messages) 85 | _maybe_filter_to_n_most_recent_images(planner_messages, self.only_n_most_recent_images) 86 | 87 | if isinstance(planner_messages[-1], dict): 88 | if not isinstance(planner_messages[-1]["content"], list): 89 | planner_messages[-1]["content"] = [planner_messages[-1]["content"]] 90 | planner_messages[-1]["content"].append(f"{OUTPUT_DIR}/screenshot_{screenshot_uuid}.png") 91 | planner_messages[-1]["content"].append(f"{OUTPUT_DIR}/screenshot_som_{screenshot_uuid}.png") 92 | 93 | start = time.time() 94 | if "gpt" in self.model or "o1" in self.model or "o3-mini" in self.model: 95 | vlm_response, token_usage = run_oai_interleaved( 96 | messages=planner_messages, 97 | system=system, 98 | model_name=self.model, 99 | api_key=self.api_key, 100 | max_tokens=self.max_tokens, 101 | provider_base_url="https://api.openai.com/v1", 102 | temperature=0, 103 | ) 104 | print(f"oai token usage: {token_usage}") 105 | self.total_token_usage += token_usage 106 | if 'gpt' in self.model: 107 | self.total_cost += (token_usage * 2.5 / 1000000) # https://openai.com/api/pricing/ 108 | elif 'o1' in self.model: 109 | self.total_cost += (token_usage * 15 / 1000000) # https://openai.com/api/pricing/ 110 | elif 'o3-mini' in self.model: 111 | self.total_cost += (token_usage * 1.1 / 1000000) # https://openai.com/api/pricing/ 112 | elif "r1" in self.model: 113 | vlm_response, token_usage = run_groq_interleaved( 114 | messages=planner_messages, 115 | system=system, 116 | model_name=self.model, 117 | api_key=self.api_key, 118 | max_tokens=self.max_tokens, 119 | ) 120 | print(f"groq token usage: {token_usage}") 121 | self.total_token_usage += token_usage 122 | self.total_cost += (token_usage * 0.99 / 1000000) 123 | elif "qwen" in self.model: 124 | vlm_response, token_usage = run_oai_interleaved( 125 | messages=planner_messages, 126 | system=system, 127 | model_name=self.model, 128 | api_key=self.api_key, 129 | max_tokens=min(2048, self.max_tokens), 130 | provider_base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", 131 | temperature=0, 132 | ) 133 | print(f"qwen token usage: {token_usage}") 134 | self.total_token_usage += token_usage 135 | self.total_cost += (token_usage * 2.2 / 1000000) # https://help.aliyun.com/zh/model-studio/getting-started/models?spm=a2c4g.11186623.0.0.74b04823CGnPv7#fe96cfb1a422a 136 | else: 137 | raise ValueError(f"Model {self.model} not supported") 138 | latency_vlm = time.time() - start 139 | self.output_callback(f"LLM: {latency_vlm:.2f}s, OmniParser: {latency_omniparser:.2f}s", sender="bot") 140 | 141 | print(f"{vlm_response}") 142 | 143 | if self.print_usage: 144 | print(f"Total token so far: {self.total_token_usage}. Total cost so far: $USD{self.total_cost:.5f}") 145 | 146 | vlm_response_json = extract_data(vlm_response, "json") 147 | vlm_response_json = json.loads(vlm_response_json) 148 | 149 | img_to_show_base64 = parsed_screen["som_image_base64"] 150 | if "Box ID" in vlm_response_json: 151 | try: 152 | bbox = parsed_screen["parsed_content_list"][int(vlm_response_json["Box ID"])]["bbox"] 153 | vlm_response_json["box_centroid_coordinate"] = [int((bbox[0] + bbox[2]) / 2 * screen_width), int((bbox[1] + bbox[3]) / 2 * screen_height)] 154 | img_to_show_data = base64.b64decode(img_to_show_base64) 155 | img_to_show = Image.open(BytesIO(img_to_show_data)) 156 | 157 | draw = ImageDraw.Draw(img_to_show) 158 | x, y = vlm_response_json["box_centroid_coordinate"] 159 | radius = 10 160 | draw.ellipse((x - radius, y - radius, x + radius, y + radius), fill='red') 161 | draw.ellipse((x - radius*3, y - radius*3, x + radius*3, y + radius*3), fill=None, outline='red', width=2) 162 | 163 | buffered = BytesIO() 164 | img_to_show.save(buffered, format="PNG") 165 | img_to_show_base64 = base64.b64encode(buffered.getvalue()).decode("utf-8") 166 | except: 167 | print(f"Error parsing: {vlm_response_json}") 168 | pass 169 | self.output_callback(f'', sender="bot") 170 | self.output_callback( 171 | f'
' 172 | f' Parsed Screen elemetns by OmniParser' 173 | f' 
{screen_info}
' 174 | f'
', 175 | sender="bot" 176 | ) 177 | vlm_plan_str = "" 178 | for key, value in vlm_response_json.items(): 179 | if key == "Reasoning": 180 | vlm_plan_str += f'{value}' 181 | else: 182 | vlm_plan_str += f'\n{key}: {value}' 183 | 184 | # construct the response so that anthropicExcutor can execute the tool 185 | response_content = [BetaTextBlock(text=vlm_plan_str, type='text')] 186 | if 'box_centroid_coordinate' in vlm_response_json: 187 | move_cursor_block = BetaToolUseBlock(id=f'toolu_{uuid.uuid4()}', 188 | input={'action': 'mouse_move', 'coordinate': vlm_response_json["box_centroid_coordinate"]}, 189 | name='computer', type='tool_use') 190 | response_content.append(move_cursor_block) 191 | 192 | if vlm_response_json["Next Action"] == "None": 193 | print("Task paused/completed.") 194 | elif vlm_response_json["Next Action"] == "type": 195 | sim_content_block = BetaToolUseBlock(id=f'toolu_{uuid.uuid4()}', 196 | input={'action': vlm_response_json["Next Action"], 'text': vlm_response_json["value"]}, 197 | name='computer', type='tool_use') 198 | response_content.append(sim_content_block) 199 | else: 200 | sim_content_block = BetaToolUseBlock(id=f'toolu_{uuid.uuid4()}', 201 | input={'action': vlm_response_json["Next Action"]}, 202 | name='computer', type='tool_use') 203 | response_content.append(sim_content_block) 204 | response_message = BetaMessage(id=f'toolu_{uuid.uuid4()}', content=response_content, model='', role='assistant', type='message', stop_reason='tool_use', usage=BetaUsage(input_tokens=0, output_tokens=0)) 205 | return response_message, vlm_response_json 206 | 207 | def _api_response_callback(self, response: APIResponse): 208 | self.api_response_callback(response) 209 | 210 | def _get_system_prompt(self, screen_info: str = ""): 211 | main_section = f""" 212 | You are using a Windows device. 213 | You are able to use a mouse and keyboard to interact with the computer based on the given task and screenshot. 214 | You can only interact with the desktop GUI (no terminal or application menu access). 215 | 216 | You may be given some history plan and actions, this is the response from the previous loop. 217 | You should carefully consider your plan base on the task, screenshot, and history actions. 218 | 219 | Here is the list of all detected bounding boxes by IDs on the screen and their description:{screen_info} 220 | 221 | Your available "Next Action" only include: 222 | - type: types a string of text. 223 | - left_click: move mouse to box id and left clicks. 224 | - right_click: move mouse to box id and right clicks. 225 | - double_click: move mouse to box id and double clicks. 226 | - hover: move mouse to box id. 227 | - scroll_up: scrolls the screen up to view previous content. 228 | - scroll_down: scrolls the screen down, when the desired button is not visible, or you need to see more content. 229 | - wait: waits for 1 second for the device to load or respond. 230 | 231 | Based on the visual information from the screenshot image and the detected bounding boxes, please determine the next action, the Box ID you should operate on (if action is one of 'type', 'hover', 'scroll_up', 'scroll_down', 'wait', there should be no Box ID field), and the value (if the action is 'type') in order to complete the task. 232 | 233 | Output format: 234 | ```json 235 | {{ 236 | "Reasoning": str, # describe what is in the current screen, taking into account the history, then describe your step-by-step thoughts on how to achieve the task, choose one action from available actions at a time. 237 | "Next Action": "action_type, action description" | "None" # one action at a time, describe it in short and precisely. 238 | "Box ID": n, 239 | "value": "xxx" # only provide value field if the action is type, else don't include value key 240 | }} 241 | ``` 242 | 243 | One Example: 244 | ```json 245 | {{ 246 | "Reasoning": "The current screen shows google result of amazon, in previous action I have searched amazon on google. Then I need to click on the first search results to go to amazon.com.", 247 | "Next Action": "left_click", 248 | "Box ID": m 249 | }} 250 | ``` 251 | 252 | Another Example: 253 | ```json 254 | {{ 255 | "Reasoning": "The current screen shows the front page of amazon. There is no previous action. Therefore I need to type "Apple watch" in the search bar.", 256 | "Next Action": "type", 257 | "Box ID": n, 258 | "value": "Apple watch" 259 | }} 260 | ``` 261 | 262 | Another Example: 263 | ```json 264 | {{ 265 | "Reasoning": "The current screen does not show 'submit' button, I need to scroll down to see if the button is available.", 266 | "Next Action": "scroll_down", 267 | }} 268 | ``` 269 | 270 | IMPORTANT NOTES: 271 | 1. You should only give a single action at a time. 272 | 273 | """ 274 | thinking_model = "r1" in self.model 275 | if not thinking_model: 276 | main_section += """ 277 | 2. You should give an analysis to the current screen, and reflect on what has been done by looking at the history, then describe your step-by-step thoughts on how to achieve the task. 278 | 279 | """ 280 | else: 281 | main_section += """ 282 | 2. In XML tags give an analysis to the current screen, and reflect on what has been done by looking at the history, then describe your step-by-step thoughts on how to achieve the task. In XML tags put the next action prediction JSON. 283 | 284 | """ 285 | main_section += """ 286 | 3. Attach the next action prediction in the "Next Action". 287 | 4. You should not include other actions, such as keyboard shortcuts. 288 | 5. When the task is completed, don't complete additional actions. You should say "Next Action": "None" in the json field. 289 | 6. The tasks involve buying multiple products or navigating through multiple pages. You should break it into subgoals and complete each subgoal one by one in the order of the instructions. 290 | 7. avoid choosing the same action/elements multiple times in a row, if it happens, reflect to yourself, what may have gone wrong, and predict a different action. 291 | 8. If you are prompted with login information page or captcha page, or you think it need user's permission to do the next action, you should say "Next Action": "None" in the json field. 292 | """ 293 | 294 | return main_section 295 | 296 | def _remove_som_images(messages): 297 | for msg in messages: 298 | msg_content = msg["content"] 299 | if isinstance(msg_content, list): 300 | msg["content"] = [ 301 | cnt for cnt in msg_content 302 | if not (isinstance(cnt, str) and 'som' in cnt and is_image_path(cnt)) 303 | ] 304 | 305 | 306 | def _maybe_filter_to_n_most_recent_images( 307 | messages: list[BetaMessageParam], 308 | images_to_keep: int, 309 | min_removal_threshold: int = 10, 310 | ): 311 | """ 312 | With the assumption that images are screenshots that are of diminishing value as 313 | the conversation progresses, remove all but the final `images_to_keep` tool_result 314 | images in place 315 | """ 316 | if images_to_keep is None: 317 | return messages 318 | 319 | total_images = 0 320 | for msg in messages: 321 | for cnt in msg.get("content", []): 322 | if isinstance(cnt, str) and is_image_path(cnt): 323 | total_images += 1 324 | elif isinstance(cnt, dict) and cnt.get("type") == "tool_result": 325 | for content in cnt.get("content", []): 326 | if isinstance(content, dict) and content.get("type") == "image": 327 | total_images += 1 328 | 329 | images_to_remove = total_images - images_to_keep 330 | 331 | for msg in messages: 332 | msg_content = msg["content"] 333 | if isinstance(msg_content, list): 334 | new_content = [] 335 | for cnt in msg_content: 336 | # Remove images from SOM or screenshot as needed 337 | if isinstance(cnt, str) and is_image_path(cnt): 338 | if images_to_remove > 0: 339 | images_to_remove -= 1 340 | continue 341 | # VLM shouldn't use anthropic screenshot tool so shouldn't have these but in case it does, remove as needed 342 | elif isinstance(cnt, dict) and cnt.get("type") == "tool_result": 343 | new_tool_result_content = [] 344 | for tool_result_entry in cnt.get("content", []): 345 | if isinstance(tool_result_entry, dict) and tool_result_entry.get("type") == "image": 346 | if images_to_remove > 0: 347 | images_to_remove -= 1 348 | continue 349 | new_tool_result_content.append(tool_result_entry) 350 | cnt["content"] = new_tool_result_content 351 | # Append fixed content to current message's content list 352 | new_content.append(cnt) 353 | msg["content"] = new_content -------------------------------------------------------------------------------- /omnitool/gradio/app.py: -------------------------------------------------------------------------------- 1 | """ 2 | python app.py --windows_host_url localhost:8006 --omniparser_server_url localhost:8000 3 | """ 4 | 5 | import os 6 | from datetime import datetime 7 | from enum import StrEnum 8 | from functools import partial 9 | from pathlib import Path 10 | from typing import cast 11 | import argparse 12 | import gradio as gr 13 | from anthropic import APIResponse 14 | from anthropic.types import TextBlock 15 | from anthropic.types.beta import BetaMessage, BetaTextBlock, BetaToolUseBlock 16 | from anthropic.types.tool_use_block import ToolUseBlock 17 | from loop import ( 18 | APIProvider, 19 | sampling_loop_sync, 20 | ) 21 | from tools import ToolResult 22 | import requests 23 | from requests.exceptions import RequestException 24 | import base64 25 | 26 | CONFIG_DIR = Path("~/.anthropic").expanduser() 27 | API_KEY_FILE = CONFIG_DIR / "api_key" 28 | 29 | INTRO_TEXT = ''' 30 | OmniParser lets you turn any vision-langauge model into an AI agent. We currently support **OpenAI (4o/o1/o3-mini), DeepSeek (R1), Qwen (2.5VL) or Anthropic Computer Use (Sonnet).** 31 | 32 | Type a message and press submit to start OmniTool. Press stop to pause, and press the trash icon in the chat to clear the message history. 33 | ''' 34 | 35 | def parse_arguments(): 36 | 37 | parser = argparse.ArgumentParser(description="Gradio App") 38 | parser.add_argument("--windows_host_url", type=str, default='localhost:8006') 39 | parser.add_argument("--omniparser_server_url", type=str, default="localhost:8000") 40 | return parser.parse_args() 41 | args = parse_arguments() 42 | 43 | 44 | class Sender(StrEnum): 45 | USER = "user" 46 | BOT = "assistant" 47 | TOOL = "tool" 48 | 49 | 50 | def setup_state(state): 51 | if "messages" not in state: 52 | state["messages"] = [] 53 | if "model" not in state: 54 | state["model"] = "omniparser + gpt-4o" 55 | if "provider" not in state: 56 | state["provider"] = "openai" 57 | if "openai_api_key" not in state: # Fetch API keys from environment variables 58 | state["openai_api_key"] = os.getenv("OPENAI_API_KEY", "") 59 | if "anthropic_api_key" not in state: 60 | state["anthropic_api_key"] = os.getenv("ANTHROPIC_API_KEY", "") 61 | if "api_key" not in state: 62 | state["api_key"] = "" 63 | if "auth_validated" not in state: 64 | state["auth_validated"] = False 65 | if "responses" not in state: 66 | state["responses"] = {} 67 | if "tools" not in state: 68 | state["tools"] = {} 69 | if "only_n_most_recent_images" not in state: 70 | state["only_n_most_recent_images"] = 2 71 | if 'chatbot_messages' not in state: 72 | state['chatbot_messages'] = [] 73 | if 'stop' not in state: 74 | state['stop'] = False 75 | 76 | async def main(state): 77 | """Render loop for Gradio""" 78 | setup_state(state) 79 | return "Setup completed" 80 | 81 | def validate_auth(provider: APIProvider, api_key: str | None): 82 | if provider == APIProvider.ANTHROPIC: 83 | if not api_key: 84 | return "Enter your Anthropic API key to continue." 85 | if provider == APIProvider.BEDROCK: 86 | import boto3 87 | 88 | if not boto3.Session().get_credentials(): 89 | return "You must have AWS credentials set up to use the Bedrock API." 90 | if provider == APIProvider.VERTEX: 91 | import google.auth 92 | from google.auth.exceptions import DefaultCredentialsError 93 | 94 | if not os.environ.get("CLOUD_ML_REGION"): 95 | return "Set the CLOUD_ML_REGION environment variable to use the Vertex API." 96 | try: 97 | google.auth.default(scopes=["https://www.googleapis.com/auth/cloud-platform"]) 98 | except DefaultCredentialsError: 99 | return "Your google cloud credentials are not set up correctly." 100 | 101 | def load_from_storage(filename: str) -> str | None: 102 | """Load data from a file in the storage directory.""" 103 | try: 104 | file_path = CONFIG_DIR / filename 105 | if file_path.exists(): 106 | data = file_path.read_text().strip() 107 | if data: 108 | return data 109 | except Exception as e: 110 | print(f"Debug: Error loading {filename}: {e}") 111 | return None 112 | 113 | def save_to_storage(filename: str, data: str) -> None: 114 | """Save data to a file in the storage directory.""" 115 | try: 116 | CONFIG_DIR.mkdir(parents=True, exist_ok=True) 117 | file_path = CONFIG_DIR / filename 118 | file_path.write_text(data) 119 | # Ensure only user can read/write the file 120 | file_path.chmod(0o600) 121 | except Exception as e: 122 | print(f"Debug: Error saving {filename}: {e}") 123 | 124 | def _api_response_callback(response: APIResponse[BetaMessage], response_state: dict): 125 | response_id = datetime.now().isoformat() 126 | response_state[response_id] = response 127 | 128 | def _tool_output_callback(tool_output: ToolResult, tool_id: str, tool_state: dict): 129 | tool_state[tool_id] = tool_output 130 | 131 | def chatbot_output_callback(message, chatbot_state, hide_images=False, sender="bot"): 132 | def _render_message(message: str | BetaTextBlock | BetaToolUseBlock | ToolResult, hide_images=False): 133 | 134 | print(f"_render_message: {str(message)[:100]}") 135 | 136 | if isinstance(message, str): 137 | return message 138 | 139 | is_tool_result = not isinstance(message, str) and ( 140 | isinstance(message, ToolResult) 141 | or message.__class__.__name__ == "ToolResult" 142 | ) 143 | if not message or ( 144 | is_tool_result 145 | and hide_images 146 | and not hasattr(message, "error") 147 | and not hasattr(message, "output") 148 | ): # return None if hide_images is True 149 | return 150 | # render tool result 151 | if is_tool_result: 152 | message = cast(ToolResult, message) 153 | if message.output: 154 | return message.output 155 | if message.error: 156 | return f"Error: {message.error}" 157 | if message.base64_image and not hide_images: 158 | # somehow can't display via gr.Image 159 | # image_data = base64.b64decode(message.base64_image) 160 | # return gr.Image(value=Image.open(io.BytesIO(image_data))) 161 | return f'' 162 | 163 | elif isinstance(message, BetaTextBlock) or isinstance(message, TextBlock): 164 | return f"Analysis: {message.text}" 165 | elif isinstance(message, BetaToolUseBlock) or isinstance(message, ToolUseBlock): 166 | # return f"Tool Use: {message.name}\nInput: {message.input}" 167 | return f"Next I will perform the following action: {message.input}" 168 | else: 169 | return message 170 | 171 | def _truncate_string(s, max_length=500): 172 | """Truncate long strings for concise printing.""" 173 | if isinstance(s, str) and len(s) > max_length: 174 | return s[:max_length] + "..." 175 | return s 176 | # processing Anthropic messages 177 | message = _render_message(message, hide_images) 178 | 179 | if sender == "bot": 180 | chatbot_state.append((None, message)) 181 | else: 182 | chatbot_state.append((message, None)) 183 | 184 | # Create a concise version of the chatbot state for printing 185 | concise_state = [(_truncate_string(user_msg), _truncate_string(bot_msg)) 186 | for user_msg, bot_msg in chatbot_state] 187 | # print(f"chatbot_output_callback chatbot_state: {concise_state} (truncated)") 188 | 189 | def valid_params(user_input, state): 190 | """Validate all requirements and return a list of error messages.""" 191 | errors = [] 192 | 193 | for server_name, url in [('Windows Host', 'localhost:5000'), ('OmniParser Server', args.omniparser_server_url)]: 194 | try: 195 | url = f'http://{url}/probe' 196 | response = requests.get(url, timeout=3) 197 | if response.status_code != 200: 198 | errors.append(f"{server_name} is not responding") 199 | except RequestException as e: 200 | errors.append(f"{server_name} is not responding") 201 | 202 | if not state["api_key"].strip(): 203 | errors.append("LLM API Key is not set") 204 | 205 | if not user_input: 206 | errors.append("no computer use request provided") 207 | 208 | return errors 209 | 210 | def process_input(user_input, state): 211 | # Reset the stop flag 212 | if state["stop"]: 213 | state["stop"] = False 214 | 215 | errors = valid_params(user_input, state) 216 | if errors: 217 | raise gr.Error("Validation errors: " + ", ".join(errors)) 218 | 219 | # Append the user message to state["messages"] 220 | state["messages"].append( 221 | { 222 | "role": Sender.USER, 223 | "content": [TextBlock(type="text", text=user_input)], 224 | } 225 | ) 226 | 227 | # Append the user's message to chatbot_messages with None for the assistant's reply 228 | state['chatbot_messages'].append((user_input, None)) 229 | yield state['chatbot_messages'] # Yield to update the chatbot UI with the user's message 230 | 231 | print("state") 232 | print(state) 233 | 234 | # Run sampling_loop_sync with the chatbot_output_callback 235 | for loop_msg in sampling_loop_sync( 236 | model=state["model"], 237 | provider=state["provider"], 238 | messages=state["messages"], 239 | output_callback=partial(chatbot_output_callback, chatbot_state=state['chatbot_messages'], hide_images=False), 240 | tool_output_callback=partial(_tool_output_callback, tool_state=state["tools"]), 241 | api_response_callback=partial(_api_response_callback, response_state=state["responses"]), 242 | api_key=state["api_key"], 243 | only_n_most_recent_images=state["only_n_most_recent_images"], 244 | max_tokens=16384, 245 | omniparser_url=args.omniparser_server_url 246 | ): 247 | if loop_msg is None or state.get("stop"): 248 | yield state['chatbot_messages'] 249 | print("End of task. Close the loop.") 250 | break 251 | 252 | yield state['chatbot_messages'] # Yield the updated chatbot_messages to update the chatbot UI 253 | 254 | def stop_app(state): 255 | state["stop"] = True 256 | return "App stopped" 257 | 258 | def get_header_image_base64(): 259 | try: 260 | # Get the absolute path to the image relative to this script 261 | script_dir = Path(__file__).parent 262 | image_path = script_dir.parent.parent / "imgs" / "header_bar_thin.png" 263 | 264 | with open(image_path, "rb") as image_file: 265 | encoded_string = base64.b64encode(image_file.read()).decode() 266 | return f'data:image/png;base64,{encoded_string}' 267 | except Exception as e: 268 | print(f"Failed to load header image: {e}") 269 | return None 270 | 271 | with gr.Blocks(theme=gr.themes.Default()) as demo: 272 | gr.HTML(""" 273 | 284 | """) 285 | state = gr.State({}) 286 | 287 | setup_state(state.value) 288 | 289 | header_image = get_header_image_base64() 290 | if header_image: 291 | gr.HTML(f'OmniTool Header', elem_classes="no-padding") 292 | gr.HTML('

OmniTool

') 293 | else: 294 | gr.Markdown("# OmniTool") 295 | 296 | if not os.getenv("HIDE_WARNING", False): 297 | gr.Markdown(INTRO_TEXT, elem_classes="markdown-text") 298 | 299 | 300 | with gr.Accordion("Settings", open=True): 301 | with gr.Row(): 302 | with gr.Column(): 303 | model = gr.Dropdown( 304 | label="Model", 305 | choices=["omniparser + gpt-4o", "omniparser + o1", "omniparser + o3-mini", "omniparser + R1", "omniparser + qwen2.5vl", "claude-3-5-sonnet-20241022", "omniparser + gpt-4o-orchestrated", "omniparser + o1-orchestrated", "omniparser + o3-mini-orchestrated", "omniparser + R1-orchestrated", "omniparser + qwen2.5vl-orchestrated"], 306 | value="omniparser + gpt-4o", 307 | interactive=True, 308 | ) 309 | with gr.Column(): 310 | only_n_images = gr.Slider( 311 | label="N most recent screenshots", 312 | minimum=0, 313 | maximum=10, 314 | step=1, 315 | value=2, 316 | interactive=True 317 | ) 318 | with gr.Row(): 319 | with gr.Column(1): 320 | provider = gr.Dropdown( 321 | label="API Provider", 322 | choices=[option.value for option in APIProvider], 323 | value="openai", 324 | interactive=False, 325 | ) 326 | with gr.Column(2): 327 | api_key = gr.Textbox( 328 | label="API Key", 329 | type="password", 330 | value=state.value.get("api_key", ""), 331 | placeholder="Paste your API key here", 332 | interactive=True, 333 | ) 334 | 335 | with gr.Row(): 336 | with gr.Column(scale=8): 337 | chat_input = gr.Textbox(show_label=False, placeholder="Type a message to send to Omniparser + X ...", container=False) 338 | with gr.Column(scale=1, min_width=50): 339 | submit_button = gr.Button(value="Send", variant="primary") 340 | with gr.Column(scale=1, min_width=50): 341 | stop_button = gr.Button(value="Stop", variant="secondary") 342 | 343 | with gr.Row(): 344 | with gr.Column(scale=2): 345 | chatbot = gr.Chatbot(label="Chatbot History", autoscroll=True, height=580) 346 | with gr.Column(scale=3): 347 | iframe = gr.HTML( 348 | f'', 349 | container=False, 350 | elem_classes="no-padding" 351 | ) 352 | 353 | def update_model(model_selection, state): 354 | state["model"] = model_selection 355 | print(f"Model updated to: {state['model']}") 356 | 357 | if model_selection == "claude-3-5-sonnet-20241022": 358 | provider_choices = [option.value for option in APIProvider if option.value != "openai"] 359 | elif model_selection in set(["omniparser + gpt-4o", "omniparser + o1", "omniparser + o3-mini", "omniparser + gpt-4o-orchestrated", "omniparser + o1-orchestrated", "omniparser + o3-mini-orchestrated"]): 360 | provider_choices = ["openai"] 361 | elif model_selection == "omniparser + R1": 362 | provider_choices = ["groq"] 363 | elif model_selection == "omniparser + qwen2.5vl": 364 | provider_choices = ["dashscope"] 365 | else: 366 | provider_choices = [option.value for option in APIProvider] 367 | default_provider_value = provider_choices[0] 368 | 369 | provider_interactive = len(provider_choices) > 1 370 | api_key_placeholder = f"{default_provider_value.title()} API Key" 371 | 372 | # Update state 373 | state["provider"] = default_provider_value 374 | state["api_key"] = state.get(f"{default_provider_value}_api_key", "") 375 | 376 | # Calls to update other components UI 377 | provider_update = gr.update( 378 | choices=provider_choices, 379 | value=default_provider_value, 380 | interactive=provider_interactive 381 | ) 382 | api_key_update = gr.update( 383 | placeholder=api_key_placeholder, 384 | value=state["api_key"] 385 | ) 386 | 387 | return provider_update, api_key_update 388 | 389 | def update_only_n_images(only_n_images_value, state): 390 | state["only_n_most_recent_images"] = only_n_images_value 391 | 392 | def update_provider(provider_value, state): 393 | # Update state 394 | state["provider"] = provider_value 395 | state["api_key"] = state.get(f"{provider_value}_api_key", "") 396 | 397 | # Calls to update other components UI 398 | api_key_update = gr.update( 399 | placeholder=f"{provider_value.title()} API Key", 400 | value=state["api_key"] 401 | ) 402 | return api_key_update 403 | 404 | def update_api_key(api_key_value, state): 405 | state["api_key"] = api_key_value 406 | state[f'{state["provider"]}_api_key'] = api_key_value 407 | 408 | def clear_chat(state): 409 | # Reset message-related state 410 | state["messages"] = [] 411 | state["responses"] = {} 412 | state["tools"] = {} 413 | state['chatbot_messages'] = [] 414 | return state['chatbot_messages'] 415 | 416 | model.change(fn=update_model, inputs=[model, state], outputs=[provider, api_key]) 417 | only_n_images.change(fn=update_only_n_images, inputs=[only_n_images, state], outputs=None) 418 | provider.change(fn=update_provider, inputs=[provider, state], outputs=api_key) 419 | api_key.change(fn=update_api_key, inputs=[api_key, state], outputs=None) 420 | chatbot.clear(fn=clear_chat, inputs=[state], outputs=[chatbot]) 421 | 422 | submit_button.click(process_input, [chat_input, state], chatbot) 423 | stop_button.click(stop_app, [state], None) 424 | 425 | if __name__ == "__main__": 426 | demo.launch(server_name="0.0.0.0", server_port=7888) -------------------------------------------------------------------------------- /omnitool/gradio/app_streamlit.py: -------------------------------------------------------------------------------- 1 | """ 2 | Streamlit implementation of the OmniTool frontend. 3 | Usage: streamlit run app_streamlit.py -- --windows_host_url localhost:8006 --omniparser_server_url localhost:8000 4 | """ 5 | 6 | import os 7 | import io 8 | import shutil 9 | import mimetypes 10 | import argparse 11 | import base64 12 | from datetime import datetime 13 | from pathlib import Path 14 | from typing import cast 15 | from enum import StrEnum 16 | import streamlit as st 17 | from anthropic import APIResponse 18 | from anthropic.types import TextBlock 19 | from anthropic.types.beta import BetaMessage, BetaTextBlock, BetaToolUseBlock 20 | from anthropic.types.tool_use_block import ToolUseBlock 21 | import requests 22 | from requests.exceptions import RequestException 23 | 24 | from loop import ( 25 | APIProvider, 26 | sampling_loop_sync, 27 | ) 28 | from tools import ToolResult 29 | 30 | # Constants and configurations 31 | CONFIG_DIR = Path("~/.anthropic").expanduser() 32 | API_KEY_FILE = CONFIG_DIR / "api_key" 33 | UPLOAD_FOLDER = Path("./uploads").absolute() 34 | UPLOAD_FOLDER.mkdir(parents=True, exist_ok=True) 35 | 36 | class Sender(StrEnum): 37 | USER = "user" 38 | BOT = "assistant" 39 | TOOL = "tool" 40 | 41 | def parse_arguments(): 42 | parser = argparse.ArgumentParser(description="Streamlit App") 43 | parser.add_argument("--windows_host_url", type=str, default='localhost:8006') 44 | parser.add_argument("--omniparser_server_url", type=str, default="localhost:8000") 45 | parser.add_argument("--upload_folder", type=str, default="./uploads") 46 | return parser.parse_known_args()[0] 47 | 48 | def initialize_session_state(): 49 | """Initialize session state variables""" 50 | if "messages" not in st.session_state: 51 | st.session_state.messages = [] 52 | if "model" not in st.session_state: 53 | st.session_state.model = "omniparser + gpt-4o-orchestrated" 54 | if "provider" not in st.session_state: 55 | st.session_state.provider = "openai" 56 | if "api_key" not in st.session_state: 57 | st.session_state.api_key = os.getenv("OPENAI_API_KEY", "") 58 | if "anthropic_api_key" not in st.session_state: 59 | st.session_state.anthropic_api_key = os.getenv("ANTHROPIC_API_KEY", "") 60 | if "only_n_most_recent_images" not in st.session_state: 61 | st.session_state.only_n_most_recent_images = 2 62 | if "responses" not in st.session_state: 63 | st.session_state.responses = {} 64 | if "tools" not in st.session_state: 65 | st.session_state.tools = {} 66 | if "uploaded_files" not in st.session_state: 67 | st.session_state.uploaded_files = [] 68 | if "selected_file" not in st.session_state: 69 | st.session_state.selected_file = "None" 70 | if "stop" not in st.session_state: 71 | st.session_state.stop = False 72 | 73 | def get_file_viewer_html(file_path=None, windows_host_url=None): 74 | """Generate HTML to view a file based on its type""" 75 | if not file_path: 76 | # Return the VNC viewer iframe 77 | return f'' 78 | 79 | file_path = Path(file_path) 80 | if not file_path.exists(): 81 | return f'
File not found: {file_path.name}
' 82 | 83 | mime_type, _ = mimetypes.guess_type(file_path) 84 | file_type = mime_type.split('/')[0] if mime_type else 'unknown' 85 | file_extension = file_path.suffix.lower() 86 | 87 | if file_type == 'image': 88 | with open(file_path, "rb") as image_file: 89 | encoded_string = base64.b64encode(image_file.read()).decode() 90 | return f'

{file_path.name}

' 91 | 92 | elif file_extension in ['.txt', '.py', '.js', '.html', '.css', '.json', '.md', '.csv'] or file_type == 'text': 93 | try: 94 | content = file_path.read_text(errors='replace') 95 | content = content.replace('&', '&').replace('<', '<').replace('>', '>') 96 | return f'

{file_path.name}

{content}
' 97 | except UnicodeDecodeError: 98 | return f'
Cannot display binary file: {file_path.name}
' 99 | 100 | else: 101 | size_kb = file_path.stat().st_size / 1024 102 | return f'

{file_path.name}

File type: {mime_type or "Unknown"}

Size: {size_kb:.2f} KB

This file type cannot be displayed in the browser.

' 103 | 104 | def handle_file_upload(uploaded_files): 105 | """Handle file uploads and store them in the upload directory""" 106 | if uploaded_files: 107 | for file in uploaded_files: 108 | file_path = UPLOAD_FOLDER / file.name 109 | with open(file_path, "wb") as f: 110 | f.write(file.getvalue()) 111 | if str(file_path) not in st.session_state.uploaded_files: 112 | st.session_state.uploaded_files.append(str(file_path)) 113 | 114 | def _api_response_callback(response: APIResponse[BetaMessage]): 115 | response_id = datetime.now().isoformat() 116 | st.session_state.responses[response_id] = response 117 | 118 | def _tool_output_callback(tool_output: ToolResult, tool_id: str): 119 | st.session_state.tools[tool_id] = tool_output 120 | 121 | def chatbot_output_callback(message, hide_images=False): 122 | def _render_message(message: str | BetaTextBlock | BetaToolUseBlock | ToolResult, hide_images=False): 123 | if isinstance(message, str): 124 | return message 125 | 126 | is_tool_result = not isinstance(message, str) and ( 127 | isinstance(message, ToolResult) 128 | or message.__class__.__name__ == "ToolResult" 129 | ) 130 | 131 | if is_tool_result: 132 | message = cast(ToolResult, message) 133 | if message.output: 134 | return message.output 135 | if message.error: 136 | return f"Error: {message.error}" 137 | if message.base64_image and not hide_images: 138 | return f'' 139 | 140 | elif isinstance(message, (BetaTextBlock, TextBlock)): 141 | return f"Next step Reasoning: {message.text}" 142 | 143 | elif isinstance(message, (BetaToolUseBlock, ToolUseBlock)): 144 | return None 145 | 146 | return message 147 | 148 | rendered_message = _render_message(message, hide_images) 149 | if rendered_message: 150 | st.session_state.messages.append({"role": "assistant", "content": rendered_message}) 151 | 152 | def main(): 153 | args = parse_arguments() 154 | initialize_session_state() 155 | 156 | # Page configuration 157 | st.set_page_config( 158 | page_title="OmniTool", 159 | page_icon="🤖", 160 | layout="wide" 161 | ) 162 | 163 | # Custom CSS 164 | st.markdown(""" 165 | 244 | """, unsafe_allow_html=True) 245 | 246 | # Header 247 | st.title("OmniTool") 248 | 249 | # Sidebar with settings 250 | with st.sidebar: 251 | st.header("Settings") 252 | 253 | # Model selection 254 | model = st.selectbox( 255 | "Model", 256 | ["omniparser + gpt-4o", "omniparser + o1", "omniparser + o3-mini", 257 | "omniparser + R1", "omniparser + qwen2.5vl", "claude-3-5-sonnet-20241022", 258 | "omniparser + gpt-4o-orchestrated", "omniparser + o1-orchestrated", 259 | "omniparser + o3-mini-orchestrated", "omniparser + R1-orchestrated", 260 | "omniparser + qwen2.5vl-orchestrated"], 261 | index=6 262 | ) 263 | st.session_state.model = model 264 | 265 | # API settings 266 | api_key = st.text_input("API Key", value=st.session_state.api_key, type="password") 267 | st.session_state.api_key = api_key 268 | 269 | # Image settings 270 | n_images = st.slider("N most recent screenshots", 0, 10, 2) 271 | st.session_state.only_n_most_recent_images = n_images 272 | 273 | # File viewer selection 274 | file_options = ["None"] 275 | if st.session_state.uploaded_files: 276 | file_options.extend([Path(f).name for f in st.session_state.uploaded_files]) 277 | 278 | selected_file = st.selectbox( 279 | "View File", 280 | options=file_options, 281 | format_func=lambda x: x 282 | ) 283 | st.session_state.selected_file = selected_file 284 | 285 | view_mode = st.radio("Display Mode", ["OmniTool Computer", "File Viewer"]) 286 | 287 | # Main content area with two columns 288 | col1, col2 = st.columns([2, 3]) 289 | 290 | # Chat interface (left column) 291 | with col1: 292 | # Chat header with title and share button 293 | col_header_1, col_header_2 = st.columns([3, 1]) 294 | with col_header_1: 295 | st.markdown("### Chat") 296 | with col_header_2: 297 | share_button = st.button("📤 Share", key="share_btn", help="Share conversation") 298 | # Apply custom styling with HTML 299 | st.markdown(""" 300 | 312 | """, unsafe_allow_html=True) 313 | 314 | # Share functionality 315 | if share_button: 316 | # Create a shareable text of the conversation 317 | conversation_text = "" 318 | for message in st.session_state.messages: 319 | if message["role"] == "user": 320 | conversation_text += f"User: {message['content']}\n\n" 321 | else: 322 | conversation_text += f"Assistant: {message['content']}\n\n" 323 | 324 | # Create a download link 325 | timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") 326 | st.download_button( 327 | label="Download Conversation", 328 | data=conversation_text, 329 | file_name=f"omnitool_conversation_{timestamp}.txt", 330 | mime="text/plain", 331 | key="download_conversation" 332 | ) 333 | 334 | # Display chat messages 335 | chat_container = st.container(height=450) 336 | with chat_container: 337 | for message in st.session_state.messages: 338 | if message["role"] == "user": 339 | st.markdown(f"**You:** {message['content']}") 340 | else: 341 | st.markdown(f"**Assistant:** {message['content']}", unsafe_allow_html=True) 342 | 343 | # Chat input and buttons 344 | user_input = st.text_input( 345 | "Type your message:", 346 | key="user_input", 347 | label_visibility="collapsed", 348 | placeholder="Send message to OmniTool..." 349 | ) 350 | 351 | # Button row with icons 352 | col1_1, col1_2, col1_3, col1_4 = st.columns([6, 1, 1, 1]) 353 | 354 | with col1_2: 355 | # Send button with icon - using arrow up icon 356 | send_button = st.button("⬆️", help="Send message", key="send_btn") 357 | # Apply custom styling with HTML 358 | st.markdown(""" 359 | 365 | """, unsafe_allow_html=True) 366 | 367 | with col1_3: 368 | # Stop button with icon 369 | stop_button = st.button("🛑", help="Stop processing", key="stop_btn") 370 | # Apply custom styling with HTML 371 | st.markdown(""" 372 | 379 | """, unsafe_allow_html=True) 380 | 381 | with col1_4: 382 | # File upload button with icon 383 | upload_button = st.button("📎", help="Upload files", key="upload_btn") 384 | # Apply custom styling with HTML 385 | st.markdown(""" 386 | 393 | """, unsafe_allow_html=True) 394 | 395 | # File upload area (hidden by default, shown when upload button is clicked) 396 | if upload_button: 397 | uploaded_files = st.file_uploader("Upload Files", accept_multiple_files=True, label_visibility="collapsed") 398 | if uploaded_files: 399 | handle_file_upload(uploaded_files) 400 | st.success(f"Uploaded {len(uploaded_files)} file(s)") 401 | # Update file options 402 | file_options = ["None"] 403 | if st.session_state.uploaded_files: 404 | file_options.extend([Path(f).name for f in st.session_state.uploaded_files]) 405 | st.rerun() 406 | 407 | # Process send button click 408 | if send_button and user_input: 409 | # Add user message to state 410 | st.session_state.messages.append({"role": "user", "content": user_input}) 411 | 412 | # Process the message through sampling_loop_sync 413 | for loop_msg in sampling_loop_sync( 414 | model=st.session_state.model, 415 | provider=st.session_state.provider, 416 | messages=[{"role": "user", "content": [TextBlock(type="text", text=msg["content"])]} for msg in st.session_state.messages], 417 | output_callback=chatbot_output_callback, 418 | tool_output_callback=_tool_output_callback, 419 | api_response_callback=_api_response_callback, 420 | api_key=st.session_state.api_key, 421 | only_n_most_recent_images=st.session_state.only_n_most_recent_images, 422 | max_tokens=16384, 423 | omniparser_url=args.omniparser_server_url, 424 | save_folder=str(UPLOAD_FOLDER) 425 | ): 426 | if loop_msg is None or st.session_state.stop: 427 | break 428 | st.rerun() 429 | 430 | # Process stop button click 431 | if stop_button: 432 | st.session_state.stop = True 433 | st.info("Processing stopped") 434 | 435 | # Viewer interface (right column) 436 | with col2: 437 | st.markdown("### Display") 438 | if view_mode == "OmniTool Computer": 439 | viewer_html = get_file_viewer_html(windows_host_url=args.windows_host_url) 440 | st.components.v1.html( 441 | viewer_html, 442 | height=600, 443 | scrolling=True 444 | ) 445 | else: # File Viewer mode 446 | if st.session_state.selected_file and st.session_state.selected_file != "None": 447 | file_path = next((f for f in st.session_state.uploaded_files 448 | if Path(f).name == st.session_state.selected_file), None) 449 | if file_path: 450 | viewer_html = get_file_viewer_html(file_path=file_path) 451 | st.components.v1.html( 452 | viewer_html, 453 | height=600, 454 | scrolling=True 455 | ) 456 | else: 457 | st.error(f"Could not find file: {st.session_state.selected_file}") 458 | else: 459 | st.info("Please select a file to view from the sidebar.") 460 | 461 | # Debug information (temporary) 462 | with st.expander("Debug Info"): 463 | st.write("View Mode:", view_mode) 464 | st.write("Selected File:", st.session_state.selected_file) 465 | st.write("Available Files:", st.session_state.uploaded_files) 466 | if view_mode == "File Viewer" and st.session_state.selected_file != "None": 467 | st.write("File Path:", file_path if 'file_path' in locals() else "Not found") 468 | 469 | if __name__ == "__main__": 470 | main() 471 | -------------------------------------------------------------------------------- /omnitool/gradio/executor/anthropic_executor.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from typing import Any, Dict, cast 3 | from collections.abc import Callable 4 | from anthropic.types.beta import ( 5 | BetaContentBlock, 6 | BetaContentBlockParam, 7 | BetaImageBlockParam, 8 | BetaMessage, 9 | BetaMessageParam, 10 | BetaTextBlockParam, 11 | BetaToolResultBlockParam, 12 | ) 13 | from anthropic.types import TextBlock 14 | from anthropic.types.beta import BetaMessage, BetaTextBlock, BetaToolUseBlock 15 | from tools import ComputerTool, ToolCollection, ToolResult 16 | 17 | 18 | class AnthropicExecutor: 19 | def __init__( 20 | self, 21 | output_callback: Callable[[BetaContentBlockParam], None], 22 | tool_output_callback: Callable[[Any, str], None], 23 | ): 24 | self.tool_collection = ToolCollection( 25 | ComputerTool() 26 | ) 27 | self.output_callback = output_callback 28 | self.tool_output_callback = tool_output_callback 29 | 30 | def __call__(self, response: BetaMessage, messages: list[BetaMessageParam]): 31 | new_message = { 32 | "role": "assistant", 33 | "content": cast(list[BetaContentBlockParam], response.content), 34 | } 35 | if new_message not in messages: 36 | messages.append(new_message) 37 | else: 38 | print("new_message already in messages, there are duplicates.") 39 | 40 | tool_result_content: list[BetaToolResultBlockParam] = [] 41 | for content_block in cast(list[BetaContentBlock], response.content): 42 | self.output_callback(content_block, sender="bot") 43 | # Execute the tool 44 | if content_block.type == "tool_use": 45 | # Run the asynchronous tool execution in a synchronous context 46 | result = asyncio.run(self.tool_collection.run( 47 | name=content_block.name, 48 | tool_input=cast(dict[str, Any], content_block.input), 49 | )) 50 | 51 | self.output_callback(result, sender="bot") 52 | 53 | tool_result_content.append( 54 | _make_api_tool_result(result, content_block.id) 55 | ) 56 | # self.tool_output_callback(result, content_block.id) 57 | 58 | # Craft messages based on the content_block 59 | # Note: to display the messages in the gradio, you should organize the messages in the following way (user message, bot message) 60 | 61 | display_messages = _message_display_callback(messages) 62 | # display_messages = [] 63 | 64 | # Send the messages to the gradio 65 | for user_msg, bot_msg in display_messages: 66 | # yield [user_msg, bot_msg], tool_result_content 67 | yield [None, None], tool_result_content 68 | 69 | if not tool_result_content: 70 | return messages 71 | 72 | return tool_result_content 73 | 74 | def _message_display_callback(messages): 75 | display_messages = [] 76 | for msg in messages: 77 | try: 78 | if isinstance(msg["content"][0], TextBlock): 79 | display_messages.append((msg["content"][0].text, None)) # User message 80 | elif isinstance(msg["content"][0], BetaTextBlock): 81 | display_messages.append((None, msg["content"][0].text)) # Bot message 82 | elif isinstance(msg["content"][0], BetaToolUseBlock): 83 | display_messages.append((None, f"Tool Use: {msg['content'][0].name}\nInput: {msg['content'][0].input}")) # Bot message 84 | elif isinstance(msg["content"][0], Dict) and msg["content"][0]["content"][-1]["type"] == "image": 85 | display_messages.append((None, f'')) # Bot message 86 | else: 87 | print(msg["content"][0]) 88 | except Exception as e: 89 | print("error", e) 90 | pass 91 | return display_messages 92 | 93 | def _make_api_tool_result( 94 | result: ToolResult, tool_use_id: str 95 | ) -> BetaToolResultBlockParam: 96 | """Convert an agent ToolResult to an API ToolResultBlockParam.""" 97 | tool_result_content: list[BetaTextBlockParam | BetaImageBlockParam] | str = [] 98 | is_error = False 99 | if result.error: 100 | is_error = True 101 | tool_result_content = _maybe_prepend_system_tool_result(result, result.error) 102 | else: 103 | if result.output: 104 | tool_result_content.append( 105 | { 106 | "type": "text", 107 | "text": _maybe_prepend_system_tool_result(result, result.output), 108 | } 109 | ) 110 | if result.base64_image: 111 | tool_result_content.append( 112 | { 113 | "type": "image", 114 | "source": { 115 | "type": "base64", 116 | "media_type": "image/png", 117 | "data": result.base64_image, 118 | }, 119 | } 120 | ) 121 | return { 122 | "type": "tool_result", 123 | "content": tool_result_content, 124 | "tool_use_id": tool_use_id, 125 | "is_error": is_error, 126 | } 127 | 128 | 129 | def _maybe_prepend_system_tool_result(result: ToolResult, result_text: str): 130 | if result.system: 131 | result_text = f"{result.system}\n{result_text}" 132 | return result_text -------------------------------------------------------------------------------- /omnitool/gradio/loop.py: -------------------------------------------------------------------------------- 1 | """ 2 | Agentic sampling loop that calls the Anthropic API and local implenmentation of anthropic-defined computer use tools. 3 | """ 4 | from collections.abc import Callable 5 | from enum import StrEnum 6 | 7 | from anthropic import APIResponse 8 | from anthropic.types import ( 9 | TextBlock, 10 | ) 11 | from anthropic.types.beta import ( 12 | BetaContentBlock, 13 | BetaMessage, 14 | BetaMessageParam 15 | ) 16 | from tools import ToolResult 17 | 18 | from agent.llm_utils.omniparserclient import OmniParserClient 19 | from agent.anthropic_agent import AnthropicActor 20 | from agent.vlm_agent import VLMAgent 21 | from agent.vlm_agent_with_orchestrator import VLMOrchestratedAgent 22 | from executor.anthropic_executor import AnthropicExecutor 23 | 24 | BETA_FLAG = "computer-use-2024-10-22" 25 | 26 | class APIProvider(StrEnum): 27 | ANTHROPIC = "anthropic" 28 | BEDROCK = "bedrock" 29 | VERTEX = "vertex" 30 | OPENAI = "openai" 31 | 32 | 33 | PROVIDER_TO_DEFAULT_MODEL_NAME: dict[APIProvider, str] = { 34 | APIProvider.ANTHROPIC: "claude-3-5-sonnet-20241022", 35 | APIProvider.BEDROCK: "anthropic.claude-3-5-sonnet-20241022-v2:0", 36 | APIProvider.VERTEX: "claude-3-5-sonnet-v2@20241022", 37 | APIProvider.OPENAI: "gpt-4o", 38 | } 39 | 40 | def sampling_loop_sync( 41 | *, 42 | model: str, 43 | provider: APIProvider | None, 44 | messages: list[BetaMessageParam], 45 | output_callback: Callable[[BetaContentBlock], None], 46 | tool_output_callback: Callable[[ToolResult, str], None], 47 | api_response_callback: Callable[[APIResponse[BetaMessage]], None], 48 | api_key: str, 49 | only_n_most_recent_images: int | None = 2, 50 | max_tokens: int = 4096, 51 | omniparser_url: str, 52 | save_folder: str = "./uploads" 53 | ): 54 | """ 55 | Synchronous agentic sampling loop for the assistant/tool interaction of computer use. 56 | """ 57 | print('in sampling_loop_sync, model:', model) 58 | omniparser_client = OmniParserClient(url=f"http://{omniparser_url}/parse/") 59 | if model == "claude-3-5-sonnet-20241022": 60 | # Register Actor and Executor 61 | actor = AnthropicActor( 62 | model=model, 63 | provider=provider, 64 | api_key=api_key, 65 | api_response_callback=api_response_callback, 66 | max_tokens=max_tokens, 67 | only_n_most_recent_images=only_n_most_recent_images 68 | ) 69 | elif model in set(["omniparser + gpt-4o", "omniparser + o1", "omniparser + o3-mini", "omniparser + R1", "omniparser + qwen2.5vl"]): 70 | actor = VLMAgent( 71 | model=model, 72 | provider=provider, 73 | api_key=api_key, 74 | api_response_callback=api_response_callback, 75 | output_callback=output_callback, 76 | max_tokens=max_tokens, 77 | only_n_most_recent_images=only_n_most_recent_images 78 | ) 79 | elif model in set(["omniparser + gpt-4o-orchestrated", "omniparser + o1-orchestrated", "omniparser + o3-mini-orchestrated", "omniparser + R1-orchestrated", "omniparser + qwen2.5vl-orchestrated"]): 80 | actor = VLMOrchestratedAgent( 81 | model=model, 82 | provider=provider, 83 | api_key=api_key, 84 | api_response_callback=api_response_callback, 85 | output_callback=output_callback, 86 | max_tokens=max_tokens, 87 | only_n_most_recent_images=only_n_most_recent_images, 88 | save_folder=save_folder 89 | ) 90 | else: 91 | raise ValueError(f"Model {model} not supported") 92 | executor = AnthropicExecutor( 93 | output_callback=output_callback, 94 | tool_output_callback=tool_output_callback, 95 | ) 96 | print(f"Model Inited: {model}, Provider: {provider}") 97 | 98 | tool_result_content = None 99 | 100 | print(f"Start the message loop. User messages: {messages}") 101 | 102 | if model == "claude-3-5-sonnet-20241022": # Anthropic loop 103 | while True: 104 | parsed_screen = omniparser_client() # parsed_screen: {"som_image_base64": dino_labled_img, "parsed_content_list": parsed_content_list, "screen_info"} 105 | screen_info_block = TextBlock(text='Below is the structured accessibility information of the current UI screen, which includes text and icons you can operate on, take these information into account when you are making the prediction for the next action. Note you will still need to take screenshot to get the image: \n' + parsed_screen['screen_info'], type='text') 106 | screen_info_dict = {"role": "user", "content": [screen_info_block]} 107 | messages.append(screen_info_dict) 108 | tools_use_needed = actor(messages=messages) 109 | 110 | for message, tool_result_content in executor(tools_use_needed, messages): 111 | yield message 112 | 113 | if not tool_result_content: 114 | return messages 115 | 116 | messages.append({"content": tool_result_content, "role": "user"}) 117 | 118 | elif model in set(["omniparser + gpt-4o", "omniparser + o1", "omniparser + o3-mini", "omniparser + R1", "omniparser + qwen2.5vl", "omniparser + gpt-4o-orchestrated", "omniparser + o1-orchestrated", "omniparser + o3-mini-orchestrated", "omniparser + R1-orchestrated", "omniparser + qwen2.5vl-orchestrated"]): 119 | while True: 120 | parsed_screen = omniparser_client() 121 | tools_use_needed, vlm_response_json = actor(messages=messages, parsed_screen=parsed_screen) 122 | 123 | for message, tool_result_content in executor(tools_use_needed, messages): 124 | yield message 125 | 126 | if not tool_result_content: 127 | return messages -------------------------------------------------------------------------------- /omnitool/gradio/tools/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import ToolResult 2 | from .collection import ToolCollection 3 | from .computer import ComputerTool 4 | from .screen_capture import get_screenshot 5 | 6 | __ALL__ = [ 7 | ComputerTool, 8 | ToolCollection, 9 | ToolResult, 10 | get_screenshot, 11 | ] 12 | -------------------------------------------------------------------------------- /omnitool/gradio/tools/base.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | from dataclasses import dataclass, fields, replace 3 | from typing import Any 4 | 5 | from anthropic.types.beta import BetaToolUnionParam 6 | 7 | 8 | class BaseAnthropicTool(metaclass=ABCMeta): 9 | """Abstract base class for Anthropic-defined tools.""" 10 | 11 | @abstractmethod 12 | def __call__(self, **kwargs) -> Any: 13 | """Executes the tool with the given arguments.""" 14 | ... 15 | 16 | @abstractmethod 17 | def to_params( 18 | self, 19 | ) -> BetaToolUnionParam: 20 | raise NotImplementedError 21 | 22 | 23 | @dataclass(kw_only=True, frozen=True) 24 | class ToolResult: 25 | """Represents the result of a tool execution.""" 26 | 27 | output: str | None = None 28 | error: str | None = None 29 | base64_image: str | None = None 30 | system: str | None = None 31 | 32 | def __bool__(self): 33 | return any(getattr(self, field.name) for field in fields(self)) 34 | 35 | def __add__(self, other: "ToolResult"): 36 | def combine_fields( 37 | field: str | None, other_field: str | None, concatenate: bool = True 38 | ): 39 | if field and other_field: 40 | if concatenate: 41 | return field + other_field 42 | raise ValueError("Cannot combine tool results") 43 | return field or other_field 44 | 45 | return ToolResult( 46 | output=combine_fields(self.output, other.output), 47 | error=combine_fields(self.error, other.error), 48 | base64_image=combine_fields(self.base64_image, other.base64_image, False), 49 | system=combine_fields(self.system, other.system), 50 | ) 51 | 52 | def replace(self, **kwargs): 53 | """Returns a new ToolResult with the given fields replaced.""" 54 | return replace(self, **kwargs) 55 | 56 | 57 | class ToolFailure(ToolResult): 58 | """A ToolResult that represents a failure.""" 59 | 60 | 61 | class ToolError(Exception): 62 | """Raised when a tool encounters an error.""" 63 | 64 | def __init__(self, message): 65 | self.message = message 66 | -------------------------------------------------------------------------------- /omnitool/gradio/tools/collection.py: -------------------------------------------------------------------------------- 1 | """Collection classes for managing multiple tools.""" 2 | 3 | from typing import Any 4 | 5 | from anthropic.types.beta import BetaToolUnionParam 6 | 7 | from .base import ( 8 | BaseAnthropicTool, 9 | ToolError, 10 | ToolFailure, 11 | ToolResult, 12 | ) 13 | 14 | 15 | class ToolCollection: 16 | """A collection of anthropic-defined tools.""" 17 | 18 | def __init__(self, *tools: BaseAnthropicTool): 19 | self.tools = tools 20 | self.tool_map = {tool.to_params()["name"]: tool for tool in tools} 21 | 22 | def to_params( 23 | self, 24 | ) -> list[BetaToolUnionParam]: 25 | return [tool.to_params() for tool in self.tools] 26 | 27 | async def run(self, *, name: str, tool_input: dict[str, Any]) -> ToolResult: 28 | tool = self.tool_map.get(name) 29 | if not tool: 30 | return ToolFailure(error=f"Tool {name} is invalid") 31 | try: 32 | return await tool(**tool_input) 33 | except ToolError as e: 34 | return ToolFailure(error=e.message) 35 | -------------------------------------------------------------------------------- /omnitool/gradio/tools/computer.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import time 3 | from enum import StrEnum 4 | from typing import Literal, TypedDict 5 | 6 | from PIL import Image 7 | 8 | from anthropic.types.beta import BetaToolComputerUse20241022Param 9 | 10 | from .base import BaseAnthropicTool, ToolError, ToolResult 11 | from .screen_capture import get_screenshot 12 | import requests 13 | import re 14 | 15 | OUTPUT_DIR = "./tmp/outputs" 16 | 17 | TYPING_DELAY_MS = 12 18 | TYPING_GROUP_SIZE = 50 19 | 20 | Action = Literal[ 21 | "key", 22 | "type", 23 | "mouse_move", 24 | "left_click", 25 | "left_click_drag", 26 | "right_click", 27 | "middle_click", 28 | "double_click", 29 | "screenshot", 30 | "cursor_position", 31 | "hover", 32 | "wait" 33 | ] 34 | 35 | 36 | class Resolution(TypedDict): 37 | width: int 38 | height: int 39 | 40 | 41 | MAX_SCALING_TARGETS: dict[str, Resolution] = { 42 | "XGA": Resolution(width=1024, height=768), # 4:3 43 | "WXGA": Resolution(width=1280, height=800), # 16:10 44 | "FWXGA": Resolution(width=1366, height=768), # ~16:9 45 | } 46 | 47 | 48 | class ScalingSource(StrEnum): 49 | COMPUTER = "computer" 50 | API = "api" 51 | 52 | 53 | class ComputerToolOptions(TypedDict): 54 | display_height_px: int 55 | display_width_px: int 56 | display_number: int | None 57 | 58 | 59 | def chunks(s: str, chunk_size: int) -> list[str]: 60 | return [s[i : i + chunk_size] for i in range(0, len(s), chunk_size)] 61 | 62 | class ComputerTool(BaseAnthropicTool): 63 | """ 64 | A tool that allows the agent to interact with the screen, keyboard, and mouse of the current computer. 65 | Adapted for Windows using 'pyautogui'. 66 | """ 67 | 68 | name: Literal["computer"] = "computer" 69 | api_type: Literal["computer_20241022"] = "computer_20241022" 70 | width: int 71 | height: int 72 | display_num: int | None 73 | 74 | _screenshot_delay = 2.0 75 | _scaling_enabled = True 76 | 77 | @property 78 | def options(self) -> ComputerToolOptions: 79 | width, height = self.scale_coordinates( 80 | ScalingSource.COMPUTER, self.width, self.height 81 | ) 82 | return { 83 | "display_width_px": width, 84 | "display_height_px": height, 85 | "display_number": self.display_num, 86 | } 87 | 88 | def to_params(self) -> BetaToolComputerUse20241022Param: 89 | return {"name": self.name, "type": self.api_type, **self.options} 90 | 91 | def __init__(self, is_scaling: bool = False): 92 | super().__init__() 93 | 94 | # Get screen width and height using Windows command 95 | self.display_num = None 96 | self.offset_x = 0 97 | self.offset_y = 0 98 | self.is_scaling = is_scaling 99 | self.width, self.height = self.get_screen_size() 100 | print(f"screen size: {self.width}, {self.height}") 101 | 102 | self.key_conversion = {"Page_Down": "pagedown", 103 | "Page_Up": "pageup", 104 | "Super_L": "win", 105 | "Escape": "esc"} 106 | 107 | 108 | async def __call__( 109 | self, 110 | *, 111 | action: Action, 112 | text: str | None = None, 113 | coordinate: tuple[int, int] | None = None, 114 | **kwargs, 115 | ): 116 | print(f"action: {action}, text: {text}, coordinate: {coordinate}, is_scaling: {self.is_scaling}") 117 | if action in ("mouse_move", "left_click_drag"): 118 | if coordinate is None: 119 | raise ToolError(f"coordinate is required for {action}") 120 | if text is not None: 121 | raise ToolError(f"text is not accepted for {action}") 122 | if not isinstance(coordinate, (list, tuple)) or len(coordinate) != 2: 123 | raise ToolError(f"{coordinate} must be a tuple of length 2") 124 | # if not all(isinstance(i, int) and i >= 0 for i in coordinate): 125 | if not all(isinstance(i, int) for i in coordinate): 126 | raise ToolError(f"{coordinate} must be a tuple of non-negative ints") 127 | 128 | if self.is_scaling: 129 | x, y = self.scale_coordinates( 130 | ScalingSource.API, coordinate[0], coordinate[1] 131 | ) 132 | else: 133 | x, y = coordinate 134 | 135 | # print(f"scaled_coordinates: {x}, {y}") 136 | # print(f"offset: {self.offset_x}, {self.offset_y}") 137 | 138 | # x += self.offset_x # TODO - check if this is needed 139 | # y += self.offset_y 140 | 141 | print(f"mouse move to {x}, {y}") 142 | 143 | if action == "mouse_move": 144 | self.send_to_vm(f"pyautogui.moveTo({x}, {y})") 145 | return ToolResult(output=f"Moved mouse to ({x}, {y})") 146 | elif action == "left_click_drag": 147 | current_x, current_y = self.send_to_vm("pyautogui.position()") 148 | self.send_to_vm(f"pyautogui.dragTo({x}, {y}, duration=0.5)") 149 | return ToolResult(output=f"Dragged mouse from ({current_x}, {current_y}) to ({x}, {y})") 150 | 151 | if action in ("key", "type"): 152 | if text is None: 153 | raise ToolError(f"text is required for {action}") 154 | if coordinate is not None: 155 | raise ToolError(f"coordinate is not accepted for {action}") 156 | if not isinstance(text, str): 157 | raise ToolError(output=f"{text} must be a string") 158 | 159 | if action == "key": 160 | # Handle key combinations 161 | keys = text.split('+') 162 | for key in keys: 163 | key = self.key_conversion.get(key.strip(), key.strip()) 164 | key = key.lower() 165 | self.send_to_vm(f"pyautogui.keyDown('{key}')") # Press down each key 166 | for key in reversed(keys): 167 | key = self.key_conversion.get(key.strip(), key.strip()) 168 | key = key.lower() 169 | self.send_to_vm(f"pyautogui.keyUp('{key}')") # Release each key in reverse order 170 | return ToolResult(output=f"Pressed keys: {text}") 171 | 172 | elif action == "type": 173 | # default click before type TODO: check if this is needed 174 | self.send_to_vm("pyautogui.click()") 175 | self.send_to_vm(f"pyautogui.typewrite('{text}', interval={TYPING_DELAY_MS / 1000})") 176 | self.send_to_vm("pyautogui.press('enter')") 177 | screenshot_base64 = (await self.screenshot()).base64_image 178 | return ToolResult(output=text, base64_image=screenshot_base64) 179 | 180 | if action in ( 181 | "left_click", 182 | "right_click", 183 | "double_click", 184 | "middle_click", 185 | "screenshot", 186 | "cursor_position", 187 | "left_press", 188 | ): 189 | if text is not None: 190 | raise ToolError(f"text is not accepted for {action}") 191 | if coordinate is not None: 192 | raise ToolError(f"coordinate is not accepted for {action}") 193 | 194 | if action == "screenshot": 195 | return await self.screenshot() 196 | elif action == "cursor_position": 197 | x, y = self.send_to_vm("pyautogui.position()") 198 | x, y = self.scale_coordinates(ScalingSource.COMPUTER, x, y) 199 | return ToolResult(output=f"X={x},Y={y}") 200 | else: 201 | if action == "left_click": 202 | self.send_to_vm("pyautogui.click()") 203 | elif action == "right_click": 204 | self.send_to_vm("pyautogui.rightClick()") 205 | elif action == "middle_click": 206 | self.send_to_vm("pyautogui.middleClick()") 207 | elif action == "double_click": 208 | self.send_to_vm("pyautogui.doubleClick()") 209 | elif action == "left_press": 210 | self.send_to_vm("pyautogui.mouseDown()") 211 | time.sleep(1) 212 | self.send_to_vm("pyautogui.mouseUp()") 213 | return ToolResult(output=f"Performed {action}") 214 | if action in ("scroll_up", "scroll_down"): 215 | if action == "scroll_up": 216 | self.send_to_vm("pyautogui.scroll(100)") 217 | elif action == "scroll_down": 218 | self.send_to_vm("pyautogui.scroll(-100)") 219 | return ToolResult(output=f"Performed {action}") 220 | if action == "hover": 221 | return ToolResult(output=f"Performed {action}") 222 | if action == "wait": 223 | time.sleep(1) 224 | return ToolResult(output=f"Performed {action}") 225 | raise ToolError(f"Invalid action: {action}") 226 | 227 | def send_to_vm(self, action: str): 228 | """ 229 | Executes a python command on the server. Only return tuple of x,y when action is "pyautogui.position()" 230 | """ 231 | prefix = "import pyautogui; pyautogui.FAILSAFE = False;" 232 | command_list = ["python", "-c", f"{prefix} {action}"] 233 | parse = action == "pyautogui.position()" 234 | if parse: 235 | command_list[-1] = f"{prefix} print({action})" 236 | 237 | try: 238 | print(f"sending to vm: {command_list}") 239 | response = requests.post( 240 | f"http://localhost:5000/execute", 241 | headers={'Content-Type': 'application/json'}, 242 | json={"command": command_list}, 243 | timeout=90 244 | ) 245 | time.sleep(0.7) # avoid async error as actions take time to complete 246 | print(f"action executed") 247 | if response.status_code != 200: 248 | raise ToolError(f"Failed to execute command. Status code: {response.status_code}") 249 | if parse: 250 | output = response.json()['output'].strip() 251 | match = re.search(r'Point\(x=(\d+),\s*y=(\d+)\)', output) 252 | if not match: 253 | raise ToolError(f"Could not parse coordinates from output: {output}") 254 | x, y = map(int, match.groups()) 255 | return x, y 256 | except requests.exceptions.RequestException as e: 257 | raise ToolError(f"An error occurred while trying to execute the command: {str(e)}") 258 | 259 | async def screenshot(self): 260 | if not hasattr(self, 'target_dimension'): 261 | screenshot = self.padding_image(screenshot) 262 | self.target_dimension = MAX_SCALING_TARGETS["WXGA"] 263 | width, height = self.target_dimension["width"], self.target_dimension["height"] 264 | screenshot, path = get_screenshot(resize=True, target_width=width, target_height=height) 265 | time.sleep(0.7) # avoid async error as actions take time to complete 266 | return ToolResult(base64_image=base64.b64encode(path.read_bytes()).decode()) 267 | 268 | def padding_image(self, screenshot): 269 | """Pad the screenshot to 16:10 aspect ratio, when the aspect ratio is not 16:10.""" 270 | _, height = screenshot.size 271 | new_width = height * 16 // 10 272 | 273 | padding_image = Image.new("RGB", (new_width, height), (255, 255, 255)) 274 | # padding to top left 275 | padding_image.paste(screenshot, (0, 0)) 276 | return padding_image 277 | 278 | def scale_coordinates(self, source: ScalingSource, x: int, y: int): 279 | """Scale coordinates to a target maximum resolution.""" 280 | if not self._scaling_enabled: 281 | return x, y 282 | ratio = self.width / self.height 283 | target_dimension = None 284 | 285 | for target_name, dimension in MAX_SCALING_TARGETS.items(): 286 | # allow some error in the aspect ratio - not ratios are exactly 16:9 287 | if abs(dimension["width"] / dimension["height"] - ratio) < 0.02: 288 | if dimension["width"] < self.width: 289 | target_dimension = dimension 290 | self.target_dimension = target_dimension 291 | # print(f"target_dimension: {target_dimension}") 292 | break 293 | 294 | if target_dimension is None: 295 | # TODO: currently we force the target to be WXGA (16:10), when it cannot find a match 296 | target_dimension = MAX_SCALING_TARGETS["WXGA"] 297 | self.target_dimension = MAX_SCALING_TARGETS["WXGA"] 298 | 299 | # should be less than 1 300 | x_scaling_factor = target_dimension["width"] / self.width 301 | y_scaling_factor = target_dimension["height"] / self.height 302 | if source == ScalingSource.API: 303 | if x > self.width or y > self.height: 304 | raise ToolError(f"Coordinates {x}, {y} are out of bounds") 305 | # scale up 306 | return round(x / x_scaling_factor), round(y / y_scaling_factor) 307 | # scale down 308 | return round(x * x_scaling_factor), round(y * y_scaling_factor) 309 | 310 | def get_screen_size(self): 311 | """Return width and height of the screen""" 312 | try: 313 | response = requests.post( 314 | f"http://localhost:5000/execute", 315 | headers={'Content-Type': 'application/json'}, 316 | json={"command": ["python", "-c", "import pyautogui; print(pyautogui.size())"]}, 317 | timeout=90 318 | ) 319 | if response.status_code != 200: 320 | raise ToolError(f"Failed to get screen size. Status code: {response.status_code}") 321 | 322 | output = response.json()['output'].strip() 323 | match = re.search(r'Size\(width=(\d+),\s*height=(\d+)\)', output) 324 | if not match: 325 | raise ToolError(f"Could not parse screen size from output: {output}") 326 | width, height = map(int, match.groups()) 327 | return width, height 328 | except requests.exceptions.RequestException as e: 329 | raise ToolError(f"An error occurred while trying to get screen size: {str(e)}") -------------------------------------------------------------------------------- /omnitool/gradio/tools/screen_capture.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from uuid import uuid4 3 | import requests 4 | from PIL import Image 5 | from .base import BaseAnthropicTool, ToolError 6 | from io import BytesIO 7 | 8 | OUTPUT_DIR = "./tmp/outputs" 9 | 10 | def get_screenshot(resize: bool = False, target_width: int = 1920, target_height: int = 1080): 11 | """Capture screenshot by requesting from HTTP endpoint - returns native resolution unless resized""" 12 | output_dir = Path(OUTPUT_DIR) 13 | output_dir.mkdir(parents=True, exist_ok=True) 14 | path = output_dir / f"screenshot_{uuid4().hex}.png" 15 | 16 | try: 17 | response = requests.get('http://localhost:5000/screenshot') 18 | if response.status_code != 200: 19 | raise ToolError(f"Failed to capture screenshot: HTTP {response.status_code}") 20 | 21 | # (1280, 800) 22 | screenshot = Image.open(BytesIO(response.content)) 23 | 24 | if resize and screenshot.size != (target_width, target_height): 25 | screenshot = screenshot.resize((target_width, target_height)) 26 | screenshot.save(path) 27 | return screenshot, path 28 | except Exception as e: 29 | raise ToolError(f"Failed to capture screenshot: {str(e)}") -------------------------------------------------------------------------------- /omnitool/omnibox/.gitignore: -------------------------------------------------------------------------------- 1 | vm/win11iso/custom.iso 2 | vm/win11storage 3 | vm/win11setup/setupscripts/firstboot_log.txt 4 | vm/win11setup/setupscripts/server/server.log 5 | -------------------------------------------------------------------------------- /omnitool/omnibox/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG VERSION_ARG="latest" 2 | FROM scratch AS build-amd64 3 | 4 | COPY --from=qemux/qemu-docker:6.08 / / 5 | 6 | ARG DEBCONF_NOWARNINGS="yes" 7 | ARG DEBIAN_FRONTEND="noninteractive" 8 | ARG DEBCONF_NONINTERACTIVE_SEEN="true" 9 | 10 | RUN set -eu && \ 11 | apt-get update && \ 12 | apt-get --no-install-recommends -y install \ 13 | bc \ 14 | jq \ 15 | curl \ 16 | 7zip \ 17 | wsdd \ 18 | samba \ 19 | xz-utils \ 20 | wimtools \ 21 | dos2unix \ 22 | cabextract \ 23 | genisoimage \ 24 | libxml2-utils \ 25 | libarchive-tools && \ 26 | apt-get clean && \ 27 | rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 28 | 29 | COPY --chmod=755 ./vm/buildcontainer /run/ 30 | RUN dos2unix /run/* 31 | 32 | COPY --chmod=755 ./vm/win11def /run/assets 33 | RUN dos2unix /run/assets/* 34 | 35 | ADD --chmod=755 https://raw.githubusercontent.com/christgau/wsdd/v0.8/src/wsdd.py /usr/sbin/wsdd 36 | ADD --chmod=664 https://github.com/qemus/virtiso-whql/releases/download/v1.9.43-0/virtio-win-1.9.43.tar.xz /drivers.txz 37 | 38 | FROM dockurr/windows-arm:${VERSION_ARG} AS build-arm64 39 | FROM build-${TARGETARCH} 40 | 41 | ARG VERSION_ARG="0.00" 42 | RUN echo "$VERSION_ARG" > /run/version 43 | 44 | EXPOSE 8006 3389 45 | 46 | ENV VERSION="win11e" 47 | 48 | ENTRYPOINT ["/usr/bin/tini", "-s", "/run/entry.sh"] -------------------------------------------------------------------------------- /omnitool/omnibox/compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | windows: 3 | image: windows-local 4 | container_name: omni-windows 5 | privileged: true 6 | environment: 7 | RAM_SIZE: "8G" 8 | CPU_CORES: "4" 9 | DISK_SIZE: "20G" 10 | devices: 11 | - /dev/kvm 12 | - /dev/net/tun 13 | cap_add: 14 | - NET_ADMIN 15 | ports: 16 | - 8006:8006 # Web Viewer access 17 | - 5000:5000 # Computer control server 18 | volumes: 19 | - ./vm/win11iso/custom.iso:/custom.iso 20 | - ./vm/win11setup/firstboot:/oem 21 | - ./vm/win11setup/setupscripts:/data 22 | - ./vm/win11storage:/storage 23 | -------------------------------------------------------------------------------- /omnitool/omnibox/scripts/manage_vm.ps1: -------------------------------------------------------------------------------- 1 | function Create-VM { 2 | if (-not (docker images windows-local -q)) { 3 | Write-Host "Image not found locally. Building..." 4 | docker build -t windows-local .. 5 | } else { 6 | Write-Host "Image found locally. Skipping build." 7 | } 8 | 9 | docker compose -f ../compose.yml up -d 10 | 11 | while ($true) { 12 | try { 13 | $response = Invoke-WebRequest -Uri "http://localhost:5000/probe" -Method GET -UseBasicParsing 14 | if ($response.StatusCode -eq 200) { 15 | break 16 | } 17 | } catch { 18 | Write-Host "Waiting for a response from the computer control server. When first building the VM storage folder this can take a while..." 19 | Start-Sleep -Seconds 5 20 | } 21 | } 22 | 23 | Write-Host "VM + server is up and running!" 24 | } 25 | 26 | function Start-LocalVM { 27 | Write-Host "Starting VM..." 28 | docker compose -f ../compose.yml start 29 | while ($true) { 30 | try { 31 | $response = Invoke-WebRequest -Uri "http://localhost:5000/probe" -Method GET -UseBasicParsing 32 | if ($response.StatusCode -eq 200) { 33 | break 34 | } 35 | } catch { 36 | Write-Host "Waiting for a response from the computer control server" 37 | Start-Sleep -Seconds 5 38 | } 39 | } 40 | Write-Host "VM started" 41 | } 42 | 43 | function Stop-LocalVM { 44 | Write-Host "Stopping VM..." 45 | docker compose -f ../compose.yml stop 46 | Write-Host "VM stopped" 47 | } 48 | 49 | function Remove-VM { 50 | Write-Host "Removing VM and associated containers..." 51 | docker compose -f ../compose.yml down 52 | Write-Host "VM removed" 53 | } 54 | 55 | if (-not $args[0]) { 56 | Write-Host "Usage: $($MyInvocation.MyCommand.Name) [create|start|stop|delete]" 57 | exit 1 58 | } 59 | 60 | switch ($args[0]) { 61 | "create" { Create-VM } 62 | "start" { Start-LocalVM } 63 | "stop" { Stop-LocalVM } 64 | "delete" { Remove-VM } 65 | default { 66 | Write-Host "Invalid option: $($args[0])" 67 | Write-Host "Usage: $($MyInvocation.MyCommand.Name) [create|start|stop|delete]" 68 | exit 1 69 | } 70 | } -------------------------------------------------------------------------------- /omnitool/omnibox/scripts/manage_vm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | create_vm() { 4 | if ! docker images windows-local -q | grep -q .; then 5 | echo "Image not found locally. Building..." 6 | docker build -t windows-local .. 7 | else 8 | echo "Image found locally. Skipping build." 9 | fi 10 | 11 | docker compose -f ../compose.yml up -d 12 | 13 | # Wait for the VM to start up 14 | while true; do 15 | response=$(curl --write-out '%{http_code}' --silent --output /dev/null localhost:5000/probe) 16 | if [ $response -eq 200 ]; then 17 | break 18 | fi 19 | echo "Waiting for a response from the computer control server. When first building the VM storage folder this can take a while..." 20 | sleep 5 21 | done 22 | 23 | echo "VM + server is up and running!" 24 | } 25 | 26 | start_vm() { 27 | echo "Starting VM..." 28 | docker compose -f ../compose.yml start 29 | while true; do 30 | response=$(curl --write-out '%{http_code}' --silent --output /dev/null localhost:5000/probe) 31 | if [ $response -eq 200 ]; then 32 | break 33 | fi 34 | echo "Waiting for a response from the computer control server" 35 | sleep 5 36 | done 37 | echo "VM started" 38 | } 39 | 40 | stop_vm() { 41 | echo "Stopping VM..." 42 | docker compose -f ../compose.yml stop 43 | echo "VM stopped" 44 | } 45 | 46 | delete_vm() { 47 | echo "Removing VM and associated containers..." 48 | docker compose -f ../compose.yml down 49 | echo "VM removed" 50 | } 51 | 52 | # Check if control parameter is provided 53 | if [ -z "$1" ]; then 54 | echo "Usage: $0 [create|start|stop|delete]" 55 | exit 1 56 | fi 57 | 58 | # Execute the appropriate function based on the control parameter 59 | case "$1" in 60 | "create") 61 | create_vm 62 | ;; 63 | "start") 64 | start_vm 65 | ;; 66 | "stop") 67 | stop_vm 68 | ;; 69 | "delete") 70 | delete_vm 71 | ;; 72 | *) 73 | echo "Invalid option: $1" 74 | echo "Usage: $0 [create|start|stop|delete]" 75 | exit 1 76 | ;; 77 | esac -------------------------------------------------------------------------------- /omnitool/omnibox/vm/buildcontainer/define.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -Eeuo pipefail 3 | 4 | : "${WIDTH:=""}" 5 | : "${HEIGHT:=""}" 6 | : "${VERIFY:=""}" 7 | : "${REGION:=""}" 8 | : "${MANUAL:=""}" 9 | : "${REMOVE:=""}" 10 | : "${VERSION:=""}" 11 | : "${DETECTED:=""}" 12 | : "${KEYBOARD:=""}" 13 | : "${LANGUAGE:=""}" 14 | : "${USERNAME:=""}" 15 | : "${PASSWORD:=""}" 16 | 17 | MIRRORS=4 18 | PLATFORM="x64" 19 | 20 | parseVersion() { 21 | 22 | if [[ "${VERSION}" == \"*\" || "${VERSION}" == \'*\' ]]; then 23 | VERSION="${VERSION:1:-1}" 24 | fi 25 | 26 | [ -z "$VERSION" ] && VERSION="win11" 27 | 28 | case "${VERSION,,}" in 29 | "11" | "11p" | "win11" | "pro11" | "win11p" | "windows11" | "windows 11" ) 30 | VERSION="win11x64" 31 | ;; 32 | "11e" | "win11e" | "windows11e" | "windows 11e" | "win11x64-enterprise-eval" ) 33 | VERSION="win11x64-enterprise-eval" 34 | ;; 35 | esac 36 | 37 | return 0 38 | } 39 | 40 | getLanguage() { 41 | 42 | local id="$1" 43 | local ret="$2" 44 | local lang="" 45 | local desc="" 46 | local culture="" 47 | 48 | case "${id,,}" in 49 | "ar" | "ar-"* ) 50 | lang="Arabic" 51 | desc="$lang" 52 | culture="ar-SA" ;; 53 | "bg" | "bg-"* ) 54 | lang="Bulgarian" 55 | desc="$lang" 56 | culture="bg-BG" ;; 57 | "cs" | "cs-"* | "cz" | "cz-"* ) 58 | lang="Czech" 59 | desc="$lang" 60 | culture="cs-CZ" ;; 61 | "da" | "da-"* | "dk" | "dk-"* ) 62 | lang="Danish" 63 | desc="$lang" 64 | culture="da-DK" ;; 65 | "de" | "de-"* ) 66 | lang="German" 67 | desc="$lang" 68 | culture="de-DE" ;; 69 | "el" | "el-"* | "gr" | "gr-"* ) 70 | lang="Greek" 71 | desc="$lang" 72 | culture="el-GR" ;; 73 | "gb" | "en-gb" ) 74 | lang="English International" 75 | desc="English" 76 | culture="en-GB" ;; 77 | "en" | "en-"* ) 78 | lang="English" 79 | desc="English" 80 | culture="en-US" ;; 81 | "mx" | "es-mx" ) 82 | lang="Spanish (Mexico)" 83 | desc="Spanish" 84 | culture="es-MX" ;; 85 | "es" | "es-"* ) 86 | lang="Spanish" 87 | desc="$lang" 88 | culture="es-ES" ;; 89 | "et" | "et-"* ) 90 | lang="Estonian" 91 | desc="$lang" 92 | culture="et-EE" ;; 93 | "fi" | "fi-"* ) 94 | lang="Finnish" 95 | desc="$lang" 96 | culture="fi-FI" ;; 97 | "ca" | "fr-ca" ) 98 | lang="French Canadian" 99 | desc="French" 100 | culture="fr-CA" ;; 101 | "fr" | "fr-"* ) 102 | lang="French" 103 | desc="$lang" 104 | culture="fr-FR" ;; 105 | "he" | "he-"* | "il" | "il-"* ) 106 | lang="Hebrew" 107 | desc="$lang" 108 | culture="he-IL" ;; 109 | "hr" | "hr-"* | "cr" | "cr-"* ) 110 | lang="Croatian" 111 | desc="$lang" 112 | culture="hr-HR" ;; 113 | "hu" | "hu-"* ) 114 | lang="Hungarian" 115 | desc="$lang" 116 | culture="hu-HU" ;; 117 | "it" | "it-"* ) 118 | lang="Italian" 119 | desc="$lang" 120 | culture="it-IT" ;; 121 | "ja" | "ja-"* | "jp" | "jp-"* ) 122 | lang="Japanese" 123 | desc="$lang" 124 | culture="ja-JP" ;; 125 | "ko" | "ko-"* | "kr" | "kr-"* ) 126 | lang="Korean" 127 | desc="$lang" 128 | culture="ko-KR" ;; 129 | "lt" | "lt-"* ) 130 | lang="Lithuanian" 131 | desc="$lang" 132 | culture="lv-LV" ;; 133 | "lv" | "lv-"* ) 134 | lang="Latvian" 135 | desc="$lang" 136 | culture="lt-LT" ;; 137 | "nb" | "nb-"* |"nn" | "nn-"* | "no" | "no-"* ) 138 | lang="Norwegian" 139 | desc="$lang" 140 | culture="nb-NO" ;; 141 | "nl" | "nl-"* ) 142 | lang="Dutch" 143 | desc="$lang" 144 | culture="nl-NL" ;; 145 | "pl" | "pl-"* ) 146 | lang="Polish" 147 | desc="$lang" 148 | culture="pl-PL" ;; 149 | "br" | "pt-br" ) 150 | lang="Brazilian Portuguese" 151 | desc="Portuguese" 152 | culture="pt-BR" ;; 153 | "pt" | "pt-"* ) 154 | lang="Portuguese" 155 | desc="$lang" 156 | culture="pt-BR" ;; 157 | "ro" | "ro-"* ) 158 | lang="Romanian" 159 | desc="$lang" 160 | culture="ro-RO" ;; 161 | "ru" | "ru-"* ) 162 | lang="Russian" 163 | desc="$lang" 164 | culture="ru-RU" ;; 165 | "sk" | "sk-"* ) 166 | lang="Slovak" 167 | desc="$lang" 168 | culture="sk-SK" ;; 169 | "sl" | "sl-"* | "si" | "si-"* ) 170 | lang="Slovenian" 171 | desc="$lang" 172 | culture="sl-SI" ;; 173 | "sr" | "sr-"* ) 174 | lang="Serbian Latin" 175 | desc="Serbian" 176 | culture="sr-Latn-RS" ;; 177 | "sv" | "sv-"* | "se" | "se-"* ) 178 | lang="Swedish" 179 | desc="$lang" 180 | culture="sv-SE" ;; 181 | "th" | "th-"* ) 182 | lang="Thai" 183 | desc="$lang" 184 | culture="th-TH" ;; 185 | "tr" | "tr-"* ) 186 | lang="Turkish" 187 | desc="$lang" 188 | culture="tr-TR" ;; 189 | "ua" | "ua-"* | "uk" | "uk-"* ) 190 | lang="Ukrainian" 191 | desc="$lang" 192 | culture="uk-UA" ;; 193 | "hk" | "zh-hk" | "cn-hk" ) 194 | lang="Chinese (Traditional)" 195 | desc="Chinese HK" 196 | culture="zh-TW" ;; 197 | "tw" | "zh-tw" | "cn-tw" ) 198 | lang="Chinese (Traditional)" 199 | desc="Chinese TW" 200 | culture="zh-TW" ;; 201 | "zh" | "zh-"* | "cn" | "cn-"* ) 202 | lang="Chinese (Simplified)" 203 | desc="Chinese" 204 | culture="zh-CN" ;; 205 | esac 206 | 207 | case "${ret,,}" in 208 | "desc" ) echo "$desc" ;; 209 | "name" ) echo "$lang" ;; 210 | "culture" ) echo "$culture" ;; 211 | *) echo "$desc";; 212 | esac 213 | 214 | return 0 215 | } 216 | 217 | parseLanguage() { 218 | 219 | REGION="${REGION//_/-/}" 220 | KEYBOARD="${KEYBOARD//_/-/}" 221 | LANGUAGE="${LANGUAGE//_/-/}" 222 | 223 | [ -z "$LANGUAGE" ] && LANGUAGE="en" 224 | 225 | case "${LANGUAGE,,}" in 226 | "arabic" | "arab" ) LANGUAGE="ar" ;; 227 | "bulgarian" | "bu" ) LANGUAGE="bg" ;; 228 | "chinese" | "cn" ) LANGUAGE="zh" ;; 229 | "croatian" | "cr" | "hrvatski" ) LANGUAGE="hr" ;; 230 | "czech" | "cz" | "cesky" ) LANGUAGE="cs" ;; 231 | "danish" | "dk" | "danske" ) LANGUAGE="da" ;; 232 | "dutch" | "nederlands" ) LANGUAGE="nl" ;; 233 | "english" | "gb" | "british" ) LANGUAGE="en" ;; 234 | "estonian" | "eesti" ) LANGUAGE="et" ;; 235 | "finnish" | "suomi" ) LANGUAGE="fi" ;; 236 | "french" | "français" | "francais" ) LANGUAGE="fr" ;; 237 | "german" | "deutsch" ) LANGUAGE="de" ;; 238 | "greek" | "gr" ) LANGUAGE="el" ;; 239 | "hebrew" | "il" ) LANGUAGE="he" ;; 240 | "hungarian" | "magyar" ) LANGUAGE="hu" ;; 241 | "italian" | "italiano" ) LANGUAGE="it" ;; 242 | "japanese" | "jp" ) LANGUAGE="ja" ;; 243 | "korean" | "kr" ) LANGUAGE="ko" ;; 244 | "latvian" | "latvijas" ) LANGUAGE="lv" ;; 245 | "lithuanian" | "lietuvos" ) LANGUAGE="lt" ;; 246 | "norwegian" | "no" | "nb" | "norsk" ) LANGUAGE="nn" ;; 247 | "polish" | "polski" ) LANGUAGE="pl" ;; 248 | "portuguese" | "pt" | "br" ) LANGUAGE="pt-br" ;; 249 | "português" | "portugues" ) LANGUAGE="pt-br" ;; 250 | "romanian" | "română" | "romana" ) LANGUAGE="ro" ;; 251 | "russian" | "ruski" ) LANGUAGE="ru" ;; 252 | "serbian" | "serbian latin" ) LANGUAGE="sr" ;; 253 | "slovak" | "slovenský" | "slovensky" ) LANGUAGE="sk" ;; 254 | "slovenian" | "si" | "slovenski" ) LANGUAGE="sl" ;; 255 | "spanish" | "espanol" | "español" ) LANGUAGE="es" ;; 256 | "swedish" | "se" | "svenska" ) LANGUAGE="sv" ;; 257 | "turkish" | "türk" | "turk" ) LANGUAGE="tr" ;; 258 | "thai" ) LANGUAGE="th" ;; 259 | "ukrainian" | "ua" ) LANGUAGE="uk" ;; 260 | esac 261 | 262 | local culture 263 | culture=$(getLanguage "$LANGUAGE" "culture") 264 | [ -n "$culture" ] && return 0 265 | 266 | error "Invalid LANGUAGE specified, value \"$LANGUAGE\" is not recognized!" 267 | return 1 268 | } 269 | 270 | printVersion() { 271 | 272 | local id="$1" 273 | local desc="$2" 274 | 275 | case "${id,,}" in 276 | "win11"* ) desc="Windows 11" ;; 277 | esac 278 | 279 | if [ -z "$desc" ]; then 280 | desc="Windows" 281 | [[ "${PLATFORM,,}" != "x64" ]] && desc+=" for ${PLATFORM}" 282 | fi 283 | 284 | echo "$desc" 285 | return 0 286 | } 287 | 288 | printEdition() { 289 | 290 | local id="$1" 291 | local desc="$2" 292 | local result="" 293 | local edition="" 294 | 295 | result=$(printVersion "$id" "x") 296 | [[ "$result" == "x" ]] && echo "$desc" && return 0 297 | 298 | case "${id,,}" in 299 | *"-enterprise" ) 300 | edition="Enterprise" 301 | ;; 302 | *"-enterprise-eval" ) 303 | edition="Enterprise (Evaluation)" 304 | ;; 305 | esac 306 | 307 | [ -n "$edition" ] && result+=" $edition" 308 | 309 | echo "$result" 310 | return 0 311 | } 312 | 313 | fromName() { 314 | 315 | local id="" 316 | local name="$1" 317 | local arch="$2" 318 | 319 | local add="" 320 | [[ "$arch" != "x64" ]] && add="$arch" 321 | 322 | case "${name,,}" in 323 | *"windows 11"* ) id="win11${arch}" ;; 324 | esac 325 | 326 | echo "$id" 327 | return 0 328 | } 329 | 330 | getVersion() { 331 | 332 | local id 333 | local name="$1" 334 | local arch="$2" 335 | 336 | id=$(fromName "$name" "$arch") 337 | 338 | case "${id,,}" in 339 | "win11"* ) 340 | case "${name,,}" in 341 | *" enterprise evaluation"* ) id="$id-enterprise-eval" ;; 342 | *" enterprise"* ) id="$id-enterprise" ;; 343 | esac 344 | ;; 345 | esac 346 | 347 | echo "$id" 348 | return 0 349 | } 350 | 351 | addFolder() { 352 | 353 | local src="$1" 354 | local folder="/oem" 355 | 356 | [ ! -d "$folder" ] && folder="/OEM" 357 | [ ! -d "$folder" ] && folder="$STORAGE/oem" 358 | [ ! -d "$folder" ] && folder="$STORAGE/OEM" 359 | [ ! -d "$folder" ] && return 0 360 | 361 | local msg="Adding OEM folder to image..." 362 | info "$msg" && html "$msg" 363 | 364 | local dest="$src/\$OEM\$/\$1/OEM" 365 | mkdir -p "$dest" || return 1 366 | cp -Lr "$folder/." "$dest" || return 1 367 | 368 | local file 369 | file=$(find "$dest" -maxdepth 1 -type f -iname install.bat | head -n 1) 370 | [ -f "$file" ] && unix2dos -q "$file" 371 | 372 | return 0 373 | } 374 | 375 | # migrateFiles() { 376 | 377 | # local base="$1" 378 | # local version="$2" 379 | # local file="" 380 | 381 | # [ -f "$base" ] && return 0 382 | 383 | # [[ "${version,,}" == "tiny10" ]] && file="tiny10_x64_23h2.iso" 384 | # [[ "${version,,}" == "tiny11" ]] && file="tiny11_2311_x64.iso" 385 | # [[ "${version,,}" == "core11" ]] && file="tiny11_core_x64_beta_1.iso" 386 | # [[ "${version,,}" == "winxpx86" ]] && file="en_windows_xp_professional_with_service_pack_3_x86_cd_x14-80428.iso" 387 | # [[ "${version,,}" == "winvistax64" ]] && file="en_windows_vista_sp2_x64_dvd_342267.iso" 388 | # [[ "${version,,}" == "win7x64" ]] && file="en_windows_7_enterprise_with_sp1_x64_dvd_u_677651.iso" 389 | 390 | # [ ! -f "$STORAGE/$file" ] && return 0 391 | # mv -f "$STORAGE/$file" "$base" || return 1 392 | 393 | # return 0 394 | # } 395 | 396 | migrateFiles() { 397 | 398 | local base="$1" 399 | local version="$2" 400 | local file="" 401 | 402 | [ -f "$base" ] && return 0 403 | 404 | [ ! -f "$STORAGE/$file" ] && return 0 405 | mv -f "$STORAGE/$file" "$base" || return 1 406 | 407 | return 0 408 | } 409 | 410 | return 0 411 | -------------------------------------------------------------------------------- /omnitool/omnibox/vm/buildcontainer/entry.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -Eeuo pipefail 3 | 4 | : "${BOOT_MODE:="windows"}" 5 | 6 | APP="OmniParser Windows" 7 | SUPPORT="https://github.com/microsoft/OmniParser" 8 | 9 | cd /run 10 | 11 | . reset.sh # Initialize system 12 | . define.sh # Define versions 13 | . install.sh # Run installation 14 | . disk.sh # Initialize disks 15 | . display.sh # Initialize graphics 16 | . network.sh # Initialize network 17 | . samba.sh # Configure samba 18 | . boot.sh # Configure boot 19 | . proc.sh # Initialize processor 20 | . power.sh # Configure shutdown 21 | . config.sh # Configure arguments 22 | 23 | trap - ERR 24 | 25 | version=$(qemu-system-x86_64 --version | head -n 1 | cut -d '(' -f 1 | awk '{ print $NF }') 26 | info "Booting ${APP}${BOOT_DESC} using QEMU v$version..." 27 | 28 | { qemu-system-x86_64 ${ARGS:+ $ARGS} >"$QEMU_OUT" 2>"$QEMU_LOG"; rc=$?; } || : 29 | (( rc != 0 )) && error "$(<"$QEMU_LOG")" && exit 15 30 | 31 | terminal 32 | ( sleep 30; boot ) & 33 | tail -fn +0 "$QEMU_LOG" 2>/dev/null & 34 | cat "$QEMU_TERM" 2> /dev/null | tee "$QEMU_PTY" & 35 | wait $! || : 36 | 37 | sleep 1 & wait $! 38 | [ ! -f "$QEMU_END" ] && finish 0 39 | -------------------------------------------------------------------------------- /omnitool/omnibox/vm/buildcontainer/power.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -Eeuo pipefail 3 | 4 | # Configure QEMU for graceful shutdown 5 | 6 | QEMU_TERM="" 7 | QEMU_PORT=7100 8 | QEMU_TIMEOUT=110 9 | QEMU_DIR="/run/shm" 10 | QEMU_PID="$QEMU_DIR/qemu.pid" 11 | QEMU_PTY="$QEMU_DIR/qemu.pty" 12 | QEMU_LOG="$QEMU_DIR/qemu.log" 13 | QEMU_OUT="$QEMU_DIR/qemu.out" 14 | QEMU_END="$QEMU_DIR/qemu.end" 15 | 16 | rm -f "$QEMU_DIR/qemu.*" 17 | touch "$QEMU_LOG" 18 | 19 | _trap() { 20 | func="$1" ; shift 21 | for sig ; do 22 | trap "$func $sig" "$sig" 23 | done 24 | } 25 | 26 | boot() { 27 | 28 | [ -f "$QEMU_END" ] && return 0 29 | 30 | if [ -s "$QEMU_PTY" ]; then 31 | if [ "$(stat -c%s "$QEMU_PTY")" -gt 7 ]; then 32 | local fail="" 33 | if [[ "${BOOT_MODE,,}" == "windows_legacy" ]]; then 34 | grep -Fq "No bootable device." "$QEMU_PTY" && fail="y" 35 | grep -Fq "BOOTMGR is missing" "$QEMU_PTY" && fail="y" 36 | fi 37 | if [ -z "$fail" ]; then 38 | info "Windows has started successfully. You can directly view the VM at http://localhost:8006/vnc.html?view_only=1&autoconnect=1&resize=scale. Wait until setup is complete before interacting manually." 39 | return 0 40 | fi 41 | fi 42 | fi 43 | 44 | error "Timeout while waiting for QEMU to boot the machine!" 45 | 46 | local pid 47 | pid=$(<"$QEMU_PID") 48 | { kill -15 "$pid" || true; } 2>/dev/null 49 | 50 | return 0 51 | } 52 | 53 | ready() { 54 | 55 | [ -f "$STORAGE/windows.boot" ] && return 0 56 | [ ! -s "$QEMU_PTY" ] && return 1 57 | 58 | if [[ "${BOOT_MODE,,}" == "windows_legacy" ]]; then 59 | local last 60 | local bios="Booting from Hard" 61 | last=$(grep "^Booting.*" "$QEMU_PTY" | tail -1) 62 | [[ "${last,,}" != "${bios,,}"* ]] && return 1 63 | grep -Fq "No bootable device." "$QEMU_PTY" && return 1 64 | grep -Fq "BOOTMGR is missing" "$QEMU_PTY" && return 1 65 | return 0 66 | fi 67 | 68 | local line="\"Windows Boot Manager\"" 69 | grep -Fq "$line" "$QEMU_PTY" && return 0 70 | 71 | return 1 72 | } 73 | 74 | finish() { 75 | 76 | local pid 77 | local reason=$1 78 | 79 | touch "$QEMU_END" 80 | 81 | if [ -s "$QEMU_PID" ]; then 82 | 83 | pid=$(<"$QEMU_PID") 84 | error "Forcefully terminating Windows, reason: $reason..." 85 | { kill -15 "$pid" || true; } 2>/dev/null 86 | 87 | while isAlive "$pid"; do 88 | sleep 1 89 | # Workaround for zombie pid 90 | [ ! -s "$QEMU_PID" ] && break 91 | done 92 | fi 93 | 94 | if [ ! -f "$STORAGE/windows.boot" ] && [ -f "$BOOT" ]; then 95 | # Remove CD-ROM ISO after install 96 | if ready; then 97 | touch "$STORAGE/windows.boot" 98 | if [[ "$REMOVE" != [Nn]* ]]; then 99 | rm -f "$BOOT" 2>/dev/null || true 100 | fi 101 | fi 102 | fi 103 | 104 | pid="/var/run/tpm.pid" 105 | [ -s "$pid" ] && pKill "$(<"$pid")" 106 | 107 | pid="/var/run/wsdd.pid" 108 | [ -s "$pid" ] && pKill "$(<"$pid")" 109 | 110 | fKill "smbd" 111 | 112 | closeNetwork 113 | 114 | sleep 0.5 115 | echo "❯ Shutdown completed!" 116 | 117 | exit "$reason" 118 | } 119 | 120 | terminal() { 121 | 122 | local dev="" 123 | 124 | if [ -s "$QEMU_OUT" ]; then 125 | 126 | local msg 127 | msg=$(<"$QEMU_OUT") 128 | 129 | if [ -n "$msg" ]; then 130 | 131 | if [[ "${msg,,}" != "char"* || "$msg" != *"serial0)" ]]; then 132 | echo "$msg" 133 | fi 134 | 135 | dev="${msg#*/dev/p}" 136 | dev="/dev/p${dev%% *}" 137 | 138 | fi 139 | fi 140 | 141 | if [ ! -c "$dev" ]; then 142 | dev=$(echo 'info chardev' | nc -q 1 -w 1 localhost "$QEMU_PORT" | tr -d '\000') 143 | dev="${dev#*serial0}" 144 | dev="${dev#*pty:}" 145 | dev="${dev%%$'\n'*}" 146 | dev="${dev%%$'\r'*}" 147 | fi 148 | 149 | if [ ! -c "$dev" ]; then 150 | error "Device '$dev' not found!" 151 | finish 34 && return 34 152 | fi 153 | 154 | QEMU_TERM="$dev" 155 | return 0 156 | } 157 | 158 | _graceful_shutdown() { 159 | 160 | local code=$? 161 | 162 | set +e 163 | 164 | if [ -f "$QEMU_END" ]; then 165 | info "Received $1 while already shutting down..." 166 | return 167 | fi 168 | 169 | touch "$QEMU_END" 170 | info "Received $1, sending ACPI shutdown signal..." 171 | 172 | if [ ! -s "$QEMU_PID" ]; then 173 | error "QEMU PID file does not exist?" 174 | finish "$code" && return "$code" 175 | fi 176 | 177 | local pid="" 178 | pid=$(<"$QEMU_PID") 179 | 180 | if ! isAlive "$pid"; then 181 | error "QEMU process does not exist?" 182 | finish "$code" && return "$code" 183 | fi 184 | 185 | if ! ready; then 186 | info "Cannot send ACPI signal during Windows setup, aborting..." 187 | finish "$code" && return "$code" 188 | fi 189 | 190 | # Send ACPI shutdown signal 191 | echo 'system_powerdown' | nc -q 1 -w 1 localhost "${QEMU_PORT}" > /dev/null 192 | 193 | local cnt=0 194 | while [ "$cnt" -lt "$QEMU_TIMEOUT" ]; do 195 | 196 | sleep 1 197 | cnt=$((cnt+1)) 198 | 199 | ! isAlive "$pid" && break 200 | # Workaround for zombie pid 201 | [ ! -s "$QEMU_PID" ] && break 202 | 203 | info "Waiting for Windows to shutdown... ($cnt/$QEMU_TIMEOUT)" 204 | 205 | # Send ACPI shutdown signal 206 | echo 'system_powerdown' | nc -q 1 -w 1 localhost "${QEMU_PORT}" > /dev/null 207 | 208 | done 209 | 210 | if [ "$cnt" -ge "$QEMU_TIMEOUT" ]; then 211 | error "Shutdown timeout reached, aborting..." 212 | fi 213 | 214 | finish "$code" && return "$code" 215 | } 216 | 217 | SERIAL="pty" 218 | MONITOR="telnet:localhost:$QEMU_PORT,server,nowait,nodelay" 219 | MONITOR+=" -daemonize -D $QEMU_LOG -pidfile $QEMU_PID" 220 | 221 | _trap _graceful_shutdown SIGTERM SIGHUP SIGINT SIGABRT SIGQUIT 222 | 223 | return 0 224 | -------------------------------------------------------------------------------- /omnitool/omnibox/vm/buildcontainer/samba.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -Eeuo pipefail 3 | 4 | : "${SAMBA:="Y"}" 5 | 6 | [[ "$SAMBA" == [Nn]* ]] && return 0 7 | [[ "$NETWORK" == [Nn]* ]] && return 0 8 | 9 | hostname="host.lan" 10 | interface="dockerbridge" 11 | 12 | if [[ "$DHCP" == [Yy1]* ]]; then 13 | hostname="$IP" 14 | interface="$VM_NET_DEV" 15 | fi 16 | 17 | addShare() { 18 | local dir="$1" 19 | local name="$2" 20 | local comment="$3" 21 | 22 | mkdir -p "$dir" || return 1 23 | 24 | if [ -z "$(ls -A "$dir")" ]; then 25 | 26 | chmod 777 "$dir" 27 | 28 | { echo "--------------------------------------------------------" 29 | echo " $APP" 30 | echo " For support visit $SUPPORT" 31 | echo "--------------------------------------------------------" 32 | echo "" 33 | echo "Using this folder you can share files with the host machine." 34 | echo "" 35 | echo "To change its location, include the following bind mount in your compose file:" 36 | echo "" 37 | echo " volumes:" 38 | echo " - \"/home/example:/${name,,}\"" 39 | echo "" 40 | echo "Or in your run command:" 41 | echo "" 42 | echo " -v \"/home/example:/${name,,}\"" 43 | echo "" 44 | echo "Replace the example path /home/example with the desired shared folder." 45 | echo "" 46 | } | unix2dos > "$dir/readme.txt" 47 | 48 | fi 49 | 50 | { echo "" 51 | echo "[$name]" 52 | echo " path = $dir" 53 | echo " comment = $comment" 54 | echo " writable = yes" 55 | echo " guest ok = yes" 56 | echo " guest only = yes" 57 | echo " force user = root" 58 | echo " force group = root" 59 | } >> "/etc/samba/smb.conf" 60 | 61 | return 0 62 | } 63 | 64 | { echo "[global]" 65 | echo " server string = Dockur" 66 | echo " netbios name = $hostname" 67 | echo " workgroup = WORKGROUP" 68 | echo " interfaces = $interface" 69 | echo " bind interfaces only = yes" 70 | echo " security = user" 71 | echo " guest account = nobody" 72 | echo " map to guest = Bad User" 73 | echo " server min protocol = NT1" 74 | echo "" 75 | echo " # disable printing services" 76 | echo " load printers = no" 77 | echo " printing = bsd" 78 | echo " printcap name = /dev/null" 79 | echo " disable spoolss = yes" 80 | } > "/etc/samba/smb.conf" 81 | 82 | share="/data" 83 | [ ! -d "$share" ] && [ -d "$STORAGE/data" ] && share="$STORAGE/data" 84 | [ ! -d "$share" ] && [ -d "/shared" ] && share="/shared" 85 | [ ! -d "$share" ] && [ -d "$STORAGE/shared" ] && share="$STORAGE/shared" 86 | 87 | addShare "$share" "Data" "Shared" || error "Failed to create shared folder!" 88 | 89 | [ -d "/data2" ] && addShare "/data2" "Data2" "Shared" 90 | [ -d "/data3" ] && addShare "/data3" "Data3" "Shared" 91 | 92 | if ! smbd; then 93 | error "Samba daemon failed to start!" 94 | smbd -i --debug-stdout || true 95 | fi 96 | 97 | if [[ "${BOOT_MODE:-}" == "windows_legacy" ]]; then 98 | # Enable NetBIOS on Windows 7 and lower 99 | if ! nmbd; then 100 | error "NetBIOS daemon failed to start!" 101 | nmbd -i --debug-stdout || true 102 | fi 103 | else 104 | # Enable Web Service Discovery on Vista and up 105 | wsdd -i "$interface" -p -n "$hostname" & 106 | echo "$!" > /var/run/wsdd.pid 107 | fi 108 | 109 | return 0 110 | -------------------------------------------------------------------------------- /omnitool/omnibox/vm/win11iso/README.md: -------------------------------------------------------------------------------- 1 | Add your Win11E setup.iso to this folder -------------------------------------------------------------------------------- /omnitool/omnibox/vm/win11setup/firstboot/install.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | SET ScriptFolder=\\host.lan\Data 4 | SET LogFile=%ScriptFolder%\firstboot_log.txt 5 | 6 | echo Running PowerShell script... > %LogFile% 7 | 8 | :: Check for PowerShell availability 9 | where powershell >> %LogFile% 2>&1 10 | if %ERRORLEVEL% neq 0 ( 11 | echo PowerShell is not available! >> %LogFile% 12 | echo PowerShell is not available! 13 | exit /b 1 14 | ) 15 | 16 | :: Add a 30-second delay 17 | echo Waiting for 30 seconds before continuing... >> %LogFile% 18 | timeout /t 30 /nobreak >> %LogFile% 2>&1 19 | 20 | :: Run PowerShell script with ExecutionPolicy Bypass and log errors 21 | echo Running setup.ps1... >> %LogFile% 22 | 23 | powershell -ExecutionPolicy Bypass -File "%ScriptFolder%\setup.ps1" >> %LogFile% 2>&1 24 | 25 | if %ERRORLEVEL% neq 0 ( 26 | echo An error occurred. See %LogFile% for details. 27 | ) else ( 28 | echo PowerShell script has completed successfully. 29 | ) 30 | 31 | echo PowerShell script has completed. -------------------------------------------------------------------------------- /omnitool/omnibox/vm/win11setup/setupscripts/on-logon.ps1: -------------------------------------------------------------------------------- 1 | $scriptFolder = "\\host.lan\Data" 2 | $pythonScriptFile = "$scriptFolder\server\main.py" 3 | $pythonServerPort = 5000 4 | 5 | # Start the flask computer use server 6 | Write-Host "Running the server on port $pythonServerPort" 7 | python $pythonScriptFile --port $pythonServerPort 8 | -------------------------------------------------------------------------------- /omnitool/omnibox/vm/win11setup/setupscripts/server/cursor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/omnitool/omnibox/vm/win11setup/setupscripts/server/cursor.png -------------------------------------------------------------------------------- /omnitool/omnibox/vm/win11setup/setupscripts/server/main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | import argparse 4 | import shlex 5 | import subprocess 6 | from flask import Flask, request, jsonify, send_file 7 | import threading 8 | import traceback 9 | import pyautogui 10 | from PIL import Image 11 | from io import BytesIO 12 | 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument("--log_file", help="log file path", type=str, 15 | default=os.path.join(os.path.dirname(__file__), "server.log")) 16 | parser.add_argument("--port", help="port", type=int, default=5000) 17 | args = parser.parse_args() 18 | 19 | logging.basicConfig(filename=args.log_file,level=logging.DEBUG, filemode='w' ) 20 | logger = logging.getLogger('werkzeug') 21 | 22 | app = Flask(__name__) 23 | 24 | computer_control_lock = threading.Lock() 25 | 26 | @app.route('/probe', methods=['GET']) 27 | def probe_endpoint(): 28 | return jsonify({"status": "Probe successful", "message": "Service is operational"}), 200 29 | 30 | @app.route('/execute', methods=['POST']) 31 | def execute_command(): 32 | # Only execute one command at a time 33 | with computer_control_lock: 34 | data = request.json 35 | # The 'command' key in the JSON request should contain the command to be executed. 36 | shell = data.get('shell', False) 37 | command = data.get('command', "" if shell else []) 38 | 39 | if isinstance(command, str) and not shell: 40 | command = shlex.split(command) 41 | 42 | # Expand user directory 43 | for i, arg in enumerate(command): 44 | if arg.startswith("~/"): 45 | command[i] = os.path.expanduser(arg) 46 | 47 | # Execute the command without any safety checks. 48 | try: 49 | result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, text=True, timeout=120) 50 | return jsonify({ 51 | 'status': 'success', 52 | 'output': result.stdout, 53 | 'error': result.stderr, 54 | 'returncode': result.returncode 55 | }) 56 | except Exception as e: 57 | logger.error("\n" + traceback.format_exc() + "\n") 58 | return jsonify({ 59 | 'status': 'error', 60 | 'message': str(e) 61 | }), 500 62 | 63 | @app.route('/screenshot', methods=['GET']) 64 | def capture_screen_with_cursor(): 65 | cursor_path = os.path.join(os.path.dirname(__file__), "cursor.png") 66 | screenshot = pyautogui.screenshot() 67 | cursor_x, cursor_y = pyautogui.position() 68 | cursor = Image.open(cursor_path) 69 | # make the cursor smaller 70 | cursor = cursor.resize((int(cursor.width / 1.5), int(cursor.height / 1.5))) 71 | screenshot.paste(cursor, (cursor_x, cursor_y), cursor) 72 | 73 | 74 | # Convert PIL Image to bytes and send 75 | img_io = BytesIO() 76 | screenshot.save(img_io, 'PNG') 77 | img_io.seek(0) 78 | return send_file(img_io, mimetype='image/png') 79 | 80 | if __name__ == '__main__': 81 | app.run(debug=True, host="0.0.0.0", port=args.port) -------------------------------------------------------------------------------- /omnitool/omnibox/vm/win11setup/setupscripts/server/requirements.txt: -------------------------------------------------------------------------------- 1 | flask 2 | PyAutoGUI -------------------------------------------------------------------------------- /omnitool/omnibox/vm/win11setup/setupscripts/setup-tools.psm1: -------------------------------------------------------------------------------- 1 | function Get-Tools { 2 | param( 3 | [string]$toolsConfigJson 4 | ) 5 | 6 | # Convert the JSON string to a PowerShell object 7 | $toolsList = $toolsConfigJson | ConvertFrom-Json 8 | 9 | return $toolsList 10 | } 11 | 12 | function Get-ToolDetails { 13 | param( 14 | $toolsList, 15 | [string]$toolName 16 | ) 17 | 18 | # Check if the program exists in the JSON data 19 | if ($toolsList.PSObject.Properties.Name -contains $toolName) { 20 | # Return the program details as a PowerShell object 21 | return $toolsList.$toolName 22 | } else { 23 | # Handle the case where the program is not found 24 | Write-Host "Program '$toolName' not found in the list." 25 | return $null 26 | } 27 | } 28 | 29 | function Invoke-DownloadFileFromAvailableMirrors { 30 | param ( 31 | [string[]]$mirrorUrls, 32 | [string]$outfile 33 | ) 34 | foreach ($url in $mirrorUrls) { 35 | try { 36 | $result = Invoke-DownloadFile -url $url -outfile $outfile 37 | if ($result -eq $true) { 38 | Write-Host "Downloaded using $url" 39 | return $true 40 | } 41 | } catch { 42 | Write-Host "Error downloading from $url. Please check and update the mirrors." 43 | } 44 | } 45 | 46 | Write-Host "Downloading from the provided mirrors failed. Please check and update the mirrors." 47 | return $false 48 | } 49 | 50 | function Invoke-DownloadFile { 51 | param ( 52 | [string]$url, 53 | [string]$outfile 54 | ) 55 | # Makes download faster by disabling progress bar 56 | $ProgressPreference = "SilentlyContinue" 57 | 58 | $retryCount = 0 59 | $maxRetries = 3 60 | $sleepSeconds = 2 61 | $maxSleepSeconds = 10 62 | $userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36" 63 | 64 | # Ensure directory exists 65 | $directory = Split-Path -Path $outfile -Parent 66 | if (-Not (Test-Path -Path $directory)) { 67 | Write-Host "Creating directory $directory..." 68 | New-Item -Path $directory -ItemType Directory -Force | Out-Null 69 | } 70 | 71 | while ($retryCount -lt $maxRetries) { 72 | try { 73 | Invoke-RestMethod -Uri $url -OutFile $outfile -Headers @{"User-Agent" = $userAgent} 74 | Write-Host "Download successful, file saved to: $outfile" 75 | break 76 | } catch { 77 | $retryCount++ 78 | Write-Host "Attempt $retryCount of $maxRetries failed. Error: $($_.Exception.Message)" 79 | Start-Sleep -Seconds $sleepSeconds 80 | $sleepSeconds = [Math]::Min($sleepSeconds * 2, $maxSleepSeconds) # Exponential backoff with a cap 81 | } 82 | } 83 | 84 | if ($retryCount -eq $maxRetries) { 85 | Write-Host "Failed to download the file after $maxRetries attempts." 86 | return $false 87 | } 88 | 89 | return $true 90 | } 91 | 92 | function Add-ToEnvPath { 93 | param ( 94 | [string]$NewPath 95 | ) 96 | 97 | # Get the current PATH environment variable 98 | $envPath = [Environment]::GetEnvironmentVariable("PATH", "Machine") 99 | 100 | # Append the new path to the existing PATH 101 | $newPath = "$envPath;$NewPath" 102 | 103 | # Set the updated PATH environment variable 104 | [Environment]::SetEnvironmentVariable("PATH", $newPath, "Machine") 105 | 106 | # Fetch updates from the shell 107 | $env:PATH += ";${newPath}" 108 | } 109 | 110 | function Register-LogonTask { 111 | param( 112 | 113 | [parameter(Mandatory = $true, ValueFromPipelineByPropertyName = $true, HelpMessage = "Name of the scheduled task")] 114 | [string] 115 | $TaskName, 116 | 117 | [parameter(Mandatory = $true, ValueFromPipelineByPropertyName = $true, HelpMessage = "Path to the .py script")] 118 | [string] 119 | $ScriptPath, 120 | 121 | [parameter(Mandatory = $false, ValueFromPipelineByPropertyName = $true, HelpMessage = "Arguments to the .py script")] 122 | [string] 123 | $Arguments = "", 124 | 125 | [parameter(Mandatory = $false, ValueFromPipelineByPropertyName = $true, HelpMessage = "Local Account username")] 126 | [string] 127 | $LocalUser, 128 | 129 | [parameter(Mandatory = $false, ValueFromPipelineByPropertyName = $true, HelpMessage = "Local Account password")] 130 | [string] 131 | $LocalPassword, 132 | 133 | [parameter(Mandatory = $false, ValueFromPipelineByPropertyName = $true, HelpMessage = "Whether to execute the command as SYSTEM")] 134 | [switch] 135 | $AsSystem = $false, 136 | 137 | [parameter(Mandatory = $false, ValueFromPipelineByPropertyName = $true, HelpMessage = "logging file")] 138 | [string] 139 | $LogFilePath 140 | ) 141 | 142 | $scriptDirectory = Split-Path $ScriptPath 143 | 144 | $taskActionArgument = "-ExecutionPolicy Bypass -windowstyle hidden -Command `"try { . '$ScriptPath' $Arguments } catch { Write `$_.Exception.Message | Out-File $($TaskName)_Log.txt } finally { } `"" 145 | $taskAction = New-ScheduledTaskAction -Execute "$PSHome\powershell.exe" -Argument $taskActionArgument -WorkingDirectory $scriptDirectory 146 | 147 | $params = @{ 148 | Force = $True 149 | Action = $taskAction 150 | RunLevel = "Highest" 151 | TaskName = $TaskName 152 | } 153 | 154 | $taskTrigger = New-ScheduledTaskTrigger -AtLogOn 155 | $params.Add("Trigger", $taskTrigger) 156 | 157 | if ($AsSystem) { 158 | $params.Add("User", "NT AUTHORITY\SYSTEM") 159 | } 160 | else { 161 | $params.Add("User", $LocalUser) 162 | if ($LocalPassword) { 163 | $params.Add("Password", $LocalPassword) 164 | } 165 | } 166 | 167 | Write-Host "Registering scheduled task '$TaskName' to run 'powershell.exe $taskActionArgument'..." 168 | Register-ScheduledTask @params 169 | } 170 | 171 | # Function to attempt pip install and handle failures 172 | function Install-PythonPackages { 173 | param ( 174 | [string]$Package = "", 175 | [string]$Arguments = "", 176 | [string]$RequirementsPath = "" 177 | ) 178 | $RetryCount = 3 179 | $currentAttempt = 0 180 | while ($currentAttempt -lt $RetryCount) { 181 | if (-not [string]::IsNullOrWhiteSpace($RequirementsPath)) { 182 | & python -m pip install --no-cache-dir -r $RequirementsPath $Arguments 183 | } else { 184 | & python -m pip install --no-cache-dir $Package $Arguments 185 | } 186 | if ($LASTEXITCODE -eq 0) { 187 | Write-Host "Installation successful." 188 | return 189 | } else { 190 | Write-Host "Attempt $($currentAttempt + 1) failed. Retrying..." 191 | Start-Sleep -Seconds 10 192 | $currentAttempt++ 193 | } 194 | } 195 | Write-Error "Failed to install after $RetryCount attempts." 196 | exit 197 | } -------------------------------------------------------------------------------- /omnitool/omnibox/vm/win11setup/setupscripts/setup.ps1: -------------------------------------------------------------------------------- 1 | $ErrorActionPreference = "Continue" # until downloading from mirrors is more stable 2 | 3 | # Section - General Setup 4 | $scriptFolder = "\\host.lan\Data" 5 | $toolsFolder = "C:\Users\$env:USERNAME\Tools" 6 | 7 | # Load the shared setup-tools module 8 | Import-Module (Join-Path $scriptFolder -ChildPath "setup-tools.psm1") 9 | 10 | # Check if profile exists 11 | if (-not (Test-Path $PROFILE)) { 12 | New-Item -ItemType File -Path $PROFILE -Force 13 | } 14 | 15 | # Create a folder where we store all the standalone executables 16 | if (-not (Test-Path $toolsFolder)) { 17 | New-Item -ItemType Directory -Path $toolsFolder -Force 18 | $envPath = [Environment]::GetEnvironmentVariable("PATH", "Machine") 19 | $newPath = "$envPath;$toolsFolder" 20 | [Environment]::SetEnvironmentVariable("PATH", $newPath, "Machine") 21 | } 22 | 23 | # Section - Tools Installation 24 | 25 | # Set TLS version to 1.2 or higher 26 | [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -bor [Net.SecurityProtocolType]::Tls13 27 | 28 | # Load the tools config json listing mirrors and aliases used for installing tools 29 | $toolsConfigJsonPath = Join-Path $scriptFolder -ChildPath "tools_config.json" 30 | $toolsConfigJson = Get-Content -Path $toolsConfigJsonPath -Raw 31 | $toolsList = Get-Tools -toolsConfigJson $toolsConfigJson 32 | 33 | ## - Python 34 | $pythonToolName = "Python" 35 | $userPythonPath = "$env:LOCALAPPDATA\Programs\Python" 36 | $pythonDetails = Get-ToolDetails -toolsList $toolsList -toolName $pythonToolName 37 | $pythonAlias = $pythonDetails.alias 38 | 39 | # Check for Python installation 40 | $pythonExecutablePath = Get-ChildItem -Path $userPythonPath -Filter python.exe -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1 -ExpandProperty FullName 41 | 42 | # Force to install Python 3.10 as the pre-installed version on Windows may not work sometimes 43 | Write-Host "Downloading Python $pythonVersion..." 44 | $pythonInstallerFilePath = "$env:TEMP\python_installer.exe" 45 | $downloadResult = Invoke-DownloadFileFromAvailableMirrors -mirrorUrls $pythonDetails.mirrors -outfile $pythonInstallerFilePath 46 | if (-not $downloadResult) { 47 | Write-Host "Failed to download Python. Please try again later or install manually." 48 | } else { 49 | Write-Host "Installing Python for current user..." 50 | Start-Process -FilePath $pythonInstallerFilePath -Args "/quiet InstallAllUsers=0 PrependPath=0" -NoNewWindow -Wait 51 | $pythonExecutablePath = "$userPythonPath\Python310\python.exe" 52 | $setAliasExpression = "Set-Alias -Name $pythonAlias -Value `"$pythonExecutablePath`"" 53 | Add-Content -Path $PROFILE -Value $setAliasExpression 54 | Invoke-Expression $setAliasExpression 55 | } 56 | 57 | ## - Git 58 | $gitToolName = "git" 59 | $gitToolDetails = Get-ToolDetails -toolsList $toolsList -toolName $gitToolName 60 | 61 | # Check for Git installation 62 | try { 63 | git --version | Out-Null 64 | Write-Host "Git is already installed." 65 | } catch { 66 | Write-Host "Git is not installed. Downloading and installing Git..." 67 | $gitInstallerFilePath = "$env:TEMP\git_installer.exe" 68 | $downloadResult = Invoke-DownloadFileFromAvailableMirrors -mirrorUrls $gitToolDetails.mirrors -outfile $gitInstallerFilePath 69 | if (-not $downloadResult) { 70 | Write-Host "Failed to download Git. Please try again later or install manually." 71 | } else { 72 | Start-Process -FilePath $gitInstallerFilePath -Args "/VERYSILENT /NORESTART /NOCANCEL /SP-" -Wait 73 | Add-ToEnvPath -NewPath "C:\Program Files\Git\bin" 74 | 75 | Write-Host "Git has been installed." 76 | } 77 | } 78 | 79 | # - 7zip 80 | $7ZipToolName = "7zip" 81 | $7ZipToolDetails = Get-ToolDetails -toolsList $toolsList -toolName $7ZipToolName 82 | Write-Host "$7ZipToolDetails" 83 | 84 | if (Get-Command 7z -ErrorAction SilentlyContinue) { 85 | Write-Host "7-Zip is already installed." 86 | } 87 | else { 88 | Write-Host "Installing 7-Zip..." 89 | 90 | $7ZipInstallerFilePath = "$env:TEMP\7_zip.exe" 91 | Write-Host "$($7ZipToolDetails.mirrors)" 92 | $downloadResult = Invoke-DownloadFileFromAvailableMirrors -mirrorUrls $7ZipToolDetails.mirrors -outfile $7ZipInstallerFilePath 93 | if (-not $downloadResult) { 94 | Write-Host "Failed to download 7-Zip. Please try again later or install manually." 95 | } else { 96 | Start-Process -FilePath $7ZipInstallerFilePath -Args "/S" -Verb RunAs -Wait 97 | Remove-Item $7ZipInstallerFilePath 98 | 99 | # add 7z to PATH 100 | Add-ToEnvPath -NewPath "${env:ProgramFiles}\7-Zip" 101 | } 102 | } 103 | 104 | # - ffpmeg 105 | $ffpmegToolName = "ffmpeg" 106 | $ffpmegToolDetails = Get-ToolDetails -toolsList $toolsList -toolName $ffpmegToolName 107 | 108 | if (Get-Command ffmpeg -ErrorAction SilentlyContinue) { 109 | Write-Host "ffmpeg is already installed." 110 | } else { 111 | Write-Host "ffmpeg is not installed. Installing it." 112 | $ffpmegInstallerFilePath = "C:\ffmpeg.7z" 113 | $downloadResult = Invoke-DownloadFileFromAvailableMirrors -mirrorUrls $ffpmegToolDetails.mirrors -outfile $ffpmegInstallerFilePath 114 | if (-not $downloadResult) { 115 | Write-Host "Failed to download ffmpeg. Please try again later or install manually." 116 | } else { 117 | Write-Host "Extracting $ffpmegInstallerFilePath..." 118 | 7z x -y -o"C:\" "C:\ffmpeg.7z" 119 | 120 | $ffmpegFolder = Get-ChildItem -Path "C:\" -Filter "ffmpeg-*" -Directory 121 | $ffmpegFolder = -join ("C:\", $ffmpegFolder) 122 | #remove ffmpeg folder if exists 123 | if (Test-Path "C:\ffmpeg") { 124 | Remove-Item -Path "C:\ffmpeg" -Recurse -Force 125 | } 126 | Rename-Item -Path "$ffmpegFolder" -NewName "ffmpeg" 127 | 128 | Write-Host "Adding ffmpeg to PATH..." 129 | Add-ToEnvPath -NewPath "C:\ffmpeg\bin" 130 | 131 | Write-Host "ffmpeg is installed" 132 | } 133 | } 134 | 135 | # Disable Edge Auto Updates 136 | Stop-Process -Name "MicrosoftEdgeUpdate" -Force -ErrorAction SilentlyContinue 137 | $edgeUpdatePath = "${env:ProgramFiles(x86)}\Microsoft\EdgeUpdate" 138 | Remove-Item -Path $edgeUpdatePath -Recurse -Force -ErrorAction SilentlyContinue 139 | Write-Host "Edge Update processes terminated and directory removed." 140 | 141 | # - Google Chrome 142 | $chromeToolName = "Google Chrome" 143 | $chromeToolDetails = Get-ToolDetails -toolsList $toolsList -toolName $chromeToolName 144 | $chromeExePath = "C:\Program Files\Google\Chrome\Application\chrome.exe" 145 | $chromeAlias = $chromeToolDetails.alias 146 | 147 | # Check if Google Chrome is already installed by its alias 148 | if (Get-Command $chromeAlias -ErrorAction SilentlyContinue) { 149 | Write-Host "Google Chrome is already installed." 150 | } else { 151 | # Download the installer to the Temp directory 152 | $chromeInstallerFilePath = "$env:TEMP\chrome_installer.exe" 153 | $downloadResult = Invoke-DownloadFileFromAvailableMirrors -mirrorUrls $chromeToolDetails.mirrors -outfile $chromeInstallerFilePath 154 | if (-not $downloadResult) { 155 | Write-Host "Failed to download Google Chrome. Please try again later or install manually." 156 | } else { 157 | # Execute the installer silently with elevated permissions 158 | Start-Process -FilePath $chromeInstallerFilePath -ArgumentList "/silent", "/install" -Verb RunAs -Wait 159 | 160 | # Remove the installer file after installation 161 | Remove-Item -Path $chromeInstallerFilePath 162 | 163 | # Set alias 164 | $setAliasExpression = "Set-Alias -Name $chromeAlias -Value `"$chromeExePath`"" 165 | Add-Content -Path $PROFILE -Value $setAliasExpression 166 | Invoke-Expression $setAliasExpression 167 | 168 | # Add Chrome to the system PATH environment variable 169 | Add-ToEnvPath -NewPath "${env:ProgramFiles}\Google\Chrome\Application" 170 | 171 | # Disable Google Chrome Auto Updates 172 | $chromeRegPath = "HKLM:\SOFTWARE\Policies\Google\Update" 173 | if (-not (Test-Path $chromeRegPath)) { 174 | New-Item -Path $chromeRegPath -Force 175 | } 176 | Set-ItemProperty -Path $chromeRegPath -Name "AutoUpdateCheckPeriodMinutes" -Value 0 177 | Set-ItemProperty -Path $chromeRegPath -Name "UpdateDefault" -Value 0 178 | } 179 | } 180 | 181 | # - LibreOffice 182 | $libreOfficeToolName = "LibreOffice" 183 | $libreOfficeToolDetails = Get-ToolDetails -toolsList $toolsList -toolName $libreOfficeToolName 184 | 185 | # Check for LibreOffice installation 186 | $installedVersion = (Get-WmiObject -Query "SELECT * FROM Win32_Product WHERE Name like 'LibreOffice%'").Version 187 | if (-not [string]::IsNullOrWhiteSpace($installedVersion)) { 188 | Write-Host "LibreOffice $version is already installed." 189 | } else { 190 | Write-Host "LibreOffice is not installed. Downloading and installing LibreOffice..." 191 | $libreOfficeInstallerFilePath = "$env:TEMP\libreOffice_installer.exe" 192 | 193 | $downloadResult = Invoke-DownloadFileFromAvailableMirrors -mirrorUrls $libreOfficeToolDetails.mirrors -outfile $libreOfficeInstallerFilePath 194 | if (-not $downloadResult) { 195 | Write-Host "Failed to download LibreOffice. Please try again later or install manually." 196 | } else { 197 | Start-Process "msiexec.exe" -ArgumentList "/i `"$libreOfficeInstallerFilePath`" /quiet" -Wait -NoNewWindow 198 | Write-Host "LibreOffice has been installed." 199 | 200 | # Add LibreOffice to the system PATH environment variable 201 | Add-ToEnvPath -NewPath "C:\Program Files\LibreOffice\program" 202 | } 203 | } 204 | 205 | # - VLC 206 | $vlcToolName = "VLC" 207 | $vlcToolDetails = Get-ToolDetails -toolsList $toolsList -toolName $vlcToolName 208 | $vlcAlias = $vlcToolDetails.alias 209 | $vlcExecutableFilePath = "C:\Program Files\VideoLAN\VLC\vlc.exe" 210 | 211 | # Check if VLC is already installed by checking the VLC command 212 | if (Test-Path $vlcExecutableFilePath) { 213 | Write-Host "VLC is already installed." 214 | } else { 215 | # Download the installer to the Temp directory 216 | $vlcInstallerFilePath = "$env:TEMP\vlc_installer.exe" 217 | $downloadResult = Invoke-DownloadFileFromAvailableMirrors -mirrorUrls $vlcToolDetails.mirrors -outfile $vlcInstallerFilePath 218 | if (-not $downloadResult) { 219 | Write-Host "Failed to download VLC. Please try again later or install manually." 220 | } else { 221 | # Execute the installer silently with elevated permissions 222 | Start-Process -FilePath $vlcInstallerFilePath -ArgumentList "/S" -Verb RunAs -Wait 223 | 224 | # Remove the installer file after installation 225 | Remove-Item -Path $vlcInstallerFilePath 226 | 227 | # Set alias 228 | $setAliasExpression = "Set-Alias -Name $vlcAlias -Value `"$vlcExecutableFilePath`"" 229 | Add-Content -Path $PROFILE -Value $setAliasExpression 230 | Invoke-Expression $setAliasExpression 231 | 232 | # Add VLC to the system PATH environment variable 233 | Add-ToEnvPath -NewPath "C:\Program Files\VideoLAN\VLC" 234 | } 235 | } 236 | 237 | # - GIMP 238 | $gimpToolName = "GIMP" 239 | $gimpToolDetails = Get-ToolDetails -toolsList $toolsList -toolName $gimpToolName 240 | $gimpAlias = $gimpToolDetails.alias 241 | $gimpExecutablePath = "C:\Program Files\GIMP 2\bin\gimp-2.10.exe" 242 | 243 | # Check if GIMP is already installed by checking the GIMP executable path 244 | if (Test-Path $gimpExecutablePath) { 245 | Write-Host "GIMP is already installed." 246 | } else { 247 | # Download the installer to the Temp directory 248 | $gimpInstallerFilePath = "$env:TEMP\gimp_installer.exe" 249 | $downloadResult = Invoke-DownloadFileFromAvailableMirrors -mirrorUrls $gimpToolDetails.mirrors -outfile $gimpInstallerFilePath 250 | if (-not $downloadResult) { 251 | Write-Host "Failed to download GIMP. Please try again later or install manually." 252 | } else { 253 | # Execute the installer silently with elevated permissions 254 | Start-Process -FilePath $gimpInstallerFilePath -ArgumentList "/VERYSILENT /ALLUSERS" -Verb RunAs -Wait 255 | 256 | # Remove the installer file after installation 257 | Remove-Item -Path $gimpInstallerFilePath 258 | 259 | # Set alias 260 | $setAliasExpression = "Set-Alias -Name $gimpAlias -Value `"$gimpExecutablePath`"" 261 | Add-Content -Path $PROFILE -Value $setAliasExpression 262 | Invoke-Expression $setAliasExpression 263 | 264 | # Add GIMP to the system PATH environment variable 265 | Add-ToEnvPath -NewPath "C:\Program Files\GIMP 2\bin" 266 | } 267 | } 268 | 269 | # - VS Code 270 | $vsCodeToolName = "VS Code" 271 | $vsCodeToolDetails = Get-ToolDetails -toolsList $toolsList -toolName $vsCodeToolName 272 | $vsCodeAlias = $gimpToolDetails.alias 273 | $vsCodeExecutablePath = "C:\Users\$env:USERNAME\AppData\Local\Programs\Microsoft VS Code\Code.exe" 274 | 275 | # Check if VS Code is already installed by checking the VS Code executable path 276 | if (Test-Path $vsCodeExecutablePath) { 277 | Write-Host "VS Code is already installed." 278 | } else { 279 | # Download the installer to the Temp directory 280 | $vsCodeInstallerFilePath = "$env:TEMP\VSCodeSetup.exe" 281 | $downloadResult = Invoke-DownloadFileFromAvailableMirrors -mirrorUrls $vsCodeToolDetails.mirrors -outfile $vsCodeInstallerFilePath 282 | if (-not $downloadResult) { 283 | Write-Host "Failed to download VS Code. Please try again later or install manually." 284 | } else { 285 | # Execute the installer silently with elevated permissions 286 | Start-Process -FilePath $vsCodeInstallerFilePath -ArgumentList "/VERYSILENT", "/mergetasks=!runcode" -Verb RunAs -Wait 287 | 288 | # Remove the installer file after installation 289 | Remove-Item -Path $vsCodeInstallerFilePath 290 | 291 | # Set alias 292 | $setAliasExpression = "Set-Alias -Name $vsCodeAlias -Value `"$vsCodeExecutablePath`"" 293 | Add-Content -Path $PROFILE -Value $setAliasExpression 294 | Invoke-Expression $setAliasExpression 295 | 296 | # Add VS Code to the system PATH environment variable 297 | Add-ToEnvPath -NewPath "C:\Users\$env:USERNAME\AppData\Local\Programs\Microsoft VS Code\bin" 298 | 299 | # Disable Visual Studio Code Auto Updates 300 | $vsCodeSettingsPath = "${env:APPDATA}\Code\User\settings.json" 301 | if (-not (Test-Path $vsCodeSettingsPath)) { 302 | # Create the directory if it doesn't exist 303 | $dirPath = Split-Path -Path $vsCodeSettingsPath -Parent 304 | if (-not (Test-Path $dirPath)) { 305 | New-Item -ItemType Directory -Path $dirPath -Force 306 | } 307 | # Initialize an empty hashtable to act as the JSON object 308 | $settingsObj = @{} 309 | $settingsObj["update.mode"] = "none" # Set update mode to none 310 | $settingsObj | ConvertTo-Json | Set-Content $vsCodeSettingsPath 311 | } else { 312 | # If the file exists, modify it 313 | $settingsObj = Get-Content $vsCodeSettingsPath | ConvertFrom-Json 314 | $settingsObj["update.mode"] = "none" 315 | $settingsObj | ConvertTo-Json | Set-Content $vsCodeSettingsPath 316 | } 317 | } 318 | } 319 | 320 | # - Thunderbird 321 | $thunderbirdToolName = "Thunderbird" 322 | $thunderbirdToolDetails = Get-ToolDetails -toolsList $toolsList -toolName $thunderbirdToolName 323 | $thunderbirdAlias = $thunderbirdToolDetails.alias 324 | $thunderbirdExecutablePath = "C:\Program Files\Mozilla Thunderbird\thunderbird.exe" 325 | 326 | # Check if Thunderbird is already installed by checking the Thunderbird executable path 327 | if (Test-Path $thunderbirdExecutablePath) { 328 | Write-Host "Thunderbird is already installed." 329 | } else { 330 | # Download the installer to the Temp directory 331 | $thunderbirdInstallerFilePath = "$env:TEMP\ThunderbirdSetup.exe" 332 | $downloadResult = Invoke-DownloadFileFromAvailableMirrors -mirrorUrls $thunderbirdToolDetails.mirrors -outfile $thunderbirdInstallerFilePath 333 | if (-not $downloadResult) { 334 | Write-Host "Failed to download Thunderbird. Please try again later or install manually." 335 | } else { 336 | # Execute the installer silently with elevated permissions 337 | Start-Process -FilePath $thunderbirdInstallerFilePath -ArgumentList "/S" -Verb RunAs -Wait 338 | 339 | # Remove the installer file after installation 340 | Remove-Item -Path $thunderbirdInstallerFilePath 341 | 342 | # Set alias 343 | $setAliasExpression = "Set-Alias -Name $thunderbirdAlias -Value `"$thunderbirdExecutablePath`"" 344 | Add-Content -Path $PROFILE -Value $setAliasExpression 345 | Invoke-Expression $setAliasExpression 346 | 347 | # Add Thunderbird to the system PATH environment variable 348 | Add-ToEnvPath -NewPath "C:\Program Files\Mozilla Thunderbird" 349 | } 350 | } 351 | 352 | # - Server Setup 353 | 354 | $pythonServerPort = 5000 355 | $onLogonTaskName = "Server_OnLogon" 356 | $requirementsFile = "$scriptFolder\server\requirements.txt" 357 | 358 | # Ensure pip is updated to the latest version 359 | Install-PythonPackages -Package "pip" -Arguments "--upgrade" 360 | 361 | Install-PythonPackages -Package "wheel" 362 | Install-PythonPackages -Package "pywinauto" 363 | 364 | # Install Python packages from requirements.txt using Python's pip module 365 | if (Test-Path $requirementsFile) { 366 | Write-Host "Installing required Python packages using pip from requirements file..." 367 | Install-PythonPackages -RequirementsPath $requirementsFile 368 | } else { 369 | Write-Error "Requirements file not found: $requirementsFile" 370 | exit 371 | } 372 | 373 | # Add a firewall rule to allow incoming connections on the specified port for the Python executable 374 | $pythonServerRuleName = "PythonHTTPServer-$pythonServerPort" 375 | if (-not (Get-NetFirewallRule -Name $pythonServerRuleName -ErrorAction SilentlyContinue)) { 376 | New-NetFirewallRule -DisplayName $pythonServerRuleName -Direction Inbound -Program $pythonExecutablePath -Protocol TCP -LocalPort $pythonServerPort -Action Allow -Profile Any 377 | Write-Host "Firewall rule added to allow traffic on port $pythonServerPort for Python" 378 | } else { 379 | Write-Host "Firewall rule already exists. $pythonServerRuleName " 380 | } 381 | 382 | $onLogonScriptPath = "$scriptFolder\on-logon.ps1" 383 | # Check if the scheduled task exists before unregistering it 384 | if (Get-ScheduledTask -TaskName $onLogonTaskName -ErrorAction SilentlyContinue) { 385 | Write-Host "Scheduled task $onLogonTaskName already exists." 386 | } else { 387 | Write-Host "Registering new task $onLogonTaskName..." 388 | Register-LogonTask -TaskName $onLogonTaskName -ScriptPath $onLogonScriptPath -LocalUser "Docker" 389 | } 390 | 391 | Start-Sleep -Seconds 10 392 | Start-ScheduledTask -TaskName $onLogonTaskName -------------------------------------------------------------------------------- /omnitool/omnibox/vm/win11setup/setupscripts/tools_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "Python": { 3 | "mirrors": [ 4 | "https://www.python.org/ftp/python/3.10.0/python-3.10.0-amd64.exe" 5 | ], 6 | "alias": "python" 7 | }, 8 | "git": { 9 | "mirrors": [ 10 | "https://github.com/git-for-windows/git/releases/download/v2.37.1.windows.1/Git-2.37.1-64-bit.exe" 11 | ] 12 | }, 13 | "7zip": { 14 | "mirrors": [ 15 | "https://www.7-zip.org/a/7z2407-x64.exe" 16 | ] 17 | }, 18 | "ffmpeg": { 19 | "mirrors": [ 20 | "https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-essentials.7z" 21 | ] 22 | }, 23 | "Google Chrome": { 24 | "mirrors": [ 25 | "https://dl.google.com/chrome/install/latest/chrome_installer.exe" 26 | ], 27 | "alias": "google-chrome" 28 | }, 29 | "LibreOffice": { 30 | "mirrors": [ 31 | "https://mirror.raiolanetworks.com/tdf/libreoffice/stable/24.8.4/win/x86_64/LibreOffice_24.8.4_Win_x86-64.msi", 32 | "https://mirrors.iu13.net/tdf/libreoffice/stable/24.8.4/win/x86_64/LibreOffice_24.8.4_Win_x86-64.msi", 33 | "https://download.documentfoundation.org/libreoffice/stable/24.8.4/win/x86_64/LibreOffice_24.8.4_Win_x86-64.msi" 34 | ] 35 | }, 36 | "VLC": { 37 | "mirrors": [ 38 | "https://ftp.free.org/mirrors/videolan/vlc/3.0.21/win64/vlc-3.0.21-win64.exe", 39 | "https://mirror.fcix.net/videolan-ftp/vlc/3.0.21/win64/vlc-3.0.21-win64.exe", 40 | "https://mirror.raiolanetworks.com/videolan/vlc/3.0.21/win64/vlc-3.0.21-win64.exe" 41 | ], 42 | "alias": "vlc" 43 | }, 44 | "GIMP": { 45 | "mirrors": [ 46 | "https://download.gimp.org/gimp/v2.10/windows/gimp-2.10.38-setup.exe" 47 | ], 48 | "alias": "gimp" 49 | }, 50 | "VS Code": { 51 | "mirrors": [ 52 | "https://update.code.visualstudio.com/latest/win32-x64-user/stable" 53 | ], 54 | "alias": "code" 55 | }, 56 | "Thunderbird": { 57 | "mirrors": [ 58 | "https://download-installer.cdn.mozilla.net/pub/thunderbird/releases/115.12.1/win64/en-US/Thunderbird%20Setup%20115.12.1.exe", 59 | "https://archive.mozilla.org/pub/thunderbird/releases/115.12.1/win64/en-US/Thunderbird%20Setup%20115.12.1.exe" 60 | ], 61 | "alias": "thunderbird" 62 | }, 63 | "Caddy Proxy": { 64 | "mirrors": [ 65 | "https://caddyserver.com/api/download?os=windows&arch=amd64" 66 | ], 67 | "alias": "caddy" 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /omnitool/omniparserserver/omniparserserver.py: -------------------------------------------------------------------------------- 1 | ''' 2 | python -m omniparserserver --som_model_path ../../weights/icon_detect/model.pt --caption_model_name florence2 --caption_model_path ../../weights/icon_caption_florence --device cuda --BOX_TRESHOLD 0.05 3 | ''' 4 | 5 | import sys 6 | import os 7 | import time 8 | from fastapi import FastAPI 9 | from pydantic import BaseModel 10 | import argparse 11 | import uvicorn 12 | root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 13 | sys.path.append(root_dir) 14 | from util.omniparser import Omniparser 15 | 16 | def parse_arguments(): 17 | parser = argparse.ArgumentParser(description='Omniparser API') 18 | parser.add_argument('--som_model_path', type=str, default='../../weights/icon_detect/model.pt', help='Path to the som model') 19 | parser.add_argument('--caption_model_name', type=str, default='florence2', help='Name of the caption model') 20 | parser.add_argument('--caption_model_path', type=str, default='../../weights/icon_caption_florence', help='Path to the caption model') 21 | parser.add_argument('--device', type=str, default='cpu', help='Device to run the model') 22 | parser.add_argument('--BOX_TRESHOLD', type=float, default=0.05, help='Threshold for box detection') 23 | parser.add_argument('--host', type=str, default='0.0.0.0', help='Host for the API') 24 | parser.add_argument('--port', type=int, default=8000, help='Port for the API') 25 | args = parser.parse_args() 26 | return args 27 | 28 | args = parse_arguments() 29 | config = vars(args) 30 | 31 | app = FastAPI() 32 | omniparser = Omniparser(config) 33 | 34 | class ParseRequest(BaseModel): 35 | base64_image: str 36 | 37 | @app.post("/parse/") 38 | async def parse(parse_request: ParseRequest): 39 | print('start parsing...') 40 | start = time.time() 41 | dino_labled_img, parsed_content_list = omniparser.parse(parse_request.base64_image) 42 | latency = time.time() - start 43 | print('time:', latency) 44 | return {"som_image_base64": dino_labled_img, "parsed_content_list": parsed_content_list, 'latency': latency} 45 | 46 | @app.get("/probe/") 47 | async def root(): 48 | return {"message": "Omniparser API ready"} 49 | 50 | if __name__ == "__main__": 51 | uvicorn.run("omniparserserver:app", host=args.host, port=args.port, reload=True) -------------------------------------------------------------------------------- /omnitool/readme.md: -------------------------------------------------------------------------------- 1 | OmniTool Header 2 | 3 | # OmniTool 4 | 5 | Control a Windows 11 VM with OmniParser + your vision model of choice. 6 | 7 | ## Highlights: 8 | 9 | 1. **OmniParser V2** is 60% faster than V1 and now understands a wide variety of OS, app and inside app icons! 10 | 2. **OmniBox** uses 50% less disk space than other Windows VMs for agent testing, whilst providing the same computer use API 11 | 3. **OmniTool** supports out of the box the following vision models - OpenAI (4o/o1/o3-mini), DeepSeek (R1), Qwen (2.5VL) or Anthropic Computer Use 12 | 13 | ## Overview 14 | 15 | There are three components: 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 |
omniparserserverFastAPI server running OmniParser V2.
omniboxA Windows 11 VM running in a Docker container.
gradioUI to provide commands and watch reasoning + execution on OmniBox
34 | 35 | ## Showcase Video 36 | | OmniParser V2 | [Watch Video](https://1drv.ms/v/c/650b027c18d5a573/EWXbVESKWo9Buu6OYCwg06wBeoM97C6EOTG6RjvWLEN1Qg?e=alnHGC) | 37 | |--------------|------------------------------------------------------------------| 38 | | OmniTool | [Watch Video](https://1drv.ms/v/c/650b027c18d5a573/EehZ7RzY69ZHn-MeQHrnnR4BCj3by-cLLpUVlxMjF4O65Q?e=8LxMgX) | 39 | 40 | 41 | ## Notes: 42 | 43 | 1. Though **OmniParser V2** can run on a CPU, we have separated this out if you want to run it fast on a GPU machine 44 | 2. The **OmniBox** Windows 11 VM docker is dependent on KVM so can only run quickly on Windows and Linux. This can run on a CPU machine (doesn't need GPU). 45 | 3. The Gradio UI can also run on a CPU machine. We suggest running **omnibox** and **gradio** on the same CPU machine and **omniparserserver** on a GPU server. 46 | 47 | ## Setup 48 | 49 | 1. **omniparserserver**: 50 | 51 | a. If you already have a conda environment for OmniParser, you can use that. Else follow the following steps to create one 52 | 53 | b. Ensure conda is installed with `conda --version` or install from the [Anaconda website](https://www.anaconda.com/download/success) 54 | 55 | c. Navigate to the root of the repo with `cd OmniParser` 56 | 57 | d. Create a conda python environment with `conda create -n "omni" python==3.12` 58 | 59 | e. Set the python environment to be used with `conda activate omni` 60 | 61 | f. Install the dependencies with `pip install -r requirements.txt` 62 | 63 | g. Continue from here if you already had the conda environment. 64 | 65 | h. Ensure you have the V2 weights downloaded in weights folder (**ensure caption weights folder is called icon_caption_florence**). If not download them with: 66 | ``` 67 | rm -rf weights/icon_detect weights/icon_caption weights/icon_caption_florence 68 | for folder in icon_caption icon_detect; do huggingface-cli download microsoft/OmniParser-v2.0 --local-dir weights --repo-type model --include "$folder/*"; done 69 | mv weights/icon_caption weights/icon_caption_florence 70 | ``` 71 | 72 | h. Navigate to the server directory with `cd OmniParser/omnitool/omniparserserver` 73 | 74 | i. Start the server with `python -m omniparserserver` 75 | 76 | 2. **omnibox**: 77 | 78 | a. Ensure you have 30GB of space remaining (5GB for ISO, 400MB for Docker container, 20GB for storage folder) 79 | 80 | b. Install Docker Desktop 81 | 82 | c. Visit [Microsoft Evaluation Center](https://info.microsoft.com/ww-landing-windows-11-enterprise.html), accept the Terms of Service, and download a **Windows 11 Enterprise Evaluation (90-day trial, English, United States)** ISO file [~6GB]. Rename the file to `custom.iso` and copy it to the directory `OmniParser/omnitool/omnibox/vm/win11iso` 83 | 84 | d. Navigate to vm management script directory with`cd OmniParser/omnitool/omnibox/scripts` 85 | 86 | e. Build the docker container [400MB] and install the ISO to a storage folder [20GB] with `./manage_vm.sh create`. The process is shown in the screenshots below and will take 20-90 mins depending on download speeds (commonly around 60 mins). When complete the terminal will show `VM + server is up and running!`. You can see the apps being installed in the VM by looking at the desktop via the NoVNC viewer (http://localhost:8006/vnc.html?view_only=1&autoconnect=1&resize=scale). The terminal window shown in the NoVNC viewer will not be open on the desktop after the setup is done. If you can see it, wait and don't click around! 87 | ![image](https://github.com/user-attachments/assets/6bd18f81-18e2-4bc5-9170-293a6699481d) 88 | 89 | f. After creating the first time it will store a save of the VM state in `vm/win11storage`. You can then manage the VM with `./manage_vm.sh start` and `./manage_vm.sh stop`. To delete the VM, use `./manage_vm.sh delete` and delete the `OmniParser/omnitool/omnibox/vm/win11storage` directory. 90 | 91 | 3. **gradio**: 92 | 93 | a. Navigate to the gradio directory with `cd OmniParser/omnitool/gradio` 94 | 95 | b. Ensure you have activated the conda python environment with `conda activate omni` 96 | 97 | c. Start the server with `python app.py --windows_host_url localhost:8006 --omniparser_server_url localhost:8000` 98 | 99 | d. Open the URL in the terminal output, set your API Key and start playing with the AI agent! 100 | 101 | ## Common setup errors 102 | ### OmniBox install taking a while 103 | If your internet speed is slow and you want a minimal VM with less preinstalled apps comment out lines 57 to 350 in this [file](https://github.com/microsoft/OmniParser/blob/master/omnitool/omnibox/vm/win11setup/setupscripts/setup.ps1) that defines all the apps to install when you first create the container + VM. Ensure that you follow factory reset instructions from the next section when creating your VM to wipe any previous omnibox setup. 104 | 105 | ### Validation errors: Windows Host is not responding 106 | If you get this error in Gradio after clicking the submit button, this indicates that the server running in the VM that accepts commands from Gradio and then moves the mouse/ keyboard isn't available. You can verify this by running `curl http://localhost:5000/probe`. Ensure your `omnibox` is fully finished setting up (should no longer have a terminal window). Refer to the omnibox section for timing on that. If you have set up your omnibox, it may be a matter of waiting a little. 107 | 108 | If waiting 10 mins doesn't help. Try stopping (`./manage_vm.sh stop`) and starting (`./manage_vm.sh start`) your omnibox VM with the script commands. 109 | 110 | Then, if that doesn't work, delete your VM (`./manage_vm.sh delete`) leaving the storage folder and then run create again. It will be fast as it will use the existing storage folder. 111 | 112 | Finally, if that still doesn't work and you want to fully reset your VM to factory settings (create a new VM): 113 | 1. run `./manage_vm.sh delete` 114 | 2. delete the `vm/win11storage` folder 115 | 3. run `./manage_vm.sh create` 116 | 117 | ### libpaddle: The specified module could not be found 118 | The OCR library used by OmniParser is Paddle that depends on C++ Redistributable on Windows. If you are on Windows ensure that you have installed it, then rerun installing the requirements.txt. More details [here](https://github.com/microsoft/OmniParser/issues/140#issuecomment-2670619168). 119 | 120 | ## Risks and Mitigations 121 | To align with the Microsoft AI principles and Responsible AI practices, we conduct risk mitigation by training the icon caption model with Responsible AI data, which helps the model avoid inferring sensitive attributes (e.g.race, religion etc.) of the individuals which happen to be in icon images as much as possible. At the same time, we encourage user to apply OmniParser only for screenshot that does not contain harmful/violent content. For the OmniTool, we conduct threat model analysis using Microsoft Threat Modeling Tool. We advise human to stay in the loop in order to minimize risk. 122 | 123 | 124 | ## Acknowledgment 125 | Kudos to the amazing resources that are invaluable in the development of our code: [Claude Computer Use](https://github.com/anthropics/anthropic-quickstarts/blob/main/computer-use-demo/README.md), [OS World](https://github.com/xlang-ai/OSWorld), [Windows Agent Arena](https://github.com/microsoft/WindowsAgentArena), and [computer_use_ootb](https://github.com/showlab/computer_use_ootb). 126 | We are grateful for helpful suggestions and feedbacks provided by Francesco Bonacci, Jianwei Yang, Dillon DuPont, Yue Wu, Anh Nguyen. 127 | Many thanks to @keyserjaya for screenshots on omnibox install. 128 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch 2 | easyocr 3 | torchvision 4 | supervision==0.18.0 5 | openai==1.3.5 6 | transformers 7 | ultralytics==8.3.70 8 | azure-identity 9 | numpy==1.26.4 10 | opencv-python 11 | opencv-python-headless 12 | gradio 13 | dill 14 | accelerate 15 | timm 16 | einops==0.8.0 17 | paddlepaddle 18 | paddleocr 19 | ruff==0.6.7 20 | pre-commit==3.8.0 21 | pytest==8.3.3 22 | pytest-asyncio==0.23.6 23 | pyautogui==0.9.54 24 | streamlit>=1.38.0 25 | anthropic[bedrock,vertex]>=0.37.1 26 | jsonschema==4.22.0 27 | boto3>=1.28.57 28 | google-auth<3,>=2 29 | screeninfo 30 | uiautomation 31 | dashscope 32 | groq -------------------------------------------------------------------------------- /util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/OmniParser/5171b092483ab3e74ca50b9357e225f9f3571f18/util/__init__.py -------------------------------------------------------------------------------- /util/box_annotator.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional, Union, Tuple 2 | 3 | import cv2 4 | import numpy as np 5 | 6 | from supervision.detection.core import Detections 7 | from supervision.draw.color import Color, ColorPalette 8 | 9 | 10 | class BoxAnnotator: 11 | """ 12 | A class for drawing bounding boxes on an image using detections provided. 13 | 14 | Attributes: 15 | color (Union[Color, ColorPalette]): The color to draw the bounding box, 16 | can be a single color or a color palette 17 | thickness (int): The thickness of the bounding box lines, default is 2 18 | text_color (Color): The color of the text on the bounding box, default is white 19 | text_scale (float): The scale of the text on the bounding box, default is 0.5 20 | text_thickness (int): The thickness of the text on the bounding box, 21 | default is 1 22 | text_padding (int): The padding around the text on the bounding box, 23 | default is 5 24 | 25 | """ 26 | 27 | def __init__( 28 | self, 29 | color: Union[Color, ColorPalette] = ColorPalette.DEFAULT, 30 | thickness: int = 3, # 1 for seeclick 2 for mind2web and 3 for demo 31 | text_color: Color = Color.BLACK, 32 | text_scale: float = 0.5, # 0.8 for mobile/web, 0.3 for desktop # 0.4 for mind2web 33 | text_thickness: int = 2, #1, # 2 for demo 34 | text_padding: int = 10, 35 | avoid_overlap: bool = True, 36 | ): 37 | self.color: Union[Color, ColorPalette] = color 38 | self.thickness: int = thickness 39 | self.text_color: Color = text_color 40 | self.text_scale: float = text_scale 41 | self.text_thickness: int = text_thickness 42 | self.text_padding: int = text_padding 43 | self.avoid_overlap: bool = avoid_overlap 44 | 45 | def annotate( 46 | self, 47 | scene: np.ndarray, 48 | detections: Detections, 49 | labels: Optional[List[str]] = None, 50 | skip_label: bool = False, 51 | image_size: Optional[Tuple[int, int]] = None, 52 | ) -> np.ndarray: 53 | """ 54 | Draws bounding boxes on the frame using the detections provided. 55 | 56 | Args: 57 | scene (np.ndarray): The image on which the bounding boxes will be drawn 58 | detections (Detections): The detections for which the 59 | bounding boxes will be drawn 60 | labels (Optional[List[str]]): An optional list of labels 61 | corresponding to each detection. If `labels` are not provided, 62 | corresponding `class_id` will be used as label. 63 | skip_label (bool): Is set to `True`, skips bounding box label annotation. 64 | Returns: 65 | np.ndarray: The image with the bounding boxes drawn on it 66 | 67 | Example: 68 | ```python 69 | import supervision as sv 70 | 71 | classes = ['person', ...] 72 | image = ... 73 | detections = sv.Detections(...) 74 | 75 | box_annotator = sv.BoxAnnotator() 76 | labels = [ 77 | f"{classes[class_id]} {confidence:0.2f}" 78 | for _, _, confidence, class_id, _ in detections 79 | ] 80 | annotated_frame = box_annotator.annotate( 81 | scene=image.copy(), 82 | detections=detections, 83 | labels=labels 84 | ) 85 | ``` 86 | """ 87 | font = cv2.FONT_HERSHEY_SIMPLEX 88 | for i in range(len(detections)): 89 | x1, y1, x2, y2 = detections.xyxy[i].astype(int) 90 | class_id = ( 91 | detections.class_id[i] if detections.class_id is not None else None 92 | ) 93 | idx = class_id if class_id is not None else i 94 | color = ( 95 | self.color.by_idx(idx) 96 | if isinstance(self.color, ColorPalette) 97 | else self.color 98 | ) 99 | cv2.rectangle( 100 | img=scene, 101 | pt1=(x1, y1), 102 | pt2=(x2, y2), 103 | color=color.as_bgr(), 104 | thickness=self.thickness, 105 | ) 106 | if skip_label: 107 | continue 108 | 109 | text = ( 110 | f"{class_id}" 111 | if (labels is None or len(detections) != len(labels)) 112 | else labels[i] 113 | ) 114 | 115 | text_width, text_height = cv2.getTextSize( 116 | text=text, 117 | fontFace=font, 118 | fontScale=self.text_scale, 119 | thickness=self.text_thickness, 120 | )[0] 121 | 122 | if not self.avoid_overlap: 123 | text_x = x1 + self.text_padding 124 | text_y = y1 - self.text_padding 125 | 126 | text_background_x1 = x1 127 | text_background_y1 = y1 - 2 * self.text_padding - text_height 128 | 129 | text_background_x2 = x1 + 2 * self.text_padding + text_width 130 | text_background_y2 = y1 131 | # text_x = x1 - self.text_padding - text_width 132 | # text_y = y1 + self.text_padding + text_height 133 | # text_background_x1 = x1 - 2 * self.text_padding - text_width 134 | # text_background_y1 = y1 135 | # text_background_x2 = x1 136 | # text_background_y2 = y1 + 2 * self.text_padding + text_height 137 | else: 138 | text_x, text_y, text_background_x1, text_background_y1, text_background_x2, text_background_y2 = get_optimal_label_pos(self.text_padding, text_width, text_height, x1, y1, x2, y2, detections, image_size) 139 | 140 | cv2.rectangle( 141 | img=scene, 142 | pt1=(text_background_x1, text_background_y1), 143 | pt2=(text_background_x2, text_background_y2), 144 | color=color.as_bgr(), 145 | thickness=cv2.FILLED, 146 | ) 147 | # import pdb; pdb.set_trace() 148 | box_color = color.as_rgb() 149 | luminance = 0.299 * box_color[0] + 0.587 * box_color[1] + 0.114 * box_color[2] 150 | text_color = (0,0,0) if luminance > 160 else (255,255,255) 151 | cv2.putText( 152 | img=scene, 153 | text=text, 154 | org=(text_x, text_y), 155 | fontFace=font, 156 | fontScale=self.text_scale, 157 | # color=self.text_color.as_rgb(), 158 | color=text_color, 159 | thickness=self.text_thickness, 160 | lineType=cv2.LINE_AA, 161 | ) 162 | return scene 163 | 164 | 165 | def box_area(box): 166 | return (box[2] - box[0]) * (box[3] - box[1]) 167 | 168 | def intersection_area(box1, box2): 169 | x1 = max(box1[0], box2[0]) 170 | y1 = max(box1[1], box2[1]) 171 | x2 = min(box1[2], box2[2]) 172 | y2 = min(box1[3], box2[3]) 173 | return max(0, x2 - x1) * max(0, y2 - y1) 174 | 175 | def IoU(box1, box2, return_max=True): 176 | intersection = intersection_area(box1, box2) 177 | union = box_area(box1) + box_area(box2) - intersection 178 | if box_area(box1) > 0 and box_area(box2) > 0: 179 | ratio1 = intersection / box_area(box1) 180 | ratio2 = intersection / box_area(box2) 181 | else: 182 | ratio1, ratio2 = 0, 0 183 | if return_max: 184 | return max(intersection / union, ratio1, ratio2) 185 | else: 186 | return intersection / union 187 | 188 | 189 | def get_optimal_label_pos(text_padding, text_width, text_height, x1, y1, x2, y2, detections, image_size): 190 | """ check overlap of text and background detection box, and get_optimal_label_pos, 191 | pos: str, position of the text, must be one of 'top left', 'top right', 'outer left', 'outer right' TODO: if all are overlapping, return the last one, i.e. outer right 192 | Threshold: default to 0.3 193 | """ 194 | 195 | def get_is_overlap(detections, text_background_x1, text_background_y1, text_background_x2, text_background_y2, image_size): 196 | is_overlap = False 197 | for i in range(len(detections)): 198 | detection = detections.xyxy[i].astype(int) 199 | if IoU([text_background_x1, text_background_y1, text_background_x2, text_background_y2], detection) > 0.3: 200 | is_overlap = True 201 | break 202 | # check if the text is out of the image 203 | if text_background_x1 < 0 or text_background_x2 > image_size[0] or text_background_y1 < 0 or text_background_y2 > image_size[1]: 204 | is_overlap = True 205 | return is_overlap 206 | 207 | # if pos == 'top left': 208 | text_x = x1 + text_padding 209 | text_y = y1 - text_padding 210 | 211 | text_background_x1 = x1 212 | text_background_y1 = y1 - 2 * text_padding - text_height 213 | 214 | text_background_x2 = x1 + 2 * text_padding + text_width 215 | text_background_y2 = y1 216 | is_overlap = get_is_overlap(detections, text_background_x1, text_background_y1, text_background_x2, text_background_y2, image_size) 217 | if not is_overlap: 218 | return text_x, text_y, text_background_x1, text_background_y1, text_background_x2, text_background_y2 219 | 220 | # elif pos == 'outer left': 221 | text_x = x1 - text_padding - text_width 222 | text_y = y1 + text_padding + text_height 223 | 224 | text_background_x1 = x1 - 2 * text_padding - text_width 225 | text_background_y1 = y1 226 | 227 | text_background_x2 = x1 228 | text_background_y2 = y1 + 2 * text_padding + text_height 229 | is_overlap = get_is_overlap(detections, text_background_x1, text_background_y1, text_background_x2, text_background_y2, image_size) 230 | if not is_overlap: 231 | return text_x, text_y, text_background_x1, text_background_y1, text_background_x2, text_background_y2 232 | 233 | 234 | # elif pos == 'outer right': 235 | text_x = x2 + text_padding 236 | text_y = y1 + text_padding + text_height 237 | 238 | text_background_x1 = x2 239 | text_background_y1 = y1 240 | 241 | text_background_x2 = x2 + 2 * text_padding + text_width 242 | text_background_y2 = y1 + 2 * text_padding + text_height 243 | 244 | is_overlap = get_is_overlap(detections, text_background_x1, text_background_y1, text_background_x2, text_background_y2, image_size) 245 | if not is_overlap: 246 | return text_x, text_y, text_background_x1, text_background_y1, text_background_x2, text_background_y2 247 | 248 | # elif pos == 'top right': 249 | text_x = x2 - text_padding - text_width 250 | text_y = y1 - text_padding 251 | 252 | text_background_x1 = x2 - 2 * text_padding - text_width 253 | text_background_y1 = y1 - 2 * text_padding - text_height 254 | 255 | text_background_x2 = x2 256 | text_background_y2 = y1 257 | 258 | is_overlap = get_is_overlap(detections, text_background_x1, text_background_y1, text_background_x2, text_background_y2, image_size) 259 | if not is_overlap: 260 | return text_x, text_y, text_background_x1, text_background_y1, text_background_x2, text_background_y2 261 | 262 | return text_x, text_y, text_background_x1, text_background_y1, text_background_x2, text_background_y2 263 | -------------------------------------------------------------------------------- /util/omniparser.py: -------------------------------------------------------------------------------- 1 | from util.utils import get_som_labeled_img, get_caption_model_processor, get_yolo_model, check_ocr_box 2 | import torch 3 | from PIL import Image 4 | import io 5 | import base64 6 | from typing import Dict 7 | class Omniparser(object): 8 | def __init__(self, config: Dict): 9 | self.config = config 10 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 11 | 12 | self.som_model = get_yolo_model(model_path=config['som_model_path']) 13 | self.caption_model_processor = get_caption_model_processor(model_name=config['caption_model_name'], model_name_or_path=config['caption_model_path'], device=device) 14 | print('Omniparser initialized!!!') 15 | 16 | def parse(self, image_base64: str): 17 | image_bytes = base64.b64decode(image_base64) 18 | image = Image.open(io.BytesIO(image_bytes)) 19 | print('image size:', image.size) 20 | 21 | box_overlay_ratio = max(image.size) / 3200 22 | draw_bbox_config = { 23 | 'text_scale': 0.8 * box_overlay_ratio, 24 | 'text_thickness': max(int(2 * box_overlay_ratio), 1), 25 | 'text_padding': max(int(3 * box_overlay_ratio), 1), 26 | 'thickness': max(int(3 * box_overlay_ratio), 1), 27 | } 28 | 29 | (text, ocr_bbox), _ = check_ocr_box(image, display_img=False, output_bb_format='xyxy', easyocr_args={'text_threshold': 0.8}, use_paddleocr=False) 30 | dino_labled_img, label_coordinates, parsed_content_list = get_som_labeled_img(image, self.som_model, BOX_TRESHOLD = self.config['BOX_TRESHOLD'], output_coord_in_ratio=True, ocr_bbox=ocr_bbox,draw_bbox_config=draw_bbox_config, caption_model_processor=self.caption_model_processor, ocr_text=text,use_local_semantics=True, iou_threshold=0.7, scale_img=False, batch_size=128) 31 | 32 | return dino_labled_img, parsed_content_list --------------------------------------------------------------------------------