├── .github
├── ISSUE_TEMPLATE
│ ├── bug-report.en-US.yml
│ ├── feature-request.en-US.yml
│ └── llm-connectivity-issue---模型连接错误.md
├── pr-labeler.yml
├── release.yml
└── workflows
│ ├── ai-evaluation.yml
│ ├── ai-unit-test.yml
│ ├── ai.yml
│ ├── ci.yml
│ ├── issue-close-require.yml
│ ├── issue-labeled.yml
│ ├── lint.yml
│ ├── pr-label.yml
│ └── release.yml
├── .gitignore
├── .husky
└── commit-msg
├── .npmrc
├── .prettierignore
├── .prettierrc
├── .vscode
└── settings.json
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── README.zh.md
├── apps
├── android-playground
│ ├── README.md
│ ├── package.json
│ ├── rsbuild.config.ts
│ ├── src
│ │ ├── App.less
│ │ ├── App.tsx
│ │ ├── adb-device
│ │ │ ├── index.less
│ │ │ └── index.tsx
│ │ ├── env.d.ts
│ │ ├── favicon.ico
│ │ ├── icons
│ │ │ ├── linked.svg
│ │ │ ├── screenshot.svg
│ │ │ └── unlink.svg
│ │ ├── index.tsx
│ │ ├── scrcpy-player
│ │ │ ├── index.less
│ │ │ └── index.tsx
│ │ └── scripts
│ │ │ └── blank_polyfill.ts
│ └── tsconfig.json
├── chrome-extension
│ ├── README.md
│ ├── package.json
│ ├── rsbuild.config.ts
│ ├── scripts
│ │ └── pack-extension.js
│ ├── src
│ │ ├── App.less
│ │ ├── App.tsx
│ │ ├── component
│ │ │ ├── playground.less
│ │ │ └── playground.tsx
│ │ ├── env.d.ts
│ │ ├── extension
│ │ │ ├── bridge.less
│ │ │ ├── bridge.tsx
│ │ │ ├── common.less
│ │ │ ├── misc.tsx
│ │ │ ├── popup.less
│ │ │ ├── popup.tsx
│ │ │ └── recorder
│ │ │ │ ├── ExportControls.tsx
│ │ │ │ ├── README.md
│ │ │ │ ├── components
│ │ │ │ ├── ProgressModal.tsx
│ │ │ │ ├── RecordDetail.tsx
│ │ │ │ ├── RecordList.tsx
│ │ │ │ ├── SessionModals.tsx
│ │ │ │ └── index.ts
│ │ │ │ ├── generators
│ │ │ │ ├── README.md
│ │ │ │ ├── index.ts
│ │ │ │ ├── playwrightGenerator.ts
│ │ │ │ ├── shared
│ │ │ │ │ ├── testGenerationUtils.ts
│ │ │ │ │ └── types.ts
│ │ │ │ └── yamlGenerator.ts
│ │ │ │ ├── hooks
│ │ │ │ ├── index.ts
│ │ │ │ ├── useLifecycleCleanup.ts
│ │ │ │ ├── useRecordingControl.ts
│ │ │ │ ├── useRecordingSession.ts
│ │ │ │ └── useTabMonitoring.ts
│ │ │ │ ├── index.tsx
│ │ │ │ ├── logger.ts
│ │ │ │ ├── recorder.less
│ │ │ │ ├── shared
│ │ │ │ └── exportControlsUtils.ts
│ │ │ │ ├── types.ts
│ │ │ │ └── utils.ts
│ │ ├── icons
│ │ │ ├── bridge.svg
│ │ │ ├── play.svg
│ │ │ ├── playground-2.svg
│ │ │ └── playground.svg
│ │ ├── index.tsx
│ │ ├── scripts
│ │ │ ├── blank_polyfill.ts
│ │ │ ├── event-recorder-bridge.ts
│ │ │ ├── stop-water-flow.ts
│ │ │ ├── water-flow.ts
│ │ │ └── worker.ts
│ │ ├── store.tsx
│ │ ├── utils.ts
│ │ └── utils
│ │ │ ├── eventOptimizer.ts
│ │ │ └── indexedDB.ts
│ ├── static
│ │ ├── fonts
│ │ │ └── open-sans
│ │ │ │ ├── Apache License.txt
│ │ │ │ ├── open-sans-10-black
│ │ │ │ ├── open-sans-10-black.fnt
│ │ │ │ └── open-sans-10-black.png
│ │ │ │ ├── open-sans-12-black
│ │ │ │ ├── open-sans-12-black.fnt
│ │ │ │ └── open-sans-12-black.png
│ │ │ │ ├── open-sans-128-black
│ │ │ │ ├── open-sans-128-black.fnt
│ │ │ │ └── open-sans-128-black.png
│ │ │ │ ├── open-sans-128-white
│ │ │ │ ├── open-sans-128-white.fnt
│ │ │ │ └── open-sans-128-white.png
│ │ │ │ ├── open-sans-14-black
│ │ │ │ ├── open-sans-14-black.fnt
│ │ │ │ └── open-sans-14-black.png
│ │ │ │ ├── open-sans-16-black
│ │ │ │ ├── open-sans-16-black.fnt
│ │ │ │ └── open-sans-16-black.png
│ │ │ │ ├── open-sans-16-white
│ │ │ │ ├── open-sans-16-white.fnt
│ │ │ │ └── open-sans-16-white.png
│ │ │ │ ├── open-sans-32-black
│ │ │ │ ├── open-sans-32-black.fnt
│ │ │ │ └── open-sans-32-black.png
│ │ │ │ ├── open-sans-32-white
│ │ │ │ ├── open-sans-32-white.fnt
│ │ │ │ └── open-sans-32-white.png
│ │ │ │ ├── open-sans-64-black
│ │ │ │ ├── open-sans-64-black.fnt
│ │ │ │ └── open-sans-64-black.png
│ │ │ │ ├── open-sans-64-white
│ │ │ │ ├── open-sans-64-white.fnt
│ │ │ │ └── open-sans-64-white.png
│ │ │ │ ├── open-sans-8-black
│ │ │ │ ├── open-sans-8-black.fnt
│ │ │ │ └── open-sans-8-black.png
│ │ │ │ └── open-sans-8-white
│ │ │ │ ├── open-sans-8-white.fnt
│ │ │ │ └── open-sans-8-white.png
│ │ ├── icon128.png
│ │ └── manifest.json
│ └── tsconfig.json
├── recorder-form
│ ├── .gitignore
│ ├── README.md
│ ├── package.json
│ ├── rsbuild.config.ts
│ ├── src
│ │ ├── App.css
│ │ ├── App.tsx
│ │ ├── components
│ │ │ └── CanvasSelector.tsx
│ │ ├── env.d.ts
│ │ └── index.tsx
│ └── tsconfig.json
├── report
│ ├── .gitignore
│ ├── README.md
│ ├── e2e
│ │ └── check-html.yaml
│ ├── package.json
│ ├── rsbuild.config.ts
│ ├── src
│ │ ├── App.less
│ │ ├── App.tsx
│ │ ├── blank_polyfill.ts
│ │ ├── components
│ │ │ ├── PlaywrightCaseSelector.tsx
│ │ │ ├── common.less
│ │ │ ├── detail-panel.less
│ │ │ ├── detail-panel.tsx
│ │ │ ├── detail-side.less
│ │ │ ├── detail-side.tsx
│ │ │ ├── global-hover-preview.less
│ │ │ ├── global-hover-preview.tsx
│ │ │ ├── open-in-playground.less
│ │ │ ├── open-in-playground.tsx
│ │ │ ├── panel-title.less
│ │ │ ├── panel-title.tsx
│ │ │ ├── pixi-loader.tsx
│ │ │ ├── playground-demo-ui-context.json
│ │ │ ├── playground.tsx
│ │ │ ├── side-item.tsx
│ │ │ ├── sidebar.less
│ │ │ ├── sidebar.tsx
│ │ │ ├── store.tsx
│ │ │ ├── timeline.less
│ │ │ ├── timeline.tsx
│ │ │ └── yaml-player-component.tsx
│ │ ├── env.d.ts
│ │ ├── index.less
│ │ ├── index.tsx
│ │ └── types.ts
│ ├── template
│ │ └── index.html
│ ├── test-data
│ │ ├── ai-shop.json
│ │ ├── ai-todo.json
│ │ ├── error.json
│ │ ├── online-order.json
│ │ ├── query-only.json
│ │ ├── search-headphone-on-ebay.json
│ │ ├── swag-lab.json
│ │ └── taobao.json
│ └── tsconfig.json
└── site
│ ├── .gitignore
│ ├── build.sh
│ ├── docs
│ ├── en
│ │ ├── API.mdx
│ │ ├── automate-with-scripts-in-yaml.mdx
│ │ ├── blog-introducing-instant-actions-and-deep-think.md
│ │ ├── blog-programming-practice-using-structured-api.md
│ │ ├── blog-support-android-automation.mdx
│ │ ├── bridge-mode-by-chrome-extension.mdx
│ │ ├── caching.mdx
│ │ ├── changelog.mdx
│ │ ├── choose-a-model.mdx
│ │ ├── common
│ │ │ ├── prepare-android.mdx
│ │ │ ├── prepare-key-for-further-use.mdx
│ │ │ ├── setup-env.mdx
│ │ │ └── start-experience.mdx
│ │ ├── data-privacy.md
│ │ ├── faq.md
│ │ ├── index.mdx
│ │ ├── integrate-with-android.mdx
│ │ ├── integrate-with-playwright.mdx
│ │ ├── integrate-with-puppeteer.mdx
│ │ ├── llm-txt.mdx
│ │ ├── mcp.mdx
│ │ ├── model-provider.mdx
│ │ ├── prompting-tips.md
│ │ ├── quick-experience-with-android.mdx
│ │ └── quick-experience.mdx
│ ├── public
│ │ ├── android-playground.png
│ │ ├── android-set-env.png
│ │ ├── android-usb-debug-en.png
│ │ ├── android-usb-debug.png
│ │ ├── blog
│ │ │ ├── 0.10.0-2.png
│ │ │ ├── 0.10.0.png
│ │ │ ├── 0.11.0-2.png
│ │ │ ├── 0.11.0.png
│ │ │ ├── 0.13.0.jpeg
│ │ │ ├── 0.5.0-2.png
│ │ │ ├── 0.5.0.png
│ │ │ ├── 0.6.0-2.png
│ │ │ ├── 0.6.0-3.png
│ │ │ ├── 0.6.0-4.png
│ │ │ ├── 0.6.0-5.png
│ │ │ ├── 0.6.0-6.png
│ │ │ ├── 0.6.0.png
│ │ │ ├── 0.9.0.png
│ │ │ ├── ai-ide-convert-prompt-result.png
│ │ │ ├── ai-ide-convert-prompt.png
│ │ │ ├── android-playground-lark-poster-cn.png
│ │ │ ├── android-playground-lark-poster-en.png
│ │ │ ├── coze-sidebar.png
│ │ │ ├── export-video.png
│ │ │ ├── logScreenshot-api.png
│ │ │ ├── report-coze-deep-think.png
│ │ │ ├── report-instant-action.png
│ │ │ └── report-planning.png
│ │ ├── bridge_in_extension.jpg
│ │ ├── cache
│ │ │ ├── no-cache-time.png
│ │ │ └── use-cache-time.png
│ │ ├── midescene-playground-entry.jpg
│ │ ├── midscene-bridge-mode.jpg
│ │ ├── midscene-extension.jpg
│ │ ├── midscene-icon.png
│ │ ├── midscene_with_text_light.png
│ │ ├── playground.png
│ │ └── report.gif
│ └── zh
│ │ ├── API.mdx
│ │ ├── automate-with-scripts-in-yaml.mdx
│ │ ├── blog-introducing-instant-actions-and-deep-think.md
│ │ ├── blog-programming-practice-using-structured-api.md
│ │ ├── blog-support-android-automation.mdx
│ │ ├── bridge-mode-by-chrome-extension.mdx
│ │ ├── caching.mdx
│ │ ├── changelog.mdx
│ │ ├── choose-a-model.mdx
│ │ ├── common
│ │ ├── prepare-android.mdx
│ │ ├── prepare-key-for-further-use.mdx
│ │ ├── setup-env.mdx
│ │ └── start-experience.mdx
│ │ ├── data-privacy.md
│ │ ├── faq.md
│ │ ├── index.mdx
│ │ ├── integrate-with-android.mdx
│ │ ├── integrate-with-playwright.mdx
│ │ ├── integrate-with-puppeteer.mdx
│ │ ├── llm-txt.mdx
│ │ ├── mcp.mdx
│ │ ├── model-provider.mdx
│ │ ├── prompting-tips.md
│ │ ├── quick-experience-with-android.mdx
│ │ └── quick-experience.mdx
│ ├── i18n.json
│ ├── package.json
│ ├── route.json
│ ├── rspress.config.ts
│ ├── styles
│ └── index.css
│ └── tsconfig.json
├── biome.json
├── commitlint.config.js
├── cspell.config.cjs
├── nx.json
├── package.json
├── packages
├── android-playground
│ ├── .gitignore
│ ├── README.md
│ ├── bin
│ │ ├── android-playground
│ │ └── server.bin
│ ├── modern.config.ts
│ ├── package.json
│ ├── src
│ │ ├── index.ts
│ │ └── scrcpy-server.ts
│ └── tsconfig.json
├── android
│ ├── .gitignore
│ ├── README.md
│ ├── bin
│ │ └── yadb
│ ├── modern.config.ts
│ ├── package.json
│ ├── src
│ │ ├── agent
│ │ │ └── index.ts
│ │ ├── index.ts
│ │ ├── page
│ │ │ └── index.ts
│ │ └── utils
│ │ │ └── index.ts
│ ├── tests
│ │ └── ai
│ │ │ ├── ebay.test.ts
│ │ │ ├── setting.test.ts
│ │ │ ├── todo.test.ts
│ │ │ └── travel.test.ts
│ ├── tsconfig.json
│ └── vitest.config.ts
├── cli
│ ├── .gitignore
│ ├── .npmignore
│ ├── LICENSE
│ ├── README.md
│ ├── bin
│ │ └── midscene
│ ├── modern.config.ts
│ ├── package.json
│ ├── src
│ │ ├── args.ts
│ │ ├── cli-utils.ts
│ │ ├── http-server.d.ts
│ │ ├── index.ts
│ │ ├── printer.ts
│ │ ├── tty-renderer.ts
│ │ └── yaml-runner.ts
│ ├── tests
│ │ ├── ai
│ │ │ ├── __snapshots__
│ │ │ │ └── bin.test.ts.snap
│ │ │ ├── bin.test.ts
│ │ │ └── bridge.test.ts
│ │ ├── midscene_scripts
│ │ │ ├── local
│ │ │ │ ├── local-error-message.yml
│ │ │ │ └── local.yml
│ │ │ └── online
│ │ │ │ └── online.yaml
│ │ ├── midscene_scripts_bridge
│ │ │ ├── current_tab
│ │ │ │ └── check_content.yaml
│ │ │ └── new_tab
│ │ │ │ ├── bing.yaml
│ │ │ │ ├── local.yml
│ │ │ │ └── open-new-tab.yaml
│ │ ├── server_root
│ │ │ └── index.html
│ │ └── unit-test
│ │ │ ├── __snapshots__
│ │ │ └── cli-utils.test.ts.snap
│ │ │ └── cli-utils.test.ts
│ ├── tsconfig.json
│ └── vitest.config.ts
├── core
│ ├── .gitignore
│ ├── .npmignore
│ ├── LICENSE
│ ├── README.md
│ ├── modern.config.ts
│ ├── package.json
│ ├── src
│ │ ├── ai-model
│ │ │ ├── action-executor.ts
│ │ │ ├── common.ts
│ │ │ ├── index.ts
│ │ │ ├── inspect.ts
│ │ │ ├── llm-planning.ts
│ │ │ ├── prompt
│ │ │ │ ├── assertion.ts
│ │ │ │ ├── common.ts
│ │ │ │ ├── describe.ts
│ │ │ │ ├── extraction.ts
│ │ │ │ ├── llm-locator.ts
│ │ │ │ ├── llm-planning.ts
│ │ │ │ ├── llm-section-locator.ts
│ │ │ │ ├── playwright-generator.ts
│ │ │ │ ├── ui-tars-locator.ts
│ │ │ │ ├── ui-tars-planning.ts
│ │ │ │ ├── util.ts
│ │ │ │ └── yaml-generator.ts
│ │ │ ├── service-caller
│ │ │ │ ├── index.ts
│ │ │ │ └── types.d.ts
│ │ │ └── ui-tars-planning.ts
│ │ ├── image
│ │ │ └── index.ts
│ │ ├── index.ts
│ │ ├── insight
│ │ │ ├── index.ts
│ │ │ └── utils.ts
│ │ ├── tree.ts
│ │ ├── types.ts
│ │ ├── utils.ts
│ │ └── yaml.ts
│ ├── tests
│ │ ├── ai
│ │ │ ├── assert
│ │ │ │ └── assert.test.ts
│ │ │ ├── connectivity.test.ts
│ │ │ ├── extract
│ │ │ │ ├── __snapshots__
│ │ │ │ │ └── extract.test.ts.snap
│ │ │ │ └── extract.test.ts
│ │ │ ├── insight
│ │ │ │ └── insight.test.ts
│ │ │ ├── llm-inspect.test.ts
│ │ │ ├── llm-planning
│ │ │ │ ├── __snapshots__
│ │ │ │ │ ├── basic.test.ts.snap
│ │ │ │ │ ├── input.test.ts.snap
│ │ │ │ │ ├── planning-input.test.ts.snap
│ │ │ │ │ └── planning.test.ts.snap
│ │ │ │ ├── basic.test.ts
│ │ │ │ └── input.test.ts
│ │ │ ├── llm-section-locator.test.ts
│ │ │ ├── parse-action.test.ts
│ │ │ └── ui-tars-planning
│ │ │ │ ├── output.png
│ │ │ │ └── plan-to-target.test.ts
│ │ ├── evaluation.ts
│ │ ├── fixtures
│ │ │ ├── baidu.png
│ │ │ ├── dump-for-utils-test.json
│ │ │ └── dump.json
│ │ ├── tsconfig.json
│ │ ├── unit-test
│ │ │ ├── __snapshots__
│ │ │ │ ├── llm-planning.test.ts.snap
│ │ │ │ └── tree.test.ts.snap
│ │ │ ├── env.test.ts
│ │ │ ├── executor
│ │ │ │ ├── __snapshots__
│ │ │ │ │ └── index.test.ts.snap
│ │ │ │ └── index.test.ts
│ │ │ ├── llm-planning.test.ts
│ │ │ ├── mocks
│ │ │ │ └── intl-mock.ts
│ │ │ ├── prompt
│ │ │ │ ├── __snapshots__
│ │ │ │ │ ├── assertion.test.ts.snap
│ │ │ │ │ ├── describe.test.ts.snap
│ │ │ │ │ └── prompt.test.ts.snap
│ │ │ │ ├── assertion.test.ts
│ │ │ │ ├── describe.test.ts
│ │ │ │ ├── playwright-generator.test.ts
│ │ │ │ ├── prompt.test.ts
│ │ │ │ └── utils.test.ts
│ │ │ ├── types-rightclick.test.ts
│ │ │ └── utils.test.ts
│ │ └── utils.ts
│ ├── third-party-licenses.txt
│ ├── tsconfig.json
│ └── vitest.config.ts
├── evaluation
│ ├── .gitignore
│ ├── README.md
│ ├── data-generator
│ │ ├── fixture.ts
│ │ ├── generator-headed.spec.ts
│ │ ├── generator-headless.spec.ts
│ │ └── utils.ts
│ ├── package.json
│ ├── page-cases
│ │ ├── assertion
│ │ │ ├── online_order.json
│ │ │ └── online_order_list.json
│ │ ├── inspect
│ │ │ ├── antd-carousel.json
│ │ │ ├── antd-carousel.json-coordinates-annotated.png
│ │ │ ├── aweme-login.json
│ │ │ ├── aweme-login.json-coordinates-annotated.png
│ │ │ ├── aweme-play.json
│ │ │ ├── aweme-play.json-coordinates-annotated.png
│ │ │ ├── online_order.json
│ │ │ ├── online_order.json-coordinates-annotated.png
│ │ │ ├── online_order_list.json
│ │ │ ├── online_order_list.json-coordinates-annotated.png
│ │ │ ├── taobao.json
│ │ │ ├── taobao.json-coordinates-annotated.png
│ │ │ ├── todo.json
│ │ │ └── todo.json-coordinates-annotated.png
│ │ ├── planning
│ │ │ ├── antd-form-vl.json
│ │ │ ├── antd-form-vl.json-planning-coordinates-annotated.png
│ │ │ ├── antd-tooltip-vl.json
│ │ │ ├── antd-tooltip-vl.json-planning-coordinates-annotated.png
│ │ │ ├── aweme-login-vl.json
│ │ │ ├── todo-vl.json
│ │ │ ├── todo-vl.json-planning-coordinates-annotated.png
│ │ │ └── todo.json
│ │ └── section-locator
│ │ │ ├── antd-tooltip.json
│ │ │ └── antd-tooltip.json-coordinates-annotated.png
│ ├── page-data
│ │ ├── antd-carousel
│ │ │ ├── element-snapshot.json
│ │ │ ├── element-tree.json
│ │ │ ├── element-tree.txt
│ │ │ ├── input.png
│ │ │ ├── output.png
│ │ │ ├── output_without_text.png
│ │ │ └── resize-output.png
│ │ ├── antd-form
│ │ │ ├── element-snapshot.json
│ │ │ ├── element-tree.json
│ │ │ ├── element-tree.txt
│ │ │ ├── input.png
│ │ │ ├── output.png
│ │ │ ├── output_without_text.png
│ │ │ └── resize-output.png
│ │ ├── antd-pagination
│ │ │ ├── element-snapshot.json
│ │ │ ├── element-tree.json
│ │ │ ├── element-tree.txt
│ │ │ ├── input.png
│ │ │ ├── output.png
│ │ │ ├── output_without_text.png
│ │ │ └── resize-output.png
│ │ ├── antd-tooltip
│ │ │ ├── element-snapshot.json
│ │ │ ├── element-tree.json
│ │ │ ├── element-tree.txt
│ │ │ ├── input.png
│ │ │ ├── output.png
│ │ │ ├── output_without_text.png
│ │ │ └── resize-output.png
│ │ ├── aweme-login
│ │ │ ├── element-snapshot.json
│ │ │ ├── element-tree.json
│ │ │ ├── element-tree.txt
│ │ │ ├── input.png
│ │ │ ├── output.png
│ │ │ ├── output_without_text.png
│ │ │ └── resize-output.png
│ │ ├── aweme-play
│ │ │ ├── element-snapshot.json
│ │ │ ├── element-tree.json
│ │ │ ├── element-tree.txt
│ │ │ ├── input.png
│ │ │ ├── output.png
│ │ │ ├── output_without_text.png
│ │ │ └── resize-output.png
│ │ ├── githubstatus
│ │ │ ├── element-snapshot.json
│ │ │ ├── element-tree.json
│ │ │ ├── element-tree.txt
│ │ │ ├── input.png
│ │ │ ├── output.png
│ │ │ ├── output_without_text.png
│ │ │ └── resize-output.png
│ │ ├── image-only
│ │ │ └── .gitignore
│ │ ├── online_order
│ │ │ ├── element-snapshot.json
│ │ │ ├── element-tree.json
│ │ │ ├── element-tree.txt
│ │ │ ├── input.png
│ │ │ ├── output.png
│ │ │ ├── output_without_text.png
│ │ │ └── resize-output.png
│ │ ├── online_order_list
│ │ │ ├── element-snapshot.json
│ │ │ ├── element-tree.json
│ │ │ ├── element-tree.txt
│ │ │ ├── input.png
│ │ │ ├── output.png
│ │ │ ├── output_without_text.png
│ │ │ └── resize-output.png
│ │ ├── taobao
│ │ │ ├── element-snapshot.json
│ │ │ ├── element-tree.json
│ │ │ ├── element-tree.txt
│ │ │ ├── input.png
│ │ │ ├── output.png
│ │ │ ├── output_without_text.png
│ │ │ └── resize-output.png
│ │ ├── todo-input-with-value
│ │ │ ├── element-snapshot.json
│ │ │ ├── element-tree.json
│ │ │ ├── element-tree.txt
│ │ │ ├── input.png
│ │ │ ├── output.png
│ │ │ ├── output_without_text.png
│ │ │ └── resize-output.png
│ │ ├── todo
│ │ │ ├── element-snapshot.json
│ │ │ ├── element-tree.json
│ │ │ ├── element-tree.txt
│ │ │ ├── input.png
│ │ │ ├── output.png
│ │ │ ├── output_without_text.png
│ │ │ └── resize-output.png
│ │ └── visualstudio
│ │ │ ├── element-snapshot.json
│ │ │ ├── element-tree.json
│ │ │ ├── element-tree.txt
│ │ │ ├── input.png
│ │ │ ├── output.png
│ │ │ ├── output_without_text.png
│ │ │ └── resize-output.png
│ ├── playwright.config.ts
│ ├── src
│ │ └── test-analyzer.ts
│ ├── tests
│ │ ├── assertion.test.ts
│ │ ├── llm-locator.test.ts
│ │ ├── llm-planning.test.ts
│ │ ├── llm-section-locator.test.ts
│ │ ├── screenspot-v2-evaluation.test.ts
│ │ └── util.ts
│ ├── tsconfig.json
│ └── vitest.config.ts
├── mcp
│ ├── .gitignore
│ ├── README.md
│ ├── package.json
│ ├── rslib.config.ts
│ ├── scripts
│ │ └── inspect.mjs
│ ├── src
│ │ ├── index.ts
│ │ ├── midscene.ts
│ │ ├── prompts.ts
│ │ ├── puppeteer.ts
│ │ ├── resources.ts
│ │ ├── tools.ts
│ │ └── utils.ts
│ ├── tests
│ │ ├── index.test.ts
│ │ ├── test-login.mjs
│ │ └── tsconfig.json
│ ├── tsconfig.json
│ └── vitest.config.ts
├── recorder
│ ├── .gitignore
│ ├── README.md
│ ├── package.json
│ ├── rslib.config.ts
│ ├── src
│ │ ├── Button.tsx
│ │ ├── RecordTimeline.css
│ │ ├── RecordTimeline.tsx
│ │ ├── button.css
│ │ ├── index.tsx
│ │ ├── recorder-iife-index.ts
│ │ └── recorder.ts
│ └── tsconfig.json
├── shared
│ ├── README.md
│ ├── modern.config.ts
│ ├── modern.inspect.config.ts
│ ├── package.json
│ ├── src
│ │ ├── common.ts
│ │ ├── constants
│ │ │ ├── example-code.ts
│ │ │ └── index.ts
│ │ ├── env.ts
│ │ ├── extractor
│ │ │ ├── constants.ts
│ │ │ ├── debug.ts
│ │ │ ├── dom-util.ts
│ │ │ ├── index.ts
│ │ │ ├── locator.ts
│ │ │ ├── tree.ts
│ │ │ ├── util.ts
│ │ │ └── web-extractor.ts
│ │ ├── img
│ │ │ ├── box-select.ts
│ │ │ ├── draw-box.ts
│ │ │ ├── get-jimp.ts
│ │ │ ├── index.ts
│ │ │ ├── info.ts
│ │ │ ├── jimp.d.ts
│ │ │ └── transform.ts
│ │ ├── index.ts
│ │ ├── logger.ts
│ │ ├── modern-app-env.d.ts
│ │ ├── node
│ │ │ └── fs.ts
│ │ ├── types
│ │ │ └── index.ts
│ │ ├── us-keyboard-layout.ts
│ │ └── utils.ts
│ ├── tests
│ │ ├── fixtures
│ │ │ ├── 2x2.jpeg
│ │ │ ├── baidu.png
│ │ │ ├── colorful.png
│ │ │ ├── dump.json
│ │ │ ├── heytea.jpeg
│ │ │ ├── icon.png
│ │ │ ├── long-text.png
│ │ │ ├── reference-of-list.png
│ │ │ ├── reference.png
│ │ │ └── table.png
│ │ ├── tsconfig.json
│ │ ├── unit-test
│ │ │ ├── __snapshots__
│ │ │ │ └── tree.test.ts.snap
│ │ │ ├── fs.test.ts
│ │ │ ├── image
│ │ │ │ ├── __snapshots__
│ │ │ │ │ └── index.test.ts.snap
│ │ │ │ └── index.test.ts
│ │ │ ├── keyboard.test.ts
│ │ │ └── tree.test.ts
│ │ └── utils.ts
│ ├── tsconfig.json
│ └── vitest.config.ts
├── visualizer
│ ├── .gitignore
│ ├── .npmignore
│ ├── README.md
│ ├── modern.config.ts
│ ├── package.json
│ ├── src
│ │ ├── blank_polyfill.ts
│ │ ├── component
│ │ │ ├── blackboard.less
│ │ │ ├── blackboard.tsx
│ │ │ ├── color.tsx
│ │ │ ├── common.less
│ │ │ ├── describer.less
│ │ │ ├── describer.tsx
│ │ │ ├── env-config.tsx
│ │ │ ├── github-star.less
│ │ │ ├── github-star.tsx
│ │ │ ├── logo.less
│ │ │ ├── logo.tsx
│ │ │ ├── misc.tsx
│ │ │ ├── pixi-loader.tsx
│ │ │ ├── player.less
│ │ │ ├── player.tsx
│ │ │ ├── playground-demo-ui-context.json
│ │ │ ├── playground
│ │ │ │ ├── ConfigSelector.tsx
│ │ │ │ ├── ContextPreview.tsx
│ │ │ │ ├── HistorySelector.tsx
│ │ │ │ ├── PlaygroundResult.tsx
│ │ │ │ ├── PromptInput.tsx
│ │ │ │ ├── ServiceModeControl.tsx
│ │ │ │ ├── index.less
│ │ │ │ ├── playground-constants.tsx
│ │ │ │ ├── playground-types.ts
│ │ │ │ ├── playground-utils.ts
│ │ │ │ ├── useServerValid.ts
│ │ │ │ └── useStaticPageAgent.ts
│ │ │ ├── replay-scripts.tsx
│ │ │ ├── shiny-text.less
│ │ │ ├── shiny-text.tsx
│ │ │ └── store
│ │ │ │ ├── history.ts
│ │ │ │ └── store.tsx
│ │ ├── extension
│ │ │ ├── jimp.d.ts
│ │ │ └── utils.ts
│ │ ├── global.d.ts
│ │ ├── icons
│ │ │ ├── close.svg
│ │ │ ├── history.svg
│ │ │ ├── magnifying-glass.svg
│ │ │ └── setting.svg
│ │ ├── index.tsx
│ │ ├── init.ts
│ │ ├── types.d.ts
│ │ └── utils.ts
│ └── tsconfig.json
└── web-integration
│ ├── .gitignore
│ ├── README.md
│ ├── bin
│ └── midscene-playground
│ ├── modern.config.ts
│ ├── package.json
│ ├── scripts
│ └── check-exports.js
│ ├── src
│ ├── bridge-mode
│ │ ├── agent-cli-side.ts
│ │ ├── browser.ts
│ │ ├── common.ts
│ │ ├── index.ts
│ │ ├── io-client.ts
│ │ ├── io-server.ts
│ │ └── page-browser-side.ts
│ ├── chrome-extension
│ │ ├── agent.ts
│ │ ├── cdpInput.ts
│ │ ├── dynamic-scripts.ts
│ │ ├── index.ts
│ │ └── page.ts
│ ├── common
│ │ ├── agent.ts
│ │ ├── page.d.ts
│ │ ├── plan-builder.ts
│ │ ├── task-cache.ts
│ │ ├── tasks.ts
│ │ ├── ui-utils.ts
│ │ └── utils.ts
│ ├── index.ts
│ ├── page.ts
│ ├── playground
│ │ ├── agent.ts
│ │ ├── bin.ts
│ │ ├── index.ts
│ │ ├── server.ts
│ │ └── static-page.ts
│ ├── playwright
│ │ ├── ai-fixture.ts
│ │ ├── index.ts
│ │ ├── page.ts
│ │ └── reporter
│ │ │ ├── index.ts
│ │ │ └── select-cache-file.ts
│ ├── puppeteer
│ │ ├── agent-launcher.ts
│ │ ├── base-page.ts
│ │ ├── index.ts
│ │ └── page.ts
│ ├── web-element.ts
│ └── yaml
│ │ ├── builder.ts
│ │ ├── index.ts
│ │ ├── player.ts
│ │ └── utils.ts
│ ├── tests
│ ├── ai
│ │ ├── bridge
│ │ │ ├── agent.test.ts
│ │ │ ├── keyboard-event.test.ts
│ │ │ ├── open-new-tab.test.ts
│ │ │ └── temp.test.ts
│ │ ├── fixtures
│ │ │ └── ui-context.json
│ │ └── web
│ │ │ ├── playwright-reporter-test
│ │ │ └── todo-report.spec.ts
│ │ │ ├── playwright
│ │ │ ├── ai-auto-todo.spec.ts
│ │ │ ├── ai-online-order.spec.ts
│ │ │ ├── ai-shop.spec.ts
│ │ │ ├── fixture.ts
│ │ │ ├── memory-release.spec.ts
│ │ │ ├── open-new-tab.spec.ts
│ │ │ └── util.ts
│ │ │ ├── puppeteer
│ │ │ ├── agent.test.ts
│ │ │ ├── e2e.test.ts
│ │ │ ├── open-new-tab.test.ts
│ │ │ ├── query.test.ts
│ │ │ ├── scroll.html
│ │ │ └── utils.ts
│ │ │ └── static
│ │ │ └── static-page.test.ts
│ ├── playwright.config.ts
│ ├── tsconfig.json
│ └── unit-test
│ │ ├── __snapshots__
│ │ ├── agent.test.ts.snap
│ │ ├── plan-builder.test.ts.snap
│ │ ├── task-cache.test.ts.snap
│ │ ├── web-extractor.test.ts.snap
│ │ └── yaml.test.ts.snap
│ │ ├── agent.test.ts
│ │ ├── bridge
│ │ └── io.test.ts
│ │ ├── fixtures
│ │ ├── cookie
│ │ │ └── httpbin.dev_cookies.json
│ │ ├── dump-with-invisible.json
│ │ ├── dump.json
│ │ ├── extractor
│ │ │ ├── child.html
│ │ │ └── scroll
│ │ │ │ ├── input.png
│ │ │ │ └── output.png
│ │ └── web-extractor
│ │ │ ├── assets
│ │ │ ├── search-dark.svg
│ │ │ └── search.svg
│ │ │ ├── child.html
│ │ │ ├── index.html
│ │ │ ├── input.png
│ │ │ ├── merge-rects.html
│ │ │ ├── output.png
│ │ │ └── scroll
│ │ │ ├── input.png
│ │ │ └── output.png
│ │ ├── http-server.d.ts
│ │ ├── page-task-executor-rightclick.test.ts
│ │ ├── plan-builder.test.ts
│ │ ├── playground-server.test.ts
│ │ ├── task-cache.test.ts
│ │ ├── util.test.ts
│ │ ├── web-extractor.test.ts
│ │ └── yaml
│ │ ├── __snapshots__
│ │ ├── player.test.ts.snap
│ │ └── utils.test.ts.snap
│ │ ├── player.test.ts
│ │ ├── server_root
│ │ └── index.html
│ │ └── utils.test.ts
│ ├── tsconfig.json
│ └── vitest.config.ts
├── pnpm-lock.yaml
├── pnpm-workspace.yaml
└── scripts
├── dictionary.txt
└── release.js
/.github/ISSUE_TEMPLATE/feature-request.en-US.yml:
--------------------------------------------------------------------------------
1 | name: '💡 Feature Request'
2 | description: Submit a new feature request to Midscene
3 | title: '[Feature]: '
4 | type: Enhancement
5 | body:
6 | - type: markdown
7 | attributes:
8 | value: |
9 | Thanks for submitting new feature requests! Before submitting, please note:
10 |
11 | - Confirmed that this is a common feature and cannot be implemented through existing APIs.
12 | - Make sure you searched in the [Issues](https://github.com/web-infra-dev/midscene/issues) and didn't find the same request.
13 |
14 | - type: textarea
15 | id: description
16 | attributes:
17 | label: What problem does this feature solve?
18 | description: Please describe the usage scenario for this feature.
19 | validations:
20 | required: true
21 |
22 | - type: textarea
23 | id: api
24 | attributes:
25 | label: What does the proposed API look like?
26 | description: Describe the new API, provide some code examples.
27 | validations:
28 | required: true
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/llm-connectivity-issue---模型连接错误.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: LLM Connectivity Issue / 模型连接错误
3 | about: How to solve the LLM connectivity problem
4 | title: "[Connectivity]"
5 | labels: ''
6 | assignees: ''
7 |
8 | ---
9 |
10 | ## Read this before open issue
11 |
12 | How to choose and config a model: https://midscenejs.com/model-provider.html
13 |
14 | Use this project to check the connection: https://github.com/web-infra-dev/midscene-example/tree/main/connectivity-test
15 |
16 | ## If the error persists, tell us these information
17 |
18 | - Where are you using Midscene.js (Chrome extension, yaml with cli, Puppeteer,…)
19 |
20 | - The version of Midscene.js or Extension
21 |
22 | - The error message
23 |
24 | - The model name and endpoint (if could be public）
25 |
26 | ## Security Check
27 |
28 | Do NOT include your API key in your issue! Revoke it immediately if it has already been leaked in your issue.
29 |
--------------------------------------------------------------------------------
/.github/pr-labeler.yml:
--------------------------------------------------------------------------------
1 | 'change: feat':
2 | - '/^(feat|types|style)/'
3 | 'change: fix':
4 | - '/^fix/'
5 | 'change: perf':
6 | - '/^perf/'
7 | 'change: breaking':
8 | - '/^breaking change/'
9 | 'change: docs':
10 | - '/^docs/'
11 |
--------------------------------------------------------------------------------
/.github/release.yml:
--------------------------------------------------------------------------------
1 | # .github/release.yml
2 |
3 | changelog:
4 | exclude:
5 | authors:
6 | # Ignore the release PR created by github-actions
7 | - github-actions
8 | categories:
9 | - title: Breaking Changes 🍭
10 | labels:
11 | - 'change: breaking'
12 | - title: New Features 🎉
13 | labels:
14 | - 'change: feat'
15 | - title: Performance 🚀
16 | labels:
17 | - 'change: perf'
18 | - title: Bug Fixes 🐞
19 | labels:
20 | - 'change: fix'
21 | - title: Document 📖
22 | labels:
23 | - 'change: docs'
24 | - title: Other Changes
25 | labels:
26 | - '*'
27 |
--------------------------------------------------------------------------------
/.github/workflows/issue-close-require.yml:
--------------------------------------------------------------------------------
1 | name: Issue Close Require
2 |
3 | on:
4 | schedule:
5 | - cron: '0 0 * * *'
6 |
7 | permissions:
8 | # Permits `actions-cool/issues-helper` to close an issue
9 | issues: write
10 | contents: read
11 |
12 | jobs:
13 | issue-close-require:
14 | runs-on: ubuntu-latest
15 | if: github.repository == 'web-infra-dev/midscene'
16 | steps:
17 | - name: need reproduction
18 | uses: actions-cool/issues-helper@a610082f8ac0cf03e357eb8dd0d5e2ba075e017e # v3.6.0
19 | with:
20 | actions: 'close-issues'
21 | labels: 'need reproduction'
22 | inactive-day: 5
23 | body: |
24 | As the issue was labelled with `need reproduction`, but no response in 5 days. This issue will be closed. Feel free to comment and reopen it if you have any further questions. For background, see [Why reproductions are required](https://antfu.me/posts/why-reproductions-are-required).
25 |
26 | 由于该 issue 被标记为 "需要重现"，但在 5 天内没有回应，因此该 issue 将被关闭。如果你有任何进一步的问题，请随时发表评论并重新打开该 issue。背景请参考 [为什么需要最小重现](https://antfu.me/posts/why-reproductions-are-required-zh)。
--------------------------------------------------------------------------------
/.github/workflows/issue-labeled.yml:
--------------------------------------------------------------------------------
1 | name: Issue Labeled
2 |
3 | on:
4 | issues:
5 | types: [labeled]
6 |
7 | permissions:
8 | contents: read
9 | # Permits `actions-cool/issues-helper` to comment on an issue
10 | issues: write
11 |
12 | jobs:
13 | reply-labeled:
14 | name: Reply need reproduction
15 | runs-on: ubuntu-latest
16 | if: github.repository == 'web-infra-dev/midscene'
17 | steps:
18 | - name: need reproduction
19 | if: github.event.label.name == 'need reproduction'
20 | uses: actions-cool/issues-helper@a610082f8ac0cf03e357eb8dd0d5e2ba075e017e # v3.6.0
21 | with:
22 | actions: 'create-comment'
23 | issue-number: ${{ github.event.issue.number }}
24 | body: |
25 | Hello @${{ github.event.issue.user.login }}. Please provide a reproduction repository or online demo. For background, see [Why reproductions are required](https://antfu.me/posts/why-reproductions-are-required). Thanks ❤️
--------------------------------------------------------------------------------
/.github/workflows/lint.yml:
--------------------------------------------------------------------------------
1 | name: Lint
2 |
3 | on:
4 | push:
5 | branches:
6 | - main
7 | pull_request:
8 | branches:
9 | - main
10 |
11 | permissions:
12 | contents: read
13 |
14 | jobs:
15 | main:
16 | runs-on: ubuntu-latest
17 | strategy:
18 | matrix:
19 | node-version: [18.19.0]
20 |
21 | env:
22 | MIDSCENE_OPENAI_INIT_CONFIG_JSON: ${{ secrets.MIDSCENE_OPENAI_INIT_CONFIG_JSON }}
23 | MIDSCENE_OPENAI_MODEL: ${{ secrets.MIDSCENE_OPENAI_MODEL }}
24 |
25 | steps:
26 | - uses: actions/checkout@v4
27 | with:
28 | fetch-depth: 0
29 |
30 | - name: Fetch all branches
31 | run: git fetch --all
32 |
33 | - name: Setup pnpm
34 | uses: pnpm/action-setup@v2
35 | with:
36 | version: 9.3.0
37 |
38 | - name: Setup Node.js
39 | uses: actions/setup-node@v4
40 | with:
41 | node-version: '18'
42 | cache: 'pnpm'
43 |
44 | - name: Install Dependencies
45 | run: pnpm install --ignore-scripts
46 |
47 | - name: Check Dependency Version
48 | run: pnpm run check-dependency-version
49 |
50 | - name: Biome lint
51 | run: npx biome check . --diagnostic-level=warn --no-errors-on-unmatched
52 |
53 |
--------------------------------------------------------------------------------
/.github/workflows/pr-label.yml:
--------------------------------------------------------------------------------
1 | name: PR Labeler
2 |
3 | on:
4 | pull_request_target:
5 | types:
6 | - opened
7 | - edited
8 |
9 | permissions:
10 | # Permits `github/issue-labeler` to add a label to a pull request
11 | pull-requests: write
12 | contents: read
13 |
14 | jobs:
15 | change-labeling:
16 | name: Labeling for changes
17 | runs-on: ubuntu-latest
18 | if: github.repository == 'web-infra-dev/midscene'
19 | steps:
20 | - uses: github/issue-labeler@v3.4
21 | with:
22 | repo-token: "${{ secrets.GITHUB_TOKEN }}"
23 | configuration-path: .github/pr-labeler.yml
24 | enable-versioned-regex: 0
25 | include-title: 1
26 | sync-labels: 1
27 |
28 |
--------------------------------------------------------------------------------
/.husky/commit-msg:
--------------------------------------------------------------------------------
1 | npx --no -- commitlint --edit "$1"
--------------------------------------------------------------------------------
/.npmrc:
--------------------------------------------------------------------------------
1 | registry=https://registry.npmjs.org/
2 | strict-peer-dependencies=false
3 | save-prefix=''
4 | save-workspace-protocol=rolling
5 | ignore-compatibility-db=true
6 | use-lockfile-v6=true
7 | puppeteer_download_base_url=https://cdn.npmmirror.com/binaries/chrome-for-testing
--------------------------------------------------------------------------------
/.prettierrc:
--------------------------------------------------------------------------------
1 | {
2 | "singleQuote": true
3 | }
--------------------------------------------------------------------------------
/.vscode/settings.json:
--------------------------------------------------------------------------------
1 | {
2 | "editor.codeActionsOnSave": {
3 | "source.organizeImports.biome": "explicit"
4 | },
5 | "editor.defaultFormatter": "biomejs.biome",
6 | "editor.formatOnSave": true,
7 | "cSpell.words": [
8 | "AITEST",
9 | "Aliyun",
10 | "aweme",
11 | "bbox",
12 | "bytedance",
13 | "deepseek",
14 | "doubao",
15 | "douyin",
16 | "fkill",
17 | "httpbin",
18 | "iconfont",
19 | "modelcontextprotocol",
20 | "openrouter",
21 | "qwen",
22 | "taobao",
23 | "targetcreated",
24 | "Volcengine",
25 | "xpaths",
26 | "Yadb"
27 | ],
28 | "[jsonc]": {
29 | "editor.defaultFormatter": "biomejs.biome"
30 | },
31 | "[plaintext]": {
32 | "editor.defaultFormatter": "esbenp.prettier-vscode"
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2024-present Bytedance, Inc. and its affiliates.
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/apps/android-playground/README.md:
--------------------------------------------------------------------------------
1 | # Midscene Android Playground
2 |
3 | Playground tool for Android cli @midscene/android.
4 |
5 | See https://midscenejs.com/ for details.
6 |
--------------------------------------------------------------------------------
/apps/android-playground/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "android-playground",
3 | "private": true,
4 | "version": "0.12.4",
5 | "type": "module",
6 | "scripts": {
7 | "build": "rsbuild build",
8 | "dev": "rsbuild dev --open",
9 | "preview": "rsbuild preview"
10 | },
11 | "dependencies": {
12 | "@ant-design/icons": "^5.3.1",
13 | "@midscene/android": "workspace:*",
14 | "@midscene/core": "workspace:*",
15 | "@midscene/shared": "workspace:*",
16 | "@midscene/visualizer": "workspace:*",
17 | "@midscene/web": "workspace:*",
18 | "@yume-chan/scrcpy": "^1.1.0",
19 | "@yume-chan/scrcpy-decoder-webcodecs": "1.1.0",
20 | "antd": "^5.21.6",
21 | "dayjs": "^1.11.11",
22 | "react": "18.3.1",
23 | "react-dom": "18.3.1",
24 | "socket.io-client": "4.8.1"
25 | },
26 | "devDependencies": {
27 | "@rsbuild/core": "^1.3.22",
28 | "@rsbuild/plugin-less": "^1.2.4",
29 | "@rsbuild/plugin-node-polyfill": "1.3.0",
30 | "@rsbuild/plugin-react": "^1.3.1",
31 | "@rsbuild/plugin-svgr": "^1.1.1",
32 | "@types/react": "^18.3.1",
33 | "@types/react-dom": "^18.3.1",
34 | "archiver": "^6.0.0",
35 | "less": "^4.2.0",
36 | "typescript": "^5.8.3"
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/apps/android-playground/src/env.d.ts:
--------------------------------------------------------------------------------
1 | /// <reference types="@rsbuild/core/types" />
2 |
3 | declare module '*.svg' {
4 | const content: string;
5 | export default content;
6 | }
7 | declare module '*.svg?react' {
8 | const ReactComponent: React.FunctionComponent<React.SVGProps<SVGSVGElement>>;
9 | export default ReactComponent;
10 | }
11 |
--------------------------------------------------------------------------------
/apps/android-playground/src/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/android-playground/src/favicon.ico
--------------------------------------------------------------------------------
/apps/android-playground/src/icons/linked.svg:
--------------------------------------------------------------------------------
1 | <svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
2 | <path fill-rule="evenodd" clip-rule="evenodd" d="M9.68001 1.99996C10.88 0.799959 12.88 0.719959 14.08 1.91996C15.28 3.11996 15.2 5.11996 14 6.31996L11.84 8.47996C11.6 8.71996 11.28 8.71996 11.04 8.47996C10.8 8.23996 10.8 7.91996 11.04 7.67996L13.2 5.51996C14 4.71996 14 3.43996 13.28 2.71996C12.56 1.99996 11.28 1.99996 10.48 2.79996L8.32001 4.95996C8.08001 5.19996 7.76001 5.19996 7.52001 4.95996C7.28001 4.71996 7.28001 4.39996 7.52001 4.15996L9.68001 1.99996ZM5.60001 13.2L7.76001 11.04C8.00001 10.8 8.32001 10.8 8.56001 11.04C8.80001 11.28 8.80001 11.6 8.56001 11.84L6.40001 14C5.20001 15.2 3.28001 15.36 2.00001 14.08C0.640014 12.8 0.800014 10.88 2.08001 9.67996L4.24001 7.51996C4.48001 7.27996 4.80001 7.27996 5.04001 7.51996C5.28001 7.75996 5.28001 8.07996 5.04001 8.31996L2.88001 10.48C2.08001 11.28 2.00001 12.48 2.80001 13.28C3.60001 14.08 4.80001 14 5.60001 13.2ZM10.24 4.95991C10.48 4.71991 10.88 4.71991 11.2 4.95991C11.44 5.19991 11.44 5.59991 11.2 5.91991L5.92001 11.1999C5.68001 11.4399 5.28001 11.4399 4.96001 11.1999C4.72001 10.9599 4.72001 10.5599 4.96001 10.2399L10.24 4.95991Z" fill="black" fill-opacity="0.25"/>
3 | </svg>
4 |
--------------------------------------------------------------------------------
/apps/android-playground/src/icons/unlink.svg:
--------------------------------------------------------------------------------
1 | <svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
2 | <path d="M14.08 1.92002C12.88 0.72002 10.88 0.80002 9.68 2.00002L7.52 4.16002C7.28 4.40002 7.28 4.72002 7.52 4.96002C7.76 5.20002 8.08 5.20002 8.32 4.96002L10.48 2.80002C11.28 2.00002 12.56 2.00002 13.28 2.72002C14 3.44002 14 4.72002 13.2 5.52002L11.04 7.68002C10.8 7.92002 10.8 8.24002 11.04 8.48002C11.28 8.72002 11.6 8.72002 11.84 8.48002L14 6.32002C15.2 5.12002 15.28 3.12002 14.08 1.92002ZM7.76 11.04L5.6 13.2C4.8 14 3.6 14.08 2.8 13.28C2 12.48 2.08 11.28 2.88 10.48L5.04 8.32002C5.28 8.08002 5.28 7.76002 5.04 7.52002C4.8 7.28002 4.48 7.28002 4.24 7.52002L2.08 9.68002C0.799999 10.88 0.639999 12.8 2 14.08C3.28 15.36 5.2 15.2 6.4 14L8.56 11.84C8.8 11.6 8.8 11.28 8.56 11.04C8.32 10.8 8 10.8 7.76 11.04ZM5.92 4.96002C5.68 4.72002 5.28 4.72002 4.96 4.96002C4.72 5.20002 4.72 5.60002 4.96 5.92002L10.24 11.2C10.48 11.44 10.88 11.44 11.2 11.2C11.44 10.96 11.44 10.56 11.2 10.24L5.92 4.96002Z" fill="#FF4550"/>
3 | </svg>
4 |
--------------------------------------------------------------------------------
/apps/android-playground/src/index.tsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import ReactDOM from 'react-dom/client';
3 | import App from './App';
4 |
5 | const rootEl = document.getElementById('root');
6 | if (rootEl) {
7 | const root = ReactDOM.createRoot(rootEl);
8 | root.render(<App />);
9 | }
10 |
--------------------------------------------------------------------------------
/apps/android-playground/src/scripts/blank_polyfill.ts:
--------------------------------------------------------------------------------
1 | export default {};
2 |
--------------------------------------------------------------------------------
/apps/android-playground/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "lib": ["DOM", "ES2020"],
4 | "jsx": "react-jsx",
5 | "target": "ES2020",
6 | "skipLibCheck": true,
7 | "useDefineForClassFields": true,
8 |
9 | /* modules */
10 | "module": "ESNext",
11 | "isolatedModules": true,
12 | "resolveJsonModule": true,
13 | "moduleResolution": "bundler",
14 | "allowImportingTsExtensions": true,
15 | "noEmit": true,
16 |
17 | /* type checking */
18 | "strict": true,
19 | "noUnusedLocals": true,
20 | "noUnusedParameters": true
21 | },
22 | "include": ["src"]
23 | }
24 |
--------------------------------------------------------------------------------
/apps/chrome-extension/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "chrome-extension",
3 | "private": true,
4 | "version": "0.12.4",
5 | "type": "module",
6 | "scripts": {
7 | "build": "rsbuild build && npm run pack-extension",
8 | "dev": "rsbuild dev --open",
9 | "preview": "rsbuild preview",
10 | "pack-extension": "node scripts/pack-extension.js"
11 | },
12 | "dependencies": {
13 | "@ant-design/icons": "^5.3.1",
14 | "@midscene/core": "workspace:*",
15 | "@midscene/recorder": "workspace:*",
16 | "@midscene/report": "workspace:*",
17 | "@midscene/shared": "workspace:*",
18 | "@midscene/visualizer": "workspace:*",
19 | "@midscene/web": "workspace:*",
20 | "antd": "^5.21.6",
21 | "canvas-confetti": "1.9.3",
22 | "dayjs": "^1.11.11",
23 | "react": "18.3.1",
24 | "react-dom": "18.3.1",
25 | "zustand": "4.5.2"
26 | },
27 | "devDependencies": {
28 | "@rsbuild/core": "^1.3.22",
29 | "@rsbuild/plugin-less": "^1.2.4",
30 | "@rsbuild/plugin-node-polyfill": "1.3.0",
31 | "@rsbuild/plugin-react": "^1.3.1",
32 | "@rsbuild/plugin-svgr": "^1.1.1",
33 | "@rsbuild/plugin-type-check": "1.2.3",
34 | "@types/chrome": "0.0.279",
35 | "@types/react": "^18.3.1",
36 | "@types/react-dom": "^18.3.1",
37 | "archiver": "^6.0.0",
38 | "less": "^4.2.0",
39 | "typescript": "^5.8.3"
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/apps/chrome-extension/src/App.less:
--------------------------------------------------------------------------------
1 | .command-form {
2 | display: flex;
3 | flex-direction: column;
4 | height: 100%;
5 | overflow: hidden;
6 | }
7 |
8 | .form-content {
9 | display: grid;
10 | grid-template-rows: auto 1fr;
11 | height: 100%;
12 | min-height: 0;
13 | position: relative;
14 | gap: 24px;
15 | }
16 |
17 | .result-container {
18 | grid-row: 2;
19 | display: flex;
20 | flex-direction: column;
21 | min-height: 0;
22 | position: relative;
23 | overflow: auto;
24 | }
25 |
26 | .result-container>div {
27 | flex: 1;
28 | height: auto;
29 | min-height: 0;
30 | }
--------------------------------------------------------------------------------
/apps/chrome-extension/src/App.tsx:
--------------------------------------------------------------------------------
1 | import './App.less';
2 | import { PlaygroundPopup } from './extension/popup';
3 |
4 | export default function App() {
5 | return <PlaygroundPopup />;
6 | }
7 |
--------------------------------------------------------------------------------
/apps/chrome-extension/src/env.d.ts:
--------------------------------------------------------------------------------
1 | /// <reference types="@rsbuild/core/types" />
2 |
3 | declare module '*.svg' {
4 | const content: string;
5 | export default content;
6 | }
7 | declare module '*.svg?react' {
8 | const ReactComponent: React.FunctionComponent<React.SVGProps<SVGSVGElement>>;
9 | export default ReactComponent;
10 | }
11 |
--------------------------------------------------------------------------------
/apps/chrome-extension/src/extension/common.less:
--------------------------------------------------------------------------------
1 | @main-text: #3b3b3b;
2 |
3 | @primary-color: #2B83FF;
4 | @main-orange: #F9483E;
5 |
6 | @side-bg: #F8F8F8;
7 | @title-bg: @side-bg;
8 | @border-color: #E5E5E5;
9 | @heavy-border-color: #888;
10 |
11 | @selected-bg: #bfc4da80;
12 | @hover-bg: #dcdcdc80;
13 |
14 | @weak-bg: #F3F3F3;
15 | @weak-text: #777;
16 | @footer-text: #CCC;
17 |
18 | @toolbar-btn-bg: #E9E9E9;
19 |
20 | @layout-space: 20px;
21 |
22 | @side-horizontal-padding: 10px;
23 | @side-vertical-spacing: 10px;
24 |
25 | @layout-extension-space-horizontal: 20px;
26 | @layout-extension-space-vertical: 20px;
27 |
28 | .clear-button-container {
29 | width: 28;
30 | height: 28;
31 | border-radius: 4px;
32 | display: flex;
33 | justify-content: center;
34 | align-items: center;
35 | margin: 4px 0;
36 | position: fixed;
37 | top: 60px;
38 | right: 12px;
39 | background: #fff;
40 | z-index: 999;
41 | border: 1px solid rgba(240, 240, 240);
42 | }
--------------------------------------------------------------------------------
/apps/chrome-extension/src/extension/misc.tsx:
--------------------------------------------------------------------------------
1 | import {
2 | ArrowRightOutlined,
3 | CheckOutlined,
4 | ClockCircleOutlined,
5 | CloseOutlined,
6 | LogoutOutlined,
7 | MinusOutlined,
8 | WarningOutlined,
9 | } from '@ant-design/icons';
10 |
11 | export const iconForStatus = (status: string): JSX.Element => {
12 | switch (status) {
13 | case 'finished':
14 | case 'passed':
15 | case 'success':
16 | case 'connected':
17 | return (
18 | <span style={{ color: '#2B8243' }}>
19 | <CheckOutlined />
20 | </span>
21 | );
22 |
23 | case 'finishedWithWarning':
24 | return (
25 | <span style={{ color: '#f7bb05' }}>
26 | <WarningOutlined />
27 | </span>
28 | );
29 | case 'failed':
30 | case 'closed':
31 | case 'timedOut':
32 | case 'interrupted':
33 | return (
34 | <span style={{ color: '#FF0A0A' }}>
35 | <CloseOutlined />
36 | </span>
37 | );
38 | case 'pending':
39 | return <ClockCircleOutlined />;
40 | case 'cancelled':
41 | case 'skipped':
42 | return <LogoutOutlined />;
43 | case 'running':
44 | return <ArrowRightOutlined />;
45 | default:
46 | return <MinusOutlined />;
47 | }
48 | };
49 |
--------------------------------------------------------------------------------
/apps/chrome-extension/src/extension/recorder/components/index.ts:
--------------------------------------------------------------------------------
1 | export { RecordList } from './RecordList';
2 | export { RecordDetail } from './RecordDetail';
3 | export { SessionModals } from './SessionModals';
4 | export { ProgressModal } from './ProgressModal';
5 |
--------------------------------------------------------------------------------
/apps/chrome-extension/src/extension/recorder/generators/index.ts:
--------------------------------------------------------------------------------
1 | // Main generators
2 | export { generatePlaywrightTest } from './playwrightGenerator';
3 | export { generateYamlTest, exportEventsToYaml } from './yamlGenerator';
4 |
5 | // Shared utilities
6 | export * from './shared/types';
7 | export * from './shared/testGenerationUtils';
8 |
--------------------------------------------------------------------------------
/apps/chrome-extension/src/extension/recorder/hooks/index.ts:
--------------------------------------------------------------------------------
1 | export { useRecordingSession } from './useRecordingSession';
2 | export { useRecordingControl } from './useRecordingControl';
3 | export { useTabMonitoring } from './useTabMonitoring';
4 | export { useLifecycleCleanup } from './useLifecycleCleanup';
5 |
--------------------------------------------------------------------------------
/apps/chrome-extension/src/icons/play.svg:
--------------------------------------------------------------------------------
1 | <svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg">
2 | <path d="M3.4668 2.05664C3.66983 1.93942 3.92001 1.93942 4.12305 2.05664L11.7012 6.43164C11.904 6.54891 12.0293 6.76565 12.0293 7C12.0293 7.23435 11.904 7.45109 11.7012 7.56836L4.12305 11.9434C3.92001 12.0606 3.66983 12.0606 3.4668 11.9434C3.26375 11.8261 3.13867 11.6095 3.13867 11.375V2.625C3.13867 2.39054 3.26375 2.17387 3.4668 2.05664Z" stroke="white" stroke-width="1.3125" stroke-linejoin="round"/>
3 | </svg>
4 |
--------------------------------------------------------------------------------
/apps/chrome-extension/src/icons/playground-2.svg:
--------------------------------------------------------------------------------
1 | <svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
2 | <path d="M3.55921 2.65821C4.0413 2.20419 4.76402 2.12006 5.33753 2.45118L16.1627 8.70118C16.6268 8.96913 16.9127 9.46411 16.9127 10C16.9127 10.5359 16.6268 11.0309 16.1627 11.2988L5.33753 17.5488C4.76402 17.88 4.0413 17.7958 3.55921 17.3418C3.07715 16.8878 2.94958 16.1714 3.24573 15.5791L6.03479 10L3.24573 4.42091C2.94958 3.82861 3.07715 3.11224 3.55921 2.65821ZM7.71253 10L4.58753 16.25L15.4127 10L4.58753 3.75001L7.71253 10Z" fill="currentColor"/>
3 | </svg>
4 |
--------------------------------------------------------------------------------
/apps/chrome-extension/src/icons/playground.svg:
--------------------------------------------------------------------------------
1 | <svg
2 | width="20"
3 | height="20"
4 | viewBox="0 0 20 20"
5 | fill="none"
6 | xmlns="http://www.w3.org/2000/svg"
7 | aria-label="Playground"
8 | >
9 | <title>Playground</title>
10 | <rect width="20" height="20" rx="10" fill="#2B83FF" />
11 | <path
12 | d="M6.86597 5.88171C7.04673 5.71147 7.3179 5.67947 7.53296 5.80359L10.781 7.67859L14.0281 9.55359C14.2021 9.65407 14.3093 9.83993 14.3093 10.0409C14.3093 10.2419 14.2021 10.4277 14.0281 10.5282L10.781 12.4032L7.53296 14.2782C7.3179 14.4023 7.04673 14.3703 6.86597 14.2001C6.68535 14.0298 6.63776 13.761 6.74878 13.5389L8.4978 10.0409L6.74878 6.54285C6.63776 6.32081 6.68535 6.05198 6.86597 5.88171Z"
13 | fill="#2B83FF"
14 | stroke="white"
15 | stroke-width="1.125"
16 | stroke-linejoin="round"
17 | />
18 | </svg>
--------------------------------------------------------------------------------
/apps/chrome-extension/src/index.tsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import ReactDOM from 'react-dom/client';
3 | import App from './App';
4 |
5 | const rootEl = document.getElementById('root');
6 | if (rootEl) {
7 | const root = ReactDOM.createRoot(rootEl);
8 | root.render(<App />);
9 | }
10 |
--------------------------------------------------------------------------------
/apps/chrome-extension/src/scripts/blank_polyfill.ts:
--------------------------------------------------------------------------------
1 | export default {};
2 |
--------------------------------------------------------------------------------
/apps/chrome-extension/src/scripts/stop-water-flow.ts:
--------------------------------------------------------------------------------
1 | if (typeof window.midsceneWaterFlowAnimation !== 'undefined') {
2 | window.midsceneWaterFlowAnimation.disable();
3 | }
4 |
--------------------------------------------------------------------------------
/apps/chrome-extension/static/fonts/open-sans/open-sans-10-black/open-sans-10-black.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/chrome-extension/static/fonts/open-sans/open-sans-10-black/open-sans-10-black.png
--------------------------------------------------------------------------------
/apps/chrome-extension/static/fonts/open-sans/open-sans-12-black/open-sans-12-black.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/chrome-extension/static/fonts/open-sans/open-sans-12-black/open-sans-12-black.png
--------------------------------------------------------------------------------
/apps/chrome-extension/static/fonts/open-sans/open-sans-128-black/open-sans-128-black.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/chrome-extension/static/fonts/open-sans/open-sans-128-black/open-sans-128-black.png
--------------------------------------------------------------------------------
/apps/chrome-extension/static/fonts/open-sans/open-sans-128-white/open-sans-128-white.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/chrome-extension/static/fonts/open-sans/open-sans-128-white/open-sans-128-white.png
--------------------------------------------------------------------------------
/apps/chrome-extension/static/fonts/open-sans/open-sans-14-black/open-sans-14-black.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/chrome-extension/static/fonts/open-sans/open-sans-14-black/open-sans-14-black.png
--------------------------------------------------------------------------------
/apps/chrome-extension/static/fonts/open-sans/open-sans-16-black/open-sans-16-black.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/chrome-extension/static/fonts/open-sans/open-sans-16-black/open-sans-16-black.png
--------------------------------------------------------------------------------
/apps/chrome-extension/static/fonts/open-sans/open-sans-16-white/open-sans-16-white.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/chrome-extension/static/fonts/open-sans/open-sans-16-white/open-sans-16-white.png
--------------------------------------------------------------------------------
/apps/chrome-extension/static/fonts/open-sans/open-sans-32-black/open-sans-32-black.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/chrome-extension/static/fonts/open-sans/open-sans-32-black/open-sans-32-black.png
--------------------------------------------------------------------------------
/apps/chrome-extension/static/fonts/open-sans/open-sans-32-white/open-sans-32-white.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/chrome-extension/static/fonts/open-sans/open-sans-32-white/open-sans-32-white.png
--------------------------------------------------------------------------------
/apps/chrome-extension/static/fonts/open-sans/open-sans-64-black/open-sans-64-black.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/chrome-extension/static/fonts/open-sans/open-sans-64-black/open-sans-64-black.png
--------------------------------------------------------------------------------
/apps/chrome-extension/static/fonts/open-sans/open-sans-64-white/open-sans-64-white.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/chrome-extension/static/fonts/open-sans/open-sans-64-white/open-sans-64-white.png
--------------------------------------------------------------------------------
/apps/chrome-extension/static/fonts/open-sans/open-sans-8-black/open-sans-8-black.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/chrome-extension/static/fonts/open-sans/open-sans-8-black/open-sans-8-black.png
--------------------------------------------------------------------------------
/apps/chrome-extension/static/fonts/open-sans/open-sans-8-white/open-sans-8-white.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/chrome-extension/static/fonts/open-sans/open-sans-8-white/open-sans-8-white.png
--------------------------------------------------------------------------------
/apps/chrome-extension/static/icon128.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/chrome-extension/static/icon128.png
--------------------------------------------------------------------------------
/apps/chrome-extension/static/manifest.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "Midscene.js",
3 | "description": "Open-source SDK for automating web pages using natural language through AI.",
4 | "version": "0.82",
5 | "manifest_version": 3,
6 | "permissions": [
7 | "activeTab",
8 | "tabs",
9 | "sidePanel",
10 | "debugger",
11 | "scripting"
12 | ],
13 | "incognito": "split",
14 | "background": {
15 | "service_worker": "./scripts/worker.js"
16 | },
17 | "host_permissions": [
18 | "<all_urls>"
19 | ],
20 | "action": {
21 | "default_icon": "icon128.png",
22 | "default_title": "Midscene.js"
23 | },
24 | "side_panel": {
25 | "default_path": "./index.html"
26 | },
27 | "icons": {
28 | "128": "icon128.png"
29 | }
30 | }
--------------------------------------------------------------------------------
/apps/chrome-extension/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "lib": ["DOM", "ES2020"],
4 | "jsx": "react-jsx",
5 | "target": "ES2020",
6 | "skipLibCheck": true,
7 | "useDefineForClassFields": true,
8 |
9 | /* modules */
10 | "module": "ESNext",
11 | "isolatedModules": true,
12 | "resolveJsonModule": true,
13 | "moduleResolution": "bundler",
14 | "allowImportingTsExtensions": true,
15 | "noEmit": true,
16 |
17 | /* type checking */
18 | "strict": true,
19 | "noUnusedLocals": false,
20 | "noUnusedParameters": false
21 | },
22 | "include": ["src"]
23 | }
24 |
--------------------------------------------------------------------------------
/apps/recorder-form/.gitignore:
--------------------------------------------------------------------------------
1 | # Local
2 | .DS_Store
3 | *.local
4 | *.log*
5 |
6 | # Dist
7 | node_modules
8 | dist/
9 |
10 | # IDE
11 | .vscode/*
12 | !.vscode/extensions.json
13 | .idea
14 |
--------------------------------------------------------------------------------
/apps/recorder-form/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "recorder-form",
3 | "version": "1.0.0",
4 | "private": true,
5 | "type": "module",
6 | "scripts": {
7 | "build": "rsbuild build",
8 | "dev": "rsbuild dev --open",
9 | "preview": "rsbuild preview"
10 | },
11 | "dependencies": {
12 | "antd": "^5.21.6",
13 | "dayjs": "^1.11.11",
14 | "react": "18.3.1",
15 | "react-dom": "18.3.1",
16 | "@midscene/recorder": "workspace:*"
17 | },
18 | "devDependencies": {
19 | "@rsbuild/plugin-node-polyfill": "1.3.0",
20 | "@rsbuild/core": "^1.3.22",
21 | "@rsbuild/plugin-react": "^1.3.1",
22 | "@types/react": "^18.3.1",
23 | "@types/react-dom": "^18.3.1",
24 | "typescript": "^5.8.3"
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/apps/recorder-form/rsbuild.config.ts:
--------------------------------------------------------------------------------
1 | import { defineConfig } from '@rsbuild/core';
2 | import { pluginNodePolyfill } from '@rsbuild/plugin-node-polyfill';
3 | import { pluginReact } from '@rsbuild/plugin-react';
4 |
5 | export default defineConfig({
6 | server: {
7 | port: 3001,
8 | },
9 | plugins: [pluginReact(), pluginNodePolyfill()],
10 | });
11 |
--------------------------------------------------------------------------------
/apps/recorder-form/src/env.d.ts:
--------------------------------------------------------------------------------
1 | /// <reference types="@rsbuild/core/types" />
2 |
--------------------------------------------------------------------------------
/apps/recorder-form/src/index.tsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import ReactDOM from 'react-dom/client';
3 | import App from './App';
4 |
5 | const rootEl = document.getElementById('root');
6 | if (rootEl) {
7 | const root = ReactDOM.createRoot(rootEl);
8 | root.render(
9 | <React.StrictMode>
10 | <App />
11 | </React.StrictMode>,
12 | );
13 | }
14 |
--------------------------------------------------------------------------------
/apps/recorder-form/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "lib": ["DOM", "ES2020"],
4 | "jsx": "react-jsx",
5 | "target": "ES2020",
6 | "noEmit": true,
7 | "skipLibCheck": true,
8 | "useDefineForClassFields": true,
9 |
10 | /* modules */
11 | "module": "ESNext",
12 | "isolatedModules": true,
13 | "resolveJsonModule": true,
14 | "moduleResolution": "bundler",
15 | "allowImportingTsExtensions": true,
16 |
17 | /* type checking */
18 | "strict": true,
19 | "noUnusedLocals": true,
20 | "noUnusedParameters": true
21 | },
22 | "include": ["src", "../../packages/record/src/RecordTimeline.tsx"]
23 | }
24 |
--------------------------------------------------------------------------------
/apps/report/.gitignore:
--------------------------------------------------------------------------------
1 | # Local
2 | .DS_Store
3 | *.local
4 | *.log*
5 |
6 | # Dist
7 | node_modules
8 | dist/
9 |
10 | # IDE
11 | .vscode/*
12 | !.vscode/extensions.json
13 | .idea
14 |
15 | # Midscene.js dump files
16 | midscene_run/report
17 | midscene_run/tmp
18 |
--------------------------------------------------------------------------------
/apps/report/e2e/check-html.yaml:
--------------------------------------------------------------------------------
1 | target:
2 | serve: ./dist
3 | url: demo.html
4 | tasks:
5 | - name: check console html
6 | flow:
7 | - ai: Click the 'Insight / Locate' on Left
8 | - sleep: 3000
9 | - aiAssert: There is a 'Open in Playground' button on the page
10 |
--------------------------------------------------------------------------------
/apps/report/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "@midscene/report",
3 | "version": "0.13.1",
4 | "private": true,
5 | "scripts": {
6 | "dev": "rsbuild dev",
7 | "build": "rsbuild build",
8 | "dev:rsdoctor": "RSDOCTOR=true rsbuild dev",
9 | "build:rsdoctor": "RSDOCTOR=true rsbuild build",
10 | "preview": "rsbuild preview",
11 | "e2e": "node ../../packages/cli/bin/midscene ./e2e/"
12 | },
13 | "dependencies": {
14 | "@ant-design/icons": "^5.3.1",
15 | "@midscene/core": "workspace:*",
16 | "@midscene/visualizer": "workspace:*",
17 | "@midscene/web": "workspace:*",
18 | "@midscene/shared": "workspace:*",
19 | "@modern-js/runtime": "2.60.6",
20 | "@rsbuild/core": "^1.3.22",
21 | "@rsbuild/plugin-less": "^1.2.4",
22 | "@rsbuild/plugin-react": "^1.3.1",
23 | "@types/chrome": "0.0.279",
24 | "antd": "^5.21.6",
25 | "pixi-filters": "6.0.5",
26 | "pixi.js": "8.1.1",
27 | "react": "18.3.1",
28 | "react-dom": "18.3.1",
29 | "react-resizable-panels": "2.0.22",
30 | "zustand": "4.5.2"
31 | },
32 | "devDependencies": {
33 | "@rsbuild/plugin-node-polyfill": "1.3.0",
34 | "@rsdoctor/rspack-plugin": "1.0.2",
35 | "@types/react": "^18.3.1",
36 | "@types/react-dom": "^18.3.1",
37 | "less": "^4.2.0",
38 | "typescript": "^5.8.3"
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/apps/report/src/App.less:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/report/src/App.less
--------------------------------------------------------------------------------
/apps/report/src/blank_polyfill.ts:
--------------------------------------------------------------------------------
1 | export default {};
2 |
--------------------------------------------------------------------------------
/apps/report/src/components/common.less:
--------------------------------------------------------------------------------
1 | @main-text: #3b3b3b;
2 |
3 | @primary-color: #06b1ab;
4 | @main-orange: #F9483E;
5 |
6 | @side-bg: #F8F8F8;
7 | @title-bg: @side-bg;
8 | @border-color: #E5E5E5;
9 | @heavy-border-color: #888;
10 |
11 | @selected-bg: #bfc4da80;
12 | @hover-bg: #dcdcdc80;
13 |
14 | @weak-bg: #F3F3F3;
15 | @weak-text: #777;
16 | @footer-text: #CCC;
17 |
18 | @toolbar-btn-bg: #E9E9E9;
19 |
20 | @layout-space: 20px;
21 |
22 | @side-horizontal-padding: 10px;
23 | @side-vertical-spacing: 10px;
24 |
25 |
26 | @layout-extension-space-horizontal: 20px;
27 | @layout-extension-space-vertical: 20px;
28 |
--------------------------------------------------------------------------------
/apps/report/src/components/global-hover-preview.less:
--------------------------------------------------------------------------------
1 | @import './common.less';
2 |
3 | @max-size: 400px;
4 | .global-hover-preview {
5 | position: fixed;
6 | display: block;
7 | max-width: @max-size;
8 | max-height: @max-size;
9 | overflow: hidden;
10 | z-index: 10;
11 | text-align: center;
12 | border: 1px solid @border-color;
13 | box-sizing: border-box;
14 | background: @side-bg;
15 | box-shadow: 1px 1px 5px 0 rgba(0, 0, 0, 0.2);
16 |
17 | img {
18 | max-width: @max-size;
19 | max-height: @max-size;
20 | width: auto;
21 | height: auto;
22 | }
23 | }
--------------------------------------------------------------------------------
/apps/report/src/components/open-in-playground.less:
--------------------------------------------------------------------------------
1 | .playground-drawer {
2 | .ant-drawer-body {
3 | padding: 0;
4 | }
5 | }
6 |
--------------------------------------------------------------------------------
/apps/report/src/components/panel-title.less:
--------------------------------------------------------------------------------
1 | @import './common.less';
2 |
3 |
4 | .panel-title {
5 | background: @title-bg;
6 | border-top: 1px solid @border-color;
7 | border-bottom: 1px solid @border-color;
8 | margin-top: -1px;
9 | padding: 2px @side-horizontal-padding;
10 | .task-list-name {
11 | font-weight: bold;
12 | }
13 | }
14 |
--------------------------------------------------------------------------------
/apps/report/src/components/panel-title.tsx:
--------------------------------------------------------------------------------
1 | import './panel-title.less';
2 |
3 | const PanelTitle = (props: {
4 | title: string;
5 | subTitle?: string;
6 | }): JSX.Element => {
7 | const subTitleEl = props.subTitle ? (
8 | <div className="task-list-sub-name">{props.subTitle}</div>
9 | ) : null;
10 | return (
11 | <div className="panel-title">
12 | <div className="task-list-name">{props.title}</div>
13 | {subTitleEl}
14 | </div>
15 | );
16 | };
17 |
18 | export default PanelTitle;
19 |
--------------------------------------------------------------------------------
/apps/report/src/components/pixi-loader.tsx:
--------------------------------------------------------------------------------
1 | import 'pixi.js/unsafe-eval';
2 | import * as PIXI from 'pixi.js';
3 |
4 | const globalTextureMap = new Map<string, PIXI.Texture>();
5 |
6 | export const loadTexture = async (img: string) => {
7 | if (globalTextureMap.has(img)) return;
8 | return PIXI.Assets.load(img).then((texture) => {
9 | globalTextureMap.set(img, texture);
10 | });
11 | };
12 |
13 | export const getTextureFromCache = (name: string) => {
14 | return globalTextureMap.get(name);
15 | };
16 |
17 | export const getTexture = async (name: string) => {
18 | if (globalTextureMap.has(name)) {
19 | return globalTextureMap.get(name);
20 | }
21 |
22 | await loadTexture(name);
23 | return globalTextureMap.get(name);
24 | };
25 |
--------------------------------------------------------------------------------
/apps/report/src/components/side-item.tsx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/report/src/components/side-item.tsx
--------------------------------------------------------------------------------
/apps/report/src/components/timeline.less:
--------------------------------------------------------------------------------
1 | @import './common.less';
2 |
3 | @base-height: 110px;
4 |
5 | .timeline-wrapper {
6 | flex-basis: @base-height;
7 | flex-grow: 0;
8 | flex-shrink: 0;
9 |
10 | width: 100%;
11 | height: 100%;
12 | border-bottom: 1px solid @border-color;
13 | position: relative;
14 | box-sizing: border-box;
15 |
16 | .timeline-canvas-wrapper {
17 | width: 100%;
18 | height: 100%;
19 | }
20 |
21 | canvas {
22 | width: 100%;
23 | height: 100%;
24 | }
25 | }
--------------------------------------------------------------------------------
/apps/report/src/env.d.ts:
--------------------------------------------------------------------------------
1 | /// <reference types="@rsbuild/core/types" />
2 |
--------------------------------------------------------------------------------
/apps/report/src/index.tsx:
--------------------------------------------------------------------------------
1 | import ReactDOM from 'react-dom/client';
2 | import { App } from './App';
3 |
4 | const rootEl = document.getElementById('root');
5 |
6 | if (rootEl) {
7 | const root = ReactDOM.createRoot(rootEl);
8 |
9 | root.render(<App />);
10 | } else {
11 | console.error('no root element found');
12 | }
13 |
--------------------------------------------------------------------------------
/apps/report/src/types.ts:
--------------------------------------------------------------------------------
1 | import type { GroupedActionDump } from '@midscene/core';
2 | import type { AnimationScript } from '@midscene/visualizer';
3 |
4 | // Core visualization types
5 | export interface ExecutionDumpWithPlaywrightAttributes
6 | extends GroupedActionDump {
7 | attributes: Record<string, any>;
8 | }
9 |
10 | export interface VisualizerProps {
11 | logoAction?: () => void;
12 | dumps?: ExecutionDumpWithPlaywrightAttributes[];
13 | }
14 |
15 | // Store types
16 | export interface StoreState {
17 | dump: GroupedActionDump | null;
18 | _executionDumpLoadId: number;
19 | replayAllMode: boolean;
20 | setReplayAllMode: (mode: boolean) => void;
21 | allExecutionAnimation: AnimationScript[] | null;
22 | insightWidth: number | null;
23 | insightHeight: number | null;
24 | setGroupedDump: (dump: GroupedActionDump) => void;
25 | reset: () => void;
26 | }
27 |
--------------------------------------------------------------------------------
/apps/report/template/index.html:
--------------------------------------------------------------------------------
1 | <!doctype html>
2 | <html>
3 | <head>
4 | <title>Report - Midscene.js</title>
5 | <link
6 | rel="icon"
7 | type="image/png"
8 | sizes="32x32"
9 | href="https://lf3-static.bytednsdoc.com/obj/eden-cn/vhaeh7vhabf/favicon-32x32.png"
10 | />
11 | </head>
12 | <body>
13 | <!-- it should be replaced by the actual content -->
14 | <div id="<%= mountId %>" style="height: 100vh; width: 100vw"></div>
15 | </body>
16 | </html>
17 |
--------------------------------------------------------------------------------
/apps/report/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "lib": ["DOM", "ES2020"],
4 | "jsx": "react-jsx",
5 | "target": "ES2020",
6 | "noEmit": true,
7 | "skipLibCheck": true,
8 | "useDefineForClassFields": true,
9 |
10 | /* modules */
11 | "module": "ESNext",
12 | "isolatedModules": true,
13 | "resolveJsonModule": true,
14 | "moduleResolution": "bundler",
15 | "allowImportingTsExtensions": true,
16 |
17 | /* type checking */
18 | "strict": true,
19 | "noUnusedLocals": true,
20 | "noUnusedParameters": true,
21 |
22 | "paths": {
23 | "@/*": ["./src/*"]
24 | }
25 | },
26 | "include": ["src"]
27 | }
28 |
--------------------------------------------------------------------------------
/apps/site/.gitignore:
--------------------------------------------------------------------------------
1 | # Logs
2 | logs
3 | *.log
4 | npm-debug.log*
5 | yarn-debug.log*
6 | yarn-error.log*
7 | pnpm-debug.log*
8 | lerna-debug.log*
9 |
10 | node_modules
11 | dist
12 | dist-ssr
13 | *.local
14 |
15 | # Editor directories and files
16 | .vscode/*
17 | !.vscode/extensions.json
18 | .idea
19 | .DS_Store
20 | *.suo
21 | *.ntvs*
22 | *.njsproj
23 | *.sln
24 | *.sw?
25 | doc_build
26 |
27 | # Midscene.js dump files
28 | midscene_run/report
29 | midscene_run/dump
30 |
--------------------------------------------------------------------------------
/apps/site/build.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -e
4 |
5 | npm i -g pnpm@7
6 |
7 | pnpm install
8 |
9 | npm run build
10 |
11 | mkdir output
12 | mkdir output_resource
13 |
14 | cp -rf doc_build/* ./output
15 | cp -rf doc_build/* ./output_resource
16 |
17 |
18 | if [ -f route.json ]; then
19 | cp route.json ./output
20 | fi
21 |
--------------------------------------------------------------------------------
/apps/site/docs/en/common/prepare-key-for-further-use.mdx:
--------------------------------------------------------------------------------
1 | Prepare the config for the AI model you want to use. You can check the supported models in [Choose a model](../choose-a-model)
--------------------------------------------------------------------------------
/apps/site/docs/en/common/setup-env.mdx:
--------------------------------------------------------------------------------
1 | ## Setup AI model service
2 |
3 | Set your model configs into the environment variables. You may refer to [choose a model](../choose-a-model) for more details.
4 |
5 | ```bash
6 | # replace with your own
7 | export OPENAI_API_KEY="sk-abcdefghijklmnopqrstuvwxyz"
8 | ```
9 |
--------------------------------------------------------------------------------
/apps/site/docs/en/common/start-experience.mdx:
--------------------------------------------------------------------------------
1 | ## Start experiencing
2 |
3 | After the configuration, you can immediately experience Midscene. There are three main tabs in the extension:
4 |
5 | - **Action**: interact with the web page. This is also known as "Auto Planning". For example:
6 | ```
7 | type Midscene in the search box
8 | click the login button
9 | ```
10 |
11 | - **Query**: extract JSON data from the web page
12 |
13 | ```
14 | extract the user id from the page, return in \{ id: string \}
15 | ```
16 |
17 | - **Assert**: validate the page
18 |
19 | ```
20 | the page title is "Midscene"
21 | ```
22 |
23 | - **Tap**: perform a single tap on the element where you want to click. This is also known as "Instant Action".
24 |
25 | ```
26 | the login button
27 | ```
28 |
29 | Enjoy !
30 |
31 | > For the different between "Auto Planning" and "Instant Action", please refer to the [API](../API.html) document.
32 |
33 | ## Want to write some code ?
34 |
35 | After experiencing, you may want to write some code to integrate Midscene. There are multiple ways to do that. Please refer to the documents below:
36 |
37 | * [Automate with Scripts in YAML](../automate-with-scripts-in-yaml)
38 |
--------------------------------------------------------------------------------
/apps/site/docs/en/data-privacy.md:
--------------------------------------------------------------------------------
1 | # Data Privacy
2 |
3 | Midscene.js is an open-source project (GitHub: [Midscene](https://github.com/web-infra-dev/midscene/)) under the MIT license. You can see all the codes in the public repository.
4 |
5 | When using Midscene.js, your page data (including the screenshot) is sent directly to the AI model provider you choose. No third-party platform will have access to this data. All you need to be concerned about is the data privacy policy of the model provider.
6 |
7 | If you prefer building Midscene.js and its Chrome Extension in your own environment instead of using the published versions, you can refer to the [Contributing Guide](https://github.com/web-infra-dev/midscene/blob/main/CONTRIBUTING.md) to find building instructions.
8 |
9 |
--------------------------------------------------------------------------------
/apps/site/docs/en/llm-txt.mdx:
--------------------------------------------------------------------------------
1 | # LLMs.txt Documentation
2 |
3 | How to get tools like Cursor, Windstatic, GitHub Copilot, ChatGPT, and Claude to understand Midscene.js.
4 |
5 | We support LLMs.txt files for making the Midscene.js documentation available to large language models.
6 |
7 | ## Directory Overview
8 |
9 | The following files are available.
10 |
11 | - [llms.txt](https://midscenejs.com/llms.txt): The main LLMs.txt file
12 | - [llms-full.txt](https://midscenejs.com/llms-full.txt): The complete documentation for Midscene.js
13 |
14 |
15 | ## Usage
16 |
17 | ### Cursor
18 |
19 | Use `@Docs` feature in Cursor to include the LLMs.txt files in your project.
20 |
21 | [Read more](https://docs.cursor.com/context/@-symbols/@-docs)
22 |
23 | ### Windstatic
24 |
25 | Reference the LLMs.txt files using `@` or in your `.windsurfrules` files.
26 |
27 | [Read more](https://docs.windsurf.com/windsurf/getting-started#memories-and-rules)
28 |
29 |
30 |
--------------------------------------------------------------------------------
/apps/site/docs/en/quick-experience-with-android.mdx:
--------------------------------------------------------------------------------
1 | import StartExperience from './common/start-experience.mdx';
2 | import PrepareAndroid from './common/prepare-android.mdx';
3 |
4 | # Quick Experience with Android
5 |
6 | By using Midscene.js playground, you can quickly experience the main features of Midscene on Android devices, without needing to write any code.
7 |
8 | 
9 |
10 | <PrepareAndroid />
11 |
12 | ## Run Playground
13 |
14 | ```bash
15 | npx --yes @midscene/android-playground
16 | ```
17 |
18 | ## Config API Key
19 |
20 | Click the gear button to enter the configuration page and paste your API key config.
21 |
22 | 
23 |
24 | Refer to [Config Model and Provider](./model-provider) document, config the API Key.
25 |
26 | <StartExperience />
27 |
28 | * [Integrate javascript SDK with Android](./integrate-with-android)
29 |
--------------------------------------------------------------------------------
/apps/site/docs/public/android-playground.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/android-playground.png
--------------------------------------------------------------------------------
/apps/site/docs/public/android-set-env.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/android-set-env.png
--------------------------------------------------------------------------------
/apps/site/docs/public/android-usb-debug-en.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/android-usb-debug-en.png
--------------------------------------------------------------------------------
/apps/site/docs/public/android-usb-debug.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/android-usb-debug.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/0.10.0-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/0.10.0-2.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/0.10.0.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/0.10.0.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/0.11.0-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/0.11.0-2.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/0.11.0.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/0.11.0.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/0.13.0.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/0.13.0.jpeg
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/0.5.0-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/0.5.0-2.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/0.5.0.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/0.5.0.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/0.6.0-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/0.6.0-2.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/0.6.0-3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/0.6.0-3.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/0.6.0-4.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/0.6.0-4.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/0.6.0-5.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/0.6.0-5.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/0.6.0-6.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/0.6.0-6.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/0.6.0.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/0.6.0.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/0.9.0.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/0.9.0.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/ai-ide-convert-prompt-result.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/ai-ide-convert-prompt-result.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/ai-ide-convert-prompt.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/ai-ide-convert-prompt.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/android-playground-lark-poster-cn.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/android-playground-lark-poster-cn.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/android-playground-lark-poster-en.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/android-playground-lark-poster-en.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/coze-sidebar.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/coze-sidebar.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/export-video.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/export-video.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/logScreenshot-api.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/logScreenshot-api.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/report-coze-deep-think.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/report-coze-deep-think.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/report-instant-action.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/report-instant-action.png
--------------------------------------------------------------------------------
/apps/site/docs/public/blog/report-planning.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/blog/report-planning.png
--------------------------------------------------------------------------------
/apps/site/docs/public/bridge_in_extension.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/bridge_in_extension.jpg
--------------------------------------------------------------------------------
/apps/site/docs/public/cache/no-cache-time.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/cache/no-cache-time.png
--------------------------------------------------------------------------------
/apps/site/docs/public/cache/use-cache-time.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/cache/use-cache-time.png
--------------------------------------------------------------------------------
/apps/site/docs/public/midescene-playground-entry.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/midescene-playground-entry.jpg
--------------------------------------------------------------------------------
/apps/site/docs/public/midscene-bridge-mode.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/midscene-bridge-mode.jpg
--------------------------------------------------------------------------------
/apps/site/docs/public/midscene-extension.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/midscene-extension.jpg
--------------------------------------------------------------------------------
/apps/site/docs/public/midscene-icon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/midscene-icon.png
--------------------------------------------------------------------------------
/apps/site/docs/public/midscene_with_text_light.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/midscene_with_text_light.png
--------------------------------------------------------------------------------
/apps/site/docs/public/playground.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/playground.png
--------------------------------------------------------------------------------
/apps/site/docs/public/report.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/apps/site/docs/public/report.gif
--------------------------------------------------------------------------------
/apps/site/docs/zh/common/prepare-key-for-further-use.mdx:
--------------------------------------------------------------------------------
1 | 准备你想要使用的 AI 模型及其 API Key。你可以在 [选择模型](../choose-a-model) 文档中查看 Midscene.js 支持的模型。
--------------------------------------------------------------------------------
/apps/site/docs/zh/common/setup-env.mdx:
--------------------------------------------------------------------------------
1 | ## 配置 AI 模型服务
2 |
3 | 将你的模型配置写入环境变量。更多信息请查看 [选择 AI 模型](../choose-a-model)。
4 |
5 | ```bash
6 | # 替换为你的 API Key
7 | export OPENAI_API_KEY="sk-abcdefghijklmnopqrstuvwxyz"
8 | ```
9 |
--------------------------------------------------------------------------------
/apps/site/docs/zh/common/start-experience.mdx:
--------------------------------------------------------------------------------
1 | ## 开始体验
2 |
3 | 配置完成后，你可以立即开始使用 Midscene。它一共有三个关键操作 Tab：
4 |
5 | - **Action**: 与网页进行交互，这就是所谓的自动规划（Auto Planning）。比如
6 | ```
7 | 在搜索框中输入 Midscene
8 | 点击登录按钮
9 | ```
10 |
11 | - **Query**: 从界面中提取 JSON 数据
12 | ```
13 | 提取页面中的用户 ID，返回 \{ id: string \}
14 | ```
15 |
16 | - **Assert**: 验证页面
17 |
18 | ```
19 | 页面标题是 Midscene
20 | ```
21 |
22 | - **Tap**: 在某个元素上点击，这就是所谓的即时操作（Instant Action）。
23 | ```
24 | 登录按钮
25 | ```
26 |
27 | 快来试试吧！
28 |
29 | > 关于自动规划（Auto Planning）和即时操作（Instant Action）的区别，请参考 [API](../API.html) 文档。
30 |
31 | ## 想将 Midscene 集成到代码？
32 |
33 | 插件体验结束后，你可能想将 Midscene 集成到代码中。这里有几种不同集成形式的文档：
34 |
35 | * [使用 YAML 格式的自动化脚本](../automate-with-scripts-in-yaml)
36 |
--------------------------------------------------------------------------------
/apps/site/docs/zh/data-privacy.md:
--------------------------------------------------------------------------------
1 | # 数据隐私
2 |
3 | Midscene.js 是一个开源项目（GitHub: [Midscene](https://github.com/web-infra-dev/midscene/))，遵循 MIT 许可证。你可以在公开仓库中查看到所有代码。
4 |
5 | 当使用 Midscene.js 时，你的页面数据（包括截图）将直接发送到你配置的 AI 模型提供商。没有第三方平台会访问这些数据。你需要关注的是模型提供商的数据隐私政策。
6 |
7 | 如果你希望在你自己的环境中构建 Midscene.js 和它的 Chrome 扩展（而不是使用我们已发布的版本），你可以参考 [贡献指南](https://github.com/web-infra-dev/midscene/blob/main/CONTRIBUTING.md) 以找到构建说明。
8 |
9 |
--------------------------------------------------------------------------------
/apps/site/docs/zh/llm-txt.mdx:
--------------------------------------------------------------------------------
1 | # LLMs.txt 文档
2 |
3 | 如何让 Cursor、Windstatic、GitHub Copilot、ChatGPT 和 Claude 等工具理解 Midscene.js。
4 |
5 | 我们支持 LLMs.txt 文件，使 Midscene.js 的文档可供大型语言模型使用。
6 |
7 | ## 目录概览
8 |
9 | 以下文件可供使用：
10 |
11 | - [llms.txt](https://midscenejs.com/llms.txt)：主要的 LLMs.txt 文件
12 | - [llms-full.txt](https://midscenejs.com/llms-full.txt)：Midscene.js 的完整文档
13 |
14 |
15 | ## 使用方法
16 |
17 | ### Cursor
18 |
19 | 在 Cursor 中使用 `@Docs` 功能来将 LLMs.txt 文件包含到你的项目中。
20 |
21 | [阅读更多](https://docs.cursor.com/context/@-symbols/@-docs)
22 |
23 | ### Windstatic
24 |
25 | 使用 `@` 或在你的 `.windsurfrules` 文件中引用 LLMs.txt 文件。
26 |
27 | [阅读更多](https://docs.windsurf.com/windsurf/getting-started#memories-and-rules)
28 |
29 |
30 |
--------------------------------------------------------------------------------
/apps/site/docs/zh/quick-experience-with-android.mdx:
--------------------------------------------------------------------------------
1 | import StartExperience from './common/start-experience.mdx';
2 | import PrepareAndroid from './common/prepare-android.mdx';
3 |
4 | # 使用 Android Playground 快速体验
5 |
6 | 通过使用 Midscene.js Android 设备，你可以快速在 Android 设备上体验 Midscene 的主要功能，而无需编写任何代码。
7 |
8 | 该 Playground 和 `@midscene/android` 包共享一份代码，因此你可以将其视为 Midscene Android SDK 的一个 Playground 或调试工具。
9 |
10 | 
11 |
12 | <PrepareAndroid />
13 |
14 | ## 启动 Playground
15 |
16 | ```bash
17 | npx --yes @midscene/android-playground
18 | ```
19 |
20 | ## 配置 API Key
21 |
22 | 点击齿轮按钮，进入配置页面：
23 |
24 | 
25 |
26 | 参考 [配置模型和服务商](./model-provider) 文档，配置 API Key。
27 |
28 | <StartExperience />
29 |
30 | * [与 Android(adb) 集成](./integrate-with-android)
31 |
--------------------------------------------------------------------------------
/apps/site/i18n.json:
--------------------------------------------------------------------------------
1 | {
2 | "gettingStarted": {
3 | "en": "Getting Started",
4 | "zh": "开始"
5 | }
6 | }
7 |
--------------------------------------------------------------------------------
/apps/site/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "doc",
3 | "version": "1.0.0",
4 | "private": true,
5 | "type": "module",
6 | "scripts": {
7 | "dev": "rspress dev",
8 | "build": "rspress build",
9 | "preview": "rspress preview"
10 | },
11 | "engines": {
12 | "pnpm": ">=9.3.0",
13 | "node": ">=18.19.0"
14 | },
15 | "packageManager": "pnpm@9.3.0",
16 | "dependencies": {
17 | "querystring": "0.2.1",
18 | "react": "18.3.1",
19 | "react-dom": "18.3.1"
20 | },
21 | "devDependencies": {
22 | "@rspress/plugin-llms": "2.0.0-beta.4",
23 | "rspress-plugin-sitemap": "1.1.2",
24 | "@types/node": "^18.0.0",
25 | "rspress": "2.0.0-beta.4"
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/apps/site/route.json:
--------------------------------------------------------------------------------
1 | {
2 | "common": {
3 | "statusPages": {
4 | "404": "404.html"
5 | }
6 | },
7 | "routes": [
8 | {
9 | "isApi": false,
10 | "entryPath": "./",
11 | "isSpa": false,
12 | "isGarr": false,
13 | "isSSR": false,
14 | "redirect": "",
15 | "regexDomain": false
16 | }
17 | ]
18 | }
19 |
--------------------------------------------------------------------------------
/apps/site/styles/index.css:
--------------------------------------------------------------------------------
1 | .rspress-logo {
2 | height: 2.6rem;
3 | }
4 |
5 | :root {
6 | --rp-c-text-1: #0b140f;
7 | }
8 |
9 | /* Footer styles */
10 |
11 | .footer {
12 | background-color: transparent;
13 | text-align: center;
14 | }
15 |
16 | .footer::before,
17 | .footer::after {
18 | content: none;
19 | border: none;
20 | }
21 |
22 | .footer-content {
23 | display: flex;
24 | justify-content: center;
25 | align-items: center;
26 | flex-wrap: wrap;
27 | gap: 15px;
28 | border: none;
29 | }
30 |
31 | .footer-logo {
32 | width: 40px;
33 | border: none;
34 | }
35 |
36 | .footer-text {
37 | margin: 0;
38 | font-size: 14px;
39 | border: none;
40 | white-space: nowrap;
41 | }
42 |
43 | @media (max-width: 768px) {
44 | .md\:text-6xl {
45 | font-size: 3.3rem;
46 | line-height: 1;
47 | }
48 | .footer-content {
49 | flex-direction: column;
50 | align-items: center;
51 | text-align: center;
52 | }
53 | .footer-logo {
54 | margin-bottom: 10px;
55 | }
56 | .footer-bottom {
57 | text-align: center;
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/apps/site/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "jsx": "react-jsx",
4 | "esModuleInterop": true
5 | },
6 | "mdx": {
7 | "checkMdx": true
8 | }
9 | }
10 |
--------------------------------------------------------------------------------
/cspell.config.cjs:
--------------------------------------------------------------------------------
1 | const { banWords } = require('cspell-ban-words');
2 |
3 | module.exports = {
4 | version: '0.2',
5 | language: 'en',
6 | files: ['**/*.{ts,tsx,js,jsx,md,mdx}'],
7 | enableFiletypes: ['mdx'],
8 | ignoreRegExpList: [
9 | // ignore markdown anchors such as [modifyRsbuildConfig](#modifyrsbuildconfig)
10 | '#.*?\\)',
11 | ],
12 | ignorePaths: [
13 | 'dist',
14 | 'dist-*',
15 | 'compiled',
16 | 'coverage',
17 | 'doc_build',
18 | 'node_modules',
19 | 'pnpm-lock.yaml',
20 | 'midscene_run',
21 | ],
22 | flagWords: banWords,
23 | dictionaries: ['dictionary'],
24 | dictionaryDefinitions: [
25 | {
26 | name: 'dictionary',
27 | path: './scripts/dictionary.txt',
28 | addWords: true,
29 | },
30 | ],
31 | };
32 |
--------------------------------------------------------------------------------
/nx.json:
--------------------------------------------------------------------------------
1 | {
2 | "$schema": "./node_modules/nx/schemas/nx-schema.json",
3 | "namedInputs": {
4 | "build": [
5 | "default",
6 | "{projectRoot}/tsconfig.json",
7 | "{projectRoot}/package.json",
8 | "{projectRoot}/modern.config.*",
9 | "{projectRoot}/scripts/**/*",
10 | "!{projectRoot}/**/*.{md,mdx}",
11 | "!{projectRoot}/vitest.config.ts",
12 | "!{projectRoot}/**/?(*.)+(spec|test).ts"
13 | ]
14 | },
15 | "targetDefaults": {
16 | "dev": {
17 | "dependsOn": ["^build"]
18 | },
19 | "build": {
20 | "dependsOn": ["^build"],
21 | "cache": true,
22 | "inputs": ["build", "^build", "{workspaceRoot}/package.json"],
23 | "outputs": ["{projectRoot}/dist"]
24 | },
25 | "build:watch": {
26 | "dependsOn": ["^build"]
27 | },
28 | "test": {
29 | "cache": false
30 | },
31 | "e2e": {
32 | "dependsOn": ["^build"]
33 | },
34 | "e2e:ui": {
35 | "dependsOn": ["^build"]
36 | }
37 | },
38 | "defaultBase": "main"
39 | }
40 |
--------------------------------------------------------------------------------
/packages/android-playground/.gitignore:
--------------------------------------------------------------------------------
1 |
2 | # Midscene.js dump files
3 | midscene_run/report
4 | midscene_run/tmp
5 | static/
--------------------------------------------------------------------------------
/packages/android-playground/README.md:
--------------------------------------------------------------------------------
1 | ## Android playground
2 |
3 | Android is driven by adb, so you need install adb first:
4 | - [CLI](https://developer.android.com/tools/adb)
5 |
6 | ```bash
7 | npx @midscene/android-playground
8 | ```
--------------------------------------------------------------------------------
/packages/android-playground/bin/android-playground:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 |
3 | require('../dist/lib/index.js');
4 |
--------------------------------------------------------------------------------
/packages/android-playground/bin/server.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/android-playground/bin/server.bin
--------------------------------------------------------------------------------
/packages/android-playground/modern.config.ts:
--------------------------------------------------------------------------------
1 | import { defineConfig, moduleTools } from '@modern-js/module-tools';
2 |
3 | export default defineConfig({
4 | plugins: [moduleTools()],
5 | buildPreset: 'npm-library',
6 | buildConfig: {
7 | input: {
8 | index: './src/index.ts',
9 | },
10 | target: 'es2020',
11 | dts: {
12 | respectExternal: true,
13 | },
14 | },
15 | });
16 |
--------------------------------------------------------------------------------
/packages/android-playground/src/index.ts:
--------------------------------------------------------------------------------
1 | import path from 'node:path';
2 | import { AndroidAgent, AndroidDevice } from '@midscene/android';
3 | import {
4 | PLAYGROUND_SERVER_PORT,
5 | SCRCPY_SERVER_PORT,
6 | } from '@midscene/shared/constants';
7 | import PlaygroundServer from '@midscene/web/midscene-server';
8 | import ScrcpyServer from './scrcpy-server';
9 |
10 | const staticDir = path.join(__dirname, '../../static');
11 | const playgroundServer = new PlaygroundServer(
12 | AndroidDevice,
13 | AndroidAgent,
14 | staticDir,
15 | );
16 | const scrcpyServer = new ScrcpyServer();
17 |
18 | const main = async () => {
19 | const { default: open } = await import('open');
20 | try {
21 | await Promise.all([
22 | playgroundServer.launch(PLAYGROUND_SERVER_PORT),
23 | scrcpyServer.launch(SCRCPY_SERVER_PORT),
24 | ]);
25 | console.log(
26 | `Midscene playground server is running on http://localhost:${playgroundServer.port}`,
27 | );
28 | open(`http://localhost:${playgroundServer.port}`);
29 | } catch (error) {
30 | console.error('Failed to start servers:', error);
31 | process.exit(1);
32 | }
33 | };
34 |
35 | main();
36 |
--------------------------------------------------------------------------------
/packages/android-playground/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "allowJs": true,
4 | "baseUrl": ".",
5 | "declaration": true,
6 | "emitDeclarationOnly": true,
7 | "esModuleInterop": true,
8 | "forceConsistentCasingInFileNames": true,
9 | "isolatedModules": true,
10 | "jsx": "preserve",
11 | "lib": ["DOM", "ESNext"],
12 | "moduleResolution": "node",
13 | "resolveJsonModule": true,
14 | "rootDir": "src",
15 | "skipLibCheck": true,
16 | "strict": true,
17 | "module": "ES2020",
18 | "target": "es2020",
19 | "types": ["node"]
20 | },
21 | "exclude": ["**/node_modules"],
22 | "include": ["src"]
23 | }
24 |
--------------------------------------------------------------------------------
/packages/android/.gitignore:
--------------------------------------------------------------------------------
1 |
2 | # Midscene.js dump files
3 | midscene_run/report
4 | midscene_run/tmp
5 |
--------------------------------------------------------------------------------
/packages/android/README.md:
--------------------------------------------------------------------------------
1 | ## Documentation
2 |
3 | Automate UI actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML.
4 |
5 | See https://midscenejs.com/ for details.
6 |
7 | ## License
8 |
9 | Midscene is MIT licensed.
--------------------------------------------------------------------------------
/packages/android/bin/yadb:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/android/bin/yadb
--------------------------------------------------------------------------------
/packages/android/modern.config.ts:
--------------------------------------------------------------------------------
1 | import { defineConfig, moduleTools } from '@modern-js/module-tools';
2 |
3 | export default defineConfig({
4 | plugins: [moduleTools()],
5 | buildPreset: 'npm-library',
6 | buildConfig: {
7 | input: {
8 | index: './src/index.ts',
9 | },
10 | target: 'es2020',
11 | dts: {
12 | respectExternal: true,
13 | },
14 | },
15 | });
16 |
--------------------------------------------------------------------------------
/packages/android/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "@midscene/android",
3 | "version": "0.21.2",
4 | "description": "Android automation library for Midscene",
5 | "keywords": [
6 | "Android UI automation",
7 | "Android AI testing",
8 | "Android automation library",
9 | "Android automation tool",
10 | "Android use"
11 | ],
12 | "main": "./dist/lib/index.js",
13 | "types": "./dist/types/index.d.ts",
14 | "files": ["bin", "dist", "README.md"],
15 | "exports": {
16 | ".": {
17 | "types": "./dist/types/index.d.ts",
18 | "default": "./dist/lib/index.js"
19 | }
20 | },
21 | "scripts": {
22 | "dev": "modern dev",
23 | "build": "modern build -c ./modern.config.ts",
24 | "build:watch": "modern build -w -c ./modern.config.ts",
25 | "test": "vitest --run",
26 | "test:u": "vitest --run -u",
27 | "test:ai": "MIDSCENE_CACHE=true npm run test"
28 | },
29 | "dependencies": {
30 | "@midscene/core": "workspace:*",
31 | "@midscene/shared": "workspace:*",
32 | "@midscene/web": "workspace:*",
33 | "appium-adb": "12.12.1"
34 | },
35 | "devDependencies": {
36 | "@modern-js/module-tools": "2.60.6",
37 | "@types/node": "^18.0.0",
38 | "dotenv": "16.4.5",
39 | "typescript": "^5.8.3",
40 | "vitest": "3.0.5"
41 | },
42 | "license": "MIT"
43 | }
44 |
--------------------------------------------------------------------------------
/packages/android/src/index.ts:
--------------------------------------------------------------------------------
1 | export { AndroidDevice } from './page';
2 | export { AndroidAgent, agentFromAdbDevice } from './agent';
3 | export { getConnectedDevices } from './utils';
4 | export { overrideAIConfig } from '@midscene/shared/env';
5 |
--------------------------------------------------------------------------------
/packages/android/src/utils/index.ts:
--------------------------------------------------------------------------------
1 | import { ADB, type Device } from 'appium-adb';
2 | import { debugPage } from '../page';
3 |
4 | export async function getConnectedDevices(): Promise<Device[]> {
5 | try {
6 | const adb = await ADB.createADB({
7 | adbExecTimeout: 60000,
8 | });
9 | const devices = await adb.getConnectedDevices();
10 |
11 | debugPage(`Found ${devices.length} connected devices: `, devices);
12 |
13 | return devices;
14 | } catch (error: any) {
15 | console.error('Failed to get device list:', error);
16 | throw new Error(
17 | `Unable to get connected Android device list, please check https://midscenejs.com/integrate-with-android.html#faq : ${error.message}`,
18 | {
19 | cause: error,
20 | },
21 | );
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/packages/android/tests/ai/setting.test.ts:
--------------------------------------------------------------------------------
1 | import { describe, it, vi } from 'vitest';
2 | import { agentFromAdbDevice, getConnectedDevices } from '../../src';
3 |
4 | vi.setConfig({
5 | testTimeout: 90 * 1000,
6 | });
7 |
8 | describe(
9 | 'android integration',
10 | async () => {
11 | await it('Android settings page demo for scroll', async () => {
12 | const devices = await getConnectedDevices();
13 | const agent = await agentFromAdbDevice(devices[0].udid, {
14 | aiActionContext:
15 | 'If any location, permission, user agreement, etc. popup, click agree. If login page pops up, close it.',
16 | });
17 |
18 | await agent.launch('com.android.settings/.Settings');
19 |
20 | await agent.aiAction('scroll list to bottom');
21 | await agent.aiAction('open "More settings"');
22 | await agent.aiAction('scroll list to bottom');
23 | await agent.aiAction('scroll list to top');
24 | await agent.aiAction('swipe down one screen');
25 | await agent.aiAction('swipe up one screen');
26 | });
27 | },
28 | 360 * 1000,
29 | );
30 |
--------------------------------------------------------------------------------
/packages/android/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "allowJs": true,
4 | "baseUrl": ".",
5 | "declaration": true,
6 | "emitDeclarationOnly": true,
7 | "esModuleInterop": true,
8 | "forceConsistentCasingInFileNames": true,
9 | "isolatedModules": true,
10 | "jsx": "preserve",
11 | "lib": ["DOM", "ESNext"],
12 | "moduleResolution": "node",
13 | "resolveJsonModule": true,
14 | "rootDir": "src",
15 | "skipLibCheck": true,
16 | "strict": true,
17 | "module": "ES2020",
18 | "target": "es2020",
19 | "types": ["node"]
20 | },
21 | "exclude": ["**/node_modules"],
22 | "include": ["src"]
23 | }
24 |
--------------------------------------------------------------------------------
/packages/android/vitest.config.ts:
--------------------------------------------------------------------------------
1 | import path from 'node:path';
2 | import dotenv from 'dotenv';
3 | import { defineConfig } from 'vitest/config';
4 | import { version } from './package.json';
5 |
6 | /**
7 | * Read environment variables from file.
8 | * https://github.com/motdotla/dotenv
9 | */
10 | dotenv.config({
11 | path: path.join(__dirname, '../../.env'),
12 | });
13 |
14 | const testFiles = ['tests/ai/**/*.test.ts'];
15 |
16 | export default defineConfig({
17 | resolve: {
18 | alias: {
19 | '@': path.resolve(__dirname, 'src'),
20 | },
21 | },
22 | test: {
23 | include: testFiles,
24 | testTimeout: 3 * 60 * 1000, // Global timeout set to 10 seconds
25 | dangerouslyIgnoreUnhandledErrors: !!process.env.CI, // showcase.test.ts is not stable
26 | },
27 | define: {
28 | __VERSION__: `'${version}'`,
29 | },
30 | });
31 |
--------------------------------------------------------------------------------
/packages/cli/.gitignore:
--------------------------------------------------------------------------------
1 |
2 | # midscene.js
3 | midscene_run/
4 |
5 | status*.json
--------------------------------------------------------------------------------
/packages/cli/.npmignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 |
3 | .pnp
4 | .pnp.js
5 | .evn.local
6 | .env.*.local
7 | .history
8 | .rts*
9 | *.log*
10 | *.pid
11 | *.pid.*
12 | *.report
13 | *.lcov
14 | lib-cov
15 |
16 | doc_build/
17 | node_modules/
18 | .npm
19 | .lock-wscript
20 | .yarn-integrity
21 | .node_repl_history
22 | .nyc_output
23 | *.tsbuildinfo
24 | .eslintcache
25 | .sonarlint
26 |
27 | coverage/
28 | release/
29 | output/
30 | output_resource/
31 |
32 | .vscode/**/*
33 | !.vscode/settings.json
34 | !.vscode/extensions.json
35 | .idea/
36 |
37 | **/*/api/typings/auto-generated
38 | **/*/adapters/**/index.ts
39 | **/*/adapters/**/index.js
40 |
41 | src/
42 | midscene_run/
43 | log/
44 | docs/
45 | tests/
--------------------------------------------------------------------------------
/packages/cli/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2024-present Bytedance, Inc. and its affiliates.
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/packages/cli/README.md:
--------------------------------------------------------------------------------
1 | ## Documentation
2 |
3 | An AI-powered automation SDK can control the page, perform assertions, and extract data in JSON format using natural language.
4 |
5 | See https://midscenejs.com/ for details.
6 |
7 | ## License
8 |
9 | Midscene is MIT licensed.
--------------------------------------------------------------------------------
/packages/cli/bin/midscene:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 |
3 | require('../dist/lib/index.js')
--------------------------------------------------------------------------------
/packages/cli/modern.config.ts:
--------------------------------------------------------------------------------
1 | import { defineConfig, moduleTools } from '@modern-js/module-tools';
2 | import { version } from './package.json';
3 |
4 | export default defineConfig({
5 | plugins: [moduleTools()],
6 | buildPreset: 'npm-library',
7 | buildConfig: {
8 | input: {
9 | index: 'src/index.ts',
10 | },
11 | externals: ['node:buffer', 'puppeteer'],
12 | target: 'es2020',
13 | define: {
14 | __VERSION__: version,
15 | },
16 | sourceMap: true,
17 | },
18 | });
19 |
--------------------------------------------------------------------------------
/packages/cli/src/args.ts:
--------------------------------------------------------------------------------
1 | import type minimist from 'minimist';
2 |
3 | export type ArgumentValueType = string | boolean | number;
4 | export function findOnlyItemInArgs(
5 | args: minimist.ParsedArgs,
6 | name: string,
7 | ): string | boolean | number | undefined {
8 | const found = args[name];
9 | if (found === undefined) {
10 | return false;
11 | }
12 |
13 | if (Array.isArray(found) && found.length > 1) {
14 | throw new Error(`Multiple values found for ${name}`);
15 | }
16 |
17 | return found;
18 | }
19 |
20 | export interface OrderedArgumentItem {
21 | name: string;
22 | value: ArgumentValueType;
23 | }
24 |
25 | export function orderMattersParse(args: string[]): OrderedArgumentItem[] {
26 | const orderedArgs: OrderedArgumentItem[] = [];
27 | args.forEach((arg, index) => {
28 | if (arg.startsWith('--')) {
29 | const key = arg.substring(2);
30 | let value: ArgumentValueType =
31 | args[index + 1] && !args[index + 1].startsWith('--')
32 | ? args[index + 1]
33 | : true;
34 |
35 | if (typeof value === 'string' && /^\d+$/.test(value)) {
36 | value = Number.parseInt(value, 10);
37 | }
38 | orderedArgs.push({ name: key, value });
39 | }
40 | });
41 |
42 | return orderedArgs;
43 | }
44 |
--------------------------------------------------------------------------------
/packages/cli/src/http-server.d.ts:
--------------------------------------------------------------------------------
1 | declare module 'http-server' {
2 | export function createServer(options: http.ServerOptions): {
3 | server: http.Server;
4 | listen: (port: number, host: string, callback: () => void) => void;
5 | };
6 | }
7 |
--------------------------------------------------------------------------------
/packages/cli/tests/ai/bridge.test.ts:
--------------------------------------------------------------------------------
1 | import { execa } from 'execa';
2 | import { describe, test } from 'vitest';
3 | const cliBin = require.resolve('../../bin/midscene');
4 |
5 | const describeIf = process.env.BRIDGE_MODE ? describe : describe.skip;
6 |
7 | describeIf(
8 | 'bridge',
9 | {
10 | timeout: 1000 * 60 * 3,
11 | },
12 | () => {
13 | test('open new tab', async () => {
14 | const params = [
15 | './tests/midscene_scripts_bridge/new_tab/open-new-tab.yaml',
16 | ];
17 | await execa(cliBin, params);
18 | });
19 | },
20 | );
21 |
--------------------------------------------------------------------------------
/packages/cli/tests/midscene_scripts/local/local-error-message.yml:
--------------------------------------------------------------------------------
1 | target:
2 | serve: ./tests/server_root
3 | url: index.html
4 | tasks:
5 | - name: check title
6 | flow:
7 | - aiAssert: the content title is "Your App"
8 | errorMessage: "something error when assert title"
9 |
--------------------------------------------------------------------------------
/packages/cli/tests/midscene_scripts/local/local.yml:
--------------------------------------------------------------------------------
1 | target:
2 | serve: ./tests/server_root
3 | url: index.html
4 | tasks:
5 | - name: check title
6 | flow:
7 | - aiAssert: the content title is "My App"
8 |
--------------------------------------------------------------------------------
/packages/cli/tests/midscene_scripts/online/online.yaml:
--------------------------------------------------------------------------------
1 | # login to sauce demo, extract the items info into a json file, and assert the price of 'Sauce Labs Fleece Jacket'
2 |
3 | web:
4 | url: https://www.saucedemo.com/
5 | output: ./midscene_output/sauce-demo-items.json
6 |
7 | tasks:
8 | - name: login
9 | flow:
10 | - aiAction: type 'standard_user' in user name input, type 'secret_sauce' in password, click 'Login'
11 |
12 | - name: extract items info
13 | flow:
14 | - aiQuery: >
15 | {name: string, price: number, actionBtnName: string, imageUrl: string}[], return item name, price and the action button name on the lower right corner of each item, and the image url of each item (like 'Remove')
16 | name: items
17 | domIncluded: true
18 | - aiAssert: The price of 'Sauce Labs Fleece Jacket' is 49.99
19 |
20 | - name: run javascript code
21 | flow:
22 | - javascript: >
23 | document.title
24 | name: page-title
25 |
--------------------------------------------------------------------------------
/packages/cli/tests/midscene_scripts_bridge/current_tab/check_content.yaml:
--------------------------------------------------------------------------------
1 | target:
2 | bridgeMode: currentTab
3 | tasks:
4 | - name: check page content
5 | flow:
6 | - aiAssert: this is a web page
7 |
--------------------------------------------------------------------------------
/packages/cli/tests/midscene_scripts_bridge/new_tab/bing.yaml:
--------------------------------------------------------------------------------
1 | target:
2 | url: https://www.bing.com
3 | bridgeMode: newTabWithUrl
4 | tasks:
5 | - name: search weather
6 | flow:
7 | - sleep: 5000
8 | - ai: input 'weather today' in input box, click search button
9 | - sleep: 5000
10 |
11 | - name: check result
12 | flow:
13 | - aiAssert: the result shows the weather info
14 |
15 | - name: evaluateJavaScript
16 | flow:
17 | - javascript: alert('finished! this is javascript')
18 |
--------------------------------------------------------------------------------
/packages/cli/tests/midscene_scripts_bridge/new_tab/local.yml:
--------------------------------------------------------------------------------
1 | target:
2 | serve: ./tests/server_root
3 | url: index.html
4 | bridgeMode: newTabWithUrl
5 | tasks:
6 | - name: check title
7 | flow:
8 | - aiAssert: the content title is "My App"
9 |
--------------------------------------------------------------------------------
/packages/cli/tests/midscene_scripts_bridge/new_tab/open-new-tab.yaml:
--------------------------------------------------------------------------------
1 | target:
2 | url: https://www.bing.com
3 | forceSameTabNavigation: true
4 | bridgeMode: newTabWithUrl
5 | closeNewTabsAfterDisconnect: true
6 |
7 | tasks:
8 | - name: search weather
9 | flow:
10 | - sleep: 5000
11 | - ai: input 'midscene github' in input box, click search button
12 | - ai: click the first result
13 | - sleep: 5000
14 |
15 | - name: check result
16 | flow:
17 | - aiAssert: the page is "midscene github"
18 |
--------------------------------------------------------------------------------
/packages/cli/tests/server_root/index.html:
--------------------------------------------------------------------------------
1 | <h1>My App</h1>
2 | <p>This is a test page</p>
3 | <div>Width:</div>
4 | <div id="j_width"></div>
5 | <div>Height:</div>
6 | <div id="j_height"></div>
7 | <script>
8 | document.getElementById('j_width').innerText = window.innerWidth;
9 | document.getElementById('j_height').innerText = window.innerHeight;
10 | </script>
11 | </body>
--------------------------------------------------------------------------------
/packages/cli/tests/unit-test/__snapshots__/cli-utils.test.ts.snap:
--------------------------------------------------------------------------------
1 | // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
2 |
3 | exports[`cli utils > match exact file 1`] = `[]`;
4 |
5 | exports[`cli utils > match files 1`] = `
6 | [
7 | "tests/midscene_scripts/local/local-error-message.yml",
8 | "tests/midscene_scripts/local/local.yml",
9 | ]
10 | `;
11 |
12 | exports[`cli utils > match folder 1`] = `
13 | [
14 | "tests/midscene_scripts/local/local-error-message.yml",
15 | "tests/midscene_scripts/local/local.yml",
16 | "tests/midscene_scripts/online/online.yaml",
17 | ]
18 | `;
19 |
--------------------------------------------------------------------------------
/packages/cli/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "baseUrl": ".",
4 | "declaration": true,
5 | "emitDeclarationOnly": true,
6 | "esModuleInterop": true,
7 | "forceConsistentCasingInFileNames": true,
8 | "isolatedModules": true,
9 | "jsx": "preserve",
10 | "lib": ["ESNext", "DOM"],
11 | "moduleResolution": "node",
12 | "paths": {
13 | "@/*": ["./src/*"]
14 | },
15 | "resolveJsonModule": true,
16 | "rootDir": ".",
17 | "skipLibCheck": true,
18 | "strict": true
19 | },
20 | "exclude": ["**/node_modules"],
21 | "include": ["src", "tests"]
22 | }
23 |
--------------------------------------------------------------------------------
/packages/cli/vitest.config.ts:
--------------------------------------------------------------------------------
1 | import path from 'node:path';
2 | import dotenv from 'dotenv';
3 | import { defineConfig } from 'vitest/config';
4 |
5 | dotenv.config({
6 | path: path.join(__dirname, '../../.env'),
7 | });
8 |
9 | const enableAiTest = Boolean(process.env.AITEST);
10 | const basicTest = ['tests/unit-test/**/*.test.ts'];
11 |
12 | export default defineConfig({
13 | test: {
14 | include: enableAiTest ? ['tests/ai/**/*.test.ts'] : basicTest,
15 | testTimeout: 3 * 60 * 1000, // Global timeout set to 3 minutes
16 | },
17 | resolve: {
18 | alias: {
19 | '@': path.resolve(__dirname, 'src'),
20 | },
21 | },
22 | });
23 |
--------------------------------------------------------------------------------
/packages/core/.gitignore:
--------------------------------------------------------------------------------
1 |
2 | # midscene.js
3 | midscene_run/
4 | report/
--------------------------------------------------------------------------------
/packages/core/.npmignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 |
3 | .pnp
4 | .pnp.js
5 | .evn.local
6 | .env.*.local
7 | .history
8 | .rts*
9 | *.log*
10 | *.pid
11 | *.pid.*
12 | *.report
13 | *.lcov
14 | lib-cov
15 |
16 | doc_build/
17 | node_modules/
18 | .npm
19 | .lock-wscript
20 | .yarn-integrity
21 | .node_repl_history
22 | .nyc_output
23 | *.tsbuildinfo
24 | .eslintcache
25 | .sonarlint
26 |
27 | coverage/
28 | release/
29 | output/
30 | output_resource/
31 |
32 | .vscode/**/*
33 | !.vscode/settings.json
34 | !.vscode/extensions.json
35 | .idea/
36 |
37 | **/*/api/typings/auto-generated
38 | **/*/adapters/**/index.ts
39 | **/*/adapters/**/index.js
40 |
41 | src/
42 | midscene_run/
43 | log/
44 | docs/
45 | tests/
--------------------------------------------------------------------------------
/packages/core/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2024-present Bytedance, Inc. and its affiliates.
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/packages/core/README.md:
--------------------------------------------------------------------------------
1 | ## Documentation
2 |
3 | Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML.
4 |
5 | See https://midscenejs.com/ for details.
6 |
7 | ## License
8 |
9 | Midscene is MIT licensed.
--------------------------------------------------------------------------------
/packages/core/modern.config.ts:
--------------------------------------------------------------------------------
1 | import { defineConfig, moduleTools } from '@modern-js/module-tools';
2 | import { version } from './package.json';
3 |
4 | export default defineConfig({
5 | plugins: [moduleTools()],
6 | buildPreset: 'npm-library',
7 | buildConfig: {
8 | input: {
9 | index: 'src/index.ts',
10 | utils: 'src/utils.ts',
11 | tree: 'src/tree.ts',
12 | 'ai-model': 'src/ai-model/index.ts',
13 | },
14 | externals: ['langsmith'],
15 | target: 'es2020',
16 | define: {
17 | __VERSION__: version,
18 | },
19 | splitting: true,
20 | sourceMap: true,
21 | dts: {
22 | respectExternal: true,
23 | },
24 | },
25 | });
26 |
--------------------------------------------------------------------------------
/packages/core/src/ai-model/index.ts:
--------------------------------------------------------------------------------
1 | export { callToGetJSONObject, call as callAi } from './service-caller/index';
2 | export { systemPromptToLocateElement } from './prompt/llm-locator';
3 | export {
4 | describeUserPage,
5 | elementByPositionWithElementInfo,
6 | } from './prompt/util';
7 | export { generatePlaywrightTest } from './prompt/playwright-generator';
8 | export { generateYamlTest } from './prompt/yaml-generator';
9 |
10 | export type { ChatCompletionMessageParam } from 'openai/resources';
11 |
12 | export {
13 | AiLocateElement,
14 | AiExtractElementInfo,
15 | AiAssert,
16 | AiLocateSection,
17 | } from './inspect';
18 |
19 | export { plan } from './llm-planning';
20 | export { callAiFn, adaptBboxToRect } from './common';
21 | export { vlmPlanning, resizeImageForUiTars } from './ui-tars-planning';
22 |
23 | export { AIActionType } from './common';
24 |
--------------------------------------------------------------------------------
/packages/core/src/ai-model/prompt/common.ts:
--------------------------------------------------------------------------------
1 | import type { vlLocateMode } from '@midscene/shared/env';
2 | export function bboxDescription(vlMode: ReturnType<typeof vlLocateMode>) {
3 | if (vlMode === 'gemini') {
4 | return '2d bounding box as [ymin, xmin, ymax, xmax]';
5 | }
6 | return '2d bounding box as [xmin, ymin, xmax, ymax]';
7 | }
8 |
--------------------------------------------------------------------------------
/packages/core/src/ai-model/prompt/describe.ts:
--------------------------------------------------------------------------------
1 | import { getPreferredLanguage } from '@midscene/shared/env';
2 |
3 | export const elementDescriberInstruction = () => {
4 | return `Describe the element in the red rectangle for precise identification. Use ${getPreferredLanguage()}.
5 |
6 | Rules:
7 | 1. Start with element type (button, input, link, etc.)
8 | 2. Include key identifiers:
9 | - Text content: "with text 'Submit'"
10 | - Visual features: "blue background", "icon only"
11 | - Position: "top-right", "below search bar"
12 | 3. Keep description under 20 words
13 | 4. Don't mention the red rectangle
14 |
15 | Return JSON:
16 | {
17 | "description": "brief element type with key identifiers",
18 | "error"?: "error message if any"
19 | }`;
20 | };
21 |
--------------------------------------------------------------------------------
/packages/core/src/ai-model/service-caller/types.d.ts:
--------------------------------------------------------------------------------
1 | declare module 'dirty-json';
2 |
--------------------------------------------------------------------------------
/packages/core/src/image/index.ts:
--------------------------------------------------------------------------------
1 | export {
2 | imageInfo,
3 | imageInfoOfBase64,
4 | base64Encoded,
5 | resizeImg,
6 | transformImgPathToBase64,
7 | saveBase64Image,
8 | zoomForGPT4o,
9 | } from '@midscene/shared/img';
10 |
--------------------------------------------------------------------------------
/packages/core/src/index.ts:
--------------------------------------------------------------------------------
1 | import { Executor } from './ai-model/action-executor';
2 | import Insight from './insight/index';
3 | import { getVersion } from './utils';
4 |
5 | export {
6 | plan,
7 | describeUserPage,
8 | AiLocateElement,
9 | AiAssert,
10 | } from './ai-model/index';
11 |
12 | export { getAIConfig, MIDSCENE_MODEL_NAME } from '@midscene/shared/env';
13 |
14 | export type * from './types';
15 | export default Insight;
16 | export { Executor, Insight, getVersion };
17 |
18 | export type {
19 | MidsceneYamlScript,
20 | MidsceneYamlTask,
21 | MidsceneYamlFlowItem,
22 | MidsceneYamlFlowItemAIRightClick,
23 | } from './yaml';
24 |
--------------------------------------------------------------------------------
/packages/core/src/insight/utils.ts:
--------------------------------------------------------------------------------
1 | import type {
2 | DumpMeta,
3 | DumpSubscriber,
4 | InsightDump,
5 | PartialInsightDumpFromSDK,
6 | } from '@/types';
7 | import { getVersion } from '@/utils';
8 | import { MIDSCENE_MODEL_NAME, getAIConfig } from '@midscene/shared/env';
9 | import { uuid } from '@midscene/shared/utils';
10 |
11 | export function emitInsightDump(
12 | data: PartialInsightDumpFromSDK,
13 | dumpSubscriber?: DumpSubscriber,
14 | ) {
15 | const baseData: DumpMeta = {
16 | sdkVersion: getVersion(),
17 | logTime: Date.now(),
18 | model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',
19 | };
20 | const finalData: InsightDump = {
21 | logId: uuid(),
22 | ...baseData,
23 | ...data,
24 | };
25 |
26 | dumpSubscriber?.(finalData);
27 | }
28 |
--------------------------------------------------------------------------------
/packages/core/src/tree.ts:
--------------------------------------------------------------------------------
1 | export {
2 | truncateText,
3 | trimAttributes,
4 | descriptionOfTree,
5 | } from '@midscene/shared/extractor';
6 |
--------------------------------------------------------------------------------
/packages/core/tests/ai/assert/assert.test.ts:
--------------------------------------------------------------------------------
1 | import { AiAssert } from '@/ai-model';
2 | import { getContextFromFixture } from 'tests/evaluation';
3 | /* eslint-disable max-lines-per-function */
4 | import { describe, expect, it, vi } from 'vitest';
5 |
6 | vi.setConfig({
7 | testTimeout: 180 * 1000,
8 | hookTimeout: 30 * 1000,
9 | });
10 |
11 | describe('assert', () => {
12 | it('todo pass', async () => {
13 | const { context } = await getContextFromFixture('todo');
14 |
15 | const {
16 | content: { pass },
17 | } = await AiAssert({
18 | assertion: 'Three tasks have been added',
19 | context,
20 | });
21 | expect(pass).toBe(true);
22 | });
23 |
24 | it('todo error', async () => {
25 | const { context } = await getContextFromFixture('todo');
26 |
27 | const {
28 | content: { pass, thought },
29 | } = await AiAssert({
30 | assertion: 'There are four tasks in the task list',
31 | context,
32 | });
33 | expect(pass).toBe(false);
34 | });
35 | });
36 |
--------------------------------------------------------------------------------
/packages/core/tests/ai/extract/__snapshots__/extract.test.ts.snap:
--------------------------------------------------------------------------------
1 | // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
2 |
3 | exports[`extract > online order 1`] = `
4 | {
5 | "data": [
6 | {
7 | "name": "Very Tangerine Blast (Original)",
8 | "price": "6.82",
9 | },
10 | {
11 | "name": "Mango Grapefruit Pops",
12 | "price": "6.54",
13 | },
14 | ],
15 | "errors": [],
16 | }
17 | `;
18 |
19 | exports[`extract > todo obj 1`] = `
20 | {
21 | "data": [
22 | {
23 | "checked": false,
24 | "text": "Learn Python",
25 | },
26 | {
27 | "checked": false,
28 | "text": "Learn Rust",
29 | },
30 | {
31 | "checked": false,
32 | "text": "Learn AI",
33 | },
34 | ],
35 | "errors": [],
36 | }
37 | `;
38 |
--------------------------------------------------------------------------------
/packages/core/tests/ai/llm-inspect.test.ts:
--------------------------------------------------------------------------------
1 | import { AiLocateElement, AiLocateSection } from '@/ai-model';
2 | import { getContextFromFixture } from 'tests/evaluation';
3 | import { expect, test, vi } from 'vitest';
4 |
5 | vi.setConfig({
6 | testTimeout: 60 * 1000,
7 | });
8 |
9 | test(
10 | 'basic inspect',
11 | async () => {
12 | const { context } = await getContextFromFixture('todo');
13 |
14 | const startTime = Date.now();
15 | const { parseResult } = await AiLocateElement({
16 | context,
17 | targetElementDescription: 'input 输入框',
18 | });
19 | expect(parseResult.elements.length).toBe(1);
20 | },
21 | {
22 | timeout: 1000000,
23 | },
24 | );
25 |
26 | test('locate section', async () => {
27 | const { context } = await getContextFromFixture('todo');
28 | const { rect } = await AiLocateSection({
29 | context,
30 | sectionDescription: '搜索框',
31 | });
32 | expect(rect).toBeDefined();
33 | });
34 |
--------------------------------------------------------------------------------
/packages/core/tests/ai/llm-planning/__snapshots__/basic.test.ts.snap:
--------------------------------------------------------------------------------
1 | // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
2 |
3 | exports[`automation - llm planning > basic run 1`] = `
4 | {
5 | "timeMs": 3500,
6 | }
7 | `;
8 |
9 | exports[`automation - llm planning > basic run 2`] = `
10 | {
11 | "value": "Enter",
12 | }
13 | `;
14 |
--------------------------------------------------------------------------------
/packages/core/tests/ai/llm-planning/__snapshots__/input.test.ts.snap:
--------------------------------------------------------------------------------
1 | // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
2 |
3 | exports[`automation - planning input > input value 1`] = `
4 | [
5 | {
6 | "locate": {
7 | "id": "okgbn",
8 | "prompt": "",
9 | },
10 | "param": {
11 | "value": "learning english",
12 | },
13 | "thought": undefined,
14 | "type": "Input",
15 | },
16 | ]
17 | `;
18 |
19 | exports[`automation - planning input > input value 2`] = `
20 | [
21 | {
22 | "locate": {
23 | "id": "okgbn",
24 | "prompt": "",
25 | },
26 | "param": {
27 | "value": "learning english",
28 | },
29 | "thought": undefined,
30 | "type": "Input",
31 | },
32 | {
33 | "locate": null,
34 | "param": {
35 | "value": "Enter",
36 | },
37 | "thought": undefined,
38 | "type": "KeyboardPress",
39 | },
40 | ]
41 | `;
42 |
43 | exports[`automation - planning input > input value Add, delete, correct and check 1`] = `
44 | [
45 | {
46 | "locate": {
47 | "id": "okgbn",
48 | "prompt": "",
49 | },
50 | "param": {
51 | "value": "Learn English tomorrow",
52 | },
53 | "thought": undefined,
54 | "type": "Input",
55 | },
56 | ]
57 | `;
58 |
--------------------------------------------------------------------------------
/packages/core/tests/ai/llm-planning/__snapshots__/planning.test.ts.snap:
--------------------------------------------------------------------------------
1 | // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
2 |
3 | exports[`automation - planning > basic run 1`] = `
4 | {
5 | "timeMs": 3500,
6 | }
7 | `;
8 |
9 | exports[`automation - planning > basic run 2`] = `
10 | {
11 | "value": "Enter",
12 | }
13 | `;
14 |
--------------------------------------------------------------------------------
/packages/core/tests/ai/llm-section-locator.test.ts:
--------------------------------------------------------------------------------
1 | import { AiLocateElement } from '@/ai-model';
2 | import { AiLocateSection } from '@/ai-model/inspect';
3 | import { saveBase64Image } from '@/image';
4 | import { getTmpFile } from '@/utils';
5 | import { vlLocateMode } from '@midscene/shared/env';
6 | import { getContextFromFixture } from 'tests/evaluation';
7 | import { expect, test } from 'vitest';
8 |
9 | test.skipIf(!vlLocateMode())(
10 | 'locate section',
11 | async () => {
12 | const { context } = await getContextFromFixture('antd-tooltip');
13 | const { rect, imageBase64 } = await AiLocateSection({
14 | context,
15 | sectionDescription: 'the version info on the top right corner',
16 | });
17 | expect(rect).toBeDefined();
18 | expect(imageBase64).toBeDefined();
19 |
20 | const tmpFile = getTmpFile('jpg');
21 | await saveBase64Image({
22 | base64Data: imageBase64!,
23 | outputPath: tmpFile!,
24 | });
25 | },
26 | {
27 | timeout: 60 * 1000,
28 | },
29 | );
30 |
--------------------------------------------------------------------------------
/packages/core/tests/ai/ui-tars-planning/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/core/tests/ai/ui-tars-planning/output.png
--------------------------------------------------------------------------------
/packages/core/tests/fixtures/baidu.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/core/tests/fixtures/baidu.png
--------------------------------------------------------------------------------
/packages/core/tests/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "extends": "../tsconfig.json",
3 | "compilerOptions": {
4 | "baseUrl": "../",
5 | "rootDir": "../"
6 | },
7 | "include": ["**/*", "../src", "evaluation.ts"],
8 | "exclude": ["**/node_modules"]
9 | }
10 |
--------------------------------------------------------------------------------
/packages/core/tests/unit-test/__snapshots__/llm-planning.test.ts.snap:
--------------------------------------------------------------------------------
1 | // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
2 |
3 | exports[`llm planning - build yaml flow > build yaml flow 1`] = `
4 | [
5 | {
6 | "aiInput": "hello",
7 | "locate": "The input box for adding a new todo",
8 | },
9 | {
10 | "aiHover": "The second item 'Learn Rust' in the task list",
11 | },
12 | {
13 | "aiTap": "The input box labeled 'What needs to be done?'",
14 | },
15 | {
16 | "aiScroll": null,
17 | "direction": "down",
18 | "distance": 500,
19 | "locate": "some button",
20 | "scrollType": "once",
21 | },
22 | ]
23 | `;
24 |
--------------------------------------------------------------------------------
/packages/core/tests/unit-test/executor/__snapshots__/index.test.ts.snap:
--------------------------------------------------------------------------------
1 | // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
2 |
3 | exports[`executor > insight - basic run 1`] = `
4 | {
5 | "element": {
6 | "attributes": {
7 | "nodeType": "CONTAINER Node",
8 | },
9 | "center": [
10 | 250,
11 | 250,
12 | ],
13 | "id": "0",
14 | "indexId": undefined,
15 | "rect": {
16 | "height": 100,
17 | "left": 200,
18 | "top": 200,
19 | "width": 100,
20 | },
21 | "xpaths": [],
22 | },
23 | }
24 | `;
25 |
--------------------------------------------------------------------------------
/packages/core/tests/unit-test/mocks/intl-mock.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * Mock file for Intl.DateTimeFormat
3 | * This mock ensures that getTimeZoneInfo always returns non-China timezone
4 | */
5 |
6 | const originalIntl = global.Intl;
7 |
8 | export function mockNonChinaTimeZone() {
9 | const mockIntl = {
10 | DateTimeFormat: () => ({
11 | resolvedOptions: () => ({
12 | timeZone: 'America/New_York', // Using US timezone as non-China example
13 | }),
14 | }),
15 | };
16 |
17 | // @ts-ignore - Overriding readonly property
18 | global.Intl = { ...originalIntl, ...mockIntl };
19 | }
20 |
21 | export function restoreIntl() {
22 | // @ts-ignore - Restoring readonly property
23 | global.Intl = originalIntl;
24 | }
25 |
--------------------------------------------------------------------------------
/packages/core/tests/unit-test/prompt/__snapshots__/describe.test.ts.snap:
--------------------------------------------------------------------------------
1 | // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
2 |
3 | exports[`elementDescriberInstruction > should return the correct instruction 1`] = `
4 | "Describe the element in the red rectangle for precise identification. Use English.
5 |
6 | Rules:
7 | 1. Start with element type (button, input, link, etc.)
8 | 2. Include key identifiers:
9 | - Text content: "with text 'Submit'"
10 | - Visual features: "blue background", "icon only"
11 | - Position: "top-right", "below search bar"
12 | 3. Keep description under 20 words
13 | 4. Don't mention the red rectangle
14 |
15 | Return JSON:
16 | {
17 | "description": "brief element type with key identifiers",
18 | "error"?: "error message if any"
19 | }"
20 | `;
21 |
--------------------------------------------------------------------------------
/packages/core/tests/unit-test/prompt/assertion.test.ts:
--------------------------------------------------------------------------------
1 | import { systemPromptToAssert } from '@/ai-model/prompt/assertion';
2 | import { describe, expect, it, vi } from 'vitest';
3 |
4 | describe('Assertion prompt', () => {
5 | vi.mock('@midscene/shared/env', () => ({
6 | getPreferredLanguage: vi.fn().mockReturnValue('English'),
7 | }));
8 |
9 | it('return default when it is not UI-Tars', () => {
10 | const prompt = systemPromptToAssert({ isUITars: false });
11 | expect(prompt).toMatchSnapshot();
12 | });
13 |
14 | it('return UI-Tars specific when it is UI-Tars', () => {
15 | const prompt = systemPromptToAssert({ isUITars: true });
16 | expect(prompt).toMatchSnapshot();
17 | });
18 | });
19 |
--------------------------------------------------------------------------------
/packages/core/tests/unit-test/prompt/describe.test.ts:
--------------------------------------------------------------------------------
1 | import { elementDescriberInstruction } from '@/ai-model/prompt/describe';
2 | import { describe, expect, it, vi } from 'vitest';
3 |
4 | describe('elementDescriberInstruction', () => {
5 | vi.mock('@midscene/shared/env', () => ({
6 | getPreferredLanguage: vi.fn().mockReturnValue('English'),
7 | }));
8 |
9 | it('should return the correct instruction', () => {
10 | expect(elementDescriberInstruction()).toMatchSnapshot();
11 | });
12 | });
13 |
--------------------------------------------------------------------------------
/packages/core/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "baseUrl": ".",
4 | "sourceMap": true,
5 | "declaration": true,
6 | "emitDeclarationOnly": true,
7 | "esModuleInterop": true,
8 | "forceConsistentCasingInFileNames": true,
9 | "isolatedModules": true,
10 | "jsx": "preserve",
11 | "lib": ["DOM", "ESNext"],
12 | "moduleResolution": "node",
13 | "paths": {
14 | "@/*": ["./src/*"]
15 | },
16 | "resolveJsonModule": true,
17 | "rootDir": "./src",
18 | "skipLibCheck": true,
19 | "strict": true,
20 | "module": "ESNext",
21 | "target": "es2018"
22 | },
23 | "exclude": ["**/node_modules"],
24 | "include": ["src", "report"]
25 | }
26 |
--------------------------------------------------------------------------------
/packages/core/vitest.config.ts:
--------------------------------------------------------------------------------
1 | import path from 'node:path';
2 | import dotenv from 'dotenv';
3 | import { defineConfig } from 'vitest/config';
4 | import { version } from './package.json';
5 |
6 | /**
7 | * Read environment variables from file.
8 | * https://github.com/motdotla/dotenv
9 | */
10 | dotenv.config({
11 | path: path.join(__dirname, '../../.env'),
12 | override: true,
13 | debug: true,
14 | });
15 |
16 | const enableAiTest = Boolean(process.env.AITEST);
17 | const basicTest = ['tests/unit-test/**/*.test.ts'];
18 |
19 | export default defineConfig({
20 | test: {
21 | include: enableAiTest ? ['tests/ai/**/**.test.ts'] : basicTest,
22 | },
23 | define: {
24 | __VERSION__: `'${version}'`,
25 | },
26 | resolve: {
27 | alias: {
28 | '@': path.resolve(__dirname, 'src'),
29 | },
30 | },
31 | });
32 |
--------------------------------------------------------------------------------
/packages/evaluation/.gitignore:
--------------------------------------------------------------------------------
1 |
2 | # Midscene.js dump files
3 | midscene_run/report
4 | midscene_run/tmp
5 | page-data/screenspot-v2
6 |
--------------------------------------------------------------------------------
/packages/evaluation/README.md:
--------------------------------------------------------------------------------
1 |
2 | ### ScreenSpot-v2
3 |
4 | ```bash
5 | pip install huggingface_hub
6 | ```
7 |
8 | ```bash
9 | npm run download-screenspot-v2
10 | ```
--------------------------------------------------------------------------------
/packages/evaluation/data-generator/fixture.ts:
--------------------------------------------------------------------------------
1 | import type { PlayWrightAiFixtureType } from '@midscene/web';
2 | import { PlaywrightAiFixture } from '@midscene/web/playwright';
3 | import { test as base } from '@playwright/test';
4 |
5 | export const test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture());
6 |
--------------------------------------------------------------------------------
/packages/evaluation/data-generator/generator-headed.spec.ts:
--------------------------------------------------------------------------------
1 | import { PlaywrightWebPage } from '@midscene/web/playwright';
2 | import { test } from './fixture';
3 | import { generateExtractData, generateTestDataPath } from './utils';
4 |
5 | function sleep(time: number) {
6 | return new Promise((resolve) => {
7 | setTimeout(() => {
8 | resolve(0);
9 | }, time);
10 | });
11 | }
12 |
13 | test('taobao', async ({ page, ai }) => {
14 | const playwrightPage = new PlaywrightWebPage(page);
15 | page.setViewportSize({ width: 1280, height: 800 });
16 |
17 | await page.goto('https://www.taobao.com/');
18 |
19 | // for --ui
20 | await sleep(5000);
21 |
22 | await generateExtractData(playwrightPage, generateTestDataPath('taobao'));
23 | });
24 |
--------------------------------------------------------------------------------
/packages/evaluation/page-cases/assertion/online_order.json:
--------------------------------------------------------------------------------
1 | {
2 | "testDataPath": "online_order",
3 | "testCases": [
4 | {
5 | "prompt": "there are three tabs in the page, named 'Menu', 'Reviews', 'Merchant'",
6 | "expected": true
7 | },
8 | {
9 | "prompt": "there is a shopping bag icon on the top right of the page",
10 | "expected": true
11 | },
12 | {
13 | "prompt": "the 'select option' button is blue",
14 | "expected": false
15 | },
16 | {
17 | "prompt": "the tab name on the right of 'Reviews' is 'Merry'",
18 | "expected": false
19 | },
20 | {
21 | "prompt": "there are three tabs in the page, named 'Home', 'Order', 'Profile'",
22 | "expected": false
23 | },
24 | {
25 | "prompt": "The shopping bag icon on the top left of the page",
26 | "expected": false
27 | },
28 | {
29 | "prompt": "There is a homepage icon on the top right of the page, instead of a shopping bag icon",
30 | "expected": false
31 | }
32 | ]
33 | }
34 |
--------------------------------------------------------------------------------
/packages/evaluation/page-cases/assertion/online_order_list.json:
--------------------------------------------------------------------------------
1 | {
2 | "testDataPath": "online_order_list",
3 | "testCases": [
4 | {
5 | "prompt": "左侧有个菜单写着‘要简单’",
6 | "expected": true
7 | },
8 | {
9 | "prompt": "左侧有个菜单写着‘要米饭’",
10 | "expected": false
11 | },
12 | {
13 | "prompt": "有一杯饮料的名字是多肉忙忙",
14 | "expected": false
15 | }
16 | ]
17 | }
18 |
--------------------------------------------------------------------------------
/packages/evaluation/page-cases/inspect/antd-carousel.json-coordinates-annotated.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-cases/inspect/antd-carousel.json-coordinates-annotated.png
--------------------------------------------------------------------------------
/packages/evaluation/page-cases/inspect/aweme-login.json-coordinates-annotated.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-cases/inspect/aweme-login.json-coordinates-annotated.png
--------------------------------------------------------------------------------
/packages/evaluation/page-cases/inspect/aweme-play.json-coordinates-annotated.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-cases/inspect/aweme-play.json-coordinates-annotated.png
--------------------------------------------------------------------------------
/packages/evaluation/page-cases/inspect/online_order.json-coordinates-annotated.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-cases/inspect/online_order.json-coordinates-annotated.png
--------------------------------------------------------------------------------
/packages/evaluation/page-cases/inspect/online_order_list.json-coordinates-annotated.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-cases/inspect/online_order_list.json-coordinates-annotated.png
--------------------------------------------------------------------------------
/packages/evaluation/page-cases/inspect/taobao.json-coordinates-annotated.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-cases/inspect/taobao.json-coordinates-annotated.png
--------------------------------------------------------------------------------
/packages/evaluation/page-cases/inspect/todo.json-coordinates-annotated.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-cases/inspect/todo.json-coordinates-annotated.png
--------------------------------------------------------------------------------
/packages/evaluation/page-cases/planning/antd-form-vl.json-planning-coordinates-annotated.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-cases/planning/antd-form-vl.json-planning-coordinates-annotated.png
--------------------------------------------------------------------------------
/packages/evaluation/page-cases/planning/antd-tooltip-vl.json-planning-coordinates-annotated.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-cases/planning/antd-tooltip-vl.json-planning-coordinates-annotated.png
--------------------------------------------------------------------------------
/packages/evaluation/page-cases/planning/aweme-login-vl.json:
--------------------------------------------------------------------------------
1 | {
2 | "testDataPath": "aweme-login",
3 | "testCases": [
4 | {
5 | "prompt": "type 'user' in the username input box, type '98' in the age input box",
6 | "log": "type 'user' in the username input box",
7 | "response_planning": {
8 | "error": "Failed to plan actions: The current screenshot does not show a username input box or an age input box. The visible fields are for phone number and verification code, which do not match the instruction requirements."
9 | }
10 | }
11 | ]
12 | }
13 |
--------------------------------------------------------------------------------
/packages/evaluation/page-cases/planning/todo-vl.json-planning-coordinates-annotated.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-cases/planning/todo-vl.json-planning-coordinates-annotated.png
--------------------------------------------------------------------------------
/packages/evaluation/page-cases/section-locator/antd-tooltip.json:
--------------------------------------------------------------------------------
1 | {
2 | "testDataPath": "antd-tooltip",
3 | "testCases": [
4 | {
5 | "prompt": "the version info on the top right corner",
6 | "annotation_index_id": 1,
7 | "response_rect": {
8 | "left": 1245,
9 | "top": 0,
10 | "width": 588,
11 | "height": 200
12 | }
13 | },
14 | {
15 | "prompt": "‘位置有12个方向’上面的一圈按钮",
16 | "annotation_index_id": 2,
17 | "response_rect": {
18 | "left": 1049,
19 | "top": 526,
20 | "width": 639,
21 | "height": 386
22 | }
23 | },
24 | {
25 | "prompt": "‘位置有12个方向’上面的 Top 按钮",
26 | "annotation_index_id": 3,
27 | "response_rect": {
28 | "left": 1049,
29 | "top": 532,
30 | "width": 464,
31 | "height": 380
32 | }
33 | },
34 | {
35 | "prompt": "‘a series of buttons under 'show/hide' switch",
36 | "annotation_index_id": 4,
37 | "response_rect": {
38 | "left": 345,
39 | "top": 754,
40 | "width": 637,
41 | "height": 326
42 | }
43 | }
44 | ]
45 | }
46 |
--------------------------------------------------------------------------------
/packages/evaluation/page-cases/section-locator/antd-tooltip.json-coordinates-annotated.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-cases/section-locator/antd-tooltip.json-coordinates-annotated.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-carousel/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-carousel/input.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-carousel/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-carousel/output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-carousel/output_without_text.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-carousel/output_without_text.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-carousel/resize-output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-carousel/resize-output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-form/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-form/input.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-form/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-form/output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-form/output_without_text.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-form/output_without_text.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-form/resize-output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-form/resize-output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-pagination/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-pagination/input.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-pagination/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-pagination/output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-pagination/output_without_text.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-pagination/output_without_text.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-pagination/resize-output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-pagination/resize-output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-tooltip/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-tooltip/input.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-tooltip/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-tooltip/output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-tooltip/output_without_text.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-tooltip/output_without_text.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/antd-tooltip/resize-output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/antd-tooltip/resize-output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/aweme-login/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/aweme-login/input.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/aweme-login/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/aweme-login/output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/aweme-login/output_without_text.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/aweme-login/output_without_text.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/aweme-login/resize-output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/aweme-login/resize-output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/aweme-play/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/aweme-play/input.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/aweme-play/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/aweme-play/output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/aweme-play/output_without_text.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/aweme-play/output_without_text.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/aweme-play/resize-output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/aweme-play/resize-output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/githubstatus/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/githubstatus/input.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/githubstatus/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/githubstatus/output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/githubstatus/output_without_text.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/githubstatus/output_without_text.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/githubstatus/resize-output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/githubstatus/resize-output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/image-only/.gitignore:
--------------------------------------------------------------------------------
1 | tmp.*
2 | input.*
--------------------------------------------------------------------------------
/packages/evaluation/page-data/online_order/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/online_order/input.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/online_order/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/online_order/output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/online_order/output_without_text.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/online_order/output_without_text.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/online_order/resize-output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/online_order/resize-output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/online_order_list/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/online_order_list/input.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/online_order_list/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/online_order_list/output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/online_order_list/output_without_text.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/online_order_list/output_without_text.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/online_order_list/resize-output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/online_order_list/resize-output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/taobao/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/taobao/input.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/taobao/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/taobao/output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/taobao/output_without_text.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/taobao/output_without_text.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/taobao/resize-output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/taobao/resize-output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/todo-input-with-value/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/todo-input-with-value/input.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/todo-input-with-value/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/todo-input-with-value/output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/todo-input-with-value/output_without_text.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/todo-input-with-value/output_without_text.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/todo-input-with-value/resize-output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/todo-input-with-value/resize-output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/todo/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/todo/input.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/todo/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/todo/output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/todo/output_without_text.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/todo/output_without_text.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/todo/resize-output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/todo/resize-output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/visualstudio/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/visualstudio/input.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/visualstudio/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/visualstudio/output.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/visualstudio/output_without_text.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/visualstudio/output_without_text.png
--------------------------------------------------------------------------------
/packages/evaluation/page-data/visualstudio/resize-output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/evaluation/page-data/visualstudio/resize-output.png
--------------------------------------------------------------------------------
/packages/evaluation/playwright.config.ts:
--------------------------------------------------------------------------------
1 | import { defineConfig, devices } from '@playwright/test';
2 |
3 | /**
4 | * See https://playwright.dev/docs/test-configuration.
5 | */
6 | export default defineConfig({
7 | timeout: 90 * 1000,
8 | /* Run tests in files in parallel */
9 | fullyParallel: false,
10 | /* Fail the build on CI if you accidentally left test.only in the source code. */
11 | forbidOnly: Boolean(process.env.CI),
12 | /* Retry on CI only */
13 | retries: 0, //process.env.CI ? 1 : 0,
14 | /* Opt out of parallel tests on CI. */
15 | workers: process.env.CI ? 1 : undefined,
16 | /* Reporter to use. See https://playwright.dev/docs/test-reporters */
17 | // reporter: 'html',
18 | /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
19 | use: {
20 | /* Base URL to use in actions like `await page.goto('/')`. */
21 | // baseURL: 'http://127.0.0.1:3000',
22 |
23 | /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
24 | trace: 'on-first-retry',
25 | deviceScaleFactor: 1, // Device scaling factor
26 | },
27 | });
28 |
--------------------------------------------------------------------------------
/packages/evaluation/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "baseUrl": ".",
4 | "declaration": true,
5 | "emitDeclarationOnly": true,
6 | "esModuleInterop": true,
7 | "forceConsistentCasingInFileNames": true,
8 | "isolatedModules": true,
9 | "jsx": "preserve",
10 | "lib": ["DOM", "ESNext"],
11 | "moduleResolution": "node",
12 | "paths": {
13 | "@/*": ["./src/*"]
14 | },
15 | "target": "ESNext",
16 | "resolveJsonModule": true,
17 | "rootDir": "./",
18 | "skipLibCheck": true,
19 | "strict": true,
20 | "module": "ESNext"
21 | },
22 | "exclude": ["node_modules"],
23 | "include": ["src", "tests", "./playwright.config.ts", "./vitest.config"]
24 | }
25 |
--------------------------------------------------------------------------------
/packages/evaluation/vitest.config.ts:
--------------------------------------------------------------------------------
1 | import path from 'node:path';
2 | import dotenv from 'dotenv';
3 | import { defineConfig } from 'vitest/config';
4 |
5 | /**
6 | * Read environment variables from file.
7 | * https://github.com/motdotla/dotenv
8 | */
9 | dotenv.config({
10 | path: path.join(__dirname, '../../.env'),
11 | override: true,
12 | debug: true,
13 | });
14 |
15 | export default defineConfig({
16 | test: {
17 | include: ['tests/**.test.ts'],
18 | },
19 | resolve: {
20 | alias: {
21 | '@': path.resolve(__dirname, 'src'),
22 | },
23 | },
24 | });
25 |
--------------------------------------------------------------------------------
/packages/mcp/.gitignore:
--------------------------------------------------------------------------------
1 |
2 | # Midscene.js dump files
3 | midscene_run/dump
4 | midscene_run/report
5 | midscene_run/tmp
6 | midscene_run/log
7 | src/inject-tp.js
--------------------------------------------------------------------------------
/packages/mcp/README.md:
--------------------------------------------------------------------------------
1 | # Midscene MCP
2 |
3 | docs: https://midscenejs.com/mcp.html
4 |
--------------------------------------------------------------------------------
/packages/mcp/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "@midscene/mcp",
3 | "version": "0.21.2",
4 | "type": "module",
5 | "exports": {
6 | ".": {
7 | "types": "./dist/index.d.ts",
8 | "import": "./dist/index.js",
9 | "require": "./dist/index.cjs"
10 | }
11 | },
12 | "bin": "dist/index.cjs",
13 | "main": "./dist/index.cjs",
14 | "module": "./dist/index.js",
15 | "types": "./dist/index.d.ts",
16 | "files": ["dist"],
17 | "scripts": {
18 | "build": "rslib build",
19 | "dev": "rslib build --watch",
20 | "test": "vitest run",
21 | "inspect": "node scripts/inspect.mjs",
22 | "inspect2": "mcp-inspector node ./dist/test2.cjs"
23 | },
24 | "devDependencies": {
25 | "@modelcontextprotocol/inspector": "0.9.0",
26 | "@rslib/core": "^0.8.0",
27 | "@types/node": "^18.0.0",
28 | "typescript": "^5.8.3",
29 | "vitest": "3.0.5",
30 | "dotenv": "16.4.5",
31 | "@midscene/web": "workspace:*",
32 | "@midscene/report": "workspace:*",
33 | "@midscene/core": "workspace:*",
34 | "@midscene/shared": "workspace:*",
35 | "@modelcontextprotocol/sdk": "1.10.2",
36 | "zod": "3.24.3"
37 | },
38 | "dependencies": {
39 | "puppeteer": "24.2.0"
40 | },
41 | "license": "MIT"
42 | }
43 |
--------------------------------------------------------------------------------
/packages/mcp/src/prompts.ts:
--------------------------------------------------------------------------------
1 | import fs from 'node:fs';
2 | import path from 'node:path';
3 | import { PLAYWRIGHT_EXAMPLE_CODE } from '@midscene/shared/constants';
4 |
5 | const apiText = fs.readFileSync(path.join(__dirname, 'API.mdx'), 'utf-8');
6 |
7 | export const PROMPTS = {
8 | PLAYWRIGHT_CODE_EXAMPLE: PLAYWRIGHT_EXAMPLE_CODE,
9 | MIDSCENE_API_DOCS: apiText,
10 | };
11 |
--------------------------------------------------------------------------------
/packages/mcp/src/utils.ts:
--------------------------------------------------------------------------------
1 | // Deep merge utility function
2 | export function deepMerge(target: any, source: any): any {
3 | const output = Object.assign({}, target);
4 | if (typeof target !== 'object' || typeof source !== 'object') return source;
5 |
6 | for (const key of Object.keys(source)) {
7 | const targetVal = target[key];
8 | const sourceVal = source[key];
9 | if (Array.isArray(targetVal) && Array.isArray(sourceVal)) {
10 | // Deduplicate args/ignoreDefaultArgs, prefer source values
11 | output[key] = Array.from(
12 | new Set([
13 | ...(key === 'args' || key === 'ignoreDefaultArgs'
14 | ? targetVal.filter(
15 | (arg: string) =>
16 | !sourceVal.some(
17 | (launchArg: string) =>
18 | arg.startsWith('--') &&
19 | launchArg.startsWith(arg.split('=')[0]),
20 | ),
21 | )
22 | : targetVal),
23 | ...sourceVal,
24 | ]),
25 | );
26 | } else if (sourceVal instanceof Object && key in target) {
27 | output[key] = deepMerge(targetVal, sourceVal);
28 | } else {
29 | output[key] = sourceVal;
30 | }
31 | }
32 | return output;
33 | }
34 |
--------------------------------------------------------------------------------
/packages/mcp/tests/index.test.ts:
--------------------------------------------------------------------------------
1 | import { expect, test } from 'vitest';
2 |
3 | test('squared', () => {
4 | expect(4).toBe(4);
5 | });
6 |
--------------------------------------------------------------------------------
/packages/mcp/tests/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "extends": "../tsconfig.json",
3 | "include": [".", "../vitest.setup.ts"]
4 | }
5 |
--------------------------------------------------------------------------------
/packages/mcp/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "lib": ["ES2021"],
4 | "module": "ESNext",
5 | "noEmit": true,
6 | "strict": true,
7 | "skipLibCheck": true,
8 | "isolatedModules": true,
9 | "resolveJsonModule": true,
10 | "moduleResolution": "bundler",
11 | "useDefineForClassFields": true,
12 | "allowImportingTsExtensions": true
13 | },
14 | "include": ["src"]
15 | }
16 |
--------------------------------------------------------------------------------
/packages/mcp/vitest.config.ts:
--------------------------------------------------------------------------------
1 | import { defineConfig } from 'vitest/config';
2 |
3 | export default defineConfig({
4 | // Configure Vitest (https://vitest.dev/config/)
5 | test: {},
6 | });
7 |
--------------------------------------------------------------------------------
/packages/recorder/.gitignore:
--------------------------------------------------------------------------------
1 | # Local
2 | .DS_Store
3 | *.local
4 | *.log*
5 |
6 | # Dist
7 | node_modules
8 | dist/
9 |
10 | # IDE
11 | .vscode/*
12 | !.vscode/extensions.json
13 | .idea
14 |
--------------------------------------------------------------------------------
/packages/recorder/README.md:
--------------------------------------------------------------------------------
1 | # Rslib project
2 |
3 | ## Setup
4 |
5 | Install the dependencies:
6 |
7 | ```bash
8 | pnpm install
9 | ```
10 |
11 | ## Get started
12 |
13 | Build the library:
14 |
15 | ```bash
16 | pnpm build
17 | ```
18 |
19 | Build the library in watch mode:
20 |
21 | ```bash
22 | pnpm dev
23 | ```
24 |
--------------------------------------------------------------------------------
/packages/recorder/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "@midscene/recorder",
3 | "version": "0.21.2",
4 | "type": "module",
5 | "exports": {
6 | ".": {
7 | "types": "./dist/index.d.ts",
8 | "import": "./dist/index.js"
9 | }
10 | },
11 | "types": "./dist/index.d.ts",
12 | "files": ["dist"],
13 | "scripts": {
14 | "build": "rslib build",
15 | "dev": "rslib build --watch"
16 | },
17 | "devDependencies": {
18 | "@rsbuild/plugin-react": "^1.3.1",
19 | "@rslib/core": "^0.8.0",
20 | "@types/react": "^18.3.1",
21 | "react": "18.3.1",
22 | "typescript": "^5.8.3"
23 | },
24 | "dependencies": {
25 | "@ant-design/icons": "^5.3.1",
26 | "antd": "^5.21.6",
27 | "dayjs": "^1.11.11",
28 | "react-dom": "18.3.1",
29 | "@midscene/shared": "workspace:*"
30 | },
31 | "peerDependencies": {
32 | "react": "18.3.1",
33 | "react-dom": "18.3.1"
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/packages/recorder/rslib.config.ts:
--------------------------------------------------------------------------------
1 | import { pluginReact } from '@rsbuild/plugin-react';
2 | import { defineConfig } from '@rslib/core';
3 |
4 | export default defineConfig({
5 | lib: [
6 | {
7 | bundle: false,
8 | dts: true,
9 | format: 'esm',
10 | source: {
11 | entry: {
12 | index: './src/**',
13 | },
14 | },
15 | },
16 | {
17 | format: 'iife',
18 | dts: false,
19 | source: {
20 | entry: {
21 | 'recorder-iife': './src/recorder-iife-index.ts',
22 | },
23 | },
24 | },
25 | ],
26 | output: {
27 | target: 'web',
28 | },
29 | plugins: [pluginReact()],
30 | });
31 |
--------------------------------------------------------------------------------
/packages/recorder/src/Button.tsx:
--------------------------------------------------------------------------------
1 | import './button.css';
2 |
3 | interface ButtonProps {
4 | primary?: boolean;
5 | backgroundColor?: string;
6 | size?: 'small' | 'medium' | 'large';
7 | label: string;
8 | onClick?: () => void;
9 | }
10 |
11 | export const Button = ({
12 | primary = false,
13 | size = 'medium',
14 | backgroundColor,
15 | label,
16 | ...props
17 | }: ButtonProps) => {
18 | const mode = primary ? 'demo-button--primary' : 'demo-button--secondary';
19 | return (
20 | <button
21 | type="button"
22 | className={['demo-button', `demo-button--${size}`, mode].join(' ')}
23 | style={{ backgroundColor }}
24 | {...props}
25 | >
26 | {label}
27 | </button>
28 | );
29 | };
30 |
--------------------------------------------------------------------------------
/packages/recorder/src/RecordTimeline.css:
--------------------------------------------------------------------------------
1 | .record-timeline-screenshot-thumbnail {
2 | transition: transform 0.2s ease-in-out, box-shadow 0.2s ease-in-out;
3 | }
4 |
5 | .record-timeline-screenshot-thumbnail:hover {
6 | transform: scale(1.05);
7 | box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15);
8 | }
9 |
10 | .record-timeline-screenshot-popover {
11 | max-width: 600px;
12 | }
13 |
14 | .record-timeline-screenshot-popover .ant-popover-inner {
15 | padding: 8px;
16 | }
17 |
18 | .record-timeline-screenshot-full {
19 | border-radius: 4px;
20 | box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
21 | }
22 |
--------------------------------------------------------------------------------
/packages/recorder/src/button.css:
--------------------------------------------------------------------------------
1 | .demo-button {
2 | font-weight: 700;
3 | border: 0;
4 | border-radius: 3em;
5 | cursor: pointer;
6 | display: inline-block;
7 | line-height: 1;
8 | }
9 |
10 | .demo-button--primary {
11 | color: white;
12 | background-color: #1ea7fd;
13 | }
14 |
15 | .demo-button--secondary {
16 | color: #333;
17 | background-color: transparent;
18 | box-shadow: rgba(0, 0, 0, 0.15) 0px 0px 0px 1px inset;
19 | }
20 |
21 | .demo-button--small {
22 | font-size: 12px;
23 | padding: 10px 16px;
24 | }
25 |
26 | .demo-button--medium {
27 | font-size: 14px;
28 | padding: 11px 20px;
29 | }
30 |
31 | .demo-button--large {
32 | font-size: 16px;
33 | padding: 12px 24px;
34 | }
35 |
--------------------------------------------------------------------------------
/packages/recorder/src/index.tsx:
--------------------------------------------------------------------------------
1 | export { Button } from './Button';
2 | export {
3 | EventRecorder,
4 | type RecordedEvent,
5 | type ChromeRecordedEvent,
6 | convertToChromeEvents,
7 | } from './recorder';
8 | export { RecordTimeline } from './RecordTimeline';
9 |
--------------------------------------------------------------------------------
/packages/recorder/src/recorder-iife-index.ts:
--------------------------------------------------------------------------------
1 | import { EventRecorder } from './recorder';
2 |
3 | declare global {
4 | interface Window {
5 | EventRecorder: typeof EventRecorder;
6 | }
7 | }
8 |
9 | window.EventRecorder = EventRecorder;
10 | // const eventRecorder = new EventRecorder((event) => {
11 | // console.log(event);
12 | // });
13 |
14 | // eventRecorder.start();
15 |
--------------------------------------------------------------------------------
/packages/recorder/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "lib": ["DOM", "ES2021"],
4 | "module": "ESNext",
5 | "jsx": "react-jsx",
6 | "strict": true,
7 | "skipLibCheck": true,
8 | "isolatedModules": true,
9 | "resolveJsonModule": true,
10 | "moduleResolution": "bundler",
11 | "useDefineForClassFields": true
12 | },
13 | "include": ["src"]
14 | }
15 |
--------------------------------------------------------------------------------
/packages/shared/README.md:
--------------------------------------------------------------------------------
1 | ## Documentation
2 |
3 | An AI-powered automation SDK can control the page, perform assertions, and extract data in JSON format using natural language.
4 |
5 | See https://midscenejs.com/ for details.
6 |
7 | ## License
8 |
9 | Midscene is MIT licensed.
--------------------------------------------------------------------------------
/packages/shared/modern.config.ts:
--------------------------------------------------------------------------------
1 | import { defineConfig, moduleTools } from '@modern-js/module-tools';
2 |
3 | export default defineConfig({
4 | plugins: [moduleTools()],
5 | buildPreset: 'npm-library',
6 | buildConfig: {
7 | input: {
8 | index: './src/index.ts',
9 | img: './src/img/index.ts',
10 | constants: './src/constants/index.ts',
11 | extractor: './src/extractor/index.ts',
12 | 'extractor-debug': './src/extractor/debug.ts',
13 | fs: './src/node/fs.ts',
14 | utils: './src/utils.ts',
15 | logger: './src/logger.ts',
16 | common: './src/common.ts',
17 | 'us-keyboard-layout': './src/us-keyboard-layout.ts',
18 | env: './src/env.ts',
19 | types: './src/types/index.ts',
20 | },
21 | target: 'es2020',
22 | dts: {
23 | respectExternal: true,
24 | },
25 | },
26 | });
27 |
--------------------------------------------------------------------------------
/packages/shared/modern.inspect.config.ts:
--------------------------------------------------------------------------------
1 | import { defineConfig, moduleTools } from '@modern-js/module-tools';
2 |
3 | // It was split into two configuration files because of a bug in the build config array
4 | export default defineConfig({
5 | plugins: [moduleTools()],
6 | buildConfig: {
7 | platform: 'browser',
8 | buildType: 'bundle',
9 | format: 'iife',
10 | dts: false,
11 | input: {
12 | htmlElement: 'src/extractor/index.ts',
13 | htmlElementDebug: 'src/extractor/debug.ts',
14 | },
15 | autoExternal: false,
16 | outDir: 'dist/script',
17 | esbuildOptions: (options) => {
18 | options.globalName = 'midscene_element_inspector';
19 | return options;
20 | },
21 | target: 'es6',
22 | },
23 | });
24 |
--------------------------------------------------------------------------------
/packages/shared/src/constants/index.ts:
--------------------------------------------------------------------------------
1 | export const TEXT_SIZE_THRESHOLD = 9;
2 |
3 | export const TEXT_MAX_SIZE = 40;
4 |
5 | export const CONTAINER_MINI_HEIGHT = 3;
6 | export const CONTAINER_MINI_WIDTH = 3;
7 |
8 | export enum NodeType {
9 | CONTAINER = 'CONTAINER Node',
10 | FORM_ITEM = 'FORM_ITEM Node',
11 | BUTTON = 'BUTTON Node',
12 | A = 'Anchor Node',
13 | IMG = 'IMG Node',
14 | TEXT = 'TEXT Node',
15 | POSITION = 'POSITION Node',
16 | }
17 |
18 | export const PLAYGROUND_SERVER_PORT = 5800;
19 | export const SCRCPY_SERVER_PORT = 5700;
20 |
21 | export const DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT = 5000;
22 | export const DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT = 2000;
23 | export const DEFAULT_WAIT_FOR_NETWORK_IDLE_TIME = 300;
24 | export const DEFAULT_WAIT_FOR_NETWORK_IDLE_CONCURRENCY = 2;
25 |
26 | export { PLAYWRIGHT_EXAMPLE_CODE, YAML_EXAMPLE_CODE } from './example-code';
27 |
--------------------------------------------------------------------------------
/packages/shared/src/extractor/constants.ts:
--------------------------------------------------------------------------------
1 | export {
2 | NodeType,
3 | TEXT_MAX_SIZE,
4 | TEXT_SIZE_THRESHOLD,
5 | } from '../constants/index';
6 |
--------------------------------------------------------------------------------
/packages/shared/src/extractor/debug.ts:
--------------------------------------------------------------------------------
1 | import { webExtractTextWithPosition } from '.';
2 | import {
3 | setExtractTextWithPositionOnWindow,
4 | setMidsceneVisibleRectOnWindow,
5 | } from './util';
6 |
7 | console.log(webExtractTextWithPosition(document.body, true));
8 | console.log(JSON.stringify(webExtractTextWithPosition(document.body, true)));
9 | setExtractTextWithPositionOnWindow();
10 | setMidsceneVisibleRectOnWindow();
11 |
--------------------------------------------------------------------------------
/packages/shared/src/extractor/index.ts:
--------------------------------------------------------------------------------
1 | import type { NodeType } from '../constants/index';
2 |
3 | export interface ElementInfo {
4 | id: string;
5 | indexId: number;
6 | nodeHashId: string;
7 | xpaths?: string[];
8 | attributes: {
9 | nodeType: NodeType;
10 | [key: string]: string;
11 | };
12 | nodeType: NodeType;
13 | content: string;
14 | rect: { left: number; top: number; width: number; height: number };
15 | center: [number, number];
16 | isVisible: boolean;
17 | }
18 |
19 | export interface ElementNode {
20 | node: ElementInfo | null;
21 | children: ElementNode[];
22 | }
23 |
24 | export {
25 | descriptionOfTree,
26 | traverseTree,
27 | treeToList,
28 | truncateText,
29 | trimAttributes,
30 | } from './tree';
31 |
32 | export { extractTextWithPosition as webExtractTextWithPosition } from './web-extractor';
33 |
34 | export { extractTreeNode as webExtractNodeTree } from './web-extractor';
35 |
36 | export { extractTreeNodeAsString as webExtractNodeTreeAsString } from './web-extractor';
37 |
38 | export { setNodeHashCacheListOnWindow, getNodeFromCacheList } from './util';
39 |
40 | export {
41 | getXpathsById,
42 | getNodeInfoByXpath,
43 | getElementInfoByXpath,
44 | } from './locator';
45 |
46 | export { generateElementByPosition } from './dom-util';
47 |
--------------------------------------------------------------------------------
/packages/shared/src/img/get-jimp.ts:
--------------------------------------------------------------------------------
1 | // @ts-ignore
2 | import type Jimp from 'jimp/browser/lib/jimp.js';
3 |
4 | const ifInBrowser = typeof window !== 'undefined';
5 | export default async function getJimp(): Promise<typeof Jimp> {
6 | if (ifInBrowser) {
7 | // @ts-ignore
8 | await import('jimp/browser/lib/jimp.js');
9 | return (window as any).Jimp;
10 | }
11 | // return Jimp;
12 | // @ts-ignore
13 | return (await import('jimp')).default;
14 | }
15 |
--------------------------------------------------------------------------------
/packages/shared/src/img/index.ts:
--------------------------------------------------------------------------------
1 | export {
2 | imageInfo,
3 | imageInfoOfBase64,
4 | bufferFromBase64,
5 | base64Encoded,
6 | isValidPNGImageBuffer,
7 | } from './info';
8 | export {
9 | trimImage,
10 | resizeImg,
11 | resizeImgBase64,
12 | transformImgPathToBase64,
13 | zoomForGPT4o,
14 | saveBase64Image,
15 | paddingToMatchBlock,
16 | paddingToMatchBlockByBase64,
17 | cropByRect,
18 | jimpFromBase64,
19 | jimpToBase64,
20 | } from './transform';
21 | export { processImageElementInfo, compositeElementInfoImg } from './box-select';
22 | export { drawBoxOnImage, savePositionImg } from './draw-box';
23 |
--------------------------------------------------------------------------------
/packages/shared/src/img/jimp.d.ts:
--------------------------------------------------------------------------------
1 | declare module 'jimp/browser/lib/jimp.js' {
2 | import Jimp from 'jimp';
3 | export default Jimp;
4 | }
5 |
--------------------------------------------------------------------------------
/packages/shared/src/index.ts:
--------------------------------------------------------------------------------
1 | export default {};
2 |
--------------------------------------------------------------------------------
/packages/shared/src/modern-app-env.d.ts:
--------------------------------------------------------------------------------
1 | /// <reference types='@modern-js/module-tools/types' />
2 |
--------------------------------------------------------------------------------
/packages/shared/src/types/index.ts:
--------------------------------------------------------------------------------
1 | import type { NodeType } from '../constants';
2 | import type { ElementInfo } from '../extractor';
3 |
4 | export interface Point {
5 | left: number;
6 | top: number;
7 | }
8 |
9 | export interface Size {
10 | width: number; // device independent window size
11 | height: number;
12 | dpr?: number; // the scale factor of the screenshots
13 | }
14 |
15 | export type Rect = Point & Size & { zoom?: number };
16 |
17 | export abstract class BaseElement {
18 | abstract id: string;
19 |
20 | abstract indexId?: number; // markerId for web
21 |
22 | abstract attributes: {
23 | nodeType: NodeType;
24 | [key: string]: string;
25 | };
26 |
27 | abstract content: string;
28 |
29 | abstract rect: Rect;
30 |
31 | abstract center: [number, number];
32 |
33 | abstract xpaths?: string[];
34 |
35 | abstract isVisible: boolean;
36 | }
37 |
38 | export interface ElementTreeNode<
39 | ElementType extends BaseElement = BaseElement,
40 | > {
41 | node: ElementType | null;
42 | children: ElementTreeNode<ElementType>[];
43 | }
44 |
45 | export interface WebElementInfo extends ElementInfo {
46 | zoom: number;
47 | }
48 |
--------------------------------------------------------------------------------
/packages/shared/tests/fixtures/2x2.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/shared/tests/fixtures/2x2.jpeg
--------------------------------------------------------------------------------
/packages/shared/tests/fixtures/baidu.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/shared/tests/fixtures/baidu.png
--------------------------------------------------------------------------------
/packages/shared/tests/fixtures/colorful.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/shared/tests/fixtures/colorful.png
--------------------------------------------------------------------------------
/packages/shared/tests/fixtures/heytea.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/shared/tests/fixtures/heytea.jpeg
--------------------------------------------------------------------------------
/packages/shared/tests/fixtures/icon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/shared/tests/fixtures/icon.png
--------------------------------------------------------------------------------
/packages/shared/tests/fixtures/long-text.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/shared/tests/fixtures/long-text.png
--------------------------------------------------------------------------------
/packages/shared/tests/fixtures/reference-of-list.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/shared/tests/fixtures/reference-of-list.png
--------------------------------------------------------------------------------
/packages/shared/tests/fixtures/reference.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/shared/tests/fixtures/reference.png
--------------------------------------------------------------------------------
/packages/shared/tests/fixtures/table.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/shared/tests/fixtures/table.png
--------------------------------------------------------------------------------
/packages/shared/tests/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "extends": "../tsconfig.json",
3 | "compilerOptions": {
4 | "baseUrl": "../",
5 | "rootDir": "../"
6 | },
7 | "include": ["**/*", "../src"],
8 | "exclude": ["**/node_modules"]
9 | }
10 |
--------------------------------------------------------------------------------
/packages/shared/tests/unit-test/fs.test.ts:
--------------------------------------------------------------------------------
1 | import { describe, expect, it } from 'vitest';
2 | import { getRunningPkgInfo } from '../../src/node/fs';
3 |
4 | describe('fs', () => {
5 | it('getRunningPkgInfo', () => {
6 | const info = getRunningPkgInfo();
7 | expect(info).toBeDefined();
8 | expect(info?.dir).toMatch(/shared$/);
9 | });
10 |
11 | it('getRunningPkgInfo - no package.json', () => {
12 | const info = getRunningPkgInfo('/home');
13 | expect(info?.dir).toEqual('/home');
14 | });
15 | });
16 |
--------------------------------------------------------------------------------
/packages/shared/tests/utils.ts:
--------------------------------------------------------------------------------
1 | import { join } from 'node:path';
2 |
3 | export function getFixture(name: string) {
4 | return join(__dirname, 'fixtures', name);
5 | }
6 |
--------------------------------------------------------------------------------
/packages/shared/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "allowJs": true,
4 | "baseUrl": ".",
5 | "declaration": true,
6 | "emitDeclarationOnly": true,
7 | "esModuleInterop": true,
8 | "forceConsistentCasingInFileNames": true,
9 | "isolatedModules": true,
10 | "jsx": "preserve",
11 | "lib": ["DOM", "ESNext"],
12 | "moduleResolution": "node",
13 | "resolveJsonModule": true,
14 | "rootDir": "src",
15 | "skipLibCheck": true,
16 | "strict": true,
17 | "module": "ES2020",
18 | "target": "es2020",
19 | "types": ["node"]
20 | },
21 | "exclude": ["**/node_modules"],
22 | "include": ["src", "playwright.config.ts"]
23 | }
24 |
--------------------------------------------------------------------------------
/packages/shared/vitest.config.ts:
--------------------------------------------------------------------------------
1 | import path from 'node:path';
2 | import dotenv from 'dotenv';
3 | import { defineConfig } from 'vitest/config';
4 |
5 | /**
6 | * Read environment variables from file.
7 | * https://github.com/motdotla/dotenv
8 | */
9 | dotenv.config({
10 | path: path.join(__dirname, '../../.env'),
11 | });
12 |
13 | const enableAiTest = Boolean(process.env.AITEST);
14 | const basicTest = ['tests/unit-test/**/*.test.ts'];
15 |
16 | export default defineConfig({
17 | test: {
18 | include: enableAiTest ? ['tests/ai/**/*.test.ts', ...basicTest] : basicTest,
19 | },
20 | resolve: {
21 | alias: {
22 | '@': path.resolve(__dirname, 'src'),
23 | },
24 | },
25 | });
26 |
--------------------------------------------------------------------------------
/packages/visualizer/.gitignore:
--------------------------------------------------------------------------------
1 | unpacked-extension/lib/
2 | unpacked-extension/scripts/
3 | unpacked-extension/pages/
4 |
5 |
6 | # Midscene.js dump files
7 | midscene_run/report
8 | midscene_run/dump
9 |
--------------------------------------------------------------------------------
/packages/visualizer/.npmignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 |
3 | .pnp
4 | .pnp.js
5 | .evn.local
6 | .env.*.local
7 | .history
8 | .rts*
9 | *.log*
10 | *.pid
11 | *.pid.*
12 | *.report
13 | *.lcov
14 | lib-cov
15 |
16 | doc_build/
17 | node_modules/
18 | .npm
19 | .lock-wscript
20 | .yarn-integrity
21 | .node_repl_history
22 | .nyc_output
23 | *.tsbuildinfo
24 | .eslintcache
25 | .sonarlint
26 |
27 | coverage/
28 | release/
29 | output/
30 | output_resource/
31 |
32 | .vscode/**/*
33 | !.vscode/settings.json
34 | !.vscode/extensions.json
35 | .idea/
36 |
37 | **/*/api/typings/auto-generated
38 | **/*/adapters/**/index.ts
39 | **/*/adapters/**/index.js
40 |
--------------------------------------------------------------------------------
/packages/visualizer/README.md:
--------------------------------------------------------------------------------
1 | ## Documentation
2 |
3 | An AI-powered automation SDK can control the page, perform assertions, and extract data in JSON format using natural language.
4 |
5 | See https://midscenejs.com/ for details.
6 |
7 | ## License
8 |
9 | Midscene is MIT licensed.
--------------------------------------------------------------------------------
/packages/visualizer/src/blank_polyfill.ts:
--------------------------------------------------------------------------------
1 | export default {};
2 |
--------------------------------------------------------------------------------
/packages/visualizer/src/component/blackboard.less:
--------------------------------------------------------------------------------
1 | @import './common.less';
2 |
3 | .blackboard {
4 | .footer {
5 | color: #aaa;
6 | }
7 |
8 | ul {
9 | padding-left: 0px;
10 | }
11 |
12 | li {
13 | list-style: none;
14 | }
15 |
16 | .bottom-tip {
17 | height: 30px;
18 | }
19 |
20 | .bottom-tip-item {
21 | max-width: 500px;
22 | color: #AAA;
23 | text-overflow: ellipsis;
24 | word-wrap: break-word;
25 | }
26 | }
27 |
28 | .blackboard-filter {
29 | margin: 10px 0;
30 | }
31 |
32 | .blackboard-main-content {
33 | canvas {
34 | width: 100%;
35 | border: 1px solid @heavy-border-color;
36 | box-sizing: border-box;
37 | }
38 | }
--------------------------------------------------------------------------------
/packages/visualizer/src/component/common.less:
--------------------------------------------------------------------------------
1 | @main-text: #3b3b3b;
2 |
3 | @primary-color: #2B83FF;
4 | @main-orange: #F9483E;
5 |
6 | @side-bg: #F8F8F8;
7 | @title-bg: @side-bg;
8 | @border-color: #E5E5E5;
9 | @heavy-border-color: #888;
10 |
11 | @selected-bg: #bfc4da80;
12 | @hover-bg: #dcdcdc80;
13 |
14 | @weak-bg: #F3F3F3;
15 | @weak-text: #777;
16 | @footer-text: #CCC;
17 |
18 | @toolbar-btn-bg: #E9E9E9;
19 |
20 | @layout-space: 20px;
21 |
22 | @side-horizontal-padding: 10px;
23 | @side-vertical-spacing: 10px;
24 |
25 |
26 | @layout-extension-space-horizontal: 20px;
27 | @layout-extension-space-vertical: 20px;
--------------------------------------------------------------------------------
/packages/visualizer/src/component/describer.less:
--------------------------------------------------------------------------------
1 | .image-describer {
2 | position: relative;
3 |
4 | .describe-text {
5 | box-sizing: border-box;
6 | position: absolute;
7 | background: #000;
8 | width: 100%;
9 | height: 30px;
10 | left: 0;
11 | bottom: 0;
12 | color: #FFF;
13 | font-size: 12px;
14 | padding: 10px;
15 | }
16 |
17 | .describe-text.success {
18 | background: #047704;
19 | }
20 |
21 | .describe-text.error {
22 | background: #870707;
23 | }
24 | }
--------------------------------------------------------------------------------
/packages/visualizer/src/component/github-star.less:
--------------------------------------------------------------------------------
1 | .github-star {
2 | height: 22px;
3 | }
--------------------------------------------------------------------------------
/packages/visualizer/src/component/github-star.tsx:
--------------------------------------------------------------------------------
1 | import './github-star.less';
2 |
3 | export const GithubStar = () => {
4 | return (
5 | <a
6 | href="https://github.com/web-infra-dev/midscene"
7 | target="_blank"
8 | rel="noreferrer"
9 | style={{ display: 'flex', alignItems: 'center' }}
10 | >
11 | <img
12 | className="github-star"
13 | src="https://img.shields.io/github/stars/web-infra-dev/midscene?style=social"
14 | alt="Github star"
15 | style={{ display: 'block' }}
16 | />
17 | </a>
18 | );
19 | };
20 |
--------------------------------------------------------------------------------
/packages/visualizer/src/component/logo.less:
--------------------------------------------------------------------------------
1 | .logo img {
2 | height: 30px;
3 | line-height: 30px;
4 | vertical-align: baseline;
5 | vertical-align: -webkit-baseline-middle;
6 | }
7 |
8 | .logo-with-star-wrapper {
9 | display: flex;
10 | flex-direction: row;
11 | justify-content: space-between;
12 | }
--------------------------------------------------------------------------------
/packages/visualizer/src/component/logo.tsx:
--------------------------------------------------------------------------------
1 | import './logo.less';
2 |
3 | export const LogoUrl =
4 | 'https://lf3-static.bytednsdoc.com/obj/eden-cn/vhaeh7vhabf/Midscene.png';
5 |
6 | export const Logo = ({ hideLogo = false }: { hideLogo?: boolean }) => {
7 | if (hideLogo) {
8 | return null;
9 | }
10 |
11 | return (
12 | <div className="logo">
13 | <a href="https://midscenejs.com/" target="_blank" rel="noreferrer">
14 | <img
15 | alt="Midscene_logo"
16 | src="https://lf3-static.bytednsdoc.com/obj/eden-cn/vhaeh7vhabf/Midscene.png"
17 | />
18 | </a>
19 | </div>
20 | );
21 | };
22 |
--------------------------------------------------------------------------------
/packages/visualizer/src/component/pixi-loader.tsx:
--------------------------------------------------------------------------------
1 | import 'pixi.js/unsafe-eval';
2 | import * as PIXI from 'pixi.js';
3 |
4 | const globalTextureMap = new Map<string, PIXI.Texture>();
5 |
6 | export const loadTexture = async (img: string) => {
7 | if (globalTextureMap.has(img)) return;
8 | return PIXI.Assets.load(img).then((texture) => {
9 | globalTextureMap.set(img, texture);
10 | });
11 | };
12 |
13 | export const getTextureFromCache = (name: string) => {
14 | return globalTextureMap.get(name);
15 | };
16 |
17 | export const getTexture = async (name: string) => {
18 | if (globalTextureMap.has(name)) {
19 | return globalTextureMap.get(name);
20 | }
21 |
22 | await loadTexture(name);
23 | return globalTextureMap.get(name);
24 | };
25 |
--------------------------------------------------------------------------------
/packages/visualizer/src/component/playground/playground-types.ts:
--------------------------------------------------------------------------------
1 | import type { GroupedActionDump, UIContext } from '@midscene/core';
2 | import type { ChromeExtensionProxyPageAgent } from '@midscene/web/chrome-extension';
3 | import type { StaticPageAgent } from '@midscene/web/playground';
4 | import type { WebUIContext } from '@midscene/web/utils';
5 |
6 | // result type
7 | export interface PlaygroundResult {
8 | result: any;
9 | dump?: GroupedActionDump | null;
10 | reportHTML?: string | null;
11 | error: string | null;
12 | }
13 |
14 | // Playground component props type
15 | export interface PlaygroundProps {
16 | getAgent: (
17 | forceSameTabNavigation?: boolean,
18 | ) => StaticPageAgent | ChromeExtensionProxyPageAgent | null;
19 | hideLogo?: boolean;
20 | showContextPreview?: boolean;
21 | dryMode?: boolean;
22 | }
23 |
24 | // static playground component props type
25 | export interface StaticPlaygroundProps {
26 | context: WebUIContext | null;
27 | }
28 |
29 | // service mode type
30 | export type ServiceModeType = 'Server' | 'In-Browser' | 'In-Browser-Extension';
31 |
32 | // run type
33 | export type RunType = 'aiAction' | 'aiQuery' | 'aiAssert' | 'aiTap';
34 |
--------------------------------------------------------------------------------
/packages/visualizer/src/component/playground/useServerValid.ts:
--------------------------------------------------------------------------------
1 | import { useEffect, useState } from 'react';
2 | import { useEnvConfig } from '../store/store';
3 | import { checkServerStatus } from './playground-utils';
4 |
5 | export const useServerValid = (shouldRun = true) => {
6 | const [serverValid, setServerValid] = useState(true);
7 | const { serviceMode } = useEnvConfig();
8 |
9 | useEffect(() => {
10 | let interruptFlag = false;
11 | if (!shouldRun) return;
12 |
13 | Promise.resolve(
14 | (async () => {
15 | while (!interruptFlag) {
16 | const status = await checkServerStatus();
17 | if (status) {
18 | setServerValid(true);
19 | } else {
20 | setServerValid(false);
21 | }
22 | // sleep 1s
23 | await new Promise((resolve) => setTimeout(resolve, 1000));
24 | }
25 | })(),
26 | );
27 |
28 | return () => {
29 | interruptFlag = true;
30 | };
31 | }, [serviceMode, shouldRun]);
32 |
33 | return serverValid;
34 | };
35 |
--------------------------------------------------------------------------------
/packages/visualizer/src/component/playground/useStaticPageAgent.ts:
--------------------------------------------------------------------------------
1 | import type { StaticPageAgent } from '@midscene/web/playground';
2 | import type { WebUIContext } from '@midscene/web/utils';
3 | import { useMemo } from 'react';
4 | import { staticAgentFromContext } from './playground-utils';
5 |
6 | export { staticAgentFromContext };
7 |
8 | export const useStaticPageAgent = (
9 | context: WebUIContext | undefined | null,
10 | ): StaticPageAgent | null => {
11 | const agent = useMemo(() => {
12 | if (!context) return null;
13 | return staticAgentFromContext(context);
14 | }, [context]);
15 |
16 | return agent;
17 | };
18 |
--------------------------------------------------------------------------------
/packages/visualizer/src/component/shiny-text.tsx:
--------------------------------------------------------------------------------
1 | import type React from 'react';
2 | import './shiny-text.less';
3 |
4 | type ColorTheme = 'blue' | 'purple' | 'green' | 'rainbow';
5 |
6 | interface ShinyTextProps {
7 | text: string;
8 | disabled?: boolean;
9 | speed?: number;
10 | className?: string;
11 | colorTheme?: ColorTheme;
12 | }
13 |
14 | const ShinyText: React.FC<ShinyTextProps> = ({
15 | text,
16 | disabled = false,
17 | speed = 5,
18 | className = '',
19 | }) => {
20 | const style = {
21 | '--animation-duration': `${speed}s`,
22 | } as React.CSSProperties;
23 |
24 | return (
25 | <div
26 | className={`shiny-text ${disabled ? 'disabled' : ''} ${className}`}
27 | style={style}
28 | >
29 | {text}
30 | </div>
31 | );
32 | };
33 |
34 | export default ShinyText;
35 |
--------------------------------------------------------------------------------
/packages/visualizer/src/extension/jimp.d.ts:
--------------------------------------------------------------------------------
1 | declare module 'jimp/browser/lib/jimp.js' {
2 | import Jimp from 'jimp';
3 | export default Jimp;
4 | }
5 |
--------------------------------------------------------------------------------
/packages/visualizer/src/global.d.ts:
--------------------------------------------------------------------------------
1 | declare module '*.svg' {
2 | export const ReactComponent: React.FunctionComponent<
3 | React.SVGProps<SVGSVGElement> & {
4 | style?: React.CSSProperties; // 确保包含 style 属性
5 | }
6 | >;
7 |
8 | // const content: string;
9 | export default ReactComponent;
10 | }
11 |
12 | declare module '*.png' {
13 | export default string;
14 | }
15 |
16 | declare module '*.svg?react' {
17 | const ReactComponent: React.FunctionComponent<
18 | React.SVGProps<SVGSVGElement> & {
19 | style?: React.CSSProperties; // 确保包含 style 属性
20 | }
21 | >;
22 | export default ReactComponent;
23 | }
24 |
--------------------------------------------------------------------------------
/packages/visualizer/src/icons/close.svg:
--------------------------------------------------------------------------------
1 | <svg width="18" height="16" viewBox="0 0 18 16" fill="none" xmlns="http://www.w3.org/2000/svg">
2 | <path d="M3.12354 2.66667L14.2863 13.3333" stroke="#333333" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
3 | <path d="M3.12354 13.3333L14.2863 2.66667" stroke="#333333" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/>
4 | </svg>
5 |
--------------------------------------------------------------------------------
/packages/visualizer/src/icons/history.svg:
--------------------------------------------------------------------------------
1 | <svg width="25" height="25" viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg">
2 | <path d="M6.63002 9.02131C3.76823 15.1469 8.82714 19.522 12.6931 19.522C16.5591 19.522 19.6931 16.388 19.6931 12.522C19.6931 8.65598 16.5591 5.52197 12.6931 5.52197C10.1024 5.52197 7.84043 6.92936 6.63002 9.02131Z" stroke="black" stroke-opacity="0.85" stroke-width="1.33" stroke-linecap="round" stroke-linejoin="round"/>
3 | <path d="M12.6948 8.32198L12.6943 12.5251L15.6621 15.4928" stroke="black" stroke-opacity="0.85" stroke-width="1.33" stroke-linecap="round" stroke-linejoin="round"/>
4 | </svg>
5 |
--------------------------------------------------------------------------------
/packages/visualizer/src/icons/magnifying-glass.svg:
--------------------------------------------------------------------------------
1 | <svg width="19" height="19" viewBox="0 0 19 19" fill="none" xmlns="http://www.w3.org/2000/svg">
2 | <g clip-path="url(#clip0_541_8555)">
3 | <path d="M8.39697 14.2909C11.9178 14.2909 14.772 11.4367 14.772 7.91589C14.772 4.39509 11.9178 1.54089 8.39697 1.54089C4.87617 1.54089 2.02197 4.39509 2.02197 7.91589C2.02197 11.4367 4.87617 14.2909 8.39697 14.2909Z" stroke="black" stroke-opacity="0.65" stroke-width="1.5" stroke-linejoin="round"/>
4 | <path d="M10.5185 5.41956C9.97563 4.87667 9.22563 4.54089 8.39718 4.54089C7.56877 4.54089 6.81877 4.87667 6.27588 5.41956" stroke="black" stroke-opacity="0.65" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
5 | <path d="M12.98 12.499L16.162 15.681" stroke="black" stroke-opacity="0.65" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
6 | </g>
7 | <defs>
8 | <clipPath id="clip0_541_8555">
9 | <rect width="18" height="18" fill="white" transform="translate(0.521973 0.0408936)"/>
10 | </clipPath>
11 | </defs>
12 | </svg>
13 |
--------------------------------------------------------------------------------
/packages/visualizer/src/init.ts:
--------------------------------------------------------------------------------
1 | // biome-ignore lint/style/useNodejsImportProtocol: <explanation>
2 | import { Buffer } from 'buffer';
3 |
4 | // To solve the '"global is not defined" in randomBytes
5 | // https://www.perplexity.ai/search/how-to-solve-global-is-not-def-xOrpDcfOSKqz_IXtwmK4_Q
6 | window.global ||= window;
7 | window.Buffer = Buffer;
8 |
9 | let sideEffect = 0;
10 |
11 | export const setSideEffect = () => {
12 | sideEffect++;
13 | return sideEffect;
14 | };
15 |
--------------------------------------------------------------------------------
/packages/visualizer/src/types.d.ts:
--------------------------------------------------------------------------------
1 | // 扩展 Window 接口以包含 global 和 Buffer 属性
2 | interface Window {
3 | global: typeof globalThis;
4 | Buffer: any;
5 | }
6 |
7 | // 版本变量
8 | declare const __VERSION__: string;
9 |
--------------------------------------------------------------------------------
/packages/visualizer/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "baseUrl": ".",
4 | "sourceMap": true,
5 | "declaration": true,
6 | "emitDeclarationOnly": true,
7 | "esModuleInterop": true,
8 | "forceConsistentCasingInFileNames": true,
9 | "isolatedModules": true,
10 | "jsx": "preserve",
11 | "lib": ["DOM", "ESNext"],
12 | "moduleResolution": "node",
13 | "paths": {
14 | "@/*": ["./src/*"]
15 | },
16 | "resolveJsonModule": true,
17 | "rootDir": "src",
18 | "skipLibCheck": true,
19 | "strict": true,
20 | "target": "ES2022",
21 | "types": ["react"]
22 | },
23 | "exclude": ["**/node_modules"],
24 | "include": ["src", "builder"]
25 | }
26 |
--------------------------------------------------------------------------------
/packages/web-integration/.gitignore:
--------------------------------------------------------------------------------
1 |
2 | # Midscene.js dump files
3 | midscene_run/report
4 | midscene_run/dump
5 |
6 |
--------------------------------------------------------------------------------
/packages/web-integration/README.md:
--------------------------------------------------------------------------------
1 | ## Documentation
2 |
3 | Automate UI actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML.
4 |
5 | See https://midscenejs.com/ for details.
6 |
7 | ## License
8 |
9 | Midscene is MIT licensed.
--------------------------------------------------------------------------------
/packages/web-integration/bin/midscene-playground:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 |
3 | require('../dist/lib/midscene-playground.js');
--------------------------------------------------------------------------------
/packages/web-integration/src/bridge-mode/browser.ts:
--------------------------------------------------------------------------------
1 | import { ExtensionBridgePageBrowserSide } from '../bridge-mode/page-browser-side';
2 |
3 | export { ExtensionBridgePageBrowserSide };
4 |
--------------------------------------------------------------------------------
/packages/web-integration/src/bridge-mode/index.ts:
--------------------------------------------------------------------------------
1 | import { AgentOverChromeBridge } from './agent-cli-side';
2 |
3 | export { AgentOverChromeBridge };
4 |
5 | export { overrideAIConfig, allConfigFromEnv } from '@midscene/shared/env';
6 |
7 | export { killRunningServer } from './io-server';
8 |
--------------------------------------------------------------------------------
/packages/web-integration/src/chrome-extension/agent.ts:
--------------------------------------------------------------------------------
1 | import { PageAgent, type PageAgentOpt } from '@/common/agent';
2 | import type ChromeExtensionProxyPage from './page';
3 |
4 | export class ChromeExtensionProxyPageAgent extends PageAgent {
5 | // biome-ignore lint/complexity/noUselessConstructor: <explanation>
6 | constructor(page: ChromeExtensionProxyPage, opts?: PageAgentOpt) {
7 | super(page, opts);
8 | }
9 | }
10 |
--------------------------------------------------------------------------------
/packages/web-integration/src/chrome-extension/index.ts:
--------------------------------------------------------------------------------
1 | import { ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED } from '../common/utils';
2 | import { ChromeExtensionProxyPageAgent } from './agent';
3 | import ChromeExtensionProxyPage from './page';
4 |
5 | export { overrideAIConfig } from '@midscene/shared/env';
6 |
7 | export {
8 | ChromeExtensionProxyPage,
9 | ChromeExtensionProxyPageAgent,
10 | ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED,
11 | };
12 |
--------------------------------------------------------------------------------
/packages/web-integration/src/index.ts:
--------------------------------------------------------------------------------
1 | export { PlaywrightAiFixture } from './playwright';
2 | export type { PlayWrightAiFixtureType } from './playwright';
3 | export type {
4 | WebPage,
5 | AndroidDevicePage,
6 | AndroidDeviceInputOpt,
7 | } from './common/page';
8 | export type { AbstractPage } from './page';
9 |
10 | export { PageAgent, type PageAgentOpt } from './common/agent';
11 | export { PuppeteerAgent } from './puppeteer';
12 | export { PlaywrightAgent } from './playwright';
13 | export { StaticPageAgent } from './playground/agent';
14 |
15 | export { ScriptPlayer, parseYamlScript } from './yaml';
16 | export { parseContextFromWebPage } from './common/utils';
17 |
--------------------------------------------------------------------------------
/packages/web-integration/src/playground/agent.ts:
--------------------------------------------------------------------------------
1 | import { PageAgent } from '@/common/agent';
2 | import type StaticPage from './static-page';
3 |
4 | export class StaticPageAgent extends PageAgent {
5 | constructor(page: StaticPage) {
6 | super(page, {});
7 | this.dryMode = true;
8 | }
9 | }
10 |
--------------------------------------------------------------------------------
/packages/web-integration/src/playground/bin.ts:
--------------------------------------------------------------------------------
1 | import { StaticPage, StaticPageAgent } from './';
2 | import PlaygroundServer from './server';
3 |
4 | const server = new PlaygroundServer(StaticPage, StaticPageAgent);
5 | Promise.resolve()
6 | .then(() => server.launch())
7 | .then(() => {
8 | console.log(
9 | `Midscene playground server is running on http://localhost:${server.port}`,
10 | );
11 | });
12 |
--------------------------------------------------------------------------------
/packages/web-integration/src/playground/index.ts:
--------------------------------------------------------------------------------
1 | import { ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED } from '../common/utils';
2 | import { StaticPageAgent } from './agent';
3 | import StaticPage from './static-page';
4 |
5 | export { StaticPageAgent, StaticPage, ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED };
6 |
--------------------------------------------------------------------------------
/packages/web-integration/src/playwright/index.ts:
--------------------------------------------------------------------------------
1 | import { PageAgent, type WebPageAgentOpt } from '@/common/agent';
2 | import type { Page as PlaywrightPage } from 'playwright';
3 | import { WebPage as PlaywrightWebPage } from './page';
4 |
5 | export type { PlayWrightAiFixtureType } from './ai-fixture';
6 | export { PlaywrightAiFixture } from './ai-fixture';
7 | export { overrideAIConfig } from '@midscene/shared/env';
8 | export { WebPage as PlaywrightWebPage } from './page';
9 | import { forceClosePopup } from '@/common/utils';
10 | import { getDebug } from '@midscene/shared/logger';
11 |
12 | const debug = getDebug('playwright:agent');
13 |
14 | export class PlaywrightAgent extends PageAgent<PlaywrightWebPage> {
15 | constructor(page: PlaywrightPage, opts?: WebPageAgentOpt) {
16 | const webPage = new PlaywrightWebPage(page, opts);
17 | super(webPage, opts);
18 |
19 | const { forceSameTabNavigation = true } = opts ?? {};
20 |
21 | if (forceSameTabNavigation) {
22 | forceClosePopup(page, debug);
23 | }
24 | }
25 |
26 | async waitForNetworkIdle(timeout = 1000) {
27 | await this.page.underlyingPage.waitForLoadState('networkidle', { timeout });
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/packages/web-integration/src/playwright/page.ts:
--------------------------------------------------------------------------------
1 | import {
2 | DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT,
3 | DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT,
4 | } from '@midscene/shared/constants';
5 | import type { Page as PlaywrightPageType } from 'playwright';
6 | import type { WebPageOpt } from '../common/agent';
7 | import { Page as BasePage } from '../puppeteer/base-page';
8 |
9 | export class WebPage extends BasePage<'playwright', PlaywrightPageType> {
10 | waitForNavigationTimeout: number;
11 | waitForNetworkIdleTimeout: number;
12 |
13 | constructor(page: PlaywrightPageType, opts?: WebPageOpt) {
14 | super(page, 'playwright', opts);
15 | const {
16 | waitForNavigationTimeout = DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT,
17 | waitForNetworkIdleTimeout = DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT,
18 | } = opts ?? {};
19 | this.waitForNavigationTimeout = waitForNavigationTimeout;
20 | this.waitForNetworkIdleTimeout = waitForNetworkIdleTimeout;
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/packages/web-integration/src/puppeteer/index.ts:
--------------------------------------------------------------------------------
1 | import { PageAgent, type WebPageAgentOpt } from '@/common/agent';
2 | import { forceClosePopup } from '@/common/utils';
3 | import { getDebug } from '@midscene/shared/logger';
4 | import type { Page as PuppeteerPage } from 'puppeteer';
5 | import type { AndroidDeviceInputOpt } from '../common/page';
6 | import { WebPage as PuppeteerWebPage } from './page';
7 |
8 | const debug = getDebug('puppeteer:agent');
9 |
10 | export { WebPage as PuppeteerWebPage } from './page';
11 | export type { AndroidDeviceInputOpt };
12 |
13 | export class PuppeteerAgent extends PageAgent<PuppeteerWebPage> {
14 | constructor(page: PuppeteerPage, opts?: WebPageAgentOpt) {
15 | const webPage = new PuppeteerWebPage(page, opts);
16 | super(webPage, opts);
17 |
18 | const { forceSameTabNavigation = true } = opts ?? {};
19 |
20 | if (forceSameTabNavigation) {
21 | forceClosePopup(page, debug);
22 | }
23 | }
24 | }
25 |
26 | export { overrideAIConfig } from '@midscene/shared/env';
27 |
28 | // Do NOT export this since it requires puppeteer
29 | // export { puppeteerAgentForTarget } from './agent-launcher';
30 |
--------------------------------------------------------------------------------
/packages/web-integration/src/yaml/builder.ts:
--------------------------------------------------------------------------------
1 | import type {
2 | MidsceneYamlScript,
3 | MidsceneYamlScriptWebEnv,
4 | MidsceneYamlTask,
5 | } from '@midscene/core';
6 | import yaml from 'js-yaml';
7 |
8 | export function buildYaml(
9 | env: MidsceneYamlScriptWebEnv,
10 | tasks: MidsceneYamlTask[],
11 | ) {
12 | const result: MidsceneYamlScript = {
13 | target: env,
14 | tasks,
15 | };
16 |
17 | return yaml.dump(result, {
18 | indent: 2,
19 | });
20 | }
21 |
--------------------------------------------------------------------------------
/packages/web-integration/src/yaml/index.ts:
--------------------------------------------------------------------------------
1 | export * from './player';
2 | export * from './builder';
3 | export * from './utils';
4 |
--------------------------------------------------------------------------------
/packages/web-integration/tests/ai/bridge/keyboard-event.test.ts:
--------------------------------------------------------------------------------
1 | import {
2 | AgentOverChromeBridge,
3 | getBridgePageInCliSide,
4 | } from '@/bridge-mode/agent-cli-side';
5 | import { describe, expect, it, vi } from 'vitest';
6 |
7 | vi.setConfig({
8 | testTimeout: 3 * 60 * 1000,
9 | });
10 | const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
11 |
12 | const describeIf = process.env.BRIDGE_MODE ? describe : describe.skip;
13 |
14 | describeIf(
15 | 'keyboard event in bridge mode',
16 | {
17 | timeout: 3 * 60 * 10,
18 | },
19 | () => {
20 | it('page in cli side scroll down', async () => {
21 | const agent = new AgentOverChromeBridge();
22 | await agent.connectNewTabWithUrl('https://www.baidu.com');
23 |
24 | await agent.aiAction('type "midscene" and hit Enter and scroll down');
25 | // sleep 3s
26 | await sleep(3000);
27 |
28 | await agent.destroy();
29 | });
30 |
31 | it('page in cli side select all text', async () => {
32 | const agent = new AgentOverChromeBridge();
33 | await agent.connectNewTabWithUrl('https://www.baidu.com');
34 |
35 | await agent.aiAction('type "Midscene" and hit Enter and select all text');
36 | // sleep 3s
37 | await sleep(3000);
38 |
39 | await agent.destroy();
40 | });
41 | },
42 | );
43 |
--------------------------------------------------------------------------------
/packages/web-integration/tests/ai/bridge/open-new-tab.test.ts:
--------------------------------------------------------------------------------
1 | import {
2 | AgentOverChromeBridge,
3 | getBridgePageInCliSide,
4 | } from '@/bridge-mode/agent-cli-side';
5 | import { sleep } from '@midscene/core/utils';
6 | import { describe, expect, it, test, vi } from 'vitest';
7 |
8 | vi.setConfig({
9 | testTimeout: 300 * 1000,
10 | });
11 |
12 | const describeIf = process.env.BRIDGE_MODE ? describe : describe.skip;
13 |
14 | describeIf('open new tab in bridge mode', () => {
15 | it(
16 | 'open new tab',
17 | {
18 | timeout: 3 * 60 * 1000,
19 | },
20 | async () => {
21 | const agent = new AgentOverChromeBridge();
22 | await agent.connectNewTabWithUrl('https://www.google.com');
23 |
24 | await agent.aiAction(
25 | 'search "midscene github" and open the first result',
26 | );
27 |
28 | // sleep 3s
29 | await sleep(5000);
30 |
31 | await agent.aiAssert('the page is "midscene github"');
32 |
33 | await agent.destroy();
34 | },
35 | );
36 | });
37 |
--------------------------------------------------------------------------------
/packages/web-integration/tests/ai/bridge/temp.test.ts:
--------------------------------------------------------------------------------
1 | import {
2 | AgentOverChromeBridge,
3 | getBridgePageInCliSide,
4 | } from '@/bridge-mode/agent-cli-side';
5 | import { sleep } from '@midscene/core/utils';
6 | import { describe, expect, it, vi } from 'vitest';
7 |
8 | vi.setConfig({
9 | testTimeout: 300 * 1000,
10 | });
11 |
12 | const describeIf = process.env.BRIDGE_MODE ? describe : describe.skip;
13 |
14 | describeIf('drag event', () => {
15 | it('agent in cli side, current tab', async () => {
16 | const agent = new AgentOverChromeBridge({
17 | cacheId: 'finish-form-and-submit',
18 | });
19 | await agent.connectCurrentTab();
20 |
21 | await sleep(2000);
22 |
23 | await agent.aiAction(
24 | 'Use the test data to complete the form,Comply with the following restrictions: 1. The Captcha code is not required 2. No need to click the register button',
25 | );
26 |
27 | await agent.destroy();
28 | });
29 | });
30 |
--------------------------------------------------------------------------------
/packages/web-integration/tests/ai/web/playwright/ai-shop.spec.ts:
--------------------------------------------------------------------------------
1 | import { test } from './fixture';
2 |
3 | test.beforeEach(async ({ page }) => {
4 | await page.goto('https://www.saucedemo.com/');
5 | await page.setViewportSize({ width: 1920, height: 1080 });
6 | });
7 |
8 | const CACHE_TIME_OUT = process.env.MIDSCENE_CACHE;
9 |
10 | test('ai shop', async ({
11 | ai,
12 | aiInput,
13 | aiAssert,
14 | aiQuery,
15 | aiTap,
16 | agentForPage,
17 | page,
18 | }) => {
19 | if (CACHE_TIME_OUT) {
20 | test.setTimeout(1000 * 1000);
21 | }
22 | // login
23 | const agent = await agentForPage(page);
24 | await aiInput('standard_user', 'in user name input');
25 | await aiInput('secret_sauce', 'in password input');
26 | await agent.aiTap('Login Button');
27 |
28 | // check the login success
29 | await aiAssert('the page title is "Swag Labs"');
30 |
31 | // add to cart
32 | await aiTap('"add to cart" for black t-shirt products');
33 |
34 | await aiTap('click right top cart icon', {
35 | deepThink: true,
36 | });
37 | });
38 |
--------------------------------------------------------------------------------
/packages/web-integration/tests/ai/web/playwright/fixture.ts:
--------------------------------------------------------------------------------
1 | import type { PlayWrightAiFixtureType } from '@/playwright/ai-fixture';
2 | import { PlaywrightAiFixture } from '@/playwright/ai-fixture';
3 | import { test as base } from '@playwright/test';
4 |
5 | export const test = base.extend<PlayWrightAiFixtureType>(
6 | PlaywrightAiFixture({
7 | waitForNetworkIdleTimeout: 10000,
8 | }),
9 | );
10 |
--------------------------------------------------------------------------------
/packages/web-integration/tests/ai/web/playwright/open-new-tab.spec.ts:
--------------------------------------------------------------------------------
1 | import { sleep } from '@midscene/core/utils';
2 | import { test } from './fixture';
3 |
4 | test.beforeEach(async ({ page }) => {
5 | await page.goto('https://cn.bing.com');
6 | });
7 |
8 | const CACHE_TIME_OUT = process.env.MIDSCENE_CACHE;
9 |
10 | test('test open new tab', async ({ page, ai, aiAssert, aiQuery }) => {
11 | if (CACHE_TIME_OUT) {
12 | test.setTimeout(200 * 1000);
13 | }
14 | await ai(
15 | 'type "midscene github" in search box, hit Enter, sleep 5s, and open the github page in result list',
16 | );
17 |
18 | await sleep(5000);
19 | await aiAssert('the page is "midscene github"');
20 | });
21 |
--------------------------------------------------------------------------------
/packages/web-integration/tests/ai/web/puppeteer/utils.ts:
--------------------------------------------------------------------------------
1 | import { PuppeteerWebPage } from '@/puppeteer';
2 | import { launchPuppeteerPage } from '@/puppeteer/agent-launcher';
3 | import type { Viewport } from 'puppeteer';
4 |
5 | export async function launchPage(
6 | url: string,
7 | opt?: {
8 | viewport?: Viewport;
9 | headless?: boolean;
10 | },
11 | ) {
12 | const { page, freeFn } = await launchPuppeteerPage(
13 | {
14 | url,
15 | viewportWidth: opt?.viewport?.width,
16 | viewportHeight: opt?.viewport?.height,
17 | viewportScale: opt?.viewport?.deviceScaleFactor,
18 | },
19 | {
20 | headed: typeof opt?.headless === 'boolean' ? !opt.headless : false,
21 | },
22 | );
23 |
24 | const originPage = page;
25 | const midscenePage = new PuppeteerWebPage(originPage);
26 |
27 | return {
28 | page: midscenePage,
29 | originPage,
30 | reset: async () => {
31 | for (const fn of freeFn) {
32 | await fn.fn();
33 | }
34 | },
35 | };
36 | }
37 |
--------------------------------------------------------------------------------
/packages/web-integration/tests/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "extends": "../tsconfig.json",
3 | "compilerOptions": {
4 | "baseUrl": "../",
5 | "rootDir": "../"
6 | },
7 | "include": [
8 | "**/*",
9 | "../src",
10 | "playwright.config.ts",
11 | "../../android/tests/setting.test.ts",
12 | "../../android/tests/todo.test.ts"
13 | ],
14 | "exclude": ["**/node_modules"]
15 | }
16 |
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/__snapshots__/yaml.test.ts.snap:
--------------------------------------------------------------------------------
1 | // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
2 |
3 | exports[`yaml > basic load 1`] = `
4 | {
5 | "target": {
6 | "url": "https://www.baidu.com",
7 | "waitForNetworkIdle": {
8 | "continueOnNetworkIdleError": true,
9 | "timeout": 1000,
10 | },
11 | },
12 | "tasks": [
13 | {
14 | "flow": [
15 | {
16 | "action": "type 'hello' in search box, hit enter",
17 | },
18 | ],
19 | "name": "search",
20 | },
21 | ],
22 | }
23 | `;
24 |
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/fixtures/cookie/httpbin.dev_cookies.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "domain": "httpbin.dev",
4 | "hostOnly": true,
5 | "httpOnly": false,
6 | "name": "midscene_foo",
7 | "path": "/",
8 | "sameSite": "unspecified",
9 | "secure": false,
10 | "session": true,
11 | "storeId": "0",
12 | "value": "bar"
13 | }
14 | ]
15 |
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/fixtures/extractor/child.html:
--------------------------------------------------------------------------------
1 | <!DOCTYPE html>
2 | <html lang="en">
3 |
4 | <head>
5 | <meta charset="UTF-8">
6 | <meta name="viewport" content="width=device-width, initial-scale=1.0">
7 | <title>Child HTML Page</title>
8 | <style>
9 | body {
10 | font-family: Arial, sans-serif;
11 | margin: 20px;
12 | }
13 |
14 | table {
15 | border-collapse: collapse;
16 | width: 50%;
17 | margin-top: 20px;
18 | }
19 |
20 | th,
21 | td {
22 | border: 1px solid #000;
23 | padding: 8px;
24 | text-align: center;
25 | }
26 |
27 | th {
28 | background-color: #f2f2f2;
29 | }
30 | </style>
31 | </head>
32 |
33 | <body>
34 | <h1>Yet Another Data Record</h1>
35 | <h2>2021-07-15 00:00:00</h2>
36 | <h2>User Name: Guess Who</h2>
37 | <table>
38 | <thead>
39 | <tr>
40 | <th>ID</th>
41 | <th>Field 2</th>
42 | <th>Field 3</th>
43 | <th>Field 4</th>
44 | <th>Field 5</th>
45 | </tr>
46 | </thead>
47 | <tbody>
48 | <tr>
49 | <td>30S</td>
50 | <td>Kace Cervantes</td>
51 | <td>Aylin Sawyer</td>
52 | <td>Jefferson Kirby</td>
53 | <td>Skyla Jefferson</td>
54 | </tr>
55 | </tbody>
56 | </table>
57 | </body>
58 |
59 | </html>
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/fixtures/extractor/scroll/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/web-integration/tests/unit-test/fixtures/extractor/scroll/input.png
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/fixtures/extractor/scroll/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/web-integration/tests/unit-test/fixtures/extractor/scroll/output.png
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/fixtures/web-extractor/assets/search-dark.svg:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1737511467416" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4171" xmlns:xlink="http://www.w3.org/1999/xlink" width="200" height="200"><path d="M690.272 475.424q0-105.728-75.136-180.864t-180.864-75.136-180.864 75.136-75.136 180.864 75.136 180.864 180.864 75.136 180.864-75.136 75.136-180.864zM982.848 950.848q0 29.728-21.728 51.424t-51.424 21.728q-30.848 0-51.424-21.728l-196-195.424q-102.272 70.848-228 70.848-81.728 0-156.288-31.712t-128.576-85.728-85.728-128.576-31.712-156.288 31.712-156.288 85.728-128.576 128.576-85.728 156.288-31.712 156.288 31.712 128.576 85.728 85.728 128.576 31.712 156.288q0 125.728-70.848 228l196 196q21.152 21.152 21.152 51.424z" p-id="4172"></path></svg>
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/fixtures/web-extractor/assets/search.svg:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1737511467416" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4171" xmlns:xlink="http://www.w3.org/1999/xlink" width="200" height="200"><path d="M690.272 475.424q0-105.728-75.136-180.864t-180.864-75.136-180.864 75.136-75.136 180.864 75.136 180.864 180.864 75.136 180.864-75.136 75.136-180.864zM982.848 950.848q0 29.728-21.728 51.424t-51.424 21.728q-30.848 0-51.424-21.728l-196-195.424q-102.272 70.848-228 70.848-81.728 0-156.288-31.712t-128.576-85.728-85.728-128.576-31.712-156.288 31.712-156.288 85.728-128.576 128.576-85.728 156.288-31.712 156.288 31.712 128.576 85.728 85.728 128.576 31.712 156.288q0 125.728-70.848 228l196 196q21.152 21.152 21.152 51.424z" p-id="4172" fill="#ffffff"></path></svg>
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/fixtures/web-extractor/child.html:
--------------------------------------------------------------------------------
1 | <!DOCTYPE html>
2 | <html lang="en">
3 |
4 | <head>
5 | <meta charset="UTF-8">
6 | <meta name="viewport" content="width=device-width, initial-scale=1.0">
7 | <title>Child Page</title>
8 | </head>
9 |
10 | <body>
11 | <h1>Child Page</h1>
12 | <p>This is a child page.</p>
13 | <button>Click me</button>
14 | <div style="width: 100px; height: 200px; background-color: #ccc;"></div>
15 | <p>Something beyong 200px</p>
16 | </body>
17 |
18 | </html>
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/fixtures/web-extractor/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/web-integration/tests/unit-test/fixtures/web-extractor/input.png
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/fixtures/web-extractor/merge-rects.html:
--------------------------------------------------------------------------------
1 | <!doctype html>
2 | <html lang="en">
3 | <head>
4 | <meta charset="UTF-8" />
5 | <meta name="viewport" content="width=device-width, initial-scale=1.0" />
6 | <title>Document</title>
7 | </head>
8 | <body>
9 | <div style="width: 100px; height: 100px">
10 | <button style="width: 20px; height: 20px">
11 | <span>Click Me(span)</span>
12 | </button>
13 | </div>
14 | <div style="width: 100px; height: 100px">
15 | <button style="width: 20px; height: 20px">Click Me(text)</button>
16 | </div>
17 | </body>
18 | </html>
19 |
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/fixtures/web-extractor/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/web-integration/tests/unit-test/fixtures/web-extractor/output.png
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/fixtures/web-extractor/scroll/input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/web-integration/tests/unit-test/fixtures/web-extractor/scroll/input.png
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/fixtures/web-extractor/scroll/output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/web-infra-dev/midscene/3d1bfb4b169dcf6aa1f413b6a57bc892253eb443/packages/web-integration/tests/unit-test/fixtures/web-extractor/scroll/output.png
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/http-server.d.ts:
--------------------------------------------------------------------------------
1 | declare module 'http-server' {
2 | export function createServer(options: http.ServerOptions): {
3 | server: http.Server;
4 | listen: (port: number, host: string, callback: () => void) => void;
5 | };
6 | }
7 |
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/yaml/__snapshots__/player.test.ts.snap:
--------------------------------------------------------------------------------
1 | // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
2 |
3 | exports[`yaml utils > basic build && load 1`] = `
4 | "target:
5 | url: https://bing.com
6 | waitForNetworkIdle:
7 | timeout: 1000
8 | continueOnNetworkIdleError: true
9 | tasks:
10 | - name: search
11 | flow:
12 | - aiAction: type "hello" in search box, hit enter
13 | "
14 | `;
15 |
16 | exports[`yaml utils > basic build && load 2`] = `
17 | {
18 | "target": {
19 | "url": "https://bing.com",
20 | "waitForNetworkIdle": {
21 | "continueOnNetworkIdleError": true,
22 | "timeout": 1000,
23 | },
24 | },
25 | "tasks": [
26 | {
27 | "flow": [
28 | {
29 | "aiAction": "type "hello" in search box, hit enter",
30 | },
31 | ],
32 | "name": "search",
33 | },
34 | ],
35 | }
36 | `;
37 |
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/yaml/__snapshots__/utils.test.ts.snap:
--------------------------------------------------------------------------------
1 | // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
2 |
3 | exports[`utils > build yaml 1`] = `
4 | "target:
5 | url: https://www.example.com
6 | tasks: []
7 | "
8 | `;
9 |
10 | exports[`utils > parseYamlScript > aiRightClick 1`] = `
11 | {
12 | "target": {
13 | "url": "sample_url",
14 | },
15 | "tasks": [
16 | {
17 | "sleep": 1000,
18 | },
19 | {
20 | "aiTap": "sample_button",
21 | },
22 | {
23 | "aiRightClick": "context menu trigger",
24 | },
25 | {
26 | "aiInput": {
27 | "aiInput": "test@example.com",
28 | "locate": "email input",
29 | },
30 | },
31 | ],
32 | }
33 | `;
34 |
35 | exports[`utils > parseYamlScript > interpolates environment variables 1`] = `
36 | {
37 | "target": {
38 | "url": "sample_url",
39 | },
40 | "tasks": [
41 | {
42 | "sleep": 1000,
43 | },
44 | {
45 | "aiTap": "sample_button",
46 | },
47 | {
48 | "aiInput": "sample_input",
49 | "locate": "input description",
50 | },
51 | {
52 | "aiInput": null,
53 | "locate": "input description",
54 | },
55 | ],
56 | }
57 | `;
58 |
--------------------------------------------------------------------------------
/packages/web-integration/tests/unit-test/yaml/server_root/index.html:
--------------------------------------------------------------------------------
1 | <h1>My App</h1>
2 | <p>This is a test page</p>
3 | <div>Width:</div>
4 | <div id="j_width"></div>
5 | <div>Height:</div>
6 | <div id="j_height"></div>
7 | <script>
8 | document.getElementById('j_width').innerText = window.innerWidth;
9 | document.getElementById('j_height').innerText = window.innerHeight;
10 | </script>
11 | </body>
--------------------------------------------------------------------------------
/packages/web-integration/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "baseUrl": ".",
4 | "declaration": true,
5 | "emitDeclarationOnly": true,
6 | "esModuleInterop": true,
7 | "forceConsistentCasingInFileNames": true,
8 | "isolatedModules": true,
9 | "jsx": "preserve",
10 | "lib": ["DOM", "ESNext"],
11 | "moduleResolution": "node",
12 | "paths": {
13 | "@/*": ["./src/*"]
14 | },
15 | "target": "es6",
16 | "resolveJsonModule": true,
17 | "rootDir": "./src",
18 | "skipLibCheck": true,
19 | "strict": true,
20 | "module": "ESNext"
21 | },
22 | "exclude": ["node_modules"],
23 | "include": ["src", "./vitest.config"]
24 | }
25 |
--------------------------------------------------------------------------------
/packages/web-integration/vitest.config.ts:
--------------------------------------------------------------------------------
1 | import path from 'node:path';
2 | import dotenv from 'dotenv';
3 | import { defineConfig } from 'vitest/config';
4 | import { version } from './package.json';
5 |
6 | /**
7 | * Read environment variables from file.
8 | * https://github.com/motdotla/dotenv
9 | */
10 | dotenv.config({
11 | path: path.join(__dirname, '../../.env'),
12 | });
13 |
14 | const aiTestType = process.env.AI_TEST_TYPE;
15 | const unitTests = ['tests/unit-test/**/*.test.ts'];
16 | const aiWebTests = [
17 | 'tests/ai/web/**/*.test.ts',
18 | 'tests/ai/bridge/**/*.test.ts',
19 | ];
20 |
21 | const testFiles = (() => {
22 | switch (aiTestType) {
23 | case 'web':
24 | return [...aiWebTests];
25 | default:
26 | return unitTests;
27 | }
28 | })();
29 |
30 | export default defineConfig({
31 | resolve: {
32 | alias: {
33 | '@': path.resolve(__dirname, 'src'),
34 | },
35 | },
36 | test: {
37 | include: testFiles,
38 | testTimeout: 3 * 60 * 1000, // Global timeout set to 3 minutes
39 | dangerouslyIgnoreUnhandledErrors: !!process.env.CI, // showcase.test.ts is not stable
40 | },
41 | define: {
42 | __VERSION__: `'${version}'`,
43 | },
44 | });
45 |
--------------------------------------------------------------------------------
/pnpm-workspace.yaml:
--------------------------------------------------------------------------------
1 | packages:
2 | - apps/*
3 | - packages/*
4 |
--------------------------------------------------------------------------------